From de43e2a8e7b36cf5e5c4e5c3f5185c85e5e03dea Mon Sep 17 00:00:00 2001 From: bluestreak Date: Sat, 14 Feb 2026 20:05:38 +0000 Subject: [PATCH 01/89] merge fallout --- .../main/java/io/questdb/client/Sender.java | 199 ++- .../cutlass/http/client/WebSocketClient.java | 775 +++++++++ .../http/client/WebSocketClientFactory.java | 137 ++ .../http/client/WebSocketClientLinux.java | 71 + .../http/client/WebSocketClientOsx.java | 75 + .../http/client/WebSocketClientWindows.java | 74 + .../http/client/WebSocketFrameHandler.java | 93 ++ .../http/client/WebSocketSendBuffer.java | 582 +++++++ .../ilpv4/client/GlobalSymbolDictionary.java | 173 ++ .../cutlass/ilpv4/client/IlpBufferWriter.java | 177 ++ .../ilpv4/client/IlpV4WebSocketEncoder.java | 790 +++++++++ .../ilpv4/client/IlpV4WebSocketSender.java | 1398 ++++++++++++++++ .../cutlass/ilpv4/client/InFlightWindow.java | 468 ++++++ .../ilpv4/client/MicrobatchBuffer.java | 501 ++++++ .../ilpv4/client/NativeBufferWriter.java | 289 ++++ .../cutlass/ilpv4/client/ResponseReader.java | 247 +++ .../ilpv4/client/WebSocketChannel.java | 668 ++++++++ .../ilpv4/client/WebSocketResponse.java | 283 ++++ .../ilpv4/client/WebSocketSendQueue.java | 693 ++++++++ .../ilpv4/protocol/IlpV4BitReader.java | 335 ++++ .../ilpv4/protocol/IlpV4BitWriter.java | 247 +++ .../ilpv4/protocol/IlpV4ColumnDef.java | 163 ++ .../ilpv4/protocol/IlpV4Constants.java | 506 ++++++ .../ilpv4/protocol/IlpV4GorillaDecoder.java | 251 +++ .../ilpv4/protocol/IlpV4GorillaEncoder.java | 235 +++ .../ilpv4/protocol/IlpV4NullBitmap.java | 310 ++++ .../ilpv4/protocol/IlpV4SchemaHash.java | 574 +++++++ .../ilpv4/protocol/IlpV4TableBuffer.java | 1424 +++++++++++++++++ .../ilpv4/protocol/IlpV4TimestampDecoder.java | 474 ++++++ .../cutlass/ilpv4/protocol/IlpV4Varint.java | 261 +++ .../cutlass/ilpv4/protocol/IlpV4ZigZag.java | 98 ++ .../ilpv4/websocket/WebSocketCloseCode.java | 178 +++ .../ilpv4/websocket/WebSocketFrameParser.java | 342 ++++ .../ilpv4/websocket/WebSocketFrameWriter.java | 281 ++++ .../ilpv4/websocket/WebSocketHandshake.java | 421 +++++ .../ilpv4/websocket/WebSocketOpcode.java | 136 ++ .../client/std/CharSequenceIntHashMap.java | 209 +++ .../io/questdb/client/std/LongHashSet.java | 155 ++ core/src/main/java/module-info.java | 3 + 39 files changed, 14292 insertions(+), 4 deletions(-) create mode 100644 core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientFactory.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientLinux.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientOsx.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientWindows.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketFrameHandler.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/client/GlobalSymbolDictionary.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/client/IlpBufferWriter.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/client/IlpV4WebSocketEncoder.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/client/IlpV4WebSocketSender.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/client/InFlightWindow.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/client/MicrobatchBuffer.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/client/NativeBufferWriter.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/client/ResponseReader.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/client/WebSocketChannel.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/client/WebSocketResponse.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/client/WebSocketSendQueue.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4BitReader.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4BitWriter.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4ColumnDef.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4Constants.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4GorillaDecoder.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4GorillaEncoder.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4NullBitmap.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4SchemaHash.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4TableBuffer.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4TimestampDecoder.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4Varint.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4ZigZag.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketCloseCode.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketFrameParser.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketFrameWriter.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketHandshake.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketOpcode.java create mode 100644 core/src/main/java/io/questdb/client/std/CharSequenceIntHashMap.java create mode 100644 core/src/main/java/io/questdb/client/std/LongHashSet.java diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index c63609f..fcc6d46 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -34,6 +34,7 @@ import io.questdb.client.cutlass.line.http.AbstractLineHttpSender; import io.questdb.client.cutlass.line.tcp.DelegatingTlsChannel; import io.questdb.client.cutlass.line.tcp.PlainTcpLineChannel; +import io.questdb.client.cutlass.ilpv4.client.IlpV4WebSocketSender; import io.questdb.client.impl.ConfStringParser; import io.questdb.client.network.NetworkFacade; import io.questdb.client.network.NetworkFacadeImpl; @@ -144,7 +145,21 @@ static LineSenderBuilder builder(CharSequence configurationString) { * @return Builder object to create a new Sender instance. */ static LineSenderBuilder builder(Transport transport) { - return new LineSenderBuilder(transport == Transport.HTTP ? LineSenderBuilder.PROTOCOL_HTTP : LineSenderBuilder.PROTOCOL_TCP); + int protocol; + switch (transport) { + case HTTP: + protocol = LineSenderBuilder.PROTOCOL_HTTP; + break; + case TCP: + protocol = LineSenderBuilder.PROTOCOL_TCP; + break; + case WEBSOCKET: + protocol = LineSenderBuilder.PROTOCOL_WEBSOCKET; + break; + default: + throw new LineSenderException("unknown transport: " + transport); + } + return new LineSenderBuilder(protocol); } /** @@ -461,7 +476,15 @@ enum Transport { * and for use-cases where HTTP transport is not suitable, when communicating with a QuestDB server over a high-latency * network */ - TCP + TCP, + + /** + * Use WebSocket transport to communicate with a QuestDB server. + *

+ * WebSocket transport uses the ILP v4 binary protocol for efficient data ingestion. + * It supports both synchronous and asynchronous modes with flow control. + */ + WEBSOCKET } /** @@ -522,6 +545,13 @@ final class LineSenderBuilder { private static final int PARAMETER_NOT_SET_EXPLICITLY = -1; private static final int PROTOCOL_HTTP = 1; private static final int PROTOCOL_TCP = 0; + private static final int PROTOCOL_WEBSOCKET = 2; + private static final int DEFAULT_WEBSOCKET_PORT = 9000; + private static final int DEFAULT_IN_FLIGHT_WINDOW_SIZE = 8; + private static final int DEFAULT_SEND_QUEUE_CAPACITY = 16; + private static final int DEFAULT_WS_AUTO_FLUSH_ROWS = 500; + private static final int DEFAULT_WS_AUTO_FLUSH_BYTES = 1024 * 1024; // 1MB + private static final long DEFAULT_WS_AUTO_FLUSH_INTERVAL_NANOS = 100_000_000L; // 100ms private final ObjList hosts = new ObjList<>(); private final IntList ports = new IntList(); private int autoFlushIntervalMillis = PARAMETER_NOT_SET_EXPLICITLY; @@ -568,6 +598,11 @@ public int getTimeout() { private char[] trustStorePassword; private String trustStorePath; private String username; + // WebSocket-specific fields + private int inFlightWindowSize = PARAMETER_NOT_SET_EXPLICITLY; + private int sendQueueCapacity = PARAMETER_NOT_SET_EXPLICITLY; + private boolean asyncMode = false; + private int autoFlushBytes = PARAMETER_NOT_SET_EXPLICITLY; private LineSenderBuilder() { @@ -733,6 +768,47 @@ public LineSenderBuilder autoFlushRows(int autoFlushRows) { return this; } + /** + * Set the maximum number of bytes per batch before auto-flushing. + *
+ * This is only used when communicating over WebSocket transport. + *
+ * Default value is 1MB. + * + * @param bytes maximum bytes per batch + * @return this instance for method chaining + */ + public LineSenderBuilder autoFlushBytes(int bytes) { + if (this.autoFlushBytes != PARAMETER_NOT_SET_EXPLICITLY) { + throw new LineSenderException("auto flush bytes was already configured") + .put("[bytes=").put(this.autoFlushBytes).put("]"); + } + if (bytes < 0) { + throw new LineSenderException("auto flush bytes cannot be negative") + .put("[bytes=").put(bytes).put("]"); + } + this.autoFlushBytes = bytes; + return this; + } + + /** + * Enable asynchronous mode for WebSocket transport. + *
+ * In async mode, rows are batched and sent asynchronously with flow control. + * This provides higher throughput at the cost of more complex error handling. + *
+ * This is only used when communicating over WebSocket transport. + *
+ * Default is synchronous mode (false). + * + * @param enabled whether to enable async mode + * @return this instance for method chaining + */ + public LineSenderBuilder asyncMode(boolean enabled) { + this.asyncMode = enabled; + return this; + } + /** * Configure capacity of an internal buffer. *

@@ -791,6 +867,39 @@ public Sender build() { username, password, maxNameLength, actualMaxRetriesNanos, maxBackoffMillis, actualMinRequestThroughput, actualAutoFlushIntervalMillis, protocolVersion); } + if (protocol == PROTOCOL_WEBSOCKET) { + if (hosts.size() != 1 || ports.size() != 1) { + throw new LineSenderException("only a single address (host:port) is supported for WebSocket transport"); + } + + int actualAutoFlushRows = autoFlushRows == PARAMETER_NOT_SET_EXPLICITLY ? DEFAULT_WS_AUTO_FLUSH_ROWS : autoFlushRows; + int actualAutoFlushBytes = autoFlushBytes == PARAMETER_NOT_SET_EXPLICITLY ? DEFAULT_WS_AUTO_FLUSH_BYTES : autoFlushBytes; + long actualAutoFlushIntervalNanos = autoFlushIntervalMillis == PARAMETER_NOT_SET_EXPLICITLY + ? DEFAULT_WS_AUTO_FLUSH_INTERVAL_NANOS + : TimeUnit.MILLISECONDS.toNanos(autoFlushIntervalMillis); + int actualInFlightWindowSize = inFlightWindowSize == PARAMETER_NOT_SET_EXPLICITLY ? DEFAULT_IN_FLIGHT_WINDOW_SIZE : inFlightWindowSize; + int actualSendQueueCapacity = sendQueueCapacity == PARAMETER_NOT_SET_EXPLICITLY ? DEFAULT_SEND_QUEUE_CAPACITY : sendQueueCapacity; + + if (asyncMode) { + return IlpV4WebSocketSender.connectAsync( + hosts.getQuick(0), + ports.getQuick(0), + tlsEnabled, + actualAutoFlushRows, + actualAutoFlushBytes, + actualAutoFlushIntervalNanos, + actualInFlightWindowSize, + actualSendQueueCapacity + ); + } else { + return IlpV4WebSocketSender.connect( + hosts.getQuick(0), + ports.getQuick(0), + tlsEnabled + ); + } + } + assert protocol == PROTOCOL_TCP; if (hosts.size() != 1 || ports.size() != 1) { @@ -1048,6 +1157,29 @@ public LineSenderBuilder httpUsernamePassword(String username, String password) return this; } + /** + * Set the maximum number of batches that can be in-flight awaiting server acknowledgment. + *
+ * This is only used when communicating over WebSocket transport with async mode enabled. + *
+ * Default value is 8. + * + * @param size maximum number of in-flight batches + * @return this instance for method chaining + */ + public LineSenderBuilder inFlightWindowSize(int size) { + if (this.inFlightWindowSize != PARAMETER_NOT_SET_EXPLICITLY) { + throw new LineSenderException("in-flight window size was already configured") + .put("[size=").put(this.inFlightWindowSize).put("]"); + } + if (size < 1) { + throw new LineSenderException("in-flight window size must be positive") + .put("[size=").put(size).put("]"); + } + this.inFlightWindowSize = size; + return this; + } + /** * Configures the maximum backoff time between retry attempts when the Sender encounters recoverable errors. *
@@ -1239,6 +1371,29 @@ public LineSenderBuilder retryTimeoutMillis(int retryTimeoutMillis) { return this; } + /** + * Set the capacity of the send queue for batches waiting to be sent. + *
+ * This is only used when communicating over WebSocket transport with async mode enabled. + *
+ * Default value is 16. + * + * @param capacity send queue capacity + * @return this instance for method chaining + */ + public LineSenderBuilder sendQueueCapacity(int capacity) { + if (this.sendQueueCapacity != PARAMETER_NOT_SET_EXPLICITLY) { + throw new LineSenderException("send queue capacity was already configured") + .put("[capacity=").put(this.sendQueueCapacity).put("]"); + } + if (capacity < 1) { + throw new LineSenderException("send queue capacity must be positive") + .put("[capacity=").put(capacity).put("]"); + } + this.sendQueueCapacity = capacity; + return this; + } + private static int getValue(CharSequence configurationString, int pos, StringSink sink, String name) { if ((pos = ConfStringParser.value(configurationString, pos, sink)) < 0) { throw new LineSenderException("invalid ").put(name).put(" [error=").put(sink).put("]"); @@ -1275,7 +1430,13 @@ private void configureDefaults() { maximumBufferCapacity = protocol == PROTOCOL_HTTP ? DEFAULT_MAXIMUM_BUFFER_CAPACITY : bufferCapacity; } if (ports.size() == 0) { - ports.add(protocol == PROTOCOL_HTTP ? DEFAULT_HTTP_PORT : DEFAULT_TCP_PORT); + if (protocol == PROTOCOL_HTTP) { + ports.add(DEFAULT_HTTP_PORT); + } else if (protocol == PROTOCOL_WEBSOCKET) { + ports.add(DEFAULT_WEBSOCKET_PORT); + } else { + ports.add(DEFAULT_TCP_PORT); + } } if (tlsValidationMode == null) { tlsValidationMode = TlsValidationMode.DEFAULT; @@ -1334,8 +1495,16 @@ private LineSenderBuilder fromConfig(CharSequence configurationString) { } else if (Chars.equals("tcps", sink)) { tcp(); tlsEnabled = true; + } else if (Chars.equals("ws", sink)) { + if (tlsEnabled) { + throw new LineSenderException("cannot use ws protocol when TLS is enabled. use wss instead"); + } + websocket(); + } else if (Chars.equals("wss", sink)) { + websocket(); + tlsEnabled = true; } else { - throw new LineSenderException("invalid schema [schema=").put(sink).put(", supported-schemas=[http, https, tcp, tcps]]"); + throw new LineSenderException("invalid schema [schema=").put(sink).put(", supported-schemas=[http, https, tcp, tcps, ws, wss]]"); } String tcpToken = null; @@ -1557,6 +1726,14 @@ private void tcp() { protocol = PROTOCOL_TCP; } + private void websocket() { + if (protocol != PARAMETER_NOT_SET_EXPLICITLY) { + throw new LineSenderException("protocol was already configured ") + .put("[protocol=").put(protocol).put("]"); + } + protocol = PROTOCOL_WEBSOCKET; + } + private void validateParameters() { if (hosts.size() == 0) { throw new LineSenderException("questdb server address not set"); @@ -1617,6 +1794,20 @@ private void validateParameters() { if (autoFlushIntervalMillis != PARAMETER_NOT_SET_EXPLICITLY) { throw new LineSenderException("auto flush interval is not supported for TCP protocol"); } + } else if (protocol == PROTOCOL_WEBSOCKET) { + if (privateKey != null) { + throw new LineSenderException("TCP authentication is not supported for WebSocket protocol"); + } + if (httpToken != null || username != null || password != null) { + // TODO: WebSocket auth not yet implemented + throw new LineSenderException("Authentication is not yet supported for WebSocket protocol"); + } + if (inFlightWindowSize != PARAMETER_NOT_SET_EXPLICITLY && !asyncMode) { + throw new LineSenderException("in-flight window size requires async mode"); + } + if (sendQueueCapacity != PARAMETER_NOT_SET_EXPLICITLY && !asyncMode) { + throw new LineSenderException("send queue capacity requires async mode"); + } } else { throw new LineSenderException("unsupported protocol ") .put("[protocol=").put(protocol).put("]"); diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java new file mode 100644 index 0000000..8f5ce83 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java @@ -0,0 +1,775 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.http.client; + +import io.questdb.client.HttpClientConfiguration; +import io.questdb.client.cutlass.ilpv4.websocket.WebSocketCloseCode; +import io.questdb.client.cutlass.ilpv4.websocket.WebSocketFrameParser; +import io.questdb.client.cutlass.ilpv4.websocket.WebSocketHandshake; +import io.questdb.client.cutlass.ilpv4.websocket.WebSocketOpcode; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import io.questdb.client.network.IOOperation; +import io.questdb.client.network.NetworkFacade; +import io.questdb.client.network.Socket; +import io.questdb.client.network.SocketFactory; +import io.questdb.client.network.TlsSessionInitFailedException; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Misc; +import io.questdb.client.std.QuietCloseable; +import io.questdb.client.std.Rnd; +import io.questdb.client.std.Unsafe; +import io.questdb.client.std.Vect; +import io.questdb.client.std.str.Utf8String; + +import java.nio.charset.StandardCharsets; +import java.util.Base64; + +import static java.util.concurrent.TimeUnit.NANOSECONDS; + +/** + * Zero-GC WebSocket client built on QuestDB's native socket infrastructure. + *

+ * This client uses native memory buffers and non-blocking I/O with + * platform-specific event notification (epoll/kqueue/select). + *

+ * Features: + *

+ *

+ * Thread safety: This class is NOT thread-safe. Each connection should be + * accessed from a single thread at a time. + */ +public abstract class WebSocketClient implements QuietCloseable { + + private static final Logger LOG = LoggerFactory.getLogger(WebSocketClient.class); + + private static final int DEFAULT_RECV_BUFFER_SIZE = 65536; + private static final int DEFAULT_SEND_BUFFER_SIZE = 65536; + + protected final NetworkFacade nf; + protected final Socket socket; + + private final WebSocketSendBuffer sendBuffer; + private final WebSocketFrameParser frameParser; + private final Rnd rnd; + private final int defaultTimeout; + + // Receive buffer (native memory) + private long recvBufPtr; + private int recvBufSize; + private int recvPos; // Write position + private int recvReadPos; // Read position + + // Connection state + private CharSequence host; + private int port; + private boolean upgraded; + private boolean closed; + + // Handshake key for verification + private String handshakeKey; + + public WebSocketClient(HttpClientConfiguration configuration, SocketFactory socketFactory) { + this.nf = configuration.getNetworkFacade(); + this.socket = socketFactory.newInstance(nf, LOG); + this.defaultTimeout = configuration.getTimeout(); + + int sendBufSize = Math.max(configuration.getInitialRequestBufferSize(), DEFAULT_SEND_BUFFER_SIZE); + int maxSendBufSize = Math.max(configuration.getMaximumRequestBufferSize(), sendBufSize); + this.sendBuffer = new WebSocketSendBuffer(sendBufSize, maxSendBufSize); + + this.recvBufSize = Math.max(configuration.getResponseBufferSize(), DEFAULT_RECV_BUFFER_SIZE); + this.recvBufPtr = Unsafe.malloc(recvBufSize, MemoryTag.NATIVE_DEFAULT); + this.recvPos = 0; + this.recvReadPos = 0; + + this.frameParser = new WebSocketFrameParser(); + this.rnd = new Rnd(System.nanoTime(), System.currentTimeMillis()); + this.upgraded = false; + this.closed = false; + } + + @Override + public void close() { + if (!closed) { + closed = true; + + // Try to send close frame + if (upgraded && !socket.isClosed()) { + try { + sendCloseFrame(WebSocketCloseCode.NORMAL_CLOSURE, null, 1000); + } catch (Exception e) { + // Ignore errors during close + } + } + + disconnect(); + sendBuffer.close(); + + if (recvBufPtr != 0) { + Unsafe.free(recvBufPtr, recvBufSize, MemoryTag.NATIVE_DEFAULT); + recvBufPtr = 0; + } + } + } + + /** + * Disconnects the socket without closing the client. + * The client can be reconnected by calling connect() again. + */ + public void disconnect() { + Misc.free(socket); + upgraded = false; + host = null; + port = 0; + recvPos = 0; + recvReadPos = 0; + } + + /** + * Connects to a WebSocket server. + * + * @param host the server hostname + * @param port the server port + * @param timeout connection timeout in milliseconds + */ + public void connect(CharSequence host, int port, int timeout) { + if (closed) { + throw new HttpClientException("WebSocket client is closed"); + } + + // Close existing connection if connecting to different host:port + if (this.host != null && (!this.host.equals(host) || this.port != port)) { + disconnect(); + } + + if (socket.isClosed()) { + doConnect(host, port, timeout); + } + + this.host = host; + this.port = port; + } + + /** + * Connects using default timeout. + */ + public void connect(CharSequence host, int port) { + connect(host, port, defaultTimeout); + } + + private void doConnect(CharSequence host, int port, int timeout) { + int fd = nf.socketTcp(true); + if (fd < 0) { + throw new HttpClientException("could not allocate a file descriptor [errno=").errno(nf.errno()).put(']'); + } + + if (nf.setTcpNoDelay(fd, true) < 0) { + LOG.info("could not disable Nagle's algorithm [fd={}, errno={}]", fd, nf.errno()); + } + + socket.of(fd); + nf.configureKeepAlive(fd); + + long addrInfo = nf.getAddrInfo(host, port); + if (addrInfo == -1) { + disconnect(); + throw new HttpClientException("could not resolve host [host=").put(host).put(']'); + } + + if (nf.connectAddrInfo(fd, addrInfo) != 0) { + int errno = nf.errno(); + nf.freeAddrInfo(addrInfo); + disconnect(); + throw new HttpClientException("could not connect [host=").put(host) + .put(", port=").put(port) + .put(", errno=").put(errno).put(']'); + } + nf.freeAddrInfo(addrInfo); + + if (nf.configureNonBlocking(fd) < 0) { + int errno = nf.errno(); + disconnect(); + throw new HttpClientException("could not configure non-blocking [fd=").put(fd) + .put(", errno=").put(errno).put(']'); + } + + if (socket.supportsTls()) { + try { + socket.startTlsSession(host); + } catch (TlsSessionInitFailedException e) { + int errno = nf.errno(); + disconnect(); + throw new HttpClientException("could not start TLS session [fd=").put(fd) + .put(", error=").put(e.getFlyweightMessage()) + .put(", errno=").put(errno).put(']'); + } + } + + setupIoWait(); + LOG.debug("Connected to [host={}, port={}]", host, port); + } + + /** + * Performs WebSocket upgrade handshake. + * + * @param path the WebSocket endpoint path (e.g., "/ws") + * @param timeout timeout in milliseconds + */ + public void upgrade(CharSequence path, int timeout) { + if (closed) { + throw new HttpClientException("WebSocket client is closed"); + } + if (socket.isClosed()) { + throw new HttpClientException("Not connected"); + } + if (upgraded) { + return; // Already upgraded + } + + // Generate random key + byte[] keyBytes = new byte[16]; + for (int i = 0; i < 16; i++) { + keyBytes[i] = (byte) rnd.nextInt(256); + } + handshakeKey = Base64.getEncoder().encodeToString(keyBytes); + + // Build upgrade request + sendBuffer.reset(); + sendBuffer.putAscii("GET "); + sendBuffer.putAscii(path); + sendBuffer.putAscii(" HTTP/1.1\r\n"); + sendBuffer.putAscii("Host: "); + sendBuffer.putAscii(host); + if ((socket.supportsTls() && port != 443) || (!socket.supportsTls() && port != 80)) { + sendBuffer.putAscii(":"); + sendBuffer.putAscii(Integer.toString(port)); + } + sendBuffer.putAscii("\r\n"); + sendBuffer.putAscii("Upgrade: websocket\r\n"); + sendBuffer.putAscii("Connection: Upgrade\r\n"); + sendBuffer.putAscii("Sec-WebSocket-Key: "); + sendBuffer.putAscii(handshakeKey); + sendBuffer.putAscii("\r\n"); + sendBuffer.putAscii("Sec-WebSocket-Version: 13\r\n"); + sendBuffer.putAscii("\r\n"); + + // Send request + long startTime = System.nanoTime(); + doSend(sendBuffer.getBufferPtr(), sendBuffer.getWritePos(), timeout); + + // Read response + int remainingTimeout = remainingTime(timeout, startTime); + readUpgradeResponse(remainingTimeout); + + upgraded = true; + sendBuffer.reset(); + LOG.debug("WebSocket upgraded [path={}]", path); + } + + /** + * Performs upgrade with default timeout. + */ + public void upgrade(CharSequence path) { + upgrade(path, defaultTimeout); + } + + private void readUpgradeResponse(int timeout) { + // Read HTTP response into receive buffer + long startTime = System.nanoTime(); + + while (true) { + int remainingTimeout = remainingTime(timeout, startTime); + int bytesRead = recvOrDie(recvBufPtr + recvPos, recvBufSize - recvPos, remainingTimeout); + if (bytesRead > 0) { + recvPos += bytesRead; + } + + // Check for end of headers (\r\n\r\n) + int headerEnd = findHeaderEnd(); + if (headerEnd > 0) { + validateUpgradeResponse(headerEnd); + // Compact buffer - move remaining data to start + int remaining = recvPos - headerEnd; + if (remaining > 0) { + Vect.memmove(recvBufPtr, recvBufPtr + headerEnd, remaining); + } + recvPos = remaining; + recvReadPos = 0; + return; + } + + if (recvPos >= recvBufSize) { + throw new HttpClientException("HTTP response too large"); + } + } + } + + private int findHeaderEnd() { + // Look for \r\n\r\n + for (int i = 0; i < recvPos - 3; i++) { + if (Unsafe.getUnsafe().getByte(recvBufPtr + i) == '\r' && + Unsafe.getUnsafe().getByte(recvBufPtr + i + 1) == '\n' && + Unsafe.getUnsafe().getByte(recvBufPtr + i + 2) == '\r' && + Unsafe.getUnsafe().getByte(recvBufPtr + i + 3) == '\n') { + return i + 4; + } + } + return -1; + } + + private void validateUpgradeResponse(int headerEnd) { + // Extract response as string for parsing + byte[] responseBytes = new byte[headerEnd]; + for (int i = 0; i < headerEnd; i++) { + responseBytes[i] = Unsafe.getUnsafe().getByte(recvBufPtr + i); + } + String response = new String(responseBytes, StandardCharsets.US_ASCII); + + // Check status line + if (!response.startsWith("HTTP/1.1 101")) { + String statusLine = response.split("\r\n")[0]; + throw new HttpClientException("WebSocket upgrade failed: ").put(statusLine); + } + + // Verify Sec-WebSocket-Accept + String expectedAccept = WebSocketHandshake.computeAcceptKey(handshakeKey); + if (!response.contains("Sec-WebSocket-Accept: " + expectedAccept)) { + throw new HttpClientException("Invalid Sec-WebSocket-Accept header"); + } + } + + // === Sending === + + /** + * Gets the send buffer for building WebSocket frames. + *

+ * Usage: + * 

+     * WebSocketSendBuffer buf = client.getSendBuffer();
+     * buf.beginBinaryFrame();
+     * buf.putLong(data);
+     * WebSocketSendBuffer.FrameInfo frame = buf.endBinaryFrame();
+     * client.sendFrame(frame, timeout);
+     * buf.reset();
+     *
+ */ + public WebSocketSendBuffer getSendBuffer() { + return sendBuffer; + } + + /** + * Sends a complete WebSocket frame. + * + * @param frame frame info from endBinaryFrame() + * @param timeout timeout in milliseconds + */ + public void sendFrame(WebSocketSendBuffer.FrameInfo frame, int timeout) { + checkConnected(); + doSend(sendBuffer.getBufferPtr() + frame.offset, frame.length, timeout); + } + + /** + * Sends a complete WebSocket frame with default timeout. + */ + public void sendFrame(WebSocketSendBuffer.FrameInfo frame) { + sendFrame(frame, defaultTimeout); + } + + /** + * Sends binary data as a WebSocket binary frame. + * + * @param dataPtr pointer to data + * @param length data length + * @param timeout timeout in milliseconds + */ + public void sendBinary(long dataPtr, int length, int timeout) { + checkConnected(); + sendBuffer.reset(); + sendBuffer.beginBinaryFrame(); + sendBuffer.putBlockOfBytes(dataPtr, length); + WebSocketSendBuffer.FrameInfo frame = sendBuffer.endBinaryFrame(); + doSend(sendBuffer.getBufferPtr() + frame.offset, frame.length, timeout); + sendBuffer.reset(); + } + + /** + * Sends binary data with default timeout. + */ + public void sendBinary(long dataPtr, int length) { + sendBinary(dataPtr, length, defaultTimeout); + } + + /** + * Sends a ping frame. + */ + public void sendPing(int timeout) { + checkConnected(); + sendBuffer.reset(); + WebSocketSendBuffer.FrameInfo frame = sendBuffer.writePingFrame(); + doSend(sendBuffer.getBufferPtr() + frame.offset, frame.length, timeout); + sendBuffer.reset(); + } + + /** + * Sends a close frame. + */ + public void sendCloseFrame(int code, String reason, int timeout) { + sendBuffer.reset(); + WebSocketSendBuffer.FrameInfo frame = sendBuffer.writeCloseFrame(code, reason); + try { + doSend(sendBuffer.getBufferPtr() + frame.offset, frame.length, timeout); + } finally { + sendBuffer.reset(); + } + } + + private void doSend(long ptr, int len, int timeout) { + long startTime = System.nanoTime(); + while (len > 0) { + int remainingTimeout = remainingTime(timeout, startTime); + ioWait(remainingTimeout, IOOperation.WRITE); + int sent = dieIfNegative(socket.send(ptr, len)); + while (socket.wantsTlsWrite()) { + remainingTimeout = remainingTime(timeout, startTime); + ioWait(remainingTimeout, IOOperation.WRITE); + dieIfNegative(socket.tlsIO(Socket.WRITE_FLAG)); + } + if (sent > 0) { + ptr += sent; + len -= sent; + } + } + } + + // === Receiving === + + /** + * Receives and processes WebSocket frames. + * + * @param handler frame handler callback + * @param timeout timeout in milliseconds + * @return true if a frame was received, false on timeout + */ + public boolean receiveFrame(WebSocketFrameHandler handler, int timeout) { + checkConnected(); + + // First, try to parse any data already in buffer + Boolean result = tryParseFrame(handler); + if (result != null) { + return result; + } + + // Need more data + long startTime = System.nanoTime(); + while (true) { + int remainingTimeout = remainingTime(timeout, startTime); + if (remainingTimeout <= 0) { + return false; // Timeout + } + + // Ensure buffer has space + if (recvPos >= recvBufSize - 1024) { + growRecvBuffer(); + } + + int bytesRead = recvOrTimeout(recvBufPtr + recvPos, recvBufSize - recvPos, remainingTimeout); + if (bytesRead <= 0) { + return false; // Timeout + } + recvPos += bytesRead; + + result = tryParseFrame(handler); + if (result != null) { + return result; + } + } + } + + /** + * Receives frame with default timeout. + */ + public boolean receiveFrame(WebSocketFrameHandler handler) { + return receiveFrame(handler, defaultTimeout); + } + + /** + * Non-blocking attempt to receive a WebSocket frame. + * Returns immediately if no complete frame is available. + * + * @param handler frame handler callback + * @return true if a frame was received, false if no data available + */ + public boolean tryReceiveFrame(WebSocketFrameHandler handler) { + checkConnected(); + + // First, try to parse any data already in buffer + Boolean result = tryParseFrame(handler); + if (result != null) { + return result; + } + + // Try one non-blocking recv + if (recvPos >= recvBufSize - 1024) { + growRecvBuffer(); + } + + int n = socket.recv(recvBufPtr + recvPos, recvBufSize - recvPos); + if (n < 0) { + throw new HttpClientException("peer disconnect [errno=").errno(nf.errno()).put(']'); + } + if (n == 0) { + return false; // No data available + } + recvPos += n; + + // Try to parse again + result = tryParseFrame(handler); + return result != null && result; + } + + private Boolean tryParseFrame(WebSocketFrameHandler handler) { + if (recvPos <= recvReadPos) { + return null; // No data + } + + frameParser.reset(); + int consumed = frameParser.parse(recvBufPtr + recvReadPos, recvBufPtr + recvPos); + + if (frameParser.getState() == WebSocketFrameParser.STATE_NEED_MORE || + frameParser.getState() == WebSocketFrameParser.STATE_NEED_PAYLOAD) { + return null; // Need more data + } + + if (frameParser.getState() == WebSocketFrameParser.STATE_ERROR) { + throw new HttpClientException("WebSocket frame parse error: ") + .put(WebSocketCloseCode.describe(frameParser.getErrorCode())); + } + + if (frameParser.getState() == WebSocketFrameParser.STATE_COMPLETE) { + long payloadPtr = recvBufPtr + recvReadPos + frameParser.getHeaderSize(); + int payloadLen = (int) frameParser.getPayloadLength(); + + // Unmask if needed (server frames should not be masked) + if (frameParser.isMasked()) { + frameParser.unmaskPayload(payloadPtr, payloadLen); + } + + // Handle frame by opcode + int opcode = frameParser.getOpcode(); + switch (opcode) { + case WebSocketOpcode.PING: + // Auto-respond with pong + sendPongFrame(payloadPtr, payloadLen); + if (handler != null) { + handler.onPing(payloadPtr, payloadLen); + } + break; + case WebSocketOpcode.PONG: + if (handler != null) { + handler.onPong(payloadPtr, payloadLen); + } + break; + case WebSocketOpcode.CLOSE: + upgraded = false; + if (handler != null) { + int closeCode = 0; + String reason = null; + if (payloadLen >= 2) { + closeCode = ((Unsafe.getUnsafe().getByte(payloadPtr) & 0xFF) << 8) + | (Unsafe.getUnsafe().getByte(payloadPtr + 1) & 0xFF); + if (payloadLen > 2) { + byte[] reasonBytes = new byte[payloadLen - 2]; + for (int i = 0; i < reasonBytes.length; i++) { + reasonBytes[i] = Unsafe.getUnsafe().getByte(payloadPtr + 2 + i); + } + reason = new String(reasonBytes, StandardCharsets.UTF_8); + } + } + handler.onClose(closeCode, reason); + } + break; + case WebSocketOpcode.BINARY: + if (handler != null) { + handler.onBinaryMessage(payloadPtr, payloadLen); + } + break; + case WebSocketOpcode.TEXT: + if (handler != null) { + handler.onTextMessage(payloadPtr, payloadLen); + } + break; + } + + // Advance read position + recvReadPos += consumed; + + // Compact buffer if needed + compactRecvBuffer(); + + return true; + } + + return false; + } + + private void sendPongFrame(long payloadPtr, int payloadLen) { + try { + sendBuffer.reset(); + WebSocketSendBuffer.FrameInfo frame = sendBuffer.writePongFrame(payloadPtr, payloadLen); + doSend(sendBuffer.getBufferPtr() + frame.offset, frame.length, 1000); // Short timeout for pong + sendBuffer.reset(); + } catch (Exception e) { + LOG.error("Failed to send pong: {}", e.getMessage()); + } + } + + private void compactRecvBuffer() { + if (recvReadPos > 0) { + int remaining = recvPos - recvReadPos; + if (remaining > 0) { + Vect.memmove(recvBufPtr, recvBufPtr + recvReadPos, remaining); + } + recvPos = remaining; + recvReadPos = 0; + } + } + + private void growRecvBuffer() { + int newSize = recvBufSize * 2; + recvBufPtr = Unsafe.realloc(recvBufPtr, recvBufSize, newSize, MemoryTag.NATIVE_DEFAULT); + recvBufSize = newSize; + } + + // === Socket I/O helpers === + + private int recvOrDie(long ptr, int len, int timeout) { + long startTime = System.nanoTime(); + int n = dieIfNegative(socket.recv(ptr, len)); + if (n == 0) { + ioWait(remainingTime(timeout, startTime), IOOperation.READ); + n = dieIfNegative(socket.recv(ptr, len)); + } + return n; + } + + private int recvOrTimeout(long ptr, int len, int timeout) { + long startTime = System.nanoTime(); + int n = socket.recv(ptr, len); + if (n < 0) { + throw new HttpClientException("peer disconnect [errno=").errno(nf.errno()).put(']'); + } + if (n == 0) { + try { + ioWait(timeout, IOOperation.READ); + } catch (HttpClientException e) { + // Timeout + return 0; + } + n = socket.recv(ptr, len); + if (n < 0) { + throw new HttpClientException("peer disconnect [errno=").errno(nf.errno()).put(']'); + } + } + return n; + } + + private int dieIfNegative(int byteCount) { + if (byteCount < 0) { + throw new HttpClientException("peer disconnect [errno=").errno(nf.errno()).put(']'); + } + return byteCount; + } + + private int remainingTime(int timeoutMillis, long startTimeNanos) { + timeoutMillis -= (int) NANOSECONDS.toMillis(System.nanoTime() - startTimeNanos); + if (timeoutMillis <= 0) { + throw new HttpClientException("timed out [errno=").errno(nf.errno()).put(']'); + } + return timeoutMillis; + } + + protected void dieWaiting(int n) { + if (n == 1) { + return; + } + if (n == 0) { + throw new HttpClientException("timed out [errno=").put(nf.errno()).put(']'); + } + throw new HttpClientException("queue error [errno=").put(nf.errno()).put(']'); + } + + private void checkConnected() { + if (closed) { + throw new HttpClientException("WebSocket client is closed"); + } + if (!upgraded) { + throw new HttpClientException("WebSocket not connected or upgraded"); + } + } + + // === State === + + /** + * Returns whether the WebSocket is connected and upgraded. + */ + public boolean isConnected() { + return upgraded && !closed && !socket.isClosed(); + } + + /** + * Returns the connected host. + */ + public CharSequence getHost() { + return host; + } + + /** + * Returns the connected port. + */ + public int getPort() { + return port; + } + + // === Platform-specific I/O === + + /** + * Waits for I/O readiness using platform-specific mechanism. + * + * @param timeout timeout in milliseconds + * @param op I/O operation (READ or WRITE) + */ + protected abstract void ioWait(int timeout, int op); + + /** + * Sets up platform-specific I/O wait mechanism after connection. + */ + protected abstract void setupIoWait(); +} diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientFactory.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientFactory.java new file mode 100644 index 0000000..9284786 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientFactory.java @@ -0,0 +1,137 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.http.client; + +import io.questdb.client.ClientTlsConfiguration; +import io.questdb.client.DefaultHttpClientConfiguration; +import io.questdb.client.HttpClientConfiguration; +import io.questdb.client.network.JavaTlsClientSocketFactory; +import io.questdb.client.network.PlainSocketFactory; +import io.questdb.client.network.SocketFactory; +import io.questdb.client.std.Os; + +/** + * Factory for creating platform-specific {@link WebSocketClient} instances. + *

+ * Usage: + * 

+ * // Plain text connection
+ * WebSocketClient client = WebSocketClientFactory.newPlainTextInstance();
+ *
+ * // TLS connection
+ * WebSocketClient client = WebSocketClientFactory.newTlsInstance(config, tlsConfig);
+ *
+ * // Connect and upgrade
+ * client.connect("localhost", 9000);
+ * client.upgrade("/ws");
+ *
+ * // Send data
+ * WebSocketSendBuffer buf = client.getSendBuffer();
+ * buf.beginBinaryFrame();
+ * buf.putLong(data);
+ * WebSocketSendBuffer.FrameInfo frame = buf.endBinaryFrame();
+ * client.sendFrame(frame);
+ * buf.reset();
+ *
+ * // Receive data
+ * client.receiveFrame(handler);
+ *
+ * client.close();
+ *
+ */ +public class WebSocketClientFactory { + + /** + * Creates a new WebSocket client with insecure TLS (no certificate validation). + *

+ * WARNING: Only use this for testing. Production code should use proper TLS validation. + * + * @return a new WebSocket client with insecure TLS + */ + public static WebSocketClient newInsecureTlsInstance() { + return newInstance(DefaultHttpClientConfiguration.INSTANCE, JavaTlsClientSocketFactory.INSECURE_NO_VALIDATION); + } + + /** + * Creates a new WebSocket client with the specified configuration and socket factory. + * + * @param configuration the HTTP client configuration + * @param socketFactory the socket factory for creating sockets + * @return a new platform-specific WebSocket client + */ + public static WebSocketClient newInstance(HttpClientConfiguration configuration, SocketFactory socketFactory) { + switch (Os.type) { + case Os.LINUX: + return new WebSocketClientLinux(configuration, socketFactory); + case Os.DARWIN: + case Os.FREEBSD: + return new WebSocketClientOsx(configuration, socketFactory); + case Os.WINDOWS: + return new WebSocketClientWindows(configuration, socketFactory); + default: + throw new UnsupportedOperationException("Unsupported platform: " + Os.type); + } + } + + /** + * Creates a new plain text WebSocket client with default configuration. + * + * @return a new plain text WebSocket client + */ + public static WebSocketClient newPlainTextInstance() { + return newPlainTextInstance(DefaultHttpClientConfiguration.INSTANCE); + } + + /** + * Creates a new plain text WebSocket client with the specified configuration. + * + * @param configuration the HTTP client configuration + * @return a new plain text WebSocket client + */ + public static WebSocketClient newPlainTextInstance(HttpClientConfiguration configuration) { + return newInstance(configuration, PlainSocketFactory.INSTANCE); + } + + /** + * Creates a new TLS WebSocket client with the specified configuration. + * + * @param configuration the HTTP client configuration + * @param tlsConfig the TLS configuration + * @return a new TLS WebSocket client + */ + public static WebSocketClient newTlsInstance(HttpClientConfiguration configuration, ClientTlsConfiguration tlsConfig) { + return newInstance(configuration, new JavaTlsClientSocketFactory(tlsConfig)); + } + + /** + * Creates a new TLS WebSocket client with default HTTP configuration. + * + * @param tlsConfig the TLS configuration + * @return a new TLS WebSocket client + */ + public static WebSocketClient newTlsInstance(ClientTlsConfiguration tlsConfig) { + return newTlsInstance(DefaultHttpClientConfiguration.INSTANCE, tlsConfig); + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientLinux.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientLinux.java new file mode 100644 index 0000000..f4ac6ba --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientLinux.java @@ -0,0 +1,71 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.http.client; + +import io.questdb.client.HttpClientConfiguration; +import io.questdb.client.network.Epoll; +import io.questdb.client.network.EpollAccessor; +import io.questdb.client.network.IOOperation; +import io.questdb.client.network.SocketFactory; +import io.questdb.client.std.Misc; + +/** + * Linux-specific WebSocket client using epoll for I/O waiting. + */ +public class WebSocketClientLinux extends WebSocketClient { + private Epoll epoll; + + public WebSocketClientLinux(HttpClientConfiguration configuration, SocketFactory socketFactory) { + super(configuration, socketFactory); + epoll = new Epoll( + configuration.getEpollFacade(), + configuration.getWaitQueueCapacity() + ); + } + + @Override + public void close() { + super.close(); + epoll = Misc.free(epoll); + } + + @Override + protected void ioWait(int timeout, int op) { + final int event = op == IOOperation.WRITE ? EpollAccessor.EPOLLOUT : EpollAccessor.EPOLLIN; + if (epoll.control(socket.getFd(), 0, EpollAccessor.EPOLL_CTL_MOD, event) < 0) { + throw new HttpClientException("internal error: epoll_ctl failure [op=").put(op) + .put(", errno=").put(nf.errno()) + .put(']'); + } + dieWaiting(epoll.poll(timeout)); + } + + @Override + protected void setupIoWait() { + if (epoll.control(socket.getFd(), 0, EpollAccessor.EPOLL_CTL_ADD, EpollAccessor.EPOLLOUT) < 0) { + throw new HttpClientException("internal error: epoll_ctl failure [cmd=add, errno=").put(nf.errno()).put(']'); + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientOsx.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientOsx.java new file mode 100644 index 0000000..d34df7c --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientOsx.java @@ -0,0 +1,75 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.http.client; + +import io.questdb.client.HttpClientConfiguration; +import io.questdb.client.network.IOOperation; +import io.questdb.client.network.Kqueue; +import io.questdb.client.network.SocketFactory; +import io.questdb.client.std.Misc; + +/** + * macOS-specific WebSocket client using kqueue for I/O waiting. + */ +public class WebSocketClientOsx extends WebSocketClient { + private Kqueue kqueue; + + public WebSocketClientOsx(HttpClientConfiguration configuration, SocketFactory socketFactory) { + super(configuration, socketFactory); + this.kqueue = new Kqueue( + configuration.getKQueueFacade(), + configuration.getWaitQueueCapacity() + ); + } + + @Override + public void close() { + super.close(); + this.kqueue = Misc.free(kqueue); + } + + @Override + protected void ioWait(int timeout, int op) { + kqueue.setWriteOffset(0); + if (op == IOOperation.READ) { + kqueue.readFD(socket.getFd(), 0); + } else { + kqueue.writeFD(socket.getFd(), 0); + } + + // 1 = always one FD, we are a single threaded network client + if (kqueue.register(1) != 0) { + throw new HttpClientException("could not register with kqueue [op=").put(op) + .put(", errno=").errno(nf.errno()) + .put(']'); + } + dieWaiting(kqueue.poll(timeout)); + } + + @Override + protected void setupIoWait() { + // no-op on macOS + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientWindows.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientWindows.java new file mode 100644 index 0000000..cdaec88 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientWindows.java @@ -0,0 +1,74 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.http.client; + +import io.questdb.client.HttpClientConfiguration; +import io.questdb.client.network.FDSet; +import io.questdb.client.network.IOOperation; +import io.questdb.client.network.SelectFacade; +import io.questdb.client.network.SocketFactory; +import io.questdb.client.std.Misc; + +/** + * Windows-specific WebSocket client using select for I/O waiting. + */ +public class WebSocketClientWindows extends WebSocketClient { + private final SelectFacade sf; + private FDSet fdSet; + + public WebSocketClientWindows(HttpClientConfiguration configuration, SocketFactory socketFactory) { + super(configuration, socketFactory); + this.fdSet = new FDSet(configuration.getWaitQueueCapacity()); + this.sf = configuration.getSelectFacade(); + } + + @Override + public void close() { + super.close(); + this.fdSet = Misc.free(fdSet); + } + + @Override + protected void ioWait(int timeout, int op) { + final long readAddr; + final long writeAddr; + fdSet.clear(); + fdSet.add(socket.getFd()); + fdSet.setCount(1); + if (op == IOOperation.READ) { + readAddr = fdSet.address(); + writeAddr = 0; + } else { + readAddr = 0; + writeAddr = fdSet.address(); + } + dieWaiting(sf.select(readAddr, writeAddr, 0, timeout)); + } + + @Override + protected void setupIoWait() { + // no-op on Windows + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketFrameHandler.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketFrameHandler.java new file mode 100644 index 0000000..aff429d --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketFrameHandler.java @@ -0,0 +1,93 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.http.client; + +/** + * Callback interface for handling received WebSocket frames. + *

+ * Implementations should process received data efficiently and avoid blocking, + * as callbacks are invoked on the I/O thread. + *

+ * Thread safety: Callbacks are invoked from the thread that called receiveFrame(). + * Implementations must handle their own synchronization if accessed from multiple threads. + */ +public interface WebSocketFrameHandler { + + /** + * Called when a binary frame is received. + * + * @param payloadPtr pointer to the payload data in native memory + * @param payloadLen length of the payload in bytes + */ + void onBinaryMessage(long payloadPtr, int payloadLen); + + /** + * Called when a text frame is received. + *

+ * Default implementation does nothing. Override if text frames need handling. + * + * @param payloadPtr pointer to the UTF-8 encoded payload in native memory + * @param payloadLen length of the payload in bytes + */ + default void onTextMessage(long payloadPtr, int payloadLen) { + // Default: ignore text frames + } + + /** + * Called when a close frame is received from the server. + *

+ * After this callback, the connection will be closed. The handler should + * perform any necessary cleanup. + * + * @param code the close status code (e.g., 1000 for normal closure) + * @param reason the close reason (may be null or empty) + */ + void onClose(int code, String reason); + + /** + * Called when a ping frame is received. + *

+ * Default implementation does nothing. The WebSocketClient automatically + * sends a pong response, so this callback is for informational purposes only. + * + * @param payloadPtr pointer to the ping payload in native memory + * @param payloadLen length of the payload in bytes + */ + default void onPing(long payloadPtr, int payloadLen) { + // Default: handled automatically by client + } + + /** + * Called when a pong frame is received. + *

+ * Default implementation does nothing. + * + * @param payloadPtr pointer to the pong payload in native memory + * @param payloadLen length of the payload in bytes + */ + default void onPong(long payloadPtr, int payloadLen) { + // Default: ignore pong frames + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java new file mode 100644 index 0000000..0eea869 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java @@ -0,0 +1,582 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.http.client; + +import io.questdb.client.cutlass.ilpv4.websocket.WebSocketFrameWriter; +import io.questdb.client.cutlass.ilpv4.websocket.WebSocketOpcode; +import io.questdb.client.cutlass.ilpv4.client.IlpBufferWriter; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Numbers; +import io.questdb.client.std.QuietCloseable; +import io.questdb.client.std.Rnd; +import io.questdb.client.std.Unsafe; +import io.questdb.client.std.Vect; + +/** + * Zero-GC WebSocket send buffer that implements {@link ArrayBufferAppender} for direct + * payload writing. Manages native memory with safe growth and handles WebSocket frame + * building (reserve header -> write payload -> patch header -> mask). + *

+ * Usage pattern: + * 

+ * buffer.beginBinaryFrame();
+ * // Write payload using ArrayBufferAppender methods
+ * buffer.putLong(value);
+ * buffer.putBlockOfBytes(ptr, len);
+ * // Finish frame and get send info
+ * FrameInfo frame = buffer.endBinaryFrame();
+ * // Send frame using socket
+ * socket.send(buffer.getBufferPtr() + frame.offset, frame.length);
+ * buffer.reset();
+ *
+ *

+ * Thread safety: This class is NOT thread-safe. Each connection should have its own buffer. + */ +public class WebSocketSendBuffer implements IlpBufferWriter, QuietCloseable { + + // Maximum header size: 2 (base) + 8 (64-bit length) + 4 (mask key) + private static final int MAX_HEADER_SIZE = 14; + + private static final int DEFAULT_INITIAL_CAPACITY = 65536; + private static final int MAX_BUFFER_SIZE = Integer.MAX_VALUE - 8; // Leave room for alignment + + private long bufPtr; + private int bufCapacity; + private int writePos; // Current write position (offset from bufPtr) + private int frameStartOffset; // Where current frame's reserved header starts + private int payloadStartOffset; // Where payload begins (frameStart + MAX_HEADER_SIZE) + + private final Rnd rnd; + private final int maxBufferSize; + private final FrameInfo frameInfo = new FrameInfo(); + + /** + * Creates a new WebSocket send buffer with default initial capacity. + */ + public WebSocketSendBuffer() { + this(DEFAULT_INITIAL_CAPACITY, MAX_BUFFER_SIZE); + } + + /** + * Creates a new WebSocket send buffer with specified initial capacity. + * + * @param initialCapacity initial buffer size in bytes + */ + public WebSocketSendBuffer(int initialCapacity) { + this(initialCapacity, MAX_BUFFER_SIZE); + } + + /** + * Creates a new WebSocket send buffer with specified initial and max capacity. + * + * @param initialCapacity initial buffer size in bytes + * @param maxBufferSize maximum buffer size in bytes + */ + public WebSocketSendBuffer(int initialCapacity, int maxBufferSize) { + this.bufCapacity = Math.max(initialCapacity, MAX_HEADER_SIZE * 2); + this.maxBufferSize = maxBufferSize; + this.bufPtr = Unsafe.malloc(bufCapacity, MemoryTag.NATIVE_DEFAULT); + this.writePos = 0; + this.frameStartOffset = 0; + this.payloadStartOffset = 0; + this.rnd = new Rnd(System.nanoTime(), System.currentTimeMillis()); + } + + @Override + public void close() { + if (bufPtr != 0) { + Unsafe.free(bufPtr, bufCapacity, MemoryTag.NATIVE_DEFAULT); + bufPtr = 0; + bufCapacity = 0; + } + } + + // === Buffer Management === + + /** + * Ensures the buffer has capacity for the specified number of additional bytes. + * May reallocate the buffer if necessary. + * + * @param additionalBytes number of additional bytes needed + */ + @Override + public void ensureCapacity(int additionalBytes) { + long requiredCapacity = (long) writePos + additionalBytes; + if (requiredCapacity > bufCapacity) { + grow(requiredCapacity); + } + } + + private void grow(long requiredCapacity) { + if (requiredCapacity > maxBufferSize) { + throw new HttpClientException("WebSocket buffer size exceeded maximum [required=") + .put(requiredCapacity) + .put(", max=") + .put(maxBufferSize) + .put(']'); + } + int newCapacity = (int) Math.min( + Numbers.ceilPow2((int) requiredCapacity), + maxBufferSize + ); + bufPtr = Unsafe.realloc(bufPtr, bufCapacity, newCapacity, MemoryTag.NATIVE_DEFAULT); + bufCapacity = newCapacity; + } + + // === ArrayBufferAppender Implementation === + + @Override + public void putByte(byte b) { + ensureCapacity(1); + Unsafe.getUnsafe().putByte(bufPtr + writePos, b); + writePos++; + } + + @Override + public void putInt(int value) { + ensureCapacity(4); + Unsafe.getUnsafe().putInt(bufPtr + writePos, value); + writePos += 4; + } + + @Override + public void putLong(long value) { + ensureCapacity(8); + Unsafe.getUnsafe().putLong(bufPtr + writePos, value); + writePos += 8; + } + + @Override + public void putDouble(double value) { + ensureCapacity(8); + Unsafe.getUnsafe().putDouble(bufPtr + writePos, value); + writePos += 8; + } + + @Override + public void putBlockOfBytes(long from, long len) { + if (len <= 0) { + return; + } + ensureCapacity((int) len); + Vect.memcpy(bufPtr + writePos, from, len); + writePos += (int) len; + } + + // === Additional write methods (not in ArrayBufferAppender but useful) === + + /** + * Writes a short value in little-endian format. + */ + public void putShort(short value) { + ensureCapacity(2); + Unsafe.getUnsafe().putShort(bufPtr + writePos, value); + writePos += 2; + } + + /** + * Writes a float value. + */ + public void putFloat(float value) { + ensureCapacity(4); + Unsafe.getUnsafe().putFloat(bufPtr + writePos, value); + writePos += 4; + } + + /** + * Writes a long value in big-endian format. + */ + public void putLongBE(long value) { + ensureCapacity(8); + Unsafe.getUnsafe().putLong(bufPtr + writePos, Long.reverseBytes(value)); + writePos += 8; + } + + /** + * Writes raw bytes from a byte array. + */ + public void putBytes(byte[] bytes, int offset, int length) { + if (length <= 0) { + return; + } + ensureCapacity(length); + for (int i = 0; i < length; i++) { + Unsafe.getUnsafe().putByte(bufPtr + writePos + i, bytes[offset + i]); + } + writePos += length; + } + + /** + * Writes an ASCII string. + */ + public void putAscii(CharSequence cs) { + if (cs == null) { + return; + } + int len = cs.length(); + ensureCapacity(len); + for (int i = 0; i < len; i++) { + Unsafe.getUnsafe().putByte(bufPtr + writePos + i, (byte) cs.charAt(i)); + } + writePos += len; + } + + // === IlpBufferWriter Implementation === + + /** + * Writes an unsigned variable-length integer (LEB128 encoding). + */ + @Override + public void putVarint(long value) { + while (value > 0x7F) { + putByte((byte) ((value & 0x7F) | 0x80)); + value >>>= 7; + } + putByte((byte) value); + } + + /** + * Writes a length-prefixed UTF-8 string. + */ + @Override + public void putString(String value) { + if (value == null || value.isEmpty()) { + putVarint(0); + return; + } + int utf8Len = IlpBufferWriter.utf8Length(value); + putVarint(utf8Len); + putUtf8(value); + } + + /** + * Writes UTF-8 encoded bytes directly without length prefix. + */ + @Override + public void putUtf8(String value) { + if (value == null || value.isEmpty()) { + return; + } + for (int i = 0, n = value.length(); i < n; i++) { + char c = value.charAt(i); + if (c < 0x80) { + putByte((byte) c); + } else if (c < 0x800) { + putByte((byte) (0xC0 | (c >> 6))); + putByte((byte) (0x80 | (c & 0x3F))); + } else if (c >= 0xD800 && c <= 0xDBFF && i + 1 < n) { + char c2 = value.charAt(++i); + int codePoint = 0x10000 + ((c - 0xD800) << 10) + (c2 - 0xDC00); + putByte((byte) (0xF0 | (codePoint >> 18))); + putByte((byte) (0x80 | ((codePoint >> 12) & 0x3F))); + putByte((byte) (0x80 | ((codePoint >> 6) & 0x3F))); + putByte((byte) (0x80 | (codePoint & 0x3F))); + } else { + putByte((byte) (0xE0 | (c >> 12))); + putByte((byte) (0x80 | ((c >> 6) & 0x3F))); + putByte((byte) (0x80 | (c & 0x3F))); + } + } + } + + /** + * Patches an int value at the specified offset. + */ + @Override + public void patchInt(int offset, int value) { + Unsafe.getUnsafe().putInt(bufPtr + offset, value); + } + + /** + * Skips the specified number of bytes, advancing the position. + */ + @Override + public void skip(int bytes) { + ensureCapacity(bytes); + writePos += bytes; + } + + /** + * Gets the current write position (number of bytes written). + */ + @Override + public int getPosition() { + return writePos; + } + + // === Frame Building === + + /** + * Begins a new binary WebSocket frame. Reserves space for the maximum header size. + * After calling this method, use ArrayBufferAppender methods to write the payload. + */ + public void beginBinaryFrame() { + beginFrame(WebSocketOpcode.BINARY); + } + + /** + * Begins a new text WebSocket frame. Reserves space for the maximum header size. + */ + public void beginTextFrame() { + beginFrame(WebSocketOpcode.TEXT); + } + + /** + * Begins a new WebSocket frame with the specified opcode. + * + * @param opcode the frame opcode + */ + public void beginFrame(int opcode) { + frameStartOffset = writePos; + // Reserve maximum header space + ensureCapacity(MAX_HEADER_SIZE); + writePos += MAX_HEADER_SIZE; + payloadStartOffset = writePos; + } + + /** + * Finishes the current binary frame, writing the header and applying masking. + * Returns information about where to find the complete frame in the buffer. + *

+ * IMPORTANT: Only call this after all payload writes are complete. The buffer + * pointer is stable after this call (no more reallocations for this frame). + * + * @return frame info containing offset and length for sending + */ + public FrameInfo endBinaryFrame() { + return endFrame(WebSocketOpcode.BINARY); + } + + /** + * Finishes the current text frame, writing the header and applying masking. + */ + public FrameInfo endTextFrame() { + return endFrame(WebSocketOpcode.TEXT); + } + + /** + * Finishes the current frame with the specified opcode. + * + * @param opcode the frame opcode + * @return frame info containing offset and length for sending + */ + public FrameInfo endFrame(int opcode) { + int payloadLen = writePos - payloadStartOffset; + + // Calculate actual header size (with mask key for client frames) + int actualHeaderSize = WebSocketFrameWriter.headerSize(payloadLen, true); + int unusedSpace = MAX_HEADER_SIZE - actualHeaderSize; + int actualFrameStart = frameStartOffset + unusedSpace; + + // Generate mask key + int maskKey = rnd.nextInt(); + + // Write header at actual position (after unused space) + WebSocketFrameWriter.writeHeader(bufPtr + actualFrameStart, true, opcode, payloadLen, maskKey); + + // Apply mask to payload + if (payloadLen > 0) { + WebSocketFrameWriter.maskPayload(bufPtr + payloadStartOffset, payloadLen, maskKey); + } + + return frameInfo.set(actualFrameStart, actualHeaderSize + payloadLen); + } + + /** + * Writes a complete ping frame (control frame, no masking needed for server). + * Note: Client frames MUST be masked per RFC 6455. This writes a masked ping. + * + * @return frame info for sending + */ + public FrameInfo writePingFrame() { + return writePingFrame(0, 0); + } + + /** + * Writes a complete ping frame with payload. + * + * @param payloadPtr pointer to ping payload + * @param payloadLen length of payload (max 125 bytes for control frames) + * @return frame info for sending + */ + public FrameInfo writePingFrame(long payloadPtr, int payloadLen) { + if (payloadLen > 125) { + throw new HttpClientException("Ping payload too large [len=").put(payloadLen).put(']'); + } + + int frameStart = writePos; + int headerSize = WebSocketFrameWriter.headerSize(payloadLen, true); + ensureCapacity(headerSize + payloadLen); + + int maskKey = rnd.nextInt(); + int written = WebSocketFrameWriter.writeHeader(bufPtr + writePos, true, WebSocketOpcode.PING, payloadLen, maskKey); + writePos += written; + + if (payloadLen > 0) { + Vect.memcpy(bufPtr + writePos, payloadPtr, payloadLen); + WebSocketFrameWriter.maskPayload(bufPtr + writePos, payloadLen, maskKey); + writePos += payloadLen; + } + + return frameInfo.set(frameStart, headerSize + payloadLen); + } + + /** + * Writes a complete pong frame. + * + * @param payloadPtr pointer to pong payload (should match received ping) + * @param payloadLen length of payload + * @return frame info for sending + */ + public FrameInfo writePongFrame(long payloadPtr, int payloadLen) { + if (payloadLen > 125) { + throw new HttpClientException("Pong payload too large [len=").put(payloadLen).put(']'); + } + + int frameStart = writePos; + int headerSize = WebSocketFrameWriter.headerSize(payloadLen, true); + ensureCapacity(headerSize + payloadLen); + + int maskKey = rnd.nextInt(); + int written = WebSocketFrameWriter.writeHeader(bufPtr + writePos, true, WebSocketOpcode.PONG, payloadLen, maskKey); + writePos += written; + + if (payloadLen > 0) { + Vect.memcpy(bufPtr + writePos, payloadPtr, payloadLen); + WebSocketFrameWriter.maskPayload(bufPtr + writePos, payloadLen, maskKey); + writePos += payloadLen; + } + + return frameInfo.set(frameStart, headerSize + payloadLen); + } + + /** + * Writes a complete close frame. + * + * @param code close status code (e.g., 1000 for normal closure) + * @param reason optional reason string (may be null) + * @return frame info for sending + */ + public FrameInfo writeCloseFrame(int code, String reason) { + int payloadLen = 2; // status code + byte[] reasonBytes = null; + if (reason != null && !reason.isEmpty()) { + reasonBytes = reason.getBytes(java.nio.charset.StandardCharsets.UTF_8); + payloadLen += reasonBytes.length; + } + + if (payloadLen > 125) { + throw new HttpClientException("Close payload too large [len=").put(payloadLen).put(']'); + } + + int frameStart = writePos; + int headerSize = WebSocketFrameWriter.headerSize(payloadLen, true); + ensureCapacity(headerSize + payloadLen); + + int maskKey = rnd.nextInt(); + int written = WebSocketFrameWriter.writeHeader(bufPtr + writePos, true, WebSocketOpcode.CLOSE, payloadLen, maskKey); + writePos += written; + + // Write status code (big-endian) + long payloadStart = bufPtr + writePos; + Unsafe.getUnsafe().putByte(payloadStart, (byte) ((code >> 8) & 0xFF)); + Unsafe.getUnsafe().putByte(payloadStart + 1, (byte) (code & 0xFF)); + writePos += 2; + + // Write reason if present + if (reasonBytes != null) { + for (byte reasonByte : reasonBytes) { + Unsafe.getUnsafe().putByte(bufPtr + writePos++, reasonByte); + } + } + + // Mask the payload (including status code and reason) + WebSocketFrameWriter.maskPayload(payloadStart, payloadLen, maskKey); + + return frameInfo.set(frameStart, headerSize + payloadLen); + } + + // === Buffer State === + + /** + * Gets the buffer pointer. Only use this for reading after frame is complete. + */ + public long getBufferPtr() { + return bufPtr; + } + + /** + * Gets the current buffer capacity. + */ + public int getCapacity() { + return bufCapacity; + } + + /** + * Gets the current write position (total bytes written since last reset). + */ + public int getWritePos() { + return writePos; + } + + /** + * Gets the payload length of the current frame being built. + */ + public int getCurrentPayloadLength() { + return writePos - payloadStartOffset; + } + + /** + * Resets the buffer for reuse. Does not deallocate memory. + */ + public void reset() { + writePos = 0; + frameStartOffset = 0; + payloadStartOffset = 0; + } + + /** + * Information about a completed WebSocket frame's location in the buffer. + * This class is mutable and reused to avoid allocations. Callers must + * extract values before calling any end*Frame() method again. + */ + public static final class FrameInfo { + /** + * Offset from buffer start where the frame begins. + */ + public int offset; + + /** + * Total length of the frame (header + payload). + */ + public int length; + + FrameInfo set(int offset, int length) { + this.offset = offset; + this.length = length; + return this; + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/GlobalSymbolDictionary.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/GlobalSymbolDictionary.java new file mode 100644 index 0000000..743c029 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/GlobalSymbolDictionary.java @@ -0,0 +1,173 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.ilpv4.client; + +import io.questdb.client.std.CharSequenceIntHashMap; +import io.questdb.client.std.ObjList; + +/** + * Global symbol dictionary that maps symbol strings to sequential integer IDs. + *

+ * This dictionary is shared across all tables and columns within a client instance. + * IDs are assigned sequentially starting from 0, ensuring contiguous ID space. + *

+ * Thread safety: This class is NOT thread-safe. External synchronization is required + * if accessed from multiple threads. + */ +public class GlobalSymbolDictionary { + + private final CharSequenceIntHashMap symbolToId; + private final ObjList idToSymbol; + + public GlobalSymbolDictionary() { + this(64); // Default initial capacity + } + + public GlobalSymbolDictionary(int initialCapacity) { + this.symbolToId = new CharSequenceIntHashMap(initialCapacity); + this.idToSymbol = new ObjList<>(initialCapacity); + } + + /** + * Gets or adds a symbol to the dictionary. + *

+ * If the symbol already exists, returns its existing ID. + * If the symbol is new, assigns the next sequential ID and returns it. + * + * @param symbol the symbol string (must not be null) + * @return the global ID for this symbol (>= 0) + * @throws IllegalArgumentException if symbol is null + */ + public int getOrAddSymbol(String symbol) { + if (symbol == null) { + throw new IllegalArgumentException("symbol cannot be null"); + } + + int existingId = symbolToId.get(symbol); + if (existingId != CharSequenceIntHashMap.NO_ENTRY_VALUE) { + return existingId; + } + + // Assign new ID + int newId = idToSymbol.size(); + symbolToId.put(symbol, newId); + idToSymbol.add(symbol); + return newId; + } + + /** + * Gets the symbol string for a given ID. + * + * @param id the symbol ID + * @return the symbol string + * @throws IndexOutOfBoundsException if id is out of range + */ + public String getSymbol(int id) { + if (id < 0 || id >= idToSymbol.size()) { + throw new IndexOutOfBoundsException("Invalid symbol ID: " + id + ", dictionary size: " + idToSymbol.size()); + } + return idToSymbol.getQuick(id); + } + + /** + * Gets the ID for an existing symbol, or -1 if not found. + * + * @param symbol the symbol string + * @return the symbol ID, or -1 if not in dictionary + */ + public int getId(String symbol) { + if (symbol == null) { + return -1; + } + int id = symbolToId.get(symbol); + return id == CharSequenceIntHashMap.NO_ENTRY_VALUE ? -1 : id; + } + + /** + * Returns the number of symbols in the dictionary. + * + * @return dictionary size + */ + public int size() { + return idToSymbol.size(); + } + + /** + * Checks if the dictionary is empty. + * + * @return true if no symbols have been added + */ + public boolean isEmpty() { + return idToSymbol.size() == 0; + } + + /** + * Checks if the dictionary contains the given symbol. + * + * @param symbol the symbol to check + * @return true if the symbol exists in the dictionary + */ + public boolean contains(String symbol) { + return symbol != null && symbolToId.get(symbol) != CharSequenceIntHashMap.NO_ENTRY_VALUE; + } + + /** + * Gets the symbols in the given ID range [fromId, toId). + *

+ * This is used to extract the delta for sending to the server. + * The range is inclusive of fromId and exclusive of toId. + * + * @param fromId start ID (inclusive) + * @param toId end ID (exclusive) + * @return array of symbols in the range, or empty array if range is invalid/empty + */ + public String[] getSymbolsInRange(int fromId, int toId) { + if (fromId < 0 || toId < fromId || fromId >= idToSymbol.size()) { + return new String[0]; + } + + int actualToId = Math.min(toId, idToSymbol.size()); + int count = actualToId - fromId; + if (count <= 0) { + return new String[0]; + } + + String[] result = new String[count]; + for (int i = 0; i < count; i++) { + result[i] = idToSymbol.getQuick(fromId + i); + } + return result; + } + + /** + * Clears all symbols from the dictionary. + *

+ * After clearing, the next symbol added will get ID 0. + */ + public void clear() { + symbolToId.clear(); + idToSymbol.clear(); + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/IlpBufferWriter.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/IlpBufferWriter.java new file mode 100644 index 0000000..1976cac --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/IlpBufferWriter.java @@ -0,0 +1,177 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.ilpv4.client; + +import io.questdb.client.cutlass.line.array.ArrayBufferAppender; + +/** + * Buffer writer interface for ILP v4 message encoding. + *

+ * This interface extends {@link ArrayBufferAppender} with additional methods + * required for encoding ILP v4 messages, including varint encoding, string + * handling, and buffer manipulation. + *

+ * Implementations include: + *

+ *

+ * All multi-byte values are written in little-endian format unless the method + * name explicitly indicates big-endian (e.g., {@link #putLongBE}). + */ +public interface IlpBufferWriter extends ArrayBufferAppender { + + // === Primitive writes (little-endian) === + + /** + * Writes a short (2 bytes, little-endian). + */ + void putShort(short value); + + /** + * Writes a float (4 bytes, little-endian). + */ + void putFloat(float value); + + // === Big-endian writes === + + /** + * Writes a long in big-endian byte order. + */ + void putLongBE(long value); + + // === Variable-length encoding === + + /** + * Writes an unsigned variable-length integer (LEB128 encoding). + *

+ * Each byte contains 7 bits of data with the high bit indicating + * whether more bytes follow. + */ + void putVarint(long value); + + // === String encoding === + + /** + * Writes a length-prefixed UTF-8 string. + *

+ * Format: varint length + UTF-8 bytes + * + * @param value the string to write (may be null or empty) + */ + void putString(String value); + + /** + * Writes UTF-8 encoded bytes directly without length prefix. + * + * @param value the string to encode (may be null or empty) + */ + void putUtf8(String value); + + // === Buffer manipulation === + + /** + * Patches an int value at the specified offset in the buffer. + *

+ * Used for updating length fields after writing content. + * + * @param offset the byte offset from buffer start + * @param value the int value to write + */ + void patchInt(int offset, int value); + + /** + * Skips the specified number of bytes, advancing the position. + *

+ * Used when data has been written directly to the buffer via + * {@link #getBufferPtr()}. + * + * @param bytes number of bytes to skip + */ + void skip(int bytes); + + /** + * Ensures the buffer has capacity for at least the specified + * additional bytes beyond the current position. + * + * @param additionalBytes number of additional bytes needed + */ + void ensureCapacity(int additionalBytes); + + /** + * Resets the buffer for reuse, setting the position to 0. + *

+ * Does not deallocate memory. + */ + void reset(); + + // === Buffer state === + + /** + * Returns the current write position (number of bytes written). + */ + int getPosition(); + + /** + * Returns the current buffer capacity in bytes. + */ + int getCapacity(); + + /** + * Returns the native memory pointer to the buffer start. + *

+ * The returned pointer is valid until the next buffer growth operation. + * Use with care and only for reading completed data. + */ + long getBufferPtr(); + + // === Utility === + + /** + * Returns the UTF-8 encoded length of a string. + * + * @param s the string (may be null) + * @return the number of bytes needed to encode the string as UTF-8 + */ + static int utf8Length(String s) { + if (s == null) return 0; + int len = 0; + for (int i = 0, n = s.length(); i < n; i++) { + char c = s.charAt(i); + if (c < 0x80) { + len++; + } else if (c < 0x800) { + len += 2; + } else if (c >= 0xD800 && c <= 0xDBFF && i + 1 < n) { + i++; + len += 4; + } else { + len += 3; + } + } + return len; + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/IlpV4WebSocketEncoder.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/IlpV4WebSocketEncoder.java new file mode 100644 index 0000000..f1826f5 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/IlpV4WebSocketEncoder.java @@ -0,0 +1,790 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.ilpv4.client; + +import io.questdb.client.cutlass.ilpv4.protocol.*; + +import io.questdb.client.cutlass.ilpv4.protocol.IlpV4ColumnDef; +import io.questdb.client.cutlass.ilpv4.protocol.IlpV4GorillaEncoder; + +import io.questdb.client.cutlass.ilpv4.protocol.IlpV4TimestampDecoder; +import io.questdb.client.std.QuietCloseable; + +import static io.questdb.client.cutlass.ilpv4.protocol.IlpV4Constants.*; + +/** + * Encodes ILP v4 messages for WebSocket transport. + *

+ * This encoder can write to either an internal {@link NativeBufferWriter} (default) + * or an external {@link IlpBufferWriter} such as {@link io.questdb.client.cutlass.http.client.WebSocketSendBuffer}. + *

+ * When using an external buffer, the encoder writes directly to it without intermediate copies, + * enabling zero-copy WebSocket frame construction. + *

+ * Usage with external buffer (zero-copy): + * 

+ * WebSocketSendBuffer buf = client.getSendBuffer();
+ * buf.beginBinaryFrame();
+ * encoder.setBuffer(buf);
+ * encoder.encode(tableData, false);
+ * FrameInfo frame = buf.endBinaryFrame();
+ * client.sendFrame(frame);
+ *
+ */ +public class IlpV4WebSocketEncoder implements QuietCloseable { + + private NativeBufferWriter ownedBuffer; + private IlpBufferWriter buffer; + private final IlpV4GorillaEncoder gorillaEncoder = new IlpV4GorillaEncoder(); + private byte flags; + + public IlpV4WebSocketEncoder() { + this.ownedBuffer = new NativeBufferWriter(); + this.buffer = ownedBuffer; + this.flags = 0; + } + + public IlpV4WebSocketEncoder(int bufferSize) { + this.ownedBuffer = new NativeBufferWriter(bufferSize); + this.buffer = ownedBuffer; + this.flags = 0; + } + + /** + * Returns the underlying buffer. + *

+ * If an external buffer was set via {@link #setBuffer(IlpBufferWriter)}, + * that buffer is returned. Otherwise, returns the internal buffer. + */ + public IlpBufferWriter getBuffer() { + return buffer; + } + + /** + * Sets an external buffer for encoding. + *

+ * When set, the encoder writes directly to this buffer instead of its internal buffer. + * The caller is responsible for managing the external buffer's lifecycle. + *

+ * Pass {@code null} to revert to using the internal buffer. + * + * @param externalBuffer the external buffer to use, or null to use internal buffer + */ + public void setBuffer(IlpBufferWriter externalBuffer) { + this.buffer = externalBuffer != null ? externalBuffer : ownedBuffer; + } + + /** + * Returns true if currently using an external buffer. + */ + public boolean isUsingExternalBuffer() { + return buffer != ownedBuffer; + } + + /** + * Resets the encoder for a new message. + *

+ * If using an external buffer, this only resets the internal state (flags). + * The external buffer's reset is the caller's responsibility. + * If using the internal buffer, resets both the buffer and internal state. + */ + public void reset() { + if (!isUsingExternalBuffer()) { + buffer.reset(); + } + } + + /** + * Sets whether Gorilla timestamp encoding is enabled. + */ + public void setGorillaEnabled(boolean enabled) { + if (enabled) { + flags |= FLAG_GORILLA; + } else { + flags &= ~FLAG_GORILLA; + } + } + + /** + * Returns true if Gorilla encoding is enabled. + */ + public boolean isGorillaEnabled() { + return (flags & FLAG_GORILLA) != 0; + } + + /** + * Encodes a complete ILP v4 message from a table buffer. + * + * @param tableBuffer the table buffer containing row data + * @param useSchemaRef whether to use schema reference mode + * @return the number of bytes written + */ + public int encode(IlpV4TableBuffer tableBuffer, boolean useSchemaRef) { + buffer.reset(); + + // Write message header with placeholder for payload length + writeHeader(1, 0); + int payloadStart = buffer.getPosition(); + + // Encode table data + encodeTable(tableBuffer, useSchemaRef); + + // Patch payload length + int payloadLength = buffer.getPosition() - payloadStart; + buffer.patchInt(8, payloadLength); + + return buffer.getPosition(); + } + + /** + * Encodes a complete ILP v4 message with delta symbol dictionary encoding. + *

+ * This method sends only new symbols (delta) since the last confirmed watermark, + * and uses global symbol IDs instead of per-column local indices. + * + * @param tableBuffer the table buffer containing row data + * @param globalDict the global symbol dictionary + * @param confirmedMaxId the highest symbol ID the server has confirmed (from ConnectionSymbolState) + * @param batchMaxId the highest symbol ID used in this batch + * @param useSchemaRef whether to use schema reference mode + * @return the number of bytes written + */ + public int encodeWithDeltaDict( + IlpV4TableBuffer tableBuffer, + GlobalSymbolDictionary globalDict, + int confirmedMaxId, + int batchMaxId, + boolean useSchemaRef + ) { + buffer.reset(); + + // Calculate delta range + int deltaStart = confirmedMaxId + 1; + int deltaCount = Math.max(0, batchMaxId - confirmedMaxId); + + // Set delta dictionary flag + byte savedFlags = flags; + flags |= FLAG_DELTA_SYMBOL_DICT; + + // Write message header with placeholder for payload length + writeHeader(1, 0); + int payloadStart = buffer.getPosition(); + + // Write symbol delta section (before tables) + buffer.putVarint(deltaStart); + buffer.putVarint(deltaCount); + for (int id = deltaStart; id < deltaStart + deltaCount; id++) { + String symbol = globalDict.getSymbol(id); + buffer.putString(symbol); + } + + // Encode table data (symbol columns will use global IDs) + encodeTableWithGlobalSymbols(tableBuffer, useSchemaRef); + + // Patch payload length + int payloadLength = buffer.getPosition() - payloadStart; + buffer.patchInt(8, payloadLength); + + // Restore flags + flags = savedFlags; + + return buffer.getPosition(); + } + + /** + * Sets the delta symbol dictionary flag. + */ + public void setDeltaSymbolDictEnabled(boolean enabled) { + if (enabled) { + flags |= FLAG_DELTA_SYMBOL_DICT; + } else { + flags &= ~FLAG_DELTA_SYMBOL_DICT; + } + } + + /** + * Returns true if delta symbol dictionary encoding is enabled. + */ + public boolean isDeltaSymbolDictEnabled() { + return (flags & FLAG_DELTA_SYMBOL_DICT) != 0; + } + + /** + * Writes the ILP v4 message header. + * + * @param tableCount number of tables in the message + * @param payloadLength payload length (can be 0 if patched later) + */ + public void writeHeader(int tableCount, int payloadLength) { + // Magic "ILP4" + buffer.putByte((byte) 'I'); + buffer.putByte((byte) 'L'); + buffer.putByte((byte) 'P'); + buffer.putByte((byte) '4'); + + // Version + buffer.putByte(VERSION_1); + + // Flags + buffer.putByte(flags); + + // Table count (uint16, little-endian) + buffer.putShort((short) tableCount); + + // Payload length (uint32, little-endian) + buffer.putInt(payloadLength); + } + + /** + * Encodes a single table from the buffer. + */ + private void encodeTable(IlpV4TableBuffer tableBuffer, boolean useSchemaRef) { + IlpV4ColumnDef[] columnDefs = tableBuffer.getColumnDefs(); + int rowCount = tableBuffer.getRowCount(); + + if (useSchemaRef) { + writeTableHeaderWithSchemaRef( + tableBuffer.getTableName(), + rowCount, + tableBuffer.getSchemaHash(), + columnDefs.length + ); + } else { + writeTableHeaderWithSchema(tableBuffer.getTableName(), rowCount, columnDefs); + } + + // Write each column's data + boolean useGorilla = isGorillaEnabled(); + for (int i = 0; i < tableBuffer.getColumnCount(); i++) { + IlpV4TableBuffer.ColumnBuffer col = tableBuffer.getColumn(i); + IlpV4ColumnDef colDef = columnDefs[i]; + encodeColumn(col, colDef, rowCount, useGorilla); + } + } + + /** + * Encodes a single table from the buffer using global symbol IDs. + * This is used with delta dictionary encoding. + */ + private void encodeTableWithGlobalSymbols(IlpV4TableBuffer tableBuffer, boolean useSchemaRef) { + IlpV4ColumnDef[] columnDefs = tableBuffer.getColumnDefs(); + int rowCount = tableBuffer.getRowCount(); + + if (useSchemaRef) { + writeTableHeaderWithSchemaRef( + tableBuffer.getTableName(), + rowCount, + tableBuffer.getSchemaHash(), + columnDefs.length + ); + } else { + writeTableHeaderWithSchema(tableBuffer.getTableName(), rowCount, columnDefs); + } + + // Write each column's data + boolean useGorilla = isGorillaEnabled(); + for (int i = 0; i < tableBuffer.getColumnCount(); i++) { + IlpV4TableBuffer.ColumnBuffer col = tableBuffer.getColumn(i); + IlpV4ColumnDef colDef = columnDefs[i]; + encodeColumnWithGlobalSymbols(col, colDef, rowCount, useGorilla); + } + } + + /** + * Writes a table header with full schema. + */ + private void writeTableHeaderWithSchema(String tableName, int rowCount, IlpV4ColumnDef[] columns) { + // Table name + buffer.putString(tableName); + + // Row count (varint) + buffer.putVarint(rowCount); + + // Column count (varint) + buffer.putVarint(columns.length); + + // Schema mode: full schema (0x00) + buffer.putByte(SCHEMA_MODE_FULL); + + // Column definitions (name + type for each) + for (IlpV4ColumnDef col : columns) { + buffer.putString(col.getName()); + buffer.putByte(col.getWireTypeCode()); + } + } + + /** + * Writes a table header with schema reference. + */ + private void writeTableHeaderWithSchemaRef(String tableName, int rowCount, long schemaHash, int columnCount) { + // Table name + buffer.putString(tableName); + + // Row count (varint) + buffer.putVarint(rowCount); + + // Column count (varint) + buffer.putVarint(columnCount); + + // Schema mode: reference (0x01) + buffer.putByte(SCHEMA_MODE_REFERENCE); + + // Schema hash (8 bytes) + buffer.putLong(schemaHash); + } + + /** + * Encodes a single column. + */ + private void encodeColumn(IlpV4TableBuffer.ColumnBuffer col, IlpV4ColumnDef colDef, int rowCount, boolean useGorilla) { + int valueCount = col.getValueCount(); + + // Write null bitmap if column is nullable + if (colDef.isNullable()) { + writeNullBitmapPacked(col.getNullBitmapPacked(), rowCount); + } + + // Write column data based on type + switch (col.getType()) { + case TYPE_BOOLEAN: + writeBooleanColumn(col.getBooleanValues(), valueCount); + break; + case TYPE_BYTE: + writeByteColumn(col.getByteValues(), valueCount); + break; + case TYPE_SHORT: + case TYPE_CHAR: + writeShortColumn(col.getShortValues(), valueCount); + break; + case TYPE_INT: + writeIntColumn(col.getIntValues(), valueCount); + break; + case TYPE_LONG: + writeLongColumn(col.getLongValues(), valueCount); + break; + case TYPE_FLOAT: + writeFloatColumn(col.getFloatValues(), valueCount); + break; + case TYPE_DOUBLE: + writeDoubleColumn(col.getDoubleValues(), valueCount); + break; + case TYPE_TIMESTAMP: + case TYPE_TIMESTAMP_NANOS: + writeTimestampColumn(col.getLongValues(), valueCount, useGorilla); + break; + case TYPE_DATE: + writeLongColumn(col.getLongValues(), valueCount); + break; + case TYPE_STRING: + case TYPE_VARCHAR: + writeStringColumn(col.getStringValues(), valueCount); + break; + case TYPE_SYMBOL: + writeSymbolColumn(col, valueCount); + break; + case TYPE_UUID: + writeUuidColumn(col.getUuidHigh(), col.getUuidLow(), valueCount); + break; + case TYPE_LONG256: + writeLong256Column(col.getLong256Values(), valueCount); + break; + case TYPE_DOUBLE_ARRAY: + writeDoubleArrayColumn(col, valueCount); + break; + case TYPE_LONG_ARRAY: + writeLongArrayColumn(col, valueCount); + break; + case TYPE_DECIMAL64: + writeDecimal64Column(col.getDecimalScale(), col.getDecimal64Values(), valueCount); + break; + case TYPE_DECIMAL128: + writeDecimal128Column(col.getDecimalScale(), col.getDecimal128High(), col.getDecimal128Low(), valueCount); + break; + case TYPE_DECIMAL256: + writeDecimal256Column(col.getDecimalScale(), + col.getDecimal256Hh(), col.getDecimal256Hl(), + col.getDecimal256Lh(), col.getDecimal256Ll(), valueCount); + break; + default: + throw new IllegalStateException("Unknown column type: " + col.getType()); + } + } + + /** + * Encodes a single column using global symbol IDs for SYMBOL type. + * All other column types are encoded the same as encodeColumn. + */ + private void encodeColumnWithGlobalSymbols(IlpV4TableBuffer.ColumnBuffer col, IlpV4ColumnDef colDef, int rowCount, boolean useGorilla) { + int valueCount = col.getValueCount(); + + // Write null bitmap if column is nullable + if (colDef.isNullable()) { + writeNullBitmapPacked(col.getNullBitmapPacked(), rowCount); + } + + // For symbol columns, use global IDs; for all others, use standard encoding + if (col.getType() == TYPE_SYMBOL) { + writeSymbolColumnWithGlobalIds(col, valueCount); + } else { + // Write column data based on type (same as encodeColumn) + switch (col.getType()) { + case TYPE_BOOLEAN: + writeBooleanColumn(col.getBooleanValues(), valueCount); + break; + case TYPE_BYTE: + writeByteColumn(col.getByteValues(), valueCount); + break; + case TYPE_SHORT: + case TYPE_CHAR: + writeShortColumn(col.getShortValues(), valueCount); + break; + case TYPE_INT: + writeIntColumn(col.getIntValues(), valueCount); + break; + case TYPE_LONG: + writeLongColumn(col.getLongValues(), valueCount); + break; + case TYPE_FLOAT: + writeFloatColumn(col.getFloatValues(), valueCount); + break; + case TYPE_DOUBLE: + writeDoubleColumn(col.getDoubleValues(), valueCount); + break; + case TYPE_TIMESTAMP: + case TYPE_TIMESTAMP_NANOS: + writeTimestampColumn(col.getLongValues(), valueCount, useGorilla); + break; + case TYPE_DATE: + writeLongColumn(col.getLongValues(), valueCount); + break; + case TYPE_STRING: + case TYPE_VARCHAR: + writeStringColumn(col.getStringValues(), valueCount); + break; + case TYPE_UUID: + writeUuidColumn(col.getUuidHigh(), col.getUuidLow(), valueCount); + break; + case TYPE_LONG256: + writeLong256Column(col.getLong256Values(), valueCount); + break; + case TYPE_DOUBLE_ARRAY: + writeDoubleArrayColumn(col, valueCount); + break; + case TYPE_LONG_ARRAY: + writeLongArrayColumn(col, valueCount); + break; + case TYPE_DECIMAL64: + writeDecimal64Column(col.getDecimalScale(), col.getDecimal64Values(), valueCount); + break; + case TYPE_DECIMAL128: + writeDecimal128Column(col.getDecimalScale(), col.getDecimal128High(), col.getDecimal128Low(), valueCount); + break; + case TYPE_DECIMAL256: + writeDecimal256Column(col.getDecimalScale(), + col.getDecimal256Hh(), col.getDecimal256Hl(), + col.getDecimal256Lh(), col.getDecimal256Ll(), valueCount); + break; + default: + throw new IllegalStateException("Unknown column type: " + col.getType()); + } + } + } + + /** + * Writes a null bitmap from bit-packed long array. + */ + private void writeNullBitmapPacked(long[] nullsPacked, int count) { + int bitmapSize = (count + 7) / 8; + + for (int byteIdx = 0; byteIdx < bitmapSize; byteIdx++) { + int longIndex = byteIdx >>> 3; + int byteInLong = byteIdx & 7; + byte b = (byte) ((nullsPacked[longIndex] >>> (byteInLong * 8)) & 0xFF); + buffer.putByte(b); + } + } + + /** + * Writes boolean column data (bit-packed). + */ + private void writeBooleanColumn(boolean[] values, int count) { + int packedSize = (count + 7) / 8; + + for (int i = 0; i < packedSize; i++) { + byte b = 0; + for (int bit = 0; bit < 8; bit++) { + int idx = i * 8 + bit; + if (idx < count && values[idx]) { + b |= (1 << bit); + } + } + buffer.putByte(b); + } + } + + private void writeByteColumn(byte[] values, int count) { + for (int i = 0; i < count; i++) { + buffer.putByte(values[i]); + } + } + + private void writeShortColumn(short[] values, int count) { + for (int i = 0; i < count; i++) { + buffer.putShort(values[i]); + } + } + + private void writeIntColumn(int[] values, int count) { + for (int i = 0; i < count; i++) { + buffer.putInt(values[i]); + } + } + + private void writeLongColumn(long[] values, int count) { + for (int i = 0; i < count; i++) { + buffer.putLong(values[i]); + } + } + + private void writeFloatColumn(float[] values, int count) { + for (int i = 0; i < count; i++) { + buffer.putFloat(values[i]); + } + } + + private void writeDoubleColumn(double[] values, int count) { + for (int i = 0; i < count; i++) { + buffer.putDouble(values[i]); + } + } + + /** + * Writes a timestamp column with optional Gorilla compression. + *

+ * When Gorilla encoding is enabled and applicable (3+ timestamps with + * delta-of-deltas fitting in 32-bit range), uses delta-of-delta compression. + * Otherwise, falls back to uncompressed encoding. + */ + private void writeTimestampColumn(long[] values, int count, boolean useGorilla) { + if (useGorilla && count > 2 && IlpV4GorillaEncoder.canUseGorilla(values, count)) { + // Write Gorilla encoding flag + buffer.putByte(IlpV4TimestampDecoder.ENCODING_GORILLA); + + // Calculate size needed and ensure buffer has capacity + int encodedSize = IlpV4GorillaEncoder.calculateEncodedSize(values, count); + buffer.ensureCapacity(encodedSize); + + // Encode timestamps to buffer + int bytesWritten = gorillaEncoder.encodeTimestamps( + buffer.getBufferPtr() + buffer.getPosition(), + buffer.getCapacity() - buffer.getPosition(), + values, + count + ); + buffer.skip(bytesWritten); + } else { + // Write uncompressed + if (useGorilla) { + buffer.putByte(IlpV4TimestampDecoder.ENCODING_UNCOMPRESSED); + } + writeLongColumn(values, count); + } + } + + /** + * Writes a string column with offset array. + */ + private void writeStringColumn(String[] strings, int count) { + // Calculate total data length + int totalDataLen = 0; + for (int i = 0; i < count; i++) { + if (strings[i] != null) { + totalDataLen += IlpBufferWriter.utf8Length(strings[i]); + } + } + + // Write offset array + int runningOffset = 0; + buffer.putInt(0); + for (int i = 0; i < count; i++) { + if (strings[i] != null) { + runningOffset += IlpBufferWriter.utf8Length(strings[i]); + } + buffer.putInt(runningOffset); + } + + // Write string data + for (int i = 0; i < count; i++) { + if (strings[i] != null) { + buffer.putUtf8(strings[i]); + } + } + } + + /** + * Writes a symbol column with dictionary. + * Format: + * - Dictionary length (varint) + * - Dictionary entries (length-prefixed UTF-8 strings) + * - Symbol indices (varints, one per value) + */ + private void writeSymbolColumn(IlpV4TableBuffer.ColumnBuffer col, int count) { + // Get symbol data from column buffer + int[] symbolIndices = col.getSymbolIndices(); + String[] dictionary = col.getSymbolDictionary(); + + // Write dictionary + buffer.putVarint(dictionary.length); + for (String symbol : dictionary) { + buffer.putString(symbol); + } + + // Write symbol indices (one per non-null value) + for (int i = 0; i < count; i++) { + buffer.putVarint(symbolIndices[i]); + } + } + + /** + * Writes a symbol column using global IDs (for delta dictionary mode). + * Format: + * - Global symbol IDs (varints, one per value) + *

+ * The dictionary is not included here because it's written at the message level + * in delta format. + */ + private void writeSymbolColumnWithGlobalIds(IlpV4TableBuffer.ColumnBuffer col, int count) { + int[] globalIds = col.getGlobalSymbolIds(); + if (globalIds == null) { + // Fall back to local indices if no global IDs stored + int[] symbolIndices = col.getSymbolIndices(); + for (int i = 0; i < count; i++) { + buffer.putVarint(symbolIndices[i]); + } + } else { + // Write global symbol IDs + for (int i = 0; i < count; i++) { + buffer.putVarint(globalIds[i]); + } + } + } + + private void writeUuidColumn(long[] highBits, long[] lowBits, int count) { + // Little-endian: lo first, then hi + for (int i = 0; i < count; i++) { + buffer.putLong(lowBits[i]); + buffer.putLong(highBits[i]); + } + } + + private void writeLong256Column(long[] values, int count) { + // Flat array: 4 longs per value, little-endian (least significant first) + // values layout: [long0, long1, long2, long3] per row + for (int i = 0; i < count * 4; i++) { + buffer.putLong(values[i]); + } + } + + private void writeDoubleArrayColumn(IlpV4TableBuffer.ColumnBuffer col, int count) { + byte[] dims = col.getArrayDims(); + int[] shapes = col.getArrayShapes(); + double[] data = col.getDoubleArrayData(); + + int shapeIdx = 0; + int dataIdx = 0; + for (int row = 0; row < count; row++) { + int nDims = dims[row]; + buffer.putByte((byte) nDims); + + int elemCount = 1; + for (int d = 0; d < nDims; d++) { + int dimLen = shapes[shapeIdx++]; + buffer.putInt(dimLen); + elemCount *= dimLen; + } + + for (int e = 0; e < elemCount; e++) { + buffer.putDouble(data[dataIdx++]); + } + } + } + + private void writeLongArrayColumn(IlpV4TableBuffer.ColumnBuffer col, int count) { + byte[] dims = col.getArrayDims(); + int[] shapes = col.getArrayShapes(); + long[] data = col.getLongArrayData(); + + int shapeIdx = 0; + int dataIdx = 0; + for (int row = 0; row < count; row++) { + int nDims = dims[row]; + buffer.putByte((byte) nDims); + + int elemCount = 1; + for (int d = 0; d < nDims; d++) { + int dimLen = shapes[shapeIdx++]; + buffer.putInt(dimLen); + elemCount *= dimLen; + } + + for (int e = 0; e < elemCount; e++) { + buffer.putLong(data[dataIdx++]); + } + } + } + + private void writeDecimal64Column(byte scale, long[] values, int count) { + buffer.putByte(scale); + for (int i = 0; i < count; i++) { + buffer.putLongBE(values[i]); + } + } + + private void writeDecimal128Column(byte scale, long[] high, long[] low, int count) { + buffer.putByte(scale); + for (int i = 0; i < count; i++) { + buffer.putLongBE(high[i]); + buffer.putLongBE(low[i]); + } + } + + private void writeDecimal256Column(byte scale, long[] hh, long[] hl, long[] lh, long[] ll, int count) { + buffer.putByte(scale); + for (int i = 0; i < count; i++) { + buffer.putLongBE(hh[i]); + buffer.putLongBE(hl[i]); + buffer.putLongBE(lh[i]); + buffer.putLongBE(ll[i]); + } + } + + @Override + public void close() { + if (ownedBuffer != null) { + ownedBuffer.close(); + ownedBuffer = null; + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/IlpV4WebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/IlpV4WebSocketSender.java new file mode 100644 index 0000000..43a0b4c --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/IlpV4WebSocketSender.java @@ -0,0 +1,1398 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.ilpv4.client; + +import io.questdb.client.cutlass.ilpv4.protocol.*; + +import io.questdb.client.Sender; +import io.questdb.client.cutlass.http.client.WebSocketClient; +import io.questdb.client.cutlass.http.client.WebSocketClientFactory; +import io.questdb.client.cutlass.http.client.WebSocketFrameHandler; + +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.line.array.DoubleArray; +import io.questdb.client.cutlass.line.array.LongArray; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import io.questdb.client.std.Chars; +import io.questdb.client.std.CharSequenceObjHashMap; +import io.questdb.client.std.LongHashSet; +import io.questdb.client.std.ObjList; +import io.questdb.client.std.Decimal128; +import io.questdb.client.std.Decimal256; +import io.questdb.client.std.Decimal64; +import io.questdb.client.std.bytes.DirectByteSlice; +import org.jetbrains.annotations.NotNull; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.concurrent.TimeUnit; + +import static io.questdb.client.cutlass.ilpv4.protocol.IlpV4Constants.*; + +/** + * ILP v4 WebSocket client sender for streaming data to QuestDB. + *

+ * This sender uses a double-buffering scheme with asynchronous I/O for high throughput: + *

+ *

+ * Configuration options: + *

+ *

+ * Example usage: + * 

+ * try (IlpV4WebSocketSender sender = IlpV4WebSocketSender.connect("localhost", 9000)) {
+ *     for (int i = 0; i < 100_000; i++) {
+ *         sender.table("metrics")
+ *               .symbol("host", "server-" + (i % 10))
+ *               .doubleColumn("cpu", Math.random() * 100)
+ *               .atNow();
+ *         // Rows are batched and sent asynchronously!
+ *     }
+ *     // flush() waits for all pending batches to be sent
+ *     sender.flush();
+ * }
+ *
+ */ +public class IlpV4WebSocketSender implements Sender { + + private static final Logger LOG = LoggerFactory.getLogger(IlpV4WebSocketSender.class); + + private static final int DEFAULT_BUFFER_SIZE = 8192; + private static final int DEFAULT_MICROBATCH_BUFFER_SIZE = 1024 * 1024; // 1MB + public static final int DEFAULT_AUTO_FLUSH_ROWS = 500; + public static final int DEFAULT_AUTO_FLUSH_BYTES = 1024 * 1024; // 1MB + public static final long DEFAULT_AUTO_FLUSH_INTERVAL_NANOS = 100_000_000L; // 100ms + public static final int DEFAULT_IN_FLIGHT_WINDOW_SIZE = InFlightWindow.DEFAULT_WINDOW_SIZE; // 8 + public static final int DEFAULT_SEND_QUEUE_CAPACITY = WebSocketSendQueue.DEFAULT_QUEUE_CAPACITY; // 16 + private static final String WRITE_PATH = "/write/v4"; + + private final String host; + private final int port; + private final boolean tlsEnabled; + private final CharSequenceObjHashMap tableBuffers; + private IlpV4TableBuffer currentTableBuffer; + private String currentTableName; + // Cached column references to avoid repeated hashmap lookups + private IlpV4TableBuffer.ColumnBuffer cachedTimestampColumn; + private IlpV4TableBuffer.ColumnBuffer cachedTimestampNanosColumn; + + // Encoder for ILP v4 messages + private final IlpV4WebSocketEncoder encoder; + + // WebSocket client (zero-GC native implementation) + private WebSocketClient client; + private boolean connected; + private boolean closed; + + // Double-buffering for async I/O + private MicrobatchBuffer buffer0; + private MicrobatchBuffer buffer1; + private MicrobatchBuffer activeBuffer; + private WebSocketSendQueue sendQueue; + + // Flow control + private InFlightWindow inFlightWindow; + + // Auto-flush configuration + private final int autoFlushRows; + private final int autoFlushBytes; + private final long autoFlushIntervalNanos; + + // Flow control configuration + private final int inFlightWindowSize; + private final int sendQueueCapacity; + + // Configuration + private boolean gorillaEnabled = true; + + // Async mode: pending row tracking + private int pendingRowCount; + private long firstPendingRowTimeNanos; + + // Batch sequence counter (must match server's messageSequence) + private long nextBatchSequence = 0; + + // Global symbol dictionary for delta encoding + private final GlobalSymbolDictionary globalSymbolDictionary; + + // Track max global symbol ID used in current batch (for delta calculation) + private int currentBatchMaxSymbolId = -1; + + // Track highest symbol ID sent to server (for delta encoding) + // Once sent over TCP, server is guaranteed to receive it (or connection dies) + private volatile int maxSentSymbolId = -1; + + // Track schema hashes that have been sent to the server (for schema reference mode) + // First time we send a schema: full schema. Subsequent times: 8-byte hash reference. + // Combined key = schemaHash XOR (tableNameHash << 32) to include table name in lookup. + private final LongHashSet sentSchemaHashes = new LongHashSet(); + + private IlpV4WebSocketSender(String host, int port, boolean tlsEnabled, int bufferSize, + int autoFlushRows, int autoFlushBytes, long autoFlushIntervalNanos, + int inFlightWindowSize, int sendQueueCapacity) { + this.host = host; + this.port = port; + this.tlsEnabled = tlsEnabled; + this.encoder = new IlpV4WebSocketEncoder(bufferSize); + this.tableBuffers = new CharSequenceObjHashMap<>(); + this.currentTableBuffer = null; + this.currentTableName = null; + this.connected = false; + this.closed = false; + this.autoFlushRows = autoFlushRows; + this.autoFlushBytes = autoFlushBytes; + this.autoFlushIntervalNanos = autoFlushIntervalNanos; + this.inFlightWindowSize = inFlightWindowSize; + this.sendQueueCapacity = sendQueueCapacity; + + // Initialize global symbol dictionary for delta encoding + this.globalSymbolDictionary = new GlobalSymbolDictionary(); + + // Initialize double-buffering if async mode (window > 1) + if (inFlightWindowSize > 1) { + int microbatchBufferSize = Math.max(DEFAULT_MICROBATCH_BUFFER_SIZE, autoFlushBytes * 2); + this.buffer0 = new MicrobatchBuffer(microbatchBufferSize, autoFlushRows, autoFlushBytes, autoFlushIntervalNanos); + this.buffer1 = new MicrobatchBuffer(microbatchBufferSize, autoFlushRows, autoFlushBytes, autoFlushIntervalNanos); + this.activeBuffer = buffer0; + } + } + + /** + * Creates a new sender and connects to the specified host and port. + * Uses synchronous mode for backward compatibility. + * + * @param host server host + * @param port server HTTP port (WebSocket upgrade happens on same port) + * @return connected sender + */ + public static IlpV4WebSocketSender connect(String host, int port) { + return connect(host, port, false); + } + + /** + * Creates a new sender with TLS and connects to the specified host and port. + * Uses synchronous mode for backward compatibility. + * + * @param host server host + * @param port server HTTP port + * @param tlsEnabled whether to use TLS + * @return connected sender + */ + public static IlpV4WebSocketSender connect(String host, int port, boolean tlsEnabled) { + IlpV4WebSocketSender sender = new IlpV4WebSocketSender( + host, port, tlsEnabled, DEFAULT_BUFFER_SIZE, + 0, 0, 0, // No auto-flush in sync mode + 1, 1 // window=1 for sync behavior, queue=1 (not used) + ); + sender.ensureConnected(); + return sender; + } + + /** + * Creates a new sender with async mode and custom configuration. + * + * @param host server host + * @param port server HTTP port + * @param tlsEnabled whether to use TLS + * @param autoFlushRows rows per batch (0 = no limit) + * @param autoFlushBytes bytes per batch (0 = no limit) + * @param autoFlushIntervalNanos age before flush in nanos (0 = no limit) + * @return connected sender + */ + public static IlpV4WebSocketSender connectAsync(String host, int port, boolean tlsEnabled, + int autoFlushRows, int autoFlushBytes, + long autoFlushIntervalNanos) { + return connectAsync(host, port, tlsEnabled, autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, + DEFAULT_IN_FLIGHT_WINDOW_SIZE, DEFAULT_SEND_QUEUE_CAPACITY); + } + + /** + * Creates a new sender with async mode and full configuration including flow control. + * + * @param host server host + * @param port server HTTP port + * @param tlsEnabled whether to use TLS + * @param autoFlushRows rows per batch (0 = no limit) + * @param autoFlushBytes bytes per batch (0 = no limit) + * @param autoFlushIntervalNanos age before flush in nanos (0 = no limit) + * @param inFlightWindowSize max batches awaiting server ACK (default: 8) + * @param sendQueueCapacity max batches waiting to send (default: 16) + * @return connected sender + */ + public static IlpV4WebSocketSender connectAsync(String host, int port, boolean tlsEnabled, + int autoFlushRows, int autoFlushBytes, + long autoFlushIntervalNanos, + int inFlightWindowSize, int sendQueueCapacity) { + IlpV4WebSocketSender sender = new IlpV4WebSocketSender( + host, port, tlsEnabled, DEFAULT_BUFFER_SIZE, + autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, + inFlightWindowSize, sendQueueCapacity + ); + sender.ensureConnected(); + return sender; + } + + /** + * Creates a new sender with async mode and default configuration. + * + * @param host server host + * @param port server HTTP port + * @param tlsEnabled whether to use TLS + * @return connected sender + */ + public static IlpV4WebSocketSender connectAsync(String host, int port, boolean tlsEnabled) { + return connectAsync(host, port, tlsEnabled, + DEFAULT_AUTO_FLUSH_ROWS, DEFAULT_AUTO_FLUSH_BYTES, DEFAULT_AUTO_FLUSH_INTERVAL_NANOS); + } + + /** + * Factory method for SenderBuilder integration. + */ + public static IlpV4WebSocketSender create( + String host, + int port, + boolean tlsEnabled, + int bufferSize, + String authToken, + String username, + String password + ) { + IlpV4WebSocketSender sender = new IlpV4WebSocketSender( + host, port, tlsEnabled, bufferSize, + 0, 0, 0, + 1, 1 // window=1 for sync behavior + ); + // TODO: Store auth credentials for connection + sender.ensureConnected(); + return sender; + } + + /** + * Creates a sender without connecting. For testing only. + *

+ * This allows unit tests to test sender logic without requiring a real server. + * + * @param host server host (not connected) + * @param port server port (not connected) + * @param inFlightWindowSize window size: 1 for sync behavior, >1 for async + * @return unconnected sender + */ + public static IlpV4WebSocketSender createForTesting(String host, int port, int inFlightWindowSize) { + return new IlpV4WebSocketSender( + host, port, false, DEFAULT_BUFFER_SIZE, + 0, 0, 0, + inFlightWindowSize, DEFAULT_SEND_QUEUE_CAPACITY + ); + // Note: does NOT call ensureConnected() + } + + /** + * Creates a sender with custom flow control settings without connecting. For testing only. + * + * @param host server host (not connected) + * @param port server port (not connected) + * @param autoFlushRows rows per batch (0 = no limit) + * @param autoFlushBytes bytes per batch (0 = no limit) + * @param autoFlushIntervalNanos age before flush in nanos (0 = no limit) + * @param inFlightWindowSize window size: 1 for sync behavior, >1 for async + * @param sendQueueCapacity max batches waiting to send + * @return unconnected sender + */ + public static IlpV4WebSocketSender createForTesting( + String host, int port, + int autoFlushRows, int autoFlushBytes, long autoFlushIntervalNanos, + int inFlightWindowSize, int sendQueueCapacity) { + return new IlpV4WebSocketSender( + host, port, false, DEFAULT_BUFFER_SIZE, + autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, + inFlightWindowSize, sendQueueCapacity + ); + // Note: does NOT call ensureConnected() + } + + private void ensureConnected() { + if (closed) { + throw new LineSenderException("Sender is closed"); + } + if (!connected) { + // Create WebSocket client using factory (zero-GC native implementation) + if (tlsEnabled) { + client = WebSocketClientFactory.newInsecureTlsInstance(); + } else { + client = WebSocketClientFactory.newPlainTextInstance(); + } + + // Connect and upgrade to WebSocket + try { + client.connect(host, port); + client.upgrade(WRITE_PATH); + } catch (Exception e) { + client.close(); + client = null; + throw new LineSenderException("Failed to connect to " + host + ":" + port, e); + } + + // a window for tracking batches awaiting ACK (both modes) + inFlightWindow = new InFlightWindow(inFlightWindowSize, InFlightWindow.DEFAULT_TIMEOUT_MS); + + // Initialize send queue for async mode (window > 1) + // The send queue handles both sending AND receiving (single I/O thread) + if (inFlightWindowSize > 1) { + sendQueue = new WebSocketSendQueue(client, inFlightWindow, + sendQueueCapacity, + WebSocketSendQueue.DEFAULT_ENQUEUE_TIMEOUT_MS, + WebSocketSendQueue.DEFAULT_SHUTDOWN_TIMEOUT_MS); + } + // Sync mode (window=1): no send queue - we send and read ACKs synchronously + + // Clear sent schema hashes - server starts fresh on each connection + sentSchemaHashes.clear(); + + connected = true; + LOG.info("Connected to WebSocket [host={}, port={}, windowSize={}]", host, port, inFlightWindowSize); + } + } + + /** + * Returns whether Gorilla encoding is enabled. + */ + public boolean isGorillaEnabled() { + return gorillaEnabled; + } + + /** + * Sets whether to use Gorilla timestamp encoding. + */ + public IlpV4WebSocketSender setGorillaEnabled(boolean enabled) { + this.gorillaEnabled = enabled; + this.encoder.setGorillaEnabled(enabled); + return this; + } + + /** + * Returns whether async mode is enabled (window size > 1). + */ + public boolean isAsyncMode() { + return inFlightWindowSize > 1; + } + + /** + * Returns the in-flight window size. + * Window=1 means sync mode, window>1 means async mode. + */ + public int getInFlightWindowSize() { + return inFlightWindowSize; + } + + /** + * Returns the send queue capacity. + */ + public int getSendQueueCapacity() { + return sendQueueCapacity; + } + + /** + * Returns the auto-flush row threshold. + */ + public int getAutoFlushRows() { + return autoFlushRows; + } + + /** + * Returns the auto-flush byte threshold. + */ + public int getAutoFlushBytes() { + return autoFlushBytes; + } + + /** + * Returns the auto-flush interval in nanoseconds. + */ + public long getAutoFlushIntervalNanos() { + return autoFlushIntervalNanos; + } + + /** + * Returns the global symbol dictionary. + * For testing and encoder integration. + */ + public GlobalSymbolDictionary getGlobalSymbolDictionary() { + return globalSymbolDictionary; + } + + /** + * Returns the max symbol ID sent to the server. + * Once sent over TCP, server is guaranteed to receive it (or connection dies). + */ + public int getMaxSentSymbolId() { + return maxSentSymbolId; + } + + // ==================== Fast-path API for high-throughput generators ==================== + // + // These methods bypass the normal fluent API to avoid per-row overhead: + // - No hashmap lookups for column names + // - No checkNotClosed()/checkTableSelected() per column + // - Direct access to column buffers + // + // Usage: + // // Setup (once) + // IlpV4TableBuffer tableBuffer = sender.getTableBuffer("q"); + // IlpV4TableBuffer.ColumnBuffer colSymbol = tableBuffer.getOrCreateColumn("s", TYPE_SYMBOL, true); + // IlpV4TableBuffer.ColumnBuffer colBid = tableBuffer.getOrCreateColumn("b", TYPE_DOUBLE, false); + // + // // Hot path (per row) + // colSymbol.addSymbolWithGlobalId(symbol, sender.getOrAddGlobalSymbol(symbol)); + // colBid.addDouble(bid); + // tableBuffer.nextRow(); + // sender.incrementPendingRowCount(); + + /** + * Gets or creates a table buffer for direct access. + * For high-throughput generators that want to bypass fluent API overhead. + */ + public IlpV4TableBuffer getTableBuffer(String tableName) { + IlpV4TableBuffer buffer = tableBuffers.get(tableName); + if (buffer == null) { + buffer = new IlpV4TableBuffer(tableName); + tableBuffers.put(tableName, buffer); + } + currentTableBuffer = buffer; + currentTableName = tableName; + return buffer; + } + + /** + * Registers a symbol in the global dictionary and returns its ID. + * For use with fast-path column buffer access. + */ + public int getOrAddGlobalSymbol(String value) { + int globalId = globalSymbolDictionary.getOrAddSymbol(value); + if (globalId > currentBatchMaxSymbolId) { + currentBatchMaxSymbolId = globalId; + } + return globalId; + } + + /** + * Increments the pending row count for auto-flush tracking. + * Call this after adding a complete row via fast-path API. + * Triggers auto-flush if any threshold is exceeded. + */ + public void incrementPendingRowCount() { + if (pendingRowCount == 0) { + firstPendingRowTimeNanos = System.nanoTime(); + } + pendingRowCount++; + + // Check if any flush threshold is exceeded (same as sendRow()) + if (shouldAutoFlush()) { + if (inFlightWindowSize > 1) { + flushPendingRows(); + } else { + // Sync mode (window=1): flush directly with ACK wait + flushSync(); + } + } + } + + // ==================== Sender interface implementation ==================== + + @Override + public IlpV4WebSocketSender table(CharSequence tableName) { + checkNotClosed(); + // Fast path: if table name matches current, skip hashmap lookup + if (currentTableName != null && currentTableBuffer != null && Chars.equals(tableName, currentTableName)) { + return this; + } + // Table changed - invalidate cached column references + cachedTimestampColumn = null; + cachedTimestampNanosColumn = null; + currentTableName = tableName.toString(); + currentTableBuffer = tableBuffers.get(currentTableName); + if (currentTableBuffer == null) { + currentTableBuffer = new IlpV4TableBuffer(currentTableName); + tableBuffers.put(currentTableName, currentTableBuffer); + } + // Both modes accumulate rows until flush + return this; + } + + @Override + public IlpV4WebSocketSender symbol(CharSequence columnName, CharSequence value) { + checkNotClosed(); + checkTableSelected(); + IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_SYMBOL, true); + + if (value != null) { + // Register symbol in global dictionary and track max ID for delta calculation + String symbolValue = value.toString(); + int globalId = globalSymbolDictionary.getOrAddSymbol(symbolValue); + if (globalId > currentBatchMaxSymbolId) { + currentBatchMaxSymbolId = globalId; + } + // Store global ID in the column buffer + col.addSymbolWithGlobalId(symbolValue, globalId); + } else { + col.addSymbol(null); + } + return this; + } + + @Override + public IlpV4WebSocketSender boolColumn(CharSequence columnName, boolean value) { + checkNotClosed(); + checkTableSelected(); + IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_BOOLEAN, false); + col.addBoolean(value); + return this; + } + + @Override + public IlpV4WebSocketSender longColumn(CharSequence columnName, long value) { + checkNotClosed(); + checkTableSelected(); + IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_LONG, false); + col.addLong(value); + return this; + } + + /** + * Adds an INT column value to the current row. + * + * @param columnName the column name + * @param value the int value + * @return this sender for method chaining + */ + public IlpV4WebSocketSender intColumn(CharSequence columnName, int value) { + checkNotClosed(); + checkTableSelected(); + IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_INT, false); + col.addInt(value); + return this; + } + + @Override + public IlpV4WebSocketSender doubleColumn(CharSequence columnName, double value) { + checkNotClosed(); + checkTableSelected(); + IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_DOUBLE, false); + col.addDouble(value); + return this; + } + + @Override + public IlpV4WebSocketSender stringColumn(CharSequence columnName, CharSequence value) { + checkNotClosed(); + checkTableSelected(); + IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_STRING, true); + col.addString(value != null ? value.toString() : null); + return this; + } + + /** + * Adds a SHORT column value to the current row. + * + * @param columnName the column name + * @param value the short value + * @return this sender for method chaining + */ + public IlpV4WebSocketSender shortColumn(CharSequence columnName, short value) { + checkNotClosed(); + checkTableSelected(); + IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_SHORT, false); + col.addShort(value); + return this; + } + + /** + * Adds a CHAR column value to the current row. + *

+ * CHAR is stored as a 2-byte UTF-16 code unit in QuestDB. + * + * @param columnName the column name + * @param value the character value + * @return this sender for method chaining + */ + public IlpV4WebSocketSender charColumn(CharSequence columnName, char value) { + checkNotClosed(); + checkTableSelected(); + IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_CHAR, false); + col.addShort((short) value); + return this; + } + + /** + * Adds a UUID column value to the current row. + * + * @param columnName the column name + * @param lo the low 64 bits of the UUID + * @param hi the high 64 bits of the UUID + * @return this sender for method chaining + */ + public IlpV4WebSocketSender uuidColumn(CharSequence columnName, long lo, long hi) { + checkNotClosed(); + checkTableSelected(); + IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_UUID, true); + col.addUuid(hi, lo); + return this; + } + + /** + * Adds a LONG256 column value to the current row. + * + * @param columnName the column name + * @param l0 the least significant 64 bits + * @param l1 the second 64 bits + * @param l2 the third 64 bits + * @param l3 the most significant 64 bits + * @return this sender for method chaining + */ + public IlpV4WebSocketSender long256Column(CharSequence columnName, long l0, long l1, long l2, long l3) { + checkNotClosed(); + checkTableSelected(); + IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_LONG256, true); + col.addLong256(l0, l1, l2, l3); + return this; + } + + @Override + public IlpV4WebSocketSender timestampColumn(CharSequence columnName, long value, ChronoUnit unit) { + checkNotClosed(); + checkTableSelected(); + if (unit == ChronoUnit.NANOS) { + IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_TIMESTAMP_NANOS, true); + col.addLong(value); + } else { + long micros = toMicros(value, unit); + IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_TIMESTAMP, true); + col.addLong(micros); + } + return this; + } + + @Override + public IlpV4WebSocketSender timestampColumn(CharSequence columnName, Instant value) { + checkNotClosed(); + checkTableSelected(); + long micros = value.getEpochSecond() * 1_000_000L + value.getNano() / 1000L; + IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_TIMESTAMP, true); + col.addLong(micros); + return this; + } + + @Override + public void at(long timestamp, ChronoUnit unit) { + checkNotClosed(); + checkTableSelected(); + if (unit == ChronoUnit.NANOS) { + atNanos(timestamp); + } else { + long micros = toMicros(timestamp, unit); + atMicros(micros); + } + } + + @Override + public void at(Instant timestamp) { + checkNotClosed(); + checkTableSelected(); + long micros = timestamp.getEpochSecond() * 1_000_000L + timestamp.getNano() / 1000L; + atMicros(micros); + } + + private void atMicros(long timestampMicros) { + // Add designated timestamp column (empty name for designated timestamp) + // Use cached reference to avoid hashmap lookup per row + if (cachedTimestampColumn == null) { + cachedTimestampColumn = currentTableBuffer.getOrCreateColumn("", TYPE_TIMESTAMP, true); + } + cachedTimestampColumn.addLong(timestampMicros); + sendRow(); + } + + private void atNanos(long timestampNanos) { + // Add designated timestamp column (empty name for designated timestamp) + // Use cached reference to avoid hashmap lookup per row + if (cachedTimestampNanosColumn == null) { + cachedTimestampNanosColumn = currentTableBuffer.getOrCreateColumn("", TYPE_TIMESTAMP_NANOS, true); + } + cachedTimestampNanosColumn.addLong(timestampNanos); + sendRow(); + } + + @Override + public void atNow() { + checkNotClosed(); + checkTableSelected(); + // Server-assigned timestamp - just send the row without designated timestamp + sendRow(); + } + + /** + * Accumulates the current row. + * Both sync and async modes buffer rows until flush (explicit or auto-flush). + * The difference is that sync mode flush() blocks until server ACKs. + */ + private void sendRow() { + ensureConnected(); + currentTableBuffer.nextRow(); + + // Both modes: accumulate rows, don't encode yet + if (pendingRowCount == 0) { + firstPendingRowTimeNanos = System.nanoTime(); + } + pendingRowCount++; + + // Check if any flush threshold is exceeded + if (shouldAutoFlush()) { + if (inFlightWindowSize > 1) { + flushPendingRows(); + } else { + // Sync mode (window=1): flush directly with ACK wait + flushSync(); + } + } + } + + /** + * Checks if any auto-flush threshold is exceeded. + */ + private boolean shouldAutoFlush() { + if (pendingRowCount <= 0) { + return false; + } + // Row limit + if (autoFlushRows > 0 && pendingRowCount >= autoFlushRows) { + return true; + } + // Time limit + if (autoFlushIntervalNanos > 0) { + long ageNanos = System.nanoTime() - firstPendingRowTimeNanos; + if (ageNanos >= autoFlushIntervalNanos) { + return true; + } + } + // Byte limit is harder to estimate without encoding, skip for now + return false; + } + + /** + * Flushes pending rows by encoding and sending them. + * Each table's rows are encoded into a separate ILP v4 message and sent as one WebSocket frame. + */ + private void flushPendingRows() { + if (pendingRowCount <= 0) { + return; + } + + LOG.debug("Flushing pending rows [count={}, tables={}]", pendingRowCount, tableBuffers.size()); + + // Ensure activeBuffer is ready for writing + // It might be in RECYCLED state if previous batch was sent but we didn't swap yet + ensureActiveBufferReady(); + + // Encode all table buffers that have data + // Iterate over the keys list directly + ObjList keys = tableBuffers.keys(); + for (int i = 0, n = keys.size(); i < n; i++) { + CharSequence tableName = keys.getQuick(i); + if (tableName == null) { + continue; // Skip null entries (shouldn't happen but be safe) + } + IlpV4TableBuffer tableBuffer = tableBuffers.get(tableName); + if (tableBuffer == null) { + continue; + } + int rowCount = tableBuffer.getRowCount(); + if (rowCount > 0) { + // Check if this schema has been sent before (use schema reference mode if so) + // Combined key includes table name since server caches by (tableName, schemaHash) + long schemaHash = tableBuffer.getSchemaHash(); + long schemaKey = schemaHash ^ ((long) tableBuffer.getTableName().hashCode() << 32); + boolean useSchemaRef = sentSchemaHashes.contains(schemaKey); + + LOG.debug("Encoding table [name={}, rows={}, maxSentSymbolId={}, batchMaxId={}, useSchemaRef={}]", tableName, rowCount, maxSentSymbolId, currentBatchMaxSymbolId, useSchemaRef); + + // Encode this table's rows with delta symbol dictionary + int messageSize = encoder.encodeWithDeltaDict( + tableBuffer, + globalSymbolDictionary, + maxSentSymbolId, + currentBatchMaxSymbolId, + useSchemaRef + ); + + // Track schema key if this was the first time sending this schema + if (!useSchemaRef) { + sentSchemaHashes.add(schemaKey); + } + IlpBufferWriter buffer = encoder.getBuffer(); + + // Copy to microbatch buffer and seal immediately + // Each ILP v4 message must be in its own WebSocket frame + activeBuffer.ensureCapacity(messageSize); + activeBuffer.write(buffer.getBufferPtr(), messageSize); + activeBuffer.incrementRowCount(); + activeBuffer.setMaxSymbolId(currentBatchMaxSymbolId); + + // Update maxSentSymbolId - once sent over TCP, server will receive it + maxSentSymbolId = currentBatchMaxSymbolId; + + // Seal and enqueue for sending + sealAndSwapBuffer(); + + // Reset table buffer and batch-level symbol tracking + tableBuffer.reset(); + currentBatchMaxSymbolId = -1; + } + } + + // Reset pending count + pendingRowCount = 0; + firstPendingRowTimeNanos = 0; + } + + /** + * Ensures the active buffer is ready for writing (in FILLING state). + * If the buffer is in RECYCLED state, resets it. If it's in use, waits for it. + */ + private void ensureActiveBufferReady() { + if (activeBuffer.isFilling()) { + return; // Already ready + } + + if (activeBuffer.isRecycled()) { + // Buffer was recycled but not reset - reset it now + activeBuffer.reset(); + return; + } + + // Buffer is in use (SEALED or SENDING) - wait for it + // Use a while loop to handle spurious wakeups and race conditions with the latch + while (activeBuffer.isInUse()) { + LOG.debug("Waiting for active buffer [id={}, state={}]", activeBuffer.getBatchId(), MicrobatchBuffer.stateName(activeBuffer.getState())); + boolean recycled = activeBuffer.awaitRecycled(30, TimeUnit.SECONDS); + if (!recycled) { + throw new LineSenderException("Timeout waiting for active buffer to be recycled"); + } + } + + // Buffer should now be RECYCLED - reset it + if (activeBuffer.isRecycled()) { + activeBuffer.reset(); + } + } + + /** + * Adds encoded data to the active microbatch buffer. + * Triggers seal and swap if buffer is full. + */ + private void addToMicrobatch(long dataPtr, int length) { + // Ensure activeBuffer is ready for writing + ensureActiveBufferReady(); + + // If current buffer can't hold the data, seal and swap + if (activeBuffer.hasData() && + activeBuffer.getBufferPos() + length > activeBuffer.getBufferCapacity()) { + sealAndSwapBuffer(); + } + + // Ensure buffer can hold the data + activeBuffer.ensureCapacity(activeBuffer.getBufferPos() + length); + + // Copy data to buffer + activeBuffer.write(dataPtr, length); + activeBuffer.incrementRowCount(); + } + + /** + * Seals the current buffer and swaps to the other buffer. + * Enqueues the sealed buffer for async sending. + */ + private void sealAndSwapBuffer() { + if (!activeBuffer.hasData()) { + return; // Nothing to send + } + + MicrobatchBuffer toSend = activeBuffer; + toSend.seal(); + + LOG.debug("Sealing buffer [id={}, rows={}, bytes={}]", toSend.getBatchId(), toSend.getRowCount(), toSend.getBufferPos()); + + // Swap to the other buffer + activeBuffer = (activeBuffer == buffer0) ? buffer1 : buffer0; + + // If the other buffer is still being sent, wait for it + // Use a while loop to handle spurious wakeups and race conditions with the latch + while (activeBuffer.isInUse()) { + LOG.debug("Waiting for buffer recycle [id={}, state={}]", activeBuffer.getBatchId(), MicrobatchBuffer.stateName(activeBuffer.getState())); + boolean recycled = activeBuffer.awaitRecycled(30, TimeUnit.SECONDS); + if (!recycled) { + throw new LineSenderException("Timeout waiting for buffer to be recycled"); + } + LOG.debug("Buffer recycled [id={}, state={}]", activeBuffer.getBatchId(), MicrobatchBuffer.stateName(activeBuffer.getState())); + } + + // Reset the new active buffer + int stateBeforeReset = activeBuffer.getState(); + LOG.debug("Resetting buffer [id={}, state={}]", activeBuffer.getBatchId(), MicrobatchBuffer.stateName(stateBeforeReset)); + activeBuffer.reset(); + + // Enqueue the sealed buffer for sending. + // If enqueue fails, roll back local state so the same batch can be retried. + try { + if (!sendQueue.enqueue(toSend)) { + throw new LineSenderException("Failed to enqueue buffer for sending"); + } + } catch (LineSenderException e) { + activeBuffer = toSend; + if (toSend.isSealed()) { + toSend.rollbackSealForRetry(); + } + throw e; + } + } + + @Override + public void flush() { + checkNotClosed(); + ensureConnected(); + + if (inFlightWindowSize > 1) { + // Async mode (window > 1): flush pending rows and wait for ACKs + flushPendingRows(); + + // Flush any remaining data in the active microbatch buffer + if (activeBuffer.hasData()) { + sealAndSwapBuffer(); + } + + // Wait for all pending batches to be sent to the server + sendQueue.flush(); + + // Wait for all in-flight batches to be acknowledged by the server + inFlightWindow.awaitEmpty(); + + LOG.debug("Flush complete [totalBatches={}, totalBytes={}, totalAcked={}]", sendQueue.getTotalBatchesSent(), sendQueue.getTotalBytesSent(), inFlightWindow.getTotalAcked()); + } else { + // Sync mode (window=1): flush pending rows and wait for ACKs synchronously + flushSync(); + } + } + + /** + * Flushes pending rows synchronously, blocking until server ACKs. + * Used in sync mode for simpler, blocking operation. + */ + private void flushSync() { + if (pendingRowCount <= 0) { + return; + } + + LOG.debug("Sync flush [pendingRows={}, tables={}]", pendingRowCount, tableBuffers.size()); + + // Encode all table buffers that have data into a single message + ObjList keys = tableBuffers.keys(); + for (int i = 0, n = keys.size(); i < n; i++) { + CharSequence tableName = keys.getQuick(i); + if (tableName == null) { + continue; + } + IlpV4TableBuffer tableBuffer = tableBuffers.get(tableName); + if (tableBuffer == null || tableBuffer.getRowCount() == 0) { + continue; + } + + // Check if this schema has been sent before (use schema reference mode if so) + // Combined key includes table name since server caches by (tableName, schemaHash) + long schemaHash = tableBuffer.getSchemaHash(); + long schemaKey = schemaHash ^ ((long) tableBuffer.getTableName().hashCode() << 32); + boolean useSchemaRef = sentSchemaHashes.contains(schemaKey); + + // Encode this table's rows with delta symbol dictionary + int messageSize = encoder.encodeWithDeltaDict( + tableBuffer, + globalSymbolDictionary, + maxSentSymbolId, + currentBatchMaxSymbolId, + useSchemaRef + ); + + // Track schema key if this was the first time sending this schema + if (!useSchemaRef) { + sentSchemaHashes.add(schemaKey); + } + + if (messageSize > 0) { + IlpBufferWriter buffer = encoder.getBuffer(); + + // Track batch in InFlightWindow before sending + long batchSequence = nextBatchSequence++; + inFlightWindow.addInFlight(batchSequence); + + // Update maxSentSymbolId - once sent over TCP, server will receive it + maxSentSymbolId = currentBatchMaxSymbolId; + + LOG.debug("Sending sync batch [seq={}, bytes={}, rows={}, maxSentSymbolId={}, useSchemaRef={}]", batchSequence, messageSize, tableBuffer.getRowCount(), maxSentSymbolId, useSchemaRef); + + // Send over WebSocket + client.sendBinary(buffer.getBufferPtr(), messageSize); + + // Wait for ACK synchronously + waitForAck(batchSequence); + } + + // Reset table buffer after sending + tableBuffer.reset(); + + // Reset batch-level symbol tracking + currentBatchMaxSymbolId = -1; + } + + // Reset pending row tracking + pendingRowCount = 0; + firstPendingRowTimeNanos = 0; + + LOG.debug("Sync flush complete [totalAcked={}]", inFlightWindow.getTotalAcked()); + } + + /** + * Waits synchronously for an ACK from the server for the specified batch. + */ + private void waitForAck(long expectedSequence) { + WebSocketResponse response = new WebSocketResponse(); + long deadline = System.currentTimeMillis() + InFlightWindow.DEFAULT_TIMEOUT_MS; + + while (System.currentTimeMillis() < deadline) { + try { + final boolean[] sawBinary = {false}; + boolean received = client.receiveFrame(new WebSocketFrameHandler() { + @Override + public void onBinaryMessage(long payloadPtr, int payloadLen) { + sawBinary[0] = true; + if (!WebSocketResponse.isStructurallyValid(payloadPtr, payloadLen)) { + throw new LineSenderException( + "Invalid ACK response payload [length=" + payloadLen + ']' + ); + } + if (!response.readFrom(payloadPtr, payloadLen)) { + throw new LineSenderException("Failed to parse ACK response"); + } + } + + @Override + public void onClose(int code, String reason) { + throw new LineSenderException("WebSocket closed while waiting for ACK: " + reason); + } + }, 1000); // 1 second timeout per read attempt + + if (received) { + // Non-binary frames (e.g. ping/pong/text) are not ACKs. + if (!sawBinary[0]) { + continue; + } + long sequence = response.getSequence(); + if (response.isSuccess()) { + // Cumulative ACK - acknowledge all batches up to this sequence + inFlightWindow.acknowledgeUpTo(sequence); + if (sequence >= expectedSequence) { + return; // Our batch was acknowledged (cumulative) + } + // Got ACK for lower sequence - continue waiting + } else { + String errorMessage = response.getErrorMessage(); + LineSenderException error = new LineSenderException( + "Server error for batch " + sequence + ": " + + response.getStatusName() + " - " + errorMessage); + inFlightWindow.fail(sequence, error); + if (sequence == expectedSequence) { + throw error; + } + } + } + } catch (LineSenderException e) { + failExpectedIfNeeded(expectedSequence, e); + throw e; + } catch (Exception e) { + LineSenderException wrapped = new LineSenderException("Error waiting for ACK: " + e.getMessage(), e); + failExpectedIfNeeded(expectedSequence, wrapped); + throw wrapped; + } + } + + LineSenderException timeout = new LineSenderException("Timeout waiting for ACK for batch " + expectedSequence); + failExpectedIfNeeded(expectedSequence, timeout); + throw timeout; + } + + private void failExpectedIfNeeded(long expectedSequence, LineSenderException error) { + if (inFlightWindow != null && inFlightWindow.getLastError() == null) { + inFlightWindow.fail(expectedSequence, error); + } + } + + @Override + public DirectByteSlice bufferView() { + throw new LineSenderException("bufferView() is not supported for WebSocket sender"); + } + + @Override + public void cancelRow() { + checkNotClosed(); + if (currentTableBuffer != null) { + currentTableBuffer.cancelCurrentRow(); + } + } + + @Override + public void reset() { + checkNotClosed(); + if (currentTableBuffer != null) { + currentTableBuffer.reset(); + } + } + + // ==================== Array methods ==================== + + @Override + public Sender doubleArray(@NotNull CharSequence name, double[] values) { + if (values == null) return this; + checkNotClosed(); + checkTableSelected(); + IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DOUBLE_ARRAY, true); + col.addDoubleArray(values); + return this; + } + + @Override + public Sender doubleArray(@NotNull CharSequence name, double[][] values) { + if (values == null) return this; + checkNotClosed(); + checkTableSelected(); + IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DOUBLE_ARRAY, true); + col.addDoubleArray(values); + return this; + } + + @Override + public Sender doubleArray(@NotNull CharSequence name, double[][][] values) { + if (values == null) return this; + checkNotClosed(); + checkTableSelected(); + IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DOUBLE_ARRAY, true); + col.addDoubleArray(values); + return this; + } + + @Override + public Sender doubleArray(CharSequence name, DoubleArray array) { + if (array == null) return this; + checkNotClosed(); + checkTableSelected(); + IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DOUBLE_ARRAY, true); + col.addDoubleArray(array); + return this; + } + + @Override + public Sender longArray(@NotNull CharSequence name, long[] values) { + if (values == null) return this; + checkNotClosed(); + checkTableSelected(); + IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_LONG_ARRAY, true); + col.addLongArray(values); + return this; + } + + @Override + public Sender longArray(@NotNull CharSequence name, long[][] values) { + if (values == null) return this; + checkNotClosed(); + checkTableSelected(); + IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_LONG_ARRAY, true); + col.addLongArray(values); + return this; + } + + @Override + public Sender longArray(@NotNull CharSequence name, long[][][] values) { + if (values == null) return this; + checkNotClosed(); + checkTableSelected(); + IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_LONG_ARRAY, true); + col.addLongArray(values); + return this; + } + + @Override + public Sender longArray(CharSequence name, LongArray array) { + if (array == null) return this; + checkNotClosed(); + checkTableSelected(); + IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_LONG_ARRAY, true); + col.addLongArray(array); + return this; + } + + // ==================== Decimal methods ==================== + + @Override + public Sender decimalColumn(CharSequence name, Decimal64 value) { + if (value == null || value.isNull()) return this; + checkNotClosed(); + checkTableSelected(); + IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DECIMAL64, true); + col.addDecimal64(value); + return this; + } + + @Override + public Sender decimalColumn(CharSequence name, Decimal128 value) { + if (value == null || value.isNull()) return this; + checkNotClosed(); + checkTableSelected(); + IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DECIMAL128, true); + col.addDecimal128(value); + return this; + } + + @Override + public Sender decimalColumn(CharSequence name, Decimal256 value) { + if (value == null || value.isNull()) return this; + checkNotClosed(); + checkTableSelected(); + IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DECIMAL256, true); + col.addDecimal256(value); + return this; + } + + @Override + public Sender decimalColumn(CharSequence name, CharSequence value) { + if (value == null || value.length() == 0) return this; + checkNotClosed(); + checkTableSelected(); + try { + java.math.BigDecimal bd = new java.math.BigDecimal(value.toString()); + Decimal256 decimal = Decimal256.fromBigDecimal(bd); + IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DECIMAL256, true); + col.addDecimal256(decimal); + } catch (Exception e) { + throw new LineSenderException("Failed to parse decimal value: " + value, e); + } + return this; + } + + // ==================== Helper methods ==================== + + private long toMicros(long value, ChronoUnit unit) { + switch (unit) { + case NANOS: + return value / 1000L; + case MICROS: + return value; + case MILLIS: + return value * 1000L; + case SECONDS: + return value * 1_000_000L; + case MINUTES: + return value * 60_000_000L; + case HOURS: + return value * 3_600_000_000L; + case DAYS: + return value * 86_400_000_000L; + default: + throw new LineSenderException("Unsupported time unit: " + unit); + } + } + + private void checkNotClosed() { + if (closed) { + throw new LineSenderException("Sender is closed"); + } + } + + private void checkTableSelected() { + if (currentTableBuffer == null) { + throw new LineSenderException("table() must be called before adding columns"); + } + } + + @Override + public void close() { + if (!closed) { + closed = true; + + // Flush any remaining data + try { + if (inFlightWindowSize > 1) { + // Async mode (window > 1): flush accumulated rows in table buffers first + flushPendingRows(); + + if (activeBuffer != null && activeBuffer.hasData()) { + sealAndSwapBuffer(); + } + if (sendQueue != null) { + sendQueue.close(); + } + } else { + // Sync mode (window=1): flush pending rows synchronously + if (pendingRowCount > 0 && client != null && client.isConnected()) { + flushSync(); + } + } + } catch (Exception e) { + LOG.error("Error during close: {}", String.valueOf(e)); + } + + // Close buffers (async mode only, window > 1) + if (buffer0 != null) { + buffer0.close(); + } + if (buffer1 != null) { + buffer1.close(); + } + + if (client != null) { + client.close(); + client = null; + } + encoder.close(); + tableBuffers.clear(); + + LOG.info("IlpV4WebSocketSender closed"); + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/InFlightWindow.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/InFlightWindow.java new file mode 100644 index 0000000..9db5456 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/InFlightWindow.java @@ -0,0 +1,468 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.ilpv4.client; + +import io.questdb.client.cutlass.line.LineSenderException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.locks.LockSupport; + +/** + * Lock-free in-flight batch tracker for the sliding window protocol. + *

+ * Concurrency model (lock-free): + *

+ * Assumptions that keep it simple and lock-free: + * + * With these constraints we can rely on volatile reads/writes (no CAS) and still + * offer blocking waits for space/empty without protecting the counters with locks. + */ +public class InFlightWindow { + + private static final Logger LOG = LoggerFactory.getLogger(InFlightWindow.class); + + public static final int DEFAULT_WINDOW_SIZE = 8; + public static final long DEFAULT_TIMEOUT_MS = 30_000; + + // Spin parameters + private static final int SPIN_TRIES = 100; + private static final long PARK_NANOS = 100_000; // 100 microseconds + + private final int maxWindowSize; + private final long timeoutMs; + + // Core state + // highestSent: the sequence number of the last batch added to the window + private volatile long highestSent = -1; + + // highestAcked: the sequence number of the last acknowledged batch (cumulative) + private volatile long highestAcked = -1; + + // Error state + private final AtomicReference lastError = new AtomicReference<>(); + private volatile long failedBatchId = -1; + + // Thread waiting for space (sender thread) + private volatile Thread waitingForSpace; + + // Thread waiting for empty (flush thread) + private volatile Thread waitingForEmpty; + + // Statistics (not strictly accurate under contention, but good enough for monitoring) + private volatile long totalAcked = 0; + private volatile long totalFailed = 0; + + /** + * Creates a new InFlightWindow with default configuration. + */ + public InFlightWindow() { + this(DEFAULT_WINDOW_SIZE, DEFAULT_TIMEOUT_MS); + } + + /** + * Creates a new InFlightWindow with custom configuration. + * + * @param maxWindowSize maximum number of batches in flight + * @param timeoutMs timeout for blocking operations + */ + public InFlightWindow(int maxWindowSize, long timeoutMs) { + if (maxWindowSize <= 0) { + throw new IllegalArgumentException("maxWindowSize must be positive"); + } + this.maxWindowSize = maxWindowSize; + this.timeoutMs = timeoutMs; + } + + /** + * Checks if there's space in the window for another batch. + * Wait-free operation. + * + * @return true if there's space, false if window is full + */ + public boolean hasWindowSpace() { + return getInFlightCount() < maxWindowSize; + } + + /** + * Tries to add a batch to the in-flight window without blocking. + * Lock-free, assuming single producer for highestSent. + * + * Called by: async producer (WebSocket I/O thread) before sending a batch. + * @param batchId the batch ID to track (must be sequential) + * @return true if added, false if window is full + */ + public boolean tryAddInFlight(long batchId) { + // Check window space first + long sent = highestSent; + long acked = highestAcked; + + if (sent - acked >= maxWindowSize) { + return false; + } + + // Sequential caller: just publish the new highestSent + highestSent = batchId; + + LOG.debug("Added to window [batchId={}, windowSize={}]", batchId, getInFlightCount()); + return true; + } + + /** + * Adds a batch to the in-flight window. + *

+ * Blocks if the window is full until space becomes available or timeout. + * Uses spin-wait with exponential backoff, then parks. Blocking is only expected + * in modes where another actor can make progress on acknowledgments. In normal + * sync usage the window size is 1 and the same thread immediately waits for the + * ACK, so this should never actually park. If a caller uses a larger window here + * it must ensure ACKs are processed on another thread; a single-threaded caller + * with window>1 would deadlock by parking while also being the only thread that + * can advance {@link #acknowledgeUpTo(long)}. + * + * Called by: sync sender thread before sending a batch (window=1). + * @param batchId the batch ID to track + * @throws LineSenderException if timeout occurs or an error was reported + */ + public void addInFlight(long batchId) { + // Check for errors first + checkError(); + + // Fast path: try to add without waiting + if (tryAddInFlightInternal(batchId)) { + return; + } + + // Slow path: need to wait for space + long deadline = System.currentTimeMillis() + timeoutMs; + int spins = 0; + + // Register as waiting thread + waitingForSpace = Thread.currentThread(); + try { + while (true) { + // Check for errors + checkError(); + + // Try to add + if (tryAddInFlightInternal(batchId)) { + return; + } + + // Check timeout + long remaining = deadline - System.currentTimeMillis(); + if (remaining <= 0) { + throw new LineSenderException("Timeout waiting for window space, window full with " + + getInFlightCount() + " batches"); + } + + // Spin or park + if (spins < SPIN_TRIES) { + Thread.onSpinWait(); + spins++; + } else { + // Park with timeout + LockSupport.parkNanos(Math.min(PARK_NANOS, remaining * 1_000_000)); + if (Thread.interrupted()) { + throw new LineSenderException("Interrupted while waiting for window space"); + } + } + } + } finally { + waitingForSpace = null; + } + } + + private boolean tryAddInFlightInternal(long batchId) { + long sent = highestSent; + long acked = highestAcked; + + if (sent - acked >= maxWindowSize) { + return false; + } + + // For sequential IDs, we just update highestSent + // The caller guarantees batchId is the next in sequence + highestSent = batchId; + + LOG.debug("Added to window [batchId={}, windowSize={}]", batchId, getInFlightCount()); + return true; + } + + /** + * Acknowledges a batch, removing it from the in-flight window. + *

+ * For sequential batch IDs, this is a cumulative acknowledgment - + * acknowledging batch N means all batches up to N are acknowledged. + * + * Called by: acker (WebSocket I/O thread) after receiving an ACK. + * @param batchId the batch ID that was acknowledged + * @return true if the batch was in flight, false if already acknowledged + */ + public boolean acknowledge(long batchId) { + return acknowledgeUpTo(batchId) > 0 || highestAcked >= batchId; + } + + /** + * Acknowledges all batches up to and including the given sequence (cumulative ACK). + * Lock-free with single consumer. + * + * Called by: acker (WebSocket I/O thread) after receiving an ACK. + * @param sequence the highest acknowledged sequence + * @return the number of batches acknowledged + */ + public int acknowledgeUpTo(long sequence) { + long sent = highestSent; + + // Nothing to acknowledge if window is empty or sequence is beyond what's sent + if (sent < 0) { + return 0; // No batches have been sent + } + + // Cap sequence at highestSent - can't acknowledge what hasn't been sent + long effectiveSequence = Math.min(sequence, sent); + + long prevAcked = highestAcked; + if (effectiveSequence <= prevAcked) { + // Already acknowledged up to this point + return 0; + } + highestAcked = effectiveSequence; + + int acknowledged = (int) (effectiveSequence - prevAcked); + totalAcked += acknowledged; + + LOG.debug("Cumulative ACK [upTo={}, acknowledged={}, remaining={}]", sequence, acknowledged, getInFlightCount()); + + // Wake up waiting threads + Thread waiter = waitingForSpace; + if (waiter != null) { + LockSupport.unpark(waiter); + } + + waiter = waitingForEmpty; + if (waiter != null && getInFlightCount() == 0) { + LockSupport.unpark(waiter); + } + + return acknowledged; + } + + /** + * Marks a batch as failed, setting an error that will be propagated to waiters. + * + * Called by: acker (WebSocket I/O thread) on error response or send failure. + * @param batchId the batch ID that failed + * @param error the error that occurred + */ + public void fail(long batchId, Throwable error) { + this.failedBatchId = batchId; + this.lastError.set(error); + totalFailed++; + + LOG.error("Batch failed [batchId={}, error={}]", batchId, String.valueOf(error)); + + wakeWaiters(); + } + + /** + * Marks all currently in-flight batches as failed. + *

+ * Used for transport-level failures (disconnect/protocol violation) where + * no further ACKs are expected and all waiters must be released. + * + * @param error terminal error to propagate + */ + public void failAll(Throwable error) { + long sent = highestSent; + long acked = highestAcked; + long inFlight = Math.max(0, sent - acked); + + this.failedBatchId = sent; + this.lastError.set(error); + totalFailed += Math.max(1, inFlight); + + LOG.error("All in-flight batches failed [inFlight={}, error={}]", inFlight, String.valueOf(error)); + + wakeWaiters(); + } + + /** + * Waits until all in-flight batches are acknowledged. + *

+ * Called by flush() to ensure all data is confirmed. + * + * Called by: waiter (flush thread), while producer/acker thread progresses. + * @throws LineSenderException if timeout occurs or an error was reported + */ + public void awaitEmpty() { + checkError(); + + // Fast path: already empty + if (getInFlightCount() == 0) { + LOG.debug("Window already empty"); + return; + } + + long deadline = System.currentTimeMillis() + timeoutMs; + int spins = 0; + + // Register as waiting thread + waitingForEmpty = Thread.currentThread(); + try { + while (getInFlightCount() > 0) { + checkError(); + + long remaining = deadline - System.currentTimeMillis(); + if (remaining <= 0) { + throw new LineSenderException("Timeout waiting for batch acknowledgments, " + + getInFlightCount() + " batches still in flight"); + } + + if (spins < SPIN_TRIES) { + Thread.onSpinWait(); + spins++; + } else { + LockSupport.parkNanos(Math.min(PARK_NANOS, remaining * 1_000_000)); + if (Thread.interrupted()) { + throw new LineSenderException("Interrupted while waiting for acknowledgments"); + } + } + } + + LOG.debug("Window empty, all batches ACKed"); + } finally { + waitingForEmpty = null; + } + } + + /** + * Returns the current number of batches in flight. + * Wait-free operation. + */ + public int getInFlightCount() { + long sent = highestSent; + long acked = highestAcked; + // Ensure non-negative (can happen during initialization) + return (int) Math.max(0, sent - acked); + } + + /** + * Returns true if the window is empty. + * Wait-free operation. + */ + public boolean isEmpty() { + return getInFlightCount() == 0; + } + + /** + * Returns true if the window is full. + * Wait-free operation. + */ + public boolean isFull() { + return getInFlightCount() >= maxWindowSize; + } + + /** + * Returns the maximum window size. + */ + public int getMaxWindowSize() { + return maxWindowSize; + } + + /** + * Returns the total number of batches acknowledged. + */ + public long getTotalAcked() { + return totalAcked; + } + + /** + * Returns the total number of batches that failed. + */ + public long getTotalFailed() { + return totalFailed; + } + + /** + * Returns the last error, or null if no error. + */ + public Throwable getLastError() { + return lastError.get(); + } + + /** + * Clears the error state. + */ + public void clearError() { + lastError.set(null); + failedBatchId = -1; + } + + /** + * Resets the window, clearing all state. + */ + public void reset() { + highestSent = -1; + highestAcked = -1; + lastError.set(null); + failedBatchId = -1; + + wakeWaiters(); + } + + private void checkError() { + Throwable error = lastError.get(); + if (error != null) { + throw new LineSenderException("Batch " + failedBatchId + " failed: " + error.getMessage(), error); + } + } + + private void wakeWaiters() { + Thread waiter = waitingForSpace; + if (waiter != null) { + LockSupport.unpark(waiter); + } + waiter = waitingForEmpty; + if (waiter != null) { + LockSupport.unpark(waiter); + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/MicrobatchBuffer.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/MicrobatchBuffer.java new file mode 100644 index 0000000..832bb66 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/MicrobatchBuffer.java @@ -0,0 +1,501 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.ilpv4.client; + +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.QuietCloseable; +import io.questdb.client.std.Unsafe; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +/** + * A buffer for accumulating ILP data into microbatches before sending. + *

+ * This class implements a state machine for buffer lifecycle management in the + * double-buffering scheme used by {@link IlpV4WebSocketSender}: + * 

+ * Buffer States:
+ * ┌─────────────┐    seal()     ┌─────────────┐    markSending()  ┌─────────────┐
+ * │   FILLING   │──────────────►│   SEALED    │──────────────────►│   SENDING   │
+ * │ (user owns) │               │ (in queue)  │                   │ (I/O owns)  │
+ * └─────────────┘               └─────────────┘                   └──────┬──────┘
+ *        ▲                                                               │
+ *        │                         markRecycled()                        │
+ *        └───────────────────────────────────────────────────────────────┘
+ *                              (after send complete)
+ *
+ *

+ * Thread safety: This class is NOT thread-safe for concurrent writes. However, it + * supports safe hand-over between user thread and I/O thread through the state + * machine. State transitions use volatile fields to ensure visibility. + */ +public class MicrobatchBuffer implements QuietCloseable { + + // Buffer states + public static final int STATE_FILLING = 0; + public static final int STATE_SEALED = 1; + public static final int STATE_SENDING = 2; + public static final int STATE_RECYCLED = 3; + + // Flush trigger thresholds + private final int maxRows; + private final int maxBytes; + private final long maxAgeNanos; + + // Native memory buffer + private long bufferPtr; + private int bufferCapacity; + private int bufferPos; + + // Row tracking + private int rowCount; + private long firstRowTimeNanos; + + // Symbol tracking for delta encoding + private int maxSymbolId = -1; + + // Batch identification + private long batchId; + private static long nextBatchId = 0; + + // State machine + private volatile int state = STATE_FILLING; + + // For waiting on recycle (user thread waits for I/O thread to finish) + // CountDownLatch is not resettable, so we create a new instance on reset() + private volatile CountDownLatch recycleLatch = new CountDownLatch(1); + + /** + * Creates a new MicrobatchBuffer with specified flush thresholds. + * + * @param initialCapacity initial buffer size in bytes + * @param maxRows maximum rows before auto-flush (0 = unlimited) + * @param maxBytes maximum bytes before auto-flush (0 = unlimited) + * @param maxAgeNanos maximum age in nanoseconds before auto-flush (0 = unlimited) + */ + public MicrobatchBuffer(int initialCapacity, int maxRows, int maxBytes, long maxAgeNanos) { + if (initialCapacity <= 0) { + throw new IllegalArgumentException("initialCapacity must be positive"); + } + this.bufferCapacity = initialCapacity; + this.bufferPtr = Unsafe.malloc(initialCapacity, MemoryTag.NATIVE_ILP_RSS); + this.bufferPos = 0; + this.rowCount = 0; + this.firstRowTimeNanos = 0; + this.maxRows = maxRows; + this.maxBytes = maxBytes; + this.maxAgeNanos = maxAgeNanos; + this.batchId = nextBatchId++; + } + + /** + * Creates a new MicrobatchBuffer with default thresholds (no auto-flush). + * + * @param initialCapacity initial buffer size in bytes + */ + public MicrobatchBuffer(int initialCapacity) { + this(initialCapacity, 0, 0, 0); + } + + // ==================== DATA OPERATIONS ==================== + + /** + * Returns the buffer pointer for writing data. + * Only valid when state is FILLING. + */ + public long getBufferPtr() { + return bufferPtr; + } + + /** + * Returns the current write position in the buffer. + */ + public int getBufferPos() { + return bufferPos; + } + + /** + * Returns the buffer capacity. + */ + public int getBufferCapacity() { + return bufferCapacity; + } + + /** + * Sets the buffer position after external writes. + * Only valid when state is FILLING. + * + * @param pos new position + */ + public void setBufferPos(int pos) { + if (state != STATE_FILLING) { + throw new IllegalStateException("Cannot set position when state is " + stateName(state)); + } + if (pos < 0 || pos > bufferCapacity) { + throw new IllegalArgumentException("Position out of bounds: " + pos); + } + this.bufferPos = pos; + } + + /** + * Ensures the buffer has at least the specified capacity. + * Grows the buffer if necessary. + * + * @param requiredCapacity minimum required capacity + */ + public void ensureCapacity(int requiredCapacity) { + if (state != STATE_FILLING) { + throw new IllegalStateException("Cannot resize when state is " + stateName(state)); + } + if (requiredCapacity > bufferCapacity) { + int newCapacity = Math.max(bufferCapacity * 2, requiredCapacity); + long newPtr = Unsafe.realloc(bufferPtr, bufferCapacity, newCapacity, MemoryTag.NATIVE_ILP_RSS); + bufferPtr = newPtr; + bufferCapacity = newCapacity; + } + } + + /** + * Writes bytes to the buffer at the current position. + * Grows the buffer if necessary. + * + * @param src source address + * @param length number of bytes to write + */ + public void write(long src, int length) { + if (state != STATE_FILLING) { + throw new IllegalStateException("Cannot write when state is " + stateName(state)); + } + ensureCapacity(bufferPos + length); + Unsafe.getUnsafe().copyMemory(src, bufferPtr + bufferPos, length); + bufferPos += length; + } + + /** + * Writes a single byte to the buffer. + * + * @param b byte to write + */ + public void writeByte(byte b) { + if (state != STATE_FILLING) { + throw new IllegalStateException("Cannot write when state is " + stateName(state)); + } + ensureCapacity(bufferPos + 1); + Unsafe.getUnsafe().putByte(bufferPtr + bufferPos, b); + bufferPos++; + } + + /** + * Increments the row count and records the first row time if this is the first row. + */ + public void incrementRowCount() { + if (state != STATE_FILLING) { + throw new IllegalStateException("Cannot increment row count when state is " + stateName(state)); + } + if (rowCount == 0) { + firstRowTimeNanos = System.nanoTime(); + } + rowCount++; + } + + /** + * Returns the number of rows in this buffer. + */ + public int getRowCount() { + return rowCount; + } + + /** + * Returns true if the buffer has any data. + */ + public boolean hasData() { + return bufferPos > 0; + } + + /** + * Returns the batch ID for this buffer. + */ + public long getBatchId() { + return batchId; + } + + /** + * Returns the maximum symbol ID used in this batch. + * Used for delta symbol dictionary tracking. + */ + public int getMaxSymbolId() { + return maxSymbolId; + } + + /** + * Sets the maximum symbol ID used in this batch. + * Used for delta symbol dictionary tracking. + */ + public void setMaxSymbolId(int maxSymbolId) { + this.maxSymbolId = maxSymbolId; + } + + // ==================== FLUSH TRIGGER CHECKS ==================== + + /** + * Checks if the buffer should be flushed based on configured thresholds. + * + * @return true if any flush threshold is exceeded + */ + public boolean shouldFlush() { + if (!hasData()) { + return false; + } + return isRowLimitExceeded() || isByteLimitExceeded() || isAgeLimitExceeded(); + } + + /** + * Checks if the row count limit has been exceeded. + */ + public boolean isRowLimitExceeded() { + return maxRows > 0 && rowCount >= maxRows; + } + + /** + * Checks if the byte size limit has been exceeded. + */ + public boolean isByteLimitExceeded() { + return maxBytes > 0 && bufferPos >= maxBytes; + } + + /** + * Checks if the age limit has been exceeded. + */ + public boolean isAgeLimitExceeded() { + if (maxAgeNanos <= 0 || rowCount == 0) { + return false; + } + long ageNanos = System.nanoTime() - firstRowTimeNanos; + return ageNanos >= maxAgeNanos; + } + + /** + * Returns the age of the first row in nanoseconds, or 0 if no rows. + */ + public long getAgeNanos() { + if (rowCount == 0) { + return 0; + } + return System.nanoTime() - firstRowTimeNanos; + } + + // ==================== STATE MACHINE ==================== + + /** + * Returns the current state. + */ + public int getState() { + return state; + } + + /** + * Returns true if the buffer is in FILLING state (available for writing). + */ + public boolean isFilling() { + return state == STATE_FILLING; + } + + /** + * Returns true if the buffer is in SEALED state (ready to send). + */ + public boolean isSealed() { + return state == STATE_SEALED; + } + + /** + * Returns true if the buffer is in SENDING state (being sent by I/O thread). + */ + public boolean isSending() { + return state == STATE_SENDING; + } + + /** + * Returns true if the buffer is in RECYCLED state (available for reset). + */ + public boolean isRecycled() { + return state == STATE_RECYCLED; + } + + /** + * Returns true if the buffer is currently in use (not available for the user thread). + */ + public boolean isInUse() { + int s = state; + return s == STATE_SEALED || s == STATE_SENDING; + } + + /** + * Seals the buffer, transitioning from FILLING to SEALED. + * After sealing, no more data can be written. + * Only the user thread should call this. + * + * @throws IllegalStateException if not in FILLING state + */ + public void seal() { + if (state != STATE_FILLING) { + throw new IllegalStateException("Cannot seal buffer in state " + stateName(state)); + } + state = STATE_SEALED; + } + + /** + * Rolls back a seal operation, transitioning from SEALED back to FILLING. + *

+ * Used when enqueue fails after a buffer has been sealed but before ownership + * was transferred to the I/O thread. + * + * @throws IllegalStateException if not in SEALED state + */ + public void rollbackSealForRetry() { + if (state != STATE_SEALED) { + throw new IllegalStateException("Cannot rollback seal in state " + stateName(state)); + } + state = STATE_FILLING; + } + + /** + * Marks the buffer as being sent, transitioning from SEALED to SENDING. + * Only the I/O thread should call this. + * + * @throws IllegalStateException if not in SEALED state + */ + public void markSending() { + if (state != STATE_SEALED) { + throw new IllegalStateException("Cannot mark sending in state " + stateName(state)); + } + state = STATE_SENDING; + } + + /** + * Marks the buffer as recycled, transitioning from SENDING to RECYCLED. + * This signals to the user thread that the buffer can be reused. + * Only the I/O thread should call this. + * + * @throws IllegalStateException if not in SENDING state + */ + public void markRecycled() { + if (state != STATE_SENDING) { + throw new IllegalStateException("Cannot mark recycled in state " + stateName(state)); + } + state = STATE_RECYCLED; + recycleLatch.countDown(); + } + + /** + * Waits for the buffer to be recycled (transition to RECYCLED state). + * Only the user thread should call this. + */ + public void awaitRecycled() { + try { + recycleLatch.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + /** + * Waits for the buffer to be recycled with a timeout. + * + * @param timeout the maximum time to wait + * @param unit the time unit + * @return true if recycled, false if timeout elapsed + */ + public boolean awaitRecycled(long timeout, TimeUnit unit) { + try { + return recycleLatch.await(timeout, unit); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } + } + + /** + * Resets the buffer to FILLING state, clearing all data. + * Only valid when in RECYCLED state or when the buffer is fresh. + * + * @throws IllegalStateException if in SEALED or SENDING state + */ + public void reset() { + int s = state; + if (s == STATE_SEALED || s == STATE_SENDING) { + throw new IllegalStateException("Cannot reset buffer in state " + stateName(s)); + } + bufferPos = 0; + rowCount = 0; + firstRowTimeNanos = 0; + maxSymbolId = -1; + batchId = nextBatchId++; + state = STATE_FILLING; + recycleLatch = new CountDownLatch(1); + } + + // ==================== LIFECYCLE ==================== + + @Override + public void close() { + if (bufferPtr != 0) { + Unsafe.free(bufferPtr, bufferCapacity, MemoryTag.NATIVE_ILP_RSS); + bufferPtr = 0; + bufferCapacity = 0; + } + } + + // ==================== UTILITIES ==================== + + /** + * Returns a human-readable name for the given state. + */ + public static String stateName(int state) { + switch (state) { + case STATE_FILLING: + return "FILLING"; + case STATE_SEALED: + return "SEALED"; + case STATE_SENDING: + return "SENDING"; + case STATE_RECYCLED: + return "RECYCLED"; + default: + return "UNKNOWN(" + state + ")"; + } + } + + @Override + public String toString() { + return "MicrobatchBuffer{" + + "batchId=" + batchId + + ", state=" + stateName(state) + + ", rows=" + rowCount + + ", bytes=" + bufferPos + + ", capacity=" + bufferCapacity + + '}'; + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/NativeBufferWriter.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/NativeBufferWriter.java new file mode 100644 index 0000000..5cab035 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/NativeBufferWriter.java @@ -0,0 +1,289 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.ilpv4.client; + +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.QuietCloseable; +import io.questdb.client.std.Unsafe; + +/** + * A simple native memory buffer writer for encoding ILP v4 messages. + *

+ * This class provides write methods similar to HttpClient.Request but writes + * to a native memory buffer that can be sent over WebSocket. + *

+ * All multi-byte values are written in little-endian format unless otherwise specified. + */ +public class NativeBufferWriter implements IlpBufferWriter, QuietCloseable { + + private static final int DEFAULT_CAPACITY = 8192; + + private long bufferPtr; + private int capacity; + private int position; + + public NativeBufferWriter() { + this(DEFAULT_CAPACITY); + } + + public NativeBufferWriter(int initialCapacity) { + this.capacity = initialCapacity; + this.bufferPtr = Unsafe.malloc(capacity, MemoryTag.NATIVE_DEFAULT); + this.position = 0; + } + + /** + * Returns the buffer pointer. + */ + @Override + public long getBufferPtr() { + return bufferPtr; + } + + /** + * Returns the current write position (number of bytes written). + */ + @Override + public int getPosition() { + return position; + } + + /** + * Resets the buffer for reuse. + */ + @Override + public void reset() { + position = 0; + } + + /** + * Writes a single byte. + */ + @Override + public void putByte(byte value) { + ensureCapacity(1); + Unsafe.getUnsafe().putByte(bufferPtr + position, value); + position++; + } + + /** + * Writes a short (2 bytes, little-endian). + */ + @Override + public void putShort(short value) { + ensureCapacity(2); + Unsafe.getUnsafe().putShort(bufferPtr + position, value); + position += 2; + } + + /** + * Writes an int (4 bytes, little-endian). + */ + @Override + public void putInt(int value) { + ensureCapacity(4); + Unsafe.getUnsafe().putInt(bufferPtr + position, value); + position += 4; + } + + /** + * Writes a long (8 bytes, little-endian). + */ + @Override + public void putLong(long value) { + ensureCapacity(8); + Unsafe.getUnsafe().putLong(bufferPtr + position, value); + position += 8; + } + + /** + * Writes a long in big-endian order. + */ + @Override + public void putLongBE(long value) { + ensureCapacity(8); + Unsafe.getUnsafe().putLong(bufferPtr + position, Long.reverseBytes(value)); + position += 8; + } + + /** + * Writes a float (4 bytes, little-endian). + */ + @Override + public void putFloat(float value) { + ensureCapacity(4); + Unsafe.getUnsafe().putFloat(bufferPtr + position, value); + position += 4; + } + + /** + * Writes a double (8 bytes, little-endian). + */ + @Override + public void putDouble(double value) { + ensureCapacity(8); + Unsafe.getUnsafe().putDouble(bufferPtr + position, value); + position += 8; + } + + /** + * Writes a block of bytes from native memory. + */ + @Override + public void putBlockOfBytes(long from, long len) { + ensureCapacity((int) len); + Unsafe.getUnsafe().copyMemory(from, bufferPtr + position, len); + position += (int) len; + } + + /** + * Writes a varint (unsigned LEB128). + */ + @Override + public void putVarint(long value) { + while (value > 0x7F) { + putByte((byte) ((value & 0x7F) | 0x80)); + value >>>= 7; + } + putByte((byte) value); + } + + /** + * Writes a length-prefixed UTF-8 string. + */ + @Override + public void putString(String value) { + if (value == null || value.isEmpty()) { + putVarint(0); + return; + } + + int utf8Len = utf8Length(value); + putVarint(utf8Len); + putUtf8(value); + } + + /** + * Writes UTF-8 bytes directly without length prefix. + */ + @Override + public void putUtf8(String value) { + if (value == null || value.isEmpty()) { + return; + } + for (int i = 0, n = value.length(); i < n; i++) { + char c = value.charAt(i); + if (c < 0x80) { + putByte((byte) c); + } else if (c < 0x800) { + putByte((byte) (0xC0 | (c >> 6))); + putByte((byte) (0x80 | (c & 0x3F))); + } else if (c >= 0xD800 && c <= 0xDBFF && i + 1 < n) { + char c2 = value.charAt(++i); + int codePoint = 0x10000 + ((c - 0xD800) << 10) + (c2 - 0xDC00); + putByte((byte) (0xF0 | (codePoint >> 18))); + putByte((byte) (0x80 | ((codePoint >> 12) & 0x3F))); + putByte((byte) (0x80 | ((codePoint >> 6) & 0x3F))); + putByte((byte) (0x80 | (codePoint & 0x3F))); + } else { + putByte((byte) (0xE0 | (c >> 12))); + putByte((byte) (0x80 | ((c >> 6) & 0x3F))); + putByte((byte) (0x80 | (c & 0x3F))); + } + } + } + + /** + * Returns the UTF-8 encoded length of a string. + */ + public static int utf8Length(String s) { + if (s == null) return 0; + int len = 0; + for (int i = 0, n = s.length(); i < n; i++) { + char c = s.charAt(i); + if (c < 0x80) { + len++; + } else if (c < 0x800) { + len += 2; + } else if (c >= 0xD800 && c <= 0xDBFF && i + 1 < n) { + i++; + len += 4; + } else { + len += 3; + } + } + return len; + } + + /** + * Patches an int value at the specified offset. + * Used for updating length fields after writing content. + */ + @Override + public void patchInt(int offset, int value) { + Unsafe.getUnsafe().putInt(bufferPtr + offset, value); + } + + /** + * Returns the current buffer capacity. + */ + @Override + public int getCapacity() { + return capacity; + } + + /** + * Skips the specified number of bytes, advancing the position. + * Used when data has been written directly to the buffer via getBufferPtr(). + * + * @param bytes number of bytes to skip + */ + @Override + public void skip(int bytes) { + position += bytes; + } + + /** + * Ensures the buffer has at least the specified additional capacity. + * + * @param needed additional bytes needed beyond current position + */ + @Override + public void ensureCapacity(int needed) { + if (position + needed > capacity) { + int newCapacity = Math.max(capacity * 2, position + needed); + bufferPtr = Unsafe.realloc(bufferPtr, capacity, newCapacity, MemoryTag.NATIVE_DEFAULT); + capacity = newCapacity; + } + } + + @Override + public void close() { + if (bufferPtr != 0) { + Unsafe.free(bufferPtr, capacity, MemoryTag.NATIVE_DEFAULT); + bufferPtr = 0; + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/ResponseReader.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/ResponseReader.java new file mode 100644 index 0000000..dca05f2 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/ResponseReader.java @@ -0,0 +1,247 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.ilpv4.client; + +import io.questdb.client.cutlass.line.LineSenderException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.QuietCloseable; +import io.questdb.client.std.Unsafe; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Reads server responses from WebSocket channel and updates InFlightWindow. + *

+ * This class runs a dedicated thread that: + *

    + *
  • Reads WebSocket frames from the server
    • + *
  • Parses binary responses containing ACK/error status
    • + *
  • Updates the InFlightWindow with acknowledgments or failures
    • + *
+ *

+ * Thread safety: This class is thread-safe. The reader thread processes + * responses independently of the sender thread. + */ +public class ResponseReader implements QuietCloseable { + + private static final Logger LOG = LoggerFactory.getLogger(ResponseReader.class); + + private static final int DEFAULT_READ_TIMEOUT_MS = 100; + private static final long DEFAULT_SHUTDOWN_TIMEOUT_MS = 5_000; + + private final WebSocketChannel channel; + private final InFlightWindow inFlightWindow; + private final Thread readerThread; + private final CountDownLatch shutdownLatch; + private final WebSocketResponse response; + + // Buffer for parsing responses + private final long parseBufferPtr; + private final int parseBufferSize; + + // State + private volatile boolean running; + private volatile Throwable lastError; + + // Statistics + private final AtomicLong totalAcks = new AtomicLong(0); + private final AtomicLong totalErrors = new AtomicLong(0); + + /** + * Creates a new response reader. + * + * @param channel the WebSocket channel to read from + * @param inFlightWindow the window to update with acknowledgments + */ + public ResponseReader(WebSocketChannel channel, InFlightWindow inFlightWindow) { + if (channel == null) { + throw new IllegalArgumentException("channel cannot be null"); + } + if (inFlightWindow == null) { + throw new IllegalArgumentException("inFlightWindow cannot be null"); + } + + this.channel = channel; + this.inFlightWindow = inFlightWindow; + this.response = new WebSocketResponse(); + + // Allocate parse buffer (enough for max response) + this.parseBufferSize = 2048; + this.parseBufferPtr = Unsafe.malloc(parseBufferSize, MemoryTag.NATIVE_DEFAULT); + + this.running = true; + this.shutdownLatch = new CountDownLatch(1); + + // Start reader thread + this.readerThread = new Thread(this::readLoop, "questdb-websocket-response-reader"); + this.readerThread.setDaemon(true); + this.readerThread.start(); + + LOG.info("Response reader started"); + } + + /** + * Returns the last error that occurred, or null if no error. + */ + public Throwable getLastError() { + return lastError; + } + + /** + * Returns true if the reader is still running. + */ + public boolean isRunning() { + return running; + } + + /** + * Returns total successful acknowledgments received. + */ + public long getTotalAcks() { + return totalAcks.get(); + } + + /** + * Returns total error responses received. + */ + public long getTotalErrors() { + return totalErrors.get(); + } + + @Override + public void close() { + if (!running) { + return; + } + + LOG.info("Closing response reader"); + + running = false; + + // Wait for reader thread to finish + try { + shutdownLatch.await(DEFAULT_SHUTDOWN_TIMEOUT_MS, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + + // Free parse buffer + if (parseBufferPtr != 0) { + Unsafe.free(parseBufferPtr, parseBufferSize, MemoryTag.NATIVE_DEFAULT); + } + + LOG.info("Response reader closed [totalAcks={}, totalErrors={}]", totalAcks.get(), totalErrors.get()); + } + + // ==================== Reader Thread ==================== + + /** + * Main read loop that processes incoming WebSocket frames. + */ + private void readLoop() { + LOG.info("Read loop started"); + + try { + while (running && channel.isConnected()) { + try { + // Non-blocking read with short timeout + boolean received = channel.receiveFrame(new ResponseHandlerImpl(), DEFAULT_READ_TIMEOUT_MS); + if (!received) { + // No frame available, continue polling + continue; + } + } catch (LineSenderException e) { + if (running) { + LOG.error("Error reading response: {}", e.getMessage()); + lastError = e; + } + // Continue trying to read unless we're shutting down + } catch (Throwable t) { + if (running) { + LOG.error("Unexpected error in read loop: {}", String.valueOf(t)); + lastError = t; + } + break; + } + } + } finally { + shutdownLatch.countDown(); + LOG.info("Read loop stopped"); + } + } + + /** + * Handler for received WebSocket frames. + */ + private class ResponseHandlerImpl implements WebSocketChannel.ResponseHandler { + + @Override + public void onBinaryMessage(long payload, int length) { + if (length < WebSocketResponse.MIN_RESPONSE_SIZE) { + LOG.error("Response too short [length={}]", length); + return; + } + + // Parse response from binary payload + if (!response.readFrom(payload, length)) { + LOG.error("Failed to parse response"); + return; + } + + long sequence = response.getSequence(); + + if (response.isSuccess()) { + // Cumulative ACK - acknowledge all batches up to this sequence + int acked = inFlightWindow.acknowledgeUpTo(sequence); + if (acked > 0) { + totalAcks.addAndGet(acked); + LOG.debug("Cumulative ACK received [upTo={}, acked={}]", sequence, acked); + } else { + LOG.debug("ACK for already-acknowledged sequences [upTo={}]", sequence); + } + } else { + // Error - fail the batch + String errorMessage = response.getErrorMessage(); + LOG.error("Error response [seq={}, status={}, error={}]", sequence, response.getStatusName(), errorMessage); + + LineSenderException error = new LineSenderException( + "Server error for batch " + sequence + ": " + + response.getStatusName() + " - " + errorMessage); + inFlightWindow.fail(sequence, error); + totalErrors.incrementAndGet(); + } + } + + @Override + public void onClose(int code, String reason) { + LOG.info("WebSocket closed by server [code={}, reason={}]", code, reason); + running = false; + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/WebSocketChannel.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/WebSocketChannel.java new file mode 100644 index 0000000..f5be4a4 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/WebSocketChannel.java @@ -0,0 +1,668 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.ilpv4.client; + +import io.questdb.client.cutlass.ilpv4.websocket.WebSocketCloseCode; +import io.questdb.client.cutlass.ilpv4.websocket.WebSocketFrameParser; +import io.questdb.client.cutlass.ilpv4.websocket.WebSocketFrameWriter; +import io.questdb.client.cutlass.ilpv4.websocket.WebSocketHandshake; +import io.questdb.client.cutlass.ilpv4.websocket.WebSocketOpcode; +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.QuietCloseable; +import io.questdb.client.std.Rnd; +import io.questdb.client.std.Unsafe; + +import javax.net.SocketFactory; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLSocketFactory; +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509TrustManager; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.Socket; +import java.net.SocketTimeoutException; +import java.nio.charset.StandardCharsets; +import java.security.SecureRandom; +import java.security.cert.X509Certificate; +import java.util.Base64; + +/** + * WebSocket client channel for ILP v4 binary streaming. + *

+ * This class handles: + *

    + *
  • HTTP upgrade handshake to establish WebSocket connection
    • + *
  • Binary frame encoding with client-side masking (RFC 6455)
    • + *
  • Ping/pong for connection keepalive
    • + *
  • Close handshake
    • + *
+ *

+ * Thread safety: This class is NOT thread-safe. It should only be accessed + * from a single thread at a time. + */ +public class WebSocketChannel implements QuietCloseable { + + private static final int DEFAULT_BUFFER_SIZE = 65536; + private static final int MAX_FRAME_HEADER_SIZE = 14; // 2 + 8 + 4 (header + extended len + mask) + + // Connection state + private final String host; + private final int port; + private final String path; + private final boolean tlsEnabled; + private final boolean tlsValidationEnabled; + + // Socket I/O + private Socket socket; + private InputStream in; + private OutputStream out; + + // Pre-allocated send buffer (native memory) + private long sendBufferPtr; + private int sendBufferSize; + + // Pre-allocated receive buffer (native memory) + private long recvBufferPtr; + private int recvBufferSize; + private int recvBufferPos; // Write position + private int recvBufferReadPos; // Read position + + // Frame parser (reused) + private final WebSocketFrameParser frameParser; + + // Random for mask key generation + private final Rnd rnd; + + // Timeouts + private int connectTimeoutMs = 10_000; + private int readTimeoutMs = 30_000; + + // State + private boolean connected; + private boolean closed; + + // Temporary byte array for handshake (allocated once) + private final byte[] handshakeBuffer = new byte[4096]; + + public WebSocketChannel(String url, boolean tlsEnabled) { + this(url, tlsEnabled, true); + } + + public WebSocketChannel(String url, boolean tlsEnabled, boolean tlsValidationEnabled) { + // Parse URL: ws://host:port/path or wss://host:port/path + String remaining = url; + if (remaining.startsWith("wss://")) { + remaining = remaining.substring(6); + this.tlsEnabled = true; + } else if (remaining.startsWith("ws://")) { + remaining = remaining.substring(5); + this.tlsEnabled = tlsEnabled; + } else { + this.tlsEnabled = tlsEnabled; + } + + int slashIdx = remaining.indexOf('/'); + String hostPort; + if (slashIdx >= 0) { + hostPort = remaining.substring(0, slashIdx); + this.path = remaining.substring(slashIdx); + } else { + hostPort = remaining; + this.path = "/"; + } + + int colonIdx = hostPort.lastIndexOf(':'); + if (colonIdx >= 0) { + this.host = hostPort.substring(0, colonIdx); + this.port = Integer.parseInt(hostPort.substring(colonIdx + 1)); + } else { + this.host = hostPort; + this.port = this.tlsEnabled ? 443 : 80; + } + + this.tlsValidationEnabled = tlsValidationEnabled; + this.frameParser = new WebSocketFrameParser(); + this.rnd = new Rnd(System.nanoTime(), System.currentTimeMillis()); + + // Allocate native buffers + this.sendBufferSize = DEFAULT_BUFFER_SIZE; + this.sendBufferPtr = Unsafe.malloc(sendBufferSize, MemoryTag.NATIVE_DEFAULT); + + this.recvBufferSize = DEFAULT_BUFFER_SIZE; + this.recvBufferPtr = Unsafe.malloc(recvBufferSize, MemoryTag.NATIVE_DEFAULT); + this.recvBufferPos = 0; + this.recvBufferReadPos = 0; + + this.connected = false; + this.closed = false; + } + + /** + * Sets the connection timeout. + */ + public WebSocketChannel setConnectTimeout(int timeoutMs) { + this.connectTimeoutMs = timeoutMs; + return this; + } + + /** + * Sets the read timeout. + */ + public WebSocketChannel setReadTimeout(int timeoutMs) { + this.readTimeoutMs = timeoutMs; + return this; + } + + /** + * Connects to the WebSocket server. + * Performs TCP connection and HTTP upgrade handshake. + */ + public void connect() { + if (connected) { + return; + } + if (closed) { + throw new LineSenderException("WebSocket channel is closed"); + } + + try { + // Create socket + SocketFactory socketFactory = tlsEnabled ? createSslSocketFactory() : SocketFactory.getDefault(); + socket = socketFactory.createSocket(); + socket.connect(new java.net.InetSocketAddress(host, port), connectTimeoutMs); + socket.setSoTimeout(readTimeoutMs); + socket.setTcpNoDelay(true); + + in = socket.getInputStream(); + out = socket.getOutputStream(); + + // Perform WebSocket handshake + performHandshake(); + + connected = true; + } catch (IOException e) { + closeQuietly(); + throw new LineSenderException("Failed to connect to WebSocket server: " + e.getMessage(), e); + } + } + + /** + * Sends binary data as a WebSocket binary frame. + * The data is read from native memory at the given pointer. + * + * @param dataPtr pointer to the data + * @param length length of data in bytes + */ + public void sendBinary(long dataPtr, int length) { + ensureConnected(); + sendFrame(WebSocketOpcode.BINARY, dataPtr, length); + } + + /** + * Sends a ping frame. + */ + public void sendPing() { + ensureConnected(); + sendFrame(WebSocketOpcode.PING, 0, 0); + } + + /** + * Receives and processes incoming frames. + * Handles ping/pong automatically. + * + * @param handler callback for received binary messages (may be null) + * @param timeoutMs read timeout in milliseconds + * @return true if a frame was received, false on timeout + */ + public boolean receiveFrame(ResponseHandler handler, int timeoutMs) { + ensureConnected(); + try { + int oldTimeout = socket.getSoTimeout(); + socket.setSoTimeout(timeoutMs); + try { + return doReceiveFrame(handler); + } finally { + socket.setSoTimeout(oldTimeout); + } + } catch (SocketTimeoutException e) { + return false; + } catch (IOException e) { + throw new LineSenderException("Failed to receive WebSocket frame: " + e.getMessage(), e); + } + } + + /** + * Sends a close frame and closes the connection. + */ + @Override + public void close() { + if (closed) { + return; + } + closed = true; + + try { + if (connected) { + // Send close frame + sendCloseFrame(WebSocketCloseCode.NORMAL_CLOSURE, null); + } + } catch (Exception e) { + // Ignore errors during close + } + + closeQuietly(); + + // Free native memory + if (sendBufferPtr != 0) { + Unsafe.free(sendBufferPtr, sendBufferSize, MemoryTag.NATIVE_DEFAULT); + sendBufferPtr = 0; + } + if (recvBufferPtr != 0) { + Unsafe.free(recvBufferPtr, recvBufferSize, MemoryTag.NATIVE_DEFAULT); + recvBufferPtr = 0; + } + } + + public boolean isConnected() { + return connected && !closed; + } + + // ==================== Private methods ==================== + + private void ensureConnected() { + if (closed) { + throw new LineSenderException("WebSocket channel is closed"); + } + if (!connected) { + throw new LineSenderException("WebSocket channel is not connected"); + } + } + + private SocketFactory createSslSocketFactory() { + try { + if (!tlsValidationEnabled) { + SSLContext sslContext = SSLContext.getInstance("TLS"); + sslContext.init(null, new TrustManager[]{new X509TrustManager() { + public void checkClientTrusted(X509Certificate[] certs, String t) {} + public void checkServerTrusted(X509Certificate[] certs, String t) {} + public X509Certificate[] getAcceptedIssuers() { return null; } + }}, new SecureRandom()); + return sslContext.getSocketFactory(); + } + return SSLSocketFactory.getDefault(); + } catch (Exception e) { + throw new LineSenderException("Failed to create SSL socket factory: " + e.getMessage(), e); + } + } + + private void performHandshake() throws IOException { + // Generate random key (16 bytes, base64 encoded = 24 chars) + byte[] keyBytes = new byte[16]; + for (int i = 0; i < 16; i++) { + keyBytes[i] = (byte) rnd.nextInt(256); + } + String key = Base64.getEncoder().encodeToString(keyBytes); + + // Build HTTP upgrade request + StringBuilder request = new StringBuilder(); + request.append("GET ").append(path).append(" HTTP/1.1\r\n"); + request.append("Host: ").append(host); + if ((tlsEnabled && port != 443) || (!tlsEnabled && port != 80)) { + request.append(":").append(port); + } + request.append("\r\n"); + request.append("Upgrade: websocket\r\n"); + request.append("Connection: Upgrade\r\n"); + request.append("Sec-WebSocket-Key: ").append(key).append("\r\n"); + request.append("Sec-WebSocket-Version: 13\r\n"); + request.append("\r\n"); + + // Send request + byte[] requestBytes = request.toString().getBytes(StandardCharsets.US_ASCII); + out.write(requestBytes); + out.flush(); + + // Read response + int responseLen = readHttpResponse(); + + // Parse response + String response = new String(handshakeBuffer, 0, responseLen, StandardCharsets.US_ASCII); + + // Check status line + if (!response.startsWith("HTTP/1.1 101")) { + throw new IOException("WebSocket handshake failed: " + response.split("\r\n")[0]); + } + + // Verify Sec-WebSocket-Accept + String expectedAccept = WebSocketHandshake.computeAcceptKey(key); + if (!response.contains("Sec-WebSocket-Accept: " + expectedAccept)) { + throw new IOException("Invalid Sec-WebSocket-Accept in handshake response"); + } + } + + private int readHttpResponse() throws IOException { + int pos = 0; + int consecutiveCrLf = 0; + + while (pos < handshakeBuffer.length) { + int b = in.read(); + if (b < 0) { + throw new IOException("Connection closed during handshake"); + } + handshakeBuffer[pos++] = (byte) b; + + // Look for \r\n\r\n + if (b == '\r' || b == '\n') { + if ((consecutiveCrLf == 0 && b == '\r') || + (consecutiveCrLf == 1 && b == '\n') || + (consecutiveCrLf == 2 && b == '\r') || + (consecutiveCrLf == 3 && b == '\n')) { + consecutiveCrLf++; + if (consecutiveCrLf == 4) { + return pos; + } + } else { + consecutiveCrLf = (b == '\r') ? 1 : 0; + } + } else { + consecutiveCrLf = 0; + } + } + throw new IOException("HTTP response too large"); + } + + private void sendFrame(int opcode, long payloadPtr, int payloadLen) { + // Generate mask key + int maskKey = rnd.nextInt(); + + // Calculate required buffer size + int headerSize = WebSocketFrameWriter.headerSize(payloadLen, true); + int frameSize = headerSize + payloadLen; + + // Ensure buffer is large enough + ensureSendBufferSize(frameSize); + + // Write frame header with mask + int headerWritten = WebSocketFrameWriter.writeHeader( + sendBufferPtr, true, opcode, payloadLen, maskKey); + + // Copy payload to buffer after header + if (payloadLen > 0) { + Unsafe.getUnsafe().copyMemory(payloadPtr, sendBufferPtr + headerWritten, payloadLen); + // Mask the payload in place + WebSocketFrameWriter.maskPayload(sendBufferPtr + headerWritten, payloadLen, maskKey); + } + + // Send frame + try { + writeToSocket(sendBufferPtr, frameSize); + } catch (IOException e) { + throw new LineSenderException("Failed to send WebSocket frame: " + e.getMessage(), e); + } + } + + private void sendCloseFrame(int code, String reason) { + int maskKey = rnd.nextInt(); + + // Close payload: 2-byte code + optional reason + int reasonLen = (reason != null) ? reason.length() : 0; + int payloadLen = 2 + reasonLen; + + int headerSize = WebSocketFrameWriter.headerSize(payloadLen, true); + int frameSize = headerSize + payloadLen; + + ensureSendBufferSize(frameSize); + + // Write header + int headerWritten = WebSocketFrameWriter.writeHeader( + sendBufferPtr, true, WebSocketOpcode.CLOSE, payloadLen, maskKey); + + // Write close code (big-endian) + long payloadStart = sendBufferPtr + headerWritten; + Unsafe.getUnsafe().putByte(payloadStart, (byte) ((code >> 8) & 0xFF)); + Unsafe.getUnsafe().putByte(payloadStart + 1, (byte) (code & 0xFF)); + + // Write reason if present + if (reason != null) { + byte[] reasonBytes = reason.getBytes(StandardCharsets.UTF_8); + for (int i = 0; i < reasonBytes.length; i++) { + Unsafe.getUnsafe().putByte(payloadStart + 2 + i, reasonBytes[i]); + } + } + + // Mask payload + WebSocketFrameWriter.maskPayload(payloadStart, payloadLen, maskKey); + + try { + writeToSocket(sendBufferPtr, frameSize); + } catch (IOException e) { + // Ignore errors during close + } + } + + private boolean doReceiveFrame(ResponseHandler handler) throws IOException { + // First, try to parse any data already in the buffer + // This handles the case where multiple frames arrived in a single TCP read + if (recvBufferPos > recvBufferReadPos) { + Boolean result = tryParseFrame(handler); + if (result != null) { + return result; + } + // result == null means we need more data, continue to read + } + + // Read more data into receive buffer + int bytesRead = readFromSocket(); + if (bytesRead <= 0) { + return false; + } + + // Try parsing again with the new data + Boolean result = tryParseFrame(handler); + return result != null && result; + } + + /** + * Tries to parse a frame from the receive buffer. + * @return true if frame processed, false if error, null if need more data + */ + private Boolean tryParseFrame(ResponseHandler handler) throws IOException { + frameParser.reset(); + int consumed = frameParser.parse( + recvBufferPtr + recvBufferReadPos, + recvBufferPtr + recvBufferPos); + + if (frameParser.getState() == WebSocketFrameParser.STATE_NEED_MORE) { + return null; // Need more data + } + + if (frameParser.getState() == WebSocketFrameParser.STATE_ERROR) { + throw new IOException("WebSocket frame parse error: " + frameParser.getErrorCode()); + } + + if (frameParser.getState() == WebSocketFrameParser.STATE_COMPLETE) { + long payloadPtr = recvBufferPtr + recvBufferReadPos + frameParser.getHeaderSize(); + int payloadLen = (int) frameParser.getPayloadLength(); + + // Handle control frames + int opcode = frameParser.getOpcode(); + switch (opcode) { + case WebSocketOpcode.PING: + sendPongFrame(payloadPtr, payloadLen); + break; + case WebSocketOpcode.PONG: + // Ignore pong + break; + case WebSocketOpcode.CLOSE: + connected = false; + if (handler != null) { + int closeCode = 0; + if (payloadLen >= 2) { + closeCode = ((Unsafe.getUnsafe().getByte(payloadPtr) & 0xFF) << 8) + | (Unsafe.getUnsafe().getByte(payloadPtr + 1) & 0xFF); + } + handler.onClose(closeCode, null); + } + break; + case WebSocketOpcode.BINARY: + if (handler != null) { + handler.onBinaryMessage(payloadPtr, payloadLen); + } + break; + case WebSocketOpcode.TEXT: + // Ignore text frames for now + break; + } + + // Advance read position + recvBufferReadPos += consumed; + + // Compact buffer if needed + if (recvBufferReadPos > 0) { + int remaining = recvBufferPos - recvBufferReadPos; + if (remaining > 0) { + Unsafe.getUnsafe().copyMemory( + recvBufferPtr + recvBufferReadPos, + recvBufferPtr, + remaining); + } + recvBufferPos = remaining; + recvBufferReadPos = 0; + } + + return true; + } + + return false; + } + + private void sendPongFrame(long pingPayloadPtr, int pingPayloadLen) { + int maskKey = rnd.nextInt(); + int headerSize = WebSocketFrameWriter.headerSize(pingPayloadLen, true); + int frameSize = headerSize + pingPayloadLen; + + ensureSendBufferSize(frameSize); + + int headerWritten = WebSocketFrameWriter.writeHeader( + sendBufferPtr, true, WebSocketOpcode.PONG, pingPayloadLen, maskKey); + + if (pingPayloadLen > 0) { + Unsafe.getUnsafe().copyMemory(pingPayloadPtr, sendBufferPtr + headerWritten, pingPayloadLen); + WebSocketFrameWriter.maskPayload(sendBufferPtr + headerWritten, pingPayloadLen, maskKey); + } + + try { + writeToSocket(sendBufferPtr, frameSize); + } catch (IOException e) { + // Ignore pong send errors + } + } + + private void ensureSendBufferSize(int required) { + if (required > sendBufferSize) { + int newSize = Math.max(required, sendBufferSize * 2); + sendBufferPtr = Unsafe.realloc(sendBufferPtr, sendBufferSize, newSize, MemoryTag.NATIVE_DEFAULT); + sendBufferSize = newSize; + } + } + + private void writeToSocket(long ptr, int len) throws IOException { + // Copy to temp array for socket write (unavoidable with OutputStream) + // Use separate write buffer to avoid race with read thread + byte[] temp = getWriteTempBuffer(len); + for (int i = 0; i < len; i++) { + temp[i] = Unsafe.getUnsafe().getByte(ptr + i); + } + out.write(temp, 0, len); + out.flush(); + } + + private int readFromSocket() throws IOException { + // Ensure space in receive buffer + int available = recvBufferSize - recvBufferPos; + if (available < 1024) { + // Grow buffer + int newSize = recvBufferSize * 2; + recvBufferPtr = Unsafe.realloc(recvBufferPtr, recvBufferSize, newSize, MemoryTag.NATIVE_DEFAULT); + recvBufferSize = newSize; + available = recvBufferSize - recvBufferPos; + } + + // Read into temp array then copy to native buffer + // Use separate read buffer to avoid race with write thread + byte[] temp = getReadTempBuffer(available); + int bytesRead = in.read(temp, 0, available); + if (bytesRead > 0) { + for (int i = 0; i < bytesRead; i++) { + Unsafe.getUnsafe().putByte(recvBufferPtr + recvBufferPos + i, temp[i]); + } + recvBufferPos += bytesRead; + } + return bytesRead; + } + + // Separate temp buffers for read and write to avoid race conditions + // between send queue thread and response reader thread + private byte[] writeTempBuffer; + private byte[] readTempBuffer; + + private byte[] getWriteTempBuffer(int minSize) { + if (writeTempBuffer == null || writeTempBuffer.length < minSize) { + writeTempBuffer = new byte[Math.max(minSize, 8192)]; + } + return writeTempBuffer; + } + + private byte[] getReadTempBuffer(int minSize) { + if (readTempBuffer == null || readTempBuffer.length < minSize) { + readTempBuffer = new byte[Math.max(minSize, 8192)]; + } + return readTempBuffer; + } + + private void closeQuietly() { + connected = false; + if (socket != null) { + try { + socket.close(); + } catch (IOException e) { + // Ignore + } + socket = null; + } + in = null; + out = null; + } + + /** + * Callback interface for received WebSocket messages. + */ + public interface ResponseHandler { + void onBinaryMessage(long payload, int length); + void onClose(int code, String reason); + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/WebSocketResponse.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/WebSocketResponse.java new file mode 100644 index 0000000..42e74af --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/WebSocketResponse.java @@ -0,0 +1,283 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.ilpv4.client; + +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; + +import java.nio.charset.StandardCharsets; + +/** + * Binary response format for WebSocket ILP v4 protocol. + *

+ * Response format (little-endian): + * 

+ * +--------+----------+------------------+
+ * | status | sequence | error (if any)   |
+ * | 1 byte | 8 bytes  | 2 bytes + UTF-8  |
+ * +--------+----------+------------------+
+ *
+ *

+ * Status codes: + *

    + *
  • 0: Success (ACK)
    • + *
  • 1: Parse error
    • + *
  • 2: Schema error
    • + *
  • 3: Write error
    • + *
  • 4: Security error
    • + *
  • 255: Internal error
    • + *
+ *

+ * The sequence number allows correlation with the original request. + * Error message is only present when status != 0. + */ +public class WebSocketResponse { + + // Status codes + public static final byte STATUS_OK = 0; + public static final byte STATUS_PARSE_ERROR = 1; + public static final byte STATUS_SCHEMA_ERROR = 2; + public static final byte STATUS_WRITE_ERROR = 3; + public static final byte STATUS_SECURITY_ERROR = 4; + public static final byte STATUS_INTERNAL_ERROR = (byte) 255; + + // Minimum response size: status (1) + sequence (8) + public static final int MIN_RESPONSE_SIZE = 9; + public static final int MIN_ERROR_RESPONSE_SIZE = 11; // status + sequence + error length + public static final int MAX_ERROR_MESSAGE_LENGTH = 1024; + + private byte status; + private long sequence; + private String errorMessage; + + public WebSocketResponse() { + this.status = STATUS_OK; + this.sequence = 0; + this.errorMessage = null; + } + + /** + * Creates a success response. + */ + public static WebSocketResponse success(long sequence) { + WebSocketResponse response = new WebSocketResponse(); + response.status = STATUS_OK; + response.sequence = sequence; + return response; + } + + /** + * Creates an error response. + */ + public static WebSocketResponse error(long sequence, byte status, String errorMessage) { + WebSocketResponse response = new WebSocketResponse(); + response.status = status; + response.sequence = sequence; + response.errorMessage = errorMessage; + return response; + } + + /** + * Validates binary response framing without allocating. + *

+ * Accepted formats: + *

    + *
  • OK: exactly 9 bytes (status + sequence)
    • + *
  • Error: exactly 11 + errorLength bytes
    • + *
+ * + * @param ptr response buffer pointer + * @param length response frame payload length + * @return true if payload structure is valid + */ + public static boolean isStructurallyValid(long ptr, int length) { + if (length < MIN_RESPONSE_SIZE) { + return false; + } + + byte status = Unsafe.getUnsafe().getByte(ptr); + if (status == STATUS_OK) { + return length == MIN_RESPONSE_SIZE; + } + + if (length < MIN_ERROR_RESPONSE_SIZE) { + return false; + } + + int msgLen = Unsafe.getUnsafe().getShort(ptr + MIN_RESPONSE_SIZE) & 0xFFFF; + return length == MIN_ERROR_RESPONSE_SIZE + msgLen; + } + + /** + * Returns true if this is a success response. + */ + public boolean isSuccess() { + return status == STATUS_OK; + } + + /** + * Returns the status code. + */ + public byte getStatus() { + return status; + } + + /** + * Returns the sequence number. + */ + public long getSequence() { + return sequence; + } + + /** + * Returns the error message, or null for success responses. + */ + public String getErrorMessage() { + return errorMessage; + } + + /** + * Returns a human-readable status name. + */ + public String getStatusName() { + switch (status) { + case STATUS_OK: + return "OK"; + case STATUS_PARSE_ERROR: + return "PARSE_ERROR"; + case STATUS_SCHEMA_ERROR: + return "SCHEMA_ERROR"; + case STATUS_WRITE_ERROR: + return "WRITE_ERROR"; + case STATUS_SECURITY_ERROR: + return "SECURITY_ERROR"; + case STATUS_INTERNAL_ERROR: + return "INTERNAL_ERROR"; + default: + return "UNKNOWN(" + (status & 0xFF) + ")"; + } + } + + /** + * Calculates the serialized size of this response. + */ + public int serializedSize() { + int size = MIN_RESPONSE_SIZE; + if (errorMessage != null && !errorMessage.isEmpty()) { + byte[] msgBytes = errorMessage.getBytes(StandardCharsets.UTF_8); + int msgLen = Math.min(msgBytes.length, MAX_ERROR_MESSAGE_LENGTH); + size += 2 + msgLen; // 2 bytes for length prefix + } + return size; + } + + /** + * Writes this response to native memory. + * + * @param ptr destination address + * @return number of bytes written + */ + public int writeTo(long ptr) { + int offset = 0; + + // Status (1 byte) + Unsafe.getUnsafe().putByte(ptr + offset, status); + offset += 1; + + // Sequence (8 bytes, little-endian) + Unsafe.getUnsafe().putLong(ptr + offset, sequence); + offset += 8; + + // Error message (if any) + if (status != STATUS_OK && errorMessage != null && !errorMessage.isEmpty()) { + byte[] msgBytes = errorMessage.getBytes(StandardCharsets.UTF_8); + int msgLen = Math.min(msgBytes.length, MAX_ERROR_MESSAGE_LENGTH); + + // Length prefix (2 bytes, little-endian) + Unsafe.getUnsafe().putShort(ptr + offset, (short) msgLen); + offset += 2; + + // Message bytes + for (int i = 0; i < msgLen; i++) { + Unsafe.getUnsafe().putByte(ptr + offset + i, msgBytes[i]); + } + offset += msgLen; + } + + return offset; + } + + /** + * Reads a response from native memory. + * + * @param ptr source address + * @param length available bytes + * @return true if successfully parsed, false if not enough data + */ + public boolean readFrom(long ptr, int length) { + if (length < MIN_RESPONSE_SIZE) { + return false; + } + + int offset = 0; + + // Status (1 byte) + status = Unsafe.getUnsafe().getByte(ptr + offset); + offset += 1; + + // Sequence (8 bytes, little-endian) + sequence = Unsafe.getUnsafe().getLong(ptr + offset); + offset += 8; + + // Error message (if status != OK and more data available) + if (status != STATUS_OK && length > offset + 2) { + int msgLen = Unsafe.getUnsafe().getShort(ptr + offset) & 0xFFFF; + offset += 2; + + if (length >= offset + msgLen && msgLen > 0) { + byte[] msgBytes = new byte[msgLen]; + for (int i = 0; i < msgLen; i++) { + msgBytes[i] = Unsafe.getUnsafe().getByte(ptr + offset + i); + } + errorMessage = new String(msgBytes, StandardCharsets.UTF_8); + offset += msgLen; + } + } else { + errorMessage = null; + } + + return true; + } + + @Override + public String toString() { + if (isSuccess()) { + return "WebSocketResponse{status=OK, seq=" + sequence + "}"; + } else { + return "WebSocketResponse{status=" + getStatusName() + ", seq=" + sequence + + ", error=" + errorMessage + "}"; + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/WebSocketSendQueue.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/WebSocketSendQueue.java new file mode 100644 index 0000000..b34926e --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/WebSocketSendQueue.java @@ -0,0 +1,693 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.ilpv4.client; + +import io.questdb.client.cutlass.http.client.WebSocketClient; +import io.questdb.client.cutlass.http.client.WebSocketFrameHandler; +import io.questdb.client.cutlass.line.LineSenderException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import io.questdb.client.std.QuietCloseable; +import org.jetbrains.annotations.Nullable; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Asynchronous I/O handler for WebSocket microbatch transmission. + *

+ * This class manages a dedicated I/O thread that handles both: + *

    + *
  • Sending batches from a bounded queue
    • + *
  • Receiving and processing server ACK responses
    • + *
+ * Using a single thread eliminates concurrency issues with the WebSocket channel. + *

+ * Thread safety: + *

    + *
  • The send queue is thread-safe for concurrent access
    • + *
  • Only the I/O thread interacts with the WebSocket channel
    • + *
  • Buffer state transitions ensure safe hand-over
    • + *
+ *

+ * Backpressure: + *

    + *
  • When the queue is full, {@link #enqueue} blocks
    • + *
  • This propagates backpressure to the user thread
    • + *
+ */ +public class WebSocketSendQueue implements QuietCloseable { + + private static final Logger LOG = LoggerFactory.getLogger(WebSocketSendQueue.class); + + // Default configuration + public static final int DEFAULT_QUEUE_CAPACITY = 16; + public static final long DEFAULT_ENQUEUE_TIMEOUT_MS = 30_000; + public static final long DEFAULT_SHUTDOWN_TIMEOUT_MS = 10_000; + + // Single pending buffer slot (double-buffering means at most 1 item in queue) + // Zero allocation - just a volatile reference handoff + private volatile MicrobatchBuffer pendingBuffer; + + // The WebSocket client for I/O (single-threaded access only) + private final WebSocketClient client; + + // Optional InFlightWindow for tracking sent batches awaiting ACK + @Nullable + private final InFlightWindow inFlightWindow; + + // The I/O thread for async send/receive + private final Thread ioThread; + + // Running state + private volatile boolean running; + private volatile boolean shuttingDown; + + // Synchronization for flush/close + private final CountDownLatch shutdownLatch; + + // Error handling + private volatile Throwable lastError; + + // Statistics - sending + private final AtomicLong totalBatchesSent = new AtomicLong(0); + private final AtomicLong totalBytesSent = new AtomicLong(0); + + // Statistics - receiving + private final AtomicLong totalAcks = new AtomicLong(0); + private final AtomicLong totalErrors = new AtomicLong(0); + + // Counter for batches currently being processed by the I/O thread + // This tracks batches that have been dequeued but not yet fully sent + private final AtomicInteger processingCount = new AtomicInteger(0); + + // Lock for all coordination between user thread and I/O thread. + // Used for: queue poll + processingCount increment atomicity, + // flush() waiting, I/O thread waiting when idle. + private final Object processingLock = new Object(); + + // Batch sequence counter (must match server's messageSequence) + private long nextBatchSequence = 0; + + // Response parsing + private final WebSocketResponse response = new WebSocketResponse(); + private final ResponseHandler responseHandler = new ResponseHandler(); + + // Configuration + private final long enqueueTimeoutMs; + private final long shutdownTimeoutMs; + + // ==================== Pending Buffer Operations (zero allocation) ==================== + + private boolean offerPending(MicrobatchBuffer buffer) { + if (pendingBuffer != null) { + return false; // slot occupied + } + pendingBuffer = buffer; + return true; + } + + private MicrobatchBuffer pollPending() { + MicrobatchBuffer buffer = pendingBuffer; + if (buffer != null) { + pendingBuffer = null; + } + return buffer; + } + + private boolean isPendingEmpty() { + return pendingBuffer == null; + } + + private int getPendingSize() { + return pendingBuffer == null ? 0 : 1; + } + + /** + * Creates a new send queue with default configuration. + * + * @param client the WebSocket client for I/O + */ + public WebSocketSendQueue(WebSocketClient client) { + this(client, null, DEFAULT_QUEUE_CAPACITY, DEFAULT_ENQUEUE_TIMEOUT_MS, DEFAULT_SHUTDOWN_TIMEOUT_MS); + } + + /** + * Creates a new send queue with InFlightWindow for tracking sent batches. + * + * @param client the WebSocket client for I/O + * @param inFlightWindow the window to track sent batches awaiting ACK (may be null) + */ + public WebSocketSendQueue(WebSocketClient client, @Nullable InFlightWindow inFlightWindow) { + this(client, inFlightWindow, DEFAULT_QUEUE_CAPACITY, DEFAULT_ENQUEUE_TIMEOUT_MS, DEFAULT_SHUTDOWN_TIMEOUT_MS); + } + + /** + * Creates a new send queue with custom configuration. + * + * @param client the WebSocket client for I/O + * @param inFlightWindow the window to track sent batches awaiting ACK (may be null) + * @param queueCapacity maximum number of pending batches + * @param enqueueTimeoutMs timeout for enqueue operations (ms) + * @param shutdownTimeoutMs timeout for graceful shutdown (ms) + */ + public WebSocketSendQueue(WebSocketClient client, @Nullable InFlightWindow inFlightWindow, + int queueCapacity, long enqueueTimeoutMs, long shutdownTimeoutMs) { + if (client == null) { + throw new IllegalArgumentException("client cannot be null"); + } + if (queueCapacity <= 0) { + throw new IllegalArgumentException("queueCapacity must be positive"); + } + + this.client = client; + this.inFlightWindow = inFlightWindow; + this.enqueueTimeoutMs = enqueueTimeoutMs; + this.shutdownTimeoutMs = shutdownTimeoutMs; + this.running = true; + this.shuttingDown = false; + this.shutdownLatch = new CountDownLatch(1); + + // Start the I/O thread (handles both sending and receiving) + this.ioThread = new Thread(this::ioLoop, "questdb-websocket-io"); + this.ioThread.setDaemon(true); + this.ioThread.start(); + + LOG.info("WebSocket I/O thread started [capacity={}]", queueCapacity); + } + + /** + * Enqueues a sealed buffer for sending. + *

+ * The buffer must be in SEALED state. After this method returns successfully, + * ownership of the buffer transfers to the send queue. + * + * @param buffer the sealed buffer to send + * @return true if enqueued successfully + * @throws LineSenderException if the buffer is not sealed or an error occurred + */ + public boolean enqueue(MicrobatchBuffer buffer) { + if (buffer == null) { + throw new IllegalArgumentException("buffer cannot be null"); + } + if (!buffer.isSealed()) { + throw new LineSenderException("Buffer must be sealed before enqueue, state=" + + MicrobatchBuffer.stateName(buffer.getState())); + } + if (!running || shuttingDown) { + throw new LineSenderException("Send queue is not running"); + } + + // Check for errors from I/O thread + checkError(); + + final long deadline = System.currentTimeMillis() + enqueueTimeoutMs; + synchronized (processingLock) { + while (true) { + if (!running || shuttingDown) { + throw new LineSenderException("Send queue is not running"); + } + checkError(); + + if (offerPending(buffer)) { + processingLock.notifyAll(); + break; + } + + long remaining = deadline - System.currentTimeMillis(); + if (remaining <= 0) { + throw new LineSenderException("Enqueue timeout after " + enqueueTimeoutMs + "ms"); + } + try { + processingLock.wait(Math.min(10, remaining)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new LineSenderException("Interrupted while enqueueing", e); + } + } + } + + LOG.debug("Enqueued batch [id={}, bytes={}, rows={}]", buffer.getBatchId(), buffer.getBufferPos(), buffer.getRowCount()); + return true; + } + + /** + * Waits for all pending batches to be sent. + *

+ * This method blocks until the queue is empty and all in-flight sends complete. + * It does not close the queue - new batches can still be enqueued after flush. + * + * @throws LineSenderException if an error occurs during flush + */ + public void flush() { + checkError(); + + long startTime = System.currentTimeMillis(); + + // Wait under lock - I/O thread will notify when processingCount decrements + synchronized (processingLock) { + while (running) { + // Atomically check: queue empty AND not processing + if (isPendingEmpty() && processingCount.get() == 0) { + break; // All done + } + + try { + processingLock.wait(10); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new LineSenderException("Interrupted while flushing", e); + } + + // Check timeout + if (System.currentTimeMillis() - startTime > enqueueTimeoutMs) { + throw new LineSenderException("Flush timeout after " + enqueueTimeoutMs + "ms, " + + "queue=" + getPendingSize() + ", processing=" + processingCount.get()); + } + + // Check for errors + checkError(); + } + } + + // If loop exited because running=false we still need to surface the root cause. + checkError(); + + LOG.debug("Flush complete"); + } + + /** + * Returns the number of batches waiting to be sent. + */ + public int getPendingCount() { + return getPendingSize(); + } + + /** + * Returns true if the queue is empty. + */ + public boolean isEmpty() { + return isPendingEmpty(); + } + + /** + * Returns true if the queue is still running. + */ + public boolean isRunning() { + return running && !shuttingDown; + } + + /** + * Returns the total number of batches sent. + */ + public long getTotalBatchesSent() { + return totalBatchesSent.get(); + } + + /** + * Returns the total number of bytes sent. + */ + public long getTotalBytesSent() { + return totalBytesSent.get(); + } + + /** + * Returns the last error that occurred in the I/O thread, or null if no error. + */ + public Throwable getLastError() { + return lastError; + } + + /** + * Closes the send queue gracefully. + *

+ * This method: + * 1. Stops accepting new batches + * 2. Waits for pending batches to be sent + * 3. Stops the I/O thread + *

+ * Note: This does NOT close the WebSocket channel - that's the caller's responsibility. + */ + @Override + public void close() { + if (!running) { + return; + } + + LOG.info("Closing WebSocket send queue [pending={}]", getPendingSize()); + + // Signal shutdown + shuttingDown = true; + + // Wait for pending batches to be sent + long startTime = System.currentTimeMillis(); + while (!isPendingEmpty()) { + if (System.currentTimeMillis() - startTime > shutdownTimeoutMs) { + LOG.error("Shutdown timeout, {} batches not sent", getPendingSize()); + break; + } + try { + Thread.sleep(10); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } + } + + // Stop the I/O thread + running = false; + + // Wake up I/O thread if it's blocked on processingLock.wait() + synchronized (processingLock) { + processingLock.notifyAll(); + } + ioThread.interrupt(); + + // Wait for I/O thread to finish + try { + shutdownLatch.await(shutdownTimeoutMs, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + + LOG.info("WebSocket send queue closed [totalBatches={}, totalBytes={}]", totalBatchesSent.get(), totalBytesSent.get()); + } + + // ==================== I/O Thread ==================== + + /** + * I/O loop states for the state machine. + *

    + *
  • IDLE: queue empty, no in-flight batches - can block waiting for work
    • + *
  • ACTIVE: have batches to send - non-blocking loop
    • + *
  • DRAINING: queue empty but ACKs pending - poll for ACKs, short wait
    • + *
+ */ + private enum IoState { + IDLE, ACTIVE, DRAINING + } + + /** + * The main I/O loop that handles both sending batches and receiving ACKs. + *

+ * Uses a state machine: + *

    + *
  • IDLE: block on processingLock.wait() until work arrives
    • + *
  • ACTIVE: non-blocking poll queue, send batches, check for ACKs
    • + *
  • DRAINING: no batches but ACKs pending - poll for ACKs with short wait
    • + *
+ */ + private void ioLoop() { + LOG.info("I/O loop started"); + + try { + while (running || !isPendingEmpty()) { + MicrobatchBuffer batch = null; + boolean hasInFlight = (inFlightWindow != null && inFlightWindow.getInFlightCount() > 0); + IoState state = computeState(hasInFlight); + + switch (state) { + case IDLE: + // Nothing to do - wait for work under lock + synchronized (processingLock) { + // Re-check under lock to avoid missed wakeup + if (isPendingEmpty() && running) { + try { + processingLock.wait(100); + } catch (InterruptedException e) { + if (!running) return; + } + } + } + break; + + case ACTIVE: + case DRAINING: + // Try to receive any pending ACKs (non-blocking) + if (hasInFlight && client.isConnected()) { + tryReceiveAcks(); + } + + // Try to dequeue and send a batch + boolean hasWindowSpace = (inFlightWindow == null || inFlightWindow.hasWindowSpace()); + if (hasWindowSpace) { + // Atomically: poll queue + increment processingCount + synchronized (processingLock) { + batch = pollPending(); + if (batch != null) { + processingCount.incrementAndGet(); + } + } + + if (batch != null) { + try { + safeSendBatch(batch); + } finally { + // Atomically: decrement + notify flush() + synchronized (processingLock) { + processingCount.decrementAndGet(); + processingLock.notifyAll(); + } + } + } + } + + // In DRAINING state with no work, short wait to avoid busy loop + if (state == IoState.DRAINING && batch == null) { + synchronized (processingLock) { + try { + processingLock.wait(10); + } catch (InterruptedException e) { + if (!running) return; + } + } + } + break; + } + } + } finally { + shutdownLatch.countDown(); + LOG.info("I/O loop stopped [totalAcks={}, totalErrors={}]", totalAcks.get(), totalErrors.get()); + } + } + + /** + * Computes the current I/O state based on queue and in-flight status. + */ + private IoState computeState(boolean hasInFlight) { + if (!isPendingEmpty()) { + return IoState.ACTIVE; + } else if (hasInFlight) { + return IoState.DRAINING; + } else { + return IoState.IDLE; + } + } + + /** + * Tries to receive ACKs from the server (non-blocking). + */ + private void tryReceiveAcks() { + try { + client.tryReceiveFrame(responseHandler); + } catch (Exception e) { + if (running) { + LOG.error("Error receiving response: {}", e.getMessage()); + failTransport(new LineSenderException("Error receiving response: " + e.getMessage(), e)); + } + } + } + + /** + * Sends a batch with error handling. Does NOT manage processingCount. + */ + private void safeSendBatch(MicrobatchBuffer batch) { + try { + sendBatch(batch); + } catch (Throwable t) { + LOG.error("Error sending batch [id={}]{}", batch.getBatchId(), "", t); + failTransport(new LineSenderException("Error sending batch " + batch.getBatchId() + ": " + t.getMessage(), t)); + // Mark as recycled even on error to allow cleanup + if (batch.isSealed()) { + batch.markSending(); + } + if (batch.isSending()) { + batch.markRecycled(); + } + } + } + + /** + * Sends a single batch over the WebSocket channel. + */ + private void sendBatch(MicrobatchBuffer batch) { + // Transition state: SEALED -> SENDING + batch.markSending(); + + // Use our own sequence counter (must match server's messageSequence) + long batchSequence = nextBatchSequence++; + int bytes = batch.getBufferPos(); + int rows = batch.getRowCount(); + + LOG.debug("Sending batch [seq={}, bytes={}, rows={}, bufferId={}]", batchSequence, bytes, rows, batch.getBatchId()); + + // Add to in-flight window BEFORE sending (so we're ready for ACK) + // Use non-blocking tryAddInFlight since we already checked window space in ioLoop + if (inFlightWindow != null) { + LOG.debug("Adding to in-flight window [seq={}, inFlight={}, max={}]", batchSequence, inFlightWindow.getInFlightCount(), inFlightWindow.getMaxWindowSize()); + if (!inFlightWindow.tryAddInFlight(batchSequence)) { + // Should not happen since we checked hasWindowSpace before polling + throw new LineSenderException("In-flight window unexpectedly full"); + } + LOG.debug("Added to in-flight window [seq={}]", batchSequence); + } + + // Send over WebSocket + LOG.debug("Calling sendBinary [seq={}]", batchSequence); + client.sendBinary(batch.getBufferPtr(), bytes); + LOG.debug("sendBinary returned [seq={}]", batchSequence); + + // Update statistics + totalBatchesSent.incrementAndGet(); + totalBytesSent.addAndGet(bytes); + + // Transition state: SENDING -> RECYCLED + batch.markRecycled(); + + LOG.debug("Batch sent and recycled [seq={}, bufferId={}]", batchSequence, batch.getBatchId()); + } + + /** + * Checks if an error occurred in the I/O thread and throws if so. + */ + private void checkError() { + Throwable error = lastError; + if (error != null) { + throw new LineSenderException("Error in send queue I/O thread: " + error.getMessage(), error); + } + } + + private void failTransport(LineSenderException error) { + Throwable rootError = lastError; + if (rootError == null) { + lastError = error; + rootError = error; + } + running = false; + shuttingDown = true; + if (inFlightWindow != null) { + inFlightWindow.failAll(rootError); + } + synchronized (processingLock) { + MicrobatchBuffer dropped = pollPending(); + if (dropped != null) { + if (dropped.isSealed()) { + dropped.markSending(); + } + if (dropped.isSending()) { + dropped.markRecycled(); + } + } + processingLock.notifyAll(); + } + } + + /** + * Returns total successful acknowledgments received. + */ + public long getTotalAcks() { + return totalAcks.get(); + } + + /** + * Returns total error responses received. + */ + public long getTotalErrors() { + return totalErrors.get(); + } + + // ==================== Response Handler ==================== + + /** + * Handler for received WebSocket frames (ACKs from server). + */ + private class ResponseHandler implements WebSocketFrameHandler { + + @Override + public void onBinaryMessage(long payloadPtr, int payloadLen) { + if (!WebSocketResponse.isStructurallyValid(payloadPtr, payloadLen)) { + LineSenderException error = new LineSenderException( + "Invalid ACK response payload [length=" + payloadLen + ']' + ); + LOG.error("Invalid ACK response payload [length={}]", payloadLen); + failTransport(error); + return; + } + + // Parse response from binary payload + if (!response.readFrom(payloadPtr, payloadLen)) { + LineSenderException error = new LineSenderException("Failed to parse ACK response"); + LOG.error("Failed to parse response"); + failTransport(error); + return; + } + + long sequence = response.getSequence(); + + if (response.isSuccess()) { + // Cumulative ACK - acknowledge all batches up to this sequence + if (inFlightWindow != null) { + int acked = inFlightWindow.acknowledgeUpTo(sequence); + if (acked > 0) { + totalAcks.addAndGet(acked); + LOG.debug("Cumulative ACK received [upTo={}, acked={}]", sequence, acked); + } else { + LOG.debug("ACK for already-acknowledged sequences [upTo={}]", sequence); + } + } + } else { + // Error - fail the batch + String errorMessage = response.getErrorMessage(); + LOG.error("Error response [seq={}, status={}, error={}]", sequence, response.getStatusName(), errorMessage); + + if (inFlightWindow != null) { + LineSenderException error = new LineSenderException( + "Server error for batch " + sequence + ": " + + response.getStatusName() + " - " + errorMessage); + inFlightWindow.fail(sequence, error); + } + totalErrors.incrementAndGet(); + } + } + + @Override + public void onClose(int code, String reason) { + LOG.info("WebSocket closed by server [code={}, reason={}]", code, reason); + failTransport(new LineSenderException("WebSocket closed by server [code=" + code + ", reason=" + reason + ']')); + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4BitReader.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4BitReader.java new file mode 100644 index 0000000..158ec3d --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4BitReader.java @@ -0,0 +1,335 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2024 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.ilpv4.protocol; + +import io.questdb.client.std.Unsafe; + +/** + * Bit-level reader for ILP v4 protocol. + *

+ * This class reads bits from a buffer in LSB-first order within each byte. + * Bits are read sequentially, spanning byte boundaries as needed. + *

+ * The implementation buffers bytes to minimize memory reads. + *

+ * Usage pattern: + * 

+ * IlpV4BitReader reader = new IlpV4BitReader();
+ * reader.reset(address, length);
+ * int bit = reader.readBit();
+ * long value = reader.readBits(numBits);
+ * long signedValue = reader.readSigned(numBits);
+ *
+ */ +public class IlpV4BitReader { + + private long startAddress; + private long currentAddress; + private long endAddress; + + // Buffer for reading bits + private long bitBuffer; + // Number of bits currently available in the buffer (0-64) + private int bitsInBuffer; + // Total bits available for reading (from reset) + private long totalBitsAvailable; + // Total bits already consumed + private long totalBitsRead; + + /** + * Creates a new bit reader. Call {@link #reset} before use. + */ + public IlpV4BitReader() { + } + + /** + * Resets the reader to read from the specified memory region. + * + * @param address the starting address + * @param length the number of bytes available to read + */ + public void reset(long address, long length) { + this.startAddress = address; + this.currentAddress = address; + this.endAddress = address + length; + this.bitBuffer = 0; + this.bitsInBuffer = 0; + this.totalBitsAvailable = length * 8L; + this.totalBitsRead = 0; + } + + /** + * Resets the reader to read from the specified byte array. + * + * @param buf the byte array + * @param offset the starting offset + * @param length the number of bytes available + */ + public void reset(byte[] buf, int offset, int length) { + // For byte array, we'll store position info differently + // This is mainly for testing - in production we use direct memory + throw new UnsupportedOperationException("Use direct memory version"); + } + + /** + * Returns the number of bits remaining to be read. + * + * @return available bits + */ + public long getAvailableBits() { + return totalBitsAvailable - totalBitsRead; + } + + /** + * Returns true if there are more bits to read. + * + * @return true if bits available + */ + public boolean hasMoreBits() { + return totalBitsRead < totalBitsAvailable; + } + + /** + * Returns the current position in bits from the start. + * + * @return bits read since reset + */ + public long getBitPosition() { + return totalBitsRead; + } + + /** + * Ensures the buffer has at least the requested number of bits. + * Loads more bytes from memory if needed. + * + * @param bitsNeeded minimum bits required in buffer + * @return true if sufficient bits available, false otherwise + */ + private boolean ensureBits(int bitsNeeded) { + while (bitsInBuffer < bitsNeeded && currentAddress < endAddress) { + byte b = Unsafe.getUnsafe().getByte(currentAddress++); + bitBuffer |= (long) (b & 0xFF) << bitsInBuffer; + bitsInBuffer += 8; + } + return bitsInBuffer >= bitsNeeded; + } + + /** + * Reads a single bit. + * + * @return 0 or 1 + * @throws IllegalStateException if no more bits available + */ + public int readBit() { + if (totalBitsRead >= totalBitsAvailable) { + throw new IllegalStateException("bit read overflow"); + } + if (!ensureBits(1)) { + throw new IllegalStateException("bit read overflow"); + } + + int bit = (int) (bitBuffer & 1); + bitBuffer >>>= 1; + bitsInBuffer--; + totalBitsRead++; + return bit; + } + + /** + * Reads multiple bits and returns them as a long (unsigned). + *

+ * Bits are returned LSB-aligned. For example, reading 4 bits might return + * 0b1101 where bit 0 is the first bit read. + * + * @param numBits number of bits to read (1-64) + * @return the value formed by the bits (unsigned) + * @throws IllegalStateException if not enough bits available + */ + public long readBits(int numBits) { + if (numBits <= 0) { + return 0; + } + if (numBits > 64) { + throw new IllegalArgumentException("Cannot read more than 64 bits at once"); + } + if (totalBitsRead + numBits > totalBitsAvailable) { + throw new IllegalStateException("bit read overflow"); + } + + long result = 0; + int bitsRemaining = numBits; + int resultShift = 0; + + while (bitsRemaining > 0) { + if (bitsInBuffer == 0) { + if (!ensureBits(Math.min(bitsRemaining, 64))) { + throw new IllegalStateException("bit read overflow"); + } + } + + int bitsToTake = Math.min(bitsRemaining, bitsInBuffer); + long mask = bitsToTake == 64 ? -1L : (1L << bitsToTake) - 1; + result |= (bitBuffer & mask) << resultShift; + + bitBuffer >>>= bitsToTake; + bitsInBuffer -= bitsToTake; + bitsRemaining -= bitsToTake; + resultShift += bitsToTake; + } + + totalBitsRead += numBits; + return result; + } + + /** + * Reads multiple bits and interprets them as a signed value using two's complement. + * + * @param numBits number of bits to read (1-64) + * @return the signed value + * @throws IllegalStateException if not enough bits available + */ + public long readSigned(int numBits) { + long unsigned = readBits(numBits); + // Sign extend: if the high bit (bit numBits-1) is set, extend the sign + if (numBits < 64 && (unsigned & (1L << (numBits - 1))) != 0) { + // Set all bits above numBits to 1 + unsigned |= -1L << numBits; + } + return unsigned; + } + + /** + * Peeks at the next bit without consuming it. + * + * @return 0 or 1, or -1 if no more bits + */ + public int peekBit() { + if (totalBitsRead >= totalBitsAvailable) { + return -1; + } + if (!ensureBits(1)) { + return -1; + } + return (int) (bitBuffer & 1); + } + + /** + * Skips the specified number of bits. + * + * @param numBits bits to skip + * @throws IllegalStateException if not enough bits available + */ + public void skipBits(int numBits) { + if (totalBitsRead + numBits > totalBitsAvailable) { + throw new IllegalStateException("bit read overflow"); + } + + // Fast path: skip bits in current buffer + if (numBits <= bitsInBuffer) { + bitBuffer >>>= numBits; + bitsInBuffer -= numBits; + totalBitsRead += numBits; + return; + } + + // Consume all buffered bits + int bitsToSkip = numBits - bitsInBuffer; + totalBitsRead += bitsInBuffer; + bitsInBuffer = 0; + bitBuffer = 0; + + // Skip whole bytes + int bytesToSkip = bitsToSkip / 8; + currentAddress += bytesToSkip; + totalBitsRead += bytesToSkip * 8L; + + // Handle remaining bits + int remainingBits = bitsToSkip % 8; + if (remainingBits > 0) { + ensureBits(remainingBits); + bitBuffer >>>= remainingBits; + bitsInBuffer -= remainingBits; + totalBitsRead += remainingBits; + } + } + + /** + * Aligns the reader to the next byte boundary by skipping any partial bits. + * + * @throws IllegalStateException if alignment fails + */ + public void alignToByte() { + int bitsToSkip = bitsInBuffer % 8; + if (bitsToSkip != 0) { + // We need to skip the remaining bits in the current partial byte + // But since we read in byte chunks, bitsInBuffer should be a multiple of 8 + // minus what we've consumed. The remainder in the conceptual stream is: + int remainder = (int) (totalBitsRead % 8); + if (remainder != 0) { + skipBits(8 - remainder); + } + } + } + + /** + * Reads a complete byte, ensuring byte alignment first. + * + * @return the byte value (0-255) + * @throws IllegalStateException if not enough data + */ + public int readByte() { + return (int) readBits(8) & 0xFF; + } + + /** + * Reads a complete 32-bit integer in little-endian order. + * + * @return the integer value + * @throws IllegalStateException if not enough data + */ + public int readInt() { + return (int) readBits(32); + } + + /** + * Reads a complete 64-bit long in little-endian order. + * + * @return the long value + * @throws IllegalStateException if not enough data + */ + public long readLong() { + return readBits(64); + } + + /** + * Returns the current byte address being read. + * Note: This is approximate due to bit buffering. + * + * @return current address + */ + public long getCurrentAddress() { + return currentAddress; + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4BitWriter.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4BitWriter.java new file mode 100644 index 0000000..8be1600 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4BitWriter.java @@ -0,0 +1,247 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2024 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.ilpv4.protocol; + +import io.questdb.client.std.Unsafe; + +/** + * Bit-level writer for ILP v4 protocol. + *

+ * This class writes bits to a buffer in LSB-first order within each byte. + * Bits are packed sequentially, spanning byte boundaries as needed. + *

+ * The implementation buffers up to 64 bits before flushing to the output buffer + * to minimize memory operations. All writes are to direct memory for performance. + *

+ * Usage pattern: + * 

+ * IlpV4BitWriter writer = new IlpV4BitWriter();
+ * writer.reset(address, capacity);
+ * writer.writeBits(value, numBits);
+ * writer.writeBits(value2, numBits2);
+ * writer.flush(); // must call before reading output
+ * long bytesWritten = writer.getPosition() - address;
+ *
+ */ +public class IlpV4BitWriter { + + private long startAddress; + private long currentAddress; + private long endAddress; + + // Buffer for accumulating bits before writing + private long bitBuffer; + // Number of bits currently in the buffer (0-63) + private int bitsInBuffer; + + /** + * Creates a new bit writer. Call {@link #reset} before use. + */ + public IlpV4BitWriter() { + } + + /** + * Resets the writer to write to the specified memory region. + * + * @param address the starting address + * @param capacity the maximum number of bytes to write + */ + public void reset(long address, long capacity) { + this.startAddress = address; + this.currentAddress = address; + this.endAddress = address + capacity; + this.bitBuffer = 0; + this.bitsInBuffer = 0; + } + + /** + * Returns the current write position (address). + * Note: Call {@link #flush()} first to ensure all buffered bits are written. + * + * @return the current address after all written data + */ + public long getPosition() { + return currentAddress; + } + + /** + * Returns the number of bits that have been written (including buffered bits). + * + * @return total bits written since reset + */ + public long getTotalBitsWritten() { + return (currentAddress - startAddress) * 8L + bitsInBuffer; + } + + /** + * Writes a single bit. + * + * @param bit the bit value (0 or 1, only LSB is used) + */ + public void writeBit(int bit) { + writeBits(bit & 1, 1); + } + + /** + * Writes multiple bits from the given value. + *

+ * Bits are taken from the LSB of the value. For example, if value=0b1101 + * and numBits=4, the bits written are 1, 0, 1, 1 (LSB to MSB order). + * + * @param value the value containing the bits (LSB-aligned) + * @param numBits number of bits to write (1-64) + */ + public void writeBits(long value, int numBits) { + if (numBits <= 0 || numBits > 64) { + return; + } + + // Mask the value to only include the requested bits + if (numBits < 64) { + value &= (1L << numBits) - 1; + } + + int bitsToWrite = numBits; + + while (bitsToWrite > 0) { + // How many bits can we fit in current buffer (max 64 total) + int availableInBuffer = 64 - bitsInBuffer; + int bitsThisRound = Math.min(bitsToWrite, availableInBuffer); + + // Add bits to the buffer + long mask = bitsThisRound == 64 ? -1L : (1L << bitsThisRound) - 1; + bitBuffer |= (value & mask) << bitsInBuffer; + bitsInBuffer += bitsThisRound; + value >>>= bitsThisRound; + bitsToWrite -= bitsThisRound; + + // Flush complete bytes from the buffer + while (bitsInBuffer >= 8) { + if (currentAddress < endAddress) { + Unsafe.getUnsafe().putByte(currentAddress++, (byte) bitBuffer); + } + bitBuffer >>>= 8; + bitsInBuffer -= 8; + } + } + } + + /** + * Writes a signed value using two's complement representation. + * + * @param value the signed value + * @param numBits number of bits to use for the representation + */ + public void writeSigned(long value, int numBits) { + // Two's complement is automatic in Java for the bit pattern + writeBits(value, numBits); + } + + /** + * Flushes any remaining bits in the buffer to memory. + *

+ * If there are partial bits (less than 8), they are written as the last byte + * with the remaining high bits set to zero. + *

+ * Must be called before reading the output or getting the final position. + */ + public void flush() { + if (bitsInBuffer > 0 && currentAddress < endAddress) { + Unsafe.getUnsafe().putByte(currentAddress++, (byte) bitBuffer); + bitBuffer = 0; + bitsInBuffer = 0; + } + } + + /** + * Finishes writing and returns the number of bytes written since reset. + *

+ * This method flushes any remaining bits and returns the total byte count. + * + * @return bytes written since reset + */ + public int finish() { + flush(); + return (int) (currentAddress - startAddress); + } + + /** + * Returns the number of bits remaining in the partial byte buffer. + * This is 0 after a flush or when aligned on a byte boundary. + * + * @return bits in buffer (0-7) + */ + public int getBitsInBuffer() { + return bitsInBuffer; + } + + /** + * Aligns the writer to the next byte boundary by padding with zeros. + * If already byte-aligned, this is a no-op. + */ + public void alignToByte() { + if (bitsInBuffer > 0) { + flush(); + } + } + + /** + * Writes a complete byte, ensuring byte alignment first. + * + * @param value the byte value + */ + public void writeByte(int value) { + alignToByte(); + if (currentAddress < endAddress) { + Unsafe.getUnsafe().putByte(currentAddress++, (byte) value); + } + } + + /** + * Writes a complete 32-bit integer in little-endian order, ensuring byte alignment first. + * + * @param value the integer value + */ + public void writeInt(int value) { + alignToByte(); + if (currentAddress + 4 <= endAddress) { + Unsafe.getUnsafe().putInt(currentAddress, value); + currentAddress += 4; + } + } + + /** + * Writes a complete 64-bit long in little-endian order, ensuring byte alignment first. + * + * @param value the long value + */ + public void writeLong(long value) { + alignToByte(); + if (currentAddress + 8 <= endAddress) { + Unsafe.getUnsafe().putLong(currentAddress, value); + currentAddress += 8; + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4ColumnDef.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4ColumnDef.java new file mode 100644 index 0000000..cea87af --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4ColumnDef.java @@ -0,0 +1,163 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2024 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.ilpv4.protocol; + +import static io.questdb.client.cutlass.ilpv4.protocol.IlpV4Constants.*; + +/** + * Represents a column definition in an ILP v4 schema. + *

+ * This class is immutable and safe for caching. + */ +public final class IlpV4ColumnDef { + private final String name; + private final byte typeCode; + private final boolean nullable; + + /** + * Creates a column definition. + * + * @param name the column name (UTF-8) + * @param typeCode the ILP v4 type code (0x01-0x0F, optionally OR'd with 0x80 for nullable) + */ + public IlpV4ColumnDef(String name, byte typeCode) { + this.name = name; + // Extract nullable flag (high bit) and base type + this.nullable = (typeCode & 0x80) != 0; + this.typeCode = (byte) (typeCode & 0x7F); + } + + /** + * Creates a column definition with explicit nullable flag. + * + * @param name the column name + * @param typeCode the base type code (0x01-0x0F) + * @param nullable whether the column is nullable + */ + public IlpV4ColumnDef(String name, byte typeCode, boolean nullable) { + this.name = name; + this.typeCode = (byte) (typeCode & 0x7F); + this.nullable = nullable; + } + + /** + * Gets the column name. + */ + public String getName() { + return name; + } + + /** + * Gets the base type code (without nullable flag). + * + * @return type code 0x01-0x0F + */ + public byte getTypeCode() { + return typeCode; + } + + /** + * Gets the wire type code (with nullable flag if applicable). + * + * @return type code as sent on wire + */ + public byte getWireTypeCode() { + return nullable ? (byte) (typeCode | 0x80) : typeCode; + } + + /** + * Returns true if this column is nullable. + */ + public boolean isNullable() { + return nullable; + } + + /** + * Returns true if this is a fixed-width type. + */ + public boolean isFixedWidth() { + return IlpV4Constants.isFixedWidthType(typeCode); + } + + /** + * Gets the fixed width in bytes for fixed-width types. + * + * @return width in bytes, or -1 for variable-width types + */ + public int getFixedWidth() { + return IlpV4Constants.getFixedTypeSize(typeCode); + } + + /** + * Gets the type name for display purposes. + */ + public String getTypeName() { + return IlpV4Constants.getTypeName(typeCode); + } + + /** + * Validates that this column definition has a valid type code. + * + * @throws IllegalArgumentException if type code is invalid + */ + public void validate() { + // Valid type codes: TYPE_BOOLEAN (0x01) through TYPE_DECIMAL256 (0x15) + // This includes all basic types, arrays, and decimals + boolean valid = (typeCode >= TYPE_BOOLEAN && typeCode <= TYPE_DECIMAL256); + if (!valid) { + throw new IllegalArgumentException( + "invalid column type code: 0x" + Integer.toHexString(typeCode) + ); + } + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + IlpV4ColumnDef that = (IlpV4ColumnDef) o; + return typeCode == that.typeCode && + nullable == that.nullable && + name.equals(that.name); + } + + @Override + public int hashCode() { + int result = name.hashCode(); + result = 31 * result + typeCode; + result = 31 * result + (nullable ? 1 : 0); + return result; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(name).append(':').append(getTypeName()); + if (nullable) { + sb.append('?'); + } + return sb.toString(); + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4Constants.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4Constants.java new file mode 100644 index 0000000..7b229f6 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4Constants.java @@ -0,0 +1,506 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2024 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.ilpv4.protocol; + +/** + * Constants for the ILP v4 binary protocol. + */ +public final class IlpV4Constants { + + // ==================== Magic Bytes ==================== + + /** + * Magic bytes for ILP v4 message: "ILP4" (ASCII). + */ + public static final int MAGIC_MESSAGE = 0x34504C49; // "ILP4" in little-endian + + /** + * Magic bytes for capability request: "ILP?" (ASCII). + */ + public static final int MAGIC_CAPABILITY_REQUEST = 0x3F504C49; // "ILP?" in little-endian + + /** + * Magic bytes for capability response: "ILP!" (ASCII). + */ + public static final int MAGIC_CAPABILITY_RESPONSE = 0x21504C49; // "ILP!" in little-endian + + /** + * Magic bytes for fallback response (old server): "ILP0" (ASCII). + */ + public static final int MAGIC_FALLBACK = 0x30504C49; // "ILP0" in little-endian + + // ==================== Header Structure ==================== + + /** + * Size of the message header in bytes. + */ + public static final int HEADER_SIZE = 12; + + /** + * Offset of magic bytes in header (4 bytes). + */ + public static final int HEADER_OFFSET_MAGIC = 0; + + /** + * Offset of version byte in header. + */ + public static final int HEADER_OFFSET_VERSION = 4; + + /** + * Offset of flags byte in header. + */ + public static final int HEADER_OFFSET_FLAGS = 5; + + /** + * Offset of table count (uint16, little-endian) in header. + */ + public static final int HEADER_OFFSET_TABLE_COUNT = 6; + + /** + * Offset of payload length (uint32, little-endian) in header. + */ + public static final int HEADER_OFFSET_PAYLOAD_LENGTH = 8; + + // ==================== Protocol Version ==================== + + /** + * Current protocol version. + */ + public static final byte VERSION_1 = 1; + + // ==================== Flag Bits ==================== + + /** + * Flag bit: LZ4 compression enabled. + */ + public static final byte FLAG_LZ4 = 0x01; + + /** + * Flag bit: Zstd compression enabled. + */ + public static final byte FLAG_ZSTD = 0x02; + + /** + * Flag bit: Gorilla timestamp encoding enabled. + */ + public static final byte FLAG_GORILLA = 0x04; + + /** + * Flag bit: Delta symbol dictionary encoding enabled. + * When set, symbol columns use global IDs and send only new dictionary entries. + */ + public static final byte FLAG_DELTA_SYMBOL_DICT = 0x08; + + /** + * Mask for compression flags (bits 0-1). + */ + public static final byte FLAG_COMPRESSION_MASK = FLAG_LZ4 | FLAG_ZSTD; + + // ==================== Column Type Codes ==================== + + /** + * Column type: BOOLEAN (1 bit per value, packed). + */ + public static final byte TYPE_BOOLEAN = 0x01; + + /** + * Column type: BYTE (int8). + */ + public static final byte TYPE_BYTE = 0x02; + + /** + * Column type: SHORT (int16, little-endian). + */ + public static final byte TYPE_SHORT = 0x03; + + /** + * Column type: INT (int32, little-endian). + */ + public static final byte TYPE_INT = 0x04; + + /** + * Column type: LONG (int64, little-endian). + */ + public static final byte TYPE_LONG = 0x05; + + /** + * Column type: FLOAT (IEEE 754 float32). + */ + public static final byte TYPE_FLOAT = 0x06; + + /** + * Column type: DOUBLE (IEEE 754 float64). + */ + public static final byte TYPE_DOUBLE = 0x07; + + /** + * Column type: STRING (length-prefixed UTF-8). + */ + public static final byte TYPE_STRING = 0x08; + + /** + * Column type: SYMBOL (dictionary-encoded string). + */ + public static final byte TYPE_SYMBOL = 0x09; + + /** + * Column type: TIMESTAMP (int64 microseconds since epoch). + * Use this for timestamps beyond nanosecond range (year > 2262). + */ + public static final byte TYPE_TIMESTAMP = 0x0A; + + /** + * Column type: TIMESTAMP_NANOS (int64 nanoseconds since epoch). + * Use this for full nanosecond precision (limited to years 1677-2262). + */ + public static final byte TYPE_TIMESTAMP_NANOS = 0x10; + + /** + * Column type: DATE (int64 milliseconds since epoch). + */ + public static final byte TYPE_DATE = 0x0B; + + /** + * Column type: UUID (16 bytes, big-endian). + */ + public static final byte TYPE_UUID = 0x0C; + + /** + * Column type: LONG256 (32 bytes, big-endian). + */ + public static final byte TYPE_LONG256 = 0x0D; + + /** + * Column type: GEOHASH (varint bits + packed geohash). + */ + public static final byte TYPE_GEOHASH = 0x0E; + + /** + * Column type: VARCHAR (length-prefixed UTF-8, aux storage). + */ + public static final byte TYPE_VARCHAR = 0x0F; + + /** + * Column type: DOUBLE_ARRAY (N-dimensional array of IEEE 754 float64). + * Wire format: [nDims (1B)] [dim1_len (4B)]...[dimN_len (4B)] [flattened values (LE)] + */ + public static final byte TYPE_DOUBLE_ARRAY = 0x11; + + /** + * Column type: LONG_ARRAY (N-dimensional array of int64). + * Wire format: [nDims (1B)] [dim1_len (4B)]...[dimN_len (4B)] [flattened values (LE)] + */ + public static final byte TYPE_LONG_ARRAY = 0x12; + + /** + * Column type: DECIMAL64 (8 bytes, 18 digits precision). + * Wire format: [scale (1B in schema)] + [big-endian unscaled value (8B)] + */ + public static final byte TYPE_DECIMAL64 = 0x13; + + /** + * Column type: DECIMAL128 (16 bytes, 38 digits precision). + * Wire format: [scale (1B in schema)] + [big-endian unscaled value (16B)] + */ + public static final byte TYPE_DECIMAL128 = 0x14; + + /** + * Column type: DECIMAL256 (32 bytes, 77 digits precision). + * Wire format: [scale (1B in schema)] + [big-endian unscaled value (32B)] + */ + public static final byte TYPE_DECIMAL256 = 0x15; + + /** + * Column type: CHAR (2-byte UTF-16 code unit). + */ + public static final byte TYPE_CHAR = 0x16; + + /** + * High bit indicating nullable column. + */ + public static final byte TYPE_NULLABLE_FLAG = (byte) 0x80; + + /** + * Mask for type code without nullable flag. + */ + public static final byte TYPE_MASK = 0x7F; + + // ==================== Schema Mode ==================== + + /** + * Schema mode: Full schema included. + */ + public static final byte SCHEMA_MODE_FULL = 0x00; + + /** + * Schema mode: Schema reference (hash lookup). + */ + public static final byte SCHEMA_MODE_REFERENCE = 0x01; + + // ==================== Response Status Codes ==================== + + /** + * Status: Batch accepted successfully. + */ + public static final byte STATUS_OK = 0x00; + + /** + * Status: Some rows failed (partial failure). + */ + public static final byte STATUS_PARTIAL = 0x01; + + /** + * Status: Schema hash not recognized. + */ + public static final byte STATUS_SCHEMA_REQUIRED = 0x02; + + /** + * Status: Column type incompatible. + */ + public static final byte STATUS_SCHEMA_MISMATCH = 0x03; + + /** + * Status: Table doesn't exist (auto-create disabled). + */ + public static final byte STATUS_TABLE_NOT_FOUND = 0x04; + + /** + * Status: Malformed message. + */ + public static final byte STATUS_PARSE_ERROR = 0x05; + + /** + * Status: Server error. + */ + public static final byte STATUS_INTERNAL_ERROR = 0x06; + + /** + * Status: Back-pressure, retry later. + */ + public static final byte STATUS_OVERLOADED = 0x07; + + // ==================== Default Limits ==================== + + /** + * Default maximum batch size in bytes (16 MB). + */ + public static final int DEFAULT_MAX_BATCH_SIZE = 16 * 1024 * 1024; + + /** + * Default maximum tables per batch. + */ + public static final int DEFAULT_MAX_TABLES_PER_BATCH = 256; + + /** + * Default maximum rows per table in a batch. + */ + public static final int DEFAULT_MAX_ROWS_PER_TABLE = 1_000_000; + + /** + * Maximum columns per table (QuestDB limit). + */ + public static final int MAX_COLUMNS_PER_TABLE = 2048; + + /** + * Maximum table name length in bytes. + */ + public static final int MAX_TABLE_NAME_LENGTH = 127; + + /** + * Maximum column name length in bytes. + */ + public static final int MAX_COLUMN_NAME_LENGTH = 127; + + /** + * Default maximum string length in bytes (1 MB). + */ + public static final int DEFAULT_MAX_STRING_LENGTH = 1024 * 1024; + + /** + * Default initial receive buffer size (64 KB). + */ + public static final int DEFAULT_INITIAL_RECV_BUFFER_SIZE = 64 * 1024; + + /** + * Maximum in-flight batches for pipelining. + */ + public static final int DEFAULT_MAX_IN_FLIGHT_BATCHES = 4; + + // ==================== Capability Negotiation ==================== + + /** + * Size of capability request in bytes. + */ + public static final int CAPABILITY_REQUEST_SIZE = 8; + + /** + * Size of capability response in bytes. + */ + public static final int CAPABILITY_RESPONSE_SIZE = 8; + + private IlpV4Constants() { + // utility class + } + + /** + * Returns true if the type code represents a fixed-width type. + * + * @param typeCode the column type code (without nullable flag) + * @return true if fixed-width + */ + public static boolean isFixedWidthType(byte typeCode) { + int code = typeCode & TYPE_MASK; + return code == TYPE_BOOLEAN || + code == TYPE_BYTE || + code == TYPE_SHORT || + code == TYPE_CHAR || + code == TYPE_INT || + code == TYPE_LONG || + code == TYPE_FLOAT || + code == TYPE_DOUBLE || + code == TYPE_TIMESTAMP || + code == TYPE_TIMESTAMP_NANOS || + code == TYPE_DATE || + code == TYPE_UUID || + code == TYPE_LONG256; + } + + /** + * Returns the size in bytes for fixed-width types. + * + * @param typeCode the column type code (without nullable flag) + * @return size in bytes, or -1 for variable-width types + */ + public static int getFixedTypeSize(byte typeCode) { + int code = typeCode & TYPE_MASK; + switch (code) { + case TYPE_BOOLEAN: + return 0; // Special: bit-packed + case TYPE_BYTE: + return 1; + case TYPE_SHORT: + case TYPE_CHAR: + return 2; + case TYPE_INT: + case TYPE_FLOAT: + return 4; + case TYPE_LONG: + case TYPE_DOUBLE: + case TYPE_TIMESTAMP: + case TYPE_TIMESTAMP_NANOS: + case TYPE_DATE: + return 8; + case TYPE_UUID: + return 16; + case TYPE_LONG256: + return 32; + default: + return -1; // Variable width + } + } + + /** + * Returns a human-readable name for the type code. + * + * @param typeCode the column type code + * @return type name + */ + public static String getTypeName(byte typeCode) { + int code = typeCode & TYPE_MASK; + boolean nullable = (typeCode & TYPE_NULLABLE_FLAG) != 0; + String name; + switch (code) { + case TYPE_BOOLEAN: + name = "BOOLEAN"; + break; + case TYPE_BYTE: + name = "BYTE"; + break; + case TYPE_SHORT: + name = "SHORT"; + break; + case TYPE_CHAR: + name = "CHAR"; + break; + case TYPE_INT: + name = "INT"; + break; + case TYPE_LONG: + name = "LONG"; + break; + case TYPE_FLOAT: + name = "FLOAT"; + break; + case TYPE_DOUBLE: + name = "DOUBLE"; + break; + case TYPE_STRING: + name = "STRING"; + break; + case TYPE_SYMBOL: + name = "SYMBOL"; + break; + case TYPE_TIMESTAMP: + name = "TIMESTAMP"; + break; + case TYPE_TIMESTAMP_NANOS: + name = "TIMESTAMP_NANOS"; + break; + case TYPE_DATE: + name = "DATE"; + break; + case TYPE_UUID: + name = "UUID"; + break; + case TYPE_LONG256: + name = "LONG256"; + break; + case TYPE_GEOHASH: + name = "GEOHASH"; + break; + case TYPE_VARCHAR: + name = "VARCHAR"; + break; + case TYPE_DOUBLE_ARRAY: + name = "DOUBLE_ARRAY"; + break; + case TYPE_LONG_ARRAY: + name = "LONG_ARRAY"; + break; + case TYPE_DECIMAL64: + name = "DECIMAL64"; + break; + case TYPE_DECIMAL128: + name = "DECIMAL128"; + break; + case TYPE_DECIMAL256: + name = "DECIMAL256"; + break; + default: + name = "UNKNOWN(" + code + ")"; + } + return nullable ? name + "?" : name; + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4GorillaDecoder.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4GorillaDecoder.java new file mode 100644 index 0000000..2471ad4 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4GorillaDecoder.java @@ -0,0 +1,251 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2024 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.ilpv4.protocol; + +/** + * Gorilla delta-of-delta decoder for timestamps in ILP v4 format. + *

+ * Gorilla encoding uses delta-of-delta compression where: + * 

+ * D = (t[n] - t[n-1]) - (t[n-1] - t[n-2])
+ *
+ * if D == 0:              write '0'              (1 bit)
+ * elif D in [-63, 64]:    write '10' + 7-bit     (9 bits)
+ * elif D in [-255, 256]:  write '110' + 9-bit    (12 bits)
+ * elif D in [-2047, 2048]: write '1110' + 12-bit (16 bits)
+ * else:                   write '1111' + 32-bit  (36 bits)
+ *
+ *

+ * The decoder reads bit-packed delta-of-delta values and reconstructs + * the original timestamp sequence. + */ +public class IlpV4GorillaDecoder { + + // Bucket boundaries (two's complement signed ranges) + private static final int BUCKET_7BIT_MIN = -63; + private static final int BUCKET_7BIT_MAX = 64; + private static final int BUCKET_9BIT_MIN = -255; + private static final int BUCKET_9BIT_MAX = 256; + private static final int BUCKET_12BIT_MIN = -2047; + private static final int BUCKET_12BIT_MAX = 2048; + + private final IlpV4BitReader bitReader; + + // State for decoding + private long prevTimestamp; + private long prevDelta; + + /** + * Creates a new Gorilla decoder. + */ + public IlpV4GorillaDecoder() { + this.bitReader = new IlpV4BitReader(); + } + + /** + * Creates a decoder using an existing bit reader. + * + * @param bitReader the bit reader to use + */ + public IlpV4GorillaDecoder(IlpV4BitReader bitReader) { + this.bitReader = bitReader; + } + + /** + * Resets the decoder with the first two timestamps. + *

+ * The first two timestamps are always stored uncompressed and are used + * to establish the initial delta for subsequent compression. + * + * @param firstTimestamp the first timestamp in the sequence + * @param secondTimestamp the second timestamp in the sequence + */ + public void reset(long firstTimestamp, long secondTimestamp) { + this.prevTimestamp = secondTimestamp; + this.prevDelta = secondTimestamp - firstTimestamp; + } + + /** + * Resets the bit reader for reading encoded delta-of-deltas. + * + * @param address the address of the encoded data + * @param length the length of the encoded data in bytes + */ + public void resetReader(long address, long length) { + bitReader.reset(address, length); + } + + /** + * Decodes the next timestamp from the bit stream. + *

+ * The encoding format is: + *

    + *
  • '0' = delta-of-delta is 0 (1 bit)
    • + *
  • '10' + 7-bit signed = delta-of-delta in [-63, 64] (9 bits)
    • + *
  • '110' + 9-bit signed = delta-of-delta in [-255, 256] (12 bits)
    • + *
  • '1110' + 12-bit signed = delta-of-delta in [-2047, 2048] (16 bits)
    • + *
  • '1111' + 32-bit signed = any other delta-of-delta (36 bits)
    • + *
+ * + * @return the decoded timestamp + */ + public long decodeNext() { + long deltaOfDelta = decodeDoD(); + long delta = prevDelta + deltaOfDelta; + long timestamp = prevTimestamp + delta; + + prevDelta = delta; + prevTimestamp = timestamp; + + return timestamp; + } + + /** + * Decodes a delta-of-delta value from the bit stream. + * + * @return the delta-of-delta value + */ + private long decodeDoD() { + int bit = bitReader.readBit(); + + if (bit == 0) { + // '0' = DoD is 0 + return 0; + } + + // bit == 1, check next bit + bit = bitReader.readBit(); + if (bit == 0) { + // '10' = 7-bit signed value + return bitReader.readSigned(7); + } + + // '11', check next bit + bit = bitReader.readBit(); + if (bit == 0) { + // '110' = 9-bit signed value + return bitReader.readSigned(9); + } + + // '111', check next bit + bit = bitReader.readBit(); + if (bit == 0) { + // '1110' = 12-bit signed value + return bitReader.readSigned(12); + } + + // '1111' = 32-bit signed value + return bitReader.readSigned(32); + } + + /** + * Returns whether there are more bits available in the reader. + * + * @return true if more bits available + */ + public boolean hasMoreBits() { + return bitReader.hasMoreBits(); + } + + /** + * Returns the number of bits remaining. + * + * @return available bits + */ + public long getAvailableBits() { + return bitReader.getAvailableBits(); + } + + /** + * Returns the current bit position (bits read since reset). + * + * @return bits read + */ + public long getBitPosition() { + return bitReader.getBitPosition(); + } + + /** + * Gets the previous timestamp (for debugging/testing). + * + * @return the last decoded timestamp + */ + public long getPrevTimestamp() { + return prevTimestamp; + } + + /** + * Gets the previous delta (for debugging/testing). + * + * @return the last computed delta + */ + public long getPrevDelta() { + return prevDelta; + } + + // ==================== Static Encoding Methods (for testing) ==================== + + /** + * Determines which bucket a delta-of-delta value falls into. + * + * @param deltaOfDelta the delta-of-delta value + * @return bucket number (0 = 1-bit, 1 = 9-bit, 2 = 12-bit, 3 = 16-bit, 4 = 36-bit) + */ + public static int getBucket(long deltaOfDelta) { + if (deltaOfDelta == 0) { + return 0; // 1-bit + } else if (deltaOfDelta >= BUCKET_7BIT_MIN && deltaOfDelta <= BUCKET_7BIT_MAX) { + return 1; // 9-bit (2 prefix + 7 value) + } else if (deltaOfDelta >= BUCKET_9BIT_MIN && deltaOfDelta <= BUCKET_9BIT_MAX) { + return 2; // 12-bit (3 prefix + 9 value) + } else if (deltaOfDelta >= BUCKET_12BIT_MIN && deltaOfDelta <= BUCKET_12BIT_MAX) { + return 3; // 16-bit (4 prefix + 12 value) + } else { + return 4; // 36-bit (4 prefix + 32 value) + } + } + + /** + * Returns the number of bits required to encode a delta-of-delta value. + * + * @param deltaOfDelta the delta-of-delta value + * @return bits required + */ + public static int getBitsRequired(long deltaOfDelta) { + int bucket = getBucket(deltaOfDelta); + switch (bucket) { + case 0: + return 1; + case 1: + return 9; + case 2: + return 12; + case 3: + return 16; + default: + return 36; + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4GorillaEncoder.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4GorillaEncoder.java new file mode 100644 index 0000000..e8f0f47 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4GorillaEncoder.java @@ -0,0 +1,235 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.ilpv4.protocol; + +import io.questdb.client.std.Unsafe; + +/** + * Gorilla delta-of-delta encoder for timestamps in ILP v4 format. + *

+ * This encoder is used by the WebSocket encoder to compress timestamp columns. + * It uses delta-of-delta compression where: + * 

+ * DoD = (t[n] - t[n-1]) - (t[n-1] - t[n-2])
+ *
+ * if DoD == 0:              write '0'              (1 bit)
+ * elif DoD in [-63, 64]:    write '10' + 7-bit     (9 bits)
+ * elif DoD in [-255, 256]:  write '110' + 9-bit    (12 bits)
+ * elif DoD in [-2047, 2048]: write '1110' + 12-bit (16 bits)
+ * else:                     write '1111' + 32-bit  (36 bits)
+ *
+ *

+ * The encoder writes first two timestamps uncompressed, then encodes + * remaining timestamps using delta-of-delta compression. + */ +public class IlpV4GorillaEncoder { + + private final IlpV4BitWriter bitWriter = new IlpV4BitWriter(); + + /** + * Creates a new Gorilla encoder. + */ + public IlpV4GorillaEncoder() { + } + + /** + * Encodes a delta-of-delta value using bucket selection. + *

+ * Prefix patterns are written LSB-first to match the decoder's read order: + *

    + *
  • '0' -> write bit 0
    • + *
  • '10' -> write bit 1, then bit 0 (0b01 as 2-bit value)
    • + *
  • '110' -> write bit 1, bit 1, bit 0 (0b011 as 3-bit value)
    • + *
  • '1110' -> write bit 1, bit 1, bit 1, bit 0 (0b0111 as 4-bit value)
    • + *
  • '1111' -> write bit 1, bit 1, bit 1, bit 1 (0b1111 as 4-bit value)
    • + *
+ * + * @param deltaOfDelta the delta-of-delta value to encode + */ + public void encodeDoD(long deltaOfDelta) { + int bucket = IlpV4GorillaDecoder.getBucket(deltaOfDelta); + switch (bucket) { + case 0: // DoD == 0 + bitWriter.writeBit(0); + break; + case 1: // [-63, 64] -> '10' + 7-bit + bitWriter.writeBits(0b01, 2); + bitWriter.writeSigned(deltaOfDelta, 7); + break; + case 2: // [-255, 256] -> '110' + 9-bit + bitWriter.writeBits(0b011, 3); + bitWriter.writeSigned(deltaOfDelta, 9); + break; + case 3: // [-2047, 2048] -> '1110' + 12-bit + bitWriter.writeBits(0b0111, 4); + bitWriter.writeSigned(deltaOfDelta, 12); + break; + default: // '1111' + 32-bit + bitWriter.writeBits(0b1111, 4); + bitWriter.writeSigned(deltaOfDelta, 32); + break; + } + } + + /** + * Encodes an array of timestamps to native memory using Gorilla compression. + *

+ * Format: + * 

+     * - First timestamp: int64 (8 bytes, little-endian)
+     * - Second timestamp: int64 (8 bytes, little-endian)
+     * - Remaining timestamps: bit-packed delta-of-delta
+     *
+ *

+ * Note: This method does NOT write the encoding flag byte. The caller is + * responsible for writing the ENCODING_GORILLA flag before calling this method. + * + * @param destAddress destination address in native memory + * @param capacity maximum number of bytes to write + * @param timestamps array of timestamp values + * @param count number of timestamps to encode + * @return number of bytes written + */ + public int encodeTimestamps(long destAddress, long capacity, long[] timestamps, int count) { + if (count == 0) { + return 0; + } + + int pos = 0; + + // Write first timestamp uncompressed + if (capacity < 8) { + return 0; // Not enough space + } + Unsafe.getUnsafe().putLong(destAddress, timestamps[0]); + pos = 8; + + if (count == 1) { + return pos; + } + + // Write second timestamp uncompressed + if (capacity < pos + 8) { + return pos; // Not enough space + } + Unsafe.getUnsafe().putLong(destAddress + pos, timestamps[1]); + pos += 8; + + if (count == 2) { + return pos; + } + + // Encode remaining with delta-of-delta + bitWriter.reset(destAddress + pos, capacity - pos); + long prevTs = timestamps[1]; + long prevDelta = timestamps[1] - timestamps[0]; + + for (int i = 2; i < count; i++) { + long delta = timestamps[i] - prevTs; + long dod = delta - prevDelta; + encodeDoD(dod); + prevDelta = delta; + prevTs = timestamps[i]; + } + + return pos + bitWriter.finish(); + } + + /** + * Checks if Gorilla encoding can be used for the given timestamps. + *

+ * Gorilla encoding uses 32-bit signed integers for delta-of-delta values, + * so it cannot encode timestamps where the delta-of-delta exceeds the + * 32-bit signed integer range. + * + * @param timestamps array of timestamp values + * @param count number of timestamps + * @return true if Gorilla encoding can be used, false otherwise + */ + public static boolean canUseGorilla(long[] timestamps, int count) { + if (count < 3) { + return true; // No DoD encoding needed for 0, 1, or 2 timestamps + } + + long prevDelta = timestamps[1] - timestamps[0]; + for (int i = 2; i < count; i++) { + long delta = timestamps[i] - timestamps[i - 1]; + long dod = delta - prevDelta; + if (dod < Integer.MIN_VALUE || dod > Integer.MAX_VALUE) { + return false; + } + prevDelta = delta; + } + return true; + } + + /** + * Calculates the encoded size in bytes for Gorilla-encoded timestamps. + *

+ * Note: This does NOT include the encoding flag byte. Add 1 byte if + * the encoding flag is needed. + * + * @param timestamps array of timestamp values + * @param count number of timestamps + * @return encoded size in bytes (excluding encoding flag) + */ + public static int calculateEncodedSize(long[] timestamps, int count) { + if (count == 0) { + return 0; + } + + int size = 8; // first timestamp + + if (count == 1) { + return size; + } + + size += 8; // second timestamp + + if (count == 2) { + return size; + } + + // Calculate bits for delta-of-delta encoding + long prevTimestamp = timestamps[1]; + long prevDelta = timestamps[1] - timestamps[0]; + int totalBits = 0; + + for (int i = 2; i < count; i++) { + long delta = timestamps[i] - prevTimestamp; + long deltaOfDelta = delta - prevDelta; + + totalBits += IlpV4GorillaDecoder.getBitsRequired(deltaOfDelta); + + prevDelta = delta; + prevTimestamp = timestamps[i]; + } + + // Round up to bytes + size += (totalBits + 7) / 8; + + return size; + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4NullBitmap.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4NullBitmap.java new file mode 100644 index 0000000..738f2dd --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4NullBitmap.java @@ -0,0 +1,310 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2024 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.ilpv4.protocol; + +import io.questdb.client.std.Unsafe; + +/** + * Utility class for reading and writing null bitmaps in ILP v4 format. + *

+ * Null bitmap format: + *

    + *
  • Size: ceil(rowCount / 8) bytes
    • + *
  • bit[i] = 1 means row[i] is NULL
    • + *
  • Bit order: LSB first within each byte
    • + *
+ *

+ * Example: For 10 rows where rows 0, 2, 9 are null: + * 

+ * Byte 0: 0b00000101 (bits 0,2 set)
+ * Byte 1: 0b00000010 (bit 1 set, which is row 9)
+ *
+ */ +public final class IlpV4NullBitmap { + + private IlpV4NullBitmap() { + // utility class + } + + /** + * Calculates the size in bytes needed for a null bitmap. + * + * @param rowCount number of rows + * @return bitmap size in bytes + */ + public static int sizeInBytes(long rowCount) { + return (int) ((rowCount + 7) / 8); + } + + /** + * Checks if a specific row is null in the bitmap (from direct memory). + * + * @param address bitmap start address + * @param rowIndex row index to check + * @return true if the row is null + */ + public static boolean isNull(long address, int rowIndex) { + int byteIndex = rowIndex >>> 3; // rowIndex / 8 + int bitIndex = rowIndex & 7; // rowIndex % 8 + byte b = Unsafe.getUnsafe().getByte(address + byteIndex); + return (b & (1 << bitIndex)) != 0; + } + + /** + * Checks if a specific row is null in the bitmap (from byte array). + * + * @param bitmap bitmap byte array + * @param offset starting offset in array + * @param rowIndex row index to check + * @return true if the row is null + */ + public static boolean isNull(byte[] bitmap, int offset, int rowIndex) { + int byteIndex = rowIndex >>> 3; + int bitIndex = rowIndex & 7; + byte b = bitmap[offset + byteIndex]; + return (b & (1 << bitIndex)) != 0; + } + + /** + * Sets a row as null in the bitmap (direct memory). + * + * @param address bitmap start address + * @param rowIndex row index to set as null + */ + public static void setNull(long address, int rowIndex) { + int byteIndex = rowIndex >>> 3; + int bitIndex = rowIndex & 7; + long addr = address + byteIndex; + byte b = Unsafe.getUnsafe().getByte(addr); + b |= (1 << bitIndex); + Unsafe.getUnsafe().putByte(addr, b); + } + + /** + * Sets a row as null in the bitmap (byte array). + * + * @param bitmap bitmap byte array + * @param offset starting offset in array + * @param rowIndex row index to set as null + */ + public static void setNull(byte[] bitmap, int offset, int rowIndex) { + int byteIndex = rowIndex >>> 3; + int bitIndex = rowIndex & 7; + bitmap[offset + byteIndex] |= (1 << bitIndex); + } + + /** + * Clears a row's null flag in the bitmap (direct memory). + * + * @param address bitmap start address + * @param rowIndex row index to clear + */ + public static void clearNull(long address, int rowIndex) { + int byteIndex = rowIndex >>> 3; + int bitIndex = rowIndex & 7; + long addr = address + byteIndex; + byte b = Unsafe.getUnsafe().getByte(addr); + b &= ~(1 << bitIndex); + Unsafe.getUnsafe().putByte(addr, b); + } + + /** + * Clears a row's null flag in the bitmap (byte array). + * + * @param bitmap bitmap byte array + * @param offset starting offset in array + * @param rowIndex row index to clear + */ + public static void clearNull(byte[] bitmap, int offset, int rowIndex) { + int byteIndex = rowIndex >>> 3; + int bitIndex = rowIndex & 7; + bitmap[offset + byteIndex] &= ~(1 << bitIndex); + } + + /** + * Counts the number of null values in the bitmap. + * + * @param address bitmap start address + * @param rowCount total number of rows + * @return count of null values + */ + public static int countNulls(long address, int rowCount) { + int count = 0; + int fullBytes = rowCount >>> 3; + int remainingBits = rowCount & 7; + + // Count full bytes + for (int i = 0; i < fullBytes; i++) { + byte b = Unsafe.getUnsafe().getByte(address + i); + count += Integer.bitCount(b & 0xFF); + } + + // Count remaining bits in last partial byte + if (remainingBits > 0) { + byte b = Unsafe.getUnsafe().getByte(address + fullBytes); + int mask = (1 << remainingBits) - 1; + count += Integer.bitCount((b & mask) & 0xFF); + } + + return count; + } + + /** + * Counts the number of null values in the bitmap (byte array). + * + * @param bitmap bitmap byte array + * @param offset starting offset in array + * @param rowCount total number of rows + * @return count of null values + */ + public static int countNulls(byte[] bitmap, int offset, int rowCount) { + int count = 0; + int fullBytes = rowCount >>> 3; + int remainingBits = rowCount & 7; + + for (int i = 0; i < fullBytes; i++) { + count += Integer.bitCount(bitmap[offset + i] & 0xFF); + } + + if (remainingBits > 0) { + byte b = bitmap[offset + fullBytes]; + int mask = (1 << remainingBits) - 1; + count += Integer.bitCount((b & mask) & 0xFF); + } + + return count; + } + + /** + * Checks if all rows are null. + * + * @param address bitmap start address + * @param rowCount total number of rows + * @return true if all rows are null + */ + public static boolean allNull(long address, int rowCount) { + int fullBytes = rowCount >>> 3; + int remainingBits = rowCount & 7; + + // Check full bytes (all bits should be 1) + for (int i = 0; i < fullBytes; i++) { + byte b = Unsafe.getUnsafe().getByte(address + i); + if ((b & 0xFF) != 0xFF) { + return false; + } + } + + // Check remaining bits + if (remainingBits > 0) { + byte b = Unsafe.getUnsafe().getByte(address + fullBytes); + int mask = (1 << remainingBits) - 1; + if ((b & mask) != mask) { + return false; + } + } + + return true; + } + + /** + * Checks if no rows are null. + * + * @param address bitmap start address + * @param rowCount total number of rows + * @return true if no rows are null + */ + public static boolean noneNull(long address, int rowCount) { + int fullBytes = rowCount >>> 3; + int remainingBits = rowCount & 7; + + // Check full bytes + for (int i = 0; i < fullBytes; i++) { + byte b = Unsafe.getUnsafe().getByte(address + i); + if (b != 0) { + return false; + } + } + + // Check remaining bits + if (remainingBits > 0) { + byte b = Unsafe.getUnsafe().getByte(address + fullBytes); + int mask = (1 << remainingBits) - 1; + if ((b & mask) != 0) { + return false; + } + } + + return true; + } + + /** + * Fills the bitmap setting all rows as null (direct memory). + * + * @param address bitmap start address + * @param rowCount total number of rows + */ + public static void fillAllNull(long address, int rowCount) { + int fullBytes = rowCount >>> 3; + int remainingBits = rowCount & 7; + + // Fill full bytes with all 1s + for (int i = 0; i < fullBytes; i++) { + Unsafe.getUnsafe().putByte(address + i, (byte) 0xFF); + } + + // Set remaining bits in last byte + if (remainingBits > 0) { + byte mask = (byte) ((1 << remainingBits) - 1); + Unsafe.getUnsafe().putByte(address + fullBytes, mask); + } + } + + /** + * Clears the bitmap setting all rows as non-null (direct memory). + * + * @param address bitmap start address + * @param rowCount total number of rows + */ + public static void fillNoneNull(long address, int rowCount) { + int sizeBytes = sizeInBytes(rowCount); + for (int i = 0; i < sizeBytes; i++) { + Unsafe.getUnsafe().putByte(address + i, (byte) 0); + } + } + + /** + * Clears the bitmap setting all rows as non-null (byte array). + * + * @param bitmap bitmap byte array + * @param offset starting offset in array + * @param rowCount total number of rows + */ + public static void fillNoneNull(byte[] bitmap, int offset, int rowCount) { + int sizeBytes = sizeInBytes(rowCount); + for (int i = 0; i < sizeBytes; i++) { + bitmap[offset + i] = 0; + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4SchemaHash.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4SchemaHash.java new file mode 100644 index 0000000..566e2f2 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4SchemaHash.java @@ -0,0 +1,574 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2024 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.ilpv4.protocol; + + +import io.questdb.client.std.Unsafe; +import io.questdb.client.std.str.DirectUtf8Sequence; +import io.questdb.client.std.str.Utf8Sequence; + +import java.nio.charset.StandardCharsets; + +/** + * XXHash64 implementation for schema hashing in ILP v4 protocol. + *

+ * The schema hash is computed over column definitions (name + type) to enable + * schema caching. When a client sends a schema reference (hash), the server + * can look up the cached schema instead of re-parsing the full schema each time. + *

+ * This is a pure Java implementation of XXHash64 based on the original algorithm + * by Yann Collet. It's optimized for small inputs typical of schema hashing. + * + * @see xxHash + */ +public final class IlpV4SchemaHash { + + // XXHash64 constants + private static final long PRIME64_1 = 0x9E3779B185EBCA87L; + private static final long PRIME64_2 = 0xC2B2AE3D27D4EB4FL; + private static final long PRIME64_3 = 0x165667B19E3779F9L; + private static final long PRIME64_4 = 0x85EBCA77C2B2AE63L; + private static final long PRIME64_5 = 0x27D4EB2F165667C5L; + + // Default seed (0 for ILP v4) + private static final long DEFAULT_SEED = 0L; + + // Thread-local Hasher to avoid allocation on every computeSchemaHash call + private static final ThreadLocal HASHER_POOL = ThreadLocal.withInitial(Hasher::new); + + private IlpV4SchemaHash() { + // utility class + } + + /** + * Computes XXHash64 of a byte array. + * + * @param data the data to hash + * @return the 64-bit hash value + */ + public static long hash(byte[] data) { + return hash(data, 0, data.length, DEFAULT_SEED); + } + + /** + * Computes XXHash64 of a byte array region. + * + * @param data the data to hash + * @param offset starting offset + * @param length number of bytes to hash + * @return the 64-bit hash value + */ + public static long hash(byte[] data, int offset, int length) { + return hash(data, offset, length, DEFAULT_SEED); + } + + /** + * Computes XXHash64 of a byte array region with custom seed. + * + * @param data the data to hash + * @param offset starting offset + * @param length number of bytes to hash + * @param seed the hash seed + * @return the 64-bit hash value + */ + public static long hash(byte[] data, int offset, int length, long seed) { + long h64; + int end = offset + length; + int pos = offset; + + if (length >= 32) { + int limit = end - 32; + long v1 = seed + PRIME64_1 + PRIME64_2; + long v2 = seed + PRIME64_2; + long v3 = seed; + long v4 = seed - PRIME64_1; + + do { + v1 = round(v1, getLong(data, pos)); + pos += 8; + v2 = round(v2, getLong(data, pos)); + pos += 8; + v3 = round(v3, getLong(data, pos)); + pos += 8; + v4 = round(v4, getLong(data, pos)); + pos += 8; + } while (pos <= limit); + + h64 = Long.rotateLeft(v1, 1) + Long.rotateLeft(v2, 7) + + Long.rotateLeft(v3, 12) + Long.rotateLeft(v4, 18); + h64 = mergeRound(h64, v1); + h64 = mergeRound(h64, v2); + h64 = mergeRound(h64, v3); + h64 = mergeRound(h64, v4); + } else { + h64 = seed + PRIME64_5; + } + + h64 += length; + + // Process remaining 8-byte blocks + while (pos + 8 <= end) { + long k1 = getLong(data, pos); + k1 *= PRIME64_2; + k1 = Long.rotateLeft(k1, 31); + k1 *= PRIME64_1; + h64 ^= k1; + h64 = Long.rotateLeft(h64, 27) * PRIME64_1 + PRIME64_4; + pos += 8; + } + + // Process remaining 4-byte block + if (pos + 4 <= end) { + h64 ^= (getInt(data, pos) & 0xFFFFFFFFL) * PRIME64_1; + h64 = Long.rotateLeft(h64, 23) * PRIME64_2 + PRIME64_3; + pos += 4; + } + + // Process remaining bytes + while (pos < end) { + h64 ^= (data[pos] & 0xFFL) * PRIME64_5; + h64 = Long.rotateLeft(h64, 11) * PRIME64_1; + pos++; + } + + return avalanche(h64); + } + + /** + * Computes XXHash64 of direct memory. + * + * @param address start address + * @param length number of bytes + * @return the 64-bit hash value + */ + public static long hash(long address, long length) { + return hash(address, length, DEFAULT_SEED); + } + + /** + * Computes XXHash64 of direct memory with custom seed. + * + * @param address start address + * @param length number of bytes + * @param seed the hash seed + * @return the 64-bit hash value + */ + public static long hash(long address, long length, long seed) { + long h64; + long end = address + length; + long pos = address; + + if (length >= 32) { + long limit = end - 32; + long v1 = seed + PRIME64_1 + PRIME64_2; + long v2 = seed + PRIME64_2; + long v3 = seed; + long v4 = seed - PRIME64_1; + + do { + v1 = round(v1, Unsafe.getUnsafe().getLong(pos)); + pos += 8; + v2 = round(v2, Unsafe.getUnsafe().getLong(pos)); + pos += 8; + v3 = round(v3, Unsafe.getUnsafe().getLong(pos)); + pos += 8; + v4 = round(v4, Unsafe.getUnsafe().getLong(pos)); + pos += 8; + } while (pos <= limit); + + h64 = Long.rotateLeft(v1, 1) + Long.rotateLeft(v2, 7) + + Long.rotateLeft(v3, 12) + Long.rotateLeft(v4, 18); + h64 = mergeRound(h64, v1); + h64 = mergeRound(h64, v2); + h64 = mergeRound(h64, v3); + h64 = mergeRound(h64, v4); + } else { + h64 = seed + PRIME64_5; + } + + h64 += length; + + // Process remaining 8-byte blocks + while (pos + 8 <= end) { + long k1 = Unsafe.getUnsafe().getLong(pos); + k1 *= PRIME64_2; + k1 = Long.rotateLeft(k1, 31); + k1 *= PRIME64_1; + h64 ^= k1; + h64 = Long.rotateLeft(h64, 27) * PRIME64_1 + PRIME64_4; + pos += 8; + } + + // Process remaining 4-byte block + if (pos + 4 <= end) { + h64 ^= (Unsafe.getUnsafe().getInt(pos) & 0xFFFFFFFFL) * PRIME64_1; + h64 = Long.rotateLeft(h64, 23) * PRIME64_2 + PRIME64_3; + pos += 4; + } + + // Process remaining bytes + while (pos < end) { + h64 ^= (Unsafe.getUnsafe().getByte(pos) & 0xFFL) * PRIME64_5; + h64 = Long.rotateLeft(h64, 11) * PRIME64_1; + pos++; + } + + return avalanche(h64); + } + + /** + * Computes the schema hash for ILP v4. + *

+ * Hash is computed over: for each column, hash(name_bytes + type_byte) + * This matches the spec in Appendix C. + * + * @param columnNames array of column names (UTF-8) + * @param columnTypes array of type codes + * @return the schema hash + */ + public static long computeSchemaHash(Utf8Sequence[] columnNames, byte[] columnTypes) { + // Use pooled hasher to avoid allocation + Hasher hasher = HASHER_POOL.get(); + hasher.reset(DEFAULT_SEED); + + for (int i = 0; i < columnNames.length; i++) { + Utf8Sequence name = columnNames[i]; + for (int j = 0, n = name.size(); j < n; j++) { + hasher.update(name.byteAt(j)); + } + hasher.update(columnTypes[i]); + } + + return hasher.getValue(); + } + + /** + * Computes the schema hash for ILP v4 using String column names. + * Note: Iterates over String chars and converts to UTF-8 bytes directly to avoid getBytes() allocation. + * + * @param columnNames array of column names + * @param columnTypes array of type codes + * @return the schema hash + */ + public static long computeSchemaHash(String[] columnNames, byte[] columnTypes) { + // Use pooled hasher to avoid allocation + Hasher hasher = HASHER_POOL.get(); + hasher.reset(DEFAULT_SEED); + + for (int i = 0; i < columnNames.length; i++) { + String name = columnNames[i]; + // Encode UTF-8 directly without allocating byte array + for (int j = 0, len = name.length(); j < len; j++) { + char c = name.charAt(j); + if (c < 0x80) { + // Single byte (ASCII) + hasher.update((byte) c); + } else if (c < 0x800) { + // Two bytes + hasher.update((byte) (0xC0 | (c >> 6))); + hasher.update((byte) (0x80 | (c & 0x3F))); + } else if (c >= 0xD800 && c <= 0xDBFF && j + 1 < len) { + // Surrogate pair (4 bytes) + char c2 = name.charAt(++j); + int codePoint = 0x10000 + ((c - 0xD800) << 10) + (c2 - 0xDC00); + hasher.update((byte) (0xF0 | (codePoint >> 18))); + hasher.update((byte) (0x80 | ((codePoint >> 12) & 0x3F))); + hasher.update((byte) (0x80 | ((codePoint >> 6) & 0x3F))); + hasher.update((byte) (0x80 | (codePoint & 0x3F))); + } else { + // Three bytes + hasher.update((byte) (0xE0 | (c >> 12))); + hasher.update((byte) (0x80 | ((c >> 6) & 0x3F))); + hasher.update((byte) (0x80 | (c & 0x3F))); + } + } + hasher.update(columnTypes[i]); + } + + return hasher.getValue(); + } + + /** + * Computes the schema hash for ILP v4 using DirectUtf8Sequence column names. + * + * @param columnNames array of column names + * @param columnTypes array of type codes + * @return the schema hash + */ + public static long computeSchemaHash(DirectUtf8Sequence[] columnNames, byte[] columnTypes) { + // Use pooled hasher to avoid allocation + Hasher hasher = HASHER_POOL.get(); + hasher.reset(DEFAULT_SEED); + + for (int i = 0; i < columnNames.length; i++) { + DirectUtf8Sequence name = columnNames[i]; + long addr = name.ptr(); + int len = name.size(); + for (int j = 0; j < len; j++) { + hasher.update(Unsafe.getUnsafe().getByte(addr + j)); + } + hasher.update(columnTypes[i]); + } + + return hasher.getValue(); + } + + /** + * Computes the schema hash directly from column buffers without intermediate arrays. + * This is the most efficient method when column data is already available. + * + * @param columns list of column buffers + * @return the schema hash + */ + public static long computeSchemaHashDirect(io.questdb.client.std.ObjList columns) { + // Use pooled hasher to avoid allocation + Hasher hasher = HASHER_POOL.get(); + hasher.reset(DEFAULT_SEED); + + for (int i = 0, n = columns.size(); i < n; i++) { + IlpV4TableBuffer.ColumnBuffer col = columns.get(i); + String name = col.getName(); + // Encode UTF-8 directly without allocating byte array + for (int j = 0, len = name.length(); j < len; j++) { + char c = name.charAt(j); + if (c < 0x80) { + hasher.update((byte) c); + } else if (c < 0x800) { + hasher.update((byte) (0xC0 | (c >> 6))); + hasher.update((byte) (0x80 | (c & 0x3F))); + } else if (c >= 0xD800 && c <= 0xDBFF && j + 1 < len) { + char c2 = name.charAt(++j); + int codePoint = 0x10000 + ((c - 0xD800) << 10) + (c2 - 0xDC00); + hasher.update((byte) (0xF0 | (codePoint >> 18))); + hasher.update((byte) (0x80 | ((codePoint >> 12) & 0x3F))); + hasher.update((byte) (0x80 | ((codePoint >> 6) & 0x3F))); + hasher.update((byte) (0x80 | (codePoint & 0x3F))); + } else { + hasher.update((byte) (0xE0 | (c >> 12))); + hasher.update((byte) (0x80 | ((c >> 6) & 0x3F))); + hasher.update((byte) (0x80 | (c & 0x3F))); + } + } + // Wire type code: type | (nullable ? 0x80 : 0) + byte wireType = (byte) (col.getType() | (col.nullable ? 0x80 : 0)); + hasher.update(wireType); + } + + return hasher.getValue(); + } + + private static long round(long acc, long input) { + acc += input * PRIME64_2; + acc = Long.rotateLeft(acc, 31); + acc *= PRIME64_1; + return acc; + } + + private static long mergeRound(long acc, long val) { + val = round(0, val); + acc ^= val; + acc = acc * PRIME64_1 + PRIME64_4; + return acc; + } + + private static long avalanche(long h64) { + h64 ^= h64 >>> 33; + h64 *= PRIME64_2; + h64 ^= h64 >>> 29; + h64 *= PRIME64_3; + h64 ^= h64 >>> 32; + return h64; + } + + private static long getLong(byte[] data, int pos) { + return ((long) data[pos] & 0xFF) | + (((long) data[pos + 1] & 0xFF) << 8) | + (((long) data[pos + 2] & 0xFF) << 16) | + (((long) data[pos + 3] & 0xFF) << 24) | + (((long) data[pos + 4] & 0xFF) << 32) | + (((long) data[pos + 5] & 0xFF) << 40) | + (((long) data[pos + 6] & 0xFF) << 48) | + (((long) data[pos + 7] & 0xFF) << 56); + } + + private static int getInt(byte[] data, int pos) { + return (data[pos] & 0xFF) | + ((data[pos + 1] & 0xFF) << 8) | + ((data[pos + 2] & 0xFF) << 16) | + ((data[pos + 3] & 0xFF) << 24); + } + + /** + * Streaming hasher for incremental hash computation. + *

+ * This is useful when building the schema hash incrementally + * as columns are processed. + */ + public static class Hasher { + private long v1, v2, v3, v4; + private long totalLen; + private final byte[] buffer = new byte[32]; + private int bufferPos; + private long seed; + + public Hasher() { + reset(DEFAULT_SEED); + } + + /** + * Resets the hasher with the given seed. + * + * @param seed the hash seed + */ + public void reset(long seed) { + this.seed = seed; + v1 = seed + PRIME64_1 + PRIME64_2; + v2 = seed + PRIME64_2; + v3 = seed; + v4 = seed - PRIME64_1; + totalLen = 0; + bufferPos = 0; + } + + /** + * Updates the hash with a single byte. + * + * @param b the byte to add + */ + public void update(byte b) { + buffer[bufferPos++] = b; + totalLen++; + + if (bufferPos == 32) { + processBuffer(); + } + } + + /** + * Updates the hash with a byte array. + * + * @param data the bytes to add + */ + public void update(byte[] data) { + update(data, 0, data.length); + } + + /** + * Updates the hash with a byte array region. + * + * @param data the bytes to add + * @param offset starting offset + * @param length number of bytes + */ + public void update(byte[] data, int offset, int length) { + totalLen += length; + + // Fill buffer first + if (bufferPos > 0) { + int toCopy = Math.min(32 - bufferPos, length); + System.arraycopy(data, offset, buffer, bufferPos, toCopy); + bufferPos += toCopy; + offset += toCopy; + length -= toCopy; + + if (bufferPos == 32) { + processBuffer(); + } + } + + // Process 32-byte blocks directly + while (length >= 32) { + v1 = round(v1, getLong(data, offset)); + v2 = round(v2, getLong(data, offset + 8)); + v3 = round(v3, getLong(data, offset + 16)); + v4 = round(v4, getLong(data, offset + 24)); + offset += 32; + length -= 32; + } + + // Buffer remaining + if (length > 0) { + System.arraycopy(data, offset, buffer, 0, length); + bufferPos = length; + } + } + + /** + * Finalizes and returns the hash value. + * + * @return the 64-bit hash + */ + public long getValue() { + long h64; + + if (totalLen >= 32) { + h64 = Long.rotateLeft(v1, 1) + Long.rotateLeft(v2, 7) + + Long.rotateLeft(v3, 12) + Long.rotateLeft(v4, 18); + h64 = mergeRound(h64, v1); + h64 = mergeRound(h64, v2); + h64 = mergeRound(h64, v3); + h64 = mergeRound(h64, v4); + } else { + h64 = seed + PRIME64_5; + } + + h64 += totalLen; + + // Process buffered data + int pos = 0; + while (pos + 8 <= bufferPos) { + long k1 = getLong(buffer, pos); + k1 *= PRIME64_2; + k1 = Long.rotateLeft(k1, 31); + k1 *= PRIME64_1; + h64 ^= k1; + h64 = Long.rotateLeft(h64, 27) * PRIME64_1 + PRIME64_4; + pos += 8; + } + + if (pos + 4 <= bufferPos) { + h64 ^= (getInt(buffer, pos) & 0xFFFFFFFFL) * PRIME64_1; + h64 = Long.rotateLeft(h64, 23) * PRIME64_2 + PRIME64_3; + pos += 4; + } + + while (pos < bufferPos) { + h64 ^= (buffer[pos] & 0xFFL) * PRIME64_5; + h64 = Long.rotateLeft(h64, 11) * PRIME64_1; + pos++; + } + + return avalanche(h64); + } + + private void processBuffer() { + v1 = round(v1, getLong(buffer, 0)); + v2 = round(v2, getLong(buffer, 8)); + v3 = round(v3, getLong(buffer, 16)); + v4 = round(v4, getLong(buffer, 24)); + bufferPos = 0; + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4TableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4TableBuffer.java new file mode 100644 index 0000000..6d1af59 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4TableBuffer.java @@ -0,0 +1,1424 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2024 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.ilpv4.protocol; + +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.line.array.ArrayBufferAppender; +import io.questdb.client.cutlass.line.array.DoubleArray; +import io.questdb.client.cutlass.line.array.LongArray; +import io.questdb.client.std.CharSequenceIntHashMap; +import io.questdb.client.std.Decimal128; +import io.questdb.client.std.Decimal256; +import io.questdb.client.std.Decimal64; +import io.questdb.client.std.Decimals; +import io.questdb.client.std.ObjList; +import io.questdb.client.std.Unsafe; + +import java.util.Arrays; + +import static io.questdb.client.cutlass.ilpv4.protocol.IlpV4Constants.*; + +/** + * Buffers rows for a single table in columnar format. + *

+ * This buffer accumulates row data column by column, allowing efficient + * encoding to the ILP v4 wire format. + */ +public class IlpV4TableBuffer { + + private final String tableName; + private final ObjList columns; + private final CharSequenceIntHashMap columnNameToIndex; + private ColumnBuffer[] fastColumns; // plain array for O(1) sequential access + private int columnAccessCursor; // tracks expected next column index + private int rowCount; + private long schemaHash; + private boolean schemaHashComputed; + private IlpV4ColumnDef[] cachedColumnDefs; + private boolean columnDefsCacheValid; + + public IlpV4TableBuffer(String tableName) { + this.tableName = tableName; + this.columns = new ObjList<>(); + this.columnNameToIndex = new CharSequenceIntHashMap(); + this.rowCount = 0; + this.schemaHash = 0; + this.schemaHashComputed = false; + this.columnDefsCacheValid = false; + } + + /** + * Returns the table name. + */ + public String getTableName() { + return tableName; + } + + /** + * Returns the number of rows buffered. + */ + public int getRowCount() { + return rowCount; + } + + /** + * Returns the number of columns. + */ + public int getColumnCount() { + return columns.size(); + } + + /** + * Returns the column at the given index. + */ + public ColumnBuffer getColumn(int index) { + return columns.get(index); + } + + /** + * Returns the column definitions (cached for efficiency). + */ + public IlpV4ColumnDef[] getColumnDefs() { + if (!columnDefsCacheValid || cachedColumnDefs == null || cachedColumnDefs.length != columns.size()) { + cachedColumnDefs = new IlpV4ColumnDef[columns.size()]; + for (int i = 0; i < columns.size(); i++) { + ColumnBuffer col = columns.get(i); + cachedColumnDefs[i] = new IlpV4ColumnDef(col.name, col.type, col.nullable); + } + columnDefsCacheValid = true; + } + return cachedColumnDefs; + } + + /** + * Gets or creates a column with the given name and type. + *

+ * Optimized for the common case where columns are accessed in the same + * order every row: a sequential cursor avoids hash map lookups entirely. + */ + public ColumnBuffer getOrCreateColumn(String name, byte type, boolean nullable) { + // Fast path: predict next column in sequence + int n = columns.size(); + if (columnAccessCursor < n) { + ColumnBuffer candidate = fastColumns[columnAccessCursor]; + if (candidate.name.equals(name)) { + columnAccessCursor++; + if (candidate.type != type) { + throw new IllegalArgumentException( + "Column type mismatch for " + name + ": existing=" + candidate.type + " new=" + type + ); + } + return candidate; + } + } + + // Slow path: hash map lookup + int idx = columnNameToIndex.get(name); + if (idx != CharSequenceIntHashMap.NO_ENTRY_VALUE) { + ColumnBuffer existing = columns.get(idx); + if (existing.type != type) { + throw new IllegalArgumentException( + "Column type mismatch for " + name + ": existing=" + existing.type + " new=" + type + ); + } + return existing; + } + + // Create new column + ColumnBuffer col = new ColumnBuffer(name, type, nullable); + int index = columns.size(); + columns.add(col); + columnNameToIndex.put(name, index); + // Update fast access array + if (fastColumns == null || index >= fastColumns.length) { + int newLen = Math.max(8, index + 4); + ColumnBuffer[] newArr = new ColumnBuffer[newLen]; + if (fastColumns != null) { + System.arraycopy(fastColumns, 0, newArr, 0, index); + } + fastColumns = newArr; + } + fastColumns[index] = col; + schemaHashComputed = false; + columnDefsCacheValid = false; + return col; + } + + /** + * Advances to the next row. + *

+ * This should be called after all column values for the current row have been set. + */ + public void nextRow() { + // Reset sequential access cursor for the next row + columnAccessCursor = 0; + // Ensure all columns have the same row count + for (int i = 0, n = columns.size(); i < n; i++) { + ColumnBuffer col = fastColumns[i]; + // If column wasn't set for this row, add a null + while (col.size < rowCount + 1) { + col.addNull(); + } + } + rowCount++; + } + + /** + * Cancels the current in-progress row. + *

+ * This removes any column values added since the last {@link #nextRow()} call. + * If no values have been added for the current row, this is a no-op. + */ + public void cancelCurrentRow() { + // Reset sequential access cursor + columnAccessCursor = 0; + // Truncate each column back to the committed row count + for (int i = 0, n = columns.size(); i < n; i++) { + ColumnBuffer col = fastColumns[i]; + col.truncateTo(rowCount); + } + } + + /** + * Returns the schema hash for this table. + *

+ * The hash is computed to match what IlpV4Schema.computeSchemaHash() produces: + * - Uses wire type codes (with nullable bit) + * - Hash is over name bytes + type code for each column + */ + public long getSchemaHash() { + if (!schemaHashComputed) { + // Compute hash directly from column buffers without intermediate arrays + schemaHash = IlpV4SchemaHash.computeSchemaHashDirect(columns); + schemaHashComputed = true; + } + return schemaHash; + } + + /** + * Resets the buffer for reuse. + */ + public void reset() { + for (int i = 0, n = columns.size(); i < n; i++) { + fastColumns[i].reset(); + } + columnAccessCursor = 0; + rowCount = 0; + } + + /** + * Clears the buffer completely, including column definitions. + */ + public void clear() { + columns.clear(); + columnNameToIndex.clear(); + fastColumns = null; + columnAccessCursor = 0; + rowCount = 0; + schemaHash = 0; + schemaHashComputed = false; + columnDefsCacheValid = false; + cachedColumnDefs = null; + } + + /** + * Column buffer for a single column. + */ + public static class ColumnBuffer { + final String name; + final byte type; + final boolean nullable; + + private int size; // Total row count (including nulls) + private int valueCount; // Actual stored values (excludes nulls) + private int capacity; + + // Storage for different types + private boolean[] booleanValues; + private byte[] byteValues; + private short[] shortValues; + private int[] intValues; + private long[] longValues; + private float[] floatValues; + private double[] doubleValues; + private String[] stringValues; + private long[] uuidHigh; + private long[] uuidLow; + // Long256 stored as flat array: 4 longs per value (avoids inner array allocation) + private long[] long256Values; + + // Array storage (double/long arrays - variable length per row) + // Each row stores: [nDims (1B)][dim1..dimN (4B each)][flattened data] + // We track per-row metadata separately from the actual data + private byte[] arrayDims; // nDims per row + private int[] arrayShapes; // Flattened shape data (all dimensions concatenated) + private int arrayShapeOffset; // Current write offset in arrayShapes + private double[] doubleArrayData; // Flattened double values + private long[] longArrayData; // Flattened long values + private int arrayDataOffset; // Current write offset in data arrays + private int arrayRowCapacity; // Capacity for array row count + + // Null tracking - bit-packed for memory efficiency (1 bit per row vs 8 bits with boolean[]) + private long[] nullBitmapPacked; + private boolean hasNulls; + + // Symbol specific + private CharSequenceIntHashMap symbolDict; + private ObjList symbolList; + private int[] symbolIndices; + + // Global symbol IDs for delta encoding (parallel to symbolIndices) + private int[] globalSymbolIds; + private int maxGlobalSymbolId = -1; + + // Decimal storage + // All values in a decimal column must share the same scale + // For Decimal64: single long per value (64-bit unscaled) + // For Decimal128: two longs per value (128-bit unscaled: high, low) + // For Decimal256: four longs per value (256-bit unscaled: hh, hl, lh, ll) + private byte decimalScale = -1; // Shared scale for column (-1 = not set) + private long[] decimal64Values; // Decimal64: one long per value + private long[] decimal128High; // Decimal128: high 64 bits + private long[] decimal128Low; // Decimal128: low 64 bits + private long[] decimal256Hh; // Decimal256: bits 255-192 + private long[] decimal256Hl; // Decimal256: bits 191-128 + private long[] decimal256Lh; // Decimal256: bits 127-64 + private long[] decimal256Ll; // Decimal256: bits 63-0 + + public ColumnBuffer(String name, byte type, boolean nullable) { + this.name = name; + this.type = type; + this.nullable = nullable; + this.size = 0; + this.valueCount = 0; + this.capacity = 16; + this.hasNulls = false; + + allocateStorage(type, capacity); + if (nullable) { + // Bit-packed: 64 bits per long, so we need (capacity + 63) / 64 longs + nullBitmapPacked = new long[(capacity + 63) >>> 6]; + } + } + + public String getName() { + return name; + } + + public byte getType() { + return type; + } + + public int getSize() { + return size; + } + + /** + * Returns the number of actual stored values (excludes nulls). + */ + public int getValueCount() { + return valueCount; + } + + public boolean hasNulls() { + return hasNulls; + } + + /** + * Returns the bit-packed null bitmap. + * Each long contains 64 bits, bit 0 of long 0 = row 0, bit 1 of long 0 = row 1, etc. + */ + public long[] getNullBitmapPacked() { + return nullBitmapPacked; + } + + /** + * Returns the null bitmap as boolean array (for backward compatibility). + * This creates a new array, so prefer getNullBitmapPacked() for efficiency. + */ + public boolean[] getNullBitmap() { + if (nullBitmapPacked == null) { + return null; + } + boolean[] result = new boolean[size]; + for (int i = 0; i < size; i++) { + result[i] = isNull(i); + } + return result; + } + + /** + * Checks if the row at the given index is null. + */ + public boolean isNull(int index) { + if (nullBitmapPacked == null) { + return false; + } + int longIndex = index >>> 6; + int bitIndex = index & 63; + return (nullBitmapPacked[longIndex] & (1L << bitIndex)) != 0; + } + + public boolean[] getBooleanValues() { + return booleanValues; + } + + public byte[] getByteValues() { + return byteValues; + } + + public short[] getShortValues() { + return shortValues; + } + + public int[] getIntValues() { + return intValues; + } + + public long[] getLongValues() { + return longValues; + } + + public float[] getFloatValues() { + return floatValues; + } + + public double[] getDoubleValues() { + return doubleValues; + } + + public String[] getStringValues() { + return stringValues; + } + + public long[] getUuidHigh() { + return uuidHigh; + } + + public long[] getUuidLow() { + return uuidLow; + } + + /** + * Returns Long256 values as flat array (4 longs per value). + * Use getLong256Value(index, component) for indexed access. + */ + public long[] getLong256Values() { + return long256Values; + } + + /** + * Returns a component of a Long256 value. + * @param index value index + * @param component component 0-3 + */ + public long getLong256Value(int index, int component) { + return long256Values[index * 4 + component]; + } + + // ==================== Decimal getters ==================== + + /** + * Returns the shared scale for this decimal column. + * Returns -1 if no values have been added yet. + */ + public byte getDecimalScale() { + return decimalScale; + } + + /** + * Returns the Decimal64 values (one long per value). + */ + public long[] getDecimal64Values() { + return decimal64Values; + } + + /** + * Returns the high 64 bits of Decimal128 values. + */ + public long[] getDecimal128High() { + return decimal128High; + } + + /** + * Returns the low 64 bits of Decimal128 values. + */ + public long[] getDecimal128Low() { + return decimal128Low; + } + + /** + * Returns bits 255-192 of Decimal256 values. + */ + public long[] getDecimal256Hh() { + return decimal256Hh; + } + + /** + * Returns bits 191-128 of Decimal256 values. + */ + public long[] getDecimal256Hl() { + return decimal256Hl; + } + + /** + * Returns bits 127-64 of Decimal256 values. + */ + public long[] getDecimal256Lh() { + return decimal256Lh; + } + + /** + * Returns bits 63-0 of Decimal256 values. + */ + public long[] getDecimal256Ll() { + return decimal256Ll; + } + + /** + * Returns the array dimensions per row (nDims for each row). + */ + public byte[] getArrayDims() { + return arrayDims; + } + + /** + * Returns the flattened array shapes (all dimension lengths concatenated). + */ + public int[] getArrayShapes() { + return arrayShapes; + } + + /** + * Returns the current write offset in arrayShapes. + */ + public int getArrayShapeOffset() { + return arrayShapeOffset; + } + + /** + * Returns the flattened double array data. + */ + public double[] getDoubleArrayData() { + return doubleArrayData; + } + + /** + * Returns the flattened long array data. + */ + public long[] getLongArrayData() { + return longArrayData; + } + + /** + * Returns the current write offset in the data arrays. + */ + public int getArrayDataOffset() { + return arrayDataOffset; + } + + /** + * Returns the symbol indices array (one index per value). + * Each index refers to a position in the symbol dictionary. + */ + public int[] getSymbolIndices() { + return symbolIndices; + } + + /** + * Returns the symbol dictionary as a String array. + * Index i in symbolIndices maps to symbolDictionary[i]. + */ + public String[] getSymbolDictionary() { + if (symbolList == null) { + return new String[0]; + } + String[] dict = new String[symbolList.size()]; + for (int i = 0; i < symbolList.size(); i++) { + dict[i] = symbolList.get(i); + } + return dict; + } + + /** + * Returns the size of the symbol dictionary. + */ + public int getSymbolDictionarySize() { + return symbolList == null ? 0 : symbolList.size(); + } + + /** + * Returns the global symbol IDs array for delta encoding. + * Returns null if no global IDs have been stored. + */ + public int[] getGlobalSymbolIds() { + return globalSymbolIds; + } + + /** + * Returns the maximum global symbol ID used in this column. + * Returns -1 if no symbols have been added with global IDs. + */ + public int getMaxGlobalSymbolId() { + return maxGlobalSymbolId; + } + + public void addBoolean(boolean value) { + ensureCapacity(); + booleanValues[valueCount++] = value; + size++; + } + + public void addByte(byte value) { + ensureCapacity(); + byteValues[valueCount++] = value; + size++; + } + + public void addShort(short value) { + ensureCapacity(); + shortValues[valueCount++] = value; + size++; + } + + public void addInt(int value) { + ensureCapacity(); + intValues[valueCount++] = value; + size++; + } + + public void addLong(long value) { + ensureCapacity(); + longValues[valueCount++] = value; + size++; + } + + public void addFloat(float value) { + ensureCapacity(); + floatValues[valueCount++] = value; + size++; + } + + public void addDouble(double value) { + ensureCapacity(); + doubleValues[valueCount++] = value; + size++; + } + + public void addString(String value) { + ensureCapacity(); + if (value == null && nullable) { + markNull(size); + // Null strings don't take space in the value buffer + size++; + } else { + stringValues[valueCount++] = value; + size++; + } + } + + public void addSymbol(String value) { + ensureCapacity(); + if (value == null) { + if (nullable) { + markNull(size); + } + // Null symbols don't take space in the value buffer + size++; + } else { + int idx = symbolDict.get(value); + if (idx == CharSequenceIntHashMap.NO_ENTRY_VALUE) { + idx = symbolList.size(); + symbolDict.put(value, idx); + symbolList.add(value); + } + symbolIndices[valueCount++] = idx; + size++; + } + } + + /** + * Adds a symbol with both local dictionary and global ID tracking. + * Used for delta dictionary encoding where global IDs are shared across all columns. + * + * @param value the symbol string + * @param globalId the global ID from GlobalSymbolDictionary + */ + public void addSymbolWithGlobalId(String value, int globalId) { + ensureCapacity(); + if (value == null) { + if (nullable) { + markNull(size); + } + size++; + } else { + // Add to local dictionary (for backward compatibility with existing encoder) + int localIdx = symbolDict.get(value); + if (localIdx == CharSequenceIntHashMap.NO_ENTRY_VALUE) { + localIdx = symbolList.size(); + symbolDict.put(value, localIdx); + symbolList.add(value); + } + symbolIndices[valueCount] = localIdx; + + // Also store global ID for delta encoding + if (globalSymbolIds == null) { + globalSymbolIds = new int[capacity]; + } + globalSymbolIds[valueCount] = globalId; + + // Track max global ID for this column + if (globalId > maxGlobalSymbolId) { + maxGlobalSymbolId = globalId; + } + + valueCount++; + size++; + } + } + + public void addUuid(long high, long low) { + ensureCapacity(); + uuidHigh[valueCount] = high; + uuidLow[valueCount] = low; + valueCount++; + size++; + } + + public void addLong256(long l0, long l1, long l2, long l3) { + ensureCapacity(); + int offset = valueCount * 4; + long256Values[offset] = l0; + long256Values[offset + 1] = l1; + long256Values[offset + 2] = l2; + long256Values[offset + 3] = l3; + valueCount++; + size++; + } + + // ==================== Decimal methods ==================== + + /** + * Adds a Decimal64 value. + * All values in a decimal column must share the same scale. + * + * @param value the Decimal64 value to add + * @throws LineSenderException if the scale doesn't match previous values + */ + public void addDecimal64(Decimal64 value) { + if (value == null || value.isNull()) { + addNull(); + return; + } + ensureCapacity(); + validateAndSetScale((byte) value.getScale()); + decimal64Values[valueCount++] = value.getValue(); + size++; + } + + /** + * Adds a Decimal128 value. + * All values in a decimal column must share the same scale. + * + * @param value the Decimal128 value to add + * @throws LineSenderException if the scale doesn't match previous values + */ + public void addDecimal128(Decimal128 value) { + if (value == null || value.isNull()) { + addNull(); + return; + } + ensureCapacity(); + validateAndSetScale((byte) value.getScale()); + decimal128High[valueCount] = value.getHigh(); + decimal128Low[valueCount] = value.getLow(); + valueCount++; + size++; + } + + /** + * Adds a Decimal256 value. + * All values in a decimal column must share the same scale. + * + * @param value the Decimal256 value to add + * @throws LineSenderException if the scale doesn't match previous values + */ + public void addDecimal256(Decimal256 value) { + if (value == null || value.isNull()) { + addNull(); + return; + } + ensureCapacity(); + validateAndSetScale((byte) value.getScale()); + decimal256Hh[valueCount] = value.getHh(); + decimal256Hl[valueCount] = value.getHl(); + decimal256Lh[valueCount] = value.getLh(); + decimal256Ll[valueCount] = value.getLl(); + valueCount++; + size++; + } + + /** + * Validates that the given scale matches the column's scale. + * If this is the first value, sets the column scale. + * + * @param scale the scale of the value being added + * @throws LineSenderException if the scale doesn't match + */ + private void validateAndSetScale(byte scale) { + if (decimalScale == -1) { + decimalScale = scale; + } else if (decimalScale != scale) { + throw new LineSenderException( + "decimal scale mismatch in column '" + name + "': expected " + + decimalScale + " but got " + scale + + ". All values in a decimal column must have the same scale." + ); + } + } + + // ==================== Array methods ==================== + + /** + * Adds a 1D double array. + */ + public void addDoubleArray(double[] values) { + if (values == null) { + addNull(); + return; + } + ensureArrayCapacity(1, values.length); + arrayDims[valueCount] = 1; + arrayShapes[arrayShapeOffset++] = values.length; + for (double v : values) { + doubleArrayData[arrayDataOffset++] = v; + } + valueCount++; + size++; + } + + /** + * Adds a 2D double array. + * @throws LineSenderException if the array is jagged (irregular shape) + */ + public void addDoubleArray(double[][] values) { + if (values == null) { + addNull(); + return; + } + int dim0 = values.length; + int dim1 = dim0 > 0 ? values[0].length : 0; + // Validate rectangular shape + for (int i = 1; i < dim0; i++) { + if (values[i].length != dim1) { + throw new LineSenderException("irregular array shape"); + } + } + ensureArrayCapacity(2, dim0 * dim1); + arrayDims[valueCount] = 2; + arrayShapes[arrayShapeOffset++] = dim0; + arrayShapes[arrayShapeOffset++] = dim1; + for (double[] row : values) { + for (double v : row) { + doubleArrayData[arrayDataOffset++] = v; + } + } + valueCount++; + size++; + } + + /** + * Adds a 3D double array. + * @throws LineSenderException if the array is jagged (irregular shape) + */ + public void addDoubleArray(double[][][] values) { + if (values == null) { + addNull(); + return; + } + int dim0 = values.length; + int dim1 = dim0 > 0 ? values[0].length : 0; + int dim2 = dim0 > 0 && dim1 > 0 ? values[0][0].length : 0; + // Validate rectangular shape + for (int i = 0; i < dim0; i++) { + if (values[i].length != dim1) { + throw new LineSenderException("irregular array shape"); + } + for (int j = 0; j < dim1; j++) { + if (values[i][j].length != dim2) { + throw new LineSenderException("irregular array shape"); + } + } + } + ensureArrayCapacity(3, dim0 * dim1 * dim2); + arrayDims[valueCount] = 3; + arrayShapes[arrayShapeOffset++] = dim0; + arrayShapes[arrayShapeOffset++] = dim1; + arrayShapes[arrayShapeOffset++] = dim2; + for (double[][] plane : values) { + for (double[] row : plane) { + for (double v : row) { + doubleArrayData[arrayDataOffset++] = v; + } + } + } + valueCount++; + size++; + } + + /** + * Adds a DoubleArray (N-dimensional wrapper). + * Uses a capturing approach to extract shape and data. + */ + public void addDoubleArray(DoubleArray array) { + if (array == null) { + addNull(); + return; + } + // Use a capturing ArrayBufferAppender to extract the data + ArrayCapture capture = new ArrayCapture(); + array.appendToBufPtr(capture); + + ensureArrayCapacity(capture.nDims, capture.doubleDataOffset); + arrayDims[valueCount] = capture.nDims; + for (int i = 0; i < capture.nDims; i++) { + arrayShapes[arrayShapeOffset++] = capture.shape[i]; + } + for (int i = 0; i < capture.doubleDataOffset; i++) { + doubleArrayData[arrayDataOffset++] = capture.doubleData[i]; + } + valueCount++; + size++; + } + + /** + * Adds a 1D long array. + */ + public void addLongArray(long[] values) { + if (values == null) { + addNull(); + return; + } + ensureArrayCapacity(1, values.length); + arrayDims[valueCount] = 1; + arrayShapes[arrayShapeOffset++] = values.length; + for (long v : values) { + longArrayData[arrayDataOffset++] = v; + } + valueCount++; + size++; + } + + /** + * Adds a 2D long array. + * @throws LineSenderException if the array is jagged (irregular shape) + */ + public void addLongArray(long[][] values) { + if (values == null) { + addNull(); + return; + } + int dim0 = values.length; + int dim1 = dim0 > 0 ? values[0].length : 0; + // Validate rectangular shape + for (int i = 1; i < dim0; i++) { + if (values[i].length != dim1) { + throw new LineSenderException("irregular array shape"); + } + } + ensureArrayCapacity(2, dim0 * dim1); + arrayDims[valueCount] = 2; + arrayShapes[arrayShapeOffset++] = dim0; + arrayShapes[arrayShapeOffset++] = dim1; + for (long[] row : values) { + for (long v : row) { + longArrayData[arrayDataOffset++] = v; + } + } + valueCount++; + size++; + } + + /** + * Adds a 3D long array. + * @throws LineSenderException if the array is jagged (irregular shape) + */ + public void addLongArray(long[][][] values) { + if (values == null) { + addNull(); + return; + } + int dim0 = values.length; + int dim1 = dim0 > 0 ? values[0].length : 0; + int dim2 = dim0 > 0 && dim1 > 0 ? values[0][0].length : 0; + // Validate rectangular shape + for (int i = 0; i < dim0; i++) { + if (values[i].length != dim1) { + throw new LineSenderException("irregular array shape"); + } + for (int j = 0; j < dim1; j++) { + if (values[i][j].length != dim2) { + throw new LineSenderException("irregular array shape"); + } + } + } + ensureArrayCapacity(3, dim0 * dim1 * dim2); + arrayDims[valueCount] = 3; + arrayShapes[arrayShapeOffset++] = dim0; + arrayShapes[arrayShapeOffset++] = dim1; + arrayShapes[arrayShapeOffset++] = dim2; + for (long[][] plane : values) { + for (long[] row : plane) { + for (long v : row) { + longArrayData[arrayDataOffset++] = v; + } + } + } + valueCount++; + size++; + } + + /** + * Adds a LongArray (N-dimensional wrapper). + * Uses a capturing approach to extract shape and data. + */ + public void addLongArray(LongArray array) { + if (array == null) { + addNull(); + return; + } + // Use a capturing ArrayBufferAppender to extract the data + ArrayCapture capture = new ArrayCapture(); + array.appendToBufPtr(capture); + + ensureArrayCapacity(capture.nDims, capture.longDataOffset); + arrayDims[valueCount] = capture.nDims; + for (int i = 0; i < capture.nDims; i++) { + arrayShapes[arrayShapeOffset++] = capture.shape[i]; + } + for (int i = 0; i < capture.longDataOffset; i++) { + longArrayData[arrayDataOffset++] = capture.longData[i]; + } + valueCount++; + size++; + } + + /** + * Ensures capacity for array storage. + * @param nDims number of dimensions for this array + * @param dataElements number of data elements + */ + private void ensureArrayCapacity(int nDims, int dataElements) { + ensureCapacity(); // For row-level capacity (arrayDims uses valueCount) + + // Ensure shape array capacity + int requiredShapeCapacity = arrayShapeOffset + nDims; + if (arrayShapes == null) { + arrayShapes = new int[Math.max(64, requiredShapeCapacity)]; + } else if (requiredShapeCapacity > arrayShapes.length) { + arrayShapes = Arrays.copyOf(arrayShapes, Math.max(arrayShapes.length * 2, requiredShapeCapacity)); + } + + // Ensure data array capacity + int requiredDataCapacity = arrayDataOffset + dataElements; + if (type == TYPE_DOUBLE_ARRAY) { + if (doubleArrayData == null) { + doubleArrayData = new double[Math.max(256, requiredDataCapacity)]; + } else if (requiredDataCapacity > doubleArrayData.length) { + doubleArrayData = Arrays.copyOf(doubleArrayData, Math.max(doubleArrayData.length * 2, requiredDataCapacity)); + } + } else if (type == TYPE_LONG_ARRAY) { + if (longArrayData == null) { + longArrayData = new long[Math.max(256, requiredDataCapacity)]; + } else if (requiredDataCapacity > longArrayData.length) { + longArrayData = Arrays.copyOf(longArrayData, Math.max(longArrayData.length * 2, requiredDataCapacity)); + } + } + } + + public void addNull() { + ensureCapacity(); + if (nullable) { + // For nullable columns, mark null in bitmap but don't store a value + markNull(size); + size++; + } else { + // For non-nullable columns, we must store a sentinel/default value + // because no null bitmap will be written + switch (type) { + case TYPE_BOOLEAN: + booleanValues[valueCount++] = false; + break; + case TYPE_BYTE: + byteValues[valueCount++] = 0; + break; + case TYPE_SHORT: + case TYPE_CHAR: + shortValues[valueCount++] = 0; + break; + case TYPE_INT: + intValues[valueCount++] = 0; + break; + case TYPE_LONG: + case TYPE_TIMESTAMP: + case TYPE_TIMESTAMP_NANOS: + case TYPE_DATE: + longValues[valueCount++] = Long.MIN_VALUE; + break; + case TYPE_FLOAT: + floatValues[valueCount++] = Float.NaN; + break; + case TYPE_DOUBLE: + doubleValues[valueCount++] = Double.NaN; + break; + case TYPE_STRING: + case TYPE_VARCHAR: + stringValues[valueCount++] = null; + break; + case TYPE_SYMBOL: + symbolIndices[valueCount++] = -1; + break; + case TYPE_UUID: + uuidHigh[valueCount] = Long.MIN_VALUE; + uuidLow[valueCount] = Long.MIN_VALUE; + valueCount++; + break; + case TYPE_LONG256: + int offset = valueCount * 4; + long256Values[offset] = Long.MIN_VALUE; + long256Values[offset + 1] = Long.MIN_VALUE; + long256Values[offset + 2] = Long.MIN_VALUE; + long256Values[offset + 3] = Long.MIN_VALUE; + valueCount++; + break; + case TYPE_DECIMAL64: + decimal64Values[valueCount++] = Decimals.DECIMAL64_NULL; + break; + case TYPE_DECIMAL128: + decimal128High[valueCount] = Decimals.DECIMAL128_HI_NULL; + decimal128Low[valueCount] = Decimals.DECIMAL128_LO_NULL; + valueCount++; + break; + case TYPE_DECIMAL256: + decimal256Hh[valueCount] = Decimals.DECIMAL256_HH_NULL; + decimal256Hl[valueCount] = Decimals.DECIMAL256_HL_NULL; + decimal256Lh[valueCount] = Decimals.DECIMAL256_LH_NULL; + decimal256Ll[valueCount] = Decimals.DECIMAL256_LL_NULL; + valueCount++; + break; + } + size++; + } + } + + private void markNull(int index) { + int longIndex = index >>> 6; + int bitIndex = index & 63; + nullBitmapPacked[longIndex] |= (1L << bitIndex); + hasNulls = true; + } + + public void reset() { + size = 0; + valueCount = 0; + hasNulls = false; + if (nullBitmapPacked != null) { + Arrays.fill(nullBitmapPacked, 0L); + } + if (symbolDict != null) { + symbolDict.clear(); + symbolList.clear(); + } + // Reset global symbol tracking + maxGlobalSymbolId = -1; + // Reset array tracking + arrayShapeOffset = 0; + arrayDataOffset = 0; + // Reset decimal scale (will be set by first non-null value) + decimalScale = -1; + } + + /** + * Truncates the column to the specified size. + * This is used to cancel uncommitted row values. + * + * @param newSize the target size (number of rows) + */ + public void truncateTo(int newSize) { + if (newSize >= size) { + return; // Nothing to truncate + } + + // Count non-null values up to newSize + int newValueCount = 0; + if (nullable && nullBitmapPacked != null) { + for (int i = 0; i < newSize; i++) { + int longIndex = i >>> 6; + int bitIndex = i & 63; + if ((nullBitmapPacked[longIndex] & (1L << bitIndex)) == 0) { + newValueCount++; + } + } + // Clear null bits for truncated rows + for (int i = newSize; i < size; i++) { + int longIndex = i >>> 6; + int bitIndex = i & 63; + nullBitmapPacked[longIndex] &= ~(1L << bitIndex); + } + // Recompute hasNulls + hasNulls = false; + for (int i = 0; i < newSize && !hasNulls; i++) { + int longIndex = i >>> 6; + int bitIndex = i & 63; + if ((nullBitmapPacked[longIndex] & (1L << bitIndex)) != 0) { + hasNulls = true; + } + } + } else { + newValueCount = newSize; + } + + size = newSize; + valueCount = newValueCount; + } + + private void ensureCapacity() { + if (size >= capacity) { + int newCapacity = capacity * 2; + growStorage(type, newCapacity); + if (nullable && nullBitmapPacked != null) { + int newLongCount = (newCapacity + 63) >>> 6; + nullBitmapPacked = Arrays.copyOf(nullBitmapPacked, newLongCount); + } + capacity = newCapacity; + } + } + + private void allocateStorage(byte type, int cap) { + switch (type) { + case TYPE_BOOLEAN: + booleanValues = new boolean[cap]; + break; + case TYPE_BYTE: + byteValues = new byte[cap]; + break; + case TYPE_SHORT: + case TYPE_CHAR: + shortValues = new short[cap]; + break; + case TYPE_INT: + intValues = new int[cap]; + break; + case TYPE_LONG: + case TYPE_TIMESTAMP: + case TYPE_TIMESTAMP_NANOS: + case TYPE_DATE: + longValues = new long[cap]; + break; + case TYPE_FLOAT: + floatValues = new float[cap]; + break; + case TYPE_DOUBLE: + doubleValues = new double[cap]; + break; + case TYPE_STRING: + case TYPE_VARCHAR: + stringValues = new String[cap]; + break; + case TYPE_SYMBOL: + symbolIndices = new int[cap]; + symbolDict = new CharSequenceIntHashMap(); + symbolList = new ObjList<>(); + break; + case TYPE_UUID: + uuidHigh = new long[cap]; + uuidLow = new long[cap]; + break; + case TYPE_LONG256: + // Flat array: 4 longs per value + long256Values = new long[cap * 4]; + break; + case TYPE_DOUBLE_ARRAY: + case TYPE_LONG_ARRAY: + // Array types: allocate per-row tracking + // Shape and data arrays are grown dynamically in ensureArrayCapacity() + arrayDims = new byte[cap]; + arrayRowCapacity = cap; + break; + case TYPE_DECIMAL64: + decimal64Values = new long[cap]; + break; + case TYPE_DECIMAL128: + decimal128High = new long[cap]; + decimal128Low = new long[cap]; + break; + case TYPE_DECIMAL256: + decimal256Hh = new long[cap]; + decimal256Hl = new long[cap]; + decimal256Lh = new long[cap]; + decimal256Ll = new long[cap]; + break; + } + } + + private void growStorage(byte type, int newCap) { + switch (type) { + case TYPE_BOOLEAN: + booleanValues = Arrays.copyOf(booleanValues, newCap); + break; + case TYPE_BYTE: + byteValues = Arrays.copyOf(byteValues, newCap); + break; + case TYPE_SHORT: + case TYPE_CHAR: + shortValues = Arrays.copyOf(shortValues, newCap); + break; + case TYPE_INT: + intValues = Arrays.copyOf(intValues, newCap); + break; + case TYPE_LONG: + case TYPE_TIMESTAMP: + case TYPE_TIMESTAMP_NANOS: + case TYPE_DATE: + longValues = Arrays.copyOf(longValues, newCap); + break; + case TYPE_FLOAT: + floatValues = Arrays.copyOf(floatValues, newCap); + break; + case TYPE_DOUBLE: + doubleValues = Arrays.copyOf(doubleValues, newCap); + break; + case TYPE_STRING: + case TYPE_VARCHAR: + stringValues = Arrays.copyOf(stringValues, newCap); + break; + case TYPE_SYMBOL: + symbolIndices = Arrays.copyOf(symbolIndices, newCap); + if (globalSymbolIds != null) { + globalSymbolIds = Arrays.copyOf(globalSymbolIds, newCap); + } + break; + case TYPE_UUID: + uuidHigh = Arrays.copyOf(uuidHigh, newCap); + uuidLow = Arrays.copyOf(uuidLow, newCap); + break; + case TYPE_LONG256: + // Flat array: 4 longs per value + long256Values = Arrays.copyOf(long256Values, newCap * 4); + break; + case TYPE_DOUBLE_ARRAY: + case TYPE_LONG_ARRAY: + // Array types: grow per-row tracking + arrayDims = Arrays.copyOf(arrayDims, newCap); + arrayRowCapacity = newCap; + // Note: shapes and data arrays are grown in ensureArrayCapacity() + break; + case TYPE_DECIMAL64: + decimal64Values = Arrays.copyOf(decimal64Values, newCap); + break; + case TYPE_DECIMAL128: + decimal128High = Arrays.copyOf(decimal128High, newCap); + decimal128Low = Arrays.copyOf(decimal128Low, newCap); + break; + case TYPE_DECIMAL256: + decimal256Hh = Arrays.copyOf(decimal256Hh, newCap); + decimal256Hl = Arrays.copyOf(decimal256Hl, newCap); + decimal256Lh = Arrays.copyOf(decimal256Lh, newCap); + decimal256Ll = Arrays.copyOf(decimal256Ll, newCap); + break; + } + } + } + + /** + * Helper class to capture array data from DoubleArray/LongArray.appendToBufPtr(). + * This implements ArrayBufferAppender to intercept the serialization and extract + * shape and data into Java arrays for storage in ColumnBuffer. + */ + private static class ArrayCapture implements ArrayBufferAppender { + byte nDims; + int[] shape = new int[32]; // Max 32 dimensions + int shapeIndex; + double[] doubleData; + int doubleDataOffset; + long[] longData; + int longDataOffset; + + @Override + public void putByte(byte b) { + if (shapeIndex == 0) { + // First byte is nDims + nDims = b; + } + } + + @Override + public void putInt(int value) { + // Shape dimensions + if (shapeIndex < nDims) { + shape[shapeIndex++] = value; + // Once we have all dimensions, compute total elements and allocate data array + if (shapeIndex == nDims) { + int totalElements = 1; + for (int i = 0; i < nDims; i++) { + totalElements *= shape[i]; + } + // Allocate both - only one will be used + doubleData = new double[totalElements]; + longData = new long[totalElements]; + } + } + } + + @Override + public void putDouble(double value) { + if (doubleData != null && doubleDataOffset < doubleData.length) { + doubleData[doubleDataOffset++] = value; + } + } + + @Override + public void putLong(long value) { + if (longData != null && longDataOffset < longData.length) { + longData[longDataOffset++] = value; + } + } + + @Override + public void putBlockOfBytes(long from, long len) { + // This is the bulk data from the array + // The AbstractArray uses this to copy raw bytes + // We need to figure out if it's doubles or longs based on context + // For now, assume doubles (8 bytes each) since DoubleArray uses this + int count = (int) (len / 8); + if (doubleData == null) { + doubleData = new double[count]; + } + for (int i = 0; i < count; i++) { + doubleData[doubleDataOffset++] = Unsafe.getUnsafe().getDouble(from + i * 8L); + } + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4TimestampDecoder.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4TimestampDecoder.java new file mode 100644 index 0000000..3b4fffa --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4TimestampDecoder.java @@ -0,0 +1,474 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2024 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.ilpv4.protocol; + +import io.questdb.client.std.Unsafe; + +/** + * Decoder for TIMESTAMP columns in ILP v4 format. + *

+ * Supports two encoding modes: + *

    + *
  • Uncompressed (0x00): array of int64 values
    • + *
  • Gorilla (0x01): delta-of-delta compressed
    • + *
+ *

+ * Gorilla format: + * 

+ * [Null bitmap if nullable]
+ * First timestamp: int64 (8 bytes, little-endian)
+ * Second timestamp: int64 (8 bytes, little-endian)
+ * Remaining timestamps: bit-packed delta-of-delta
+ *
+ */ +public final class IlpV4TimestampDecoder { + + /** + * Encoding flag for uncompressed timestamps. + */ + public static final byte ENCODING_UNCOMPRESSED = 0x00; + + /** + * Encoding flag for Gorilla-encoded timestamps. + */ + public static final byte ENCODING_GORILLA = 0x01; + + public static final IlpV4TimestampDecoder INSTANCE = new IlpV4TimestampDecoder(); + + private final IlpV4GorillaDecoder gorillaDecoder = new IlpV4GorillaDecoder(); + + private IlpV4TimestampDecoder() { + } + + /** + * Decodes timestamp column data from native memory. + * + * @param sourceAddress source address in native memory + * @param sourceLength length of source data in bytes + * @param rowCount number of rows to decode + * @param nullable whether the column is nullable + * @param sink sink to receive decoded values + * @return number of bytes consumed + */ + public int decode(long sourceAddress, int sourceLength, int rowCount, boolean nullable, ColumnSink sink) { + if (rowCount == 0) { + return 0; + } + + int offset = 0; + + // Parse null bitmap if nullable + long nullBitmapAddress = 0; + if (nullable) { + int nullBitmapSize = IlpV4NullBitmap.sizeInBytes(rowCount); + if (offset + nullBitmapSize > sourceLength) { + throw new IllegalArgumentException("insufficient data for null bitmap"); + } + nullBitmapAddress = sourceAddress + offset; + offset += nullBitmapSize; + } + + // Read encoding flag + if (offset + 1 > sourceLength) { + throw new IllegalArgumentException("insufficient data for encoding flag"); + } + byte encoding = Unsafe.getUnsafe().getByte(sourceAddress + offset); + offset++; + + if (encoding == ENCODING_UNCOMPRESSED) { + offset = decodeUncompressed(sourceAddress, sourceLength, offset, rowCount, nullable, nullBitmapAddress, sink); + } else if (encoding == ENCODING_GORILLA) { + offset = decodeGorilla(sourceAddress, sourceLength, offset, rowCount, nullable, nullBitmapAddress, sink); + } else { + throw new IllegalArgumentException("unknown timestamp encoding: " + encoding); + } + + return offset; + } + + private int decodeUncompressed(long sourceAddress, int sourceLength, int offset, int rowCount, + boolean nullable, long nullBitmapAddress, ColumnSink sink) { + // Count nulls to determine actual value count + int nullCount = 0; + if (nullable) { + nullCount = IlpV4NullBitmap.countNulls(nullBitmapAddress, rowCount); + } + int valueCount = rowCount - nullCount; + + // Uncompressed: valueCount * 8 bytes + int valuesSize = valueCount * 8; + if (offset + valuesSize > sourceLength) { + throw new IllegalArgumentException("insufficient data for uncompressed timestamps"); + } + + long valuesAddress = sourceAddress + offset; + int valueOffset = 0; + for (int i = 0; i < rowCount; i++) { + if (nullable && IlpV4NullBitmap.isNull(nullBitmapAddress, i)) { + sink.putNull(i); + } else { + long value = Unsafe.getUnsafe().getLong(valuesAddress + (long) valueOffset * 8); + sink.putLong(i, value); + valueOffset++; + } + } + + return offset + valuesSize; + } + + private int decodeGorilla(long sourceAddress, int sourceLength, int offset, int rowCount, + boolean nullable, long nullBitmapAddress, ColumnSink sink) { + // Count nulls to determine actual value count + int nullCount = 0; + if (nullable) { + nullCount = IlpV4NullBitmap.countNulls(nullBitmapAddress, rowCount); + } + int valueCount = rowCount - nullCount; + + if (valueCount == 0) { + // All nulls + for (int i = 0; i < rowCount; i++) { + sink.putNull(i); + } + return offset; + } + + // First timestamp: 8 bytes + if (offset + 8 > sourceLength) { + throw new IllegalArgumentException("insufficient data for first timestamp"); + } + long firstTimestamp = Unsafe.getUnsafe().getLong(sourceAddress + offset); + offset += 8; + + if (valueCount == 1) { + // Only one non-null value, output it at the appropriate row position + for (int i = 0; i < rowCount; i++) { + if (nullable && IlpV4NullBitmap.isNull(nullBitmapAddress, i)) { + sink.putNull(i); + } else { + sink.putLong(i, firstTimestamp); + } + } + return offset; + } + + // Second timestamp: 8 bytes + if (offset + 8 > sourceLength) { + throw new IllegalArgumentException("insufficient data for second timestamp"); + } + long secondTimestamp = Unsafe.getUnsafe().getLong(sourceAddress + offset); + offset += 8; + + if (valueCount == 2) { + // Two non-null values + int valueIdx = 0; + for (int i = 0; i < rowCount; i++) { + if (nullable && IlpV4NullBitmap.isNull(nullBitmapAddress, i)) { + sink.putNull(i); + } else { + sink.putLong(i, valueIdx == 0 ? firstTimestamp : secondTimestamp); + valueIdx++; + } + } + return offset; + } + + // Remaining timestamps: bit-packed delta-of-delta + // Reset the Gorilla decoder with the initial state + gorillaDecoder.reset(firstTimestamp, secondTimestamp); + + // Calculate remaining bytes for bit data + int remainingBytes = sourceLength - offset; + gorillaDecoder.resetReader(sourceAddress + offset, remainingBytes); + + // Decode timestamps and distribute to rows + int valueIdx = 0; + for (int i = 0; i < rowCount; i++) { + if (nullable && IlpV4NullBitmap.isNull(nullBitmapAddress, i)) { + sink.putNull(i); + } else { + long timestamp; + if (valueIdx == 0) { + timestamp = firstTimestamp; + } else if (valueIdx == 1) { + timestamp = secondTimestamp; + } else { + timestamp = gorillaDecoder.decodeNext(); + } + sink.putLong(i, timestamp); + valueIdx++; + } + } + + // Calculate how many bytes were consumed + // The bit reader has consumed some bits; round up to bytes + long bitsRead = gorillaDecoder.getAvailableBits(); + long totalBits = remainingBytes * 8L; + long bitsConsumed = totalBits - bitsRead; + int bytesConsumed = (int) ((bitsConsumed + 7) / 8); + + return offset + bytesConsumed; + } + + /** + * Returns the expected size for decoding. + * + * @param rowCount number of rows + * @param nullable whether the column is nullable + * @return expected size in bytes + */ + public int expectedSize(int rowCount, boolean nullable) { + // Minimum size: just encoding flag + uncompressed timestamps + int size = 1; // encoding flag + if (nullable) { + size += IlpV4NullBitmap.sizeInBytes(rowCount); + } + size += rowCount * 8; // worst case: uncompressed + return size; + } + + // ==================== Static Encoding Methods (for testing) ==================== + + /** + * Encodes timestamps in uncompressed format to direct memory. + * Only non-null values are written. + * + * @param destAddress destination address + * @param timestamps timestamp values + * @param nulls null flags (can be null if not nullable) + * @return address after encoded data + */ + public static long encodeUncompressed(long destAddress, long[] timestamps, boolean[] nulls) { + int rowCount = timestamps.length; + boolean nullable = nulls != null; + long pos = destAddress; + + // Write null bitmap if nullable + if (nullable) { + int bitmapSize = IlpV4NullBitmap.sizeInBytes(rowCount); + IlpV4NullBitmap.fillNoneNull(pos, rowCount); + for (int i = 0; i < rowCount; i++) { + if (nulls[i]) { + IlpV4NullBitmap.setNull(pos, i); + } + } + pos += bitmapSize; + } + + // Write encoding flag + Unsafe.getUnsafe().putByte(pos++, ENCODING_UNCOMPRESSED); + + // Write only non-null timestamps + for (int i = 0; i < rowCount; i++) { + if (nullable && nulls[i]) continue; + Unsafe.getUnsafe().putLong(pos, timestamps[i]); + pos += 8; + } + + return pos; + } + + /** + * Encodes timestamps in Gorilla format to direct memory. + * Only non-null values are encoded. + * + * @param destAddress destination address + * @param timestamps timestamp values + * @param nulls null flags (can be null if not nullable) + * @return address after encoded data + */ + public static long encodeGorilla(long destAddress, long[] timestamps, boolean[] nulls) { + int rowCount = timestamps.length; + boolean nullable = nulls != null; + long pos = destAddress; + + // Write null bitmap if nullable + if (nullable) { + int bitmapSize = IlpV4NullBitmap.sizeInBytes(rowCount); + IlpV4NullBitmap.fillNoneNull(pos, rowCount); + for (int i = 0; i < rowCount; i++) { + if (nulls[i]) { + IlpV4NullBitmap.setNull(pos, i); + } + } + pos += bitmapSize; + } + + // Count non-null values + int valueCount = 0; + for (int i = 0; i < rowCount; i++) { + if (!nullable || !nulls[i]) valueCount++; + } + + // Write encoding flag + Unsafe.getUnsafe().putByte(pos++, ENCODING_GORILLA); + + if (valueCount == 0) { + return pos; + } + + // Build array of non-null values + long[] nonNullValues = new long[valueCount]; + int idx = 0; + for (int i = 0; i < rowCount; i++) { + if (nullable && nulls[i]) continue; + nonNullValues[idx++] = timestamps[i]; + } + + // Write first timestamp + Unsafe.getUnsafe().putLong(pos, nonNullValues[0]); + pos += 8; + + if (valueCount == 1) { + return pos; + } + + // Write second timestamp + Unsafe.getUnsafe().putLong(pos, nonNullValues[1]); + pos += 8; + + if (valueCount == 2) { + return pos; + } + + // Encode remaining timestamps using Gorilla + IlpV4BitWriter bitWriter = new IlpV4BitWriter(); + bitWriter.reset(pos, 1024 * 1024); // 1MB max for bit data + + long prevTimestamp = nonNullValues[1]; + long prevDelta = nonNullValues[1] - nonNullValues[0]; + + for (int i = 2; i < valueCount; i++) { + long delta = nonNullValues[i] - prevTimestamp; + long deltaOfDelta = delta - prevDelta; + + encodeDoD(bitWriter, deltaOfDelta); + + prevDelta = delta; + prevTimestamp = nonNullValues[i]; + } + + // Flush remaining bits + int bytesWritten = bitWriter.finish(); + pos += bytesWritten; + + return pos; + } + + /** + * Encodes a delta-of-delta value to the bit writer. + *

+ * Prefix patterns are written LSB-first to match the decoder's read order: + * - '0' -> write bit 0 + * - '10' -> write bit 1, then bit 0 (0b01 as 2-bit value) + * - '110' -> write bit 1, bit 1, bit 0 (0b011 as 3-bit value) + * - '1110' -> write bit 1, bit 1, bit 1, bit 0 (0b0111 as 4-bit value) + * - '1111' -> write bit 1, bit 1, bit 1, bit 1 (0b1111 as 4-bit value) + */ + private static void encodeDoD(IlpV4BitWriter writer, long deltaOfDelta) { + if (deltaOfDelta == 0) { + // '0' = DoD is 0 + writer.writeBit(0); + } else if (deltaOfDelta >= -63 && deltaOfDelta <= 64) { + // '10' prefix: first bit read=1, second bit read=0 -> write as 0b01 (LSB-first) + writer.writeBits(0b01, 2); + writer.writeSigned(deltaOfDelta, 7); + } else if (deltaOfDelta >= -255 && deltaOfDelta <= 256) { + // '110' prefix: bits read as 1,1,0 -> write as 0b011 (LSB-first) + writer.writeBits(0b011, 3); + writer.writeSigned(deltaOfDelta, 9); + } else if (deltaOfDelta >= -2047 && deltaOfDelta <= 2048) { + // '1110' prefix: bits read as 1,1,1,0 -> write as 0b0111 (LSB-first) + writer.writeBits(0b0111, 4); + writer.writeSigned(deltaOfDelta, 12); + } else { + // '1111' prefix: bits read as 1,1,1,1 -> write as 0b1111 (LSB-first) + writer.writeBits(0b1111, 4); + writer.writeSigned(deltaOfDelta, 32); + } + } + + /** + * Calculates the encoded size in bytes for Gorilla-encoded timestamps. + * + * @param timestamps timestamp values + * @param nullable whether column is nullable + * @return encoded size in bytes + */ + public static int calculateGorillaSize(long[] timestamps, boolean nullable) { + int rowCount = timestamps.length; + int size = 0; + + if (nullable) { + size += IlpV4NullBitmap.sizeInBytes(rowCount); + } + + size += 1; // encoding flag + + if (rowCount == 0) { + return size; + } + + size += 8; // first timestamp + + if (rowCount == 1) { + return size; + } + + size += 8; // second timestamp + + if (rowCount == 2) { + return size; + } + + // Calculate bits for delta-of-delta encoding + long prevTimestamp = timestamps[1]; + long prevDelta = timestamps[1] - timestamps[0]; + int totalBits = 0; + + for (int i = 2; i < rowCount; i++) { + long delta = timestamps[i] - prevTimestamp; + long deltaOfDelta = delta - prevDelta; + + totalBits += IlpV4GorillaDecoder.getBitsRequired(deltaOfDelta); + + prevDelta = delta; + prevTimestamp = timestamps[i]; + } + + // Round up to bytes + size += (totalBits + 7) / 8; + + return size; + } + + /** + * Sink interface for receiving decoded column values. + */ + public interface ColumnSink { + void putLong(int rowIndex, long value); + void putNull(int rowIndex); + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4Varint.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4Varint.java new file mode 100644 index 0000000..cd150d4 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4Varint.java @@ -0,0 +1,261 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2024 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.ilpv4.protocol; + +import io.questdb.client.std.Unsafe; + +/** + * Variable-length integer encoding/decoding utilities for ILP v4 protocol. + * Uses unsigned LEB128 (Little Endian Base 128) encoding. + *

+ * The encoding scheme: + * - Values are split into 7-bit groups + * - Each byte uses the high bit (0x80) as a continuation flag + * - If high bit is set, more bytes follow + * - If high bit is clear, this is the last byte + *

+ * This implementation is designed for zero-allocation on hot paths. + */ +public final class IlpV4Varint { + + /** + * Maximum number of bytes needed to encode a 64-bit varint. + * ceil(64/7) = 10 bytes + */ + public static final int MAX_VARINT_BYTES = 10; + + /** + * Continuation bit mask - set in all bytes except the last. + */ + private static final int CONTINUATION_BIT = 0x80; + + /** + * Data mask - lower 7 bits of each byte. + */ + private static final int DATA_MASK = 0x7F; + + private IlpV4Varint() { + // utility class + } + + /** + * Calculates the number of bytes needed to encode the given value. + * + * @param value the value to measure (treated as unsigned) + * @return number of bytes needed (1-10) + */ + public static int encodedLength(long value) { + if (value == 0) { + return 1; + } + // Count leading zeros to determine the number of bits needed + int bits = 64 - Long.numberOfLeadingZeros(value); + // Each byte encodes 7 bits, round up + return (bits + 6) / 7; + } + + /** + * Encodes a long value as a varint into the given byte array. + * + * @param buf the buffer to write to + * @param pos the position to start writing + * @param value the value to encode (treated as unsigned) + * @return the new position after the encoded bytes + */ + public static int encode(byte[] buf, int pos, long value) { + while ((value & ~DATA_MASK) != 0) { + buf[pos++] = (byte) ((value & DATA_MASK) | CONTINUATION_BIT); + value >>>= 7; + } + buf[pos++] = (byte) value; + return pos; + } + + /** + * Encodes a long value as a varint to direct memory. + * + * @param address the memory address to write to + * @param value the value to encode (treated as unsigned) + * @return the new address after the encoded bytes + */ + public static long encode(long address, long value) { + while ((value & ~DATA_MASK) != 0) { + Unsafe.getUnsafe().putByte(address++, (byte) ((value & DATA_MASK) | CONTINUATION_BIT)); + value >>>= 7; + } + Unsafe.getUnsafe().putByte(address++, (byte) value); + return address; + } + + /** + * Decodes a varint from the given byte array. + * + * @param buf the buffer to read from + * @param pos the position to start reading + * @return the decoded value + * @throws IllegalArgumentException if the varint is malformed (too many bytes) + */ + public static long decode(byte[] buf, int pos) { + return decode(buf, pos, buf.length); + } + + /** + * Decodes a varint from the given byte array with bounds checking. + * + * @param buf the buffer to read from + * @param pos the position to start reading + * @param limit the maximum position to read (exclusive) + * @return the decoded value + * @throws IllegalArgumentException if the varint is malformed or buffer underflows + */ + public static long decode(byte[] buf, int pos, int limit) { + long result = 0; + int shift = 0; + int bytesRead = 0; + byte b; + + do { + if (pos >= limit) { + throw new IllegalArgumentException("incomplete varint"); + } + if (bytesRead >= MAX_VARINT_BYTES) { + throw new IllegalArgumentException("varint overflow"); + } + b = buf[pos++]; + result |= (long) (b & DATA_MASK) << shift; + shift += 7; + bytesRead++; + } while ((b & CONTINUATION_BIT) != 0); + + return result; + } + + /** + * Decodes a varint from direct memory. + * + * @param address the memory address to read from + * @param limit the maximum address to read (exclusive) + * @return the decoded value + * @throws IllegalArgumentException if the varint is malformed or buffer underflows + */ + public static long decode(long address, long limit) { + long result = 0; + int shift = 0; + int bytesRead = 0; + byte b; + + do { + if (address >= limit) { + throw new IllegalArgumentException("incomplete varint"); + } + if (bytesRead >= MAX_VARINT_BYTES) { + throw new IllegalArgumentException("varint overflow"); + } + b = Unsafe.getUnsafe().getByte(address++); + result |= (long) (b & DATA_MASK) << shift; + shift += 7; + bytesRead++; + } while ((b & CONTINUATION_BIT) != 0); + + return result; + } + + /** + * Result holder for decoding varints when the number of bytes consumed matters. + * This class is mutable and should be reused to avoid allocations. + */ + public static class DecodeResult { + public long value; + public int bytesRead; + + public void reset() { + value = 0; + bytesRead = 0; + } + } + + /** + * Decodes a varint from a byte array and stores both value and bytes consumed. + * + * @param buf the buffer to read from + * @param pos the position to start reading + * @param limit the maximum position to read (exclusive) + * @param result the result holder (must not be null) + * @throws IllegalArgumentException if the varint is malformed or buffer underflows + */ + public static void decode(byte[] buf, int pos, int limit, DecodeResult result) { + long value = 0; + int shift = 0; + int bytesRead = 0; + byte b; + + do { + if (pos >= limit) { + throw new IllegalArgumentException("incomplete varint"); + } + if (bytesRead >= MAX_VARINT_BYTES) { + throw new IllegalArgumentException("varint overflow"); + } + b = buf[pos++]; + value |= (long) (b & DATA_MASK) << shift; + shift += 7; + bytesRead++; + } while ((b & CONTINUATION_BIT) != 0); + + result.value = value; + result.bytesRead = bytesRead; + } + + /** + * Decodes a varint from direct memory and stores both value and bytes consumed. + * + * @param address the memory address to read from + * @param limit the maximum address to read (exclusive) + * @param result the result holder (must not be null) + * @throws IllegalArgumentException if the varint is malformed or buffer underflows + */ + public static void decode(long address, long limit, DecodeResult result) { + long value = 0; + int shift = 0; + int bytesRead = 0; + byte b; + + do { + if (address >= limit) { + throw new IllegalArgumentException("incomplete varint"); + } + if (bytesRead >= MAX_VARINT_BYTES) { + throw new IllegalArgumentException("varint overflow"); + } + b = Unsafe.getUnsafe().getByte(address++); + value |= (long) (b & DATA_MASK) << shift; + shift += 7; + bytesRead++; + } while ((b & CONTINUATION_BIT) != 0); + + result.value = value; + result.bytesRead = bytesRead; + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4ZigZag.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4ZigZag.java new file mode 100644 index 0000000..b0542e2 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4ZigZag.java @@ -0,0 +1,98 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2024 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.ilpv4.protocol; + +/** + * ZigZag encoding/decoding for signed integers. + *

+ * ZigZag encoding maps signed integers to unsigned integers so that + * numbers with small absolute value have small encoded values. + *

+ * The encoding works as follows: + * 

+ *  0 ->  0
+ * -1 ->  1
+ *  1 ->  2
+ * -2 ->  3
+ *  2 ->  4
+ * ...
+ *
+ *

+ * Formula: + * 

+ * encode(n) = (n << 1) ^ (n >> 63)  // for 64-bit
+ * decode(n) = (n >>> 1) ^ -(n & 1)
+ *
+ *

+ * This is useful when combined with varint encoding because small + * negative numbers like -1 become small positive numbers (1), which + * encode efficiently as varints. + */ +public final class IlpV4ZigZag { + + private IlpV4ZigZag() { + // utility class + } + + /** + * Encodes a signed 64-bit integer using ZigZag encoding. + * + * @param value the signed value to encode + * @return the ZigZag encoded value (unsigned interpretation) + */ + public static long encode(long value) { + return (value << 1) ^ (value >> 63); + } + + /** + * Decodes a ZigZag encoded 64-bit integer. + * + * @param value the ZigZag encoded value + * @return the original signed value + */ + public static long decode(long value) { + return (value >>> 1) ^ -(value & 1); + } + + /** + * Encodes a signed 32-bit integer using ZigZag encoding. + * + * @param value the signed value to encode + * @return the ZigZag encoded value (unsigned interpretation) + */ + public static int encode(int value) { + return (value << 1) ^ (value >> 31); + } + + /** + * Decodes a ZigZag encoded 32-bit integer. + * + * @param value the ZigZag encoded value + * @return the original signed value + */ + public static int decode(int value) { + return (value >>> 1) ^ -(value & 1); + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketCloseCode.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketCloseCode.java new file mode 100644 index 0000000..2cb001c --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketCloseCode.java @@ -0,0 +1,178 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.ilpv4.websocket; + +/** + * WebSocket close status codes as defined in RFC 6455. + */ +public final class WebSocketCloseCode { + /** + * Normal closure (1000). + * The connection successfully completed whatever purpose for which it was created. + */ + public static final int NORMAL_CLOSURE = 1000; + + /** + * Going away (1001). + * The endpoint is going away, e.g., server shutting down or browser navigating away. + */ + public static final int GOING_AWAY = 1001; + + /** + * Protocol error (1002). + * The endpoint is terminating the connection due to a protocol error. + */ + public static final int PROTOCOL_ERROR = 1002; + + /** + * Unsupported data (1003). + * The endpoint received a type of data it cannot accept. + */ + public static final int UNSUPPORTED_DATA = 1003; + + /** + * Reserved (1004). + * Reserved for future use. + */ + public static final int RESERVED = 1004; + + /** + * No status received (1005). + * Reserved value. MUST NOT be sent in a Close frame. + */ + public static final int NO_STATUS_RECEIVED = 1005; + + /** + * Abnormal closure (1006). + * Reserved value. MUST NOT be sent in a Close frame. + * Used to indicate that a connection was closed abnormally. + */ + public static final int ABNORMAL_CLOSURE = 1006; + + /** + * Invalid frame payload data (1007). + * The endpoint received a message with invalid payload data. + */ + public static final int INVALID_PAYLOAD_DATA = 1007; + + /** + * Policy violation (1008). + * The endpoint received a message that violates its policy. + */ + public static final int POLICY_VIOLATION = 1008; + + /** + * Message too big (1009). + * The endpoint received a message that is too big to process. + */ + public static final int MESSAGE_TOO_BIG = 1009; + + /** + * Mandatory extension (1010). + * The client expected the server to negotiate one or more extensions. + */ + public static final int MANDATORY_EXTENSION = 1010; + + /** + * Internal server error (1011). + * The server encountered an unexpected condition that prevented it from fulfilling the request. + */ + public static final int INTERNAL_ERROR = 1011; + + /** + * TLS handshake (1015). + * Reserved value. MUST NOT be sent in a Close frame. + * Used to indicate that the connection was closed due to TLS handshake failure. + */ + public static final int TLS_HANDSHAKE = 1015; + + private WebSocketCloseCode() { + // Constants class + } + + /** + * Checks if a close code is valid for use in a Close frame. + * Codes 1005 and 1006 are reserved and must not be sent. + * + * @param code the close code + * @return true if the code can be sent in a Close frame + */ + public static boolean isValidForSending(int code) { + if (code < 1000) { + return false; + } + if (code == NO_STATUS_RECEIVED || code == ABNORMAL_CLOSURE || code == TLS_HANDSHAKE) { + return false; + } + // 1000-2999 are defined by RFC 6455 + // 3000-3999 are reserved for libraries/frameworks + // 4000-4999 are reserved for applications + return code < 5000; + } + + /** + * Returns a human-readable description of the close code. + * + * @param code the close code + * @return the description + */ + public static String describe(int code) { + switch (code) { + case NORMAL_CLOSURE: + return "Normal Closure"; + case GOING_AWAY: + return "Going Away"; + case PROTOCOL_ERROR: + return "Protocol Error"; + case UNSUPPORTED_DATA: + return "Unsupported Data"; + case RESERVED: + return "Reserved"; + case NO_STATUS_RECEIVED: + return "No Status Received"; + case ABNORMAL_CLOSURE: + return "Abnormal Closure"; + case INVALID_PAYLOAD_DATA: + return "Invalid Payload Data"; + case POLICY_VIOLATION: + return "Policy Violation"; + case MESSAGE_TOO_BIG: + return "Message Too Big"; + case MANDATORY_EXTENSION: + return "Mandatory Extension"; + case INTERNAL_ERROR: + return "Internal Error"; + case TLS_HANDSHAKE: + return "TLS Handshake"; + default: + if (code >= 3000 && code < 4000) { + return "Library/Framework Code (" + code + ")"; + } else if (code >= 4000 && code < 5000) { + return "Application Code (" + code + ")"; + } + return "Unknown (" + code + ")"; + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketFrameParser.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketFrameParser.java new file mode 100644 index 0000000..53d02b3 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketFrameParser.java @@ -0,0 +1,342 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.ilpv4.websocket; + +import io.questdb.client.std.Unsafe; + +/** + * Zero-allocation WebSocket frame parser. + * Parses WebSocket frames according to RFC 6455. + * + *

The parser operates on raw memory buffers and maintains minimal state. + * It can parse frames incrementally when data arrives in chunks. + * + *

Thread safety: This class is NOT thread-safe. Each connection should + * have its own parser instance. + */ +public class WebSocketFrameParser { + /** + * Initial state, waiting for frame header. + */ + public static final int STATE_HEADER = 0; + + /** + * Need more data to complete parsing. + */ + public static final int STATE_NEED_MORE = 1; + + /** + * Header parsed, need payload data. + */ + public static final int STATE_NEED_PAYLOAD = 2; + + /** + * Frame completely parsed. + */ + public static final int STATE_COMPLETE = 3; + + /** + * Error state - frame is invalid. + */ + public static final int STATE_ERROR = 4; + + // Frame header bits + private static final int FIN_BIT = 0x80; + private static final int RSV_BITS = 0x70; + private static final int OPCODE_MASK = 0x0F; + private static final int MASK_BIT = 0x80; + private static final int LENGTH_MASK = 0x7F; + + // Control frame max payload size (RFC 6455) + private static final int MAX_CONTROL_FRAME_PAYLOAD = 125; + + // Parsed frame data + private boolean fin; + private int opcode; + private boolean masked; + private int maskKey; + private long payloadLength; + private int headerSize; + + // Parser state + private int state = STATE_HEADER; + private int errorCode; + + // Configuration + private boolean serverMode = false; // If true, expect masked frames from clients + private boolean strictMode = false; // If true, reject non-minimal length encodings + + /** + * Parses a WebSocket frame from the given buffer. + * + * @param buf the start of the buffer + * @param limit the end of the buffer (exclusive) + * @return the number of bytes consumed, or 0 if more data is needed + */ + public int parse(long buf, long limit) { + long available = limit - buf; + + if (available < 2) { + state = STATE_NEED_MORE; + return 0; + } + + // Parse first two bytes + int byte0 = Unsafe.getUnsafe().getByte(buf) & 0xFF; + int byte1 = Unsafe.getUnsafe().getByte(buf + 1) & 0xFF; + + // Check reserved bits (must be 0 unless extension negotiated) + if ((byte0 & RSV_BITS) != 0) { + state = STATE_ERROR; + errorCode = WebSocketCloseCode.PROTOCOL_ERROR; + return 0; + } + + fin = (byte0 & FIN_BIT) != 0; + opcode = byte0 & OPCODE_MASK; + + // Validate opcode + if (!WebSocketOpcode.isValid(opcode)) { + state = STATE_ERROR; + errorCode = WebSocketCloseCode.PROTOCOL_ERROR; + return 0; + } + + // Control frames must not be fragmented + if (WebSocketOpcode.isControlFrame(opcode) && !fin) { + state = STATE_ERROR; + errorCode = WebSocketCloseCode.PROTOCOL_ERROR; + return 0; + } + + masked = (byte1 & MASK_BIT) != 0; + int lengthField = byte1 & LENGTH_MASK; + + // Validate masking based on mode + if (serverMode && !masked) { + // Client frames MUST be masked + state = STATE_ERROR; + errorCode = WebSocketCloseCode.PROTOCOL_ERROR; + return 0; + } + if (!serverMode && masked) { + // Server frames MUST NOT be masked + state = STATE_ERROR; + errorCode = WebSocketCloseCode.PROTOCOL_ERROR; + return 0; + } + + // Calculate header size and payload length + int offset = 2; + + if (lengthField <= 125) { + payloadLength = lengthField; + } else if (lengthField == 126) { + // 16-bit extended length + if (available < 4) { + state = STATE_NEED_MORE; + return 0; + } + int high = Unsafe.getUnsafe().getByte(buf + 2) & 0xFF; + int low = Unsafe.getUnsafe().getByte(buf + 3) & 0xFF; + payloadLength = (high << 8) | low; + + // Strict mode: reject non-minimal encodings + if (strictMode && payloadLength < 126) { + state = STATE_ERROR; + errorCode = WebSocketCloseCode.PROTOCOL_ERROR; + return 0; + } + + offset = 4; + } else { + // 64-bit extended length + if (available < 10) { + state = STATE_NEED_MORE; + return 0; + } + payloadLength = Long.reverseBytes(Unsafe.getUnsafe().getLong(buf + 2)); + + // Strict mode: reject non-minimal encodings + if (strictMode && payloadLength <= 65535) { + state = STATE_ERROR; + errorCode = WebSocketCloseCode.PROTOCOL_ERROR; + return 0; + } + + // MSB must be 0 (no negative lengths) + if (payloadLength < 0) { + state = STATE_ERROR; + errorCode = WebSocketCloseCode.PROTOCOL_ERROR; + return 0; + } + + offset = 10; + } + + // Control frames must not have payload > 125 bytes + if (WebSocketOpcode.isControlFrame(opcode) && payloadLength > MAX_CONTROL_FRAME_PAYLOAD) { + state = STATE_ERROR; + errorCode = WebSocketCloseCode.PROTOCOL_ERROR; + return 0; + } + + // Close frame with 1 byte payload is invalid (must be 0 or >= 2) + if (opcode == WebSocketOpcode.CLOSE && payloadLength == 1) { + state = STATE_ERROR; + errorCode = WebSocketCloseCode.PROTOCOL_ERROR; + return 0; + } + + // Parse mask key if present + if (masked) { + if (available < offset + 4) { + state = STATE_NEED_MORE; + return 0; + } + maskKey = Unsafe.getUnsafe().getInt(buf + offset); + offset += 4; + } else { + maskKey = 0; + } + + headerSize = offset; + + // Check if we have the complete payload + long totalFrameSize = headerSize + payloadLength; + if (available < totalFrameSize) { + state = STATE_NEED_PAYLOAD; + return headerSize; + } + + state = STATE_COMPLETE; + return (int) totalFrameSize; + } + + /** + * Unmasks the payload data in place. + * + * @param buf the start of the payload data + * @param len the length of the payload + */ + public void unmaskPayload(long buf, long len) { + if (!masked || maskKey == 0) { + return; + } + + // Process 8 bytes at a time when possible for better performance + long i = 0; + long longMask = ((long) maskKey << 32) | (maskKey & 0xFFFFFFFFL); + + // Process 8-byte chunks + while (i + 8 <= len) { + long value = Unsafe.getUnsafe().getLong(buf + i); + Unsafe.getUnsafe().putLong(buf + i, value ^ longMask); + i += 8; + } + + // Process 4-byte chunk if remaining + if (i + 4 <= len) { + int value = Unsafe.getUnsafe().getInt(buf + i); + Unsafe.getUnsafe().putInt(buf + i, value ^ maskKey); + i += 4; + } + + // Process remaining bytes + while (i < len) { + byte b = Unsafe.getUnsafe().getByte(buf + i); + int shift = ((int) (i % 4)) << 3; // 0, 8, 16, or 24 + byte maskByte = (byte) ((maskKey >> shift) & 0xFF); + Unsafe.getUnsafe().putByte(buf + i, (byte) (b ^ maskByte)); + i++; + } + } + + /** + * Resets the parser state for parsing a new frame. + */ + public void reset() { + state = STATE_HEADER; + fin = false; + opcode = 0; + masked = false; + maskKey = 0; + payloadLength = 0; + headerSize = 0; + errorCode = 0; + } + + // Getters + + public boolean isFin() { + return fin; + } + + public int getOpcode() { + return opcode; + } + + public boolean isMasked() { + return masked; + } + + public int getMaskKey() { + return maskKey; + } + + public long getPayloadLength() { + return payloadLength; + } + + public int getHeaderSize() { + return headerSize; + } + + public int getState() { + return state; + } + + public int getErrorCode() { + return errorCode; + } + + // Setters for configuration + + public void setServerMode(boolean serverMode) { + this.serverMode = serverMode; + } + + public void setStrictMode(boolean strictMode) { + this.strictMode = strictMode; + } + + /** + * Sets the mask key for unmasking. Used in testing. + */ + public void setMaskKey(int maskKey) { + this.maskKey = maskKey; + this.masked = true; + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketFrameWriter.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketFrameWriter.java new file mode 100644 index 0000000..d94bcd5 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketFrameWriter.java @@ -0,0 +1,281 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.ilpv4.websocket; + +import io.questdb.client.std.Unsafe; + +import java.nio.charset.StandardCharsets; + +/** + * Zero-allocation WebSocket frame writer. + * Writes WebSocket frames according to RFC 6455. + * + *

All methods are static utilities that write directly to memory buffers. + * + *

Thread safety: This class is thread-safe as it contains no mutable state. + */ +public final class WebSocketFrameWriter { + // Frame header bits + private static final int FIN_BIT = 0x80; + private static final int MASK_BIT = 0x80; + + private WebSocketFrameWriter() { + // Static utility class + } + + /** + * Writes a WebSocket frame header to the buffer. + * + * @param buf the buffer to write to + * @param fin true if this is the final frame + * @param opcode the frame opcode + * @param payloadLength the payload length + * @param masked true if the payload should be masked + * @return the number of bytes written (header size) + */ + public static int writeHeader(long buf, boolean fin, int opcode, long payloadLength, boolean masked) { + int offset = 0; + + // First byte: FIN + opcode + int byte0 = (fin ? FIN_BIT : 0) | (opcode & 0x0F); + Unsafe.getUnsafe().putByte(buf + offset++, (byte) byte0); + + // Second byte: MASK + payload length + int maskBit = masked ? MASK_BIT : 0; + + if (payloadLength <= 125) { + Unsafe.getUnsafe().putByte(buf + offset++, (byte) (maskBit | payloadLength)); + } else if (payloadLength <= 65535) { + Unsafe.getUnsafe().putByte(buf + offset++, (byte) (maskBit | 126)); + Unsafe.getUnsafe().putByte(buf + offset++, (byte) ((payloadLength >> 8) & 0xFF)); + Unsafe.getUnsafe().putByte(buf + offset++, (byte) (payloadLength & 0xFF)); + } else { + Unsafe.getUnsafe().putByte(buf + offset++, (byte) (maskBit | 127)); + Unsafe.getUnsafe().putLong(buf + offset, Long.reverseBytes(payloadLength)); + offset += 8; + } + + return offset; + } + + /** + * Writes a WebSocket frame header with optional mask key. + * + * @param buf the buffer to write to + * @param fin true if this is the final frame + * @param opcode the frame opcode + * @param payloadLength the payload length + * @param maskKey the mask key (only used if masked is true) + * @return the number of bytes written (header size including mask key) + */ + public static int writeHeader(long buf, boolean fin, int opcode, long payloadLength, int maskKey) { + int offset = writeHeader(buf, fin, opcode, payloadLength, true); + Unsafe.getUnsafe().putInt(buf + offset, maskKey); + return offset + 4; + } + + /** + * Calculates the header size for a given payload length and masking. + * + * @param payloadLength the payload length + * @param masked true if the payload will be masked + * @return the header size in bytes + */ + public static int headerSize(long payloadLength, boolean masked) { + int size; + if (payloadLength <= 125) { + size = 2; + } else if (payloadLength <= 65535) { + size = 4; + } else { + size = 10; + } + return masked ? size + 4 : size; + } + + /** + * Writes the payload for a Close frame. + * + * @param buf the buffer to write to (after the header) + * @param code the close status code + * @param reason the close reason (may be null) + * @return the number of bytes written + */ + public static int writeClosePayload(long buf, int code, String reason) { + // Write status code in network byte order (big-endian) + Unsafe.getUnsafe().putShort(buf, Short.reverseBytes((short) code)); + int offset = 2; + + // Write reason if provided + if (reason != null && !reason.isEmpty()) { + byte[] reasonBytes = reason.getBytes(StandardCharsets.UTF_8); + for (byte reasonByte : reasonBytes) { + Unsafe.getUnsafe().putByte(buf + offset++, reasonByte); + } + } + + return offset; + } + + /** + * Writes a complete Close frame to the buffer. + * + * @param buf the buffer to write to + * @param code the close status code + * @param reason the close reason (may be null) + * @return the total number of bytes written (header + payload) + */ + public static int writeCloseFrame(long buf, int code, String reason) { + int payloadLen = 2; // status code + if (reason != null && !reason.isEmpty()) { + payloadLen += reason.getBytes(StandardCharsets.UTF_8).length; + } + + int headerLen = writeHeader(buf, true, WebSocketOpcode.CLOSE, payloadLen, false); + int payloadOffset = writeClosePayload(buf + headerLen, code, reason); + + return headerLen + payloadOffset; + } + + /** + * Writes a complete Ping frame to the buffer. + * + * @param buf the buffer to write to + * @param payload the ping payload + * @param payloadOff offset into payload array + * @param payloadLen length of payload to write + * @return the total number of bytes written + */ + public static int writePingFrame(long buf, byte[] payload, int payloadOff, int payloadLen) { + int headerLen = writeHeader(buf, true, WebSocketOpcode.PING, payloadLen, false); + + // Copy payload + for (int i = 0; i < payloadLen; i++) { + Unsafe.getUnsafe().putByte(buf + headerLen + i, payload[payloadOff + i]); + } + + return headerLen + payloadLen; + } + + /** + * Writes a complete Pong frame to the buffer. + * + * @param buf the buffer to write to + * @param payload the pong payload (should match the received ping) + * @param payloadOff offset into payload array + * @param payloadLen length of payload to write + * @return the total number of bytes written + */ + public static int writePongFrame(long buf, byte[] payload, int payloadOff, int payloadLen) { + int headerLen = writeHeader(buf, true, WebSocketOpcode.PONG, payloadLen, false); + + // Copy payload + for (int i = 0; i < payloadLen; i++) { + Unsafe.getUnsafe().putByte(buf + headerLen + i, payload[payloadOff + i]); + } + + return headerLen + payloadLen; + } + + /** + * Writes a Pong frame with payload from a memory address. + * + * @param buf the buffer to write to + * @param payloadPtr pointer to the ping payload to echo + * @param payloadLen length of payload + * @return the total number of bytes written + */ + public static int writePongFrame(long buf, long payloadPtr, int payloadLen) { + int headerLen = writeHeader(buf, true, WebSocketOpcode.PONG, payloadLen, false); + + // Copy payload from memory + Unsafe.getUnsafe().copyMemory(payloadPtr, buf + headerLen, payloadLen); + + return headerLen + payloadLen; + } + + /** + * Writes a binary frame with payload from a memory address. + * + * @param buf the buffer to write to + * @param payloadPtr pointer to the payload data + * @param payloadLen length of payload + * @return the total number of bytes written + */ + public static int writeBinaryFrame(long buf, long payloadPtr, int payloadLen) { + int headerLen = writeHeader(buf, true, WebSocketOpcode.BINARY, payloadLen, false); + + // Copy payload from memory + Unsafe.getUnsafe().copyMemory(payloadPtr, buf + headerLen, payloadLen); + + return headerLen + payloadLen; + } + + /** + * Writes a binary frame header only (for when payload is written separately). + * + * @param buf the buffer to write to + * @param payloadLen length of payload that will follow + * @return the header size in bytes + */ + public static int writeBinaryFrameHeader(long buf, int payloadLen) { + return writeHeader(buf, true, WebSocketOpcode.BINARY, payloadLen, false); + } + + /** + * Masks payload data in place using XOR with the given mask key. + * + * @param buf the payload buffer + * @param len the payload length + * @param maskKey the 4-byte mask key + */ + public static void maskPayload(long buf, long len, int maskKey) { + // Process 8 bytes at a time when possible + long i = 0; + long longMask = ((long) maskKey << 32) | (maskKey & 0xFFFFFFFFL); + + // Process 8-byte chunks + while (i + 8 <= len) { + long value = Unsafe.getUnsafe().getLong(buf + i); + Unsafe.getUnsafe().putLong(buf + i, value ^ longMask); + i += 8; + } + + // Process 4-byte chunk if remaining + if (i + 4 <= len) { + int value = Unsafe.getUnsafe().getInt(buf + i); + Unsafe.getUnsafe().putInt(buf + i, value ^ maskKey); + i += 4; + } + + // Process remaining bytes (0-3 bytes) - extract mask byte inline to avoid allocation + while (i < len) { + byte b = Unsafe.getUnsafe().getByte(buf + i); + int maskByte = (maskKey >> (((int) i & 3) << 3)) & 0xFF; + Unsafe.getUnsafe().putByte(buf + i, (byte) (b ^ maskByte)); + i++; + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketHandshake.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketHandshake.java new file mode 100644 index 0000000..5248942 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketHandshake.java @@ -0,0 +1,421 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.ilpv4.websocket; + +import io.questdb.client.std.Unsafe; +import io.questdb.client.std.str.Utf8Sequence; +import io.questdb.client.std.str.Utf8String; +import io.questdb.client.std.str.Utf8s; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Base64; + +/** + * WebSocket handshake processing as defined in RFC 6455. + * Provides utilities for validating WebSocket upgrade requests and + * generating proper handshake responses. + */ +public final class WebSocketHandshake { + /** + * The WebSocket magic GUID used in the Sec-WebSocket-Accept calculation. + */ + public static final String WEBSOCKET_GUID = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"; + + /** + * The required WebSocket version (RFC 6455). + */ + public static final int WEBSOCKET_VERSION = 13; + + // Header names (case-insensitive) + public static final Utf8String HEADER_UPGRADE = new Utf8String("Upgrade"); + public static final Utf8String HEADER_CONNECTION = new Utf8String("Connection"); + public static final Utf8String HEADER_SEC_WEBSOCKET_KEY = new Utf8String("Sec-WebSocket-Key"); + public static final Utf8String HEADER_SEC_WEBSOCKET_VERSION = new Utf8String("Sec-WebSocket-Version"); + public static final Utf8String HEADER_SEC_WEBSOCKET_PROTOCOL = new Utf8String("Sec-WebSocket-Protocol"); + public static final Utf8String HEADER_SEC_WEBSOCKET_ACCEPT = new Utf8String("Sec-WebSocket-Accept"); + + // Header values + public static final Utf8String VALUE_WEBSOCKET = new Utf8String("websocket"); + public static final Utf8String VALUE_UPGRADE = new Utf8String("upgrade"); + + // Response template + private static final byte[] RESPONSE_PREFIX = + "HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: ".getBytes(StandardCharsets.US_ASCII); + private static final byte[] RESPONSE_SUFFIX = "\r\n\r\n".getBytes(StandardCharsets.US_ASCII); + + // Thread-local SHA-1 digest for computing Sec-WebSocket-Accept + private static final ThreadLocal SHA1_DIGEST = ThreadLocal.withInitial(() -> { + try { + return MessageDigest.getInstance("SHA-1"); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException("SHA-1 not available", e); + } + }); + + private WebSocketHandshake() { + // Static utility class + } + + /** + * Checks if the given header indicates a WebSocket upgrade request. + * + * @param upgradeHeader the value of the Upgrade header + * @return true if this is a WebSocket upgrade request + */ + public static boolean isWebSocketUpgrade(Utf8Sequence upgradeHeader) { + return upgradeHeader != null && Utf8s.equalsIgnoreCaseAscii(upgradeHeader, VALUE_WEBSOCKET); + } + + /** + * Checks if the Connection header contains "upgrade". + * + * @param connectionHeader the value of the Connection header + * @return true if the connection should be upgraded + */ + public static boolean isConnectionUpgrade(Utf8Sequence connectionHeader) { + if (connectionHeader == null) { + return false; + } + // Connection header may contain multiple values, e.g., "keep-alive, Upgrade" + // Perform case-insensitive substring search + return containsIgnoreCaseAscii(connectionHeader, VALUE_UPGRADE); + } + + /** + * Checks if the sequence contains the given substring (case-insensitive). + */ + private static boolean containsIgnoreCaseAscii(Utf8Sequence seq, Utf8Sequence substring) { + int seqLen = seq.size(); + int subLen = substring.size(); + + if (subLen > seqLen) { + return false; + } + if (subLen == 0) { + return true; + } + + outer: + for (int i = 0; i <= seqLen - subLen; i++) { + for (int j = 0; j < subLen; j++) { + byte a = seq.byteAt(i + j); + byte b = substring.byteAt(j); + // Convert to lowercase for comparison + if (a >= 'A' && a <= 'Z') { + a = (byte) (a + 32); + } + if (b >= 'A' && b <= 'Z') { + b = (byte) (b + 32); + } + if (a != b) { + continue outer; + } + } + return true; + } + return false; + } + + /** + * Validates the WebSocket version. + * + * @param versionHeader the Sec-WebSocket-Version header value + * @return true if the version is valid (13) + */ + public static boolean isValidVersion(Utf8Sequence versionHeader) { + if (versionHeader == null || versionHeader.size() == 0) { + return false; + } + // Parse the version number + try { + int version = 0; + for (int i = 0; i < versionHeader.size(); i++) { + byte b = versionHeader.byteAt(i); + if (b < '0' || b > '9') { + return false; + } + version = version * 10 + (b - '0'); + } + return version == WEBSOCKET_VERSION; + } catch (Exception e) { + return false; + } + } + + /** + * Validates the Sec-WebSocket-Key header. + * The key must be a base64-encoded 16-byte value. + * + * @param key the Sec-WebSocket-Key header value + * @return true if the key is valid + */ + public static boolean isValidKey(Utf8Sequence key) { + if (key == null) { + return false; + } + // Base64-encoded 16-byte value should be exactly 24 characters + // (16 bytes = 128 bits = 22 base64 chars + 2 padding = 24) + int size = key.size(); + if (size != 24) { + return false; + } + // Basic validation: check that all characters are valid base64 + for (int i = 0; i < size; i++) { + byte b = key.byteAt(i); + boolean valid = (b >= 'A' && b <= 'Z') || (b >= 'a' && b <= 'z') || + (b >= '0' && b <= '9') || b == '+' || b == '/' || b == '='; + if (!valid) { + return false; + } + } + return true; + } + + /** + * Computes the Sec-WebSocket-Accept value for the given key. + * + * @param key the Sec-WebSocket-Key from the client + * @return the base64-encoded SHA-1 hash to send in the response + */ + public static String computeAcceptKey(Utf8Sequence key) { + MessageDigest sha1 = SHA1_DIGEST.get(); + sha1.reset(); + + // Concatenate key + GUID + byte[] keyBytes = new byte[key.size()]; + for (int i = 0; i < key.size(); i++) { + keyBytes[i] = key.byteAt(i); + } + sha1.update(keyBytes); + sha1.update(WEBSOCKET_GUID.getBytes(StandardCharsets.US_ASCII)); + + // Compute SHA-1 hash and base64 encode + byte[] hash = sha1.digest(); + return Base64.getEncoder().encodeToString(hash); + } + + /** + * Computes the Sec-WebSocket-Accept value for the given key string. + * + * @param key the Sec-WebSocket-Key from the client + * @return the base64-encoded SHA-1 hash to send in the response + */ + public static String computeAcceptKey(String key) { + MessageDigest sha1 = SHA1_DIGEST.get(); + sha1.reset(); + + // Concatenate key + GUID + sha1.update(key.getBytes(StandardCharsets.US_ASCII)); + sha1.update(WEBSOCKET_GUID.getBytes(StandardCharsets.US_ASCII)); + + // Compute SHA-1 hash and base64 encode + byte[] hash = sha1.digest(); + return Base64.getEncoder().encodeToString(hash); + } + + /** + * Writes the WebSocket handshake response to the given buffer. + * + * @param buf the buffer to write to + * @param acceptKey the computed Sec-WebSocket-Accept value + * @return the number of bytes written + */ + public static int writeResponse(long buf, String acceptKey) { + int offset = 0; + + // Write prefix + for (byte b : RESPONSE_PREFIX) { + Unsafe.getUnsafe().putByte(buf + offset++, b); + } + + // Write accept key + byte[] acceptBytes = acceptKey.getBytes(StandardCharsets.US_ASCII); + for (byte b : acceptBytes) { + Unsafe.getUnsafe().putByte(buf + offset++, b); + } + + // Write suffix + for (byte b : RESPONSE_SUFFIX) { + Unsafe.getUnsafe().putByte(buf + offset++, b); + } + + return offset; + } + + /** + * Returns the size of the handshake response for the given accept key. + * + * @param acceptKey the computed accept key + * @return the total response size in bytes + */ + public static int responseSize(String acceptKey) { + return RESPONSE_PREFIX.length + acceptKey.length() + RESPONSE_SUFFIX.length; + } + + /** + * Writes the WebSocket handshake response with an optional subprotocol. + * + * @param buf the buffer to write to + * @param acceptKey the computed Sec-WebSocket-Accept value + * @param protocol the negotiated subprotocol (may be null or empty) + * @return the number of bytes written + */ + public static int writeResponseWithProtocol(long buf, String acceptKey, String protocol) { + int offset = 0; + + // Write prefix + for (byte b : RESPONSE_PREFIX) { + Unsafe.getUnsafe().putByte(buf + offset++, b); + } + + // Write accept key + byte[] acceptBytes = acceptKey.getBytes(StandardCharsets.US_ASCII); + for (byte b : acceptBytes) { + Unsafe.getUnsafe().putByte(buf + offset++, b); + } + + // Write protocol header if present + if (protocol != null && !protocol.isEmpty()) { + byte[] protocolHeader = ("\r\nSec-WebSocket-Protocol: " + protocol).getBytes(StandardCharsets.US_ASCII); + for (byte b : protocolHeader) { + Unsafe.getUnsafe().putByte(buf + offset++, b); + } + } + + // Write suffix + for (byte b : RESPONSE_SUFFIX) { + Unsafe.getUnsafe().putByte(buf + offset++, b); + } + + return offset; + } + + /** + * Returns the size of the handshake response with an optional subprotocol. + * + * @param acceptKey the computed accept key + * @param protocol the negotiated subprotocol (may be null or empty) + * @return the total response size in bytes + */ + public static int responseSizeWithProtocol(String acceptKey, String protocol) { + int size = RESPONSE_PREFIX.length + acceptKey.length() + RESPONSE_SUFFIX.length; + if (protocol != null && !protocol.isEmpty()) { + size += "\r\nSec-WebSocket-Protocol: ".length() + protocol.length(); + } + return size; + } + + /** + * Writes a 400 Bad Request response. + * + * @param buf the buffer to write to + * @param reason the reason for the bad request + * @return the number of bytes written + */ + public static int writeBadRequestResponse(long buf, String reason) { + int offset = 0; + + byte[] statusLine = "HTTP/1.1 400 Bad Request\r\n".getBytes(StandardCharsets.US_ASCII); + for (byte b : statusLine) { + Unsafe.getUnsafe().putByte(buf + offset++, b); + } + + byte[] contentType = "Content-Type: text/plain\r\n".getBytes(StandardCharsets.US_ASCII); + for (byte b : contentType) { + Unsafe.getUnsafe().putByte(buf + offset++, b); + } + + byte[] reasonBytes = reason != null ? reason.getBytes(StandardCharsets.UTF_8) : new byte[0]; + byte[] contentLength = ("Content-Length: " + reasonBytes.length + "\r\n\r\n").getBytes(StandardCharsets.US_ASCII); + for (byte b : contentLength) { + Unsafe.getUnsafe().putByte(buf + offset++, b); + } + + for (byte b : reasonBytes) { + Unsafe.getUnsafe().putByte(buf + offset++, b); + } + + return offset; + } + + /** + * Writes a 426 Upgrade Required response indicating unsupported WebSocket version. + * + * @param buf the buffer to write to + * @return the number of bytes written + */ + public static int writeVersionNotSupportedResponse(long buf) { + int offset = 0; + + byte[] statusLine = "HTTP/1.1 426 Upgrade Required\r\n".getBytes(StandardCharsets.US_ASCII); + for (byte b : statusLine) { + Unsafe.getUnsafe().putByte(buf + offset++, b); + } + + byte[] versionHeader = "Sec-WebSocket-Version: 13\r\n".getBytes(StandardCharsets.US_ASCII); + for (byte b : versionHeader) { + Unsafe.getUnsafe().putByte(buf + offset++, b); + } + + byte[] contentLength = "Content-Length: 0\r\n\r\n".getBytes(StandardCharsets.US_ASCII); + for (byte b : contentLength) { + Unsafe.getUnsafe().putByte(buf + offset++, b); + } + + return offset; + } + + /** + * Validates all required headers for a WebSocket upgrade request. + * + * @param upgradeHeader the Upgrade header value + * @param connectionHeader the Connection header value + * @param keyHeader the Sec-WebSocket-Key header value + * @param versionHeader the Sec-WebSocket-Version header value + * @return null if valid, or an error message describing the problem + */ + public static String validate( + Utf8Sequence upgradeHeader, + Utf8Sequence connectionHeader, + Utf8Sequence keyHeader, + Utf8Sequence versionHeader + ) { + if (!isWebSocketUpgrade(upgradeHeader)) { + return "Missing or invalid Upgrade header"; + } + if (!isConnectionUpgrade(connectionHeader)) { + return "Missing or invalid Connection header"; + } + if (!isValidKey(keyHeader)) { + return "Missing or invalid Sec-WebSocket-Key header"; + } + if (!isValidVersion(versionHeader)) { + return "Unsupported WebSocket version"; + } + return null; + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketOpcode.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketOpcode.java new file mode 100644 index 0000000..03fe188 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketOpcode.java @@ -0,0 +1,136 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.ilpv4.websocket; + +/** + * WebSocket frame opcodes as defined in RFC 6455. + */ +public final class WebSocketOpcode { + /** + * Continuation frame (0x0). + * Used for fragmented messages after the initial frame. + */ + public static final int CONTINUATION = 0x00; + + /** + * Text frame (0x1). + * Payload is UTF-8 encoded text. + */ + public static final int TEXT = 0x01; + + /** + * Binary frame (0x2). + * Payload is arbitrary binary data. + */ + public static final int BINARY = 0x02; + + // Reserved non-control frames: 0x3-0x7 + + /** + * Connection close frame (0x8). + * Indicates that the endpoint wants to close the connection. + */ + public static final int CLOSE = 0x08; + + /** + * Ping frame (0x9). + * Used for keep-alive and connection health checks. + */ + public static final int PING = 0x09; + + /** + * Pong frame (0xA). + * Response to a ping frame. + */ + public static final int PONG = 0x0A; + + // Reserved control frames: 0xB-0xF + + private WebSocketOpcode() { + // Constants class + } + + /** + * Checks if the opcode is a control frame. + * Control frames are CLOSE (0x8), PING (0x9), and PONG (0xA). + * + * @param opcode the opcode to check + * @return true if the opcode is a control frame + */ + public static boolean isControlFrame(int opcode) { + return (opcode & 0x08) != 0; + } + + /** + * Checks if the opcode is a data frame. + * Data frames are CONTINUATION (0x0), TEXT (0x1), and BINARY (0x2). + * + * @param opcode the opcode to check + * @return true if the opcode is a data frame + */ + public static boolean isDataFrame(int opcode) { + return opcode <= 0x02; + } + + /** + * Checks if the opcode is valid according to RFC 6455. + * + * @param opcode the opcode to check + * @return true if the opcode is valid + */ + public static boolean isValid(int opcode) { + return opcode == CONTINUATION + || opcode == TEXT + || opcode == BINARY + || opcode == CLOSE + || opcode == PING + || opcode == PONG; + } + + /** + * Returns a human-readable name for the opcode. + * + * @param opcode the opcode + * @return the opcode name + */ + public static String name(int opcode) { + switch (opcode) { + case CONTINUATION: + return "CONTINUATION"; + case TEXT: + return "TEXT"; + case BINARY: + return "BINARY"; + case CLOSE: + return "CLOSE"; + case PING: + return "PING"; + case PONG: + return "PONG"; + default: + return "UNKNOWN(" + opcode + ")"; + } + } +} diff --git a/core/src/main/java/io/questdb/client/std/CharSequenceIntHashMap.java b/core/src/main/java/io/questdb/client/std/CharSequenceIntHashMap.java new file mode 100644 index 0000000..d299423 --- /dev/null +++ b/core/src/main/java/io/questdb/client/std/CharSequenceIntHashMap.java @@ -0,0 +1,209 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.std; + +import org.jetbrains.annotations.NotNull; + +import java.util.Arrays; + + +public class CharSequenceIntHashMap extends AbstractCharSequenceHashSet { + public static final int NO_ENTRY_VALUE = -1; + private final ObjList list; + private final int noEntryValue; + private int[] values; + + public CharSequenceIntHashMap() { + this(8); + } + + public CharSequenceIntHashMap(int initialCapacity) { + this(initialCapacity, 0.4, NO_ENTRY_VALUE); + } + + public CharSequenceIntHashMap(int initialCapacity, double loadFactor, int noEntryValue) { + super(initialCapacity, loadFactor); + this.noEntryValue = noEntryValue; + this.list = new ObjList<>(capacity); + values = new int[keys.length]; + clear(); + } + + @Override + public final void clear() { + super.clear(); + list.clear(); + Arrays.fill(values, noEntryValue); + } + + public int get(@NotNull CharSequence key) { + return valueAt(keyIndex(key)); + } + + public void inc(@NotNull CharSequence key) { + int index = keyIndex(key); + if (index < 0) { + values[-index - 1]++; + } else { + putAt0(index, Chars.toString(key), 1); + } + } + + public ObjList keys() { + return list; + } + + public boolean put(@NotNull CharSequence key, int value) { + return putAt(keyIndex(key), key, value); + } + + public void putAll(@NotNull CharSequenceIntHashMap other) { + CharSequence[] otherKeys = other.keys; + int[] otherValues = other.values; + for (int i = 0, n = otherKeys.length; i < n; i++) { + if (otherKeys[i] != noEntryKey) { + put(otherKeys[i], otherValues[i]); + } + } + } + + public boolean putAt(int index, @NotNull CharSequence key, int value) { + if (index < 0) { + values[-index - 1] = value; + return false; + } + final String keyString = Chars.toString(key); + putAt0(index, keyString, value); + list.add(keyString); + return true; + } + + public void putIfAbsent(@NotNull CharSequence key, int value) { + int index = keyIndex(key); + if (index > -1) { + String keyString = Chars.toString(key); + putAt0(index, keyString, value); + list.add(keyString); + } + } + + public void removeAt(int index) { + if (index < 0) { + int from = -index - 1; + CharSequence key = keys[from]; + erase(from); + free++; + + // after we have freed up a slot + // consider non-empty keys directly below + // they may have been a direct hit but because + // directly hit slot wasn't empty these keys would + // have moved. + // + // After slot is freed these keys require re-hash + from = (from + 1) & mask; + for ( + CharSequence k = keys[from]; + k != noEntryKey; + from = (from + 1) & mask, k = keys[from] + ) { + int idealHit = Hash.spread(Chars.hashCode(k)) & mask; + if (idealHit != from) { + int to; + if (keys[idealHit] != noEntryKey) { + to = probe0(k, idealHit); + } else { + to = idealHit; + } + + if (to > -1) { + move(from, to); + } + } + } + + list.remove(key); + } + } + + public int valueAt(int index) { + int index1 = -index - 1; + return index < 0 ? values[index1] : noEntryValue; + } + + public int valueQuick(int index) { + return get(list.getQuick(index)); + } + + private void putAt0(int index, CharSequence key, int value) { + keys[index] = key; + values[index] = value; + if (--free == 0) { + rehash(); + } + } + + private void rehash() { + int[] oldValues = values; + CharSequence[] oldKeys = keys; + int size = capacity - free; + capacity = capacity * 2; + free = capacity - size; + mask = Numbers.ceilPow2((int) (capacity / loadFactor)) - 1; + this.keys = new CharSequence[mask + 1]; + this.values = new int[mask + 1]; + for (int i = oldKeys.length - 1; i > -1; i--) { + CharSequence key = oldKeys[i]; + if (key != null) { + final int index = keyIndex(key); + keys[index] = key; + values[index] = oldValues[i]; + } + } + } + + private void erase(int index) { + keys[index] = noEntryKey; + values[index] = noEntryValue; + } + + private void move(int from, int to) { + keys[to] = keys[from]; + values[to] = values[from]; + erase(from); + } + + private int probe0(CharSequence key, int index) { + do { + index = (index + 1) & mask; + if (keys[index] == noEntryKey) { + return index; + } + if (Chars.equals(key, keys[index])) { + return -index - 1; + } + } while (true); + } +} diff --git a/core/src/main/java/io/questdb/client/std/LongHashSet.java b/core/src/main/java/io/questdb/client/std/LongHashSet.java new file mode 100644 index 0000000..4ec2648 --- /dev/null +++ b/core/src/main/java/io/questdb/client/std/LongHashSet.java @@ -0,0 +1,155 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.std; + +import io.questdb.client.std.str.CharSink; +import io.questdb.client.std.str.Sinkable; +import org.jetbrains.annotations.NotNull; + +import java.util.Arrays; + + +public class LongHashSet extends AbstractLongHashSet implements Sinkable { + + public static final double DEFAULT_LOAD_FACTOR = 0.4; + private static final int MIN_INITIAL_CAPACITY = 16; + private final LongList list; + + public LongHashSet() { + this(MIN_INITIAL_CAPACITY); + } + + public LongHashSet(int initialCapacity) { + this(initialCapacity, DEFAULT_LOAD_FACTOR, noEntryKey); + } + + public LongHashSet(int initialCapacity, double loadFactor, long noKeyValue) { + super(initialCapacity, loadFactor, noKeyValue); + list = new LongList(free); + clear(); + } + + /** + * Adds key to hash set preserving key uniqueness. + * + * @param key key to be added. + * @return false if key is already in the set and true otherwise. + */ + public boolean add(long key) { + int index = keyIndex(key); + if (index < 0) { + return false; + } + + addAt(index, key); + return true; + } + + public void addAt(int index, long key) { + keys[index] = key; + list.add(key); + if (--free < 1) { + rehash(); + } + } + + public final void clear() { + free = capacity; + Arrays.fill(keys, noEntryKeyValue); + list.clear(); + } + + public boolean contains(long key) { + return keyIndex(key) < 0; + } + + public long get(int index) { + return list.getQuick(index); + } + + public long getLast() { + return list.getLast(); + } + + public void removeAt(int index) { + if (index < 0) { + long key = keys[-index - 1]; + super.removeAt(index); + listRemove(key); + } + } + + @Override + public void toSink(@NotNull CharSink sink) { + list.toSink(sink); + } + + @Override + public String toString() { + return list.toString(); + } + + private void listRemove(long v) { + int sz = list.size(); + for (int i = 0; i < sz; i++) { + if (list.getQuick(i) == v) { + // shift remaining elements left + for (int j = i + 1; j < sz; j++) { + list.setQuick(j - 1, list.getQuick(j)); + } + list.setPos(sz - 1); + return; + } + } + } + + private void rehash() { + int newCapacity = capacity * 2; + free = capacity = newCapacity; + int len = Numbers.ceilPow2((int) (newCapacity / loadFactor)); + this.keys = new long[len]; + Arrays.fill(keys, noEntryKeyValue); + mask = len - 1; + int n = list.size(); + free -= n; + for (int i = 0; i < n; i++) { + long key = list.getQuick(i); + int keyIndex = keyIndex(key); + keys[keyIndex] = key; + } + } + + @Override + protected void erase(int index) { + keys[index] = noEntryKeyValue; + } + + @Override + protected void move(int from, int to) { + keys[to] = keys[from]; + erase(from); + } + +} diff --git a/core/src/main/java/module-info.java b/core/src/main/java/module-info.java index fa5bc48..45b319c 100644 --- a/core/src/main/java/module-info.java +++ b/core/src/main/java/module-info.java @@ -56,4 +56,7 @@ exports io.questdb.client.cairo.arr; exports io.questdb.client.cutlass.line.array; exports io.questdb.client.cutlass.line.udp; + exports io.questdb.client.cutlass.ilpv4.client; + exports io.questdb.client.cutlass.ilpv4.protocol; + exports io.questdb.client.cutlass.ilpv4.websocket; } From dee93bdeeef54b91c836f2f85c652afa27e2a802 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Sat, 14 Feb 2026 20:27:34 +0000 Subject: [PATCH 02/89] tidy --- .../questdb/client/cutlass/http/client/WebSocketClient.java | 5 ++--- .../client/cutlass/http/client/WebSocketSendBuffer.java | 3 ++- .../client/cutlass/ilpv4/client/WebSocketResponse.java | 1 - .../client/cutlass/ilpv4/protocol/IlpV4SchemaHash.java | 2 -- 4 files changed, 4 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java index 8f5ce83..cc64a63 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java @@ -29,8 +29,6 @@ import io.questdb.client.cutlass.ilpv4.websocket.WebSocketFrameParser; import io.questdb.client.cutlass.ilpv4.websocket.WebSocketHandshake; import io.questdb.client.cutlass.ilpv4.websocket.WebSocketOpcode; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import io.questdb.client.network.IOOperation; import io.questdb.client.network.NetworkFacade; import io.questdb.client.network.Socket; @@ -42,7 +40,8 @@ import io.questdb.client.std.Rnd; import io.questdb.client.std.Unsafe; import io.questdb.client.std.Vect; -import io.questdb.client.std.str.Utf8String; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.nio.charset.StandardCharsets; import java.util.Base64; diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java index 0eea869..20c43c1 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java @@ -27,6 +27,7 @@ import io.questdb.client.cutlass.ilpv4.websocket.WebSocketFrameWriter; import io.questdb.client.cutlass.ilpv4.websocket.WebSocketOpcode; import io.questdb.client.cutlass.ilpv4.client.IlpBufferWriter; +import io.questdb.client.cutlass.line.array.ArrayBufferAppender; import io.questdb.client.std.MemoryTag; import io.questdb.client.std.Numbers; import io.questdb.client.std.QuietCloseable; @@ -137,7 +138,7 @@ private void grow(long requiredCapacity) { .put(maxBufferSize) .put(']'); } - int newCapacity = (int) Math.min( + int newCapacity = Math.min( Numbers.ceilPow2((int) requiredCapacity), maxBufferSize ); diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/WebSocketResponse.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/WebSocketResponse.java index 42e74af..2c29baa 100644 --- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/WebSocketResponse.java +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/WebSocketResponse.java @@ -24,7 +24,6 @@ package io.questdb.client.cutlass.ilpv4.client; -import io.questdb.client.std.MemoryTag; import io.questdb.client.std.Unsafe; import java.nio.charset.StandardCharsets; diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4SchemaHash.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4SchemaHash.java index 566e2f2..9996d7f 100644 --- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4SchemaHash.java +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4SchemaHash.java @@ -29,8 +29,6 @@ import io.questdb.client.std.str.DirectUtf8Sequence; import io.questdb.client.std.str.Utf8Sequence; -import java.nio.charset.StandardCharsets; - /** * XXHash64 implementation for schema hashing in ILP v4 protocol. *

From ab2f5b58ebdaecf60eddad1209fd5ed17a772a9b Mon Sep 17 00:00:00 2001 From: bluestreak Date: Sat, 14 Feb 2026 21:11:38 +0000 Subject: [PATCH 03/89] move client test --- .../test/LineSenderBuilderWebSocketTest.java | 777 ++++++++++++++++++ 1 file changed, 777 insertions(+) create mode 100644 core/src/test/java/io/questdb/client/test/LineSenderBuilderWebSocketTest.java diff --git a/core/src/test/java/io/questdb/client/test/LineSenderBuilderWebSocketTest.java b/core/src/test/java/io/questdb/client/test/LineSenderBuilderWebSocketTest.java new file mode 100644 index 0000000..736aec4 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/LineSenderBuilderWebSocketTest.java @@ -0,0 +1,777 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test; + +import io.questdb.client.Sender; +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Ignore; +import org.junit.Test; + +/** + * Tests for WebSocket transport support in the Sender.builder() API. + * These tests verify the builder configuration and validation, + * not actual WebSocket connectivity (which requires a running server). + */ +public class LineSenderBuilderWebSocketTest extends AbstractTest { + + private static final String LOCALHOST = "localhost"; + + @Test + public void testAddressConfiguration() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST + ":9000"); + Assert.assertNotNull(builder); + } + + @Test + public void testAddressEmpty_fails() { + assertThrows("address cannot be empty", + () -> Sender.builder(Sender.Transport.WEBSOCKET).address("")); + } + + @Test + public void testAddressEndsWithColon_fails() { + assertThrows("invalid address", + () -> Sender.builder(Sender.Transport.WEBSOCKET).address("foo:")); + } + + @Test + public void testAddressNull_fails() { + assertThrows("null", + () -> Sender.builder(Sender.Transport.WEBSOCKET).address(null)); + } + + // ==================== Transport Selection Tests ==================== + + @Test + public void testAddressWithoutPort_usesDefaultPort9000() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST); + Assert.assertNotNull(builder); + } + + @Test + public void testAsyncModeCanBeSetMultipleTimes() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .asyncMode(true) + .asyncMode(false); + Assert.assertNotNull(builder); + } + + // ==================== Address Configuration Tests ==================== + + @Test + public void testAsyncModeDisabled() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .asyncMode(false); + Assert.assertNotNull(builder); + } + + @Test + public void testAsyncModeEnabled() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .asyncMode(true); + Assert.assertNotNull(builder); + } + + @Test + public void testAsyncModeWithAllOptions() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .asyncMode(true) + .autoFlushRows(500) + .autoFlushBytes(512 * 1024) + .autoFlushIntervalMillis(50) + .inFlightWindowSize(8) + .sendQueueCapacity(16); + Assert.assertNotNull(builder); + } + + @Test + public void testAutoFlushBytes() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .autoFlushBytes(1024 * 1024); + Assert.assertNotNull(builder); + } + + @Test + public void testAutoFlushBytesDoubleSet_fails() { + assertThrows("already configured", + () -> Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .autoFlushBytes(1024) + .autoFlushBytes(2048)); + } + + @Test + public void testAutoFlushBytesNegative_fails() { + assertThrows("cannot be negative", + () -> Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .autoFlushBytes(-1)); + } + + @Test + public void testAutoFlushBytesZero() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .autoFlushBytes(0); + Assert.assertNotNull(builder); + } + + @Test + public void testAutoFlushIntervalMillis() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .autoFlushIntervalMillis(100); + Assert.assertNotNull(builder); + } + + @Test + public void testAutoFlushIntervalMillisDoubleSet_fails() { + assertThrows("already configured", + () -> Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .autoFlushIntervalMillis(100) + .autoFlushIntervalMillis(200)); + } + + // ==================== TLS Configuration Tests ==================== + + @Test + public void testAutoFlushIntervalMillisNegative_fails() { + assertThrows("cannot be negative", + () -> Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .autoFlushIntervalMillis(-1)); + } + + @Test + public void testAutoFlushIntervalMillisZero_fails() { + assertThrows("cannot be negative", + () -> Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .autoFlushIntervalMillis(0)); + } + + @Test + public void testAutoFlushRows() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .autoFlushRows(1000); + Assert.assertNotNull(builder); + } + + @Test + public void testAutoFlushRowsDoubleSet_fails() { + assertThrows("already configured", + () -> Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .autoFlushRows(100) + .autoFlushRows(200)); + } + + @Test + public void testAutoFlushRowsNegative_fails() { + assertThrows("cannot be negative", + () -> Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .autoFlushRows(-1)); + } + + // ==================== Async Mode Tests ==================== + + @Test + public void testAutoFlushRowsZero_disablesRowBasedAutoFlush() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .autoFlushRows(0); + Assert.assertNotNull(builder); + } + + @Test + public void testBufferCapacity() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .bufferCapacity(128 * 1024); + Assert.assertNotNull(builder); + } + + @Test + public void testBufferCapacityDoubleSet_fails() { + assertThrows("already configured", + () -> Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .bufferCapacity(1024) + .bufferCapacity(2048)); + } + + // ==================== Auto Flush Rows Tests ==================== + + @Test + public void testBufferCapacityNegative_fails() { + assertThrows("cannot be negative", + () -> Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .bufferCapacity(-1)); + } + + @Test + public void testBuilderWithWebSocketTransport() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET); + Assert.assertNotNull("Builder should be created for WebSocket transport", builder); + } + + @Test + public void testBuilderWithWebSocketTransportCreatesCorrectSenderType() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST + ":9000"), + "connect", "Failed" + ); + } + + @Test + public void testConnectionRefused() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST + ":19999"), + "connect", "Failed" + ); + } + + // ==================== Auto Flush Bytes Tests ==================== + + @Test + public void testCustomTrustStore_butTlsNotEnabled_fails() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .advancedTls().customTrustStore("/some/path", "password".toCharArray()) + .address(LOCALHOST), + "TLS was not enabled"); + } + + @Test + @Ignore("Disable auto flush may need different semantics for WebSocket") + public void testDisableAutoFlush_semantics() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .disableAutoFlush(); + Assert.assertNotNull(builder); + } + + @Test + public void testDnsResolutionFailure() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address("this-domain-does-not-exist-i-hope-better-to-use-a-silly-tld.silly-tld:9000"), + "resolve", "connect", "Failed" + ); + } + + @Test + public void testDuplicateAddresses_fails() { + assertThrows("duplicated addresses", + () -> Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST + ":9000") + .address(LOCALHOST + ":9000")); + } + + // ==================== Auto Flush Interval Tests ==================== + + @Test + @Ignore("TCP authentication is not supported for WebSocket protocol") + public void testEnableAuth_notSupported() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .enableAuth("keyId") + .authToken("token"), + "not supported for WebSocket"); + } + + @Test + public void testFullAsyncConfiguration() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .asyncMode(true) + .autoFlushRows(1000) + .autoFlushBytes(1024 * 1024) + .autoFlushIntervalMillis(100) + .inFlightWindowSize(16) + .sendQueueCapacity(32); + Assert.assertNotNull(builder); + } + + @Test + public void testFullAsyncConfigurationWithTls() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .enableTls() + .advancedTls().disableCertificateValidation() + .asyncMode(true) + .autoFlushRows(1000) + .autoFlushBytes(1024 * 1024) + .inFlightWindowSize(16) + .sendQueueCapacity(32); + Assert.assertNotNull(builder); + } + + @Test + @Ignore("HTTP path is HTTP-specific and may not apply to WebSocket") + public void testHttpPath_mayNotApply() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .httpPath("/custom/path"); + Assert.assertNotNull(builder); + } + + // ==================== In-Flight Window Size Tests ==================== + + @Test + @Ignore("HTTP timeout is HTTP-specific and may not apply to WebSocket") + public void testHttpTimeout_mayNotApply() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .httpTimeoutMillis(5000); + Assert.assertNotNull(builder); + } + + @Test + public void testHttpToken_fails() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .httpToken("token"), + "not yet supported"); + } + + @Test + @Ignore("HTTP token authentication is not yet supported for WebSocket protocol") + public void testHttpToken_notYetSupported() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .httpToken("token"), + "not yet supported"); + } + + @Test + public void testInFlightWindowSizeDoubleSet_fails() { + assertThrows("already configured", + () -> Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .asyncMode(true) + .inFlightWindowSize(8) + .inFlightWindowSize(16)); + } + + @Test + public void testInFlightWindowSizeNegative_fails() { + assertThrows("must be positive", + () -> Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .asyncMode(true) + .inFlightWindowSize(-1)); + } + + // ==================== Send Queue Capacity Tests ==================== + + @Test + public void testInFlightWindowSizeZero_fails() { + assertThrows("must be positive", + () -> Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .asyncMode(true) + .inFlightWindowSize(0)); + } + + @Test + public void testInFlightWindowSize_withAsyncMode() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .asyncMode(true) + .inFlightWindowSize(16); + Assert.assertNotNull(builder); + } + + @Test + public void testInFlightWindowSize_withoutAsyncMode_fails() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .inFlightWindowSize(16), + "requires async mode"); + } + + @Test + public void testInvalidPort_fails() { + assertThrows("invalid port", + () -> Sender.builder(Sender.Transport.WEBSOCKET).address(LOCALHOST + ":99999")); + } + + @Test + public void testInvalidSchema_fails() { + assertBadConfig("invalid::addr=localhost:9000;", "invalid schema [schema=invalid, supported-schemas=[http, https, tcp, tcps, ws, wss]]"); + } + + // ==================== Combined Async Configuration Tests ==================== + + @Test + public void testMalformedPortInAddress_fails() { + assertThrows("cannot parse a port from the address", + () -> Sender.builder(Sender.Transport.WEBSOCKET).address("foo:nonsense12334")); + } + + @Test + @Ignore("Max backoff is HTTP-specific and may not apply to WebSocket") + public void testMaxBackoff_mayNotApply() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .maxBackoffMillis(1000); + Assert.assertNotNull(builder); + } + + // ==================== Config String Tests (ws:// and wss://) ==================== + + @Test + public void testMaxNameLength() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .maxNameLength(256); + Assert.assertNotNull(builder); + } + + @Test + public void testMaxNameLengthDoubleSet_fails() { + assertThrows("already configured", + () -> Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .maxNameLength(128) + .maxNameLength(256)); + } + + @Test + public void testMaxNameLengthTooSmall_fails() { + assertThrows("at least 16 bytes", + () -> Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .maxNameLength(10)); + } + + @Test + @Ignore("Min request throughput is HTTP-specific and may not apply to WebSocket") + public void testMinRequestThroughput_mayNotApply() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .minRequestThroughput(10000); + Assert.assertNotNull(builder); + } + + @Test + public void testMultipleAddresses_fails() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST + ":9000") + .address(LOCALHOST + ":9001"), + "single address"); + } + + @Test + public void testNoAddress_fails() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET), + "address not set"); + } + + @Test + public void testPortMismatch_fails() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST + ":9000") + .port(9001), + "mismatch"); + } + + // ==================== Buffer Configuration Tests ==================== + + @Test + @Ignore("Protocol version is for ILP text protocol, WebSocket uses ILP v4 binary protocol") + public void testProtocolVersion_notApplicable() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .protocolVersion(Sender.PROTOCOL_VERSION_V2); + Assert.assertNotNull(builder); + } + + @Test + @Ignore("Retry timeout is HTTP-specific and may not apply to WebSocket") + public void testRetryTimeout_mayNotApply() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .retryTimeoutMillis(5000); + Assert.assertNotNull(builder); + } + + @Test + public void testSendQueueCapacityDoubleSet_fails() { + assertThrows("already configured", + () -> Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .asyncMode(true) + .sendQueueCapacity(16) + .sendQueueCapacity(32)); + } + + // ==================== Unsupported Features (TCP Authentication) ==================== + + @Test + public void testSendQueueCapacityNegative_fails() { + assertThrows("must be positive", + () -> Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .asyncMode(true) + .sendQueueCapacity(-1)); + } + + @Test + public void testSendQueueCapacityZero_fails() { + assertThrows("must be positive", + () -> Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .asyncMode(true) + .sendQueueCapacity(0)); + } + + // ==================== Unsupported Features (HTTP Token Authentication) ==================== + + @Test + public void testSendQueueCapacity_withAsyncMode() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .asyncMode(true) + .sendQueueCapacity(32); + Assert.assertNotNull(builder); + } + + @Test + public void testSendQueueCapacity_withoutAsyncMode_fails() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .sendQueueCapacity(32), + "requires async mode"); + } + + // ==================== Unsupported Features (Username/Password Authentication) ==================== + + @Test + public void testSyncModeDoesNotAllowInFlightWindowSize() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .asyncMode(false) + .inFlightWindowSize(16), + "requires async mode"); + } + + @Test + public void testSyncModeDoesNotAllowSendQueueCapacity() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .asyncMode(false) + .sendQueueCapacity(32), + "requires async mode"); + } + + // ==================== Unsupported Features (HTTP-specific options) ==================== + + @Test + public void testSyncModeIsDefault() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST); + Assert.assertNotNull(builder); + } + + @Test + public void testTcpAuth_fails() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .enableAuth("keyId") + .authToken("5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48"), + "not supported for WebSocket"); + } + + @Test + public void testTlsDoubleSet_fails() { + assertThrows("already enabled", + () -> Sender.builder(Sender.Transport.WEBSOCKET) + .enableTls() + .enableTls()); + } + + @Test + public void testTlsEnabled() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .enableTls(); + Assert.assertNotNull(builder); + } + + @Test + public void testTlsValidationDisabled() { + Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .enableTls() + .advancedTls().disableCertificateValidation(); + Assert.assertNotNull(builder); + } + + @Test + public void testTlsValidationDisabled_butTlsNotEnabled_fails() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .advancedTls().disableCertificateValidation() + .address(LOCALHOST), + "TLS was not enabled"); + } + + // ==================== Unsupported Features (Protocol Version) ==================== + + @Test + public void testUsernamePassword_fails() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .httpUsernamePassword("user", "pass"), + "not yet supported"); + } + + // ==================== Config String Unsupported Options ==================== + + @Test + @Ignore("Username/password authentication is not yet supported for WebSocket protocol") + public void testUsernamePassword_notYetSupported() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .httpUsernamePassword("user", "pass"), + "not yet supported"); + } + + @Test + public void testWsConfigString() { + assertBadConfig("ws::addr=localhost:9000;", "connect", "Failed"); + } + + // ==================== Edge Cases ==================== + + @Test + public void testWsConfigString_missingAddr_fails() { + assertBadConfig("ws::addr=localhost;", "connect", "Failed"); + assertBadConfig("ws::foo=bar;", "addr is missing"); + } + + @Test + public void testWsConfigString_protocolAlreadyConfigured_fails() { + assertThrowsAny( + Sender.builder("ws::addr=localhost:9000;") + .enableTls(), + "TLS", "connect", "Failed" + ); + } + + @Test + public void testWsConfigString_uppercaseNotSupported() { + assertBadConfig("WS::addr=localhost:9000;", "invalid schema"); + } + + @Test + @Ignore("Token authentication in ws config string is not yet supported") + public void testWsConfigString_withToken_notYetSupported() { + assertBadConfig("ws::addr=localhost:9000;token=mytoken;", "not yet supported"); + } + + @Test + @Ignore("Username/password in ws config string is not yet supported") + public void testWsConfigString_withUsernamePassword_notYetSupported() { + assertBadConfig("ws::addr=localhost:9000;username=user;password=pass;", "not yet supported"); + } + + // ==================== Connection Tests ==================== + + @Test + public void testWssConfigString() { + assertBadConfig("wss::addr=localhost:9000;tls_verify=unsafe_off;", "connect", "Failed", "SSL"); + } + + @Test + public void testWssConfigString_uppercaseNotSupported() { + assertBadConfig("WSS::addr=localhost:9000;", "invalid schema"); + } + + // ==================== Sync vs Async Mode Tests ==================== + + @SuppressWarnings("resource") + private static void assertBadConfig(String config, String... anyOf) { + assertThrowsAny(() -> Sender.fromConfig(config), anyOf); + } + + private static void assertThrows(String expectedSubstring, Runnable action) { + try { + action.run(); + Assert.fail("Expected LineSenderException containing '" + expectedSubstring + "'"); + } catch (LineSenderException e) { + TestUtils.assertContains(e.getMessage(), expectedSubstring); + } + } + + private static void assertThrowsAny(Sender.LineSenderBuilder builder, String... anyOf) { + assertThrowsAny(builder::build, anyOf); + } + + private static void assertThrowsAny(Runnable action, String... anyOf) { + try { + action.run(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + for (String s : anyOf) { + if (msg.contains(s)) { + return; + } + } + Assert.fail("Expected message containing one of [" + String.join(", ", anyOf) + "] but got: " + msg); + } + } +} From 73ee623630ec47e2df403ef1f1a0f5030356493d Mon Sep 17 00:00:00 2001 From: bluestreak Date: Sat, 14 Feb 2026 21:54:40 +0000 Subject: [PATCH 04/89] more tidy --- .../client/BuildInformationHolder.java | 6 +- .../ilpv4/client/IlpV4WebSocketEncoder.java | 593 ++++++------- .../ilpv4/protocol/IlpV4GorillaDecoder.java | 251 ------ .../ilpv4/protocol/IlpV4GorillaEncoder.java | 53 +- .../ilpv4/protocol/IlpV4TimestampDecoder.java | 474 ----------- .../client/test/LineSenderBuilderTest.java | 799 ------------------ .../cutlass/line/LineSenderBuilderTest.java | 490 +++++++++++ .../line}/LineSenderBuilderWebSocketTest.java | 3 +- core/src/test/java/module-info.java | 1 + 9 files changed, 846 insertions(+), 1824 deletions(-) delete mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4GorillaDecoder.java delete mode 100644 core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4TimestampDecoder.java delete mode 100644 core/src/test/java/io/questdb/client/test/LineSenderBuilderTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/line/LineSenderBuilderTest.java rename core/src/test/java/io/questdb/client/test/{ => cutlass/line}/LineSenderBuilderWebSocketTest.java (99%) diff --git a/core/src/main/java/io/questdb/client/BuildInformationHolder.java b/core/src/main/java/io/questdb/client/BuildInformationHolder.java index e18f961..825a101 100644 --- a/core/src/main/java/io/questdb/client/BuildInformationHolder.java +++ b/core/src/main/java/io/questdb/client/BuildInformationHolder.java @@ -41,7 +41,7 @@ public BuildInformationHolder(Class clazz) { String swVersion; try { final Attributes manifestAttributes = getManifestAttributes(clazz); - swVersion = getAttr(manifestAttributes, "QuestDB-Client-Version", "[DEVELOPMENT]"); + swVersion = getAttr(manifestAttributes, "[DEVELOPMENT]"); } catch (IOException e) { swVersion = UNKNOWN; } @@ -57,8 +57,8 @@ public String getSwVersion() { return swVersion; } - private static String getAttr(final Attributes manifestAttributes, String attributeName, String defaultValue) { - final String value = manifestAttributes.getValue(attributeName); + private static String getAttr(final Attributes manifestAttributes, String defaultValue) { + final String value = manifestAttributes.getValue("QuestDB-Client-Version"); return value != null ? value : defaultValue; } diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/IlpV4WebSocketEncoder.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/IlpV4WebSocketEncoder.java index f1826f5..e98a1df 100644 --- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/IlpV4WebSocketEncoder.java +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/IlpV4WebSocketEncoder.java @@ -24,12 +24,9 @@ package io.questdb.client.cutlass.ilpv4.client; -import io.questdb.client.cutlass.ilpv4.protocol.*; - import io.questdb.client.cutlass.ilpv4.protocol.IlpV4ColumnDef; import io.questdb.client.cutlass.ilpv4.protocol.IlpV4GorillaEncoder; - -import io.questdb.client.cutlass.ilpv4.protocol.IlpV4TimestampDecoder; +import io.questdb.client.cutlass.ilpv4.protocol.IlpV4TableBuffer; import io.questdb.client.std.QuietCloseable; import static io.questdb.client.cutlass.ilpv4.protocol.IlpV4Constants.*; @@ -55,10 +52,18 @@ */ public class IlpV4WebSocketEncoder implements QuietCloseable { - private NativeBufferWriter ownedBuffer; - private IlpBufferWriter buffer; + /** + * Encoding flag for Gorilla-encoded timestamps. + */ + public static final byte ENCODING_GORILLA = 0x01; + /** + * Encoding flag for uncompressed timestamps. + */ + public static final byte ENCODING_UNCOMPRESSED = 0x00; private final IlpV4GorillaEncoder gorillaEncoder = new IlpV4GorillaEncoder(); + private IlpBufferWriter buffer; private byte flags; + private NativeBufferWriter ownedBuffer; public IlpV4WebSocketEncoder() { this.ownedBuffer = new NativeBufferWriter(); @@ -72,68 +77,14 @@ public IlpV4WebSocketEncoder(int bufferSize) { this.flags = 0; } - /** - * Returns the underlying buffer. - *

- * If an external buffer was set via {@link #setBuffer(IlpBufferWriter)}, - * that buffer is returned. Otherwise, returns the internal buffer. - */ - public IlpBufferWriter getBuffer() { - return buffer; - } - - /** - * Sets an external buffer for encoding. - *

- * When set, the encoder writes directly to this buffer instead of its internal buffer. - * The caller is responsible for managing the external buffer's lifecycle. - *

- * Pass {@code null} to revert to using the internal buffer. - * - * @param externalBuffer the external buffer to use, or null to use internal buffer - */ - public void setBuffer(IlpBufferWriter externalBuffer) { - this.buffer = externalBuffer != null ? externalBuffer : ownedBuffer; - } - - /** - * Returns true if currently using an external buffer. - */ - public boolean isUsingExternalBuffer() { - return buffer != ownedBuffer; - } - - /** - * Resets the encoder for a new message. - *

- * If using an external buffer, this only resets the internal state (flags). - * The external buffer's reset is the caller's responsibility. - * If using the internal buffer, resets both the buffer and internal state. - */ - public void reset() { - if (!isUsingExternalBuffer()) { - buffer.reset(); - } - } - - /** - * Sets whether Gorilla timestamp encoding is enabled. - */ - public void setGorillaEnabled(boolean enabled) { - if (enabled) { - flags |= FLAG_GORILLA; - } else { - flags &= ~FLAG_GORILLA; + @Override + public void close() { + if (ownedBuffer != null) { + ownedBuffer.close(); + ownedBuffer = null; } } - /** - * Returns true if Gorilla encoding is enabled. - */ - public boolean isGorillaEnabled() { - return (flags & FLAG_GORILLA) != 0; - } - /** * Encodes a complete ILP v4 message from a table buffer. * @@ -164,11 +115,11 @@ public int encode(IlpV4TableBuffer tableBuffer, boolean useSchemaRef) { * This method sends only new symbols (delta) since the last confirmed watermark, * and uses global symbol IDs instead of per-column local indices. * - * @param tableBuffer the table buffer containing row data - * @param globalDict the global symbol dictionary - * @param confirmedMaxId the highest symbol ID the server has confirmed (from ConnectionSymbolState) - * @param batchMaxId the highest symbol ID used in this batch - * @param useSchemaRef whether to use schema reference mode + * @param tableBuffer the table buffer containing row data + * @param globalDict the global symbol dictionary + * @param confirmedMaxId the highest symbol ID the server has confirmed (from ConnectionSymbolState) + * @param batchMaxId the highest symbol ID used in this batch + * @param useSchemaRef whether to use schema reference mode * @return the number of bytes written */ public int encodeWithDeltaDict( @@ -214,14 +165,13 @@ public int encodeWithDeltaDict( } /** - * Sets the delta symbol dictionary flag. + * Returns the underlying buffer. + *

+ * If an external buffer was set via {@link #setBuffer(IlpBufferWriter)}, + * that buffer is returned. Otherwise, returns the internal buffer. */ - public void setDeltaSymbolDictEnabled(boolean enabled) { - if (enabled) { - flags |= FLAG_DELTA_SYMBOL_DICT; - } else { - flags &= ~FLAG_DELTA_SYMBOL_DICT; - } + public IlpBufferWriter getBuffer() { + return buffer; } /** @@ -232,127 +182,92 @@ public boolean isDeltaSymbolDictEnabled() { } /** - * Writes the ILP v4 message header. - * - * @param tableCount number of tables in the message - * @param payloadLength payload length (can be 0 if patched later) + * Returns true if Gorilla encoding is enabled. */ - public void writeHeader(int tableCount, int payloadLength) { - // Magic "ILP4" - buffer.putByte((byte) 'I'); - buffer.putByte((byte) 'L'); - buffer.putByte((byte) 'P'); - buffer.putByte((byte) '4'); - - // Version - buffer.putByte(VERSION_1); - - // Flags - buffer.putByte(flags); - - // Table count (uint16, little-endian) - buffer.putShort((short) tableCount); - - // Payload length (uint32, little-endian) - buffer.putInt(payloadLength); + public boolean isGorillaEnabled() { + return (flags & FLAG_GORILLA) != 0; } /** - * Encodes a single table from the buffer. + * Returns true if currently using an external buffer. */ - private void encodeTable(IlpV4TableBuffer tableBuffer, boolean useSchemaRef) { - IlpV4ColumnDef[] columnDefs = tableBuffer.getColumnDefs(); - int rowCount = tableBuffer.getRowCount(); - - if (useSchemaRef) { - writeTableHeaderWithSchemaRef( - tableBuffer.getTableName(), - rowCount, - tableBuffer.getSchemaHash(), - columnDefs.length - ); - } else { - writeTableHeaderWithSchema(tableBuffer.getTableName(), rowCount, columnDefs); - } + public boolean isUsingExternalBuffer() { + return buffer != ownedBuffer; + } - // Write each column's data - boolean useGorilla = isGorillaEnabled(); - for (int i = 0; i < tableBuffer.getColumnCount(); i++) { - IlpV4TableBuffer.ColumnBuffer col = tableBuffer.getColumn(i); - IlpV4ColumnDef colDef = columnDefs[i]; - encodeColumn(col, colDef, rowCount, useGorilla); + /** + * Resets the encoder for a new message. + *

+ * If using an external buffer, this only resets the internal state (flags). + * The external buffer's reset is the caller's responsibility. + * If using the internal buffer, resets both the buffer and internal state. + */ + public void reset() { + if (!isUsingExternalBuffer()) { + buffer.reset(); } } /** - * Encodes a single table from the buffer using global symbol IDs. - * This is used with delta dictionary encoding. + * Sets an external buffer for encoding. + *

+ * When set, the encoder writes directly to this buffer instead of its internal buffer. + * The caller is responsible for managing the external buffer's lifecycle. + *

+ * Pass {@code null} to revert to using the internal buffer. + * + * @param externalBuffer the external buffer to use, or null to use internal buffer */ - private void encodeTableWithGlobalSymbols(IlpV4TableBuffer tableBuffer, boolean useSchemaRef) { - IlpV4ColumnDef[] columnDefs = tableBuffer.getColumnDefs(); - int rowCount = tableBuffer.getRowCount(); + public void setBuffer(IlpBufferWriter externalBuffer) { + this.buffer = externalBuffer != null ? externalBuffer : ownedBuffer; + } - if (useSchemaRef) { - writeTableHeaderWithSchemaRef( - tableBuffer.getTableName(), - rowCount, - tableBuffer.getSchemaHash(), - columnDefs.length - ); + /** + * Sets the delta symbol dictionary flag. + */ + public void setDeltaSymbolDictEnabled(boolean enabled) { + if (enabled) { + flags |= FLAG_DELTA_SYMBOL_DICT; } else { - writeTableHeaderWithSchema(tableBuffer.getTableName(), rowCount, columnDefs); - } - - // Write each column's data - boolean useGorilla = isGorillaEnabled(); - for (int i = 0; i < tableBuffer.getColumnCount(); i++) { - IlpV4TableBuffer.ColumnBuffer col = tableBuffer.getColumn(i); - IlpV4ColumnDef colDef = columnDefs[i]; - encodeColumnWithGlobalSymbols(col, colDef, rowCount, useGorilla); + flags &= ~FLAG_DELTA_SYMBOL_DICT; } } /** - * Writes a table header with full schema. + * Sets whether Gorilla timestamp encoding is enabled. */ - private void writeTableHeaderWithSchema(String tableName, int rowCount, IlpV4ColumnDef[] columns) { - // Table name - buffer.putString(tableName); - - // Row count (varint) - buffer.putVarint(rowCount); - - // Column count (varint) - buffer.putVarint(columns.length); - - // Schema mode: full schema (0x00) - buffer.putByte(SCHEMA_MODE_FULL); - - // Column definitions (name + type for each) - for (IlpV4ColumnDef col : columns) { - buffer.putString(col.getName()); - buffer.putByte(col.getWireTypeCode()); + public void setGorillaEnabled(boolean enabled) { + if (enabled) { + flags |= FLAG_GORILLA; + } else { + flags &= ~FLAG_GORILLA; } } /** - * Writes a table header with schema reference. + * Writes the ILP v4 message header. + * + * @param tableCount number of tables in the message + * @param payloadLength payload length (can be 0 if patched later) */ - private void writeTableHeaderWithSchemaRef(String tableName, int rowCount, long schemaHash, int columnCount) { - // Table name - buffer.putString(tableName); + public void writeHeader(int tableCount, int payloadLength) { + // Magic "ILP4" + buffer.putByte((byte) 'I'); + buffer.putByte((byte) 'L'); + buffer.putByte((byte) 'P'); + buffer.putByte((byte) '4'); - // Row count (varint) - buffer.putVarint(rowCount); + // Version + buffer.putByte(VERSION_1); - // Column count (varint) - buffer.putVarint(columnCount); + // Flags + buffer.putByte(flags); - // Schema mode: reference (0x01) - buffer.putByte(SCHEMA_MODE_REFERENCE); + // Table count (uint16, little-endian) + buffer.putShort((short) tableCount); - // Schema hash (8 bytes) - buffer.putLong(schemaHash); + // Payload length (uint32, little-endian) + buffer.putInt(payloadLength); } /** @@ -513,20 +428,61 @@ private void encodeColumnWithGlobalSymbols(IlpV4TableBuffer.ColumnBuffer col, Il } /** - * Writes a null bitmap from bit-packed long array. + * Encodes a single table from the buffer. */ - private void writeNullBitmapPacked(long[] nullsPacked, int count) { - int bitmapSize = (count + 7) / 8; - - for (int byteIdx = 0; byteIdx < bitmapSize; byteIdx++) { - int longIndex = byteIdx >>> 3; - int byteInLong = byteIdx & 7; - byte b = (byte) ((nullsPacked[longIndex] >>> (byteInLong * 8)) & 0xFF); - buffer.putByte(b); - } - } + private void encodeTable(IlpV4TableBuffer tableBuffer, boolean useSchemaRef) { + IlpV4ColumnDef[] columnDefs = tableBuffer.getColumnDefs(); + int rowCount = tableBuffer.getRowCount(); - /** + if (useSchemaRef) { + writeTableHeaderWithSchemaRef( + tableBuffer.getTableName(), + rowCount, + tableBuffer.getSchemaHash(), + columnDefs.length + ); + } else { + writeTableHeaderWithSchema(tableBuffer.getTableName(), rowCount, columnDefs); + } + + // Write each column's data + boolean useGorilla = isGorillaEnabled(); + for (int i = 0; i < tableBuffer.getColumnCount(); i++) { + IlpV4TableBuffer.ColumnBuffer col = tableBuffer.getColumn(i); + IlpV4ColumnDef colDef = columnDefs[i]; + encodeColumn(col, colDef, rowCount, useGorilla); + } + } + + /** + * Encodes a single table from the buffer using global symbol IDs. + * This is used with delta dictionary encoding. + */ + private void encodeTableWithGlobalSymbols(IlpV4TableBuffer tableBuffer, boolean useSchemaRef) { + IlpV4ColumnDef[] columnDefs = tableBuffer.getColumnDefs(); + int rowCount = tableBuffer.getRowCount(); + + if (useSchemaRef) { + writeTableHeaderWithSchemaRef( + tableBuffer.getTableName(), + rowCount, + tableBuffer.getSchemaHash(), + columnDefs.length + ); + } else { + writeTableHeaderWithSchema(tableBuffer.getTableName(), rowCount, columnDefs); + } + + // Write each column's data + boolean useGorilla = isGorillaEnabled(); + for (int i = 0; i < tableBuffer.getColumnCount(); i++) { + IlpV4TableBuffer.ColumnBuffer col = tableBuffer.getColumn(i); + IlpV4ColumnDef colDef = columnDefs[i]; + encodeColumnWithGlobalSymbols(col, colDef, rowCount, useGorilla); + } + } + + /** * Writes boolean column data (bit-packed). */ private void writeBooleanColumn(boolean[] values, int count) { @@ -550,21 +506,58 @@ private void writeByteColumn(byte[] values, int count) { } } - private void writeShortColumn(short[] values, int count) { + private void writeDecimal128Column(byte scale, long[] high, long[] low, int count) { + buffer.putByte(scale); for (int i = 0; i < count; i++) { - buffer.putShort(values[i]); + buffer.putLongBE(high[i]); + buffer.putLongBE(low[i]); } } - private void writeIntColumn(int[] values, int count) { + private void writeDecimal256Column(byte scale, long[] hh, long[] hl, long[] lh, long[] ll, int count) { + buffer.putByte(scale); for (int i = 0; i < count; i++) { - buffer.putInt(values[i]); + buffer.putLongBE(hh[i]); + buffer.putLongBE(hl[i]); + buffer.putLongBE(lh[i]); + buffer.putLongBE(ll[i]); } } - private void writeLongColumn(long[] values, int count) { + private void writeDecimal64Column(byte scale, long[] values, int count) { + buffer.putByte(scale); for (int i = 0; i < count; i++) { - buffer.putLong(values[i]); + buffer.putLongBE(values[i]); + } + } + + private void writeDoubleArrayColumn(IlpV4TableBuffer.ColumnBuffer col, int count) { + byte[] dims = col.getArrayDims(); + int[] shapes = col.getArrayShapes(); + double[] data = col.getDoubleArrayData(); + + int shapeIdx = 0; + int dataIdx = 0; + for (int row = 0; row < count; row++) { + int nDims = dims[row]; + buffer.putByte((byte) nDims); + + int elemCount = 1; + for (int d = 0; d < nDims; d++) { + int dimLen = shapes[shapeIdx++]; + buffer.putInt(dimLen); + elemCount *= dimLen; + } + + for (int e = 0; e < elemCount; e++) { + buffer.putDouble(data[dataIdx++]); + } + } + } + + private void writeDoubleColumn(double[] values, int count) { + for (int i = 0; i < count; i++) { + buffer.putDouble(values[i]); } } @@ -574,42 +567,67 @@ private void writeFloatColumn(float[] values, int count) { } } - private void writeDoubleColumn(double[] values, int count) { + private void writeIntColumn(int[] values, int count) { for (int i = 0; i < count; i++) { - buffer.putDouble(values[i]); + buffer.putInt(values[i]); + } + } + + private void writeLong256Column(long[] values, int count) { + // Flat array: 4 longs per value, little-endian (least significant first) + // values layout: [long0, long1, long2, long3] per row + for (int i = 0; i < count * 4; i++) { + buffer.putLong(values[i]); + } + } + + private void writeLongArrayColumn(IlpV4TableBuffer.ColumnBuffer col, int count) { + byte[] dims = col.getArrayDims(); + int[] shapes = col.getArrayShapes(); + long[] data = col.getLongArrayData(); + + int shapeIdx = 0; + int dataIdx = 0; + for (int row = 0; row < count; row++) { + int nDims = dims[row]; + buffer.putByte((byte) nDims); + + int elemCount = 1; + for (int d = 0; d < nDims; d++) { + int dimLen = shapes[shapeIdx++]; + buffer.putInt(dimLen); + elemCount *= dimLen; + } + + for (int e = 0; e < elemCount; e++) { + buffer.putLong(data[dataIdx++]); + } + } + } + + private void writeLongColumn(long[] values, int count) { + for (int i = 0; i < count; i++) { + buffer.putLong(values[i]); } } /** - * Writes a timestamp column with optional Gorilla compression. - *

- * When Gorilla encoding is enabled and applicable (3+ timestamps with - * delta-of-deltas fitting in 32-bit range), uses delta-of-delta compression. - * Otherwise, falls back to uncompressed encoding. + * Writes a null bitmap from bit-packed long array. */ - private void writeTimestampColumn(long[] values, int count, boolean useGorilla) { - if (useGorilla && count > 2 && IlpV4GorillaEncoder.canUseGorilla(values, count)) { - // Write Gorilla encoding flag - buffer.putByte(IlpV4TimestampDecoder.ENCODING_GORILLA); + private void writeNullBitmapPacked(long[] nullsPacked, int count) { + int bitmapSize = (count + 7) / 8; - // Calculate size needed and ensure buffer has capacity - int encodedSize = IlpV4GorillaEncoder.calculateEncodedSize(values, count); - buffer.ensureCapacity(encodedSize); + for (int byteIdx = 0; byteIdx < bitmapSize; byteIdx++) { + int longIndex = byteIdx >>> 3; + int byteInLong = byteIdx & 7; + byte b = (byte) ((nullsPacked[longIndex] >>> (byteInLong * 8)) & 0xFF); + buffer.putByte(b); + } + } - // Encode timestamps to buffer - int bytesWritten = gorillaEncoder.encodeTimestamps( - buffer.getBufferPtr() + buffer.getPosition(), - buffer.getCapacity() - buffer.getPosition(), - values, - count - ); - buffer.skip(bytesWritten); - } else { - // Write uncompressed - if (useGorilla) { - buffer.putByte(IlpV4TimestampDecoder.ENCODING_UNCOMPRESSED); - } - writeLongColumn(values, count); + private void writeShortColumn(short[] values, int count) { + for (int i = 0; i < count; i++) { + buffer.putShort(values[i]); } } @@ -691,100 +709,87 @@ private void writeSymbolColumnWithGlobalIds(IlpV4TableBuffer.ColumnBuffer col, i } } - private void writeUuidColumn(long[] highBits, long[] lowBits, int count) { - // Little-endian: lo first, then hi - for (int i = 0; i < count; i++) { - buffer.putLong(lowBits[i]); - buffer.putLong(highBits[i]); - } - } - - private void writeLong256Column(long[] values, int count) { - // Flat array: 4 longs per value, little-endian (least significant first) - // values layout: [long0, long1, long2, long3] per row - for (int i = 0; i < count * 4; i++) { - buffer.putLong(values[i]); - } - } + /** + * Writes a table header with full schema. + */ + private void writeTableHeaderWithSchema(String tableName, int rowCount, IlpV4ColumnDef[] columns) { + // Table name + buffer.putString(tableName); - private void writeDoubleArrayColumn(IlpV4TableBuffer.ColumnBuffer col, int count) { - byte[] dims = col.getArrayDims(); - int[] shapes = col.getArrayShapes(); - double[] data = col.getDoubleArrayData(); + // Row count (varint) + buffer.putVarint(rowCount); - int shapeIdx = 0; - int dataIdx = 0; - for (int row = 0; row < count; row++) { - int nDims = dims[row]; - buffer.putByte((byte) nDims); + // Column count (varint) + buffer.putVarint(columns.length); - int elemCount = 1; - for (int d = 0; d < nDims; d++) { - int dimLen = shapes[shapeIdx++]; - buffer.putInt(dimLen); - elemCount *= dimLen; - } + // Schema mode: full schema (0x00) + buffer.putByte(SCHEMA_MODE_FULL); - for (int e = 0; e < elemCount; e++) { - buffer.putDouble(data[dataIdx++]); - } + // Column definitions (name + type for each) + for (IlpV4ColumnDef col : columns) { + buffer.putString(col.getName()); + buffer.putByte(col.getWireTypeCode()); } } - private void writeLongArrayColumn(IlpV4TableBuffer.ColumnBuffer col, int count) { - byte[] dims = col.getArrayDims(); - int[] shapes = col.getArrayShapes(); - long[] data = col.getLongArrayData(); + /** + * Writes a table header with schema reference. + */ + private void writeTableHeaderWithSchemaRef(String tableName, int rowCount, long schemaHash, int columnCount) { + // Table name + buffer.putString(tableName); - int shapeIdx = 0; - int dataIdx = 0; - for (int row = 0; row < count; row++) { - int nDims = dims[row]; - buffer.putByte((byte) nDims); + // Row count (varint) + buffer.putVarint(rowCount); - int elemCount = 1; - for (int d = 0; d < nDims; d++) { - int dimLen = shapes[shapeIdx++]; - buffer.putInt(dimLen); - elemCount *= dimLen; - } + // Column count (varint) + buffer.putVarint(columnCount); - for (int e = 0; e < elemCount; e++) { - buffer.putLong(data[dataIdx++]); - } - } - } + // Schema mode: reference (0x01) + buffer.putByte(SCHEMA_MODE_REFERENCE); - private void writeDecimal64Column(byte scale, long[] values, int count) { - buffer.putByte(scale); - for (int i = 0; i < count; i++) { - buffer.putLongBE(values[i]); - } + // Schema hash (8 bytes) + buffer.putLong(schemaHash); } - private void writeDecimal128Column(byte scale, long[] high, long[] low, int count) { - buffer.putByte(scale); - for (int i = 0; i < count; i++) { - buffer.putLongBE(high[i]); - buffer.putLongBE(low[i]); - } - } + /** + * Writes a timestamp column with optional Gorilla compression. + *

+ * When Gorilla encoding is enabled and applicable (3+ timestamps with + * delta-of-deltas fitting in 32-bit range), uses delta-of-delta compression. + * Otherwise, falls back to uncompressed encoding. + */ + private void writeTimestampColumn(long[] values, int count, boolean useGorilla) { + if (useGorilla && count > 2 && IlpV4GorillaEncoder.canUseGorilla(values, count)) { + // Write Gorilla encoding flag + buffer.putByte(ENCODING_GORILLA); - private void writeDecimal256Column(byte scale, long[] hh, long[] hl, long[] lh, long[] ll, int count) { - buffer.putByte(scale); - for (int i = 0; i < count; i++) { - buffer.putLongBE(hh[i]); - buffer.putLongBE(hl[i]); - buffer.putLongBE(lh[i]); - buffer.putLongBE(ll[i]); + // Calculate size needed and ensure buffer has capacity + int encodedSize = IlpV4GorillaEncoder.calculateEncodedSize(values, count); + buffer.ensureCapacity(encodedSize); + + // Encode timestamps to buffer + int bytesWritten = gorillaEncoder.encodeTimestamps( + buffer.getBufferPtr() + buffer.getPosition(), + buffer.getCapacity() - buffer.getPosition(), + values, + count + ); + buffer.skip(bytesWritten); + } else { + // Write uncompressed + if (useGorilla) { + buffer.putByte(ENCODING_UNCOMPRESSED); + } + writeLongColumn(values, count); } } - @Override - public void close() { - if (ownedBuffer != null) { - ownedBuffer.close(); - ownedBuffer = null; + private void writeUuidColumn(long[] highBits, long[] lowBits, int count) { + // Little-endian: lo first, then hi + for (int i = 0; i < count; i++) { + buffer.putLong(lowBits[i]); + buffer.putLong(highBits[i]); } } } diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4GorillaDecoder.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4GorillaDecoder.java deleted file mode 100644 index 2471ad4..0000000 --- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4GorillaDecoder.java +++ /dev/null @@ -1,251 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2024 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.cutlass.ilpv4.protocol; - -/** - * Gorilla delta-of-delta decoder for timestamps in ILP v4 format. - *

- * Gorilla encoding uses delta-of-delta compression where: - * 

- * D = (t[n] - t[n-1]) - (t[n-1] - t[n-2])
- *
- * if D == 0:              write '0'              (1 bit)
- * elif D in [-63, 64]:    write '10' + 7-bit     (9 bits)
- * elif D in [-255, 256]:  write '110' + 9-bit    (12 bits)
- * elif D in [-2047, 2048]: write '1110' + 12-bit (16 bits)
- * else:                   write '1111' + 32-bit  (36 bits)
- *
- *

- * The decoder reads bit-packed delta-of-delta values and reconstructs - * the original timestamp sequence. - */ -public class IlpV4GorillaDecoder { - - // Bucket boundaries (two's complement signed ranges) - private static final int BUCKET_7BIT_MIN = -63; - private static final int BUCKET_7BIT_MAX = 64; - private static final int BUCKET_9BIT_MIN = -255; - private static final int BUCKET_9BIT_MAX = 256; - private static final int BUCKET_12BIT_MIN = -2047; - private static final int BUCKET_12BIT_MAX = 2048; - - private final IlpV4BitReader bitReader; - - // State for decoding - private long prevTimestamp; - private long prevDelta; - - /** - * Creates a new Gorilla decoder. - */ - public IlpV4GorillaDecoder() { - this.bitReader = new IlpV4BitReader(); - } - - /** - * Creates a decoder using an existing bit reader. - * - * @param bitReader the bit reader to use - */ - public IlpV4GorillaDecoder(IlpV4BitReader bitReader) { - this.bitReader = bitReader; - } - - /** - * Resets the decoder with the first two timestamps. - *

- * The first two timestamps are always stored uncompressed and are used - * to establish the initial delta for subsequent compression. - * - * @param firstTimestamp the first timestamp in the sequence - * @param secondTimestamp the second timestamp in the sequence - */ - public void reset(long firstTimestamp, long secondTimestamp) { - this.prevTimestamp = secondTimestamp; - this.prevDelta = secondTimestamp - firstTimestamp; - } - - /** - * Resets the bit reader for reading encoded delta-of-deltas. - * - * @param address the address of the encoded data - * @param length the length of the encoded data in bytes - */ - public void resetReader(long address, long length) { - bitReader.reset(address, length); - } - - /** - * Decodes the next timestamp from the bit stream. - *

- * The encoding format is: - *

    - *
  • '0' = delta-of-delta is 0 (1 bit)
    • - *
  • '10' + 7-bit signed = delta-of-delta in [-63, 64] (9 bits)
    • - *
  • '110' + 9-bit signed = delta-of-delta in [-255, 256] (12 bits)
    • - *
  • '1110' + 12-bit signed = delta-of-delta in [-2047, 2048] (16 bits)
    • - *
  • '1111' + 32-bit signed = any other delta-of-delta (36 bits)
    • - *
- * - * @return the decoded timestamp - */ - public long decodeNext() { - long deltaOfDelta = decodeDoD(); - long delta = prevDelta + deltaOfDelta; - long timestamp = prevTimestamp + delta; - - prevDelta = delta; - prevTimestamp = timestamp; - - return timestamp; - } - - /** - * Decodes a delta-of-delta value from the bit stream. - * - * @return the delta-of-delta value - */ - private long decodeDoD() { - int bit = bitReader.readBit(); - - if (bit == 0) { - // '0' = DoD is 0 - return 0; - } - - // bit == 1, check next bit - bit = bitReader.readBit(); - if (bit == 0) { - // '10' = 7-bit signed value - return bitReader.readSigned(7); - } - - // '11', check next bit - bit = bitReader.readBit(); - if (bit == 0) { - // '110' = 9-bit signed value - return bitReader.readSigned(9); - } - - // '111', check next bit - bit = bitReader.readBit(); - if (bit == 0) { - // '1110' = 12-bit signed value - return bitReader.readSigned(12); - } - - // '1111' = 32-bit signed value - return bitReader.readSigned(32); - } - - /** - * Returns whether there are more bits available in the reader. - * - * @return true if more bits available - */ - public boolean hasMoreBits() { - return bitReader.hasMoreBits(); - } - - /** - * Returns the number of bits remaining. - * - * @return available bits - */ - public long getAvailableBits() { - return bitReader.getAvailableBits(); - } - - /** - * Returns the current bit position (bits read since reset). - * - * @return bits read - */ - public long getBitPosition() { - return bitReader.getBitPosition(); - } - - /** - * Gets the previous timestamp (for debugging/testing). - * - * @return the last decoded timestamp - */ - public long getPrevTimestamp() { - return prevTimestamp; - } - - /** - * Gets the previous delta (for debugging/testing). - * - * @return the last computed delta - */ - public long getPrevDelta() { - return prevDelta; - } - - // ==================== Static Encoding Methods (for testing) ==================== - - /** - * Determines which bucket a delta-of-delta value falls into. - * - * @param deltaOfDelta the delta-of-delta value - * @return bucket number (0 = 1-bit, 1 = 9-bit, 2 = 12-bit, 3 = 16-bit, 4 = 36-bit) - */ - public static int getBucket(long deltaOfDelta) { - if (deltaOfDelta == 0) { - return 0; // 1-bit - } else if (deltaOfDelta >= BUCKET_7BIT_MIN && deltaOfDelta <= BUCKET_7BIT_MAX) { - return 1; // 9-bit (2 prefix + 7 value) - } else if (deltaOfDelta >= BUCKET_9BIT_MIN && deltaOfDelta <= BUCKET_9BIT_MAX) { - return 2; // 12-bit (3 prefix + 9 value) - } else if (deltaOfDelta >= BUCKET_12BIT_MIN && deltaOfDelta <= BUCKET_12BIT_MAX) { - return 3; // 16-bit (4 prefix + 12 value) - } else { - return 4; // 36-bit (4 prefix + 32 value) - } - } - - /** - * Returns the number of bits required to encode a delta-of-delta value. - * - * @param deltaOfDelta the delta-of-delta value - * @return bits required - */ - public static int getBitsRequired(long deltaOfDelta) { - int bucket = getBucket(deltaOfDelta); - switch (bucket) { - case 0: - return 1; - case 1: - return 9; - case 2: - return 12; - case 3: - return 16; - default: - return 36; - } - } -} diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4GorillaEncoder.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4GorillaEncoder.java index e8f0f47..84e4334 100644 --- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4GorillaEncoder.java +++ b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4GorillaEncoder.java @@ -46,6 +46,13 @@ */ public class IlpV4GorillaEncoder { + private static final int BUCKET_12BIT_MAX = 2048; + private static final int BUCKET_12BIT_MIN = -2047; + private static final int BUCKET_7BIT_MAX = 64; + // Bucket boundaries (two's complement signed ranges) + private static final int BUCKET_7BIT_MIN = -63; + private static final int BUCKET_9BIT_MAX = 256; + private static final int BUCKET_9BIT_MIN = -255; private final IlpV4BitWriter bitWriter = new IlpV4BitWriter(); /** @@ -54,6 +61,48 @@ public class IlpV4GorillaEncoder { public IlpV4GorillaEncoder() { } + /** + * Returns the number of bits required to encode a delta-of-delta value. + * + * @param deltaOfDelta the delta-of-delta value + * @return bits required + */ + public static int getBitsRequired(long deltaOfDelta) { + int bucket = getBucket(deltaOfDelta); + switch (bucket) { + case 0: + return 1; + case 1: + return 9; + case 2: + return 12; + case 3: + return 16; + default: + return 36; + } + } + + /** + * Determines which bucket a delta-of-delta value falls into. + * + * @param deltaOfDelta the delta-of-delta value + * @return bucket number (0 = 1-bit, 1 = 9-bit, 2 = 12-bit, 3 = 16-bit, 4 = 36-bit) + */ + public static int getBucket(long deltaOfDelta) { + if (deltaOfDelta == 0) { + return 0; // 1-bit + } else if (deltaOfDelta >= BUCKET_7BIT_MIN && deltaOfDelta <= BUCKET_7BIT_MAX) { + return 1; // 9-bit (2 prefix + 7 value) + } else if (deltaOfDelta >= BUCKET_9BIT_MIN && deltaOfDelta <= BUCKET_9BIT_MAX) { + return 2; // 12-bit (3 prefix + 9 value) + } else if (deltaOfDelta >= BUCKET_12BIT_MIN && deltaOfDelta <= BUCKET_12BIT_MAX) { + return 3; // 16-bit (4 prefix + 12 value) + } else { + return 4; // 36-bit (4 prefix + 32 value) + } + } + /** * Encodes a delta-of-delta value using bucket selection. *

@@ -69,7 +118,7 @@ public IlpV4GorillaEncoder() { * @param deltaOfDelta the delta-of-delta value to encode */ public void encodeDoD(long deltaOfDelta) { - int bucket = IlpV4GorillaDecoder.getBucket(deltaOfDelta); + int bucket = getBucket(deltaOfDelta); switch (bucket) { case 0: // DoD == 0 bitWriter.writeBit(0); @@ -221,7 +270,7 @@ public static int calculateEncodedSize(long[] timestamps, int count) { long delta = timestamps[i] - prevTimestamp; long deltaOfDelta = delta - prevDelta; - totalBits += IlpV4GorillaDecoder.getBitsRequired(deltaOfDelta); + totalBits += getBitsRequired(deltaOfDelta); prevDelta = delta; prevTimestamp = timestamps[i]; diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4TimestampDecoder.java b/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4TimestampDecoder.java deleted file mode 100644 index 3b4fffa..0000000 --- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4TimestampDecoder.java +++ /dev/null @@ -1,474 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2024 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.cutlass.ilpv4.protocol; - -import io.questdb.client.std.Unsafe; - -/** - * Decoder for TIMESTAMP columns in ILP v4 format. - *

- * Supports two encoding modes: - *

    - *
  • Uncompressed (0x00): array of int64 values
    • - *
  • Gorilla (0x01): delta-of-delta compressed
    • - *
- *

- * Gorilla format: - * 

- * [Null bitmap if nullable]
- * First timestamp: int64 (8 bytes, little-endian)
- * Second timestamp: int64 (8 bytes, little-endian)
- * Remaining timestamps: bit-packed delta-of-delta
- *
- */ -public final class IlpV4TimestampDecoder { - - /** - * Encoding flag for uncompressed timestamps. - */ - public static final byte ENCODING_UNCOMPRESSED = 0x00; - - /** - * Encoding flag for Gorilla-encoded timestamps. - */ - public static final byte ENCODING_GORILLA = 0x01; - - public static final IlpV4TimestampDecoder INSTANCE = new IlpV4TimestampDecoder(); - - private final IlpV4GorillaDecoder gorillaDecoder = new IlpV4GorillaDecoder(); - - private IlpV4TimestampDecoder() { - } - - /** - * Decodes timestamp column data from native memory. - * - * @param sourceAddress source address in native memory - * @param sourceLength length of source data in bytes - * @param rowCount number of rows to decode - * @param nullable whether the column is nullable - * @param sink sink to receive decoded values - * @return number of bytes consumed - */ - public int decode(long sourceAddress, int sourceLength, int rowCount, boolean nullable, ColumnSink sink) { - if (rowCount == 0) { - return 0; - } - - int offset = 0; - - // Parse null bitmap if nullable - long nullBitmapAddress = 0; - if (nullable) { - int nullBitmapSize = IlpV4NullBitmap.sizeInBytes(rowCount); - if (offset + nullBitmapSize > sourceLength) { - throw new IllegalArgumentException("insufficient data for null bitmap"); - } - nullBitmapAddress = sourceAddress + offset; - offset += nullBitmapSize; - } - - // Read encoding flag - if (offset + 1 > sourceLength) { - throw new IllegalArgumentException("insufficient data for encoding flag"); - } - byte encoding = Unsafe.getUnsafe().getByte(sourceAddress + offset); - offset++; - - if (encoding == ENCODING_UNCOMPRESSED) { - offset = decodeUncompressed(sourceAddress, sourceLength, offset, rowCount, nullable, nullBitmapAddress, sink); - } else if (encoding == ENCODING_GORILLA) { - offset = decodeGorilla(sourceAddress, sourceLength, offset, rowCount, nullable, nullBitmapAddress, sink); - } else { - throw new IllegalArgumentException("unknown timestamp encoding: " + encoding); - } - - return offset; - } - - private int decodeUncompressed(long sourceAddress, int sourceLength, int offset, int rowCount, - boolean nullable, long nullBitmapAddress, ColumnSink sink) { - // Count nulls to determine actual value count - int nullCount = 0; - if (nullable) { - nullCount = IlpV4NullBitmap.countNulls(nullBitmapAddress, rowCount); - } - int valueCount = rowCount - nullCount; - - // Uncompressed: valueCount * 8 bytes - int valuesSize = valueCount * 8; - if (offset + valuesSize > sourceLength) { - throw new IllegalArgumentException("insufficient data for uncompressed timestamps"); - } - - long valuesAddress = sourceAddress + offset; - int valueOffset = 0; - for (int i = 0; i < rowCount; i++) { - if (nullable && IlpV4NullBitmap.isNull(nullBitmapAddress, i)) { - sink.putNull(i); - } else { - long value = Unsafe.getUnsafe().getLong(valuesAddress + (long) valueOffset * 8); - sink.putLong(i, value); - valueOffset++; - } - } - - return offset + valuesSize; - } - - private int decodeGorilla(long sourceAddress, int sourceLength, int offset, int rowCount, - boolean nullable, long nullBitmapAddress, ColumnSink sink) { - // Count nulls to determine actual value count - int nullCount = 0; - if (nullable) { - nullCount = IlpV4NullBitmap.countNulls(nullBitmapAddress, rowCount); - } - int valueCount = rowCount - nullCount; - - if (valueCount == 0) { - // All nulls - for (int i = 0; i < rowCount; i++) { - sink.putNull(i); - } - return offset; - } - - // First timestamp: 8 bytes - if (offset + 8 > sourceLength) { - throw new IllegalArgumentException("insufficient data for first timestamp"); - } - long firstTimestamp = Unsafe.getUnsafe().getLong(sourceAddress + offset); - offset += 8; - - if (valueCount == 1) { - // Only one non-null value, output it at the appropriate row position - for (int i = 0; i < rowCount; i++) { - if (nullable && IlpV4NullBitmap.isNull(nullBitmapAddress, i)) { - sink.putNull(i); - } else { - sink.putLong(i, firstTimestamp); - } - } - return offset; - } - - // Second timestamp: 8 bytes - if (offset + 8 > sourceLength) { - throw new IllegalArgumentException("insufficient data for second timestamp"); - } - long secondTimestamp = Unsafe.getUnsafe().getLong(sourceAddress + offset); - offset += 8; - - if (valueCount == 2) { - // Two non-null values - int valueIdx = 0; - for (int i = 0; i < rowCount; i++) { - if (nullable && IlpV4NullBitmap.isNull(nullBitmapAddress, i)) { - sink.putNull(i); - } else { - sink.putLong(i, valueIdx == 0 ? firstTimestamp : secondTimestamp); - valueIdx++; - } - } - return offset; - } - - // Remaining timestamps: bit-packed delta-of-delta - // Reset the Gorilla decoder with the initial state - gorillaDecoder.reset(firstTimestamp, secondTimestamp); - - // Calculate remaining bytes for bit data - int remainingBytes = sourceLength - offset; - gorillaDecoder.resetReader(sourceAddress + offset, remainingBytes); - - // Decode timestamps and distribute to rows - int valueIdx = 0; - for (int i = 0; i < rowCount; i++) { - if (nullable && IlpV4NullBitmap.isNull(nullBitmapAddress, i)) { - sink.putNull(i); - } else { - long timestamp; - if (valueIdx == 0) { - timestamp = firstTimestamp; - } else if (valueIdx == 1) { - timestamp = secondTimestamp; - } else { - timestamp = gorillaDecoder.decodeNext(); - } - sink.putLong(i, timestamp); - valueIdx++; - } - } - - // Calculate how many bytes were consumed - // The bit reader has consumed some bits; round up to bytes - long bitsRead = gorillaDecoder.getAvailableBits(); - long totalBits = remainingBytes * 8L; - long bitsConsumed = totalBits - bitsRead; - int bytesConsumed = (int) ((bitsConsumed + 7) / 8); - - return offset + bytesConsumed; - } - - /** - * Returns the expected size for decoding. - * - * @param rowCount number of rows - * @param nullable whether the column is nullable - * @return expected size in bytes - */ - public int expectedSize(int rowCount, boolean nullable) { - // Minimum size: just encoding flag + uncompressed timestamps - int size = 1; // encoding flag - if (nullable) { - size += IlpV4NullBitmap.sizeInBytes(rowCount); - } - size += rowCount * 8; // worst case: uncompressed - return size; - } - - // ==================== Static Encoding Methods (for testing) ==================== - - /** - * Encodes timestamps in uncompressed format to direct memory. - * Only non-null values are written. - * - * @param destAddress destination address - * @param timestamps timestamp values - * @param nulls null flags (can be null if not nullable) - * @return address after encoded data - */ - public static long encodeUncompressed(long destAddress, long[] timestamps, boolean[] nulls) { - int rowCount = timestamps.length; - boolean nullable = nulls != null; - long pos = destAddress; - - // Write null bitmap if nullable - if (nullable) { - int bitmapSize = IlpV4NullBitmap.sizeInBytes(rowCount); - IlpV4NullBitmap.fillNoneNull(pos, rowCount); - for (int i = 0; i < rowCount; i++) { - if (nulls[i]) { - IlpV4NullBitmap.setNull(pos, i); - } - } - pos += bitmapSize; - } - - // Write encoding flag - Unsafe.getUnsafe().putByte(pos++, ENCODING_UNCOMPRESSED); - - // Write only non-null timestamps - for (int i = 0; i < rowCount; i++) { - if (nullable && nulls[i]) continue; - Unsafe.getUnsafe().putLong(pos, timestamps[i]); - pos += 8; - } - - return pos; - } - - /** - * Encodes timestamps in Gorilla format to direct memory. - * Only non-null values are encoded. - * - * @param destAddress destination address - * @param timestamps timestamp values - * @param nulls null flags (can be null if not nullable) - * @return address after encoded data - */ - public static long encodeGorilla(long destAddress, long[] timestamps, boolean[] nulls) { - int rowCount = timestamps.length; - boolean nullable = nulls != null; - long pos = destAddress; - - // Write null bitmap if nullable - if (nullable) { - int bitmapSize = IlpV4NullBitmap.sizeInBytes(rowCount); - IlpV4NullBitmap.fillNoneNull(pos, rowCount); - for (int i = 0; i < rowCount; i++) { - if (nulls[i]) { - IlpV4NullBitmap.setNull(pos, i); - } - } - pos += bitmapSize; - } - - // Count non-null values - int valueCount = 0; - for (int i = 0; i < rowCount; i++) { - if (!nullable || !nulls[i]) valueCount++; - } - - // Write encoding flag - Unsafe.getUnsafe().putByte(pos++, ENCODING_GORILLA); - - if (valueCount == 0) { - return pos; - } - - // Build array of non-null values - long[] nonNullValues = new long[valueCount]; - int idx = 0; - for (int i = 0; i < rowCount; i++) { - if (nullable && nulls[i]) continue; - nonNullValues[idx++] = timestamps[i]; - } - - // Write first timestamp - Unsafe.getUnsafe().putLong(pos, nonNullValues[0]); - pos += 8; - - if (valueCount == 1) { - return pos; - } - - // Write second timestamp - Unsafe.getUnsafe().putLong(pos, nonNullValues[1]); - pos += 8; - - if (valueCount == 2) { - return pos; - } - - // Encode remaining timestamps using Gorilla - IlpV4BitWriter bitWriter = new IlpV4BitWriter(); - bitWriter.reset(pos, 1024 * 1024); // 1MB max for bit data - - long prevTimestamp = nonNullValues[1]; - long prevDelta = nonNullValues[1] - nonNullValues[0]; - - for (int i = 2; i < valueCount; i++) { - long delta = nonNullValues[i] - prevTimestamp; - long deltaOfDelta = delta - prevDelta; - - encodeDoD(bitWriter, deltaOfDelta); - - prevDelta = delta; - prevTimestamp = nonNullValues[i]; - } - - // Flush remaining bits - int bytesWritten = bitWriter.finish(); - pos += bytesWritten; - - return pos; - } - - /** - * Encodes a delta-of-delta value to the bit writer. - *

- * Prefix patterns are written LSB-first to match the decoder's read order: - * - '0' -> write bit 0 - * - '10' -> write bit 1, then bit 0 (0b01 as 2-bit value) - * - '110' -> write bit 1, bit 1, bit 0 (0b011 as 3-bit value) - * - '1110' -> write bit 1, bit 1, bit 1, bit 0 (0b0111 as 4-bit value) - * - '1111' -> write bit 1, bit 1, bit 1, bit 1 (0b1111 as 4-bit value) - */ - private static void encodeDoD(IlpV4BitWriter writer, long deltaOfDelta) { - if (deltaOfDelta == 0) { - // '0' = DoD is 0 - writer.writeBit(0); - } else if (deltaOfDelta >= -63 && deltaOfDelta <= 64) { - // '10' prefix: first bit read=1, second bit read=0 -> write as 0b01 (LSB-first) - writer.writeBits(0b01, 2); - writer.writeSigned(deltaOfDelta, 7); - } else if (deltaOfDelta >= -255 && deltaOfDelta <= 256) { - // '110' prefix: bits read as 1,1,0 -> write as 0b011 (LSB-first) - writer.writeBits(0b011, 3); - writer.writeSigned(deltaOfDelta, 9); - } else if (deltaOfDelta >= -2047 && deltaOfDelta <= 2048) { - // '1110' prefix: bits read as 1,1,1,0 -> write as 0b0111 (LSB-first) - writer.writeBits(0b0111, 4); - writer.writeSigned(deltaOfDelta, 12); - } else { - // '1111' prefix: bits read as 1,1,1,1 -> write as 0b1111 (LSB-first) - writer.writeBits(0b1111, 4); - writer.writeSigned(deltaOfDelta, 32); - } - } - - /** - * Calculates the encoded size in bytes for Gorilla-encoded timestamps. - * - * @param timestamps timestamp values - * @param nullable whether column is nullable - * @return encoded size in bytes - */ - public static int calculateGorillaSize(long[] timestamps, boolean nullable) { - int rowCount = timestamps.length; - int size = 0; - - if (nullable) { - size += IlpV4NullBitmap.sizeInBytes(rowCount); - } - - size += 1; // encoding flag - - if (rowCount == 0) { - return size; - } - - size += 8; // first timestamp - - if (rowCount == 1) { - return size; - } - - size += 8; // second timestamp - - if (rowCount == 2) { - return size; - } - - // Calculate bits for delta-of-delta encoding - long prevTimestamp = timestamps[1]; - long prevDelta = timestamps[1] - timestamps[0]; - int totalBits = 0; - - for (int i = 2; i < rowCount; i++) { - long delta = timestamps[i] - prevTimestamp; - long deltaOfDelta = delta - prevDelta; - - totalBits += IlpV4GorillaDecoder.getBitsRequired(deltaOfDelta); - - prevDelta = delta; - prevTimestamp = timestamps[i]; - } - - // Round up to bytes - size += (totalBits + 7) / 8; - - return size; - } - - /** - * Sink interface for receiving decoded column values. - */ - public interface ColumnSink { - void putLong(int rowIndex, long value); - void putNull(int rowIndex); - } -} diff --git a/core/src/test/java/io/questdb/client/test/LineSenderBuilderTest.java b/core/src/test/java/io/questdb/client/test/LineSenderBuilderTest.java deleted file mode 100644 index fe72c01..0000000 --- a/core/src/test/java/io/questdb/client/test/LineSenderBuilderTest.java +++ /dev/null @@ -1,799 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.test; - -import io.questdb.client.Sender; -import io.questdb.client.cutlass.line.LineSenderException; -import io.questdb.client.test.tools.TestUtils; -import org.junit.Assert; -import org.junit.Test; - -import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; -import static org.junit.Assert.fail; - -/** - * Unit tests for LineSenderBuilder that don't require a running QuestDB instance. - * Tests that require an actual QuestDB connection have been moved to integration tests. - */ -public class LineSenderBuilderTest { - private static final String AUTH_TOKEN_KEY1 = "UvuVb1USHGRRT08gEnwN2zGZrvM4MsLQ5brgF6SVkAw="; - private static final String LOCALHOST = "localhost"; - private static final char[] TRUSTSTORE_PASSWORD = "questdb".toCharArray(); - private static final String TRUSTSTORE_PATH = "/keystore/server.keystore"; - - @Test - public void testAddressDoubleSet_firstAddressThenAddress() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP).address(LOCALHOST); - try { - builder.address("127.0.0.1"); - builder.build(); - fail("should not allow double host set"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "mismatch"); - } - }); - } - - @Test - public void testAddressEmpty() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP); - try { - builder.address(""); - fail("empty address should fail"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "address cannot be empty"); - } - }); - } - - @Test - public void testAddressEndsWithColon() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP); - try { - builder.address("foo:"); - fail("should fail when address ends with colon"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "invalid address"); - } - }); - } - - @Test - public void testAddressNull() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP); - try { - builder.address(null); - fail("null address should fail"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "null"); - } - }); - } - - @Test - public void testAuthDoubleSet() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP).enableAuth("foo").authToken(AUTH_TOKEN_KEY1); - try { - builder.enableAuth("bar"); - fail("should not allow double auth set"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "already configured"); - } - }); - } - - @Test - public void testAuthTooSmallBuffer() throws Exception { - assertMemoryLeak(() -> { - try { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP) - .enableAuth("foo").authToken(AUTH_TOKEN_KEY1).address(LOCALHOST + ":9001") - .bufferCapacity(1); - builder.build(); - fail("tiny buffer should NOT be allowed as it wont fit auth challenge"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "minimalCapacity"); - TestUtils.assertContains(e.getMessage(), "requestedCapacity"); - } - }); - } - - @Test - public void testAuthWithBadToken() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder.AuthBuilder builder = Sender.builder(Sender.Transport.TCP).enableAuth("foo"); - try { - builder.authToken("bar token"); - fail("bad token should not be imported"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "could not import token"); - } - }); - } - - @Test - public void testAutoFlushIntervalMustBePositive() { - try (Sender ignored = Sender.builder(Sender.Transport.HTTP).autoFlushIntervalMillis(0).build()) { - fail("auto-flush must be positive"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "auto flush interval cannot be negative [autoFlushIntervalMillis=0]"); - } - - try (Sender ignored = Sender.builder(Sender.Transport.HTTP).autoFlushIntervalMillis(-1).build()) { - fail("auto-flush must be positive"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "auto flush interval cannot be negative [autoFlushIntervalMillis=-1]"); - } - } - - @Test - public void testAutoFlushIntervalNotSupportedForTcp() throws Exception { - assertMemoryLeak(() -> { - try { - Sender.builder(Sender.Transport.TCP).address(LOCALHOST).autoFlushIntervalMillis(1).build(); - fail("auto flush interval should not be supported for TCP"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "auto flush interval is not supported for TCP protocol"); - } - }); - } - - @Test - public void testAutoFlushInterval_afterAutoFlushDisabled() throws Exception { - assertMemoryLeak(() -> { - try { - Sender.builder(Sender.Transport.HTTP).disableAutoFlush().autoFlushIntervalMillis(1); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "cannot set auto flush interval when interval based auto-flush is already disabled"); - } - }); - } - - @Test - public void testAutoFlushInterval_doubleConfiguration() throws Exception { - assertMemoryLeak(() -> { - try { - Sender.builder(Sender.Transport.HTTP).autoFlushIntervalMillis(1).autoFlushIntervalMillis(1); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "auto flush interval was already configured [autoFlushIntervalMillis=1]"); - } - }); - } - - @Test - public void testAutoFlushRowsCannotBeNegative() { - try (Sender ignored = Sender.builder(Sender.Transport.HTTP).autoFlushRows(-1).build()) { - fail("auto-flush must be positive"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "auto flush rows cannot be negative [autoFlushRows=-1]"); - } - } - - @Test - public void testAutoFlushRowsNotSupportedForTcp() throws Exception { - assertMemoryLeak(() -> { - try { - Sender.builder(Sender.Transport.TCP).address(LOCALHOST).autoFlushRows(1).build(); - fail("auto flush rows should not be supported for TCP"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "auto flush rows is not supported for TCP protocol"); - } - }); - } - - @Test - public void testAutoFlushRows_doubleConfiguration() throws Exception { - assertMemoryLeak(() -> { - try { - Sender.builder(Sender.Transport.HTTP).autoFlushRows(1).autoFlushRows(1); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "auto flush rows was already configured [autoFlushRows=1]"); - } - }); - } - - @Test - public void testBufferSizeDoubleSet() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP).bufferCapacity(1024); - try { - builder.bufferCapacity(1024); - fail("should not allow double buffer capacity set"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "already configured"); - } - }); - } - - @Test - public void testConfStringValidation() throws Exception { - assertMemoryLeak(() -> { - assertConfStrError("foo", "invalid schema [schema=foo, supported-schemas=[http, https, tcp, tcps]]"); - assertConfStrError("badschema::addr=bar;", "invalid schema [schema=badschema, supported-schemas=[http, https, tcp, tcps]]"); - assertConfStrError("http::addr=localhost:-1;", "invalid port [port=-1]"); - assertConfStrError("http::auto_flush=on;", "addr is missing"); - assertConfStrError("http::addr=localhost;tls_roots=/some/path;", "tls_roots was configured, but tls_roots_password is missing"); - assertConfStrError("http::addr=localhost;tls_roots_password=hunter123;", "tls_roots_password was configured, but tls_roots is missing"); - assertConfStrError("tcp::addr=localhost;user=foo;", "token cannot be empty nor null"); - assertConfStrError("tcp::addr=localhost;username=foo;", "token cannot be empty nor null"); - assertConfStrError("tcp::addr=localhost;token=foo;", "TCP token is configured, but user is missing"); - assertConfStrError("http::addr=localhost;user=foo;", "password cannot be empty nor null"); - assertConfStrError("http::addr=localhost;username=foo;", "password cannot be empty nor null"); - assertConfStrError("http::addr=localhost;pass=foo;", "HTTP password is configured, but username is missing"); - assertConfStrError("http::addr=localhost;password=foo;", "HTTP password is configured, but username is missing"); - assertConfStrError("tcp::addr=localhost;pass=foo;", "password is not supported for TCP protocol"); - assertConfStrError("tcp::addr=localhost;password=foo;", "password is not supported for TCP protocol"); - assertConfStrError("tcp::addr=localhost;retry_timeout=;", "retry_timeout cannot be empty"); - assertConfStrError("tcp::addr=localhost;max_buf_size=;", "max_buf_size cannot be empty"); - assertConfStrError("tcp::addr=localhost;init_buf_size=;", "init_buf_size cannot be empty"); - assertConfStrError("http::addr=localhost:8080;tls_verify=unsafe_off;", "TLS validation disabled, but TLS was not enabled"); - assertConfStrError("http::addr=localhost:8080;tls_verify=bad;", "invalid tls_verify [value=bad, allowed-values=[on, unsafe_off]]"); - assertConfStrError("tcps::addr=localhost;pass=unsafe_off;", "password is not supported for TCP protocol"); - assertConfStrError("tcps::addr=localhost;password=unsafe_off;", "password is not supported for TCP protocol"); - assertConfStrError("http::addr=localhost:8080;max_buf_size=-32;", "maximum buffer capacity cannot be less than initial buffer capacity [maximumBufferCapacity=-32, initialBufferCapacity=65536]"); - assertConfStrError("http::addr=localhost:8080;max_buf_size=notanumber;", "invalid max_buf_size [value=notanumber]"); - assertConfStrError("http::addr=localhost:8080;init_buf_size=notanumber;", "invalid init_buf_size [value=notanumber]"); - assertConfStrError("http::addr=localhost:8080;init_buf_size=-42;", "buffer capacity cannot be negative [capacity=-42]"); - assertConfStrError("http::addr=localhost:8080;auto_flush_rows=0;", "invalid auto_flush_rows [value=0]"); - assertConfStrError("http::addr=localhost:8080;auto_flush_rows=notanumber;", "invalid auto_flush_rows [value=notanumber]"); - assertConfStrError("http::addr=localhost:8080;auto_flush=invalid;", "invalid auto_flush [value=invalid, allowed-values=[on, off]]"); - assertConfStrError("http::addr=localhost:8080;auto_flush=off;auto_flush_rows=100;", "cannot set auto flush rows when auto-flush is already disabled"); - assertConfStrError("http::addr=localhost:8080;auto_flush_rows=100;auto_flush=off;", "auto flush rows was already configured [autoFlushRows=100]"); - assertConfStrError("HTTP::addr=localhost;", "invalid schema [schema=HTTP, supported-schemas=[http, https, tcp, tcps]]"); - assertConfStrError("HTTPS::addr=localhost;", "invalid schema [schema=HTTPS, supported-schemas=[http, https, tcp, tcps]]"); - assertConfStrError("TCP::addr=localhost;", "invalid schema [schema=TCP, supported-schemas=[http, https, tcp, tcps]]"); - assertConfStrError("TCPS::addr=localhost;", "invalid schema [schema=TCPS, supported-schemas=[http, https, tcp, tcps]]"); - assertConfStrError("http::addr=localhost;auto_flush=off;auto_flush_interval=1;", "cannot set auto flush interval when interval based auto-flush is already disabled"); - assertConfStrError("http::addr=localhost;auto_flush=off;auto_flush_rows=1;", "cannot set auto flush rows when auto-flush is already disabled"); - assertConfStrError("http::addr=localhost;auto_flush_bytes=1024;", "auto_flush_bytes is only supported for TCP transport"); - assertConfStrError("http::addr=localhost;protocol_version=10", "current client only supports protocol version 1(text format for all datatypes), 2(binary format for part datatypes), 3(decimal datatype) or explicitly unset"); - assertConfStrError("http::addr=localhost:48884;max_name_len=10;", "max_name_len must be at least 16 bytes [max_name_len=10]"); - - assertConfStrOk("addr=localhost:8080", "auto_flush_rows=100", "protocol_version=1"); - assertConfStrOk("addr=localhost:8080", "auto_flush=on", "auto_flush_rows=100", "protocol_version=2"); - assertConfStrOk("addr=localhost:8080", "auto_flush_rows=100", "auto_flush=on", "protocol_version=2"); - assertConfStrOk("addr=localhost", "auto_flush=on", "protocol_version=2"); - assertConfStrOk("addr=localhost:8080", "max_name_len=1024", "protocol_version=2"); - - assertConfStrError("tcp::addr=localhost;auto_flush_bytes=1024;init_buf_size=2048;", "TCP transport requires init_buf_size and auto_flush_bytes to be set to the same value [init_buf_size=2048, auto_flush_bytes=1024]"); - assertConfStrError("tcp::addr=localhost;init_buf_size=1024;auto_flush_bytes=2048;", "TCP transport requires init_buf_size and auto_flush_bytes to be set to the same value [init_buf_size=1024, auto_flush_bytes=2048]"); - assertConfStrError("tcp::addr=localhost;auto_flush_bytes=off;", "TCP transport must have auto_flush_bytes enabled"); - - assertConfStrOk("http::addr=localhost;auto_flush=off;protocol_version=2;"); - assertConfStrOk("http::addr=localhost;protocol_version=2;"); - assertConfStrOk("http::addr=localhost;auto_flush_interval=off;protocol_version=2;"); - assertConfStrOk("http::addr=localhost;auto_flush_rows=off;protocol_version=2;"); - assertConfStrOk("http::addr=localhost;auto_flush_interval=off;auto_flush_rows=off;protocol_version=1;"); - assertConfStrOk("http::addr=localhost;auto_flush_interval=off;auto_flush_rows=1;protocol_version=2;"); - assertConfStrOk("http::addr=localhost;auto_flush_rows=off;auto_flush_interval=1;protocol_version=2;"); - assertConfStrOk("http::addr=localhost;auto_flush_interval=off;auto_flush_rows=off;auto_flush=off;protocol_version=2;"); - assertConfStrOk("http::addr=localhost;auto_flush=off;auto_flush_interval=off;auto_flush_rows=off;protocol_version=1;"); - assertConfStrOk("http::addr=localhost:8080;protocol_version=2;"); - assertConfStrOk("http::addr=localhost:8080;token=foo;protocol_version=2;"); - assertConfStrOk("http::addr=localhost:8080;token=foo=bar;protocol_version=2;"); - assertConfStrOk("addr=localhost:8080", "token=foo", "retry_timeout=1000", "max_buf_size=1000000", "protocol_version=2"); - assertConfStrOk("addr=localhost:8080", "token=foo", "retry_timeout=1000", "max_buf_size=1000000", "protocol_version=1"); - assertConfStrOk("http::addr=localhost:8080;token=foo;max_buf_size=1000000;retry_timeout=1000;protocol_version=2;"); - assertConfStrOk("https::addr=localhost:8080;tls_verify=unsafe_off;auto_flush_rows=100;protocol_version=2;"); - assertConfStrOk("https::addr=localhost:8080;tls_verify=unsafe_off;auto_flush_rows=100;protocol_version=2;max_name_len=256;"); - assertConfStrOk("https::addr=localhost:8080;tls_verify=on;protocol_version=2;"); - assertConfStrOk("https::addr=localhost:8080;tls_verify=on;protocol_version=3;"); - assertConfStrError("https::addr=2001:0db8:85a3:0000:0000:8a2e:0370:7334;tls_verify=on;", "cannot parse a port from the address, use IPv4 address or a domain name [address=2001:0db8:85a3:0000:0000:8a2e:0370:7334]"); - assertConfStrError("https::addr=[2001:0db8:85a3:0000:0000:8a2e:0370:7334]:9000;tls_verify=on;", "cannot parse a port from the address, use IPv4 address or a domain name [address=[2001:0db8:85a3:0000:0000:8a2e:0370:7334]:9000]"); - }); - } - - @Test - public void testCustomTruststoreButTlsNotEnabled() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP) - .advancedTls().customTrustStore(TRUSTSTORE_PATH, TRUSTSTORE_PASSWORD) - .address(LOCALHOST); - try { - builder.build(); - fail("should fail when custom trust store configured, but TLS not enabled"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "TLS was not enabled"); - } - }); - } - - @Test - public void testCustomTruststoreDoubleSet() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP).advancedTls().customTrustStore(TRUSTSTORE_PATH, TRUSTSTORE_PASSWORD); - try { - builder.advancedTls().customTrustStore(TRUSTSTORE_PATH, TRUSTSTORE_PASSWORD); - fail("should not allow double custom trust store set"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "already configured"); - } - }); - } - - @Test - public void testCustomTruststorePasswordCannotBeNull() { - try { - Sender.builder(Sender.Transport.TCP).advancedTls().customTrustStore(TRUSTSTORE_PATH, null); - fail("should not allow null trust store password"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "trust store password cannot be null"); - } - } - - @Test - public void testCustomTruststorePathCannotBeBlank() { - try { - Sender.builder(Sender.Transport.TCP).advancedTls().customTrustStore("", TRUSTSTORE_PASSWORD); - fail("should not allow blank trust store path"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "trust store path cannot be empty nor null"); - } - - try { - Sender.builder(Sender.Transport.TCP).advancedTls().customTrustStore(null, TRUSTSTORE_PASSWORD); - fail("should not allow null trust store path"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "trust store path cannot be empty nor null"); - } - } - - @Test - public void testDisableAutoFlushNotSupportedForTcp() throws Exception { - assertMemoryLeak(() -> { - try (Sender ignored = Sender.builder(Sender.Transport.TCP).address(LOCALHOST).disableAutoFlush().build()) { - fail("TCP does not support disabling auto-flush"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "auto-flush is not supported for TCP protocol"); - } - }); - } - - @Test - public void testDnsResolutionFail() throws Exception { - assertMemoryLeak(() -> { - try (Sender ignored = Sender.builder(Sender.Transport.TCP).address("this-domain-does-not-exist-i-hope-better-to-use-a-silly-tld.silly-tld").build()) { - fail("dns resolution errors should fail fast"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "could not resolve"); - } - }); - } - - @Test - public void testDuplicatedAddresses() throws Exception { - assertMemoryLeak(() -> { - try { - Sender.builder(Sender.Transport.TCP).address("localhost:9000").address("localhost:9000"); - Assert.fail("should not allow multiple addresses"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "duplicated addresses are not allowed [address=localhost:9000]"); - } - - try { - Sender.builder(Sender.Transport.TCP).address("localhost:9000").address("localhost:9001").address("localhost:9000"); - Assert.fail("should not allow multiple addresses"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "duplicated addresses are not allowed [address=localhost:9000]"); - } - }); - } - - @Test - public void testDuplicatedAddressesWithDifferentPortsAllowed() throws Exception { - assertMemoryLeak(() -> { - Sender.builder(Sender.Transport.TCP).address("localhost:9000").address("localhost:9001"); - }); - } - - @Test - public void testDuplicatedAddressesWithNoPortsAllowed() throws Exception { - assertMemoryLeak(() -> { - Sender.builder(Sender.Transport.TCP).address("localhost:9000").address("localhost"); - Sender.builder(Sender.Transport.TCP).address("localhost").address("localhost:9000"); - }); - } - - @Test - public void testFailFastWhenSetCustomTrustStoreTwice() { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP).advancedTls().customTrustStore(TRUSTSTORE_PATH, TRUSTSTORE_PASSWORD); - try { - builder.advancedTls().customTrustStore(TRUSTSTORE_PATH, TRUSTSTORE_PASSWORD); - fail("should not allow double custom trust store set"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "already configured"); - } - } - - @Test - public void testFirstTlsValidationDisabledThenCustomTruststore() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP) - .advancedTls().disableCertificateValidation(); - try { - builder.advancedTls().customTrustStore(TRUSTSTORE_PATH, TRUSTSTORE_PASSWORD); - fail("should not allow custom truststore when TLS validation was disabled disabled"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "TLS validation was already disabled"); - } - }); - } - - @Test - public void testHostNorAddressSet() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP); - try { - builder.build(); - fail("not host should fail"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "server address not set"); - } - }); - } - - @Test - public void testHttpTokenNotSupportedForTcp() throws Exception { - assertMemoryLeak(() -> { - try { - Sender.builder(Sender.Transport.TCP).address(LOCALHOST).httpToken("foo").build(); - fail("HTTP token should not be supported for TCP"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "HTTP token authentication is not supported for TCP protocol"); - } - }); - } - - @Test - public void testInvalidHttpTimeout() throws Exception { - assertMemoryLeak(() -> { - try { - Sender.builder(Sender.Transport.HTTP).address("someurl").httpTimeoutMillis(0); - fail("should fail with bad http time"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "HTTP timeout must be positive [timeout=0]"); - } - - try { - Sender.builder(Sender.Transport.HTTP).address("someurl").httpTimeoutMillis(-1); - fail("should fail with bad http time"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "HTTP timeout must be positive [timeout=-1]"); - } - - try { - Sender.builder(Sender.Transport.HTTP).address("someurl").httpTimeoutMillis(100).httpTimeoutMillis(200); - fail("should fail with bad http time"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "HTTP timeout was already configured [timeout=100]"); - } - - try { - Sender.builder(Sender.Transport.TCP).address("localhost").httpTimeoutMillis(5000).build(); - fail("should fail with bad http time"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "HTTP timeout is not supported for TCP protocol"); - } - }); - } - - @Test - public void testInvalidRetryTimeout() { - try { - Sender.builder(Sender.Transport.HTTP).retryTimeoutMillis(-1); - Assert.fail(); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "retry timeout cannot be negative [retryTimeoutMillis=-1]"); - } - - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.HTTP).retryTimeoutMillis(100); - try { - builder.retryTimeoutMillis(200); - Assert.fail(); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "retry timeout was already configured [retryTimeoutMillis=100]"); - } - } - - @Test - public void testMalformedPortInAddress() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP); - try { - builder.address("foo:nonsense12334"); - fail("should fail with malformated port"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "cannot parse a port from the address"); - } - }); - } - - @Test - public void testMaxRequestBufferSizeCannotBeLessThanDefault() throws Exception { - assertMemoryLeak(() -> { - try (Sender ignored = Sender.builder(Sender.Transport.HTTP) - .address("localhost:1") - .maxBufferCapacity(65535) - .build() - ) { - Assert.fail(); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "maximum buffer capacity cannot be less than initial buffer capacity [maximumBufferCapacity=65535, initialBufferCapacity=65536]"); - } - }); - } - - @Test - public void testMaxRequestBufferSizeCannotBeLessThanInitialBufferSize() throws Exception { - assertMemoryLeak(() -> { - try (Sender ignored = Sender.builder(Sender.Transport.HTTP) - .address("localhost:1") - .maxBufferCapacity(100_000) - .bufferCapacity(200_000) - .build() - ) { - Assert.fail(); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "maximum buffer capacity cannot be less than initial buffer capacity [maximumBufferCapacity=100000, initialBufferCapacity=200000]"); - } - }); - } - - @Test - public void testMaxRetriesNotSupportedForTcp() throws Exception { - assertMemoryLeak(() -> { - try { - Sender.builder(Sender.Transport.TCP).address(LOCALHOST).retryTimeoutMillis(100).build(); - fail("max retries should not be supported for TCP"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "retrying is not supported for TCP protocol"); - } - }); - } - - @Test - public void testMinRequestThroughputCannotBeNegative() throws Exception { - assertMemoryLeak(() -> { - try { - Sender.builder(Sender.Transport.HTTP).address(LOCALHOST).minRequestThroughput(-100).build(); - fail("minimum request throughput must not be negative"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "minimum request throughput must not be negative [minRequestThroughput=-100]"); - } - }); - } - - @Test - public void testMinRequestThroughputNotSupportedForTcp() throws Exception { - assertMemoryLeak(() -> { - try { - Sender.builder(Sender.Transport.TCP).address(LOCALHOST).minRequestThroughput(1).build(); - fail("min request throughput is not be supported for TCP and the builder should fail-fast"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "minimum request throughput is not supported for TCP protocol"); - } - }); - } - - @Test - public void testPlainAuth_connectionRefused() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP) - .enableAuth("foo").authToken(AUTH_TOKEN_KEY1).address(LOCALHOST + ":19003"); - try { - builder.build(); - fail("connection refused should fail fast"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "could not connect"); - } - }); - } - - @Test - public void testPlainOldTokenNotSupportedForHttpProtocol() throws Exception { - assertMemoryLeak(() -> { - try { - Sender.builder(Sender.Transport.HTTP).address("localhost:9000").enableAuth("key").authToken(AUTH_TOKEN_KEY1).build(); - fail("HTTP token should not be supported for TCP"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "old token authentication is not supported for HTTP protocol"); - } - }); - } - - @Test - public void testPlain_connectionRefused() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP).address(LOCALHOST + ":19003"); - try { - builder.build(); - fail("connection refused should fail fast"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "could not connect"); - } - }); - } - - @Test - public void testPortDoubleSet_firstAddressThenPort() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP).address(LOCALHOST + ":9000"); - try { - builder.port(9000); - builder.build(); - fail("should not allow double port set"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "mismatch"); - } - }); - } - - @Test - public void testPortDoubleSet_firstPortThenAddress() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP).port(9000); - try { - builder.address(LOCALHOST + ":9000"); - builder.build(); - fail("should not allow double port set"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "mismatch"); - } - }); - } - - @Test - public void testPortDoubleSet_firstPortThenPort() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP).port(9000); - try { - builder.port(9000); - builder.build(); - fail("should not allow double port set"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "questdb server address not set"); - } - }); - } - - @Test - public void testSmallMaxNameLen() throws Exception { - assertMemoryLeak(() -> { - try { - Sender.LineSenderBuilder ignored = Sender - .builder(Sender.Transport.TCP) - .maxNameLength(10); - fail("should not allow double buffer capacity set"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "max_name_len must be at least 16 bytes [max_name_len=10]"); - } - }); - } - - @Test - public void testTlsDoubleSet() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP).enableTls(); - try { - builder.enableTls(); - fail("should not allow double tls set"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "already enabled"); - } - }); - } - - @Test - public void testTlsValidationDisabledButTlsNotEnabled() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP) - .advancedTls().disableCertificateValidation() - .address(LOCALHOST); - try { - builder.build(); - fail("should fail when TLS validation is disabled, but TLS not enabled"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "TLS was not enabled"); - } - }); - } - - @Test - public void testTlsValidationDisabledDoubleSet() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP) - .advancedTls().disableCertificateValidation(); - try { - builder.advancedTls().disableCertificateValidation(); - fail("should not allow double TLS validation disabled"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "TLS validation was already disabled"); - } - }); - } - - @Test - public void testTls_connectionRefused() throws Exception { - assertMemoryLeak(() -> { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.TCP).enableTls().address(LOCALHOST + ":19003"); - try { - builder.build(); - fail("connection refused should fail fast"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "could not connect"); - } - }); - } - - @Test - public void testUsernamePasswordAuthNotSupportedForTcp() throws Exception { - assertMemoryLeak(() -> { - try { - Sender.builder(Sender.Transport.TCP).address(LOCALHOST).httpUsernamePassword("foo", "bar").build(); - fail("HTTP token should not be supported for TCP"); - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), "username/password authentication is not supported for TCP protocol"); - } - }); - } - - private static void assertConfStrError(String conf, String expectedError) { - try { - try (Sender ignored = Sender.fromConfig(conf)) { - fail("should fail with bad conf string"); - } - } catch (LineSenderException e) { - TestUtils.assertContains(e.getMessage(), expectedError); - } - } - - private static void assertConfStrOk(String... params) { - StringBuilder sb = new StringBuilder(); - sb.append("http").append("::"); - shuffle(params); - for (int i = 0; i < params.length; i++) { - sb.append(params[i]).append(";"); - } - assertConfStrOk(sb.toString()); - } - - private static void assertConfStrOk(String conf) { - Sender.fromConfig(conf).close(); - } - - private static void shuffle(String[] input) { - for (int i = 0; i < input.length; i++) { - int j = (int) (Math.random() * input.length); - String tmp = input[i]; - input[i] = input[j]; - input[j] = tmp; - } - } -} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/line/LineSenderBuilderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/line/LineSenderBuilderTest.java new file mode 100644 index 0000000..bf9f5c7 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/line/LineSenderBuilderTest.java @@ -0,0 +1,490 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.line; + +import io.questdb.client.Sender; +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Test; + +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; +import static org.junit.Assert.fail; + +/** + * Unit tests for LineSenderBuilder that don't require a running QuestDB instance. + * Tests that require an actual QuestDB connection have been moved to integration tests. + */ +public class LineSenderBuilderTest { + private static final String AUTH_TOKEN_KEY1 = "UvuVb1USHGRRT08gEnwN2zGZrvM4MsLQ5brgF6SVkAw="; + private static final String LOCALHOST = "localhost"; + private static final char[] TRUSTSTORE_PASSWORD = "questdb".toCharArray(); + private static final String TRUSTSTORE_PATH = "/keystore/server.keystore"; + + @Test + public void testAddressDoubleSet_firstAddressThenAddress() throws Exception { + assertMemoryLeak(() -> assertThrows("mismatch", + Sender.builder(Sender.Transport.TCP).address(LOCALHOST).address("127.0.0.1"))); + } + + @Test + public void testAddressEmpty() throws Exception { + assertMemoryLeak(() -> assertThrows("address cannot be empty", + () -> Sender.builder(Sender.Transport.TCP).address(""))); + } + + @Test + public void testAddressEndsWithColon() throws Exception { + assertMemoryLeak(() -> assertThrows("invalid address", + () -> Sender.builder(Sender.Transport.TCP).address("foo:"))); + } + + @Test + public void testAddressNull() throws Exception { + assertMemoryLeak(() -> assertThrows("null", + () -> Sender.builder(Sender.Transport.TCP).address(null))); + } + + @Test + public void testAuthDoubleSet() throws Exception { + assertMemoryLeak(() -> assertThrows("already configured", + () -> Sender.builder(Sender.Transport.TCP).enableAuth("foo").authToken(AUTH_TOKEN_KEY1).enableAuth("bar"))); + } + + @Test + public void testAuthTooSmallBuffer() throws Exception { + assertMemoryLeak(() -> assertThrows("minimalCapacity", + Sender.builder(Sender.Transport.TCP) + .enableAuth("foo").authToken(AUTH_TOKEN_KEY1).address(LOCALHOST + ":9001") + .bufferCapacity(1))); + } + + @Test + public void testAuthWithBadToken() throws Exception { + assertMemoryLeak(() -> assertThrows("could not import token", + () -> Sender.builder(Sender.Transport.TCP).enableAuth("foo").authToken("bar token"))); + } + + @Test + public void testAutoFlushIntervalMustBePositive() { + assertThrows("auto flush interval cannot be negative [autoFlushIntervalMillis=0]", + () -> Sender.builder(Sender.Transport.HTTP).autoFlushIntervalMillis(0)); + assertThrows("auto flush interval cannot be negative [autoFlushIntervalMillis=-1]", + () -> Sender.builder(Sender.Transport.HTTP).autoFlushIntervalMillis(-1)); + } + + @Test + public void testAutoFlushIntervalNotSupportedForTcp() throws Exception { + assertMemoryLeak(() -> assertThrows("auto flush interval is not supported for TCP protocol", + Sender.builder(Sender.Transport.TCP).address(LOCALHOST).autoFlushIntervalMillis(1))); + } + + @Test + public void testAutoFlushInterval_afterAutoFlushDisabled() throws Exception { + assertMemoryLeak(() -> assertThrows("cannot set auto flush interval when interval based auto-flush is already disabled", + () -> Sender.builder(Sender.Transport.HTTP).disableAutoFlush().autoFlushIntervalMillis(1))); + } + + @Test + public void testAutoFlushInterval_doubleConfiguration() throws Exception { + assertMemoryLeak(() -> assertThrows("auto flush interval was already configured [autoFlushIntervalMillis=1]", + () -> Sender.builder(Sender.Transport.HTTP).autoFlushIntervalMillis(1).autoFlushIntervalMillis(1))); + } + + @Test + public void testAutoFlushRowsCannotBeNegative() { + assertThrows("auto flush rows cannot be negative [autoFlushRows=-1]", + () -> Sender.builder(Sender.Transport.HTTP).autoFlushRows(-1)); + } + + @Test + public void testAutoFlushRowsNotSupportedForTcp() throws Exception { + assertMemoryLeak(() -> assertThrows("auto flush rows is not supported for TCP protocol", + Sender.builder(Sender.Transport.TCP).address(LOCALHOST).autoFlushRows(1))); + } + + @Test + public void testAutoFlushRows_doubleConfiguration() throws Exception { + assertMemoryLeak(() -> assertThrows("auto flush rows was already configured [autoFlushRows=1]", + () -> Sender.builder(Sender.Transport.HTTP).autoFlushRows(1).autoFlushRows(1))); + } + + @Test + public void testBufferSizeDoubleSet() throws Exception { + assertMemoryLeak(() -> assertThrows("already configured", + () -> Sender.builder(Sender.Transport.TCP).bufferCapacity(1024).bufferCapacity(1024))); + } + + @Test + public void testConfStringValidation() throws Exception { + assertMemoryLeak(() -> { + assertConfStrError("foo", "invalid schema [schema=foo, supported-schemas=[http, https, tcp, tcps, ws, wss]]"); + assertConfStrError("badschema::addr=bar;", "invalid schema [schema=badschema, supported-schemas=[http, https, tcp, tcps, ws, wss]]"); + assertConfStrError("http::addr=localhost:-1;", "invalid port [port=-1]"); + assertConfStrError("http::auto_flush=on;", "addr is missing"); + assertConfStrError("http::addr=localhost;tls_roots=/some/path;", "tls_roots was configured, but tls_roots_password is missing"); + assertConfStrError("http::addr=localhost;tls_roots_password=hunter123;", "tls_roots_password was configured, but tls_roots is missing"); + assertConfStrError("tcp::addr=localhost;user=foo;", "token cannot be empty nor null"); + assertConfStrError("tcp::addr=localhost;username=foo;", "token cannot be empty nor null"); + assertConfStrError("tcp::addr=localhost;token=foo;", "TCP token is configured, but user is missing"); + assertConfStrError("http::addr=localhost;user=foo;", "password cannot be empty nor null"); + assertConfStrError("http::addr=localhost;username=foo;", "password cannot be empty nor null"); + assertConfStrError("http::addr=localhost;pass=foo;", "HTTP password is configured, but username is missing"); + assertConfStrError("http::addr=localhost;password=foo;", "HTTP password is configured, but username is missing"); + assertConfStrError("tcp::addr=localhost;pass=foo;", "password is not supported for TCP protocol"); + assertConfStrError("tcp::addr=localhost;password=foo;", "password is not supported for TCP protocol"); + assertConfStrError("tcp::addr=localhost;retry_timeout=;", "retry_timeout cannot be empty"); + assertConfStrError("tcp::addr=localhost;max_buf_size=;", "max_buf_size cannot be empty"); + assertConfStrError("tcp::addr=localhost;init_buf_size=;", "init_buf_size cannot be empty"); + assertConfStrError("http::addr=localhost:8080;tls_verify=unsafe_off;", "TLS validation disabled, but TLS was not enabled"); + assertConfStrError("http::addr=localhost:8080;tls_verify=bad;", "invalid tls_verify [value=bad, allowed-values=[on, unsafe_off]]"); + assertConfStrError("tcps::addr=localhost;pass=unsafe_off;", "password is not supported for TCP protocol"); + assertConfStrError("tcps::addr=localhost;password=unsafe_off;", "password is not supported for TCP protocol"); + assertConfStrError("http::addr=localhost:8080;max_buf_size=-32;", "maximum buffer capacity cannot be less than initial buffer capacity [maximumBufferCapacity=-32, initialBufferCapacity=65536]"); + assertConfStrError("http::addr=localhost:8080;max_buf_size=notanumber;", "invalid max_buf_size [value=notanumber]"); + assertConfStrError("http::addr=localhost:8080;init_buf_size=notanumber;", "invalid init_buf_size [value=notanumber]"); + assertConfStrError("http::addr=localhost:8080;init_buf_size=-42;", "buffer capacity cannot be negative [capacity=-42]"); + assertConfStrError("http::addr=localhost:8080;auto_flush_rows=0;", "invalid auto_flush_rows [value=0]"); + assertConfStrError("http::addr=localhost:8080;auto_flush_rows=notanumber;", "invalid auto_flush_rows [value=notanumber]"); + assertConfStrError("http::addr=localhost:8080;auto_flush=invalid;", "invalid auto_flush [value=invalid, allowed-values=[on, off]]"); + assertConfStrError("http::addr=localhost:8080;auto_flush=off;auto_flush_rows=100;", "cannot set auto flush rows when auto-flush is already disabled"); + assertConfStrError("http::addr=localhost:8080;auto_flush_rows=100;auto_flush=off;", "auto flush rows was already configured [autoFlushRows=100]"); + assertConfStrError("HTTP::addr=localhost;", "invalid schema [schema=HTTP, supported-schemas=[http, https, tcp, tcps, ws, wss]]"); + assertConfStrError("HTTPS::addr=localhost;", "invalid schema [schema=HTTPS, supported-schemas=[http, https, tcp, tcps, ws, wss]]"); + assertConfStrError("TCP::addr=localhost;", "invalid schema [schema=TCP, supported-schemas=[http, https, tcp, tcps, ws, wss]]"); + assertConfStrError("TCPS::addr=localhost;", "invalid schema [schema=TCPS, supported-schemas=[http, https, tcp, tcps, ws, wss]]"); + assertConfStrError("http::addr=localhost;auto_flush=off;auto_flush_interval=1;", "cannot set auto flush interval when interval based auto-flush is already disabled"); + assertConfStrError("http::addr=localhost;auto_flush=off;auto_flush_rows=1;", "cannot set auto flush rows when auto-flush is already disabled"); + assertConfStrError("http::addr=localhost;auto_flush_bytes=1024;", "auto_flush_bytes is only supported for TCP transport"); + assertConfStrError("http::addr=localhost;protocol_version=10", "current client only supports protocol version 1(text format for all datatypes), 2(binary format for part datatypes), 3(decimal datatype) or explicitly unset"); + assertConfStrError("http::addr=localhost:48884;max_name_len=10;", "max_name_len must be at least 16 bytes [max_name_len=10]"); + + assertConfStrOk("addr=localhost:8080", "auto_flush_rows=100", "protocol_version=1"); + assertConfStrOk("addr=localhost:8080", "auto_flush=on", "auto_flush_rows=100", "protocol_version=2"); + assertConfStrOk("addr=localhost:8080", "auto_flush_rows=100", "auto_flush=on", "protocol_version=2"); + assertConfStrOk("addr=localhost", "auto_flush=on", "protocol_version=2"); + assertConfStrOk("addr=localhost:8080", "max_name_len=1024", "protocol_version=2"); + + assertConfStrError("tcp::addr=localhost;auto_flush_bytes=1024;init_buf_size=2048;", "TCP transport requires init_buf_size and auto_flush_bytes to be set to the same value [init_buf_size=2048, auto_flush_bytes=1024]"); + assertConfStrError("tcp::addr=localhost;init_buf_size=1024;auto_flush_bytes=2048;", "TCP transport requires init_buf_size and auto_flush_bytes to be set to the same value [init_buf_size=1024, auto_flush_bytes=2048]"); + assertConfStrError("tcp::addr=localhost;auto_flush_bytes=off;", "TCP transport must have auto_flush_bytes enabled"); + + assertConfStrOk("http::addr=localhost;auto_flush=off;protocol_version=2;"); + assertConfStrOk("http::addr=localhost;protocol_version=2;"); + assertConfStrOk("http::addr=localhost;auto_flush_interval=off;protocol_version=2;"); + assertConfStrOk("http::addr=localhost;auto_flush_rows=off;protocol_version=2;"); + assertConfStrOk("http::addr=localhost;auto_flush_interval=off;auto_flush_rows=off;protocol_version=1;"); + assertConfStrOk("http::addr=localhost;auto_flush_interval=off;auto_flush_rows=1;protocol_version=2;"); + assertConfStrOk("http::addr=localhost;auto_flush_rows=off;auto_flush_interval=1;protocol_version=2;"); + assertConfStrOk("http::addr=localhost;auto_flush_interval=off;auto_flush_rows=off;auto_flush=off;protocol_version=2;"); + assertConfStrOk("http::addr=localhost;auto_flush=off;auto_flush_interval=off;auto_flush_rows=off;protocol_version=1;"); + assertConfStrOk("http::addr=localhost:8080;protocol_version=2;"); + assertConfStrOk("http::addr=localhost:8080;token=foo;protocol_version=2;"); + assertConfStrOk("http::addr=localhost:8080;token=foo=bar;protocol_version=2;"); + assertConfStrOk("addr=localhost:8080", "token=foo", "retry_timeout=1000", "max_buf_size=1000000", "protocol_version=2"); + assertConfStrOk("addr=localhost:8080", "token=foo", "retry_timeout=1000", "max_buf_size=1000000", "protocol_version=1"); + assertConfStrOk("http::addr=localhost:8080;token=foo;max_buf_size=1000000;retry_timeout=1000;protocol_version=2;"); + assertConfStrOk("https::addr=localhost:8080;tls_verify=unsafe_off;auto_flush_rows=100;protocol_version=2;"); + assertConfStrOk("https::addr=localhost:8080;tls_verify=unsafe_off;auto_flush_rows=100;protocol_version=2;max_name_len=256;"); + assertConfStrOk("https::addr=localhost:8080;tls_verify=on;protocol_version=2;"); + assertConfStrOk("https::addr=localhost:8080;tls_verify=on;protocol_version=3;"); + assertConfStrError("https::addr=2001:0db8:85a3:0000:0000:8a2e:0370:7334;tls_verify=on;", "cannot parse a port from the address, use IPv4 address or a domain name [address=2001:0db8:85a3:0000:0000:8a2e:0370:7334]"); + assertConfStrError("https::addr=[2001:0db8:85a3:0000:0000:8a2e:0370:7334]:9000;tls_verify=on;", "cannot parse a port from the address, use IPv4 address or a domain name [address=[2001:0db8:85a3:0000:0000:8a2e:0370:7334]:9000]"); + }); + } + + @Test + public void testCustomTruststoreButTlsNotEnabled() throws Exception { + assertMemoryLeak(() -> assertThrows("TLS was not enabled", + Sender.builder(Sender.Transport.TCP) + .advancedTls().customTrustStore(TRUSTSTORE_PATH, TRUSTSTORE_PASSWORD) + .address(LOCALHOST))); + } + + @Test + public void testCustomTruststoreDoubleSet() throws Exception { + assertMemoryLeak(() -> assertThrows("already configured", + () -> Sender.builder(Sender.Transport.TCP) + .advancedTls().customTrustStore(TRUSTSTORE_PATH, TRUSTSTORE_PASSWORD) + .advancedTls().customTrustStore(TRUSTSTORE_PATH, TRUSTSTORE_PASSWORD))); + } + + @Test + public void testCustomTruststorePasswordCannotBeNull() { + assertThrows("trust store password cannot be null", + () -> Sender.builder(Sender.Transport.TCP).advancedTls().customTrustStore(TRUSTSTORE_PATH, null)); + } + + @Test + public void testCustomTruststorePathCannotBeBlank() { + assertThrows("trust store path cannot be empty nor null", + () -> Sender.builder(Sender.Transport.TCP).advancedTls().customTrustStore("", TRUSTSTORE_PASSWORD)); + assertThrows("trust store path cannot be empty nor null", + () -> Sender.builder(Sender.Transport.TCP).advancedTls().customTrustStore(null, TRUSTSTORE_PASSWORD)); + } + + @Test + public void testDisableAutoFlushNotSupportedForTcp() throws Exception { + assertMemoryLeak(() -> assertThrows("auto-flush is not supported for TCP protocol", + Sender.builder(Sender.Transport.TCP).address(LOCALHOST).disableAutoFlush())); + } + + @Test + public void testDnsResolutionFail() throws Exception { + assertMemoryLeak(() -> assertThrows("could not resolve", + Sender.builder(Sender.Transport.TCP).address("this-domain-does-not-exist-i-hope-better-to-use-a-silly-tld.silly-tld"))); + } + + @Test + public void testDuplicatedAddresses() throws Exception { + assertMemoryLeak(() -> { + assertThrows("duplicated addresses are not allowed [address=localhost:9000]", + () -> Sender.builder(Sender.Transport.TCP).address("localhost:9000").address("localhost:9000")); + assertThrows("duplicated addresses are not allowed [address=localhost:9000]", + () -> Sender.builder(Sender.Transport.TCP).address("localhost:9000").address("localhost:9001").address("localhost:9000")); + }); + } + + @Test + public void testDuplicatedAddressesWithDifferentPortsAllowed() throws Exception { + assertMemoryLeak(() -> Sender.builder(Sender.Transport.TCP).address("localhost:9000").address("localhost:9001")); + } + + @Test + public void testDuplicatedAddressesWithNoPortsAllowed() throws Exception { + assertMemoryLeak(() -> { + Sender.builder(Sender.Transport.TCP).address("localhost:9000").address("localhost"); + Sender.builder(Sender.Transport.TCP).address("localhost").address("localhost:9000"); + }); + } + + @Test + public void testFailFastWhenSetCustomTrustStoreTwice() { + assertThrows("already configured", + () -> Sender.builder(Sender.Transport.TCP) + .advancedTls().customTrustStore(TRUSTSTORE_PATH, TRUSTSTORE_PASSWORD) + .advancedTls().customTrustStore(TRUSTSTORE_PATH, TRUSTSTORE_PASSWORD)); + } + + @Test + public void testFirstTlsValidationDisabledThenCustomTruststore() throws Exception { + assertMemoryLeak(() -> assertThrows("TLS validation was already disabled", + () -> Sender.builder(Sender.Transport.TCP) + .advancedTls().disableCertificateValidation() + .advancedTls().customTrustStore(TRUSTSTORE_PATH, TRUSTSTORE_PASSWORD))); + } + + @Test + public void testHostNorAddressSet() throws Exception { + assertMemoryLeak(() -> assertThrows("server address not set", + Sender.builder(Sender.Transport.TCP))); + } + + @Test + public void testHttpTokenNotSupportedForTcp() throws Exception { + assertMemoryLeak(() -> assertThrows("HTTP token authentication is not supported for TCP protocol", + Sender.builder(Sender.Transport.TCP).address(LOCALHOST).httpToken("foo"))); + } + + @Test + public void testInvalidHttpTimeout() throws Exception { + assertMemoryLeak(() -> { + assertThrows("HTTP timeout must be positive [timeout=0]", + () -> Sender.builder(Sender.Transport.HTTP).address("someurl").httpTimeoutMillis(0)); + assertThrows("HTTP timeout must be positive [timeout=-1]", + () -> Sender.builder(Sender.Transport.HTTP).address("someurl").httpTimeoutMillis(-1)); + assertThrows("HTTP timeout was already configured [timeout=100]", + () -> Sender.builder(Sender.Transport.HTTP).address("someurl").httpTimeoutMillis(100).httpTimeoutMillis(200)); + assertThrows("HTTP timeout is not supported for TCP protocol", + Sender.builder(Sender.Transport.TCP).address("localhost").httpTimeoutMillis(5000)); + }); + } + + @Test + public void testInvalidRetryTimeout() { + assertThrows("retry timeout cannot be negative [retryTimeoutMillis=-1]", + () -> Sender.builder(Sender.Transport.HTTP).retryTimeoutMillis(-1)); + assertThrows("retry timeout was already configured [retryTimeoutMillis=100]", + () -> Sender.builder(Sender.Transport.HTTP).retryTimeoutMillis(100).retryTimeoutMillis(200)); + } + + @Test + public void testMalformedPortInAddress() throws Exception { + assertMemoryLeak(() -> assertThrows("cannot parse a port from the address", + () -> Sender.builder(Sender.Transport.TCP).address("foo:nonsense12334"))); + } + + @Test + public void testMaxRequestBufferSizeCannotBeLessThanDefault() throws Exception { + assertMemoryLeak(() -> assertThrows("maximum buffer capacity cannot be less than initial buffer capacity [maximumBufferCapacity=65535, initialBufferCapacity=65536]", + Sender.builder(Sender.Transport.HTTP).address("localhost:1").maxBufferCapacity(65535))); + } + + @Test + public void testMaxRequestBufferSizeCannotBeLessThanInitialBufferSize() throws Exception { + assertMemoryLeak(() -> assertThrows("maximum buffer capacity cannot be less than initial buffer capacity [maximumBufferCapacity=100000, initialBufferCapacity=200000]", + Sender.builder(Sender.Transport.HTTP).address("localhost:1").maxBufferCapacity(100_000).bufferCapacity(200_000))); + } + + @Test + public void testMaxRetriesNotSupportedForTcp() throws Exception { + assertMemoryLeak(() -> assertThrows("retrying is not supported for TCP protocol", + Sender.builder(Sender.Transport.TCP).address(LOCALHOST).retryTimeoutMillis(100))); + } + + @Test + public void testMinRequestThroughputCannotBeNegative() throws Exception { + assertMemoryLeak(() -> assertThrows("minimum request throughput must not be negative [minRequestThroughput=-100]", + Sender.builder(Sender.Transport.HTTP).address(LOCALHOST).minRequestThroughput(-100))); + } + + @Test + public void testMinRequestThroughputNotSupportedForTcp() throws Exception { + assertMemoryLeak(() -> assertThrows("minimum request throughput is not supported for TCP protocol", + Sender.builder(Sender.Transport.TCP).address(LOCALHOST).minRequestThroughput(1))); + } + + @Test + public void testPlainAuth_connectionRefused() throws Exception { + assertMemoryLeak(() -> assertThrows("could not connect", + Sender.builder(Sender.Transport.TCP) + .enableAuth("foo").authToken(AUTH_TOKEN_KEY1).address(LOCALHOST + ":19003"))); + } + + @Test + public void testPlainOldTokenNotSupportedForHttpProtocol() throws Exception { + assertMemoryLeak(() -> assertThrows("old token authentication is not supported for HTTP protocol", + Sender.builder(Sender.Transport.HTTP).address("localhost:9000").enableAuth("key").authToken(AUTH_TOKEN_KEY1))); + } + + @Test + public void testPlain_connectionRefused() throws Exception { + assertMemoryLeak(() -> assertThrows("could not connect", + Sender.builder(Sender.Transport.TCP).address(LOCALHOST + ":19003"))); + } + + @Test + public void testPortDoubleSet_firstAddressThenPort() throws Exception { + assertMemoryLeak(() -> assertThrows("mismatch", + Sender.builder(Sender.Transport.TCP).address(LOCALHOST + ":9000").port(9000))); + } + + @Test + public void testPortDoubleSet_firstPortThenAddress() throws Exception { + assertMemoryLeak(() -> assertThrows("mismatch", + Sender.builder(Sender.Transport.TCP).port(9000).address(LOCALHOST + ":9000"))); + } + + @Test + public void testPortDoubleSet_firstPortThenPort() throws Exception { + assertMemoryLeak(() -> assertThrows("questdb server address not set", + Sender.builder(Sender.Transport.TCP).port(9000).port(9000))); + } + + @Test + public void testSmallMaxNameLen() throws Exception { + assertMemoryLeak(() -> assertThrows("max_name_len must be at least 16 bytes [max_name_len=10]", + () -> Sender.builder(Sender.Transport.TCP).maxNameLength(10))); + } + + @Test + public void testTlsDoubleSet() throws Exception { + assertMemoryLeak(() -> assertThrows("already enabled", + () -> Sender.builder(Sender.Transport.TCP).enableTls().enableTls())); + } + + @Test + public void testTlsValidationDisabledButTlsNotEnabled() throws Exception { + assertMemoryLeak(() -> assertThrows("TLS was not enabled", + Sender.builder(Sender.Transport.TCP) + .advancedTls().disableCertificateValidation() + .address(LOCALHOST))); + } + + @Test + public void testTlsValidationDisabledDoubleSet() throws Exception { + assertMemoryLeak(() -> assertThrows("TLS validation was already disabled", + () -> Sender.builder(Sender.Transport.TCP) + .advancedTls().disableCertificateValidation() + .advancedTls().disableCertificateValidation())); + } + + @Test + public void testTls_connectionRefused() throws Exception { + assertMemoryLeak(() -> assertThrows("could not connect", + Sender.builder(Sender.Transport.TCP).enableTls().address(LOCALHOST + ":19003"))); + } + + @Test + public void testUsernamePasswordAuthNotSupportedForTcp() throws Exception { + assertMemoryLeak(() -> assertThrows("username/password authentication is not supported for TCP protocol", + Sender.builder(Sender.Transport.TCP).address(LOCALHOST).httpUsernamePassword("foo", "bar"))); + } + + private static void assertConfStrError(String conf, String expectedError) { + try { + try (Sender ignored = Sender.fromConfig(conf)) { + fail("should fail with bad conf string"); + } + } catch (LineSenderException e) { + TestUtils.assertContains(e.getMessage(), expectedError); + } + } + + private static void assertConfStrOk(String... params) { + StringBuilder sb = new StringBuilder(); + sb.append("http").append("::"); + shuffle(params); + for (int i = 0; i < params.length; i++) { + sb.append(params[i]).append(";"); + } + assertConfStrOk(sb.toString()); + } + + private static void assertConfStrOk(String conf) { + Sender.fromConfig(conf).close(); + } + + private static void assertThrows(String expectedSubstring, Sender.LineSenderBuilder builder) { + assertThrows(expectedSubstring, builder::build); + } + + private static void assertThrows(String expectedSubstring, Runnable action) { + try { + action.run(); + fail("Expected LineSenderException containing '" + expectedSubstring + "'"); + } catch (LineSenderException e) { + TestUtils.assertContains(e.getMessage(), expectedSubstring); + } + } + + private static void shuffle(String[] input) { + for (int i = 0; i < input.length; i++) { + int j = (int) (Math.random() * input.length); + String tmp = input[i]; + input[i] = input[j]; + input[j] = tmp; + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/LineSenderBuilderWebSocketTest.java b/core/src/test/java/io/questdb/client/test/cutlass/line/LineSenderBuilderWebSocketTest.java similarity index 99% rename from core/src/test/java/io/questdb/client/test/LineSenderBuilderWebSocketTest.java rename to core/src/test/java/io/questdb/client/test/cutlass/line/LineSenderBuilderWebSocketTest.java index 736aec4..92c79d0 100644 --- a/core/src/test/java/io/questdb/client/test/LineSenderBuilderWebSocketTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/line/LineSenderBuilderWebSocketTest.java @@ -22,10 +22,11 @@ * ******************************************************************************/ -package io.questdb.client.test; +package io.questdb.client.test.cutlass.line; import io.questdb.client.Sender; import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.test.AbstractTest; import io.questdb.client.test.tools.TestUtils; import org.junit.Assert; import org.junit.Ignore; diff --git a/core/src/test/java/module-info.java b/core/src/test/java/module-info.java index 3e33568..86341d8 100644 --- a/core/src/test/java/module-info.java +++ b/core/src/test/java/module-info.java @@ -35,4 +35,5 @@ exports io.questdb.client.test; exports io.questdb.client.test.cairo; + exports io.questdb.client.test.cutlass.line; } From 49ecfa727e2b0d1b9f2a7102cf37e06086857cb1 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Sat, 14 Feb 2026 22:35:15 +0000 Subject: [PATCH 05/89] ilpv4 -> QWP (QuestDB Wire Protocol) --- .../main/java/io/questdb/client/Sender.java | 6 +- .../cutlass/http/client/WebSocketClient.java | 8 +- .../http/client/WebSocketSendBuffer.java | 12 +- .../client/GlobalSymbolDictionary.java | 2 +- .../{ilpv4 => qwp}/client/InFlightWindow.java | 2 +- .../client/MicrobatchBuffer.java | 4 +- .../client/NativeBufferWriter.java | 4 +- .../client/QwpBufferWriter.java} | 4 +- .../client/QwpWebSocketEncoder.java} | 72 +- .../client/QwpWebSocketSender.java} | 154 ++-- .../{ilpv4 => qwp}/client/ResponseReader.java | 2 +- .../client/WebSocketChannel.java | 12 +- .../client/WebSocketResponse.java | 2 +- .../client/WebSocketSendQueue.java | 2 +- .../protocol/QwpBitReader.java} | 8 +- .../protocol/QwpBitWriter.java} | 8 +- .../protocol/QwpColumnDef.java} | 18 +- .../protocol/QwpConstants.java} | 6 +- .../protocol/QwpGorillaEncoder.java} | 10 +- .../protocol/QwpNullBitmap.java} | 6 +- .../protocol/QwpSchemaHash.java} | 10 +- .../protocol/QwpTableBuffer.java} | 20 +- .../protocol/QwpVarint.java} | 6 +- .../protocol/QwpZigZag.java} | 6 +- .../websocket/WebSocketCloseCode.java | 2 +- .../websocket/WebSocketFrameParser.java | 2 +- .../websocket/WebSocketFrameWriter.java | 2 +- .../websocket/WebSocketHandshake.java | 2 +- .../websocket/WebSocketOpcode.java | 2 +- core/src/main/java/module-info.java | 6 +- .../qwp/client/InFlightWindowTest.java | 832 ++++++++++++++++++ 31 files changed, 1032 insertions(+), 200 deletions(-) rename core/src/main/java/io/questdb/client/cutlass/{ilpv4 => qwp}/client/GlobalSymbolDictionary.java (99%) rename core/src/main/java/io/questdb/client/cutlass/{ilpv4 => qwp}/client/InFlightWindow.java (99%) rename core/src/main/java/io/questdb/client/cutlass/{ilpv4 => qwp}/client/MicrobatchBuffer.java (99%) rename core/src/main/java/io/questdb/client/cutlass/{ilpv4 => qwp}/client/NativeBufferWriter.java (98%) rename core/src/main/java/io/questdb/client/cutlass/{ilpv4/client/IlpBufferWriter.java => qwp/client/QwpBufferWriter.java} (97%) rename core/src/main/java/io/questdb/client/cutlass/{ilpv4/client/IlpV4WebSocketEncoder.java => qwp/client/QwpWebSocketEncoder.java} (90%) rename core/src/main/java/io/questdb/client/cutlass/{ilpv4/client/IlpV4WebSocketSender.java => qwp/client/QwpWebSocketSender.java} (87%) rename core/src/main/java/io/questdb/client/cutlass/{ilpv4 => qwp}/client/ResponseReader.java (99%) rename core/src/main/java/io/questdb/client/cutlass/{ilpv4 => qwp}/client/WebSocketChannel.java (98%) rename core/src/main/java/io/questdb/client/cutlass/{ilpv4 => qwp}/client/WebSocketResponse.java (99%) rename core/src/main/java/io/questdb/client/cutlass/{ilpv4 => qwp}/client/WebSocketSendQueue.java (99%) rename core/src/main/java/io/questdb/client/cutlass/{ilpv4/protocol/IlpV4BitReader.java => qwp/protocol/QwpBitReader.java} (98%) rename core/src/main/java/io/questdb/client/cutlass/{ilpv4/protocol/IlpV4BitWriter.java => qwp/protocol/QwpBitWriter.java} (97%) rename core/src/main/java/io/questdb/client/cutlass/{ilpv4/protocol/IlpV4ColumnDef.java => qwp/protocol/QwpColumnDef.java} (89%) rename core/src/main/java/io/questdb/client/cutlass/{ilpv4/protocol/IlpV4Constants.java => qwp/protocol/QwpConstants.java} (99%) rename core/src/main/java/io/questdb/client/cutlass/{ilpv4/protocol/IlpV4GorillaEncoder.java => qwp/protocol/QwpGorillaEncoder.java} (97%) rename core/src/main/java/io/questdb/client/cutlass/{ilpv4/protocol/IlpV4NullBitmap.java => qwp/protocol/QwpNullBitmap.java} (98%) rename core/src/main/java/io/questdb/client/cutlass/{ilpv4/protocol/IlpV4SchemaHash.java => qwp/protocol/QwpSchemaHash.java} (98%) rename core/src/main/java/io/questdb/client/cutlass/{ilpv4/protocol/IlpV4TableBuffer.java => qwp/protocol/QwpTableBuffer.java} (98%) rename core/src/main/java/io/questdb/client/cutlass/{ilpv4/protocol/IlpV4Varint.java => qwp/protocol/QwpVarint.java} (98%) rename core/src/main/java/io/questdb/client/cutlass/{ilpv4/protocol/IlpV4ZigZag.java => qwp/protocol/QwpZigZag.java} (96%) rename core/src/main/java/io/questdb/client/cutlass/{ilpv4 => qwp}/websocket/WebSocketCloseCode.java (99%) rename core/src/main/java/io/questdb/client/cutlass/{ilpv4 => qwp}/websocket/WebSocketFrameParser.java (99%) rename core/src/main/java/io/questdb/client/cutlass/{ilpv4 => qwp}/websocket/WebSocketFrameWriter.java (99%) rename core/src/main/java/io/questdb/client/cutlass/{ilpv4 => qwp}/websocket/WebSocketHandshake.java (99%) rename core/src/main/java/io/questdb/client/cutlass/{ilpv4 => qwp}/websocket/WebSocketOpcode.java (98%) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/InFlightWindowTest.java diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index fcc6d46..b2b3c19 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -34,7 +34,7 @@ import io.questdb.client.cutlass.line.http.AbstractLineHttpSender; import io.questdb.client.cutlass.line.tcp.DelegatingTlsChannel; import io.questdb.client.cutlass.line.tcp.PlainTcpLineChannel; -import io.questdb.client.cutlass.ilpv4.client.IlpV4WebSocketSender; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; import io.questdb.client.impl.ConfStringParser; import io.questdb.client.network.NetworkFacade; import io.questdb.client.network.NetworkFacadeImpl; @@ -881,7 +881,7 @@ public Sender build() { int actualSendQueueCapacity = sendQueueCapacity == PARAMETER_NOT_SET_EXPLICITLY ? DEFAULT_SEND_QUEUE_CAPACITY : sendQueueCapacity; if (asyncMode) { - return IlpV4WebSocketSender.connectAsync( + return QwpWebSocketSender.connectAsync( hosts.getQuick(0), ports.getQuick(0), tlsEnabled, @@ -892,7 +892,7 @@ public Sender build() { actualSendQueueCapacity ); } else { - return IlpV4WebSocketSender.connect( + return QwpWebSocketSender.connect( hosts.getQuick(0), ports.getQuick(0), tlsEnabled diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java index cc64a63..ab8f696 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java @@ -25,10 +25,10 @@ package io.questdb.client.cutlass.http.client; import io.questdb.client.HttpClientConfiguration; -import io.questdb.client.cutlass.ilpv4.websocket.WebSocketCloseCode; -import io.questdb.client.cutlass.ilpv4.websocket.WebSocketFrameParser; -import io.questdb.client.cutlass.ilpv4.websocket.WebSocketHandshake; -import io.questdb.client.cutlass.ilpv4.websocket.WebSocketOpcode; +import io.questdb.client.cutlass.qwp.websocket.WebSocketCloseCode; +import io.questdb.client.cutlass.qwp.websocket.WebSocketFrameParser; +import io.questdb.client.cutlass.qwp.websocket.WebSocketHandshake; +import io.questdb.client.cutlass.qwp.websocket.WebSocketOpcode; import io.questdb.client.network.IOOperation; import io.questdb.client.network.NetworkFacade; import io.questdb.client.network.Socket; diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java index 20c43c1..5075b9f 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java @@ -24,9 +24,9 @@ package io.questdb.client.cutlass.http.client; -import io.questdb.client.cutlass.ilpv4.websocket.WebSocketFrameWriter; -import io.questdb.client.cutlass.ilpv4.websocket.WebSocketOpcode; -import io.questdb.client.cutlass.ilpv4.client.IlpBufferWriter; +import io.questdb.client.cutlass.qwp.websocket.WebSocketFrameWriter; +import io.questdb.client.cutlass.qwp.websocket.WebSocketOpcode; +import io.questdb.client.cutlass.qwp.client.QwpBufferWriter; import io.questdb.client.cutlass.line.array.ArrayBufferAppender; import io.questdb.client.std.MemoryTag; import io.questdb.client.std.Numbers; @@ -55,7 +55,7 @@ *

* Thread safety: This class is NOT thread-safe. Each connection should have its own buffer. */ -public class WebSocketSendBuffer implements IlpBufferWriter, QuietCloseable { +public class WebSocketSendBuffer implements QwpBufferWriter, QuietCloseable { // Maximum header size: 2 (base) + 8 (64-bit length) + 4 (mask key) private static final int MAX_HEADER_SIZE = 14; @@ -244,7 +244,7 @@ public void putAscii(CharSequence cs) { writePos += len; } - // === IlpBufferWriter Implementation === + // === QwpBufferWriter Implementation === /** * Writes an unsigned variable-length integer (LEB128 encoding). @@ -267,7 +267,7 @@ public void putString(String value) { putVarint(0); return; } - int utf8Len = IlpBufferWriter.utf8Length(value); + int utf8Len = QwpBufferWriter.utf8Length(value); putVarint(utf8Len); putUtf8(value); } diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/GlobalSymbolDictionary.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java similarity index 99% rename from core/src/main/java/io/questdb/client/cutlass/ilpv4/client/GlobalSymbolDictionary.java rename to core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java index 743c029..4d75211 100644 --- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/GlobalSymbolDictionary.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java @@ -22,7 +22,7 @@ * ******************************************************************************/ -package io.questdb.client.cutlass.ilpv4.client; +package io.questdb.client.cutlass.qwp.client; import io.questdb.client.std.CharSequenceIntHashMap; import io.questdb.client.std.ObjList; diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/InFlightWindow.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/InFlightWindow.java similarity index 99% rename from core/src/main/java/io/questdb/client/cutlass/ilpv4/client/InFlightWindow.java rename to core/src/main/java/io/questdb/client/cutlass/qwp/client/InFlightWindow.java index 9db5456..cffd93e 100644 --- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/InFlightWindow.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/InFlightWindow.java @@ -22,7 +22,7 @@ * ******************************************************************************/ -package io.questdb.client.cutlass.ilpv4.client; +package io.questdb.client.cutlass.qwp.client; import io.questdb.client.cutlass.line.LineSenderException; import org.slf4j.Logger; diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/MicrobatchBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java similarity index 99% rename from core/src/main/java/io/questdb/client/cutlass/ilpv4/client/MicrobatchBuffer.java rename to core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java index 832bb66..9bcf738 100644 --- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/MicrobatchBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java @@ -22,7 +22,7 @@ * ******************************************************************************/ -package io.questdb.client.cutlass.ilpv4.client; +package io.questdb.client.cutlass.qwp.client; import io.questdb.client.std.MemoryTag; import io.questdb.client.std.QuietCloseable; @@ -35,7 +35,7 @@ * A buffer for accumulating ILP data into microbatches before sending. *

* This class implements a state machine for buffer lifecycle management in the - * double-buffering scheme used by {@link IlpV4WebSocketSender}: + * double-buffering scheme used by {@link QwpWebSocketSender}: * 

  * Buffer States:
  * ┌─────────────┐    seal()     ┌─────────────┐    markSending()  ┌─────────────┐
diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/NativeBufferWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java
similarity index 98%
rename from core/src/main/java/io/questdb/client/cutlass/ilpv4/client/NativeBufferWriter.java
rename to core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java
index 5cab035..ee264ed 100644
--- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/NativeBufferWriter.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java
@@ -22,7 +22,7 @@
  *
  ******************************************************************************/
 
-package io.questdb.client.cutlass.ilpv4.client;
+package io.questdb.client.cutlass.qwp.client;
 
 import io.questdb.client.std.MemoryTag;
 import io.questdb.client.std.QuietCloseable;
@@ -36,7 +36,7 @@
  * 

  * All multi-byte values are written in little-endian format unless otherwise specified.
  */
-public class NativeBufferWriter implements IlpBufferWriter, QuietCloseable {
+public class NativeBufferWriter implements QwpBufferWriter, QuietCloseable {
 
     private static final int DEFAULT_CAPACITY = 8192;
 
diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/IlpBufferWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpBufferWriter.java
similarity index 97%
rename from core/src/main/java/io/questdb/client/cutlass/ilpv4/client/IlpBufferWriter.java
rename to core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpBufferWriter.java
index 1976cac..05ef0ea 100644
--- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/IlpBufferWriter.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpBufferWriter.java
@@ -22,7 +22,7 @@
  *
  ******************************************************************************/
 
-package io.questdb.client.cutlass.ilpv4.client;
+package io.questdb.client.cutlass.qwp.client;
 
 import io.questdb.client.cutlass.line.array.ArrayBufferAppender;
 
@@ -42,7 +42,7 @@
  * All multi-byte values are written in little-endian format unless the method
  * name explicitly indicates big-endian (e.g., {@link #putLongBE}).
  */
-public interface IlpBufferWriter extends ArrayBufferAppender {
+public interface QwpBufferWriter extends ArrayBufferAppender {
 
     // === Primitive writes (little-endian) ===
 
diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/IlpV4WebSocketEncoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java
similarity index 90%
rename from core/src/main/java/io/questdb/client/cutlass/ilpv4/client/IlpV4WebSocketEncoder.java
rename to core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java
index e98a1df..c71e158 100644
--- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/IlpV4WebSocketEncoder.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java
@@ -22,20 +22,20 @@
  *
  ******************************************************************************/
 
-package io.questdb.client.cutlass.ilpv4.client;
+package io.questdb.client.cutlass.qwp.client;
 
-import io.questdb.client.cutlass.ilpv4.protocol.IlpV4ColumnDef;
-import io.questdb.client.cutlass.ilpv4.protocol.IlpV4GorillaEncoder;
-import io.questdb.client.cutlass.ilpv4.protocol.IlpV4TableBuffer;
+import io.questdb.client.cutlass.qwp.protocol.QwpColumnDef;
+import io.questdb.client.cutlass.qwp.protocol.QwpGorillaEncoder;
+import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer;
 import io.questdb.client.std.QuietCloseable;
 
-import static io.questdb.client.cutlass.ilpv4.protocol.IlpV4Constants.*;
+import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.*;
 
 /**
  * Encodes ILP v4 messages for WebSocket transport.
  * 

  * This encoder can write to either an internal {@link NativeBufferWriter} (default)
- * or an external {@link IlpBufferWriter} such as {@link io.questdb.client.cutlass.http.client.WebSocketSendBuffer}.
+ * or an external {@link QwpBufferWriter} such as {@link io.questdb.client.cutlass.http.client.WebSocketSendBuffer}.
  * 

  * When using an external buffer, the encoder writes directly to it without intermediate copies,
  * enabling zero-copy WebSocket frame construction.
@@ -50,7 +50,7 @@
  * client.sendFrame(frame);
  *
*/ -public class IlpV4WebSocketEncoder implements QuietCloseable { +public class QwpWebSocketEncoder implements QuietCloseable { /** * Encoding flag for Gorilla-encoded timestamps. @@ -60,18 +60,18 @@ public class IlpV4WebSocketEncoder implements QuietCloseable { * Encoding flag for uncompressed timestamps. */ public static final byte ENCODING_UNCOMPRESSED = 0x00; - private final IlpV4GorillaEncoder gorillaEncoder = new IlpV4GorillaEncoder(); - private IlpBufferWriter buffer; + private final QwpGorillaEncoder gorillaEncoder = new QwpGorillaEncoder(); + private QwpBufferWriter buffer; private byte flags; private NativeBufferWriter ownedBuffer; - public IlpV4WebSocketEncoder() { + public QwpWebSocketEncoder() { this.ownedBuffer = new NativeBufferWriter(); this.buffer = ownedBuffer; this.flags = 0; } - public IlpV4WebSocketEncoder(int bufferSize) { + public QwpWebSocketEncoder(int bufferSize) { this.ownedBuffer = new NativeBufferWriter(bufferSize); this.buffer = ownedBuffer; this.flags = 0; @@ -92,7 +92,7 @@ public void close() { * @param useSchemaRef whether to use schema reference mode * @return the number of bytes written */ - public int encode(IlpV4TableBuffer tableBuffer, boolean useSchemaRef) { + public int encode(QwpTableBuffer tableBuffer, boolean useSchemaRef) { buffer.reset(); // Write message header with placeholder for payload length @@ -123,7 +123,7 @@ public int encode(IlpV4TableBuffer tableBuffer, boolean useSchemaRef) { * @return the number of bytes written */ public int encodeWithDeltaDict( - IlpV4TableBuffer tableBuffer, + QwpTableBuffer tableBuffer, GlobalSymbolDictionary globalDict, int confirmedMaxId, int batchMaxId, @@ -167,10 +167,10 @@ public int encodeWithDeltaDict( /** * Returns the underlying buffer. *

- * If an external buffer was set via {@link #setBuffer(IlpBufferWriter)}, + * If an external buffer was set via {@link #setBuffer(QwpBufferWriter)}, * that buffer is returned. Otherwise, returns the internal buffer. */ - public IlpBufferWriter getBuffer() { + public QwpBufferWriter getBuffer() { return buffer; } @@ -218,7 +218,7 @@ public void reset() { * * @param externalBuffer the external buffer to use, or null to use internal buffer */ - public void setBuffer(IlpBufferWriter externalBuffer) { + public void setBuffer(QwpBufferWriter externalBuffer) { this.buffer = externalBuffer != null ? externalBuffer : ownedBuffer; } @@ -273,7 +273,7 @@ public void writeHeader(int tableCount, int payloadLength) { /** * Encodes a single column. */ - private void encodeColumn(IlpV4TableBuffer.ColumnBuffer col, IlpV4ColumnDef colDef, int rowCount, boolean useGorilla) { + private void encodeColumn(QwpTableBuffer.ColumnBuffer col, QwpColumnDef colDef, int rowCount, boolean useGorilla) { int valueCount = col.getValueCount(); // Write null bitmap if column is nullable @@ -351,7 +351,7 @@ private void encodeColumn(IlpV4TableBuffer.ColumnBuffer col, IlpV4ColumnDef colD * Encodes a single column using global symbol IDs for SYMBOL type. * All other column types are encoded the same as encodeColumn. */ - private void encodeColumnWithGlobalSymbols(IlpV4TableBuffer.ColumnBuffer col, IlpV4ColumnDef colDef, int rowCount, boolean useGorilla) { + private void encodeColumnWithGlobalSymbols(QwpTableBuffer.ColumnBuffer col, QwpColumnDef colDef, int rowCount, boolean useGorilla) { int valueCount = col.getValueCount(); // Write null bitmap if column is nullable @@ -430,8 +430,8 @@ private void encodeColumnWithGlobalSymbols(IlpV4TableBuffer.ColumnBuffer col, Il /** * Encodes a single table from the buffer. */ - private void encodeTable(IlpV4TableBuffer tableBuffer, boolean useSchemaRef) { - IlpV4ColumnDef[] columnDefs = tableBuffer.getColumnDefs(); + private void encodeTable(QwpTableBuffer tableBuffer, boolean useSchemaRef) { + QwpColumnDef[] columnDefs = tableBuffer.getColumnDefs(); int rowCount = tableBuffer.getRowCount(); if (useSchemaRef) { @@ -448,8 +448,8 @@ private void encodeTable(IlpV4TableBuffer tableBuffer, boolean useSchemaRef) { // Write each column's data boolean useGorilla = isGorillaEnabled(); for (int i = 0; i < tableBuffer.getColumnCount(); i++) { - IlpV4TableBuffer.ColumnBuffer col = tableBuffer.getColumn(i); - IlpV4ColumnDef colDef = columnDefs[i]; + QwpTableBuffer.ColumnBuffer col = tableBuffer.getColumn(i); + QwpColumnDef colDef = columnDefs[i]; encodeColumn(col, colDef, rowCount, useGorilla); } } @@ -458,8 +458,8 @@ private void encodeTable(IlpV4TableBuffer tableBuffer, boolean useSchemaRef) { * Encodes a single table from the buffer using global symbol IDs. * This is used with delta dictionary encoding. */ - private void encodeTableWithGlobalSymbols(IlpV4TableBuffer tableBuffer, boolean useSchemaRef) { - IlpV4ColumnDef[] columnDefs = tableBuffer.getColumnDefs(); + private void encodeTableWithGlobalSymbols(QwpTableBuffer tableBuffer, boolean useSchemaRef) { + QwpColumnDef[] columnDefs = tableBuffer.getColumnDefs(); int rowCount = tableBuffer.getRowCount(); if (useSchemaRef) { @@ -476,8 +476,8 @@ private void encodeTableWithGlobalSymbols(IlpV4TableBuffer tableBuffer, boolean // Write each column's data boolean useGorilla = isGorillaEnabled(); for (int i = 0; i < tableBuffer.getColumnCount(); i++) { - IlpV4TableBuffer.ColumnBuffer col = tableBuffer.getColumn(i); - IlpV4ColumnDef colDef = columnDefs[i]; + QwpTableBuffer.ColumnBuffer col = tableBuffer.getColumn(i); + QwpColumnDef colDef = columnDefs[i]; encodeColumnWithGlobalSymbols(col, colDef, rowCount, useGorilla); } } @@ -531,7 +531,7 @@ private void writeDecimal64Column(byte scale, long[] values, int count) { } } - private void writeDoubleArrayColumn(IlpV4TableBuffer.ColumnBuffer col, int count) { + private void writeDoubleArrayColumn(QwpTableBuffer.ColumnBuffer col, int count) { byte[] dims = col.getArrayDims(); int[] shapes = col.getArrayShapes(); double[] data = col.getDoubleArrayData(); @@ -581,7 +581,7 @@ private void writeLong256Column(long[] values, int count) { } } - private void writeLongArrayColumn(IlpV4TableBuffer.ColumnBuffer col, int count) { + private void writeLongArrayColumn(QwpTableBuffer.ColumnBuffer col, int count) { byte[] dims = col.getArrayDims(); int[] shapes = col.getArrayShapes(); long[] data = col.getLongArrayData(); @@ -639,7 +639,7 @@ private void writeStringColumn(String[] strings, int count) { int totalDataLen = 0; for (int i = 0; i < count; i++) { if (strings[i] != null) { - totalDataLen += IlpBufferWriter.utf8Length(strings[i]); + totalDataLen += QwpBufferWriter.utf8Length(strings[i]); } } @@ -648,7 +648,7 @@ private void writeStringColumn(String[] strings, int count) { buffer.putInt(0); for (int i = 0; i < count; i++) { if (strings[i] != null) { - runningOffset += IlpBufferWriter.utf8Length(strings[i]); + runningOffset += QwpBufferWriter.utf8Length(strings[i]); } buffer.putInt(runningOffset); } @@ -668,7 +668,7 @@ private void writeStringColumn(String[] strings, int count) { * - Dictionary entries (length-prefixed UTF-8 strings) * - Symbol indices (varints, one per value) */ - private void writeSymbolColumn(IlpV4TableBuffer.ColumnBuffer col, int count) { + private void writeSymbolColumn(QwpTableBuffer.ColumnBuffer col, int count) { // Get symbol data from column buffer int[] symbolIndices = col.getSymbolIndices(); String[] dictionary = col.getSymbolDictionary(); @@ -693,7 +693,7 @@ private void writeSymbolColumn(IlpV4TableBuffer.ColumnBuffer col, int count) { * The dictionary is not included here because it's written at the message level * in delta format. */ - private void writeSymbolColumnWithGlobalIds(IlpV4TableBuffer.ColumnBuffer col, int count) { + private void writeSymbolColumnWithGlobalIds(QwpTableBuffer.ColumnBuffer col, int count) { int[] globalIds = col.getGlobalSymbolIds(); if (globalIds == null) { // Fall back to local indices if no global IDs stored @@ -712,7 +712,7 @@ private void writeSymbolColumnWithGlobalIds(IlpV4TableBuffer.ColumnBuffer col, i /** * Writes a table header with full schema. */ - private void writeTableHeaderWithSchema(String tableName, int rowCount, IlpV4ColumnDef[] columns) { + private void writeTableHeaderWithSchema(String tableName, int rowCount, QwpColumnDef[] columns) { // Table name buffer.putString(tableName); @@ -726,7 +726,7 @@ private void writeTableHeaderWithSchema(String tableName, int rowCount, IlpV4Col buffer.putByte(SCHEMA_MODE_FULL); // Column definitions (name + type for each) - for (IlpV4ColumnDef col : columns) { + for (QwpColumnDef col : columns) { buffer.putString(col.getName()); buffer.putByte(col.getWireTypeCode()); } @@ -760,12 +760,12 @@ private void writeTableHeaderWithSchemaRef(String tableName, int rowCount, long * Otherwise, falls back to uncompressed encoding. */ private void writeTimestampColumn(long[] values, int count, boolean useGorilla) { - if (useGorilla && count > 2 && IlpV4GorillaEncoder.canUseGorilla(values, count)) { + if (useGorilla && count > 2 && QwpGorillaEncoder.canUseGorilla(values, count)) { // Write Gorilla encoding flag buffer.putByte(ENCODING_GORILLA); // Calculate size needed and ensure buffer has capacity - int encodedSize = IlpV4GorillaEncoder.calculateEncodedSize(values, count); + int encodedSize = QwpGorillaEncoder.calculateEncodedSize(values, count); buffer.ensureCapacity(encodedSize); // Encode timestamps to buffer diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/IlpV4WebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java similarity index 87% rename from core/src/main/java/io/questdb/client/cutlass/ilpv4/client/IlpV4WebSocketSender.java rename to core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 43a0b4c..6b0343f 100644 --- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/IlpV4WebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -22,9 +22,9 @@ * ******************************************************************************/ -package io.questdb.client.cutlass.ilpv4.client; +package io.questdb.client.cutlass.qwp.client; -import io.questdb.client.cutlass.ilpv4.protocol.*; +import io.questdb.client.cutlass.qwp.protocol.*; import io.questdb.client.Sender; import io.questdb.client.cutlass.http.client.WebSocketClient; @@ -50,7 +50,7 @@ import java.time.temporal.ChronoUnit; import java.util.concurrent.TimeUnit; -import static io.questdb.client.cutlass.ilpv4.protocol.IlpV4Constants.*; +import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.*; /** * ILP v4 WebSocket client sender for streaming data to QuestDB. @@ -72,7 +72,7 @@ *

* Example usage: * 

- * try (IlpV4WebSocketSender sender = IlpV4WebSocketSender.connect("localhost", 9000)) {
+ * try (QwpWebSocketSender sender = QwpWebSocketSender.connect("localhost", 9000)) {
  *     for (int i = 0; i < 100_000; i++) {
  *         sender.table("metrics")
  *               .symbol("host", "server-" + (i % 10))
@@ -85,9 +85,9 @@
  * }
  *
*/ -public class IlpV4WebSocketSender implements Sender { +public class QwpWebSocketSender implements Sender { - private static final Logger LOG = LoggerFactory.getLogger(IlpV4WebSocketSender.class); + private static final Logger LOG = LoggerFactory.getLogger(QwpWebSocketSender.class); private static final int DEFAULT_BUFFER_SIZE = 8192; private static final int DEFAULT_MICROBATCH_BUFFER_SIZE = 1024 * 1024; // 1MB @@ -101,15 +101,15 @@ public class IlpV4WebSocketSender implements Sender { private final String host; private final int port; private final boolean tlsEnabled; - private final CharSequenceObjHashMap tableBuffers; - private IlpV4TableBuffer currentTableBuffer; + private final CharSequenceObjHashMap tableBuffers; + private QwpTableBuffer currentTableBuffer; private String currentTableName; // Cached column references to avoid repeated hashmap lookups - private IlpV4TableBuffer.ColumnBuffer cachedTimestampColumn; - private IlpV4TableBuffer.ColumnBuffer cachedTimestampNanosColumn; + private QwpTableBuffer.ColumnBuffer cachedTimestampColumn; + private QwpTableBuffer.ColumnBuffer cachedTimestampNanosColumn; // Encoder for ILP v4 messages - private final IlpV4WebSocketEncoder encoder; + private final QwpWebSocketEncoder encoder; // WebSocket client (zero-GC native implementation) private WebSocketClient client; @@ -159,13 +159,13 @@ public class IlpV4WebSocketSender implements Sender { // Combined key = schemaHash XOR (tableNameHash << 32) to include table name in lookup. private final LongHashSet sentSchemaHashes = new LongHashSet(); - private IlpV4WebSocketSender(String host, int port, boolean tlsEnabled, int bufferSize, + private QwpWebSocketSender(String host, int port, boolean tlsEnabled, int bufferSize, int autoFlushRows, int autoFlushBytes, long autoFlushIntervalNanos, int inFlightWindowSize, int sendQueueCapacity) { this.host = host; this.port = port; this.tlsEnabled = tlsEnabled; - this.encoder = new IlpV4WebSocketEncoder(bufferSize); + this.encoder = new QwpWebSocketEncoder(bufferSize); this.tableBuffers = new CharSequenceObjHashMap<>(); this.currentTableBuffer = null; this.currentTableName = null; @@ -197,7 +197,7 @@ private IlpV4WebSocketSender(String host, int port, boolean tlsEnabled, int buff * @param port server HTTP port (WebSocket upgrade happens on same port) * @return connected sender */ - public static IlpV4WebSocketSender connect(String host, int port) { + public static QwpWebSocketSender connect(String host, int port) { return connect(host, port, false); } @@ -210,8 +210,8 @@ public static IlpV4WebSocketSender connect(String host, int port) { * @param tlsEnabled whether to use TLS * @return connected sender */ - public static IlpV4WebSocketSender connect(String host, int port, boolean tlsEnabled) { - IlpV4WebSocketSender sender = new IlpV4WebSocketSender( + public static QwpWebSocketSender connect(String host, int port, boolean tlsEnabled) { + QwpWebSocketSender sender = new QwpWebSocketSender( host, port, tlsEnabled, DEFAULT_BUFFER_SIZE, 0, 0, 0, // No auto-flush in sync mode 1, 1 // window=1 for sync behavior, queue=1 (not used) @@ -231,7 +231,7 @@ public static IlpV4WebSocketSender connect(String host, int port, boolean tlsEna * @param autoFlushIntervalNanos age before flush in nanos (0 = no limit) * @return connected sender */ - public static IlpV4WebSocketSender connectAsync(String host, int port, boolean tlsEnabled, + public static QwpWebSocketSender connectAsync(String host, int port, boolean tlsEnabled, int autoFlushRows, int autoFlushBytes, long autoFlushIntervalNanos) { return connectAsync(host, port, tlsEnabled, autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, @@ -251,11 +251,11 @@ public static IlpV4WebSocketSender connectAsync(String host, int port, boolean t * @param sendQueueCapacity max batches waiting to send (default: 16) * @return connected sender */ - public static IlpV4WebSocketSender connectAsync(String host, int port, boolean tlsEnabled, + public static QwpWebSocketSender connectAsync(String host, int port, boolean tlsEnabled, int autoFlushRows, int autoFlushBytes, long autoFlushIntervalNanos, int inFlightWindowSize, int sendQueueCapacity) { - IlpV4WebSocketSender sender = new IlpV4WebSocketSender( + QwpWebSocketSender sender = new QwpWebSocketSender( host, port, tlsEnabled, DEFAULT_BUFFER_SIZE, autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, inFlightWindowSize, sendQueueCapacity @@ -272,7 +272,7 @@ public static IlpV4WebSocketSender connectAsync(String host, int port, boolean t * @param tlsEnabled whether to use TLS * @return connected sender */ - public static IlpV4WebSocketSender connectAsync(String host, int port, boolean tlsEnabled) { + public static QwpWebSocketSender connectAsync(String host, int port, boolean tlsEnabled) { return connectAsync(host, port, tlsEnabled, DEFAULT_AUTO_FLUSH_ROWS, DEFAULT_AUTO_FLUSH_BYTES, DEFAULT_AUTO_FLUSH_INTERVAL_NANOS); } @@ -280,7 +280,7 @@ public static IlpV4WebSocketSender connectAsync(String host, int port, boolean t /** * Factory method for SenderBuilder integration. */ - public static IlpV4WebSocketSender create( + public static QwpWebSocketSender create( String host, int port, boolean tlsEnabled, @@ -289,7 +289,7 @@ public static IlpV4WebSocketSender create( String username, String password ) { - IlpV4WebSocketSender sender = new IlpV4WebSocketSender( + QwpWebSocketSender sender = new QwpWebSocketSender( host, port, tlsEnabled, bufferSize, 0, 0, 0, 1, 1 // window=1 for sync behavior @@ -309,8 +309,8 @@ public static IlpV4WebSocketSender create( * @param inFlightWindowSize window size: 1 for sync behavior, >1 for async * @return unconnected sender */ - public static IlpV4WebSocketSender createForTesting(String host, int port, int inFlightWindowSize) { - return new IlpV4WebSocketSender( + public static QwpWebSocketSender createForTesting(String host, int port, int inFlightWindowSize) { + return new QwpWebSocketSender( host, port, false, DEFAULT_BUFFER_SIZE, 0, 0, 0, inFlightWindowSize, DEFAULT_SEND_QUEUE_CAPACITY @@ -330,11 +330,11 @@ public static IlpV4WebSocketSender createForTesting(String host, int port, int i * @param sendQueueCapacity max batches waiting to send * @return unconnected sender */ - public static IlpV4WebSocketSender createForTesting( + public static QwpWebSocketSender createForTesting( String host, int port, int autoFlushRows, int autoFlushBytes, long autoFlushIntervalNanos, int inFlightWindowSize, int sendQueueCapacity) { - return new IlpV4WebSocketSender( + return new QwpWebSocketSender( host, port, false, DEFAULT_BUFFER_SIZE, autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, inFlightWindowSize, sendQueueCapacity @@ -395,7 +395,7 @@ public boolean isGorillaEnabled() { /** * Sets whether to use Gorilla timestamp encoding. */ - public IlpV4WebSocketSender setGorillaEnabled(boolean enabled) { + public QwpWebSocketSender setGorillaEnabled(boolean enabled) { this.gorillaEnabled = enabled; this.encoder.setGorillaEnabled(enabled); return this; @@ -469,9 +469,9 @@ public int getMaxSentSymbolId() { // // Usage: // // Setup (once) - // IlpV4TableBuffer tableBuffer = sender.getTableBuffer("q"); - // IlpV4TableBuffer.ColumnBuffer colSymbol = tableBuffer.getOrCreateColumn("s", TYPE_SYMBOL, true); - // IlpV4TableBuffer.ColumnBuffer colBid = tableBuffer.getOrCreateColumn("b", TYPE_DOUBLE, false); + // QwpTableBuffer tableBuffer = sender.getTableBuffer("q"); + // QwpTableBuffer.ColumnBuffer colSymbol = tableBuffer.getOrCreateColumn("s", TYPE_SYMBOL, true); + // QwpTableBuffer.ColumnBuffer colBid = tableBuffer.getOrCreateColumn("b", TYPE_DOUBLE, false); // // // Hot path (per row) // colSymbol.addSymbolWithGlobalId(symbol, sender.getOrAddGlobalSymbol(symbol)); @@ -483,10 +483,10 @@ public int getMaxSentSymbolId() { * Gets or creates a table buffer for direct access. * For high-throughput generators that want to bypass fluent API overhead. */ - public IlpV4TableBuffer getTableBuffer(String tableName) { - IlpV4TableBuffer buffer = tableBuffers.get(tableName); + public QwpTableBuffer getTableBuffer(String tableName) { + QwpTableBuffer buffer = tableBuffers.get(tableName); if (buffer == null) { - buffer = new IlpV4TableBuffer(tableName); + buffer = new QwpTableBuffer(tableName); tableBuffers.put(tableName, buffer); } currentTableBuffer = buffer; @@ -531,7 +531,7 @@ public void incrementPendingRowCount() { // ==================== Sender interface implementation ==================== @Override - public IlpV4WebSocketSender table(CharSequence tableName) { + public QwpWebSocketSender table(CharSequence tableName) { checkNotClosed(); // Fast path: if table name matches current, skip hashmap lookup if (currentTableName != null && currentTableBuffer != null && Chars.equals(tableName, currentTableName)) { @@ -543,7 +543,7 @@ public IlpV4WebSocketSender table(CharSequence tableName) { currentTableName = tableName.toString(); currentTableBuffer = tableBuffers.get(currentTableName); if (currentTableBuffer == null) { - currentTableBuffer = new IlpV4TableBuffer(currentTableName); + currentTableBuffer = new QwpTableBuffer(currentTableName); tableBuffers.put(currentTableName, currentTableBuffer); } // Both modes accumulate rows until flush @@ -551,10 +551,10 @@ public IlpV4WebSocketSender table(CharSequence tableName) { } @Override - public IlpV4WebSocketSender symbol(CharSequence columnName, CharSequence value) { + public QwpWebSocketSender symbol(CharSequence columnName, CharSequence value) { checkNotClosed(); checkTableSelected(); - IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_SYMBOL, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_SYMBOL, true); if (value != null) { // Register symbol in global dictionary and track max ID for delta calculation @@ -572,19 +572,19 @@ public IlpV4WebSocketSender symbol(CharSequence columnName, CharSequence value) } @Override - public IlpV4WebSocketSender boolColumn(CharSequence columnName, boolean value) { + public QwpWebSocketSender boolColumn(CharSequence columnName, boolean value) { checkNotClosed(); checkTableSelected(); - IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_BOOLEAN, false); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_BOOLEAN, false); col.addBoolean(value); return this; } @Override - public IlpV4WebSocketSender longColumn(CharSequence columnName, long value) { + public QwpWebSocketSender longColumn(CharSequence columnName, long value) { checkNotClosed(); checkTableSelected(); - IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_LONG, false); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_LONG, false); col.addLong(value); return this; } @@ -596,28 +596,28 @@ public IlpV4WebSocketSender longColumn(CharSequence columnName, long value) { * @param value the int value * @return this sender for method chaining */ - public IlpV4WebSocketSender intColumn(CharSequence columnName, int value) { + public QwpWebSocketSender intColumn(CharSequence columnName, int value) { checkNotClosed(); checkTableSelected(); - IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_INT, false); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_INT, false); col.addInt(value); return this; } @Override - public IlpV4WebSocketSender doubleColumn(CharSequence columnName, double value) { + public QwpWebSocketSender doubleColumn(CharSequence columnName, double value) { checkNotClosed(); checkTableSelected(); - IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_DOUBLE, false); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_DOUBLE, false); col.addDouble(value); return this; } @Override - public IlpV4WebSocketSender stringColumn(CharSequence columnName, CharSequence value) { + public QwpWebSocketSender stringColumn(CharSequence columnName, CharSequence value) { checkNotClosed(); checkTableSelected(); - IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_STRING, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_STRING, true); col.addString(value != null ? value.toString() : null); return this; } @@ -629,10 +629,10 @@ public IlpV4WebSocketSender stringColumn(CharSequence columnName, CharSequence v * @param value the short value * @return this sender for method chaining */ - public IlpV4WebSocketSender shortColumn(CharSequence columnName, short value) { + public QwpWebSocketSender shortColumn(CharSequence columnName, short value) { checkNotClosed(); checkTableSelected(); - IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_SHORT, false); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_SHORT, false); col.addShort(value); return this; } @@ -646,10 +646,10 @@ public IlpV4WebSocketSender shortColumn(CharSequence columnName, short value) { * @param value the character value * @return this sender for method chaining */ - public IlpV4WebSocketSender charColumn(CharSequence columnName, char value) { + public QwpWebSocketSender charColumn(CharSequence columnName, char value) { checkNotClosed(); checkTableSelected(); - IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_CHAR, false); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_CHAR, false); col.addShort((short) value); return this; } @@ -662,10 +662,10 @@ public IlpV4WebSocketSender charColumn(CharSequence columnName, char value) { * @param hi the high 64 bits of the UUID * @return this sender for method chaining */ - public IlpV4WebSocketSender uuidColumn(CharSequence columnName, long lo, long hi) { + public QwpWebSocketSender uuidColumn(CharSequence columnName, long lo, long hi) { checkNotClosed(); checkTableSelected(); - IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_UUID, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_UUID, true); col.addUuid(hi, lo); return this; } @@ -680,35 +680,35 @@ public IlpV4WebSocketSender uuidColumn(CharSequence columnName, long lo, long hi * @param l3 the most significant 64 bits * @return this sender for method chaining */ - public IlpV4WebSocketSender long256Column(CharSequence columnName, long l0, long l1, long l2, long l3) { + public QwpWebSocketSender long256Column(CharSequence columnName, long l0, long l1, long l2, long l3) { checkNotClosed(); checkTableSelected(); - IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_LONG256, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_LONG256, true); col.addLong256(l0, l1, l2, l3); return this; } @Override - public IlpV4WebSocketSender timestampColumn(CharSequence columnName, long value, ChronoUnit unit) { + public QwpWebSocketSender timestampColumn(CharSequence columnName, long value, ChronoUnit unit) { checkNotClosed(); checkTableSelected(); if (unit == ChronoUnit.NANOS) { - IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_TIMESTAMP_NANOS, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_TIMESTAMP_NANOS, true); col.addLong(value); } else { long micros = toMicros(value, unit); - IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_TIMESTAMP, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_TIMESTAMP, true); col.addLong(micros); } return this; } @Override - public IlpV4WebSocketSender timestampColumn(CharSequence columnName, Instant value) { + public QwpWebSocketSender timestampColumn(CharSequence columnName, Instant value) { checkNotClosed(); checkTableSelected(); long micros = value.getEpochSecond() * 1_000_000L + value.getNano() / 1000L; - IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_TIMESTAMP, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_TIMESTAMP, true); col.addLong(micros); return this; } @@ -832,7 +832,7 @@ private void flushPendingRows() { if (tableName == null) { continue; // Skip null entries (shouldn't happen but be safe) } - IlpV4TableBuffer tableBuffer = tableBuffers.get(tableName); + QwpTableBuffer tableBuffer = tableBuffers.get(tableName); if (tableBuffer == null) { continue; } @@ -859,7 +859,7 @@ private void flushPendingRows() { if (!useSchemaRef) { sentSchemaHashes.add(schemaKey); } - IlpBufferWriter buffer = encoder.getBuffer(); + QwpBufferWriter buffer = encoder.getBuffer(); // Copy to microbatch buffer and seal immediately // Each ILP v4 message must be in its own WebSocket frame @@ -1031,7 +1031,7 @@ private void flushSync() { if (tableName == null) { continue; } - IlpV4TableBuffer tableBuffer = tableBuffers.get(tableName); + QwpTableBuffer tableBuffer = tableBuffers.get(tableName); if (tableBuffer == null || tableBuffer.getRowCount() == 0) { continue; } @@ -1057,7 +1057,7 @@ private void flushSync() { } if (messageSize > 0) { - IlpBufferWriter buffer = encoder.getBuffer(); + QwpBufferWriter buffer = encoder.getBuffer(); // Track batch in InFlightWindow before sending long batchSequence = nextBatchSequence++; @@ -1192,7 +1192,7 @@ public Sender doubleArray(@NotNull CharSequence name, double[] values) { if (values == null) return this; checkNotClosed(); checkTableSelected(); - IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DOUBLE_ARRAY, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DOUBLE_ARRAY, true); col.addDoubleArray(values); return this; } @@ -1202,7 +1202,7 @@ public Sender doubleArray(@NotNull CharSequence name, double[][] values) { if (values == null) return this; checkNotClosed(); checkTableSelected(); - IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DOUBLE_ARRAY, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DOUBLE_ARRAY, true); col.addDoubleArray(values); return this; } @@ -1212,7 +1212,7 @@ public Sender doubleArray(@NotNull CharSequence name, double[][][] values) { if (values == null) return this; checkNotClosed(); checkTableSelected(); - IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DOUBLE_ARRAY, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DOUBLE_ARRAY, true); col.addDoubleArray(values); return this; } @@ -1222,7 +1222,7 @@ public Sender doubleArray(CharSequence name, DoubleArray array) { if (array == null) return this; checkNotClosed(); checkTableSelected(); - IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DOUBLE_ARRAY, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DOUBLE_ARRAY, true); col.addDoubleArray(array); return this; } @@ -1232,7 +1232,7 @@ public Sender longArray(@NotNull CharSequence name, long[] values) { if (values == null) return this; checkNotClosed(); checkTableSelected(); - IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_LONG_ARRAY, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_LONG_ARRAY, true); col.addLongArray(values); return this; } @@ -1242,7 +1242,7 @@ public Sender longArray(@NotNull CharSequence name, long[][] values) { if (values == null) return this; checkNotClosed(); checkTableSelected(); - IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_LONG_ARRAY, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_LONG_ARRAY, true); col.addLongArray(values); return this; } @@ -1252,7 +1252,7 @@ public Sender longArray(@NotNull CharSequence name, long[][][] values) { if (values == null) return this; checkNotClosed(); checkTableSelected(); - IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_LONG_ARRAY, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_LONG_ARRAY, true); col.addLongArray(values); return this; } @@ -1262,7 +1262,7 @@ public Sender longArray(CharSequence name, LongArray array) { if (array == null) return this; checkNotClosed(); checkTableSelected(); - IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_LONG_ARRAY, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_LONG_ARRAY, true); col.addLongArray(array); return this; } @@ -1274,7 +1274,7 @@ public Sender decimalColumn(CharSequence name, Decimal64 value) { if (value == null || value.isNull()) return this; checkNotClosed(); checkTableSelected(); - IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DECIMAL64, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DECIMAL64, true); col.addDecimal64(value); return this; } @@ -1284,7 +1284,7 @@ public Sender decimalColumn(CharSequence name, Decimal128 value) { if (value == null || value.isNull()) return this; checkNotClosed(); checkTableSelected(); - IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DECIMAL128, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DECIMAL128, true); col.addDecimal128(value); return this; } @@ -1294,7 +1294,7 @@ public Sender decimalColumn(CharSequence name, Decimal256 value) { if (value == null || value.isNull()) return this; checkNotClosed(); checkTableSelected(); - IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DECIMAL256, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DECIMAL256, true); col.addDecimal256(value); return this; } @@ -1307,7 +1307,7 @@ public Sender decimalColumn(CharSequence name, CharSequence value) { try { java.math.BigDecimal bd = new java.math.BigDecimal(value.toString()); Decimal256 decimal = Decimal256.fromBigDecimal(bd); - IlpV4TableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DECIMAL256, true); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DECIMAL256, true); col.addDecimal256(decimal); } catch (Exception e) { throw new LineSenderException("Failed to parse decimal value: " + value, e); @@ -1392,7 +1392,7 @@ public void close() { encoder.close(); tableBuffers.clear(); - LOG.info("IlpV4WebSocketSender closed"); + LOG.info("QwpWebSocketSender closed"); } } } diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/ResponseReader.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/ResponseReader.java similarity index 99% rename from core/src/main/java/io/questdb/client/cutlass/ilpv4/client/ResponseReader.java rename to core/src/main/java/io/questdb/client/cutlass/qwp/client/ResponseReader.java index dca05f2..2b78429 100644 --- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/ResponseReader.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/ResponseReader.java @@ -22,7 +22,7 @@ * ******************************************************************************/ -package io.questdb.client.cutlass.ilpv4.client; +package io.questdb.client.cutlass.qwp.client; import io.questdb.client.cutlass.line.LineSenderException; import org.slf4j.Logger; diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/WebSocketChannel.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketChannel.java similarity index 98% rename from core/src/main/java/io/questdb/client/cutlass/ilpv4/client/WebSocketChannel.java rename to core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketChannel.java index f5be4a4..8774ac2 100644 --- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/WebSocketChannel.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketChannel.java @@ -22,13 +22,13 @@ * ******************************************************************************/ -package io.questdb.client.cutlass.ilpv4.client; +package io.questdb.client.cutlass.qwp.client; -import io.questdb.client.cutlass.ilpv4.websocket.WebSocketCloseCode; -import io.questdb.client.cutlass.ilpv4.websocket.WebSocketFrameParser; -import io.questdb.client.cutlass.ilpv4.websocket.WebSocketFrameWriter; -import io.questdb.client.cutlass.ilpv4.websocket.WebSocketHandshake; -import io.questdb.client.cutlass.ilpv4.websocket.WebSocketOpcode; +import io.questdb.client.cutlass.qwp.websocket.WebSocketCloseCode; +import io.questdb.client.cutlass.qwp.websocket.WebSocketFrameParser; +import io.questdb.client.cutlass.qwp.websocket.WebSocketFrameWriter; +import io.questdb.client.cutlass.qwp.websocket.WebSocketHandshake; +import io.questdb.client.cutlass.qwp.websocket.WebSocketOpcode; import io.questdb.client.cutlass.line.LineSenderException; import io.questdb.client.std.MemoryTag; import io.questdb.client.std.QuietCloseable; diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/WebSocketResponse.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java similarity index 99% rename from core/src/main/java/io/questdb/client/cutlass/ilpv4/client/WebSocketResponse.java rename to core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java index 2c29baa..35a5f77 100644 --- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/WebSocketResponse.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java @@ -22,7 +22,7 @@ * ******************************************************************************/ -package io.questdb.client.cutlass.ilpv4.client; +package io.questdb.client.cutlass.qwp.client; import io.questdb.client.std.Unsafe; diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/WebSocketSendQueue.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketSendQueue.java similarity index 99% rename from core/src/main/java/io/questdb/client/cutlass/ilpv4/client/WebSocketSendQueue.java rename to core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketSendQueue.java index b34926e..e47c1d8 100644 --- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/client/WebSocketSendQueue.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketSendQueue.java @@ -22,7 +22,7 @@ * ******************************************************************************/ -package io.questdb.client.cutlass.ilpv4.client; +package io.questdb.client.cutlass.qwp.client; import io.questdb.client.cutlass.http.client.WebSocketClient; import io.questdb.client.cutlass.http.client.WebSocketFrameHandler; diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4BitReader.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitReader.java similarity index 98% rename from core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4BitReader.java rename to core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitReader.java index 158ec3d..bd99092 100644 --- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4BitReader.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitReader.java @@ -22,7 +22,7 @@ * ******************************************************************************/ -package io.questdb.client.cutlass.ilpv4.protocol; +package io.questdb.client.cutlass.qwp.protocol; import io.questdb.client.std.Unsafe; @@ -36,14 +36,14 @@ *

* Usage pattern: * 

- * IlpV4BitReader reader = new IlpV4BitReader();
+ * QwpBitReader reader = new QwpBitReader();
  * reader.reset(address, length);
  * int bit = reader.readBit();
  * long value = reader.readBits(numBits);
  * long signedValue = reader.readSigned(numBits);
  *
*/ -public class IlpV4BitReader { +public class QwpBitReader { private long startAddress; private long currentAddress; @@ -61,7 +61,7 @@ public class IlpV4BitReader { /** * Creates a new bit reader. Call {@link #reset} before use. */ - public IlpV4BitReader() { + public QwpBitReader() { } /** diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4BitWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitWriter.java similarity index 97% rename from core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4BitWriter.java rename to core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitWriter.java index 8be1600..624b083 100644 --- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4BitWriter.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitWriter.java @@ -22,7 +22,7 @@ * ******************************************************************************/ -package io.questdb.client.cutlass.ilpv4.protocol; +package io.questdb.client.cutlass.qwp.protocol; import io.questdb.client.std.Unsafe; @@ -37,7 +37,7 @@ *

* Usage pattern: * 

- * IlpV4BitWriter writer = new IlpV4BitWriter();
+ * QwpBitWriter writer = new QwpBitWriter();
  * writer.reset(address, capacity);
  * writer.writeBits(value, numBits);
  * writer.writeBits(value2, numBits2);
@@ -45,7 +45,7 @@
  * long bytesWritten = writer.getPosition() - address;
  *
*/ -public class IlpV4BitWriter { +public class QwpBitWriter { private long startAddress; private long currentAddress; @@ -59,7 +59,7 @@ public class IlpV4BitWriter { /** * Creates a new bit writer. Call {@link #reset} before use. */ - public IlpV4BitWriter() { + public QwpBitWriter() { } /** diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4ColumnDef.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java similarity index 89% rename from core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4ColumnDef.java rename to core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java index cea87af..b9d9a26 100644 --- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4ColumnDef.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java @@ -22,16 +22,16 @@ * ******************************************************************************/ -package io.questdb.client.cutlass.ilpv4.protocol; +package io.questdb.client.cutlass.qwp.protocol; -import static io.questdb.client.cutlass.ilpv4.protocol.IlpV4Constants.*; +import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.*; /** * Represents a column definition in an ILP v4 schema. *

* This class is immutable and safe for caching. */ -public final class IlpV4ColumnDef { +public final class QwpColumnDef { private final String name; private final byte typeCode; private final boolean nullable; @@ -42,7 +42,7 @@ public final class IlpV4ColumnDef { * @param name the column name (UTF-8) * @param typeCode the ILP v4 type code (0x01-0x0F, optionally OR'd with 0x80 for nullable) */ - public IlpV4ColumnDef(String name, byte typeCode) { + public QwpColumnDef(String name, byte typeCode) { this.name = name; // Extract nullable flag (high bit) and base type this.nullable = (typeCode & 0x80) != 0; @@ -56,7 +56,7 @@ public IlpV4ColumnDef(String name, byte typeCode) { * @param typeCode the base type code (0x01-0x0F) * @param nullable whether the column is nullable */ - public IlpV4ColumnDef(String name, byte typeCode, boolean nullable) { + public QwpColumnDef(String name, byte typeCode, boolean nullable) { this.name = name; this.typeCode = (byte) (typeCode & 0x7F); this.nullable = nullable; @@ -98,7 +98,7 @@ public boolean isNullable() { * Returns true if this is a fixed-width type. */ public boolean isFixedWidth() { - return IlpV4Constants.isFixedWidthType(typeCode); + return QwpConstants.isFixedWidthType(typeCode); } /** @@ -107,14 +107,14 @@ public boolean isFixedWidth() { * @return width in bytes, or -1 for variable-width types */ public int getFixedWidth() { - return IlpV4Constants.getFixedTypeSize(typeCode); + return QwpConstants.getFixedTypeSize(typeCode); } /** * Gets the type name for display purposes. */ public String getTypeName() { - return IlpV4Constants.getTypeName(typeCode); + return QwpConstants.getTypeName(typeCode); } /** @@ -137,7 +137,7 @@ public void validate() { public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - IlpV4ColumnDef that = (IlpV4ColumnDef) o; + QwpColumnDef that = (QwpColumnDef) o; return typeCode == that.typeCode && nullable == that.nullable && name.equals(that.name); diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4Constants.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java similarity index 99% rename from core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4Constants.java rename to core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java index 7b229f6..622dbdd 100644 --- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4Constants.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java @@ -22,12 +22,12 @@ * ******************************************************************************/ -package io.questdb.client.cutlass.ilpv4.protocol; +package io.questdb.client.cutlass.qwp.protocol; /** * Constants for the ILP v4 binary protocol. */ -public final class IlpV4Constants { +public final class QwpConstants { // ==================== Magic Bytes ==================== @@ -360,7 +360,7 @@ public final class IlpV4Constants { */ public static final int CAPABILITY_RESPONSE_SIZE = 8; - private IlpV4Constants() { + private QwpConstants() { // utility class } diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4GorillaEncoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java similarity index 97% rename from core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4GorillaEncoder.java rename to core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java index 84e4334..5281dbc 100644 --- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4GorillaEncoder.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java @@ -22,7 +22,7 @@ * ******************************************************************************/ -package io.questdb.client.cutlass.ilpv4.protocol; +package io.questdb.client.cutlass.qwp.protocol; import io.questdb.client.std.Unsafe; @@ -44,7 +44,7 @@ * The encoder writes first two timestamps uncompressed, then encodes * remaining timestamps using delta-of-delta compression. */ -public class IlpV4GorillaEncoder { +public class QwpGorillaEncoder { private static final int BUCKET_12BIT_MAX = 2048; private static final int BUCKET_12BIT_MIN = -2047; @@ -53,12 +53,12 @@ public class IlpV4GorillaEncoder { private static final int BUCKET_7BIT_MIN = -63; private static final int BUCKET_9BIT_MAX = 256; private static final int BUCKET_9BIT_MIN = -255; - private final IlpV4BitWriter bitWriter = new IlpV4BitWriter(); + private final QwpBitWriter bitWriter = new QwpBitWriter(); /** * Creates a new Gorilla encoder. */ - public IlpV4GorillaEncoder() { + public QwpGorillaEncoder() { } /** @@ -166,7 +166,7 @@ public int encodeTimestamps(long destAddress, long capacity, long[] timestamps, return 0; } - int pos = 0; + int pos; // Write first timestamp uncompressed if (capacity < 8) { diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4NullBitmap.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpNullBitmap.java similarity index 98% rename from core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4NullBitmap.java rename to core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpNullBitmap.java index 738f2dd..90cf944 100644 --- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4NullBitmap.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpNullBitmap.java @@ -22,7 +22,7 @@ * ******************************************************************************/ -package io.questdb.client.cutlass.ilpv4.protocol; +package io.questdb.client.cutlass.qwp.protocol; import io.questdb.client.std.Unsafe; @@ -42,9 +42,9 @@ * Byte 1: 0b00000010 (bit 1 set, which is row 9) * */ -public final class IlpV4NullBitmap { +public final class QwpNullBitmap { - private IlpV4NullBitmap() { + private QwpNullBitmap() { // utility class } diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4SchemaHash.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java similarity index 98% rename from core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4SchemaHash.java rename to core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java index 9996d7f..9edc3f6 100644 --- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4SchemaHash.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java @@ -22,7 +22,7 @@ * ******************************************************************************/ -package io.questdb.client.cutlass.ilpv4.protocol; +package io.questdb.client.cutlass.qwp.protocol; import io.questdb.client.std.Unsafe; @@ -41,7 +41,7 @@ * * @see xxHash */ -public final class IlpV4SchemaHash { +public final class QwpSchemaHash { // XXHash64 constants private static final long PRIME64_1 = 0x9E3779B185EBCA87L; @@ -56,7 +56,7 @@ public final class IlpV4SchemaHash { // Thread-local Hasher to avoid allocation on every computeSchemaHash call private static final ThreadLocal HASHER_POOL = ThreadLocal.withInitial(Hasher::new); - private IlpV4SchemaHash() { + private QwpSchemaHash() { // utility class } @@ -340,13 +340,13 @@ public static long computeSchemaHash(DirectUtf8Sequence[] columnNames, byte[] co * @param columns list of column buffers * @return the schema hash */ - public static long computeSchemaHashDirect(io.questdb.client.std.ObjList columns) { + public static long computeSchemaHashDirect(io.questdb.client.std.ObjList columns) { // Use pooled hasher to avoid allocation Hasher hasher = HASHER_POOL.get(); hasher.reset(DEFAULT_SEED); for (int i = 0, n = columns.size(); i < n; i++) { - IlpV4TableBuffer.ColumnBuffer col = columns.get(i); + QwpTableBuffer.ColumnBuffer col = columns.get(i); String name = col.getName(); // Encode UTF-8 directly without allocating byte array for (int j = 0, len = name.length(); j < len; j++) { diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4TableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java similarity index 98% rename from core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4TableBuffer.java rename to core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index 6d1af59..68f75ec 100644 --- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4TableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -22,7 +22,7 @@ * ******************************************************************************/ -package io.questdb.client.cutlass.ilpv4.protocol; +package io.questdb.client.cutlass.qwp.protocol; import io.questdb.client.cutlass.line.LineSenderException; import io.questdb.client.cutlass.line.array.ArrayBufferAppender; @@ -38,7 +38,7 @@ import java.util.Arrays; -import static io.questdb.client.cutlass.ilpv4.protocol.IlpV4Constants.*; +import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.*; /** * Buffers rows for a single table in columnar format. @@ -46,7 +46,7 @@ * This buffer accumulates row data column by column, allowing efficient * encoding to the ILP v4 wire format. */ -public class IlpV4TableBuffer { +public class QwpTableBuffer { private final String tableName; private final ObjList columns; @@ -56,10 +56,10 @@ public class IlpV4TableBuffer { private int rowCount; private long schemaHash; private boolean schemaHashComputed; - private IlpV4ColumnDef[] cachedColumnDefs; + private QwpColumnDef[] cachedColumnDefs; private boolean columnDefsCacheValid; - public IlpV4TableBuffer(String tableName) { + public QwpTableBuffer(String tableName) { this.tableName = tableName; this.columns = new ObjList<>(); this.columnNameToIndex = new CharSequenceIntHashMap(); @@ -100,12 +100,12 @@ public ColumnBuffer getColumn(int index) { /** * Returns the column definitions (cached for efficiency). */ - public IlpV4ColumnDef[] getColumnDefs() { + public QwpColumnDef[] getColumnDefs() { if (!columnDefsCacheValid || cachedColumnDefs == null || cachedColumnDefs.length != columns.size()) { - cachedColumnDefs = new IlpV4ColumnDef[columns.size()]; + cachedColumnDefs = new QwpColumnDef[columns.size()]; for (int i = 0; i < columns.size(); i++) { ColumnBuffer col = columns.get(i); - cachedColumnDefs[i] = new IlpV4ColumnDef(col.name, col.type, col.nullable); + cachedColumnDefs[i] = new QwpColumnDef(col.name, col.type, col.nullable); } columnDefsCacheValid = true; } @@ -204,14 +204,14 @@ public void cancelCurrentRow() { /** * Returns the schema hash for this table. *

- * The hash is computed to match what IlpV4Schema.computeSchemaHash() produces: + * The hash is computed to match what QwpSchema.computeSchemaHash() produces: * - Uses wire type codes (with nullable bit) * - Hash is over name bytes + type code for each column */ public long getSchemaHash() { if (!schemaHashComputed) { // Compute hash directly from column buffers without intermediate arrays - schemaHash = IlpV4SchemaHash.computeSchemaHashDirect(columns); + schemaHash = QwpSchemaHash.computeSchemaHashDirect(columns); schemaHashComputed = true; } return schemaHash; diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4Varint.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpVarint.java similarity index 98% rename from core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4Varint.java rename to core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpVarint.java index cd150d4..3b98a70 100644 --- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4Varint.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpVarint.java @@ -22,7 +22,7 @@ * ******************************************************************************/ -package io.questdb.client.cutlass.ilpv4.protocol; +package io.questdb.client.cutlass.qwp.protocol; import io.questdb.client.std.Unsafe; @@ -38,7 +38,7 @@ *

* This implementation is designed for zero-allocation on hot paths. */ -public final class IlpV4Varint { +public final class QwpVarint { /** * Maximum number of bytes needed to encode a 64-bit varint. @@ -56,7 +56,7 @@ public final class IlpV4Varint { */ private static final int DATA_MASK = 0x7F; - private IlpV4Varint() { + private QwpVarint() { // utility class } diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4ZigZag.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpZigZag.java similarity index 96% rename from core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4ZigZag.java rename to core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpZigZag.java index b0542e2..44e596d 100644 --- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/protocol/IlpV4ZigZag.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpZigZag.java @@ -22,7 +22,7 @@ * ******************************************************************************/ -package io.questdb.client.cutlass.ilpv4.protocol; +package io.questdb.client.cutlass.qwp.protocol; /** * ZigZag encoding/decoding for signed integers. @@ -50,9 +50,9 @@ * negative numbers like -1 become small positive numbers (1), which * encode efficiently as varints. */ -public final class IlpV4ZigZag { +public final class QwpZigZag { - private IlpV4ZigZag() { + private QwpZigZag() { // utility class } diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketCloseCode.java b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketCloseCode.java similarity index 99% rename from core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketCloseCode.java rename to core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketCloseCode.java index 2cb001c..629767f 100644 --- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketCloseCode.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketCloseCode.java @@ -22,7 +22,7 @@ * ******************************************************************************/ -package io.questdb.client.cutlass.ilpv4.websocket; +package io.questdb.client.cutlass.qwp.websocket; /** * WebSocket close status codes as defined in RFC 6455. diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketFrameParser.java b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameParser.java similarity index 99% rename from core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketFrameParser.java rename to core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameParser.java index 53d02b3..fb980ec 100644 --- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketFrameParser.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameParser.java @@ -22,7 +22,7 @@ * ******************************************************************************/ -package io.questdb.client.cutlass.ilpv4.websocket; +package io.questdb.client.cutlass.qwp.websocket; import io.questdb.client.std.Unsafe; diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketFrameWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameWriter.java similarity index 99% rename from core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketFrameWriter.java rename to core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameWriter.java index d94bcd5..e4d423b 100644 --- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketFrameWriter.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameWriter.java @@ -22,7 +22,7 @@ * ******************************************************************************/ -package io.questdb.client.cutlass.ilpv4.websocket; +package io.questdb.client.cutlass.qwp.websocket; import io.questdb.client.std.Unsafe; diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketHandshake.java b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketHandshake.java similarity index 99% rename from core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketHandshake.java rename to core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketHandshake.java index 5248942..0bbcddd 100644 --- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketHandshake.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketHandshake.java @@ -22,7 +22,7 @@ * ******************************************************************************/ -package io.questdb.client.cutlass.ilpv4.websocket; +package io.questdb.client.cutlass.qwp.websocket; import io.questdb.client.std.Unsafe; import io.questdb.client.std.str.Utf8Sequence; diff --git a/core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketOpcode.java b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketOpcode.java similarity index 98% rename from core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketOpcode.java rename to core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketOpcode.java index 03fe188..40466ec 100644 --- a/core/src/main/java/io/questdb/client/cutlass/ilpv4/websocket/WebSocketOpcode.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketOpcode.java @@ -22,7 +22,7 @@ * ******************************************************************************/ -package io.questdb.client.cutlass.ilpv4.websocket; +package io.questdb.client.cutlass.qwp.websocket; /** * WebSocket frame opcodes as defined in RFC 6455. diff --git a/core/src/main/java/module-info.java b/core/src/main/java/module-info.java index 45b319c..cf4c93b 100644 --- a/core/src/main/java/module-info.java +++ b/core/src/main/java/module-info.java @@ -56,7 +56,7 @@ exports io.questdb.client.cairo.arr; exports io.questdb.client.cutlass.line.array; exports io.questdb.client.cutlass.line.udp; - exports io.questdb.client.cutlass.ilpv4.client; - exports io.questdb.client.cutlass.ilpv4.protocol; - exports io.questdb.client.cutlass.ilpv4.websocket; + exports io.questdb.client.cutlass.qwp.client; + exports io.questdb.client.cutlass.qwp.protocol; + exports io.questdb.client.cutlass.qwp.websocket; } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/InFlightWindowTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/InFlightWindowTest.java new file mode 100644 index 0000000..023cf13 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/InFlightWindowTest.java @@ -0,0 +1,832 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.InFlightWindow; +import org.junit.Test; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.Assert.*; + +/** + * Tests for InFlightWindow. + *

+ * The window assumes sequential batch IDs and cumulative acknowledgments. It + * tracks only the range [lastAcked+1, highestSent] rather than individual batch + * IDs. + */ +public class InFlightWindowTest { + + @Test + public void testBasicAddAndAcknowledge() { + InFlightWindow window = new InFlightWindow(8, 1000); + + assertTrue(window.isEmpty()); + assertEquals(0, window.getInFlightCount()); + + // Add a batch (sequential: 0) + window.addInFlight(0); + assertFalse(window.isEmpty()); + assertEquals(1, window.getInFlightCount()); + + // Acknowledge it (cumulative ACK up to 0) + assertTrue(window.acknowledge(0)); + assertTrue(window.isEmpty()); + assertEquals(0, window.getInFlightCount()); + assertEquals(1, window.getTotalAcked()); + } + + @Test + public void testMultipleBatches() { + InFlightWindow window = new InFlightWindow(8, 1000); + + // Add sequential batches 0-4 + for (long i = 0; i < 5; i++) { + window.addInFlight(i); + } + assertEquals(5, window.getInFlightCount()); + + // Cumulative ACK up to 2 (acknowledges 0, 1, 2) + assertEquals(3, window.acknowledgeUpTo(2)); + assertEquals(2, window.getInFlightCount()); + + // Cumulative ACK up to 4 (acknowledges 3, 4) + assertEquals(2, window.acknowledgeUpTo(4)); + assertTrue(window.isEmpty()); + assertEquals(5, window.getTotalAcked()); + } + + @Test + public void testAcknowledgeAlreadyAcked() { + InFlightWindow window = new InFlightWindow(8, 1000); + + window.addInFlight(0); + window.addInFlight(1); + + // ACK up to 1 + assertTrue(window.acknowledge(1)); + assertTrue(window.isEmpty()); + + // ACK for already acknowledged sequence returns true (idempotent) + assertTrue(window.acknowledge(0)); + assertTrue(window.acknowledge(1)); + assertTrue(window.isEmpty()); + } + + @Test + public void testWindowFull() { + InFlightWindow window = new InFlightWindow(3, 1000); + + // Fill the window + window.addInFlight(0); + window.addInFlight(1); + window.addInFlight(2); + + assertTrue(window.isFull()); + assertEquals(3, window.getInFlightCount()); + + // Free slots by ACKing + window.acknowledgeUpTo(1); + assertFalse(window.isFull()); + assertEquals(1, window.getInFlightCount()); + } + + @Test + public void testWindowBlocksWhenFull() throws Exception { + InFlightWindow window = new InFlightWindow(2, 5000); + + // Fill the window + window.addInFlight(0); + window.addInFlight(1); + + AtomicBoolean blocked = new AtomicBoolean(true); + CountDownLatch started = new CountDownLatch(1); + CountDownLatch finished = new CountDownLatch(1); + + // Start thread that will block + Thread addThread = new Thread(() -> { + started.countDown(); + window.addInFlight(2); + blocked.set(false); + finished.countDown(); + }); + addThread.start(); + + // Wait for thread to start and block + assertTrue(started.await(1, TimeUnit.SECONDS)); + Thread.sleep(100); // Give time to block + assertTrue(blocked.get()); + + // Free a slot + window.acknowledge(0); + + // Thread should complete + assertTrue(finished.await(1, TimeUnit.SECONDS)); + assertFalse(blocked.get()); + assertEquals(2, window.getInFlightCount()); + } + + @Test + public void testWindowBlocksTimeout() { + InFlightWindow window = new InFlightWindow(2, 100); // 100ms timeout + + // Fill the window + window.addInFlight(0); + window.addInFlight(1); + + // Try to add another - should timeout + long start = System.currentTimeMillis(); + try { + window.addInFlight(2); + fail("Expected timeout exception"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("Timeout")); + } + long elapsed = System.currentTimeMillis() - start; + assertTrue("Should have waited at least 100ms", elapsed >= 90); + } + + @Test + public void testAwaitEmpty() throws Exception { + InFlightWindow window = new InFlightWindow(8, 5000); + + window.addInFlight(0); + window.addInFlight(1); + window.addInFlight(2); + + AtomicBoolean waiting = new AtomicBoolean(true); + CountDownLatch started = new CountDownLatch(1); + CountDownLatch finished = new CountDownLatch(1); + + // Start thread waiting for empty + Thread waitThread = new Thread(() -> { + started.countDown(); + window.awaitEmpty(); + waiting.set(false); + finished.countDown(); + }); + waitThread.start(); + + assertTrue(started.await(1, TimeUnit.SECONDS)); + Thread.sleep(100); + assertTrue(waiting.get()); + + // Cumulative ACK all batches + window.acknowledgeUpTo(2); + assertTrue(finished.await(1, TimeUnit.SECONDS)); + assertFalse(waiting.get()); + } + + @Test + public void testAwaitEmptyTimeout() { + InFlightWindow window = new InFlightWindow(8, 100); // 100ms timeout + + window.addInFlight(0); + + long start = System.currentTimeMillis(); + try { + window.awaitEmpty(); + fail("Expected timeout exception"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("Timeout")); + } + long elapsed = System.currentTimeMillis() - start; + assertTrue("Should have waited at least 100ms", elapsed >= 90); + } + + @Test + public void testAwaitEmptyAlreadyEmpty() { + InFlightWindow window = new InFlightWindow(8, 1000); + + // Should return immediately + window.awaitEmpty(); + assertTrue(window.isEmpty()); + } + + @Test + public void testFailBatch() { + InFlightWindow window = new InFlightWindow(8, 1000); + + window.addInFlight(0); + window.addInFlight(1); + + // Fail batch 0 + RuntimeException error = new RuntimeException("Test error"); + window.fail(0, error); + + assertEquals(1, window.getTotalFailed()); + assertNotNull(window.getLastError()); + } + + @Test + public void testFailPropagatesError() { + InFlightWindow window = new InFlightWindow(8, 1000); + + window.addInFlight(0); + window.fail(0, new RuntimeException("Test error")); + + // Subsequent operations should throw + try { + window.addInFlight(1); + fail("Expected exception due to error"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("failed")); + } + + try { + window.awaitEmpty(); + fail("Expected exception due to error"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("failed")); + } + } + + @Test + public void testFailAllPropagatesError() { + InFlightWindow window = new InFlightWindow(8, 1000); + + window.addInFlight(0); + window.addInFlight(1); + window.failAll(new RuntimeException("Transport down")); + + try { + window.awaitEmpty(); + fail("Expected exception due to failAll"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("failed")); + assertTrue(e.getMessage().contains("Transport down")); + } + } + + @Test + public void testClearError() { + InFlightWindow window = new InFlightWindow(8, 1000); + + window.addInFlight(0); + window.fail(0, new RuntimeException("Test error")); + + assertNotNull(window.getLastError()); + + window.clearError(); + assertNull(window.getLastError()); + + // Should work again + window.addInFlight(1); + assertEquals(2, window.getInFlightCount()); // 0 and 1 both in window (fail doesn't remove) + } + + @Test + public void testReset() { + InFlightWindow window = new InFlightWindow(8, 1000); + + window.addInFlight(0); + window.addInFlight(1); + window.fail(2, new RuntimeException("Test")); + + window.reset(); + + assertTrue(window.isEmpty()); + assertNull(window.getLastError()); + assertEquals(0, window.getInFlightCount()); + } + + @Test + public void testConcurrentAddAndAck() throws Exception { + InFlightWindow window = new InFlightWindow(4, 5000); + int numOperations = 100; + CountDownLatch done = new CountDownLatch(2); + AtomicReference error = new AtomicReference<>(); + AtomicInteger highestAdded = new AtomicInteger(-1); + + // Sender thread + Thread sender = new Thread(() -> { + try { + for (int i = 0; i < numOperations; i++) { + window.addInFlight(i); + highestAdded.set(i); + Thread.sleep(1); // Small delay + } + } catch (Throwable t) { + error.set(t); + } finally { + done.countDown(); + } + }); + + // ACK thread (cumulative ACKs) + Thread acker = new Thread(() -> { + try { + Thread.sleep(10); // Let sender get ahead + int lastAcked = -1; + while (lastAcked < numOperations - 1) { + int highest = highestAdded.get(); + if (highest > lastAcked) { + window.acknowledgeUpTo(highest); + lastAcked = highest; + } + Thread.sleep(1); + } + } catch (Throwable t) { + error.set(t); + } finally { + done.countDown(); + } + }); + + sender.start(); + acker.start(); + + assertTrue(done.await(10, TimeUnit.SECONDS)); + assertNull(error.get()); + assertTrue(window.isEmpty()); + assertEquals(numOperations, window.getTotalAcked()); + } + + @Test + public void testFailWakesBlockedAdder() throws Exception { + InFlightWindow window = new InFlightWindow(2, 5000); + + // Fill the window + window.addInFlight(0); + window.addInFlight(1); + + CountDownLatch started = new CountDownLatch(1); + AtomicReference caught = new AtomicReference<>(); + + // Thread that will block on add + Thread addThread = new Thread(() -> { + started.countDown(); + try { + window.addInFlight(2); + } catch (LineSenderException e) { + caught.set(e); + } + }); + addThread.start(); + + assertTrue(started.await(1, TimeUnit.SECONDS)); + Thread.sleep(100); // Let it block + + // Fail a batch - should wake the blocked thread + window.fail(0, new RuntimeException("Test error")); + + addThread.join(1000); + assertFalse(addThread.isAlive()); + assertNotNull(caught.get()); + assertTrue(caught.get().getMessage().contains("failed")); + } + + @Test + public void testFailWakesAwaitEmpty() throws Exception { + InFlightWindow window = new InFlightWindow(8, 5000); + + window.addInFlight(0); + + CountDownLatch started = new CountDownLatch(1); + AtomicReference caught = new AtomicReference<>(); + + // Thread waiting for empty + Thread waitThread = new Thread(() -> { + started.countDown(); + try { + window.awaitEmpty(); + } catch (LineSenderException e) { + caught.set(e); + } + }); + waitThread.start(); + + assertTrue(started.await(1, TimeUnit.SECONDS)); + Thread.sleep(100); // Let it block + + // Fail a batch - should wake the blocked thread + window.fail(0, new RuntimeException("Test error")); + + waitThread.join(1000); + assertFalse(waitThread.isAlive()); + assertNotNull(caught.get()); + assertTrue(caught.get().getMessage().contains("failed")); + } + + @Test(expected = IllegalArgumentException.class) + public void testInvalidWindowSize() { + new InFlightWindow(0, 1000); + } + + @Test + public void testGetMaxWindowSize() { + InFlightWindow window = new InFlightWindow(16, 1000); + assertEquals(16, window.getMaxWindowSize()); + } + + @Test + public void testRapidAddAndAck() { + InFlightWindow window = new InFlightWindow(16, 5000); + + // Rapid add and ack in same thread + for (int i = 0; i < 10000; i++) { + window.addInFlight(i); + assertTrue(window.acknowledge(i)); + } + + assertTrue(window.isEmpty()); + assertEquals(10000, window.getTotalAcked()); + } + + @Test + public void testFillAndDrainRepeatedly() { + InFlightWindow window = new InFlightWindow(4, 1000); + + int batchId = 0; + for (int cycle = 0; cycle < 100; cycle++) { + // Fill + int startBatch = batchId; + for (int i = 0; i < 4; i++) { + window.addInFlight(batchId++); + } + assertTrue(window.isFull()); + assertEquals(4, window.getInFlightCount()); + + // Drain with cumulative ACK + window.acknowledgeUpTo(batchId - 1); + assertTrue(window.isEmpty()); + } + + assertEquals(400, window.getTotalAcked()); + } + + @Test + public void testMultipleResets() { + InFlightWindow window = new InFlightWindow(8, 1000); + + for (int cycle = 0; cycle < 10; cycle++) { + window.addInFlight(cycle); + window.reset(); + + assertTrue(window.isEmpty()); + assertNull(window.getLastError()); + } + } + + @Test + public void testFailThenClearThenAdd() { + InFlightWindow window = new InFlightWindow(8, 1000); + + window.addInFlight(0); + window.fail(0, new RuntimeException("Error")); + + // Should not be able to add + try { + window.addInFlight(1); + fail("Expected exception"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("failed")); + } + + // Clear error + window.clearError(); + + // Should work now + window.addInFlight(1); + assertEquals(2, window.getInFlightCount()); + } + + @Test + public void testDefaultWindowSize() { + InFlightWindow window = new InFlightWindow(); + assertEquals(InFlightWindow.DEFAULT_WINDOW_SIZE, window.getMaxWindowSize()); + } + + @Test + public void testSmallestPossibleWindow() { + InFlightWindow window = new InFlightWindow(1, 1000); + + window.addInFlight(0); + assertTrue(window.isFull()); + + window.acknowledge(0); + assertFalse(window.isFull()); + } + + @Test + public void testVeryLargeWindow() { + InFlightWindow window = new InFlightWindow(10000, 1000); + + // Add many batches + for (int i = 0; i < 5000; i++) { + window.addInFlight(i); + } + assertEquals(5000, window.getInFlightCount()); + assertFalse(window.isFull()); + + // ACK half + window.acknowledgeUpTo(2499); + assertEquals(2500, window.getInFlightCount()); + } + + @Test + public void testZeroBatchId() { + InFlightWindow window = new InFlightWindow(8, 1000); + + window.addInFlight(0); + assertEquals(1, window.getInFlightCount()); + + assertTrue(window.acknowledge(0)); + assertTrue(window.isEmpty()); + } + + // ==================== CUMULATIVE ACK TESTS ==================== + + @Test + public void testAcknowledgeUpToBasic() { + InFlightWindow window = new InFlightWindow(16, 1000); + + // Add batches 0-9 + for (int i = 0; i < 10; i++) { + window.addInFlight(i); + } + assertEquals(10, window.getInFlightCount()); + + // ACK up to 5 (should remove 0-5, leaving 6-9) + int acked = window.acknowledgeUpTo(5); + assertEquals(6, acked); + assertEquals(4, window.getInFlightCount()); + assertEquals(6, window.getTotalAcked()); + } + + @Test + public void testAcknowledgeUpToIdempotent() { + InFlightWindow window = new InFlightWindow(16, 1000); + + window.addInFlight(0); + window.addInFlight(1); + window.addInFlight(2); + + // First ACK + assertEquals(3, window.acknowledgeUpTo(2)); + assertTrue(window.isEmpty()); + + // Duplicate ACK - should be no-op + assertEquals(0, window.acknowledgeUpTo(2)); + assertTrue(window.isEmpty()); + + // ACK with lower sequence - should be no-op + assertEquals(0, window.acknowledgeUpTo(1)); + assertTrue(window.isEmpty()); + } + + @Test + public void testAcknowledgeUpToWakesBlockedAdder() throws Exception { + InFlightWindow window = new InFlightWindow(3, 5000); + + // Fill the window + window.addInFlight(0); + window.addInFlight(1); + window.addInFlight(2); + assertTrue(window.isFull()); + + AtomicBoolean blocked = new AtomicBoolean(true); + CountDownLatch started = new CountDownLatch(1); + CountDownLatch finished = new CountDownLatch(1); + + // Start thread that will block + Thread addThread = new Thread(() -> { + started.countDown(); + window.addInFlight(3); + blocked.set(false); + finished.countDown(); + }); + addThread.start(); + + assertTrue(started.await(1, TimeUnit.SECONDS)); + Thread.sleep(100); // Give time to block + assertTrue(blocked.get()); + + // Cumulative ACK frees multiple slots + window.acknowledgeUpTo(1); // Removes 0 and 1 + + // Thread should complete + assertTrue(finished.await(1, TimeUnit.SECONDS)); + assertFalse(blocked.get()); + assertEquals(2, window.getInFlightCount()); // batch 2 and 3 + } + + @Test + public void testAcknowledgeUpToWakesAwaitEmpty() throws Exception { + InFlightWindow window = new InFlightWindow(16, 5000); + + window.addInFlight(0); + window.addInFlight(1); + window.addInFlight(2); + + AtomicBoolean waiting = new AtomicBoolean(true); + CountDownLatch started = new CountDownLatch(1); + CountDownLatch finished = new CountDownLatch(1); + + // Start thread waiting for empty + Thread waitThread = new Thread(() -> { + started.countDown(); + window.awaitEmpty(); + waiting.set(false); + finished.countDown(); + }); + waitThread.start(); + + assertTrue(started.await(1, TimeUnit.SECONDS)); + Thread.sleep(100); + assertTrue(waiting.get()); + + // Single cumulative ACK clears all + window.acknowledgeUpTo(2); + + assertTrue(finished.await(1, TimeUnit.SECONDS)); + assertFalse(waiting.get()); + assertTrue(window.isEmpty()); + } + + @Test + public void testAcknowledgeUpToEmpty() { + InFlightWindow window = new InFlightWindow(16, 1000); + + // ACK on empty window should be no-op + assertEquals(0, window.acknowledgeUpTo(100)); + assertTrue(window.isEmpty()); + } + + @Test + public void testAcknowledgeUpToAllBatches() { + InFlightWindow window = new InFlightWindow(16, 1000); + + // Add batches + for (int i = 0; i < 10; i++) { + window.addInFlight(i); + } + + // ACK all with high sequence + int acked = window.acknowledgeUpTo(Long.MAX_VALUE); + assertEquals(10, acked); + assertTrue(window.isEmpty()); + } + + @Test + public void testConcurrentAddAndCumulativeAck() throws Exception { + InFlightWindow window = new InFlightWindow(100, 10000); + int numBatches = 500; + CountDownLatch done = new CountDownLatch(2); + AtomicReference error = new AtomicReference<>(); + AtomicInteger highestAdded = new AtomicInteger(-1); + + // Sender thread + Thread sender = new Thread(() -> { + try { + for (int i = 0; i < numBatches; i++) { + window.addInFlight(i); + highestAdded.set(i); + } + } catch (Throwable t) { + error.set(t); + } finally { + done.countDown(); + } + }); + + // ACK thread using cumulative ACKs + Thread acker = new Thread(() -> { + try { + int lastAcked = -1; + while (lastAcked < numBatches - 1) { + int highest = highestAdded.get(); + if (highest > lastAcked) { + window.acknowledgeUpTo(highest); + lastAcked = highest; + } else { + Thread.sleep(1); + } + } + } catch (Throwable t) { + error.set(t); + } finally { + done.countDown(); + } + }); + + sender.start(); + acker.start(); + + assertTrue(done.await(30, TimeUnit.SECONDS)); + assertNull(error.get()); + assertTrue(window.isEmpty()); + assertEquals(numBatches, window.getTotalAcked()); + } + + @Test + public void testTryAddInFlight() { + InFlightWindow window = new InFlightWindow(2, 1000); + + // Should succeed + assertTrue(window.tryAddInFlight(0)); + assertTrue(window.tryAddInFlight(1)); + + // Should fail - window full + assertFalse(window.tryAddInFlight(2)); + + // After ACK, should succeed + window.acknowledge(0); + assertTrue(window.tryAddInFlight(2)); + } + + @Test + public void testHasWindowSpace() { + InFlightWindow window = new InFlightWindow(2, 1000); + + assertTrue(window.hasWindowSpace()); + window.addInFlight(0); + assertTrue(window.hasWindowSpace()); + window.addInFlight(1); + assertFalse(window.hasWindowSpace()); + + window.acknowledge(0); + assertTrue(window.hasWindowSpace()); + } + + @Test + public void testHighConcurrencyStress() throws Exception { + InFlightWindow window = new InFlightWindow(8, 30000); + int numBatches = 10000; + CountDownLatch done = new CountDownLatch(2); + AtomicReference error = new AtomicReference<>(); + AtomicInteger highestAdded = new AtomicInteger(-1); + + // Fast sender thread + Thread sender = new Thread(() -> { + try { + for (int i = 0; i < numBatches; i++) { + window.addInFlight(i); + highestAdded.set(i); + } + } catch (Throwable t) { + error.set(t); + } finally { + done.countDown(); + } + }); + + // Fast ACK thread + Thread acker = new Thread(() -> { + try { + int lastAcked = -1; + while (lastAcked < numBatches - 1) { + int highest = highestAdded.get(); + if (highest > lastAcked) { + window.acknowledgeUpTo(highest); + lastAcked = highest; + } + // No sleep - maximum contention + } + } catch (Throwable t) { + error.set(t); + } finally { + done.countDown(); + } + }); + + sender.start(); + acker.start(); + + assertTrue(done.await(60, TimeUnit.SECONDS)); + if (error.get() != null) { + error.get().printStackTrace(); + } + assertNull(error.get()); + assertTrue(window.isEmpty()); + assertEquals(numBatches, window.getTotalAcked()); + } +} From 10c306ab596c58c1b72637fc0ab622cd52d62335 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Sat, 14 Feb 2026 23:02:52 +0000 Subject: [PATCH 06/89] tidy --- .../client/test/cutlass/line/LineSenderBuilderTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/line/LineSenderBuilderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/line/LineSenderBuilderTest.java index bf9f5c7..e684e52 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/line/LineSenderBuilderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/line/LineSenderBuilderTest.java @@ -338,7 +338,7 @@ public void testMalformedPortInAddress() throws Exception { @Test public void testMaxRequestBufferSizeCannotBeLessThanDefault() throws Exception { assertMemoryLeak(() -> assertThrows("maximum buffer capacity cannot be less than initial buffer capacity [maximumBufferCapacity=65535, initialBufferCapacity=65536]", - Sender.builder(Sender.Transport.HTTP).address("localhost:1").maxBufferCapacity(65535))); + () -> Sender.builder(Sender.Transport.HTTP).address("localhost:1").maxBufferCapacity(65535))); } @Test @@ -350,13 +350,13 @@ public void testMaxRequestBufferSizeCannotBeLessThanInitialBufferSize() throws E @Test public void testMaxRetriesNotSupportedForTcp() throws Exception { assertMemoryLeak(() -> assertThrows("retrying is not supported for TCP protocol", - Sender.builder(Sender.Transport.TCP).address(LOCALHOST).retryTimeoutMillis(100))); + () -> Sender.builder(Sender.Transport.TCP).address(LOCALHOST).retryTimeoutMillis(100))); } @Test public void testMinRequestThroughputCannotBeNegative() throws Exception { assertMemoryLeak(() -> assertThrows("minimum request throughput must not be negative [minRequestThroughput=-100]", - Sender.builder(Sender.Transport.HTTP).address(LOCALHOST).minRequestThroughput(-100))); + () -> Sender.builder(Sender.Transport.HTTP).address(LOCALHOST).minRequestThroughput(-100))); } @Test From f12a6f99a477cea7b8ff3a84beff428ce7b6c416 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Sun, 15 Feb 2026 00:29:09 +0000 Subject: [PATCH 07/89] wip --- .../qwp/client/QwpWebSocketSender.java | 17 +- .../questdb/client/test/AbstractQdbTest.java | 2 +- .../LineSenderBuilderWebSocketTest.java | 2 +- .../cutlass/qwp/client/QwpSenderTest.java | 952 ++++++++++++++++++ core/src/test/java/module-info.java | 1 + 5 files changed, 971 insertions(+), 3 deletions(-) rename core/src/test/java/io/questdb/client/test/cutlass/{line => qwp/client}/LineSenderBuilderWebSocketTest.java (99%) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 6b0343f..4a70fb1 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -613,6 +613,21 @@ public QwpWebSocketSender doubleColumn(CharSequence columnName, double value) { return this; } + /** + * Adds a FLOAT column value to the current row. + * + * @param columnName the column name + * @param value the float value + * @return this sender for method chaining + */ + public QwpWebSocketSender floatColumn(CharSequence columnName, float value) { + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_FLOAT, false); + col.addFloat(value); + return this; + } + @Override public QwpWebSocketSender stringColumn(CharSequence columnName, CharSequence value) { checkNotClosed(); @@ -1258,7 +1273,7 @@ public Sender longArray(@NotNull CharSequence name, long[][][] values) { } @Override - public Sender longArray(CharSequence name, LongArray array) { + public Sender longArray(@NotNull CharSequence name, LongArray array) { if (array == null) return this; checkNotClosed(); checkTableSelected(); diff --git a/core/src/test/java/io/questdb/client/test/AbstractQdbTest.java b/core/src/test/java/io/questdb/client/test/AbstractQdbTest.java index 60f3ed2..81242bd 100644 --- a/core/src/test/java/io/questdb/client/test/AbstractQdbTest.java +++ b/core/src/test/java/io/questdb/client/test/AbstractQdbTest.java @@ -426,7 +426,7 @@ protected static String getPgUser() { * Get whether a QuestDB instance is running locally. */ protected static boolean getQuestDBRunning() { - return getConfigBool("QUESTDB_RUNNING", "questdb.running", false); + return getConfigBool("QUESTDB_RUNNING", "questdb.running", true); } /** diff --git a/core/src/test/java/io/questdb/client/test/cutlass/line/LineSenderBuilderWebSocketTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java similarity index 99% rename from core/src/test/java/io/questdb/client/test/cutlass/line/LineSenderBuilderWebSocketTest.java rename to core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java index 92c79d0..ee80665 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/line/LineSenderBuilderWebSocketTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java @@ -22,7 +22,7 @@ * ******************************************************************************/ -package io.questdb.client.test.cutlass.line; +package io.questdb.client.test.cutlass.qwp.client; import io.questdb.client.Sender; import io.questdb.client.cutlass.line.LineSenderException; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java new file mode 100644 index 0000000..4867640 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java @@ -0,0 +1,952 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; +import io.questdb.client.std.Decimal256; +import io.questdb.client.test.cutlass.line.AbstractLineSenderTest; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.time.temporal.ChronoUnit; +import java.util.UUID; + +/** + * Integration tests for the QWP (QuestDB Wire Protocol) WebSocket sender. + *

+ * Tests verify that all QWP native types arrive correctly (exact type match) + * and that reasonable type coercions work (e.g., client sends INT but server + * column is LONG). + *

+ * Tests are skipped if no QuestDB instance is running + * ({@code -Dquestdb.running=true}). + */ +public class QwpSenderTest extends AbstractLineSenderTest { + + @BeforeClass + public static void setUpStatic() { + AbstractLineSenderTest.setUpStatic(); + } + + // === Exact Type Match Tests === + + @Test + public void testBoolean() throws Exception { + String table = "test_qwp_boolean"; + useTable(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .boolColumn("b", true) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .boolColumn("b", false) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "b\ttimestamp\n" + + "true\t1970-01-01T00:00:01.000000000Z\n" + + "false\t1970-01-01T00:00:02.000000000Z\n", + "SELECT b, timestamp FROM " + table + " ORDER BY timestamp"); + } + + @Test + public void testByte() throws Exception { + String table = "test_qwp_byte"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "b BYTE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .shortColumn("b", (short) -1) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("b", (short) 0) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("b", (short) 127) + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + } + + @Test + public void testChar() throws Exception { + String table = "test_qwp_char"; + useTable(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .charColumn("c", 'A') + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .charColumn("c", '\u00FC') // ü + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .charColumn("c", '\u4E2D') // 中 + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "c\ttimestamp\n" + + "A\t1970-01-01T00:00:01.000000000Z\n" + + "\u00FC\t1970-01-01T00:00:02.000000000Z\n" + + "\u4E2D\t1970-01-01T00:00:03.000000000Z\n", + "SELECT c, timestamp FROM " + table + " ORDER BY timestamp"); + } + + @Test + public void testDecimal() throws Exception { + String table = "test_qwp_decimal"; + useTable(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .decimalColumn("d", "123.45") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .decimalColumn("d", "-999.99") + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .decimalColumn("d", "0.01") + .at(3_000_000, ChronoUnit.MICROS); + sender.table(table) + .decimalColumn("d", Decimal256.fromLong(42_000, 3)) + .at(4_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 4); + } + + @Test + public void testDouble() throws Exception { + String table = "test_qwp_double"; + useTable(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .doubleColumn("d", 42.5) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .doubleColumn("d", -1.0E10) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .doubleColumn("d", Double.MAX_VALUE) + .at(3_000_000, ChronoUnit.MICROS); + sender.table(table) + .doubleColumn("d", Double.MIN_VALUE) + .at(4_000_000, ChronoUnit.MICROS); + // NaN and Inf should be stored as null + sender.table(table) + .doubleColumn("d", Double.NaN) + .at(5_000_000, ChronoUnit.MICROS); + sender.table(table) + .doubleColumn("d", Double.POSITIVE_INFINITY) + .at(6_000_000, ChronoUnit.MICROS); + sender.table(table) + .doubleColumn("d", Double.NEGATIVE_INFINITY) + .at(7_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 7); + assertSqlEventually( + "d\ttimestamp\n" + + "42.5\t1970-01-01T00:00:01.000000000Z\n" + + "-1.0E10\t1970-01-01T00:00:02.000000000Z\n" + + "1.7976931348623157E308\t1970-01-01T00:00:03.000000000Z\n" + + "4.9E-324\t1970-01-01T00:00:04.000000000Z\n" + + "null\t1970-01-01T00:00:05.000000000Z\n" + + "null\t1970-01-01T00:00:06.000000000Z\n" + + "null\t1970-01-01T00:00:07.000000000Z\n", + "SELECT d, timestamp FROM " + table + " ORDER BY timestamp"); + } + + @Test + public void testDoubleArray() throws Exception { + String table = "test_qwp_double_array"; + useTable(table); + + double[] arr1d = createDoubleArray(5); + double[][] arr2d = createDoubleArray(2, 3); + double[][][] arr3d = createDoubleArray(1, 2, 3); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .doubleArray("a1", arr1d) + .doubleArray("a2", arr2d) + .doubleArray("a3", arr3d) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 1); + } + + @Test + public void testDoubleToDecimal() throws Exception { + String table = "test_qwp_double_to_decimal"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(10, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .doubleColumn("d", 123.45) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .doubleColumn("d", -42.10) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "123.45\t1970-01-01T00:00:01.000000000Z\n" + + "-42.10\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testDoubleToFloat() throws Exception { + String table = "test_qwp_double_to_float"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "f FLOAT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .doubleColumn("f", 1.5) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .doubleColumn("f", -42.25) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + } + + @Test + public void testFloat() throws Exception { + String table = "test_qwp_float"; + useTable(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .floatColumn("f", 1.5f) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .floatColumn("f", -42.25f) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .floatColumn("f", 0.0f) + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + } + + @Test + public void testFloatToDouble() throws Exception { + String table = "test_qwp_float_to_double"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DOUBLE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .floatColumn("d", 1.5f) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .floatColumn("d", -42.25f) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "1.5\t1970-01-01T00:00:01.000000000Z\n" + + "-42.25\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testInt() throws Exception { + String table = "test_qwp_int"; + useTable(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + // Integer.MIN_VALUE is the null sentinel for INT + sender.table(table) + .intColumn("i", Integer.MIN_VALUE) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("i", 0) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("i", Integer.MAX_VALUE) + .at(3_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("i", -42) + .at(4_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 4); + assertSqlEventually( + "i\ttimestamp\n" + + "null\t1970-01-01T00:00:01.000000000Z\n" + + "0\t1970-01-01T00:00:02.000000000Z\n" + + "2147483647\t1970-01-01T00:00:03.000000000Z\n" + + "-42\t1970-01-01T00:00:04.000000000Z\n", + "SELECT i, timestamp FROM " + table + " ORDER BY timestamp"); + } + + @Test + public void testIntToDecimal() throws Exception { + String table = "test_qwp_int_to_decimal"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(6, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("d", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testIntToDouble() throws Exception { + String table = "test_qwp_int_to_double"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DOUBLE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("d", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "42.0\t1970-01-01T00:00:01.000000000Z\n" + + "-100.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testIntToLong() throws Exception { + String table = "test_qwp_int_to_long"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "l LONG, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("l", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("l", Integer.MAX_VALUE) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("l", -1) + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "l\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "2147483647\t1970-01-01T00:00:02.000000000Z\n" + + "-1\t1970-01-01T00:00:03.000000000Z\n", + "SELECT l, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testLong() throws Exception { + String table = "test_qwp_long"; + useTable(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + // Long.MIN_VALUE is the null sentinel for LONG + sender.table(table) + .longColumn("l", Long.MIN_VALUE) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("l", 0) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("l", Long.MAX_VALUE) + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "l\ttimestamp\n" + + "null\t1970-01-01T00:00:01.000000000Z\n" + + "0\t1970-01-01T00:00:02.000000000Z\n" + + "9223372036854775807\t1970-01-01T00:00:03.000000000Z\n", + "SELECT l, timestamp FROM " + table + " ORDER BY timestamp"); + } + + @Test + public void testLong256() throws Exception { + String table = "test_qwp_long256"; + useTable(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + // All zeros + sender.table(table) + .long256Column("v", 0, 0, 0, 0) + .at(1_000_000, ChronoUnit.MICROS); + // Mixed values + sender.table(table) + .long256Column("v", 1, 2, 3, 4) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + } + + @Test + public void testLongToDecimal() throws Exception { + String table = "test_qwp_long_to_decimal"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(10, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("d", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("d", -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testLongToInt() throws Exception { + String table = "test_qwp_long_to_int"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "i INT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + // Value in INT range should succeed + sender.table(table) + .longColumn("i", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("i", -1) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "i\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-1\t1970-01-01T00:00:02.000000000Z\n", + "SELECT i, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testLongToShort() throws Exception { + String table = "test_qwp_long_to_short"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "s SHORT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + // Value in SHORT range should succeed + sender.table(table) + .longColumn("s", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("s", -1) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + } + + @Test + public void testMultipleRowsAndBatching() throws Exception { + String table = "test_qwp_multiple_rows"; + useTable(table); + + int rowCount = 1000; + try (QwpWebSocketSender sender = createQwpSender()) { + for (int i = 0; i < rowCount; i++) { + sender.table(table) + .symbol("sym", "s" + (i % 10)) + .longColumn("val", i) + .doubleColumn("dbl", i * 1.5) + .at((long) (i + 1) * 1_000_000, ChronoUnit.MICROS); + } + sender.flush(); + } + + assertTableSizeEventually(table, rowCount); + } + + @Test + public void testShort() throws Exception { + String table = "test_qwp_short"; + useTable(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + // Short.MIN_VALUE is the null sentinel for SHORT + sender.table(table) + .shortColumn("s", Short.MIN_VALUE) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("s", (short) 0) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("s", Short.MAX_VALUE) + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + } + + @Test + public void testShortToInt() throws Exception { + String table = "test_qwp_short_to_int"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "i INT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .shortColumn("i", (short) 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("i", Short.MAX_VALUE) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "i\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "32767\t1970-01-01T00:00:02.000000000Z\n", + "SELECT i, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testShortToLong() throws Exception { + String table = "test_qwp_short_to_long"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "l LONG, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .shortColumn("l", (short) 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("l", Short.MAX_VALUE) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "l\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "32767\t1970-01-01T00:00:02.000000000Z\n", + "SELECT l, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testString() throws Exception { + String table = "test_qwp_string"; + useTable(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("s", "hello world") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("s", "non-ascii \u00E4\u00F6\u00FC") + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("s", "") + .at(3_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("s", null) + .at(4_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 4); + assertSqlEventually( + "s\ttimestamp\n" + + "hello world\t1970-01-01T00:00:01.000000000Z\n" + + "non-ascii \u00E4\u00F6\u00FC\t1970-01-01T00:00:02.000000000Z\n" + + "\t1970-01-01T00:00:03.000000000Z\n" + + "null\t1970-01-01T00:00:04.000000000Z\n", + "SELECT s, timestamp FROM " + table + " ORDER BY timestamp"); + } + + @Test + public void testStringToChar() throws Exception { + String table = "test_qwp_string_to_char"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "c CHAR, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("c", "A") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("c", "Hello") + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "c\tts\n" + + "A\t1970-01-01T00:00:01.000000000Z\n" + + "H\t1970-01-01T00:00:02.000000000Z\n", + "SELECT c, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testStringToSymbol() throws Exception { + String table = "test_qwp_string_to_symbol"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "s SYMBOL, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("s", "hello") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("s", "world") + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "s\tts\n" + + "hello\t1970-01-01T00:00:01.000000000Z\n" + + "world\t1970-01-01T00:00:02.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testStringToUuid() throws Exception { + String table = "test_qwp_string_to_uuid"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "u UUID, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("u", "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 1); + assertSqlEventually( + "u\tts\n" + + "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11\t1970-01-01T00:00:01.000000000Z\n", + "SELECT u, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testSymbol() throws Exception { + String table = "test_qwp_symbol"; + useTable(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .symbol("s", "alpha") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .symbol("s", "beta") + .at(2_000_000, ChronoUnit.MICROS); + // repeated value reuses dictionary entry + sender.table(table) + .symbol("s", "alpha") + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "s\ttimestamp\n" + + "alpha\t1970-01-01T00:00:01.000000000Z\n" + + "beta\t1970-01-01T00:00:02.000000000Z\n" + + "alpha\t1970-01-01T00:00:03.000000000Z\n", + "SELECT s, timestamp FROM " + table + " ORDER BY timestamp"); + } + + @Test + public void testTimestampMicros() throws Exception { + String table = "test_qwp_timestamp_micros"; + useTable(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + long tsMicros = 1_645_747_200_000_000L; // 2022-02-25T00:00:00Z in micros + sender.table(table) + .timestampColumn("ts_col", tsMicros, ChronoUnit.MICROS) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 1); + assertSqlEventually( + "ts_col\ttimestamp\n" + + "2022-02-25T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z\n", + "SELECT ts_col, timestamp FROM " + table); + } + + @Test + public void testTimestampMicrosToNanos() throws Exception { + String table = "test_qwp_timestamp_micros_to_nanos"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "ts_col TIMESTAMP WITH TIME ZONE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + long tsMicros = 1_645_747_200_000_000L; // 2022-02-25T00:00:00Z + sender.table(table) + .timestampColumn("ts_col", tsMicros, ChronoUnit.MICROS) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 1); + } + + @Test + public void testTimestampNanos() throws Exception { + String table = "test_qwp_timestamp_nanos"; + useTable(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + long tsNanos = 1_645_747_200_000_000_000L; // 2022-02-25T00:00:00Z in nanos + sender.table(table) + .timestampColumn("ts_col", tsNanos, ChronoUnit.NANOS) + .at(tsNanos, ChronoUnit.NANOS); + sender.flush(); + } + + assertTableSizeEventually(table, 1); + } + + @Test + public void testTimestampNanosToMicros() throws Exception { + String table = "test_qwp_timestamp_nanos_to_micros"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "ts_col TIMESTAMP, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + long tsNanos = 1_645_747_200_123_456_789L; + sender.table(table) + .timestampColumn("ts_col", tsNanos, ChronoUnit.NANOS) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 1); + // Nanoseconds truncated to microseconds + assertSqlEventually( + "ts_col\tts\n" + + "2022-02-25T00:00:00.123456000Z\t1970-01-01T00:00:01.000000000Z\n", + "SELECT ts_col, ts FROM " + table); + } + + @Test + public void testUuid() throws Exception { + String table = "test_qwp_uuid"; + useTable(table); + + UUID uuid1 = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); + UUID uuid2 = UUID.fromString("11111111-2222-3333-4444-555555555555"); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .uuidColumn("u", uuid1.getLeastSignificantBits(), uuid1.getMostSignificantBits()) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .uuidColumn("u", uuid2.getLeastSignificantBits(), uuid2.getMostSignificantBits()) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "u\ttimestamp\n" + + "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11\t1970-01-01T00:00:01.000000000Z\n" + + "11111111-2222-3333-4444-555555555555\t1970-01-01T00:00:02.000000000Z\n", + "SELECT u, timestamp FROM " + table + " ORDER BY timestamp"); + } + + @Test + public void testWriteAllTypesInOneRow() throws Exception { + String table = "test_qwp_all_types"; + useTable(table); + + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); + double[] arr1d = {1.0, 2.0, 3.0}; + long tsMicros = 1_645_747_200_000_000L; // 2022-02-25T00:00:00Z + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .symbol("sym", "test_symbol") + .boolColumn("bool_col", true) + .shortColumn("short_col", (short) 42) + .intColumn("int_col", 100_000) + .longColumn("long_col", 1_000_000_000L) + .floatColumn("float_col", 2.5f) + .doubleColumn("double_col", 3.14) + .stringColumn("string_col", "hello") + .charColumn("char_col", 'Z') + .timestampColumn("ts_col", tsMicros, ChronoUnit.MICROS) + .uuidColumn("uuid_col", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .long256Column("long256_col", 1, 0, 0, 0) + .doubleArray("arr_col", arr1d) + .decimalColumn("decimal_col", "99.99") + .at(tsMicros, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 1); + } + + // === Helper Methods === + + private QwpWebSocketSender createQwpSender() { + return QwpWebSocketSender.connect(getQuestDbHost(), getHttpPort()); + } +} diff --git a/core/src/test/java/module-info.java b/core/src/test/java/module-info.java index 86341d8..7e39674 100644 --- a/core/src/test/java/module-info.java +++ b/core/src/test/java/module-info.java @@ -36,4 +36,5 @@ exports io.questdb.client.test; exports io.questdb.client.test.cairo; exports io.questdb.client.test.cutlass.line; + exports io.questdb.client.test.cutlass.qwp.client; } From d442198405b95e724c0fd6b1ebf123cfc741fcac Mon Sep 17 00:00:00 2001 From: bluestreak Date: Sun, 15 Feb 2026 01:24:18 +0000 Subject: [PATCH 08/89] wip2 --- .../cutlass/qwp/client/QwpSenderTest.java | 480 +++++++++++++++++- 1 file changed, 474 insertions(+), 6 deletions(-) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java index 4867640..cbb5c8d 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java @@ -111,10 +111,10 @@ public void testChar() throws Exception { .charColumn("c", 'A') .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .charColumn("c", '\u00FC') // ü + .charColumn("c", 'ü') // ü .at(2_000_000, ChronoUnit.MICROS); sender.table(table) - .charColumn("c", '\u4E2D') // 中 + .charColumn("c", '中') // 中 .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } @@ -123,8 +123,8 @@ public void testChar() throws Exception { assertSqlEventually( "c\ttimestamp\n" + "A\t1970-01-01T00:00:01.000000000Z\n" + - "\u00FC\t1970-01-01T00:00:02.000000000Z\n" + - "\u4E2D\t1970-01-01T00:00:03.000000000Z\n", + "ü\t1970-01-01T00:00:02.000000000Z\n" + + "中\t1970-01-01T00:00:03.000000000Z\n", "SELECT c, timestamp FROM " + table + " ORDER BY timestamp"); } @@ -377,6 +377,166 @@ public void testIntToDecimal() throws Exception { "SELECT d, ts FROM " + table + " ORDER BY ts"); } + @Test + public void testIntToDecimal128() throws Exception { + String table = "test_qwp_int_to_decimal128"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(38, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("d", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", 0) + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n" + + "0.00\t1970-01-01T00:00:03.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testIntToDecimal16() throws Exception { + String table = "test_qwp_int_to_decimal16"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(4, 1), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("d", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", 0) + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "d\tts\n" + + "42.0\t1970-01-01T00:00:01.000000000Z\n" + + "-100.0\t1970-01-01T00:00:02.000000000Z\n" + + "0.0\t1970-01-01T00:00:03.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testIntToDecimal256() throws Exception { + String table = "test_qwp_int_to_decimal256"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(76, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("d", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", 0) + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n" + + "0.00\t1970-01-01T00:00:03.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testIntToDecimal64() throws Exception { + String table = "test_qwp_int_to_decimal64"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(18, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("d", Integer.MAX_VALUE) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", 0) + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "d\tts\n" + + "2147483647.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n" + + "0.00\t1970-01-01T00:00:03.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testIntToDecimal8() throws Exception { + String table = "test_qwp_int_to_decimal8"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(2, 1), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("d", 5) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", -9) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", 0) + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "d\tts\n" + + "5.0\t1970-01-01T00:00:01.000000000Z\n" + + "-9.0\t1970-01-01T00:00:02.000000000Z\n" + + "0.0\t1970-01-01T00:00:03.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + @Test public void testIntToDouble() throws Exception { String table = "test_qwp_int_to_double"; @@ -513,6 +673,146 @@ public void testLongToDecimal() throws Exception { "SELECT d, ts FROM " + table + " ORDER BY ts"); } + @Test + public void testLongToDecimal128() throws Exception { + String table = "test_qwp_long_to_decimal128"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(38, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("d", 1_000_000_000L) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("d", -1_000_000_000L) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "1000000000.00\t1970-01-01T00:00:01.000000000Z\n" + + "-1000000000.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testLongToDecimal16() throws Exception { + String table = "test_qwp_long_to_decimal16"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(4, 1), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("d", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("d", -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "42.0\t1970-01-01T00:00:01.000000000Z\n" + + "-100.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testLongToDecimal256() throws Exception { + String table = "test_qwp_long_to_decimal256"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(76, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("d", Long.MAX_VALUE) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("d", -1_000_000_000_000L) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "9223372036854775807.00\t1970-01-01T00:00:01.000000000Z\n" + + "-1000000000000.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testLongToDecimal32() throws Exception { + String table = "test_qwp_long_to_decimal32"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(6, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("d", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("d", -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testLongToDecimal8() throws Exception { + String table = "test_qwp_long_to_decimal8"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(2, 1), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("d", 5) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("d", -9) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "5.0\t1970-01-01T00:00:01.000000000Z\n" + + "-9.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + @Test public void testLongToInt() throws Exception { String table = "test_qwp_long_to_int"; @@ -608,6 +908,174 @@ public void testShort() throws Exception { assertTableSizeEventually(table, 3); } + @Test + public void testShortToDecimal128() throws Exception { + String table = "test_qwp_short_to_decimal128"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(38, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .shortColumn("d", Short.MAX_VALUE) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("d", Short.MIN_VALUE) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "32767.00\t1970-01-01T00:00:01.000000000Z\n" + + "-32768.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testShortToDecimal16() throws Exception { + String table = "test_qwp_short_to_decimal16"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(4, 1), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .shortColumn("d", (short) 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("d", (short) -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "42.0\t1970-01-01T00:00:01.000000000Z\n" + + "-100.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testShortToDecimal256() throws Exception { + String table = "test_qwp_short_to_decimal256"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(76, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .shortColumn("d", (short) 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("d", (short) -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testShortToDecimal32() throws Exception { + String table = "test_qwp_short_to_decimal32"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(6, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .shortColumn("d", (short) 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("d", (short) -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testShortToDecimal64() throws Exception { + String table = "test_qwp_short_to_decimal64"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(18, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .shortColumn("d", (short) 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("d", (short) -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testShortToDecimal8() throws Exception { + String table = "test_qwp_short_to_decimal8"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(2, 1), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .shortColumn("d", (short) 5) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("d", (short) -9) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "5.0\t1970-01-01T00:00:01.000000000Z\n" + + "-9.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + @Test public void testShortToInt() throws Exception { String table = "test_qwp_short_to_int"; @@ -674,7 +1142,7 @@ public void testString() throws Exception { .stringColumn("s", "hello world") .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .stringColumn("s", "non-ascii \u00E4\u00F6\u00FC") + .stringColumn("s", "non-ascii äöü") .at(2_000_000, ChronoUnit.MICROS); sender.table(table) .stringColumn("s", "") @@ -689,7 +1157,7 @@ public void testString() throws Exception { assertSqlEventually( "s\ttimestamp\n" + "hello world\t1970-01-01T00:00:01.000000000Z\n" + - "non-ascii \u00E4\u00F6\u00FC\t1970-01-01T00:00:02.000000000Z\n" + + "non-ascii äöü\t1970-01-01T00:00:02.000000000Z\n" + "\t1970-01-01T00:00:03.000000000Z\n" + "null\t1970-01-01T00:00:04.000000000Z\n", "SELECT s, timestamp FROM " + table + " ORDER BY timestamp"); From ea798b80124bf28fb64c7bc1c1523f3f00ac869d Mon Sep 17 00:00:00 2001 From: bluestreak Date: Sun, 15 Feb 2026 03:14:19 +0000 Subject: [PATCH 09/89] wip3 --- .../cutlass/qwp/client/QwpSenderTest.java | 1781 +++++++++++++++-- 1 file changed, 1564 insertions(+), 217 deletions(-) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java index cbb5c8d..cf73ee3 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java @@ -24,9 +24,11 @@ package io.questdb.client.test.cutlass.qwp.client; +import io.questdb.client.cutlass.line.LineSenderException; import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; import io.questdb.client.std.Decimal256; import io.questdb.client.test.cutlass.line.AbstractLineSenderTest; +import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; @@ -245,126 +247,1448 @@ public void testDoubleToDecimal() throws Exception { "SELECT d, ts FROM " + table + " ORDER BY ts"); } + @Test + public void testDoubleToDecimalPrecisionLossError() throws Exception { + String table = "test_qwp_double_to_decimal_prec"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(10, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .doubleColumn("d", 123.456) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected precision loss error but got: " + msg, + msg.contains("loses precision") && msg.contains("123.456") && msg.contains("scale=2") + ); + } + } + + @Test + public void testDoubleToByte() throws Exception { + String table = "test_qwp_double_to_byte"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "b BYTE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .doubleColumn("b", 42.0) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .doubleColumn("b", -100.0) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "b\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-100\t1970-01-01T00:00:02.000000000Z\n", + "SELECT b, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testDoubleToBytePrecisionLossError() throws Exception { + String table = "test_qwp_double_to_byte_prec"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "b BYTE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .doubleColumn("b", 42.5) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected precision loss error but got: " + msg, + msg.contains("loses precision") && msg.contains("42.5") + ); + } + } + + @Test + public void testDoubleToByteOverflowError() throws Exception { + String table = "test_qwp_double_to_byte_ovf"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "b BYTE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .doubleColumn("b", 200.0) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected overflow error but got: " + msg, + msg.contains("integer value 200 out of range for BYTE") + ); + } + } + @Test public void testDoubleToFloat() throws Exception { String table = "test_qwp_double_to_float"; useTable(table); execute("CREATE TABLE " + table + " (" + - "f FLOAT, " + + "f FLOAT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .doubleColumn("f", 1.5) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .doubleColumn("f", -42.25) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + } + + @Test + public void testDoubleToInt() throws Exception { + String table = "test_qwp_double_to_int"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "i INT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .doubleColumn("i", 100_000.0) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .doubleColumn("i", -42.0) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "i\tts\n" + + "100000\t1970-01-01T00:00:01.000000000Z\n" + + "-42\t1970-01-01T00:00:02.000000000Z\n", + "SELECT i, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testDoubleToIntPrecisionLossError() throws Exception { + String table = "test_qwp_double_to_int_prec"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "i INT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .doubleColumn("i", 3.14) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected precision loss error but got: " + msg, + msg.contains("loses precision") && msg.contains("3.14") + ); + } + } + + @Test + public void testDoubleToLong() throws Exception { + String table = "test_qwp_double_to_long"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "l LONG, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .doubleColumn("l", 1_000_000.0) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .doubleColumn("l", -42.0) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "l\tts\n" + + "1000000\t1970-01-01T00:00:01.000000000Z\n" + + "-42\t1970-01-01T00:00:02.000000000Z\n", + "SELECT l, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testDoubleToString() throws Exception { + String table = "test_qwp_double_to_string"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "s STRING, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .doubleColumn("s", 3.14) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .doubleColumn("s", -42.0) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "s\tts\n" + + "3.14\t1970-01-01T00:00:01.000000000Z\n" + + "-42.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testDoubleToSymbol() throws Exception { + String table = "test_qwp_double_to_symbol"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "sym SYMBOL, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .doubleColumn("sym", 3.14) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 1); + assertSqlEventually( + "sym\tts\n" + + "3.14\t1970-01-01T00:00:01.000000000Z\n", + "SELECT sym, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testDoubleToVarchar() throws Exception { + String table = "test_qwp_double_to_varchar"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "v VARCHAR, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .doubleColumn("v", 3.14) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .doubleColumn("v", -42.0) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "v\tts\n" + + "3.14\t1970-01-01T00:00:01.000000000Z\n" + + "-42.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testFloat() throws Exception { + String table = "test_qwp_float"; + useTable(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .floatColumn("f", 1.5f) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .floatColumn("f", -42.25f) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .floatColumn("f", 0.0f) + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + } + + @Test + public void testFloatToDecimal() throws Exception { + String table = "test_qwp_float_to_decimal"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(10, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .floatColumn("d", 1.5f) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .floatColumn("d", -42.25f) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "1.50\t1970-01-01T00:00:01.000000000Z\n" + + "-42.25\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testFloatToDecimalPrecisionLossError() throws Exception { + String table = "test_qwp_float_to_decimal_prec"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(10, 1), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .floatColumn("d", 1.25f) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected precision loss error but got: " + msg, + msg.contains("loses precision") && msg.contains("scale=1") + ); + } + } + + @Test + public void testFloatToDouble() throws Exception { + String table = "test_qwp_float_to_double"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DOUBLE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .floatColumn("d", 1.5f) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .floatColumn("d", -42.25f) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "1.5\t1970-01-01T00:00:01.000000000Z\n" + + "-42.25\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testFloatToInt() throws Exception { + String table = "test_qwp_float_to_int"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "i INT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .floatColumn("i", 42.0f) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .floatColumn("i", -100.0f) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "i\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-100\t1970-01-01T00:00:02.000000000Z\n", + "SELECT i, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testFloatToIntPrecisionLossError() throws Exception { + String table = "test_qwp_float_to_int_prec"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "i INT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .floatColumn("i", 3.14f) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected precision loss error but got: " + msg, + msg.contains("loses precision") + ); + } + } + + @Test + public void testFloatToLong() throws Exception { + String table = "test_qwp_float_to_long"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "l LONG, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .floatColumn("l", 1000.0f) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 1); + assertSqlEventually( + "l\tts\n" + + "1000\t1970-01-01T00:00:01.000000000Z\n", + "SELECT l, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testFloatToString() throws Exception { + String table = "test_qwp_float_to_string"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "s STRING, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .floatColumn("s", 1.5f) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 1); + assertSqlEventually( + "s\tts\n" + + "1.5\t1970-01-01T00:00:01.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testFloatToSymbol() throws Exception { + String table = "test_qwp_float_to_symbol"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "sym SYMBOL, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .floatColumn("sym", 1.5f) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 1); + assertSqlEventually( + "sym\tts\n" + + "1.5\t1970-01-01T00:00:01.000000000Z\n", + "SELECT sym, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testFloatToVarchar() throws Exception { + String table = "test_qwp_float_to_varchar"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "v VARCHAR, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .floatColumn("v", 1.5f) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 1); + assertSqlEventually( + "v\tts\n" + + "1.5\t1970-01-01T00:00:01.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testInt() throws Exception { + String table = "test_qwp_int"; + useTable(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + // Integer.MIN_VALUE is the null sentinel for INT + sender.table(table) + .intColumn("i", Integer.MIN_VALUE) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("i", 0) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("i", Integer.MAX_VALUE) + .at(3_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("i", -42) + .at(4_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 4); + assertSqlEventually( + "i\ttimestamp\n" + + "null\t1970-01-01T00:00:01.000000000Z\n" + + "0\t1970-01-01T00:00:02.000000000Z\n" + + "2147483647\t1970-01-01T00:00:03.000000000Z\n" + + "-42\t1970-01-01T00:00:04.000000000Z\n", + "SELECT i, timestamp FROM " + table + " ORDER BY timestamp"); + } + + @Test + public void testIntToBooleanCoercionError() throws Exception { + String table = "test_qwp_int_to_boolean_error"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "b BOOLEAN, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("b", 1) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected error mentioning INT and BOOLEAN but got: " + msg, + msg.contains("INT") && msg.contains("BOOLEAN") + ); + } + } + + @Test + public void testIntToByte() throws Exception { + String table = "test_qwp_int_to_byte"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "b BYTE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("b", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("b", -128) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("b", 127) + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "b\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-128\t1970-01-01T00:00:02.000000000Z\n" + + "127\t1970-01-01T00:00:03.000000000Z\n", + "SELECT b, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testIntToByteOverflowError() throws Exception { + String table = "test_qwp_int_to_byte_overflow"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "b BYTE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("b", 128) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected overflow error but got: " + msg, + msg.contains("integer value 128 out of range for BYTE") + ); + } + } + + @Test + public void testIntToCharCoercionError() throws Exception { + String table = "test_qwp_int_to_char_error"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "c CHAR, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("c", 65) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected error mentioning INT and CHAR but got: " + msg, + msg.contains("INT") && msg.contains("CHAR") + ); + } + } + + @Test + public void testIntToDate() throws Exception { + String table = "test_qwp_int_to_date"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DATE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + // 86_400_000 millis = 1 day + sender.table(table) + .intColumn("d", 86_400_000) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", 0) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "1970-01-02T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z\n" + + "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testIntToDecimal() throws Exception { + String table = "test_qwp_int_to_decimal"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(6, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("d", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testIntToDecimal128() throws Exception { + String table = "test_qwp_int_to_decimal128"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(38, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("d", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", 0) + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n" + + "0.00\t1970-01-01T00:00:03.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testIntToDecimal16() throws Exception { + String table = "test_qwp_int_to_decimal16"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(4, 1), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("d", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", 0) + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "d\tts\n" + + "42.0\t1970-01-01T00:00:01.000000000Z\n" + + "-100.0\t1970-01-01T00:00:02.000000000Z\n" + + "0.0\t1970-01-01T00:00:03.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testIntToDecimal256() throws Exception { + String table = "test_qwp_int_to_decimal256"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(76, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("d", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", 0) + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n" + + "0.00\t1970-01-01T00:00:03.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testIntToDecimal64() throws Exception { + String table = "test_qwp_int_to_decimal64"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(18, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("d", Integer.MAX_VALUE) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", 0) + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "d\tts\n" + + "2147483647.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n" + + "0.00\t1970-01-01T00:00:03.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testIntToDecimal8() throws Exception { + String table = "test_qwp_int_to_decimal8"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(2, 1), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("d", 5) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", -9) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", 0) + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "d\tts\n" + + "5.0\t1970-01-01T00:00:01.000000000Z\n" + + "-9.0\t1970-01-01T00:00:02.000000000Z\n" + + "0.0\t1970-01-01T00:00:03.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testIntToDouble() throws Exception { + String table = "test_qwp_int_to_double"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DOUBLE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("d", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "42.0\t1970-01-01T00:00:01.000000000Z\n" + + "-100.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testIntToFloat() throws Exception { + String table = "test_qwp_int_to_float"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "f FLOAT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("f", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("f", -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("f", 0) + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "f\tts\n" + + "42.0\t1970-01-01T00:00:01.000000000Z\n" + + "-100.0\t1970-01-01T00:00:02.000000000Z\n" + + "0.0\t1970-01-01T00:00:03.000000000Z\n", + "SELECT f, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testIntToGeoHashCoercionError() throws Exception { + String table = "test_qwp_int_to_geohash_error"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "g GEOHASH(4c), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("g", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error mentioning INT but got: " + msg, + msg.contains("type coercion from INT to") && msg.contains("is not supported") + ); + } + } + + @Test + public void testIntToLong() throws Exception { + String table = "test_qwp_int_to_long"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "l LONG, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("l", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("l", Integer.MAX_VALUE) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("l", -1) + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "l\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "2147483647\t1970-01-01T00:00:02.000000000Z\n" + + "-1\t1970-01-01T00:00:03.000000000Z\n", + "SELECT l, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testIntToLong256CoercionError() throws Exception { + String table = "test_qwp_int_to_long256_error"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "v LONG256, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("v", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from INT to LONG256 is not supported") + ); + } + } + + @Test + public void testIntToShort() throws Exception { + String table = "test_qwp_int_to_short"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "s SHORT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("s", 1000) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("s", -32768) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("s", 32767) + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "s\tts\n" + + "1000\t1970-01-01T00:00:01.000000000Z\n" + + "-32768\t1970-01-01T00:00:02.000000000Z\n" + + "32767\t1970-01-01T00:00:03.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testIntToShortOverflowError() throws Exception { + String table = "test_qwp_int_to_short_overflow"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "s SHORT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("s", 32768) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected overflow error but got: " + msg, + msg.contains("integer value 32768 out of range for SHORT") + ); + } + } + + @Test + public void testIntToString() throws Exception { + String table = "test_qwp_int_to_string"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "s STRING, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("f", 1.5) + .intColumn("s", 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .doubleColumn("f", -42.25) + .intColumn("s", -100) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("s", 0) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); + assertSqlEventually( + "s\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-100\t1970-01-01T00:00:02.000000000Z\n" + + "0\t1970-01-01T00:00:03.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testFloat() throws Exception { - String table = "test_qwp_float"; + public void testIntToSymbol() throws Exception { + String table = "test_qwp_int_to_symbol"; useTable(table); + execute("CREATE TABLE " + table + " (" + + "s SYMBOL, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .floatColumn("f", 1.5f) + .intColumn("s", 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .floatColumn("f", -42.25f) + .intColumn("s", -1) .at(2_000_000, ChronoUnit.MICROS); sender.table(table) - .floatColumn("f", 0.0f) + .intColumn("s", 0) .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 3); + assertSqlEventually( + "s\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-1\t1970-01-01T00:00:02.000000000Z\n" + + "0\t1970-01-01T00:00:03.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testFloatToDouble() throws Exception { - String table = "test_qwp_float_to_double"; + public void testIntToTimestamp() throws Exception { + String table = "test_qwp_int_to_timestamp"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DOUBLE, " + + "t TIMESTAMP, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { + // 1_000_000 micros = 1 second sender.table(table) - .floatColumn("d", 1.5f) + .intColumn("t", 1_000_000) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .floatColumn("d", -42.25f) + .intColumn("t", 0) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "1.5\t1970-01-01T00:00:01.000000000Z\n" + - "-42.25\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + "t\tts\n" + + "1970-01-01T00:00:01.000000000Z\t1970-01-01T00:00:01.000000000Z\n" + + "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", + "SELECT t, ts FROM " + table + " ORDER BY ts"); } @Test - public void testInt() throws Exception { - String table = "test_qwp_int"; + public void testIntToUuidCoercionError() throws Exception { + String table = "test_qwp_int_to_uuid_error"; useTable(table); + execute("CREATE TABLE " + table + " (" + + "u UUID, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - // Integer.MIN_VALUE is the null sentinel for INT sender.table(table) - .intColumn("i", Integer.MIN_VALUE) + .intColumn("u", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from INT to UUID is not supported") + ); + } + } + + @Test + public void testIntToVarchar() throws Exception { + String table = "test_qwp_int_to_varchar"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "v VARCHAR, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("v", 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .intColumn("i", 0) + .intColumn("v", -100) .at(2_000_000, ChronoUnit.MICROS); sender.table(table) - .intColumn("i", Integer.MAX_VALUE) + .intColumn("v", Integer.MAX_VALUE) .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "v\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-100\t1970-01-01T00:00:02.000000000Z\n" + + "2147483647\t1970-01-01T00:00:03.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testLong() throws Exception { + String table = "test_qwp_long"; + useTable(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + // Long.MIN_VALUE is the null sentinel for LONG sender.table(table) - .intColumn("i", -42) - .at(4_000_000, ChronoUnit.MICROS); + .longColumn("l", Long.MIN_VALUE) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("l", 0) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("l", Long.MAX_VALUE) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 4); + assertTableSizeEventually(table, 3); assertSqlEventually( - "i\ttimestamp\n" + + "l\ttimestamp\n" + "null\t1970-01-01T00:00:01.000000000Z\n" + "0\t1970-01-01T00:00:02.000000000Z\n" + - "2147483647\t1970-01-01T00:00:03.000000000Z\n" + - "-42\t1970-01-01T00:00:04.000000000Z\n", - "SELECT i, timestamp FROM " + table + " ORDER BY timestamp"); + "9223372036854775807\t1970-01-01T00:00:03.000000000Z\n", + "SELECT l, timestamp FROM " + table + " ORDER BY timestamp"); } @Test - public void testIntToDecimal() throws Exception { - String table = "test_qwp_int_to_decimal"; + public void testLong256() throws Exception { + String table = "test_qwp_long256"; + useTable(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + // All zeros + sender.table(table) + .long256Column("v", 0, 0, 0, 0) + .at(1_000_000, ChronoUnit.MICROS); + // Mixed values + sender.table(table) + .long256Column("v", 1, 2, 3, 4) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + } + + @Test + public void testLongToBooleanCoercionError() throws Exception { + String table = "test_qwp_long_to_boolean_error"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(6, 2), " + + "b BOOLEAN, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("d", 42) + .longColumn("b", 1) .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected error mentioning LONG and BOOLEAN but got: " + msg, + msg.contains("LONG") && msg.contains("BOOLEAN") + ); + } + } + + @Test + public void testLongToByte() throws Exception { + String table = "test_qwp_long_to_byte"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "b BYTE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("d", -100) + .longColumn("b", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("b", -128) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("b", 127) + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "b\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-128\t1970-01-01T00:00:02.000000000Z\n" + + "127\t1970-01-01T00:00:03.000000000Z\n", + "SELECT b, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testLongToByteOverflowError() throws Exception { + String table = "test_qwp_long_to_byte_overflow"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "b BYTE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("b", 128) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected overflow error but got: " + msg, + msg.contains("integer value 128 out of range for BYTE") + ); + } + } + + @Test + public void testLongToCharCoercionError() throws Exception { + String table = "test_qwp_long_to_char_error"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "c CHAR, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("c", 65) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected error mentioning LONG and CHAR but got: " + msg, + msg.contains("LONG") && msg.contains("CHAR") + ); + } + } + + @Test + public void testLongToDate() throws Exception { + String table = "test_qwp_long_to_date"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DATE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("d", 86_400_000L) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("d", 0L) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "1970-01-02T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z\n" + + "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testLongToDecimal() throws Exception { + String table = "test_qwp_long_to_decimal"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(10, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("d", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("d", -100) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } @@ -378,8 +1702,8 @@ public void testIntToDecimal() throws Exception { } @Test - public void testIntToDecimal128() throws Exception { - String table = "test_qwp_int_to_decimal128"; + public void testLongToDecimal128() throws Exception { + String table = "test_qwp_long_to_decimal128"; useTable(table); execute("CREATE TABLE " + table + " (" + "d DECIMAL(38, 2), " + @@ -389,29 +1713,25 @@ public void testIntToDecimal128() throws Exception { try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("d", 42) + .longColumn("d", 1_000_000_000L) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .intColumn("d", -100) + .longColumn("d", -1_000_000_000L) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("d", 0) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 2); assertSqlEventually( "d\tts\n" + - "42.00\t1970-01-01T00:00:01.000000000Z\n" + - "-100.00\t1970-01-01T00:00:02.000000000Z\n" + - "0.00\t1970-01-01T00:00:03.000000000Z\n", + "1000000000.00\t1970-01-01T00:00:01.000000000Z\n" + + "-1000000000.00\t1970-01-01T00:00:02.000000000Z\n", "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToDecimal16() throws Exception { - String table = "test_qwp_int_to_decimal16"; + public void testLongToDecimal16() throws Exception { + String table = "test_qwp_long_to_decimal16"; useTable(table); execute("CREATE TABLE " + table + " (" + "d DECIMAL(4, 1), " + @@ -421,29 +1741,25 @@ public void testIntToDecimal16() throws Exception { try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("d", 42) + .longColumn("d", 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .intColumn("d", -100) + .longColumn("d", -100) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("d", 0) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 2); assertSqlEventually( "d\tts\n" + "42.0\t1970-01-01T00:00:01.000000000Z\n" + - "-100.0\t1970-01-01T00:00:02.000000000Z\n" + - "0.0\t1970-01-01T00:00:03.000000000Z\n", + "-100.0\t1970-01-01T00:00:02.000000000Z\n", "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToDecimal256() throws Exception { - String table = "test_qwp_int_to_decimal256"; + public void testLongToDecimal256() throws Exception { + String table = "test_qwp_long_to_decimal256"; useTable(table); execute("CREATE TABLE " + table + " (" + "d DECIMAL(76, 2), " + @@ -453,61 +1769,53 @@ public void testIntToDecimal256() throws Exception { try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("d", 42) + .longColumn("d", Long.MAX_VALUE) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .intColumn("d", -100) + .longColumn("d", -1_000_000_000_000L) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("d", 0) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 2); assertSqlEventually( "d\tts\n" + - "42.00\t1970-01-01T00:00:01.000000000Z\n" + - "-100.00\t1970-01-01T00:00:02.000000000Z\n" + - "0.00\t1970-01-01T00:00:03.000000000Z\n", + "9223372036854775807.00\t1970-01-01T00:00:01.000000000Z\n" + + "-1000000000000.00\t1970-01-01T00:00:02.000000000Z\n", "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToDecimal64() throws Exception { - String table = "test_qwp_int_to_decimal64"; + public void testLongToDecimal32() throws Exception { + String table = "test_qwp_long_to_decimal32"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(18, 2), " + + "d DECIMAL(6, 2), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("d", Integer.MAX_VALUE) + .longColumn("d", 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .intColumn("d", -100) + .longColumn("d", -100) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("d", 0) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 2); assertSqlEventually( "d\tts\n" + - "2147483647.00\t1970-01-01T00:00:01.000000000Z\n" + - "-100.00\t1970-01-01T00:00:02.000000000Z\n" + - "0.00\t1970-01-01T00:00:03.000000000Z\n", + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n", "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToDecimal8() throws Exception { - String table = "test_qwp_int_to_decimal8"; + public void testLongToDecimal8() throws Exception { + String table = "test_qwp_long_to_decimal8"; useTable(table); execute("CREATE TABLE " + table + " (" + "d DECIMAL(2, 1), " + @@ -517,29 +1825,25 @@ public void testIntToDecimal8() throws Exception { try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("d", 5) + .longColumn("d", 5) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .intColumn("d", -9) + .longColumn("d", -9) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("d", 0) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 2); assertSqlEventually( "d\tts\n" + "5.0\t1970-01-01T00:00:01.000000000Z\n" + - "-9.0\t1970-01-01T00:00:02.000000000Z\n" + - "0.0\t1970-01-01T00:00:03.000000000Z\n", + "-9.0\t1970-01-01T00:00:02.000000000Z\n", "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToDouble() throws Exception { - String table = "test_qwp_int_to_double"; + public void testLongToDouble() throws Exception { + String table = "test_qwp_long_to_double"; useTable(table); execute("CREATE TABLE " + table + " (" + "d DOUBLE, " + @@ -549,10 +1853,10 @@ public void testIntToDouble() throws Exception { try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("d", 42) + .longColumn("d", 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .intColumn("d", -100) + .longColumn("d", -100) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } @@ -566,304 +1870,321 @@ public void testIntToDouble() throws Exception { } @Test - public void testIntToLong() throws Exception { - String table = "test_qwp_int_to_long"; + public void testLongToFloat() throws Exception { + String table = "test_qwp_long_to_float"; useTable(table); execute("CREATE TABLE " + table + " (" + - "l LONG, " + + "f FLOAT, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("l", 42) + .longColumn("f", 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .intColumn("l", Integer.MAX_VALUE) + .longColumn("f", -100) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("l", -1) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 2); assertSqlEventually( - "l\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "2147483647\t1970-01-01T00:00:02.000000000Z\n" + - "-1\t1970-01-01T00:00:03.000000000Z\n", - "SELECT l, ts FROM " + table + " ORDER BY ts"); + "f\tts\n" + + "42.0\t1970-01-01T00:00:01.000000000Z\n" + + "-100.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT f, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLong() throws Exception { - String table = "test_qwp_long"; + public void testLongToGeoHashCoercionError() throws Exception { + String table = "test_qwp_long_to_geohash_error"; useTable(table); + execute("CREATE TABLE " + table + " (" + + "g GEOHASH(4c), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - // Long.MIN_VALUE is the null sentinel for LONG sender.table(table) - .longColumn("l", Long.MIN_VALUE) + .longColumn("g", 42) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .longColumn("l", 0) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .longColumn("l", Long.MAX_VALUE) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error mentioning LONG but got: " + msg, + msg.contains("type coercion from LONG to") && msg.contains("is not supported") + ); } - - assertTableSizeEventually(table, 3); - assertSqlEventually( - "l\ttimestamp\n" + - "null\t1970-01-01T00:00:01.000000000Z\n" + - "0\t1970-01-01T00:00:02.000000000Z\n" + - "9223372036854775807\t1970-01-01T00:00:03.000000000Z\n", - "SELECT l, timestamp FROM " + table + " ORDER BY timestamp"); } @Test - public void testLong256() throws Exception { - String table = "test_qwp_long256"; + public void testLongToInt() throws Exception { + String table = "test_qwp_long_to_int"; useTable(table); + execute("CREATE TABLE " + table + " (" + + "i INT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - // All zeros + // Value in INT range should succeed sender.table(table) - .long256Column("v", 0, 0, 0, 0) + .longColumn("i", 42) .at(1_000_000, ChronoUnit.MICROS); - // Mixed values sender.table(table) - .long256Column("v", 1, 2, 3, 4) + .longColumn("i", -1) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); + assertSqlEventually( + "i\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-1\t1970-01-01T00:00:02.000000000Z\n", + "SELECT i, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToDecimal() throws Exception { - String table = "test_qwp_long_to_decimal"; + public void testLongToIntOverflowError() throws Exception { + String table = "test_qwp_long_to_int_overflow"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(10, 2), " + + "i INT, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("d", 42) + .longColumn("i", (long) Integer.MAX_VALUE + 1) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .longColumn("d", -100) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected overflow error but got: " + msg, + msg.contains("integer value 2147483648 out of range for INT") + ); } + } - assertTableSizeEventually(table, 2); - assertSqlEventually( - "d\tts\n" + - "42.00\t1970-01-01T00:00:01.000000000Z\n" + - "-100.00\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + @Test + public void testLongToLong256CoercionError() throws Exception { + String table = "test_qwp_long_to_long256_error"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "v LONG256, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("v", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from LONG to LONG256 is not supported") + ); + } } @Test - public void testLongToDecimal128() throws Exception { - String table = "test_qwp_long_to_decimal128"; + public void testLongToShort() throws Exception { + String table = "test_qwp_long_to_short"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(38, 2), " + + "s SHORT, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { + // Value in SHORT range should succeed sender.table(table) - .longColumn("d", 1_000_000_000L) + .longColumn("s", 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("d", -1_000_000_000L) + .longColumn("s", -1) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); - assertSqlEventually( - "d\tts\n" + - "1000000000.00\t1970-01-01T00:00:01.000000000Z\n" + - "-1000000000.00\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToDecimal16() throws Exception { - String table = "test_qwp_long_to_decimal16"; + public void testLongToShortOverflowError() throws Exception { + String table = "test_qwp_long_to_short_overflow"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(4, 1), " + + "s SHORT, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("d", 42) + .longColumn("s", 32768) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .longColumn("d", -100) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected overflow error but got: " + msg, + msg.contains("integer value 32768 out of range for SHORT") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "d\tts\n" + - "42.0\t1970-01-01T00:00:01.000000000Z\n" + - "-100.0\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToDecimal256() throws Exception { - String table = "test_qwp_long_to_decimal256"; + public void testLongToString() throws Exception { + String table = "test_qwp_long_to_string"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(76, 2), " + + "s STRING, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("d", Long.MAX_VALUE) + .longColumn("s", 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("d", -1_000_000_000_000L) + .longColumn("s", Long.MAX_VALUE) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "9223372036854775807.00\t1970-01-01T00:00:01.000000000Z\n" + - "-1000000000000.00\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + "s\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "9223372036854775807\t1970-01-01T00:00:02.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToDecimal32() throws Exception { - String table = "test_qwp_long_to_decimal32"; + public void testLongToSymbol() throws Exception { + String table = "test_qwp_long_to_symbol"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(6, 2), " + + "s SYMBOL, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("d", 42) + .longColumn("s", 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("d", -100) + .longColumn("s", -1) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "42.00\t1970-01-01T00:00:01.000000000Z\n" + - "-100.00\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + "s\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-1\t1970-01-01T00:00:02.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToDecimal8() throws Exception { - String table = "test_qwp_long_to_decimal8"; + public void testLongToTimestamp() throws Exception { + String table = "test_qwp_long_to_timestamp"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(2, 1), " + + "t TIMESTAMP, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("d", 5) + .longColumn("t", 1_000_000L) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("d", -9) + .longColumn("t", 0L) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "5.0\t1970-01-01T00:00:01.000000000Z\n" + - "-9.0\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + "t\tts\n" + + "1970-01-01T00:00:01.000000000Z\t1970-01-01T00:00:01.000000000Z\n" + + "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", + "SELECT t, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToInt() throws Exception { - String table = "test_qwp_long_to_int"; + public void testLongToUuidCoercionError() throws Exception { + String table = "test_qwp_long_to_uuid_error"; useTable(table); execute("CREATE TABLE " + table + " (" + - "i INT, " + + "u UUID, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - // Value in INT range should succeed sender.table(table) - .longColumn("i", 42) + .longColumn("u", 42) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .longColumn("i", -1) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from LONG to UUID is not supported") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "i\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-1\t1970-01-01T00:00:02.000000000Z\n", - "SELECT i, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToShort() throws Exception { - String table = "test_qwp_long_to_short"; + public void testLongToVarchar() throws Exception { + String table = "test_qwp_long_to_varchar"; useTable(table); execute("CREATE TABLE " + table + " (" + - "s SHORT, " + + "v VARCHAR, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - // Value in SHORT range should succeed sender.table(table) - .longColumn("s", 42) + .longColumn("v", 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("s", -1) + .longColumn("v", Long.MAX_VALUE) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); + assertSqlEventually( + "v\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "9223372036854775807\t1970-01-01T00:00:02.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); } @Test @@ -1380,6 +2701,32 @@ public void testUuid() throws Exception { "SELECT u, timestamp FROM " + table + " ORDER BY timestamp"); } + @Test + public void testUuidToShortCoercionError() throws Exception { + String table = "test_qwp_uuid_to_short_error"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "s SHORT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); + sender.table(table) + .uuidColumn("s", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from UUID to SHORT is not supported") + ); + } + } + @Test public void testWriteAllTypesInOneRow() throws Exception { String table = "test_qwp_all_types"; From 3e444a04a63a03a5753861926b9b5d3cb5852e0c Mon Sep 17 00:00:00 2001 From: bluestreak Date: Sun, 15 Feb 2026 03:41:14 +0000 Subject: [PATCH 10/89] wip4 --- .../qwp/client/QwpWebSocketSender.java | 15 + .../cutlass/qwp/protocol/QwpTableBuffer.java | 75 +- .../cutlass/qwp/client/QwpSenderTest.java | 3081 ++++++++++++----- 3 files changed, 2221 insertions(+), 950 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 4a70fb1..d428dbb 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -596,6 +596,21 @@ public QwpWebSocketSender longColumn(CharSequence columnName, long value) { * @param value the int value * @return this sender for method chaining */ + /** + * Adds a BYTE column value to the current row. + * + * @param columnName the column name + * @param value the byte value + * @return this sender for method chaining + */ + public QwpWebSocketSender byteColumn(CharSequence columnName, byte value) { + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_BYTE, false); + col.addByte(value); + return this; + } + public QwpWebSocketSender intColumn(CharSequence columnName, int value) { checkNotClosed(); checkTableSelected(); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index 68f75ec..f70dd98 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -299,6 +299,7 @@ public static class ColumnBuffer { // For Decimal128: two longs per value (128-bit unscaled: high, low) // For Decimal256: four longs per value (256-bit unscaled: hh, hl, lh, ll) private byte decimalScale = -1; // Shared scale for column (-1 = not set) + private final Decimal256 rescaleTemp = new Decimal256(); // Reusable temp for rescaling private long[] decimal64Values; // Decimal64: one long per value private long[] decimal128High; // Decimal128: high 64 bits private long[] decimal128Low; // Decimal128: low 64 bits @@ -722,10 +723,10 @@ public void addLong256(long l0, long l1, long l2, long l3) { /** * Adds a Decimal64 value. - * All values in a decimal column must share the same scale. + * If the value's scale differs from the column's established scale, + * the value is automatically rescaled to match. * * @param value the Decimal64 value to add - * @throws LineSenderException if the scale doesn't match previous values */ public void addDecimal64(Decimal64 value) { if (value == null || value.isNull()) { @@ -733,17 +734,26 @@ public void addDecimal64(Decimal64 value) { return; } ensureCapacity(); - validateAndSetScale((byte) value.getScale()); - decimal64Values[valueCount++] = value.getValue(); + if (decimalScale == -1) { + decimalScale = (byte) value.getScale(); + decimal64Values[valueCount++] = value.getValue(); + } else if (decimalScale != value.getScale()) { + rescaleTemp.ofRaw(value.getValue()); + rescaleTemp.setScale(value.getScale()); + rescaleTemp.rescale(decimalScale); + decimal64Values[valueCount++] = rescaleTemp.getLl(); + } else { + decimal64Values[valueCount++] = value.getValue(); + } size++; } /** * Adds a Decimal128 value. - * All values in a decimal column must share the same scale. + * If the value's scale differs from the column's established scale, + * the value is automatically rescaled to match. * * @param value the Decimal128 value to add - * @throws LineSenderException if the scale doesn't match previous values */ public void addDecimal128(Decimal128 value) { if (value == null || value.isNull()) { @@ -751,7 +761,18 @@ public void addDecimal128(Decimal128 value) { return; } ensureCapacity(); - validateAndSetScale((byte) value.getScale()); + if (decimalScale == -1) { + decimalScale = (byte) value.getScale(); + } else if (decimalScale != value.getScale()) { + rescaleTemp.ofRaw(value.getHigh(), value.getLow()); + rescaleTemp.setScale(value.getScale()); + rescaleTemp.rescale(decimalScale); + decimal128High[valueCount] = rescaleTemp.getLh(); + decimal128Low[valueCount] = rescaleTemp.getLl(); + valueCount++; + size++; + return; + } decimal128High[valueCount] = value.getHigh(); decimal128Low[valueCount] = value.getLow(); valueCount++; @@ -760,10 +781,10 @@ public void addDecimal128(Decimal128 value) { /** * Adds a Decimal256 value. - * All values in a decimal column must share the same scale. + * If the value's scale differs from the column's established scale, + * the value is automatically rescaled to match. * * @param value the Decimal256 value to add - * @throws LineSenderException if the scale doesn't match previous values */ public void addDecimal256(Decimal256 value) { if (value == null || value.isNull()) { @@ -771,32 +792,20 @@ public void addDecimal256(Decimal256 value) { return; } ensureCapacity(); - validateAndSetScale((byte) value.getScale()); - decimal256Hh[valueCount] = value.getHh(); - decimal256Hl[valueCount] = value.getHl(); - decimal256Lh[valueCount] = value.getLh(); - decimal256Ll[valueCount] = value.getLl(); - valueCount++; - size++; - } - - /** - * Validates that the given scale matches the column's scale. - * If this is the first value, sets the column scale. - * - * @param scale the scale of the value being added - * @throws LineSenderException if the scale doesn't match - */ - private void validateAndSetScale(byte scale) { + Decimal256 src = value; if (decimalScale == -1) { - decimalScale = scale; - } else if (decimalScale != scale) { - throw new LineSenderException( - "decimal scale mismatch in column '" + name + "': expected " + - decimalScale + " but got " + scale + - ". All values in a decimal column must have the same scale." - ); + decimalScale = (byte) value.getScale(); + } else if (decimalScale != value.getScale()) { + rescaleTemp.copyFrom(value); + rescaleTemp.rescale(decimalScale); + src = rescaleTemp; } + decimal256Hh[valueCount] = src.getHh(); + decimal256Hl[valueCount] = src.getHl(); + decimal256Lh[valueCount] = src.getLh(); + decimal256Ll[valueCount] = src.getLl(); + valueCount++; + size++; } // ==================== Array methods ==================== diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java index cf73ee3..b4e61c6 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java @@ -26,6 +26,8 @@ import io.questdb.client.cutlass.line.LineSenderException; import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; +import io.questdb.client.std.Decimal64; +import io.questdb.client.std.Decimal128; import io.questdb.client.std.Decimal256; import io.questdb.client.test.cutlass.line.AbstractLineSenderTest; import org.junit.Assert; @@ -52,189 +54,214 @@ public static void setUpStatic() { AbstractLineSenderTest.setUpStatic(); } - // === Exact Type Match Tests === + // === BYTE coercion tests === @Test - public void testBoolean() throws Exception { - String table = "test_qwp_boolean"; + public void testByteToBooleanCoercionError() throws Exception { + String table = "test_qwp_byte_to_boolean_error"; useTable(table); + execute("CREATE TABLE " + table + " (" + + "b BOOLEAN, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .boolColumn("b", true) + .byteColumn("b", (byte) 1) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .boolColumn("b", false) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected error mentioning BYTE and BOOLEAN but got: " + msg, + msg.contains("BYTE") && msg.contains("BOOLEAN") + ); } + } - assertTableSizeEventually(table, 2); - assertSqlEventually( - "b\ttimestamp\n" + - "true\t1970-01-01T00:00:01.000000000Z\n" + - "false\t1970-01-01T00:00:02.000000000Z\n", - "SELECT b, timestamp FROM " + table + " ORDER BY timestamp"); + @Test + public void testByteToCharCoercionError() throws Exception { + String table = "test_qwp_byte_to_char_error"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "c CHAR, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .byteColumn("c", (byte) 65) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected error mentioning BYTE and CHAR but got: " + msg, + msg.contains("BYTE") && msg.contains("CHAR") + ); + } } @Test - public void testByte() throws Exception { - String table = "test_qwp_byte"; + public void testByteToDate() throws Exception { + String table = "test_qwp_byte_to_date"; useTable(table); execute("CREATE TABLE " + table + " (" + - "b BYTE, " + + "d DATE, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("b", (short) -1) + .byteColumn("d", (byte) 100) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .shortColumn("b", (short) 0) + .byteColumn("d", (byte) 0) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("b", (short) 127) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "1970-01-01T00:00:00.100000000Z\t1970-01-01T00:00:01.000000000Z\n" + + "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testChar() throws Exception { - String table = "test_qwp_char"; + public void testByteToDecimal() throws Exception { + String table = "test_qwp_byte_to_decimal"; useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(6, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .charColumn("c", 'A') + .byteColumn("d", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .charColumn("c", 'ü') // ü + .byteColumn("d", (byte) -100) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .charColumn("c", '中') // 中 - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 2); assertSqlEventually( - "c\ttimestamp\n" + - "A\t1970-01-01T00:00:01.000000000Z\n" + - "ü\t1970-01-01T00:00:02.000000000Z\n" + - "中\t1970-01-01T00:00:03.000000000Z\n", - "SELECT c, timestamp FROM " + table + " ORDER BY timestamp"); + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDecimal() throws Exception { - String table = "test_qwp_decimal"; + public void testByteToDecimal128() throws Exception { + String table = "test_qwp_byte_to_decimal128"; useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(38, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .decimalColumn("d", "123.45") + .byteColumn("d", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .decimalColumn("d", "-999.99") + .byteColumn("d", (byte) -1) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .decimalColumn("d", "0.01") - .at(3_000_000, ChronoUnit.MICROS); - sender.table(table) - .decimalColumn("d", Decimal256.fromLong(42_000, 3)) - .at(4_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 4); + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-1.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDouble() throws Exception { - String table = "test_qwp_double"; + public void testByteToDecimal16() throws Exception { + String table = "test_qwp_byte_to_decimal16"; useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(4, 1), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("d", 42.5) + .byteColumn("d", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .doubleColumn("d", -1.0E10) + .byteColumn("d", (byte) -9) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .doubleColumn("d", Double.MAX_VALUE) - .at(3_000_000, ChronoUnit.MICROS); - sender.table(table) - .doubleColumn("d", Double.MIN_VALUE) - .at(4_000_000, ChronoUnit.MICROS); - // NaN and Inf should be stored as null - sender.table(table) - .doubleColumn("d", Double.NaN) - .at(5_000_000, ChronoUnit.MICROS); - sender.table(table) - .doubleColumn("d", Double.POSITIVE_INFINITY) - .at(6_000_000, ChronoUnit.MICROS); - sender.table(table) - .doubleColumn("d", Double.NEGATIVE_INFINITY) - .at(7_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 7); + assertTableSizeEventually(table, 2); assertSqlEventually( - "d\ttimestamp\n" + - "42.5\t1970-01-01T00:00:01.000000000Z\n" + - "-1.0E10\t1970-01-01T00:00:02.000000000Z\n" + - "1.7976931348623157E308\t1970-01-01T00:00:03.000000000Z\n" + - "4.9E-324\t1970-01-01T00:00:04.000000000Z\n" + - "null\t1970-01-01T00:00:05.000000000Z\n" + - "null\t1970-01-01T00:00:06.000000000Z\n" + - "null\t1970-01-01T00:00:07.000000000Z\n", - "SELECT d, timestamp FROM " + table + " ORDER BY timestamp"); + "d\tts\n" + + "42.0\t1970-01-01T00:00:01.000000000Z\n" + + "-9.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDoubleArray() throws Exception { - String table = "test_qwp_double_array"; + public void testByteToDecimal256() throws Exception { + String table = "test_qwp_byte_to_decimal256"; useTable(table); - - double[] arr1d = createDoubleArray(5); - double[][] arr2d = createDoubleArray(2, 3); - double[][][] arr3d = createDoubleArray(1, 2, 3); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(76, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleArray("a1", arr1d) - .doubleArray("a2", arr2d) - .doubleArray("a3", arr3d) + .byteColumn("d", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .byteColumn("d", (byte) -1) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 1); + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-1.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDoubleToDecimal() throws Exception { - String table = "test_qwp_double_to_decimal"; + public void testByteToDecimal64() throws Exception { + String table = "test_qwp_byte_to_decimal64"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(10, 2), " + + "d DECIMAL(18, 2), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("d", 123.45) + .byteColumn("d", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .doubleColumn("d", -42.10) + .byteColumn("d", (byte) -1) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } @@ -242,221 +269,244 @@ public void testDoubleToDecimal() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( "d\tts\n" + - "123.45\t1970-01-01T00:00:01.000000000Z\n" + - "-42.10\t1970-01-01T00:00:02.000000000Z\n", + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-1.00\t1970-01-01T00:00:02.000000000Z\n", "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDoubleToDecimalPrecisionLossError() throws Exception { - String table = "test_qwp_double_to_decimal_prec"; + public void testByteToDecimal8() throws Exception { + String table = "test_qwp_byte_to_decimal8"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(10, 2), " + + "d DECIMAL(2, 1), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("d", 123.456) + .byteColumn("d", (byte) 5) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .byteColumn("d", (byte) -9) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected precision loss error but got: " + msg, - msg.contains("loses precision") && msg.contains("123.456") && msg.contains("scale=2") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "5.0\t1970-01-01T00:00:01.000000000Z\n" + + "-9.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDoubleToByte() throws Exception { - String table = "test_qwp_double_to_byte"; + public void testByteToDouble() throws Exception { + String table = "test_qwp_byte_to_double"; useTable(table); execute("CREATE TABLE " + table + " (" + - "b BYTE, " + + "d DOUBLE, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("b", 42.0) + .byteColumn("d", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .doubleColumn("b", -100.0) + .byteColumn("d", (byte) -100) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "b\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-100\t1970-01-01T00:00:02.000000000Z\n", - "SELECT b, ts FROM " + table + " ORDER BY ts"); + "d\tts\n" + + "42.0\t1970-01-01T00:00:01.000000000Z\n" + + "-100.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDoubleToBytePrecisionLossError() throws Exception { - String table = "test_qwp_double_to_byte_prec"; + public void testByteToFloat() throws Exception { + String table = "test_qwp_byte_to_float"; useTable(table); execute("CREATE TABLE " + table + " (" + - "b BYTE, " + + "f FLOAT, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("b", 42.5) + .byteColumn("f", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .byteColumn("f", (byte) -100) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected precision loss error but got: " + msg, - msg.contains("loses precision") && msg.contains("42.5") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "f\tts\n" + + "42.0\t1970-01-01T00:00:01.000000000Z\n" + + "-100.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT f, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDoubleToByteOverflowError() throws Exception { - String table = "test_qwp_double_to_byte_ovf"; + public void testByteToGeoHashCoercionError() throws Exception { + String table = "test_qwp_byte_to_geohash_error"; useTable(table); execute("CREATE TABLE " + table + " (" + - "b BYTE, " + + "g GEOHASH(4c), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("b", 200.0) + .byteColumn("g", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected overflow error but got: " + msg, - msg.contains("integer value 200 out of range for BYTE") + "Expected coercion error mentioning BYTE but got: " + msg, + msg.contains("type coercion from BYTE to") && msg.contains("is not supported") ); } } @Test - public void testDoubleToFloat() throws Exception { - String table = "test_qwp_double_to_float"; + public void testByteToInt() throws Exception { + String table = "test_qwp_byte_to_int"; useTable(table); execute("CREATE TABLE " + table + " (" + - "f FLOAT, " + + "i INT, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("f", 1.5) + .byteColumn("i", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .doubleColumn("f", -42.25) + .byteColumn("i", Byte.MAX_VALUE) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .byteColumn("i", Byte.MIN_VALUE) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); + assertSqlEventually( + "i\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "127\t1970-01-01T00:00:02.000000000Z\n" + + "-128\t1970-01-01T00:00:03.000000000Z\n", + "SELECT i, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDoubleToInt() throws Exception { - String table = "test_qwp_double_to_int"; + public void testByteToLong() throws Exception { + String table = "test_qwp_byte_to_long"; useTable(table); execute("CREATE TABLE " + table + " (" + - "i INT, " + + "l LONG, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("i", 100_000.0) + .byteColumn("l", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .doubleColumn("i", -42.0) + .byteColumn("l", Byte.MAX_VALUE) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .byteColumn("l", Byte.MIN_VALUE) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); assertSqlEventually( - "i\tts\n" + - "100000\t1970-01-01T00:00:01.000000000Z\n" + - "-42\t1970-01-01T00:00:02.000000000Z\n", - "SELECT i, ts FROM " + table + " ORDER BY ts"); + "l\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "127\t1970-01-01T00:00:02.000000000Z\n" + + "-128\t1970-01-01T00:00:03.000000000Z\n", + "SELECT l, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDoubleToIntPrecisionLossError() throws Exception { - String table = "test_qwp_double_to_int_prec"; + public void testByteToLong256CoercionError() throws Exception { + String table = "test_qwp_byte_to_long256_error"; useTable(table); execute("CREATE TABLE " + table + " (" + - "i INT, " + + "v LONG256, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("i", 3.14) + .byteColumn("v", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected precision loss error but got: " + msg, - msg.contains("loses precision") && msg.contains("3.14") + "Expected coercion error but got: " + msg, + msg.contains("type coercion from BYTE to LONG256 is not supported") ); } } @Test - public void testDoubleToLong() throws Exception { - String table = "test_qwp_double_to_long"; + public void testByteToShort() throws Exception { + String table = "test_qwp_byte_to_short"; useTable(table); execute("CREATE TABLE " + table + " (" + - "l LONG, " + + "s SHORT, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("l", 1_000_000.0) + .byteColumn("s", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .doubleColumn("l", -42.0) + .byteColumn("s", Byte.MIN_VALUE) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .byteColumn("s", Byte.MAX_VALUE) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); assertSqlEventually( - "l\tts\n" + - "1000000\t1970-01-01T00:00:01.000000000Z\n" + - "-42\t1970-01-01T00:00:02.000000000Z\n", - "SELECT l, ts FROM " + table + " ORDER BY ts"); + "s\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-128\t1970-01-01T00:00:02.000000000Z\n" + + "127\t1970-01-01T00:00:03.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDoubleToString() throws Exception { - String table = "test_qwp_double_to_string"; + public void testByteToString() throws Exception { + String table = "test_qwp_byte_to_string"; useTable(table); execute("CREATE TABLE " + table + " (" + "s STRING, " + @@ -466,385 +516,366 @@ public void testDoubleToString() throws Exception { try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("s", 3.14) + .byteColumn("s", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .doubleColumn("s", -42.0) + .byteColumn("s", (byte) -100) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .byteColumn("s", (byte) 0) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); assertSqlEventually( "s\tts\n" + - "3.14\t1970-01-01T00:00:01.000000000Z\n" + - "-42.0\t1970-01-01T00:00:02.000000000Z\n", + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-100\t1970-01-01T00:00:02.000000000Z\n" + + "0\t1970-01-01T00:00:03.000000000Z\n", "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDoubleToSymbol() throws Exception { - String table = "test_qwp_double_to_symbol"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "sym SYMBOL, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .doubleColumn("sym", 3.14) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 1); - assertSqlEventually( - "sym\tts\n" + - "3.14\t1970-01-01T00:00:01.000000000Z\n", - "SELECT sym, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testDoubleToVarchar() throws Exception { - String table = "test_qwp_double_to_varchar"; + public void testByteToSymbol() throws Exception { + String table = "test_qwp_byte_to_symbol"; useTable(table); execute("CREATE TABLE " + table + " (" + - "v VARCHAR, " + + "s SYMBOL, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("v", 3.14) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .doubleColumn("v", -42.0) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "v\tts\n" + - "3.14\t1970-01-01T00:00:01.000000000Z\n" + - "-42.0\t1970-01-01T00:00:02.000000000Z\n", - "SELECT v, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testFloat() throws Exception { - String table = "test_qwp_float"; - useTable(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .floatColumn("f", 1.5f) + .byteColumn("s", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .floatColumn("f", -42.25f) + .byteColumn("s", (byte) -1) .at(2_000_000, ChronoUnit.MICROS); sender.table(table) - .floatColumn("f", 0.0f) + .byteColumn("s", (byte) 0) .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 3); + assertSqlEventually( + "s\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-1\t1970-01-01T00:00:02.000000000Z\n" + + "0\t1970-01-01T00:00:03.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testFloatToDecimal() throws Exception { - String table = "test_qwp_float_to_decimal"; + public void testByteToTimestamp() throws Exception { + String table = "test_qwp_byte_to_timestamp"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(10, 2), " + + "t TIMESTAMP, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .floatColumn("d", 1.5f) + .byteColumn("t", (byte) 100) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .floatColumn("d", -42.25f) + .byteColumn("t", (byte) 0) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "1.50\t1970-01-01T00:00:01.000000000Z\n" + - "-42.25\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + "t\tts\n" + + "1970-01-01T00:00:00.000100000Z\t1970-01-01T00:00:01.000000000Z\n" + + "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", + "SELECT t, ts FROM " + table + " ORDER BY ts"); } @Test - public void testFloatToDecimalPrecisionLossError() throws Exception { - String table = "test_qwp_float_to_decimal_prec"; + public void testByteToUuidCoercionError() throws Exception { + String table = "test_qwp_byte_to_uuid_error"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(10, 1), " + + "u UUID, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .floatColumn("d", 1.25f) + .byteColumn("u", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected precision loss error but got: " + msg, - msg.contains("loses precision") && msg.contains("scale=1") + "Expected coercion error but got: " + msg, + msg.contains("type coercion from BYTE to UUID is not supported") ); } } @Test - public void testFloatToDouble() throws Exception { - String table = "test_qwp_float_to_double"; + public void testByteToVarchar() throws Exception { + String table = "test_qwp_byte_to_varchar"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DOUBLE, " + + "v VARCHAR, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .floatColumn("d", 1.5f) + .byteColumn("v", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .floatColumn("d", -42.25f) + .byteColumn("v", (byte) -100) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .byteColumn("v", Byte.MAX_VALUE) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); assertSqlEventually( - "d\tts\n" + - "1.5\t1970-01-01T00:00:01.000000000Z\n" + - "-42.25\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + "v\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-100\t1970-01-01T00:00:02.000000000Z\n" + + "127\t1970-01-01T00:00:03.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); } + // === Exact Type Match Tests === + @Test - public void testFloatToInt() throws Exception { - String table = "test_qwp_float_to_int"; + public void testBoolean() throws Exception { + String table = "test_qwp_boolean"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "i INT, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .floatColumn("i", 42.0f) + .boolColumn("b", true) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .floatColumn("i", -100.0f) + .boolColumn("b", false) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "i\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-100\t1970-01-01T00:00:02.000000000Z\n", - "SELECT i, ts FROM " + table + " ORDER BY ts"); + "b\ttimestamp\n" + + "true\t1970-01-01T00:00:01.000000000Z\n" + + "false\t1970-01-01T00:00:02.000000000Z\n", + "SELECT b, timestamp FROM " + table + " ORDER BY timestamp"); } @Test - public void testFloatToIntPrecisionLossError() throws Exception { - String table = "test_qwp_float_to_int_prec"; + public void testByte() throws Exception { + String table = "test_qwp_byte"; useTable(table); execute("CREATE TABLE " + table + " (" + - "i INT, " + + "b BYTE, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .floatColumn("i", 3.14f) + .shortColumn("b", (short) -1) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("b", (short) 0) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("b", (short) 127) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected precision loss error but got: " + msg, - msg.contains("loses precision") - ); } + + assertTableSizeEventually(table, 3); } @Test - public void testFloatToLong() throws Exception { - String table = "test_qwp_float_to_long"; + public void testChar() throws Exception { + String table = "test_qwp_char"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "l LONG, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .floatColumn("l", 1000.0f) + .charColumn("c", 'A') .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .charColumn("c", 'ü') // ü + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .charColumn("c", '中') // 中 + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 1); + assertTableSizeEventually(table, 3); assertSqlEventually( - "l\tts\n" + - "1000\t1970-01-01T00:00:01.000000000Z\n", - "SELECT l, ts FROM " + table + " ORDER BY ts"); + "c\ttimestamp\n" + + "A\t1970-01-01T00:00:01.000000000Z\n" + + "ü\t1970-01-01T00:00:02.000000000Z\n" + + "中\t1970-01-01T00:00:03.000000000Z\n", + "SELECT c, timestamp FROM " + table + " ORDER BY timestamp"); } @Test - public void testFloatToString() throws Exception { - String table = "test_qwp_float_to_string"; + public void testDecimal() throws Exception { + String table = "test_qwp_decimal"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "s STRING, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .floatColumn("s", 1.5f) + .decimalColumn("d", "123.45") .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .decimalColumn("d", "-999.99") + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .decimalColumn("d", "0.01") + .at(3_000_000, ChronoUnit.MICROS); + sender.table(table) + .decimalColumn("d", Decimal256.fromLong(42_000, 2)) + .at(4_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 1); - assertSqlEventually( - "s\tts\n" + - "1.5\t1970-01-01T00:00:01.000000000Z\n", - "SELECT s, ts FROM " + table + " ORDER BY ts"); + assertTableSizeEventually(table, 4); } @Test - public void testFloatToSymbol() throws Exception { - String table = "test_qwp_float_to_symbol"; + public void testDouble() throws Exception { + String table = "test_qwp_double"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "sym SYMBOL, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .floatColumn("sym", 1.5f) + .doubleColumn("d", 42.5) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .doubleColumn("d", -1.0E10) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .doubleColumn("d", Double.MAX_VALUE) + .at(3_000_000, ChronoUnit.MICROS); + sender.table(table) + .doubleColumn("d", Double.MIN_VALUE) + .at(4_000_000, ChronoUnit.MICROS); + // NaN and Inf should be stored as null + sender.table(table) + .doubleColumn("d", Double.NaN) + .at(5_000_000, ChronoUnit.MICROS); + sender.table(table) + .doubleColumn("d", Double.POSITIVE_INFINITY) + .at(6_000_000, ChronoUnit.MICROS); + sender.table(table) + .doubleColumn("d", Double.NEGATIVE_INFINITY) + .at(7_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 1); + assertTableSizeEventually(table, 7); assertSqlEventually( - "sym\tts\n" + - "1.5\t1970-01-01T00:00:01.000000000Z\n", - "SELECT sym, ts FROM " + table + " ORDER BY ts"); + "d\ttimestamp\n" + + "42.5\t1970-01-01T00:00:01.000000000Z\n" + + "-1.0E10\t1970-01-01T00:00:02.000000000Z\n" + + "1.7976931348623157E308\t1970-01-01T00:00:03.000000000Z\n" + + "4.9E-324\t1970-01-01T00:00:04.000000000Z\n" + + "null\t1970-01-01T00:00:05.000000000Z\n" + + "null\t1970-01-01T00:00:06.000000000Z\n" + + "null\t1970-01-01T00:00:07.000000000Z\n", + "SELECT d, timestamp FROM " + table + " ORDER BY timestamp"); } @Test - public void testFloatToVarchar() throws Exception { - String table = "test_qwp_float_to_varchar"; + public void testDoubleArray() throws Exception { + String table = "test_qwp_double_array"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "v VARCHAR, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); + + double[] arr1d = createDoubleArray(5); + double[][] arr2d = createDoubleArray(2, 3); + double[][][] arr3d = createDoubleArray(1, 2, 3); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .floatColumn("v", 1.5f) + .doubleArray("a1", arr1d) + .doubleArray("a2", arr2d) + .doubleArray("a3", arr3d) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 1); - assertSqlEventually( - "v\tts\n" + - "1.5\t1970-01-01T00:00:01.000000000Z\n", - "SELECT v, ts FROM " + table + " ORDER BY ts"); } @Test - public void testInt() throws Exception { - String table = "test_qwp_int"; + public void testDoubleToDecimal() throws Exception { + String table = "test_qwp_double_to_decimal"; useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(10, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - // Integer.MIN_VALUE is the null sentinel for INT sender.table(table) - .intColumn("i", Integer.MIN_VALUE) + .doubleColumn("d", 123.45) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .intColumn("i", 0) + .doubleColumn("d", -42.10) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("i", Integer.MAX_VALUE) - .at(3_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("i", -42) - .at(4_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 4); + assertTableSizeEventually(table, 2); assertSqlEventually( - "i\ttimestamp\n" + - "null\t1970-01-01T00:00:01.000000000Z\n" + - "0\t1970-01-01T00:00:02.000000000Z\n" + - "2147483647\t1970-01-01T00:00:03.000000000Z\n" + - "-42\t1970-01-01T00:00:04.000000000Z\n", - "SELECT i, timestamp FROM " + table + " ORDER BY timestamp"); + "d\tts\n" + + "123.45\t1970-01-01T00:00:01.000000000Z\n" + + "-42.10\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToBooleanCoercionError() throws Exception { - String table = "test_qwp_int_to_boolean_error"; + public void testDoubleToDecimalPrecisionLossError() throws Exception { + String table = "test_qwp_double_to_decimal_prec"; useTable(table); execute("CREATE TABLE " + table + " (" + - "b BOOLEAN, " + + "d DECIMAL(10, 2), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("b", 1) + .doubleColumn("d", 123.456) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected error mentioning INT and BOOLEAN but got: " + msg, - msg.contains("INT") && msg.contains("BOOLEAN") + "Expected precision loss error but got: " + msg, + msg.contains("loses precision") && msg.contains("123.456") && msg.contains("scale=2") ); } } @Test - public void testIntToByte() throws Exception { - String table = "test_qwp_int_to_byte"; + public void testDoubleToByte() throws Exception { + String table = "test_qwp_double_to_byte"; useTable(table); execute("CREATE TABLE " + table + " (" + "b BYTE, " + @@ -854,29 +885,25 @@ public void testIntToByte() throws Exception { try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("b", 42) + .doubleColumn("b", 42.0) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .intColumn("b", -128) + .doubleColumn("b", -100.0) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("b", 127) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 2); assertSqlEventually( "b\tts\n" + "42\t1970-01-01T00:00:01.000000000Z\n" + - "-128\t1970-01-01T00:00:02.000000000Z\n" + - "127\t1970-01-01T00:00:03.000000000Z\n", + "-100\t1970-01-01T00:00:02.000000000Z\n", "SELECT b, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToByteOverflowError() throws Exception { - String table = "test_qwp_int_to_byte_overflow"; + public void testDoubleToBytePrecisionLossError() throws Exception { + String table = "test_qwp_double_to_byte_prec"; useTable(table); execute("CREATE TABLE " + table + " (" + "b BYTE, " + @@ -886,661 +913,514 @@ public void testIntToByteOverflowError() throws Exception { try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("b", 128) + .doubleColumn("b", 42.5) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected overflow error but got: " + msg, - msg.contains("integer value 128 out of range for BYTE") + "Expected precision loss error but got: " + msg, + msg.contains("loses precision") && msg.contains("42.5") ); } } @Test - public void testIntToCharCoercionError() throws Exception { - String table = "test_qwp_int_to_char_error"; + public void testDoubleToByteOverflowError() throws Exception { + String table = "test_qwp_double_to_byte_ovf"; useTable(table); execute("CREATE TABLE " + table + " (" + - "c CHAR, " + + "b BYTE, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("c", 65) + .doubleColumn("b", 200.0) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected error mentioning INT and CHAR but got: " + msg, - msg.contains("INT") && msg.contains("CHAR") + "Expected overflow error but got: " + msg, + msg.contains("integer value 200 out of range for BYTE") ); } } @Test - public void testIntToDate() throws Exception { - String table = "test_qwp_int_to_date"; + public void testDoubleToFloat() throws Exception { + String table = "test_qwp_double_to_float"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DATE, " + + "f FLOAT, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - // 86_400_000 millis = 1 day sender.table(table) - .intColumn("d", 86_400_000) + .doubleColumn("f", 1.5) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .intColumn("d", 0) + .doubleColumn("f", -42.25) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); - assertSqlEventually( - "d\tts\n" + - "1970-01-02T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z\n" + - "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToDecimal() throws Exception { - String table = "test_qwp_int_to_decimal"; + public void testDoubleToInt() throws Exception { + String table = "test_qwp_double_to_int"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(6, 2), " + + "i INT, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("d", 42) + .doubleColumn("i", 100_000.0) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .intColumn("d", -100) + .doubleColumn("i", -42.0) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "42.00\t1970-01-01T00:00:01.000000000Z\n" + - "-100.00\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + "i\tts\n" + + "100000\t1970-01-01T00:00:01.000000000Z\n" + + "-42\t1970-01-01T00:00:02.000000000Z\n", + "SELECT i, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToDecimal128() throws Exception { - String table = "test_qwp_int_to_decimal128"; + public void testDoubleToIntPrecisionLossError() throws Exception { + String table = "test_qwp_double_to_int_prec"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(38, 2), " + + "i INT, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("d", 42) + .doubleColumn("i", 3.14) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("d", -100) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("d", 0) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected precision loss error but got: " + msg, + msg.contains("loses precision") && msg.contains("3.14") + ); } - - assertTableSizeEventually(table, 3); - assertSqlEventually( - "d\tts\n" + - "42.00\t1970-01-01T00:00:01.000000000Z\n" + - "-100.00\t1970-01-01T00:00:02.000000000Z\n" + - "0.00\t1970-01-01T00:00:03.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToDecimal16() throws Exception { - String table = "test_qwp_int_to_decimal16"; + public void testDoubleToLong() throws Exception { + String table = "test_qwp_double_to_long"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(4, 1), " + + "l LONG, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("d", 42) + .doubleColumn("l", 1_000_000.0) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .intColumn("d", -100) + .doubleColumn("l", -42.0) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("d", 0) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "42.0\t1970-01-01T00:00:01.000000000Z\n" + - "-100.0\t1970-01-01T00:00:02.000000000Z\n" + - "0.0\t1970-01-01T00:00:03.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + "l\tts\n" + + "1000000\t1970-01-01T00:00:01.000000000Z\n" + + "-42\t1970-01-01T00:00:02.000000000Z\n", + "SELECT l, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToDecimal256() throws Exception { - String table = "test_qwp_int_to_decimal256"; + public void testDoubleToString() throws Exception { + String table = "test_qwp_double_to_string"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(76, 2), " + + "s STRING, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("d", 42) + .doubleColumn("s", 3.14) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .intColumn("d", -100) + .doubleColumn("s", -42.0) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("d", 0) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "42.00\t1970-01-01T00:00:01.000000000Z\n" + - "-100.00\t1970-01-01T00:00:02.000000000Z\n" + - "0.00\t1970-01-01T00:00:03.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + "s\tts\n" + + "3.14\t1970-01-01T00:00:01.000000000Z\n" + + "-42.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToDecimal64() throws Exception { - String table = "test_qwp_int_to_decimal64"; + public void testDoubleToSymbol() throws Exception { + String table = "test_qwp_double_to_symbol"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(18, 2), " + + "sym SYMBOL, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("d", Integer.MAX_VALUE) + .doubleColumn("sym", 3.14) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("d", -100) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("d", 0) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 1); assertSqlEventually( - "d\tts\n" + - "2147483647.00\t1970-01-01T00:00:01.000000000Z\n" + - "-100.00\t1970-01-01T00:00:02.000000000Z\n" + - "0.00\t1970-01-01T00:00:03.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + "sym\tts\n" + + "3.14\t1970-01-01T00:00:01.000000000Z\n", + "SELECT sym, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToDecimal8() throws Exception { - String table = "test_qwp_int_to_decimal8"; + public void testDoubleToVarchar() throws Exception { + String table = "test_qwp_double_to_varchar"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(2, 1), " + + "v VARCHAR, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("d", 5) + .doubleColumn("v", 3.14) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .intColumn("d", -9) + .doubleColumn("v", -42.0) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("d", 0) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "5.0\t1970-01-01T00:00:01.000000000Z\n" + - "-9.0\t1970-01-01T00:00:02.000000000Z\n" + - "0.0\t1970-01-01T00:00:03.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + "v\tts\n" + + "3.14\t1970-01-01T00:00:01.000000000Z\n" + + "-42.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToDouble() throws Exception { - String table = "test_qwp_int_to_double"; + public void testFloat() throws Exception { + String table = "test_qwp_float"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DOUBLE, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("d", 42) + .floatColumn("f", 1.5f) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .intColumn("d", -100) + .floatColumn("f", -42.25f) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .floatColumn("f", 0.0f) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); - assertSqlEventually( - "d\tts\n" + - "42.0\t1970-01-01T00:00:01.000000000Z\n" + - "-100.0\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + assertTableSizeEventually(table, 3); } @Test - public void testIntToFloat() throws Exception { - String table = "test_qwp_int_to_float"; + public void testFloatToDecimal() throws Exception { + String table = "test_qwp_float_to_decimal"; useTable(table); execute("CREATE TABLE " + table + " (" + - "f FLOAT, " + + "d DECIMAL(10, 2), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("f", 42) + .floatColumn("d", 1.5f) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .intColumn("f", -100) + .floatColumn("d", -42.25f) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("f", 0) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 2); assertSqlEventually( - "f\tts\n" + - "42.0\t1970-01-01T00:00:01.000000000Z\n" + - "-100.0\t1970-01-01T00:00:02.000000000Z\n" + - "0.0\t1970-01-01T00:00:03.000000000Z\n", - "SELECT f, ts FROM " + table + " ORDER BY ts"); + "d\tts\n" + + "1.50\t1970-01-01T00:00:01.000000000Z\n" + + "-42.25\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToGeoHashCoercionError() throws Exception { - String table = "test_qwp_int_to_geohash_error"; + public void testFloatToDecimalPrecisionLossError() throws Exception { + String table = "test_qwp_float_to_decimal_prec"; useTable(table); execute("CREATE TABLE " + table + " (" + - "g GEOHASH(4c), " + + "d DECIMAL(10, 1), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("g", 42) + .floatColumn("d", 1.25f) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected coercion error mentioning INT but got: " + msg, - msg.contains("type coercion from INT to") && msg.contains("is not supported") + "Expected precision loss error but got: " + msg, + msg.contains("loses precision") && msg.contains("scale=1") ); } } @Test - public void testIntToLong() throws Exception { - String table = "test_qwp_int_to_long"; + public void testFloatToDouble() throws Exception { + String table = "test_qwp_float_to_double"; useTable(table); execute("CREATE TABLE " + table + " (" + - "l LONG, " + + "d DOUBLE, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("l", 42) + .floatColumn("d", 1.5f) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .intColumn("l", Integer.MAX_VALUE) + .floatColumn("d", -42.25f) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("l", -1) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 2); assertSqlEventually( - "l\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "2147483647\t1970-01-01T00:00:02.000000000Z\n" + - "-1\t1970-01-01T00:00:03.000000000Z\n", - "SELECT l, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testIntToLong256CoercionError() throws Exception { - String table = "test_qwp_int_to_long256_error"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "v LONG256, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .intColumn("v", 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from INT to LONG256 is not supported") - ); - } + "d\tts\n" + + "1.5\t1970-01-01T00:00:01.000000000Z\n" + + "-42.25\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToShort() throws Exception { - String table = "test_qwp_int_to_short"; + public void testFloatToInt() throws Exception { + String table = "test_qwp_float_to_int"; useTable(table); execute("CREATE TABLE " + table + " (" + - "s SHORT, " + + "i INT, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("s", 1000) + .floatColumn("i", 42.0f) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .intColumn("s", -32768) + .floatColumn("i", -100.0f) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("s", 32767) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 2); assertSqlEventually( - "s\tts\n" + - "1000\t1970-01-01T00:00:01.000000000Z\n" + - "-32768\t1970-01-01T00:00:02.000000000Z\n" + - "32767\t1970-01-01T00:00:03.000000000Z\n", - "SELECT s, ts FROM " + table + " ORDER BY ts"); + "i\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-100\t1970-01-01T00:00:02.000000000Z\n", + "SELECT i, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToShortOverflowError() throws Exception { - String table = "test_qwp_int_to_short_overflow"; + public void testFloatToIntPrecisionLossError() throws Exception { + String table = "test_qwp_float_to_int_prec"; useTable(table); execute("CREATE TABLE " + table + " (" + - "s SHORT, " + + "i INT, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("s", 32768) + .floatColumn("i", 3.14f) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected overflow error but got: " + msg, - msg.contains("integer value 32768 out of range for SHORT") + "Expected precision loss error but got: " + msg, + msg.contains("loses precision") ); } } @Test - public void testIntToString() throws Exception { - String table = "test_qwp_int_to_string"; + public void testFloatToLong() throws Exception { + String table = "test_qwp_float_to_long"; useTable(table); execute("CREATE TABLE " + table + " (" + - "s STRING, " + + "l LONG, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("s", 42) + .floatColumn("l", 1000.0f) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("s", -100) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("s", 0) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 1); assertSqlEventually( - "s\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-100\t1970-01-01T00:00:02.000000000Z\n" + - "0\t1970-01-01T00:00:03.000000000Z\n", - "SELECT s, ts FROM " + table + " ORDER BY ts"); + "l\tts\n" + + "1000\t1970-01-01T00:00:01.000000000Z\n", + "SELECT l, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToSymbol() throws Exception { - String table = "test_qwp_int_to_symbol"; + public void testFloatToString() throws Exception { + String table = "test_qwp_float_to_string"; useTable(table); execute("CREATE TABLE " + table + " (" + - "s SYMBOL, " + + "s STRING, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("s", 42) + .floatColumn("s", 1.5f) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("s", -1) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("s", 0) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 1); assertSqlEventually( "s\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-1\t1970-01-01T00:00:02.000000000Z\n" + - "0\t1970-01-01T00:00:03.000000000Z\n", + "1.5\t1970-01-01T00:00:01.000000000Z\n", "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToTimestamp() throws Exception { - String table = "test_qwp_int_to_timestamp"; + public void testFloatToSymbol() throws Exception { + String table = "test_qwp_float_to_symbol"; useTable(table); execute("CREATE TABLE " + table + " (" + - "t TIMESTAMP, " + + "sym SYMBOL, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - // 1_000_000 micros = 1 second sender.table(table) - .intColumn("t", 1_000_000) + .floatColumn("sym", 1.5f) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("t", 0) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 1); assertSqlEventually( - "t\tts\n" + - "1970-01-01T00:00:01.000000000Z\t1970-01-01T00:00:01.000000000Z\n" + - "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", - "SELECT t, ts FROM " + table + " ORDER BY ts"); + "sym\tts\n" + + "1.5\t1970-01-01T00:00:01.000000000Z\n", + "SELECT sym, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToUuidCoercionError() throws Exception { - String table = "test_qwp_int_to_uuid_error"; + public void testFloatToVarchar() throws Exception { + String table = "test_qwp_float_to_varchar"; useTable(table); execute("CREATE TABLE " + table + " (" + - "u UUID, " + + "v VARCHAR, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("u", 42) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from INT to UUID is not supported") - ); - } - } - - @Test - public void testIntToVarchar() throws Exception { - String table = "test_qwp_int_to_varchar"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "v VARCHAR, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .intColumn("v", 42) + .floatColumn("v", 1.5f) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("v", -100) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("v", Integer.MAX_VALUE) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 1); assertSqlEventually( "v\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-100\t1970-01-01T00:00:02.000000000Z\n" + - "2147483647\t1970-01-01T00:00:03.000000000Z\n", + "1.5\t1970-01-01T00:00:01.000000000Z\n", "SELECT v, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLong() throws Exception { - String table = "test_qwp_long"; + public void testInt() throws Exception { + String table = "test_qwp_int"; useTable(table); try (QwpWebSocketSender sender = createQwpSender()) { - // Long.MIN_VALUE is the null sentinel for LONG + // Integer.MIN_VALUE is the null sentinel for INT sender.table(table) - .longColumn("l", Long.MIN_VALUE) + .intColumn("i", Integer.MIN_VALUE) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("l", 0) + .intColumn("i", 0) .at(2_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("l", Long.MAX_VALUE) + .intColumn("i", Integer.MAX_VALUE) .at(3_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("i", -42) + .at(4_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 4); assertSqlEventually( - "l\ttimestamp\n" + + "i\ttimestamp\n" + "null\t1970-01-01T00:00:01.000000000Z\n" + "0\t1970-01-01T00:00:02.000000000Z\n" + - "9223372036854775807\t1970-01-01T00:00:03.000000000Z\n", - "SELECT l, timestamp FROM " + table + " ORDER BY timestamp"); - } - - @Test - public void testLong256() throws Exception { - String table = "test_qwp_long256"; - useTable(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - // All zeros - sender.table(table) - .long256Column("v", 0, 0, 0, 0) - .at(1_000_000, ChronoUnit.MICROS); - // Mixed values - sender.table(table) - .long256Column("v", 1, 2, 3, 4) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); + "2147483647\t1970-01-01T00:00:03.000000000Z\n" + + "-42\t1970-01-01T00:00:04.000000000Z\n", + "SELECT i, timestamp FROM " + table + " ORDER BY timestamp"); } @Test - public void testLongToBooleanCoercionError() throws Exception { - String table = "test_qwp_long_to_boolean_error"; + public void testIntToBooleanCoercionError() throws Exception { + String table = "test_qwp_int_to_boolean_error"; useTable(table); execute("CREATE TABLE " + table + " (" + "b BOOLEAN, " + @@ -1550,22 +1430,22 @@ public void testLongToBooleanCoercionError() throws Exception { try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("b", 1) + .intColumn("b", 1) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected error mentioning LONG and BOOLEAN but got: " + msg, - msg.contains("LONG") && msg.contains("BOOLEAN") + "Expected error mentioning INT and BOOLEAN but got: " + msg, + msg.contains("INT") && msg.contains("BOOLEAN") ); } } @Test - public void testLongToByte() throws Exception { - String table = "test_qwp_long_to_byte"; + public void testIntToByte() throws Exception { + String table = "test_qwp_int_to_byte"; useTable(table); execute("CREATE TABLE " + table + " (" + "b BYTE, " + @@ -1575,13 +1455,13 @@ public void testLongToByte() throws Exception { try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("b", 42) + .intColumn("b", 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("b", -128) + .intColumn("b", -128) .at(2_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("b", 127) + .intColumn("b", 127) .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } @@ -1596,8 +1476,8 @@ public void testLongToByte() throws Exception { } @Test - public void testLongToByteOverflowError() throws Exception { - String table = "test_qwp_long_to_byte_overflow"; + public void testIntToByteOverflowError() throws Exception { + String table = "test_qwp_int_to_byte_overflow"; useTable(table); execute("CREATE TABLE " + table + " (" + "b BYTE, " + @@ -1607,7 +1487,7 @@ public void testLongToByteOverflowError() throws Exception { try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("b", 128) + .intColumn("b", 128) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -1621,8 +1501,8 @@ public void testLongToByteOverflowError() throws Exception { } @Test - public void testLongToCharCoercionError() throws Exception { - String table = "test_qwp_long_to_char_error"; + public void testIntToCharCoercionError() throws Exception { + String table = "test_qwp_int_to_char_error"; useTable(table); execute("CREATE TABLE " + table + " (" + "c CHAR, " + @@ -1632,22 +1512,22 @@ public void testLongToCharCoercionError() throws Exception { try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("c", 65) + .intColumn("c", 65) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected error mentioning LONG and CHAR but got: " + msg, - msg.contains("LONG") && msg.contains("CHAR") + "Expected error mentioning INT and CHAR but got: " + msg, + msg.contains("INT") && msg.contains("CHAR") ); } } @Test - public void testLongToDate() throws Exception { - String table = "test_qwp_long_to_date"; + public void testIntToDate() throws Exception { + String table = "test_qwp_int_to_date"; useTable(table); execute("CREATE TABLE " + table + " (" + "d DATE, " + @@ -1656,11 +1536,12 @@ public void testLongToDate() throws Exception { assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { + // 86_400_000 millis = 1 day sender.table(table) - .longColumn("d", 86_400_000L) + .intColumn("d", 86_400_000) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("d", 0L) + .intColumn("d", 0) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } @@ -1674,21 +1555,21 @@ public void testLongToDate() throws Exception { } @Test - public void testLongToDecimal() throws Exception { - String table = "test_qwp_long_to_decimal"; + public void testIntToDecimal() throws Exception { + String table = "test_qwp_int_to_decimal"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(10, 2), " + + "d DECIMAL(6, 2), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("d", 42) + .intColumn("d", 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("d", -100) + .intColumn("d", -100) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } @@ -1702,8 +1583,8 @@ public void testLongToDecimal() throws Exception { } @Test - public void testLongToDecimal128() throws Exception { - String table = "test_qwp_long_to_decimal128"; + public void testIntToDecimal128() throws Exception { + String table = "test_qwp_int_to_decimal128"; useTable(table); execute("CREATE TABLE " + table + " (" + "d DECIMAL(38, 2), " + @@ -1713,25 +1594,29 @@ public void testLongToDecimal128() throws Exception { try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("d", 1_000_000_000L) + .intColumn("d", 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("d", -1_000_000_000L) + .intColumn("d", -100) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", 0) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); assertSqlEventually( "d\tts\n" + - "1000000000.00\t1970-01-01T00:00:01.000000000Z\n" + - "-1000000000.00\t1970-01-01T00:00:02.000000000Z\n", + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n" + + "0.00\t1970-01-01T00:00:03.000000000Z\n", "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToDecimal16() throws Exception { - String table = "test_qwp_long_to_decimal16"; + public void testIntToDecimal16() throws Exception { + String table = "test_qwp_int_to_decimal16"; useTable(table); execute("CREATE TABLE " + table + " (" + "d DECIMAL(4, 1), " + @@ -1741,25 +1626,29 @@ public void testLongToDecimal16() throws Exception { try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("d", 42) + .intColumn("d", 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("d", -100) + .intColumn("d", -100) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", 0) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); assertSqlEventually( "d\tts\n" + "42.0\t1970-01-01T00:00:01.000000000Z\n" + - "-100.0\t1970-01-01T00:00:02.000000000Z\n", + "-100.0\t1970-01-01T00:00:02.000000000Z\n" + + "0.0\t1970-01-01T00:00:03.000000000Z\n", "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToDecimal256() throws Exception { - String table = "test_qwp_long_to_decimal256"; + public void testIntToDecimal256() throws Exception { + String table = "test_qwp_int_to_decimal256"; useTable(table); execute("CREATE TABLE " + table + " (" + "d DECIMAL(76, 2), " + @@ -1769,53 +1658,61 @@ public void testLongToDecimal256() throws Exception { try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("d", Long.MAX_VALUE) + .intColumn("d", 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("d", -1_000_000_000_000L) + .intColumn("d", -100) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", 0) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); assertSqlEventually( "d\tts\n" + - "9223372036854775807.00\t1970-01-01T00:00:01.000000000Z\n" + - "-1000000000000.00\t1970-01-01T00:00:02.000000000Z\n", + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n" + + "0.00\t1970-01-01T00:00:03.000000000Z\n", "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToDecimal32() throws Exception { - String table = "test_qwp_long_to_decimal32"; + public void testIntToDecimal64() throws Exception { + String table = "test_qwp_int_to_decimal64"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(6, 2), " + + "d DECIMAL(18, 2), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("d", 42) + .intColumn("d", Integer.MAX_VALUE) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("d", -100) + .intColumn("d", -100) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", 0) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); assertSqlEventually( "d\tts\n" + - "42.00\t1970-01-01T00:00:01.000000000Z\n" + - "-100.00\t1970-01-01T00:00:02.000000000Z\n", + "2147483647.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n" + + "0.00\t1970-01-01T00:00:03.000000000Z\n", "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToDecimal8() throws Exception { - String table = "test_qwp_long_to_decimal8"; + public void testIntToDecimal8() throws Exception { + String table = "test_qwp_int_to_decimal8"; useTable(table); execute("CREATE TABLE " + table + " (" + "d DECIMAL(2, 1), " + @@ -1825,25 +1722,29 @@ public void testLongToDecimal8() throws Exception { try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("d", 5) + .intColumn("d", 5) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("d", -9) + .intColumn("d", -9) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", 0) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); assertSqlEventually( "d\tts\n" + "5.0\t1970-01-01T00:00:01.000000000Z\n" + - "-9.0\t1970-01-01T00:00:02.000000000Z\n", + "-9.0\t1970-01-01T00:00:02.000000000Z\n" + + "0.0\t1970-01-01T00:00:03.000000000Z\n", "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToDouble() throws Exception { - String table = "test_qwp_long_to_double"; + public void testIntToDouble() throws Exception { + String table = "test_qwp_int_to_double"; useTable(table); execute("CREATE TABLE " + table + " (" + "d DOUBLE, " + @@ -1853,10 +1754,10 @@ public void testLongToDouble() throws Exception { try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("d", 42) + .intColumn("d", 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("d", -100) + .intColumn("d", -100) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } @@ -1870,8 +1771,8 @@ public void testLongToDouble() throws Exception { } @Test - public void testLongToFloat() throws Exception { - String table = "test_qwp_long_to_float"; + public void testIntToFloat() throws Exception { + String table = "test_qwp_int_to_float"; useTable(table); execute("CREATE TABLE " + table + " (" + "f FLOAT, " + @@ -1881,25 +1782,29 @@ public void testLongToFloat() throws Exception { try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("f", 42) + .intColumn("f", 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("f", -100) + .intColumn("f", -100) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("f", 0) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); assertSqlEventually( "f\tts\n" + "42.0\t1970-01-01T00:00:01.000000000Z\n" + - "-100.0\t1970-01-01T00:00:02.000000000Z\n", + "-100.0\t1970-01-01T00:00:02.000000000Z\n" + + "0.0\t1970-01-01T00:00:03.000000000Z\n", "SELECT f, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToGeoHashCoercionError() throws Exception { - String table = "test_qwp_long_to_geohash_error"; + public void testIntToGeoHashCoercionError() throws Exception { + String table = "test_qwp_int_to_geohash_error"; useTable(table); execute("CREATE TABLE " + table + " (" + "g GEOHASH(4c), " + @@ -1909,61 +1814,121 @@ public void testLongToGeoHashCoercionError() throws Exception { try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("g", 42) + .intColumn("g", 42) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected coercion error mentioning LONG but got: " + msg, - msg.contains("type coercion from LONG to") && msg.contains("is not supported") + "Expected coercion error mentioning INT but got: " + msg, + msg.contains("type coercion from INT to") && msg.contains("is not supported") ); } } @Test - public void testLongToInt() throws Exception { - String table = "test_qwp_long_to_int"; + public void testIntToLong() throws Exception { + String table = "test_qwp_int_to_long"; useTable(table); execute("CREATE TABLE " + table + " (" + - "i INT, " + + "l LONG, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - // Value in INT range should succeed sender.table(table) - .longColumn("i", 42) + .intColumn("l", 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("i", -1) + .intColumn("l", Integer.MAX_VALUE) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("l", -1) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); assertSqlEventually( - "i\tts\n" + + "l\tts\n" + "42\t1970-01-01T00:00:01.000000000Z\n" + - "-1\t1970-01-01T00:00:02.000000000Z\n", - "SELECT i, ts FROM " + table + " ORDER BY ts"); + "2147483647\t1970-01-01T00:00:02.000000000Z\n" + + "-1\t1970-01-01T00:00:03.000000000Z\n", + "SELECT l, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToIntOverflowError() throws Exception { - String table = "test_qwp_long_to_int_overflow"; + public void testIntToLong256CoercionError() throws Exception { + String table = "test_qwp_int_to_long256_error"; useTable(table); execute("CREATE TABLE " + table + " (" + - "i INT, " + + "v LONG256, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("i", (long) Integer.MAX_VALUE + 1) + .intColumn("v", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from INT to LONG256 is not supported") + ); + } + } + + @Test + public void testIntToShort() throws Exception { + String table = "test_qwp_int_to_short"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "s SHORT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("s", 1000) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("s", -32768) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("s", 32767) + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "s\tts\n" + + "1000\t1970-01-01T00:00:01.000000000Z\n" + + "-32768\t1970-01-01T00:00:02.000000000Z\n" + + "32767\t1970-01-01T00:00:03.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testIntToShortOverflowError() throws Exception { + String table = "test_qwp_int_to_short_overflow"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "s SHORT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("s", 32768) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -1971,24 +1936,117 @@ public void testLongToIntOverflowError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected overflow error but got: " + msg, - msg.contains("integer value 2147483648 out of range for INT") + msg.contains("integer value 32768 out of range for SHORT") ); } } @Test - public void testLongToLong256CoercionError() throws Exception { - String table = "test_qwp_long_to_long256_error"; + public void testIntToString() throws Exception { + String table = "test_qwp_int_to_string"; useTable(table); execute("CREATE TABLE " + table + " (" + - "v LONG256, " + + "s STRING, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("v", 42) + .intColumn("s", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("s", -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("s", 0) + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "s\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-100\t1970-01-01T00:00:02.000000000Z\n" + + "0\t1970-01-01T00:00:03.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testIntToSymbol() throws Exception { + String table = "test_qwp_int_to_symbol"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "s SYMBOL, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("s", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("s", -1) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("s", 0) + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "s\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-1\t1970-01-01T00:00:02.000000000Z\n" + + "0\t1970-01-01T00:00:03.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testIntToTimestamp() throws Exception { + String table = "test_qwp_int_to_timestamp"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "t TIMESTAMP, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + // 1_000_000 micros = 1 second + sender.table(table) + .intColumn("t", 1_000_000) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("t", 0) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "t\tts\n" + + "1970-01-01T00:00:01.000000000Z\t1970-01-01T00:00:01.000000000Z\n" + + "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", + "SELECT t, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testIntToUuidCoercionError() throws Exception { + String table = "test_qwp_int_to_uuid_error"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "u UUID, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .intColumn("u", 42) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -1996,255 +2054,1130 @@ public void testLongToLong256CoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("type coercion from LONG to LONG256 is not supported") + msg.contains("type coercion from INT to UUID is not supported") ); } } @Test - public void testLongToShort() throws Exception { - String table = "test_qwp_long_to_short"; + public void testIntToVarchar() throws Exception { + String table = "test_qwp_int_to_varchar"; useTable(table); execute("CREATE TABLE " + table + " (" + - "s SHORT, " + + "v VARCHAR, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - // Value in SHORT range should succeed sender.table(table) - .longColumn("s", 42) + .intColumn("v", 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("s", -1) + .intColumn("v", -100) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("v", Integer.MAX_VALUE) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); + assertSqlEventually( + "v\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-100\t1970-01-01T00:00:02.000000000Z\n" + + "2147483647\t1970-01-01T00:00:03.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testLong() throws Exception { + String table = "test_qwp_long"; + useTable(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + // Long.MIN_VALUE is the null sentinel for LONG + sender.table(table) + .longColumn("l", Long.MIN_VALUE) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("l", 0) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("l", Long.MAX_VALUE) + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "l\ttimestamp\n" + + "null\t1970-01-01T00:00:01.000000000Z\n" + + "0\t1970-01-01T00:00:02.000000000Z\n" + + "9223372036854775807\t1970-01-01T00:00:03.000000000Z\n", + "SELECT l, timestamp FROM " + table + " ORDER BY timestamp"); + } + + @Test + public void testLong256() throws Exception { + String table = "test_qwp_long256"; + useTable(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + // All zeros + sender.table(table) + .long256Column("v", 0, 0, 0, 0) + .at(1_000_000, ChronoUnit.MICROS); + // Mixed values + sender.table(table) + .long256Column("v", 1, 2, 3, 4) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + } + + @Test + public void testLongToBooleanCoercionError() throws Exception { + String table = "test_qwp_long_to_boolean_error"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "b BOOLEAN, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("b", 1) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected error mentioning LONG and BOOLEAN but got: " + msg, + msg.contains("LONG") && msg.contains("BOOLEAN") + ); + } + } + + @Test + public void testLongToByte() throws Exception { + String table = "test_qwp_long_to_byte"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "b BYTE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("b", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("b", -128) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("b", 127) + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "b\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-128\t1970-01-01T00:00:02.000000000Z\n" + + "127\t1970-01-01T00:00:03.000000000Z\n", + "SELECT b, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testLongToByteOverflowError() throws Exception { + String table = "test_qwp_long_to_byte_overflow"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "b BYTE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("b", 128) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected overflow error but got: " + msg, + msg.contains("integer value 128 out of range for BYTE") + ); + } + } + + @Test + public void testLongToCharCoercionError() throws Exception { + String table = "test_qwp_long_to_char_error"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "c CHAR, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("c", 65) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected error mentioning LONG and CHAR but got: " + msg, + msg.contains("LONG") && msg.contains("CHAR") + ); + } + } + + @Test + public void testLongToDate() throws Exception { + String table = "test_qwp_long_to_date"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DATE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("d", 86_400_000L) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("d", 0L) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "1970-01-02T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z\n" + + "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testLongToDecimal() throws Exception { + String table = "test_qwp_long_to_decimal"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(10, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("d", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("d", -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testLongToDecimal128() throws Exception { + String table = "test_qwp_long_to_decimal128"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(38, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("d", 1_000_000_000L) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("d", -1_000_000_000L) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "1000000000.00\t1970-01-01T00:00:01.000000000Z\n" + + "-1000000000.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testLongToDecimal16() throws Exception { + String table = "test_qwp_long_to_decimal16"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(4, 1), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("d", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("d", -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "42.0\t1970-01-01T00:00:01.000000000Z\n" + + "-100.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testLongToDecimal256() throws Exception { + String table = "test_qwp_long_to_decimal256"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(76, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("d", Long.MAX_VALUE) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("d", -1_000_000_000_000L) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "9223372036854775807.00\t1970-01-01T00:00:01.000000000Z\n" + + "-1000000000000.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testLongToDecimal32() throws Exception { + String table = "test_qwp_long_to_decimal32"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(6, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("d", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("d", -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testLongToDecimal8() throws Exception { + String table = "test_qwp_long_to_decimal8"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(2, 1), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("d", 5) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("d", -9) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "5.0\t1970-01-01T00:00:01.000000000Z\n" + + "-9.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testLongToDouble() throws Exception { + String table = "test_qwp_long_to_double"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DOUBLE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("d", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("d", -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "42.0\t1970-01-01T00:00:01.000000000Z\n" + + "-100.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testLongToFloat() throws Exception { + String table = "test_qwp_long_to_float"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "f FLOAT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("f", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("f", -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "f\tts\n" + + "42.0\t1970-01-01T00:00:01.000000000Z\n" + + "-100.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT f, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testLongToGeoHashCoercionError() throws Exception { + String table = "test_qwp_long_to_geohash_error"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "g GEOHASH(4c), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("g", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error mentioning LONG but got: " + msg, + msg.contains("type coercion from LONG to") && msg.contains("is not supported") + ); + } + } + + @Test + public void testLongToInt() throws Exception { + String table = "test_qwp_long_to_int"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "i INT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + // Value in INT range should succeed + sender.table(table) + .longColumn("i", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("i", -1) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "i\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-1\t1970-01-01T00:00:02.000000000Z\n", + "SELECT i, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testLongToIntOverflowError() throws Exception { + String table = "test_qwp_long_to_int_overflow"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "i INT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("i", (long) Integer.MAX_VALUE + 1) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected overflow error but got: " + msg, + msg.contains("integer value 2147483648 out of range for INT") + ); + } + } + + @Test + public void testLongToLong256CoercionError() throws Exception { + String table = "test_qwp_long_to_long256_error"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "v LONG256, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("v", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from LONG to LONG256 is not supported") + ); + } + } + + @Test + public void testLongToShort() throws Exception { + String table = "test_qwp_long_to_short"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "s SHORT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + // Value in SHORT range should succeed + sender.table(table) + .longColumn("s", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("s", -1) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + } + + @Test + public void testLongToShortOverflowError() throws Exception { + String table = "test_qwp_long_to_short_overflow"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "s SHORT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("s", 32768) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected overflow error but got: " + msg, + msg.contains("integer value 32768 out of range for SHORT") + ); + } + } + + @Test + public void testLongToString() throws Exception { + String table = "test_qwp_long_to_string"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "s STRING, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("s", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("s", Long.MAX_VALUE) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "s\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "9223372036854775807\t1970-01-01T00:00:02.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testLongToSymbol() throws Exception { + String table = "test_qwp_long_to_symbol"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "s SYMBOL, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("s", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("s", -1) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "s\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-1\t1970-01-01T00:00:02.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testLongToTimestamp() throws Exception { + String table = "test_qwp_long_to_timestamp"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "t TIMESTAMP, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("t", 1_000_000L) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("t", 0L) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "t\tts\n" + + "1970-01-01T00:00:01.000000000Z\t1970-01-01T00:00:01.000000000Z\n" + + "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", + "SELECT t, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testLongToUuidCoercionError() throws Exception { + String table = "test_qwp_long_to_uuid_error"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "u UUID, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("u", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from LONG to UUID is not supported") + ); + } + } + + @Test + public void testLongToVarchar() throws Exception { + String table = "test_qwp_long_to_varchar"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "v VARCHAR, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .longColumn("v", 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("v", Long.MAX_VALUE) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "v\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "9223372036854775807\t1970-01-01T00:00:02.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testMultipleRowsAndBatching() throws Exception { + String table = "test_qwp_multiple_rows"; + useTable(table); + + int rowCount = 1000; + try (QwpWebSocketSender sender = createQwpSender()) { + for (int i = 0; i < rowCount; i++) { + sender.table(table) + .symbol("sym", "s" + (i % 10)) + .longColumn("val", i) + .doubleColumn("dbl", i * 1.5) + .at((long) (i + 1) * 1_000_000, ChronoUnit.MICROS); + } + sender.flush(); + } + + assertTableSizeEventually(table, rowCount); + } + + @Test + public void testShort() throws Exception { + String table = "test_qwp_short"; + useTable(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + // Short.MIN_VALUE is the null sentinel for SHORT + sender.table(table) + .shortColumn("s", Short.MIN_VALUE) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("s", (short) 0) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("s", Short.MAX_VALUE) + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + } + + @Test + public void testShortToDecimal128() throws Exception { + String table = "test_qwp_short_to_decimal128"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(38, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .shortColumn("d", Short.MAX_VALUE) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("d", Short.MIN_VALUE) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "32767.00\t1970-01-01T00:00:01.000000000Z\n" + + "-32768.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testShortToDecimal16() throws Exception { + String table = "test_qwp_short_to_decimal16"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(4, 1), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .shortColumn("d", (short) 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("d", (short) -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "42.0\t1970-01-01T00:00:01.000000000Z\n" + + "-100.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testShortToDecimal256() throws Exception { + String table = "test_qwp_short_to_decimal256"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(76, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .shortColumn("d", (short) 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("d", (short) -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testShortToDecimal32() throws Exception { + String table = "test_qwp_short_to_decimal32"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(6, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .shortColumn("d", (short) 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("d", (short) -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToShortOverflowError() throws Exception { - String table = "test_qwp_long_to_short_overflow"; + public void testShortToDecimal64() throws Exception { + String table = "test_qwp_short_to_decimal64"; useTable(table); execute("CREATE TABLE " + table + " (" + - "s SHORT, " + + "d DECIMAL(18, 2), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("s", 32768) + .shortColumn("d", (short) 42) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("d", (short) -100) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected overflow error but got: " + msg, - msg.contains("integer value 32768 out of range for SHORT") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToString() throws Exception { - String table = "test_qwp_long_to_string"; + public void testShortToDecimal8() throws Exception { + String table = "test_qwp_short_to_decimal8"; useTable(table); execute("CREATE TABLE " + table + " (" + - "s STRING, " + + "d DECIMAL(2, 1), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("s", 42) + .shortColumn("d", (short) 5) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("s", Long.MAX_VALUE) + .shortColumn("d", (short) -9) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "s\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "9223372036854775807\t1970-01-01T00:00:02.000000000Z\n", - "SELECT s, ts FROM " + table + " ORDER BY ts"); + "d\tts\n" + + "5.0\t1970-01-01T00:00:01.000000000Z\n" + + "-9.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToSymbol() throws Exception { - String table = "test_qwp_long_to_symbol"; + public void testShortToInt() throws Exception { + String table = "test_qwp_short_to_int"; useTable(table); execute("CREATE TABLE " + table + " (" + - "s SYMBOL, " + + "i INT, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("s", 42) + .shortColumn("i", (short) 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("s", -1) + .shortColumn("i", Short.MAX_VALUE) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "s\tts\n" + + "i\tts\n" + "42\t1970-01-01T00:00:01.000000000Z\n" + - "-1\t1970-01-01T00:00:02.000000000Z\n", - "SELECT s, ts FROM " + table + " ORDER BY ts"); + "32767\t1970-01-01T00:00:02.000000000Z\n", + "SELECT i, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToTimestamp() throws Exception { - String table = "test_qwp_long_to_timestamp"; + public void testShortToLong() throws Exception { + String table = "test_qwp_short_to_long"; useTable(table); execute("CREATE TABLE " + table + " (" + - "t TIMESTAMP, " + + "l LONG, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("t", 1_000_000L) + .shortColumn("l", (short) 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("t", 0L) + .shortColumn("l", Short.MAX_VALUE) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "t\tts\n" + - "1970-01-01T00:00:01.000000000Z\t1970-01-01T00:00:01.000000000Z\n" + - "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", - "SELECT t, ts FROM " + table + " ORDER BY ts"); + "l\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "32767\t1970-01-01T00:00:02.000000000Z\n", + "SELECT l, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToUuidCoercionError() throws Exception { - String table = "test_qwp_long_to_uuid_error"; + public void testShortToBooleanCoercionError() throws Exception { + String table = "test_qwp_short_to_boolean_error"; useTable(table); execute("CREATE TABLE " + table + " (" + - "u UUID, " + + "b BOOLEAN, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("u", 42) + .shortColumn("b", (short) 1) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from LONG to UUID is not supported") + "Expected error mentioning SHORT and BOOLEAN but got: " + msg, + msg.contains("SHORT") && msg.contains("BOOLEAN") ); } } @Test - public void testLongToVarchar() throws Exception { - String table = "test_qwp_long_to_varchar"; + public void testShortToByte() throws Exception { + String table = "test_qwp_short_to_byte"; useTable(table); execute("CREATE TABLE " + table + " (" + - "v VARCHAR, " + + "b BYTE, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("v", 42) + .shortColumn("b", (short) 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("v", Long.MAX_VALUE) + .shortColumn("b", (short) -128) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("b", (short) 127) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); assertSqlEventually( - "v\tts\n" + + "b\tts\n" + "42\t1970-01-01T00:00:01.000000000Z\n" + - "9223372036854775807\t1970-01-01T00:00:02.000000000Z\n", - "SELECT v, ts FROM " + table + " ORDER BY ts"); + "-128\t1970-01-01T00:00:02.000000000Z\n" + + "127\t1970-01-01T00:00:03.000000000Z\n", + "SELECT b, ts FROM " + table + " ORDER BY ts"); } @Test - public void testMultipleRowsAndBatching() throws Exception { - String table = "test_qwp_multiple_rows"; + public void testShortToByteOverflowError() throws Exception { + String table = "test_qwp_short_to_byte_overflow"; useTable(table); + execute("CREATE TABLE " + table + " (" + + "b BYTE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); - int rowCount = 1000; try (QwpWebSocketSender sender = createQwpSender()) { - for (int i = 0; i < rowCount; i++) { - sender.table(table) - .symbol("sym", "s" + (i % 10)) - .longColumn("val", i) - .doubleColumn("dbl", i * 1.5) - .at((long) (i + 1) * 1_000_000, ChronoUnit.MICROS); - } + sender.table(table) + .shortColumn("b", (short) 128) + .at(1_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected overflow error but got: " + msg, + msg.contains("integer value 128 out of range for BYTE") + ); } - - assertTableSizeEventually(table, rowCount); } @Test - public void testShort() throws Exception { - String table = "test_qwp_short"; + public void testShortToCharCoercionError() throws Exception { + String table = "test_qwp_short_to_char_error"; useTable(table); + execute("CREATE TABLE " + table + " (" + + "c CHAR, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - // Short.MIN_VALUE is the null sentinel for SHORT sender.table(table) - .shortColumn("s", Short.MIN_VALUE) + .shortColumn("c", (short) 65) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("s", (short) 0) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("s", Short.MAX_VALUE) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected error mentioning SHORT and CHAR but got: " + msg, + msg.contains("SHORT") && msg.contains("CHAR") + ); } - - assertTableSizeEventually(table, 3); } @Test - public void testShortToDecimal128() throws Exception { - String table = "test_qwp_short_to_decimal128"; + public void testShortToDate() throws Exception { + String table = "test_qwp_short_to_date"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(38, 2), " + + "d DATE, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { + // 1000 millis = 1 second sender.table(table) - .shortColumn("d", Short.MAX_VALUE) + .shortColumn("d", (short) 1000) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .shortColumn("d", Short.MIN_VALUE) + .shortColumn("d", (short) 0) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } @@ -2252,17 +3185,17 @@ public void testShortToDecimal128() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( "d\tts\n" + - "32767.00\t1970-01-01T00:00:01.000000000Z\n" + - "-32768.00\t1970-01-01T00:00:02.000000000Z\n", + "1970-01-01T00:00:01.000000000Z\t1970-01-01T00:00:01.000000000Z\n" + + "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testShortToDecimal16() throws Exception { - String table = "test_qwp_short_to_decimal16"; + public void testShortToDouble() throws Exception { + String table = "test_qwp_short_to_double"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(4, 1), " + + "d DOUBLE, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); @@ -2286,171 +3219,230 @@ public void testShortToDecimal16() throws Exception { } @Test - public void testShortToDecimal256() throws Exception { - String table = "test_qwp_short_to_decimal256"; + public void testShortToFloat() throws Exception { + String table = "test_qwp_short_to_float"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(76, 2), " + + "f FLOAT, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("d", (short) 42) + .shortColumn("f", (short) 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .shortColumn("d", (short) -100) + .shortColumn("f", (short) -100) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "42.00\t1970-01-01T00:00:01.000000000Z\n" + - "-100.00\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + "f\tts\n" + + "42.0\t1970-01-01T00:00:01.000000000Z\n" + + "-100.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT f, ts FROM " + table + " ORDER BY ts"); } @Test - public void testShortToDecimal32() throws Exception { - String table = "test_qwp_short_to_decimal32"; + public void testShortToGeoHashCoercionError() throws Exception { + String table = "test_qwp_short_to_geohash_error"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(6, 2), " + + "g GEOHASH(4c), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("d", (short) 42) + .shortColumn("g", (short) 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error mentioning SHORT but got: " + msg, + msg.contains("type coercion from SHORT to") && msg.contains("is not supported") + ); + } + } + + @Test + public void testShortToLong256CoercionError() throws Exception { + String table = "test_qwp_short_to_long256_error"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "v LONG256, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .shortColumn("v", (short) 42) .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from SHORT to LONG256 is not supported") + ); + } + } + + @Test + public void testShortToString() throws Exception { + String table = "test_qwp_short_to_string"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "s STRING, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("d", (short) -100) + .shortColumn("s", (short) 42) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("s", (short) -100) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("s", (short) 0) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); assertSqlEventually( - "d\tts\n" + - "42.00\t1970-01-01T00:00:01.000000000Z\n" + - "-100.00\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + "s\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-100\t1970-01-01T00:00:02.000000000Z\n" + + "0\t1970-01-01T00:00:03.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testShortToDecimal64() throws Exception { - String table = "test_qwp_short_to_decimal64"; + public void testShortToSymbol() throws Exception { + String table = "test_qwp_short_to_symbol"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(18, 2), " + + "s SYMBOL, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("d", (short) 42) + .shortColumn("s", (short) 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .shortColumn("d", (short) -100) + .shortColumn("s", (short) -1) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("s", (short) 0) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); assertSqlEventually( - "d\tts\n" + - "42.00\t1970-01-01T00:00:01.000000000Z\n" + - "-100.00\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + "s\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-1\t1970-01-01T00:00:02.000000000Z\n" + + "0\t1970-01-01T00:00:03.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testShortToDecimal8() throws Exception { - String table = "test_qwp_short_to_decimal8"; + public void testShortToTimestamp() throws Exception { + String table = "test_qwp_short_to_timestamp"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(2, 1), " + + "t TIMESTAMP, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("d", (short) 5) + .shortColumn("t", (short) 1000) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .shortColumn("d", (short) -9) + .shortColumn("t", (short) 0) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "5.0\t1970-01-01T00:00:01.000000000Z\n" + - "-9.0\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + "t\tts\n" + + "1970-01-01T00:00:00.001000000Z\t1970-01-01T00:00:01.000000000Z\n" + + "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", + "SELECT t, ts FROM " + table + " ORDER BY ts"); } @Test - public void testShortToInt() throws Exception { - String table = "test_qwp_short_to_int"; + public void testShortToUuidCoercionError() throws Exception { + String table = "test_qwp_short_to_uuid_error"; useTable(table); execute("CREATE TABLE " + table + " (" + - "i INT, " + + "u UUID, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("i", (short) 42) + .shortColumn("u", (short) 42) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("i", Short.MAX_VALUE) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from SHORT to UUID is not supported") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "i\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "32767\t1970-01-01T00:00:02.000000000Z\n", - "SELECT i, ts FROM " + table + " ORDER BY ts"); } @Test - public void testShortToLong() throws Exception { - String table = "test_qwp_short_to_long"; + public void testShortToVarchar() throws Exception { + String table = "test_qwp_short_to_varchar"; useTable(table); execute("CREATE TABLE " + table + " (" + - "l LONG, " + + "v VARCHAR, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("l", (short) 42) + .shortColumn("v", (short) 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .shortColumn("l", Short.MAX_VALUE) + .shortColumn("v", (short) -100) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("v", Short.MAX_VALUE) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); assertSqlEventually( - "l\tts\n" + + "v\tts\n" + "42\t1970-01-01T00:00:01.000000000Z\n" + - "32767\t1970-01-01T00:00:02.000000000Z\n", - "SELECT l, ts FROM " + table + " ORDER BY ts"); + "-100\t1970-01-01T00:00:02.000000000Z\n" + + "32767\t1970-01-01T00:00:03.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); } @Test @@ -2759,6 +3751,261 @@ public void testWriteAllTypesInOneRow() throws Exception { assertTableSizeEventually(table, 1); } + // === Decimal cross-width coercion tests === + + @Test + public void testDecimal256ToDecimal64() throws Exception { + String table = "test_qwp_dec256_to_dec64"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(18, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + // Send DECIMAL256 wire type to DECIMAL64 column + sender.table(table) + .decimalColumn("d", Decimal256.fromLong(12345, 2)) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .decimalColumn("d", Decimal256.fromLong(-9999, 2)) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "123.45\t1970-01-01T00:00:01.000000000Z\n" + + "-99.99\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testDecimal256ToDecimal128() throws Exception { + String table = "test_qwp_dec256_to_dec128"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(38, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .decimalColumn("d", Decimal256.fromLong(12345, 2)) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .decimalColumn("d", Decimal256.fromLong(-9999, 2)) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "123.45\t1970-01-01T00:00:01.000000000Z\n" + + "-99.99\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testDecimal64ToDecimal128() throws Exception { + String table = "test_qwp_dec64_to_dec128"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(38, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + // Send DECIMAL64 wire type to DECIMAL128 column (widening) + sender.table(table) + .decimalColumn("d", Decimal64.fromLong(12345, 2)) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .decimalColumn("d", Decimal64.fromLong(-9999, 2)) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "123.45\t1970-01-01T00:00:01.000000000Z\n" + + "-99.99\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testDecimal64ToDecimal256() throws Exception { + String table = "test_qwp_dec64_to_dec256"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(76, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .decimalColumn("d", Decimal64.fromLong(12345, 2)) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .decimalColumn("d", Decimal64.fromLong(-9999, 2)) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "123.45\t1970-01-01T00:00:01.000000000Z\n" + + "-99.99\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testDecimal128ToDecimal64() throws Exception { + String table = "test_qwp_dec128_to_dec64"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(18, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .decimalColumn("d", Decimal128.fromLong(12345, 2)) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .decimalColumn("d", Decimal128.fromLong(-9999, 2)) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "123.45\t1970-01-01T00:00:01.000000000Z\n" + + "-99.99\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testDecimal128ToDecimal256() throws Exception { + String table = "test_qwp_dec128_to_dec256"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(76, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .decimalColumn("d", Decimal128.fromLong(12345, 2)) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .decimalColumn("d", Decimal128.fromLong(-9999, 2)) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "123.45\t1970-01-01T00:00:01.000000000Z\n" + + "-99.99\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testDecimalRescale() throws Exception { + String table = "test_qwp_decimal_rescale"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(18, 4), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + // Send scale=2 wire data to scale=4 column: server should rescale + sender.table(table) + .decimalColumn("d", Decimal64.fromLong(12345, 2)) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .decimalColumn("d", Decimal64.fromLong(-100, 2)) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "123.4500\t1970-01-01T00:00:01.000000000Z\n" + + "-1.0000\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testDecimal256ToDecimal64OverflowError() throws Exception { + String table = "test_qwp_dec256_to_dec64_overflow"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(18, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + // Create a value that fits in Decimal256 but overflows Decimal64 + // Decimal256 with hi bits set will overflow 64-bit storage + Decimal256 bigValue = Decimal256.fromBigDecimal(new java.math.BigDecimal("99999999999999999999.99")); + sender.table(table) + .decimalColumn("d", bigValue) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected overflow error but got: " + msg, + msg.contains("decimal value overflows") + ); + } + } + + @Test + public void testDecimal256ToDecimal8OverflowError() throws Exception { + String table = "test_qwp_dec256_to_dec8_overflow"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(2, 1), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + // 999.9 with scale=1 → unscaled 9999, which doesn't fit in a byte (-128..127) + sender.table(table) + .decimalColumn("d", Decimal256.fromLong(9999, 1)) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected overflow error but got: " + msg, + msg.contains("decimal value overflows") + ); + } + } + // === Helper Methods === private QwpWebSocketSender createQwpSender() { From dd4bb6232e99a0c14c69a99b823acd1025fce6a7 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Mon, 16 Feb 2026 00:07:18 +0000 Subject: [PATCH 11/89] wip 5 --- .../cutlass/qwp/client/QwpSenderTest.java | 4032 ++++++++++++++++- 1 file changed, 4030 insertions(+), 2 deletions(-) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java index b4e61c6..6b7e8d9 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java @@ -3609,13 +3609,13 @@ public void testTimestampMicrosToNanos() throws Exception { String table = "test_qwp_timestamp_micros_to_nanos"; useTable(table); execute("CREATE TABLE " + table + " (" + - "ts_col TIMESTAMP WITH TIME ZONE, " + + "ts_col TIMESTAMP_NS, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - long tsMicros = 1_645_747_200_000_000L; // 2022-02-25T00:00:00Z + long tsMicros = 1_645_747_200_111_111L; // 2022-02-25T00:00:00Z sender.table(table) .timestampColumn("ts_col", tsMicros, ChronoUnit.MICROS) .at(1_000_000, ChronoUnit.MICROS); @@ -3623,6 +3623,11 @@ public void testTimestampMicrosToNanos() throws Exception { } assertTableSizeEventually(table, 1); + // Microseconds scaled to nanoseconds + assertSqlEventually( + "ts_col\tts\n" + + "2022-02-25T00:00:00.111111000Z\t1970-01-01T00:00:01.000000000Z\n", + "SELECT ts_col, ts FROM " + table); } @Test @@ -4006,6 +4011,4029 @@ public void testDecimal256ToDecimal8OverflowError() throws Exception { } } + @Test + public void testStringToBoolean() throws Exception { + String table = "test_qwp_string_to_boolean"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "b BOOLEAN, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("b", "true") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("b", "false") + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("b", "1") + .at(3_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("b", "0") + .at(4_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("b", "TRUE") + .at(5_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 5); + assertSqlEventually( + "b\tts\n" + + "true\t1970-01-01T00:00:01.000000000Z\n" + + "false\t1970-01-01T00:00:02.000000000Z\n" + + "true\t1970-01-01T00:00:03.000000000Z\n" + + "false\t1970-01-01T00:00:04.000000000Z\n" + + "true\t1970-01-01T00:00:05.000000000Z\n", + "SELECT b, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testStringToBooleanParseError() throws Exception { + String table = "test_qwp_string_to_boolean_err"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "b BOOLEAN, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("b", "yes") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected parse error but got: " + msg, + msg.contains("cannot parse boolean from string") + ); + } + } + + @Test + public void testStringToByte() throws Exception { + String table = "test_qwp_string_to_byte"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "b BYTE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("b", "42") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("b", "-128") + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("b", "127") + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "b\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-128\t1970-01-01T00:00:02.000000000Z\n" + + "127\t1970-01-01T00:00:03.000000000Z\n", + "SELECT b, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testStringToByteParseError() throws Exception { + String table = "test_qwp_string_to_byte_err"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "b BYTE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("b", "abc") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected parse error but got: " + msg, + msg.contains("cannot parse BYTE from string") + ); + } + } + + @Test + public void testStringToDate() throws Exception { + String table = "test_qwp_string_to_date"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DATE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("d", "2022-02-25T00:00:00.000Z") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 1); + assertSqlEventually( + "d\tts\n" + + "2022-02-25T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testStringToDecimal64() throws Exception { + String table = "test_qwp_string_to_dec64"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(18, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("d", "123.45") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("d", "-99.99") + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "123.45\t1970-01-01T00:00:01.000000000Z\n" + + "-99.99\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testStringToDecimal128() throws Exception { + String table = "test_qwp_string_to_dec128"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(38, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("d", "123.45") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("d", "-99.99") + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "123.45\t1970-01-01T00:00:01.000000000Z\n" + + "-99.99\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testStringToDecimal256() throws Exception { + String table = "test_qwp_string_to_dec256"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(76, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("d", "123.45") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("d", "-99.99") + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "123.45\t1970-01-01T00:00:01.000000000Z\n" + + "-99.99\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testStringToDouble() throws Exception { + String table = "test_qwp_string_to_double"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DOUBLE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("d", "3.14") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("d", "-2.718") + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "3.14\t1970-01-01T00:00:01.000000000Z\n" + + "-2.718\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testStringToFloat() throws Exception { + String table = "test_qwp_string_to_float"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "f FLOAT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("f", "3.14") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("f", "-2.5") + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "f\tts\n" + + "3.14\t1970-01-01T00:00:01.000000000Z\n" + + "-2.5\t1970-01-01T00:00:02.000000000Z\n", + "SELECT f, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testStringToGeoHash() throws Exception { + String table = "test_qwp_string_to_geohash"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "g GEOHASH(5c), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("g", "s24se") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("g", "u33dc") + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "g\tts\n" + + "s24se\t1970-01-01T00:00:01.000000000Z\n" + + "u33dc\t1970-01-01T00:00:02.000000000Z\n", + "SELECT g, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testStringToInt() throws Exception { + String table = "test_qwp_string_to_int"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "i INT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("i", "42") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("i", "-100") + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("i", "0") + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "i\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-100\t1970-01-01T00:00:02.000000000Z\n" + + "0\t1970-01-01T00:00:03.000000000Z\n", + "SELECT i, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testStringToLong() throws Exception { + String table = "test_qwp_string_to_long"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "l LONG, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("l", "1000000000000") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("l", "-1") + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "l\tts\n" + + "1000000000000\t1970-01-01T00:00:01.000000000Z\n" + + "-1\t1970-01-01T00:00:02.000000000Z\n", + "SELECT l, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testStringToLong256() throws Exception { + String table = "test_qwp_string_to_long256"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "l LONG256, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("l", "0x01") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 1); + assertSqlEventually( + "l\tts\n" + + "0x01\t1970-01-01T00:00:01.000000000Z\n", + "SELECT l, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testStringToShort() throws Exception { + String table = "test_qwp_string_to_short"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "s SHORT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("s", "1000") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("s", "-32768") + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("s", "32767") + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "s\tts\n" + + "1000\t1970-01-01T00:00:01.000000000Z\n" + + "-32768\t1970-01-01T00:00:02.000000000Z\n" + + "32767\t1970-01-01T00:00:03.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testStringToTimestamp() throws Exception { + String table = "test_qwp_string_to_timestamp"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "t TIMESTAMP, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("t", "2022-02-25T00:00:00.000000Z") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 1); + assertSqlEventually( + "t\tts\n" + + "2022-02-25T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z\n", + "SELECT t, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testBoolToString() throws Exception { + String table = "test_qwp_bool_to_string"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "s STRING, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .boolColumn("s", true) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .boolColumn("s", false) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "s\tts\n" + + "true\t1970-01-01T00:00:01.000000000Z\n" + + "false\t1970-01-01T00:00:02.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testBoolToVarchar() throws Exception { + String table = "test_qwp_bool_to_varchar"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "v VARCHAR, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .boolColumn("v", true) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .boolColumn("v", false) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "v\tts\n" + + "true\t1970-01-01T00:00:01.000000000Z\n" + + "false\t1970-01-01T00:00:02.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testDecimalToString() throws Exception { + String table = "test_qwp_decimal_to_string"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "s STRING, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .decimalColumn("s", Decimal64.fromLong(12345, 2)) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .decimalColumn("s", Decimal64.fromLong(-9999, 2)) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "s\tts\n" + + "123.45\t1970-01-01T00:00:01.000000000Z\n" + + "-99.99\t1970-01-01T00:00:02.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testDecimalToVarchar() throws Exception { + String table = "test_qwp_decimal_to_varchar"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "v VARCHAR, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .decimalColumn("v", Decimal64.fromLong(12345, 2)) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .decimalColumn("v", Decimal64.fromLong(-9999, 2)) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "v\tts\n" + + "123.45\t1970-01-01T00:00:01.000000000Z\n" + + "-99.99\t1970-01-01T00:00:02.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testSymbolToString() throws Exception { + String table = "test_qwp_symbol_to_string"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "s STRING, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .symbol("s", "hello") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .symbol("s", "world") + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "s\tts\n" + + "hello\t1970-01-01T00:00:01.000000000Z\n" + + "world\t1970-01-01T00:00:02.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testSymbolToVarchar() throws Exception { + String table = "test_qwp_symbol_to_varchar"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "v VARCHAR, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .symbol("v", "hello") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .symbol("v", "world") + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "v\tts\n" + + "hello\t1970-01-01T00:00:01.000000000Z\n" + + "world\t1970-01-01T00:00:02.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testTimestampToString() throws Exception { + String table = "test_qwp_timestamp_to_string"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "s STRING, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + long tsMicros = 1_645_747_200_000_000L; // 2022-02-25T00:00:00Z in micros + sender.table(table) + .timestampColumn("s", tsMicros, ChronoUnit.MICROS) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 1); + assertSqlEventually( + "s\tts\n" + + "2022-02-25T00:00:00.000Z\t1970-01-01T00:00:01.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testTimestampToVarchar() throws Exception { + String table = "test_qwp_timestamp_to_varchar"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "v VARCHAR, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + long tsMicros = 1_645_747_200_000_000L; // 2022-02-25T00:00:00Z in micros + sender.table(table) + .timestampColumn("v", tsMicros, ChronoUnit.MICROS) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 1); + assertSqlEventually( + "v\tts\n" + + "2022-02-25T00:00:00.000Z\t1970-01-01T00:00:01.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testCharToString() throws Exception { + String table = "test_qwp_char_to_string"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "s STRING, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .charColumn("s", 'A') + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .charColumn("s", 'Z') + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "s\tts\n" + + "A\t1970-01-01T00:00:01.000000000Z\n" + + "Z\t1970-01-01T00:00:02.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testCharToVarchar() throws Exception { + String table = "test_qwp_char_to_varchar"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "v VARCHAR, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .charColumn("v", 'A') + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .charColumn("v", 'Z') + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "v\tts\n" + + "A\t1970-01-01T00:00:01.000000000Z\n" + + "Z\t1970-01-01T00:00:02.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testDoubleToShort() throws Exception { + String table = "test_qwp_double_to_short"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "v SHORT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .doubleColumn("v", 100.0) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .doubleColumn("v", -200.0) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "v\tts\n" + + "100\t1970-01-01T00:00:01.000000000Z\n" + + "-200\t1970-01-01T00:00:02.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testFloatToByte() throws Exception { + String table = "test_qwp_float_to_byte"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "v BYTE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .floatColumn("v", 7.0f) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .floatColumn("v", -100.0f) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "v\tts\n" + + "7\t1970-01-01T00:00:01.000000000Z\n" + + "-100\t1970-01-01T00:00:02.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testFloatToShort() throws Exception { + String table = "test_qwp_float_to_short"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "v SHORT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .floatColumn("v", 42.0f) + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .floatColumn("v", -1000.0f) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "v\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-1000\t1970-01-01T00:00:02.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testLong256ToString() throws Exception { + String table = "test_qwp_long256_to_string"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "s STRING, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .long256Column("s", 1, 2, 3, 4) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 1); + assertSqlEventually( + "s\tts\n" + + "0x04000000000000000300000000000000020000000000000001\t1970-01-01T00:00:01.000000000Z\n", + "SELECT s, ts FROM " + table); + } + + @Test + public void testLong256ToVarchar() throws Exception { + String table = "test_qwp_long256_to_varchar"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "v VARCHAR, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .long256Column("v", 1, 2, 3, 4) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 1); + assertSqlEventually( + "v\tts\n" + + "0x04000000000000000300000000000000020000000000000001\t1970-01-01T00:00:01.000000000Z\n", + "SELECT v, ts FROM " + table); + } + + @Test + public void testStringToDecimal8() throws Exception { + String table = "test_qwp_string_to_dec8"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(2, 1), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("d", "1.5") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("d", "-9.9") + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "1.5\t1970-01-01T00:00:01.000000000Z\n" + + "-9.9\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testStringToDecimal16() throws Exception { + String table = "test_qwp_string_to_dec16"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(4, 1), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("d", "12.5") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("d", "-99.9") + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "12.5\t1970-01-01T00:00:01.000000000Z\n" + + "-99.9\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testStringToDecimal32() throws Exception { + String table = "test_qwp_string_to_dec32"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(6, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("d", "1234.56") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("d", "-999.99") + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "1234.56\t1970-01-01T00:00:01.000000000Z\n" + + "-999.99\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testStringToTimestampNs() throws Exception { + String table = "test_qwp_string_to_timestamp_ns"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "ts_col TIMESTAMP_NS, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("ts_col", "2022-02-25T00:00:00.000000Z") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 1); + assertSqlEventually( + "ts_col\tts\n" + + "2022-02-25T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z\n", + "SELECT ts_col, ts FROM " + table); + } + + @Test + public void testUuidToString() throws Exception { + String table = "test_qwp_uuid_to_string"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "s STRING, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .uuidColumn("s", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 1); + assertSqlEventually( + "s\tts\n" + + "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11\t1970-01-01T00:00:01.000000000Z\n", + "SELECT s, ts FROM " + table); + } + + @Test + public void testUuidToVarchar() throws Exception { + String table = "test_qwp_uuid_to_varchar"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "v VARCHAR, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 1); + assertSqlEventually( + "v\tts\n" + + "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11\t1970-01-01T00:00:01.000000000Z\n", + "SELECT v, ts FROM " + table); + } + + // === SYMBOL negative coercion tests === + + @Test + public void testSymbolToBooleanCoercionError() throws Exception { + String table = "test_qwp_symbol_to_boolean_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .symbol("v", "hello") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write SYMBOL") && msg.contains("BOOLEAN") + ); + } + } + + @Test + public void testSymbolToByteCoercionError() throws Exception { + String table = "test_qwp_symbol_to_byte_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v BYTE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .symbol("v", "hello") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write SYMBOL") && msg.contains("BYTE") + ); + } + } + + @Test + public void testSymbolToCharCoercionError() throws Exception { + String table = "test_qwp_symbol_to_char_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v CHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .symbol("v", "hello") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write SYMBOL") && msg.contains("CHAR") + ); + } + } + + @Test + public void testSymbolToDateCoercionError() throws Exception { + String table = "test_qwp_symbol_to_date_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .symbol("v", "hello") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write SYMBOL") && msg.contains("DATE") + ); + } + } + + @Test + public void testSymbolToDecimalCoercionError() throws Exception { + String table = "test_qwp_symbol_to_decimal_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v DECIMAL(18,2), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .symbol("v", "hello") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write SYMBOL") && msg.contains("DECIMAL") + ); + } + } + + @Test + public void testSymbolToDoubleCoercionError() throws Exception { + String table = "test_qwp_symbol_to_double_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v DOUBLE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .symbol("v", "hello") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write SYMBOL") && msg.contains("DOUBLE") + ); + } + } + + @Test + public void testSymbolToFloatCoercionError() throws Exception { + String table = "test_qwp_symbol_to_float_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .symbol("v", "hello") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write SYMBOL") && msg.contains("FLOAT") + ); + } + } + + @Test + public void testSymbolToGeoHashCoercionError() throws Exception { + String table = "test_qwp_symbol_to_geohash_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .symbol("v", "hello") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write SYMBOL") && msg.contains("GEOHASH") + ); + } + } + + @Test + public void testSymbolToIntCoercionError() throws Exception { + String table = "test_qwp_symbol_to_int_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .symbol("v", "hello") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write SYMBOL") && msg.contains("INT") + ); + } + } + + @Test + public void testSymbolToLongCoercionError() throws Exception { + String table = "test_qwp_symbol_to_long_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v LONG, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .symbol("v", "hello") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write SYMBOL") && msg.contains("LONG") + ); + } + } + + @Test + public void testSymbolToLong256CoercionError() throws Exception { + String table = "test_qwp_symbol_to_long256_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .symbol("v", "hello") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write SYMBOL") && msg.contains("LONG256") + ); + } + } + + @Test + public void testSymbolToShortCoercionError() throws Exception { + String table = "test_qwp_symbol_to_short_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v SHORT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .symbol("v", "hello") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write SYMBOL") && msg.contains("SHORT") + ); + } + } + + @Test + public void testSymbolToTimestampCoercionError() throws Exception { + String table = "test_qwp_symbol_to_timestamp_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v TIMESTAMP, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .symbol("v", "hello") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write SYMBOL") && msg.contains("TIMESTAMP") + ); + } + } + + @Test + public void testSymbolToTimestampNsCoercionError() throws Exception { + String table = "test_qwp_symbol_to_timestamp_ns_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v TIMESTAMP_NS, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .symbol("v", "hello") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write SYMBOL") && msg.contains("TIMESTAMP") + ); + } + } + + @Test + public void testSymbolToUuidCoercionError() throws Exception { + String table = "test_qwp_symbol_to_uuid_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .symbol("v", "hello") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write SYMBOL") && msg.contains("UUID") + ); + } + } + + // === Null coercion tests === + + @Test + public void testNullStringToBoolean() throws Exception { + String table = "test_qwp_null_string_to_boolean"; + useTable(table); + execute("CREATE TABLE " + table + " (b BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("b", "true") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("b", null) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + assertTableSizeEventually(table, 2); + assertSqlEventually( + "b\tts\n" + + "true\t1970-01-01T00:00:01.000000000Z\n" + + "false\t1970-01-01T00:00:02.000000000Z\n", + "SELECT b, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testNullStringToChar() throws Exception { + String table = "test_qwp_null_string_to_char"; + useTable(table); + execute("CREATE TABLE " + table + " (c CHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("c", "A") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("c", null) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + assertTableSizeEventually(table, 2); + assertSqlEventually( + "c\tts\n" + + "A\t1970-01-01T00:00:01.000000000Z\n" + + "null\t1970-01-01T00:00:02.000000000Z\n", + "SELECT c, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testNullStringToDate() throws Exception { + String table = "test_qwp_null_string_to_date"; + useTable(table); + execute("CREATE TABLE " + table + " (d DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("d", "2022-02-25T00:00:00.000Z") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("d", null) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "2022-02-25T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z\n" + + "null\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testNullStringToDecimal() throws Exception { + String table = "test_qwp_null_string_to_decimal"; + useTable(table); + execute("CREATE TABLE " + table + " (d DECIMAL(18,2), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("d", "123.45") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("d", null) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "123.45\t1970-01-01T00:00:01.000000000Z\n" + + "null\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testNullStringToGeoHash() throws Exception { + String table = "test_qwp_null_string_to_geohash"; + useTable(table); + execute("CREATE TABLE " + table + " (g GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("g", "s09wh") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("g", null) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + assertTableSizeEventually(table, 2); + assertSqlEventually( + "g\tts\n" + + "s09wh\t1970-01-01T00:00:01.000000000Z\n" + + "null\t1970-01-01T00:00:02.000000000Z\n", + "SELECT g, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testNullStringToLong256() throws Exception { + String table = "test_qwp_null_string_to_long256"; + useTable(table); + execute("CREATE TABLE " + table + " (l LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("l", "0x01") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("l", null) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + assertTableSizeEventually(table, 2); + assertSqlEventually( + "l\tts\n" + + "0x01\t1970-01-01T00:00:01.000000000Z\n" + + "null\t1970-01-01T00:00:02.000000000Z\n", + "SELECT l, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testNullStringToNumeric() throws Exception { + String table = "test_qwp_null_string_to_numeric"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "i INT, " + + "l LONG, " + + "d DOUBLE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("i", "42") + .stringColumn("l", "100") + .stringColumn("d", "3.14") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("i", null) + .stringColumn("l", null) + .stringColumn("d", null) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + assertTableSizeEventually(table, 2); + assertSqlEventually( + "i\tl\td\tts\n" + + "42\t100\t3.14\t1970-01-01T00:00:01.000000000Z\n" + + "null\tnull\tnull\t1970-01-01T00:00:02.000000000Z\n", + "SELECT i, l, d, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testNullStringToSymbol() throws Exception { + String table = "test_qwp_null_string_to_symbol"; + useTable(table); + execute("CREATE TABLE " + table + " (s SYMBOL, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("s", "alpha") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("s", null) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + assertTableSizeEventually(table, 2); + assertSqlEventually( + "s\tts\n" + + "alpha\t1970-01-01T00:00:01.000000000Z\n" + + "null\t1970-01-01T00:00:02.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testNullStringToTimestamp() throws Exception { + String table = "test_qwp_null_string_to_timestamp"; + useTable(table); + execute("CREATE TABLE " + table + " (t TIMESTAMP, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("t", "2022-02-25T00:00:00.000000Z") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("t", null) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + assertTableSizeEventually(table, 2); + assertSqlEventually( + "t\tts\n" + + "2022-02-25T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z\n" + + "null\t1970-01-01T00:00:02.000000000Z\n", + "SELECT t, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testNullStringToTimestampNs() throws Exception { + String table = "test_qwp_null_string_to_timestamp_ns"; + useTable(table); + execute("CREATE TABLE " + table + " (t TIMESTAMP_NS, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("t", "2022-02-25T00:00:00.000000Z") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("t", null) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + assertTableSizeEventually(table, 2); + assertSqlEventually( + "t\tts\n" + + "2022-02-25T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z\n" + + "null\t1970-01-01T00:00:02.000000000Z\n", + "SELECT t, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testNullStringToUuid() throws Exception { + String table = "test_qwp_null_string_to_uuid"; + useTable(table); + execute("CREATE TABLE " + table + " (u UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("u", "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("u", null) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + assertTableSizeEventually(table, 2); + assertSqlEventually( + "u\tts\n" + + "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11\t1970-01-01T00:00:01.000000000Z\n" + + "null\t1970-01-01T00:00:02.000000000Z\n", + "SELECT u, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testNullSymbolToString() throws Exception { + String table = "test_qwp_null_symbol_to_string"; + useTable(table); + execute("CREATE TABLE " + table + " (s STRING, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .symbol("s", "hello") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .symbol("s", null) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + assertTableSizeEventually(table, 2); + assertSqlEventually( + "s\tts\n" + + "hello\t1970-01-01T00:00:01.000000000Z\n" + + "null\t1970-01-01T00:00:02.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testNullSymbolToVarchar() throws Exception { + String table = "test_qwp_null_symbol_to_varchar"; + useTable(table); + execute("CREATE TABLE " + table + " (v VARCHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .symbol("v", "hello") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .symbol("v", null) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + assertTableSizeEventually(table, 2); + assertSqlEventually( + "v\tts\n" + + "hello\t1970-01-01T00:00:01.000000000Z\n" + + "null\t1970-01-01T00:00:02.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); + } + + // === BOOLEAN negative tests === + + @Test + public void testBooleanToByteCoercionError() throws Exception { + String table = "test_qwp_boolean_to_byte_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v BYTE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .boolColumn("v", true) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write BOOLEAN") && msg.contains("BYTE") + ); + } + } + + @Test + public void testBooleanToShortCoercionError() throws Exception { + String table = "test_qwp_boolean_to_short_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v SHORT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .boolColumn("v", true) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write BOOLEAN") && msg.contains("SHORT") + ); + } + } + + @Test + public void testBooleanToIntCoercionError() throws Exception { + String table = "test_qwp_boolean_to_int_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .boolColumn("v", true) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write BOOLEAN") && msg.contains("INT") + ); + } + } + + @Test + public void testBooleanToLongCoercionError() throws Exception { + String table = "test_qwp_boolean_to_long_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v LONG, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .boolColumn("v", true) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write BOOLEAN") && msg.contains("LONG") + ); + } + } + + @Test + public void testBooleanToFloatCoercionError() throws Exception { + String table = "test_qwp_boolean_to_float_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .boolColumn("v", true) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write BOOLEAN") && msg.contains("FLOAT") + ); + } + } + + @Test + public void testBooleanToDoubleCoercionError() throws Exception { + String table = "test_qwp_boolean_to_double_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v DOUBLE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .boolColumn("v", true) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write BOOLEAN") && msg.contains("DOUBLE") + ); + } + } + + @Test + public void testBooleanToDateCoercionError() throws Exception { + String table = "test_qwp_boolean_to_date_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .boolColumn("v", true) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write BOOLEAN") && msg.contains("DATE") + ); + } + } + + @Test + public void testBooleanToUuidCoercionError() throws Exception { + String table = "test_qwp_boolean_to_uuid_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .boolColumn("v", true) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write BOOLEAN") && msg.contains("UUID") + ); + } + } + + @Test + public void testBooleanToLong256CoercionError() throws Exception { + String table = "test_qwp_boolean_to_long256_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .boolColumn("v", true) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write BOOLEAN") && msg.contains("LONG256") + ); + } + } + + @Test + public void testBooleanToGeoHashCoercionError() throws Exception { + String table = "test_qwp_boolean_to_geohash_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .boolColumn("v", true) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write BOOLEAN") && msg.contains("GEOHASH") + ); + } + } + + @Test + public void testBooleanToTimestampCoercionError() throws Exception { + String table = "test_qwp_boolean_to_timestamp_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v TIMESTAMP, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .boolColumn("v", true) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write BOOLEAN") && msg.contains("TIMESTAMP") + ); + } + } + + @Test + public void testBooleanToTimestampNsCoercionError() throws Exception { + String table = "test_qwp_boolean_to_timestamp_ns_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v TIMESTAMP_NS, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .boolColumn("v", true) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write BOOLEAN") && msg.contains("TIMESTAMP") + ); + } + } + + @Test + public void testBooleanToCharCoercionError() throws Exception { + String table = "test_qwp_boolean_to_char_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v CHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .boolColumn("v", true) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write BOOLEAN") && msg.contains("CHAR") + ); + } + } + + @Test + public void testBooleanToSymbolCoercionError() throws Exception { + String table = "test_qwp_boolean_to_symbol_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v SYMBOL, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .boolColumn("v", true) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write BOOLEAN") && msg.contains("SYMBOL") + ); + } + } + + @Test + public void testBooleanToDecimalCoercionError() throws Exception { + String table = "test_qwp_boolean_to_decimal_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v DECIMAL(18,2), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .boolColumn("v", true) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write BOOLEAN") && msg.contains("DECIMAL") + ); + } + } + + // === FLOAT negative tests === + + @Test + public void testFloatToBooleanCoercionError() throws Exception { + String table = "test_qwp_float_to_boolean_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .floatColumn("v", 1.5f) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write FLOAT") && msg.contains("BOOLEAN") + ); + } + } + + @Test + public void testFloatToCharCoercionError() throws Exception { + String table = "test_qwp_float_to_char_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v CHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .floatColumn("v", 1.5f) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write FLOAT") && msg.contains("CHAR") + ); + } + } + + @Test + public void testFloatToDateCoercionError() throws Exception { + String table = "test_qwp_float_to_date_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .floatColumn("v", 1.5f) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from FLOAT to") && msg.contains("is not supported") + ); + } + } + + @Test + public void testFloatToGeoHashCoercionError() throws Exception { + String table = "test_qwp_float_to_geohash_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .floatColumn("v", 1.5f) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from FLOAT to") && msg.contains("is not supported") + ); + } + } + + @Test + public void testFloatToUuidCoercionError() throws Exception { + String table = "test_qwp_float_to_uuid_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .floatColumn("v", 1.5f) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from FLOAT to") && msg.contains("is not supported") + ); + } + } + + @Test + public void testFloatToLong256CoercionError() throws Exception { + String table = "test_qwp_float_to_long256_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .floatColumn("v", 1.5f) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from FLOAT to") && msg.contains("is not supported") + ); + } + } + + // === DOUBLE negative tests === + + @Test + public void testDoubleToBooleanCoercionError() throws Exception { + String table = "test_qwp_double_to_boolean_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .doubleColumn("v", 3.14) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DOUBLE") && msg.contains("BOOLEAN") + ); + } + } + + @Test + public void testDoubleToCharCoercionError() throws Exception { + String table = "test_qwp_double_to_char_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v CHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .doubleColumn("v", 3.14) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DOUBLE") && msg.contains("CHAR") + ); + } + } + + @Test + public void testDoubleToDateCoercionError() throws Exception { + String table = "test_qwp_double_to_date_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .doubleColumn("v", 3.14) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from DOUBLE to") && msg.contains("is not supported") + ); + } + } + + @Test + public void testDoubleToGeoHashCoercionError() throws Exception { + String table = "test_qwp_double_to_geohash_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .doubleColumn("v", 3.14) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from DOUBLE to") && msg.contains("is not supported") + ); + } + } + + @Test + public void testDoubleToUuidCoercionError() throws Exception { + String table = "test_qwp_double_to_uuid_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .doubleColumn("v", 3.14) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from DOUBLE to") && msg.contains("is not supported") + ); + } + } + + @Test + public void testDoubleToLong256CoercionError() throws Exception { + String table = "test_qwp_double_to_long256_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .doubleColumn("v", 3.14) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from DOUBLE to") && msg.contains("is not supported") + ); + } + } + + // ==================== CHAR negative tests ==================== + + @Test + public void testCharToBooleanCoercionError() throws Exception { + String table = "test_qwp_char_to_boolean_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .charColumn("v", 'A') + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write") && msg.contains("BOOLEAN") + ); + } + } + + @Test + public void testCharToSymbolCoercionError() throws Exception { + String table = "test_qwp_char_to_symbol_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v SYMBOL, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .charColumn("v", 'A') + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write") && msg.contains("SYMBOL") + ); + } + } + + @Test + public void testCharToByteCoercionError() throws Exception { + String table = "test_qwp_char_to_byte_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v BYTE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .charColumn("v", 'A') + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("not supported") && msg.contains("BYTE") + ); + } + } + + @Test + public void testCharToShortCoercionError() throws Exception { + String table = "test_qwp_char_to_short_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v SHORT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .charColumn("v", 'A') + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("not supported") && msg.contains("SHORT") + ); + } + } + + @Test + public void testCharToIntCoercionError() throws Exception { + String table = "test_qwp_char_to_int_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .charColumn("v", 'A') + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("not supported") && msg.contains("INT") + ); + } + } + + @Test + public void testCharToLongCoercionError() throws Exception { + String table = "test_qwp_char_to_long_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v LONG, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .charColumn("v", 'A') + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("not supported") && msg.contains("LONG") + ); + } + } + + @Test + public void testCharToFloatCoercionError() throws Exception { + String table = "test_qwp_char_to_float_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .charColumn("v", 'A') + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("not supported") && msg.contains("FLOAT") + ); + } + } + + @Test + public void testCharToDoubleCoercionError() throws Exception { + String table = "test_qwp_char_to_double_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v DOUBLE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .charColumn("v", 'A') + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("not supported") && msg.contains("DOUBLE") + ); + } + } + + @Test + public void testCharToDateCoercionError() throws Exception { + String table = "test_qwp_char_to_date_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .charColumn("v", 'A') + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("not supported") && msg.contains("DATE") + ); + } + } + + @Test + public void testCharToUuidCoercionError() throws Exception { + String table = "test_qwp_char_to_uuid_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .charColumn("v", 'A') + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("not supported") && msg.contains("UUID") + ); + } + } + + @Test + public void testCharToLong256CoercionError() throws Exception { + String table = "test_qwp_char_to_long256_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .charColumn("v", 'A') + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("not supported") && msg.contains("LONG256") + ); + } + } + + @Test + public void testCharToGeoHashCoercionError() throws Exception { + String table = "test_qwp_char_to_geohash_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .charColumn("v", 'A') + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("GEOHASH") + ); + } + } + + // ==================== LONG256 negative tests ==================== + + @Test + public void testLong256ToBooleanCoercionError() throws Exception { + String table = "test_qwp_long256_to_boolean_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .long256Column("v", 1L, 0L, 0L, 0L) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write LONG256") && msg.contains("BOOLEAN") + ); + } + } + + @Test + public void testLong256ToCharCoercionError() throws Exception { + String table = "test_qwp_long256_to_char_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v CHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .long256Column("v", 1L, 0L, 0L, 0L) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write LONG256") && msg.contains("CHAR") + ); + } + } + + @Test + public void testLong256ToSymbolCoercionError() throws Exception { + String table = "test_qwp_long256_to_symbol_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v SYMBOL, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .long256Column("v", 1L, 0L, 0L, 0L) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write LONG256") && msg.contains("SYMBOL") + ); + } + } + + @Test + public void testLong256ToByteCoercionError() throws Exception { + String table = "test_qwp_long256_to_byte_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v BYTE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .long256Column("v", 1L, 0L, 0L, 0L) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") + ); + } + } + + @Test + public void testLong256ToShortCoercionError() throws Exception { + String table = "test_qwp_long256_to_short_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v SHORT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .long256Column("v", 1L, 0L, 0L, 0L) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") + ); + } + } + + @Test + public void testLong256ToIntCoercionError() throws Exception { + String table = "test_qwp_long256_to_int_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .long256Column("v", 1L, 0L, 0L, 0L) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") + ); + } + } + + @Test + public void testLong256ToLongCoercionError() throws Exception { + String table = "test_qwp_long256_to_long_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v LONG, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .long256Column("v", 1L, 0L, 0L, 0L) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") + ); + } + } + + @Test + public void testLong256ToFloatCoercionError() throws Exception { + String table = "test_qwp_long256_to_float_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .long256Column("v", 1L, 0L, 0L, 0L) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") + ); + } + } + + @Test + public void testLong256ToDoubleCoercionError() throws Exception { + String table = "test_qwp_long256_to_double_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v DOUBLE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .long256Column("v", 1L, 0L, 0L, 0L) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") + ); + } + } + + @Test + public void testLong256ToDateCoercionError() throws Exception { + String table = "test_qwp_long256_to_date_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .long256Column("v", 1L, 0L, 0L, 0L) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") + ); + } + } + + @Test + public void testLong256ToUuidCoercionError() throws Exception { + String table = "test_qwp_long256_to_uuid_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .long256Column("v", 1L, 0L, 0L, 0L) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") + ); + } + } + + @Test + public void testLong256ToGeoHashCoercionError() throws Exception { + String table = "test_qwp_long256_to_geohash_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .long256Column("v", 1L, 0L, 0L, 0L) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") + ); + } + } + + // ==================== UUID negative tests ==================== + + @Test + public void testUuidToBooleanCoercionError() throws Exception { + String table = "test_qwp_uuid_to_boolean_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); + sender.table(table) + .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write UUID") && msg.contains("BOOLEAN") + ); + } + } + + @Test + public void testUuidToCharCoercionError() throws Exception { + String table = "test_qwp_uuid_to_char_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v CHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); + sender.table(table) + .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write UUID") && msg.contains("CHAR") + ); + } + } + + @Test + public void testUuidToSymbolCoercionError() throws Exception { + String table = "test_qwp_uuid_to_symbol_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v SYMBOL, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); + sender.table(table) + .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write UUID") && msg.contains("SYMBOL") + ); + } + } + + @Test + public void testUuidToByteCoercionError() throws Exception { + String table = "test_qwp_uuid_to_byte_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v BYTE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); + sender.table(table) + .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from UUID to") && msg.contains("is not supported") + ); + } + } + + @Test + public void testUuidToIntCoercionError() throws Exception { + String table = "test_qwp_uuid_to_int_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); + sender.table(table) + .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from UUID to") && msg.contains("is not supported") + ); + } + } + + @Test + public void testUuidToLongCoercionError() throws Exception { + String table = "test_qwp_uuid_to_long_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v LONG, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); + sender.table(table) + .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from UUID to") && msg.contains("is not supported") + ); + } + } + + @Test + public void testUuidToFloatCoercionError() throws Exception { + String table = "test_qwp_uuid_to_float_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); + sender.table(table) + .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from UUID to") && msg.contains("is not supported") + ); + } + } + + @Test + public void testUuidToDoubleCoercionError() throws Exception { + String table = "test_qwp_uuid_to_double_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v DOUBLE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); + sender.table(table) + .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from UUID to") && msg.contains("is not supported") + ); + } + } + + @Test + public void testUuidToDateCoercionError() throws Exception { + String table = "test_qwp_uuid_to_date_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); + sender.table(table) + .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from UUID to") && msg.contains("is not supported") + ); + } + } + + @Test + public void testUuidToLong256CoercionError() throws Exception { + String table = "test_qwp_uuid_to_long256_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); + sender.table(table) + .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from UUID to") && msg.contains("is not supported") + ); + } + } + + @Test + public void testUuidToGeoHashCoercionError() throws Exception { + String table = "test_qwp_uuid_to_geohash_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); + sender.table(table) + .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from UUID to") && msg.contains("is not supported") + ); + } + } + + // === TIMESTAMP negative coercion tests === + + @Test + public void testTimestampToBooleanCoercionError() throws Exception { + String table = "test_qwp_timestamp_to_boolean_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write TIMESTAMP") && msg.contains("BOOLEAN") + ); + } + } + + @Test + public void testTimestampToByteCoercionError() throws Exception { + String table = "test_qwp_timestamp_to_byte_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v BYTE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write TIMESTAMP") && msg.contains("BYTE") + ); + } + } + + @Test + public void testTimestampToShortCoercionError() throws Exception { + String table = "test_qwp_timestamp_to_short_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v SHORT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write TIMESTAMP") && msg.contains("SHORT") + ); + } + } + + @Test + public void testTimestampToIntCoercionError() throws Exception { + String table = "test_qwp_timestamp_to_int_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write TIMESTAMP") && msg.contains("INT") + ); + } + } + + @Test + public void testTimestampToLongCoercionError() throws Exception { + String table = "test_qwp_timestamp_to_long_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v LONG, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write TIMESTAMP") && msg.contains("LONG") + ); + } + } + + @Test + public void testTimestampToFloatCoercionError() throws Exception { + String table = "test_qwp_timestamp_to_float_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write TIMESTAMP") && msg.contains("FLOAT") + ); + } + } + + @Test + public void testTimestampToDoubleCoercionError() throws Exception { + String table = "test_qwp_timestamp_to_double_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v DOUBLE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write TIMESTAMP") && msg.contains("DOUBLE") + ); + } + } + + @Test + public void testTimestampToDateCoercionError() throws Exception { + String table = "test_qwp_timestamp_to_date_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write TIMESTAMP") && msg.contains("DATE") + ); + } + } + + @Test + public void testTimestampToUuidCoercionError() throws Exception { + String table = "test_qwp_timestamp_to_uuid_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write TIMESTAMP") && msg.contains("UUID") + ); + } + } + + @Test + public void testTimestampToLong256CoercionError() throws Exception { + String table = "test_qwp_timestamp_to_long256_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write TIMESTAMP") && msg.contains("LONG256") + ); + } + } + + @Test + public void testTimestampToGeoHashCoercionError() throws Exception { + String table = "test_qwp_timestamp_to_geohash_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write TIMESTAMP") && msg.contains("GEOHASH") + ); + } + } + + @Test + public void testTimestampToCharCoercionError() throws Exception { + String table = "test_qwp_timestamp_to_char_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v CHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write TIMESTAMP") && msg.contains("CHAR") + ); + } + } + + @Test + public void testTimestampToSymbolCoercionError() throws Exception { + String table = "test_qwp_timestamp_to_symbol_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v SYMBOL, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write TIMESTAMP") && msg.contains("SYMBOL") + ); + } + } + + @Test + public void testTimestampToDecimalCoercionError() throws Exception { + String table = "test_qwp_timestamp_to_decimal_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v DECIMAL(18,2), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write TIMESTAMP") && msg.contains("DECIMAL") + ); + } + } + + // === DECIMAL negative coercion tests === + + @Test + public void testDecimalToBooleanCoercionError() throws Exception { + String table = "test_qwp_decimal_to_boolean_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DECIMAL64") && msg.contains("BOOLEAN") + ); + } + } + + @Test + public void testDecimalToByteCoercionError() throws Exception { + String table = "test_qwp_decimal_to_byte_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v BYTE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DECIMAL64") && msg.contains("BYTE") + ); + } + } + + @Test + public void testDecimalToShortCoercionError() throws Exception { + String table = "test_qwp_decimal_to_short_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v SHORT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DECIMAL64") && msg.contains("SHORT") + ); + } + } + + @Test + public void testDecimalToIntCoercionError() throws Exception { + String table = "test_qwp_decimal_to_int_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DECIMAL64") && msg.contains("INT") + ); + } + } + + @Test + public void testDecimalToLongCoercionError() throws Exception { + String table = "test_qwp_decimal_to_long_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v LONG, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DECIMAL64") && msg.contains("LONG") + ); + } + } + + @Test + public void testDecimalToFloatCoercionError() throws Exception { + String table = "test_qwp_decimal_to_float_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DECIMAL64") && msg.contains("FLOAT") + ); + } + } + + @Test + public void testDecimalToDoubleCoercionError() throws Exception { + String table = "test_qwp_decimal_to_double_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v DOUBLE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DECIMAL64") && msg.contains("DOUBLE") + ); + } + } + + @Test + public void testDecimalToDateCoercionError() throws Exception { + String table = "test_qwp_decimal_to_date_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DECIMAL64") && msg.contains("DATE") + ); + } + } + + @Test + public void testDecimalToUuidCoercionError() throws Exception { + String table = "test_qwp_decimal_to_uuid_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DECIMAL64") && msg.contains("UUID") + ); + } + } + + @Test + public void testDecimalToLong256CoercionError() throws Exception { + String table = "test_qwp_decimal_to_long256_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DECIMAL64") && msg.contains("LONG256") + ); + } + } + + @Test + public void testDecimalToGeoHashCoercionError() throws Exception { + String table = "test_qwp_decimal_to_geohash_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DECIMAL64") && msg.contains("GEOHASH") + ); + } + } + + @Test + public void testDecimalToTimestampCoercionError() throws Exception { + String table = "test_qwp_decimal_to_timestamp_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v TIMESTAMP, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DECIMAL64") && msg.contains("TIMESTAMP") + ); + } + } + + @Test + public void testDecimalToTimestampNsCoercionError() throws Exception { + String table = "test_qwp_decimal_to_timestamp_ns_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v TIMESTAMP_NS, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DECIMAL64") && msg.contains("TIMESTAMP") + ); + } + } + + @Test + public void testDecimalToCharCoercionError() throws Exception { + String table = "test_qwp_decimal_to_char_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v CHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DECIMAL64") && msg.contains("CHAR") + ); + } + } + + @Test + public void testDecimalToSymbolCoercionError() throws Exception { + String table = "test_qwp_decimal_to_symbol_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v SYMBOL, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DECIMAL64") && msg.contains("SYMBOL") + ); + } + } + + // === DOUBLE_ARRAY negative coercion tests === + + @Test + public void testDoubleArrayToIntCoercionError() throws Exception { + String table = "test_qwp_doublearray_to_int_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .doubleArray("v", new double[]{1.0, 2.0}) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DOUBLE_ARRAY") && msg.contains("INT") + ); + } + } + + @Test + public void testDoubleArrayToStringCoercionError() throws Exception { + String table = "test_qwp_doublearray_to_string_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v STRING, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .doubleArray("v", new double[]{1.0, 2.0}) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DOUBLE_ARRAY") && msg.contains("STRING") + ); + } + } + + @Test + public void testDoubleArrayToSymbolCoercionError() throws Exception { + String table = "test_qwp_doublearray_to_symbol_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v SYMBOL, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .doubleArray("v", new double[]{1.0, 2.0}) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DOUBLE_ARRAY") && msg.contains("SYMBOL") + ); + } + } + + @Test + public void testDoubleArrayToTimestampCoercionError() throws Exception { + String table = "test_qwp_doublearray_to_timestamp_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v TIMESTAMP, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .doubleArray("v", new double[]{1.0, 2.0}) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DOUBLE_ARRAY") && msg.contains("TIMESTAMP") + ); + } + } + + // ==================== Additional null coercion tests ==================== + + @Test + public void testNullStringToVarchar() throws Exception { + String table = "test_qwp_null_string_to_varchar"; + useTable(table); + execute("CREATE TABLE " + table + " (v VARCHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("v", "hello") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("v", null) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + assertTableSizeEventually(table, 2); + assertSqlEventually( + "v\tts\n" + + "hello\t1970-01-01T00:00:01.000000000Z\n" + + "null\t1970-01-01T00:00:02.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testNullSymbolToSymbol() throws Exception { + String table = "test_qwp_null_symbol_to_symbol"; + useTable(table); + execute("CREATE TABLE " + table + " (s SYMBOL, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .symbol("s", "alpha") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .symbol("s", null) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + assertTableSizeEventually(table, 2); + assertSqlEventually( + "s\tts\n" + + "alpha\t1970-01-01T00:00:01.000000000Z\n" + + "null\t1970-01-01T00:00:02.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testNullStringToByte() throws Exception { + String table = "test_qwp_null_string_to_byte"; + useTable(table); + execute("CREATE TABLE " + table + " (b BYTE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("b", "42") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("b", null) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + assertTableSizeEventually(table, 2); + assertSqlEventually( + "b\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT b, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testNullStringToShort() throws Exception { + String table = "test_qwp_null_string_to_short"; + useTable(table); + execute("CREATE TABLE " + table + " (s SHORT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("s", "42") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("s", null) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + assertTableSizeEventually(table, 2); + assertSqlEventually( + "s\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testNullStringToFloat() throws Exception { + String table = "test_qwp_null_string_to_float"; + useTable(table); + execute("CREATE TABLE " + table + " (f FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("f", "3.14") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("f", null) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + assertTableSizeEventually(table, 2); + assertSqlEventually( + "f\tts\n" + + "3.14\t1970-01-01T00:00:01.000000000Z\n" + + "null\t1970-01-01T00:00:02.000000000Z\n", + "SELECT f, ts FROM " + table + " ORDER BY ts"); + } + + // ==================== Additional positive coercion test ==================== + + @Test + public void testStringToVarchar() throws Exception { + String table = "test_qwp_string_to_varchar"; + useTable(table); + execute("CREATE TABLE " + table + " (v VARCHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("v", "hello") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("v", "world") + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + assertTableSizeEventually(table, 2); + assertSqlEventually( + "v\tts\n" + + "hello\t1970-01-01T00:00:01.000000000Z\n" + + "world\t1970-01-01T00:00:02.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); + } + + // ==================== Additional parse error tests ==================== + + @Test + public void testStringToIntParseError() throws Exception { + String table = "test_qwp_string_to_int_parse_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("v", "not_a_number") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected parse error but got: " + msg, + msg.contains("cannot parse INT from string") && msg.contains("not_a_number") + ); + } + } + + @Test + public void testStringToLongParseError() throws Exception { + String table = "test_qwp_string_to_long_parse_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v LONG, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("v", "not_a_number") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected parse error but got: " + msg, + msg.contains("cannot parse LONG from string") && msg.contains("not_a_number") + ); + } + } + + @Test + public void testStringToShortParseError() throws Exception { + String table = "test_qwp_string_to_short_parse_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v SHORT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("v", "not_a_number") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected parse error but got: " + msg, + msg.contains("cannot parse SHORT from string") && msg.contains("not_a_number") + ); + } + } + + @Test + public void testStringToFloatParseError() throws Exception { + String table = "test_qwp_string_to_float_parse_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("v", "not_a_number") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected parse error but got: " + msg, + msg.contains("cannot parse FLOAT from string") && msg.contains("not_a_number") + ); + } + } + + @Test + public void testStringToDoubleParseError() throws Exception { + String table = "test_qwp_string_to_double_parse_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v DOUBLE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("v", "not_a_number") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected parse error but got: " + msg, + msg.contains("cannot parse DOUBLE from string") && msg.contains("not_a_number") + ); + } + } + + @Test + public void testStringToDateParseError() throws Exception { + String table = "test_qwp_string_to_date_parse_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("v", "not_a_date") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected parse error but got: " + msg, + msg.contains("cannot parse DATE from string") && msg.contains("not_a_date") + ); + } + } + + @Test + public void testStringToTimestampParseError() throws Exception { + String table = "test_qwp_string_to_timestamp_parse_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v TIMESTAMP, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("v", "not_a_timestamp") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected parse error but got: " + msg, + msg.contains("cannot parse timestamp from string") && msg.contains("not_a_timestamp") + ); + } + } + + @Test + public void testStringToUuidParseError() throws Exception { + String table = "test_qwp_string_to_uuid_parse_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("v", "not-a-uuid") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected parse error but got: " + msg, + msg.contains("cannot parse UUID from string") && msg.contains("not-a-uuid") + ); + } + } + + @Test + public void testStringToLong256ParseError() throws Exception { + String table = "test_qwp_string_to_long256_parse_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("v", "not_a_long256") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected parse error but got: " + msg, + msg.contains("cannot parse long256 from string") && msg.contains("not_a_long256") + ); + } + } + + @Test + public void testStringToGeoHashParseError() throws Exception { + String table = "test_qwp_string_to_geohash_parse_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("v", "!!!") + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected parse error but got: " + msg, + msg.contains("cannot parse geohash from string") && msg.contains("!!!") + ); + } + } + // === Helper Methods === private QwpWebSocketSender createQwpSender() { From f609b5a795fc85f5cba107a39a8a5c0c541b5221 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Sun, 22 Feb 2026 00:28:59 +0000 Subject: [PATCH 12/89] wip 9 --- .../questdb/client/test/cutlass/qwp/client/QwpSenderTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java index 6b7e8d9..02106f0 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java @@ -868,7 +868,7 @@ public void testDoubleToDecimalPrecisionLossError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected precision loss error but got: " + msg, - msg.contains("loses precision") && msg.contains("123.456") && msg.contains("scale=2") + msg.contains("cannot be converted to") && msg.contains("123.456") && msg.contains("scale=2") ); } } @@ -1204,7 +1204,7 @@ public void testFloatToDecimalPrecisionLossError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected precision loss error but got: " + msg, - msg.contains("loses precision") && msg.contains("scale=1") + msg.contains("cannot be converted to") && msg.contains("scale=1") ); } } From eb9531ddf9e85623e79856c987ee32f19832ebb7 Mon Sep 17 00:00:00 2001 From: bluestreak Date: Sun, 22 Feb 2026 02:56:03 +0000 Subject: [PATCH 13/89] wip 11 --- .../qwp/client/QwpWebSocketEncoder.java | 446 ++--- .../qwp/client/QwpWebSocketSender.java | 11 + .../qwp/protocol/OffHeapAppendMemory.java | 161 ++ .../cutlass/qwp/protocol/QwpTableBuffer.java | 1606 +++++++---------- .../qwp/protocol/OffHeapAppendMemoryTest.java | 266 +++ 5 files changed, 1249 insertions(+), 1241 deletions(-) create mode 100644 core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/OffHeapAppendMemoryTest.java diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java index c71e158..ef473de 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java @@ -28,37 +28,26 @@ import io.questdb.client.cutlass.qwp.protocol.QwpGorillaEncoder; import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; import io.questdb.client.std.QuietCloseable; +import io.questdb.client.std.Unsafe; import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.*; /** * Encodes ILP v4 messages for WebSocket transport. *

- * This encoder can write to either an internal {@link NativeBufferWriter} (default) - * or an external {@link QwpBufferWriter} such as {@link io.questdb.client.cutlass.http.client.WebSocketSendBuffer}. + * This encoder reads column data from off-heap {@link io.questdb.client.cutlass.qwp.protocol.OffHeapAppendMemory} + * buffers in {@link QwpTableBuffer.ColumnBuffer} and uses bulk {@code putBlockOfBytes} for fixed-width + * types where wire format matches native byte order. *

- * When using an external buffer, the encoder writes directly to it without intermediate copies, - * enabling zero-copy WebSocket frame construction. + * Types that use bulk copy (native byte-order on wire): + * BYTE, SHORT, INT, LONG, FLOAT, DOUBLE, DATE, UUID, LONG256 *

- * Usage with external buffer (zero-copy): - * 

- * WebSocketSendBuffer buf = client.getSendBuffer();
- * buf.beginBinaryFrame();
- * encoder.setBuffer(buf);
- * encoder.encode(tableData, false);
- * FrameInfo frame = buf.endBinaryFrame();
- * client.sendFrame(frame);
- *
+ * Types that require element-by-element encoding: + * BOOLEAN (bit-packed on wire), TIMESTAMP (Gorilla), DECIMAL64/128/256 (big-endian on wire) */ public class QwpWebSocketEncoder implements QuietCloseable { - /** - * Encoding flag for Gorilla-encoded timestamps. - */ public static final byte ENCODING_GORILLA = 0x01; - /** - * Encoding flag for uncompressed timestamps. - */ public static final byte ENCODING_UNCOMPRESSED = 0x00; private final QwpGorillaEncoder gorillaEncoder = new QwpGorillaEncoder(); private QwpBufferWriter buffer; @@ -85,43 +74,16 @@ public void close() { } } - /** - * Encodes a complete ILP v4 message from a table buffer. - * - * @param tableBuffer the table buffer containing row data - * @param useSchemaRef whether to use schema reference mode - * @return the number of bytes written - */ public int encode(QwpTableBuffer tableBuffer, boolean useSchemaRef) { buffer.reset(); - - // Write message header with placeholder for payload length writeHeader(1, 0); int payloadStart = buffer.getPosition(); - - // Encode table data encodeTable(tableBuffer, useSchemaRef); - - // Patch payload length int payloadLength = buffer.getPosition() - payloadStart; buffer.patchInt(8, payloadLength); - return buffer.getPosition(); } - /** - * Encodes a complete ILP v4 message with delta symbol dictionary encoding. - *

- * This method sends only new symbols (delta) since the last confirmed watermark, - * and uses global symbol IDs instead of per-column local indices. - * - * @param tableBuffer the table buffer containing row data - * @param globalDict the global symbol dictionary - * @param confirmedMaxId the highest symbol ID the server has confirmed (from ConnectionSymbolState) - * @param batchMaxId the highest symbol ID used in this batch - * @param useSchemaRef whether to use schema reference mode - * @return the number of bytes written - */ public int encodeWithDeltaDict( QwpTableBuffer tableBuffer, GlobalSymbolDictionary globalDict, @@ -130,101 +92,51 @@ public int encodeWithDeltaDict( boolean useSchemaRef ) { buffer.reset(); - - // Calculate delta range int deltaStart = confirmedMaxId + 1; int deltaCount = Math.max(0, batchMaxId - confirmedMaxId); - - // Set delta dictionary flag byte savedFlags = flags; flags |= FLAG_DELTA_SYMBOL_DICT; - - // Write message header with placeholder for payload length writeHeader(1, 0); int payloadStart = buffer.getPosition(); - - // Write symbol delta section (before tables) buffer.putVarint(deltaStart); buffer.putVarint(deltaCount); for (int id = deltaStart; id < deltaStart + deltaCount; id++) { String symbol = globalDict.getSymbol(id); buffer.putString(symbol); } - - // Encode table data (symbol columns will use global IDs) encodeTableWithGlobalSymbols(tableBuffer, useSchemaRef); - - // Patch payload length int payloadLength = buffer.getPosition() - payloadStart; buffer.patchInt(8, payloadLength); - - // Restore flags flags = savedFlags; - return buffer.getPosition(); } - /** - * Returns the underlying buffer. - *

- * If an external buffer was set via {@link #setBuffer(QwpBufferWriter)}, - * that buffer is returned. Otherwise, returns the internal buffer. - */ public QwpBufferWriter getBuffer() { return buffer; } - /** - * Returns true if delta symbol dictionary encoding is enabled. - */ public boolean isDeltaSymbolDictEnabled() { return (flags & FLAG_DELTA_SYMBOL_DICT) != 0; } - /** - * Returns true if Gorilla encoding is enabled. - */ public boolean isGorillaEnabled() { return (flags & FLAG_GORILLA) != 0; } - /** - * Returns true if currently using an external buffer. - */ public boolean isUsingExternalBuffer() { return buffer != ownedBuffer; } - /** - * Resets the encoder for a new message. - *

- * If using an external buffer, this only resets the internal state (flags). - * The external buffer's reset is the caller's responsibility. - * If using the internal buffer, resets both the buffer and internal state. - */ public void reset() { if (!isUsingExternalBuffer()) { buffer.reset(); } } - /** - * Sets an external buffer for encoding. - *

- * When set, the encoder writes directly to this buffer instead of its internal buffer. - * The caller is responsible for managing the external buffer's lifecycle. - *

- * Pass {@code null} to revert to using the internal buffer. - * - * @param externalBuffer the external buffer to use, or null to use internal buffer - */ public void setBuffer(QwpBufferWriter externalBuffer) { this.buffer = externalBuffer != null ? externalBuffer : ownedBuffer; } - /** - * Sets the delta symbol dictionary flag. - */ public void setDeltaSymbolDictEnabled(boolean enabled) { if (enabled) { flags |= FLAG_DELTA_SYMBOL_DICT; @@ -233,9 +145,6 @@ public void setDeltaSymbolDictEnabled(boolean enabled) { } } - /** - * Sets whether Gorilla timestamp encoding is enabled. - */ public void setGorillaEnabled(boolean enabled) { if (enabled) { flags |= FLAG_GORILLA; @@ -244,73 +153,54 @@ public void setGorillaEnabled(boolean enabled) { } } - /** - * Writes the ILP v4 message header. - * - * @param tableCount number of tables in the message - * @param payloadLength payload length (can be 0 if patched later) - */ public void writeHeader(int tableCount, int payloadLength) { - // Magic "ILP4" buffer.putByte((byte) 'I'); buffer.putByte((byte) 'L'); buffer.putByte((byte) 'P'); buffer.putByte((byte) '4'); - - // Version buffer.putByte(VERSION_1); - - // Flags buffer.putByte(flags); - - // Table count (uint16, little-endian) buffer.putShort((short) tableCount); - - // Payload length (uint32, little-endian) buffer.putInt(payloadLength); } - /** - * Encodes a single column. - */ private void encodeColumn(QwpTableBuffer.ColumnBuffer col, QwpColumnDef colDef, int rowCount, boolean useGorilla) { int valueCount = col.getValueCount(); + long dataAddr = col.getDataAddress(); - // Write null bitmap if column is nullable if (colDef.isNullable()) { - writeNullBitmapPacked(col.getNullBitmapPacked(), rowCount); + writeNullBitmap(col, rowCount); } - // Write column data based on type switch (col.getType()) { case TYPE_BOOLEAN: - writeBooleanColumn(col.getBooleanValues(), valueCount); + writeBooleanColumn(dataAddr, valueCount); break; case TYPE_BYTE: - writeByteColumn(col.getByteValues(), valueCount); + buffer.putBlockOfBytes(dataAddr, valueCount); break; case TYPE_SHORT: case TYPE_CHAR: - writeShortColumn(col.getShortValues(), valueCount); + buffer.putBlockOfBytes(dataAddr, (long) valueCount * 2); break; case TYPE_INT: - writeIntColumn(col.getIntValues(), valueCount); + buffer.putBlockOfBytes(dataAddr, (long) valueCount * 4); break; case TYPE_LONG: - writeLongColumn(col.getLongValues(), valueCount); + buffer.putBlockOfBytes(dataAddr, (long) valueCount * 8); break; case TYPE_FLOAT: - writeFloatColumn(col.getFloatValues(), valueCount); + buffer.putBlockOfBytes(dataAddr, (long) valueCount * 4); break; case TYPE_DOUBLE: - writeDoubleColumn(col.getDoubleValues(), valueCount); + buffer.putBlockOfBytes(dataAddr, (long) valueCount * 8); break; case TYPE_TIMESTAMP: case TYPE_TIMESTAMP_NANOS: - writeTimestampColumn(col.getLongValues(), valueCount, useGorilla); + writeTimestampColumn(dataAddr, valueCount, useGorilla); break; case TYPE_DATE: - writeLongColumn(col.getLongValues(), valueCount); + buffer.putBlockOfBytes(dataAddr, (long) valueCount * 8); break; case TYPE_STRING: case TYPE_VARCHAR: @@ -320,10 +210,12 @@ private void encodeColumn(QwpTableBuffer.ColumnBuffer col, QwpColumnDef colDef, writeSymbolColumn(col, valueCount); break; case TYPE_UUID: - writeUuidColumn(col.getUuidHigh(), col.getUuidLow(), valueCount); + // Stored as lo+hi contiguously, matching wire order + buffer.putBlockOfBytes(dataAddr, (long) valueCount * 16); break; case TYPE_LONG256: - writeLong256Column(col.getLong256Values(), valueCount); + // Stored as 4 contiguous longs per value + buffer.putBlockOfBytes(dataAddr, (long) valueCount * 32); break; case TYPE_DOUBLE_ARRAY: writeDoubleArrayColumn(col, valueCount); @@ -332,77 +224,70 @@ private void encodeColumn(QwpTableBuffer.ColumnBuffer col, QwpColumnDef colDef, writeLongArrayColumn(col, valueCount); break; case TYPE_DECIMAL64: - writeDecimal64Column(col.getDecimalScale(), col.getDecimal64Values(), valueCount); + writeDecimal64Column(col.getDecimalScale(), dataAddr, valueCount); break; case TYPE_DECIMAL128: - writeDecimal128Column(col.getDecimalScale(), col.getDecimal128High(), col.getDecimal128Low(), valueCount); + writeDecimal128Column(col.getDecimalScale(), dataAddr, valueCount); break; case TYPE_DECIMAL256: - writeDecimal256Column(col.getDecimalScale(), - col.getDecimal256Hh(), col.getDecimal256Hl(), - col.getDecimal256Lh(), col.getDecimal256Ll(), valueCount); + writeDecimal256Column(col.getDecimalScale(), dataAddr, valueCount); break; default: throw new IllegalStateException("Unknown column type: " + col.getType()); } } - /** - * Encodes a single column using global symbol IDs for SYMBOL type. - * All other column types are encoded the same as encodeColumn. - */ private void encodeColumnWithGlobalSymbols(QwpTableBuffer.ColumnBuffer col, QwpColumnDef colDef, int rowCount, boolean useGorilla) { int valueCount = col.getValueCount(); - // Write null bitmap if column is nullable if (colDef.isNullable()) { - writeNullBitmapPacked(col.getNullBitmapPacked(), rowCount); + writeNullBitmap(col, rowCount); } - // For symbol columns, use global IDs; for all others, use standard encoding if (col.getType() == TYPE_SYMBOL) { writeSymbolColumnWithGlobalIds(col, valueCount); } else { - // Write column data based on type (same as encodeColumn) + // Delegate to standard encoding for all other types + long dataAddr = col.getDataAddress(); switch (col.getType()) { case TYPE_BOOLEAN: - writeBooleanColumn(col.getBooleanValues(), valueCount); + writeBooleanColumn(dataAddr, valueCount); break; case TYPE_BYTE: - writeByteColumn(col.getByteValues(), valueCount); + buffer.putBlockOfBytes(dataAddr, valueCount); break; case TYPE_SHORT: case TYPE_CHAR: - writeShortColumn(col.getShortValues(), valueCount); + buffer.putBlockOfBytes(dataAddr, (long) valueCount * 2); break; case TYPE_INT: - writeIntColumn(col.getIntValues(), valueCount); + buffer.putBlockOfBytes(dataAddr, (long) valueCount * 4); break; case TYPE_LONG: - writeLongColumn(col.getLongValues(), valueCount); + buffer.putBlockOfBytes(dataAddr, (long) valueCount * 8); break; case TYPE_FLOAT: - writeFloatColumn(col.getFloatValues(), valueCount); + buffer.putBlockOfBytes(dataAddr, (long) valueCount * 4); break; case TYPE_DOUBLE: - writeDoubleColumn(col.getDoubleValues(), valueCount); + buffer.putBlockOfBytes(dataAddr, (long) valueCount * 8); break; case TYPE_TIMESTAMP: case TYPE_TIMESTAMP_NANOS: - writeTimestampColumn(col.getLongValues(), valueCount, useGorilla); + writeTimestampColumn(dataAddr, valueCount, useGorilla); break; case TYPE_DATE: - writeLongColumn(col.getLongValues(), valueCount); + buffer.putBlockOfBytes(dataAddr, (long) valueCount * 8); break; case TYPE_STRING: case TYPE_VARCHAR: writeStringColumn(col.getStringValues(), valueCount); break; case TYPE_UUID: - writeUuidColumn(col.getUuidHigh(), col.getUuidLow(), valueCount); + buffer.putBlockOfBytes(dataAddr, (long) valueCount * 16); break; case TYPE_LONG256: - writeLong256Column(col.getLong256Values(), valueCount); + buffer.putBlockOfBytes(dataAddr, (long) valueCount * 32); break; case TYPE_DOUBLE_ARRAY: writeDoubleArrayColumn(col, valueCount); @@ -411,15 +296,13 @@ private void encodeColumnWithGlobalSymbols(QwpTableBuffer.ColumnBuffer col, QwpC writeLongArrayColumn(col, valueCount); break; case TYPE_DECIMAL64: - writeDecimal64Column(col.getDecimalScale(), col.getDecimal64Values(), valueCount); + writeDecimal64Column(col.getDecimalScale(), dataAddr, valueCount); break; case TYPE_DECIMAL128: - writeDecimal128Column(col.getDecimalScale(), col.getDecimal128High(), col.getDecimal128Low(), valueCount); + writeDecimal128Column(col.getDecimalScale(), dataAddr, valueCount); break; case TYPE_DECIMAL256: - writeDecimal256Column(col.getDecimalScale(), - col.getDecimal256Hh(), col.getDecimal256Hl(), - col.getDecimal256Lh(), col.getDecimal256Ll(), valueCount); + writeDecimal256Column(col.getDecimalScale(), dataAddr, valueCount); break; default: throw new IllegalStateException("Unknown column type: " + col.getType()); @@ -427,9 +310,6 @@ private void encodeColumnWithGlobalSymbols(QwpTableBuffer.ColumnBuffer col, QwpC } } - /** - * Encodes a single table from the buffer. - */ private void encodeTable(QwpTableBuffer tableBuffer, boolean useSchemaRef) { QwpColumnDef[] columnDefs = tableBuffer.getColumnDefs(); int rowCount = tableBuffer.getRowCount(); @@ -445,7 +325,6 @@ private void encodeTable(QwpTableBuffer tableBuffer, boolean useSchemaRef) { writeTableHeaderWithSchema(tableBuffer.getTableName(), rowCount, columnDefs); } - // Write each column's data boolean useGorilla = isGorillaEnabled(); for (int i = 0; i < tableBuffer.getColumnCount(); i++) { QwpTableBuffer.ColumnBuffer col = tableBuffer.getColumn(i); @@ -454,10 +333,6 @@ private void encodeTable(QwpTableBuffer tableBuffer, boolean useSchemaRef) { } } - /** - * Encodes a single table from the buffer using global symbol IDs. - * This is used with delta dictionary encoding. - */ private void encodeTableWithGlobalSymbols(QwpTableBuffer tableBuffer, boolean useSchemaRef) { QwpColumnDef[] columnDefs = tableBuffer.getColumnDefs(); int rowCount = tableBuffer.getRowCount(); @@ -473,7 +348,6 @@ private void encodeTableWithGlobalSymbols(QwpTableBuffer tableBuffer, boolean us writeTableHeaderWithSchema(tableBuffer.getTableName(), rowCount, columnDefs); } - // Write each column's data boolean useGorilla = isGorillaEnabled(); for (int i = 0; i < tableBuffer.getColumnCount(); i++) { QwpTableBuffer.ColumnBuffer col = tableBuffer.getColumn(i); @@ -483,16 +357,16 @@ private void encodeTableWithGlobalSymbols(QwpTableBuffer tableBuffer, boolean us } /** - * Writes boolean column data (bit-packed). + * Writes boolean column data (bit-packed on wire). + * Reads individual bytes from off-heap and packs into bits. */ - private void writeBooleanColumn(boolean[] values, int count) { + private void writeBooleanColumn(long addr, int count) { int packedSize = (count + 7) / 8; - for (int i = 0; i < packedSize; i++) { byte b = 0; for (int bit = 0; bit < 8; bit++) { int idx = i * 8 + bit; - if (idx < count && values[idx]) { + if (idx < count && Unsafe.getUnsafe().getByte(addr + idx) != 0) { b |= (1 << bit); } } @@ -500,34 +374,44 @@ private void writeBooleanColumn(boolean[] values, int count) { } } - private void writeByteColumn(byte[] values, int count) { - for (int i = 0; i < count; i++) { - buffer.putByte(values[i]); - } - } - - private void writeDecimal128Column(byte scale, long[] high, long[] low, int count) { + /** + * Writes Decimal128 values in big-endian wire format. + * Reads hi/lo pairs from off-heap (stored as hi, lo per value). + */ + private void writeDecimal128Column(byte scale, long addr, int count) { buffer.putByte(scale); for (int i = 0; i < count; i++) { - buffer.putLongBE(high[i]); - buffer.putLongBE(low[i]); + long offset = (long) i * 16; + long hi = Unsafe.getUnsafe().getLong(addr + offset); + long lo = Unsafe.getUnsafe().getLong(addr + offset + 8); + buffer.putLongBE(hi); + buffer.putLongBE(lo); } } - private void writeDecimal256Column(byte scale, long[] hh, long[] hl, long[] lh, long[] ll, int count) { + /** + * Writes Decimal256 values in big-endian wire format. + * Reads hh/hl/lh/ll quads from off-heap (stored contiguously per value). + */ + private void writeDecimal256Column(byte scale, long addr, int count) { buffer.putByte(scale); for (int i = 0; i < count; i++) { - buffer.putLongBE(hh[i]); - buffer.putLongBE(hl[i]); - buffer.putLongBE(lh[i]); - buffer.putLongBE(ll[i]); + long offset = (long) i * 32; + buffer.putLongBE(Unsafe.getUnsafe().getLong(addr + offset)); + buffer.putLongBE(Unsafe.getUnsafe().getLong(addr + offset + 8)); + buffer.putLongBE(Unsafe.getUnsafe().getLong(addr + offset + 16)); + buffer.putLongBE(Unsafe.getUnsafe().getLong(addr + offset + 24)); } } - private void writeDecimal64Column(byte scale, long[] values, int count) { + /** + * Writes Decimal64 values in big-endian wire format. + * Reads longs from off-heap. + */ + private void writeDecimal64Column(byte scale, long addr, int count) { buffer.putByte(scale); for (int i = 0; i < count; i++) { - buffer.putLongBE(values[i]); + buffer.putLongBE(Unsafe.getUnsafe().getLong(addr + (long) i * 8)); } } @@ -555,32 +439,6 @@ private void writeDoubleArrayColumn(QwpTableBuffer.ColumnBuffer col, int count) } } - private void writeDoubleColumn(double[] values, int count) { - for (int i = 0; i < count; i++) { - buffer.putDouble(values[i]); - } - } - - private void writeFloatColumn(float[] values, int count) { - for (int i = 0; i < count; i++) { - buffer.putFloat(values[i]); - } - } - - private void writeIntColumn(int[] values, int count) { - for (int i = 0; i < count; i++) { - buffer.putInt(values[i]); - } - } - - private void writeLong256Column(long[] values, int count) { - // Flat array: 4 longs per value, little-endian (least significant first) - // values layout: [long0, long1, long2, long3] per row - for (int i = 0; i < count * 4; i++) { - buffer.putLong(values[i]); - } - } - private void writeLongArrayColumn(QwpTableBuffer.ColumnBuffer col, int count) { byte[] dims = col.getArrayDims(); int[] shapes = col.getArrayShapes(); @@ -605,37 +463,26 @@ private void writeLongArrayColumn(QwpTableBuffer.ColumnBuffer col, int count) { } } - private void writeLongColumn(long[] values, int count) { - for (int i = 0; i < count; i++) { - buffer.putLong(values[i]); - } - } - /** - * Writes a null bitmap from bit-packed long array. + * Writes a null bitmap from off-heap memory. + * On little-endian platforms, the byte layout of the long-packed bitmap + * in memory matches the wire format, enabling bulk copy. */ - private void writeNullBitmapPacked(long[] nullsPacked, int count) { - int bitmapSize = (count + 7) / 8; - - for (int byteIdx = 0; byteIdx < bitmapSize; byteIdx++) { - int longIndex = byteIdx >>> 3; - int byteInLong = byteIdx & 7; - byte b = (byte) ((nullsPacked[longIndex] >>> (byteInLong * 8)) & 0xFF); - buffer.putByte(b); - } - } - - private void writeShortColumn(short[] values, int count) { - for (int i = 0; i < count; i++) { - buffer.putShort(values[i]); + private void writeNullBitmap(QwpTableBuffer.ColumnBuffer col, int rowCount) { + long nullAddr = col.getNullBitmapAddress(); + if (nullAddr != 0) { + int bitmapSize = (rowCount + 7) / 8; + buffer.putBlockOfBytes(nullAddr, bitmapSize); + } else { + // Non-nullable column shouldn't reach here, but write zeros as fallback + int bitmapSize = (rowCount + 7) / 8; + for (int i = 0; i < bitmapSize; i++) { + buffer.putByte((byte) 0); + } } } - /** - * Writes a string column with offset array. - */ private void writeStringColumn(String[] strings, int count) { - // Calculate total data length int totalDataLen = 0; for (int i = 0; i < count; i++) { if (strings[i] != null) { @@ -643,7 +490,6 @@ private void writeStringColumn(String[] strings, int count) { } } - // Write offset array int runningOffset = 0; buffer.putInt(0); for (int i = 0; i < count; i++) { @@ -653,7 +499,6 @@ private void writeStringColumn(String[] strings, int count) { buffer.putInt(runningOffset); } - // Write string data for (int i = 0; i < count; i++) { if (strings[i] != null) { buffer.putUtf8(strings[i]); @@ -663,133 +508,98 @@ private void writeStringColumn(String[] strings, int count) { /** * Writes a symbol column with dictionary. - * Format: - * - Dictionary length (varint) - * - Dictionary entries (length-prefixed UTF-8 strings) - * - Symbol indices (varints, one per value) + * Reads local symbol indices from off-heap data buffer. */ private void writeSymbolColumn(QwpTableBuffer.ColumnBuffer col, int count) { - // Get symbol data from column buffer - int[] symbolIndices = col.getSymbolIndices(); + long dataAddr = col.getDataAddress(); String[] dictionary = col.getSymbolDictionary(); - // Write dictionary buffer.putVarint(dictionary.length); for (String symbol : dictionary) { buffer.putString(symbol); } - // Write symbol indices (one per non-null value) for (int i = 0; i < count; i++) { - buffer.putVarint(symbolIndices[i]); + int idx = Unsafe.getUnsafe().getInt(dataAddr + (long) i * 4); + buffer.putVarint(idx); } } /** * Writes a symbol column using global IDs (for delta dictionary mode). - * Format: - * - Global symbol IDs (varints, one per value) - *

- * The dictionary is not included here because it's written at the message level - * in delta format. + * Reads from auxiliary data buffer if available, otherwise falls back to local indices. */ private void writeSymbolColumnWithGlobalIds(QwpTableBuffer.ColumnBuffer col, int count) { - int[] globalIds = col.getGlobalSymbolIds(); - if (globalIds == null) { - // Fall back to local indices if no global IDs stored - int[] symbolIndices = col.getSymbolIndices(); + long auxAddr = col.getAuxDataAddress(); + if (auxAddr == 0) { + // Fall back to local indices + long dataAddr = col.getDataAddress(); for (int i = 0; i < count; i++) { - buffer.putVarint(symbolIndices[i]); + int idx = Unsafe.getUnsafe().getInt(dataAddr + (long) i * 4); + buffer.putVarint(idx); } } else { - // Write global symbol IDs for (int i = 0; i < count; i++) { - buffer.putVarint(globalIds[i]); + int globalId = Unsafe.getUnsafe().getInt(auxAddr + (long) i * 4); + buffer.putVarint(globalId); } } } - /** - * Writes a table header with full schema. - */ private void writeTableHeaderWithSchema(String tableName, int rowCount, QwpColumnDef[] columns) { - // Table name buffer.putString(tableName); - - // Row count (varint) buffer.putVarint(rowCount); - - // Column count (varint) buffer.putVarint(columns.length); - - // Schema mode: full schema (0x00) buffer.putByte(SCHEMA_MODE_FULL); - - // Column definitions (name + type for each) for (QwpColumnDef col : columns) { buffer.putString(col.getName()); buffer.putByte(col.getWireTypeCode()); } } - /** - * Writes a table header with schema reference. - */ private void writeTableHeaderWithSchemaRef(String tableName, int rowCount, long schemaHash, int columnCount) { - // Table name buffer.putString(tableName); - - // Row count (varint) buffer.putVarint(rowCount); - - // Column count (varint) buffer.putVarint(columnCount); - - // Schema mode: reference (0x01) buffer.putByte(SCHEMA_MODE_REFERENCE); - - // Schema hash (8 bytes) buffer.putLong(schemaHash); } /** * Writes a timestamp column with optional Gorilla compression. - *

- * When Gorilla encoding is enabled and applicable (3+ timestamps with - * delta-of-deltas fitting in 32-bit range), uses delta-of-delta compression. - * Otherwise, falls back to uncompressed encoding. + * Reads longs from off-heap. For Gorilla encoding, creates a temporary + * on-heap array since the Gorilla encoder requires long[]. */ - private void writeTimestampColumn(long[] values, int count, boolean useGorilla) { - if (useGorilla && count > 2 && QwpGorillaEncoder.canUseGorilla(values, count)) { - // Write Gorilla encoding flag - buffer.putByte(ENCODING_GORILLA); - - // Calculate size needed and ensure buffer has capacity - int encodedSize = QwpGorillaEncoder.calculateEncodedSize(values, count); - buffer.ensureCapacity(encodedSize); - - // Encode timestamps to buffer - int bytesWritten = gorillaEncoder.encodeTimestamps( - buffer.getBufferPtr() + buffer.getPosition(), - buffer.getCapacity() - buffer.getPosition(), - values, - count - ); - buffer.skip(bytesWritten); + private void writeTimestampColumn(long addr, int count, boolean useGorilla) { + if (useGorilla && count > 2) { + // Extract to temp array for Gorilla encoder (which requires long[]) + long[] values = new long[count]; + for (int i = 0; i < count; i++) { + values[i] = Unsafe.getUnsafe().getLong(addr + (long) i * 8); + } + + if (QwpGorillaEncoder.canUseGorilla(values, count)) { + buffer.putByte(ENCODING_GORILLA); + int encodedSize = QwpGorillaEncoder.calculateEncodedSize(values, count); + buffer.ensureCapacity(encodedSize); + int bytesWritten = gorillaEncoder.encodeTimestamps( + buffer.getBufferPtr() + buffer.getPosition(), + buffer.getCapacity() - buffer.getPosition(), + values, + count + ); + buffer.skip(bytesWritten); + } else { + buffer.putByte(ENCODING_UNCOMPRESSED); + // Bulk copy for uncompressed path + buffer.putBlockOfBytes(addr, (long) count * 8); + } } else { - // Write uncompressed if (useGorilla) { buffer.putByte(ENCODING_UNCOMPRESSED); } - writeLongColumn(values, count); - } - } - - private void writeUuidColumn(long[] highBits, long[] lowBits, int count) { - // Little-endian: lo first, then hi - for (int i = 0; i < count; i++) { - buffer.putLong(lowBits[i]); - buffer.putLong(highBits[i]); + // Bulk copy for uncompressed timestamps + buffer.putBlockOfBytes(addr, (long) count * 8); } } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index d428dbb..dabda70 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -1420,6 +1420,17 @@ public void close() { client = null; } encoder.close(); + // Close all table buffers to free off-heap column memory + ObjList keys = tableBuffers.keys(); + for (int i = 0, n = keys.size(); i < n; i++) { + CharSequence key = keys.getQuick(i); + if (key != null) { + QwpTableBuffer tb = tableBuffers.get(key); + if (tb != null) { + tb.close(); + } + } + } tableBuffers.clear(); LOG.info("QwpWebSocketSender closed"); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java new file mode 100644 index 0000000..f4c14cc --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java @@ -0,0 +1,161 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.protocol; + +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.QuietCloseable; +import io.questdb.client.std.Unsafe; + +/** + * Lightweight append-only off-heap buffer for columnar data storage. + *

+ * This buffer provides typed append operations (putByte, putShort, etc.) backed by + * native memory allocated via {@link Unsafe}. Memory is tracked under + * {@link MemoryTag#NATIVE_ILP_RSS} for precise accounting. + *

+ * Growth strategy: capacity doubles on each resize via {@link Unsafe#realloc}. + */ +public class OffHeapAppendMemory implements QuietCloseable { + + private static final int DEFAULT_INITIAL_CAPACITY = 128; + + private long pageAddress; + private long appendAddress; + private long capacity; + + public OffHeapAppendMemory() { + this(DEFAULT_INITIAL_CAPACITY); + } + + public OffHeapAppendMemory(long initialCapacity) { + this.capacity = Math.max(initialCapacity, 8); + this.pageAddress = Unsafe.malloc(this.capacity, MemoryTag.NATIVE_ILP_RSS); + this.appendAddress = pageAddress; + } + + /** + * Returns the append offset (number of bytes written). + */ + public long getAppendOffset() { + return appendAddress - pageAddress; + } + + /** + * Returns the base address of the buffer. + */ + public long pageAddress() { + return pageAddress; + } + + /** + * Returns the address at the given byte offset from the start. + */ + public long addressOf(long offset) { + return pageAddress + offset; + } + + /** + * Resets the append position to 0 without freeing memory. + */ + public void truncate() { + appendAddress = pageAddress; + } + + /** + * Sets the append position to the given byte offset. + * Used for truncateTo operations on column buffers. + */ + public void jumpTo(long offset) { + appendAddress = pageAddress + offset; + } + + public void putByte(byte value) { + ensureCapacity(1); + Unsafe.getUnsafe().putByte(appendAddress, value); + appendAddress++; + } + + public void putBoolean(boolean value) { + putByte(value ? (byte) 1 : (byte) 0); + } + + public void putShort(short value) { + ensureCapacity(2); + Unsafe.getUnsafe().putShort(appendAddress, value); + appendAddress += 2; + } + + public void putInt(int value) { + ensureCapacity(4); + Unsafe.getUnsafe().putInt(appendAddress, value); + appendAddress += 4; + } + + public void putLong(long value) { + ensureCapacity(8); + Unsafe.getUnsafe().putLong(appendAddress, value); + appendAddress += 8; + } + + public void putFloat(float value) { + ensureCapacity(4); + Unsafe.getUnsafe().putFloat(appendAddress, value); + appendAddress += 4; + } + + public void putDouble(double value) { + ensureCapacity(8); + Unsafe.getUnsafe().putDouble(appendAddress, value); + appendAddress += 8; + } + + /** + * Advances the append position by the given number of bytes without writing. + */ + public void skip(long bytes) { + ensureCapacity(bytes); + appendAddress += bytes; + } + + @Override + public void close() { + if (pageAddress != 0) { + Unsafe.free(pageAddress, capacity, MemoryTag.NATIVE_ILP_RSS); + pageAddress = 0; + appendAddress = 0; + capacity = 0; + } + } + + private void ensureCapacity(long needed) { + long used = appendAddress - pageAddress; + if (used + needed > capacity) { + long newCapacity = Math.max(capacity * 2, used + needed); + pageAddress = Unsafe.realloc(pageAddress, capacity, newCapacity, MemoryTag.NATIVE_ILP_RSS); + capacity = newCapacity; + appendAddress = pageAddress + used; + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index f70dd98..9e97b4c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -6,7 +6,7 @@ * \__\_\\__,_|\___||___/\__|____/|____/ * * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2024 QuestDB + * Copyright (c) 2019-2026 QuestDB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -33,8 +33,11 @@ import io.questdb.client.std.Decimal256; import io.questdb.client.std.Decimal64; import io.questdb.client.std.Decimals; +import io.questdb.client.std.MemoryTag; import io.questdb.client.std.ObjList; +import io.questdb.client.std.QuietCloseable; import io.questdb.client.std.Unsafe; +import io.questdb.client.std.Vect; import java.util.Arrays; @@ -43,10 +46,11 @@ /** * Buffers rows for a single table in columnar format. *

- * This buffer accumulates row data column by column, allowing efficient - * encoding to the ILP v4 wire format. + * Fixed-width column data is stored off-heap via {@link OffHeapAppendMemory} for zero-GC + * buffering and bulk copy to network buffers. Variable-width data (strings, symbol + * dictionaries, arrays) remains on-heap. */ -public class QwpTableBuffer { +public class QwpTableBuffer implements QuietCloseable { private final String tableName; private final ObjList columns; @@ -70,24 +74,43 @@ public QwpTableBuffer(String tableName) { } /** - * Returns the table name. + * Cancels the current in-progress row. + *

+ * This removes any column values added since the last {@link #nextRow()} call. + * If no values have been added for the current row, this is a no-op. */ - public String getTableName() { - return tableName; + public void cancelCurrentRow() { + // Reset sequential access cursor + columnAccessCursor = 0; + // Truncate each column back to the committed row count + for (int i = 0, n = columns.size(); i < n; i++) { + ColumnBuffer col = fastColumns[i]; + col.truncateTo(rowCount); + } } /** - * Returns the number of rows buffered. + * Clears the buffer completely, including column definitions. + * Frees all off-heap memory. */ - public int getRowCount() { - return rowCount; + public void clear() { + for (int i = 0, n = columns.size(); i < n; i++) { + columns.get(i).close(); + } + columns.clear(); + columnNameToIndex.clear(); + fastColumns = null; + columnAccessCursor = 0; + rowCount = 0; + schemaHash = 0; + schemaHashComputed = false; + columnDefsCacheValid = false; + cachedColumnDefs = null; } - /** - * Returns the number of columns. - */ - public int getColumnCount() { - return columns.size(); + @Override + public void close() { + clear(); } /** @@ -97,6 +120,13 @@ public ColumnBuffer getColumn(int index) { return columns.get(index); } + /** + * Returns the number of columns. + */ + public int getColumnCount() { + return columns.size(); + } + /** * Returns the column definitions (cached for efficiency). */ @@ -167,38 +197,10 @@ public ColumnBuffer getOrCreateColumn(String name, byte type, boolean nullable) } /** - * Advances to the next row. - *

- * This should be called after all column values for the current row have been set. - */ - public void nextRow() { - // Reset sequential access cursor for the next row - columnAccessCursor = 0; - // Ensure all columns have the same row count - for (int i = 0, n = columns.size(); i < n; i++) { - ColumnBuffer col = fastColumns[i]; - // If column wasn't set for this row, add a null - while (col.size < rowCount + 1) { - col.addNull(); - } - } - rowCount++; - } - - /** - * Cancels the current in-progress row. - *

- * This removes any column values added since the last {@link #nextRow()} call. - * If no values have been added for the current row, this is a no-op. + * Returns the number of rows buffered. */ - public void cancelCurrentRow() { - // Reset sequential access cursor - columnAccessCursor = 0; - // Truncate each column back to the committed row count - for (int i = 0, n = columns.size(); i < n; i++) { - ColumnBuffer col = fastColumns[i]; - col.truncateTo(rowCount); - } + public int getRowCount() { + return rowCount; } /** @@ -218,7 +220,33 @@ public long getSchemaHash() { } /** - * Resets the buffer for reuse. + * Returns the table name. + */ + public String getTableName() { + return tableName; + } + + /** + * Advances to the next row. + *

+ * This should be called after all column values for the current row have been set. + */ + public void nextRow() { + // Reset sequential access cursor for the next row + columnAccessCursor = 0; + // Ensure all columns have the same row count + for (int i = 0, n = columns.size(); i < n; i++) { + ColumnBuffer col = fastColumns[i]; + // If column wasn't set for this row, add a null + while (col.size < rowCount + 1) { + col.addNull(); + } + } + rowCount++; + } + + /** + * Resets the buffer for reuse. Keeps column definitions and allocated memory. */ public void reset() { for (int i = 0, n = columns.size(); i < n; i++) { @@ -229,639 +257,228 @@ public void reset() { } /** - * Clears the buffer completely, including column definitions. + * Returns the element size in bytes for a fixed-width column type. + * Returns 0 for variable-width types (string, arrays). */ - public void clear() { - columns.clear(); - columnNameToIndex.clear(); - fastColumns = null; - columnAccessCursor = 0; - rowCount = 0; - schemaHash = 0; - schemaHashComputed = false; - columnDefsCacheValid = false; - cachedColumnDefs = null; + static int elementSize(byte type) { + switch (type) { + case TYPE_BOOLEAN: + case TYPE_BYTE: + return 1; + case TYPE_SHORT: + case TYPE_CHAR: + return 2; + case TYPE_INT: + case TYPE_SYMBOL: + case TYPE_FLOAT: + return 4; + case TYPE_LONG: + case TYPE_TIMESTAMP: + case TYPE_TIMESTAMP_NANOS: + case TYPE_DATE: + case TYPE_DECIMAL64: + case TYPE_DOUBLE: + return 8; + case TYPE_UUID: + case TYPE_DECIMAL128: + return 16; + case TYPE_LONG256: + case TYPE_DECIMAL256: + return 32; + default: + return 0; + } } /** * Column buffer for a single column. + *

+ * Fixed-width data is stored off-heap in {@link OffHeapAppendMemory} for zero-GC + * operation and efficient bulk copy to network buffers. */ - public static class ColumnBuffer { + public static class ColumnBuffer implements QuietCloseable { final String name; final byte type; final boolean nullable; + final int elemSize; private int size; // Total row count (including nulls) private int valueCount; // Actual stored values (excludes nulls) - private int capacity; - - // Storage for different types - private boolean[] booleanValues; - private byte[] byteValues; - private short[] shortValues; - private int[] intValues; - private long[] longValues; - private float[] floatValues; - private double[] doubleValues; - private String[] stringValues; - private long[] uuidHigh; - private long[] uuidLow; - // Long256 stored as flat array: 4 longs per value (avoids inner array allocation) - private long[] long256Values; - // Array storage (double/long arrays - variable length per row) - // Each row stores: [nDims (1B)][dim1..dimN (4B each)][flattened data] - // We track per-row metadata separately from the actual data - private byte[] arrayDims; // nDims per row - private int[] arrayShapes; // Flattened shape data (all dimensions concatenated) - private int arrayShapeOffset; // Current write offset in arrayShapes - private double[] doubleArrayData; // Flattened double values - private long[] longArrayData; // Flattened long values - private int arrayDataOffset; // Current write offset in data arrays - private int arrayRowCapacity; // Capacity for array row count - - // Null tracking - bit-packed for memory efficiency (1 bit per row vs 8 bits with boolean[]) - private long[] nullBitmapPacked; + // Off-heap data buffer for fixed-width types + private OffHeapAppendMemory dataBuffer; + + // Off-heap auxiliary buffer for global symbol IDs (SYMBOL type only) + private OffHeapAppendMemory auxBuffer; + + // Off-heap null bitmap (bit-packed, 1 bit per row) + private long nullBufPtr; + private int nullBufCapRows; private boolean hasNulls; - // Symbol specific + // On-heap capacity for variable-width arrays (string values, array dims) + private int onHeapCapacity; + + // On-heap storage for variable-width types + private String[] stringValues; + + // Array storage (double/long arrays - variable length per row) + private byte[] arrayDims; + private int[] arrayShapes; + private int arrayShapeOffset; + private double[] doubleArrayData; + private long[] longArrayData; + private int arrayDataOffset; + + // Symbol specific (dictionary stays on-heap) private CharSequenceIntHashMap symbolDict; private ObjList symbolList; - private int[] symbolIndices; - - // Global symbol IDs for delta encoding (parallel to symbolIndices) - private int[] globalSymbolIds; private int maxGlobalSymbolId = -1; // Decimal storage - // All values in a decimal column must share the same scale - // For Decimal64: single long per value (64-bit unscaled) - // For Decimal128: two longs per value (128-bit unscaled: high, low) - // For Decimal256: four longs per value (256-bit unscaled: hh, hl, lh, ll) - private byte decimalScale = -1; // Shared scale for column (-1 = not set) - private final Decimal256 rescaleTemp = new Decimal256(); // Reusable temp for rescaling - private long[] decimal64Values; // Decimal64: one long per value - private long[] decimal128High; // Decimal128: high 64 bits - private long[] decimal128Low; // Decimal128: low 64 bits - private long[] decimal256Hh; // Decimal256: bits 255-192 - private long[] decimal256Hl; // Decimal256: bits 191-128 - private long[] decimal256Lh; // Decimal256: bits 127-64 - private long[] decimal256Ll; // Decimal256: bits 63-0 + private byte decimalScale = -1; + private final Decimal256 rescaleTemp = new Decimal256(); public ColumnBuffer(String name, byte type, boolean nullable) { this.name = name; this.type = type; this.nullable = nullable; + this.elemSize = elementSize(type); this.size = 0; this.valueCount = 0; - this.capacity = 16; this.hasNulls = false; + this.onHeapCapacity = 16; - allocateStorage(type, capacity); + allocateStorage(type); if (nullable) { - // Bit-packed: 64 bits per long, so we need (capacity + 63) / 64 longs - nullBitmapPacked = new long[(capacity + 63) >>> 6]; + nullBufCapRows = 64; // multiple of 64 + long sizeBytes = (long) nullBufCapRows >>> 3; + nullBufPtr = Unsafe.calloc(sizeBytes, MemoryTag.NATIVE_ILP_RSS); } } - public String getName() { - return name; - } - - public byte getType() { - return type; - } - - public int getSize() { - return size; - } - - /** - * Returns the number of actual stored values (excludes nulls). - */ - public int getValueCount() { - return valueCount; - } - - public boolean hasNulls() { - return hasNulls; + public void addBoolean(boolean value) { + dataBuffer.putByte(value ? (byte) 1 : (byte) 0); + valueCount++; + size++; } - /** - * Returns the bit-packed null bitmap. - * Each long contains 64 bits, bit 0 of long 0 = row 0, bit 1 of long 0 = row 1, etc. - */ - public long[] getNullBitmapPacked() { - return nullBitmapPacked; + public void addByte(byte value) { + dataBuffer.putByte(value); + valueCount++; + size++; } - /** - * Returns the null bitmap as boolean array (for backward compatibility). - * This creates a new array, so prefer getNullBitmapPacked() for efficiency. - */ - public boolean[] getNullBitmap() { - if (nullBitmapPacked == null) { - return null; + public void addDecimal128(Decimal128 value) { + if (value == null || value.isNull()) { + addNull(); + return; } - boolean[] result = new boolean[size]; - for (int i = 0; i < size; i++) { - result[i] = isNull(i); + if (decimalScale == -1) { + decimalScale = (byte) value.getScale(); + } else if (decimalScale != value.getScale()) { + rescaleTemp.ofRaw(value.getHigh(), value.getLow()); + rescaleTemp.setScale(value.getScale()); + rescaleTemp.rescale(decimalScale); + dataBuffer.putLong(rescaleTemp.getLh()); + dataBuffer.putLong(rescaleTemp.getLl()); + valueCount++; + size++; + return; } - return result; + dataBuffer.putLong(value.getHigh()); + dataBuffer.putLong(value.getLow()); + valueCount++; + size++; } - /** - * Checks if the row at the given index is null. - */ - public boolean isNull(int index) { - if (nullBitmapPacked == null) { - return false; + public void addDecimal256(Decimal256 value) { + if (value == null || value.isNull()) { + addNull(); + return; } - int longIndex = index >>> 6; - int bitIndex = index & 63; - return (nullBitmapPacked[longIndex] & (1L << bitIndex)) != 0; - } - - public boolean[] getBooleanValues() { - return booleanValues; - } - - public byte[] getByteValues() { - return byteValues; - } - - public short[] getShortValues() { - return shortValues; - } - - public int[] getIntValues() { - return intValues; + Decimal256 src = value; + if (decimalScale == -1) { + decimalScale = (byte) value.getScale(); + } else if (decimalScale != value.getScale()) { + rescaleTemp.copyFrom(value); + rescaleTemp.rescale(decimalScale); + src = rescaleTemp; + } + dataBuffer.putLong(src.getHh()); + dataBuffer.putLong(src.getHl()); + dataBuffer.putLong(src.getLh()); + dataBuffer.putLong(src.getLl()); + valueCount++; + size++; } - public long[] getLongValues() { - return longValues; + public void addDecimal64(Decimal64 value) { + if (value == null || value.isNull()) { + addNull(); + return; + } + if (decimalScale == -1) { + decimalScale = (byte) value.getScale(); + dataBuffer.putLong(value.getValue()); + } else if (decimalScale != value.getScale()) { + rescaleTemp.ofRaw(value.getValue()); + rescaleTemp.setScale(value.getScale()); + rescaleTemp.rescale(decimalScale); + dataBuffer.putLong(rescaleTemp.getLl()); + } else { + dataBuffer.putLong(value.getValue()); + } + valueCount++; + size++; } - public float[] getFloatValues() { - return floatValues; + public void addDouble(double value) { + dataBuffer.putDouble(value); + valueCount++; + size++; } - public double[] getDoubleValues() { - return doubleValues; + public void addDoubleArray(double[] values) { + if (values == null) { + addNull(); + return; + } + ensureArrayCapacity(1, values.length); + arrayDims[valueCount] = 1; + arrayShapes[arrayShapeOffset++] = values.length; + for (double v : values) { + doubleArrayData[arrayDataOffset++] = v; + } + valueCount++; + size++; } - public String[] getStringValues() { - return stringValues; + public void addDoubleArray(double[][] values) { + if (values == null) { + addNull(); + return; + } + int dim0 = values.length; + int dim1 = dim0 > 0 ? values[0].length : 0; + for (int i = 1; i < dim0; i++) { + if (values[i].length != dim1) { + throw new LineSenderException("irregular array shape"); + } + } + ensureArrayCapacity(2, dim0 * dim1); + arrayDims[valueCount] = 2; + arrayShapes[arrayShapeOffset++] = dim0; + arrayShapes[arrayShapeOffset++] = dim1; + for (double[] row : values) { + for (double v : row) { + doubleArrayData[arrayDataOffset++] = v; + } + } + valueCount++; + size++; } - public long[] getUuidHigh() { - return uuidHigh; - } - - public long[] getUuidLow() { - return uuidLow; - } - - /** - * Returns Long256 values as flat array (4 longs per value). - * Use getLong256Value(index, component) for indexed access. - */ - public long[] getLong256Values() { - return long256Values; - } - - /** - * Returns a component of a Long256 value. - * @param index value index - * @param component component 0-3 - */ - public long getLong256Value(int index, int component) { - return long256Values[index * 4 + component]; - } - - // ==================== Decimal getters ==================== - - /** - * Returns the shared scale for this decimal column. - * Returns -1 if no values have been added yet. - */ - public byte getDecimalScale() { - return decimalScale; - } - - /** - * Returns the Decimal64 values (one long per value). - */ - public long[] getDecimal64Values() { - return decimal64Values; - } - - /** - * Returns the high 64 bits of Decimal128 values. - */ - public long[] getDecimal128High() { - return decimal128High; - } - - /** - * Returns the low 64 bits of Decimal128 values. - */ - public long[] getDecimal128Low() { - return decimal128Low; - } - - /** - * Returns bits 255-192 of Decimal256 values. - */ - public long[] getDecimal256Hh() { - return decimal256Hh; - } - - /** - * Returns bits 191-128 of Decimal256 values. - */ - public long[] getDecimal256Hl() { - return decimal256Hl; - } - - /** - * Returns bits 127-64 of Decimal256 values. - */ - public long[] getDecimal256Lh() { - return decimal256Lh; - } - - /** - * Returns bits 63-0 of Decimal256 values. - */ - public long[] getDecimal256Ll() { - return decimal256Ll; - } - - /** - * Returns the array dimensions per row (nDims for each row). - */ - public byte[] getArrayDims() { - return arrayDims; - } - - /** - * Returns the flattened array shapes (all dimension lengths concatenated). - */ - public int[] getArrayShapes() { - return arrayShapes; - } - - /** - * Returns the current write offset in arrayShapes. - */ - public int getArrayShapeOffset() { - return arrayShapeOffset; - } - - /** - * Returns the flattened double array data. - */ - public double[] getDoubleArrayData() { - return doubleArrayData; - } - - /** - * Returns the flattened long array data. - */ - public long[] getLongArrayData() { - return longArrayData; - } - - /** - * Returns the current write offset in the data arrays. - */ - public int getArrayDataOffset() { - return arrayDataOffset; - } - - /** - * Returns the symbol indices array (one index per value). - * Each index refers to a position in the symbol dictionary. - */ - public int[] getSymbolIndices() { - return symbolIndices; - } - - /** - * Returns the symbol dictionary as a String array. - * Index i in symbolIndices maps to symbolDictionary[i]. - */ - public String[] getSymbolDictionary() { - if (symbolList == null) { - return new String[0]; - } - String[] dict = new String[symbolList.size()]; - for (int i = 0; i < symbolList.size(); i++) { - dict[i] = symbolList.get(i); - } - return dict; - } - - /** - * Returns the size of the symbol dictionary. - */ - public int getSymbolDictionarySize() { - return symbolList == null ? 0 : symbolList.size(); - } - - /** - * Returns the global symbol IDs array for delta encoding. - * Returns null if no global IDs have been stored. - */ - public int[] getGlobalSymbolIds() { - return globalSymbolIds; - } - - /** - * Returns the maximum global symbol ID used in this column. - * Returns -1 if no symbols have been added with global IDs. - */ - public int getMaxGlobalSymbolId() { - return maxGlobalSymbolId; - } - - public void addBoolean(boolean value) { - ensureCapacity(); - booleanValues[valueCount++] = value; - size++; - } - - public void addByte(byte value) { - ensureCapacity(); - byteValues[valueCount++] = value; - size++; - } - - public void addShort(short value) { - ensureCapacity(); - shortValues[valueCount++] = value; - size++; - } - - public void addInt(int value) { - ensureCapacity(); - intValues[valueCount++] = value; - size++; - } - - public void addLong(long value) { - ensureCapacity(); - longValues[valueCount++] = value; - size++; - } - - public void addFloat(float value) { - ensureCapacity(); - floatValues[valueCount++] = value; - size++; - } - - public void addDouble(double value) { - ensureCapacity(); - doubleValues[valueCount++] = value; - size++; - } - - public void addString(String value) { - ensureCapacity(); - if (value == null && nullable) { - markNull(size); - // Null strings don't take space in the value buffer - size++; - } else { - stringValues[valueCount++] = value; - size++; - } - } - - public void addSymbol(String value) { - ensureCapacity(); - if (value == null) { - if (nullable) { - markNull(size); - } - // Null symbols don't take space in the value buffer - size++; - } else { - int idx = symbolDict.get(value); - if (idx == CharSequenceIntHashMap.NO_ENTRY_VALUE) { - idx = symbolList.size(); - symbolDict.put(value, idx); - symbolList.add(value); - } - symbolIndices[valueCount++] = idx; - size++; - } - } - - /** - * Adds a symbol with both local dictionary and global ID tracking. - * Used for delta dictionary encoding where global IDs are shared across all columns. - * - * @param value the symbol string - * @param globalId the global ID from GlobalSymbolDictionary - */ - public void addSymbolWithGlobalId(String value, int globalId) { - ensureCapacity(); - if (value == null) { - if (nullable) { - markNull(size); - } - size++; - } else { - // Add to local dictionary (for backward compatibility with existing encoder) - int localIdx = symbolDict.get(value); - if (localIdx == CharSequenceIntHashMap.NO_ENTRY_VALUE) { - localIdx = symbolList.size(); - symbolDict.put(value, localIdx); - symbolList.add(value); - } - symbolIndices[valueCount] = localIdx; - - // Also store global ID for delta encoding - if (globalSymbolIds == null) { - globalSymbolIds = new int[capacity]; - } - globalSymbolIds[valueCount] = globalId; - - // Track max global ID for this column - if (globalId > maxGlobalSymbolId) { - maxGlobalSymbolId = globalId; - } - - valueCount++; - size++; - } - } - - public void addUuid(long high, long low) { - ensureCapacity(); - uuidHigh[valueCount] = high; - uuidLow[valueCount] = low; - valueCount++; - size++; - } - - public void addLong256(long l0, long l1, long l2, long l3) { - ensureCapacity(); - int offset = valueCount * 4; - long256Values[offset] = l0; - long256Values[offset + 1] = l1; - long256Values[offset + 2] = l2; - long256Values[offset + 3] = l3; - valueCount++; - size++; - } - - // ==================== Decimal methods ==================== - - /** - * Adds a Decimal64 value. - * If the value's scale differs from the column's established scale, - * the value is automatically rescaled to match. - * - * @param value the Decimal64 value to add - */ - public void addDecimal64(Decimal64 value) { - if (value == null || value.isNull()) { - addNull(); - return; - } - ensureCapacity(); - if (decimalScale == -1) { - decimalScale = (byte) value.getScale(); - decimal64Values[valueCount++] = value.getValue(); - } else if (decimalScale != value.getScale()) { - rescaleTemp.ofRaw(value.getValue()); - rescaleTemp.setScale(value.getScale()); - rescaleTemp.rescale(decimalScale); - decimal64Values[valueCount++] = rescaleTemp.getLl(); - } else { - decimal64Values[valueCount++] = value.getValue(); - } - size++; - } - - /** - * Adds a Decimal128 value. - * If the value's scale differs from the column's established scale, - * the value is automatically rescaled to match. - * - * @param value the Decimal128 value to add - */ - public void addDecimal128(Decimal128 value) { - if (value == null || value.isNull()) { - addNull(); - return; - } - ensureCapacity(); - if (decimalScale == -1) { - decimalScale = (byte) value.getScale(); - } else if (decimalScale != value.getScale()) { - rescaleTemp.ofRaw(value.getHigh(), value.getLow()); - rescaleTemp.setScale(value.getScale()); - rescaleTemp.rescale(decimalScale); - decimal128High[valueCount] = rescaleTemp.getLh(); - decimal128Low[valueCount] = rescaleTemp.getLl(); - valueCount++; - size++; - return; - } - decimal128High[valueCount] = value.getHigh(); - decimal128Low[valueCount] = value.getLow(); - valueCount++; - size++; - } - - /** - * Adds a Decimal256 value. - * If the value's scale differs from the column's established scale, - * the value is automatically rescaled to match. - * - * @param value the Decimal256 value to add - */ - public void addDecimal256(Decimal256 value) { - if (value == null || value.isNull()) { - addNull(); - return; - } - ensureCapacity(); - Decimal256 src = value; - if (decimalScale == -1) { - decimalScale = (byte) value.getScale(); - } else if (decimalScale != value.getScale()) { - rescaleTemp.copyFrom(value); - rescaleTemp.rescale(decimalScale); - src = rescaleTemp; - } - decimal256Hh[valueCount] = src.getHh(); - decimal256Hl[valueCount] = src.getHl(); - decimal256Lh[valueCount] = src.getLh(); - decimal256Ll[valueCount] = src.getLl(); - valueCount++; - size++; - } - - // ==================== Array methods ==================== - - /** - * Adds a 1D double array. - */ - public void addDoubleArray(double[] values) { - if (values == null) { - addNull(); - return; - } - ensureArrayCapacity(1, values.length); - arrayDims[valueCount] = 1; - arrayShapes[arrayShapeOffset++] = values.length; - for (double v : values) { - doubleArrayData[arrayDataOffset++] = v; - } - valueCount++; - size++; - } - - /** - * Adds a 2D double array. - * @throws LineSenderException if the array is jagged (irregular shape) - */ - public void addDoubleArray(double[][] values) { - if (values == null) { - addNull(); - return; - } - int dim0 = values.length; - int dim1 = dim0 > 0 ? values[0].length : 0; - // Validate rectangular shape - for (int i = 1; i < dim0; i++) { - if (values[i].length != dim1) { - throw new LineSenderException("irregular array shape"); - } - } - ensureArrayCapacity(2, dim0 * dim1); - arrayDims[valueCount] = 2; - arrayShapes[arrayShapeOffset++] = dim0; - arrayShapes[arrayShapeOffset++] = dim1; - for (double[] row : values) { - for (double v : row) { - doubleArrayData[arrayDataOffset++] = v; - } - } - valueCount++; - size++; - } - - /** - * Adds a 3D double array. - * @throws LineSenderException if the array is jagged (irregular shape) - */ public void addDoubleArray(double[][][] values) { if (values == null) { addNull(); @@ -870,7 +487,6 @@ public void addDoubleArray(double[][][] values) { int dim0 = values.length; int dim1 = dim0 > 0 ? values[0].length : 0; int dim2 = dim0 > 0 && dim1 > 0 ? values[0][0].length : 0; - // Validate rectangular shape for (int i = 0; i < dim0; i++) { if (values[i].length != dim1) { throw new LineSenderException("irregular array shape"); @@ -897,16 +513,11 @@ public void addDoubleArray(double[][][] values) { size++; } - /** - * Adds a DoubleArray (N-dimensional wrapper). - * Uses a capturing approach to extract shape and data. - */ public void addDoubleArray(DoubleArray array) { if (array == null) { addNull(); return; } - // Use a capturing ArrayBufferAppender to extract the data ArrayCapture capture = new ArrayCapture(); array.appendToBufPtr(capture); @@ -922,9 +533,33 @@ public void addDoubleArray(DoubleArray array) { size++; } - /** - * Adds a 1D long array. - */ + public void addFloat(float value) { + dataBuffer.putFloat(value); + valueCount++; + size++; + } + + public void addInt(int value) { + dataBuffer.putInt(value); + valueCount++; + size++; + } + + public void addLong(long value) { + dataBuffer.putLong(value); + valueCount++; + size++; + } + + public void addLong256(long l0, long l1, long l2, long l3) { + dataBuffer.putLong(l0); + dataBuffer.putLong(l1); + dataBuffer.putLong(l2); + dataBuffer.putLong(l3); + valueCount++; + size++; + } + public void addLongArray(long[] values) { if (values == null) { addNull(); @@ -940,10 +575,6 @@ public void addLongArray(long[] values) { size++; } - /** - * Adds a 2D long array. - * @throws LineSenderException if the array is jagged (irregular shape) - */ public void addLongArray(long[][] values) { if (values == null) { addNull(); @@ -951,7 +582,6 @@ public void addLongArray(long[][] values) { } int dim0 = values.length; int dim1 = dim0 > 0 ? values[0].length : 0; - // Validate rectangular shape for (int i = 1; i < dim0; i++) { if (values[i].length != dim1) { throw new LineSenderException("irregular array shape"); @@ -970,10 +600,6 @@ public void addLongArray(long[][] values) { size++; } - /** - * Adds a 3D long array. - * @throws LineSenderException if the array is jagged (irregular shape) - */ public void addLongArray(long[][][] values) { if (values == null) { addNull(); @@ -982,7 +608,6 @@ public void addLongArray(long[][][] values) { int dim0 = values.length; int dim1 = dim0 > 0 ? values[0].length : 0; int dim2 = dim0 > 0 && dim1 > 0 ? values[0][0].length : 0; - // Validate rectangular shape for (int i = 0; i < dim0; i++) { if (values[i].length != dim1) { throw new LineSenderException("irregular array shape"); @@ -1009,199 +634,360 @@ public void addLongArray(long[][][] values) { size++; } + public void addLongArray(LongArray array) { + if (array == null) { + addNull(); + return; + } + ArrayCapture capture = new ArrayCapture(); + array.appendToBufPtr(capture); + + ensureArrayCapacity(capture.nDims, capture.longDataOffset); + arrayDims[valueCount] = capture.nDims; + for (int i = 0; i < capture.nDims; i++) { + arrayShapes[arrayShapeOffset++] = capture.shape[i]; + } + for (int i = 0; i < capture.longDataOffset; i++) { + longArrayData[arrayDataOffset++] = capture.longData[i]; + } + valueCount++; + size++; + } + + public void addNull() { + if (nullable) { + ensureNullCapacity(size + 1); + markNull(size); + size++; + } else { + // For non-nullable columns, store a sentinel/default value + switch (type) { + case TYPE_BOOLEAN: + dataBuffer.putByte((byte) 0); + break; + case TYPE_BYTE: + dataBuffer.putByte((byte) 0); + break; + case TYPE_SHORT: + case TYPE_CHAR: + dataBuffer.putShort((short) 0); + break; + case TYPE_INT: + dataBuffer.putInt(0); + break; + case TYPE_LONG: + case TYPE_TIMESTAMP: + case TYPE_TIMESTAMP_NANOS: + case TYPE_DATE: + dataBuffer.putLong(Long.MIN_VALUE); + break; + case TYPE_FLOAT: + dataBuffer.putFloat(Float.NaN); + break; + case TYPE_DOUBLE: + dataBuffer.putDouble(Double.NaN); + break; + case TYPE_STRING: + case TYPE_VARCHAR: + ensureOnHeapCapacity(); + stringValues[valueCount] = null; + break; + case TYPE_SYMBOL: + dataBuffer.putInt(-1); + break; + case TYPE_UUID: + dataBuffer.putLong(Long.MIN_VALUE); + dataBuffer.putLong(Long.MIN_VALUE); + break; + case TYPE_LONG256: + dataBuffer.putLong(Long.MIN_VALUE); + dataBuffer.putLong(Long.MIN_VALUE); + dataBuffer.putLong(Long.MIN_VALUE); + dataBuffer.putLong(Long.MIN_VALUE); + break; + case TYPE_DECIMAL64: + dataBuffer.putLong(Decimals.DECIMAL64_NULL); + break; + case TYPE_DECIMAL128: + dataBuffer.putLong(Decimals.DECIMAL128_HI_NULL); + dataBuffer.putLong(Decimals.DECIMAL128_LO_NULL); + break; + case TYPE_DECIMAL256: + dataBuffer.putLong(Decimals.DECIMAL256_HH_NULL); + dataBuffer.putLong(Decimals.DECIMAL256_HL_NULL); + dataBuffer.putLong(Decimals.DECIMAL256_LH_NULL); + dataBuffer.putLong(Decimals.DECIMAL256_LL_NULL); + break; + } + valueCount++; + size++; + } + } + + public void addShort(short value) { + dataBuffer.putShort(value); + valueCount++; + size++; + } + + public void addString(String value) { + if (value == null && nullable) { + ensureNullCapacity(size + 1); + markNull(size); + size++; + } else { + ensureOnHeapCapacity(); + stringValues[valueCount++] = value; + size++; + } + } + + public void addSymbol(String value) { + if (value == null) { + if (nullable) { + ensureNullCapacity(size + 1); + markNull(size); + } + size++; + } else { + int idx = symbolDict.get(value); + if (idx == CharSequenceIntHashMap.NO_ENTRY_VALUE) { + idx = symbolList.size(); + symbolDict.put(value, idx); + symbolList.add(value); + } + dataBuffer.putInt(idx); + valueCount++; + size++; + } + } + + public void addSymbolWithGlobalId(String value, int globalId) { + if (value == null) { + if (nullable) { + ensureNullCapacity(size + 1); + markNull(size); + } + size++; + } else { + int localIdx = symbolDict.get(value); + if (localIdx == CharSequenceIntHashMap.NO_ENTRY_VALUE) { + localIdx = symbolList.size(); + symbolDict.put(value, localIdx); + symbolList.add(value); + } + dataBuffer.putInt(localIdx); + + if (auxBuffer == null) { + auxBuffer = new OffHeapAppendMemory(64); + } + auxBuffer.putInt(globalId); + + if (globalId > maxGlobalSymbolId) { + maxGlobalSymbolId = globalId; + } + + valueCount++; + size++; + } + } + + public void addUuid(long high, long low) { + // Store in wire order: lo first, hi second + dataBuffer.putLong(low); + dataBuffer.putLong(high); + valueCount++; + size++; + } + + @Override + public void close() { + if (dataBuffer != null) { + dataBuffer.close(); + dataBuffer = null; + } + if (auxBuffer != null) { + auxBuffer.close(); + auxBuffer = null; + } + if (nullBufPtr != 0) { + Unsafe.free(nullBufPtr, (long) nullBufCapRows >>> 3, MemoryTag.NATIVE_ILP_RSS); + nullBufPtr = 0; + nullBufCapRows = 0; + } + } + + public int getArrayDataOffset() { + return arrayDataOffset; + } + + public byte[] getArrayDims() { + return arrayDims; + } + + public int[] getArrayShapes() { + return arrayShapes; + } + + public int getArrayShapeOffset() { + return arrayShapeOffset; + } + + /** + * Returns the off-heap address of the auxiliary data buffer (global symbol IDs). + * Returns 0 if no auxiliary data exists. + */ + public long getAuxDataAddress() { + return auxBuffer != null ? auxBuffer.pageAddress() : 0; + } + + /** + * Returns the off-heap address of the column data buffer. + */ + public long getDataAddress() { + return dataBuffer != null ? dataBuffer.pageAddress() : 0; + } + + /** + * Returns the number of bytes of data in the off-heap buffer. + */ + public long getDataSize() { + return dataBuffer != null ? dataBuffer.getAppendOffset() : 0; + } + + public byte getDecimalScale() { + return decimalScale; + } + + public double[] getDoubleArrayData() { + return doubleArrayData; + } + + public long[] getLongArrayData() { + return longArrayData; + } + + public int getMaxGlobalSymbolId() { + return maxGlobalSymbolId; + } + + public String getName() { + return name; + } + /** - * Adds a LongArray (N-dimensional wrapper). - * Uses a capturing approach to extract shape and data. + * Returns the off-heap address of the null bitmap. + * Returns 0 for non-nullable columns. */ - public void addLongArray(LongArray array) { - if (array == null) { - addNull(); - return; - } - // Use a capturing ArrayBufferAppender to extract the data - ArrayCapture capture = new ArrayCapture(); - array.appendToBufPtr(capture); + public long getNullBitmapAddress() { + return nullBufPtr; + } - ensureArrayCapacity(capture.nDims, capture.longDataOffset); - arrayDims[valueCount] = capture.nDims; - for (int i = 0; i < capture.nDims; i++) { - arrayShapes[arrayShapeOffset++] = capture.shape[i]; + /** + * Returns the bit-packed null bitmap as a long array. + * This creates a new array from off-heap data. + */ + public long[] getNullBitmapPacked() { + if (nullBufPtr == 0) { + return null; } - for (int i = 0; i < capture.longDataOffset; i++) { - longArrayData[arrayDataOffset++] = capture.longData[i]; + int longCount = (size + 63) >>> 6; + long[] result = new long[longCount]; + for (int i = 0; i < longCount; i++) { + result[i] = Unsafe.getUnsafe().getLong(nullBufPtr + (long) i * 8); } - valueCount++; - size++; + return result; } - /** - * Ensures capacity for array storage. - * @param nDims number of dimensions for this array - * @param dataElements number of data elements - */ - private void ensureArrayCapacity(int nDims, int dataElements) { - ensureCapacity(); // For row-level capacity (arrayDims uses valueCount) + public int getSize() { + return size; + } - // Ensure shape array capacity - int requiredShapeCapacity = arrayShapeOffset + nDims; - if (arrayShapes == null) { - arrayShapes = new int[Math.max(64, requiredShapeCapacity)]; - } else if (requiredShapeCapacity > arrayShapes.length) { - arrayShapes = Arrays.copyOf(arrayShapes, Math.max(arrayShapes.length * 2, requiredShapeCapacity)); - } + public String[] getStringValues() { + return stringValues; + } - // Ensure data array capacity - int requiredDataCapacity = arrayDataOffset + dataElements; - if (type == TYPE_DOUBLE_ARRAY) { - if (doubleArrayData == null) { - doubleArrayData = new double[Math.max(256, requiredDataCapacity)]; - } else if (requiredDataCapacity > doubleArrayData.length) { - doubleArrayData = Arrays.copyOf(doubleArrayData, Math.max(doubleArrayData.length * 2, requiredDataCapacity)); - } - } else if (type == TYPE_LONG_ARRAY) { - if (longArrayData == null) { - longArrayData = new long[Math.max(256, requiredDataCapacity)]; - } else if (requiredDataCapacity > longArrayData.length) { - longArrayData = Arrays.copyOf(longArrayData, Math.max(longArrayData.length * 2, requiredDataCapacity)); - } + public String[] getSymbolDictionary() { + if (symbolList == null) { + return new String[0]; } + String[] dict = new String[symbolList.size()]; + for (int i = 0; i < symbolList.size(); i++) { + dict[i] = symbolList.get(i); + } + return dict; } - public void addNull() { - ensureCapacity(); - if (nullable) { - // For nullable columns, mark null in bitmap but don't store a value - markNull(size); - size++; - } else { - // For non-nullable columns, we must store a sentinel/default value - // because no null bitmap will be written - switch (type) { - case TYPE_BOOLEAN: - booleanValues[valueCount++] = false; - break; - case TYPE_BYTE: - byteValues[valueCount++] = 0; - break; - case TYPE_SHORT: - case TYPE_CHAR: - shortValues[valueCount++] = 0; - break; - case TYPE_INT: - intValues[valueCount++] = 0; - break; - case TYPE_LONG: - case TYPE_TIMESTAMP: - case TYPE_TIMESTAMP_NANOS: - case TYPE_DATE: - longValues[valueCount++] = Long.MIN_VALUE; - break; - case TYPE_FLOAT: - floatValues[valueCount++] = Float.NaN; - break; - case TYPE_DOUBLE: - doubleValues[valueCount++] = Double.NaN; - break; - case TYPE_STRING: - case TYPE_VARCHAR: - stringValues[valueCount++] = null; - break; - case TYPE_SYMBOL: - symbolIndices[valueCount++] = -1; - break; - case TYPE_UUID: - uuidHigh[valueCount] = Long.MIN_VALUE; - uuidLow[valueCount] = Long.MIN_VALUE; - valueCount++; - break; - case TYPE_LONG256: - int offset = valueCount * 4; - long256Values[offset] = Long.MIN_VALUE; - long256Values[offset + 1] = Long.MIN_VALUE; - long256Values[offset + 2] = Long.MIN_VALUE; - long256Values[offset + 3] = Long.MIN_VALUE; - valueCount++; - break; - case TYPE_DECIMAL64: - decimal64Values[valueCount++] = Decimals.DECIMAL64_NULL; - break; - case TYPE_DECIMAL128: - decimal128High[valueCount] = Decimals.DECIMAL128_HI_NULL; - decimal128Low[valueCount] = Decimals.DECIMAL128_LO_NULL; - valueCount++; - break; - case TYPE_DECIMAL256: - decimal256Hh[valueCount] = Decimals.DECIMAL256_HH_NULL; - decimal256Hl[valueCount] = Decimals.DECIMAL256_HL_NULL; - decimal256Lh[valueCount] = Decimals.DECIMAL256_LH_NULL; - decimal256Ll[valueCount] = Decimals.DECIMAL256_LL_NULL; - valueCount++; - break; - } - size++; - } + public int getSymbolDictionarySize() { + return symbolList == null ? 0 : symbolList.size(); } - private void markNull(int index) { - int longIndex = index >>> 6; + public byte getType() { + return type; + } + + public int getValueCount() { + return valueCount; + } + + public boolean hasNulls() { + return hasNulls; + } + + public boolean isNull(int index) { + if (nullBufPtr == 0) { + return false; + } + long longAddr = nullBufPtr + ((long) (index >>> 6)) * 8; int bitIndex = index & 63; - nullBitmapPacked[longIndex] |= (1L << bitIndex); - hasNulls = true; + return (Unsafe.getUnsafe().getLong(longAddr) & (1L << bitIndex)) != 0; } public void reset() { size = 0; valueCount = 0; hasNulls = false; - if (nullBitmapPacked != null) { - Arrays.fill(nullBitmapPacked, 0L); + if (dataBuffer != null) { + dataBuffer.truncate(); + } + if (auxBuffer != null) { + auxBuffer.truncate(); + } + if (nullBufPtr != 0) { + Vect.memset(nullBufPtr, (long) nullBufCapRows >>> 3, 0); } if (symbolDict != null) { symbolDict.clear(); symbolList.clear(); } - // Reset global symbol tracking maxGlobalSymbolId = -1; - // Reset array tracking arrayShapeOffset = 0; arrayDataOffset = 0; - // Reset decimal scale (will be set by first non-null value) decimalScale = -1; } - /** - * Truncates the column to the specified size. - * This is used to cancel uncommitted row values. - * - * @param newSize the target size (number of rows) - */ public void truncateTo(int newSize) { if (newSize >= size) { - return; // Nothing to truncate + return; } - // Count non-null values up to newSize int newValueCount = 0; - if (nullable && nullBitmapPacked != null) { + if (nullable && nullBufPtr != 0) { for (int i = 0; i < newSize; i++) { - int longIndex = i >>> 6; - int bitIndex = i & 63; - if ((nullBitmapPacked[longIndex] & (1L << bitIndex)) == 0) { + if (!isNull(i)) { newValueCount++; } } // Clear null bits for truncated rows for (int i = newSize; i < size; i++) { - int longIndex = i >>> 6; + long longAddr = nullBufPtr + ((long) (i >>> 6)) * 8; int bitIndex = i & 63; - nullBitmapPacked[longIndex] &= ~(1L << bitIndex); + long current = Unsafe.getUnsafe().getLong(longAddr); + Unsafe.getUnsafe().putLong(longAddr, current & ~(1L << bitIndex)); } - // Recompute hasNulls hasNulls = false; for (int i = 0; i < newSize && !hasNulls; i++) { - int longIndex = i >>> 6; - int bitIndex = i & 63; - if ((nullBitmapPacked[longIndex] & (1L << bitIndex)) != 0) { + if (isNull(i)) { hasNulls = true; } } @@ -1211,223 +997,197 @@ public void truncateTo(int newSize) { size = newSize; valueCount = newValueCount; - } - private void ensureCapacity() { - if (size >= capacity) { - int newCapacity = capacity * 2; - growStorage(type, newCapacity); - if (nullable && nullBitmapPacked != null) { - int newLongCount = (newCapacity + 63) >>> 6; - nullBitmapPacked = Arrays.copyOf(nullBitmapPacked, newLongCount); - } - capacity = newCapacity; + // Rewind off-heap data buffer + if (dataBuffer != null && elemSize > 0) { + dataBuffer.jumpTo((long) newValueCount * elemSize); + } + + // Rewind aux buffer (symbol global IDs) + if (auxBuffer != null) { + auxBuffer.jumpTo((long) newValueCount * 4); } } - private void allocateStorage(byte type, int cap) { + private void allocateStorage(byte type) { switch (type) { case TYPE_BOOLEAN: - booleanValues = new boolean[cap]; - break; case TYPE_BYTE: - byteValues = new byte[cap]; + dataBuffer = new OffHeapAppendMemory(16); break; case TYPE_SHORT: case TYPE_CHAR: - shortValues = new short[cap]; + dataBuffer = new OffHeapAppendMemory(32); break; case TYPE_INT: - intValues = new int[cap]; + dataBuffer = new OffHeapAppendMemory(64); break; case TYPE_LONG: case TYPE_TIMESTAMP: case TYPE_TIMESTAMP_NANOS: case TYPE_DATE: - longValues = new long[cap]; + dataBuffer = new OffHeapAppendMemory(128); break; case TYPE_FLOAT: - floatValues = new float[cap]; + dataBuffer = new OffHeapAppendMemory(64); break; case TYPE_DOUBLE: - doubleValues = new double[cap]; + dataBuffer = new OffHeapAppendMemory(128); break; case TYPE_STRING: case TYPE_VARCHAR: - stringValues = new String[cap]; + stringValues = new String[onHeapCapacity]; break; case TYPE_SYMBOL: - symbolIndices = new int[cap]; + dataBuffer = new OffHeapAppendMemory(64); symbolDict = new CharSequenceIntHashMap(); symbolList = new ObjList<>(); break; case TYPE_UUID: - uuidHigh = new long[cap]; - uuidLow = new long[cap]; + dataBuffer = new OffHeapAppendMemory(256); break; case TYPE_LONG256: - // Flat array: 4 longs per value - long256Values = new long[cap * 4]; + dataBuffer = new OffHeapAppendMemory(512); break; case TYPE_DOUBLE_ARRAY: case TYPE_LONG_ARRAY: - // Array types: allocate per-row tracking - // Shape and data arrays are grown dynamically in ensureArrayCapacity() - arrayDims = new byte[cap]; - arrayRowCapacity = cap; + arrayDims = new byte[onHeapCapacity]; break; case TYPE_DECIMAL64: - decimal64Values = new long[cap]; + dataBuffer = new OffHeapAppendMemory(128); break; case TYPE_DECIMAL128: - decimal128High = new long[cap]; - decimal128Low = new long[cap]; + dataBuffer = new OffHeapAppendMemory(256); break; case TYPE_DECIMAL256: - decimal256Hh = new long[cap]; - decimal256Hl = new long[cap]; - decimal256Lh = new long[cap]; - decimal256Ll = new long[cap]; + dataBuffer = new OffHeapAppendMemory(512); break; } } - private void growStorage(byte type, int newCap) { - switch (type) { - case TYPE_BOOLEAN: - booleanValues = Arrays.copyOf(booleanValues, newCap); - break; - case TYPE_BYTE: - byteValues = Arrays.copyOf(byteValues, newCap); - break; - case TYPE_SHORT: - case TYPE_CHAR: - shortValues = Arrays.copyOf(shortValues, newCap); - break; - case TYPE_INT: - intValues = Arrays.copyOf(intValues, newCap); - break; - case TYPE_LONG: - case TYPE_TIMESTAMP: - case TYPE_TIMESTAMP_NANOS: - case TYPE_DATE: - longValues = Arrays.copyOf(longValues, newCap); - break; - case TYPE_FLOAT: - floatValues = Arrays.copyOf(floatValues, newCap); - break; - case TYPE_DOUBLE: - doubleValues = Arrays.copyOf(doubleValues, newCap); - break; - case TYPE_STRING: - case TYPE_VARCHAR: - stringValues = Arrays.copyOf(stringValues, newCap); - break; - case TYPE_SYMBOL: - symbolIndices = Arrays.copyOf(symbolIndices, newCap); - if (globalSymbolIds != null) { - globalSymbolIds = Arrays.copyOf(globalSymbolIds, newCap); - } - break; - case TYPE_UUID: - uuidHigh = Arrays.copyOf(uuidHigh, newCap); - uuidLow = Arrays.copyOf(uuidLow, newCap); - break; - case TYPE_LONG256: - // Flat array: 4 longs per value - long256Values = Arrays.copyOf(long256Values, newCap * 4); - break; - case TYPE_DOUBLE_ARRAY: - case TYPE_LONG_ARRAY: - // Array types: grow per-row tracking - arrayDims = Arrays.copyOf(arrayDims, newCap); - arrayRowCapacity = newCap; - // Note: shapes and data arrays are grown in ensureArrayCapacity() - break; - case TYPE_DECIMAL64: - decimal64Values = Arrays.copyOf(decimal64Values, newCap); - break; - case TYPE_DECIMAL128: - decimal128High = Arrays.copyOf(decimal128High, newCap); - decimal128Low = Arrays.copyOf(decimal128Low, newCap); - break; - case TYPE_DECIMAL256: - decimal256Hh = Arrays.copyOf(decimal256Hh, newCap); - decimal256Hl = Arrays.copyOf(decimal256Hl, newCap); - decimal256Lh = Arrays.copyOf(decimal256Lh, newCap); - decimal256Ll = Arrays.copyOf(decimal256Ll, newCap); - break; + private void ensureArrayCapacity(int nDims, int dataElements) { + // Ensure per-row array dims capacity + if (valueCount >= arrayDims.length) { + arrayDims = Arrays.copyOf(arrayDims, arrayDims.length * 2); + } + + // Ensure null bitmap capacity + if (nullable) { + ensureNullCapacity(size + 1); + } + + // Ensure shape array capacity + int requiredShapeCapacity = arrayShapeOffset + nDims; + if (arrayShapes == null) { + arrayShapes = new int[Math.max(64, requiredShapeCapacity)]; + } else if (requiredShapeCapacity > arrayShapes.length) { + arrayShapes = Arrays.copyOf(arrayShapes, Math.max(arrayShapes.length * 2, requiredShapeCapacity)); + } + + // Ensure data array capacity + int requiredDataCapacity = arrayDataOffset + dataElements; + if (type == TYPE_DOUBLE_ARRAY) { + if (doubleArrayData == null) { + doubleArrayData = new double[Math.max(256, requiredDataCapacity)]; + } else if (requiredDataCapacity > doubleArrayData.length) { + doubleArrayData = Arrays.copyOf(doubleArrayData, Math.max(doubleArrayData.length * 2, requiredDataCapacity)); + } + } else if (type == TYPE_LONG_ARRAY) { + if (longArrayData == null) { + longArrayData = new long[Math.max(256, requiredDataCapacity)]; + } else if (requiredDataCapacity > longArrayData.length) { + longArrayData = Arrays.copyOf(longArrayData, Math.max(longArrayData.length * 2, requiredDataCapacity)); + } } } + + private void ensureNullCapacity(int rows) { + if (rows > nullBufCapRows) { + int newCapRows = Math.max(nullBufCapRows * 2, ((rows + 63) >>> 6) << 6); + long newSizeBytes = (long) newCapRows >>> 3; + long oldSizeBytes = (long) nullBufCapRows >>> 3; + nullBufPtr = Unsafe.realloc(nullBufPtr, oldSizeBytes, newSizeBytes, MemoryTag.NATIVE_ILP_RSS); + Vect.memset(nullBufPtr + oldSizeBytes, newSizeBytes - oldSizeBytes, 0); + nullBufCapRows = newCapRows; + } + } + + private void ensureOnHeapCapacity() { + if (valueCount >= onHeapCapacity) { + int newCapacity = onHeapCapacity * 2; + if (stringValues != null) { + stringValues = Arrays.copyOf(stringValues, newCapacity); + } + onHeapCapacity = newCapacity; + } + } + + private void markNull(int index) { + long longAddr = nullBufPtr + ((long) (index >>> 6)) * 8; + int bitIndex = index & 63; + long current = Unsafe.getUnsafe().getLong(longAddr); + Unsafe.getUnsafe().putLong(longAddr, current | (1L << bitIndex)); + hasNulls = true; + } } /** * Helper class to capture array data from DoubleArray/LongArray.appendToBufPtr(). - * This implements ArrayBufferAppender to intercept the serialization and extract - * shape and data into Java arrays for storage in ColumnBuffer. */ private static class ArrayCapture implements ArrayBufferAppender { byte nDims; - int[] shape = new int[32]; // Max 32 dimensions + int[] shape = new int[32]; int shapeIndex; double[] doubleData; int doubleDataOffset; long[] longData; int longDataOffset; + @Override + public void putBlockOfBytes(long from, long len) { + int count = (int) (len / 8); + if (doubleData == null) { + doubleData = new double[count]; + } + for (int i = 0; i < count; i++) { + doubleData[doubleDataOffset++] = Unsafe.getUnsafe().getDouble(from + i * 8L); + } + } + @Override public void putByte(byte b) { if (shapeIndex == 0) { - // First byte is nDims nDims = b; } } + @Override + public void putDouble(double value) { + if (doubleData != null && doubleDataOffset < doubleData.length) { + doubleData[doubleDataOffset++] = value; + } + } + @Override public void putInt(int value) { - // Shape dimensions if (shapeIndex < nDims) { shape[shapeIndex++] = value; - // Once we have all dimensions, compute total elements and allocate data array if (shapeIndex == nDims) { int totalElements = 1; for (int i = 0; i < nDims; i++) { totalElements *= shape[i]; } - // Allocate both - only one will be used doubleData = new double[totalElements]; longData = new long[totalElements]; } } } - @Override - public void putDouble(double value) { - if (doubleData != null && doubleDataOffset < doubleData.length) { - doubleData[doubleDataOffset++] = value; - } - } - @Override public void putLong(long value) { if (longData != null && longDataOffset < longData.length) { longData[longDataOffset++] = value; } } - - @Override - public void putBlockOfBytes(long from, long len) { - // This is the bulk data from the array - // The AbstractArray uses this to copy raw bytes - // We need to figure out if it's doubles or longs based on context - // For now, assume doubles (8 bytes each) since DoubleArray uses this - int count = (int) (len / 8); - if (doubleData == null) { - doubleData = new double[count]; - } - for (int i = 0; i < count; i++) { - doubleData[doubleDataOffset++] = Unsafe.getUnsafe().getDouble(from + i * 8L); - } - } } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/OffHeapAppendMemoryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/OffHeapAppendMemoryTest.java new file mode 100644 index 0000000..96c39a3 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/OffHeapAppendMemoryTest.java @@ -0,0 +1,266 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.cutlass.qwp.protocol.OffHeapAppendMemory; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import org.junit.Test; + +import static org.junit.Assert.*; + +public class OffHeapAppendMemoryTest { + + @Test + public void testPutAndReadByte() { + long before = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putByte((byte) 42); + mem.putByte((byte) -1); + mem.putByte((byte) 0); + + assertEquals(3, mem.getAppendOffset()); + assertEquals(42, Unsafe.getUnsafe().getByte(mem.addressOf(0))); + assertEquals(-1, Unsafe.getUnsafe().getByte(mem.addressOf(1))); + assertEquals(0, Unsafe.getUnsafe().getByte(mem.addressOf(2))); + } + long after = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); + assertEquals(before, after); + } + + @Test + public void testPutAndReadShort() { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putShort((short) 12_345); + mem.putShort(Short.MIN_VALUE); + mem.putShort(Short.MAX_VALUE); + + assertEquals(6, mem.getAppendOffset()); + assertEquals(12_345, Unsafe.getUnsafe().getShort(mem.addressOf(0))); + assertEquals(Short.MIN_VALUE, Unsafe.getUnsafe().getShort(mem.addressOf(2))); + assertEquals(Short.MAX_VALUE, Unsafe.getUnsafe().getShort(mem.addressOf(4))); + } + } + + @Test + public void testPutAndReadInt() { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putInt(100_000); + mem.putInt(Integer.MIN_VALUE); + + assertEquals(8, mem.getAppendOffset()); + assertEquals(100_000, Unsafe.getUnsafe().getInt(mem.addressOf(0))); + assertEquals(Integer.MIN_VALUE, Unsafe.getUnsafe().getInt(mem.addressOf(4))); + } + } + + @Test + public void testPutAndReadLong() { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putLong(1_000_000_000_000L); + mem.putLong(Long.MIN_VALUE); + + assertEquals(16, mem.getAppendOffset()); + assertEquals(1_000_000_000_000L, Unsafe.getUnsafe().getLong(mem.addressOf(0))); + assertEquals(Long.MIN_VALUE, Unsafe.getUnsafe().getLong(mem.addressOf(8))); + } + } + + @Test + public void testPutAndReadFloat() { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putFloat(3.14f); + mem.putFloat(Float.NaN); + + assertEquals(8, mem.getAppendOffset()); + assertEquals(3.14f, Unsafe.getUnsafe().getFloat(mem.addressOf(0)), 0.0f); + assertTrue(Float.isNaN(Unsafe.getUnsafe().getFloat(mem.addressOf(4)))); + } + } + + @Test + public void testPutAndReadDouble() { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putDouble(2.718281828); + mem.putDouble(Double.NaN); + + assertEquals(16, mem.getAppendOffset()); + assertEquals(2.718281828, Unsafe.getUnsafe().getDouble(mem.addressOf(0)), 0.0); + assertTrue(Double.isNaN(Unsafe.getUnsafe().getDouble(mem.addressOf(8)))); + } + } + + @Test + public void testPutBoolean() { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putBoolean(true); + mem.putBoolean(false); + mem.putBoolean(true); + + assertEquals(3, mem.getAppendOffset()); + assertEquals(1, Unsafe.getUnsafe().getByte(mem.addressOf(0))); + assertEquals(0, Unsafe.getUnsafe().getByte(mem.addressOf(1))); + assertEquals(1, Unsafe.getUnsafe().getByte(mem.addressOf(2))); + } + } + + @Test + public void testGrowth() { + long before = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); + try (OffHeapAppendMemory mem = new OffHeapAppendMemory(8)) { + // Write more data than initial capacity to force growth + for (int i = 0; i < 100; i++) { + mem.putLong(i); + } + + assertEquals(800, mem.getAppendOffset()); + for (int i = 0; i < 100; i++) { + assertEquals(i, Unsafe.getUnsafe().getLong(mem.addressOf((long) i * 8))); + } + } + long after = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); + assertEquals(before, after); + } + + @Test + public void testTruncate() { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putInt(1); + mem.putInt(2); + mem.putInt(3); + assertEquals(12, mem.getAppendOffset()); + + mem.truncate(); + assertEquals(0, mem.getAppendOffset()); + + // Can write again after truncate + mem.putInt(42); + assertEquals(4, mem.getAppendOffset()); + assertEquals(42, Unsafe.getUnsafe().getInt(mem.addressOf(0))); + } + } + + @Test + public void testJumpTo() { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putLong(100); + mem.putLong(200); + mem.putLong(300); + assertEquals(24, mem.getAppendOffset()); + + // Jump back to offset 8 (after first long) + mem.jumpTo(8); + assertEquals(8, mem.getAppendOffset()); + + // Write new value at offset 8 + mem.putLong(999); + assertEquals(16, mem.getAppendOffset()); + assertEquals(100, Unsafe.getUnsafe().getLong(mem.addressOf(0))); + assertEquals(999, Unsafe.getUnsafe().getLong(mem.addressOf(8))); + } + } + + @Test + public void testSkip() { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putInt(1); + mem.skip(8); + mem.putInt(2); + + assertEquals(16, mem.getAppendOffset()); + assertEquals(1, Unsafe.getUnsafe().getInt(mem.addressOf(0))); + assertEquals(2, Unsafe.getUnsafe().getInt(mem.addressOf(12))); + } + } + + @Test + public void testPageAddress() { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + assertTrue(mem.pageAddress() != 0); + assertEquals(mem.pageAddress(), mem.addressOf(0)); + mem.putLong(42); + assertEquals(mem.pageAddress() + 8, mem.addressOf(8)); + } + } + + @Test + public void testCloseFreesMemory() { + long before = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); + OffHeapAppendMemory mem = new OffHeapAppendMemory(1024); + long during = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); + assertTrue(during > before); + + mem.close(); + long after = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); + assertEquals(before, after); + } + + @Test + public void testDoubleCloseIsSafe() { + OffHeapAppendMemory mem = new OffHeapAppendMemory(); + mem.putInt(42); + mem.close(); + mem.close(); // should not throw + } + + @Test + public void testMixedTypes() { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putByte((byte) 1); + mem.putShort((short) 2); + mem.putInt(3); + mem.putLong(4L); + mem.putFloat(5.0f); + mem.putDouble(6.0); + + long addr = mem.pageAddress(); + assertEquals(1, Unsafe.getUnsafe().getByte(addr)); + assertEquals(2, Unsafe.getUnsafe().getShort(addr + 1)); + assertEquals(3, Unsafe.getUnsafe().getInt(addr + 3)); + assertEquals(4L, Unsafe.getUnsafe().getLong(addr + 7)); + assertEquals(5.0f, Unsafe.getUnsafe().getFloat(addr + 15), 0.0f); + assertEquals(6.0, Unsafe.getUnsafe().getDouble(addr + 19), 0.0); + assertEquals(27, mem.getAppendOffset()); + } + } + + @Test + public void testLargeGrowth() { + long before = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); + try (OffHeapAppendMemory mem = new OffHeapAppendMemory(8)) { + // Write 10000 doubles to stress growth + for (int i = 0; i < 10_000; i++) { + mem.putDouble(i * 1.1); + } + assertEquals(80_000, mem.getAppendOffset()); + + // Verify first and last values + assertEquals(0.0, Unsafe.getUnsafe().getDouble(mem.addressOf(0)), 0.0); + assertEquals(9999 * 1.1, Unsafe.getUnsafe().getDouble(mem.addressOf(79_992)), 0.001); + } + long after = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); + assertEquals(before, after); + } +} From 7f16328b1bdb03e946a1d515a02a38248711897b Mon Sep 17 00:00:00 2001 From: bluestreak Date: Sun, 22 Feb 2026 03:14:36 +0000 Subject: [PATCH 14/89] wip 12 --- .../qwp/client/QwpWebSocketEncoder.java | 17 +- .../qwp/client/QwpWebSocketSender.java | 1504 ++++++++--------- .../qwp/protocol/QwpGorillaEncoder.java | 47 +- 3 files changed, 781 insertions(+), 787 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java index ef473de..8f8d2ac 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java @@ -567,38 +567,29 @@ private void writeTableHeaderWithSchemaRef(String tableName, int rowCount, long /** * Writes a timestamp column with optional Gorilla compression. - * Reads longs from off-heap. For Gorilla encoding, creates a temporary - * on-heap array since the Gorilla encoder requires long[]. + * Reads longs directly from off-heap — zero heap allocation. */ private void writeTimestampColumn(long addr, int count, boolean useGorilla) { if (useGorilla && count > 2) { - // Extract to temp array for Gorilla encoder (which requires long[]) - long[] values = new long[count]; - for (int i = 0; i < count; i++) { - values[i] = Unsafe.getUnsafe().getLong(addr + (long) i * 8); - } - - if (QwpGorillaEncoder.canUseGorilla(values, count)) { + if (QwpGorillaEncoder.canUseGorilla(addr, count)) { buffer.putByte(ENCODING_GORILLA); - int encodedSize = QwpGorillaEncoder.calculateEncodedSize(values, count); + int encodedSize = QwpGorillaEncoder.calculateEncodedSize(addr, count); buffer.ensureCapacity(encodedSize); int bytesWritten = gorillaEncoder.encodeTimestamps( buffer.getBufferPtr() + buffer.getPosition(), buffer.getCapacity() - buffer.getPosition(), - values, + addr, count ); buffer.skip(bytesWritten); } else { buffer.putByte(ENCODING_UNCOMPRESSED); - // Bulk copy for uncompressed path buffer.putBlockOfBytes(addr, (long) count * 8); } } else { if (useGorilla) { buffer.putByte(ENCODING_UNCOMPRESSED); } - // Bulk copy for uncompressed timestamps buffer.putBlockOfBytes(addr, (long) count * 8); } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index dabda70..68dc19d 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -24,27 +24,25 @@ package io.questdb.client.cutlass.qwp.client; -import io.questdb.client.cutlass.qwp.protocol.*; - import io.questdb.client.Sender; import io.questdb.client.cutlass.http.client.WebSocketClient; import io.questdb.client.cutlass.http.client.WebSocketClientFactory; import io.questdb.client.cutlass.http.client.WebSocketFrameHandler; - import io.questdb.client.cutlass.line.LineSenderException; import io.questdb.client.cutlass.line.array.DoubleArray; import io.questdb.client.cutlass.line.array.LongArray; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import io.questdb.client.std.Chars; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; import io.questdb.client.std.CharSequenceObjHashMap; -import io.questdb.client.std.LongHashSet; -import io.questdb.client.std.ObjList; +import io.questdb.client.std.Chars; import io.questdb.client.std.Decimal128; import io.questdb.client.std.Decimal256; import io.questdb.client.std.Decimal64; +import io.questdb.client.std.LongHashSet; +import io.questdb.client.std.ObjList; import io.questdb.client.std.bytes.DirectByteSlice; import org.jetbrains.annotations.NotNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.time.Instant; import java.time.temporal.ChronoUnit; @@ -87,81 +85,74 @@ */ public class QwpWebSocketSender implements Sender { - private static final Logger LOG = LoggerFactory.getLogger(QwpWebSocketSender.class); - - private static final int DEFAULT_BUFFER_SIZE = 8192; - private static final int DEFAULT_MICROBATCH_BUFFER_SIZE = 1024 * 1024; // 1MB - public static final int DEFAULT_AUTO_FLUSH_ROWS = 500; public static final int DEFAULT_AUTO_FLUSH_BYTES = 1024 * 1024; // 1MB public static final long DEFAULT_AUTO_FLUSH_INTERVAL_NANOS = 100_000_000L; // 100ms + public static final int DEFAULT_AUTO_FLUSH_ROWS = 500; public static final int DEFAULT_IN_FLIGHT_WINDOW_SIZE = InFlightWindow.DEFAULT_WINDOW_SIZE; // 8 public static final int DEFAULT_SEND_QUEUE_CAPACITY = WebSocketSendQueue.DEFAULT_QUEUE_CAPACITY; // 16 + private static final int DEFAULT_BUFFER_SIZE = 8192; + private static final int DEFAULT_MICROBATCH_BUFFER_SIZE = 1024 * 1024; // 1MB + private static final Logger LOG = LoggerFactory.getLogger(QwpWebSocketSender.class); private static final String WRITE_PATH = "/write/v4"; - + private final int autoFlushBytes; + private final long autoFlushIntervalNanos; + // Auto-flush configuration + private final int autoFlushRows; + // Encoder for ILP v4 messages + private final QwpWebSocketEncoder encoder; + // Global symbol dictionary for delta encoding + private final GlobalSymbolDictionary globalSymbolDictionary; private final String host; + // Flow control configuration + private final int inFlightWindowSize; private final int port; - private final boolean tlsEnabled; + private final int sendQueueCapacity; + // Track schema hashes that have been sent to the server (for schema reference mode) + // First time we send a schema: full schema. Subsequent times: 8-byte hash reference. + // Combined key = schemaHash XOR (tableNameHash << 32) to include table name in lookup. + private final LongHashSet sentSchemaHashes = new LongHashSet(); private final CharSequenceObjHashMap tableBuffers; - private QwpTableBuffer currentTableBuffer; - private String currentTableName; + private final boolean tlsEnabled; + private MicrobatchBuffer activeBuffer; + // Double-buffering for async I/O + private MicrobatchBuffer buffer0; + private MicrobatchBuffer buffer1; // Cached column references to avoid repeated hashmap lookups private QwpTableBuffer.ColumnBuffer cachedTimestampColumn; private QwpTableBuffer.ColumnBuffer cachedTimestampNanosColumn; - - // Encoder for ILP v4 messages - private final QwpWebSocketEncoder encoder; - // WebSocket client (zero-GC native implementation) private WebSocketClient client; - private boolean connected; private boolean closed; - - // Double-buffering for async I/O - private MicrobatchBuffer buffer0; - private MicrobatchBuffer buffer1; - private MicrobatchBuffer activeBuffer; - private WebSocketSendQueue sendQueue; - - // Flow control - private InFlightWindow inFlightWindow; - - // Auto-flush configuration - private final int autoFlushRows; - private final int autoFlushBytes; - private final long autoFlushIntervalNanos; - - // Flow control configuration - private final int inFlightWindowSize; - private final int sendQueueCapacity; - - // Configuration - private boolean gorillaEnabled = true; - - // Async mode: pending row tracking - private int pendingRowCount; - private long firstPendingRowTimeNanos; - - // Batch sequence counter (must match server's messageSequence) - private long nextBatchSequence = 0; - - // Global symbol dictionary for delta encoding - private final GlobalSymbolDictionary globalSymbolDictionary; - + private boolean connected; // Track max global symbol ID used in current batch (for delta calculation) private int currentBatchMaxSymbolId = -1; - + private QwpTableBuffer currentTableBuffer; + private String currentTableName; + private long firstPendingRowTimeNanos; + // Configuration + private boolean gorillaEnabled = true; + // Flow control + private InFlightWindow inFlightWindow; // Track highest symbol ID sent to server (for delta encoding) // Once sent over TCP, server is guaranteed to receive it (or connection dies) private volatile int maxSentSymbolId = -1; + // Batch sequence counter (must match server's messageSequence) + private long nextBatchSequence = 0; + // Async mode: pending row tracking + private int pendingRowCount; + private WebSocketSendQueue sendQueue; - // Track schema hashes that have been sent to the server (for schema reference mode) - // First time we send a schema: full schema. Subsequent times: 8-byte hash reference. - // Combined key = schemaHash XOR (tableNameHash << 32) to include table name in lookup. - private final LongHashSet sentSchemaHashes = new LongHashSet(); - - private QwpWebSocketSender(String host, int port, boolean tlsEnabled, int bufferSize, - int autoFlushRows, int autoFlushBytes, long autoFlushIntervalNanos, - int inFlightWindowSize, int sendQueueCapacity) { + private QwpWebSocketSender( + String host, + int port, + boolean tlsEnabled, + int bufferSize, + int autoFlushRows, + int autoFlushBytes, + long autoFlushIntervalNanos, + int inFlightWindowSize, + int sendQueueCapacity + ) { this.host = host; this.port = port; this.tlsEnabled = tlsEnabled; @@ -223,17 +214,17 @@ public static QwpWebSocketSender connect(String host, int port, boolean tlsEnabl /** * Creates a new sender with async mode and custom configuration. * - * @param host server host - * @param port server HTTP port - * @param tlsEnabled whether to use TLS - * @param autoFlushRows rows per batch (0 = no limit) - * @param autoFlushBytes bytes per batch (0 = no limit) - * @param autoFlushIntervalNanos age before flush in nanos (0 = no limit) + * @param host server host + * @param port server HTTP port + * @param tlsEnabled whether to use TLS + * @param autoFlushRows rows per batch (0 = no limit) + * @param autoFlushBytes bytes per batch (0 = no limit) + * @param autoFlushIntervalNanos age before flush in nanos (0 = no limit) * @return connected sender */ public static QwpWebSocketSender connectAsync(String host, int port, boolean tlsEnabled, - int autoFlushRows, int autoFlushBytes, - long autoFlushIntervalNanos) { + int autoFlushRows, int autoFlushBytes, + long autoFlushIntervalNanos) { return connectAsync(host, port, tlsEnabled, autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, DEFAULT_IN_FLIGHT_WINDOW_SIZE, DEFAULT_SEND_QUEUE_CAPACITY); } @@ -241,20 +232,26 @@ public static QwpWebSocketSender connectAsync(String host, int port, boolean tls /** * Creates a new sender with async mode and full configuration including flow control. * - * @param host server host - * @param port server HTTP port - * @param tlsEnabled whether to use TLS - * @param autoFlushRows rows per batch (0 = no limit) - * @param autoFlushBytes bytes per batch (0 = no limit) - * @param autoFlushIntervalNanos age before flush in nanos (0 = no limit) - * @param inFlightWindowSize max batches awaiting server ACK (default: 8) - * @param sendQueueCapacity max batches waiting to send (default: 16) + * @param host server host + * @param port server HTTP port + * @param tlsEnabled whether to use TLS + * @param autoFlushRows rows per batch (0 = no limit) + * @param autoFlushBytes bytes per batch (0 = no limit) + * @param autoFlushIntervalNanos age before flush in nanos (0 = no limit) + * @param inFlightWindowSize max batches awaiting server ACK (default: 8) + * @param sendQueueCapacity max batches waiting to send (default: 16) * @return connected sender */ - public static QwpWebSocketSender connectAsync(String host, int port, boolean tlsEnabled, - int autoFlushRows, int autoFlushBytes, - long autoFlushIntervalNanos, - int inFlightWindowSize, int sendQueueCapacity) { + public static QwpWebSocketSender connectAsync( + String host, + int port, + boolean tlsEnabled, + int autoFlushRows, + int autoFlushBytes, + long autoFlushIntervalNanos, + int inFlightWindowSize, + int sendQueueCapacity + ) { QwpWebSocketSender sender = new QwpWebSocketSender( host, port, tlsEnabled, DEFAULT_BUFFER_SIZE, autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, @@ -304,8 +301,8 @@ public static QwpWebSocketSender create( *

* This allows unit tests to test sender logic without requiring a real server. * - * @param host server host (not connected) - * @param port server port (not connected) + * @param host server host (not connected) + * @param port server port (not connected) * @param inFlightWindowSize window size: 1 for sync behavior, >1 for async * @return unconnected sender */ @@ -342,122 +339,163 @@ public static QwpWebSocketSender createForTesting( // Note: does NOT call ensureConnected() } - private void ensureConnected() { - if (closed) { - throw new LineSenderException("Sender is closed"); + @Override + public void at(long timestamp, ChronoUnit unit) { + checkNotClosed(); + checkTableSelected(); + if (unit == ChronoUnit.NANOS) { + atNanos(timestamp); + } else { + long micros = toMicros(timestamp, unit); + atMicros(micros); } - if (!connected) { - // Create WebSocket client using factory (zero-GC native implementation) - if (tlsEnabled) { - client = WebSocketClientFactory.newInsecureTlsInstance(); - } else { - client = WebSocketClientFactory.newPlainTextInstance(); - } - - // Connect and upgrade to WebSocket - try { - client.connect(host, port); - client.upgrade(WRITE_PATH); - } catch (Exception e) { - client.close(); - client = null; - throw new LineSenderException("Failed to connect to " + host + ":" + port, e); - } - - // a window for tracking batches awaiting ACK (both modes) - inFlightWindow = new InFlightWindow(inFlightWindowSize, InFlightWindow.DEFAULT_TIMEOUT_MS); - - // Initialize send queue for async mode (window > 1) - // The send queue handles both sending AND receiving (single I/O thread) - if (inFlightWindowSize > 1) { - sendQueue = new WebSocketSendQueue(client, inFlightWindow, - sendQueueCapacity, - WebSocketSendQueue.DEFAULT_ENQUEUE_TIMEOUT_MS, - WebSocketSendQueue.DEFAULT_SHUTDOWN_TIMEOUT_MS); - } - // Sync mode (window=1): no send queue - we send and read ACKs synchronously - - // Clear sent schema hashes - server starts fresh on each connection - sentSchemaHashes.clear(); + } - connected = true; - LOG.info("Connected to WebSocket [host={}, port={}, windowSize={}]", host, port, inFlightWindowSize); - } + @Override + public void at(Instant timestamp) { + checkNotClosed(); + checkTableSelected(); + long micros = timestamp.getEpochSecond() * 1_000_000L + timestamp.getNano() / 1000L; + atMicros(micros); } - /** - * Returns whether Gorilla encoding is enabled. - */ - public boolean isGorillaEnabled() { - return gorillaEnabled; + @Override + public void atNow() { + checkNotClosed(); + checkTableSelected(); + // Server-assigned timestamp - just send the row without designated timestamp + sendRow(); } - /** - * Sets whether to use Gorilla timestamp encoding. - */ - public QwpWebSocketSender setGorillaEnabled(boolean enabled) { - this.gorillaEnabled = enabled; - this.encoder.setGorillaEnabled(enabled); + @Override + public QwpWebSocketSender boolColumn(CharSequence columnName, boolean value) { + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_BOOLEAN, false); + col.addBoolean(value); return this; } - /** - * Returns whether async mode is enabled (window size > 1). - */ - public boolean isAsyncMode() { - return inFlightWindowSize > 1; + @Override + public DirectByteSlice bufferView() { + throw new LineSenderException("bufferView() is not supported for WebSocket sender"); } /** - * Returns the in-flight window size. - * Window=1 means sync mode, window>1 means async mode. + * Adds a BYTE column value to the current row. + * + * @param columnName the column name + * @param value the byte value + * @return this sender for method chaining */ - public int getInFlightWindowSize() { - return inFlightWindowSize; + public QwpWebSocketSender byteColumn(CharSequence columnName, byte value) { + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_BYTE, false); + col.addByte(value); + return this; } - /** - * Returns the send queue capacity. - */ - public int getSendQueueCapacity() { - return sendQueueCapacity; + @Override + public void cancelRow() { + checkNotClosed(); + if (currentTableBuffer != null) { + currentTableBuffer.cancelCurrentRow(); + } } /** - * Returns the auto-flush row threshold. + * Adds a CHAR column value to the current row. + *

+ * CHAR is stored as a 2-byte UTF-16 code unit in QuestDB. + * + * @param columnName the column name + * @param value the character value + * @return this sender for method chaining */ - public int getAutoFlushRows() { - return autoFlushRows; + public QwpWebSocketSender charColumn(CharSequence columnName, char value) { + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_CHAR, false); + col.addShort((short) value); + return this; } - /** - * Returns the auto-flush byte threshold. - */ - public int getAutoFlushBytes() { - return autoFlushBytes; - } + @Override + public void close() { + if (!closed) { + closed = true; - /** - * Returns the auto-flush interval in nanoseconds. - */ - public long getAutoFlushIntervalNanos() { - return autoFlushIntervalNanos; + // Flush any remaining data + try { + if (inFlightWindowSize > 1) { + // Async mode (window > 1): flush accumulated rows in table buffers first + flushPendingRows(); + + if (activeBuffer != null && activeBuffer.hasData()) { + sealAndSwapBuffer(); + } + if (sendQueue != null) { + sendQueue.close(); + } + } else { + // Sync mode (window=1): flush pending rows synchronously + if (pendingRowCount > 0 && client != null && client.isConnected()) { + flushSync(); + } + } + } catch (Exception e) { + LOG.error("Error during close: {}", String.valueOf(e)); + } + + // Close buffers (async mode only, window > 1) + if (buffer0 != null) { + buffer0.close(); + } + if (buffer1 != null) { + buffer1.close(); + } + + if (client != null) { + client.close(); + client = null; + } + encoder.close(); + // Close all table buffers to free off-heap column memory + ObjList keys = tableBuffers.keys(); + for (int i = 0, n = keys.size(); i < n; i++) { + CharSequence key = keys.getQuick(i); + if (key != null) { + QwpTableBuffer tb = tableBuffers.get(key); + if (tb != null) { + tb.close(); + } + } + } + tableBuffers.clear(); + + LOG.info("QwpWebSocketSender closed"); + } } - /** - * Returns the global symbol dictionary. - * For testing and encoder integration. - */ - public GlobalSymbolDictionary getGlobalSymbolDictionary() { - return globalSymbolDictionary; + @Override + public Sender decimalColumn(CharSequence name, Decimal64 value) { + if (value == null || value.isNull()) return this; + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DECIMAL64, true); + col.addDecimal64(value); + return this; } - /** - * Returns the max symbol ID sent to the server. - * Once sent over TCP, server is guaranteed to receive it (or connection dies). - */ - public int getMaxSentSymbolId() { - return maxSentSymbolId; + @Override + public Sender decimalColumn(CharSequence name, Decimal128 value) { + if (value == null || value.isNull()) return this; + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DECIMAL128, true); + col.addDecimal128(value); + return this; } // ==================== Fast-path API for high-throughput generators ==================== @@ -479,143 +517,71 @@ public int getMaxSentSymbolId() { // tableBuffer.nextRow(); // sender.incrementPendingRowCount(); - /** - * Gets or creates a table buffer for direct access. - * For high-throughput generators that want to bypass fluent API overhead. - */ - public QwpTableBuffer getTableBuffer(String tableName) { - QwpTableBuffer buffer = tableBuffers.get(tableName); - if (buffer == null) { - buffer = new QwpTableBuffer(tableName); - tableBuffers.put(tableName, buffer); - } - currentTableBuffer = buffer; - currentTableName = tableName; - return buffer; - } - - /** - * Registers a symbol in the global dictionary and returns its ID. - * For use with fast-path column buffer access. - */ - public int getOrAddGlobalSymbol(String value) { - int globalId = globalSymbolDictionary.getOrAddSymbol(value); - if (globalId > currentBatchMaxSymbolId) { - currentBatchMaxSymbolId = globalId; - } - return globalId; - } - - /** - * Increments the pending row count for auto-flush tracking. - * Call this after adding a complete row via fast-path API. - * Triggers auto-flush if any threshold is exceeded. - */ - public void incrementPendingRowCount() { - if (pendingRowCount == 0) { - firstPendingRowTimeNanos = System.nanoTime(); - } - pendingRowCount++; - - // Check if any flush threshold is exceeded (same as sendRow()) - if (shouldAutoFlush()) { - if (inFlightWindowSize > 1) { - flushPendingRows(); - } else { - // Sync mode (window=1): flush directly with ACK wait - flushSync(); - } - } - } - - // ==================== Sender interface implementation ==================== - @Override - public QwpWebSocketSender table(CharSequence tableName) { + public Sender decimalColumn(CharSequence name, Decimal256 value) { + if (value == null || value.isNull()) return this; checkNotClosed(); - // Fast path: if table name matches current, skip hashmap lookup - if (currentTableName != null && currentTableBuffer != null && Chars.equals(tableName, currentTableName)) { - return this; - } - // Table changed - invalidate cached column references - cachedTimestampColumn = null; - cachedTimestampNanosColumn = null; - currentTableName = tableName.toString(); - currentTableBuffer = tableBuffers.get(currentTableName); - if (currentTableBuffer == null) { - currentTableBuffer = new QwpTableBuffer(currentTableName); - tableBuffers.put(currentTableName, currentTableBuffer); - } - // Both modes accumulate rows until flush + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DECIMAL256, true); + col.addDecimal256(value); return this; } @Override - public QwpWebSocketSender symbol(CharSequence columnName, CharSequence value) { + public Sender decimalColumn(CharSequence name, CharSequence value) { + if (value == null || value.length() == 0) return this; checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_SYMBOL, true); - - if (value != null) { - // Register symbol in global dictionary and track max ID for delta calculation - String symbolValue = value.toString(); - int globalId = globalSymbolDictionary.getOrAddSymbol(symbolValue); - if (globalId > currentBatchMaxSymbolId) { - currentBatchMaxSymbolId = globalId; - } - // Store global ID in the column buffer - col.addSymbolWithGlobalId(symbolValue, globalId); - } else { - col.addSymbol(null); + try { + java.math.BigDecimal bd = new java.math.BigDecimal(value.toString()); + Decimal256 decimal = Decimal256.fromBigDecimal(bd); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DECIMAL256, true); + col.addDecimal256(decimal); + } catch (Exception e) { + throw new LineSenderException("Failed to parse decimal value: " + value, e); } return this; } @Override - public QwpWebSocketSender boolColumn(CharSequence columnName, boolean value) { + public Sender doubleArray(@NotNull CharSequence name, double[] values) { + if (values == null) return this; checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_BOOLEAN, false); - col.addBoolean(value); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DOUBLE_ARRAY, true); + col.addDoubleArray(values); return this; } + // ==================== Sender interface implementation ==================== + @Override - public QwpWebSocketSender longColumn(CharSequence columnName, long value) { + public Sender doubleArray(@NotNull CharSequence name, double[][] values) { + if (values == null) return this; checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_LONG, false); - col.addLong(value); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DOUBLE_ARRAY, true); + col.addDoubleArray(values); return this; } - /** - * Adds an INT column value to the current row. - * - * @param columnName the column name - * @param value the int value - * @return this sender for method chaining - */ - /** - * Adds a BYTE column value to the current row. - * - * @param columnName the column name - * @param value the byte value - * @return this sender for method chaining - */ - public QwpWebSocketSender byteColumn(CharSequence columnName, byte value) { + @Override + public Sender doubleArray(@NotNull CharSequence name, double[][][] values) { + if (values == null) return this; checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_BYTE, false); - col.addByte(value); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DOUBLE_ARRAY, true); + col.addDoubleArray(values); return this; } - public QwpWebSocketSender intColumn(CharSequence columnName, int value) { + @Override + public Sender doubleArray(CharSequence name, DoubleArray array) { + if (array == null) return this; checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_INT, false); - col.addInt(value); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DOUBLE_ARRAY, true); + col.addDoubleArray(array); return this; } @@ -628,6 +594,14 @@ public QwpWebSocketSender doubleColumn(CharSequence columnName, double value) { return this; } + /** + * Adds an INT column value to the current row. + * + * @param columnName the column name + * @param value the int value + * @return this sender for method chaining + */ + /** * Adds a FLOAT column value to the current row. * @@ -644,62 +618,155 @@ public QwpWebSocketSender floatColumn(CharSequence columnName, float value) { } @Override - public QwpWebSocketSender stringColumn(CharSequence columnName, CharSequence value) { + public void flush() { checkNotClosed(); - checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_STRING, true); - col.addString(value != null ? value.toString() : null); - return this; - } + ensureConnected(); - /** - * Adds a SHORT column value to the current row. - * - * @param columnName the column name - * @param value the short value - * @return this sender for method chaining - */ - public QwpWebSocketSender shortColumn(CharSequence columnName, short value) { - checkNotClosed(); - checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_SHORT, false); - col.addShort(value); - return this; - } + if (inFlightWindowSize > 1) { + // Async mode (window > 1): flush pending rows and wait for ACKs + flushPendingRows(); - /** - * Adds a CHAR column value to the current row. - *

- * CHAR is stored as a 2-byte UTF-16 code unit in QuestDB. - * - * @param columnName the column name - * @param value the character value - * @return this sender for method chaining + // Flush any remaining data in the active microbatch buffer + if (activeBuffer.hasData()) { + sealAndSwapBuffer(); + } + + // Wait for all pending batches to be sent to the server + sendQueue.flush(); + + // Wait for all in-flight batches to be acknowledged by the server + inFlightWindow.awaitEmpty(); + + LOG.debug("Flush complete [totalBatches={}, totalBytes={}, totalAcked={}]", sendQueue.getTotalBatchesSent(), sendQueue.getTotalBytesSent(), inFlightWindow.getTotalAcked()); + } else { + // Sync mode (window=1): flush pending rows and wait for ACKs synchronously + flushSync(); + } + } + + /** + * Returns the auto-flush byte threshold. */ - public QwpWebSocketSender charColumn(CharSequence columnName, char value) { - checkNotClosed(); - checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_CHAR, false); - col.addShort((short) value); - return this; + public int getAutoFlushBytes() { + return autoFlushBytes; } /** - * Adds a UUID column value to the current row. - * - * @param columnName the column name - * @param lo the low 64 bits of the UUID - * @param hi the high 64 bits of the UUID - * @return this sender for method chaining + * Returns the auto-flush interval in nanoseconds. */ - public QwpWebSocketSender uuidColumn(CharSequence columnName, long lo, long hi) { + public long getAutoFlushIntervalNanos() { + return autoFlushIntervalNanos; + } + + /** + * Returns the auto-flush row threshold. + */ + public int getAutoFlushRows() { + return autoFlushRows; + } + + /** + * Returns the global symbol dictionary. + * For testing and encoder integration. + */ + public GlobalSymbolDictionary getGlobalSymbolDictionary() { + return globalSymbolDictionary; + } + + /** + * Returns the in-flight window size. + * Window=1 means sync mode, window>1 means async mode. + */ + public int getInFlightWindowSize() { + return inFlightWindowSize; + } + + /** + * Returns the max symbol ID sent to the server. + * Once sent over TCP, server is guaranteed to receive it (or connection dies). + */ + public int getMaxSentSymbolId() { + return maxSentSymbolId; + } + + /** + * Registers a symbol in the global dictionary and returns its ID. + * For use with fast-path column buffer access. + */ + public int getOrAddGlobalSymbol(String value) { + int globalId = globalSymbolDictionary.getOrAddSymbol(value); + if (globalId > currentBatchMaxSymbolId) { + currentBatchMaxSymbolId = globalId; + } + return globalId; + } + + /** + * Returns the send queue capacity. + */ + public int getSendQueueCapacity() { + return sendQueueCapacity; + } + + /** + * Gets or creates a table buffer for direct access. + * For high-throughput generators that want to bypass fluent API overhead. + */ + public QwpTableBuffer getTableBuffer(String tableName) { + QwpTableBuffer buffer = tableBuffers.get(tableName); + if (buffer == null) { + buffer = new QwpTableBuffer(tableName); + tableBuffers.put(tableName, buffer); + } + currentTableBuffer = buffer; + currentTableName = tableName; + return buffer; + } + + /** + * Increments the pending row count for auto-flush tracking. + * Call this after adding a complete row via fast-path API. + * Triggers auto-flush if any threshold is exceeded. + */ + public void incrementPendingRowCount() { + if (pendingRowCount == 0) { + firstPendingRowTimeNanos = System.nanoTime(); + } + pendingRowCount++; + + // Check if any flush threshold is exceeded (same as sendRow()) + if (shouldAutoFlush()) { + if (inFlightWindowSize > 1) { + flushPendingRows(); + } else { + // Sync mode (window=1): flush directly with ACK wait + flushSync(); + } + } + } + + public QwpWebSocketSender intColumn(CharSequence columnName, int value) { checkNotClosed(); checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_UUID, true); - col.addUuid(hi, lo); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_INT, false); + col.addInt(value); return this; } + /** + * Returns whether async mode is enabled (window size > 1). + */ + public boolean isAsyncMode() { + return inFlightWindowSize > 1; + } + + /** + * Returns whether Gorilla encoding is enabled. + */ + public boolean isGorillaEnabled() { + return gorillaEnabled; + } + /** * Adds a LONG256 column value to the current row. * @@ -718,6 +785,137 @@ public QwpWebSocketSender long256Column(CharSequence columnName, long l0, long l return this; } + @Override + public Sender longArray(@NotNull CharSequence name, long[] values) { + if (values == null) return this; + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_LONG_ARRAY, true); + col.addLongArray(values); + return this; + } + + @Override + public Sender longArray(@NotNull CharSequence name, long[][] values) { + if (values == null) return this; + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_LONG_ARRAY, true); + col.addLongArray(values); + return this; + } + + @Override + public Sender longArray(@NotNull CharSequence name, long[][][] values) { + if (values == null) return this; + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_LONG_ARRAY, true); + col.addLongArray(values); + return this; + } + + @Override + public Sender longArray(@NotNull CharSequence name, LongArray array) { + if (array == null) return this; + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_LONG_ARRAY, true); + col.addLongArray(array); + return this; + } + + @Override + public QwpWebSocketSender longColumn(CharSequence columnName, long value) { + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_LONG, false); + col.addLong(value); + return this; + } + + @Override + public void reset() { + checkNotClosed(); + if (currentTableBuffer != null) { + currentTableBuffer.reset(); + } + } + + /** + * Sets whether to use Gorilla timestamp encoding. + */ + public QwpWebSocketSender setGorillaEnabled(boolean enabled) { + this.gorillaEnabled = enabled; + this.encoder.setGorillaEnabled(enabled); + return this; + } + + /** + * Adds a SHORT column value to the current row. + * + * @param columnName the column name + * @param value the short value + * @return this sender for method chaining + */ + public QwpWebSocketSender shortColumn(CharSequence columnName, short value) { + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_SHORT, false); + col.addShort(value); + return this; + } + + @Override + public QwpWebSocketSender stringColumn(CharSequence columnName, CharSequence value) { + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_STRING, true); + col.addString(value != null ? value.toString() : null); + return this; + } + + @Override + public QwpWebSocketSender symbol(CharSequence columnName, CharSequence value) { + checkNotClosed(); + checkTableSelected(); + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_SYMBOL, true); + + if (value != null) { + // Register symbol in global dictionary and track max ID for delta calculation + String symbolValue = value.toString(); + int globalId = globalSymbolDictionary.getOrAddSymbol(symbolValue); + if (globalId > currentBatchMaxSymbolId) { + currentBatchMaxSymbolId = globalId; + } + // Store global ID in the column buffer + col.addSymbolWithGlobalId(symbolValue, globalId); + } else { + col.addSymbol(null); + } + return this; + } + + @Override + public QwpWebSocketSender table(CharSequence tableName) { + checkNotClosed(); + // Fast path: if table name matches current, skip hashmap lookup + if (currentTableName != null && currentTableBuffer != null && Chars.equals(tableName, currentTableName)) { + return this; + } + // Table changed - invalidate cached column references + cachedTimestampColumn = null; + cachedTimestampNanosColumn = null; + currentTableName = tableName.toString(); + currentTableBuffer = tableBuffers.get(currentTableName); + if (currentTableBuffer == null) { + currentTableBuffer = new QwpTableBuffer(currentTableName); + tableBuffers.put(currentTableName, currentTableBuffer); + } + // Both modes accumulate rows until flush + return this; + } + @Override public QwpWebSocketSender timestampColumn(CharSequence columnName, long value, ChronoUnit unit) { checkNotClosed(); @@ -743,24 +941,44 @@ public QwpWebSocketSender timestampColumn(CharSequence columnName, Instant value return this; } - @Override - public void at(long timestamp, ChronoUnit unit) { + // ==================== Array methods ==================== + + /** + * Adds a UUID column value to the current row. + * + * @param columnName the column name + * @param lo the low 64 bits of the UUID + * @param hi the high 64 bits of the UUID + * @return this sender for method chaining + */ + public QwpWebSocketSender uuidColumn(CharSequence columnName, long lo, long hi) { checkNotClosed(); checkTableSelected(); - if (unit == ChronoUnit.NANOS) { - atNanos(timestamp); - } else { - long micros = toMicros(timestamp, unit); - atMicros(micros); - } + QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName.toString(), TYPE_UUID, true); + col.addUuid(hi, lo); + return this; } - @Override - public void at(Instant timestamp) { - checkNotClosed(); - checkTableSelected(); - long micros = timestamp.getEpochSecond() * 1_000_000L + timestamp.getNano() / 1000L; - atMicros(micros); + /** + * Adds encoded data to the active microbatch buffer. + * Triggers seal and swap if buffer is full. + */ + private void addToMicrobatch(long dataPtr, int length) { + // Ensure activeBuffer is ready for writing + ensureActiveBufferReady(); + + // If current buffer can't hold the data, seal and swap + if (activeBuffer.hasData() && + activeBuffer.getBufferPos() + length > activeBuffer.getBufferCapacity()) { + sealAndSwapBuffer(); + } + + // Ensure buffer can hold the data + activeBuffer.ensureCapacity(activeBuffer.getBufferPos() + length); + + // Copy data to buffer + activeBuffer.write(dataPtr, length); + activeBuffer.incrementRowCount(); } private void atMicros(long timestampMicros) { @@ -783,60 +1001,98 @@ private void atNanos(long timestampNanos) { sendRow(); } - @Override - public void atNow() { - checkNotClosed(); - checkTableSelected(); - // Server-assigned timestamp - just send the row without designated timestamp - sendRow(); + private void checkNotClosed() { + if (closed) { + throw new LineSenderException("Sender is closed"); + } + } + + private void checkTableSelected() { + if (currentTableBuffer == null) { + throw new LineSenderException("table() must be called before adding columns"); + } } /** - * Accumulates the current row. - * Both sync and async modes buffer rows until flush (explicit or auto-flush). - * The difference is that sync mode flush() blocks until server ACKs. + * Ensures the active buffer is ready for writing (in FILLING state). + * If the buffer is in RECYCLED state, resets it. If it's in use, waits for it. */ - private void sendRow() { - ensureConnected(); - currentTableBuffer.nextRow(); + private void ensureActiveBufferReady() { + if (activeBuffer.isFilling()) { + return; // Already ready + } - // Both modes: accumulate rows, don't encode yet - if (pendingRowCount == 0) { - firstPendingRowTimeNanos = System.nanoTime(); + if (activeBuffer.isRecycled()) { + // Buffer was recycled but not reset - reset it now + activeBuffer.reset(); + return; } - pendingRowCount++; - // Check if any flush threshold is exceeded - if (shouldAutoFlush()) { - if (inFlightWindowSize > 1) { - flushPendingRows(); - } else { - // Sync mode (window=1): flush directly with ACK wait - flushSync(); + // Buffer is in use (SEALED or SENDING) - wait for it + // Use a while loop to handle spurious wakeups and race conditions with the latch + while (activeBuffer.isInUse()) { + LOG.debug("Waiting for active buffer [id={}, state={}]", activeBuffer.getBatchId(), MicrobatchBuffer.stateName(activeBuffer.getState())); + boolean recycled = activeBuffer.awaitRecycled(30, TimeUnit.SECONDS); + if (!recycled) { + throw new LineSenderException("Timeout waiting for active buffer to be recycled"); } } - } - /** - * Checks if any auto-flush threshold is exceeded. - */ - private boolean shouldAutoFlush() { - if (pendingRowCount <= 0) { - return false; + // Buffer should now be RECYCLED - reset it + if (activeBuffer.isRecycled()) { + activeBuffer.reset(); } - // Row limit - if (autoFlushRows > 0 && pendingRowCount >= autoFlushRows) { - return true; + } + + private void ensureConnected() { + if (closed) { + throw new LineSenderException("Sender is closed"); } - // Time limit - if (autoFlushIntervalNanos > 0) { - long ageNanos = System.nanoTime() - firstPendingRowTimeNanos; - if (ageNanos >= autoFlushIntervalNanos) { - return true; + if (!connected) { + // Create WebSocket client using factory (zero-GC native implementation) + if (tlsEnabled) { + client = WebSocketClientFactory.newInsecureTlsInstance(); + } else { + client = WebSocketClientFactory.newPlainTextInstance(); + } + + // Connect and upgrade to WebSocket + try { + client.connect(host, port); + client.upgrade(WRITE_PATH); + } catch (Exception e) { + client.close(); + client = null; + throw new LineSenderException("Failed to connect to " + host + ":" + port, e); + } + + // a window for tracking batches awaiting ACK (both modes) + inFlightWindow = new InFlightWindow(inFlightWindowSize, InFlightWindow.DEFAULT_TIMEOUT_MS); + + // Initialize send queue for async mode (window > 1) + // The send queue handles both sending AND receiving (single I/O thread) + if (inFlightWindowSize > 1) { + sendQueue = new WebSocketSendQueue(client, inFlightWindow, + sendQueueCapacity, + WebSocketSendQueue.DEFAULT_ENQUEUE_TIMEOUT_MS, + WebSocketSendQueue.DEFAULT_SHUTDOWN_TIMEOUT_MS); } + // Sync mode (window=1): no send queue - we send and read ACKs synchronously + + // Clear sent schema hashes - server starts fresh on each connection + sentSchemaHashes.clear(); + + connected = true; + LOG.info("Connected to WebSocket [host={}, port={}, windowSize={}]", host, port, inFlightWindowSize); + } + } + + // ==================== Decimal methods ==================== + + private void failExpectedIfNeeded(long expectedSequence, LineSenderException error) { + if (inFlightWindow != null && inFlightWindow.getLastError() == null) { + inFlightWindow.fail(expectedSequence, error); } - // Byte limit is harder to estimate without encoding, skip for now - return false; } /** @@ -916,56 +1172,79 @@ private void flushPendingRows() { } /** - * Ensures the active buffer is ready for writing (in FILLING state). - * If the buffer is in RECYCLED state, resets it. If it's in use, waits for it. + * Flushes pending rows synchronously, blocking until server ACKs. + * Used in sync mode for simpler, blocking operation. */ - private void ensureActiveBufferReady() { - if (activeBuffer.isFilling()) { - return; // Already ready - } - - if (activeBuffer.isRecycled()) { - // Buffer was recycled but not reset - reset it now - activeBuffer.reset(); + private void flushSync() { + if (pendingRowCount <= 0) { return; } - // Buffer is in use (SEALED or SENDING) - wait for it - // Use a while loop to handle spurious wakeups and race conditions with the latch - while (activeBuffer.isInUse()) { - LOG.debug("Waiting for active buffer [id={}, state={}]", activeBuffer.getBatchId(), MicrobatchBuffer.stateName(activeBuffer.getState())); - boolean recycled = activeBuffer.awaitRecycled(30, TimeUnit.SECONDS); - if (!recycled) { - throw new LineSenderException("Timeout waiting for active buffer to be recycled"); + LOG.debug("Sync flush [pendingRows={}, tables={}]", pendingRowCount, tableBuffers.size()); + + // Encode all table buffers that have data into a single message + ObjList keys = tableBuffers.keys(); + for (int i = 0, n = keys.size(); i < n; i++) { + CharSequence tableName = keys.getQuick(i); + if (tableName == null) { + continue; + } + QwpTableBuffer tableBuffer = tableBuffers.get(tableName); + if (tableBuffer == null || tableBuffer.getRowCount() == 0) { + continue; } - } - // Buffer should now be RECYCLED - reset it - if (activeBuffer.isRecycled()) { - activeBuffer.reset(); - } - } + // Check if this schema has been sent before (use schema reference mode if so) + // Combined key includes table name since server caches by (tableName, schemaHash) + long schemaHash = tableBuffer.getSchemaHash(); + long schemaKey = schemaHash ^ ((long) tableBuffer.getTableName().hashCode() << 32); + boolean useSchemaRef = sentSchemaHashes.contains(schemaKey); - /** - * Adds encoded data to the active microbatch buffer. - * Triggers seal and swap if buffer is full. - */ - private void addToMicrobatch(long dataPtr, int length) { - // Ensure activeBuffer is ready for writing - ensureActiveBufferReady(); + // Encode this table's rows with delta symbol dictionary + int messageSize = encoder.encodeWithDeltaDict( + tableBuffer, + globalSymbolDictionary, + maxSentSymbolId, + currentBatchMaxSymbolId, + useSchemaRef + ); - // If current buffer can't hold the data, seal and swap - if (activeBuffer.hasData() && - activeBuffer.getBufferPos() + length > activeBuffer.getBufferCapacity()) { - sealAndSwapBuffer(); + // Track schema key if this was the first time sending this schema + if (!useSchemaRef) { + sentSchemaHashes.add(schemaKey); + } + + if (messageSize > 0) { + QwpBufferWriter buffer = encoder.getBuffer(); + + // Track batch in InFlightWindow before sending + long batchSequence = nextBatchSequence++; + inFlightWindow.addInFlight(batchSequence); + + // Update maxSentSymbolId - once sent over TCP, server will receive it + maxSentSymbolId = currentBatchMaxSymbolId; + + LOG.debug("Sending sync batch [seq={}, bytes={}, rows={}, maxSentSymbolId={}, useSchemaRef={}]", batchSequence, messageSize, tableBuffer.getRowCount(), maxSentSymbolId, useSchemaRef); + + // Send over WebSocket + client.sendBinary(buffer.getBufferPtr(), messageSize); + + // Wait for ACK synchronously + waitForAck(batchSequence); + } + + // Reset table buffer after sending + tableBuffer.reset(); + + // Reset batch-level symbol tracking + currentBatchMaxSymbolId = -1; } - // Ensure buffer can hold the data - activeBuffer.ensureCapacity(activeBuffer.getBufferPos() + length); + // Reset pending row tracking + pendingRowCount = 0; + firstPendingRowTimeNanos = 0; - // Copy data to buffer - activeBuffer.write(dataPtr, length); - activeBuffer.incrementRowCount(); + LOG.debug("Sync flush complete [totalAcked={}]", inFlightWindow.getTotalAcked()); } /** @@ -1016,107 +1295,75 @@ private void sealAndSwapBuffer() { } } - @Override - public void flush() { - checkNotClosed(); + // ==================== Helper methods ==================== + + /** + * Accumulates the current row. + * Both sync and async modes buffer rows until flush (explicit or auto-flush). + * The difference is that sync mode flush() blocks until server ACKs. + */ + private void sendRow() { ensureConnected(); + currentTableBuffer.nextRow(); - if (inFlightWindowSize > 1) { - // Async mode (window > 1): flush pending rows and wait for ACKs - flushPendingRows(); + // Both modes: accumulate rows, don't encode yet + if (pendingRowCount == 0) { + firstPendingRowTimeNanos = System.nanoTime(); + } + pendingRowCount++; - // Flush any remaining data in the active microbatch buffer - if (activeBuffer.hasData()) { - sealAndSwapBuffer(); + // Check if any flush threshold is exceeded + if (shouldAutoFlush()) { + if (inFlightWindowSize > 1) { + flushPendingRows(); + } else { + // Sync mode (window=1): flush directly with ACK wait + flushSync(); } - - // Wait for all pending batches to be sent to the server - sendQueue.flush(); - - // Wait for all in-flight batches to be acknowledged by the server - inFlightWindow.awaitEmpty(); - - LOG.debug("Flush complete [totalBatches={}, totalBytes={}, totalAcked={}]", sendQueue.getTotalBatchesSent(), sendQueue.getTotalBytesSent(), inFlightWindow.getTotalAcked()); - } else { - // Sync mode (window=1): flush pending rows and wait for ACKs synchronously - flushSync(); } } /** - * Flushes pending rows synchronously, blocking until server ACKs. - * Used in sync mode for simpler, blocking operation. + * Checks if any auto-flush threshold is exceeded. */ - private void flushSync() { + private boolean shouldAutoFlush() { if (pendingRowCount <= 0) { - return; + return false; } - - LOG.debug("Sync flush [pendingRows={}, tables={}]", pendingRowCount, tableBuffers.size()); - - // Encode all table buffers that have data into a single message - ObjList keys = tableBuffers.keys(); - for (int i = 0, n = keys.size(); i < n; i++) { - CharSequence tableName = keys.getQuick(i); - if (tableName == null) { - continue; - } - QwpTableBuffer tableBuffer = tableBuffers.get(tableName); - if (tableBuffer == null || tableBuffer.getRowCount() == 0) { - continue; - } - - // Check if this schema has been sent before (use schema reference mode if so) - // Combined key includes table name since server caches by (tableName, schemaHash) - long schemaHash = tableBuffer.getSchemaHash(); - long schemaKey = schemaHash ^ ((long) tableBuffer.getTableName().hashCode() << 32); - boolean useSchemaRef = sentSchemaHashes.contains(schemaKey); - - // Encode this table's rows with delta symbol dictionary - int messageSize = encoder.encodeWithDeltaDict( - tableBuffer, - globalSymbolDictionary, - maxSentSymbolId, - currentBatchMaxSymbolId, - useSchemaRef - ); - - // Track schema key if this was the first time sending this schema - if (!useSchemaRef) { - sentSchemaHashes.add(schemaKey); - } - - if (messageSize > 0) { - QwpBufferWriter buffer = encoder.getBuffer(); - - // Track batch in InFlightWindow before sending - long batchSequence = nextBatchSequence++; - inFlightWindow.addInFlight(batchSequence); - - // Update maxSentSymbolId - once sent over TCP, server will receive it - maxSentSymbolId = currentBatchMaxSymbolId; - - LOG.debug("Sending sync batch [seq={}, bytes={}, rows={}, maxSentSymbolId={}, useSchemaRef={}]", batchSequence, messageSize, tableBuffer.getRowCount(), maxSentSymbolId, useSchemaRef); - - // Send over WebSocket - client.sendBinary(buffer.getBufferPtr(), messageSize); - - // Wait for ACK synchronously - waitForAck(batchSequence); + // Row limit + if (autoFlushRows > 0 && pendingRowCount >= autoFlushRows) { + return true; + } + // Time limit + if (autoFlushIntervalNanos > 0) { + long ageNanos = System.nanoTime() - firstPendingRowTimeNanos; + if (ageNanos >= autoFlushIntervalNanos) { + return true; } + } + // Byte limit is harder to estimate without encoding, skip for now + return false; + } - // Reset table buffer after sending - tableBuffer.reset(); - - // Reset batch-level symbol tracking - currentBatchMaxSymbolId = -1; + private long toMicros(long value, ChronoUnit unit) { + switch (unit) { + case NANOS: + return value / 1000L; + case MICROS: + return value; + case MILLIS: + return value * 1000L; + case SECONDS: + return value * 1_000_000L; + case MINUTES: + return value * 60_000_000L; + case HOURS: + return value * 3_600_000_000L; + case DAYS: + return value * 86_400_000_000L; + default: + throw new LineSenderException("Unsupported time unit: " + unit); } - - // Reset pending row tracking - pendingRowCount = 0; - firstPendingRowTimeNanos = 0; - - LOG.debug("Sync flush complete [totalAcked={}]", inFlightWindow.getTotalAcked()); } /** @@ -1187,253 +1434,4 @@ public void onClose(int code, String reason) { failExpectedIfNeeded(expectedSequence, timeout); throw timeout; } - - private void failExpectedIfNeeded(long expectedSequence, LineSenderException error) { - if (inFlightWindow != null && inFlightWindow.getLastError() == null) { - inFlightWindow.fail(expectedSequence, error); - } - } - - @Override - public DirectByteSlice bufferView() { - throw new LineSenderException("bufferView() is not supported for WebSocket sender"); - } - - @Override - public void cancelRow() { - checkNotClosed(); - if (currentTableBuffer != null) { - currentTableBuffer.cancelCurrentRow(); - } - } - - @Override - public void reset() { - checkNotClosed(); - if (currentTableBuffer != null) { - currentTableBuffer.reset(); - } - } - - // ==================== Array methods ==================== - - @Override - public Sender doubleArray(@NotNull CharSequence name, double[] values) { - if (values == null) return this; - checkNotClosed(); - checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DOUBLE_ARRAY, true); - col.addDoubleArray(values); - return this; - } - - @Override - public Sender doubleArray(@NotNull CharSequence name, double[][] values) { - if (values == null) return this; - checkNotClosed(); - checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DOUBLE_ARRAY, true); - col.addDoubleArray(values); - return this; - } - - @Override - public Sender doubleArray(@NotNull CharSequence name, double[][][] values) { - if (values == null) return this; - checkNotClosed(); - checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DOUBLE_ARRAY, true); - col.addDoubleArray(values); - return this; - } - - @Override - public Sender doubleArray(CharSequence name, DoubleArray array) { - if (array == null) return this; - checkNotClosed(); - checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DOUBLE_ARRAY, true); - col.addDoubleArray(array); - return this; - } - - @Override - public Sender longArray(@NotNull CharSequence name, long[] values) { - if (values == null) return this; - checkNotClosed(); - checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_LONG_ARRAY, true); - col.addLongArray(values); - return this; - } - - @Override - public Sender longArray(@NotNull CharSequence name, long[][] values) { - if (values == null) return this; - checkNotClosed(); - checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_LONG_ARRAY, true); - col.addLongArray(values); - return this; - } - - @Override - public Sender longArray(@NotNull CharSequence name, long[][][] values) { - if (values == null) return this; - checkNotClosed(); - checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_LONG_ARRAY, true); - col.addLongArray(values); - return this; - } - - @Override - public Sender longArray(@NotNull CharSequence name, LongArray array) { - if (array == null) return this; - checkNotClosed(); - checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_LONG_ARRAY, true); - col.addLongArray(array); - return this; - } - - // ==================== Decimal methods ==================== - - @Override - public Sender decimalColumn(CharSequence name, Decimal64 value) { - if (value == null || value.isNull()) return this; - checkNotClosed(); - checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DECIMAL64, true); - col.addDecimal64(value); - return this; - } - - @Override - public Sender decimalColumn(CharSequence name, Decimal128 value) { - if (value == null || value.isNull()) return this; - checkNotClosed(); - checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DECIMAL128, true); - col.addDecimal128(value); - return this; - } - - @Override - public Sender decimalColumn(CharSequence name, Decimal256 value) { - if (value == null || value.isNull()) return this; - checkNotClosed(); - checkTableSelected(); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DECIMAL256, true); - col.addDecimal256(value); - return this; - } - - @Override - public Sender decimalColumn(CharSequence name, CharSequence value) { - if (value == null || value.length() == 0) return this; - checkNotClosed(); - checkTableSelected(); - try { - java.math.BigDecimal bd = new java.math.BigDecimal(value.toString()); - Decimal256 decimal = Decimal256.fromBigDecimal(bd); - QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DECIMAL256, true); - col.addDecimal256(decimal); - } catch (Exception e) { - throw new LineSenderException("Failed to parse decimal value: " + value, e); - } - return this; - } - - // ==================== Helper methods ==================== - - private long toMicros(long value, ChronoUnit unit) { - switch (unit) { - case NANOS: - return value / 1000L; - case MICROS: - return value; - case MILLIS: - return value * 1000L; - case SECONDS: - return value * 1_000_000L; - case MINUTES: - return value * 60_000_000L; - case HOURS: - return value * 3_600_000_000L; - case DAYS: - return value * 86_400_000_000L; - default: - throw new LineSenderException("Unsupported time unit: " + unit); - } - } - - private void checkNotClosed() { - if (closed) { - throw new LineSenderException("Sender is closed"); - } - } - - private void checkTableSelected() { - if (currentTableBuffer == null) { - throw new LineSenderException("table() must be called before adding columns"); - } - } - - @Override - public void close() { - if (!closed) { - closed = true; - - // Flush any remaining data - try { - if (inFlightWindowSize > 1) { - // Async mode (window > 1): flush accumulated rows in table buffers first - flushPendingRows(); - - if (activeBuffer != null && activeBuffer.hasData()) { - sealAndSwapBuffer(); - } - if (sendQueue != null) { - sendQueue.close(); - } - } else { - // Sync mode (window=1): flush pending rows synchronously - if (pendingRowCount > 0 && client != null && client.isConnected()) { - flushSync(); - } - } - } catch (Exception e) { - LOG.error("Error during close: {}", String.valueOf(e)); - } - - // Close buffers (async mode only, window > 1) - if (buffer0 != null) { - buffer0.close(); - } - if (buffer1 != null) { - buffer1.close(); - } - - if (client != null) { - client.close(); - client = null; - } - encoder.close(); - // Close all table buffers to free off-heap column memory - ObjList keys = tableBuffers.keys(); - for (int i = 0, n = keys.size(); i < n; i++) { - CharSequence key = keys.getQuick(i); - if (key != null) { - QwpTableBuffer tb = tableBuffers.get(key); - if (tb != null) { - tb.close(); - } - } - } - tableBuffers.clear(); - - LOG.info("QwpWebSocketSender closed"); - } - } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java index 5281dbc..8f59b38 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java @@ -143,7 +143,7 @@ public void encodeDoD(long deltaOfDelta) { } /** - * Encodes an array of timestamps to native memory using Gorilla compression. + * Encodes timestamps from off-heap memory using Gorilla compression. *

* Format: * 

@@ -157,11 +157,11 @@ public void encodeDoD(long deltaOfDelta) {
      *
      * @param destAddress destination address in native memory
      * @param capacity    maximum number of bytes to write
-     * @param timestamps  array of timestamp values
+     * @param srcAddress  source address of contiguous int64 timestamps in native memory
      * @param count       number of timestamps to encode
      * @return number of bytes written
      */
-    public int encodeTimestamps(long destAddress, long capacity, long[] timestamps, int count) {
+    public int encodeTimestamps(long destAddress, long capacity, long srcAddress, int count) {
         if (count == 0) {
             return 0;
         }
@@ -172,7 +172,8 @@ public int encodeTimestamps(long destAddress, long capacity, long[] timestamps,
         if (capacity < 8) {
             return 0; // Not enough space
         }
-        Unsafe.getUnsafe().putLong(destAddress, timestamps[0]);
+        long ts0 = Unsafe.getUnsafe().getLong(srcAddress);
+        Unsafe.getUnsafe().putLong(destAddress, ts0);
         pos = 8;
 
         if (count == 1) {
@@ -183,7 +184,8 @@ public int encodeTimestamps(long destAddress, long capacity, long[] timestamps,
         if (capacity < pos + 8) {
             return pos; // Not enough space
         }
-        Unsafe.getUnsafe().putLong(destAddress + pos, timestamps[1]);
+        long ts1 = Unsafe.getUnsafe().getLong(srcAddress + 8);
+        Unsafe.getUnsafe().putLong(destAddress + pos, ts1);
         pos += 8;
 
         if (count == 2) {
@@ -192,39 +194,41 @@ public int encodeTimestamps(long destAddress, long capacity, long[] timestamps,
 
         // Encode remaining with delta-of-delta
         bitWriter.reset(destAddress + pos, capacity - pos);
-        long prevTs = timestamps[1];
-        long prevDelta = timestamps[1] - timestamps[0];
+        long prevTs = ts1;
+        long prevDelta = ts1 - ts0;
 
         for (int i = 2; i < count; i++) {
-            long delta = timestamps[i] - prevTs;
+            long ts = Unsafe.getUnsafe().getLong(srcAddress + (long) i * 8);
+            long delta = ts - prevTs;
             long dod = delta - prevDelta;
             encodeDoD(dod);
             prevDelta = delta;
-            prevTs = timestamps[i];
+            prevTs = ts;
         }
 
         return pos + bitWriter.finish();
     }
 
     /**
-     * Checks if Gorilla encoding can be used for the given timestamps.
+     * Checks if Gorilla encoding can be used for timestamps stored off-heap.
      * 

      * Gorilla encoding uses 32-bit signed integers for delta-of-delta values,
      * so it cannot encode timestamps where the delta-of-delta exceeds the
      * 32-bit signed integer range.
      *
-     * @param timestamps array of timestamp values
+     * @param srcAddress source address of contiguous int64 timestamps in native memory
      * @param count      number of timestamps
      * @return true if Gorilla encoding can be used, false otherwise
      */
-    public static boolean canUseGorilla(long[] timestamps, int count) {
+    public static boolean canUseGorilla(long srcAddress, int count) {
         if (count < 3) {
             return true; // No DoD encoding needed for 0, 1, or 2 timestamps
         }
 
-        long prevDelta = timestamps[1] - timestamps[0];
+        long prevDelta = Unsafe.getUnsafe().getLong(srcAddress + 8) - Unsafe.getUnsafe().getLong(srcAddress);
         for (int i = 2; i < count; i++) {
-            long delta = timestamps[i] - timestamps[i - 1];
+            long delta = Unsafe.getUnsafe().getLong(srcAddress + (long) i * 8)
+                    - Unsafe.getUnsafe().getLong(srcAddress + (long) (i - 1) * 8);
             long dod = delta - prevDelta;
             if (dod < Integer.MIN_VALUE || dod > Integer.MAX_VALUE) {
                 return false;
@@ -235,16 +239,16 @@ public static boolean canUseGorilla(long[] timestamps, int count) {
     }
 
     /**
-     * Calculates the encoded size in bytes for Gorilla-encoded timestamps.
+     * Calculates the encoded size in bytes for Gorilla-encoded timestamps stored off-heap.
      * 

      * Note: This does NOT include the encoding flag byte. Add 1 byte if
      * the encoding flag is needed.
      *
-     * @param timestamps array of timestamp values
+     * @param srcAddress source address of contiguous int64 timestamps in native memory
      * @param count      number of timestamps
      * @return encoded size in bytes (excluding encoding flag)
      */
-    public static int calculateEncodedSize(long[] timestamps, int count) {
+    public static int calculateEncodedSize(long srcAddress, int count) {
         if (count == 0) {
             return 0;
         }
@@ -262,18 +266,19 @@ public static int calculateEncodedSize(long[] timestamps, int count) {
         }
 
         // Calculate bits for delta-of-delta encoding
-        long prevTimestamp = timestamps[1];
-        long prevDelta = timestamps[1] - timestamps[0];
+        long prevTimestamp = Unsafe.getUnsafe().getLong(srcAddress + 8);
+        long prevDelta = prevTimestamp - Unsafe.getUnsafe().getLong(srcAddress);
         int totalBits = 0;
 
         for (int i = 2; i < count; i++) {
-            long delta = timestamps[i] - prevTimestamp;
+            long ts = Unsafe.getUnsafe().getLong(srcAddress + (long) i * 8);
+            long delta = ts - prevTimestamp;
             long deltaOfDelta = delta - prevDelta;
 
             totalBits += getBitsRequired(deltaOfDelta);
 
             prevDelta = delta;
-            prevTimestamp = timestamps[i];
+            prevTimestamp = ts;
         }
 
         // Round up to bytes

From 87950812067c6b26ad3f0ab87f1c04f77c8bdb43 Mon Sep 17 00:00:00 2001
From: Jaromir Hamala 
Date: Mon, 23 Feb 2026 10:30:34 +0100
Subject: [PATCH 15/89] move test workloads/benchmarks to the client maven
 module

---
 .../line/tcp/v4/QwpAllocationTestClient.java  | 379 ++++++++++++++++
 .../line/tcp/v4/StacBenchmarkClient.java      | 424 ++++++++++++++++++
 2 files changed, 803 insertions(+)
 create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java
 create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/StacBenchmarkClient.java

diff --git a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java
new file mode 100644
index 0000000..5eab4bf
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java
@@ -0,0 +1,379 @@
+/*******************************************************************************
+ *     ___                  _   ____  ____
+ *    / _ \ _   _  ___  ___| |_|  _ \| __ )
+ *   | | | | | | |/ _ \/ __| __| | | |  _ \
+ *   | |_| | |_| |  __/\__ \ |_| |_| | |_) |
+ *    \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ *  Copyright (c) 2014-2019 Appsicle
+ *  Copyright (c) 2019-2024 QuestDB
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.cutlass.line.tcp.v4;
+
+import io.questdb.client.Sender;
+
+import java.io.IOException;
+import java.time.temporal.ChronoUnit;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Test client for ILP allocation profiling.
+ * 

+ * Supports 3 protocol modes:
+ * 
    
+ *   
  • ilp-tcp: Old ILP text protocol over TCP (port 9009)
    • 
+ *   
  • ilp-http: Old ILP text protocol over HTTP (port 9000)
    • 
+ *   
  • qwp-websocket: New QWP binary protocol over WebSocket (port 9000)
    • 
+ * 

+ * 

+ * Sends rows with various column types to exercise all code paths.
+ * Run with an allocation profiler (async-profiler, JFR, etc.) to find hotspots.
+ * 

+ * Usage:
+ * 
+ * java -cp ... QwpAllocationTestClient [options]
+ *
+ * Options:
+ *   --protocol=PROTOCOL   Protocol: ilp-tcp, ilp-http, qwp-websocket (default: qwp-websocket)
+ *   --host=HOST           Server host (default: localhost)
+ *   --port=PORT           Server port (default: 9009 for TCP, 9000 for HTTP)
+ *   --rows=N              Total rows to send (default: 10000000)
+ *   --batch=N             Batch/flush size (default: 10000)
+ *   --warmup=N            Warmup rows (default: 100000)
+ *   --report=N            Report progress every N rows (default: 1000000)
+ *   --no-warmup           Skip warmup phase
+ *   --help                Show this help
+ *
+ * Examples:
+ *   QwpAllocationTestClient --protocol=qwp-websocket --rows=1000000 --batch=5000
+ *   QwpAllocationTestClient --protocol=ilp-tcp --host=remote-server --port=9009
+ * 

+ */
+public class QwpAllocationTestClient {
+
+    // Protocol modes
+    private static final String PROTOCOL_ILP_TCP = "ilp-tcp";
+    private static final String PROTOCOL_ILP_HTTP = "ilp-http";
+    private static final String PROTOCOL_QWP_WEBSOCKET = "qwp-websocket";
+
+
+    // Default configuration
+    private static final String DEFAULT_HOST = "localhost";
+    private static final int DEFAULT_ROWS = 80_000_000;
+    private static final int DEFAULT_BATCH_SIZE = 10_000;
+    private static final int DEFAULT_FLUSH_BYTES = 0; // 0 = use protocol default
+    private static final long DEFAULT_FLUSH_INTERVAL_MS = 0; // 0 = use protocol default
+    private static final int DEFAULT_IN_FLIGHT_WINDOW = 0; // 0 = use protocol default (8)
+    private static final int DEFAULT_SEND_QUEUE = 0; // 0 = use protocol default (16)
+    private static final int DEFAULT_WARMUP_ROWS = 100_000;
+    private static final int DEFAULT_REPORT_INTERVAL = 1_000_000;
+
+    // Pre-computed test data to avoid allocation during the test
+    private static final String[] SYMBOLS = {
+            "AAPL", "GOOGL", "MSFT", "AMZN", "META", "NVDA", "TSLA", "BRK.A", "JPM", "JNJ",
+            "V", "PG", "UNH", "HD", "MA", "DIS", "PYPL", "BAC", "ADBE", "CMCSA"
+    };
+
+    private static final String[] STRINGS = {
+            "New York", "London", "Tokyo", "Paris", "Berlin", "Sydney", "Toronto", "Singapore",
+            "Hong Kong", "Dubai", "Mumbai", "Shanghai", "Moscow", "Seoul", "Bangkok",
+            "Amsterdam", "Zurich", "Frankfurt", "Milan", "Madrid"
+    };
+
+    public static void main(String[] args) {
+        // Parse command-line options
+        String protocol = PROTOCOL_QWP_WEBSOCKET;
+        String host = DEFAULT_HOST;
+        int port = -1; // -1 means use default for protocol
+        int totalRows = DEFAULT_ROWS;
+        int batchSize = DEFAULT_BATCH_SIZE;
+        int flushBytes = DEFAULT_FLUSH_BYTES;
+        long flushIntervalMs = DEFAULT_FLUSH_INTERVAL_MS;
+        int inFlightWindow = DEFAULT_IN_FLIGHT_WINDOW;
+        int sendQueue = DEFAULT_SEND_QUEUE;
+        int warmupRows = DEFAULT_WARMUP_ROWS;
+        int reportInterval = DEFAULT_REPORT_INTERVAL;
+
+        for (String arg : args) {
+            if (arg.equals("--help") || arg.equals("-h")) {
+                printUsage();
+                System.exit(0);
+            } else if (arg.startsWith("--protocol=")) {
+                protocol = arg.substring("--protocol=".length()).toLowerCase();
+            } else if (arg.startsWith("--host=")) {
+                host = arg.substring("--host=".length());
+            } else if (arg.startsWith("--port=")) {
+                port = Integer.parseInt(arg.substring("--port=".length()));
+            } else if (arg.startsWith("--rows=")) {
+                totalRows = Integer.parseInt(arg.substring("--rows=".length()));
+            } else if (arg.startsWith("--batch=")) {
+                batchSize = Integer.parseInt(arg.substring("--batch=".length()));
+            } else if (arg.startsWith("--flush-bytes=")) {
+                flushBytes = Integer.parseInt(arg.substring("--flush-bytes=".length()));
+            } else if (arg.startsWith("--flush-interval-ms=")) {
+                flushIntervalMs = Long.parseLong(arg.substring("--flush-interval-ms=".length()));
+            } else if (arg.startsWith("--in-flight-window=")) {
+                inFlightWindow = Integer.parseInt(arg.substring("--in-flight-window=".length()));
+            } else if (arg.startsWith("--send-queue=")) {
+                sendQueue = Integer.parseInt(arg.substring("--send-queue=".length()));
+            } else if (arg.startsWith("--warmup=")) {
+                warmupRows = Integer.parseInt(arg.substring("--warmup=".length()));
+            } else if (arg.startsWith("--report=")) {
+                reportInterval = Integer.parseInt(arg.substring("--report=".length()));
+            } else if (arg.equals("--no-warmup")) {
+                warmupRows = 0;
+            } else if (!arg.startsWith("--")) {
+                // Legacy positional args: protocol [host] [port] [rows]
+                protocol = arg.toLowerCase();
+            } else {
+                System.err.println("Unknown option: " + arg);
+                printUsage();
+                System.exit(1);
+            }
+        }
+
+        // Use default port if not specified
+        if (port == -1) {
+            port = getDefaultPort(protocol);
+        }
+
+        System.out.println("ILP Allocation Test Client");
+        System.out.println("==========================");
+        System.out.println("Protocol: " + protocol);
+        System.out.println("Host: " + host);
+        System.out.println("Port: " + port);
+        System.out.println("Total rows: " + String.format("%,d", totalRows));
+        System.out.println("Batch size (rows): " + String.format("%,d", batchSize) + (batchSize == 0 ? " (default)" : ""));
+        System.out.println("Flush bytes: " + (flushBytes == 0 ? "(default)" : String.format("%,d", flushBytes)));
+        System.out.println("Flush interval: " + (flushIntervalMs == 0 ? "(default)" : flushIntervalMs + " ms"));
+        System.out.println("In-flight window: " + (inFlightWindow == 0 ? "(default: 8)" : inFlightWindow));
+        System.out.println("Send queue: " + (sendQueue == 0 ? "(default: 16)" : sendQueue));
+        System.out.println("Warmup rows: " + String.format("%,d", warmupRows));
+        System.out.println("Report interval: " + String.format("%,d", reportInterval));
+        System.out.println();
+
+        try {
+            runTest(protocol, host, port, totalRows, batchSize, flushBytes, flushIntervalMs,
+                    inFlightWindow, sendQueue, warmupRows, reportInterval);
+        } catch (Exception e) {
+            System.err.println("Error: " + e.getMessage());
+            e.printStackTrace(System.err);
+            System.exit(1);
+        }
+    }
+
+    private static void printUsage() {
+        System.out.println("ILP Allocation Test Client");
+        System.out.println();
+        System.out.println("Usage: QwpAllocationTestClient [options]");
+        System.out.println();
+        System.out.println("Options:");
+        System.out.println("  --protocol=PROTOCOL      Protocol to use (default: qwp-websocket)");
+        System.out.println("  --host=HOST              Server host (default: localhost)");
+        System.out.println("  --port=PORT              Server port (default: 9009 for TCP, 9000 for HTTP/WebSocket)");
+        System.out.println("  --rows=N                 Total rows to send (default: 80000000)");
+        System.out.println("  --batch=N                Auto-flush after N rows (default: 10000)");
+        System.out.println("  --flush-bytes=N          Auto-flush after N bytes (default: protocol default)");
+        System.out.println("  --flush-interval-ms=N    Auto-flush after N ms (default: protocol default)");
+        System.out.println("  --in-flight-window=N     Max batches awaiting server ACK (default: 8, WebSocket only)");
+        System.out.println("  --send-queue=N           Max batches waiting to send (default: 16, WebSocket only)");
+        System.out.println("  --warmup=N               Warmup rows (default: 100000)");
+        System.out.println("  --report=N               Report progress every N rows (default: 1000000)");
+        System.out.println("  --no-warmup              Skip warmup phase");
+        System.out.println("  --help                   Show this help");
+        System.out.println();
+        System.out.println("Protocols:");
+        System.out.println("  ilp-tcp          Old ILP text protocol over TCP (default port: 9009)");
+        System.out.println("  ilp-http         Old ILP text protocol over HTTP (default port: 9000)");
+        System.out.println("  qwp-websocket    New QWP binary protocol over WebSocket (default port: 9000)");
+        System.out.println();
+        System.out.println("Examples:");
+        System.out.println("  QwpAllocationTestClient --protocol=qwp-websocket --rows=1000000 --batch=5000");
+        System.out.println("  QwpAllocationTestClient --protocol=ilp-tcp --host=remote-server");
+        System.out.println("  QwpAllocationTestClient --protocol=ilp-tcp --rows=100000 --no-warmup");
+    }
+
+    private static int getDefaultPort(String protocol) {
+        if (PROTOCOL_ILP_HTTP.equals(protocol) || PROTOCOL_QWP_WEBSOCKET.equals(protocol)) {
+            return 9000;
+        }
+        return 9009;
+    }
+
+    private static void runTest(String protocol, String host, int port, int totalRows,
+                                  int batchSize, int flushBytes, long flushIntervalMs,
+                                  int inFlightWindow, int sendQueue,
+                                  int warmupRows, int reportInterval) throws IOException {
+        System.out.println("Connecting to " + host + ":" + port + "...");
+
+        try (Sender sender = createSender(protocol, host, port, batchSize, flushBytes, flushIntervalMs,
+                inFlightWindow, sendQueue)) {
+            System.out.println("Connected! Protocol: " + protocol);
+            System.out.println();
+
+            // Warm-up phase
+            if (warmupRows > 0) {
+                System.out.println("Warming up (" + String.format("%,d", warmupRows) + " rows)...");
+                long warmupStart = System.nanoTime();
+                for (int i = 0; i < warmupRows; i++) {
+                    sendRow(sender, i);
+                }
+                sender.flush();
+                long warmupTime = System.nanoTime() - warmupStart;
+                System.out.println("Warmup complete in " + TimeUnit.NANOSECONDS.toMillis(warmupTime) + " ms");
+                System.out.println();
+
+                // Give GC a chance to clean up warmup allocations
+                System.gc();
+                Thread.sleep(100);
+            }
+
+            // Main test phase
+            System.out.println("Starting main test (" + String.format("%,d", totalRows) + " rows)...");
+            if (reportInterval > 0 && reportInterval <= totalRows) {
+                System.out.println("Progress will be reported every " + String.format("%,d", reportInterval) + " rows");
+            }
+            System.out.println();
+
+            long startTime = System.nanoTime();
+            long lastReportTime = startTime;
+            int lastReportRows = 0;
+
+            for (int i = 0; i < totalRows; i++) {
+                sendRow(sender, i);
+
+                // Report progress
+                if (reportInterval > 0 && (i + 1) % reportInterval == 0) {
+                    long now = System.nanoTime();
+                    long elapsedSinceReport = now - lastReportTime;
+                    int rowsSinceReport = (i + 1) - lastReportRows;
+                    double rowsPerSec = rowsSinceReport / (elapsedSinceReport / 1_000_000_000.0);
+
+                    System.out.printf("Progress: %,d / %,d rows (%.1f%%) - %.0f rows/sec%n",
+                            i + 1, totalRows,
+                            (i + 1) * 100.0 / totalRows,
+                            rowsPerSec);
+
+                    lastReportTime = now;
+                    lastReportRows = i + 1;
+                }
+            }
+
+            // Final flush
+            sender.flush();
+
+            long endTime = System.nanoTime();
+            long totalTime = endTime - startTime;
+            double totalSeconds = totalTime / 1_000_000_000.0;
+            double rowsPerSecond = totalRows / totalSeconds;
+
+            System.out.println();
+            System.out.println("Test Complete!");
+            System.out.println("==============");
+            System.out.println("Protocol: " + protocol);
+            System.out.println("Total rows: " + String.format("%,d", totalRows));
+            System.out.println("Batch size: " + String.format("%,d", batchSize));
+            System.out.println("Total time: " + String.format("%.2f", totalSeconds) + " seconds");
+            System.out.println("Throughput: " + String.format("%,.0f", rowsPerSecond) + " rows/second");
+            System.out.println("Data rate (before compression): " + String.format("%.2f", ((long)totalRows * estimatedRowSize()) / (1024.0 * 1024.0 * totalSeconds)) + " MB/s (estimated)");
+
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+            throw new IOException("Interrupted", e);
+        }
+    }
+
+    private static Sender createSender(String protocol, String host, int port,
+                                         int batchSize, int flushBytes, long flushIntervalMs,
+                                         int inFlightWindow, int sendQueue) {
+        switch (protocol) {
+            case PROTOCOL_ILP_TCP:
+                return Sender.builder(Sender.Transport.TCP)
+                        .address(host)
+                        .port(port)
+                        .build();
+            case PROTOCOL_ILP_HTTP:
+                return Sender.builder(Sender.Transport.HTTP)
+                        .address(host)
+                        .port(port)
+                        .autoFlushRows(batchSize)
+                        .build();
+            case PROTOCOL_QWP_WEBSOCKET:
+                Sender.LineSenderBuilder b = Sender.builder(Sender.Transport.WEBSOCKET)
+                        .address(host)
+                        .port(port)
+                        .asyncMode(true);
+                if (batchSize > 0) b.autoFlushRows(batchSize);
+                if (flushBytes > 0) b.autoFlushBytes(flushBytes);
+                if (flushIntervalMs > 0) b.autoFlushIntervalMillis((int) flushIntervalMs);
+                if (inFlightWindow > 0) b.inFlightWindowSize(inFlightWindow);
+                if (sendQueue > 0) b.sendQueueCapacity(sendQueue);
+                return b.build();
+            default:
+                throw new IllegalArgumentException("Unknown protocol: " + protocol +
+                        ". Use one of: ilp-tcp, ilp-http, qwp-websocket");
+        }
+    }
+
+    private static void sendRow(Sender sender, int rowIndex) {
+        // Base timestamp with small variations
+        long baseTimestamp = 1704067200000000L; // 2024-01-01 00:00:00 UTC in micros
+        long timestamp = baseTimestamp + (rowIndex * 1000L) + (rowIndex % 100);
+
+        sender.table("ilp_alloc_test")
+                // Symbol columns
+                .symbol("exchange", SYMBOLS[rowIndex % SYMBOLS.length])
+                .symbol("currency", rowIndex % 2 == 0 ? "USD" : "EUR")
+
+                // Numeric columns
+                .longColumn("trade_id", rowIndex)
+                .longColumn("volume", 100 + (rowIndex % 10000))
+                .doubleColumn("price", 100.0 + (rowIndex % 1000) * 0.01)
+                .doubleColumn("bid", 99.5 + (rowIndex % 1000) * 0.01)
+                .doubleColumn("ask", 100.5 + (rowIndex % 1000) * 0.01)
+                .longColumn("sequence", rowIndex % 1000000)
+                .doubleColumn("spread", 0.5 + (rowIndex % 100) * 0.01)
+
+                // String column
+                .stringColumn("venue", STRINGS[rowIndex % STRINGS.length])
+
+                // Boolean column
+                .boolColumn("is_buy", rowIndex % 2 == 0)
+
+                // Additional timestamp column
+                .timestampColumn("event_time", timestamp - 1000, ChronoUnit.MICROS)
+
+                // Designated timestamp
+                .at(timestamp, ChronoUnit.MICROS);
+    }
+
+    /**
+     * Estimates the size of a single row in bytes for throughput calculation.
+     */
+    private static int estimatedRowSize() {
+        // Rough estimate (binary protocol):
+        // - 2 symbols: ~10 bytes each = 20 bytes
+        // - 3 longs: 8 bytes each = 24 bytes
+        // - 4 doubles: 8 bytes each = 32 bytes
+        // - 1 string: ~10 bytes average
+        // - 1 boolean: 1 byte
+        // - 2 timestamps: 8 bytes each = 16 bytes
+        // - Overhead: ~20 bytes
+        // Total: ~123 bytes
+        return 123;
+    }
+}
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/StacBenchmarkClient.java b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/StacBenchmarkClient.java
new file mode 100644
index 0000000..abbf41f
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/StacBenchmarkClient.java
@@ -0,0 +1,424 @@
+/*******************************************************************************
+ *     ___                  _   ____  ____
+ *    / _ \ _   _  ___  ___| |_|  _ \| __ )
+ *   | | | | | | |/ _ \/ __| __| | | |  _ \
+ *   | |_| | |_| |  __/\__ \ |_| |_| | |_) |
+ *    \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ *  Copyright (c) 2014-2019 Appsicle
+ *  Copyright (c) 2019-2024 QuestDB
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.cutlass.line.tcp.v4;
+
+import io.questdb.client.Sender;
+
+import java.io.IOException;
+import java.time.temporal.ChronoUnit;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * STAC benchmark ingestion test client.
+ * 

+ * Tests ingestion performance for a STAC-like quotes table with this schema:
+ * 
+ * CREATE TABLE q (
+ *     s SYMBOL,     -- 4-letter ticker symbol (8512 unique)
+ *     x CHAR,       -- exchange code
+ *     b FLOAT,      -- bid price
+ *     a FLOAT,      -- ask price
+ *     v SHORT,      -- bid volume
+ *     w SHORT,      -- ask volume
+ *     m BOOLEAN,    -- market flag
+ *     T TIMESTAMP   -- designated timestamp
+ * ) timestamp(T) PARTITION BY DAY WAL;
+ * 

+ * 

+ * The table MUST be pre-created before running this test so the server uses
+ * the correct narrow column types (FLOAT, SHORT, CHAR). Otherwise ILP
+ * auto-creation would use DOUBLE, LONG, STRING.
+ * 

+ * Supports 3 protocol modes:
+ * 
    
+ *   
  • ilp-tcp: Old ILP text protocol over TCP (port 9009)
    • 
+ *   
  • ilp-http: Old ILP text protocol over HTTP (port 9000)
    • 
+ *   
  • qwp-websocket: New QWP binary protocol over WebSocket (port 9000)
    • 
+ * 

+ */
+public class StacBenchmarkClient {
+
+    private static final String PROTOCOL_ILP_TCP = "ilp-tcp";
+    private static final String PROTOCOL_ILP_HTTP = "ilp-http";
+    private static final String PROTOCOL_QWP_WEBSOCKET = "qwp-websocket";
+
+    private static final String DEFAULT_HOST = "localhost";
+    private static final int DEFAULT_ROWS = 80_000_000;
+    private static final int DEFAULT_BATCH_SIZE = 10_000;
+    private static final int DEFAULT_FLUSH_BYTES = 0;
+    private static final long DEFAULT_FLUSH_INTERVAL_MS = 0;
+    private static final int DEFAULT_IN_FLIGHT_WINDOW = 0;
+    private static final int DEFAULT_SEND_QUEUE = 0;
+    private static final int DEFAULT_WARMUP_ROWS = 100_000;
+    private static final int DEFAULT_REPORT_INTERVAL = 1_000_000;
+    private static final String DEFAULT_TABLE = "q";
+
+    // 8512 unique 4-letter symbols, as per STAC NYSE benchmark
+    private static final int SYMBOL_COUNT = 8512;
+    private static final String[] SYMBOLS = generateSymbols(SYMBOL_COUNT);
+
+    // Exchange codes (single characters)
+    private static final char[] EXCHANGES = {'N', 'Q', 'A', 'B', 'C', 'D', 'P', 'Z'};
+    // Pre-computed single-char strings to avoid allocation
+    private static final String[] EXCHANGE_STRINGS = new String[EXCHANGES.length];
+
+    static {
+        for (int i = 0; i < EXCHANGES.length; i++) {
+            EXCHANGE_STRINGS[i] = String.valueOf(EXCHANGES[i]);
+        }
+    }
+
+    // Pre-computed bid base prices per symbol (to generate realistic spreads)
+    private static final float[] BASE_PRICES = generateBasePrices(SYMBOL_COUNT);
+
+    public static void main(String[] args) {
+        String protocol = PROTOCOL_QWP_WEBSOCKET;
+        String host = DEFAULT_HOST;
+        int port = -1;
+        int totalRows = DEFAULT_ROWS;
+        int batchSize = DEFAULT_BATCH_SIZE;
+        int flushBytes = DEFAULT_FLUSH_BYTES;
+        long flushIntervalMs = DEFAULT_FLUSH_INTERVAL_MS;
+        int inFlightWindow = DEFAULT_IN_FLIGHT_WINDOW;
+        int sendQueue = DEFAULT_SEND_QUEUE;
+        int warmupRows = DEFAULT_WARMUP_ROWS;
+        int reportInterval = DEFAULT_REPORT_INTERVAL;
+        String table = DEFAULT_TABLE;
+
+        for (String arg : args) {
+            if (arg.equals("--help") || arg.equals("-h")) {
+                printUsage();
+                System.exit(0);
+            } else if (arg.startsWith("--protocol=")) {
+                protocol = arg.substring("--protocol=".length()).toLowerCase();
+            } else if (arg.startsWith("--host=")) {
+                host = arg.substring("--host=".length());
+            } else if (arg.startsWith("--port=")) {
+                port = Integer.parseInt(arg.substring("--port=".length()));
+            } else if (arg.startsWith("--rows=")) {
+                totalRows = Integer.parseInt(arg.substring("--rows=".length()));
+            } else if (arg.startsWith("--batch=")) {
+                batchSize = Integer.parseInt(arg.substring("--batch=".length()));
+            } else if (arg.startsWith("--flush-bytes=")) {
+                flushBytes = Integer.parseInt(arg.substring("--flush-bytes=".length()));
+            } else if (arg.startsWith("--flush-interval-ms=")) {
+                flushIntervalMs = Long.parseLong(arg.substring("--flush-interval-ms=".length()));
+            } else if (arg.startsWith("--in-flight-window=")) {
+                inFlightWindow = Integer.parseInt(arg.substring("--in-flight-window=".length()));
+            } else if (arg.startsWith("--send-queue=")) {
+                sendQueue = Integer.parseInt(arg.substring("--send-queue=".length()));
+            } else if (arg.startsWith("--warmup=")) {
+                warmupRows = Integer.parseInt(arg.substring("--warmup=".length()));
+            } else if (arg.startsWith("--report=")) {
+                reportInterval = Integer.parseInt(arg.substring("--report=".length()));
+            } else if (arg.startsWith("--table=")) {
+                table = arg.substring("--table=".length());
+            } else if (arg.equals("--no-warmup")) {
+                warmupRows = 0;
+            } else {
+                System.err.println("Unknown option: " + arg);
+                printUsage();
+                System.exit(1);
+            }
+        }
+
+        if (port == -1) {
+            port = getDefaultPort(protocol);
+        }
+
+        System.out.println("STAC Benchmark Ingestion Client");
+        System.out.println("================================");
+        System.out.println("Protocol: " + protocol);
+        System.out.println("Host: " + host);
+        System.out.println("Port: " + port);
+        System.out.println("Table: " + table);
+        System.out.println("Total rows: " + String.format("%,d", totalRows));
+        System.out.println("Batch size (rows): " + String.format("%,d", batchSize) + (batchSize == 0 ? " (default)" : ""));
+        System.out.println("Flush bytes: " + (flushBytes == 0 ? "(default)" : String.format("%,d", flushBytes)));
+        System.out.println("Flush interval: " + (flushIntervalMs == 0 ? "(default)" : flushIntervalMs + " ms"));
+        System.out.println("In-flight window: " + (inFlightWindow == 0 ? "(default: 8)" : inFlightWindow));
+        System.out.println("Send queue: " + (sendQueue == 0 ? "(default: 16)" : sendQueue));
+        System.out.println("Warmup rows: " + String.format("%,d", warmupRows));
+        System.out.println("Report interval: " + String.format("%,d", reportInterval));
+        System.out.println("Symbols: " + String.format("%,d", SYMBOL_COUNT) + " unique 4-letter tickers");
+        System.out.println();
+
+        try {
+            runTest(protocol, host, port, table, totalRows, batchSize, flushBytes, flushIntervalMs,
+                    inFlightWindow, sendQueue, warmupRows, reportInterval);
+        } catch (Exception e) {
+            System.err.println("Error: " + e.getMessage());
+            e.printStackTrace();
+            System.exit(1);
+        }
+    }
+
+    private static void printUsage() {
+        System.out.println("STAC Benchmark Ingestion Client");
+        System.out.println();
+        System.out.println("Tests ingestion performance for a STAC-like quotes table.");
+        System.out.println("The table must be pre-created with the correct schema.");
+        System.out.println();
+        System.out.println("Usage: StacBenchmarkClient [options]");
+        System.out.println();
+        System.out.println("Options:");
+        System.out.println("  --protocol=PROTOCOL      Protocol to use (default: qwp-websocket)");
+        System.out.println("  --host=HOST              Server host (default: localhost)");
+        System.out.println("  --port=PORT              Server port (default: 9009 for TCP, 9000 for HTTP/WebSocket)");
+        System.out.println("  --table=TABLE            Table name (default: q)");
+        System.out.println("  --rows=N                 Total rows to send (default: 80000000)");
+        System.out.println("  --batch=N                Auto-flush after N rows (default: 10000)");
+        System.out.println("  --flush-bytes=N          Auto-flush after N bytes (default: protocol default)");
+        System.out.println("  --flush-interval-ms=N    Auto-flush after N ms (default: protocol default)");
+        System.out.println("  --in-flight-window=N     Max batches awaiting server ACK (default: 8, WebSocket only)");
+        System.out.println("  --send-queue=N           Max batches waiting to send (default: 16, WebSocket only)");
+        System.out.println("  --warmup=N               Warmup rows (default: 100000)");
+        System.out.println("  --report=N               Report progress every N rows (default: 1000000)");
+        System.out.println("  --no-warmup              Skip warmup phase");
+        System.out.println("  --help                   Show this help");
+        System.out.println();
+        System.out.println("Protocols:");
+        System.out.println("  ilp-tcp          Old ILP text protocol over TCP (default port: 9009)");
+        System.out.println("  ilp-http         Old ILP text protocol over HTTP (default port: 9000)");
+        System.out.println("  qwp-websocket    New QWP binary protocol over WebSocket (default port: 9000)");
+        System.out.println();
+        System.out.println("Table schema (must be pre-created):");
+        System.out.println("  CREATE TABLE q (");
+        System.out.println("      s SYMBOL, x CHAR, b FLOAT, a FLOAT,");
+        System.out.println("      v SHORT, w SHORT, m BOOLEAN, T TIMESTAMP");
+        System.out.println("  ) timestamp(T) PARTITION BY DAY WAL;");
+    }
+
+    private static int getDefaultPort(String protocol) {
+        if (PROTOCOL_ILP_HTTP.equals(protocol) || PROTOCOL_QWP_WEBSOCKET.equals(protocol)) {
+            return 9000;
+        }
+        return 9009;
+    }
+
+    private static void runTest(String protocol, String host, int port, String table,
+                                int totalRows, int batchSize, int flushBytes, long flushIntervalMs,
+                                int inFlightWindow, int sendQueue,
+                                int warmupRows, int reportInterval) throws IOException {
+        System.out.println("Connecting to " + host + ":" + port + "...");
+
+        try (Sender sender = createSender(protocol, host, port, batchSize, flushBytes, flushIntervalMs,
+                inFlightWindow, sendQueue)) {
+            System.out.println("Connected! Protocol: " + protocol);
+            System.out.println();
+
+            // Warmup phase
+            if (warmupRows > 0) {
+                System.out.println("Warming up (" + String.format("%,d", warmupRows) + " rows)...");
+                long warmupStart = System.nanoTime();
+                for (int i = 0; i < warmupRows; i++) {
+                    sendQuoteRow(sender, table, i);
+                }
+                sender.flush();
+                long warmupTime = System.nanoTime() - warmupStart;
+                double warmupRowsPerSec = warmupRows / (warmupTime / 1_000_000_000.0);
+                System.out.printf("Warmup complete in %d ms (%.0f rows/sec)%n",
+                        TimeUnit.NANOSECONDS.toMillis(warmupTime), warmupRowsPerSec);
+                System.out.println();
+
+                System.gc();
+                Thread.sleep(100);
+            }
+
+            // Main test phase
+            System.out.println("Starting main test (" + String.format("%,d", totalRows) + " rows)...");
+            if (reportInterval > 0 && reportInterval <= totalRows) {
+                System.out.println("Progress will be reported every " + String.format("%,d", reportInterval) + " rows");
+            }
+            System.out.println();
+
+            long startTime = System.nanoTime();
+            long lastReportTime = startTime;
+            int lastReportRows = 0;
+
+            for (int i = 0; i < totalRows; i++) {
+                sendQuoteRow(sender, table, i);
+
+                if (reportInterval > 0 && (i + 1) % reportInterval == 0) {
+                    long now = System.nanoTime();
+                    long elapsedSinceReport = now - lastReportTime;
+                    int rowsSinceReport = (i + 1) - lastReportRows;
+                    double rowsPerSec = rowsSinceReport / (elapsedSinceReport / 1_000_000_000.0);
+                    long totalElapsed = now - startTime;
+                    double overallRowsPerSec = (i + 1) / (totalElapsed / 1_000_000_000.0);
+
+                    System.out.printf("Progress: %,d / %,d rows (%.1f%%) - %.0f rows/sec (interval) - %.0f rows/sec (overall)%n",
+                            i + 1, totalRows,
+                            (i + 1) * 100.0 / totalRows,
+                            rowsPerSec, overallRowsPerSec);
+
+                    lastReportTime = now;
+                    lastReportRows = i + 1;
+                }
+            }
+
+            sender.flush();
+
+            long endTime = System.nanoTime();
+            long totalTime = endTime - startTime;
+            double totalSeconds = totalTime / 1_000_000_000.0;
+            double rowsPerSecond = totalRows / totalSeconds;
+
+            System.out.println();
+            System.out.println("Test Complete!");
+            System.out.println("==============");
+            System.out.println("Protocol: " + protocol);
+            System.out.println("Table: " + table);
+            System.out.println("Total rows: " + String.format("%,d", totalRows));
+            System.out.println("Batch size: " + String.format("%,d", batchSize));
+            System.out.println("Total time: " + String.format("%.2f", totalSeconds) + " seconds");
+            System.out.println("Throughput: " + String.format("%,.0f", rowsPerSecond) + " rows/second");
+            System.out.println("Data rate (before compression): " + String.format("%.2f",
+                    ((long) totalRows * ESTIMATED_ROW_SIZE) / (1024.0 * 1024.0 * totalSeconds)) + " MB/s (estimated)");
+
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+            throw new IOException("Interrupted", e);
+        }
+    }
+
+    private static Sender createSender(String protocol, String host, int port,
+                                       int batchSize, int flushBytes, long flushIntervalMs,
+                                       int inFlightWindow, int sendQueue) {
+        switch (protocol) {
+            case PROTOCOL_ILP_TCP:
+                return Sender.builder(Sender.Transport.TCP)
+                        .address(host)
+                        .port(port)
+                        .build();
+            case PROTOCOL_ILP_HTTP:
+                return Sender.builder(Sender.Transport.HTTP)
+                        .address(host)
+                        .port(port)
+                        .autoFlushRows(batchSize)
+                        .build();
+            case PROTOCOL_QWP_WEBSOCKET:
+                Sender.LineSenderBuilder b = Sender.builder(Sender.Transport.WEBSOCKET)
+                        .address(host)
+                        .port(port)
+                        .asyncMode(true);
+                if (batchSize > 0) b.autoFlushRows(batchSize);
+                if (flushBytes > 0) b.autoFlushBytes(flushBytes);
+                if (flushIntervalMs > 0) b.autoFlushIntervalMillis((int) flushIntervalMs);
+                if (inFlightWindow > 0) b.inFlightWindowSize(inFlightWindow);
+                if (sendQueue > 0) b.sendQueueCapacity(sendQueue);
+                return b.build();
+            default:
+                throw new IllegalArgumentException("Unknown protocol: " + protocol +
+                        ". Use one of: ilp-tcp, ilp-http, qwp-websocket");
+        }
+    }
+
+    /**
+     * Sends a single quote row matching the STAC schema.
+     * 

+     * Schema: s SYMBOL, x CHAR, b FLOAT, a FLOAT, v SHORT, w SHORT, m BOOLEAN, T TIMESTAMP
+     * 

+     * The server downcasts doubleColumn->FLOAT, longColumn->SHORT, stringColumn->CHAR
+     * when the table is pre-created with the correct schema.
+     */
+    private static void sendQuoteRow(Sender sender, String table, int rowIndex) {
+        int symbolIdx = rowIndex % SYMBOL_COUNT;
+        int exchangeIdx = rowIndex % EXCHANGES.length;
+
+        // Bid/ask prices: base price with small variation
+        float basePrice = BASE_PRICES[symbolIdx];
+        // Use rowIndex bits for fast pseudo-random variation without Random object
+        float variation = ((rowIndex * 7 + symbolIdx * 13) % 200 - 100) * 0.01f;
+        float bid = basePrice + variation;
+        float ask = bid + 0.01f + (rowIndex % 10) * 0.01f; // spread: 1-10 cents
+
+        // Volumes: 100-32000 range fits SHORT
+        short bidVol = (short) (100 + ((rowIndex * 3 + symbolIdx) % 31901));
+        short askVol = (short) (100 + ((rowIndex * 7 + symbolIdx * 5) % 31901));
+
+        // Timestamp: 1 day of data with microsecond precision
+        // 86,400,000,000 micros per day, spread across totalRows
+        long baseTimestamp = 1704067200000000L; // 2024-01-01 00:00:00 UTC in micros
+        long timestamp = baseTimestamp + (rowIndex * 10L) + (rowIndex % 7);
+
+        sender.table(table)
+                .symbol("s", SYMBOLS[symbolIdx])
+                .stringColumn("x", EXCHANGE_STRINGS[exchangeIdx])
+                .doubleColumn("b", bid)
+                .doubleColumn("a", ask)
+                .longColumn("v", bidVol)
+                .longColumn("w", askVol)
+                .boolColumn("m", (rowIndex & 1) == 0)
+                .at(timestamp, ChronoUnit.MICROS);
+    }
+
+    /**
+     * Generates N unique 4-letter symbols.
+     * Uses combinations of uppercase letters to produce predictable, reproducible symbols.
+     */
+    private static String[] generateSymbols(int count) {
+        String[] symbols = new String[count];
+        int idx = 0;
+        // 26^4 = 456,976 possible 4-letter combinations, far more than 8512
+        outer:
+        for (char a = 'A'; a <= 'Z' && idx < count; a++) {
+            for (char b = 'A'; b <= 'Z' && idx < count; b++) {
+                for (char c = 'A'; c <= 'Z' && idx < count; c++) {
+                    for (char d = 'A'; d <= 'Z' && idx < count; d++) {
+                        symbols[idx++] = new String(new char[]{a, b, c, d});
+                        if (idx >= count) break outer;
+                    }
+                }
+            }
+        }
+        return symbols;
+    }
+
+    /**
+     * Generates pseudo-random base prices for each symbol.
+     * Prices range from $1 to $500 to simulate realistic stock prices.
+     */
+    private static float[] generateBasePrices(int count) {
+        float[] prices = new float[count];
+        Random rng = new Random(42); // fixed seed for reproducibility
+        for (int i = 0; i < count; i++) {
+            prices[i] = 1.0f + rng.nextFloat() * 499.0f;
+        }
+        return prices;
+    }
+
+    // Estimated row size for throughput calculation:
+    // - 1 symbol: ~6 bytes (4-char + overhead)
+    // - 1 char: 2 bytes
+    // - 2 floats: 4 bytes each = 8 bytes
+    // - 2 shorts: 2 bytes each = 4 bytes
+    // - 1 boolean: 1 byte
+    // - 1 timestamp: 8 bytes
+    // - overhead: ~10 bytes
+    // Total: ~39 bytes
+    private static final int ESTIMATED_ROW_SIZE = 39;
+}
\ No newline at end of file

From 992cd8ed63da5fc24aa392e9b6767c1debeba431 Mon Sep 17 00:00:00 2001
From: bluestreak 
Date: Mon, 23 Feb 2026 21:47:41 +0000
Subject: [PATCH 16/89] wip 13

---
 .../qwp/client/QwpWebSocketEncoder.java       |  36 +--
 .../qwp/protocol/OffHeapAppendMemory.java     |  32 ++
 .../cutlass/qwp/protocol/QwpTableBuffer.java  | 298 +++++++++---------
 .../qwp/protocol/OffHeapAppendMemoryTest.java |  77 +++++
 4 files changed, 276 insertions(+), 167 deletions(-)

diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java
index 8f8d2ac..0ec22b2 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java
@@ -24,6 +24,7 @@
 
 package io.questdb.client.cutlass.qwp.client;
 
+import io.questdb.client.cutlass.line.LineSenderException;
 import io.questdb.client.cutlass.qwp.protocol.QwpColumnDef;
 import io.questdb.client.cutlass.qwp.protocol.QwpGorillaEncoder;
 import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer;
@@ -204,7 +205,7 @@ private void encodeColumn(QwpTableBuffer.ColumnBuffer col, QwpColumnDef colDef,
                 break;
             case TYPE_STRING:
             case TYPE_VARCHAR:
-                writeStringColumn(col.getStringValues(), valueCount);
+                writeStringColumn(col, valueCount);
                 break;
             case TYPE_SYMBOL:
                 writeSymbolColumn(col, valueCount);
@@ -233,7 +234,7 @@ private void encodeColumn(QwpTableBuffer.ColumnBuffer col, QwpColumnDef colDef,
                 writeDecimal256Column(col.getDecimalScale(), dataAddr, valueCount);
                 break;
             default:
-                throw new IllegalStateException("Unknown column type: " + col.getType());
+                throw new LineSenderException("Unknown column type: " + col.getType());
         }
     }
 
@@ -281,7 +282,7 @@ private void encodeColumnWithGlobalSymbols(QwpTableBuffer.ColumnBuffer col, QwpC
                     break;
                 case TYPE_STRING:
                 case TYPE_VARCHAR:
-                    writeStringColumn(col.getStringValues(), valueCount);
+                    writeStringColumn(col, valueCount);
                     break;
                 case TYPE_UUID:
                     buffer.putBlockOfBytes(dataAddr, (long) valueCount * 16);
@@ -305,7 +306,7 @@ private void encodeColumnWithGlobalSymbols(QwpTableBuffer.ColumnBuffer col, QwpC
                     writeDecimal256Column(col.getDecimalScale(), dataAddr, valueCount);
                     break;
                 default:
-                    throw new IllegalStateException("Unknown column type: " + col.getType());
+                    throw new LineSenderException("Unknown column type: " + col.getType());
             }
         }
     }
@@ -482,28 +483,11 @@ private void writeNullBitmap(QwpTableBuffer.ColumnBuffer col, int rowCount) {
         }
     }
 
-    private void writeStringColumn(String[] strings, int count) {
-        int totalDataLen = 0;
-        for (int i = 0; i < count; i++) {
-            if (strings[i] != null) {
-                totalDataLen += QwpBufferWriter.utf8Length(strings[i]);
-            }
-        }
-
-        int runningOffset = 0;
-        buffer.putInt(0);
-        for (int i = 0; i < count; i++) {
-            if (strings[i] != null) {
-                runningOffset += QwpBufferWriter.utf8Length(strings[i]);
-            }
-            buffer.putInt(runningOffset);
-        }
-
-        for (int i = 0; i < count; i++) {
-            if (strings[i] != null) {
-                buffer.putUtf8(strings[i]);
-            }
-        }
+    private void writeStringColumn(QwpTableBuffer.ColumnBuffer col, int valueCount) {
+        // Offset array: (valueCount + 1) int32 values, pre-built in wire format
+        buffer.putBlockOfBytes(col.getStringOffsetsAddress(), (long) (valueCount + 1) * 4);
+        // UTF-8 data: raw bytes, contiguous
+        buffer.putBlockOfBytes(col.getStringDataAddress(), col.getStringDataSize());
     }
 
     /**
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java
index f4c14cc..5830ea7 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java
@@ -131,6 +131,38 @@ public void putDouble(double value) {
         appendAddress += 8;
     }
 
+    /**
+     * Encodes a Java String to UTF-8 directly into the off-heap buffer.
+     * Pre-ensures worst-case capacity to avoid per-byte checks.
+     */
+    public void putUtf8(String value) {
+        if (value == null || value.isEmpty()) {
+            return;
+        }
+        int len = value.length();
+        ensureCapacity((long) len * 4); // worst case: all supplementary chars
+        for (int i = 0; i < len; i++) {
+            char c = value.charAt(i);
+            if (c < 0x80) {
+                Unsafe.getUnsafe().putByte(appendAddress++, (byte) c);
+            } else if (c < 0x800) {
+                Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0xC0 | (c >> 6)));
+                Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0x80 | (c & 0x3F)));
+            } else if (c >= 0xD800 && c <= 0xDBFF && i + 1 < len) {
+                char c2 = value.charAt(++i);
+                int codePoint = 0x10000 + ((c - 0xD800) << 10) + (c2 - 0xDC00);
+                Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0xF0 | (codePoint >> 18)));
+                Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0x80 | ((codePoint >> 12) & 0x3F)));
+                Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0x80 | ((codePoint >> 6) & 0x3F)));
+                Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0x80 | (codePoint & 0x3F)));
+            } else {
+                Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0xE0 | (c >> 12)));
+                Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0x80 | ((c >> 6) & 0x3F)));
+                Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0x80 | (c & 0x3F)));
+            }
+        }
+    }
+
     /**
      * Advances the append position by the given number of bytes without writing.
      */
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java
index 9e97b4c..b2c434c 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java
@@ -52,16 +52,16 @@
  */
 public class QwpTableBuffer implements QuietCloseable {
 
-    private final String tableName;
-    private final ObjList columns;
     private final CharSequenceIntHashMap columnNameToIndex;
-    private ColumnBuffer[] fastColumns; // plain array for O(1) sequential access
+    private final ObjList columns;
+    private final String tableName;
+    private QwpColumnDef[] cachedColumnDefs;
     private int columnAccessCursor; // tracks expected next column index
+    private boolean columnDefsCacheValid;
+    private ColumnBuffer[] fastColumns; // plain array for O(1) sequential access
     private int rowCount;
     private long schemaHash;
     private boolean schemaHashComputed;
-    private QwpColumnDef[] cachedColumnDefs;
-    private boolean columnDefsCacheValid;
 
     public QwpTableBuffer(String tableName) {
         this.tableName = tableName;
@@ -156,7 +156,7 @@ public ColumnBuffer getOrCreateColumn(String name, byte type, boolean nullable)
             if (candidate.name.equals(name)) {
                 columnAccessCursor++;
                 if (candidate.type != type) {
-                    throw new IllegalArgumentException(
+                    throw new LineSenderException(
                             "Column type mismatch for " + name + ": existing=" + candidate.type + " new=" + type
                     );
                 }
@@ -169,7 +169,7 @@ public ColumnBuffer getOrCreateColumn(String name, byte type, boolean nullable)
         if (idx != CharSequenceIntHashMap.NO_ENTRY_VALUE) {
             ColumnBuffer existing = columns.get(idx);
             if (existing.type != type) {
-                throw new IllegalArgumentException(
+                throw new LineSenderException(
                         "Column type mismatch for " + name + ": existing=" + existing.type + " new=" + type
                 );
             }
@@ -290,6 +290,66 @@ static int elementSize(byte type) {
         }
     }
 
+    /**
+     * Helper class to capture array data from DoubleArray/LongArray.appendToBufPtr().
+     */
+    private static class ArrayCapture implements ArrayBufferAppender {
+        double[] doubleData;
+        int doubleDataOffset;
+        long[] longData;
+        int longDataOffset;
+        byte nDims;
+        int[] shape = new int[32];
+        int shapeIndex;
+
+        @Override
+        public void putBlockOfBytes(long from, long len) {
+            int count = (int) (len / 8);
+            if (doubleData == null) {
+                doubleData = new double[count];
+            }
+            for (int i = 0; i < count; i++) {
+                doubleData[doubleDataOffset++] = Unsafe.getUnsafe().getDouble(from + i * 8L);
+            }
+        }
+
+        @Override
+        public void putByte(byte b) {
+            if (shapeIndex == 0) {
+                nDims = b;
+            }
+        }
+
+        @Override
+        public void putDouble(double value) {
+            if (doubleData != null && doubleDataOffset < doubleData.length) {
+                doubleData[doubleDataOffset++] = value;
+            }
+        }
+
+        @Override
+        public void putInt(int value) {
+            if (shapeIndex < nDims) {
+                shape[shapeIndex++] = value;
+                if (shapeIndex == nDims) {
+                    int totalElements = 1;
+                    for (int i = 0; i < nDims; i++) {
+                        totalElements *= shape[i];
+                    }
+                    doubleData = new double[totalElements];
+                    longData = new long[totalElements];
+                }
+            }
+        }
+
+        @Override
+        public void putLong(long value) {
+            if (longData != null && longDataOffset < longData.length) {
+                longData[longDataOffset++] = value;
+            }
+        }
+    }
+
     /**
      * Column buffer for a single column.
      * 

@@ -297,47 +357,37 @@ static int elementSize(byte type) {
      * operation and efficient bulk copy to network buffers.
      */
     public static class ColumnBuffer implements QuietCloseable {
+        final int elemSize;
         final String name;
-        final byte type;
         final boolean nullable;
-        final int elemSize;
-
-        private int size;         // Total row count (including nulls)
-        private int valueCount;   // Actual stored values (excludes nulls)
-
-        // Off-heap data buffer for fixed-width types
-        private OffHeapAppendMemory dataBuffer;
-
-        // Off-heap auxiliary buffer for global symbol IDs (SYMBOL type only)
-        private OffHeapAppendMemory auxBuffer;
-
-        // Off-heap null bitmap (bit-packed, 1 bit per row)
-        private long nullBufPtr;
-        private int nullBufCapRows;
-        private boolean hasNulls;
-
-        // On-heap capacity for variable-width arrays (string values, array dims)
-        private int onHeapCapacity;
-
-        // On-heap storage for variable-width types
-        private String[] stringValues;
-
+        final byte type;
+        private final Decimal256 rescaleTemp = new Decimal256();
+        private int arrayDataOffset;
         // Array storage (double/long arrays - variable length per row)
         private byte[] arrayDims;
-        private int[] arrayShapes;
         private int arrayShapeOffset;
+        private int[] arrayShapes;
+        // Off-heap auxiliary buffer for global symbol IDs (SYMBOL type only)
+        private OffHeapAppendMemory auxBuffer;
+        // Off-heap data buffer for fixed-width types
+        private OffHeapAppendMemory dataBuffer;
+        // Decimal storage
+        private byte decimalScale = -1;
         private double[] doubleArrayData;
+        private boolean hasNulls;
         private long[] longArrayData;
-        private int arrayDataOffset;
-
+        private int maxGlobalSymbolId = -1;
+        private int nullBufCapRows;
+        // Off-heap null bitmap (bit-packed, 1 bit per row)
+        private long nullBufPtr;
+        private int size;         // Total row count (including nulls)
+        private OffHeapAppendMemory stringData;
+        // Off-heap storage for string/varchar column data
+        private OffHeapAppendMemory stringOffsets;
         // Symbol specific (dictionary stays on-heap)
         private CharSequenceIntHashMap symbolDict;
         private ObjList symbolList;
-        private int maxGlobalSymbolId = -1;
-
-        // Decimal storage
-        private byte decimalScale = -1;
-        private final Decimal256 rescaleTemp = new Decimal256();
+        private int valueCount;   // Actual stored values (excludes nulls)
 
         public ColumnBuffer(String name, byte type, boolean nullable) {
             this.name = name;
@@ -347,7 +397,6 @@ public ColumnBuffer(String name, byte type, boolean nullable) {
             this.size = 0;
             this.valueCount = 0;
             this.hasNulls = false;
-            this.onHeapCapacity = 16;
 
             allocateStorage(type);
             if (nullable) {
@@ -358,12 +407,14 @@ public ColumnBuffer(String name, byte type, boolean nullable) {
         }
 
         public void addBoolean(boolean value) {
+            ensureNullBitmapForNonNull();
             dataBuffer.putByte(value ? (byte) 1 : (byte) 0);
             valueCount++;
             size++;
         }
 
         public void addByte(byte value) {
+            ensureNullBitmapForNonNull();
             dataBuffer.putByte(value);
             valueCount++;
             size++;
@@ -374,6 +425,7 @@ public void addDecimal128(Decimal128 value) {
                 addNull();
                 return;
             }
+            ensureNullBitmapForNonNull();
             if (decimalScale == -1) {
                 decimalScale = (byte) value.getScale();
             } else if (decimalScale != value.getScale()) {
@@ -397,6 +449,7 @@ public void addDecimal256(Decimal256 value) {
                 addNull();
                 return;
             }
+            ensureNullBitmapForNonNull();
             Decimal256 src = value;
             if (decimalScale == -1) {
                 decimalScale = (byte) value.getScale();
@@ -418,6 +471,7 @@ public void addDecimal64(Decimal64 value) {
                 addNull();
                 return;
             }
+            ensureNullBitmapForNonNull();
             if (decimalScale == -1) {
                 decimalScale = (byte) value.getScale();
                 dataBuffer.putLong(value.getValue());
@@ -434,6 +488,7 @@ public void addDecimal64(Decimal64 value) {
         }
 
         public void addDouble(double value) {
+            ensureNullBitmapForNonNull();
             dataBuffer.putDouble(value);
             valueCount++;
             size++;
@@ -534,24 +589,28 @@ public void addDoubleArray(DoubleArray array) {
         }
 
         public void addFloat(float value) {
+            ensureNullBitmapForNonNull();
             dataBuffer.putFloat(value);
             valueCount++;
             size++;
         }
 
         public void addInt(int value) {
+            ensureNullBitmapForNonNull();
             dataBuffer.putInt(value);
             valueCount++;
             size++;
         }
 
         public void addLong(long value) {
+            ensureNullBitmapForNonNull();
             dataBuffer.putLong(value);
             valueCount++;
             size++;
         }
 
         public void addLong256(long l0, long l1, long l2, long l3) {
+            ensureNullBitmapForNonNull();
             dataBuffer.putLong(l0);
             dataBuffer.putLong(l1);
             dataBuffer.putLong(l2);
@@ -689,8 +748,7 @@ public void addNull() {
                         break;
                     case TYPE_STRING:
                     case TYPE_VARCHAR:
-                        ensureOnHeapCapacity();
-                        stringValues[valueCount] = null;
+                        stringOffsets.putInt((int) stringData.getAppendOffset());
                         break;
                     case TYPE_SYMBOL:
                         dataBuffer.putInt(-1);
@@ -725,6 +783,7 @@ public void addNull() {
         }
 
         public void addShort(short value) {
+            ensureNullBitmapForNonNull();
             dataBuffer.putShort(value);
             valueCount++;
             size++;
@@ -734,12 +793,15 @@ public void addString(String value) {
             if (value == null && nullable) {
                 ensureNullCapacity(size + 1);
                 markNull(size);
-                size++;
             } else {
-                ensureOnHeapCapacity();
-                stringValues[valueCount++] = value;
-                size++;
+                ensureNullBitmapForNonNull();
+                if (value != null) {
+                    stringData.putUtf8(value);
+                }
+                stringOffsets.putInt((int) stringData.getAppendOffset());
+                valueCount++;
             }
+            size++;
         }
 
         public void addSymbol(String value) {
@@ -748,8 +810,8 @@ public void addSymbol(String value) {
                     ensureNullCapacity(size + 1);
                     markNull(size);
                 }
-                size++;
             } else {
+                ensureNullBitmapForNonNull();
                 int idx = symbolDict.get(value);
                 if (idx == CharSequenceIntHashMap.NO_ENTRY_VALUE) {
                     idx = symbolList.size();
@@ -758,8 +820,8 @@ public void addSymbol(String value) {
                 }
                 dataBuffer.putInt(idx);
                 valueCount++;
-                size++;
             }
+            size++;
         }
 
         public void addSymbolWithGlobalId(String value, int globalId) {
@@ -770,6 +832,7 @@ public void addSymbolWithGlobalId(String value, int globalId) {
                 }
                 size++;
             } else {
+                ensureNullBitmapForNonNull();
                 int localIdx = symbolDict.get(value);
                 if (localIdx == CharSequenceIntHashMap.NO_ENTRY_VALUE) {
                     localIdx = symbolList.size();
@@ -793,6 +856,7 @@ public void addSymbolWithGlobalId(String value, int globalId) {
         }
 
         public void addUuid(long high, long low) {
+            ensureNullBitmapForNonNull();
             // Store in wire order: lo first, hi second
             dataBuffer.putLong(low);
             dataBuffer.putLong(high);
@@ -810,6 +874,14 @@ public void close() {
                 auxBuffer.close();
                 auxBuffer = null;
             }
+            if (stringOffsets != null) {
+                stringOffsets.close();
+                stringOffsets = null;
+            }
+            if (stringData != null) {
+                stringData.close();
+                stringData = null;
+            }
             if (nullBufPtr != 0) {
                 Unsafe.free(nullBufPtr, (long) nullBufCapRows >>> 3, MemoryTag.NATIVE_ILP_RSS);
                 nullBufPtr = 0;
@@ -825,14 +897,14 @@ public byte[] getArrayDims() {
             return arrayDims;
         }
 
-        public int[] getArrayShapes() {
-            return arrayShapes;
-        }
-
         public int getArrayShapeOffset() {
             return arrayShapeOffset;
         }
 
+        public int[] getArrayShapes() {
+            return arrayShapes;
+        }
+
         /**
          * Returns the off-heap address of the auxiliary data buffer (global symbol IDs).
          * Returns 0 if no auxiliary data exists.
@@ -883,28 +955,20 @@ public long getNullBitmapAddress() {
             return nullBufPtr;
         }
 
-        /**
-         * Returns the bit-packed null bitmap as a long array.
-         * This creates a new array from off-heap data.
-         */
-        public long[] getNullBitmapPacked() {
-            if (nullBufPtr == 0) {
-                return null;
-            }
-            int longCount = (size + 63) >>> 6;
-            long[] result = new long[longCount];
-            for (int i = 0; i < longCount; i++) {
-                result[i] = Unsafe.getUnsafe().getLong(nullBufPtr + (long) i * 8);
-            }
-            return result;
-        }
-
         public int getSize() {
             return size;
         }
 
-        public String[] getStringValues() {
-            return stringValues;
+        public long getStringDataAddress() {
+            return stringData != null ? stringData.pageAddress() : 0;
+        }
+
+        public long getStringDataSize() {
+            return stringData != null ? stringData.getAppendOffset() : 0;
+        }
+
+        public long getStringOffsetsAddress() {
+            return stringOffsets != null ? stringOffsets.pageAddress() : 0;
         }
 
         public String[] getSymbolDictionary() {
@@ -953,6 +1017,13 @@ public void reset() {
             if (auxBuffer != null) {
                 auxBuffer.truncate();
             }
+            if (stringOffsets != null) {
+                stringOffsets.truncate();
+                stringOffsets.putInt(0); // re-seed initial 0 offset
+            }
+            if (stringData != null) {
+                stringData.truncate();
+            }
             if (nullBufPtr != 0) {
                 Vect.memset(nullBufPtr, (long) nullBufCapRows >>> 3, 0);
             }
@@ -1003,6 +1074,13 @@ public void truncateTo(int newSize) {
                 dataBuffer.jumpTo((long) newValueCount * elemSize);
             }
 
+            // Rewind string buffers
+            if (stringOffsets != null) {
+                int dataOffset = Unsafe.getUnsafe().getInt(stringOffsets.pageAddress() + (long) newValueCount * 4);
+                stringData.jumpTo(dataOffset);
+                stringOffsets.jumpTo((long) (newValueCount + 1) * 4);
+            }
+
             // Rewind aux buffer (symbol global IDs)
             if (auxBuffer != null) {
                 auxBuffer.jumpTo((long) newValueCount * 4);
@@ -1036,7 +1114,9 @@ private void allocateStorage(byte type) {
                     break;
                 case TYPE_STRING:
                 case TYPE_VARCHAR:
-                    stringValues = new String[onHeapCapacity];
+                    stringOffsets = new OffHeapAppendMemory(64);
+                    stringOffsets.putInt(0); // seed initial 0 offset
+                    stringData = new OffHeapAppendMemory(256);
                     break;
                 case TYPE_SYMBOL:
                     dataBuffer = new OffHeapAppendMemory(64);
@@ -1051,7 +1131,7 @@ private void allocateStorage(byte type) {
                     break;
                 case TYPE_DOUBLE_ARRAY:
                 case TYPE_LONG_ARRAY:
-                    arrayDims = new byte[onHeapCapacity];
+                    arrayDims = new byte[16];
                     break;
                 case TYPE_DECIMAL64:
                     dataBuffer = new OffHeapAppendMemory(128);
@@ -1101,6 +1181,12 @@ private void ensureArrayCapacity(int nDims, int dataElements) {
             }
         }
 
+        private void ensureNullBitmapForNonNull() {
+            if (nullBufPtr != 0) {
+                ensureNullCapacity(size + 1);
+            }
+        }
+
         private void ensureNullCapacity(int rows) {
             if (rows > nullBufCapRows) {
                 int newCapRows = Math.max(nullBufCapRows * 2, ((rows + 63) >>> 6) << 6);
@@ -1112,16 +1198,6 @@ private void ensureNullCapacity(int rows) {
             }
         }
 
-        private void ensureOnHeapCapacity() {
-            if (valueCount >= onHeapCapacity) {
-                int newCapacity = onHeapCapacity * 2;
-                if (stringValues != null) {
-                    stringValues = Arrays.copyOf(stringValues, newCapacity);
-                }
-                onHeapCapacity = newCapacity;
-            }
-        }
-
         private void markNull(int index) {
             long longAddr = nullBufPtr + ((long) (index >>> 6)) * 8;
             int bitIndex = index & 63;
@@ -1130,64 +1206,4 @@ private void markNull(int index) {
             hasNulls = true;
         }
     }
-
-    /**
-     * Helper class to capture array data from DoubleArray/LongArray.appendToBufPtr().
-     */
-    private static class ArrayCapture implements ArrayBufferAppender {
-        byte nDims;
-        int[] shape = new int[32];
-        int shapeIndex;
-        double[] doubleData;
-        int doubleDataOffset;
-        long[] longData;
-        int longDataOffset;
-
-        @Override
-        public void putBlockOfBytes(long from, long len) {
-            int count = (int) (len / 8);
-            if (doubleData == null) {
-                doubleData = new double[count];
-            }
-            for (int i = 0; i < count; i++) {
-                doubleData[doubleDataOffset++] = Unsafe.getUnsafe().getDouble(from + i * 8L);
-            }
-        }
-
-        @Override
-        public void putByte(byte b) {
-            if (shapeIndex == 0) {
-                nDims = b;
-            }
-        }
-
-        @Override
-        public void putDouble(double value) {
-            if (doubleData != null && doubleDataOffset < doubleData.length) {
-                doubleData[doubleDataOffset++] = value;
-            }
-        }
-
-        @Override
-        public void putInt(int value) {
-            if (shapeIndex < nDims) {
-                shape[shapeIndex++] = value;
-                if (shapeIndex == nDims) {
-                    int totalElements = 1;
-                    for (int i = 0; i < nDims; i++) {
-                        totalElements *= shape[i];
-                    }
-                    doubleData = new double[totalElements];
-                    longData = new long[totalElements];
-                }
-            }
-        }
-
-        @Override
-        public void putLong(long value) {
-            if (longData != null && longDataOffset < longData.length) {
-                longData[longDataOffset++] = value;
-            }
-        }
-    }
 }
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/OffHeapAppendMemoryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/OffHeapAppendMemoryTest.java
index 96c39a3..073bf27 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/OffHeapAppendMemoryTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/OffHeapAppendMemoryTest.java
@@ -263,4 +263,81 @@ public void testLargeGrowth() {
         long after = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS);
         assertEquals(before, after);
     }
+
+    @Test
+    public void testPutUtf8Ascii() {
+        long before = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS);
+        try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) {
+            mem.putUtf8("hello");
+            assertEquals(5, mem.getAppendOffset());
+            assertEquals('h', Unsafe.getUnsafe().getByte(mem.addressOf(0)));
+            assertEquals('e', Unsafe.getUnsafe().getByte(mem.addressOf(1)));
+            assertEquals('l', Unsafe.getUnsafe().getByte(mem.addressOf(2)));
+            assertEquals('l', Unsafe.getUnsafe().getByte(mem.addressOf(3)));
+            assertEquals('o', Unsafe.getUnsafe().getByte(mem.addressOf(4)));
+        }
+        long after = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS);
+        assertEquals(before, after);
+    }
+
+    @Test
+    public void testPutUtf8Empty() {
+        try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) {
+            mem.putUtf8("");
+            assertEquals(0, mem.getAppendOffset());
+        }
+    }
+
+    @Test
+    public void testPutUtf8MultiByte() {
+        try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) {
+            // 2-byte: U+00E9 (e-acute) = C3 A9
+            mem.putUtf8("\u00E9");
+            assertEquals(2, mem.getAppendOffset());
+            assertEquals((byte) 0xC3, Unsafe.getUnsafe().getByte(mem.addressOf(0)));
+            assertEquals((byte) 0xA9, Unsafe.getUnsafe().getByte(mem.addressOf(1)));
+        }
+    }
+
+    @Test
+    public void testPutUtf8Null() {
+        try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) {
+            mem.putUtf8(null);
+            assertEquals(0, mem.getAppendOffset());
+        }
+    }
+
+    @Test
+    public void testPutUtf8SurrogatePairs() {
+        try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) {
+            // U+1F600 (grinning face) = F0 9F 98 80
+            mem.putUtf8("\uD83D\uDE00");
+            assertEquals(4, mem.getAppendOffset());
+            assertEquals((byte) 0xF0, Unsafe.getUnsafe().getByte(mem.addressOf(0)));
+            assertEquals((byte) 0x9F, Unsafe.getUnsafe().getByte(mem.addressOf(1)));
+            assertEquals((byte) 0x98, Unsafe.getUnsafe().getByte(mem.addressOf(2)));
+            assertEquals((byte) 0x80, Unsafe.getUnsafe().getByte(mem.addressOf(3)));
+        }
+    }
+
+    @Test
+    public void testPutUtf8ThreeByte() {
+        try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) {
+            // 3-byte: U+4E16 (CJK character) = E4 B8 96
+            mem.putUtf8("\u4E16");
+            assertEquals(3, mem.getAppendOffset());
+            assertEquals((byte) 0xE4, Unsafe.getUnsafe().getByte(mem.addressOf(0)));
+            assertEquals((byte) 0xB8, Unsafe.getUnsafe().getByte(mem.addressOf(1)));
+            assertEquals((byte) 0x96, Unsafe.getUnsafe().getByte(mem.addressOf(2)));
+        }
+    }
+
+    @Test
+    public void testPutUtf8Mixed() {
+        try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) {
+            // Mix: ASCII "A" (1 byte) + e-acute (2 bytes) + CJK (3 bytes) + emoji (4 bytes) = 10 bytes
+            mem.putUtf8("A\u00E9\u4E16\uD83D\uDE00");
+            assertEquals(10, mem.getAppendOffset());
+        }
+    }
 }

From 332b66f44436c3d2750f37acdd3ea21a14e85ad4 Mon Sep 17 00:00:00 2001
From: Marko Topolnik 
Date: Tue, 24 Feb 2026 11:22:03 +0100
Subject: [PATCH 17/89] Fix pong frame clobbering in-progress send buffer

sendPongFrame() used the shared sendBuffer, calling reset()
which destroyed any partially-built frame the caller had in
progress via getSendBuffer(). This could happen when a PING
arrived during receiveFrame()/tryReceiveFrame() while the
caller was mid-way through constructing a data frame.

Add a dedicated 256-byte controlFrameBuffer for sending pong
responses. RFC 6455 limits control frame payloads to 125 bytes
plus a 14-byte max header, so 256 bytes is sufficient and never
needs to grow.

Co-Authored-By: Claude Opus 4.6 
---
 .../cutlass/http/client/WebSocketClient.java       | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java
index ab8f696..cfe53dc 100644
--- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java
+++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java
@@ -76,6 +76,7 @@ public abstract class WebSocketClient implements QuietCloseable {
     protected final Socket socket;
 
     private final WebSocketSendBuffer sendBuffer;
+    private final WebSocketSendBuffer controlFrameBuffer;
     private final WebSocketFrameParser frameParser;
     private final Rnd rnd;
     private final int defaultTimeout;
@@ -103,6 +104,10 @@ public WebSocketClient(HttpClientConfiguration configuration, SocketFactory sock
         int sendBufSize = Math.max(configuration.getInitialRequestBufferSize(), DEFAULT_SEND_BUFFER_SIZE);
         int maxSendBufSize = Math.max(configuration.getMaximumRequestBufferSize(), sendBufSize);
         this.sendBuffer = new WebSocketSendBuffer(sendBufSize, maxSendBufSize);
+        // Control frames (ping/pong/close) have max 125-byte payload + 14-byte header.
+        // This dedicated buffer prevents sendPongFrame from clobbering an in-progress
+        // frame being built in the main sendBuffer.
+        this.controlFrameBuffer = new WebSocketSendBuffer(256, 256);
 
         this.recvBufSize = Math.max(configuration.getResponseBufferSize(), DEFAULT_RECV_BUFFER_SIZE);
         this.recvBufPtr = Unsafe.malloc(recvBufSize, MemoryTag.NATIVE_DEFAULT);
@@ -131,6 +136,7 @@ public void close() {
 
             disconnect();
             sendBuffer.close();
+            controlFrameBuffer.close();
 
             if (recvBufPtr != 0) {
                 Unsafe.free(recvBufPtr, recvBufSize, MemoryTag.NATIVE_DEFAULT);
@@ -641,10 +647,10 @@ private Boolean tryParseFrame(WebSocketFrameHandler handler) {
 
     private void sendPongFrame(long payloadPtr, int payloadLen) {
         try {
-            sendBuffer.reset();
-            WebSocketSendBuffer.FrameInfo frame = sendBuffer.writePongFrame(payloadPtr, payloadLen);
-            doSend(sendBuffer.getBufferPtr() + frame.offset, frame.length, 1000); // Short timeout for pong
-            sendBuffer.reset();
+            controlFrameBuffer.reset();
+            WebSocketSendBuffer.FrameInfo frame = controlFrameBuffer.writePongFrame(payloadPtr, payloadLen);
+            doSend(controlFrameBuffer.getBufferPtr() + frame.offset, frame.length, 1000);
+            controlFrameBuffer.reset();
         } catch (Exception e) {
             LOG.error("Failed to send pong: {}", e.getMessage());
         }

From 82955d057840c692181fd03402ff5e386cd7665e Mon Sep 17 00:00:00 2001
From: Marko Topolnik 
Date: Tue, 24 Feb 2026 12:04:38 +0100
Subject: [PATCH 18/89] Fix buffer overrun in WebSocket close frame

sendCloseFrame() used reason.length() (UTF-16 code units) to
calculate the payload size, but wrote reason.getBytes(UTF_8)
(UTF-8 bytes) into the buffer. For non-ASCII close reasons,
UTF-8 encoding can be longer than the UTF-16 length, causing
writes past the declared payload size. This corrupted the
frame header length, the masking range, and could overrun the
allocated buffer.

Compute the UTF-8 byte array upfront and use its length for
all sizing calculations.

Co-Authored-By: Claude Opus 4.6 
---
 .../client/cutlass/qwp/client/WebSocketChannel.java        | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketChannel.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketChannel.java
index 8774ac2..f9584f4 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketChannel.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketChannel.java
@@ -429,7 +429,9 @@ private void sendCloseFrame(int code, String reason) {
         int maskKey = rnd.nextInt();
 
         // Close payload: 2-byte code + optional reason
-        int reasonLen = (reason != null) ? reason.length() : 0;
+        // Compute UTF-8 bytes upfront so payload length is correct
+        byte[] reasonBytes = (reason != null) ? reason.getBytes(StandardCharsets.UTF_8) : null;
+        int reasonLen = (reasonBytes != null) ? reasonBytes.length : 0;
         int payloadLen = 2 + reasonLen;
 
         int headerSize = WebSocketFrameWriter.headerSize(payloadLen, true);
@@ -447,8 +449,7 @@ private void sendCloseFrame(int code, String reason) {
         Unsafe.getUnsafe().putByte(payloadStart + 1, (byte) (code & 0xFF));
 
         // Write reason if present
-        if (reason != null) {
-            byte[] reasonBytes = reason.getBytes(StandardCharsets.UTF_8);
+        if (reasonBytes != null) {
             for (int i = 0; i < reasonBytes.length; i++) {
                 Unsafe.getUnsafe().putByte(payloadStart + 2 + i, reasonBytes[i]);
             }

From 8439f45581e9f9c4454c5c01c932994c4dc84af7 Mon Sep 17 00:00:00 2001
From: Marko Topolnik 
Date: Tue, 24 Feb 2026 12:10:55 +0100
Subject: [PATCH 19/89] Echo close frame on WebSocket close (RFC 6455)

When receiving a CLOSE frame from the server, the client now echoes
a close frame back before marking the connection as no longer
upgraded. This is required by RFC 6455 Section 5.5.1.

The close code parsing was moved out of the handler-null check so
the code is always available for the echo. The echo uses the
dedicated controlFrameBuffer to avoid clobbering any in-progress
frame in the main send buffer.

Co-Authored-By: Claude Opus 4.6 
---
 .../cutlass/http/client/WebSocketClient.java  | 40 +++++++++++++------
 1 file changed, 27 insertions(+), 13 deletions(-)

diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java
index cfe53dc..ddc4a02 100644
--- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java
+++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java
@@ -603,21 +603,24 @@ private Boolean tryParseFrame(WebSocketFrameHandler handler) {
                     }
                     break;
                 case WebSocketOpcode.CLOSE:
-                    upgraded = false;
-                    if (handler != null) {
-                        int closeCode = 0;
-                        String reason = null;
-                        if (payloadLen >= 2) {
-                            closeCode = ((Unsafe.getUnsafe().getByte(payloadPtr) & 0xFF) << 8)
-                                    | (Unsafe.getUnsafe().getByte(payloadPtr + 1) & 0xFF);
-                            if (payloadLen > 2) {
-                                byte[] reasonBytes = new byte[payloadLen - 2];
-                                for (int i = 0; i < reasonBytes.length; i++) {
-                                    reasonBytes[i] = Unsafe.getUnsafe().getByte(payloadPtr + 2 + i);
-                                }
-                                reason = new String(reasonBytes, StandardCharsets.UTF_8);
+                    int closeCode = 0;
+                    String reason = null;
+                    if (payloadLen >= 2) {
+                        closeCode = ((Unsafe.getUnsafe().getByte(payloadPtr) & 0xFF) << 8)
+                                | (Unsafe.getUnsafe().getByte(payloadPtr + 1) & 0xFF);
+                        if (payloadLen > 2) {
+                            byte[] reasonBytes = new byte[payloadLen - 2];
+                            for (int i = 0; i < reasonBytes.length; i++) {
+                                reasonBytes[i] = Unsafe.getUnsafe().getByte(payloadPtr + 2 + i);
                             }
+                            reason = new String(reasonBytes, StandardCharsets.UTF_8);
                         }
+                    }
+                    // RFC 6455 Section 5.5.1: echo a close frame back before
+                    // marking the connection as no longer upgraded
+                    sendCloseFrameEcho(closeCode);
+                    upgraded = false;
+                    if (handler != null) {
                         handler.onClose(closeCode, reason);
                     }
                     break;
@@ -645,6 +648,17 @@ private Boolean tryParseFrame(WebSocketFrameHandler handler) {
         return false;
     }
 
+    private void sendCloseFrameEcho(int code) {
+        try {
+            controlFrameBuffer.reset();
+            WebSocketSendBuffer.FrameInfo frame = controlFrameBuffer.writeCloseFrame(code, null);
+            doSend(controlFrameBuffer.getBufferPtr() + frame.offset, frame.length, 1000);
+            controlFrameBuffer.reset();
+        } catch (Exception e) {
+            LOG.error("Failed to echo close frame: {}", e.getMessage());
+        }
+    }
+
     private void sendPongFrame(long payloadPtr, int payloadLen) {
         try {
             controlFrameBuffer.reset();

From c41aa58b8e112d2ddc60f3cdc2c0ff996f7403c1 Mon Sep 17 00:00:00 2001
From: Marko Topolnik 
Date: Tue, 24 Feb 2026 12:15:38 +0100
Subject: [PATCH 20/89] Add WebSocket fragmentation support (RFC 6455)

Handle CONTINUATION frames (opcode 0x0) in tryParseFrame()
which were previously silently dropped. Fragment payloads are
accumulated in a lazily-allocated native memory buffer and
delivered as a complete message to the handler when the final
FIN=1 frame arrives.

The FIN bit is now checked on TEXT/BINARY frames: FIN=0 starts
fragment accumulation, FIN=1 delivers immediately. Protocol
errors are raised for continuation without an initial fragment
and for overlapping fragmented messages.

The fragment buffer is freed in close() and the fragmentation
state is reset on disconnect().

Co-Authored-By: Claude Opus 4.6 
---
 .../cutlass/http/client/WebSocketClient.java  | 71 +++++++++++++++++--
 1 file changed, 66 insertions(+), 5 deletions(-)

diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java
index ddc4a02..804c540 100644
--- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java
+++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java
@@ -93,6 +93,12 @@ public abstract class WebSocketClient implements QuietCloseable {
     private boolean upgraded;
     private boolean closed;
 
+    // Fragmentation state (RFC 6455 Section 5.4)
+    private int fragmentOpcode = -1;   // opcode of first fragment, -1 = not in a fragmented message
+    private long fragmentBufPtr;       // native buffer for accumulating fragment payloads
+    private int fragmentBufSize;
+    private int fragmentBufPos;
+
     // Handshake key for verification
     private String handshakeKey;
 
@@ -138,6 +144,11 @@ public void close() {
             sendBuffer.close();
             controlFrameBuffer.close();
 
+            if (fragmentBufPtr != 0) {
+                Unsafe.free(fragmentBufPtr, fragmentBufSize, MemoryTag.NATIVE_DEFAULT);
+                fragmentBufPtr = 0;
+            }
+
             if (recvBufPtr != 0) {
                 Unsafe.free(recvBufPtr, recvBufSize, MemoryTag.NATIVE_DEFAULT);
                 recvBufPtr = 0;
@@ -156,6 +167,7 @@ public void disconnect() {
         port = 0;
         recvPos = 0;
         recvReadPos = 0;
+        resetFragmentState();
     }
 
     /**
@@ -625,13 +637,40 @@ private Boolean tryParseFrame(WebSocketFrameHandler handler) {
                     }
                     break;
                 case WebSocketOpcode.BINARY:
-                    if (handler != null) {
-                        handler.onBinaryMessage(payloadPtr, payloadLen);
+                case WebSocketOpcode.TEXT:
+                    if (frameParser.isFin()) {
+                        if (fragmentOpcode != -1) {
+                            throw new HttpClientException("WebSocket protocol error: new data frame during fragmented message");
+                        }
+                        if (handler != null) {
+                            if (opcode == WebSocketOpcode.BINARY) {
+                                handler.onBinaryMessage(payloadPtr, payloadLen);
+                            } else {
+                                handler.onTextMessage(payloadPtr, payloadLen);
+                            }
+                        }
+                    } else {
+                        if (fragmentOpcode != -1) {
+                            throw new HttpClientException("WebSocket protocol error: new data frame during fragmented message");
+                        }
+                        fragmentOpcode = opcode;
+                        appendToFragmentBuffer(payloadPtr, payloadLen);
                     }
                     break;
-                case WebSocketOpcode.TEXT:
-                    if (handler != null) {
-                        handler.onTextMessage(payloadPtr, payloadLen);
+                case WebSocketOpcode.CONTINUATION:
+                    if (fragmentOpcode == -1) {
+                        throw new HttpClientException("WebSocket protocol error: continuation frame without initial fragment");
+                    }
+                    appendToFragmentBuffer(payloadPtr, payloadLen);
+                    if (frameParser.isFin()) {
+                        if (handler != null) {
+                            if (fragmentOpcode == WebSocketOpcode.BINARY) {
+                                handler.onBinaryMessage(fragmentBufPtr, fragmentBufPos);
+                            } else {
+                                handler.onTextMessage(fragmentBufPtr, fragmentBufPos);
+                            }
+                        }
+                        resetFragmentState();
                     }
                     break;
             }
@@ -670,6 +709,28 @@ private void sendPongFrame(long payloadPtr, int payloadLen) {
         }
     }
 
+    private void appendToFragmentBuffer(long payloadPtr, int payloadLen) {
+        if (payloadLen == 0) {
+            return;
+        }
+        int required = fragmentBufPos + payloadLen;
+        if (fragmentBufPtr == 0) {
+            fragmentBufSize = Math.max(required, DEFAULT_RECV_BUFFER_SIZE);
+            fragmentBufPtr = Unsafe.malloc(fragmentBufSize, MemoryTag.NATIVE_DEFAULT);
+        } else if (required > fragmentBufSize) {
+            int newSize = Math.max(fragmentBufSize * 2, required);
+            fragmentBufPtr = Unsafe.realloc(fragmentBufPtr, fragmentBufSize, newSize, MemoryTag.NATIVE_DEFAULT);
+            fragmentBufSize = newSize;
+        }
+        Vect.memmove(fragmentBufPtr + fragmentBufPos, payloadPtr, payloadLen);
+        fragmentBufPos += payloadLen;
+    }
+
+    private void resetFragmentState() {
+        fragmentOpcode = -1;
+        fragmentBufPos = 0;
+    }
+
     private void compactRecvBuffer() {
         if (recvReadPos > 0) {
             int remaining = recvPos - recvReadPos;

From 8457d1b333d8678e41aa69eff6323c5892c5518a Mon Sep 17 00:00:00 2001
From: Marko Topolnik 
Date: Tue, 24 Feb 2026 12:34:29 +0100
Subject: [PATCH 21/89] Cap recv buffer growth to prevent OOM

Add a configurable maximum size for the WebSocket receive buffer,
mirroring the pattern already used by WebSocketSendBuffer. Previously,
growRecvBuffer() doubled the buffer without any upper bound, allowing
a malicious server to trigger out-of-memory by sending arbitrarily
large frames.

Add getMaximumResponseBufferSize() to HttpClientConfiguration
(defaulting to Integer.MAX_VALUE for backwards compatibility) and
enforce the limit in both growRecvBuffer() and
appendToFragmentBuffer(), which had the same unbounded growth issue
for fragmented messages.

Co-Authored-By: Claude Opus 4.6 
---
 .../client/HttpClientConfiguration.java       |  4 ++++
 .../cutlass/http/client/WebSocketClient.java  | 21 ++++++++++++++++++-
 2 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/core/src/main/java/io/questdb/client/HttpClientConfiguration.java b/core/src/main/java/io/questdb/client/HttpClientConfiguration.java
index c2f644e..b02d485 100644
--- a/core/src/main/java/io/questdb/client/HttpClientConfiguration.java
+++ b/core/src/main/java/io/questdb/client/HttpClientConfiguration.java
@@ -54,6 +54,10 @@ default int getMaximumRequestBufferSize() {
         return Integer.MAX_VALUE;
     }
 
+    default int getMaximumResponseBufferSize() {
+        return Integer.MAX_VALUE;
+    }
+
     default NetworkFacade getNetworkFacade() {
         return NetworkFacadeImpl.INSTANCE;
     }
diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java
index 804c540..5028dcf 100644
--- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java
+++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java
@@ -80,6 +80,7 @@ public abstract class WebSocketClient implements QuietCloseable {
     private final WebSocketFrameParser frameParser;
     private final Rnd rnd;
     private final int defaultTimeout;
+    private final int maxRecvBufSize;
 
     // Receive buffer (native memory)
     private long recvBufPtr;
@@ -116,6 +117,7 @@ public WebSocketClient(HttpClientConfiguration configuration, SocketFactory sock
         this.controlFrameBuffer = new WebSocketSendBuffer(256, 256);
 
         this.recvBufSize = Math.max(configuration.getResponseBufferSize(), DEFAULT_RECV_BUFFER_SIZE);
+        this.maxRecvBufSize = Math.max(configuration.getMaximumResponseBufferSize(), recvBufSize);
         this.recvBufPtr = Unsafe.malloc(recvBufSize, MemoryTag.NATIVE_DEFAULT);
         this.recvPos = 0;
         this.recvReadPos = 0;
@@ -714,11 +716,18 @@ private void appendToFragmentBuffer(long payloadPtr, int payloadLen) {
             return;
         }
         int required = fragmentBufPos + payloadLen;
+        if (required > maxRecvBufSize) {
+            throw new HttpClientException("WebSocket fragment buffer size exceeded maximum [required=")
+                    .put(required)
+                    .put(", max=")
+                    .put(maxRecvBufSize)
+                    .put(']');
+        }
         if (fragmentBufPtr == 0) {
             fragmentBufSize = Math.max(required, DEFAULT_RECV_BUFFER_SIZE);
             fragmentBufPtr = Unsafe.malloc(fragmentBufSize, MemoryTag.NATIVE_DEFAULT);
         } else if (required > fragmentBufSize) {
-            int newSize = Math.max(fragmentBufSize * 2, required);
+            int newSize = Math.min(Math.max(fragmentBufSize * 2, required), maxRecvBufSize);
             fragmentBufPtr = Unsafe.realloc(fragmentBufPtr, fragmentBufSize, newSize, MemoryTag.NATIVE_DEFAULT);
             fragmentBufSize = newSize;
         }
@@ -744,6 +753,16 @@ private void compactRecvBuffer() {
 
     private void growRecvBuffer() {
         int newSize = recvBufSize * 2;
+        if (newSize > maxRecvBufSize) {
+            if (recvBufSize >= maxRecvBufSize) {
+                throw new HttpClientException("WebSocket receive buffer size exceeded maximum [current=")
+                        .put(recvBufSize)
+                        .put(", max=")
+                        .put(maxRecvBufSize)
+                        .put(']');
+            }
+            newSize = maxRecvBufSize;
+        }
         recvBufPtr = Unsafe.realloc(recvBufPtr, recvBufSize, newSize, MemoryTag.NATIVE_DEFAULT);
         recvBufSize = newSize;
     }

From 1dc87d71d7b39a0b9cdda4da3a527099dd87a7bc Mon Sep 17 00:00:00 2001
From: Marko Topolnik 
Date: Tue, 24 Feb 2026 12:48:31 +0100
Subject: [PATCH 22/89] Use ephemeral ports in WebSocket builder tests

Tests that expect connection failure were hardcoding ports
(9000, 19999) which could collide with running services. When
a QuestDB server is running on port 9000, the WebSocket
connection succeeds and the test fails with "Expected
LineSenderException".

Replace hardcoded ports with dynamically allocated ephemeral
ports via ServerSocket(0). The port is bound and immediately
closed, guaranteeing nothing is listening when the test tries
to connect.

Affected tests:
- testBuilderWithWebSocketTransportCreatesCorrectSenderType
- testConnectionRefused
- testWsConfigString
- testWsConfigString_missingAddr_fails
- testWsConfigString_protocolAlreadyConfigured_fails

Co-Authored-By: Claude Opus 4.6 
---
 .../LineSenderBuilderWebSocketTest.java       | 34 +++++++++++++------
 1 file changed, 24 insertions(+), 10 deletions(-)

diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java
index ee80665..4fb8121 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java
@@ -252,19 +252,24 @@ public void testBuilderWithWebSocketTransport() {
     }
 
     @Test
-    public void testBuilderWithWebSocketTransportCreatesCorrectSenderType() {
+    public void testBuilderWithWebSocketTransportCreatesCorrectSenderType() throws Exception {
+        int port;
+        try (java.net.ServerSocket s = new java.net.ServerSocket(0)) {
+            port = s.getLocalPort();
+        }
         assertThrowsAny(
                 Sender.builder(Sender.Transport.WEBSOCKET)
-                        .address(LOCALHOST + ":9000"),
+                        .address(LOCALHOST + ":" + port),
                 "connect", "Failed"
         );
     }
 
     @Test
-    public void testConnectionRefused() {
+    public void testConnectionRefused() throws Exception {
+        int port = findUnusedPort();
         assertThrowsAny(
                 Sender.builder(Sender.Transport.WEBSOCKET)
-                        .address(LOCALHOST + ":19999"),
+                        .address(LOCALHOST + ":" + port),
                 "connect", "Failed"
         );
     }
@@ -691,22 +696,25 @@ public void testUsernamePassword_notYetSupported() {
     }
 
     @Test
-    public void testWsConfigString() {
-        assertBadConfig("ws::addr=localhost:9000;", "connect", "Failed");
+    public void testWsConfigString() throws Exception {
+        int port = findUnusedPort();
+        assertBadConfig("ws::addr=localhost:" + port + ";", "connect", "Failed");
     }
 
     // ==================== Edge Cases ====================
 
     @Test
-    public void testWsConfigString_missingAddr_fails() {
-        assertBadConfig("ws::addr=localhost;", "connect", "Failed");
+    public void testWsConfigString_missingAddr_fails() throws Exception {
+        int port = findUnusedPort();
+        assertBadConfig("ws::addr=localhost:" + port + ";", "connect", "Failed");
         assertBadConfig("ws::foo=bar;", "addr is missing");
     }
 
     @Test
-    public void testWsConfigString_protocolAlreadyConfigured_fails() {
+    public void testWsConfigString_protocolAlreadyConfigured_fails() throws Exception {
+        int port = findUnusedPort();
         assertThrowsAny(
-                Sender.builder("ws::addr=localhost:9000;")
+                Sender.builder("ws::addr=localhost:" + port + ";")
                         .enableTls(),
                 "TLS", "connect", "Failed"
         );
@@ -761,6 +769,12 @@ private static void assertThrowsAny(Sender.LineSenderBuilder builder, String...
         assertThrowsAny(builder::build, anyOf);
     }
 
+    private static int findUnusedPort() throws Exception {
+        try (java.net.ServerSocket s = new java.net.ServerSocket(0)) {
+            return s.getLocalPort();
+        }
+    }
+
     private static void assertThrowsAny(Runnable action, String... anyOf) {
         try {
             action.run();

From 00c145b98372c4eb16edd7ebb55f67b7966fe1ff Mon Sep 17 00:00:00 2001
From: Marko Topolnik 
Date: Tue, 24 Feb 2026 13:35:40 +0100
Subject: [PATCH 23/89] Auto-cleanup test code

---
 .../questdb/client/test/AbstractQdbTest.java  | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/core/src/test/java/io/questdb/client/test/AbstractQdbTest.java b/core/src/test/java/io/questdb/client/test/AbstractQdbTest.java
index 81242bd..244488f 100644
--- a/core/src/test/java/io/questdb/client/test/AbstractQdbTest.java
+++ b/core/src/test/java/io/questdb/client/test/AbstractQdbTest.java
@@ -31,7 +31,6 @@
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Assert;
-import org.junit.Assume;
 import org.junit.Before;
 import org.junit.BeforeClass;
 
@@ -210,10 +209,10 @@ public static void setUpStatic() {
             System.err.printf("CLEANING UP TEST TABLES%n");
             // Cleanup all test tables before starting tests
             try (Connection conn = getPgConnection();
-                    Statement readStmt = conn.createStatement();
-                    Statement stmt = conn.createStatement();
-                    ResultSet rs = readStmt
-                            .executeQuery("SELECT table_name FROM tables() WHERE table_name LIKE 'test_%'")) {
+                 Statement readStmt = conn.createStatement();
+                 Statement stmt = conn.createStatement();
+                 ResultSet rs = readStmt
+                         .executeQuery("SELECT table_name FROM tables() WHERE table_name LIKE 'test_%'")) {
                 while (rs.next()) {
                     String tableName = rs.getString(1);
                     try {
@@ -458,7 +457,7 @@ protected static Connection initPgConnection() throws SQLException {
     protected void assertSqlEventually(CharSequence expected, String sql) throws Exception {
         assertEventually(() -> {
             try (Statement statement = getPgConnection().createStatement();
-                    ResultSet rs = statement.executeQuery(sql)) {
+                 ResultSet rs = statement.executeQuery(sql)) {
                 sink.clear();
                 printToSink(sink, rs);
                 TestUtils.assertEquals(expected, sink);
@@ -474,8 +473,8 @@ protected void assertSqlEventually(CharSequence expected, String sql) throws Exc
     protected void assertTableExistsEventually(CharSequence tableName) throws Exception {
         assertEventually(() -> {
             try (Statement stmt = getPgConnection().createStatement();
-                    ResultSet rs = stmt.executeQuery(
-                            String.format("SELECT COUNT(*) AS cnt FROM tables() WHERE table_name = '%s'", tableName))) {
+                 ResultSet rs = stmt.executeQuery(
+                         String.format("SELECT COUNT(*) AS cnt FROM tables() WHERE table_name = '%s'", tableName))) {
                 Assert.assertTrue(rs.next());
                 final long actualSize = rs.getLong(1);
                 Assert.assertEquals(1, actualSize);
@@ -546,7 +545,7 @@ protected List> executeQuery(String sql) throws SQLException
         List> results = new ArrayList<>();
 
         try (Statement stmt = getPgConnection().createStatement();
-                ResultSet rs = stmt.executeQuery(sql)) {
+             ResultSet rs = stmt.executeQuery(sql)) {
 
             ResultSetMetaData metaData = rs.getMetaData();
             int columnCount = metaData.getColumnCount();
@@ -595,7 +594,7 @@ protected String queryTableAsTsv(String tableName, String orderBy) throws SQLExc
         }
 
         try (Statement stmt = getPgConnection().createStatement();
-                ResultSet rs = stmt.executeQuery(sql)) {
+             ResultSet rs = stmt.executeQuery(sql)) {
 
             ResultSetMetaData metaData = rs.getMetaData();
             int columnCount = metaData.getColumnCount();

From dbca59054609682d5f7987be78cfe9a3e3311f09 Mon Sep 17 00:00:00 2001
From: Marko Topolnik 
Date: Tue, 24 Feb 2026 13:35:56 +0100
Subject: [PATCH 24/89] Don't default QUESTDB_RUNNING to true

---
 core/src/test/java/io/questdb/client/test/AbstractQdbTest.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/test/java/io/questdb/client/test/AbstractQdbTest.java b/core/src/test/java/io/questdb/client/test/AbstractQdbTest.java
index 244488f..e984f19 100644
--- a/core/src/test/java/io/questdb/client/test/AbstractQdbTest.java
+++ b/core/src/test/java/io/questdb/client/test/AbstractQdbTest.java
@@ -425,7 +425,7 @@ protected static String getPgUser() {
      * Get whether a QuestDB instance is running locally.
      */
     protected static boolean getQuestDBRunning() {
-        return getConfigBool("QUESTDB_RUNNING", "questdb.running", true);
+        return getConfigBool("QUESTDB_RUNNING", "questdb.running", false);
     }
 
     /**

From 34cc15496cf52bf41bcc958b1bcb7e838f83559e Mon Sep 17 00:00:00 2001
From: Marko Topolnik 
Date: Tue, 24 Feb 2026 13:41:32 +0100
Subject: [PATCH 25/89] Fix case-sensitive header check in WebSocket handshake

The Sec-WebSocket-Accept header validation used case-sensitive
String.contains(), which violates RFC 7230 (HTTP headers are
case-insensitive). A server sending the header in a different
casing (e.g., sec-websocket-accept) would cause the handshake
to fail.

Replace with a containsHeaderValue() helper that uses
String.regionMatches(ignoreCase=true) for the header name
lookup, avoiding both the case-sensitivity bug and unnecessary
string allocation from toLowerCase().

Co-Authored-By: Claude Opus 4.6 
---
 .../cutlass/http/client/WebSocketClient.java  | 21 +++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java
index 5028dcf..fec618f 100644
--- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java
+++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java
@@ -378,13 +378,30 @@ private void validateUpgradeResponse(int headerEnd) {
             throw new HttpClientException("WebSocket upgrade failed: ").put(statusLine);
         }
 
-        // Verify Sec-WebSocket-Accept
+        // Verify Sec-WebSocket-Accept (case-insensitive per RFC 7230)
         String expectedAccept = WebSocketHandshake.computeAcceptKey(handshakeKey);
-        if (!response.contains("Sec-WebSocket-Accept: " + expectedAccept)) {
+        if (!containsHeaderValue(response, "Sec-WebSocket-Accept:", expectedAccept)) {
             throw new HttpClientException("Invalid Sec-WebSocket-Accept header");
         }
     }
 
+    private static boolean containsHeaderValue(String response, String headerName, String expectedValue) {
+        int headerLen = headerName.length();
+        int responseLen = response.length();
+        for (int i = 0; i <= responseLen - headerLen; i++) {
+            if (response.regionMatches(true, i, headerName, 0, headerLen)) {
+                int valueStart = i + headerLen;
+                int lineEnd = response.indexOf('\r', valueStart);
+                if (lineEnd < 0) {
+                    lineEnd = responseLen;
+                }
+                String actualValue = response.substring(valueStart, lineEnd).trim();
+                return actualValue.equals(expectedValue);
+            }
+        }
+        return false;
+    }
+
     // === Sending ===
 
     /**

From 2834b03d083a73cf152c3681e1a19381e00adbb7 Mon Sep 17 00:00:00 2001
From: Marko Topolnik 
Date: Tue, 24 Feb 2026 14:16:20 +0100
Subject: [PATCH 26/89] Use bulk copyMemory for WebSocket I/O

Replace byte-by-byte native-heap copies in writeToSocket and
readFromSocket with Unsafe.copyMemory(), using the 5-argument
form that bridges native memory and Java byte arrays via
Unsafe.BYTE_OFFSET.

Add WebSocketChannelTest with a local echo server that verifies
data integrity through the copy paths across various payload
sizes and patterns.

Co-Authored-By: Claude Opus 4.6 
---
 .../cutlass/qwp/client/WebSocketChannel.java  |   8 +-
 .../qwp/client/WebSocketChannelTest.java      | 430 ++++++++++++++++++
 2 files changed, 432 insertions(+), 6 deletions(-)
 create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketChannelTest.java

diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketChannel.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketChannel.java
index f9584f4..415ee4b 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketChannel.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketChannel.java
@@ -595,9 +595,7 @@ private void writeToSocket(long ptr, int len) throws IOException {
         // Copy to temp array for socket write (unavoidable with OutputStream)
         // Use separate write buffer to avoid race with read thread
         byte[] temp = getWriteTempBuffer(len);
-        for (int i = 0; i < len; i++) {
-            temp[i] = Unsafe.getUnsafe().getByte(ptr + i);
-        }
+        Unsafe.getUnsafe().copyMemory(null, ptr, temp, Unsafe.BYTE_OFFSET, len);
         out.write(temp, 0, len);
         out.flush();
     }
@@ -618,9 +616,7 @@ private int readFromSocket() throws IOException {
         byte[] temp = getReadTempBuffer(available);
         int bytesRead = in.read(temp, 0, available);
         if (bytesRead > 0) {
-            for (int i = 0; i < bytesRead; i++) {
-                Unsafe.getUnsafe().putByte(recvBufferPtr + recvBufferPos + i, temp[i]);
-            }
+            Unsafe.getUnsafe().copyMemory(temp, Unsafe.BYTE_OFFSET, null, recvBufferPtr + recvBufferPos, bytesRead);
             recvBufferPos += bytesRead;
         }
         return bytesRead;
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketChannelTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketChannelTest.java
new file mode 100644
index 0000000..607b442
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketChannelTest.java
@@ -0,0 +1,430 @@
+/*******************************************************************************
+ *     ___                  _   ____  ____
+ *    / _ \ _   _  ___  ___| |_|  _ \| __ )
+ *   | | | | | | |/ _ \/ __| __| | | |  _ \
+ *   | |_| | |_| |  __/\__ \ |_| |_| | |_) |
+ *    \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ *  Copyright (c) 2014-2019 Appsicle
+ *  Copyright (c) 2019-2026 QuestDB
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.cutlass.qwp.client;
+
+import io.questdb.client.cutlass.qwp.client.WebSocketChannel;
+import io.questdb.client.cutlass.qwp.websocket.WebSocketHandshake;
+import io.questdb.client.cutlass.qwp.websocket.WebSocketOpcode;
+import io.questdb.client.std.MemoryTag;
+import io.questdb.client.std.Unsafe;
+import io.questdb.client.test.AbstractTest;
+import io.questdb.client.test.tools.TestUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.BufferedOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.nio.charset.StandardCharsets;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Tests for WebSocketChannel's native-heap memory copy paths.
+ * Exercises writeToSocket (native to heap) and readFromSocket (heap to native)
+ * through a local echo server.
+ */
+public class WebSocketChannelTest extends AbstractTest {
+
+    @Test
+    public void testBinaryRoundTripSmallPayload() throws Exception {
+        TestUtils.assertMemoryLeak(() -> assertBinaryRoundTrip(13));
+    }
+
+    @Test
+    public void testBinaryRoundTripMediumPayload() throws Exception {
+        TestUtils.assertMemoryLeak(() -> assertBinaryRoundTrip(4096));
+    }
+
+    @Test
+    public void testBinaryRoundTripLargePayload() throws Exception {
+        TestUtils.assertMemoryLeak(() -> {
+            // Large payload that exercises bulk copyMemory across many cache lines.
+            // Kept under 32KB so the echo response arrives in a single TCP read
+            // on loopback (avoids a pre-existing bug in doReceiveFrame with
+            // partial frame assembly).
+            assertBinaryRoundTrip(30_000);
+        });
+    }
+
+    @Test
+    public void testBinaryRoundTripAllByteValues() throws Exception {
+        TestUtils.assertMemoryLeak(() -> {
+            int len = 256;
+            long sendPtr = Unsafe.malloc(len, MemoryTag.NATIVE_DEFAULT);
+            try {
+                for (int i = 0; i < len; i++) {
+                    Unsafe.getUnsafe().putByte(sendPtr + i, (byte) i);
+                }
+                assertBinaryRoundTrip(sendPtr, len);
+            } finally {
+                Unsafe.free(sendPtr, len, MemoryTag.NATIVE_DEFAULT);
+            }
+        });
+    }
+
+    @Test
+    public void testBinaryRoundTripRepeatedFrames() throws Exception {
+        TestUtils.assertMemoryLeak(() -> {
+            int payloadLen = 1000;
+            int frameCount = 10;
+            long sendPtr = Unsafe.malloc(payloadLen, MemoryTag.NATIVE_DEFAULT);
+            try (EchoServer server = new EchoServer()) {
+                server.start();
+                WebSocketChannel channel = new WebSocketChannel(
+                        "localhost:" + server.getPort() + "/", false
+                );
+                try {
+                    channel.setConnectTimeout(5000);
+                    channel.setReadTimeout(5000);
+                    channel.connect();
+
+                    for (int f = 0; f < frameCount; f++) {
+                        for (int i = 0; i < payloadLen; i++) {
+                            Unsafe.getUnsafe().putByte(sendPtr + i, (byte) (i + f));
+                        }
+                        channel.sendBinary(sendPtr, payloadLen);
+
+                        ReceivedPayload received = new ReceivedPayload();
+                        boolean ok = receiveWithRetry(channel, received, 5000);
+                        server.assertNoError();
+                        Assert.assertTrue("frame " + f + ": expected response", ok);
+                        Assert.assertEquals("frame " + f + ": length", payloadLen, received.length);
+
+                        for (int i = 0; i < payloadLen; i++) {
+                            Assert.assertEquals(
+                                    "frame " + f + " byte " + i,
+                                    (byte) (i + f),
+                                    Unsafe.getUnsafe().getByte(received.ptr + i)
+                            );
+                        }
+                    }
+                } finally {
+                    channel.close();
+                }
+                server.assertNoError();
+            } finally {
+                Unsafe.free(sendPtr, payloadLen, MemoryTag.NATIVE_DEFAULT);
+            }
+        });
+    }
+
+    private void assertBinaryRoundTrip(int payloadLen) throws Exception {
+        long sendPtr = Unsafe.malloc(payloadLen, MemoryTag.NATIVE_DEFAULT);
+        try {
+            for (int i = 0; i < payloadLen; i++) {
+                Unsafe.getUnsafe().putByte(sendPtr + i, (byte) (i & 0xFF));
+            }
+            assertBinaryRoundTrip(sendPtr, payloadLen);
+        } finally {
+            Unsafe.free(sendPtr, payloadLen, MemoryTag.NATIVE_DEFAULT);
+        }
+    }
+
+    private void assertBinaryRoundTrip(long sendPtr, int payloadLen) throws Exception {
+        try (EchoServer server = new EchoServer()) {
+            server.start();
+            WebSocketChannel channel = new WebSocketChannel(
+                    "localhost:" + server.getPort() + "/", false
+            );
+            try {
+                channel.setConnectTimeout(5000);
+                channel.setReadTimeout(5000);
+                channel.connect();
+
+                // Send exercises writeToSocket (native to heap via copyMemory)
+                channel.sendBinary(sendPtr, payloadLen);
+
+                // Receive exercises readFromSocket (heap to native via copyMemory)
+                ReceivedPayload received = new ReceivedPayload();
+                boolean ok = receiveWithRetry(channel, received, 5000);
+
+                // Check server error before client assertions
+                server.assertNoError();
+                Assert.assertTrue("expected a frame back from echo server", ok);
+                Assert.assertEquals("payload length mismatch", payloadLen, received.length);
+
+                for (int i = 0; i < payloadLen; i++) {
+                    byte expected = Unsafe.getUnsafe().getByte(sendPtr + i);
+                    byte actual = Unsafe.getUnsafe().getByte(received.ptr + i);
+                    Assert.assertEquals("byte mismatch at offset " + i, expected, actual);
+                }
+            } finally {
+                channel.close();
+            }
+            server.assertNoError();
+        }
+    }
+
+    /**
+     * Calls receiveFrame in a loop to handle the case where doReceiveFrame
+     * needs multiple reads to assemble a complete frame (e.g. header and
+     * payload arrive in separate TCP segments).
+     */
+    private static boolean receiveWithRetry(WebSocketChannel channel, ReceivedPayload handler, int timeoutMs) {
+        long deadline = System.currentTimeMillis() + timeoutMs;
+        while (System.currentTimeMillis() < deadline) {
+            int remaining = (int) (deadline - System.currentTimeMillis());
+            if (remaining <= 0) {
+                break;
+            }
+            if (channel.receiveFrame(handler, remaining)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    private static class ReceivedPayload implements WebSocketChannel.ResponseHandler {
+        long ptr;
+        int length;
+
+        @Override
+        public void onBinaryMessage(long payload, int length) {
+            this.ptr = payload;
+            this.length = length;
+        }
+
+        @Override
+        public void onClose(int code, String reason) {
+        }
+    }
+
+    /**
+     * Minimal WebSocket echo server. Accepts one connection, completes the
+     * HTTP upgrade handshake, then echoes every binary frame back unmasked.
+     * All echo writes use a single byte array to avoid TCP fragmentation.
+     */
+    private static class EchoServer implements AutoCloseable {
+        private static final Pattern KEY_PATTERN =
+                Pattern.compile("Sec-WebSocket-Key:\\s*(.+?)\\r\\n");
+
+        private final ServerSocket serverSocket;
+        private final AtomicReference error = new AtomicReference<>();
+        private Thread thread;
+
+        EchoServer() throws IOException {
+            serverSocket = new ServerSocket(0);
+        }
+
+        int getPort() {
+            return serverSocket.getLocalPort();
+        }
+
+        void start() {
+            thread = new Thread(this::run, "ws-echo-server");
+            thread.setDaemon(true);
+            thread.start();
+        }
+
+        void assertNoError() {
+            Throwable t = error.get();
+            if (t != null) {
+                throw new AssertionError("echo server error", t);
+            }
+        }
+
+        @Override
+        public void close() throws Exception {
+            serverSocket.close();
+            if (thread != null) {
+                thread.join(5000);
+            }
+        }
+
+        private void run() {
+            try (Socket client = serverSocket.accept()) {
+                client.setSoTimeout(10_000);
+                client.setTcpNoDelay(true);
+                InputStream in = client.getInputStream();
+                OutputStream out = new BufferedOutputStream(client.getOutputStream());
+
+                completeHandshake(in, out);
+                echoFrames(in, out);
+            } catch (IOException e) {
+                if (!serverSocket.isClosed()) {
+                    error.set(e);
+                }
+            } catch (Throwable t) {
+                error.set(t);
+            }
+        }
+
+        private void completeHandshake(InputStream in, OutputStream out) throws IOException {
+            byte[] buf = new byte[4096];
+            int pos = 0;
+
+            while (pos < buf.length) {
+                int b = in.read();
+                if (b < 0) {
+                    throw new IOException("connection closed during handshake");
+                }
+                buf[pos++] = (byte) b;
+                if (pos >= 4
+                        && buf[pos - 4] == '\r' && buf[pos - 3] == '\n'
+                        && buf[pos - 2] == '\r' && buf[pos - 1] == '\n') {
+                    break;
+                }
+            }
+
+            String request = new String(buf, 0, pos, StandardCharsets.US_ASCII);
+            Matcher m = KEY_PATTERN.matcher(request);
+            if (!m.find()) {
+                throw new IOException("no Sec-WebSocket-Key in request:\n" + request);
+            }
+            String clientKey = m.group(1).trim();
+            String acceptKey = WebSocketHandshake.computeAcceptKey(clientKey);
+
+            String response = "HTTP/1.1 101 Switching Protocols\r\n"
+                    + "Upgrade: websocket\r\n"
+                    + "Connection: Upgrade\r\n"
+                    + "Sec-WebSocket-Accept: " + acceptKey + "\r\n"
+                    + "\r\n";
+            out.write(response.getBytes(StandardCharsets.US_ASCII));
+            out.flush();
+        }
+
+        private void echoFrames(InputStream in, OutputStream out) throws IOException {
+            byte[] readBuf = new byte[256 * 1024];
+
+            while (true) {
+                int pos = 0;
+                while (pos < 2) {
+                    int n = in.read(readBuf, pos, readBuf.length - pos);
+                    if (n < 0) {
+                        return;
+                    }
+                    pos += n;
+                }
+
+                int byte0 = readBuf[0] & 0xFF;
+                int byte1 = readBuf[1] & 0xFF;
+                int opcode = byte0 & 0x0F;
+                boolean masked = (byte1 & 0x80) != 0;
+                int lengthField = byte1 & 0x7F;
+
+                int headerSize = 2;
+                long payloadLength;
+                if (lengthField <= 125) {
+                    payloadLength = lengthField;
+                } else if (lengthField == 126) {
+                    while (pos < 4) {
+                        int n = in.read(readBuf, pos, readBuf.length - pos);
+                        if (n < 0) return;
+                        pos += n;
+                    }
+                    payloadLength = ((readBuf[2] & 0xFF) << 8) | (readBuf[3] & 0xFF);
+                    headerSize = 4;
+                } else {
+                    while (pos < 10) {
+                        int n = in.read(readBuf, pos, readBuf.length - pos);
+                        if (n < 0) return;
+                        pos += n;
+                    }
+                    payloadLength = 0;
+                    for (int i = 0; i < 8; i++) {
+                        payloadLength = (payloadLength << 8) | (readBuf[2 + i] & 0xFF);
+                    }
+                    headerSize = 10;
+                }
+
+                if (masked) {
+                    headerSize += 4;
+                }
+
+                int totalFrameSize = (int) (headerSize + payloadLength);
+
+                if (totalFrameSize > readBuf.length) {
+                    byte[] newBuf = new byte[totalFrameSize];
+                    System.arraycopy(readBuf, 0, newBuf, 0, pos);
+                    readBuf = newBuf;
+                }
+
+                while (pos < totalFrameSize) {
+                    int n = in.read(readBuf, pos, totalFrameSize - pos);
+                    if (n < 0) return;
+                    pos += n;
+                }
+
+                if (opcode == WebSocketOpcode.CLOSE) {
+                    return;
+                }
+
+                if (opcode != WebSocketOpcode.BINARY && opcode != WebSocketOpcode.TEXT) {
+                    continue;
+                }
+
+                // Unmask payload in place
+                if (masked) {
+                    int maskKeyOffset = headerSize - 4;
+                    byte m0 = readBuf[maskKeyOffset];
+                    byte m1 = readBuf[maskKeyOffset + 1];
+                    byte m2 = readBuf[maskKeyOffset + 2];
+                    byte m3 = readBuf[maskKeyOffset + 3];
+                    for (int i = 0; i < (int) payloadLength; i++) {
+                        switch (i & 3) {
+                            case 0: readBuf[headerSize + i] ^= m0; break;
+                            case 1: readBuf[headerSize + i] ^= m1; break;
+                            case 2: readBuf[headerSize + i] ^= m2; break;
+                            case 3: readBuf[headerSize + i] ^= m3; break;
+                        }
+                    }
+                }
+
+                // Build complete unmasked response frame in a single array
+                byte[] responseHeader;
+                if (payloadLength <= 125) {
+                    responseHeader = new byte[]{
+                            (byte) (0x80 | opcode),
+                            (byte) payloadLength
+                    };
+                } else if (payloadLength <= 65535) {
+                    responseHeader = new byte[]{
+                            (byte) (0x80 | opcode),
+                            126,
+                            (byte) ((payloadLength >> 8) & 0xFF),
+                            (byte) (payloadLength & 0xFF)
+                    };
+                } else {
+                    responseHeader = new byte[10];
+                    responseHeader[0] = (byte) (0x80 | opcode);
+                    responseHeader[1] = 127;
+                    for (int i = 0; i < 8; i++) {
+                        responseHeader[2 + i] = (byte) ((payloadLength >> (56 - i * 8)) & 0xFF);
+                    }
+                }
+
+                // Single write: header + payload together via BufferedOutputStream
+                out.write(responseHeader);
+                out.write(readBuf, headerSize, (int) payloadLength);
+                out.flush();
+            }
+        }
+    }
+}

From 6a94139fa601a9398374d9d983b4a3e64fcda890 Mon Sep 17 00:00:00 2001
From: Marko Topolnik 
Date: Tue, 24 Feb 2026 14:29:15 +0100
Subject: [PATCH 27/89] Fix delta dict corruption on send failure

Move maxSentSymbolId and sentSchemaHashes updates to after the
send/enqueue succeeds in both async and sync flush paths. Previously
these were updated before the send, so if sealAndSwapBuffer() threw
(async) or sendBinary()/waitForAck() threw (sync), the next batch's
delta dictionary would omit symbols the server never received,
silently corrupting subsequent data.

Also move sentSchemaHashes.add() inside the messageSize > 0 guard
in the sync path, where it was incorrectly marking schemas as sent
even when no data was produced.

Co-Authored-By: Claude Opus 4.6 
---
 .../qwp/client/QwpWebSocketSender.java        | 35 +++----
 .../qwp/client/QwpDeltaDictRollbackTest.java  | 93 +++++++++++++++++++
 2 files changed, 112 insertions(+), 16 deletions(-)
 create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpDeltaDictRollbackTest.java

diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
index 68dc19d..cf697d4 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
@@ -1141,10 +1141,6 @@ private void flushPendingRows() {
                         useSchemaRef
                 );
 
-                // Track schema key if this was the first time sending this schema
-                if (!useSchemaRef) {
-                    sentSchemaHashes.add(schemaKey);
-                }
                 QwpBufferWriter buffer = encoder.getBuffer();
 
                 // Copy to microbatch buffer and seal immediately
@@ -1154,12 +1150,18 @@ private void flushPendingRows() {
                 activeBuffer.incrementRowCount();
                 activeBuffer.setMaxSymbolId(currentBatchMaxSymbolId);
 
-                // Update maxSentSymbolId - once sent over TCP, server will receive it
-                maxSentSymbolId = currentBatchMaxSymbolId;
-
                 // Seal and enqueue for sending
                 sealAndSwapBuffer();
 
+                // Update sent state only after successful enqueue.
+                // If sealAndSwapBuffer() threw, these remain unchanged so the
+                // next batch's delta dictionary will correctly re-include the
+                // symbols and schema that the server never received.
+                maxSentSymbolId = currentBatchMaxSymbolId;
+                if (!useSchemaRef) {
+                    sentSchemaHashes.add(schemaKey);
+                }
+
                 // Reset table buffer and batch-level symbol tracking
                 tableBuffer.reset();
                 currentBatchMaxSymbolId = -1;
@@ -1209,11 +1211,6 @@ private void flushSync() {
                     useSchemaRef
             );
 
-            // Track schema key if this was the first time sending this schema
-            if (!useSchemaRef) {
-                sentSchemaHashes.add(schemaKey);
-            }
-
             if (messageSize > 0) {
                 QwpBufferWriter buffer = encoder.getBuffer();
 
@@ -1221,16 +1218,22 @@ private void flushSync() {
                 long batchSequence = nextBatchSequence++;
                 inFlightWindow.addInFlight(batchSequence);
 
-                // Update maxSentSymbolId - once sent over TCP, server will receive it
-                maxSentSymbolId = currentBatchMaxSymbolId;
-
-                LOG.debug("Sending sync batch [seq={}, bytes={}, rows={}, maxSentSymbolId={}, useSchemaRef={}]", batchSequence, messageSize, tableBuffer.getRowCount(), maxSentSymbolId, useSchemaRef);
+                LOG.debug("Sending sync batch [seq={}, bytes={}, rows={}, maxSentSymbolId={}, useSchemaRef={}]", batchSequence, messageSize, tableBuffer.getRowCount(), currentBatchMaxSymbolId, useSchemaRef);
 
                 // Send over WebSocket
                 client.sendBinary(buffer.getBufferPtr(), messageSize);
 
                 // Wait for ACK synchronously
                 waitForAck(batchSequence);
+
+                // Update sent state only after successful send + ACK.
+                // If sendBinary() or waitForAck() threw, these remain unchanged
+                // so the next batch's delta dictionary will correctly re-include
+                // the symbols and schema that the server never received.
+                maxSentSymbolId = currentBatchMaxSymbolId;
+                if (!useSchemaRef) {
+                    sentSchemaHashes.add(schemaKey);
+                }
             }
 
             // Reset table buffer after sending
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpDeltaDictRollbackTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpDeltaDictRollbackTest.java
new file mode 100644
index 0000000..b26f488
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpDeltaDictRollbackTest.java
@@ -0,0 +1,93 @@
+/*******************************************************************************
+ *     ___                  _   ____  ____
+ *    / _ \ _   _  ___  ___| |_|  _ \| __ )
+ *   | | | | | | |/ _ \/ __| __| | | |  _ \
+ *   | |_| | |_| |  __/\__ \ |_| |_| | |_) |
+ *    \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ *  Copyright (c) 2014-2019 Appsicle
+ *  Copyright (c) 2019-2026 QuestDB
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.cutlass.qwp.client;
+
+import io.questdb.client.cutlass.qwp.client.InFlightWindow;
+import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender;
+import io.questdb.client.test.AbstractTest;
+import io.questdb.client.test.tools.TestUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.lang.reflect.Field;
+import java.time.temporal.ChronoUnit;
+
+/**
+ * Verifies that maxSentSymbolId and sentSchemaHashes are not updated
+ * when the send fails, so the next batch's delta dictionary correctly
+ * re-includes symbols the server never received.
+ */
+public class QwpDeltaDictRollbackTest extends AbstractTest {
+
+    @Test
+    public void testSyncFlushFailureDoesNotAdvanceMaxSentSymbolId() throws Exception {
+        TestUtils.assertMemoryLeak(() -> {
+            // Sync mode (window=1), not connected to any server
+            QwpWebSocketSender sender = QwpWebSocketSender.createForTesting("localhost", 0, 1);
+            try {
+                // Bypass ensureConnected() by marking as connected.
+                // Leave client null so sendBinary() will throw.
+                setField(sender, "connected", true);
+                setField(sender, "inFlightWindow", new InFlightWindow(1, InFlightWindow.DEFAULT_TIMEOUT_MS));
+
+                // Buffer a row with a symbol — this registers symbol id 0
+                // in the global dictionary and sets currentBatchMaxSymbolId = 0
+                sender.table("t")
+                        .symbol("s", "val1")
+                        .at(1, ChronoUnit.MICROS);
+
+                // maxSentSymbolId should still be -1 (nothing sent yet)
+                Assert.assertEquals(-1, sender.getMaxSentSymbolId());
+
+                // flush() -> flushSync() -> encode succeeds -> client.sendBinary() throws NPE
+                // because client is null (we never actually connected)
+                try {
+                    sender.flush();
+                    Assert.fail("Expected NullPointerException from null client");
+                } catch (NullPointerException expected) {
+                    // sendBinary() on null client
+                }
+
+                // The fix: maxSentSymbolId must remain -1 because the send failed.
+                // Without the fix, it would have been advanced to 0 before the throw,
+                // causing the next batch's delta dictionary to omit symbol "val1".
+                Assert.assertEquals(
+                        "maxSentSymbolId must not advance when send fails",
+                        -1, sender.getMaxSentSymbolId()
+                );
+            } finally {
+                // Mark as not connected so close() doesn't try to flush again
+                setField(sender, "connected", false);
+                sender.close();
+            }
+        });
+    }
+
+    private static void setField(Object target, String fieldName, Object value) throws Exception {
+        Field f = target.getClass().getDeclaredField(fieldName);
+        f.setAccessible(true);
+        f.set(target, value);
+    }
+}

From 6d7a104b2b024e313230fb0cb05ffac37e46450c Mon Sep 17 00:00:00 2001
From: Marko Topolnik 
Date: Tue, 24 Feb 2026 14:44:32 +0100
Subject: [PATCH 28/89] Fix TYPE_CHAR validation in QwpColumnDef

The validate() range check used TYPE_DECIMAL256 (0x15) as the upper
bound, which excluded TYPE_CHAR (0x16). CHAR columns would throw
IllegalArgumentException on validation.

Extend the upper bound to TYPE_CHAR and add tests covering all valid
type codes, nullable CHAR, and invalid type rejection.

Co-Authored-By: Claude Opus 4.6 
---
 .../cutlass/qwp/protocol/QwpColumnDef.java    |  6 +-
 .../qwp/protocol/QwpColumnDefTest.java        | 97 +++++++++++++++++++
 2 files changed, 100 insertions(+), 3 deletions(-)
 create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpColumnDefTest.java

diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java
index b9d9a26..a7257dd 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java
@@ -123,9 +123,9 @@ public String getTypeName() {
      * @throws IllegalArgumentException if type code is invalid
      */
     public void validate() {
-        // Valid type codes: TYPE_BOOLEAN (0x01) through TYPE_DECIMAL256 (0x15)
-        // This includes all basic types, arrays, and decimals
-        boolean valid = (typeCode >= TYPE_BOOLEAN && typeCode <= TYPE_DECIMAL256);
+        // Valid type codes: TYPE_BOOLEAN (0x01) through TYPE_CHAR (0x16)
+        // This includes all basic types, arrays, decimals, and char
+        boolean valid = (typeCode >= TYPE_BOOLEAN && typeCode <= TYPE_CHAR);
         if (!valid) {
             throw new IllegalArgumentException(
                     "invalid column type code: 0x" + Integer.toHexString(typeCode)
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpColumnDefTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpColumnDefTest.java
new file mode 100644
index 0000000..2bd28b9
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpColumnDefTest.java
@@ -0,0 +1,97 @@
+/*******************************************************************************
+ *     ___                  _   ____  ____
+ *    / _ \ _   _  ___  ___| |_|  _ \| __ )
+ *   | | | | | | |/ _ \/ __| __| | | |  _ \
+ *   | |_| | |_| |  __/\__ \ |_| |_| | |_) |
+ *    \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ *  Copyright (c) 2014-2019 Appsicle
+ *  Copyright (c) 2019-2026 QuestDB
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.cutlass.qwp.protocol;
+
+import io.questdb.client.cutlass.qwp.protocol.QwpColumnDef;
+import io.questdb.client.cutlass.qwp.protocol.QwpConstants;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+public class QwpColumnDefTest {
+
+    @Test
+    public void testValidateAcceptsAllValidTypes() {
+        byte[] validTypes = {
+                QwpConstants.TYPE_BOOLEAN,
+                QwpConstants.TYPE_BYTE,
+                QwpConstants.TYPE_SHORT,
+                QwpConstants.TYPE_INT,
+                QwpConstants.TYPE_LONG,
+                QwpConstants.TYPE_FLOAT,
+                QwpConstants.TYPE_DOUBLE,
+                QwpConstants.TYPE_STRING,
+                QwpConstants.TYPE_SYMBOL,
+                QwpConstants.TYPE_TIMESTAMP,
+                QwpConstants.TYPE_DATE,
+                QwpConstants.TYPE_UUID,
+                QwpConstants.TYPE_LONG256,
+                QwpConstants.TYPE_GEOHASH,
+                QwpConstants.TYPE_VARCHAR,
+                QwpConstants.TYPE_TIMESTAMP_NANOS,
+                QwpConstants.TYPE_DOUBLE_ARRAY,
+                QwpConstants.TYPE_LONG_ARRAY,
+                QwpConstants.TYPE_DECIMAL64,
+                QwpConstants.TYPE_DECIMAL128,
+                QwpConstants.TYPE_DECIMAL256,
+                QwpConstants.TYPE_CHAR,
+        };
+        for (byte type : validTypes) {
+            QwpColumnDef col = new QwpColumnDef("col", type);
+            col.validate(); // must not throw
+        }
+    }
+
+    @Test
+    public void testValidateCharType() {
+        // TYPE_CHAR (0x16) must pass validation
+        QwpColumnDef col = new QwpColumnDef("ch", QwpConstants.TYPE_CHAR);
+        col.validate();
+        assertEquals("CHAR", col.getTypeName());
+        assertEquals(QwpConstants.TYPE_CHAR, col.getTypeCode());
+    }
+
+    @Test
+    public void testValidateNullableCharType() {
+        // TYPE_CHAR with nullable flag must also pass
+        byte nullableChar = (byte) (QwpConstants.TYPE_CHAR | QwpConstants.TYPE_NULLABLE_FLAG);
+        QwpColumnDef col = new QwpColumnDef("ch", nullableChar);
+        col.validate();
+        assertTrue(col.isNullable());
+        assertEquals(QwpConstants.TYPE_CHAR, col.getTypeCode());
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testValidateRejectsInvalidType() {
+        QwpColumnDef col = new QwpColumnDef("bad", (byte) 0x17);
+        col.validate();
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testValidateRejectsZeroType() {
+        QwpColumnDef col = new QwpColumnDef("bad", (byte) 0x00);
+        col.validate();
+    }
+}

From bed5c32b126f2e26c44c8d70d9716f4bfaee0fbe Mon Sep 17 00:00:00 2001
From: Marko Topolnik 
Date: Tue, 24 Feb 2026 14:48:50 +0100
Subject: [PATCH 29/89] Throw LineSenderException for token in ws/wss config

Replace raw AssertionError with LineSenderException when a token
parameter is provided in ws:: or wss:: configuration strings. The
else branch in config string parsing was unreachable when the code
only supported HTTP and TCP, but became reachable after WebSocket
support was added. Users now get a clear "token is not supported
for WebSocket protocol" error instead of a cryptic AssertionError.

Add test assertions for both ws:: and wss:: schemas with token.

Co-Authored-By: Claude Opus 4.6 
---
 core/src/main/java/io/questdb/client/Sender.java                | 2 +-
 .../questdb/client/test/cutlass/line/LineSenderBuilderTest.java | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java
index b2b3c19..308da3a 100644
--- a/core/src/main/java/io/questdb/client/Sender.java
+++ b/core/src/main/java/io/questdb/client/Sender.java
@@ -1583,7 +1583,7 @@ private LineSenderBuilder fromConfig(CharSequence configurationString) {
                     } else if (protocol == PROTOCOL_HTTP) {
                         httpToken(sink.toString());
                     } else {
-                        throw new AssertionError();
+                        throw new LineSenderException("token is not supported for WebSocket protocol");
                     }
                 } else if (Chars.equals("retry_timeout", sink)) {
                     pos = getValue(configurationString, pos, sink, "retry_timeout");
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/line/LineSenderBuilderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/line/LineSenderBuilderTest.java
index e684e52..0fcad48 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/line/LineSenderBuilderTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/line/LineSenderBuilderTest.java
@@ -179,6 +179,8 @@ public void testConfStringValidation() throws Exception {
             assertConfStrError("http::addr=localhost;auto_flush_bytes=1024;", "auto_flush_bytes is only supported for TCP transport");
             assertConfStrError("http::addr=localhost;protocol_version=10", "current client only supports protocol version 1(text format for all datatypes), 2(binary format for part datatypes), 3(decimal datatype) or explicitly unset");
             assertConfStrError("http::addr=localhost:48884;max_name_len=10;", "max_name_len must be at least 16 bytes [max_name_len=10]");
+            assertConfStrError("ws::addr=localhost;token=foo;", "token is not supported for WebSocket protocol");
+            assertConfStrError("wss::addr=localhost;token=foo;", "token is not supported for WebSocket protocol");
 
             assertConfStrOk("addr=localhost:8080", "auto_flush_rows=100", "protocol_version=1");
             assertConfStrOk("addr=localhost:8080", "auto_flush=on", "auto_flush_rows=100", "protocol_version=2");

From 924e26af16eb435f220d73297aeef16830939142 Mon Sep 17 00:00:00 2001
From: Marko Topolnik 
Date: Tue, 24 Feb 2026 14:57:16 +0100
Subject: [PATCH 30/89] Fix racy batch ID counter in MicrobatchBuffer

The static nextBatchId field was a plain long incremented with ++,
which is a non-atomic read-modify-write. Multiple threads creating
or resetting MicrobatchBuffer instances concurrently (e.g., several
Sender instances, or a user thread resetting while another constructs)
could read the same value and produce duplicate batch IDs.

Replace the plain long with an AtomicLong and use getAndIncrement()
in both the constructor and reset() to guarantee uniqueness.

Add MicrobatchBufferTest with two concurrent tests that confirm
batch ID uniqueness under contention from 8 threads.

Co-Authored-By: Claude Opus 4.6 
---
 .../cutlass/qwp/client/MicrobatchBuffer.java  |   7 +-
 .../qwp/client/MicrobatchBufferTest.java      | 124 ++++++++++++++++++
 2 files changed, 128 insertions(+), 3 deletions(-)
 create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/MicrobatchBufferTest.java

diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java
index 9bcf738..4ef2af4 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java
@@ -30,6 +30,7 @@
 
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
 
 /**
  * A buffer for accumulating ILP data into microbatches before sending.
@@ -79,7 +80,7 @@ public class MicrobatchBuffer implements QuietCloseable {
 
     // Batch identification
     private long batchId;
-    private static long nextBatchId = 0;
+    private static final AtomicLong nextBatchId = new AtomicLong();
 
     // State machine
     private volatile int state = STATE_FILLING;
@@ -108,7 +109,7 @@ public MicrobatchBuffer(int initialCapacity, int maxRows, int maxBytes, long max
         this.maxRows = maxRows;
         this.maxBytes = maxBytes;
         this.maxAgeNanos = maxAgeNanos;
-        this.batchId = nextBatchId++;
+        this.batchId = nextBatchId.getAndIncrement();
     }
 
     /**
@@ -452,7 +453,7 @@ public void reset() {
         rowCount = 0;
         firstRowTimeNanos = 0;
         maxSymbolId = -1;
-        batchId = nextBatchId++;
+        batchId = nextBatchId.getAndIncrement();
         state = STATE_FILLING;
         recycleLatch = new CountDownLatch(1);
     }
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/MicrobatchBufferTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/MicrobatchBufferTest.java
new file mode 100644
index 0000000..cbc81ef
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/MicrobatchBufferTest.java
@@ -0,0 +1,124 @@
+/*******************************************************************************
+ *     ___                  _   ____  ____
+ *    / _ \ _   _  ___  ___| |_|  _ \| __ )
+ *   | | | | | | |/ _ \/ __| __| | | |  _ \
+ *   | |_| | |_| |  __/\__ \ |_| |_| | |_) |
+ *    \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ *  Copyright (c) 2014-2019 Appsicle
+ *  Copyright (c) 2019-2026 QuestDB
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.cutlass.qwp.client;
+
+import io.questdb.client.cutlass.qwp.client.MicrobatchBuffer;
+import org.junit.Test;
+
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.Assert.*;
+
+public class MicrobatchBufferTest {
+
+    @Test
+    public void testConcurrentBatchIdUniqueness() throws Exception {
+        int threadCount = 8;
+        int buffersPerThread = 500;
+        int totalBuffers = threadCount * buffersPerThread;
+        Set batchIds = ConcurrentHashMap.newKeySet();
+        CountDownLatch startLatch = new CountDownLatch(1);
+        CountDownLatch doneLatch = new CountDownLatch(threadCount);
+
+        Thread[] threads = new Thread[threadCount];
+        for (int t = 0; t < threadCount; t++) {
+            threads[t] = new Thread(() -> {
+                try {
+                    startLatch.await();
+                } catch (InterruptedException e) {
+                    Thread.currentThread().interrupt();
+                    return;
+                }
+                try {
+                    for (int i = 0; i < buffersPerThread; i++) {
+                        MicrobatchBuffer buf = new MicrobatchBuffer(64);
+                        batchIds.add(buf.getBatchId());
+                        buf.close();
+                    }
+                } finally {
+                    doneLatch.countDown();
+                }
+            });
+            threads[t].start();
+        }
+
+        startLatch.countDown();
+        assertTrue("Threads did not finish in time", doneLatch.await(30, TimeUnit.SECONDS));
+
+        assertEquals(
+                "Duplicate batch IDs detected: expected " + totalBuffers + " unique IDs but got " + batchIds.size(),
+                totalBuffers,
+                batchIds.size()
+        );
+    }
+
+    @Test
+    public void testConcurrentResetBatchIdUniqueness() throws Exception {
+        int threadCount = 8;
+        int resetsPerThread = 500;
+        int totalIds = threadCount * resetsPerThread;
+        Set batchIds = ConcurrentHashMap.newKeySet();
+        CountDownLatch startLatch = new CountDownLatch(1);
+        CountDownLatch doneLatch = new CountDownLatch(threadCount);
+
+        Thread[] threads = new Thread[threadCount];
+        for (int t = 0; t < threadCount; t++) {
+            threads[t] = new Thread(() -> {
+                try {
+                    startLatch.await();
+                } catch (InterruptedException e) {
+                    Thread.currentThread().interrupt();
+                    return;
+                }
+                try {
+                    MicrobatchBuffer buf = new MicrobatchBuffer(64);
+                    for (int i = 0; i < resetsPerThread; i++) {
+                        buf.seal();
+                        buf.markSending();
+                        buf.markRecycled();
+                        buf.reset();
+                        batchIds.add(buf.getBatchId());
+                    }
+                    buf.close();
+                } finally {
+                    doneLatch.countDown();
+                }
+            });
+            threads[t].start();
+        }
+
+        startLatch.countDown();
+        assertTrue("Threads did not finish in time", doneLatch.await(30, TimeUnit.SECONDS));
+
+        assertEquals(
+                "Duplicate batch IDs from reset(): expected " + totalIds + " unique IDs but got " + batchIds.size(),
+                totalIds,
+                batchIds.size()
+        );
+    }
+}

From 34e0cf74910cf44a360b058e4a81d408ebd17bc4 Mon Sep 17 00:00:00 2001
From: Marko Topolnik 
Date: Tue, 24 Feb 2026 15:16:31 +0100
Subject: [PATCH 31/89] Throw on buffer overflow in QwpBitWriter and Gorilla
 encoder

QwpBitWriter's write methods (writeBits, writeByte, writeInt,
writeLong, flush) silently dropped data when the buffer was full
instead of signaling an error. The same pattern existed in
QwpGorillaEncoder.encodeTimestamps(), which returned partial
byte counts when capacity was insufficient for the first or
second uncompressed timestamp.

Replace all silent-drop guards with LineSenderException throws
so callers get a clear error instead of silently corrupted data.
Add QwpBitWriterTest covering overflow for each write method
and both Gorilla encoder early-return paths.

Co-Authored-By: Claude Opus 4.6 
---
 .../cutlass/qwp/protocol/QwpBitWriter.java    |  30 ++-
 .../qwp/protocol/QwpGorillaEncoder.java       |   5 +-
 .../qwp/protocol/QwpBitWriterTest.java        | 191 ++++++++++++++++++
 3 files changed, 213 insertions(+), 13 deletions(-)
 create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpBitWriterTest.java

diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitWriter.java
index 624b083..af30761 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitWriter.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitWriter.java
@@ -24,6 +24,7 @@
 
 package io.questdb.client.cutlass.qwp.protocol;
 
+import io.questdb.client.cutlass.line.LineSenderException;
 import io.questdb.client.std.Unsafe;
 
 /**
@@ -139,9 +140,10 @@ public void writeBits(long value, int numBits) {
 
             // Flush complete bytes from the buffer
             while (bitsInBuffer >= 8) {
-                if (currentAddress < endAddress) {
-                    Unsafe.getUnsafe().putByte(currentAddress++, (byte) bitBuffer);
+                if (currentAddress >= endAddress) {
+                    throw new LineSenderException("QwpBitWriter buffer overflow");
                 }
+                Unsafe.getUnsafe().putByte(currentAddress++, (byte) bitBuffer);
                 bitBuffer >>>= 8;
                 bitsInBuffer -= 8;
             }
@@ -168,7 +170,10 @@ public void writeSigned(long value, int numBits) {
      * Must be called before reading the output or getting the final position.
      */
     public void flush() {
-        if (bitsInBuffer > 0 && currentAddress < endAddress) {
+        if (bitsInBuffer > 0) {
+            if (currentAddress >= endAddress) {
+                throw new LineSenderException("QwpBitWriter buffer overflow");
+            }
             Unsafe.getUnsafe().putByte(currentAddress++, (byte) bitBuffer);
             bitBuffer = 0;
             bitsInBuffer = 0;
@@ -214,9 +219,10 @@ public void alignToByte() {
      */
     public void writeByte(int value) {
         alignToByte();
-        if (currentAddress < endAddress) {
-            Unsafe.getUnsafe().putByte(currentAddress++, (byte) value);
+        if (currentAddress >= endAddress) {
+            throw new LineSenderException("QwpBitWriter buffer overflow");
         }
+        Unsafe.getUnsafe().putByte(currentAddress++, (byte) value);
     }
 
     /**
@@ -226,10 +232,11 @@ public void writeByte(int value) {
      */
     public void writeInt(int value) {
         alignToByte();
-        if (currentAddress + 4 <= endAddress) {
-            Unsafe.getUnsafe().putInt(currentAddress, value);
-            currentAddress += 4;
+        if (currentAddress + 4 > endAddress) {
+            throw new LineSenderException("QwpBitWriter buffer overflow");
         }
+        Unsafe.getUnsafe().putInt(currentAddress, value);
+        currentAddress += 4;
     }
 
     /**
@@ -239,9 +246,10 @@ public void writeInt(int value) {
      */
     public void writeLong(long value) {
         alignToByte();
-        if (currentAddress + 8 <= endAddress) {
-            Unsafe.getUnsafe().putLong(currentAddress, value);
-            currentAddress += 8;
+        if (currentAddress + 8 > endAddress) {
+            throw new LineSenderException("QwpBitWriter buffer overflow");
         }
+        Unsafe.getUnsafe().putLong(currentAddress, value);
+        currentAddress += 8;
     }
 }
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java
index 8f59b38..912af2d 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java
@@ -24,6 +24,7 @@
 
 package io.questdb.client.cutlass.qwp.protocol;
 
+import io.questdb.client.cutlass.line.LineSenderException;
 import io.questdb.client.std.Unsafe;
 
 /**
@@ -170,7 +171,7 @@ public int encodeTimestamps(long destAddress, long capacity, long srcAddress, in
 
         // Write first timestamp uncompressed
         if (capacity < 8) {
-            return 0; // Not enough space
+            throw new LineSenderException("Gorilla encoder buffer overflow");
         }
         long ts0 = Unsafe.getUnsafe().getLong(srcAddress);
         Unsafe.getUnsafe().putLong(destAddress, ts0);
@@ -182,7 +183,7 @@ public int encodeTimestamps(long destAddress, long capacity, long srcAddress, in
 
         // Write second timestamp uncompressed
         if (capacity < pos + 8) {
-            return pos; // Not enough space
+            throw new LineSenderException("Gorilla encoder buffer overflow");
         }
         long ts1 = Unsafe.getUnsafe().getLong(srcAddress + 8);
         Unsafe.getUnsafe().putLong(destAddress + pos, ts1);
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpBitWriterTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpBitWriterTest.java
new file mode 100644
index 0000000..11f4c36
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpBitWriterTest.java
@@ -0,0 +1,191 @@
+/*******************************************************************************
+ *     ___                  _   ____  ____
+ *    / _ \ _   _  ___  ___| |_|  _ \| __ )
+ *   | | | | | | |/ _ \/ __| __| | | |  _ \
+ *   | |_| | |_| |  __/\__ \ |_| |_| | |_) |
+ *    \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ *  Copyright (c) 2014-2019 Appsicle
+ *  Copyright (c) 2019-2026 QuestDB
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.cutlass.qwp.protocol;
+
+import io.questdb.client.cutlass.line.LineSenderException;
+import io.questdb.client.cutlass.qwp.protocol.QwpBitWriter;
+import io.questdb.client.cutlass.qwp.protocol.QwpGorillaEncoder;
+import io.questdb.client.std.MemoryTag;
+import io.questdb.client.std.Unsafe;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+public class QwpBitWriterTest {
+
+    @Test
+    public void testWriteBitsThrowsOnOverflow() {
+        long ptr = Unsafe.malloc(4, MemoryTag.NATIVE_ILP_RSS);
+        try {
+            QwpBitWriter writer = new QwpBitWriter();
+            writer.reset(ptr, 4);
+            // Fill the buffer (32 bits = 4 bytes)
+            writer.writeBits(0xFFFF_FFFFL, 32);
+            // Next write should throw — buffer is full
+            try {
+                writer.writeBits(1, 8);
+                fail("expected LineSenderException on buffer overflow");
+            } catch (LineSenderException e) {
+                assertTrue(e.getMessage().contains("buffer overflow"));
+            }
+        } finally {
+            Unsafe.free(ptr, 4, MemoryTag.NATIVE_ILP_RSS);
+        }
+    }
+
+    @Test
+    public void testWriteByteThrowsOnOverflow() {
+        long ptr = Unsafe.malloc(1, MemoryTag.NATIVE_ILP_RSS);
+        try {
+            QwpBitWriter writer = new QwpBitWriter();
+            writer.reset(ptr, 1);
+            writer.writeByte(0x42);
+            try {
+                writer.writeByte(0x43);
+                fail("expected LineSenderException on buffer overflow");
+            } catch (LineSenderException e) {
+                assertTrue(e.getMessage().contains("buffer overflow"));
+            }
+        } finally {
+            Unsafe.free(ptr, 1, MemoryTag.NATIVE_ILP_RSS);
+        }
+    }
+
+    @Test
+    public void testWriteIntThrowsOnOverflow() {
+        long ptr = Unsafe.malloc(4, MemoryTag.NATIVE_ILP_RSS);
+        try {
+            QwpBitWriter writer = new QwpBitWriter();
+            writer.reset(ptr, 4);
+            writer.writeInt(42);
+            try {
+                writer.writeInt(99);
+                fail("expected LineSenderException on buffer overflow");
+            } catch (LineSenderException e) {
+                assertTrue(e.getMessage().contains("buffer overflow"));
+            }
+        } finally {
+            Unsafe.free(ptr, 4, MemoryTag.NATIVE_ILP_RSS);
+        }
+    }
+
+    @Test
+    public void testWriteLongThrowsOnOverflow() {
+        long ptr = Unsafe.malloc(8, MemoryTag.NATIVE_ILP_RSS);
+        try {
+            QwpBitWriter writer = new QwpBitWriter();
+            writer.reset(ptr, 8);
+            writer.writeLong(42L);
+            try {
+                writer.writeLong(99L);
+                fail("expected LineSenderException on buffer overflow");
+            } catch (LineSenderException e) {
+                assertTrue(e.getMessage().contains("buffer overflow"));
+            }
+        } finally {
+            Unsafe.free(ptr, 8, MemoryTag.NATIVE_ILP_RSS);
+        }
+    }
+
+    @Test
+    public void testFlushThrowsOnOverflow() {
+        long ptr = Unsafe.malloc(1, MemoryTag.NATIVE_ILP_RSS);
+        try {
+            QwpBitWriter writer = new QwpBitWriter();
+            writer.reset(ptr, 1);
+            // Write 8 bits to fill the single byte
+            writer.writeBits(0xFF, 8);
+            // Write a few more bits that sit in the bit buffer
+            writer.writeBits(0x3, 4);
+            // Flush should throw because there's no room for the partial byte
+            try {
+                writer.flush();
+                fail("expected LineSenderException on buffer overflow during flush");
+            } catch (LineSenderException e) {
+                assertTrue(e.getMessage().contains("buffer overflow"));
+            }
+        } finally {
+            Unsafe.free(ptr, 1, MemoryTag.NATIVE_ILP_RSS);
+        }
+    }
+
+    // --- QwpGorillaEncoder overflow tests ---
+
+    @Test
+    public void testGorillaEncoderThrowsOnInsufficientCapacityForFirstTimestamp() {
+        // Source: 1 timestamp (8 bytes), dest: only 4 bytes
+        long src = Unsafe.malloc(8, MemoryTag.NATIVE_ILP_RSS);
+        long dst = Unsafe.malloc(4, MemoryTag.NATIVE_ILP_RSS);
+        try {
+            Unsafe.getUnsafe().putLong(src, 1_000_000L);
+            QwpGorillaEncoder encoder = new QwpGorillaEncoder();
+            try {
+                encoder.encodeTimestamps(dst, 4, src, 1);
+                fail("expected LineSenderException on buffer overflow");
+            } catch (LineSenderException e) {
+                assertTrue(e.getMessage().contains("buffer overflow"));
+            }
+        } finally {
+            Unsafe.free(src, 8, MemoryTag.NATIVE_ILP_RSS);
+            Unsafe.free(dst, 4, MemoryTag.NATIVE_ILP_RSS);
+        }
+    }
+
+    @Test
+    public void testGorillaEncoderThrowsOnInsufficientCapacityForSecondTimestamp() {
+        // Source: 2 timestamps (16 bytes), dest: only 12 bytes (enough for first, not second)
+        long src = Unsafe.malloc(16, MemoryTag.NATIVE_ILP_RSS);
+        long dst = Unsafe.malloc(12, MemoryTag.NATIVE_ILP_RSS);
+        try {
+            Unsafe.getUnsafe().putLong(src, 1_000_000L);
+            Unsafe.getUnsafe().putLong(src + 8, 2_000_000L);
+            QwpGorillaEncoder encoder = new QwpGorillaEncoder();
+            try {
+                encoder.encodeTimestamps(dst, 12, src, 2);
+                fail("expected LineSenderException on buffer overflow");
+            } catch (LineSenderException e) {
+                assertTrue(e.getMessage().contains("buffer overflow"));
+            }
+        } finally {
+            Unsafe.free(src, 16, MemoryTag.NATIVE_ILP_RSS);
+            Unsafe.free(dst, 12, MemoryTag.NATIVE_ILP_RSS);
+        }
+    }
+
+    @Test
+    public void testWriteBitsWithinCapacitySucceeds() {
+        long ptr = Unsafe.malloc(8, MemoryTag.NATIVE_ILP_RSS);
+        try {
+            QwpBitWriter writer = new QwpBitWriter();
+            writer.reset(ptr, 8);
+            writer.writeBits(0xDEAD_BEEF_CAFE_BABEL, 64);
+            writer.flush();
+            assertEquals(8, writer.getPosition() - ptr);
+            assertEquals(0xDEAD_BEEF_CAFE_BABEL, Unsafe.getUnsafe().getLong(ptr));
+        } finally {
+            Unsafe.free(ptr, 8, MemoryTag.NATIVE_ILP_RSS);
+        }
+    }
+}

From ba49b6b7f53be2860d3f4f7525c7cc18cd98f16f Mon Sep 17 00:00:00 2001
From: Marko Topolnik 
Date: Tue, 24 Feb 2026 15:31:44 +0100
Subject: [PATCH 32/89] Add NativeBufferWriter tests

Add test coverage for NativeBufferWriter, which previously had no
tests. The tests exercise skip(), patchInt(), and ensureCapacity()
mechanics, including the skip-then-patch pattern used by
QwpWebSocketEncoder to reserve and backfill length fields.

skip() and patchInt() were flagged as missing bounds checks, but
investigation showed both are internal-only methods called exclusively
by QwpWebSocketEncoder with structurally guaranteed-safe arguments:
skip()'s single caller already ensures capacity before the call, and
patchInt()'s two callers always pass the hardcoded offset 8 within a
12-byte header. Adding runtime checks would be dead branches on the
hot path, so only test coverage was added.

Co-Authored-By: Claude Opus 4.6 
---
 .../qwp/client/NativeBufferWriterTest.java    | 89 +++++++++++++++++++
 1 file changed, 89 insertions(+)
 create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java

diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java
new file mode 100644
index 0000000..46eae77
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java
@@ -0,0 +1,89 @@
+/*******************************************************************************
+ *     ___                  _   ____  ____
+ *    / _ \ _   _  ___  ___| |_|  _ \| __ )
+ *   | | | | | | |/ _ \/ __| __| | | |  _ \
+ *   | |_| | |_| |  __/\__ \ |_| |_| | |_) |
+ *    \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ *  Copyright (c) 2014-2019 Appsicle
+ *  Copyright (c) 2019-2026 QuestDB
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.cutlass.qwp.client;
+
+import io.questdb.client.cutlass.qwp.client.NativeBufferWriter;
+import io.questdb.client.std.Unsafe;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class NativeBufferWriterTest {
+
+    @Test
+    public void testPatchIntAtLastValidOffset() {
+        try (NativeBufferWriter writer = new NativeBufferWriter(16)) {
+            writer.putLong(0L); // 8 bytes, position = 8
+            // Patch at offset 4 covers bytes [4..7], exactly at the boundary
+            writer.patchInt(4, 0x1234);
+            assertEquals(0x1234, Unsafe.getUnsafe().getInt(writer.getBufferPtr() + 4));
+        }
+    }
+
+    @Test
+    public void testPatchIntAtValidOffset() {
+        try (NativeBufferWriter writer = new NativeBufferWriter(16)) {
+            writer.putInt(0); // placeholder at offset 0
+            writer.putInt(0xBEEF); // data at offset 4
+            // Patch the placeholder
+            writer.patchInt(0, 0xCAFE);
+            assertEquals(0xCAFE, Unsafe.getUnsafe().getInt(writer.getBufferPtr()));
+            assertEquals(0xBEEF, Unsafe.getUnsafe().getInt(writer.getBufferPtr() + 4));
+        }
+    }
+
+    @Test
+    public void testSkipAdvancesPosition() {
+        try (NativeBufferWriter writer = new NativeBufferWriter(16)) {
+            writer.skip(4);
+            assertEquals(4, writer.getPosition());
+            writer.skip(8);
+            assertEquals(12, writer.getPosition());
+        }
+    }
+
+    @Test
+    public void testSkipThenPatchInt() {
+        try (NativeBufferWriter writer = new NativeBufferWriter(8)) {
+            int patchOffset = writer.getPosition();
+            writer.skip(4); // reserve space for a length field
+            writer.putInt(0xDEAD);
+            // Patch the reserved space
+            writer.patchInt(patchOffset, 4);
+            assertEquals(0x4, Unsafe.getUnsafe().getInt(writer.getBufferPtr() + patchOffset));
+            assertEquals(0xDEAD, Unsafe.getUnsafe().getInt(writer.getBufferPtr() + 4));
+        }
+    }
+
+    @Test
+    public void testEnsureCapacityGrowsBuffer() {
+        try (NativeBufferWriter writer = new NativeBufferWriter(16)) {
+            assertEquals(16, writer.getCapacity());
+            writer.ensureCapacity(32);
+            assertTrue(writer.getCapacity() >= 32);
+        }
+    }
+}

From 21aa7bb03a94caded767902441416b3957f259d4 Mon Sep 17 00:00:00 2001
From: Marko Topolnik 
Date: Tue, 24 Feb 2026 15:46:36 +0100
Subject: [PATCH 33/89] Fix stale array offsets after cancelRow truncation

truncateTo() rewound off-heap buffers (data, strings, aux) but
forgot to rewind arrayShapeOffset and arrayDataOffset. After
cancelling a row with array data and adding a replacement row,
the new values were written at a gap past where the encoder
reads, causing the encoder to serialize stale cancelled data.

Fix by walking the retained values to recompute both offsets,
matching the same traversal pattern the encoder uses. Add tests
that verify the encoder would read correct data after cancel
for 1D double arrays, 1D long arrays, and 2D double arrays.

Co-Authored-By: Claude Opus 4.6 
---
 .../cutlass/qwp/protocol/QwpTableBuffer.java  |  16 ++
 .../qwp/protocol/QwpTableBufferTest.java      | 216 ++++++++++++++++++
 2 files changed, 232 insertions(+)
 create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java

diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java
index b2c434c..ef1a090 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java
@@ -1085,6 +1085,22 @@ public void truncateTo(int newSize) {
             if (auxBuffer != null) {
                 auxBuffer.jumpTo((long) newValueCount * 4);
             }
+
+            // Rewind array offsets by walking the retained values
+            if (arrayDims != null) {
+                int newShapeOffset = 0;
+                int newDataOffset = 0;
+                for (int i = 0; i < newValueCount; i++) {
+                    int nDims = arrayDims[i];
+                    int elemCount = 1;
+                    for (int d = 0; d < nDims; d++) {
+                        elemCount *= arrayShapes[newShapeOffset++];
+                    }
+                    newDataOffset += elemCount;
+                }
+                arrayShapeOffset = newShapeOffset;
+                arrayDataOffset = newDataOffset;
+            }
         }
 
         private void allocateStorage(byte type) {
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java
new file mode 100644
index 0000000..f11e7fe
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java
@@ -0,0 +1,216 @@
+/*******************************************************************************
+ *     ___                  _   ____  ____
+ *    / _ \ _   _  ___  ___| |_|  _ \| __ )
+ *   | | | | | | |/ _ \/ __| __| | | |  _ \
+ *   | |_| | |_| |  __/\__ \ |_| |_| | |_) |
+ *    \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ *  Copyright (c) 2014-2019 Appsicle
+ *  Copyright (c) 2019-2026 QuestDB
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.cutlass.qwp.protocol;
+
+import io.questdb.client.cutlass.qwp.protocol.QwpConstants;
+import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+public class QwpTableBufferTest {
+
+    /**
+     * Simulates the encoder's walk over array data — the same logic as
+     * QwpWebSocketEncoder.writeDoubleArrayColumn(). Returns the flat
+     * double values the encoder would serialize for the given column.
+     */
+    private static double[] readDoubleArraysLikeEncoder(QwpTableBuffer.ColumnBuffer col) {
+        byte[] dims = col.getArrayDims();
+        int[] shapes = col.getArrayShapes();
+        double[] data = col.getDoubleArrayData();
+        int count = col.getValueCount();
+
+        // First pass: count total elements
+        int totalElements = 0;
+        int shapeIdx = 0;
+        for (int row = 0; row < count; row++) {
+            int nDims = dims[row];
+            int elemCount = 1;
+            for (int d = 0; d < nDims; d++) {
+                elemCount *= shapes[shapeIdx++];
+            }
+            totalElements += elemCount;
+        }
+
+        // Second pass: collect values
+        double[] result = new double[totalElements];
+        shapeIdx = 0;
+        int dataIdx = 0;
+        int resultIdx = 0;
+        for (int row = 0; row < count; row++) {
+            int nDims = dims[row];
+            int elemCount = 1;
+            for (int d = 0; d < nDims; d++) {
+                elemCount *= shapes[shapeIdx++];
+            }
+            for (int e = 0; e < elemCount; e++) {
+                result[resultIdx++] = data[dataIdx++];
+            }
+        }
+        return result;
+    }
+
+    /**
+     * Same as above but for long arrays (mirrors QwpWebSocketEncoder.writeLongArrayColumn()).
+     */
+    private static long[] readLongArraysLikeEncoder(QwpTableBuffer.ColumnBuffer col) {
+        byte[] dims = col.getArrayDims();
+        int[] shapes = col.getArrayShapes();
+        long[] data = col.getLongArrayData();
+        int count = col.getValueCount();
+
+        int totalElements = 0;
+        int shapeIdx = 0;
+        for (int row = 0; row < count; row++) {
+            int nDims = dims[row];
+            int elemCount = 1;
+            for (int d = 0; d < nDims; d++) {
+                elemCount *= shapes[shapeIdx++];
+            }
+            totalElements += elemCount;
+        }
+
+        long[] result = new long[totalElements];
+        shapeIdx = 0;
+        int dataIdx = 0;
+        int resultIdx = 0;
+        for (int row = 0; row < count; row++) {
+            int nDims = dims[row];
+            int elemCount = 1;
+            for (int d = 0; d < nDims; d++) {
+                elemCount *= shapes[shapeIdx++];
+            }
+            for (int e = 0; e < elemCount; e++) {
+                result[resultIdx++] = data[dataIdx++];
+            }
+        }
+        return result;
+    }
+
+    @Test
+    public void testCancelRowRewindsDoubleArrayOffsets() {
+        try (QwpTableBuffer table = new QwpTableBuffer("test")) {
+            // Row 0: committed with [1.0, 2.0]
+            QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false);
+            col.addDoubleArray(new double[]{1.0, 2.0});
+            table.nextRow();
+
+            // Row 1: committed with [3.0, 4.0]
+            col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false);
+            col.addDoubleArray(new double[]{3.0, 4.0});
+            table.nextRow();
+
+            // Start row 2 with [5.0, 6.0] — then cancel it
+            col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false);
+            col.addDoubleArray(new double[]{5.0, 6.0});
+            table.cancelCurrentRow();
+
+            // Add replacement row 2 with [7.0, 8.0]
+            col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false);
+            col.addDoubleArray(new double[]{7.0, 8.0});
+            table.nextRow();
+
+            assertEquals(3, table.getRowCount());
+            assertEquals(3, col.getValueCount());
+
+            // Walk the arrays exactly as the encoder would
+            double[] encoded = readDoubleArraysLikeEncoder(col);
+            assertArrayEquals(
+                    new double[]{1.0, 2.0, 3.0, 4.0, 7.0, 8.0},
+                    encoded,
+                    0.0
+            );
+        }
+    }
+
+    @Test
+    public void testCancelRowRewindsLongArrayOffsets() {
+        try (QwpTableBuffer table = new QwpTableBuffer("test")) {
+            // Row 0: committed with [10, 20]
+            QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_LONG_ARRAY, false);
+            col.addLongArray(new long[]{10, 20});
+            table.nextRow();
+
+            // Start row 1 with [30, 40] — then cancel it
+            col = table.getOrCreateColumn("arr", QwpConstants.TYPE_LONG_ARRAY, false);
+            col.addLongArray(new long[]{30, 40});
+            table.cancelCurrentRow();
+
+            // Add replacement row 1 with [50, 60]
+            col = table.getOrCreateColumn("arr", QwpConstants.TYPE_LONG_ARRAY, false);
+            col.addLongArray(new long[]{50, 60});
+            table.nextRow();
+
+            assertEquals(2, table.getRowCount());
+            assertEquals(2, col.getValueCount());
+
+            long[] encoded = readLongArraysLikeEncoder(col);
+            assertArrayEquals(new long[]{10, 20, 50, 60}, encoded);
+        }
+    }
+
+    @Test
+    public void testCancelRowRewindsMultiDimArrayOffsets() {
+        try (QwpTableBuffer table = new QwpTableBuffer("test")) {
+            // Row 0: committed 2D array [[1.0, 2.0], [3.0, 4.0]]
+            QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false);
+            col.addDoubleArray(new double[][]{{1.0, 2.0}, {3.0, 4.0}});
+            table.nextRow();
+
+            // Start row 1 with 2D array [[5.0, 6.0], [7.0, 8.0]] — cancel
+            col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false);
+            col.addDoubleArray(new double[][]{{5.0, 6.0}, {7.0, 8.0}});
+            table.cancelCurrentRow();
+
+            // Replacement row 1 with [[9.0, 10.0], [11.0, 12.0]]
+            col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false);
+            col.addDoubleArray(new double[][]{{9.0, 10.0}, {11.0, 12.0}});
+            table.nextRow();
+
+            assertEquals(2, table.getRowCount());
+            assertEquals(2, col.getValueCount());
+
+            // Verify shapes are correct (2 dims per row, each [2, 2])
+            int[] shapes = col.getArrayShapes();
+            byte[] dims = col.getArrayDims();
+            assertEquals(2, dims[0]);
+            assertEquals(2, dims[1]);
+            // Row 0 shapes: [2, 2]
+            assertEquals(2, shapes[0]);
+            assertEquals(2, shapes[1]);
+            // Row 1 shapes must be the replacement [2, 2], not stale data
+            assertEquals(2, shapes[2]);
+            assertEquals(2, shapes[3]);
+
+            double[] encoded = readDoubleArraysLikeEncoder(col);
+            assertArrayEquals(
+                    new double[]{1.0, 2.0, 3.0, 4.0, 9.0, 10.0, 11.0, 12.0},
+                    encoded,
+                    0.0
+            );
+        }
+    }
+}

From 1ef6d9ebd327ae37dcf5738d2cf82c7334129787 Mon Sep 17 00:00:00 2001
From: Marko Topolnik 
Date: Tue, 24 Feb 2026 16:03:16 +0100
Subject: [PATCH 34/89] Pass auto-flush config to sync-mode WebSocket sender

The builder computed actualAutoFlushRows, actualAutoFlushBytes, and
actualAutoFlushIntervalNanos but the sync-mode WebSocket path called
QwpWebSocketSender.connect(host, port, tls), which hardcoded all
three to 0. This silently disabled auto-flush for every sync-mode
WebSocket sender, regardless of user configuration.

Add a connect() overload that accepts auto-flush parameters and
update the builder to call it. Also update createForTesting(h, p,
windowSize) to use default auto-flush values instead of zeros, so
it mirrors the production connect() path.

Co-Authored-By: Claude Opus 4.6 
---
 .../main/java/io/questdb/client/Sender.java   |  5 +++-
 .../qwp/client/QwpWebSocketSender.java        | 26 +++++++++++++++---
 .../LineSenderBuilderWebSocketTest.java       | 27 +++++++++++++++++++
 3 files changed, 54 insertions(+), 4 deletions(-)

diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java
index 308da3a..3bd08d2 100644
--- a/core/src/main/java/io/questdb/client/Sender.java
+++ b/core/src/main/java/io/questdb/client/Sender.java
@@ -895,7 +895,10 @@ public Sender build() {
                     return QwpWebSocketSender.connect(
                             hosts.getQuick(0),
                             ports.getQuick(0),
-                            tlsEnabled
+                            tlsEnabled,
+                            actualAutoFlushRows,
+                            actualAutoFlushBytes,
+                            actualAutoFlushIntervalNanos
                     );
                 }
             }
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
index cf697d4..4deb020 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
@@ -194,7 +194,7 @@ public static QwpWebSocketSender connect(String host, int port) {
 
     /**
      * Creates a new sender with TLS and connects to the specified host and port.
-     * Uses synchronous mode for backward compatibility.
+     * Uses synchronous mode with default auto-flush settings.
      *
      * @param host       server host
      * @param port       server HTTP port
@@ -202,9 +202,28 @@ public static QwpWebSocketSender connect(String host, int port) {
      * @return connected sender
      */
     public static QwpWebSocketSender connect(String host, int port, boolean tlsEnabled) {
+        return connect(host, port, tlsEnabled,
+                DEFAULT_AUTO_FLUSH_ROWS, DEFAULT_AUTO_FLUSH_BYTES, DEFAULT_AUTO_FLUSH_INTERVAL_NANOS);
+    }
+
+    /**
+     * Creates a new sender with TLS and connects to the specified host and port.
+     * Uses synchronous mode with custom auto-flush settings.
+     *
+     * @param host                   server host
+     * @param port                   server HTTP port
+     * @param tlsEnabled             whether to use TLS
+     * @param autoFlushRows          rows per batch (0 = no limit)
+     * @param autoFlushBytes         bytes per batch (0 = no limit)
+     * @param autoFlushIntervalNanos age before flush in nanos (0 = no limit)
+     * @return connected sender
+     */
+    public static QwpWebSocketSender connect(String host, int port, boolean tlsEnabled,
+                                              int autoFlushRows, int autoFlushBytes,
+                                              long autoFlushIntervalNanos) {
         QwpWebSocketSender sender = new QwpWebSocketSender(
                 host, port, tlsEnabled, DEFAULT_BUFFER_SIZE,
-                0, 0, 0, // No auto-flush in sync mode
+                autoFlushRows, autoFlushBytes, autoFlushIntervalNanos,
                 1, 1    // window=1 for sync behavior, queue=1 (not used)
         );
         sender.ensureConnected();
@@ -300,6 +319,7 @@ public static QwpWebSocketSender create(
      * Creates a sender without connecting. For testing only.
      * 

      * This allows unit tests to test sender logic without requiring a real server.
+     * Uses default auto-flush settings.
      *
      * @param host               server host (not connected)
      * @param port               server port (not connected)
@@ -309,7 +329,7 @@ public static QwpWebSocketSender create(
     public static QwpWebSocketSender createForTesting(String host, int port, int inFlightWindowSize) {
         return new QwpWebSocketSender(
                 host, port, false, DEFAULT_BUFFER_SIZE,
-                0, 0, 0,
+                DEFAULT_AUTO_FLUSH_ROWS, DEFAULT_AUTO_FLUSH_BYTES, DEFAULT_AUTO_FLUSH_INTERVAL_NANOS,
                 inFlightWindowSize, DEFAULT_SEND_QUEUE_CAPACITY
         );
         // Note: does NOT call ensureConnected()
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java
index 4fb8121..4b31040 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java
@@ -26,6 +26,7 @@
 
 import io.questdb.client.Sender;
 import io.questdb.client.cutlass.line.LineSenderException;
+import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender;
 import io.questdb.client.test.AbstractTest;
 import io.questdb.client.test.tools.TestUtils;
 import org.junit.Assert;
@@ -621,6 +622,32 @@ public void testSyncModeDoesNotAllowSendQueueCapacity() {
 
     // ==================== Unsupported Features (HTTP-specific options) ====================
 
+    @Test
+    public void testSyncModeAutoFlushDefaults() throws Exception {
+        // Regression test: sync-mode connect() must not hardcode autoFlush to 0.
+        // createForTesting(host, port, windowSize) mirrors what connect(h,p,tls)
+        // creates internally. Verify it uses sensible defaults.
+        TestUtils.assertMemoryLeak(() -> {
+            QwpWebSocketSender sender = QwpWebSocketSender.createForTesting("localhost", 0, 1);
+            try {
+                Assert.assertEquals(
+                        QwpWebSocketSender.DEFAULT_AUTO_FLUSH_ROWS,
+                        sender.getAutoFlushRows()
+                );
+                Assert.assertEquals(
+                        QwpWebSocketSender.DEFAULT_AUTO_FLUSH_BYTES,
+                        sender.getAutoFlushBytes()
+                );
+                Assert.assertEquals(
+                        QwpWebSocketSender.DEFAULT_AUTO_FLUSH_INTERVAL_NANOS,
+                        sender.getAutoFlushIntervalNanos()
+                );
+            } finally {
+                sender.close();
+            }
+        });
+    }
+
     @Test
     public void testSyncModeIsDefault() {
         Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET)

From 38d96c76b0c0bf2f148a90b2730a8331aca9850e Mon Sep 17 00:00:00 2001
From: Marko Topolnik 
Date: Tue, 24 Feb 2026 16:10:52 +0100
Subject: [PATCH 35/89] Add DECIMAL64/128/256 to isFixedWidthType and
 getFixedTypeSize

These decimal types are fixed-width (8, 16, and 32 bytes respectively)
but were missing from both isFixedWidthType() and getFixedTypeSize()
in QwpConstants. This caused them to be incorrectly classified as
variable-width types.

Co-Authored-By: Claude Opus 4.6 
---
 .../questdb/client/cutlass/qwp/protocol/QwpConstants.java | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java
index 622dbdd..2a40a35 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java
@@ -384,7 +384,10 @@ public static boolean isFixedWidthType(byte typeCode) {
                 code == TYPE_TIMESTAMP_NANOS ||
                 code == TYPE_DATE ||
                 code == TYPE_UUID ||
-                code == TYPE_LONG256;
+                code == TYPE_LONG256 ||
+                code == TYPE_DECIMAL64 ||
+                code == TYPE_DECIMAL128 ||
+                code == TYPE_DECIMAL256;
     }
 
     /**
@@ -411,10 +414,13 @@ public static int getFixedTypeSize(byte typeCode) {
             case TYPE_TIMESTAMP:
             case TYPE_TIMESTAMP_NANOS:
             case TYPE_DATE:
+            case TYPE_DECIMAL64:
                 return 8;
             case TYPE_UUID:
+            case TYPE_DECIMAL128:
                 return 16;
             case TYPE_LONG256:
+            case TYPE_DECIMAL256:
                 return 32;
             default:
                 return -1; // Variable width

From e65c882f7332306f385930b546a289095d92eca0 Mon Sep 17 00:00:00 2001
From: Marko Topolnik 
Date: Tue, 24 Feb 2026 16:29:12 +0100
Subject: [PATCH 36/89] Pool ArrayCapture to eliminate per-row allocations

Make ArrayCapture a reusable instance field of ColumnBuffer instead
of allocating a new one (plus int[32], double[N], long[N]) on every
addDoubleArray(DoubleArray) / addLongArray(LongArray) call. The
pooled instance is reset() between uses; its internal data arrays
grow but never shrink, so repeated same-shaped rows allocate nothing
after the first call.

Also fix two pre-existing issues in ArrayCapture:
- putBlockOfBytes() always read native memory as doubles via
  getDouble(), so addLongArray(LongArray) would copy zero data
  elements (longDataOffset was never incremented). Now dispatches
  on a forLong flag to use getLong() for long arrays.
- putInt() unconditionally allocated both doubleData and longData
  arrays even though only one type is ever used per call. Now
  allocates only the needed array type.

Add tests covering the DoubleArray wrapper path (multiple rows,
shrinking sizes, varying dimensionality) and multi-row long array
data isolation.

Co-Authored-By: Claude Opus 4.6 
---
 .../cutlass/qwp/protocol/QwpTableBuffer.java  |  77 +++++---
 .../qwp/protocol/QwpTableBufferTest.java      | 164 ++++++++++++++++++
 2 files changed, 216 insertions(+), 25 deletions(-)

diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java
index ef1a090..b56cd28 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java
@@ -296,20 +296,38 @@ static int elementSize(byte type) {
     private static class ArrayCapture implements ArrayBufferAppender {
         double[] doubleData;
         int doubleDataOffset;
+        private boolean forLong;
         long[] longData;
         int longDataOffset;
         byte nDims;
-        int[] shape = new int[32];
-        int shapeIndex;
+        final int[] shape = new int[32];
+        private int shapeIndex;
+
+        void reset(boolean forLong) {
+            this.forLong = forLong;
+            shapeIndex = 0;
+            nDims = 0;
+            doubleDataOffset = 0;
+            longDataOffset = 0;
+        }
 
         @Override
         public void putBlockOfBytes(long from, long len) {
             int count = (int) (len / 8);
-            if (doubleData == null) {
-                doubleData = new double[count];
-            }
-            for (int i = 0; i < count; i++) {
-                doubleData[doubleDataOffset++] = Unsafe.getUnsafe().getDouble(from + i * 8L);
+            if (forLong) {
+                if (longData == null || longData.length < count) {
+                    longData = new long[count];
+                }
+                for (int i = 0; i < count; i++) {
+                    longData[longDataOffset++] = Unsafe.getUnsafe().getLong(from + i * 8L);
+                }
+            } else {
+                if (doubleData == null || doubleData.length < count) {
+                    doubleData = new double[count];
+                }
+                for (int i = 0; i < count; i++) {
+                    doubleData[doubleDataOffset++] = Unsafe.getUnsafe().getDouble(from + i * 8L);
+                }
             }
         }
 
@@ -336,8 +354,15 @@ public void putInt(int value) {
                     for (int i = 0; i < nDims; i++) {
                         totalElements *= shape[i];
                     }
-                    doubleData = new double[totalElements];
-                    longData = new long[totalElements];
+                    if (forLong) {
+                        if (longData == null || longData.length < totalElements) {
+                            longData = new long[totalElements];
+                        }
+                    } else {
+                        if (doubleData == null || doubleData.length < totalElements) {
+                            doubleData = new double[totalElements];
+                        }
+                    }
                 }
             }
         }
@@ -362,6 +387,7 @@ public static class ColumnBuffer implements QuietCloseable {
         final boolean nullable;
         final byte type;
         private final Decimal256 rescaleTemp = new Decimal256();
+        private ArrayCapture arrayCapture;
         private int arrayDataOffset;
         // Array storage (double/long arrays - variable length per row)
         private byte[] arrayDims;
@@ -573,16 +599,16 @@ public void addDoubleArray(DoubleArray array) {
                 addNull();
                 return;
             }
-            ArrayCapture capture = new ArrayCapture();
-            array.appendToBufPtr(capture);
+            arrayCapture.reset(false);
+            array.appendToBufPtr(arrayCapture);
 
-            ensureArrayCapacity(capture.nDims, capture.doubleDataOffset);
-            arrayDims[valueCount] = capture.nDims;
-            for (int i = 0; i < capture.nDims; i++) {
-                arrayShapes[arrayShapeOffset++] = capture.shape[i];
+            ensureArrayCapacity(arrayCapture.nDims, arrayCapture.doubleDataOffset);
+            arrayDims[valueCount] = arrayCapture.nDims;
+            for (int i = 0; i < arrayCapture.nDims; i++) {
+                arrayShapes[arrayShapeOffset++] = arrayCapture.shape[i];
             }
-            for (int i = 0; i < capture.doubleDataOffset; i++) {
-                doubleArrayData[arrayDataOffset++] = capture.doubleData[i];
+            for (int i = 0; i < arrayCapture.doubleDataOffset; i++) {
+                doubleArrayData[arrayDataOffset++] = arrayCapture.doubleData[i];
             }
             valueCount++;
             size++;
@@ -698,16 +724,16 @@ public void addLongArray(LongArray array) {
                 addNull();
                 return;
             }
-            ArrayCapture capture = new ArrayCapture();
-            array.appendToBufPtr(capture);
+            arrayCapture.reset(true);
+            array.appendToBufPtr(arrayCapture);
 
-            ensureArrayCapacity(capture.nDims, capture.longDataOffset);
-            arrayDims[valueCount] = capture.nDims;
-            for (int i = 0; i < capture.nDims; i++) {
-                arrayShapes[arrayShapeOffset++] = capture.shape[i];
+            ensureArrayCapacity(arrayCapture.nDims, arrayCapture.longDataOffset);
+            arrayDims[valueCount] = arrayCapture.nDims;
+            for (int i = 0; i < arrayCapture.nDims; i++) {
+                arrayShapes[arrayShapeOffset++] = arrayCapture.shape[i];
             }
-            for (int i = 0; i < capture.longDataOffset; i++) {
-                longArrayData[arrayDataOffset++] = capture.longData[i];
+            for (int i = 0; i < arrayCapture.longDataOffset; i++) {
+                longArrayData[arrayDataOffset++] = arrayCapture.longData[i];
             }
             valueCount++;
             size++;
@@ -1148,6 +1174,7 @@ private void allocateStorage(byte type) {
                 case TYPE_DOUBLE_ARRAY:
                 case TYPE_LONG_ARRAY:
                     arrayDims = new byte[16];
+                    arrayCapture = new ArrayCapture();
                     break;
                 case TYPE_DECIMAL64:
                     dataBuffer = new OffHeapAppendMemory(128);
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java
index f11e7fe..501893e 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java
@@ -24,6 +24,7 @@
 
 package io.questdb.client.test.cutlass.qwp.protocol;
 
+import io.questdb.client.cutlass.line.array.DoubleArray;
 import io.questdb.client.cutlass.qwp.protocol.QwpConstants;
 import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer;
 import org.junit.Test;
@@ -213,4 +214,167 @@ public void testCancelRowRewindsMultiDimArrayOffsets() {
             );
         }
     }
+
+    @Test
+    public void testDoubleArrayWrapperMultipleRows() {
+        try (QwpTableBuffer table = new QwpTableBuffer("test");
+             DoubleArray arr = new DoubleArray(3)) {
+            QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false);
+
+            arr.append(1.0).append(2.0).append(3.0);
+            col.addDoubleArray(arr);
+            table.nextRow();
+
+            // DoubleArray auto-wraps, so just append next row's data
+            arr.append(4.0).append(5.0).append(6.0);
+            col.addDoubleArray(arr);
+            table.nextRow();
+
+            arr.append(7.0).append(8.0).append(9.0);
+            col.addDoubleArray(arr);
+            table.nextRow();
+
+            assertEquals(3, col.getValueCount());
+            double[] encoded = readDoubleArraysLikeEncoder(col);
+            assertArrayEquals(
+                    new double[]{1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0},
+                    encoded,
+                    0.0
+            );
+
+            byte[] dims = col.getArrayDims();
+            int[] shapes = col.getArrayShapes();
+            for (int i = 0; i < 3; i++) {
+                assertEquals(1, dims[i]);
+                assertEquals(3, shapes[i]);
+            }
+        }
+    }
+
+    @Test
+    public void testDoubleArrayWrapperShrinkingSize() {
+        try (QwpTableBuffer table = new QwpTableBuffer("test")) {
+            QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false);
+
+            // Row 0: large array (5 elements)
+            try (DoubleArray big = new DoubleArray(5)) {
+                big.append(1.0).append(2.0).append(3.0).append(4.0).append(5.0);
+                col.addDoubleArray(big);
+                table.nextRow();
+            }
+
+            // Row 1: smaller array (2 elements) — must not see leftover data from row 0
+            try (DoubleArray small = new DoubleArray(2)) {
+                small.append(10.0).append(20.0);
+                col.addDoubleArray(small);
+                table.nextRow();
+            }
+
+            assertEquals(2, col.getValueCount());
+            double[] encoded = readDoubleArraysLikeEncoder(col);
+            assertArrayEquals(
+                    new double[]{1.0, 2.0, 3.0, 4.0, 5.0, 10.0, 20.0},
+                    encoded,
+                    0.0
+            );
+
+            int[] shapes = col.getArrayShapes();
+            assertEquals(5, shapes[0]);
+            assertEquals(2, shapes[1]);
+        }
+    }
+
+    @Test
+    public void testDoubleArrayWrapperVaryingDimensionality() {
+        try (QwpTableBuffer table = new QwpTableBuffer("test")) {
+            QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false);
+
+            // Row 0: 2D array (2x2)
+            try (DoubleArray matrix = new DoubleArray(2, 2)) {
+                matrix.append(1.0).append(2.0).append(3.0).append(4.0);
+                col.addDoubleArray(matrix);
+                table.nextRow();
+            }
+
+            // Row 1: 1D array (3 elements) — different dimensionality
+            try (DoubleArray vec = new DoubleArray(3)) {
+                vec.append(10.0).append(20.0).append(30.0);
+                col.addDoubleArray(vec);
+                table.nextRow();
+            }
+
+            assertEquals(2, col.getValueCount());
+
+            byte[] dims = col.getArrayDims();
+            assertEquals(2, dims[0]);
+            assertEquals(1, dims[1]);
+
+            int[] shapes = col.getArrayShapes();
+            // Row 0: shape [2, 2]
+            assertEquals(2, shapes[0]);
+            assertEquals(2, shapes[1]);
+            // Row 1: shape [3]
+            assertEquals(3, shapes[2]);
+
+            double[] encoded = readDoubleArraysLikeEncoder(col);
+            assertArrayEquals(
+                    new double[]{1.0, 2.0, 3.0, 4.0, 10.0, 20.0, 30.0},
+                    encoded,
+                    0.0
+            );
+        }
+    }
+
+    @Test
+    public void testLongArrayMultipleRows() {
+        try (QwpTableBuffer table = new QwpTableBuffer("test")) {
+            QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_LONG_ARRAY, false);
+
+            col.addLongArray(new long[]{10, 20, 30});
+            table.nextRow();
+
+            col.addLongArray(new long[]{40, 50, 60});
+            table.nextRow();
+
+            col.addLongArray(new long[]{70, 80, 90});
+            table.nextRow();
+
+            assertEquals(3, col.getValueCount());
+            long[] encoded = readLongArraysLikeEncoder(col);
+            assertArrayEquals(
+                    new long[]{10, 20, 30, 40, 50, 60, 70, 80, 90},
+                    encoded
+            );
+
+            byte[] dims = col.getArrayDims();
+            int[] shapes = col.getArrayShapes();
+            for (int i = 0; i < 3; i++) {
+                assertEquals(1, dims[i]);
+                assertEquals(3, shapes[i]);
+            }
+        }
+    }
+
+    @Test
+    public void testLongArrayShrinkingSize() {
+        try (QwpTableBuffer table = new QwpTableBuffer("test")) {
+            QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_LONG_ARRAY, false);
+
+            // Row 0: large array (4 elements)
+            col.addLongArray(new long[]{100, 200, 300, 400});
+            table.nextRow();
+
+            // Row 1: smaller array (2 elements) — must not see leftover data from row 0
+            col.addLongArray(new long[]{10, 20});
+            table.nextRow();
+
+            assertEquals(2, col.getValueCount());
+            long[] encoded = readLongArraysLikeEncoder(col);
+            assertArrayEquals(new long[]{100, 200, 300, 400, 10, 20}, encoded);
+
+            int[] shapes = col.getArrayShapes();
+            assertEquals(4, shapes[0]);
+            assertEquals(2, shapes[1]);
+        }
+    }
 }

From 4c0c2fff679ecdbea019494f1d54a80ce8a53964 Mon Sep 17 00:00:00 2001
From: Marko Topolnik 
Date: Tue, 24 Feb 2026 16:33:59 +0100
Subject: [PATCH 37/89] Remove dead parseBuffer allocation in ResponseReader

The parseBufferPtr field (2KB native memory via Unsafe.malloc) was
allocated in the constructor and freed in close(), but never actually
read from or written to. The readLoop() receives response data
through channel.receiveFrame() which passes the channel's own buffer
pointer directly to ResponseHandlerImpl.onBinaryMessage(), bypassing
parseBufferPtr entirely.

Remove the unused fields, allocation, deallocation, and the
now-unnecessary Unsafe and MemoryTag imports.

Co-Authored-By: Claude Opus 4.6 
---
 .../client/cutlass/qwp/client/ResponseReader.java | 15 ---------------
 1 file changed, 15 deletions(-)

diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/ResponseReader.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/ResponseReader.java
index 2b78429..1d2b89f 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/ResponseReader.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/ResponseReader.java
@@ -27,9 +27,7 @@
 import io.questdb.client.cutlass.line.LineSenderException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import io.questdb.client.std.MemoryTag;
 import io.questdb.client.std.QuietCloseable;
-import io.questdb.client.std.Unsafe;
 
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
@@ -61,10 +59,6 @@ public class ResponseReader implements QuietCloseable {
     private final CountDownLatch shutdownLatch;
     private final WebSocketResponse response;
 
-    // Buffer for parsing responses
-    private final long parseBufferPtr;
-    private final int parseBufferSize;
-
     // State
     private volatile boolean running;
     private volatile Throwable lastError;
@@ -91,10 +85,6 @@ public ResponseReader(WebSocketChannel channel, InFlightWindow inFlightWindow) {
         this.inFlightWindow = inFlightWindow;
         this.response = new WebSocketResponse();
 
-        // Allocate parse buffer (enough for max response)
-        this.parseBufferSize = 2048;
-        this.parseBufferPtr = Unsafe.malloc(parseBufferSize, MemoryTag.NATIVE_DEFAULT);
-
         this.running = true;
         this.shutdownLatch = new CountDownLatch(1);
 
@@ -151,11 +141,6 @@ public void close() {
             Thread.currentThread().interrupt();
         }
 
-        // Free parse buffer
-        if (parseBufferPtr != 0) {
-            Unsafe.free(parseBufferPtr, parseBufferSize, MemoryTag.NATIVE_DEFAULT);
-        }
-
         LOG.info("Response reader closed [totalAcks={}, totalErrors={}]", totalAcks.get(), totalErrors.get());
     }
 

From f71a1eed9876295dd123939356904775349a1364 Mon Sep 17 00:00:00 2001
From: Marko Topolnik 
Date: Tue, 24 Feb 2026 16:37:29 +0100
Subject: [PATCH 38/89] Auto-clean up QwpBitReader

---
 .../cutlass/qwp/protocol/QwpBitReader.java    | 211 +++++++++---------
 1 file changed, 104 insertions(+), 107 deletions(-)

diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitReader.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitReader.java
index bd99092..9241d70 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitReader.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitReader.java
@@ -45,14 +45,12 @@
  */
 public class QwpBitReader {
 
-    private long startAddress;
-    private long currentAddress;
-    private long endAddress;
-
     // Buffer for reading bits
     private long bitBuffer;
     // Number of bits currently available in the buffer (0-64)
     private int bitsInBuffer;
+    private long currentAddress;
+    private long endAddress;
     // Total bits available for reading (from reset)
     private long totalBitsAvailable;
     // Total bits already consumed
@@ -65,75 +63,73 @@ public QwpBitReader() {
     }
 
     /**
-     * Resets the reader to read from the specified memory region.
+     * Aligns the reader to the next byte boundary by skipping any partial bits.
      *
-     * @param address the starting address
-     * @param length  the number of bytes available to read
+     * @throws IllegalStateException if alignment fails
      */
-    public void reset(long address, long length) {
-        this.startAddress = address;
-        this.currentAddress = address;
-        this.endAddress = address + length;
-        this.bitBuffer = 0;
-        this.bitsInBuffer = 0;
-        this.totalBitsAvailable = length * 8L;
-        this.totalBitsRead = 0;
+    public void alignToByte() {
+        int bitsToSkip = bitsInBuffer % 8;
+        if (bitsToSkip != 0) {
+            // We need to skip the remaining bits in the current partial byte
+            // But since we read in byte chunks, bitsInBuffer should be a multiple of 8
+            // minus what we've consumed. The remainder in the conceptual stream is:
+            int remainder = (int) (totalBitsRead % 8);
+            if (remainder != 0) {
+                skipBits(8 - remainder);
+            }
+        }
     }
 
     /**
-     * Resets the reader to read from the specified byte array.
+     * Returns the number of bits remaining to be read.
      *
-     * @param buf    the byte array
-     * @param offset the starting offset
-     * @param length the number of bytes available
+     * @return available bits
      */
-    public void reset(byte[] buf, int offset, int length) {
-        // For byte array, we'll store position info differently
-        // This is mainly for testing - in production we use direct memory
-        throw new UnsupportedOperationException("Use direct memory version");
+    public long getAvailableBits() {
+        return totalBitsAvailable - totalBitsRead;
     }
 
     /**
-     * Returns the number of bits remaining to be read.
+     * Returns the current position in bits from the start.
      *
-     * @return available bits
+     * @return bits read since reset
      */
-    public long getAvailableBits() {
-        return totalBitsAvailable - totalBitsRead;
+    public long getBitPosition() {
+        return totalBitsRead;
     }
 
     /**
-     * Returns true if there are more bits to read.
+     * Returns the current byte address being read.
+     * Note: This is approximate due to bit buffering.
      *
-     * @return true if bits available
+     * @return current address
      */
-    public boolean hasMoreBits() {
-        return totalBitsRead < totalBitsAvailable;
+    public long getCurrentAddress() {
+        return currentAddress;
     }
 
     /**
-     * Returns the current position in bits from the start.
+     * Returns true if there are more bits to read.
      *
-     * @return bits read since reset
+     * @return true if bits available
      */
-    public long getBitPosition() {
-        return totalBitsRead;
+    public boolean hasMoreBits() {
+        return totalBitsRead < totalBitsAvailable;
     }
 
     /**
-     * Ensures the buffer has at least the requested number of bits.
-     * Loads more bytes from memory if needed.
+     * Peeks at the next bit without consuming it.
      *
-     * @param bitsNeeded minimum bits required in buffer
-     * @return true if sufficient bits available, false otherwise
+     * @return 0 or 1, or -1 if no more bits
      */
-    private boolean ensureBits(int bitsNeeded) {
-        while (bitsInBuffer < bitsNeeded && currentAddress < endAddress) {
-            byte b = Unsafe.getUnsafe().getByte(currentAddress++);
-            bitBuffer |= (long) (b & 0xFF) << bitsInBuffer;
-            bitsInBuffer += 8;
+    public int peekBit() {
+        if (totalBitsRead >= totalBitsAvailable) {
+            return -1;
         }
-        return bitsInBuffer >= bitsNeeded;
+        if (!ensureBits(1)) {
+            return -1;
+        }
+        return (int) (bitBuffer & 1);
     }
 
     /**
@@ -203,6 +199,36 @@ public long readBits(int numBits) {
         return result;
     }
 
+    /**
+     * Reads a complete byte, ensuring byte alignment first.
+     *
+     * @return the byte value (0-255)
+     * @throws IllegalStateException if not enough data
+     */
+    public int readByte() {
+        return (int) readBits(8) & 0xFF;
+    }
+
+    /**
+     * Reads a complete 32-bit integer in little-endian order.
+     *
+     * @return the integer value
+     * @throws IllegalStateException if not enough data
+     */
+    public int readInt() {
+        return (int) readBits(32);
+    }
+
+    /**
+     * Reads a complete 64-bit long in little-endian order.
+     *
+     * @return the long value
+     * @throws IllegalStateException if not enough data
+     */
+    public long readLong() {
+        return readBits(64);
+    }
+
     /**
      * Reads multiple bits and interprets them as a signed value using two's complement.
      *
@@ -221,18 +247,31 @@ public long readSigned(int numBits) {
     }
 
     /**
-     * Peeks at the next bit without consuming it.
+     * Resets the reader to read from the specified memory region.
      *
-     * @return 0 or 1, or -1 if no more bits
+     * @param address the starting address
+     * @param length  the number of bytes available to read
      */
-    public int peekBit() {
-        if (totalBitsRead >= totalBitsAvailable) {
-            return -1;
-        }
-        if (!ensureBits(1)) {
-            return -1;
-        }
-        return (int) (bitBuffer & 1);
+    public void reset(long address, long length) {
+        this.currentAddress = address;
+        this.endAddress = address + length;
+        this.bitBuffer = 0;
+        this.bitsInBuffer = 0;
+        this.totalBitsAvailable = length * 8L;
+        this.totalBitsRead = 0;
+    }
+
+    /**
+     * Resets the reader to read from the specified byte array.
+     *
+     * @param buf    the byte array
+     * @param offset the starting offset
+     * @param length the number of bytes available
+     */
+    public void reset(byte[] buf, int offset, int length) {
+        // For byte array, we'll store position info differently
+        // This is mainly for testing - in production we use direct memory
+        throw new UnsupportedOperationException("Use direct memory version");
     }
 
     /**
@@ -276,60 +315,18 @@ public void skipBits(int numBits) {
     }
 
     /**
-     * Aligns the reader to the next byte boundary by skipping any partial bits.
+     * Ensures the buffer has at least the requested number of bits.
+     * Loads more bytes from memory if needed.
      *
-     * @throws IllegalStateException if alignment fails
+     * @param bitsNeeded minimum bits required in buffer
+     * @return true if sufficient bits available, false otherwise
      */
-    public void alignToByte() {
-        int bitsToSkip = bitsInBuffer % 8;
-        if (bitsToSkip != 0) {
-            // We need to skip the remaining bits in the current partial byte
-            // But since we read in byte chunks, bitsInBuffer should be a multiple of 8
-            // minus what we've consumed. The remainder in the conceptual stream is:
-            int remainder = (int) (totalBitsRead % 8);
-            if (remainder != 0) {
-                skipBits(8 - remainder);
-            }
+    private boolean ensureBits(int bitsNeeded) {
+        while (bitsInBuffer < bitsNeeded && currentAddress < endAddress) {
+            byte b = Unsafe.getUnsafe().getByte(currentAddress++);
+            bitBuffer |= (long) (b & 0xFF) << bitsInBuffer;
+            bitsInBuffer += 8;
         }
-    }
-
-    /**
-     * Reads a complete byte, ensuring byte alignment first.
-     *
-     * @return the byte value (0-255)
-     * @throws IllegalStateException if not enough data
-     */
-    public int readByte() {
-        return (int) readBits(8) & 0xFF;
-    }
-
-    /**
-     * Reads a complete 32-bit integer in little-endian order.
-     *
-     * @return the integer value
-     * @throws IllegalStateException if not enough data
-     */
-    public int readInt() {
-        return (int) readBits(32);
-    }
-
-    /**
-     * Reads a complete 64-bit long in little-endian order.
-     *
-     * @return the long value
-     * @throws IllegalStateException if not enough data
-     */
-    public long readLong() {
-        return readBits(64);
-    }
-
-    /**
-     * Returns the current byte address being read.
-     * Note: This is approximate due to bit buffering.
-     *
-     * @return current address
-     */
-    public long getCurrentAddress() {
-        return currentAddress;
+        return bitsInBuffer >= bitsNeeded;
     }
 }

From b85d8903abcb82f26d84b558775ed67dd625e1e4 Mon Sep 17 00:00:00 2001
From: Marko Topolnik 
Date: Wed, 25 Feb 2026 09:28:59 +0100
Subject: [PATCH 39/89] Sort members alphabetically, remove section headings

Reorder class members (fields, methods) alphabetically by kind
and visibility across all java-questdb-client source and test
files. This follows the project convention of alphabetical
member ordering.

Remove decorative section-heading comments (// ===, // ---,
// ====================) that no longer serve a purpose after
alphabetical reordering. Incorporate the orphaned "Fast-path
API" comment block into the QwpWebSocketSender class Javadoc.

Co-Authored-By: Claude Opus 4.6 
---
 .../main/java/io/questdb/client/Sender.java   |  120 +-
 .../cutlass/http/client/WebSocketClient.java  |  899 +--
 .../http/client/WebSocketFrameHandler.java    |   24 +-
 .../http/client/WebSocketSendBuffer.java      |  492 +-
 .../client/cutlass/json/JsonLexer.java        |    3 +-
 .../cutlass/line/AbstractLineSender.java      |    4 +-
 .../qwp/client/GlobalSymbolDictionary.java    |   98 +-
 .../cutlass/qwp/client/InFlightWindow.java    |  337 +-
 .../cutlass/qwp/client/MicrobatchBuffer.java  |  409 +-
 .../qwp/client/NativeBufferWriter.java        |  192 +-
 .../cutlass/qwp/client/QwpBufferWriter.java   |  148 +-
 .../qwp/client/QwpWebSocketSender.java        |   67 +-
 .../cutlass/qwp/client/ResponseReader.java    |   68 +-
 .../cutlass/qwp/client/WebSocketChannel.java  |  460 +-
 .../cutlass/qwp/client/WebSocketResponse.java |  146 +-
 .../qwp/client/WebSocketSendQueue.java        |  364 +-
 .../qwp/protocol/OffHeapAppendMemory.java     |   85 +-
 .../cutlass/qwp/protocol/QwpBitWriter.java    |  149 +-
 .../cutlass/qwp/protocol/QwpColumnDef.java    |   91 +-
 .../cutlass/qwp/protocol/QwpConstants.java    |  378 +-
 .../qwp/protocol/QwpGorillaEncoder.java       |  156 +-
 .../cutlass/qwp/protocol/QwpNullBitmap.java   |  212 +-
 .../cutlass/qwp/protocol/QwpSchemaHash.java   |  506 +-
 .../cutlass/qwp/protocol/QwpTableBuffer.java  |   20 +-
 .../cutlass/qwp/protocol/QwpVarint.java       |  126 +-
 .../cutlass/qwp/protocol/QwpZigZag.java       |   24 +-
 .../qwp/websocket/WebSocketCloseCode.java     |  118 +-
 .../qwp/websocket/WebSocketFrameParser.java   |  175 +-
 .../qwp/websocket/WebSocketFrameWriter.java   |  260 +-
 .../qwp/websocket/WebSocketHandshake.java     |  350 +-
 .../qwp/websocket/WebSocketOpcode.java        |   27 +-
 .../java/io/questdb/client/network/Net.java   |   26 +-
 .../client/std/CharSequenceIntHashMap.java    |   46 +-
 .../client/std/bytes/DirectByteSink.java      |    3 +-
 .../line/tcp/v4/QwpAllocationTestClient.java  |  148 +-
 .../line/tcp/v4/StacBenchmarkClient.java      |  201 +-
 .../qwp/client/InFlightWindowTest.java        |  872 +-
 .../LineSenderBuilderWebSocketTest.java       |   94 +-
 .../qwp/client/MicrobatchBufferTest.java      |    3 +-
 .../qwp/client/NativeBufferWriterTest.java    |   18 +-
 .../cutlass/qwp/client/QwpSenderTest.java     | 7112 ++++++++---------
 .../qwp/client/WebSocketChannelTest.java      |  197 +-
 .../qwp/protocol/OffHeapAppendMemoryTest.java |  303 +-
 .../qwp/protocol/QwpBitWriterTest.java        |  136 +-
 .../qwp/protocol/QwpColumnDefTest.java        |    3 +-
 .../qwp/protocol/QwpTableBufferTest.java      |  159 +-
 46 files changed, 7771 insertions(+), 8058 deletions(-)

diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java
index 3bd08d2..c998b78 100644
--- a/core/src/main/java/io/questdb/client/Sender.java
+++ b/core/src/main/java/io/questdb/client/Sender.java
@@ -531,12 +531,18 @@ final class LineSenderBuilder {
         private static final int DEFAULT_BUFFER_CAPACITY = 64 * 1024;
         private static final int DEFAULT_HTTP_PORT = 9000;
         private static final int DEFAULT_HTTP_TIMEOUT = 30_000;
+        private static final int DEFAULT_IN_FLIGHT_WINDOW_SIZE = 8;
         private static final int DEFAULT_MAXIMUM_BUFFER_CAPACITY = 100 * 1024 * 1024;
         private static final int DEFAULT_MAX_BACKOFF_MILLIS = 1_000;
         private static final int DEFAULT_MAX_NAME_LEN = 127;
         private static final long DEFAULT_MAX_RETRY_NANOS = TimeUnit.SECONDS.toNanos(10); // keep sync with the contract of the configuration method
         private static final long DEFAULT_MIN_REQUEST_THROUGHPUT = 100 * 1024; // 100KB/s, keep in sync with the contract of the configuration method
+        private static final int DEFAULT_SEND_QUEUE_CAPACITY = 16;
         private static final int DEFAULT_TCP_PORT = 9009;
+        private static final int DEFAULT_WEBSOCKET_PORT = 9000;
+        private static final int DEFAULT_WS_AUTO_FLUSH_BYTES = 1024 * 1024; // 1MB
+        private static final long DEFAULT_WS_AUTO_FLUSH_INTERVAL_NANOS = 100_000_000L; // 100ms
+        private static final int DEFAULT_WS_AUTO_FLUSH_ROWS = 500;
         private static final int MIN_BUFFER_SIZE = AuthUtils.CHALLENGE_LEN + 1; // challenge size + 1;
         // The PARAMETER_NOT_SET_EXPLICITLY constant is used to detect if a parameter was set explicitly in configuration parameters
         // where it matters. This is needed to detect invalid combinations of parameters. Why?
@@ -546,14 +552,10 @@ final class LineSenderBuilder {
         private static final int PROTOCOL_HTTP = 1;
         private static final int PROTOCOL_TCP = 0;
         private static final int PROTOCOL_WEBSOCKET = 2;
-        private static final int DEFAULT_WEBSOCKET_PORT = 9000;
-        private static final int DEFAULT_IN_FLIGHT_WINDOW_SIZE = 8;
-        private static final int DEFAULT_SEND_QUEUE_CAPACITY = 16;
-        private static final int DEFAULT_WS_AUTO_FLUSH_ROWS = 500;
-        private static final int DEFAULT_WS_AUTO_FLUSH_BYTES = 1024 * 1024; // 1MB
-        private static final long DEFAULT_WS_AUTO_FLUSH_INTERVAL_NANOS = 100_000_000L; // 100ms
         private final ObjList hosts = new ObjList<>();
         private final IntList ports = new IntList();
+        private boolean asyncMode = false;
+        private int autoFlushBytes = PARAMETER_NOT_SET_EXPLICITLY;
         private int autoFlushIntervalMillis = PARAMETER_NOT_SET_EXPLICITLY;
         private int autoFlushRows = PARAMETER_NOT_SET_EXPLICITLY;
         private int bufferCapacity = PARAMETER_NOT_SET_EXPLICITLY;
@@ -561,6 +563,8 @@ final class LineSenderBuilder {
         private String httpSettingsPath;
         private int httpTimeout = PARAMETER_NOT_SET_EXPLICITLY;
         private String httpToken;
+        // WebSocket-specific fields
+        private int inFlightWindowSize = PARAMETER_NOT_SET_EXPLICITLY;
         private String keyId;
         private int maxBackoffMillis = PARAMETER_NOT_SET_EXPLICITLY;
         private int maxNameLength = PARAMETER_NOT_SET_EXPLICITLY;
@@ -592,17 +596,13 @@ public int getTimeout() {
         private int protocol = PARAMETER_NOT_SET_EXPLICITLY;
         private int protocolVersion = PARAMETER_NOT_SET_EXPLICITLY;
         private int retryTimeoutMillis = PARAMETER_NOT_SET_EXPLICITLY;
+        private int sendQueueCapacity = PARAMETER_NOT_SET_EXPLICITLY;
         private boolean shouldDestroyPrivKey;
         private boolean tlsEnabled;
         private TlsValidationMode tlsValidationMode;
         private char[] trustStorePassword;
         private String trustStorePath;
         private String username;
-        // WebSocket-specific fields
-        private int inFlightWindowSize = PARAMETER_NOT_SET_EXPLICITLY;
-        private int sendQueueCapacity = PARAMETER_NOT_SET_EXPLICITLY;
-        private boolean asyncMode = false;
-        private int autoFlushBytes = PARAMETER_NOT_SET_EXPLICITLY;
 
         private LineSenderBuilder() {
 
@@ -693,6 +693,47 @@ public AdvancedTlsSettings advancedTls() {
             return new AdvancedTlsSettings();
         }
 
+        /**
+         * Enable asynchronous mode for WebSocket transport.
+         * 

+         * In async mode, rows are batched and sent asynchronously with flow control.
+         * This provides higher throughput at the cost of more complex error handling.
+         * 

+         * This is only used when communicating over WebSocket transport.
+         * 

+         * Default is synchronous mode (false).
+         *
+         * @param enabled whether to enable async mode
+         * @return this instance for method chaining
+         */
+        public LineSenderBuilder asyncMode(boolean enabled) {
+            this.asyncMode = enabled;
+            return this;
+        }
+
+        /**
+         * Set the maximum number of bytes per batch before auto-flushing.
+         * 

+         * This is only used when communicating over WebSocket transport.
+         * 

+         * Default value is 1MB.
+         *
+         * @param bytes maximum bytes per batch
+         * @return this instance for method chaining
+         */
+        public LineSenderBuilder autoFlushBytes(int bytes) {
+            if (this.autoFlushBytes != PARAMETER_NOT_SET_EXPLICITLY) {
+                throw new LineSenderException("auto flush bytes was already configured")
+                        .put("[bytes=").put(this.autoFlushBytes).put("]");
+            }
+            if (bytes < 0) {
+                throw new LineSenderException("auto flush bytes cannot be negative")
+                        .put("[bytes=").put(bytes).put("]");
+            }
+            this.autoFlushBytes = bytes;
+            return this;
+        }
+
         /**
          * Set the interval in milliseconds at which the Sender automatically flushes its buffer.
          * 

@@ -768,47 +809,6 @@ public LineSenderBuilder autoFlushRows(int autoFlushRows) {
             return this;
         }
 
-        /**
-         * Set the maximum number of bytes per batch before auto-flushing.
-         * 

-         * This is only used when communicating over WebSocket transport.
-         * 

-         * Default value is 1MB.
-         *
-         * @param bytes maximum bytes per batch
-         * @return this instance for method chaining
-         */
-        public LineSenderBuilder autoFlushBytes(int bytes) {
-            if (this.autoFlushBytes != PARAMETER_NOT_SET_EXPLICITLY) {
-                throw new LineSenderException("auto flush bytes was already configured")
-                        .put("[bytes=").put(this.autoFlushBytes).put("]");
-            }
-            if (bytes < 0) {
-                throw new LineSenderException("auto flush bytes cannot be negative")
-                        .put("[bytes=").put(bytes).put("]");
-            }
-            this.autoFlushBytes = bytes;
-            return this;
-        }
-
-        /**
-         * Enable asynchronous mode for WebSocket transport.
-         * 

-         * In async mode, rows are batched and sent asynchronously with flow control.
-         * This provides higher throughput at the cost of more complex error handling.
-         * 

-         * This is only used when communicating over WebSocket transport.
-         * 

-         * Default is synchronous mode (false).
-         *
-         * @param enabled whether to enable async mode
-         * @return this instance for method chaining
-         */
-        public LineSenderBuilder asyncMode(boolean enabled) {
-            this.asyncMode = enabled;
-            return this;
-        }
-
         /**
          * Configure capacity of an internal buffer.
          * 

@@ -1729,14 +1729,6 @@ private void tcp() {
             protocol = PROTOCOL_TCP;
         }
 
-        private void websocket() {
-            if (protocol != PARAMETER_NOT_SET_EXPLICITLY) {
-                throw new LineSenderException("protocol was already configured ")
-                        .put("[protocol=").put(protocol).put("]");
-            }
-            protocol = PROTOCOL_WEBSOCKET;
-        }
-
         private void validateParameters() {
             if (hosts.size() == 0) {
                 throw new LineSenderException("questdb server address not set");
@@ -1817,6 +1809,14 @@ private void validateParameters() {
             }
         }
 
+        private void websocket() {
+            if (protocol != PARAMETER_NOT_SET_EXPLICITLY) {
+                throw new LineSenderException("protocol was already configured ")
+                        .put("[protocol=").put(protocol).put("]");
+            }
+            protocol = PROTOCOL_WEBSOCKET;
+        }
+
         public class AdvancedTlsSettings {
             /**
              * Configure a custom truststore. This is only needed when using {@link #enableTls()} when your default
diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java
index fec618f..a0cf1a3 100644
--- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java
+++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java
@@ -67,41 +67,34 @@
  */
 public abstract class WebSocketClient implements QuietCloseable {
 
-    private static final Logger LOG = LoggerFactory.getLogger(WebSocketClient.class);
-
     private static final int DEFAULT_RECV_BUFFER_SIZE = 65536;
     private static final int DEFAULT_SEND_BUFFER_SIZE = 65536;
-
+    private static final Logger LOG = LoggerFactory.getLogger(WebSocketClient.class);
     protected final NetworkFacade nf;
     protected final Socket socket;
-
-    private final WebSocketSendBuffer sendBuffer;
     private final WebSocketSendBuffer controlFrameBuffer;
-    private final WebSocketFrameParser frameParser;
-    private final Rnd rnd;
     private final int defaultTimeout;
+    private final WebSocketFrameParser frameParser;
     private final int maxRecvBufSize;
-
+    private final Rnd rnd;
+    private final WebSocketSendBuffer sendBuffer;
+    private boolean closed;
+    private int fragmentBufPos;
+    private long fragmentBufPtr;       // native buffer for accumulating fragment payloads
+    private int fragmentBufSize;
+    // Fragmentation state (RFC 6455 Section 5.4)
+    private int fragmentOpcode = -1;   // opcode of first fragment, -1 = not in a fragmented message
+    // Handshake key for verification
+    private String handshakeKey;
+    // Connection state
+    private CharSequence host;
+    private int port;
     // Receive buffer (native memory)
     private long recvBufPtr;
     private int recvBufSize;
     private int recvPos;      // Write position
     private int recvReadPos;  // Read position
-
-    // Connection state
-    private CharSequence host;
-    private int port;
     private boolean upgraded;
-    private boolean closed;
-
-    // Fragmentation state (RFC 6455 Section 5.4)
-    private int fragmentOpcode = -1;   // opcode of first fragment, -1 = not in a fragmented message
-    private long fragmentBufPtr;       // native buffer for accumulating fragment payloads
-    private int fragmentBufSize;
-    private int fragmentBufPos;
-
-    // Handshake key for verification
-    private String handshakeKey;
 
     public WebSocketClient(HttpClientConfiguration configuration, SocketFactory socketFactory) {
         this.nf = configuration.getNetworkFacade();
@@ -158,20 +151,6 @@ public void close() {
         }
     }
 
-    /**
-     * Disconnects the socket without closing the client.
-     * The client can be reconnected by calling connect() again.
-     */
-    public void disconnect() {
-        Misc.free(socket);
-        upgraded = false;
-        host = null;
-        port = 0;
-        recvPos = 0;
-        recvReadPos = 0;
-        resetFragmentState();
-    }
-
     /**
      * Connects to a WebSocket server.
      *
@@ -204,206 +183,34 @@ public void connect(CharSequence host, int port) {
         connect(host, port, defaultTimeout);
     }
 
-    private void doConnect(CharSequence host, int port, int timeout) {
-        int fd = nf.socketTcp(true);
-        if (fd < 0) {
-            throw new HttpClientException("could not allocate a file descriptor [errno=").errno(nf.errno()).put(']');
-        }
-
-        if (nf.setTcpNoDelay(fd, true) < 0) {
-            LOG.info("could not disable Nagle's algorithm [fd={}, errno={}]", fd, nf.errno());
-        }
-
-        socket.of(fd);
-        nf.configureKeepAlive(fd);
-
-        long addrInfo = nf.getAddrInfo(host, port);
-        if (addrInfo == -1) {
-            disconnect();
-            throw new HttpClientException("could not resolve host [host=").put(host).put(']');
-        }
-
-        if (nf.connectAddrInfo(fd, addrInfo) != 0) {
-            int errno = nf.errno();
-            nf.freeAddrInfo(addrInfo);
-            disconnect();
-            throw new HttpClientException("could not connect [host=").put(host)
-                    .put(", port=").put(port)
-                    .put(", errno=").put(errno).put(']');
-        }
-        nf.freeAddrInfo(addrInfo);
-
-        if (nf.configureNonBlocking(fd) < 0) {
-            int errno = nf.errno();
-            disconnect();
-            throw new HttpClientException("could not configure non-blocking [fd=").put(fd)
-                    .put(", errno=").put(errno).put(']');
-        }
-
-        if (socket.supportsTls()) {
-            try {
-                socket.startTlsSession(host);
-            } catch (TlsSessionInitFailedException e) {
-                int errno = nf.errno();
-                disconnect();
-                throw new HttpClientException("could not start TLS session [fd=").put(fd)
-                        .put(", error=").put(e.getFlyweightMessage())
-                        .put(", errno=").put(errno).put(']');
-            }
-        }
-
-        setupIoWait();
-        LOG.debug("Connected to [host={}, port={}]", host, port);
-    }
-
     /**
-     * Performs WebSocket upgrade handshake.
-     *
-     * @param path    the WebSocket endpoint path (e.g., "/ws")
-     * @param timeout timeout in milliseconds
+     * Disconnects the socket without closing the client.
+     * The client can be reconnected by calling connect() again.
      */
-    public void upgrade(CharSequence path, int timeout) {
-        if (closed) {
-            throw new HttpClientException("WebSocket client is closed");
-        }
-        if (socket.isClosed()) {
-            throw new HttpClientException("Not connected");
-        }
-        if (upgraded) {
-            return; // Already upgraded
-        }
-
-        // Generate random key
-        byte[] keyBytes = new byte[16];
-        for (int i = 0; i < 16; i++) {
-            keyBytes[i] = (byte) rnd.nextInt(256);
-        }
-        handshakeKey = Base64.getEncoder().encodeToString(keyBytes);
-
-        // Build upgrade request
-        sendBuffer.reset();
-        sendBuffer.putAscii("GET ");
-        sendBuffer.putAscii(path);
-        sendBuffer.putAscii(" HTTP/1.1\r\n");
-        sendBuffer.putAscii("Host: ");
-        sendBuffer.putAscii(host);
-        if ((socket.supportsTls() && port != 443) || (!socket.supportsTls() && port != 80)) {
-            sendBuffer.putAscii(":");
-            sendBuffer.putAscii(Integer.toString(port));
-        }
-        sendBuffer.putAscii("\r\n");
-        sendBuffer.putAscii("Upgrade: websocket\r\n");
-        sendBuffer.putAscii("Connection: Upgrade\r\n");
-        sendBuffer.putAscii("Sec-WebSocket-Key: ");
-        sendBuffer.putAscii(handshakeKey);
-        sendBuffer.putAscii("\r\n");
-        sendBuffer.putAscii("Sec-WebSocket-Version: 13\r\n");
-        sendBuffer.putAscii("\r\n");
-
-        // Send request
-        long startTime = System.nanoTime();
-        doSend(sendBuffer.getBufferPtr(), sendBuffer.getWritePos(), timeout);
-
-        // Read response
-        int remainingTimeout = remainingTime(timeout, startTime);
-        readUpgradeResponse(remainingTimeout);
-
-        upgraded = true;
-        sendBuffer.reset();
-        LOG.debug("WebSocket upgraded [path={}]", path);
+    public void disconnect() {
+        Misc.free(socket);
+        upgraded = false;
+        host = null;
+        port = 0;
+        recvPos = 0;
+        recvReadPos = 0;
+        resetFragmentState();
     }
 
     /**
-     * Performs upgrade with default timeout.
+     * Returns the connected host.
      */
-    public void upgrade(CharSequence path) {
-        upgrade(path, defaultTimeout);
-    }
-
-    private void readUpgradeResponse(int timeout) {
-        // Read HTTP response into receive buffer
-        long startTime = System.nanoTime();
-
-        while (true) {
-            int remainingTimeout = remainingTime(timeout, startTime);
-            int bytesRead = recvOrDie(recvBufPtr + recvPos, recvBufSize - recvPos, remainingTimeout);
-            if (bytesRead > 0) {
-                recvPos += bytesRead;
-            }
-
-            // Check for end of headers (\r\n\r\n)
-            int headerEnd = findHeaderEnd();
-            if (headerEnd > 0) {
-                validateUpgradeResponse(headerEnd);
-                // Compact buffer - move remaining data to start
-                int remaining = recvPos - headerEnd;
-                if (remaining > 0) {
-                    Vect.memmove(recvBufPtr, recvBufPtr + headerEnd, remaining);
-                }
-                recvPos = remaining;
-                recvReadPos = 0;
-                return;
-            }
-
-            if (recvPos >= recvBufSize) {
-                throw new HttpClientException("HTTP response too large");
-            }
-        }
-    }
-
-    private int findHeaderEnd() {
-        // Look for \r\n\r\n
-        for (int i = 0; i < recvPos - 3; i++) {
-            if (Unsafe.getUnsafe().getByte(recvBufPtr + i) == '\r' &&
-                Unsafe.getUnsafe().getByte(recvBufPtr + i + 1) == '\n' &&
-                Unsafe.getUnsafe().getByte(recvBufPtr + i + 2) == '\r' &&
-                Unsafe.getUnsafe().getByte(recvBufPtr + i + 3) == '\n') {
-                return i + 4;
-            }
-        }
-        return -1;
-    }
-
-    private void validateUpgradeResponse(int headerEnd) {
-        // Extract response as string for parsing
-        byte[] responseBytes = new byte[headerEnd];
-        for (int i = 0; i < headerEnd; i++) {
-            responseBytes[i] = Unsafe.getUnsafe().getByte(recvBufPtr + i);
-        }
-        String response = new String(responseBytes, StandardCharsets.US_ASCII);
-
-        // Check status line
-        if (!response.startsWith("HTTP/1.1 101")) {
-            String statusLine = response.split("\r\n")[0];
-            throw new HttpClientException("WebSocket upgrade failed: ").put(statusLine);
-        }
-
-        // Verify Sec-WebSocket-Accept (case-insensitive per RFC 7230)
-        String expectedAccept = WebSocketHandshake.computeAcceptKey(handshakeKey);
-        if (!containsHeaderValue(response, "Sec-WebSocket-Accept:", expectedAccept)) {
-            throw new HttpClientException("Invalid Sec-WebSocket-Accept header");
-        }
+    public CharSequence getHost() {
+        return host;
     }
 
-    private static boolean containsHeaderValue(String response, String headerName, String expectedValue) {
-        int headerLen = headerName.length();
-        int responseLen = response.length();
-        for (int i = 0; i <= responseLen - headerLen; i++) {
-            if (response.regionMatches(true, i, headerName, 0, headerLen)) {
-                int valueStart = i + headerLen;
-                int lineEnd = response.indexOf('\r', valueStart);
-                if (lineEnd < 0) {
-                    lineEnd = responseLen;
-                }
-                String actualValue = response.substring(valueStart, lineEnd).trim();
-                return actualValue.equals(expectedValue);
-            }
-        }
-        return false;
+    /**
+     * Returns the connected port.
+     */
+    public int getPort() {
+        return port;
     }
 
-    // === Sending ===
-
     /**
      * Gets the send buffer for building WebSocket frames.
      * 

@@ -422,21 +229,59 @@ public WebSocketSendBuffer getSendBuffer() {
     }
 
     /**
-     * Sends a complete WebSocket frame.
+     * Returns whether the WebSocket is connected and upgraded.
+     */
+    public boolean isConnected() {
+        return upgraded && !closed && !socket.isClosed();
+    }
+
+    /**
+     * Receives and processes WebSocket frames.
      *
-     * @param frame   frame info from endBinaryFrame()
+     * @param handler frame handler callback
      * @param timeout timeout in milliseconds
+     * @return true if a frame was received, false on timeout
      */
-    public void sendFrame(WebSocketSendBuffer.FrameInfo frame, int timeout) {
+    public boolean receiveFrame(WebSocketFrameHandler handler, int timeout) {
         checkConnected();
-        doSend(sendBuffer.getBufferPtr() + frame.offset, frame.length, timeout);
+
+        // First, try to parse any data already in buffer
+        Boolean result = tryParseFrame(handler);
+        if (result != null) {
+            return result;
+        }
+
+        // Need more data
+        long startTime = System.nanoTime();
+        while (true) {
+            int remainingTimeout = remainingTime(timeout, startTime);
+            if (remainingTimeout <= 0) {
+                return false; // Timeout
+            }
+
+            // Ensure buffer has space
+            if (recvPos >= recvBufSize - 1024) {
+                growRecvBuffer();
+            }
+
+            int bytesRead = recvOrTimeout(recvBufPtr + recvPos, recvBufSize - recvPos, remainingTimeout);
+            if (bytesRead <= 0) {
+                return false; // Timeout
+            }
+            recvPos += bytesRead;
+
+            result = tryParseFrame(handler);
+            if (result != null) {
+                return result;
+            }
+        }
     }
 
     /**
-     * Sends a complete WebSocket frame with default timeout.
+     * Receives frame with default timeout.
      */
-    public void sendFrame(WebSocketSendBuffer.FrameInfo frame) {
-        sendFrame(frame, defaultTimeout);
+    public boolean receiveFrame(WebSocketFrameHandler handler) {
+        return receiveFrame(handler, defaultTimeout);
     }
 
     /**
@@ -463,6 +308,37 @@ public void sendBinary(long dataPtr, int length) {
         sendBinary(dataPtr, length, defaultTimeout);
     }
 
+    /**
+     * Sends a close frame.
+     */
+    public void sendCloseFrame(int code, String reason, int timeout) {
+        sendBuffer.reset();
+        WebSocketSendBuffer.FrameInfo frame = sendBuffer.writeCloseFrame(code, reason);
+        try {
+            doSend(sendBuffer.getBufferPtr() + frame.offset, frame.length, timeout);
+        } finally {
+            sendBuffer.reset();
+        }
+    }
+
+    /**
+     * Sends a complete WebSocket frame.
+     *
+     * @param frame   frame info from endBinaryFrame()
+     * @param timeout timeout in milliseconds
+     */
+    public void sendFrame(WebSocketSendBuffer.FrameInfo frame, int timeout) {
+        checkConnected();
+        doSend(sendBuffer.getBufferPtr() + frame.offset, frame.length, timeout);
+    }
+
+    /**
+     * Sends a complete WebSocket frame with default timeout.
+     */
+    public void sendFrame(WebSocketSendBuffer.FrameInfo frame) {
+        sendFrame(frame, defaultTimeout);
+    }
+
     /**
      * Sends a ping frame.
      */
@@ -475,16 +351,222 @@ public void sendPing(int timeout) {
     }
 
     /**
-     * Sends a close frame.
+     * Non-blocking attempt to receive a WebSocket frame.
+     * Returns immediately if no complete frame is available.
+     *
+     * @param handler frame handler callback
+     * @return true if a frame was received, false if no data available
      */
-    public void sendCloseFrame(int code, String reason, int timeout) {
+    public boolean tryReceiveFrame(WebSocketFrameHandler handler) {
+        checkConnected();
+
+        // First, try to parse any data already in buffer
+        Boolean result = tryParseFrame(handler);
+        if (result != null) {
+            return result;
+        }
+
+        // Try one non-blocking recv
+        if (recvPos >= recvBufSize - 1024) {
+            growRecvBuffer();
+        }
+
+        int n = socket.recv(recvBufPtr + recvPos, recvBufSize - recvPos);
+        if (n < 0) {
+            throw new HttpClientException("peer disconnect [errno=").errno(nf.errno()).put(']');
+        }
+        if (n == 0) {
+            return false; // No data available
+        }
+        recvPos += n;
+
+        // Try to parse again
+        result = tryParseFrame(handler);
+        return result != null && result;
+    }
+
+    /**
+     * Performs WebSocket upgrade handshake.
+     *
+     * @param path    the WebSocket endpoint path (e.g., "/ws")
+     * @param timeout timeout in milliseconds
+     */
+    public void upgrade(CharSequence path, int timeout) {
+        if (closed) {
+            throw new HttpClientException("WebSocket client is closed");
+        }
+        if (socket.isClosed()) {
+            throw new HttpClientException("Not connected");
+        }
+        if (upgraded) {
+            return; // Already upgraded
+        }
+
+        // Generate random key
+        byte[] keyBytes = new byte[16];
+        for (int i = 0; i < 16; i++) {
+            keyBytes[i] = (byte) rnd.nextInt(256);
+        }
+        handshakeKey = Base64.getEncoder().encodeToString(keyBytes);
+
+        // Build upgrade request
         sendBuffer.reset();
-        WebSocketSendBuffer.FrameInfo frame = sendBuffer.writeCloseFrame(code, reason);
-        try {
-            doSend(sendBuffer.getBufferPtr() + frame.offset, frame.length, timeout);
-        } finally {
-            sendBuffer.reset();
+        sendBuffer.putAscii("GET ");
+        sendBuffer.putAscii(path);
+        sendBuffer.putAscii(" HTTP/1.1\r\n");
+        sendBuffer.putAscii("Host: ");
+        sendBuffer.putAscii(host);
+        if ((socket.supportsTls() && port != 443) || (!socket.supportsTls() && port != 80)) {
+            sendBuffer.putAscii(":");
+            sendBuffer.putAscii(Integer.toString(port));
+        }
+        sendBuffer.putAscii("\r\n");
+        sendBuffer.putAscii("Upgrade: websocket\r\n");
+        sendBuffer.putAscii("Connection: Upgrade\r\n");
+        sendBuffer.putAscii("Sec-WebSocket-Key: ");
+        sendBuffer.putAscii(handshakeKey);
+        sendBuffer.putAscii("\r\n");
+        sendBuffer.putAscii("Sec-WebSocket-Version: 13\r\n");
+        sendBuffer.putAscii("\r\n");
+
+        // Send request
+        long startTime = System.nanoTime();
+        doSend(sendBuffer.getBufferPtr(), sendBuffer.getWritePos(), timeout);
+
+        // Read response
+        int remainingTimeout = remainingTime(timeout, startTime);
+        readUpgradeResponse(remainingTimeout);
+
+        upgraded = true;
+        sendBuffer.reset();
+        LOG.debug("WebSocket upgraded [path={}]", path);
+    }
+
+    /**
+     * Performs upgrade with default timeout.
+     */
+    public void upgrade(CharSequence path) {
+        upgrade(path, defaultTimeout);
+    }
+
+    private static boolean containsHeaderValue(String response, String headerName, String expectedValue) {
+        int headerLen = headerName.length();
+        int responseLen = response.length();
+        for (int i = 0; i <= responseLen - headerLen; i++) {
+            if (response.regionMatches(true, i, headerName, 0, headerLen)) {
+                int valueStart = i + headerLen;
+                int lineEnd = response.indexOf('\r', valueStart);
+                if (lineEnd < 0) {
+                    lineEnd = responseLen;
+                }
+                String actualValue = response.substring(valueStart, lineEnd).trim();
+                return actualValue.equals(expectedValue);
+            }
+        }
+        return false;
+    }
+
+    private void appendToFragmentBuffer(long payloadPtr, int payloadLen) {
+        if (payloadLen == 0) {
+            return;
+        }
+        int required = fragmentBufPos + payloadLen;
+        if (required > maxRecvBufSize) {
+            throw new HttpClientException("WebSocket fragment buffer size exceeded maximum [required=")
+                    .put(required)
+                    .put(", max=")
+                    .put(maxRecvBufSize)
+                    .put(']');
+        }
+        if (fragmentBufPtr == 0) {
+            fragmentBufSize = Math.max(required, DEFAULT_RECV_BUFFER_SIZE);
+            fragmentBufPtr = Unsafe.malloc(fragmentBufSize, MemoryTag.NATIVE_DEFAULT);
+        } else if (required > fragmentBufSize) {
+            int newSize = Math.min(Math.max(fragmentBufSize * 2, required), maxRecvBufSize);
+            fragmentBufPtr = Unsafe.realloc(fragmentBufPtr, fragmentBufSize, newSize, MemoryTag.NATIVE_DEFAULT);
+            fragmentBufSize = newSize;
+        }
+        Vect.memmove(fragmentBufPtr + fragmentBufPos, payloadPtr, payloadLen);
+        fragmentBufPos += payloadLen;
+    }
+
+    private void checkConnected() {
+        if (closed) {
+            throw new HttpClientException("WebSocket client is closed");
+        }
+        if (!upgraded) {
+            throw new HttpClientException("WebSocket not connected or upgraded");
+        }
+    }
+
+    private void compactRecvBuffer() {
+        if (recvReadPos > 0) {
+            int remaining = recvPos - recvReadPos;
+            if (remaining > 0) {
+                Vect.memmove(recvBufPtr, recvBufPtr + recvReadPos, remaining);
+            }
+            recvPos = remaining;
+            recvReadPos = 0;
+        }
+    }
+
+    private int dieIfNegative(int byteCount) {
+        if (byteCount < 0) {
+            throw new HttpClientException("peer disconnect [errno=").errno(nf.errno()).put(']');
         }
+        return byteCount;
+    }
+
+    private void doConnect(CharSequence host, int port, int timeout) {
+        int fd = nf.socketTcp(true);
+        if (fd < 0) {
+            throw new HttpClientException("could not allocate a file descriptor [errno=").errno(nf.errno()).put(']');
+        }
+
+        if (nf.setTcpNoDelay(fd, true) < 0) {
+            LOG.info("could not disable Nagle's algorithm [fd={}, errno={}]", fd, nf.errno());
+        }
+
+        socket.of(fd);
+        nf.configureKeepAlive(fd);
+
+        long addrInfo = nf.getAddrInfo(host, port);
+        if (addrInfo == -1) {
+            disconnect();
+            throw new HttpClientException("could not resolve host [host=").put(host).put(']');
+        }
+
+        if (nf.connectAddrInfo(fd, addrInfo) != 0) {
+            int errno = nf.errno();
+            nf.freeAddrInfo(addrInfo);
+            disconnect();
+            throw new HttpClientException("could not connect [host=").put(host)
+                    .put(", port=").put(port)
+                    .put(", errno=").put(errno).put(']');
+        }
+        nf.freeAddrInfo(addrInfo);
+
+        if (nf.configureNonBlocking(fd) < 0) {
+            int errno = nf.errno();
+            disconnect();
+            throw new HttpClientException("could not configure non-blocking [fd=").put(fd)
+                    .put(", errno=").put(errno).put(']');
+        }
+
+        if (socket.supportsTls()) {
+            try {
+                socket.startTlsSession(host);
+            } catch (TlsSessionInitFailedException e) {
+                int errno = nf.errno();
+                disconnect();
+                throw new HttpClientException("could not start TLS session [fd=").put(fd)
+                        .put(", error=").put(e.getFlyweightMessage())
+                        .put(", errno=").put(errno).put(']');
+            }
+        }
+
+        setupIoWait();
+        LOG.debug("Connected to [host={}, port={}]", host, port);
     }
 
     private void doSend(long ptr, int len, int timeout) {
@@ -505,90 +587,130 @@ private void doSend(long ptr, int len, int timeout) {
         }
     }
 
-    // === Receiving ===
-
-    /**
-     * Receives and processes WebSocket frames.
-     *
-     * @param handler frame handler callback
-     * @param timeout timeout in milliseconds
-     * @return true if a frame was received, false on timeout
-     */
-    public boolean receiveFrame(WebSocketFrameHandler handler, int timeout) {
-        checkConnected();
+    private int findHeaderEnd() {
+        // Look for \r\n\r\n
+        for (int i = 0; i < recvPos - 3; i++) {
+            if (Unsafe.getUnsafe().getByte(recvBufPtr + i) == '\r' &&
+                    Unsafe.getUnsafe().getByte(recvBufPtr + i + 1) == '\n' &&
+                    Unsafe.getUnsafe().getByte(recvBufPtr + i + 2) == '\r' &&
+                    Unsafe.getUnsafe().getByte(recvBufPtr + i + 3) == '\n') {
+                return i + 4;
+            }
+        }
+        return -1;
+    }
 
-        // First, try to parse any data already in buffer
-        Boolean result = tryParseFrame(handler);
-        if (result != null) {
-            return result;
+    private void growRecvBuffer() {
+        int newSize = recvBufSize * 2;
+        if (newSize > maxRecvBufSize) {
+            if (recvBufSize >= maxRecvBufSize) {
+                throw new HttpClientException("WebSocket receive buffer size exceeded maximum [current=")
+                        .put(recvBufSize)
+                        .put(", max=")
+                        .put(maxRecvBufSize)
+                        .put(']');
+            }
+            newSize = maxRecvBufSize;
         }
+        recvBufPtr = Unsafe.realloc(recvBufPtr, recvBufSize, newSize, MemoryTag.NATIVE_DEFAULT);
+        recvBufSize = newSize;
+    }
 
-        // Need more data
+    private void readUpgradeResponse(int timeout) {
+        // Read HTTP response into receive buffer
         long startTime = System.nanoTime();
+
         while (true) {
             int remainingTimeout = remainingTime(timeout, startTime);
-            if (remainingTimeout <= 0) {
-                return false; // Timeout
-            }
-
-            // Ensure buffer has space
-            if (recvPos >= recvBufSize - 1024) {
-                growRecvBuffer();
-            }
-
-            int bytesRead = recvOrTimeout(recvBufPtr + recvPos, recvBufSize - recvPos, remainingTimeout);
-            if (bytesRead <= 0) {
-                return false; // Timeout
+            int bytesRead = recvOrDie(recvBufPtr + recvPos, recvBufSize - recvPos, remainingTimeout);
+            if (bytesRead > 0) {
+                recvPos += bytesRead;
             }
-            recvPos += bytesRead;
 
-            result = tryParseFrame(handler);
-            if (result != null) {
-                return result;
+            // Check for end of headers (\r\n\r\n)
+            int headerEnd = findHeaderEnd();
+            if (headerEnd > 0) {
+                validateUpgradeResponse(headerEnd);
+                // Compact buffer - move remaining data to start
+                int remaining = recvPos - headerEnd;
+                if (remaining > 0) {
+                    Vect.memmove(recvBufPtr, recvBufPtr + headerEnd, remaining);
+                }
+                recvPos = remaining;
+                recvReadPos = 0;
+                return;
             }
-        }
-    }
-
-    /**
-     * Receives frame with default timeout.
-     */
-    public boolean receiveFrame(WebSocketFrameHandler handler) {
-        return receiveFrame(handler, defaultTimeout);
-    }
-
-    /**
-     * Non-blocking attempt to receive a WebSocket frame.
-     * Returns immediately if no complete frame is available.
-     *
-     * @param handler frame handler callback
-     * @return true if a frame was received, false if no data available
-     */
-    public boolean tryReceiveFrame(WebSocketFrameHandler handler) {
-        checkConnected();
 
-        // First, try to parse any data already in buffer
-        Boolean result = tryParseFrame(handler);
-        if (result != null) {
-            return result;
+            if (recvPos >= recvBufSize) {
+                throw new HttpClientException("HTTP response too large");
+            }
         }
+    }
 
-        // Try one non-blocking recv
-        if (recvPos >= recvBufSize - 1024) {
-            growRecvBuffer();
+    private int recvOrDie(long ptr, int len, int timeout) {
+        long startTime = System.nanoTime();
+        int n = dieIfNegative(socket.recv(ptr, len));
+        if (n == 0) {
+            ioWait(remainingTime(timeout, startTime), IOOperation.READ);
+            n = dieIfNegative(socket.recv(ptr, len));
         }
+        return n;
+    }
 
-        int n = socket.recv(recvBufPtr + recvPos, recvBufSize - recvPos);
+    private int recvOrTimeout(long ptr, int len, int timeout) {
+        long startTime = System.nanoTime();
+        int n = socket.recv(ptr, len);
         if (n < 0) {
             throw new HttpClientException("peer disconnect [errno=").errno(nf.errno()).put(']');
         }
         if (n == 0) {
-            return false; // No data available
+            try {
+                ioWait(timeout, IOOperation.READ);
+            } catch (HttpClientException e) {
+                // Timeout
+                return 0;
+            }
+            n = socket.recv(ptr, len);
+            if (n < 0) {
+                throw new HttpClientException("peer disconnect [errno=").errno(nf.errno()).put(']');
+            }
         }
-        recvPos += n;
+        return n;
+    }
 
-        // Try to parse again
-        result = tryParseFrame(handler);
-        return result != null && result;
+    private int remainingTime(int timeoutMillis, long startTimeNanos) {
+        timeoutMillis -= (int) NANOSECONDS.toMillis(System.nanoTime() - startTimeNanos);
+        if (timeoutMillis <= 0) {
+            throw new HttpClientException("timed out [errno=").errno(nf.errno()).put(']');
+        }
+        return timeoutMillis;
+    }
+
+    private void resetFragmentState() {
+        fragmentOpcode = -1;
+        fragmentBufPos = 0;
+    }
+
+    private void sendCloseFrameEcho(int code) {
+        try {
+            controlFrameBuffer.reset();
+            WebSocketSendBuffer.FrameInfo frame = controlFrameBuffer.writeCloseFrame(code, null);
+            doSend(controlFrameBuffer.getBufferPtr() + frame.offset, frame.length, 1000);
+            controlFrameBuffer.reset();
+        } catch (Exception e) {
+            LOG.error("Failed to echo close frame: {}", e.getMessage());
+        }
+    }
+
+    private void sendPongFrame(long payloadPtr, int payloadLen) {
+        try {
+            controlFrameBuffer.reset();
+            WebSocketSendBuffer.FrameInfo frame = controlFrameBuffer.writePongFrame(payloadPtr, payloadLen);
+            doSend(controlFrameBuffer.getBufferPtr() + frame.offset, frame.length, 1000);
+            controlFrameBuffer.reset();
+        } catch (Exception e) {
+            LOG.error("Failed to send pong: {}", e.getMessage());
+        }
     }
 
     private Boolean tryParseFrame(WebSocketFrameHandler handler) {
@@ -600,7 +722,7 @@ private Boolean tryParseFrame(WebSocketFrameHandler handler) {
         int consumed = frameParser.parse(recvBufPtr + recvReadPos, recvBufPtr + recvPos);
 
         if (frameParser.getState() == WebSocketFrameParser.STATE_NEED_MORE ||
-            frameParser.getState() == WebSocketFrameParser.STATE_NEED_PAYLOAD) {
+                frameParser.getState() == WebSocketFrameParser.STATE_NEED_PAYLOAD) {
             return null; // Need more data
         }
 
@@ -706,130 +828,25 @@ private Boolean tryParseFrame(WebSocketFrameHandler handler) {
         return false;
     }
 
-    private void sendCloseFrameEcho(int code) {
-        try {
-            controlFrameBuffer.reset();
-            WebSocketSendBuffer.FrameInfo frame = controlFrameBuffer.writeCloseFrame(code, null);
-            doSend(controlFrameBuffer.getBufferPtr() + frame.offset, frame.length, 1000);
-            controlFrameBuffer.reset();
-        } catch (Exception e) {
-            LOG.error("Failed to echo close frame: {}", e.getMessage());
-        }
-    }
-
-    private void sendPongFrame(long payloadPtr, int payloadLen) {
-        try {
-            controlFrameBuffer.reset();
-            WebSocketSendBuffer.FrameInfo frame = controlFrameBuffer.writePongFrame(payloadPtr, payloadLen);
-            doSend(controlFrameBuffer.getBufferPtr() + frame.offset, frame.length, 1000);
-            controlFrameBuffer.reset();
-        } catch (Exception e) {
-            LOG.error("Failed to send pong: {}", e.getMessage());
-        }
-    }
-
-    private void appendToFragmentBuffer(long payloadPtr, int payloadLen) {
-        if (payloadLen == 0) {
-            return;
-        }
-        int required = fragmentBufPos + payloadLen;
-        if (required > maxRecvBufSize) {
-            throw new HttpClientException("WebSocket fragment buffer size exceeded maximum [required=")
-                    .put(required)
-                    .put(", max=")
-                    .put(maxRecvBufSize)
-                    .put(']');
-        }
-        if (fragmentBufPtr == 0) {
-            fragmentBufSize = Math.max(required, DEFAULT_RECV_BUFFER_SIZE);
-            fragmentBufPtr = Unsafe.malloc(fragmentBufSize, MemoryTag.NATIVE_DEFAULT);
-        } else if (required > fragmentBufSize) {
-            int newSize = Math.min(Math.max(fragmentBufSize * 2, required), maxRecvBufSize);
-            fragmentBufPtr = Unsafe.realloc(fragmentBufPtr, fragmentBufSize, newSize, MemoryTag.NATIVE_DEFAULT);
-            fragmentBufSize = newSize;
-        }
-        Vect.memmove(fragmentBufPtr + fragmentBufPos, payloadPtr, payloadLen);
-        fragmentBufPos += payloadLen;
-    }
-
-    private void resetFragmentState() {
-        fragmentOpcode = -1;
-        fragmentBufPos = 0;
-    }
-
-    private void compactRecvBuffer() {
-        if (recvReadPos > 0) {
-            int remaining = recvPos - recvReadPos;
-            if (remaining > 0) {
-                Vect.memmove(recvBufPtr, recvBufPtr + recvReadPos, remaining);
-            }
-            recvPos = remaining;
-            recvReadPos = 0;
-        }
-    }
-
-    private void growRecvBuffer() {
-        int newSize = recvBufSize * 2;
-        if (newSize > maxRecvBufSize) {
-            if (recvBufSize >= maxRecvBufSize) {
-                throw new HttpClientException("WebSocket receive buffer size exceeded maximum [current=")
-                        .put(recvBufSize)
-                        .put(", max=")
-                        .put(maxRecvBufSize)
-                        .put(']');
-            }
-            newSize = maxRecvBufSize;
-        }
-        recvBufPtr = Unsafe.realloc(recvBufPtr, recvBufSize, newSize, MemoryTag.NATIVE_DEFAULT);
-        recvBufSize = newSize;
-    }
-
-    // === Socket I/O helpers ===
-
-    private int recvOrDie(long ptr, int len, int timeout) {
-        long startTime = System.nanoTime();
-        int n = dieIfNegative(socket.recv(ptr, len));
-        if (n == 0) {
-            ioWait(remainingTime(timeout, startTime), IOOperation.READ);
-            n = dieIfNegative(socket.recv(ptr, len));
-        }
-        return n;
-    }
-
-    private int recvOrTimeout(long ptr, int len, int timeout) {
-        long startTime = System.nanoTime();
-        int n = socket.recv(ptr, len);
-        if (n < 0) {
-            throw new HttpClientException("peer disconnect [errno=").errno(nf.errno()).put(']');
-        }
-        if (n == 0) {
-            try {
-                ioWait(timeout, IOOperation.READ);
-            } catch (HttpClientException e) {
-                // Timeout
-                return 0;
-            }
-            n = socket.recv(ptr, len);
-            if (n < 0) {
-                throw new HttpClientException("peer disconnect [errno=").errno(nf.errno()).put(']');
-            }
+    private void validateUpgradeResponse(int headerEnd) {
+        // Extract response as string for parsing
+        byte[] responseBytes = new byte[headerEnd];
+        for (int i = 0; i < headerEnd; i++) {
+            responseBytes[i] = Unsafe.getUnsafe().getByte(recvBufPtr + i);
         }
-        return n;
-    }
+        String response = new String(responseBytes, StandardCharsets.US_ASCII);
 
-    private int dieIfNegative(int byteCount) {
-        if (byteCount < 0) {
-            throw new HttpClientException("peer disconnect [errno=").errno(nf.errno()).put(']');
+        // Check status line
+        if (!response.startsWith("HTTP/1.1 101")) {
+            String statusLine = response.split("\r\n")[0];
+            throw new HttpClientException("WebSocket upgrade failed: ").put(statusLine);
         }
-        return byteCount;
-    }
 
-    private int remainingTime(int timeoutMillis, long startTimeNanos) {
-        timeoutMillis -= (int) NANOSECONDS.toMillis(System.nanoTime() - startTimeNanos);
-        if (timeoutMillis <= 0) {
-            throw new HttpClientException("timed out [errno=").errno(nf.errno()).put(']');
+        // Verify Sec-WebSocket-Accept (case-insensitive per RFC 7230)
+        String expectedAccept = WebSocketHandshake.computeAcceptKey(handshakeKey);
+        if (!containsHeaderValue(response, "Sec-WebSocket-Accept:", expectedAccept)) {
+            throw new HttpClientException("Invalid Sec-WebSocket-Accept header");
         }
-        return timeoutMillis;
     }
 
     protected void dieWaiting(int n) {
@@ -842,40 +859,6 @@ protected void dieWaiting(int n) {
         throw new HttpClientException("queue error [errno=").put(nf.errno()).put(']');
     }
 
-    private void checkConnected() {
-        if (closed) {
-            throw new HttpClientException("WebSocket client is closed");
-        }
-        if (!upgraded) {
-            throw new HttpClientException("WebSocket not connected or upgraded");
-        }
-    }
-
-    // === State ===
-
-    /**
-     * Returns whether the WebSocket is connected and upgraded.
-     */
-    public boolean isConnected() {
-        return upgraded && !closed && !socket.isClosed();
-    }
-
-    /**
-     * Returns the connected host.
-     */
-    public CharSequence getHost() {
-        return host;
-    }
-
-    /**
-     * Returns the connected port.
-     */
-    public int getPort() {
-        return port;
-    }
-
-    // === Platform-specific I/O ===
-
     /**
      * Waits for I/O readiness using platform-specific mechanism.
      *
diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketFrameHandler.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketFrameHandler.java
index aff429d..e3682f5 100644
--- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketFrameHandler.java
+++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketFrameHandler.java
@@ -43,18 +43,6 @@ public interface WebSocketFrameHandler {
      */
     void onBinaryMessage(long payloadPtr, int payloadLen);
 
-    /**
-     * Called when a text frame is received.
-     * 

-     * Default implementation does nothing. Override if text frames need handling.
-     *
-     * @param payloadPtr pointer to the UTF-8 encoded payload in native memory
-     * @param payloadLen length of the payload in bytes
-     */
-    default void onTextMessage(long payloadPtr, int payloadLen) {
-        // Default: ignore text frames
-    }
-
     /**
      * Called when a close frame is received from the server.
      * 

@@ -90,4 +78,16 @@ default void onPing(long payloadPtr, int payloadLen) {
     default void onPong(long payloadPtr, int payloadLen) {
         // Default: ignore pong frames
     }
+
+    /**
+     * Called when a text frame is received.
+     * 

+     * Default implementation does nothing. Override if text frames need handling.
+     *
+     * @param payloadPtr pointer to the UTF-8 encoded payload in native memory
+     * @param payloadLen length of the payload in bytes
+     */
+    default void onTextMessage(long payloadPtr, int payloadLen) {
+        // Default: ignore text frames
+    }
 }
diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java
index 5075b9f..861de15 100644
--- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java
+++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java
@@ -24,10 +24,10 @@
 
 package io.questdb.client.cutlass.http.client;
 
+import io.questdb.client.cutlass.line.array.ArrayBufferAppender;
+import io.questdb.client.cutlass.qwp.client.QwpBufferWriter;
 import io.questdb.client.cutlass.qwp.websocket.WebSocketFrameWriter;
 import io.questdb.client.cutlass.qwp.websocket.WebSocketOpcode;
-import io.questdb.client.cutlass.qwp.client.QwpBufferWriter;
-import io.questdb.client.cutlass.line.array.ArrayBufferAppender;
 import io.questdb.client.std.MemoryTag;
 import io.questdb.client.std.Numbers;
 import io.questdb.client.std.QuietCloseable;
@@ -57,21 +57,18 @@
  */
 public class WebSocketSendBuffer implements QwpBufferWriter, QuietCloseable {
 
-    // Maximum header size: 2 (base) + 8 (64-bit length) + 4 (mask key)
-    private static final int MAX_HEADER_SIZE = 14;
-
     private static final int DEFAULT_INITIAL_CAPACITY = 65536;
     private static final int MAX_BUFFER_SIZE = Integer.MAX_VALUE - 8; // Leave room for alignment
-
-    private long bufPtr;
+    // Maximum header size: 2 (base) + 8 (64-bit length) + 4 (mask key)
+    private static final int MAX_HEADER_SIZE = 14;
+    private final FrameInfo frameInfo = new FrameInfo();
+    private final int maxBufferSize;
+    private final Rnd rnd;
     private int bufCapacity;
-    private int writePos;           // Current write position (offset from bufPtr)
+    private long bufPtr;
     private int frameStartOffset;   // Where current frame's reserved header starts
     private int payloadStartOffset; // Where payload begins (frameStart + MAX_HEADER_SIZE)
-
-    private final Rnd rnd;
-    private final int maxBufferSize;
-    private final FrameInfo frameInfo = new FrameInfo();
+    private int writePos;           // Current write position (offset from bufPtr)
 
     /**
      * Creates a new WebSocket send buffer with default initial capacity.
@@ -105,6 +102,34 @@ public WebSocketSendBuffer(int initialCapacity, int maxBufferSize) {
         this.rnd = new Rnd(System.nanoTime(), System.currentTimeMillis());
     }
 
+    /**
+     * Begins a new binary WebSocket frame. Reserves space for the maximum header size.
+     * After calling this method, use ArrayBufferAppender methods to write the payload.
+     */
+    public void beginBinaryFrame() {
+        beginFrame(WebSocketOpcode.BINARY);
+    }
+
+    /**
+     * Begins a new WebSocket frame with the specified opcode.
+     *
+     * @param opcode the frame opcode
+     */
+    public void beginFrame(int opcode) {
+        frameStartOffset = writePos;
+        // Reserve maximum header space
+        ensureCapacity(MAX_HEADER_SIZE);
+        writePos += MAX_HEADER_SIZE;
+        payloadStartOffset = writePos;
+    }
+
+    /**
+     * Begins a new text WebSocket frame. Reserves space for the maximum header size.
+     */
+    public void beginTextFrame() {
+        beginFrame(WebSocketOpcode.TEXT);
+    }
+
     @Override
     public void close() {
         if (bufPtr != 0) {
@@ -114,7 +139,53 @@ public void close() {
         }
     }
 
-    // === Buffer Management ===
+    /**
+     * Finishes the current binary frame, writing the header and applying masking.
+     * Returns information about where to find the complete frame in the buffer.
+     * 

+     * IMPORTANT: Only call this after all payload writes are complete. The buffer
+     * pointer is stable after this call (no more reallocations for this frame).
+     *
+     * @return frame info containing offset and length for sending
+     */
+    public FrameInfo endBinaryFrame() {
+        return endFrame(WebSocketOpcode.BINARY);
+    }
+
+    /**
+     * Finishes the current frame with the specified opcode.
+     *
+     * @param opcode the frame opcode
+     * @return frame info containing offset and length for sending
+     */
+    public FrameInfo endFrame(int opcode) {
+        int payloadLen = writePos - payloadStartOffset;
+
+        // Calculate actual header size (with mask key for client frames)
+        int actualHeaderSize = WebSocketFrameWriter.headerSize(payloadLen, true);
+        int unusedSpace = MAX_HEADER_SIZE - actualHeaderSize;
+        int actualFrameStart = frameStartOffset + unusedSpace;
+
+        // Generate mask key
+        int maskKey = rnd.nextInt();
+
+        // Write header at actual position (after unused space)
+        WebSocketFrameWriter.writeHeader(bufPtr + actualFrameStart, true, opcode, payloadLen, maskKey);
+
+        // Apply mask to payload
+        if (payloadLen > 0) {
+            WebSocketFrameWriter.maskPayload(bufPtr + payloadStartOffset, payloadLen, maskKey);
+        }
+
+        return frameInfo.set(actualFrameStart, actualHeaderSize + payloadLen);
+    }
+
+    /**
+     * Finishes the current text frame, writing the header and applying masking.
+     */
+    public FrameInfo endTextFrame() {
+        return endFrame(WebSocketOpcode.TEXT);
+    }
 
     /**
      * Ensures the buffer has capacity for the specified number of additional bytes.
@@ -130,50 +201,63 @@ public void ensureCapacity(int additionalBytes) {
         }
     }
 
-    private void grow(long requiredCapacity) {
-        if (requiredCapacity > maxBufferSize) {
-            throw new HttpClientException("WebSocket buffer size exceeded maximum [required=")
-                    .put(requiredCapacity)
-                    .put(", max=")
-                    .put(maxBufferSize)
-                    .put(']');
-        }
-        int newCapacity = Math.min(
-                Numbers.ceilPow2((int) requiredCapacity),
-                maxBufferSize
-        );
-        bufPtr = Unsafe.realloc(bufPtr, bufCapacity, newCapacity, MemoryTag.NATIVE_DEFAULT);
-        bufCapacity = newCapacity;
+    /**
+     * Gets the buffer pointer. Only use this for reading after frame is complete.
+     */
+    public long getBufferPtr() {
+        return bufPtr;
     }
 
-    // === ArrayBufferAppender Implementation ===
+    /**
+     * Gets the current buffer capacity.
+     */
+    public int getCapacity() {
+        return bufCapacity;
+    }
 
-    @Override
-    public void putByte(byte b) {
-        ensureCapacity(1);
-        Unsafe.getUnsafe().putByte(bufPtr + writePos, b);
-        writePos++;
+    /**
+     * Gets the payload length of the current frame being built.
+     */
+    public int getCurrentPayloadLength() {
+        return writePos - payloadStartOffset;
     }
 
+    /**
+     * Gets the current write position (number of bytes written).
+     */
     @Override
-    public void putInt(int value) {
-        ensureCapacity(4);
-        Unsafe.getUnsafe().putInt(bufPtr + writePos, value);
-        writePos += 4;
+    public int getPosition() {
+        return writePos;
     }
 
-    @Override
-    public void putLong(long value) {
-        ensureCapacity(8);
-        Unsafe.getUnsafe().putLong(bufPtr + writePos, value);
-        writePos += 8;
+    /**
+     * Gets the current write position (total bytes written since last reset).
+     */
+    public int getWritePos() {
+        return writePos;
     }
 
+    /**
+     * Patches an int value at the specified offset.
+     */
     @Override
-    public void putDouble(double value) {
-        ensureCapacity(8);
-        Unsafe.getUnsafe().putDouble(bufPtr + writePos, value);
-        writePos += 8;
+    public void patchInt(int offset, int value) {
+        Unsafe.getUnsafe().putInt(bufPtr + offset, value);
+    }
+
+    /**
+     * Writes an ASCII string.
+     */
+    public void putAscii(CharSequence cs) {
+        if (cs == null) {
+            return;
+        }
+        int len = cs.length();
+        ensureCapacity(len);
+        for (int i = 0; i < len; i++) {
+            Unsafe.getUnsafe().putByte(bufPtr + writePos + i, (byte) cs.charAt(i));
+        }
+        writePos += len;
     }
 
     @Override
@@ -186,15 +270,32 @@ public void putBlockOfBytes(long from, long len) {
         writePos += (int) len;
     }
 
-    // === Additional write methods (not in ArrayBufferAppender but useful) ===
+    @Override
+    public void putByte(byte b) {
+        ensureCapacity(1);
+        Unsafe.getUnsafe().putByte(bufPtr + writePos, b);
+        writePos++;
+    }
 
     /**
-     * Writes a short value in little-endian format.
+     * Writes raw bytes from a byte array.
      */
-    public void putShort(short value) {
-        ensureCapacity(2);
-        Unsafe.getUnsafe().putShort(bufPtr + writePos, value);
-        writePos += 2;
+    public void putBytes(byte[] bytes, int offset, int length) {
+        if (length <= 0) {
+            return;
+        }
+        ensureCapacity(length);
+        for (int i = 0; i < length; i++) {
+            Unsafe.getUnsafe().putByte(bufPtr + writePos + i, bytes[offset + i]);
+        }
+        writePos += length;
+    }
+
+    @Override
+    public void putDouble(double value) {
+        ensureCapacity(8);
+        Unsafe.getUnsafe().putDouble(bufPtr + writePos, value);
+        writePos += 8;
     }
 
     /**
@@ -206,6 +307,20 @@ public void putFloat(float value) {
         writePos += 4;
     }
 
+    @Override
+    public void putInt(int value) {
+        ensureCapacity(4);
+        Unsafe.getUnsafe().putInt(bufPtr + writePos, value);
+        writePos += 4;
+    }
+
+    @Override
+    public void putLong(long value) {
+        ensureCapacity(8);
+        Unsafe.getUnsafe().putLong(bufPtr + writePos, value);
+        writePos += 8;
+    }
+
     /**
      * Writes a long value in big-endian format.
      */
@@ -216,46 +331,12 @@ public void putLongBE(long value) {
     }
 
     /**
-     * Writes raw bytes from a byte array.
-     */
-    public void putBytes(byte[] bytes, int offset, int length) {
-        if (length <= 0) {
-            return;
-        }
-        ensureCapacity(length);
-        for (int i = 0; i < length; i++) {
-            Unsafe.getUnsafe().putByte(bufPtr + writePos + i, bytes[offset + i]);
-        }
-        writePos += length;
-    }
-
-    /**
-     * Writes an ASCII string.
-     */
-    public void putAscii(CharSequence cs) {
-        if (cs == null) {
-            return;
-        }
-        int len = cs.length();
-        ensureCapacity(len);
-        for (int i = 0; i < len; i++) {
-            Unsafe.getUnsafe().putByte(bufPtr + writePos + i, (byte) cs.charAt(i));
-        }
-        writePos += len;
-    }
-
-    // === QwpBufferWriter Implementation ===
-
-    /**
-     * Writes an unsigned variable-length integer (LEB128 encoding).
+     * Writes a short value in little-endian format.
      */
-    @Override
-    public void putVarint(long value) {
-        while (value > 0x7F) {
-            putByte((byte) ((value & 0x7F) | 0x80));
-            value >>>= 7;
-        }
-        putByte((byte) value);
+    public void putShort(short value) {
+        ensureCapacity(2);
+        Unsafe.getUnsafe().putShort(bufPtr + writePos, value);
+        writePos += 2;
     }
 
     /**
@@ -303,106 +384,79 @@ public void putUtf8(String value) {
     }
 
     /**
-     * Patches an int value at the specified offset.
+     * Writes an unsigned variable-length integer (LEB128 encoding).
      */
     @Override
-    public void patchInt(int offset, int value) {
-        Unsafe.getUnsafe().putInt(bufPtr + offset, value);
+    public void putVarint(long value) {
+        while (value > 0x7F) {
+            putByte((byte) ((value & 0x7F) | 0x80));
+            value >>>= 7;
+        }
+        putByte((byte) value);
     }
 
     /**
-     * Skips the specified number of bytes, advancing the position.
+     * Resets the buffer for reuse. Does not deallocate memory.
      */
-    @Override
-    public void skip(int bytes) {
-        ensureCapacity(bytes);
-        writePos += bytes;
+    public void reset() {
+        writePos = 0;
+        frameStartOffset = 0;
+        payloadStartOffset = 0;
     }
 
     /**
-     * Gets the current write position (number of bytes written).
+     * Skips the specified number of bytes, advancing the position.
      */
     @Override
-    public int getPosition() {
-        return writePos;
-    }
-
-    // === Frame Building ===
-
-    /**
-     * Begins a new binary WebSocket frame. Reserves space for the maximum header size.
-     * After calling this method, use ArrayBufferAppender methods to write the payload.
-     */
-    public void beginBinaryFrame() {
-        beginFrame(WebSocketOpcode.BINARY);
-    }
-
-    /**
-     * Begins a new text WebSocket frame. Reserves space for the maximum header size.
-     */
-    public void beginTextFrame() {
-        beginFrame(WebSocketOpcode.TEXT);
-    }
-
-    /**
-     * Begins a new WebSocket frame with the specified opcode.
-     *
-     * @param opcode the frame opcode
-     */
-    public void beginFrame(int opcode) {
-        frameStartOffset = writePos;
-        // Reserve maximum header space
-        ensureCapacity(MAX_HEADER_SIZE);
-        writePos += MAX_HEADER_SIZE;
-        payloadStartOffset = writePos;
+    public void skip(int bytes) {
+        ensureCapacity(bytes);
+        writePos += bytes;
     }
 
     /**
-     * Finishes the current binary frame, writing the header and applying masking.
-     * Returns information about where to find the complete frame in the buffer.
-     * 

-     * IMPORTANT: Only call this after all payload writes are complete. The buffer
-     * pointer is stable after this call (no more reallocations for this frame).
+     * Writes a complete close frame.
      *
-     * @return frame info containing offset and length for sending
-     */
-    public FrameInfo endBinaryFrame() {
-        return endFrame(WebSocketOpcode.BINARY);
-    }
-
-    /**
-     * Finishes the current text frame, writing the header and applying masking.
+     * @param code   close status code (e.g., 1000 for normal closure)
+     * @param reason optional reason string (may be null)
+     * @return frame info for sending
      */
-    public FrameInfo endTextFrame() {
-        return endFrame(WebSocketOpcode.TEXT);
-    }
+    public FrameInfo writeCloseFrame(int code, String reason) {
+        int payloadLen = 2; // status code
+        byte[] reasonBytes = null;
+        if (reason != null && !reason.isEmpty()) {
+            reasonBytes = reason.getBytes(java.nio.charset.StandardCharsets.UTF_8);
+            payloadLen += reasonBytes.length;
+        }
 
-    /**
-     * Finishes the current frame with the specified opcode.
-     *
-     * @param opcode the frame opcode
-     * @return frame info containing offset and length for sending
-     */
-    public FrameInfo endFrame(int opcode) {
-        int payloadLen = writePos - payloadStartOffset;
+        if (payloadLen > 125) {
+            throw new HttpClientException("Close payload too large [len=").put(payloadLen).put(']');
+        }
 
-        // Calculate actual header size (with mask key for client frames)
-        int actualHeaderSize = WebSocketFrameWriter.headerSize(payloadLen, true);
-        int unusedSpace = MAX_HEADER_SIZE - actualHeaderSize;
-        int actualFrameStart = frameStartOffset + unusedSpace;
+        int frameStart = writePos;
+        int headerSize = WebSocketFrameWriter.headerSize(payloadLen, true);
+        ensureCapacity(headerSize + payloadLen);
 
-        // Generate mask key
         int maskKey = rnd.nextInt();
+        int written = WebSocketFrameWriter.writeHeader(bufPtr + writePos, true, WebSocketOpcode.CLOSE, payloadLen, maskKey);
+        writePos += written;
 
-        // Write header at actual position (after unused space)
-        WebSocketFrameWriter.writeHeader(bufPtr + actualFrameStart, true, opcode, payloadLen, maskKey);
+        // Write status code (big-endian)
+        long payloadStart = bufPtr + writePos;
+        Unsafe.getUnsafe().putByte(payloadStart, (byte) ((code >> 8) & 0xFF));
+        Unsafe.getUnsafe().putByte(payloadStart + 1, (byte) (code & 0xFF));
+        writePos += 2;
 
-        // Apply mask to payload
-        if (payloadLen > 0) {
-            WebSocketFrameWriter.maskPayload(bufPtr + payloadStartOffset, payloadLen, maskKey);
+        // Write reason if present
+        if (reasonBytes != null) {
+            for (byte reasonByte : reasonBytes) {
+                Unsafe.getUnsafe().putByte(bufPtr + writePos++, reasonByte);
+            }
         }
 
-        return frameInfo.set(actualFrameStart, actualHeaderSize + payloadLen);
+        // Mask the payload (including status code and reason)
+        WebSocketFrameWriter.maskPayload(payloadStart, payloadLen, maskKey);
+
+        return frameInfo.set(frameStart, headerSize + payloadLen);
     }
 
     /**
@@ -473,89 +527,20 @@ public FrameInfo writePongFrame(long payloadPtr, int payloadLen) {
         return frameInfo.set(frameStart, headerSize + payloadLen);
     }
 
-    /**
-     * Writes a complete close frame.
-     *
-     * @param code   close status code (e.g., 1000 for normal closure)
-     * @param reason optional reason string (may be null)
-     * @return frame info for sending
-     */
-    public FrameInfo writeCloseFrame(int code, String reason) {
-        int payloadLen = 2; // status code
-        byte[] reasonBytes = null;
-        if (reason != null && !reason.isEmpty()) {
-            reasonBytes = reason.getBytes(java.nio.charset.StandardCharsets.UTF_8);
-            payloadLen += reasonBytes.length;
-        }
-
-        if (payloadLen > 125) {
-            throw new HttpClientException("Close payload too large [len=").put(payloadLen).put(']');
-        }
-
-        int frameStart = writePos;
-        int headerSize = WebSocketFrameWriter.headerSize(payloadLen, true);
-        ensureCapacity(headerSize + payloadLen);
-
-        int maskKey = rnd.nextInt();
-        int written = WebSocketFrameWriter.writeHeader(bufPtr + writePos, true, WebSocketOpcode.CLOSE, payloadLen, maskKey);
-        writePos += written;
-
-        // Write status code (big-endian)
-        long payloadStart = bufPtr + writePos;
-        Unsafe.getUnsafe().putByte(payloadStart, (byte) ((code >> 8) & 0xFF));
-        Unsafe.getUnsafe().putByte(payloadStart + 1, (byte) (code & 0xFF));
-        writePos += 2;
-
-        // Write reason if present
-        if (reasonBytes != null) {
-            for (byte reasonByte : reasonBytes) {
-                Unsafe.getUnsafe().putByte(bufPtr + writePos++, reasonByte);
-            }
+    private void grow(long requiredCapacity) {
+        if (requiredCapacity > maxBufferSize) {
+            throw new HttpClientException("WebSocket buffer size exceeded maximum [required=")
+                    .put(requiredCapacity)
+                    .put(", max=")
+                    .put(maxBufferSize)
+                    .put(']');
         }
-
-        // Mask the payload (including status code and reason)
-        WebSocketFrameWriter.maskPayload(payloadStart, payloadLen, maskKey);
-
-        return frameInfo.set(frameStart, headerSize + payloadLen);
-    }
-
-    // === Buffer State ===
-
-    /**
-     * Gets the buffer pointer. Only use this for reading after frame is complete.
-     */
-    public long getBufferPtr() {
-        return bufPtr;
-    }
-
-    /**
-     * Gets the current buffer capacity.
-     */
-    public int getCapacity() {
-        return bufCapacity;
-    }
-
-    /**
-     * Gets the current write position (total bytes written since last reset).
-     */
-    public int getWritePos() {
-        return writePos;
-    }
-
-    /**
-     * Gets the payload length of the current frame being built.
-     */
-    public int getCurrentPayloadLength() {
-        return writePos - payloadStartOffset;
-    }
-
-    /**
-     * Resets the buffer for reuse. Does not deallocate memory.
-     */
-    public void reset() {
-        writePos = 0;
-        frameStartOffset = 0;
-        payloadStartOffset = 0;
+        int newCapacity = Math.min(
+                Numbers.ceilPow2((int) requiredCapacity),
+                maxBufferSize
+        );
+        bufPtr = Unsafe.realloc(bufPtr, bufCapacity, newCapacity, MemoryTag.NATIVE_DEFAULT);
+        bufCapacity = newCapacity;
     }
 
     /**
@@ -564,15 +549,14 @@ public void reset() {
      * extract values before calling any end*Frame() method again.
      */
     public static final class FrameInfo {
-        /**
-         * Offset from buffer start where the frame begins.
-         */
-        public int offset;
-
         /**
          * Total length of the frame (header + payload).
          */
         public int length;
+        /**
+         * Offset from buffer start where the frame begins.
+         */
+        public int offset;
 
         FrameInfo set(int offset, int length) {
             this.offset = offset;
diff --git a/core/src/main/java/io/questdb/client/cutlass/json/JsonLexer.java b/core/src/main/java/io/questdb/client/cutlass/json/JsonLexer.java
index 8b4caf0..1f0cc05 100644
--- a/core/src/main/java/io/questdb/client/cutlass/json/JsonLexer.java
+++ b/core/src/main/java/io/questdb/client/cutlass/json/JsonLexer.java
@@ -65,6 +65,7 @@ public class JsonLexer implements Mutable, Closeable {
     private boolean quoted = false;
     private int state = S_START;
     private boolean useCache = false;
+
     public JsonLexer(int cacheSize, int cacheSizeLimit) {
         this.cacheSizeLimit = cacheSizeLimit;
         // if cacheSizeLimit is 0 or negative, the cache is disabled
@@ -398,4 +399,4 @@ private void utf8DecodeCacheAndBuffer(long lo, long hi, int position) throws Jso
         unquotedTerminators.add('{');
         unquotedTerminators.add('[');
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/io/questdb/client/cutlass/line/AbstractLineSender.java b/core/src/main/java/io/questdb/client/cutlass/line/AbstractLineSender.java
index 0ed3937..11274a1 100644
--- a/core/src/main/java/io/questdb/client/cutlass/line/AbstractLineSender.java
+++ b/core/src/main/java/io/questdb/client/cutlass/line/AbstractLineSender.java
@@ -368,7 +368,7 @@ private static int findEOL(long ptr, int len) {
 
     private byte[] receiveChallengeBytes() {
         int n = 0;
-        for (;;) {
+        for (; ; ) {
             int rc = lineChannel.receive(ptr + n, capacity - n);
             if (rc < 0) {
                 int errno = lineChannel.errno();
@@ -505,4 +505,4 @@ protected AbstractLineSender writeFieldName(CharSequence name) {
         }
         throw new LineSenderException("table expected");
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java
index 4d75211..ace342e 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java
@@ -38,8 +38,8 @@
  */
 public class GlobalSymbolDictionary {
 
-    private final CharSequenceIntHashMap symbolToId;
     private final ObjList idToSymbol;
+    private final CharSequenceIntHashMap symbolToId;
 
     public GlobalSymbolDictionary() {
         this(64); // Default initial capacity
@@ -50,6 +50,40 @@ public GlobalSymbolDictionary(int initialCapacity) {
         this.idToSymbol = new ObjList<>(initialCapacity);
     }
 
+    /**
+     * Clears all symbols from the dictionary.
+     * 

+     * After clearing, the next symbol added will get ID 0.
+     */
+    public void clear() {
+        symbolToId.clear();
+        idToSymbol.clear();
+    }
+
+    /**
+     * Checks if the dictionary contains the given symbol.
+     *
+     * @param symbol the symbol to check
+     * @return true if the symbol exists in the dictionary
+     */
+    public boolean contains(String symbol) {
+        return symbol != null && symbolToId.get(symbol) != CharSequenceIntHashMap.NO_ENTRY_VALUE;
+    }
+
+    /**
+     * Gets the ID for an existing symbol, or -1 if not found.
+     *
+     * @param symbol the symbol string
+     * @return the symbol ID, or -1 if not in dictionary
+     */
+    public int getId(String symbol) {
+        if (symbol == null) {
+            return -1;
+        }
+        int id = symbolToId.get(symbol);
+        return id == CharSequenceIntHashMap.NO_ENTRY_VALUE ? -1 : id;
+    }
+
     /**
      * Gets or adds a symbol to the dictionary.
      * 

@@ -91,48 +125,6 @@ public String getSymbol(int id) {
         return idToSymbol.getQuick(id);
     }
 
-    /**
-     * Gets the ID for an existing symbol, or -1 if not found.
-     *
-     * @param symbol the symbol string
-     * @return the symbol ID, or -1 if not in dictionary
-     */
-    public int getId(String symbol) {
-        if (symbol == null) {
-            return -1;
-        }
-        int id = symbolToId.get(symbol);
-        return id == CharSequenceIntHashMap.NO_ENTRY_VALUE ? -1 : id;
-    }
-
-    /**
-     * Returns the number of symbols in the dictionary.
-     *
-     * @return dictionary size
-     */
-    public int size() {
-        return idToSymbol.size();
-    }
-
-    /**
-     * Checks if the dictionary is empty.
-     *
-     * @return true if no symbols have been added
-     */
-    public boolean isEmpty() {
-        return idToSymbol.size() == 0;
-    }
-
-    /**
-     * Checks if the dictionary contains the given symbol.
-     *
-     * @param symbol the symbol to check
-     * @return true if the symbol exists in the dictionary
-     */
-    public boolean contains(String symbol) {
-        return symbol != null && symbolToId.get(symbol) != CharSequenceIntHashMap.NO_ENTRY_VALUE;
-    }
-
     /**
      * Gets the symbols in the given ID range [fromId, toId).
      * 

@@ -162,12 +154,20 @@ public String[] getSymbolsInRange(int fromId, int toId) {
     }
 
     /**
-     * Clears all symbols from the dictionary.
-     * 

-     * After clearing, the next symbol added will get ID 0.
+     * Checks if the dictionary is empty.
+     *
+     * @return true if no symbols have been added
      */
-    public void clear() {
-        symbolToId.clear();
-        idToSymbol.clear();
+    public boolean isEmpty() {
+        return idToSymbol.size() == 0;
+    }
+
+    /**
+     * Returns the number of symbols in the dictionary.
+     *
+     * @return dictionary size
+     */
+    public int size() {
+        return idToSymbol.size();
     }
 }
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/InFlightWindow.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/InFlightWindow.java
index cffd93e..869d242 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/InFlightWindow.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/InFlightWindow.java
@@ -57,38 +57,29 @@
  */
 public class InFlightWindow {
 
-    private static final Logger LOG = LoggerFactory.getLogger(InFlightWindow.class);
-
-    public static final int DEFAULT_WINDOW_SIZE = 8;
     public static final long DEFAULT_TIMEOUT_MS = 30_000;
-
+    public static final int DEFAULT_WINDOW_SIZE = 8;
+    private static final Logger LOG = LoggerFactory.getLogger(InFlightWindow.class);
+    private static final long PARK_NANOS = 100_000; // 100 microseconds
     // Spin parameters
     private static final int SPIN_TRIES = 100;
-    private static final long PARK_NANOS = 100_000; // 100 microseconds
-
+    // Error state
+    private final AtomicReference lastError = new AtomicReference<>();
     private final int maxWindowSize;
     private final long timeoutMs;
-
+    private volatile long failedBatchId = -1;
+    // highestAcked: the sequence number of the last acknowledged batch (cumulative)
+    private volatile long highestAcked = -1;
     // Core state
     // highestSent: the sequence number of the last batch added to the window
     private volatile long highestSent = -1;
-
-    // highestAcked: the sequence number of the last acknowledged batch (cumulative)
-    private volatile long highestAcked = -1;
-
-    // Error state
-    private final AtomicReference lastError = new AtomicReference<>();
-    private volatile long failedBatchId = -1;
-
-    // Thread waiting for space (sender thread)
-    private volatile Thread waitingForSpace;
-
-    // Thread waiting for empty (flush thread)
-    private volatile Thread waitingForEmpty;
-
     // Statistics (not strictly accurate under contention, but good enough for monitoring)
     private volatile long totalAcked = 0;
     private volatile long totalFailed = 0;
+    // Thread waiting for empty (flush thread)
+    private volatile Thread waitingForEmpty;
+    // Thread waiting for space (sender thread)
+    private volatile Thread waitingForSpace;
 
     /**
      * Creates a new InFlightWindow with default configuration.
@@ -112,37 +103,64 @@ public InFlightWindow(int maxWindowSize, long timeoutMs) {
     }
 
     /**
-     * Checks if there's space in the window for another batch.
-     * Wait-free operation.
+     * Acknowledges a batch, removing it from the in-flight window.
+     * 

+     * For sequential batch IDs, this is a cumulative acknowledgment -
+     * acknowledging batch N means all batches up to N are acknowledged.
+     * 

+     * Called by: acker (WebSocket I/O thread) after receiving an ACK.
      *
-     * @return true if there's space, false if window is full
+     * @param batchId the batch ID that was acknowledged
+     * @return true if the batch was in flight, false if already acknowledged
      */
-    public boolean hasWindowSpace() {
-        return getInFlightCount() < maxWindowSize;
+    public boolean acknowledge(long batchId) {
+        return acknowledgeUpTo(batchId) > 0 || highestAcked >= batchId;
     }
 
     /**
-     * Tries to add a batch to the in-flight window without blocking.
-     * Lock-free, assuming single producer for highestSent.
+     * Acknowledges all batches up to and including the given sequence (cumulative ACK).
+     * Lock-free with single consumer.
+     * 

+     * Called by: acker (WebSocket I/O thread) after receiving an ACK.
      *
-     * Called by: async producer (WebSocket I/O thread) before sending a batch.
-     * @param batchId the batch ID to track (must be sequential)
-     * @return true if added, false if window is full
+     * @param sequence the highest acknowledged sequence
+     * @return the number of batches acknowledged
      */
-    public boolean tryAddInFlight(long batchId) {
-        // Check window space first
+    public int acknowledgeUpTo(long sequence) {
         long sent = highestSent;
-        long acked = highestAcked;
 
-        if (sent - acked >= maxWindowSize) {
-            return false;
+        // Nothing to acknowledge if window is empty or sequence is beyond what's sent
+        if (sent < 0) {
+            return 0; // No batches have been sent
         }
 
-        // Sequential caller: just publish the new highestSent
-        highestSent = batchId;
+        // Cap sequence at highestSent - can't acknowledge what hasn't been sent
+        long effectiveSequence = Math.min(sequence, sent);
 
-        LOG.debug("Added to window [batchId={}, windowSize={}]", batchId, getInFlightCount());
-        return true;
+        long prevAcked = highestAcked;
+        if (effectiveSequence <= prevAcked) {
+            // Already acknowledged up to this point
+            return 0;
+        }
+        highestAcked = effectiveSequence;
+
+        int acknowledged = (int) (effectiveSequence - prevAcked);
+        totalAcked += acknowledged;
+
+        LOG.debug("Cumulative ACK [upTo={}, acknowledged={}, remaining={}]", sequence, acknowledged, getInFlightCount());
+
+        // Wake up waiting threads
+        Thread waiter = waitingForSpace;
+        if (waiter != null) {
+            LockSupport.unpark(waiter);
+        }
+
+        waiter = waitingForEmpty;
+        if (waiter != null && getInFlightCount() == 0) {
+            LockSupport.unpark(waiter);
+        }
+
+        return acknowledged;
     }
 
     /**
@@ -156,8 +174,9 @@ public boolean tryAddInFlight(long batchId) {
      * it must ensure ACKs are processed on another thread; a single-threaded caller
      * with window>1 would deadlock by parking while also being the only thread that
      * can advance {@link #acknowledgeUpTo(long)}.
-     *
+     * 

      * Called by: sync sender thread before sending a batch (window=1).
+     *
      * @param batchId the batch ID to track
      * @throws LineSenderException if timeout occurs or an error was reported
      */
@@ -210,85 +229,69 @@ public void addInFlight(long batchId) {
         }
     }
 
-    private boolean tryAddInFlightInternal(long batchId) {
-        long sent = highestSent;
-        long acked = highestAcked;
-
-        if (sent - acked >= maxWindowSize) {
-            return false;
-        }
-
-        // For sequential IDs, we just update highestSent
-        // The caller guarantees batchId is the next in sequence
-        highestSent = batchId;
-
-        LOG.debug("Added to window [batchId={}, windowSize={}]", batchId, getInFlightCount());
-        return true;
-    }
-
     /**
-     * Acknowledges a batch, removing it from the in-flight window.
+     * Waits until all in-flight batches are acknowledged.
      * 

-     * For sequential batch IDs, this is a cumulative acknowledgment -
-     * acknowledging batch N means all batches up to N are acknowledged.
-     *
-     * Called by: acker (WebSocket I/O thread) after receiving an ACK.
-     * @param batchId the batch ID that was acknowledged
-     * @return true if the batch was in flight, false if already acknowledged
-     */
-    public boolean acknowledge(long batchId) {
-        return acknowledgeUpTo(batchId) > 0 || highestAcked >= batchId;
-    }
-
-    /**
-     * Acknowledges all batches up to and including the given sequence (cumulative ACK).
-     * Lock-free with single consumer.
+     * Called by flush() to ensure all data is confirmed.
+     * 

+     * Called by: waiter (flush thread), while producer/acker thread progresses.
      *
-     * Called by: acker (WebSocket I/O thread) after receiving an ACK.
-     * @param sequence the highest acknowledged sequence
-     * @return the number of batches acknowledged
+     * @throws LineSenderException if timeout occurs or an error was reported
      */
-    public int acknowledgeUpTo(long sequence) {
-        long sent = highestSent;
+    public void awaitEmpty() {
+        checkError();
 
-        // Nothing to acknowledge if window is empty or sequence is beyond what's sent
-        if (sent < 0) {
-            return 0; // No batches have been sent
+        // Fast path: already empty
+        if (getInFlightCount() == 0) {
+            LOG.debug("Window already empty");
+            return;
         }
 
-        // Cap sequence at highestSent - can't acknowledge what hasn't been sent
-        long effectiveSequence = Math.min(sequence, sent);
-
-        long prevAcked = highestAcked;
-        if (effectiveSequence <= prevAcked) {
-            // Already acknowledged up to this point
-            return 0;
-        }
-        highestAcked = effectiveSequence;
+        long deadline = System.currentTimeMillis() + timeoutMs;
+        int spins = 0;
 
-        int acknowledged = (int) (effectiveSequence - prevAcked);
-        totalAcked += acknowledged;
+        // Register as waiting thread
+        waitingForEmpty = Thread.currentThread();
+        try {
+            while (getInFlightCount() > 0) {
+                checkError();
 
-        LOG.debug("Cumulative ACK [upTo={}, acknowledged={}, remaining={}]", sequence, acknowledged, getInFlightCount());
+                long remaining = deadline - System.currentTimeMillis();
+                if (remaining <= 0) {
+                    throw new LineSenderException("Timeout waiting for batch acknowledgments, " +
+                            getInFlightCount() + " batches still in flight");
+                }
 
-        // Wake up waiting threads
-        Thread waiter = waitingForSpace;
-        if (waiter != null) {
-            LockSupport.unpark(waiter);
-        }
+                if (spins < SPIN_TRIES) {
+                    Thread.onSpinWait();
+                    spins++;
+                } else {
+                    LockSupport.parkNanos(Math.min(PARK_NANOS, remaining * 1_000_000));
+                    if (Thread.interrupted()) {
+                        throw new LineSenderException("Interrupted while waiting for acknowledgments");
+                    }
+                }
+            }
 
-        waiter = waitingForEmpty;
-        if (waiter != null && getInFlightCount() == 0) {
-            LockSupport.unpark(waiter);
+            LOG.debug("Window empty, all batches ACKed");
+        } finally {
+            waitingForEmpty = null;
         }
+    }
 
-        return acknowledged;
+    /**
+     * Clears the error state.
+     */
+    public void clearError() {
+        lastError.set(null);
+        failedBatchId = -1;
     }
 
     /**
      * Marks a batch as failed, setting an error that will be propagated to waiters.
-     *
+     * 

      * Called by: acker (WebSocket I/O thread) on error response or send failure.
+     *
      * @param batchId the batch ID that failed
      * @param error   the error that occurred
      */
@@ -324,55 +327,6 @@ public void failAll(Throwable error) {
         wakeWaiters();
     }
 
-    /**
-     * Waits until all in-flight batches are acknowledged.
-     * 

-     * Called by flush() to ensure all data is confirmed.
-     *
-     * Called by: waiter (flush thread), while producer/acker thread progresses.
-     * @throws LineSenderException if timeout occurs or an error was reported
-     */
-    public void awaitEmpty() {
-        checkError();
-
-        // Fast path: already empty
-        if (getInFlightCount() == 0) {
-            LOG.debug("Window already empty");
-            return;
-        }
-
-        long deadline = System.currentTimeMillis() + timeoutMs;
-        int spins = 0;
-
-        // Register as waiting thread
-        waitingForEmpty = Thread.currentThread();
-        try {
-            while (getInFlightCount() > 0) {
-                checkError();
-
-                long remaining = deadline - System.currentTimeMillis();
-                if (remaining <= 0) {
-                    throw new LineSenderException("Timeout waiting for batch acknowledgments, " +
-                            getInFlightCount() + " batches still in flight");
-                }
-
-                if (spins < SPIN_TRIES) {
-                    Thread.onSpinWait();
-                    spins++;
-                } else {
-                    LockSupport.parkNanos(Math.min(PARK_NANOS, remaining * 1_000_000));
-                    if (Thread.interrupted()) {
-                        throw new LineSenderException("Interrupted while waiting for acknowledgments");
-                    }
-                }
-            }
-
-            LOG.debug("Window empty, all batches ACKed");
-        } finally {
-            waitingForEmpty = null;
-        }
-    }
-
     /**
      * Returns the current number of batches in flight.
      * Wait-free operation.
@@ -385,19 +339,10 @@ public int getInFlightCount() {
     }
 
     /**
-     * Returns true if the window is empty.
-     * Wait-free operation.
-     */
-    public boolean isEmpty() {
-        return getInFlightCount() == 0;
-    }
-
-    /**
-     * Returns true if the window is full.
-     * Wait-free operation.
+     * Returns the last error, or null if no error.
      */
-    public boolean isFull() {
-        return getInFlightCount() >= maxWindowSize;
+    public Throwable getLastError() {
+        return lastError.get();
     }
 
     /**
@@ -422,18 +367,29 @@ public long getTotalFailed() {
     }
 
     /**
-     * Returns the last error, or null if no error.
+     * Checks if there's space in the window for another batch.
+     * Wait-free operation.
+     *
+     * @return true if there's space, false if window is full
      */
-    public Throwable getLastError() {
-        return lastError.get();
+    public boolean hasWindowSpace() {
+        return getInFlightCount() < maxWindowSize;
     }
 
     /**
-     * Clears the error state.
+     * Returns true if the window is empty.
+     * Wait-free operation.
      */
-    public void clearError() {
-        lastError.set(null);
-        failedBatchId = -1;
+    public boolean isEmpty() {
+        return getInFlightCount() == 0;
+    }
+
+    /**
+     * Returns true if the window is full.
+     * Wait-free operation.
+     */
+    public boolean isFull() {
+        return getInFlightCount() >= maxWindowSize;
     }
 
     /**
@@ -448,6 +404,31 @@ public void reset() {
         wakeWaiters();
     }
 
+    /**
+     * Tries to add a batch to the in-flight window without blocking.
+     * Lock-free, assuming single producer for highestSent.
+     * 

+     * Called by: async producer (WebSocket I/O thread) before sending a batch.
+     *
+     * @param batchId the batch ID to track (must be sequential)
+     * @return true if added, false if window is full
+     */
+    public boolean tryAddInFlight(long batchId) {
+        // Check window space first
+        long sent = highestSent;
+        long acked = highestAcked;
+
+        if (sent - acked >= maxWindowSize) {
+            return false;
+        }
+
+        // Sequential caller: just publish the new highestSent
+        highestSent = batchId;
+
+        LOG.debug("Added to window [batchId={}, windowSize={}]", batchId, getInFlightCount());
+        return true;
+    }
+
     private void checkError() {
         Throwable error = lastError.get();
         if (error != null) {
@@ -455,6 +436,22 @@ private void checkError() {
         }
     }
 
+    private boolean tryAddInFlightInternal(long batchId) {
+        long sent = highestSent;
+        long acked = highestAcked;
+
+        if (sent - acked >= maxWindowSize) {
+            return false;
+        }
+
+        // For sequential IDs, we just update highestSent
+        // The caller guarantees batchId is the next in sequence
+        highestSent = batchId;
+
+        LOG.debug("Added to window [batchId={}, windowSize={}]", batchId, getInFlightCount());
+        return true;
+    }
+
     private void wakeWaiters() {
         Thread waiter = waitingForSpace;
         if (waiter != null) {
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java
index 4ef2af4..e7efe6f 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java
@@ -57,37 +57,30 @@ public class MicrobatchBuffer implements QuietCloseable {
 
     // Buffer states
     public static final int STATE_FILLING = 0;
+    public static final int STATE_RECYCLED = 3;
     public static final int STATE_SEALED = 1;
     public static final int STATE_SENDING = 2;
-    public static final int STATE_RECYCLED = 3;
-
+    private static final AtomicLong nextBatchId = new AtomicLong();
+    private final long maxAgeNanos;
+    private final int maxBytes;
     // Flush trigger thresholds
     private final int maxRows;
-    private final int maxBytes;
-    private final long maxAgeNanos;
-
-    // Native memory buffer
-    private long bufferPtr;
+    // Batch identification
+    private long batchId;
     private int bufferCapacity;
     private int bufferPos;
-
-    // Row tracking
-    private int rowCount;
+    // Native memory buffer
+    private long bufferPtr;
     private long firstRowTimeNanos;
-
     // Symbol tracking for delta encoding
     private int maxSymbolId = -1;
-
-    // Batch identification
-    private long batchId;
-    private static final AtomicLong nextBatchId = new AtomicLong();
-
-    // State machine
-    private volatile int state = STATE_FILLING;
-
     // For waiting on recycle (user thread waits for I/O thread to finish)
     // CountDownLatch is not resettable, so we create a new instance on reset()
     private volatile CountDownLatch recycleLatch = new CountDownLatch(1);
+    // Row tracking
+    private int rowCount;
+    // State machine
+    private volatile int state = STATE_FILLING;
 
     /**
      * Creates a new MicrobatchBuffer with specified flush thresholds.
@@ -121,44 +114,59 @@ public MicrobatchBuffer(int initialCapacity) {
         this(initialCapacity, 0, 0, 0);
     }
 
-    // ==================== DATA OPERATIONS ====================
-
     /**
-     * Returns the buffer pointer for writing data.
-     * Only valid when state is FILLING.
-     */
-    public long getBufferPtr() {
-        return bufferPtr;
-    }
-
-    /**
-     * Returns the current write position in the buffer.
+     * Returns a human-readable name for the given state.
      */
-    public int getBufferPos() {
-        return bufferPos;
+    public static String stateName(int state) {
+        switch (state) {
+            case STATE_FILLING:
+                return "FILLING";
+            case STATE_SEALED:
+                return "SEALED";
+            case STATE_SENDING:
+                return "SENDING";
+            case STATE_RECYCLED:
+                return "RECYCLED";
+            default:
+                return "UNKNOWN(" + state + ")";
+        }
     }
 
     /**
-     * Returns the buffer capacity.
+     * Waits for the buffer to be recycled (transition to RECYCLED state).
+     * Only the user thread should call this.
      */
-    public int getBufferCapacity() {
-        return bufferCapacity;
+    public void awaitRecycled() {
+        try {
+            recycleLatch.await();
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+        }
     }
 
     /**
-     * Sets the buffer position after external writes.
-     * Only valid when state is FILLING.
+     * Waits for the buffer to be recycled with a timeout.
      *
-     * @param pos new position
+     * @param timeout the maximum time to wait
+     * @param unit    the time unit
+     * @return true if recycled, false if timeout elapsed
      */
-    public void setBufferPos(int pos) {
-        if (state != STATE_FILLING) {
-            throw new IllegalStateException("Cannot set position when state is " + stateName(state));
+    public boolean awaitRecycled(long timeout, TimeUnit unit) {
+        try {
+            return recycleLatch.await(timeout, unit);
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+            return false;
         }
-        if (pos < 0 || pos > bufferCapacity) {
-            throw new IllegalArgumentException("Position out of bounds: " + pos);
+    }
+
+    @Override
+    public void close() {
+        if (bufferPtr != 0) {
+            Unsafe.free(bufferPtr, bufferCapacity, MemoryTag.NATIVE_ILP_RSS);
+            bufferPtr = 0;
+            bufferCapacity = 0;
         }
-        this.bufferPos = pos;
     }
 
     /**
@@ -180,67 +188,42 @@ public void ensureCapacity(int requiredCapacity) {
     }
 
     /**
-     * Writes bytes to the buffer at the current position.
-     * Grows the buffer if necessary.
-     *
-     * @param src    source address
-     * @param length number of bytes to write
-     */
-    public void write(long src, int length) {
-        if (state != STATE_FILLING) {
-            throw new IllegalStateException("Cannot write when state is " + stateName(state));
-        }
-        ensureCapacity(bufferPos + length);
-        Unsafe.getUnsafe().copyMemory(src, bufferPtr + bufferPos, length);
-        bufferPos += length;
-    }
-
-    /**
-     * Writes a single byte to the buffer.
-     *
-     * @param b byte to write
+     * Returns the age of the first row in nanoseconds, or 0 if no rows.
      */
-    public void writeByte(byte b) {
-        if (state != STATE_FILLING) {
-            throw new IllegalStateException("Cannot write when state is " + stateName(state));
+    public long getAgeNanos() {
+        if (rowCount == 0) {
+            return 0;
         }
-        ensureCapacity(bufferPos + 1);
-        Unsafe.getUnsafe().putByte(bufferPtr + bufferPos, b);
-        bufferPos++;
+        return System.nanoTime() - firstRowTimeNanos;
     }
 
     /**
-     * Increments the row count and records the first row time if this is the first row.
+     * Returns the batch ID for this buffer.
      */
-    public void incrementRowCount() {
-        if (state != STATE_FILLING) {
-            throw new IllegalStateException("Cannot increment row count when state is " + stateName(state));
-        }
-        if (rowCount == 0) {
-            firstRowTimeNanos = System.nanoTime();
-        }
-        rowCount++;
+    public long getBatchId() {
+        return batchId;
     }
 
     /**
-     * Returns the number of rows in this buffer.
+     * Returns the buffer capacity.
      */
-    public int getRowCount() {
-        return rowCount;
+    public int getBufferCapacity() {
+        return bufferCapacity;
     }
 
     /**
-     * Returns true if the buffer has any data.
+     * Returns the current write position in the buffer.
      */
-    public boolean hasData() {
-        return bufferPos > 0;
+    public int getBufferPos() {
+        return bufferPos;
     }
 
     /**
-     * Returns the batch ID for this buffer.
+     * Returns the buffer pointer for writing data.
+     * Only valid when state is FILLING.
      */
-    public long getBatchId() {
-        return batchId;
+    public long getBufferPtr() {
+        return bufferPtr;
     }
 
     /**
@@ -252,39 +235,37 @@ public int getMaxSymbolId() {
     }
 
     /**
-     * Sets the maximum symbol ID used in this batch.
-     * Used for delta symbol dictionary tracking.
+     * Returns the number of rows in this buffer.
      */
-    public void setMaxSymbolId(int maxSymbolId) {
-        this.maxSymbolId = maxSymbolId;
+    public int getRowCount() {
+        return rowCount;
     }
 
-    // ==================== FLUSH TRIGGER CHECKS ====================
-
     /**
-     * Checks if the buffer should be flushed based on configured thresholds.
-     *
-     * @return true if any flush threshold is exceeded
+     * Returns the current state.
      */
-    public boolean shouldFlush() {
-        if (!hasData()) {
-            return false;
-        }
-        return isRowLimitExceeded() || isByteLimitExceeded() || isAgeLimitExceeded();
+    public int getState() {
+        return state;
     }
 
     /**
-     * Checks if the row count limit has been exceeded.
+     * Returns true if the buffer has any data.
      */
-    public boolean isRowLimitExceeded() {
-        return maxRows > 0 && rowCount >= maxRows;
+    public boolean hasData() {
+        return bufferPos > 0;
     }
 
     /**
-     * Checks if the byte size limit has been exceeded.
+     * Increments the row count and records the first row time if this is the first row.
      */
-    public boolean isByteLimitExceeded() {
-        return maxBytes > 0 && bufferPos >= maxBytes;
+    public void incrementRowCount() {
+        if (state != STATE_FILLING) {
+            throw new IllegalStateException("Cannot increment row count when state is " + stateName(state));
+        }
+        if (rowCount == 0) {
+            firstRowTimeNanos = System.nanoTime();
+        }
+        rowCount++;
     }
 
     /**
@@ -299,22 +280,10 @@ public boolean isAgeLimitExceeded() {
     }
 
     /**
-     * Returns the age of the first row in nanoseconds, or 0 if no rows.
-     */
-    public long getAgeNanos() {
-        if (rowCount == 0) {
-            return 0;
-        }
-        return System.nanoTime() - firstRowTimeNanos;
-    }
-
-    // ==================== STATE MACHINE ====================
-
-    /**
-     * Returns the current state.
+     * Checks if the byte size limit has been exceeded.
      */
-    public int getState() {
-        return state;
+    public boolean isByteLimitExceeded() {
+        return maxBytes > 0 && bufferPos >= maxBytes;
     }
 
     /**
@@ -325,17 +294,11 @@ public boolean isFilling() {
     }
 
     /**
-     * Returns true if the buffer is in SEALED state (ready to send).
-     */
-    public boolean isSealed() {
-        return state == STATE_SEALED;
-    }
-
-    /**
-     * Returns true if the buffer is in SENDING state (being sent by I/O thread).
+     * Returns true if the buffer is currently in use (not available for the user thread).
      */
-    public boolean isSending() {
-        return state == STATE_SENDING;
+    public boolean isInUse() {
+        int s = state;
+        return s == STATE_SEALED || s == STATE_SENDING;
     }
 
     /**
@@ -346,53 +309,24 @@ public boolean isRecycled() {
     }
 
     /**
-     * Returns true if the buffer is currently in use (not available for the user thread).
-     */
-    public boolean isInUse() {
-        int s = state;
-        return s == STATE_SEALED || s == STATE_SENDING;
-    }
-
-    /**
-     * Seals the buffer, transitioning from FILLING to SEALED.
-     * After sealing, no more data can be written.
-     * Only the user thread should call this.
-     *
-     * @throws IllegalStateException if not in FILLING state
+     * Checks if the row count limit has been exceeded.
      */
-    public void seal() {
-        if (state != STATE_FILLING) {
-            throw new IllegalStateException("Cannot seal buffer in state " + stateName(state));
-        }
-        state = STATE_SEALED;
+    public boolean isRowLimitExceeded() {
+        return maxRows > 0 && rowCount >= maxRows;
     }
 
     /**
-     * Rolls back a seal operation, transitioning from SEALED back to FILLING.
-     * 

-     * Used when enqueue fails after a buffer has been sealed but before ownership
-     * was transferred to the I/O thread.
-     *
-     * @throws IllegalStateException if not in SEALED state
+     * Returns true if the buffer is in SEALED state (ready to send).
      */
-    public void rollbackSealForRetry() {
-        if (state != STATE_SEALED) {
-            throw new IllegalStateException("Cannot rollback seal in state " + stateName(state));
-        }
-        state = STATE_FILLING;
+    public boolean isSealed() {
+        return state == STATE_SEALED;
     }
 
     /**
-     * Marks the buffer as being sent, transitioning from SEALED to SENDING.
-     * Only the I/O thread should call this.
-     *
-     * @throws IllegalStateException if not in SEALED state
+     * Returns true if the buffer is in SENDING state (being sent by I/O thread).
      */
-    public void markSending() {
-        if (state != STATE_SEALED) {
-            throw new IllegalStateException("Cannot mark sending in state " + stateName(state));
-        }
-        state = STATE_SENDING;
+    public boolean isSending() {
+        return state == STATE_SENDING;
     }
 
     /**
@@ -411,31 +345,16 @@ public void markRecycled() {
     }
 
     /**
-     * Waits for the buffer to be recycled (transition to RECYCLED state).
-     * Only the user thread should call this.
-     */
-    public void awaitRecycled() {
-        try {
-            recycleLatch.await();
-        } catch (InterruptedException e) {
-            Thread.currentThread().interrupt();
-        }
-    }
-
-    /**
-     * Waits for the buffer to be recycled with a timeout.
+     * Marks the buffer as being sent, transitioning from SEALED to SENDING.
+     * Only the I/O thread should call this.
      *
-     * @param timeout the maximum time to wait
-     * @param unit    the time unit
-     * @return true if recycled, false if timeout elapsed
+     * @throws IllegalStateException if not in SEALED state
      */
-    public boolean awaitRecycled(long timeout, TimeUnit unit) {
-        try {
-            return recycleLatch.await(timeout, unit);
-        } catch (InterruptedException e) {
-            Thread.currentThread().interrupt();
-            return false;
+    public void markSending() {
+        if (state != STATE_SEALED) {
+            throw new IllegalStateException("Cannot mark sending in state " + stateName(state));
         }
+        state = STATE_SENDING;
     }
 
     /**
@@ -458,35 +377,69 @@ public void reset() {
         recycleLatch = new CountDownLatch(1);
     }
 
-    // ==================== LIFECYCLE ====================
+    /**
+     * Rolls back a seal operation, transitioning from SEALED back to FILLING.
+     * 

+     * Used when enqueue fails after a buffer has been sealed but before ownership
+     * was transferred to the I/O thread.
+     *
+     * @throws IllegalStateException if not in SEALED state
+     */
+    public void rollbackSealForRetry() {
+        if (state != STATE_SEALED) {
+            throw new IllegalStateException("Cannot rollback seal in state " + stateName(state));
+        }
+        state = STATE_FILLING;
+    }
+
+    /**
+     * Seals the buffer, transitioning from FILLING to SEALED.
+     * After sealing, no more data can be written.
+     * Only the user thread should call this.
+     *
+     * @throws IllegalStateException if not in FILLING state
+     */
+    public void seal() {
+        if (state != STATE_FILLING) {
+            throw new IllegalStateException("Cannot seal buffer in state " + stateName(state));
+        }
+        state = STATE_SEALED;
+    }
 
-    @Override
-    public void close() {
-        if (bufferPtr != 0) {
-            Unsafe.free(bufferPtr, bufferCapacity, MemoryTag.NATIVE_ILP_RSS);
-            bufferPtr = 0;
-            bufferCapacity = 0;
+    /**
+     * Sets the buffer position after external writes.
+     * Only valid when state is FILLING.
+     *
+     * @param pos new position
+     */
+    public void setBufferPos(int pos) {
+        if (state != STATE_FILLING) {
+            throw new IllegalStateException("Cannot set position when state is " + stateName(state));
+        }
+        if (pos < 0 || pos > bufferCapacity) {
+            throw new IllegalArgumentException("Position out of bounds: " + pos);
         }
+        this.bufferPos = pos;
     }
 
-    // ==================== UTILITIES ====================
+    /**
+     * Sets the maximum symbol ID used in this batch.
+     * Used for delta symbol dictionary tracking.
+     */
+    public void setMaxSymbolId(int maxSymbolId) {
+        this.maxSymbolId = maxSymbolId;
+    }
 
     /**
-     * Returns a human-readable name for the given state.
+     * Checks if the buffer should be flushed based on configured thresholds.
+     *
+     * @return true if any flush threshold is exceeded
      */
-    public static String stateName(int state) {
-        switch (state) {
-            case STATE_FILLING:
-                return "FILLING";
-            case STATE_SEALED:
-                return "SEALED";
-            case STATE_SENDING:
-                return "SENDING";
-            case STATE_RECYCLED:
-                return "RECYCLED";
-            default:
-                return "UNKNOWN(" + state + ")";
+    public boolean shouldFlush() {
+        if (!hasData()) {
+            return false;
         }
+        return isRowLimitExceeded() || isByteLimitExceeded() || isAgeLimitExceeded();
     }
 
     @Override
@@ -499,4 +452,34 @@ public String toString() {
                 ", capacity=" + bufferCapacity +
                 '}';
     }
+
+    /**
+     * Writes bytes to the buffer at the current position.
+     * Grows the buffer if necessary.
+     *
+     * @param src    source address
+     * @param length number of bytes to write
+     */
+    public void write(long src, int length) {
+        if (state != STATE_FILLING) {
+            throw new IllegalStateException("Cannot write when state is " + stateName(state));
+        }
+        ensureCapacity(bufferPos + length);
+        Unsafe.getUnsafe().copyMemory(src, bufferPtr + bufferPos, length);
+        bufferPos += length;
+    }
+
+    /**
+     * Writes a single byte to the buffer.
+     *
+     * @param b byte to write
+     */
+    public void writeByte(byte b) {
+        if (state != STATE_FILLING) {
+            throw new IllegalStateException("Cannot write when state is " + stateName(state));
+        }
+        ensureCapacity(bufferPos + 1);
+        Unsafe.getUnsafe().putByte(bufferPtr + bufferPos, b);
+        bufferPos++;
+    }
 }
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java
index ee264ed..a11f70f 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java
@@ -54,6 +54,50 @@ public NativeBufferWriter(int initialCapacity) {
         this.position = 0;
     }
 
+    /**
+     * Returns the UTF-8 encoded length of a string.
+     */
+    public static int utf8Length(String s) {
+        if (s == null) return 0;
+        int len = 0;
+        for (int i = 0, n = s.length(); i < n; i++) {
+            char c = s.charAt(i);
+            if (c < 0x80) {
+                len++;
+            } else if (c < 0x800) {
+                len += 2;
+            } else if (c >= 0xD800 && c <= 0xDBFF && i + 1 < n) {
+                i++;
+                len += 4;
+            } else {
+                len += 3;
+            }
+        }
+        return len;
+    }
+
+    @Override
+    public void close() {
+        if (bufferPtr != 0) {
+            Unsafe.free(bufferPtr, capacity, MemoryTag.NATIVE_DEFAULT);
+            bufferPtr = 0;
+        }
+    }
+
+    /**
+     * Ensures the buffer has at least the specified additional capacity.
+     *
+     * @param needed additional bytes needed beyond current position
+     */
+    @Override
+    public void ensureCapacity(int needed) {
+        if (position + needed > capacity) {
+            int newCapacity = Math.max(capacity * 2, position + needed);
+            bufferPtr = Unsafe.realloc(bufferPtr, capacity, newCapacity, MemoryTag.NATIVE_DEFAULT);
+            capacity = newCapacity;
+        }
+    }
+
     /**
      * Returns the buffer pointer.
      */
@@ -62,6 +106,14 @@ public long getBufferPtr() {
         return bufferPtr;
     }
 
+    /**
+     * Returns the current buffer capacity.
+     */
+    @Override
+    public int getCapacity() {
+        return capacity;
+    }
+
     /**
      * Returns the current write position (number of bytes written).
      */
@@ -71,11 +123,22 @@ public int getPosition() {
     }
 
     /**
-     * Resets the buffer for reuse.
+     * Patches an int value at the specified offset.
+     * Used for updating length fields after writing content.
      */
     @Override
-    public void reset() {
-        position = 0;
+    public void patchInt(int offset, int value) {
+        Unsafe.getUnsafe().putInt(bufferPtr + offset, value);
+    }
+
+    /**
+     * Writes a block of bytes from native memory.
+     */
+    @Override
+    public void putBlockOfBytes(long from, long len) {
+        ensureCapacity((int) len);
+        Unsafe.getUnsafe().copyMemory(from, bufferPtr + position, len);
+        position += (int) len;
     }
 
     /**
@@ -89,13 +152,23 @@ public void putByte(byte value) {
     }
 
     /**
-     * Writes a short (2 bytes, little-endian).
+     * Writes a double (8 bytes, little-endian).
      */
     @Override
-    public void putShort(short value) {
-        ensureCapacity(2);
-        Unsafe.getUnsafe().putShort(bufferPtr + position, value);
-        position += 2;
+    public void putDouble(double value) {
+        ensureCapacity(8);
+        Unsafe.getUnsafe().putDouble(bufferPtr + position, value);
+        position += 8;
+    }
+
+    /**
+     * Writes a float (4 bytes, little-endian).
+     */
+    @Override
+    public void putFloat(float value) {
+        ensureCapacity(4);
+        Unsafe.getUnsafe().putFloat(bufferPtr + position, value);
+        position += 4;
     }
 
     /**
@@ -129,45 +202,13 @@ public void putLongBE(long value) {
     }
 
     /**
-     * Writes a float (4 bytes, little-endian).
-     */
-    @Override
-    public void putFloat(float value) {
-        ensureCapacity(4);
-        Unsafe.getUnsafe().putFloat(bufferPtr + position, value);
-        position += 4;
-    }
-
-    /**
-     * Writes a double (8 bytes, little-endian).
-     */
-    @Override
-    public void putDouble(double value) {
-        ensureCapacity(8);
-        Unsafe.getUnsafe().putDouble(bufferPtr + position, value);
-        position += 8;
-    }
-
-    /**
-     * Writes a block of bytes from native memory.
-     */
-    @Override
-    public void putBlockOfBytes(long from, long len) {
-        ensureCapacity((int) len);
-        Unsafe.getUnsafe().copyMemory(from, bufferPtr + position, len);
-        position += (int) len;
-    }
-
-    /**
-     * Writes a varint (unsigned LEB128).
+     * Writes a short (2 bytes, little-endian).
      */
     @Override
-    public void putVarint(long value) {
-        while (value > 0x7F) {
-            putByte((byte) ((value & 0x7F) | 0x80));
-            value >>>= 7;
-        }
-        putByte((byte) value);
+    public void putShort(short value) {
+        ensureCapacity(2);
+        Unsafe.getUnsafe().putShort(bufferPtr + position, value);
+        position += 2;
     }
 
     /**
@@ -216,42 +257,23 @@ public void putUtf8(String value) {
     }
 
     /**
-     * Returns the UTF-8 encoded length of a string.
-     */
-    public static int utf8Length(String s) {
-        if (s == null) return 0;
-        int len = 0;
-        for (int i = 0, n = s.length(); i < n; i++) {
-            char c = s.charAt(i);
-            if (c < 0x80) {
-                len++;
-            } else if (c < 0x800) {
-                len += 2;
-            } else if (c >= 0xD800 && c <= 0xDBFF && i + 1 < n) {
-                i++;
-                len += 4;
-            } else {
-                len += 3;
-            }
-        }
-        return len;
-    }
-
-    /**
-     * Patches an int value at the specified offset.
-     * Used for updating length fields after writing content.
+     * Writes a varint (unsigned LEB128).
      */
     @Override
-    public void patchInt(int offset, int value) {
-        Unsafe.getUnsafe().putInt(bufferPtr + offset, value);
+    public void putVarint(long value) {
+        while (value > 0x7F) {
+            putByte((byte) ((value & 0x7F) | 0x80));
+            value >>>= 7;
+        }
+        putByte((byte) value);
     }
 
     /**
-     * Returns the current buffer capacity.
+     * Resets the buffer for reuse.
      */
     @Override
-    public int getCapacity() {
-        return capacity;
+    public void reset() {
+        position = 0;
     }
 
     /**
@@ -264,26 +286,4 @@ public int getCapacity() {
     public void skip(int bytes) {
         position += bytes;
     }
-
-    /**
-     * Ensures the buffer has at least the specified additional capacity.
-     *
-     * @param needed additional bytes needed beyond current position
-     */
-    @Override
-    public void ensureCapacity(int needed) {
-        if (position + needed > capacity) {
-            int newCapacity = Math.max(capacity * 2, position + needed);
-            bufferPtr = Unsafe.realloc(bufferPtr, capacity, newCapacity, MemoryTag.NATIVE_DEFAULT);
-            capacity = newCapacity;
-        }
-    }
-
-    @Override
-    public void close() {
-        if (bufferPtr != 0) {
-            Unsafe.free(bufferPtr, capacity, MemoryTag.NATIVE_DEFAULT);
-            bufferPtr = 0;
-        }
-    }
 }
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpBufferWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpBufferWriter.java
index 05ef0ea..c50cdf6 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpBufferWriter.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpBufferWriter.java
@@ -44,54 +44,56 @@
  */
 public interface QwpBufferWriter extends ArrayBufferAppender {
 
-    // === Primitive writes (little-endian) ===
-
     /**
-     * Writes a short (2 bytes, little-endian).
-     */
-    void putShort(short value);
-
-    /**
-     * Writes a float (4 bytes, little-endian).
+     * Returns the UTF-8 encoded length of a string.
+     *
+     * @param s the string (may be null)
+     * @return the number of bytes needed to encode the string as UTF-8
      */
-    void putFloat(float value);
-
-    // === Big-endian writes ===
+    static int utf8Length(String s) {
+        if (s == null) return 0;
+        int len = 0;
+        for (int i = 0, n = s.length(); i < n; i++) {
+            char c = s.charAt(i);
+            if (c < 0x80) {
+                len++;
+            } else if (c < 0x800) {
+                len += 2;
+            } else if (c >= 0xD800 && c <= 0xDBFF && i + 1 < n) {
+                i++;
+                len += 4;
+            } else {
+                len += 3;
+            }
+        }
+        return len;
+    }
 
     /**
-     * Writes a long in big-endian byte order.
+     * Ensures the buffer has capacity for at least the specified
+     * additional bytes beyond the current position.
+     *
+     * @param additionalBytes number of additional bytes needed
      */
-    void putLongBE(long value);
-
-    // === Variable-length encoding ===
+    void ensureCapacity(int additionalBytes);
 
     /**
-     * Writes an unsigned variable-length integer (LEB128 encoding).
+     * Returns the native memory pointer to the buffer start.
      * 

-     * Each byte contains 7 bits of data with the high bit indicating
-     * whether more bytes follow.
+     * The returned pointer is valid until the next buffer growth operation.
+     * Use with care and only for reading completed data.
      */
-    void putVarint(long value);
-
-    // === String encoding ===
+    long getBufferPtr();
 
     /**
-     * Writes a length-prefixed UTF-8 string.
-     * 

-     * Format: varint length + UTF-8 bytes
-     *
-     * @param value the string to write (may be null or empty)
+     * Returns the current buffer capacity in bytes.
      */
-    void putString(String value);
+    int getCapacity();
 
     /**
-     * Writes UTF-8 encoded bytes directly without length prefix.
-     *
-     * @param value the string to encode (may be null or empty)
+     * Returns the current write position (number of bytes written).
      */
-    void putUtf8(String value);
-
-    // === Buffer manipulation ===
+    int getPosition();
 
     /**
      * Patches an int value at the specified offset in the buffer.
@@ -104,74 +106,58 @@ public interface QwpBufferWriter extends ArrayBufferAppender {
     void patchInt(int offset, int value);
 
     /**
-     * Skips the specified number of bytes, advancing the position.
-     * 

-     * Used when data has been written directly to the buffer via
-     * {@link #getBufferPtr()}.
-     *
-     * @param bytes number of bytes to skip
+     * Writes a float (4 bytes, little-endian).
      */
-    void skip(int bytes);
+    void putFloat(float value);
 
     /**
-     * Ensures the buffer has capacity for at least the specified
-     * additional bytes beyond the current position.
-     *
-     * @param additionalBytes number of additional bytes needed
+     * Writes a long in big-endian byte order.
      */
-    void ensureCapacity(int additionalBytes);
+    void putLongBE(long value);
 
     /**
-     * Resets the buffer for reuse, setting the position to 0.
-     * 

-     * Does not deallocate memory.
+     * Writes a short (2 bytes, little-endian).
      */
-    void reset();
-
-    // === Buffer state ===
+    void putShort(short value);
 
     /**
-     * Returns the current write position (number of bytes written).
+     * Writes a length-prefixed UTF-8 string.
+     * 

+     * Format: varint length + UTF-8 bytes
+     *
+     * @param value the string to write (may be null or empty)
      */
-    int getPosition();
+    void putString(String value);
 
     /**
-     * Returns the current buffer capacity in bytes.
+     * Writes UTF-8 encoded bytes directly without length prefix.
+     *
+     * @param value the string to encode (may be null or empty)
      */
-    int getCapacity();
+    void putUtf8(String value);
 
     /**
-     * Returns the native memory pointer to the buffer start.
+     * Writes an unsigned variable-length integer (LEB128 encoding).
      * 

-     * The returned pointer is valid until the next buffer growth operation.
-     * Use with care and only for reading completed data.
+     * Each byte contains 7 bits of data with the high bit indicating
+     * whether more bytes follow.
      */
-    long getBufferPtr();
+    void putVarint(long value);
 
-    // === Utility ===
+    /**
+     * Resets the buffer for reuse, setting the position to 0.
+     * 

+     * Does not deallocate memory.
+     */
+    void reset();
 
     /**
-     * Returns the UTF-8 encoded length of a string.
+     * Skips the specified number of bytes, advancing the position.
+     * 

+     * Used when data has been written directly to the buffer via
+     * {@link #getBufferPtr()}.
      *
-     * @param s the string (may be null)
-     * @return the number of bytes needed to encode the string as UTF-8
+     * @param bytes number of bytes to skip
      */
-    static int utf8Length(String s) {
-        if (s == null) return 0;
-        int len = 0;
-        for (int i = 0, n = s.length(); i < n; i++) {
-            char c = s.charAt(i);
-            if (c < 0x80) {
-                len++;
-            } else if (c < 0x800) {
-                len += 2;
-            } else if (c >= 0xD800 && c <= 0xDBFF && i + 1 < n) {
-                i++;
-                len += 4;
-            } else {
-                len += 3;
-            }
-        }
-        return len;
-    }
+    void skip(int bytes);
 }
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
index 4deb020..b2aa9a3 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
@@ -50,6 +50,7 @@
 
 import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.*;
 
+
 /**
  * ILP v4 WebSocket client sender for streaming data to QuestDB.
  * 

@@ -82,6 +83,25 @@
  *     sender.flush();
  * }
  *
+ *

+ * Fast-path API for high-throughput generators + *

+ * For maximum throughput, bypass the fluent API to avoid per-row overhead + * (no column-name hashmap lookups, no {@code checkNotClosed()}/{@code checkTableSelected()} + * per column, direct access to column buffers). Use {@link #getTableBuffer(String)}, + * {@link #getOrAddGlobalSymbol(String)}, and {@link #incrementPendingRowCount()}: + * 

+ * // Setup (once)
+ * QwpTableBuffer tableBuffer = sender.getTableBuffer("q");
+ * QwpTableBuffer.ColumnBuffer colSymbol = tableBuffer.getOrCreateColumn("s", TYPE_SYMBOL, true);
+ * QwpTableBuffer.ColumnBuffer colBid = tableBuffer.getOrCreateColumn("b", TYPE_DOUBLE, false);
+ *
+ * // Hot path (per row)
+ * colSymbol.addSymbolWithGlobalId(symbol, sender.getOrAddGlobalSymbol(symbol));
+ * colBid.addDouble(bid);
+ * tableBuffer.nextRow();
+ * sender.incrementPendingRowCount();
+ *
*/ public class QwpWebSocketSender implements Sender { @@ -219,8 +239,8 @@ public static QwpWebSocketSender connect(String host, int port, boolean tlsEnabl * @return connected sender */ public static QwpWebSocketSender connect(String host, int port, boolean tlsEnabled, - int autoFlushRows, int autoFlushBytes, - long autoFlushIntervalNanos) { + int autoFlushRows, int autoFlushBytes, + long autoFlushIntervalNanos) { QwpWebSocketSender sender = new QwpWebSocketSender( host, port, tlsEnabled, DEFAULT_BUFFER_SIZE, autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, @@ -518,25 +538,6 @@ public Sender decimalColumn(CharSequence name, Decimal128 value) { return this; } - // ==================== Fast-path API for high-throughput generators ==================== - // - // These methods bypass the normal fluent API to avoid per-row overhead: - // - No hashmap lookups for column names - // - No checkNotClosed()/checkTableSelected() per column - // - Direct access to column buffers - // - // Usage: - // // Setup (once) - // QwpTableBuffer tableBuffer = sender.getTableBuffer("q"); - // QwpTableBuffer.ColumnBuffer colSymbol = tableBuffer.getOrCreateColumn("s", TYPE_SYMBOL, true); - // QwpTableBuffer.ColumnBuffer colBid = tableBuffer.getOrCreateColumn("b", TYPE_DOUBLE, false); - // - // // Hot path (per row) - // colSymbol.addSymbolWithGlobalId(symbol, sender.getOrAddGlobalSymbol(symbol)); - // colBid.addDouble(bid); - // tableBuffer.nextRow(); - // sender.incrementPendingRowCount(); - @Override public Sender decimalColumn(CharSequence name, Decimal256 value) { if (value == null || value.isNull()) return this; @@ -573,8 +574,6 @@ public Sender doubleArray(@NotNull CharSequence name, double[] values) { return this; } - // ==================== Sender interface implementation ==================== - @Override public Sender doubleArray(@NotNull CharSequence name, double[][] values) { if (values == null) return this; @@ -614,14 +613,6 @@ public QwpWebSocketSender doubleColumn(CharSequence columnName, double value) { return this; } - /** - * Adds an INT column value to the current row. - * - * @param columnName the column name - * @param value the int value - * @return this sender for method chaining - */ - /** * Adds a FLOAT column value to the current row. * @@ -765,6 +756,14 @@ public void incrementPendingRowCount() { } } + + /** + * Adds an INT column value to the current row. + * + * @param columnName the column name + * @param value the int value + * @return this sender for method chaining + */ public QwpWebSocketSender intColumn(CharSequence columnName, int value) { checkNotClosed(); checkTableSelected(); @@ -961,8 +960,6 @@ public QwpWebSocketSender timestampColumn(CharSequence columnName, Instant value return this; } - // ==================== Array methods ==================== - /** * Adds a UUID column value to the current row. * @@ -1107,8 +1104,6 @@ private void ensureConnected() { } } - // ==================== Decimal methods ==================== - private void failExpectedIfNeeded(long expectedSequence, LineSenderException error) { if (inFlightWindow != null && inFlightWindow.getLastError() == null) { inFlightWindow.fail(expectedSequence, error); @@ -1318,8 +1313,6 @@ private void sealAndSwapBuffer() { } } - // ==================== Helper methods ==================== - /** * Accumulates the current row. * Both sync and async modes buffer rows until flush (explicit or auto-flush). diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/ResponseReader.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/ResponseReader.java index 1d2b89f..552e372 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/ResponseReader.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/ResponseReader.java @@ -25,9 +25,9 @@ package io.questdb.client.cutlass.qwp.client; import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.std.QuietCloseable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import io.questdb.client.std.QuietCloseable; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -48,24 +48,20 @@ */ public class ResponseReader implements QuietCloseable { - private static final Logger LOG = LoggerFactory.getLogger(ResponseReader.class); - private static final int DEFAULT_READ_TIMEOUT_MS = 100; private static final long DEFAULT_SHUTDOWN_TIMEOUT_MS = 5_000; - + private static final Logger LOG = LoggerFactory.getLogger(ResponseReader.class); private final WebSocketChannel channel; private final InFlightWindow inFlightWindow; private final Thread readerThread; - private final CountDownLatch shutdownLatch; private final WebSocketResponse response; - - // State - private volatile boolean running; - private volatile Throwable lastError; - + private final CountDownLatch shutdownLatch; // Statistics private final AtomicLong totalAcks = new AtomicLong(0); private final AtomicLong totalErrors = new AtomicLong(0); + private volatile Throwable lastError; + // State + private volatile boolean running; /** * Creates a new response reader. @@ -96,6 +92,26 @@ public ResponseReader(WebSocketChannel channel, InFlightWindow inFlightWindow) { LOG.info("Response reader started"); } + @Override + public void close() { + if (!running) { + return; + } + + LOG.info("Closing response reader"); + + running = false; + + // Wait for reader thread to finish + try { + shutdownLatch.await(DEFAULT_SHUTDOWN_TIMEOUT_MS, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + + LOG.info("Response reader closed [totalAcks={}, totalErrors={}]", totalAcks.get(), totalErrors.get()); + } + /** * Returns the last error that occurred, or null if no error. */ @@ -103,13 +119,6 @@ public Throwable getLastError() { return lastError; } - /** - * Returns true if the reader is still running. - */ - public boolean isRunning() { - return running; - } - /** * Returns total successful acknowledgments received. */ @@ -124,28 +133,13 @@ public long getTotalErrors() { return totalErrors.get(); } - @Override - public void close() { - if (!running) { - return; - } - - LOG.info("Closing response reader"); - - running = false; - - // Wait for reader thread to finish - try { - shutdownLatch.await(DEFAULT_SHUTDOWN_TIMEOUT_MS, TimeUnit.MILLISECONDS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - - LOG.info("Response reader closed [totalAcks={}, totalErrors={}]", totalAcks.get(), totalErrors.get()); + /** + * Returns true if the reader is still running. + */ + public boolean isRunning() { + return running; } - // ==================== Reader Thread ==================== - /** * Main read loop that processes incoming WebSocket frames. */ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketChannel.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketChannel.java index 415ee4b..da64ca9 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketChannel.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketChannel.java @@ -24,12 +24,12 @@ package io.questdb.client.cutlass.qwp.client; +import io.questdb.client.cutlass.line.LineSenderException; import io.questdb.client.cutlass.qwp.websocket.WebSocketCloseCode; import io.questdb.client.cutlass.qwp.websocket.WebSocketFrameParser; import io.questdb.client.cutlass.qwp.websocket.WebSocketFrameWriter; import io.questdb.client.cutlass.qwp.websocket.WebSocketHandshake; import io.questdb.client.cutlass.qwp.websocket.WebSocketOpcode; -import io.questdb.client.cutlass.line.LineSenderException; import io.questdb.client.std.MemoryTag; import io.questdb.client.std.QuietCloseable; import io.questdb.client.std.Rnd; @@ -68,45 +68,40 @@ public class WebSocketChannel implements QuietCloseable { private static final int DEFAULT_BUFFER_SIZE = 65536; private static final int MAX_FRAME_HEADER_SIZE = 14; // 2 + 8 + 4 (header + extended len + mask) - + // Frame parser (reused) + private final WebSocketFrameParser frameParser; + // Temporary byte array for handshake (allocated once) + private final byte[] handshakeBuffer = new byte[4096]; // Connection state private final String host; - private final int port; private final String path; + private final int port; + // Random for mask key generation + private final Rnd rnd; private final boolean tlsEnabled; private final boolean tlsValidationEnabled; - - // Socket I/O - private Socket socket; + private boolean closed; + // Timeouts + private int connectTimeoutMs = 10_000; + // State + private boolean connected; private InputStream in; private OutputStream out; - - // Pre-allocated send buffer (native memory) - private long sendBufferPtr; - private int sendBufferSize; - + private byte[] readTempBuffer; + private int readTimeoutMs = 30_000; + private int recvBufferPos; // Write position // Pre-allocated receive buffer (native memory) private long recvBufferPtr; - private int recvBufferSize; - private int recvBufferPos; // Write position private int recvBufferReadPos; // Read position - - // Frame parser (reused) - private final WebSocketFrameParser frameParser; - - // Random for mask key generation - private final Rnd rnd; - - // Timeouts - private int connectTimeoutMs = 10_000; - private int readTimeoutMs = 30_000; - - // State - private boolean connected; - private boolean closed; - - // Temporary byte array for handshake (allocated once) - private final byte[] handshakeBuffer = new byte[4096]; + private int recvBufferSize; + // Pre-allocated send buffer (native memory) + private long sendBufferPtr; + private int sendBufferSize; + // Socket I/O + private Socket socket; + // Separate temp buffers for read and write to avoid race conditions + // between send queue thread and response reader thread + private byte[] writeTempBuffer; public WebSocketChannel(String url, boolean tlsEnabled) { this(url, tlsEnabled, true); @@ -162,19 +157,35 @@ public WebSocketChannel(String url, boolean tlsEnabled, boolean tlsValidationEna } /** - * Sets the connection timeout. + * Sends a close frame and closes the connection. */ - public WebSocketChannel setConnectTimeout(int timeoutMs) { - this.connectTimeoutMs = timeoutMs; - return this; - } + @Override + public void close() { + if (closed) { + return; + } + closed = true; - /** - * Sets the read timeout. - */ - public WebSocketChannel setReadTimeout(int timeoutMs) { - this.readTimeoutMs = timeoutMs; - return this; + try { + if (connected) { + // Send close frame + sendCloseFrame(WebSocketCloseCode.NORMAL_CLOSURE, null); + } + } catch (Exception e) { + // Ignore errors during close + } + + closeQuietly(); + + // Free native memory + if (sendBufferPtr != 0) { + Unsafe.free(sendBufferPtr, sendBufferSize, MemoryTag.NATIVE_DEFAULT); + sendBufferPtr = 0; + } + if (recvBufferPtr != 0) { + Unsafe.free(recvBufferPtr, recvBufferSize, MemoryTag.NATIVE_DEFAULT); + recvBufferPtr = 0; + } } /** @@ -210,31 +221,15 @@ public void connect() { } } - /** - * Sends binary data as a WebSocket binary frame. - * The data is read from native memory at the given pointer. - * - * @param dataPtr pointer to the data - * @param length length of data in bytes - */ - public void sendBinary(long dataPtr, int length) { - ensureConnected(); - sendFrame(WebSocketOpcode.BINARY, dataPtr, length); - } - - /** - * Sends a ping frame. - */ - public void sendPing() { - ensureConnected(); - sendFrame(WebSocketOpcode.PING, 0, 0); + public boolean isConnected() { + return connected && !closed; } /** * Receives and processes incoming frames. * Handles ping/pong automatically. * - * @param handler callback for received binary messages (may be null) + * @param handler callback for received binary messages (may be null) * @param timeoutMs read timeout in milliseconds * @return true if a frame was received, false on timeout */ @@ -256,43 +251,100 @@ public boolean receiveFrame(ResponseHandler handler, int timeoutMs) { } /** - * Sends a close frame and closes the connection. + * Sends binary data as a WebSocket binary frame. + * The data is read from native memory at the given pointer. + * + * @param dataPtr pointer to the data + * @param length length of data in bytes */ - @Override - public void close() { - if (closed) { - return; + public void sendBinary(long dataPtr, int length) { + ensureConnected(); + sendFrame(WebSocketOpcode.BINARY, dataPtr, length); + } + + /** + * Sends a ping frame. + */ + public void sendPing() { + ensureConnected(); + sendFrame(WebSocketOpcode.PING, 0, 0); + } + + /** + * Sets the connection timeout. + */ + public WebSocketChannel setConnectTimeout(int timeoutMs) { + this.connectTimeoutMs = timeoutMs; + return this; + } + + /** + * Sets the read timeout. + */ + public WebSocketChannel setReadTimeout(int timeoutMs) { + this.readTimeoutMs = timeoutMs; + return this; + } + + private void closeQuietly() { + connected = false; + if (socket != null) { + try { + socket.close(); + } catch (IOException e) { + // Ignore + } + socket = null; } - closed = true; + in = null; + out = null; + } + private SocketFactory createSslSocketFactory() { try { - if (connected) { - // Send close frame - sendCloseFrame(WebSocketCloseCode.NORMAL_CLOSURE, null); + if (!tlsValidationEnabled) { + SSLContext sslContext = SSLContext.getInstance("TLS"); + sslContext.init(null, new TrustManager[]{new X509TrustManager() { + public void checkClientTrusted(X509Certificate[] certs, String t) { + } + + public void checkServerTrusted(X509Certificate[] certs, String t) { + } + + public X509Certificate[] getAcceptedIssuers() { + return null; + } + }}, new SecureRandom()); + return sslContext.getSocketFactory(); } + return SSLSocketFactory.getDefault(); } catch (Exception e) { - // Ignore errors during close + throw new LineSenderException("Failed to create SSL socket factory: " + e.getMessage(), e); } + } - closeQuietly(); - - // Free native memory - if (sendBufferPtr != 0) { - Unsafe.free(sendBufferPtr, sendBufferSize, MemoryTag.NATIVE_DEFAULT); - sendBufferPtr = 0; + private boolean doReceiveFrame(ResponseHandler handler) throws IOException { + // First, try to parse any data already in the buffer + // This handles the case where multiple frames arrived in a single TCP read + if (recvBufferPos > recvBufferReadPos) { + Boolean result = tryParseFrame(handler); + if (result != null) { + return result; + } + // result == null means we need more data, continue to read } - if (recvBufferPtr != 0) { - Unsafe.free(recvBufferPtr, recvBufferSize, MemoryTag.NATIVE_DEFAULT); - recvBufferPtr = 0; + + // Read more data into receive buffer + int bytesRead = readFromSocket(); + if (bytesRead <= 0) { + return false; } - } - public boolean isConnected() { - return connected && !closed; + // Try parsing again with the new data + Boolean result = tryParseFrame(handler); + return result != null && result; } - // ==================== Private methods ==================== - private void ensureConnected() { if (closed) { throw new LineSenderException("WebSocket channel is closed"); @@ -302,21 +354,26 @@ private void ensureConnected() { } } - private SocketFactory createSslSocketFactory() { - try { - if (!tlsValidationEnabled) { - SSLContext sslContext = SSLContext.getInstance("TLS"); - sslContext.init(null, new TrustManager[]{new X509TrustManager() { - public void checkClientTrusted(X509Certificate[] certs, String t) {} - public void checkServerTrusted(X509Certificate[] certs, String t) {} - public X509Certificate[] getAcceptedIssuers() { return null; } - }}, new SecureRandom()); - return sslContext.getSocketFactory(); - } - return SSLSocketFactory.getDefault(); - } catch (Exception e) { - throw new LineSenderException("Failed to create SSL socket factory: " + e.getMessage(), e); + private void ensureSendBufferSize(int required) { + if (required > sendBufferSize) { + int newSize = Math.max(required, sendBufferSize * 2); + sendBufferPtr = Unsafe.realloc(sendBufferPtr, sendBufferSize, newSize, MemoryTag.NATIVE_DEFAULT); + sendBufferSize = newSize; + } + } + + private byte[] getReadTempBuffer(int minSize) { + if (readTempBuffer == null || readTempBuffer.length < minSize) { + readTempBuffer = new byte[Math.max(minSize, 8192)]; + } + return readTempBuffer; + } + + private byte[] getWriteTempBuffer(int minSize) { + if (writeTempBuffer == null || writeTempBuffer.length < minSize) { + writeTempBuffer = new byte[Math.max(minSize, 8192)]; } + return writeTempBuffer; } private void performHandshake() throws IOException { @@ -364,6 +421,28 @@ private void performHandshake() throws IOException { } } + private int readFromSocket() throws IOException { + // Ensure space in receive buffer + int available = recvBufferSize - recvBufferPos; + if (available < 1024) { + // Grow buffer + int newSize = recvBufferSize * 2; + recvBufferPtr = Unsafe.realloc(recvBufferPtr, recvBufferSize, newSize, MemoryTag.NATIVE_DEFAULT); + recvBufferSize = newSize; + available = recvBufferSize - recvBufferPos; + } + + // Read into temp array then copy to native buffer + // Use separate read buffer to avoid race with write thread + byte[] temp = getReadTempBuffer(available); + int bytesRead = in.read(temp, 0, available); + if (bytesRead > 0) { + Unsafe.getUnsafe().copyMemory(temp, Unsafe.BYTE_OFFSET, null, recvBufferPtr + recvBufferPos, bytesRead); + recvBufferPos += bytesRead; + } + return bytesRead; + } + private int readHttpResponse() throws IOException { int pos = 0; int consecutiveCrLf = 0; @@ -378,9 +457,9 @@ private int readHttpResponse() throws IOException { // Look for \r\n\r\n if (b == '\r' || b == '\n') { if ((consecutiveCrLf == 0 && b == '\r') || - (consecutiveCrLf == 1 && b == '\n') || - (consecutiveCrLf == 2 && b == '\r') || - (consecutiveCrLf == 3 && b == '\n')) { + (consecutiveCrLf == 1 && b == '\n') || + (consecutiveCrLf == 2 && b == '\r') || + (consecutiveCrLf == 3 && b == '\n')) { consecutiveCrLf++; if (consecutiveCrLf == 4) { return pos; @@ -395,36 +474,6 @@ private int readHttpResponse() throws IOException { throw new IOException("HTTP response too large"); } - private void sendFrame(int opcode, long payloadPtr, int payloadLen) { - // Generate mask key - int maskKey = rnd.nextInt(); - - // Calculate required buffer size - int headerSize = WebSocketFrameWriter.headerSize(payloadLen, true); - int frameSize = headerSize + payloadLen; - - // Ensure buffer is large enough - ensureSendBufferSize(frameSize); - - // Write frame header with mask - int headerWritten = WebSocketFrameWriter.writeHeader( - sendBufferPtr, true, opcode, payloadLen, maskKey); - - // Copy payload to buffer after header - if (payloadLen > 0) { - Unsafe.getUnsafe().copyMemory(payloadPtr, sendBufferPtr + headerWritten, payloadLen); - // Mask the payload in place - WebSocketFrameWriter.maskPayload(sendBufferPtr + headerWritten, payloadLen, maskKey); - } - - // Send frame - try { - writeToSocket(sendBufferPtr, frameSize); - } catch (IOException e) { - throw new LineSenderException("Failed to send WebSocket frame: " + e.getMessage(), e); - } - } - private void sendCloseFrame(int code, String reason) { int maskKey = rnd.nextInt(); @@ -465,30 +514,61 @@ private void sendCloseFrame(int code, String reason) { } } - private boolean doReceiveFrame(ResponseHandler handler) throws IOException { - // First, try to parse any data already in the buffer - // This handles the case where multiple frames arrived in a single TCP read - if (recvBufferPos > recvBufferReadPos) { - Boolean result = tryParseFrame(handler); - if (result != null) { - return result; - } - // result == null means we need more data, continue to read + private void sendFrame(int opcode, long payloadPtr, int payloadLen) { + // Generate mask key + int maskKey = rnd.nextInt(); + + // Calculate required buffer size + int headerSize = WebSocketFrameWriter.headerSize(payloadLen, true); + int frameSize = headerSize + payloadLen; + + // Ensure buffer is large enough + ensureSendBufferSize(frameSize); + + // Write frame header with mask + int headerWritten = WebSocketFrameWriter.writeHeader( + sendBufferPtr, true, opcode, payloadLen, maskKey); + + // Copy payload to buffer after header + if (payloadLen > 0) { + Unsafe.getUnsafe().copyMemory(payloadPtr, sendBufferPtr + headerWritten, payloadLen); + // Mask the payload in place + WebSocketFrameWriter.maskPayload(sendBufferPtr + headerWritten, payloadLen, maskKey); } - // Read more data into receive buffer - int bytesRead = readFromSocket(); - if (bytesRead <= 0) { - return false; + // Send frame + try { + writeToSocket(sendBufferPtr, frameSize); + } catch (IOException e) { + throw new LineSenderException("Failed to send WebSocket frame: " + e.getMessage(), e); } + } - // Try parsing again with the new data - Boolean result = tryParseFrame(handler); - return result != null && result; + private void sendPongFrame(long pingPayloadPtr, int pingPayloadLen) { + int maskKey = rnd.nextInt(); + int headerSize = WebSocketFrameWriter.headerSize(pingPayloadLen, true); + int frameSize = headerSize + pingPayloadLen; + + ensureSendBufferSize(frameSize); + + int headerWritten = WebSocketFrameWriter.writeHeader( + sendBufferPtr, true, WebSocketOpcode.PONG, pingPayloadLen, maskKey); + + if (pingPayloadLen > 0) { + Unsafe.getUnsafe().copyMemory(pingPayloadPtr, sendBufferPtr + headerWritten, pingPayloadLen); + WebSocketFrameWriter.maskPayload(sendBufferPtr + headerWritten, pingPayloadLen, maskKey); + } + + try { + writeToSocket(sendBufferPtr, frameSize); + } catch (IOException e) { + // Ignore pong send errors + } } /** * Tries to parse a frame from the receive buffer. + * * @return true if frame processed, false if error, null if need more data */ private Boolean tryParseFrame(ResponseHandler handler) throws IOException { @@ -561,36 +641,6 @@ private Boolean tryParseFrame(ResponseHandler handler) throws IOException { return false; } - private void sendPongFrame(long pingPayloadPtr, int pingPayloadLen) { - int maskKey = rnd.nextInt(); - int headerSize = WebSocketFrameWriter.headerSize(pingPayloadLen, true); - int frameSize = headerSize + pingPayloadLen; - - ensureSendBufferSize(frameSize); - - int headerWritten = WebSocketFrameWriter.writeHeader( - sendBufferPtr, true, WebSocketOpcode.PONG, pingPayloadLen, maskKey); - - if (pingPayloadLen > 0) { - Unsafe.getUnsafe().copyMemory(pingPayloadPtr, sendBufferPtr + headerWritten, pingPayloadLen); - WebSocketFrameWriter.maskPayload(sendBufferPtr + headerWritten, pingPayloadLen, maskKey); - } - - try { - writeToSocket(sendBufferPtr, frameSize); - } catch (IOException e) { - // Ignore pong send errors - } - } - - private void ensureSendBufferSize(int required) { - if (required > sendBufferSize) { - int newSize = Math.max(required, sendBufferSize * 2); - sendBufferPtr = Unsafe.realloc(sendBufferPtr, sendBufferSize, newSize, MemoryTag.NATIVE_DEFAULT); - sendBufferSize = newSize; - } - } - private void writeToSocket(long ptr, int len) throws IOException { // Copy to temp array for socket write (unavoidable with OutputStream) // Use separate write buffer to avoid race with read thread @@ -600,66 +650,12 @@ private void writeToSocket(long ptr, int len) throws IOException { out.flush(); } - private int readFromSocket() throws IOException { - // Ensure space in receive buffer - int available = recvBufferSize - recvBufferPos; - if (available < 1024) { - // Grow buffer - int newSize = recvBufferSize * 2; - recvBufferPtr = Unsafe.realloc(recvBufferPtr, recvBufferSize, newSize, MemoryTag.NATIVE_DEFAULT); - recvBufferSize = newSize; - available = recvBufferSize - recvBufferPos; - } - - // Read into temp array then copy to native buffer - // Use separate read buffer to avoid race with write thread - byte[] temp = getReadTempBuffer(available); - int bytesRead = in.read(temp, 0, available); - if (bytesRead > 0) { - Unsafe.getUnsafe().copyMemory(temp, Unsafe.BYTE_OFFSET, null, recvBufferPtr + recvBufferPos, bytesRead); - recvBufferPos += bytesRead; - } - return bytesRead; - } - - // Separate temp buffers for read and write to avoid race conditions - // between send queue thread and response reader thread - private byte[] writeTempBuffer; - private byte[] readTempBuffer; - - private byte[] getWriteTempBuffer(int minSize) { - if (writeTempBuffer == null || writeTempBuffer.length < minSize) { - writeTempBuffer = new byte[Math.max(minSize, 8192)]; - } - return writeTempBuffer; - } - - private byte[] getReadTempBuffer(int minSize) { - if (readTempBuffer == null || readTempBuffer.length < minSize) { - readTempBuffer = new byte[Math.max(minSize, 8192)]; - } - return readTempBuffer; - } - - private void closeQuietly() { - connected = false; - if (socket != null) { - try { - socket.close(); - } catch (IOException e) { - // Ignore - } - socket = null; - } - in = null; - out = null; - } - /** * Callback interface for received WebSocket messages. */ public interface ResponseHandler { void onBinaryMessage(long payload, int length); + void onClose(int code, String reason); } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java index 35a5f77..e1c1e6b 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java @@ -54,22 +54,20 @@ */ public class WebSocketResponse { + public static final int MAX_ERROR_MESSAGE_LENGTH = 1024; + public static final int MIN_ERROR_RESPONSE_SIZE = 11; // status + sequence + error length + // Minimum response size: status (1) + sequence (8) + public static final int MIN_RESPONSE_SIZE = 9; + public static final byte STATUS_INTERNAL_ERROR = (byte) 255; // Status codes public static final byte STATUS_OK = 0; public static final byte STATUS_PARSE_ERROR = 1; public static final byte STATUS_SCHEMA_ERROR = 2; - public static final byte STATUS_WRITE_ERROR = 3; public static final byte STATUS_SECURITY_ERROR = 4; - public static final byte STATUS_INTERNAL_ERROR = (byte) 255; - - // Minimum response size: status (1) + sequence (8) - public static final int MIN_RESPONSE_SIZE = 9; - public static final int MIN_ERROR_RESPONSE_SIZE = 11; // status + sequence + error length - public static final int MAX_ERROR_MESSAGE_LENGTH = 1024; - - private byte status; - private long sequence; + public static final byte STATUS_WRITE_ERROR = 3; private String errorMessage; + private long sequence; + private byte status; public WebSocketResponse() { this.status = STATUS_OK; @@ -77,16 +75,6 @@ public WebSocketResponse() { this.errorMessage = null; } - /** - * Creates a success response. - */ - public static WebSocketResponse success(long sequence) { - WebSocketResponse response = new WebSocketResponse(); - response.status = STATUS_OK; - response.sequence = sequence; - return response; - } - /** * Creates an error response. */ @@ -130,17 +118,20 @@ public static boolean isStructurallyValid(long ptr, int length) { } /** - * Returns true if this is a success response. + * Creates a success response. */ - public boolean isSuccess() { - return status == STATUS_OK; + public static WebSocketResponse success(long sequence) { + WebSocketResponse response = new WebSocketResponse(); + response.status = STATUS_OK; + response.sequence = sequence; + return response; } /** - * Returns the status code. + * Returns the error message, or null for success responses. */ - public byte getStatus() { - return status; + public String getErrorMessage() { + return errorMessage; } /** @@ -151,10 +142,10 @@ public long getSequence() { } /** - * Returns the error message, or null for success responses. + * Returns the status code. */ - public String getErrorMessage() { - return errorMessage; + public byte getStatus() { + return status; } /** @@ -180,52 +171,10 @@ public String getStatusName() { } /** - * Calculates the serialized size of this response. - */ - public int serializedSize() { - int size = MIN_RESPONSE_SIZE; - if (errorMessage != null && !errorMessage.isEmpty()) { - byte[] msgBytes = errorMessage.getBytes(StandardCharsets.UTF_8); - int msgLen = Math.min(msgBytes.length, MAX_ERROR_MESSAGE_LENGTH); - size += 2 + msgLen; // 2 bytes for length prefix - } - return size; - } - - /** - * Writes this response to native memory. - * - * @param ptr destination address - * @return number of bytes written + * Returns true if this is a success response. */ - public int writeTo(long ptr) { - int offset = 0; - - // Status (1 byte) - Unsafe.getUnsafe().putByte(ptr + offset, status); - offset += 1; - - // Sequence (8 bytes, little-endian) - Unsafe.getUnsafe().putLong(ptr + offset, sequence); - offset += 8; - - // Error message (if any) - if (status != STATUS_OK && errorMessage != null && !errorMessage.isEmpty()) { - byte[] msgBytes = errorMessage.getBytes(StandardCharsets.UTF_8); - int msgLen = Math.min(msgBytes.length, MAX_ERROR_MESSAGE_LENGTH); - - // Length prefix (2 bytes, little-endian) - Unsafe.getUnsafe().putShort(ptr + offset, (short) msgLen); - offset += 2; - - // Message bytes - for (int i = 0; i < msgLen; i++) { - Unsafe.getUnsafe().putByte(ptr + offset + i, msgBytes[i]); - } - offset += msgLen; - } - - return offset; + public boolean isSuccess() { + return status == STATUS_OK; } /** @@ -270,6 +219,19 @@ public boolean readFrom(long ptr, int length) { return true; } + /** + * Calculates the serialized size of this response. + */ + public int serializedSize() { + int size = MIN_RESPONSE_SIZE; + if (errorMessage != null && !errorMessage.isEmpty()) { + byte[] msgBytes = errorMessage.getBytes(StandardCharsets.UTF_8); + int msgLen = Math.min(msgBytes.length, MAX_ERROR_MESSAGE_LENGTH); + size += 2 + msgLen; // 2 bytes for length prefix + } + return size; + } + @Override public String toString() { if (isSuccess()) { @@ -279,4 +241,40 @@ public String toString() { ", error=" + errorMessage + "}"; } } + + /** + * Writes this response to native memory. + * + * @param ptr destination address + * @return number of bytes written + */ + public int writeTo(long ptr) { + int offset = 0; + + // Status (1 byte) + Unsafe.getUnsafe().putByte(ptr + offset, status); + offset += 1; + + // Sequence (8 bytes, little-endian) + Unsafe.getUnsafe().putLong(ptr + offset, sequence); + offset += 8; + + // Error message (if any) + if (status != STATUS_OK && errorMessage != null && !errorMessage.isEmpty()) { + byte[] msgBytes = errorMessage.getBytes(StandardCharsets.UTF_8); + int msgLen = Math.min(msgBytes.length, MAX_ERROR_MESSAGE_LENGTH); + + // Length prefix (2 bytes, little-endian) + Unsafe.getUnsafe().putShort(ptr + offset, (short) msgLen); + offset += 2; + + // Message bytes + for (int i = 0; i < msgLen; i++) { + Unsafe.getUnsafe().putByte(ptr + offset + i, msgBytes[i]); + } + offset += msgLen; + } + + return offset; + } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketSendQueue.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketSendQueue.java index e47c1d8..9a8a8bb 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketSendQueue.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketSendQueue.java @@ -27,10 +27,10 @@ import io.questdb.client.cutlass.http.client.WebSocketClient; import io.questdb.client.cutlass.http.client.WebSocketFrameHandler; import io.questdb.client.cutlass.line.LineSenderException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import io.questdb.client.std.QuietCloseable; import org.jetbrains.annotations.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -62,90 +62,50 @@ */ public class WebSocketSendQueue implements QuietCloseable { - private static final Logger LOG = LoggerFactory.getLogger(WebSocketSendQueue.class); - + public static final long DEFAULT_ENQUEUE_TIMEOUT_MS = 30_000; // Default configuration public static final int DEFAULT_QUEUE_CAPACITY = 16; - public static final long DEFAULT_ENQUEUE_TIMEOUT_MS = 30_000; public static final long DEFAULT_SHUTDOWN_TIMEOUT_MS = 10_000; - - // Single pending buffer slot (double-buffering means at most 1 item in queue) - // Zero allocation - just a volatile reference handoff - private volatile MicrobatchBuffer pendingBuffer; - + private static final Logger LOG = LoggerFactory.getLogger(WebSocketSendQueue.class); // The WebSocket client for I/O (single-threaded access only) private final WebSocketClient client; - + // Configuration + private final long enqueueTimeoutMs; // Optional InFlightWindow for tracking sent batches awaiting ACK @Nullable private final InFlightWindow inFlightWindow; // The I/O thread for async send/receive private final Thread ioThread; - - // Running state - private volatile boolean running; - private volatile boolean shuttingDown; - - // Synchronization for flush/close - private final CountDownLatch shutdownLatch; - - // Error handling - private volatile Throwable lastError; - - // Statistics - sending - private final AtomicLong totalBatchesSent = new AtomicLong(0); - private final AtomicLong totalBytesSent = new AtomicLong(0); - - // Statistics - receiving - private final AtomicLong totalAcks = new AtomicLong(0); - private final AtomicLong totalErrors = new AtomicLong(0); - // Counter for batches currently being processed by the I/O thread // This tracks batches that have been dequeued but not yet fully sent private final AtomicInteger processingCount = new AtomicInteger(0); - // Lock for all coordination between user thread and I/O thread. // Used for: queue poll + processingCount increment atomicity, // flush() waiting, I/O thread waiting when idle. private final Object processingLock = new Object(); - - // Batch sequence counter (must match server's messageSequence) - private long nextBatchSequence = 0; - // Response parsing private final WebSocketResponse response = new WebSocketResponse(); private final ResponseHandler responseHandler = new ResponseHandler(); - - // Configuration - private final long enqueueTimeoutMs; + // Synchronization for flush/close + private final CountDownLatch shutdownLatch; private final long shutdownTimeoutMs; - - // ==================== Pending Buffer Operations (zero allocation) ==================== - - private boolean offerPending(MicrobatchBuffer buffer) { - if (pendingBuffer != null) { - return false; // slot occupied - } - pendingBuffer = buffer; - return true; - } - - private MicrobatchBuffer pollPending() { - MicrobatchBuffer buffer = pendingBuffer; - if (buffer != null) { - pendingBuffer = null; - } - return buffer; - } - - private boolean isPendingEmpty() { - return pendingBuffer == null; - } - - private int getPendingSize() { - return pendingBuffer == null ? 0 : 1; - } + // Statistics - receiving + private final AtomicLong totalAcks = new AtomicLong(0); + // Statistics - sending + private final AtomicLong totalBatchesSent = new AtomicLong(0); + private final AtomicLong totalBytesSent = new AtomicLong(0); + private final AtomicLong totalErrors = new AtomicLong(0); + // Error handling + private volatile Throwable lastError; + // Batch sequence counter (must match server's messageSequence) + private long nextBatchSequence = 0; + // Single pending buffer slot (double-buffering means at most 1 item in queue) + // Zero allocation - just a volatile reference handoff + private volatile MicrobatchBuffer pendingBuffer; + // Running state + private volatile boolean running; + private volatile boolean shuttingDown; /** * Creates a new send queue with default configuration. @@ -200,6 +160,61 @@ public WebSocketSendQueue(WebSocketClient client, @Nullable InFlightWindow inFli LOG.info("WebSocket I/O thread started [capacity={}]", queueCapacity); } + /** + * Closes the send queue gracefully. + *

+ * This method: + * 1. Stops accepting new batches + * 2. Waits for pending batches to be sent + * 3. Stops the I/O thread + *

+ * Note: This does NOT close the WebSocket channel - that's the caller's responsibility. + */ + @Override + public void close() { + if (!running) { + return; + } + + LOG.info("Closing WebSocket send queue [pending={}]", getPendingSize()); + + // Signal shutdown + shuttingDown = true; + + // Wait for pending batches to be sent + long startTime = System.currentTimeMillis(); + while (!isPendingEmpty()) { + if (System.currentTimeMillis() - startTime > shutdownTimeoutMs) { + LOG.error("Shutdown timeout, {} batches not sent", getPendingSize()); + break; + } + try { + Thread.sleep(10); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } + } + + // Stop the I/O thread + running = false; + + // Wake up I/O thread if it's blocked on processingLock.wait() + synchronized (processingLock) { + processingLock.notifyAll(); + } + ioThread.interrupt(); + + // Wait for I/O thread to finish + try { + shutdownLatch.await(shutdownTimeoutMs, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + + LOG.info("WebSocket send queue closed [totalBatches={}, totalBytes={}]", totalBatchesSent.get(), totalBytesSent.get()); + } + /** * Enqueues a sealed buffer for sending. *

@@ -301,24 +316,24 @@ public void flush() { } /** - * Returns the number of batches waiting to be sent. + * Returns the last error that occurred in the I/O thread, or null if no error. */ - public int getPendingCount() { - return getPendingSize(); + public Throwable getLastError() { + return lastError; } /** - * Returns true if the queue is empty. + * Returns the number of batches waiting to be sent. */ - public boolean isEmpty() { - return isPendingEmpty(); + public int getPendingCount() { + return getPendingSize(); } /** - * Returns true if the queue is still running. + * Returns total successful acknowledgments received. */ - public boolean isRunning() { - return running && !shuttingDown; + public long getTotalAcks() { + return totalAcks.get(); } /** @@ -336,79 +351,76 @@ public long getTotalBytesSent() { } /** - * Returns the last error that occurred in the I/O thread, or null if no error. + * Returns total error responses received. */ - public Throwable getLastError() { - return lastError; + public long getTotalErrors() { + return totalErrors.get(); } /** - * Closes the send queue gracefully. - *

- * This method: - * 1. Stops accepting new batches - * 2. Waits for pending batches to be sent - * 3. Stops the I/O thread - *

- * Note: This does NOT close the WebSocket channel - that's the caller's responsibility. + * Returns true if the queue is empty. */ - @Override - public void close() { - if (!running) { - return; - } + public boolean isEmpty() { + return isPendingEmpty(); + } - LOG.info("Closing WebSocket send queue [pending={}]", getPendingSize()); + /** + * Returns true if the queue is still running. + */ + public boolean isRunning() { + return running && !shuttingDown; + } - // Signal shutdown - shuttingDown = true; + /** + * Checks if an error occurred in the I/O thread and throws if so. + */ + private void checkError() { + Throwable error = lastError; + if (error != null) { + throw new LineSenderException("Error in send queue I/O thread: " + error.getMessage(), error); + } + } - // Wait for pending batches to be sent - long startTime = System.currentTimeMillis(); - while (!isPendingEmpty()) { - if (System.currentTimeMillis() - startTime > shutdownTimeoutMs) { - LOG.error("Shutdown timeout, {} batches not sent", getPendingSize()); - break; - } - try { - Thread.sleep(10); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - break; - } + /** + * Computes the current I/O state based on queue and in-flight status. + */ + private IoState computeState(boolean hasInFlight) { + if (!isPendingEmpty()) { + return IoState.ACTIVE; + } else if (hasInFlight) { + return IoState.DRAINING; + } else { + return IoState.IDLE; } + } - // Stop the I/O thread + private void failTransport(LineSenderException error) { + Throwable rootError = lastError; + if (rootError == null) { + lastError = error; + rootError = error; + } running = false; - - // Wake up I/O thread if it's blocked on processingLock.wait() + shuttingDown = true; + if (inFlightWindow != null) { + inFlightWindow.failAll(rootError); + } synchronized (processingLock) { + MicrobatchBuffer dropped = pollPending(); + if (dropped != null) { + if (dropped.isSealed()) { + dropped.markSending(); + } + if (dropped.isSending()) { + dropped.markRecycled(); + } + } processingLock.notifyAll(); } - ioThread.interrupt(); - - // Wait for I/O thread to finish - try { - shutdownLatch.await(shutdownTimeoutMs, TimeUnit.MILLISECONDS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - - LOG.info("WebSocket send queue closed [totalBatches={}, totalBytes={}]", totalBatchesSent.get(), totalBytesSent.get()); } - // ==================== I/O Thread ==================== - - /** - * I/O loop states for the state machine. - *

    - *
  • IDLE: queue empty, no in-flight batches - can block waiting for work
    • - *
  • ACTIVE: have batches to send - non-blocking loop
    • - *
  • DRAINING: queue empty but ACKs pending - poll for ACKs, short wait
    • - *
- */ - private enum IoState { - IDLE, ACTIVE, DRAINING + private int getPendingSize() { + return pendingBuffer == null ? 0 : 1; } /** @@ -495,31 +507,24 @@ private void ioLoop() { } } - /** - * Computes the current I/O state based on queue and in-flight status. - */ - private IoState computeState(boolean hasInFlight) { - if (!isPendingEmpty()) { - return IoState.ACTIVE; - } else if (hasInFlight) { - return IoState.DRAINING; - } else { - return IoState.IDLE; + private boolean isPendingEmpty() { + return pendingBuffer == null; + } + + private boolean offerPending(MicrobatchBuffer buffer) { + if (pendingBuffer != null) { + return false; // slot occupied } + pendingBuffer = buffer; + return true; } - /** - * Tries to receive ACKs from the server (non-blocking). - */ - private void tryReceiveAcks() { - try { - client.tryReceiveFrame(responseHandler); - } catch (Exception e) { - if (running) { - LOG.error("Error receiving response: {}", e.getMessage()); - failTransport(new LineSenderException("Error receiving response: " + e.getMessage(), e)); - } + private MicrobatchBuffer pollPending() { + MicrobatchBuffer buffer = pendingBuffer; + if (buffer != null) { + pendingBuffer = null; } + return buffer; } /** @@ -582,56 +587,31 @@ private void sendBatch(MicrobatchBuffer batch) { } /** - * Checks if an error occurred in the I/O thread and throws if so. + * Tries to receive ACKs from the server (non-blocking). */ - private void checkError() { - Throwable error = lastError; - if (error != null) { - throw new LineSenderException("Error in send queue I/O thread: " + error.getMessage(), error); - } - } - - private void failTransport(LineSenderException error) { - Throwable rootError = lastError; - if (rootError == null) { - lastError = error; - rootError = error; - } - running = false; - shuttingDown = true; - if (inFlightWindow != null) { - inFlightWindow.failAll(rootError); - } - synchronized (processingLock) { - MicrobatchBuffer dropped = pollPending(); - if (dropped != null) { - if (dropped.isSealed()) { - dropped.markSending(); - } - if (dropped.isSending()) { - dropped.markRecycled(); - } + private void tryReceiveAcks() { + try { + client.tryReceiveFrame(responseHandler); + } catch (Exception e) { + if (running) { + LOG.error("Error receiving response: {}", e.getMessage()); + failTransport(new LineSenderException("Error receiving response: " + e.getMessage(), e)); } - processingLock.notifyAll(); } } /** - * Returns total successful acknowledgments received. - */ - public long getTotalAcks() { - return totalAcks.get(); - } - - /** - * Returns total error responses received. + * I/O loop states for the state machine. + *
    + *
  • IDLE: queue empty, no in-flight batches - can block waiting for work
    • + *
  • ACTIVE: have batches to send - non-blocking loop
    • + *
  • DRAINING: queue empty but ACKs pending - poll for ACKs, short wait
    • + *
*/ - public long getTotalErrors() { - return totalErrors.get(); + private enum IoState { + IDLE, ACTIVE, DRAINING } - // ==================== Response Handler ==================== - /** * Handler for received WebSocket frames (ACKs from server). */ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java index 5830ea7..b73d88c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java @@ -40,10 +40,9 @@ public class OffHeapAppendMemory implements QuietCloseable { private static final int DEFAULT_INITIAL_CAPACITY = 128; - - private long pageAddress; private long appendAddress; private long capacity; + private long pageAddress; public OffHeapAppendMemory() { this(DEFAULT_INITIAL_CAPACITY); @@ -55,20 +54,6 @@ public OffHeapAppendMemory(long initialCapacity) { this.appendAddress = pageAddress; } - /** - * Returns the append offset (number of bytes written). - */ - public long getAppendOffset() { - return appendAddress - pageAddress; - } - - /** - * Returns the base address of the buffer. - */ - public long pageAddress() { - return pageAddress; - } - /** * Returns the address at the given byte offset from the start. */ @@ -76,11 +61,21 @@ public long addressOf(long offset) { return pageAddress + offset; } + @Override + public void close() { + if (pageAddress != 0) { + Unsafe.free(pageAddress, capacity, MemoryTag.NATIVE_ILP_RSS); + pageAddress = 0; + appendAddress = 0; + capacity = 0; + } + } + /** - * Resets the append position to 0 without freeing memory. + * Returns the append offset (number of bytes written). */ - public void truncate() { - appendAddress = pageAddress; + public long getAppendOffset() { + return appendAddress - pageAddress; } /** @@ -91,20 +86,33 @@ public void jumpTo(long offset) { appendAddress = pageAddress + offset; } + /** + * Returns the base address of the buffer. + */ + public long pageAddress() { + return pageAddress; + } + + public void putBoolean(boolean value) { + putByte(value ? (byte) 1 : (byte) 0); + } + public void putByte(byte value) { ensureCapacity(1); Unsafe.getUnsafe().putByte(appendAddress, value); appendAddress++; } - public void putBoolean(boolean value) { - putByte(value ? (byte) 1 : (byte) 0); + public void putDouble(double value) { + ensureCapacity(8); + Unsafe.getUnsafe().putDouble(appendAddress, value); + appendAddress += 8; } - public void putShort(short value) { - ensureCapacity(2); - Unsafe.getUnsafe().putShort(appendAddress, value); - appendAddress += 2; + public void putFloat(float value) { + ensureCapacity(4); + Unsafe.getUnsafe().putFloat(appendAddress, value); + appendAddress += 4; } public void putInt(int value) { @@ -119,16 +127,10 @@ public void putLong(long value) { appendAddress += 8; } - public void putFloat(float value) { - ensureCapacity(4); - Unsafe.getUnsafe().putFloat(appendAddress, value); - appendAddress += 4; - } - - public void putDouble(double value) { - ensureCapacity(8); - Unsafe.getUnsafe().putDouble(appendAddress, value); - appendAddress += 8; + public void putShort(short value) { + ensureCapacity(2); + Unsafe.getUnsafe().putShort(appendAddress, value); + appendAddress += 2; } /** @@ -171,14 +173,11 @@ public void skip(long bytes) { appendAddress += bytes; } - @Override - public void close() { - if (pageAddress != 0) { - Unsafe.free(pageAddress, capacity, MemoryTag.NATIVE_ILP_RSS); - pageAddress = 0; - appendAddress = 0; - capacity = 0; - } + /** + * Resets the append position to 0 without freeing memory. + */ + public void truncate() { + appendAddress = pageAddress; } private void ensureCapacity(long needed) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitWriter.java index af30761..3f18d2d 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitWriter.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitWriter.java @@ -48,14 +48,13 @@ */ public class QwpBitWriter { - private long startAddress; - private long currentAddress; - private long endAddress; - // Buffer for accumulating bits before writing private long bitBuffer; // Number of bits currently in the buffer (0-63) private int bitsInBuffer; + private long currentAddress; + private long endAddress; + private long startAddress; /** * Creates a new bit writer. Call {@link #reset} before use. @@ -64,17 +63,54 @@ public QwpBitWriter() { } /** - * Resets the writer to write to the specified memory region. + * Aligns the writer to the next byte boundary by padding with zeros. + * If already byte-aligned, this is a no-op. + */ + public void alignToByte() { + if (bitsInBuffer > 0) { + flush(); + } + } + + /** + * Finishes writing and returns the number of bytes written since reset. + *

+ * This method flushes any remaining bits and returns the total byte count. * - * @param address the starting address - * @param capacity the maximum number of bytes to write + * @return bytes written since reset */ - public void reset(long address, long capacity) { - this.startAddress = address; - this.currentAddress = address; - this.endAddress = address + capacity; - this.bitBuffer = 0; - this.bitsInBuffer = 0; + public int finish() { + flush(); + return (int) (currentAddress - startAddress); + } + + /** + * Flushes any remaining bits in the buffer to memory. + *

+ * If there are partial bits (less than 8), they are written as the last byte + * with the remaining high bits set to zero. + *

+ * Must be called before reading the output or getting the final position. + */ + public void flush() { + if (bitsInBuffer > 0) { + if (currentAddress >= endAddress) { + throw new LineSenderException("QwpBitWriter buffer overflow"); + } + Unsafe.getUnsafe().putByte(currentAddress++, (byte) bitBuffer); + bitBuffer = 0; + bitsInBuffer = 0; + } + } + + /** + * Returns the number of bits remaining in the partial byte buffer. + * This is 0 after a flush or when aligned on a byte boundary. + * + * @return bits in buffer (0-7) + */ + public int getBitsInBuffer() { + return bitsInBuffer; } /** @@ -96,6 +132,20 @@ public long getTotalBitsWritten() { return (currentAddress - startAddress) * 8L + bitsInBuffer; } + /** + * Resets the writer to write to the specified memory region. + * + * @param address the starting address + * @param capacity the maximum number of bytes to write + */ + public void reset(long address, long capacity) { + this.startAddress = address; + this.currentAddress = address; + this.endAddress = address + capacity; + this.bitBuffer = 0; + this.bitsInBuffer = 0; + } + /** * Writes a single bit. * @@ -150,68 +200,6 @@ public void writeBits(long value, int numBits) { } } - /** - * Writes a signed value using two's complement representation. - * - * @param value the signed value - * @param numBits number of bits to use for the representation - */ - public void writeSigned(long value, int numBits) { - // Two's complement is automatic in Java for the bit pattern - writeBits(value, numBits); - } - - /** - * Flushes any remaining bits in the buffer to memory. - *

- * If there are partial bits (less than 8), they are written as the last byte - * with the remaining high bits set to zero. - *

- * Must be called before reading the output or getting the final position. - */ - public void flush() { - if (bitsInBuffer > 0) { - if (currentAddress >= endAddress) { - throw new LineSenderException("QwpBitWriter buffer overflow"); - } - Unsafe.getUnsafe().putByte(currentAddress++, (byte) bitBuffer); - bitBuffer = 0; - bitsInBuffer = 0; - } - } - - /** - * Finishes writing and returns the number of bytes written since reset. - *

- * This method flushes any remaining bits and returns the total byte count. - * - * @return bytes written since reset - */ - public int finish() { - flush(); - return (int) (currentAddress - startAddress); - } - - /** - * Returns the number of bits remaining in the partial byte buffer. - * This is 0 after a flush or when aligned on a byte boundary. - * - * @return bits in buffer (0-7) - */ - public int getBitsInBuffer() { - return bitsInBuffer; - } - - /** - * Aligns the writer to the next byte boundary by padding with zeros. - * If already byte-aligned, this is a no-op. - */ - public void alignToByte() { - if (bitsInBuffer > 0) { - flush(); - } - } - /** * Writes a complete byte, ensuring byte alignment first. * @@ -252,4 +240,15 @@ public void writeLong(long value) { Unsafe.getUnsafe().putLong(currentAddress, value); currentAddress += 8; } + + /** + * Writes a signed value using two's complement representation. + * + * @param value the signed value + * @param numBits number of bits to use for the representation + */ + public void writeSigned(long value, int numBits) { + // Two's complement is automatic in Java for the bit pattern + writeBits(value, numBits); + } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java index a7257dd..f59a009 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java @@ -24,7 +24,8 @@ package io.questdb.client.cutlass.qwp.protocol; -import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.*; +import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.TYPE_BOOLEAN; +import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.TYPE_CHAR; /** * Represents a column definition in an ILP v4 schema. @@ -33,8 +34,8 @@ */ public final class QwpColumnDef { private final String name; - private final byte typeCode; private final boolean nullable; + private final byte typeCode; /** * Creates a column definition. @@ -62,6 +63,25 @@ public QwpColumnDef(String name, byte typeCode, boolean nullable) { this.nullable = nullable; } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + QwpColumnDef that = (QwpColumnDef) o; + return typeCode == that.typeCode && + nullable == that.nullable && + name.equals(that.name); + } + + /** + * Gets the fixed width in bytes for fixed-width types. + * + * @return width in bytes, or -1 for variable-width types + */ + public int getFixedWidth() { + return QwpConstants.getFixedTypeSize(typeCode); + } + /** * Gets the column name. */ @@ -78,6 +98,13 @@ public byte getTypeCode() { return typeCode; } + /** + * Gets the type name for display purposes. + */ + public String getTypeName() { + return QwpConstants.getTypeName(typeCode); + } + /** * Gets the wire type code (with nullable flag if applicable). * @@ -87,11 +114,12 @@ public byte getWireTypeCode() { return nullable ? (byte) (typeCode | 0x80) : typeCode; } - /** - * Returns true if this column is nullable. - */ - public boolean isNullable() { - return nullable; + @Override + public int hashCode() { + int result = name.hashCode(); + result = 31 * result + typeCode; + result = 31 * result + (nullable ? 1 : 0); + return result; } /** @@ -102,19 +130,20 @@ public boolean isFixedWidth() { } /** - * Gets the fixed width in bytes for fixed-width types. - * - * @return width in bytes, or -1 for variable-width types + * Returns true if this column is nullable. */ - public int getFixedWidth() { - return QwpConstants.getFixedTypeSize(typeCode); + public boolean isNullable() { + return nullable; } - /** - * Gets the type name for display purposes. - */ - public String getTypeName() { - return QwpConstants.getTypeName(typeCode); + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(name).append(':').append(getTypeName()); + if (nullable) { + sb.append('?'); + } + return sb.toString(); } /** @@ -132,32 +161,4 @@ public void validate() { ); } } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - QwpColumnDef that = (QwpColumnDef) o; - return typeCode == that.typeCode && - nullable == that.nullable && - name.equals(that.name); - } - - @Override - public int hashCode() { - int result = name.hashCode(); - result = 31 * result + typeCode; - result = 31 * result + (nullable ? 1 : 0); - return result; - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append(name).append(':').append(getTypeName()); - if (nullable) { - sb.append('?'); - } - return sb.toString(); - } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java index 2a40a35..34f9b2d 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java @@ -29,68 +29,48 @@ */ public final class QwpConstants { - // ==================== Magic Bytes ==================== - - /** - * Magic bytes for ILP v4 message: "ILP4" (ASCII). - */ - public static final int MAGIC_MESSAGE = 0x34504C49; // "ILP4" in little-endian - /** - * Magic bytes for capability request: "ILP?" (ASCII). + * Size of capability request in bytes. */ - public static final int MAGIC_CAPABILITY_REQUEST = 0x3F504C49; // "ILP?" in little-endian - + public static final int CAPABILITY_REQUEST_SIZE = 8; /** - * Magic bytes for capability response: "ILP!" (ASCII). + * Size of capability response in bytes. */ - public static final int MAGIC_CAPABILITY_RESPONSE = 0x21504C49; // "ILP!" in little-endian - + public static final int CAPABILITY_RESPONSE_SIZE = 8; /** - * Magic bytes for fallback response (old server): "ILP0" (ASCII). + * Default initial receive buffer size (64 KB). */ - public static final int MAGIC_FALLBACK = 0x30504C49; // "ILP0" in little-endian - - // ==================== Header Structure ==================== - + public static final int DEFAULT_INITIAL_RECV_BUFFER_SIZE = 64 * 1024; /** - * Size of the message header in bytes. + * Default maximum batch size in bytes (16 MB). */ - public static final int HEADER_SIZE = 12; + public static final int DEFAULT_MAX_BATCH_SIZE = 16 * 1024 * 1024; /** - * Offset of magic bytes in header (4 bytes). + * Maximum in-flight batches for pipelining. */ - public static final int HEADER_OFFSET_MAGIC = 0; - + public static final int DEFAULT_MAX_IN_FLIGHT_BATCHES = 4; /** - * Offset of version byte in header. + * Default maximum rows per table in a batch. */ - public static final int HEADER_OFFSET_VERSION = 4; - + public static final int DEFAULT_MAX_ROWS_PER_TABLE = 1_000_000; /** - * Offset of flags byte in header. + * Default maximum string length in bytes (1 MB). */ - public static final int HEADER_OFFSET_FLAGS = 5; - + public static final int DEFAULT_MAX_STRING_LENGTH = 1024 * 1024; /** - * Offset of table count (uint16, little-endian) in header. + * Default maximum tables per batch. */ - public static final int HEADER_OFFSET_TABLE_COUNT = 6; - + public static final int DEFAULT_MAX_TABLES_PER_BATCH = 256; /** - * Offset of payload length (uint32, little-endian) in header. + * Flag bit: Delta symbol dictionary encoding enabled. + * When set, symbol columns use global IDs and send only new dictionary entries. */ - public static final int HEADER_OFFSET_PAYLOAD_LENGTH = 8; - - // ==================== Protocol Version ==================== - + public static final byte FLAG_DELTA_SYMBOL_DICT = 0x08; /** - * Current protocol version. + * Flag bit: Gorilla timestamp encoding enabled. */ - public static final byte VERSION_1 = 1; - - // ==================== Flag Bits ==================== + public static final byte FLAG_GORILLA = 0x04; /** * Flag bit: LZ4 compression enabled. @@ -101,295 +81,219 @@ public final class QwpConstants { * Flag bit: Zstd compression enabled. */ public static final byte FLAG_ZSTD = 0x02; - /** - * Flag bit: Gorilla timestamp encoding enabled. + * Mask for compression flags (bits 0-1). */ - public static final byte FLAG_GORILLA = 0x04; - + public static final byte FLAG_COMPRESSION_MASK = FLAG_LZ4 | FLAG_ZSTD; /** - * Flag bit: Delta symbol dictionary encoding enabled. - * When set, symbol columns use global IDs and send only new dictionary entries. + * Offset of flags byte in header. */ - public static final byte FLAG_DELTA_SYMBOL_DICT = 0x08; - + public static final int HEADER_OFFSET_FLAGS = 5; /** - * Mask for compression flags (bits 0-1). + * Offset of magic bytes in header (4 bytes). */ - public static final byte FLAG_COMPRESSION_MASK = FLAG_LZ4 | FLAG_ZSTD; - - // ==================== Column Type Codes ==================== - + public static final int HEADER_OFFSET_MAGIC = 0; /** - * Column type: BOOLEAN (1 bit per value, packed). + * Offset of payload length (uint32, little-endian) in header. */ - public static final byte TYPE_BOOLEAN = 0x01; + public static final int HEADER_OFFSET_PAYLOAD_LENGTH = 8; /** - * Column type: BYTE (int8). + * Offset of table count (uint16, little-endian) in header. */ - public static final byte TYPE_BYTE = 0x02; - + public static final int HEADER_OFFSET_TABLE_COUNT = 6; /** - * Column type: SHORT (int16, little-endian). + * Offset of version byte in header. */ - public static final byte TYPE_SHORT = 0x03; - + public static final int HEADER_OFFSET_VERSION = 4; /** - * Column type: INT (int32, little-endian). + * Size of the message header in bytes. */ - public static final byte TYPE_INT = 0x04; - + public static final int HEADER_SIZE = 12; /** - * Column type: LONG (int64, little-endian). + * Magic bytes for capability request: "ILP?" (ASCII). */ - public static final byte TYPE_LONG = 0x05; - + public static final int MAGIC_CAPABILITY_REQUEST = 0x3F504C49; // "ILP?" in little-endian /** - * Column type: FLOAT (IEEE 754 float32). + * Magic bytes for capability response: "ILP!" (ASCII). */ - public static final byte TYPE_FLOAT = 0x06; - + public static final int MAGIC_CAPABILITY_RESPONSE = 0x21504C49; // "ILP!" in little-endian /** - * Column type: DOUBLE (IEEE 754 float64). + * Magic bytes for fallback response (old server): "ILP0" (ASCII). */ - public static final byte TYPE_DOUBLE = 0x07; - + public static final int MAGIC_FALLBACK = 0x30504C49; // "ILP0" in little-endian /** - * Column type: STRING (length-prefixed UTF-8). + * Magic bytes for ILP v4 message: "ILP4" (ASCII). */ - public static final byte TYPE_STRING = 0x08; - + public static final int MAGIC_MESSAGE = 0x34504C49; // "ILP4" in little-endian /** - * Column type: SYMBOL (dictionary-encoded string). + * Maximum columns per table (QuestDB limit). */ - public static final byte TYPE_SYMBOL = 0x09; - + public static final int MAX_COLUMNS_PER_TABLE = 2048; /** - * Column type: TIMESTAMP (int64 microseconds since epoch). - * Use this for timestamps beyond nanosecond range (year > 2262). + * Maximum column name length in bytes. */ - public static final byte TYPE_TIMESTAMP = 0x0A; - + public static final int MAX_COLUMN_NAME_LENGTH = 127; /** - * Column type: TIMESTAMP_NANOS (int64 nanoseconds since epoch). - * Use this for full nanosecond precision (limited to years 1677-2262). + * Maximum table name length in bytes. */ - public static final byte TYPE_TIMESTAMP_NANOS = 0x10; - + public static final int MAX_TABLE_NAME_LENGTH = 127; /** - * Column type: DATE (int64 milliseconds since epoch). + * Schema mode: Full schema included. */ - public static final byte TYPE_DATE = 0x0B; - + public static final byte SCHEMA_MODE_FULL = 0x00; /** - * Column type: UUID (16 bytes, big-endian). + * Schema mode: Schema reference (hash lookup). */ - public static final byte TYPE_UUID = 0x0C; - + public static final byte SCHEMA_MODE_REFERENCE = 0x01; /** - * Column type: LONG256 (32 bytes, big-endian). + * Status: Server error. */ - public static final byte TYPE_LONG256 = 0x0D; - + public static final byte STATUS_INTERNAL_ERROR = 0x06; /** - * Column type: GEOHASH (varint bits + packed geohash). + * Status: Batch accepted successfully. */ - public static final byte TYPE_GEOHASH = 0x0E; - + public static final byte STATUS_OK = 0x00; /** - * Column type: VARCHAR (length-prefixed UTF-8, aux storage). + * Status: Back-pressure, retry later. */ - public static final byte TYPE_VARCHAR = 0x0F; - + public static final byte STATUS_OVERLOADED = 0x07; /** - * Column type: DOUBLE_ARRAY (N-dimensional array of IEEE 754 float64). - * Wire format: [nDims (1B)] [dim1_len (4B)]...[dimN_len (4B)] [flattened values (LE)] + * Status: Malformed message. */ - public static final byte TYPE_DOUBLE_ARRAY = 0x11; - + public static final byte STATUS_PARSE_ERROR = 0x05; /** - * Column type: LONG_ARRAY (N-dimensional array of int64). - * Wire format: [nDims (1B)] [dim1_len (4B)]...[dimN_len (4B)] [flattened values (LE)] + * Status: Some rows failed (partial failure). */ - public static final byte TYPE_LONG_ARRAY = 0x12; - + public static final byte STATUS_PARTIAL = 0x01; /** - * Column type: DECIMAL64 (8 bytes, 18 digits precision). - * Wire format: [scale (1B in schema)] + [big-endian unscaled value (8B)] + * Status: Column type incompatible. */ - public static final byte TYPE_DECIMAL64 = 0x13; - + public static final byte STATUS_SCHEMA_MISMATCH = 0x03; /** - * Column type: DECIMAL128 (16 bytes, 38 digits precision). - * Wire format: [scale (1B in schema)] + [big-endian unscaled value (16B)] + * Status: Schema hash not recognized. */ - public static final byte TYPE_DECIMAL128 = 0x14; - + public static final byte STATUS_SCHEMA_REQUIRED = 0x02; /** - * Column type: DECIMAL256 (32 bytes, 77 digits precision). - * Wire format: [scale (1B in schema)] + [big-endian unscaled value (32B)] + * Status: Table doesn't exist (auto-create disabled). */ - public static final byte TYPE_DECIMAL256 = 0x15; - + public static final byte STATUS_TABLE_NOT_FOUND = 0x04; /** - * Column type: CHAR (2-byte UTF-16 code unit). + * Column type: BOOLEAN (1 bit per value, packed). */ - public static final byte TYPE_CHAR = 0x16; - + public static final byte TYPE_BOOLEAN = 0x01; /** - * High bit indicating nullable column. + * Column type: BYTE (int8). */ - public static final byte TYPE_NULLABLE_FLAG = (byte) 0x80; - + public static final byte TYPE_BYTE = 0x02; /** - * Mask for type code without nullable flag. + * Column type: CHAR (2-byte UTF-16 code unit). */ - public static final byte TYPE_MASK = 0x7F; - - // ==================== Schema Mode ==================== - + public static final byte TYPE_CHAR = 0x16; /** - * Schema mode: Full schema included. + * Column type: DATE (int64 milliseconds since epoch). */ - public static final byte SCHEMA_MODE_FULL = 0x00; + public static final byte TYPE_DATE = 0x0B; /** - * Schema mode: Schema reference (hash lookup). + * Column type: DECIMAL128 (16 bytes, 38 digits precision). + * Wire format: [scale (1B in schema)] + [big-endian unscaled value (16B)] */ - public static final byte SCHEMA_MODE_REFERENCE = 0x01; - - // ==================== Response Status Codes ==================== - + public static final byte TYPE_DECIMAL128 = 0x14; /** - * Status: Batch accepted successfully. + * Column type: DECIMAL256 (32 bytes, 77 digits precision). + * Wire format: [scale (1B in schema)] + [big-endian unscaled value (32B)] */ - public static final byte STATUS_OK = 0x00; + public static final byte TYPE_DECIMAL256 = 0x15; /** - * Status: Some rows failed (partial failure). + * Column type: DECIMAL64 (8 bytes, 18 digits precision). + * Wire format: [scale (1B in schema)] + [big-endian unscaled value (8B)] */ - public static final byte STATUS_PARTIAL = 0x01; - + public static final byte TYPE_DECIMAL64 = 0x13; /** - * Status: Schema hash not recognized. + * Column type: DOUBLE (IEEE 754 float64). */ - public static final byte STATUS_SCHEMA_REQUIRED = 0x02; - + public static final byte TYPE_DOUBLE = 0x07; /** - * Status: Column type incompatible. + * Column type: DOUBLE_ARRAY (N-dimensional array of IEEE 754 float64). + * Wire format: [nDims (1B)] [dim1_len (4B)]...[dimN_len (4B)] [flattened values (LE)] */ - public static final byte STATUS_SCHEMA_MISMATCH = 0x03; - + public static final byte TYPE_DOUBLE_ARRAY = 0x11; /** - * Status: Table doesn't exist (auto-create disabled). + * Column type: FLOAT (IEEE 754 float32). */ - public static final byte STATUS_TABLE_NOT_FOUND = 0x04; - + public static final byte TYPE_FLOAT = 0x06; /** - * Status: Malformed message. + * Column type: GEOHASH (varint bits + packed geohash). */ - public static final byte STATUS_PARSE_ERROR = 0x05; - + public static final byte TYPE_GEOHASH = 0x0E; /** - * Status: Server error. + * Column type: INT (int32, little-endian). */ - public static final byte STATUS_INTERNAL_ERROR = 0x06; - + public static final byte TYPE_INT = 0x04; /** - * Status: Back-pressure, retry later. + * Column type: LONG (int64, little-endian). */ - public static final byte STATUS_OVERLOADED = 0x07; - - // ==================== Default Limits ==================== - + public static final byte TYPE_LONG = 0x05; /** - * Default maximum batch size in bytes (16 MB). + * Column type: LONG256 (32 bytes, big-endian). */ - public static final int DEFAULT_MAX_BATCH_SIZE = 16 * 1024 * 1024; + public static final byte TYPE_LONG256 = 0x0D; /** - * Default maximum tables per batch. + * Column type: LONG_ARRAY (N-dimensional array of int64). + * Wire format: [nDims (1B)] [dim1_len (4B)]...[dimN_len (4B)] [flattened values (LE)] */ - public static final int DEFAULT_MAX_TABLES_PER_BATCH = 256; - + public static final byte TYPE_LONG_ARRAY = 0x12; /** - * Default maximum rows per table in a batch. + * Mask for type code without nullable flag. */ - public static final int DEFAULT_MAX_ROWS_PER_TABLE = 1_000_000; - + public static final byte TYPE_MASK = 0x7F; /** - * Maximum columns per table (QuestDB limit). + * High bit indicating nullable column. */ - public static final int MAX_COLUMNS_PER_TABLE = 2048; - + public static final byte TYPE_NULLABLE_FLAG = (byte) 0x80; /** - * Maximum table name length in bytes. + * Column type: SHORT (int16, little-endian). */ - public static final int MAX_TABLE_NAME_LENGTH = 127; - + public static final byte TYPE_SHORT = 0x03; /** - * Maximum column name length in bytes. + * Column type: STRING (length-prefixed UTF-8). */ - public static final int MAX_COLUMN_NAME_LENGTH = 127; - + public static final byte TYPE_STRING = 0x08; /** - * Default maximum string length in bytes (1 MB). + * Column type: SYMBOL (dictionary-encoded string). */ - public static final int DEFAULT_MAX_STRING_LENGTH = 1024 * 1024; - + public static final byte TYPE_SYMBOL = 0x09; /** - * Default initial receive buffer size (64 KB). + * Column type: TIMESTAMP (int64 microseconds since epoch). + * Use this for timestamps beyond nanosecond range (year > 2262). */ - public static final int DEFAULT_INITIAL_RECV_BUFFER_SIZE = 64 * 1024; - + public static final byte TYPE_TIMESTAMP = 0x0A; /** - * Maximum in-flight batches for pipelining. + * Column type: TIMESTAMP_NANOS (int64 nanoseconds since epoch). + * Use this for full nanosecond precision (limited to years 1677-2262). */ - public static final int DEFAULT_MAX_IN_FLIGHT_BATCHES = 4; - - // ==================== Capability Negotiation ==================== - + public static final byte TYPE_TIMESTAMP_NANOS = 0x10; /** - * Size of capability request in bytes. + * Column type: UUID (16 bytes, big-endian). */ - public static final int CAPABILITY_REQUEST_SIZE = 8; + public static final byte TYPE_UUID = 0x0C; /** - * Size of capability response in bytes. + * Column type: VARCHAR (length-prefixed UTF-8, aux storage). */ - public static final int CAPABILITY_RESPONSE_SIZE = 8; + public static final byte TYPE_VARCHAR = 0x0F; + /** + * Current protocol version. + */ + public static final byte VERSION_1 = 1; private QwpConstants() { // utility class } - /** - * Returns true if the type code represents a fixed-width type. - * - * @param typeCode the column type code (without nullable flag) - * @return true if fixed-width - */ - public static boolean isFixedWidthType(byte typeCode) { - int code = typeCode & TYPE_MASK; - return code == TYPE_BOOLEAN || - code == TYPE_BYTE || - code == TYPE_SHORT || - code == TYPE_CHAR || - code == TYPE_INT || - code == TYPE_LONG || - code == TYPE_FLOAT || - code == TYPE_DOUBLE || - code == TYPE_TIMESTAMP || - code == TYPE_TIMESTAMP_NANOS || - code == TYPE_DATE || - code == TYPE_UUID || - code == TYPE_LONG256 || - code == TYPE_DECIMAL64 || - code == TYPE_DECIMAL128 || - code == TYPE_DECIMAL256; - } - /** * Returns the size in bytes for fixed-width types. * @@ -509,4 +413,30 @@ public static String getTypeName(byte typeCode) { } return nullable ? name + "?" : name; } + + /** + * Returns true if the type code represents a fixed-width type. + * + * @param typeCode the column type code (without nullable flag) + * @return true if fixed-width + */ + public static boolean isFixedWidthType(byte typeCode) { + int code = typeCode & TYPE_MASK; + return code == TYPE_BOOLEAN || + code == TYPE_BYTE || + code == TYPE_SHORT || + code == TYPE_CHAR || + code == TYPE_INT || + code == TYPE_LONG || + code == TYPE_FLOAT || + code == TYPE_DOUBLE || + code == TYPE_TIMESTAMP || + code == TYPE_TIMESTAMP_NANOS || + code == TYPE_DATE || + code == TYPE_UUID || + code == TYPE_LONG256 || + code == TYPE_DECIMAL64 || + code == TYPE_DECIMAL128 || + code == TYPE_DECIMAL256; + } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java index 912af2d..082f6b2 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java @@ -62,6 +62,84 @@ public class QwpGorillaEncoder { public QwpGorillaEncoder() { } + /** + * Calculates the encoded size in bytes for Gorilla-encoded timestamps stored off-heap. + *

+ * Note: This does NOT include the encoding flag byte. Add 1 byte if + * the encoding flag is needed. + * + * @param srcAddress source address of contiguous int64 timestamps in native memory + * @param count number of timestamps + * @return encoded size in bytes (excluding encoding flag) + */ + public static int calculateEncodedSize(long srcAddress, int count) { + if (count == 0) { + return 0; + } + + int size = 8; // first timestamp + + if (count == 1) { + return size; + } + + size += 8; // second timestamp + + if (count == 2) { + return size; + } + + // Calculate bits for delta-of-delta encoding + long prevTimestamp = Unsafe.getUnsafe().getLong(srcAddress + 8); + long prevDelta = prevTimestamp - Unsafe.getUnsafe().getLong(srcAddress); + int totalBits = 0; + + for (int i = 2; i < count; i++) { + long ts = Unsafe.getUnsafe().getLong(srcAddress + (long) i * 8); + long delta = ts - prevTimestamp; + long deltaOfDelta = delta - prevDelta; + + totalBits += getBitsRequired(deltaOfDelta); + + prevDelta = delta; + prevTimestamp = ts; + } + + // Round up to bytes + size += (totalBits + 7) / 8; + + return size; + } + + /** + * Checks if Gorilla encoding can be used for timestamps stored off-heap. + *

+ * Gorilla encoding uses 32-bit signed integers for delta-of-delta values, + * so it cannot encode timestamps where the delta-of-delta exceeds the + * 32-bit signed integer range. + * + * @param srcAddress source address of contiguous int64 timestamps in native memory + * @param count number of timestamps + * @return true if Gorilla encoding can be used, false otherwise + */ + public static boolean canUseGorilla(long srcAddress, int count) { + if (count < 3) { + return true; // No DoD encoding needed for 0, 1, or 2 timestamps + } + + long prevDelta = Unsafe.getUnsafe().getLong(srcAddress + 8) - Unsafe.getUnsafe().getLong(srcAddress); + for (int i = 2; i < count; i++) { + long delta = Unsafe.getUnsafe().getLong(srcAddress + (long) i * 8) + - Unsafe.getUnsafe().getLong(srcAddress + (long) (i - 1) * 8); + long dod = delta - prevDelta; + if (dod < Integer.MIN_VALUE || dod > Integer.MAX_VALUE) { + return false; + } + prevDelta = delta; + } + return true; + } + /** * Returns the number of bits required to encode a delta-of-delta value. * @@ -209,82 +287,4 @@ public int encodeTimestamps(long destAddress, long capacity, long srcAddress, in return pos + bitWriter.finish(); } - - /** - * Checks if Gorilla encoding can be used for timestamps stored off-heap. - *

- * Gorilla encoding uses 32-bit signed integers for delta-of-delta values, - * so it cannot encode timestamps where the delta-of-delta exceeds the - * 32-bit signed integer range. - * - * @param srcAddress source address of contiguous int64 timestamps in native memory - * @param count number of timestamps - * @return true if Gorilla encoding can be used, false otherwise - */ - public static boolean canUseGorilla(long srcAddress, int count) { - if (count < 3) { - return true; // No DoD encoding needed for 0, 1, or 2 timestamps - } - - long prevDelta = Unsafe.getUnsafe().getLong(srcAddress + 8) - Unsafe.getUnsafe().getLong(srcAddress); - for (int i = 2; i < count; i++) { - long delta = Unsafe.getUnsafe().getLong(srcAddress + (long) i * 8) - - Unsafe.getUnsafe().getLong(srcAddress + (long) (i - 1) * 8); - long dod = delta - prevDelta; - if (dod < Integer.MIN_VALUE || dod > Integer.MAX_VALUE) { - return false; - } - prevDelta = delta; - } - return true; - } - - /** - * Calculates the encoded size in bytes for Gorilla-encoded timestamps stored off-heap. - *

- * Note: This does NOT include the encoding flag byte. Add 1 byte if - * the encoding flag is needed. - * - * @param srcAddress source address of contiguous int64 timestamps in native memory - * @param count number of timestamps - * @return encoded size in bytes (excluding encoding flag) - */ - public static int calculateEncodedSize(long srcAddress, int count) { - if (count == 0) { - return 0; - } - - int size = 8; // first timestamp - - if (count == 1) { - return size; - } - - size += 8; // second timestamp - - if (count == 2) { - return size; - } - - // Calculate bits for delta-of-delta encoding - long prevTimestamp = Unsafe.getUnsafe().getLong(srcAddress + 8); - long prevDelta = prevTimestamp - Unsafe.getUnsafe().getLong(srcAddress); - int totalBits = 0; - - for (int i = 2; i < count; i++) { - long ts = Unsafe.getUnsafe().getLong(srcAddress + (long) i * 8); - long delta = ts - prevTimestamp; - long deltaOfDelta = delta - prevDelta; - - totalBits += getBitsRequired(deltaOfDelta); - - prevDelta = delta; - prevTimestamp = ts; - } - - // Round up to bytes - size += (totalBits + 7) / 8; - - return size; - } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpNullBitmap.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpNullBitmap.java index 90cf944..dd6020e 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpNullBitmap.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpNullBitmap.java @@ -49,70 +49,34 @@ private QwpNullBitmap() { } /** - * Calculates the size in bytes needed for a null bitmap. - * - * @param rowCount number of rows - * @return bitmap size in bytes - */ - public static int sizeInBytes(long rowCount) { - return (int) ((rowCount + 7) / 8); - } - - /** - * Checks if a specific row is null in the bitmap (from direct memory). + * Checks if all rows are null. * * @param address bitmap start address - * @param rowIndex row index to check - * @return true if the row is null + * @param rowCount total number of rows + * @return true if all rows are null */ - public static boolean isNull(long address, int rowIndex) { - int byteIndex = rowIndex >>> 3; // rowIndex / 8 - int bitIndex = rowIndex & 7; // rowIndex % 8 - byte b = Unsafe.getUnsafe().getByte(address + byteIndex); - return (b & (1 << bitIndex)) != 0; - } + public static boolean allNull(long address, int rowCount) { + int fullBytes = rowCount >>> 3; + int remainingBits = rowCount & 7; - /** - * Checks if a specific row is null in the bitmap (from byte array). - * - * @param bitmap bitmap byte array - * @param offset starting offset in array - * @param rowIndex row index to check - * @return true if the row is null - */ - public static boolean isNull(byte[] bitmap, int offset, int rowIndex) { - int byteIndex = rowIndex >>> 3; - int bitIndex = rowIndex & 7; - byte b = bitmap[offset + byteIndex]; - return (b & (1 << bitIndex)) != 0; - } + // Check full bytes (all bits should be 1) + for (int i = 0; i < fullBytes; i++) { + byte b = Unsafe.getUnsafe().getByte(address + i); + if ((b & 0xFF) != 0xFF) { + return false; + } + } - /** - * Sets a row as null in the bitmap (direct memory). - * - * @param address bitmap start address - * @param rowIndex row index to set as null - */ - public static void setNull(long address, int rowIndex) { - int byteIndex = rowIndex >>> 3; - int bitIndex = rowIndex & 7; - long addr = address + byteIndex; - byte b = Unsafe.getUnsafe().getByte(addr); - b |= (1 << bitIndex); - Unsafe.getUnsafe().putByte(addr, b); - } + // Check remaining bits + if (remainingBits > 0) { + byte b = Unsafe.getUnsafe().getByte(address + fullBytes); + int mask = (1 << remainingBits) - 1; + if ((b & mask) != mask) { + return false; + } + } - /** - * Sets a row as null in the bitmap (byte array). - * - * @param bitmap bitmap byte array - * @param offset starting offset in array - * @param rowIndex row index to set as null - */ - public static void setNull(byte[] bitmap, int offset, int rowIndex) { - int byteIndex = rowIndex >>> 3; - int bitIndex = rowIndex & 7; - bitmap[offset + byteIndex] |= (1 << bitIndex); + return true; } /** @@ -198,34 +162,81 @@ public static int countNulls(byte[] bitmap, int offset, int rowCount) { } /** - * Checks if all rows are null. + * Fills the bitmap setting all rows as null (direct memory). * * @param address bitmap start address * @param rowCount total number of rows - * @return true if all rows are null */ - public static boolean allNull(long address, int rowCount) { + public static void fillAllNull(long address, int rowCount) { int fullBytes = rowCount >>> 3; int remainingBits = rowCount & 7; - // Check full bytes (all bits should be 1) + // Fill full bytes with all 1s for (int i = 0; i < fullBytes; i++) { - byte b = Unsafe.getUnsafe().getByte(address + i); - if ((b & 0xFF) != 0xFF) { - return false; - } + Unsafe.getUnsafe().putByte(address + i, (byte) 0xFF); } - // Check remaining bits + // Set remaining bits in last byte if (remainingBits > 0) { - byte b = Unsafe.getUnsafe().getByte(address + fullBytes); - int mask = (1 << remainingBits) - 1; - if ((b & mask) != mask) { - return false; - } + byte mask = (byte) ((1 << remainingBits) - 1); + Unsafe.getUnsafe().putByte(address + fullBytes, mask); } + } - return true; + /** + * Clears the bitmap setting all rows as non-null (direct memory). + * + * @param address bitmap start address + * @param rowCount total number of rows + */ + public static void fillNoneNull(long address, int rowCount) { + int sizeBytes = sizeInBytes(rowCount); + for (int i = 0; i < sizeBytes; i++) { + Unsafe.getUnsafe().putByte(address + i, (byte) 0); + } + } + + /** + * Clears the bitmap setting all rows as non-null (byte array). + * + * @param bitmap bitmap byte array + * @param offset starting offset in array + * @param rowCount total number of rows + */ + public static void fillNoneNull(byte[] bitmap, int offset, int rowCount) { + int sizeBytes = sizeInBytes(rowCount); + for (int i = 0; i < sizeBytes; i++) { + bitmap[offset + i] = 0; + } + } + + /** + * Checks if a specific row is null in the bitmap (from direct memory). + * + * @param address bitmap start address + * @param rowIndex row index to check + * @return true if the row is null + */ + public static boolean isNull(long address, int rowIndex) { + int byteIndex = rowIndex >>> 3; // rowIndex / 8 + int bitIndex = rowIndex & 7; // rowIndex % 8 + byte b = Unsafe.getUnsafe().getByte(address + byteIndex); + return (b & (1 << bitIndex)) != 0; + } + + /** + * Checks if a specific row is null in the bitmap (from byte array). + * + * @param bitmap bitmap byte array + * @param offset starting offset in array + * @param rowIndex row index to check + * @return true if the row is null + */ + public static boolean isNull(byte[] bitmap, int offset, int rowIndex) { + int byteIndex = rowIndex >>> 3; + int bitIndex = rowIndex & 7; + byte b = bitmap[offset + byteIndex]; + return (b & (1 << bitIndex)) != 0; } /** @@ -260,51 +271,40 @@ public static boolean noneNull(long address, int rowCount) { } /** - * Fills the bitmap setting all rows as null (direct memory). + * Sets a row as null in the bitmap (direct memory). * * @param address bitmap start address - * @param rowCount total number of rows + * @param rowIndex row index to set as null */ - public static void fillAllNull(long address, int rowCount) { - int fullBytes = rowCount >>> 3; - int remainingBits = rowCount & 7; - - // Fill full bytes with all 1s - for (int i = 0; i < fullBytes; i++) { - Unsafe.getUnsafe().putByte(address + i, (byte) 0xFF); - } - - // Set remaining bits in last byte - if (remainingBits > 0) { - byte mask = (byte) ((1 << remainingBits) - 1); - Unsafe.getUnsafe().putByte(address + fullBytes, mask); - } + public static void setNull(long address, int rowIndex) { + int byteIndex = rowIndex >>> 3; + int bitIndex = rowIndex & 7; + long addr = address + byteIndex; + byte b = Unsafe.getUnsafe().getByte(addr); + b |= (1 << bitIndex); + Unsafe.getUnsafe().putByte(addr, b); } /** - * Clears the bitmap setting all rows as non-null (direct memory). + * Sets a row as null in the bitmap (byte array). * - * @param address bitmap start address - * @param rowCount total number of rows + * @param bitmap bitmap byte array + * @param offset starting offset in array + * @param rowIndex row index to set as null */ - public static void fillNoneNull(long address, int rowCount) { - int sizeBytes = sizeInBytes(rowCount); - for (int i = 0; i < sizeBytes; i++) { - Unsafe.getUnsafe().putByte(address + i, (byte) 0); - } + public static void setNull(byte[] bitmap, int offset, int rowIndex) { + int byteIndex = rowIndex >>> 3; + int bitIndex = rowIndex & 7; + bitmap[offset + byteIndex] |= (1 << bitIndex); } /** - * Clears the bitmap setting all rows as non-null (byte array). + * Calculates the size in bytes needed for a null bitmap. * - * @param bitmap bitmap byte array - * @param offset starting offset in array - * @param rowCount total number of rows + * @param rowCount number of rows + * @return bitmap size in bytes */ - public static void fillNoneNull(byte[] bitmap, int offset, int rowCount) { - int sizeBytes = sizeInBytes(rowCount); - for (int i = 0; i < sizeBytes; i++) { - bitmap[offset + i] = 0; - } + public static int sizeInBytes(long rowCount) { + return (int) ((rowCount + 7) / 8); } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java index 9edc3f6..dd6388a 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java @@ -43,199 +43,21 @@ */ public final class QwpSchemaHash { + // Default seed (0 for ILP v4) + private static final long DEFAULT_SEED = 0L; // XXHash64 constants private static final long PRIME64_1 = 0x9E3779B185EBCA87L; private static final long PRIME64_2 = 0xC2B2AE3D27D4EB4FL; + // Thread-local Hasher to avoid allocation on every computeSchemaHash call + private static final ThreadLocal HASHER_POOL = ThreadLocal.withInitial(Hasher::new); private static final long PRIME64_3 = 0x165667B19E3779F9L; private static final long PRIME64_4 = 0x85EBCA77C2B2AE63L; private static final long PRIME64_5 = 0x27D4EB2F165667C5L; - // Default seed (0 for ILP v4) - private static final long DEFAULT_SEED = 0L; - - // Thread-local Hasher to avoid allocation on every computeSchemaHash call - private static final ThreadLocal HASHER_POOL = ThreadLocal.withInitial(Hasher::new); - private QwpSchemaHash() { // utility class } - /** - * Computes XXHash64 of a byte array. - * - * @param data the data to hash - * @return the 64-bit hash value - */ - public static long hash(byte[] data) { - return hash(data, 0, data.length, DEFAULT_SEED); - } - - /** - * Computes XXHash64 of a byte array region. - * - * @param data the data to hash - * @param offset starting offset - * @param length number of bytes to hash - * @return the 64-bit hash value - */ - public static long hash(byte[] data, int offset, int length) { - return hash(data, offset, length, DEFAULT_SEED); - } - - /** - * Computes XXHash64 of a byte array region with custom seed. - * - * @param data the data to hash - * @param offset starting offset - * @param length number of bytes to hash - * @param seed the hash seed - * @return the 64-bit hash value - */ - public static long hash(byte[] data, int offset, int length, long seed) { - long h64; - int end = offset + length; - int pos = offset; - - if (length >= 32) { - int limit = end - 32; - long v1 = seed + PRIME64_1 + PRIME64_2; - long v2 = seed + PRIME64_2; - long v3 = seed; - long v4 = seed - PRIME64_1; - - do { - v1 = round(v1, getLong(data, pos)); - pos += 8; - v2 = round(v2, getLong(data, pos)); - pos += 8; - v3 = round(v3, getLong(data, pos)); - pos += 8; - v4 = round(v4, getLong(data, pos)); - pos += 8; - } while (pos <= limit); - - h64 = Long.rotateLeft(v1, 1) + Long.rotateLeft(v2, 7) + - Long.rotateLeft(v3, 12) + Long.rotateLeft(v4, 18); - h64 = mergeRound(h64, v1); - h64 = mergeRound(h64, v2); - h64 = mergeRound(h64, v3); - h64 = mergeRound(h64, v4); - } else { - h64 = seed + PRIME64_5; - } - - h64 += length; - - // Process remaining 8-byte blocks - while (pos + 8 <= end) { - long k1 = getLong(data, pos); - k1 *= PRIME64_2; - k1 = Long.rotateLeft(k1, 31); - k1 *= PRIME64_1; - h64 ^= k1; - h64 = Long.rotateLeft(h64, 27) * PRIME64_1 + PRIME64_4; - pos += 8; - } - - // Process remaining 4-byte block - if (pos + 4 <= end) { - h64 ^= (getInt(data, pos) & 0xFFFFFFFFL) * PRIME64_1; - h64 = Long.rotateLeft(h64, 23) * PRIME64_2 + PRIME64_3; - pos += 4; - } - - // Process remaining bytes - while (pos < end) { - h64 ^= (data[pos] & 0xFFL) * PRIME64_5; - h64 = Long.rotateLeft(h64, 11) * PRIME64_1; - pos++; - } - - return avalanche(h64); - } - - /** - * Computes XXHash64 of direct memory. - * - * @param address start address - * @param length number of bytes - * @return the 64-bit hash value - */ - public static long hash(long address, long length) { - return hash(address, length, DEFAULT_SEED); - } - - /** - * Computes XXHash64 of direct memory with custom seed. - * - * @param address start address - * @param length number of bytes - * @param seed the hash seed - * @return the 64-bit hash value - */ - public static long hash(long address, long length, long seed) { - long h64; - long end = address + length; - long pos = address; - - if (length >= 32) { - long limit = end - 32; - long v1 = seed + PRIME64_1 + PRIME64_2; - long v2 = seed + PRIME64_2; - long v3 = seed; - long v4 = seed - PRIME64_1; - - do { - v1 = round(v1, Unsafe.getUnsafe().getLong(pos)); - pos += 8; - v2 = round(v2, Unsafe.getUnsafe().getLong(pos)); - pos += 8; - v3 = round(v3, Unsafe.getUnsafe().getLong(pos)); - pos += 8; - v4 = round(v4, Unsafe.getUnsafe().getLong(pos)); - pos += 8; - } while (pos <= limit); - - h64 = Long.rotateLeft(v1, 1) + Long.rotateLeft(v2, 7) + - Long.rotateLeft(v3, 12) + Long.rotateLeft(v4, 18); - h64 = mergeRound(h64, v1); - h64 = mergeRound(h64, v2); - h64 = mergeRound(h64, v3); - h64 = mergeRound(h64, v4); - } else { - h64 = seed + PRIME64_5; - } - - h64 += length; - - // Process remaining 8-byte blocks - while (pos + 8 <= end) { - long k1 = Unsafe.getUnsafe().getLong(pos); - k1 *= PRIME64_2; - k1 = Long.rotateLeft(k1, 31); - k1 *= PRIME64_1; - h64 ^= k1; - h64 = Long.rotateLeft(h64, 27) * PRIME64_1 + PRIME64_4; - pos += 8; - } - - // Process remaining 4-byte block - if (pos + 4 <= end) { - h64 ^= (Unsafe.getUnsafe().getInt(pos) & 0xFFFFFFFFL) * PRIME64_1; - h64 = Long.rotateLeft(h64, 23) * PRIME64_2 + PRIME64_3; - pos += 4; - } - - // Process remaining bytes - while (pos < end) { - h64 ^= (Unsafe.getUnsafe().getByte(pos) & 0xFFL) * PRIME64_5; - h64 = Long.rotateLeft(h64, 11) * PRIME64_1; - pos++; - } - - return avalanche(h64); - } - /** * Computes the schema hash for ILP v4. *

@@ -377,18 +199,180 @@ public static long computeSchemaHashDirect(io.questdb.client.std.ObjList= 32) { + long limit = end - 32; + long v1 = seed + PRIME64_1 + PRIME64_2; + long v2 = seed + PRIME64_2; + long v3 = seed; + long v4 = seed - PRIME64_1; + + do { + v1 = round(v1, Unsafe.getUnsafe().getLong(pos)); + pos += 8; + v2 = round(v2, Unsafe.getUnsafe().getLong(pos)); + pos += 8; + v3 = round(v3, Unsafe.getUnsafe().getLong(pos)); + pos += 8; + v4 = round(v4, Unsafe.getUnsafe().getLong(pos)); + pos += 8; + } while (pos <= limit); + + h64 = Long.rotateLeft(v1, 1) + Long.rotateLeft(v2, 7) + + Long.rotateLeft(v3, 12) + Long.rotateLeft(v4, 18); + h64 = mergeRound(h64, v1); + h64 = mergeRound(h64, v2); + h64 = mergeRound(h64, v3); + h64 = mergeRound(h64, v4); + } else { + h64 = seed + PRIME64_5; + } + + h64 += length; + + // Process remaining 8-byte blocks + while (pos + 8 <= end) { + long k1 = Unsafe.getUnsafe().getLong(pos); + k1 *= PRIME64_2; + k1 = Long.rotateLeft(k1, 31); + k1 *= PRIME64_1; + h64 ^= k1; + h64 = Long.rotateLeft(h64, 27) * PRIME64_1 + PRIME64_4; + pos += 8; + } + + // Process remaining 4-byte block + if (pos + 4 <= end) { + h64 ^= (Unsafe.getUnsafe().getInt(pos) & 0xFFFFFFFFL) * PRIME64_1; + h64 = Long.rotateLeft(h64, 23) * PRIME64_2 + PRIME64_3; + pos += 4; + } + + // Process remaining bytes + while (pos < end) { + h64 ^= (Unsafe.getUnsafe().getByte(pos) & 0xFFL) * PRIME64_5; + h64 = Long.rotateLeft(h64, 11) * PRIME64_1; + pos++; + } + + return avalanche(h64); } - private static long mergeRound(long acc, long val) { - val = round(0, val); - acc ^= val; - acc = acc * PRIME64_1 + PRIME64_4; - return acc; + /** + * Computes XXHash64 of a byte array. + * + * @param data the data to hash + * @return the 64-bit hash value + */ + public static long hash(byte[] data) { + return hash(data, 0, data.length, DEFAULT_SEED); + } + + /** + * Computes XXHash64 of a byte array region. + * + * @param data the data to hash + * @param offset starting offset + * @param length number of bytes to hash + * @return the 64-bit hash value + */ + public static long hash(byte[] data, int offset, int length) { + return hash(data, offset, length, DEFAULT_SEED); + } + + /** + * Computes XXHash64 of a byte array region with custom seed. + * + * @param data the data to hash + * @param offset starting offset + * @param length number of bytes to hash + * @param seed the hash seed + * @return the 64-bit hash value + */ + public static long hash(byte[] data, int offset, int length, long seed) { + long h64; + int end = offset + length; + int pos = offset; + + if (length >= 32) { + int limit = end - 32; + long v1 = seed + PRIME64_1 + PRIME64_2; + long v2 = seed + PRIME64_2; + long v3 = seed; + long v4 = seed - PRIME64_1; + + do { + v1 = round(v1, getLong(data, pos)); + pos += 8; + v2 = round(v2, getLong(data, pos)); + pos += 8; + v3 = round(v3, getLong(data, pos)); + pos += 8; + v4 = round(v4, getLong(data, pos)); + pos += 8; + } while (pos <= limit); + + h64 = Long.rotateLeft(v1, 1) + Long.rotateLeft(v2, 7) + + Long.rotateLeft(v3, 12) + Long.rotateLeft(v4, 18); + h64 = mergeRound(h64, v1); + h64 = mergeRound(h64, v2); + h64 = mergeRound(h64, v3); + h64 = mergeRound(h64, v4); + } else { + h64 = seed + PRIME64_5; + } + + h64 += length; + + // Process remaining 8-byte blocks + while (pos + 8 <= end) { + long k1 = getLong(data, pos); + k1 *= PRIME64_2; + k1 = Long.rotateLeft(k1, 31); + k1 *= PRIME64_1; + h64 ^= k1; + h64 = Long.rotateLeft(h64, 27) * PRIME64_1 + PRIME64_4; + pos += 8; + } + + // Process remaining 4-byte block + if (pos + 4 <= end) { + h64 ^= (getInt(data, pos) & 0xFFFFFFFFL) * PRIME64_1; + h64 = Long.rotateLeft(h64, 23) * PRIME64_2 + PRIME64_3; + pos += 4; + } + + // Process remaining bytes + while (pos < end) { + h64 ^= (data[pos] & 0xFFL) * PRIME64_5; + h64 = Long.rotateLeft(h64, 11) * PRIME64_1; + pos++; + } + + return avalanche(h64); + } + + /** + * Computes XXHash64 of direct memory. + * + * @param address start address + * @param length number of bytes + * @return the 64-bit hash value + */ + public static long hash(long address, long length) { + return hash(address, length, DEFAULT_SEED); } private static long avalanche(long h64) { @@ -400,6 +384,13 @@ private static long avalanche(long h64) { return h64; } + private static int getInt(byte[] data, int pos) { + return (data[pos] & 0xFF) | + ((data[pos + 1] & 0xFF) << 8) | + ((data[pos + 2] & 0xFF) << 16) | + ((data[pos + 3] & 0xFF) << 24); + } + private static long getLong(byte[] data, int pos) { return ((long) data[pos] & 0xFF) | (((long) data[pos + 1] & 0xFF) << 8) | @@ -411,11 +402,18 @@ private static long getLong(byte[] data, int pos) { (((long) data[pos + 7] & 0xFF) << 56); } - private static int getInt(byte[] data, int pos) { - return (data[pos] & 0xFF) | - ((data[pos + 1] & 0xFF) << 8) | - ((data[pos + 2] & 0xFF) << 16) | - ((data[pos + 3] & 0xFF) << 24); + private static long mergeRound(long acc, long val) { + val = round(0, val); + acc ^= val; + acc = acc * PRIME64_1 + PRIME64_4; + return acc; + } + + private static long round(long acc, long input) { + acc += input * PRIME64_2; + acc = Long.rotateLeft(acc, 31); + acc *= PRIME64_1; + return acc; } /** @@ -425,16 +423,64 @@ private static int getInt(byte[] data, int pos) { * as columns are processed. */ public static class Hasher { - private long v1, v2, v3, v4; - private long totalLen; private final byte[] buffer = new byte[32]; private int bufferPos; private long seed; + private long totalLen; + private long v1, v2, v3, v4; public Hasher() { reset(DEFAULT_SEED); } + /** + * Finalizes and returns the hash value. + * + * @return the 64-bit hash + */ + public long getValue() { + long h64; + + if (totalLen >= 32) { + h64 = Long.rotateLeft(v1, 1) + Long.rotateLeft(v2, 7) + + Long.rotateLeft(v3, 12) + Long.rotateLeft(v4, 18); + h64 = mergeRound(h64, v1); + h64 = mergeRound(h64, v2); + h64 = mergeRound(h64, v3); + h64 = mergeRound(h64, v4); + } else { + h64 = seed + PRIME64_5; + } + + h64 += totalLen; + + // Process buffered data + int pos = 0; + while (pos + 8 <= bufferPos) { + long k1 = getLong(buffer, pos); + k1 *= PRIME64_2; + k1 = Long.rotateLeft(k1, 31); + k1 *= PRIME64_1; + h64 ^= k1; + h64 = Long.rotateLeft(h64, 27) * PRIME64_1 + PRIME64_4; + pos += 8; + } + + if (pos + 4 <= bufferPos) { + h64 ^= (getInt(buffer, pos) & 0xFFFFFFFFL) * PRIME64_1; + h64 = Long.rotateLeft(h64, 23) * PRIME64_2 + PRIME64_3; + pos += 4; + } + + while (pos < bufferPos) { + h64 ^= (buffer[pos] & 0xFFL) * PRIME64_5; + h64 = Long.rotateLeft(h64, 11) * PRIME64_1; + pos++; + } + + return avalanche(h64); + } + /** * Resets the hasher with the given seed. * @@ -450,20 +496,6 @@ public void reset(long seed) { bufferPos = 0; } - /** - * Updates the hash with a single byte. - * - * @param b the byte to add - */ - public void update(byte b) { - buffer[bufferPos++] = b; - totalLen++; - - if (bufferPos == 32) { - processBuffer(); - } - } - /** * Updates the hash with a byte array. * @@ -514,51 +546,17 @@ public void update(byte[] data, int offset, int length) { } /** - * Finalizes and returns the hash value. + * Updates the hash with a single byte. * - * @return the 64-bit hash + * @param b the byte to add */ - public long getValue() { - long h64; - - if (totalLen >= 32) { - h64 = Long.rotateLeft(v1, 1) + Long.rotateLeft(v2, 7) + - Long.rotateLeft(v3, 12) + Long.rotateLeft(v4, 18); - h64 = mergeRound(h64, v1); - h64 = mergeRound(h64, v2); - h64 = mergeRound(h64, v3); - h64 = mergeRound(h64, v4); - } else { - h64 = seed + PRIME64_5; - } - - h64 += totalLen; - - // Process buffered data - int pos = 0; - while (pos + 8 <= bufferPos) { - long k1 = getLong(buffer, pos); - k1 *= PRIME64_2; - k1 = Long.rotateLeft(k1, 31); - k1 *= PRIME64_1; - h64 ^= k1; - h64 = Long.rotateLeft(h64, 27) * PRIME64_1 + PRIME64_4; - pos += 8; - } - - if (pos + 4 <= bufferPos) { - h64 ^= (getInt(buffer, pos) & 0xFFFFFFFFL) * PRIME64_1; - h64 = Long.rotateLeft(h64, 23) * PRIME64_2 + PRIME64_3; - pos += 4; - } + public void update(byte b) { + buffer[bufferPos++] = b; + totalLen++; - while (pos < bufferPos) { - h64 ^= (buffer[pos] & 0xFFL) * PRIME64_5; - h64 = Long.rotateLeft(h64, 11) * PRIME64_1; - pos++; + if (bufferPos == 32) { + processBuffer(); } - - return avalanche(h64); } private void processBuffer() { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index b56cd28..4e47c1b 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -294,23 +294,15 @@ static int elementSize(byte type) { * Helper class to capture array data from DoubleArray/LongArray.appendToBufPtr(). */ private static class ArrayCapture implements ArrayBufferAppender { + final int[] shape = new int[32]; double[] doubleData; int doubleDataOffset; - private boolean forLong; long[] longData; int longDataOffset; byte nDims; - final int[] shape = new int[32]; + private boolean forLong; private int shapeIndex; - void reset(boolean forLong) { - this.forLong = forLong; - shapeIndex = 0; - nDims = 0; - doubleDataOffset = 0; - longDataOffset = 0; - } - @Override public void putBlockOfBytes(long from, long len) { int count = (int) (len / 8); @@ -373,6 +365,14 @@ public void putLong(long value) { longData[longDataOffset++] = value; } } + + void reset(boolean forLong) { + this.forLong = forLong; + shapeIndex = 0; + nDims = 0; + doubleDataOffset = 0; + longDataOffset = 0; + } } /** diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpVarint.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpVarint.java index 3b98a70..ad675f4 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpVarint.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpVarint.java @@ -60,55 +60,6 @@ private QwpVarint() { // utility class } - /** - * Calculates the number of bytes needed to encode the given value. - * - * @param value the value to measure (treated as unsigned) - * @return number of bytes needed (1-10) - */ - public static int encodedLength(long value) { - if (value == 0) { - return 1; - } - // Count leading zeros to determine the number of bits needed - int bits = 64 - Long.numberOfLeadingZeros(value); - // Each byte encodes 7 bits, round up - return (bits + 6) / 7; - } - - /** - * Encodes a long value as a varint into the given byte array. - * - * @param buf the buffer to write to - * @param pos the position to start writing - * @param value the value to encode (treated as unsigned) - * @return the new position after the encoded bytes - */ - public static int encode(byte[] buf, int pos, long value) { - while ((value & ~DATA_MASK) != 0) { - buf[pos++] = (byte) ((value & DATA_MASK) | CONTINUATION_BIT); - value >>>= 7; - } - buf[pos++] = (byte) value; - return pos; - } - - /** - * Encodes a long value as a varint to direct memory. - * - * @param address the memory address to write to - * @param value the value to encode (treated as unsigned) - * @return the new address after the encoded bytes - */ - public static long encode(long address, long value) { - while ((value & ~DATA_MASK) != 0) { - Unsafe.getUnsafe().putByte(address++, (byte) ((value & DATA_MASK) | CONTINUATION_BIT)); - value >>>= 7; - } - Unsafe.getUnsafe().putByte(address++, (byte) value); - return address; - } - /** * Decodes a varint from the given byte array. * @@ -182,20 +133,6 @@ public static long decode(long address, long limit) { return result; } - /** - * Result holder for decoding varints when the number of bytes consumed matters. - * This class is mutable and should be reused to avoid allocations. - */ - public static class DecodeResult { - public long value; - public int bytesRead; - - public void reset() { - value = 0; - bytesRead = 0; - } - } - /** * Decodes a varint from a byte array and stores both value and bytes consumed. * @@ -258,4 +195,67 @@ public static void decode(long address, long limit, DecodeResult result) { result.value = value; result.bytesRead = bytesRead; } + + /** + * Encodes a long value as a varint to direct memory. + * + * @param address the memory address to write to + * @param value the value to encode (treated as unsigned) + * @return the new address after the encoded bytes + */ + public static long encode(long address, long value) { + while ((value & ~DATA_MASK) != 0) { + Unsafe.getUnsafe().putByte(address++, (byte) ((value & DATA_MASK) | CONTINUATION_BIT)); + value >>>= 7; + } + Unsafe.getUnsafe().putByte(address++, (byte) value); + return address; + } + + /** + * Encodes a long value as a varint into the given byte array. + * + * @param buf the buffer to write to + * @param pos the position to start writing + * @param value the value to encode (treated as unsigned) + * @return the new position after the encoded bytes + */ + public static int encode(byte[] buf, int pos, long value) { + while ((value & ~DATA_MASK) != 0) { + buf[pos++] = (byte) ((value & DATA_MASK) | CONTINUATION_BIT); + value >>>= 7; + } + buf[pos++] = (byte) value; + return pos; + } + + /** + * Calculates the number of bytes needed to encode the given value. + * + * @param value the value to measure (treated as unsigned) + * @return number of bytes needed (1-10) + */ + public static int encodedLength(long value) { + if (value == 0) { + return 1; + } + // Count leading zeros to determine the number of bits needed + int bits = 64 - Long.numberOfLeadingZeros(value); + // Each byte encodes 7 bits, round up + return (bits + 6) / 7; + } + + /** + * Result holder for decoding varints when the number of bytes consumed matters. + * This class is mutable and should be reused to avoid allocations. + */ + public static class DecodeResult { + public int bytesRead; + public long value; + + public void reset() { + value = 0; + bytesRead = 0; + } + } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpZigZag.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpZigZag.java index 44e596d..f113460 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpZigZag.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpZigZag.java @@ -57,22 +57,22 @@ private QwpZigZag() { } /** - * Encodes a signed 64-bit integer using ZigZag encoding. + * Decodes a ZigZag encoded 64-bit integer. * - * @param value the signed value to encode - * @return the ZigZag encoded value (unsigned interpretation) + * @param value the ZigZag encoded value + * @return the original signed value */ - public static long encode(long value) { - return (value << 1) ^ (value >> 63); + public static long decode(long value) { + return (value >>> 1) ^ -(value & 1); } /** - * Decodes a ZigZag encoded 64-bit integer. + * Decodes a ZigZag encoded 32-bit integer. * * @param value the ZigZag encoded value * @return the original signed value */ - public static long decode(long value) { + public static int decode(int value) { return (value >>> 1) ^ -(value & 1); } @@ -87,12 +87,12 @@ public static int encode(int value) { } /** - * Decodes a ZigZag encoded 32-bit integer. + * Encodes a signed 64-bit integer using ZigZag encoding. * - * @param value the ZigZag encoded value - * @return the original signed value + * @param value the signed value to encode + * @return the ZigZag encoded value (unsigned interpretation) */ - public static int decode(int value) { - return (value >>> 1) ^ -(value & 1); + public static long encode(long value) { + return (value << 1) ^ (value >> 63); } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketCloseCode.java b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketCloseCode.java index 629767f..83253aa 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketCloseCode.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketCloseCode.java @@ -29,109 +29,77 @@ */ public final class WebSocketCloseCode { /** - * Normal closure (1000). - * The connection successfully completed whatever purpose for which it was created. + * Abnormal closure (1006). + * Reserved value. MUST NOT be sent in a Close frame. + * Used to indicate that a connection was closed abnormally. */ - public static final int NORMAL_CLOSURE = 1000; - + public static final int ABNORMAL_CLOSURE = 1006; /** * Going away (1001). * The endpoint is going away, e.g., server shutting down or browser navigating away. */ public static final int GOING_AWAY = 1001; - /** - * Protocol error (1002). - * The endpoint is terminating the connection due to a protocol error. + * Internal server error (1011). + * The server encountered an unexpected condition that prevented it from fulfilling the request. */ - public static final int PROTOCOL_ERROR = 1002; - + public static final int INTERNAL_ERROR = 1011; /** - * Unsupported data (1003). - * The endpoint received a type of data it cannot accept. + * Invalid frame payload data (1007). + * The endpoint received a message with invalid payload data. */ - public static final int UNSUPPORTED_DATA = 1003; - + public static final int INVALID_PAYLOAD_DATA = 1007; /** - * Reserved (1004). - * Reserved for future use. + * Mandatory extension (1010). + * The client expected the server to negotiate one or more extensions. */ - public static final int RESERVED = 1004; - + public static final int MANDATORY_EXTENSION = 1010; /** - * No status received (1005). - * Reserved value. MUST NOT be sent in a Close frame. + * Message too big (1009). + * The endpoint received a message that is too big to process. */ - public static final int NO_STATUS_RECEIVED = 1005; - + public static final int MESSAGE_TOO_BIG = 1009; /** - * Abnormal closure (1006). - * Reserved value. MUST NOT be sent in a Close frame. - * Used to indicate that a connection was closed abnormally. + * Normal closure (1000). + * The connection successfully completed whatever purpose for which it was created. */ - public static final int ABNORMAL_CLOSURE = 1006; - + public static final int NORMAL_CLOSURE = 1000; /** - * Invalid frame payload data (1007). - * The endpoint received a message with invalid payload data. + * No status received (1005). + * Reserved value. MUST NOT be sent in a Close frame. */ - public static final int INVALID_PAYLOAD_DATA = 1007; - + public static final int NO_STATUS_RECEIVED = 1005; /** * Policy violation (1008). * The endpoint received a message that violates its policy. */ public static final int POLICY_VIOLATION = 1008; - - /** - * Message too big (1009). - * The endpoint received a message that is too big to process. - */ - public static final int MESSAGE_TOO_BIG = 1009; - /** - * Mandatory extension (1010). - * The client expected the server to negotiate one or more extensions. + * Protocol error (1002). + * The endpoint is terminating the connection due to a protocol error. */ - public static final int MANDATORY_EXTENSION = 1010; - + public static final int PROTOCOL_ERROR = 1002; /** - * Internal server error (1011). - * The server encountered an unexpected condition that prevented it from fulfilling the request. + * Reserved (1004). + * Reserved for future use. */ - public static final int INTERNAL_ERROR = 1011; - + public static final int RESERVED = 1004; /** * TLS handshake (1015). * Reserved value. MUST NOT be sent in a Close frame. * Used to indicate that the connection was closed due to TLS handshake failure. */ public static final int TLS_HANDSHAKE = 1015; + /** + * Unsupported data (1003). + * The endpoint received a type of data it cannot accept. + */ + public static final int UNSUPPORTED_DATA = 1003; private WebSocketCloseCode() { // Constants class } - /** - * Checks if a close code is valid for use in a Close frame. - * Codes 1005 and 1006 are reserved and must not be sent. - * - * @param code the close code - * @return true if the code can be sent in a Close frame - */ - public static boolean isValidForSending(int code) { - if (code < 1000) { - return false; - } - if (code == NO_STATUS_RECEIVED || code == ABNORMAL_CLOSURE || code == TLS_HANDSHAKE) { - return false; - } - // 1000-2999 are defined by RFC 6455 - // 3000-3999 are reserved for libraries/frameworks - // 4000-4999 are reserved for applications - return code < 5000; - } - /** * Returns a human-readable description of the close code. * @@ -175,4 +143,24 @@ public static String describe(int code) { return "Unknown (" + code + ")"; } } + + /** + * Checks if a close code is valid for use in a Close frame. + * Codes 1005 and 1006 are reserved and must not be sent. + * + * @param code the close code + * @return true if the code can be sent in a Close frame + */ + public static boolean isValidForSending(int code) { + if (code < 1000) { + return false; + } + if (code == NO_STATUS_RECEIVED || code == ABNORMAL_CLOSURE || code == TLS_HANDSHAKE) { + return false; + } + // 1000-2999 are defined by RFC 6455 + // 3000-3999 are reserved for libraries/frameworks + // 4000-4999 are reserved for applications + return code < 5000; + } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameParser.java b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameParser.java index fb980ec..e9d2d54 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameParser.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameParser.java @@ -37,57 +37,82 @@ * have its own parser instance. */ public class WebSocketFrameParser { + /** + * Frame completely parsed. + */ + public static final int STATE_COMPLETE = 3; + /** + * Error state - frame is invalid. + */ + public static final int STATE_ERROR = 4; /** * Initial state, waiting for frame header. */ public static final int STATE_HEADER = 0; - /** * Need more data to complete parsing. */ public static final int STATE_NEED_MORE = 1; - /** * Header parsed, need payload data. */ public static final int STATE_NEED_PAYLOAD = 2; - - /** - * Frame completely parsed. - */ - public static final int STATE_COMPLETE = 3; - - /** - * Error state - frame is invalid. - */ - public static final int STATE_ERROR = 4; - // Frame header bits private static final int FIN_BIT = 0x80; - private static final int RSV_BITS = 0x70; - private static final int OPCODE_MASK = 0x0F; - private static final int MASK_BIT = 0x80; private static final int LENGTH_MASK = 0x7F; - + private static final int MASK_BIT = 0x80; // Control frame max payload size (RFC 6455) private static final int MAX_CONTROL_FRAME_PAYLOAD = 125; - + private static final int OPCODE_MASK = 0x0F; + private static final int RSV_BITS = 0x70; + private int errorCode; // Parsed frame data private boolean fin; - private int opcode; - private boolean masked; + private int headerSize; private int maskKey; + private boolean masked; + private int opcode; private long payloadLength; - private int headerSize; - - // Parser state - private int state = STATE_HEADER; - private int errorCode; - // Configuration private boolean serverMode = false; // If true, expect masked frames from clients + // Parser state + private int state = STATE_HEADER; private boolean strictMode = false; // If true, reject non-minimal length encodings + public int getErrorCode() { + return errorCode; + } + + public int getHeaderSize() { + return headerSize; + } + + public int getMaskKey() { + return maskKey; + } + + // Getters + + public int getOpcode() { + return opcode; + } + + public long getPayloadLength() { + return payloadLength; + } + + public int getState() { + return state; + } + + public boolean isFin() { + return fin; + } + + public boolean isMasked() { + return masked; + } + /** * Parses a WebSocket frame from the given buffer. * @@ -235,6 +260,38 @@ public int parse(long buf, long limit) { return (int) totalFrameSize; } + /** + * Resets the parser state for parsing a new frame. + */ + public void reset() { + state = STATE_HEADER; + fin = false; + opcode = 0; + masked = false; + maskKey = 0; + payloadLength = 0; + headerSize = 0; + errorCode = 0; + } + + /** + * Sets the mask key for unmasking. Used in testing. + */ + public void setMaskKey(int maskKey) { + this.maskKey = maskKey; + this.masked = true; + } + + // Setters for configuration + + public void setServerMode(boolean serverMode) { + this.serverMode = serverMode; + } + + public void setStrictMode(boolean strictMode) { + this.strictMode = strictMode; + } + /** * Unmasks the payload data in place. * @@ -273,70 +330,4 @@ public void unmaskPayload(long buf, long len) { i++; } } - - /** - * Resets the parser state for parsing a new frame. - */ - public void reset() { - state = STATE_HEADER; - fin = false; - opcode = 0; - masked = false; - maskKey = 0; - payloadLength = 0; - headerSize = 0; - errorCode = 0; - } - - // Getters - - public boolean isFin() { - return fin; - } - - public int getOpcode() { - return opcode; - } - - public boolean isMasked() { - return masked; - } - - public int getMaskKey() { - return maskKey; - } - - public long getPayloadLength() { - return payloadLength; - } - - public int getHeaderSize() { - return headerSize; - } - - public int getState() { - return state; - } - - public int getErrorCode() { - return errorCode; - } - - // Setters for configuration - - public void setServerMode(boolean serverMode) { - this.serverMode = serverMode; - } - - public void setStrictMode(boolean strictMode) { - this.strictMode = strictMode; - } - - /** - * Sets the mask key for unmasking. Used in testing. - */ - public void setMaskKey(int maskKey) { - this.maskKey = maskKey; - this.masked = true; - } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameWriter.java index e4d423b..892fed4 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameWriter.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameWriter.java @@ -46,73 +46,105 @@ private WebSocketFrameWriter() { } /** - * Writes a WebSocket frame header to the buffer. + * Calculates the header size for a given payload length and masking. * - * @param buf the buffer to write to - * @param fin true if this is the final frame - * @param opcode the frame opcode * @param payloadLength the payload length - * @param masked true if the payload should be masked - * @return the number of bytes written (header size) + * @param masked true if the payload will be masked + * @return the header size in bytes */ - public static int writeHeader(long buf, boolean fin, int opcode, long payloadLength, boolean masked) { - int offset = 0; - - // First byte: FIN + opcode - int byte0 = (fin ? FIN_BIT : 0) | (opcode & 0x0F); - Unsafe.getUnsafe().putByte(buf + offset++, (byte) byte0); - - // Second byte: MASK + payload length - int maskBit = masked ? MASK_BIT : 0; - + public static int headerSize(long payloadLength, boolean masked) { + int size; if (payloadLength <= 125) { - Unsafe.getUnsafe().putByte(buf + offset++, (byte) (maskBit | payloadLength)); + size = 2; } else if (payloadLength <= 65535) { - Unsafe.getUnsafe().putByte(buf + offset++, (byte) (maskBit | 126)); - Unsafe.getUnsafe().putByte(buf + offset++, (byte) ((payloadLength >> 8) & 0xFF)); - Unsafe.getUnsafe().putByte(buf + offset++, (byte) (payloadLength & 0xFF)); + size = 4; } else { - Unsafe.getUnsafe().putByte(buf + offset++, (byte) (maskBit | 127)); - Unsafe.getUnsafe().putLong(buf + offset, Long.reverseBytes(payloadLength)); - offset += 8; + size = 10; } + return masked ? size + 4 : size; + } - return offset; + /** + * Masks payload data in place using XOR with the given mask key. + * + * @param buf the payload buffer + * @param len the payload length + * @param maskKey the 4-byte mask key + */ + public static void maskPayload(long buf, long len, int maskKey) { + // Process 8 bytes at a time when possible + long i = 0; + long longMask = ((long) maskKey << 32) | (maskKey & 0xFFFFFFFFL); + + // Process 8-byte chunks + while (i + 8 <= len) { + long value = Unsafe.getUnsafe().getLong(buf + i); + Unsafe.getUnsafe().putLong(buf + i, value ^ longMask); + i += 8; + } + + // Process 4-byte chunk if remaining + if (i + 4 <= len) { + int value = Unsafe.getUnsafe().getInt(buf + i); + Unsafe.getUnsafe().putInt(buf + i, value ^ maskKey); + i += 4; + } + + // Process remaining bytes (0-3 bytes) - extract mask byte inline to avoid allocation + while (i < len) { + byte b = Unsafe.getUnsafe().getByte(buf + i); + int maskByte = (maskKey >> (((int) i & 3) << 3)) & 0xFF; + Unsafe.getUnsafe().putByte(buf + i, (byte) (b ^ maskByte)); + i++; + } } /** - * Writes a WebSocket frame header with optional mask key. + * Writes a binary frame with payload from a memory address. * - * @param buf the buffer to write to - * @param fin true if this is the final frame - * @param opcode the frame opcode - * @param payloadLength the payload length - * @param maskKey the mask key (only used if masked is true) - * @return the number of bytes written (header size including mask key) + * @param buf the buffer to write to + * @param payloadPtr pointer to the payload data + * @param payloadLen length of payload + * @return the total number of bytes written */ - public static int writeHeader(long buf, boolean fin, int opcode, long payloadLength, int maskKey) { - int offset = writeHeader(buf, fin, opcode, payloadLength, true); - Unsafe.getUnsafe().putInt(buf + offset, maskKey); - return offset + 4; + public static int writeBinaryFrame(long buf, long payloadPtr, int payloadLen) { + int headerLen = writeHeader(buf, true, WebSocketOpcode.BINARY, payloadLen, false); + + // Copy payload from memory + Unsafe.getUnsafe().copyMemory(payloadPtr, buf + headerLen, payloadLen); + + return headerLen + payloadLen; } /** - * Calculates the header size for a given payload length and masking. + * Writes a binary frame header only (for when payload is written separately). * - * @param payloadLength the payload length - * @param masked true if the payload will be masked + * @param buf the buffer to write to + * @param payloadLen length of payload that will follow * @return the header size in bytes */ - public static int headerSize(long payloadLength, boolean masked) { - int size; - if (payloadLength <= 125) { - size = 2; - } else if (payloadLength <= 65535) { - size = 4; - } else { - size = 10; + public static int writeBinaryFrameHeader(long buf, int payloadLen) { + return writeHeader(buf, true, WebSocketOpcode.BINARY, payloadLen, false); + } + + /** + * Writes a complete Close frame to the buffer. + * + * @param buf the buffer to write to + * @param code the close status code + * @param reason the close reason (may be null) + * @return the total number of bytes written (header + payload) + */ + public static int writeCloseFrame(long buf, int code, String reason) { + int payloadLen = 2; // status code + if (reason != null && !reason.isEmpty()) { + payloadLen += reason.getBytes(StandardCharsets.UTF_8).length; } - return masked ? size + 4 : size; + + int headerLen = writeHeader(buf, true, WebSocketOpcode.CLOSE, payloadLen, false); + int payloadOffset = writeClosePayload(buf + headerLen, code, reason); + + return headerLen + payloadOffset; } /** @@ -140,32 +172,63 @@ public static int writeClosePayload(long buf, int code, String reason) { } /** - * Writes a complete Close frame to the buffer. + * Writes a WebSocket frame header to the buffer. * - * @param buf the buffer to write to - * @param code the close status code - * @param reason the close reason (may be null) - * @return the total number of bytes written (header + payload) + * @param buf the buffer to write to + * @param fin true if this is the final frame + * @param opcode the frame opcode + * @param payloadLength the payload length + * @param masked true if the payload should be masked + * @return the number of bytes written (header size) */ - public static int writeCloseFrame(long buf, int code, String reason) { - int payloadLen = 2; // status code - if (reason != null && !reason.isEmpty()) { - payloadLen += reason.getBytes(StandardCharsets.UTF_8).length; + public static int writeHeader(long buf, boolean fin, int opcode, long payloadLength, boolean masked) { + int offset = 0; + + // First byte: FIN + opcode + int byte0 = (fin ? FIN_BIT : 0) | (opcode & 0x0F); + Unsafe.getUnsafe().putByte(buf + offset++, (byte) byte0); + + // Second byte: MASK + payload length + int maskBit = masked ? MASK_BIT : 0; + + if (payloadLength <= 125) { + Unsafe.getUnsafe().putByte(buf + offset++, (byte) (maskBit | payloadLength)); + } else if (payloadLength <= 65535) { + Unsafe.getUnsafe().putByte(buf + offset++, (byte) (maskBit | 126)); + Unsafe.getUnsafe().putByte(buf + offset++, (byte) ((payloadLength >> 8) & 0xFF)); + Unsafe.getUnsafe().putByte(buf + offset++, (byte) (payloadLength & 0xFF)); + } else { + Unsafe.getUnsafe().putByte(buf + offset++, (byte) (maskBit | 127)); + Unsafe.getUnsafe().putLong(buf + offset, Long.reverseBytes(payloadLength)); + offset += 8; } - int headerLen = writeHeader(buf, true, WebSocketOpcode.CLOSE, payloadLen, false); - int payloadOffset = writeClosePayload(buf + headerLen, code, reason); + return offset; + } - return headerLen + payloadOffset; + /** + * Writes a WebSocket frame header with optional mask key. + * + * @param buf the buffer to write to + * @param fin true if this is the final frame + * @param opcode the frame opcode + * @param payloadLength the payload length + * @param maskKey the mask key (only used if masked is true) + * @return the number of bytes written (header size including mask key) + */ + public static int writeHeader(long buf, boolean fin, int opcode, long payloadLength, int maskKey) { + int offset = writeHeader(buf, fin, opcode, payloadLength, true); + Unsafe.getUnsafe().putInt(buf + offset, maskKey); + return offset + 4; } /** * Writes a complete Ping frame to the buffer. * - * @param buf the buffer to write to - * @param payload the ping payload - * @param payloadOff offset into payload array - * @param payloadLen length of payload to write + * @param buf the buffer to write to + * @param payload the ping payload + * @param payloadOff offset into payload array + * @param payloadLen length of payload to write * @return the total number of bytes written */ public static int writePingFrame(long buf, byte[] payload, int payloadOff, int payloadLen) { @@ -182,10 +245,10 @@ public static int writePingFrame(long buf, byte[] payload, int payloadOff, int p /** * Writes a complete Pong frame to the buffer. * - * @param buf the buffer to write to - * @param payload the pong payload (should match the received ping) - * @param payloadOff offset into payload array - * @param payloadLen length of payload to write + * @param buf the buffer to write to + * @param payload the pong payload (should match the received ping) + * @param payloadOff offset into payload array + * @param payloadLen length of payload to write * @return the total number of bytes written */ public static int writePongFrame(long buf, byte[] payload, int payloadOff, int payloadLen) { @@ -215,67 +278,4 @@ public static int writePongFrame(long buf, long payloadPtr, int payloadLen) { return headerLen + payloadLen; } - - /** - * Writes a binary frame with payload from a memory address. - * - * @param buf the buffer to write to - * @param payloadPtr pointer to the payload data - * @param payloadLen length of payload - * @return the total number of bytes written - */ - public static int writeBinaryFrame(long buf, long payloadPtr, int payloadLen) { - int headerLen = writeHeader(buf, true, WebSocketOpcode.BINARY, payloadLen, false); - - // Copy payload from memory - Unsafe.getUnsafe().copyMemory(payloadPtr, buf + headerLen, payloadLen); - - return headerLen + payloadLen; - } - - /** - * Writes a binary frame header only (for when payload is written separately). - * - * @param buf the buffer to write to - * @param payloadLen length of payload that will follow - * @return the header size in bytes - */ - public static int writeBinaryFrameHeader(long buf, int payloadLen) { - return writeHeader(buf, true, WebSocketOpcode.BINARY, payloadLen, false); - } - - /** - * Masks payload data in place using XOR with the given mask key. - * - * @param buf the payload buffer - * @param len the payload length - * @param maskKey the 4-byte mask key - */ - public static void maskPayload(long buf, long len, int maskKey) { - // Process 8 bytes at a time when possible - long i = 0; - long longMask = ((long) maskKey << 32) | (maskKey & 0xFFFFFFFFL); - - // Process 8-byte chunks - while (i + 8 <= len) { - long value = Unsafe.getUnsafe().getLong(buf + i); - Unsafe.getUnsafe().putLong(buf + i, value ^ longMask); - i += 8; - } - - // Process 4-byte chunk if remaining - if (i + 4 <= len) { - int value = Unsafe.getUnsafe().getInt(buf + i); - Unsafe.getUnsafe().putInt(buf + i, value ^ maskKey); - i += 4; - } - - // Process remaining bytes (0-3 bytes) - extract mask byte inline to avoid allocation - while (i < len) { - byte b = Unsafe.getUnsafe().getByte(buf + i); - int maskByte = (maskKey >> (((int) i & 3) << 3)) & 0xFF; - Unsafe.getUnsafe().putByte(buf + i, (byte) (b ^ maskByte)); - i++; - } - } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketHandshake.java b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketHandshake.java index 0bbcddd..e87cb1d 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketHandshake.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketHandshake.java @@ -40,28 +40,24 @@ * generating proper handshake responses. */ public final class WebSocketHandshake { + public static final Utf8String HEADER_CONNECTION = new Utf8String("Connection"); + public static final Utf8String HEADER_SEC_WEBSOCKET_ACCEPT = new Utf8String("Sec-WebSocket-Accept"); + public static final Utf8String HEADER_SEC_WEBSOCKET_KEY = new Utf8String("Sec-WebSocket-Key"); + public static final Utf8String HEADER_SEC_WEBSOCKET_PROTOCOL = new Utf8String("Sec-WebSocket-Protocol"); + public static final Utf8String HEADER_SEC_WEBSOCKET_VERSION = new Utf8String("Sec-WebSocket-Version"); + // Header names (case-insensitive) + public static final Utf8String HEADER_UPGRADE = new Utf8String("Upgrade"); + public static final Utf8String VALUE_UPGRADE = new Utf8String("upgrade"); + // Header values + public static final Utf8String VALUE_WEBSOCKET = new Utf8String("websocket"); /** * The WebSocket magic GUID used in the Sec-WebSocket-Accept calculation. */ public static final String WEBSOCKET_GUID = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"; - /** * The required WebSocket version (RFC 6455). */ public static final int WEBSOCKET_VERSION = 13; - - // Header names (case-insensitive) - public static final Utf8String HEADER_UPGRADE = new Utf8String("Upgrade"); - public static final Utf8String HEADER_CONNECTION = new Utf8String("Connection"); - public static final Utf8String HEADER_SEC_WEBSOCKET_KEY = new Utf8String("Sec-WebSocket-Key"); - public static final Utf8String HEADER_SEC_WEBSOCKET_VERSION = new Utf8String("Sec-WebSocket-Version"); - public static final Utf8String HEADER_SEC_WEBSOCKET_PROTOCOL = new Utf8String("Sec-WebSocket-Protocol"); - public static final Utf8String HEADER_SEC_WEBSOCKET_ACCEPT = new Utf8String("Sec-WebSocket-Accept"); - - // Header values - public static final Utf8String VALUE_WEBSOCKET = new Utf8String("websocket"); - public static final Utf8String VALUE_UPGRADE = new Utf8String("upgrade"); - // Response template private static final byte[] RESPONSE_PREFIX = "HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: ".getBytes(StandardCharsets.US_ASCII); @@ -81,13 +77,45 @@ private WebSocketHandshake() { } /** - * Checks if the given header indicates a WebSocket upgrade request. + * Computes the Sec-WebSocket-Accept value for the given key. * - * @param upgradeHeader the value of the Upgrade header - * @return true if this is a WebSocket upgrade request + * @param key the Sec-WebSocket-Key from the client + * @return the base64-encoded SHA-1 hash to send in the response */ - public static boolean isWebSocketUpgrade(Utf8Sequence upgradeHeader) { - return upgradeHeader != null && Utf8s.equalsIgnoreCaseAscii(upgradeHeader, VALUE_WEBSOCKET); + public static String computeAcceptKey(Utf8Sequence key) { + MessageDigest sha1 = SHA1_DIGEST.get(); + sha1.reset(); + + // Concatenate key + GUID + byte[] keyBytes = new byte[key.size()]; + for (int i = 0; i < key.size(); i++) { + keyBytes[i] = key.byteAt(i); + } + sha1.update(keyBytes); + sha1.update(WEBSOCKET_GUID.getBytes(StandardCharsets.US_ASCII)); + + // Compute SHA-1 hash and base64 encode + byte[] hash = sha1.digest(); + return Base64.getEncoder().encodeToString(hash); + } + + /** + * Computes the Sec-WebSocket-Accept value for the given key string. + * + * @param key the Sec-WebSocket-Key from the client + * @return the base64-encoded SHA-1 hash to send in the response + */ + public static String computeAcceptKey(String key) { + MessageDigest sha1 = SHA1_DIGEST.get(); + sha1.reset(); + + // Concatenate key + GUID + sha1.update(key.getBytes(StandardCharsets.US_ASCII)); + sha1.update(WEBSOCKET_GUID.getBytes(StandardCharsets.US_ASCII)); + + // Compute SHA-1 hash and base64 encode + byte[] hash = sha1.digest(); + return Base64.getEncoder().encodeToString(hash); } /** @@ -106,38 +134,32 @@ public static boolean isConnectionUpgrade(Utf8Sequence connectionHeader) { } /** - * Checks if the sequence contains the given substring (case-insensitive). + * Validates the Sec-WebSocket-Key header. + * The key must be a base64-encoded 16-byte value. + * + * @param key the Sec-WebSocket-Key header value + * @return true if the key is valid */ - private static boolean containsIgnoreCaseAscii(Utf8Sequence seq, Utf8Sequence substring) { - int seqLen = seq.size(); - int subLen = substring.size(); - - if (subLen > seqLen) { + public static boolean isValidKey(Utf8Sequence key) { + if (key == null) { return false; } - if (subLen == 0) { - return true; + // Base64-encoded 16-byte value should be exactly 24 characters + // (16 bytes = 128 bits = 22 base64 chars + 2 padding = 24) + int size = key.size(); + if (size != 24) { + return false; } - - outer: - for (int i = 0; i <= seqLen - subLen; i++) { - for (int j = 0; j < subLen; j++) { - byte a = seq.byteAt(i + j); - byte b = substring.byteAt(j); - // Convert to lowercase for comparison - if (a >= 'A' && a <= 'Z') { - a = (byte) (a + 32); - } - if (b >= 'A' && b <= 'Z') { - b = (byte) (b + 32); - } - if (a != b) { - continue outer; - } + // Basic validation: check that all characters are valid base64 + for (int i = 0; i < size; i++) { + byte b = key.byteAt(i); + boolean valid = (b >= 'A' && b <= 'Z') || (b >= 'a' && b <= 'z') || + (b >= '0' && b <= '9') || b == '+' || b == '/' || b == '='; + if (!valid) { + return false; } - return true; } - return false; + return true; } /** @@ -167,74 +189,101 @@ public static boolean isValidVersion(Utf8Sequence versionHeader) { } /** - * Validates the Sec-WebSocket-Key header. - * The key must be a base64-encoded 16-byte value. + * Checks if the given header indicates a WebSocket upgrade request. * - * @param key the Sec-WebSocket-Key header value - * @return true if the key is valid + * @param upgradeHeader the value of the Upgrade header + * @return true if this is a WebSocket upgrade request */ - public static boolean isValidKey(Utf8Sequence key) { - if (key == null) { - return false; - } - // Base64-encoded 16-byte value should be exactly 24 characters - // (16 bytes = 128 bits = 22 base64 chars + 2 padding = 24) - int size = key.size(); - if (size != 24) { - return false; - } - // Basic validation: check that all characters are valid base64 - for (int i = 0; i < size; i++) { - byte b = key.byteAt(i); - boolean valid = (b >= 'A' && b <= 'Z') || (b >= 'a' && b <= 'z') || - (b >= '0' && b <= '9') || b == '+' || b == '/' || b == '='; - if (!valid) { - return false; - } - } - return true; + public static boolean isWebSocketUpgrade(Utf8Sequence upgradeHeader) { + return upgradeHeader != null && Utf8s.equalsIgnoreCaseAscii(upgradeHeader, VALUE_WEBSOCKET); } /** - * Computes the Sec-WebSocket-Accept value for the given key. + * Returns the size of the handshake response for the given accept key. * - * @param key the Sec-WebSocket-Key from the client - * @return the base64-encoded SHA-1 hash to send in the response + * @param acceptKey the computed accept key + * @return the total response size in bytes */ - public static String computeAcceptKey(Utf8Sequence key) { - MessageDigest sha1 = SHA1_DIGEST.get(); - sha1.reset(); + public static int responseSize(String acceptKey) { + return RESPONSE_PREFIX.length + acceptKey.length() + RESPONSE_SUFFIX.length; + } - // Concatenate key + GUID - byte[] keyBytes = new byte[key.size()]; - for (int i = 0; i < key.size(); i++) { - keyBytes[i] = key.byteAt(i); + /** + * Returns the size of the handshake response with an optional subprotocol. + * + * @param acceptKey the computed accept key + * @param protocol the negotiated subprotocol (may be null or empty) + * @return the total response size in bytes + */ + public static int responseSizeWithProtocol(String acceptKey, String protocol) { + int size = RESPONSE_PREFIX.length + acceptKey.length() + RESPONSE_SUFFIX.length; + if (protocol != null && !protocol.isEmpty()) { + size += "\r\nSec-WebSocket-Protocol: ".length() + protocol.length(); } - sha1.update(keyBytes); - sha1.update(WEBSOCKET_GUID.getBytes(StandardCharsets.US_ASCII)); + return size; + } - // Compute SHA-1 hash and base64 encode - byte[] hash = sha1.digest(); - return Base64.getEncoder().encodeToString(hash); + /** + * Validates all required headers for a WebSocket upgrade request. + * + * @param upgradeHeader the Upgrade header value + * @param connectionHeader the Connection header value + * @param keyHeader the Sec-WebSocket-Key header value + * @param versionHeader the Sec-WebSocket-Version header value + * @return null if valid, or an error message describing the problem + */ + public static String validate( + Utf8Sequence upgradeHeader, + Utf8Sequence connectionHeader, + Utf8Sequence keyHeader, + Utf8Sequence versionHeader + ) { + if (!isWebSocketUpgrade(upgradeHeader)) { + return "Missing or invalid Upgrade header"; + } + if (!isConnectionUpgrade(connectionHeader)) { + return "Missing or invalid Connection header"; + } + if (!isValidKey(keyHeader)) { + return "Missing or invalid Sec-WebSocket-Key header"; + } + if (!isValidVersion(versionHeader)) { + return "Unsupported WebSocket version"; + } + return null; } /** - * Computes the Sec-WebSocket-Accept value for the given key string. + * Writes a 400 Bad Request response. * - * @param key the Sec-WebSocket-Key from the client - * @return the base64-encoded SHA-1 hash to send in the response + * @param buf the buffer to write to + * @param reason the reason for the bad request + * @return the number of bytes written */ - public static String computeAcceptKey(String key) { - MessageDigest sha1 = SHA1_DIGEST.get(); - sha1.reset(); + public static int writeBadRequestResponse(long buf, String reason) { + int offset = 0; - // Concatenate key + GUID - sha1.update(key.getBytes(StandardCharsets.US_ASCII)); - sha1.update(WEBSOCKET_GUID.getBytes(StandardCharsets.US_ASCII)); + byte[] statusLine = "HTTP/1.1 400 Bad Request\r\n".getBytes(StandardCharsets.US_ASCII); + for (byte b : statusLine) { + Unsafe.getUnsafe().putByte(buf + offset++, b); + } - // Compute SHA-1 hash and base64 encode - byte[] hash = sha1.digest(); - return Base64.getEncoder().encodeToString(hash); + byte[] contentType = "Content-Type: text/plain\r\n".getBytes(StandardCharsets.US_ASCII); + for (byte b : contentType) { + Unsafe.getUnsafe().putByte(buf + offset++, b); + } + + byte[] reasonBytes = reason != null ? reason.getBytes(StandardCharsets.UTF_8) : new byte[0]; + byte[] contentLength = ("Content-Length: " + reasonBytes.length + "\r\n\r\n").getBytes(StandardCharsets.US_ASCII); + for (byte b : contentLength) { + Unsafe.getUnsafe().putByte(buf + offset++, b); + } + + for (byte b : reasonBytes) { + Unsafe.getUnsafe().putByte(buf + offset++, b); + } + + return offset; } /** @@ -266,16 +315,6 @@ public static int writeResponse(long buf, String acceptKey) { return offset; } - /** - * Returns the size of the handshake response for the given accept key. - * - * @param acceptKey the computed accept key - * @return the total response size in bytes - */ - public static int responseSize(String acceptKey) { - return RESPONSE_PREFIX.length + acceptKey.length() + RESPONSE_SUFFIX.length; - } - /** * Writes the WebSocket handshake response with an optional subprotocol. * @@ -314,54 +353,6 @@ public static int writeResponseWithProtocol(long buf, String acceptKey, String p return offset; } - /** - * Returns the size of the handshake response with an optional subprotocol. - * - * @param acceptKey the computed accept key - * @param protocol the negotiated subprotocol (may be null or empty) - * @return the total response size in bytes - */ - public static int responseSizeWithProtocol(String acceptKey, String protocol) { - int size = RESPONSE_PREFIX.length + acceptKey.length() + RESPONSE_SUFFIX.length; - if (protocol != null && !protocol.isEmpty()) { - size += "\r\nSec-WebSocket-Protocol: ".length() + protocol.length(); - } - return size; - } - - /** - * Writes a 400 Bad Request response. - * - * @param buf the buffer to write to - * @param reason the reason for the bad request - * @return the number of bytes written - */ - public static int writeBadRequestResponse(long buf, String reason) { - int offset = 0; - - byte[] statusLine = "HTTP/1.1 400 Bad Request\r\n".getBytes(StandardCharsets.US_ASCII); - for (byte b : statusLine) { - Unsafe.getUnsafe().putByte(buf + offset++, b); - } - - byte[] contentType = "Content-Type: text/plain\r\n".getBytes(StandardCharsets.US_ASCII); - for (byte b : contentType) { - Unsafe.getUnsafe().putByte(buf + offset++, b); - } - - byte[] reasonBytes = reason != null ? reason.getBytes(StandardCharsets.UTF_8) : new byte[0]; - byte[] contentLength = ("Content-Length: " + reasonBytes.length + "\r\n\r\n").getBytes(StandardCharsets.US_ASCII); - for (byte b : contentLength) { - Unsafe.getUnsafe().putByte(buf + offset++, b); - } - - for (byte b : reasonBytes) { - Unsafe.getUnsafe().putByte(buf + offset++, b); - } - - return offset; - } - /** * Writes a 426 Upgrade Required response indicating unsupported WebSocket version. * @@ -390,32 +381,37 @@ public static int writeVersionNotSupportedResponse(long buf) { } /** - * Validates all required headers for a WebSocket upgrade request. - * - * @param upgradeHeader the Upgrade header value - * @param connectionHeader the Connection header value - * @param keyHeader the Sec-WebSocket-Key header value - * @param versionHeader the Sec-WebSocket-Version header value - * @return null if valid, or an error message describing the problem + * Checks if the sequence contains the given substring (case-insensitive). */ - public static String validate( - Utf8Sequence upgradeHeader, - Utf8Sequence connectionHeader, - Utf8Sequence keyHeader, - Utf8Sequence versionHeader - ) { - if (!isWebSocketUpgrade(upgradeHeader)) { - return "Missing or invalid Upgrade header"; - } - if (!isConnectionUpgrade(connectionHeader)) { - return "Missing or invalid Connection header"; + private static boolean containsIgnoreCaseAscii(Utf8Sequence seq, Utf8Sequence substring) { + int seqLen = seq.size(); + int subLen = substring.size(); + + if (subLen > seqLen) { + return false; } - if (!isValidKey(keyHeader)) { - return "Missing or invalid Sec-WebSocket-Key header"; + if (subLen == 0) { + return true; } - if (!isValidVersion(versionHeader)) { - return "Unsupported WebSocket version"; + + outer: + for (int i = 0; i <= seqLen - subLen; i++) { + for (int j = 0; j < subLen; j++) { + byte a = seq.byteAt(i + j); + byte b = substring.byteAt(j); + // Convert to lowercase for comparison + if (a >= 'A' && a <= 'Z') { + a = (byte) (a + 32); + } + if (b >= 'A' && b <= 'Z') { + b = (byte) (b + 32); + } + if (a != b) { + continue outer; + } + } + return true; } - return null; + return false; } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketOpcode.java b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketOpcode.java index 40466ec..f2fead7 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketOpcode.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketOpcode.java @@ -28,43 +28,38 @@ * WebSocket frame opcodes as defined in RFC 6455. */ public final class WebSocketOpcode { - /** - * Continuation frame (0x0). - * Used for fragmented messages after the initial frame. - */ - public static final int CONTINUATION = 0x00; - - /** - * Text frame (0x1). - * Payload is UTF-8 encoded text. - */ - public static final int TEXT = 0x01; - /** * Binary frame (0x2). * Payload is arbitrary binary data. */ public static final int BINARY = 0x02; - - // Reserved non-control frames: 0x3-0x7 - /** * Connection close frame (0x8). * Indicates that the endpoint wants to close the connection. */ public static final int CLOSE = 0x08; + /** + * Continuation frame (0x0). + * Used for fragmented messages after the initial frame. + */ + public static final int CONTINUATION = 0x00; + // Reserved non-control frames: 0x3-0x7 /** * Ping frame (0x9). * Used for keep-alive and connection health checks. */ public static final int PING = 0x09; - /** * Pong frame (0xA). * Response to a ping frame. */ public static final int PONG = 0x0A; + /** + * Text frame (0x1). + * Payload is UTF-8 encoded text. + */ + public static final int TEXT = 0x01; // Reserved control frames: 0xB-0xF diff --git a/core/src/main/java/io/questdb/client/network/Net.java b/core/src/main/java/io/questdb/client/network/Net.java index 1f35299..a3f7939 100644 --- a/core/src/main/java/io/questdb/client/network/Net.java +++ b/core/src/main/java/io/questdb/client/network/Net.java @@ -106,18 +106,6 @@ public static long getAddrInfo(CharSequence host, int port) { } } - private static long getAddrInfo(DirectUtf8Sequence host, int port) { - return getAddrInfo(host.ptr(), port); - } - - private static long getAddrInfo(long lpszHost, int port) { - long addrInfo = getAddrInfo0(lpszHost, port); - if (addrInfo != -1) { - ADDR_INFO_COUNTER.incrementAndGet(); - } - return addrInfo; - } - public native static int getSndBuf(int fd); public static void init() { @@ -163,6 +151,18 @@ public static long sockaddr(int ipv4address, int port) { private static native void freeSockAddr0(long sockaddr); + private static long getAddrInfo(DirectUtf8Sequence host, int port) { + return getAddrInfo(host.ptr(), port); + } + + private static long getAddrInfo(long lpszHost, int port) { + long addrInfo = getAddrInfo0(lpszHost, port); + if (addrInfo != -1) { + ADDR_INFO_COUNTER.incrementAndGet(); + } + return addrInfo; + } + private static native long getAddrInfo0(long lpszHost, int port); private static native int getEwouldblock(); @@ -186,4 +186,4 @@ public static long sockaddr(int ipv4address, int port) { MMSGHDR_BUFFER_LENGTH_OFFSET = -1L; } } -} \ No newline at end of file +} diff --git a/core/src/main/java/io/questdb/client/std/CharSequenceIntHashMap.java b/core/src/main/java/io/questdb/client/std/CharSequenceIntHashMap.java index d299423..932d1eb 100644 --- a/core/src/main/java/io/questdb/client/std/CharSequenceIntHashMap.java +++ b/core/src/main/java/io/questdb/client/std/CharSequenceIntHashMap.java @@ -157,6 +157,29 @@ public int valueQuick(int index) { return get(list.getQuick(index)); } + private void erase(int index) { + keys[index] = noEntryKey; + values[index] = noEntryValue; + } + + private void move(int from, int to) { + keys[to] = keys[from]; + values[to] = values[from]; + erase(from); + } + + private int probe0(CharSequence key, int index) { + do { + index = (index + 1) & mask; + if (keys[index] == noEntryKey) { + return index; + } + if (Chars.equals(key, keys[index])) { + return -index - 1; + } + } while (true); + } + private void putAt0(int index, CharSequence key, int value) { keys[index] = key; values[index] = value; @@ -183,27 +206,4 @@ private void rehash() { } } } - - private void erase(int index) { - keys[index] = noEntryKey; - values[index] = noEntryValue; - } - - private void move(int from, int to) { - keys[to] = keys[from]; - values[to] = values[from]; - erase(from); - } - - private int probe0(CharSequence key, int index) { - do { - index = (index + 1) & mask; - if (keys[index] == noEntryKey) { - return index; - } - if (Chars.equals(key, keys[index])) { - return -index - 1; - } - } while (true); - } } diff --git a/core/src/main/java/io/questdb/client/std/bytes/DirectByteSink.java b/core/src/main/java/io/questdb/client/std/bytes/DirectByteSink.java index e9f1f23..3d8b130 100644 --- a/core/src/main/java/io/questdb/client/std/bytes/DirectByteSink.java +++ b/core/src/main/java/io/questdb/client/std/bytes/DirectByteSink.java @@ -71,6 +71,7 @@ public long ptr() { return impl; } }; + public DirectByteSink(long initialCapacity, int memoryTag) { this(initialCapacity, memoryTag, false); } @@ -275,4 +276,4 @@ private void setImplPtr(long ptr) { static { Os.init(); } -} \ No newline at end of file +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java index 5eab4bf..88715be 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java @@ -65,34 +65,30 @@ */ public class QwpAllocationTestClient { - // Protocol modes - private static final String PROTOCOL_ILP_TCP = "ilp-tcp"; - private static final String PROTOCOL_ILP_HTTP = "ilp-http"; - private static final String PROTOCOL_QWP_WEBSOCKET = "qwp-websocket"; - - - // Default configuration - private static final String DEFAULT_HOST = "localhost"; - private static final int DEFAULT_ROWS = 80_000_000; private static final int DEFAULT_BATCH_SIZE = 10_000; private static final int DEFAULT_FLUSH_BYTES = 0; // 0 = use protocol default private static final long DEFAULT_FLUSH_INTERVAL_MS = 0; // 0 = use protocol default + // Default configuration + private static final String DEFAULT_HOST = "localhost"; private static final int DEFAULT_IN_FLIGHT_WINDOW = 0; // 0 = use protocol default (8) + private static final int DEFAULT_REPORT_INTERVAL = 1_000_000; + private static final int DEFAULT_ROWS = 80_000_000; private static final int DEFAULT_SEND_QUEUE = 0; // 0 = use protocol default (16) private static final int DEFAULT_WARMUP_ROWS = 100_000; - private static final int DEFAULT_REPORT_INTERVAL = 1_000_000; - - // Pre-computed test data to avoid allocation during the test - private static final String[] SYMBOLS = { - "AAPL", "GOOGL", "MSFT", "AMZN", "META", "NVDA", "TSLA", "BRK.A", "JPM", "JNJ", - "V", "PG", "UNH", "HD", "MA", "DIS", "PYPL", "BAC", "ADBE", "CMCSA" - }; - + private static final String PROTOCOL_ILP_HTTP = "ilp-http"; + // Protocol modes + private static final String PROTOCOL_ILP_TCP = "ilp-tcp"; + private static final String PROTOCOL_QWP_WEBSOCKET = "qwp-websocket"; private static final String[] STRINGS = { "New York", "London", "Tokyo", "Paris", "Berlin", "Sydney", "Toronto", "Singapore", "Hong Kong", "Dubai", "Mumbai", "Shanghai", "Moscow", "Seoul", "Bangkok", "Amsterdam", "Zurich", "Frankfurt", "Milan", "Madrid" }; + // Pre-computed test data to avoid allocation during the test + private static final String[] SYMBOLS = { + "AAPL", "GOOGL", "MSFT", "AMZN", "META", "NVDA", "TSLA", "BRK.A", "JPM", "JNJ", + "V", "PG", "UNH", "HD", "MA", "DIS", "PYPL", "BAC", "ADBE", "CMCSA" + }; public static void main(String[] args) { // Parse command-line options @@ -176,6 +172,61 @@ public static void main(String[] args) { } } + private static Sender createSender(String protocol, String host, int port, + int batchSize, int flushBytes, long flushIntervalMs, + int inFlightWindow, int sendQueue) { + switch (protocol) { + case PROTOCOL_ILP_TCP: + return Sender.builder(Sender.Transport.TCP) + .address(host) + .port(port) + .build(); + case PROTOCOL_ILP_HTTP: + return Sender.builder(Sender.Transport.HTTP) + .address(host) + .port(port) + .autoFlushRows(batchSize) + .build(); + case PROTOCOL_QWP_WEBSOCKET: + Sender.LineSenderBuilder b = Sender.builder(Sender.Transport.WEBSOCKET) + .address(host) + .port(port) + .asyncMode(true); + if (batchSize > 0) b.autoFlushRows(batchSize); + if (flushBytes > 0) b.autoFlushBytes(flushBytes); + if (flushIntervalMs > 0) b.autoFlushIntervalMillis((int) flushIntervalMs); + if (inFlightWindow > 0) b.inFlightWindowSize(inFlightWindow); + if (sendQueue > 0) b.sendQueueCapacity(sendQueue); + return b.build(); + default: + throw new IllegalArgumentException("Unknown protocol: " + protocol + + ". Use one of: ilp-tcp, ilp-http, qwp-websocket"); + } + } + + /** + * Estimates the size of a single row in bytes for throughput calculation. + */ + private static int estimatedRowSize() { + // Rough estimate (binary protocol): + // - 2 symbols: ~10 bytes each = 20 bytes + // - 3 longs: 8 bytes each = 24 bytes + // - 4 doubles: 8 bytes each = 32 bytes + // - 1 string: ~10 bytes average + // - 1 boolean: 1 byte + // - 2 timestamps: 8 bytes each = 16 bytes + // - Overhead: ~20 bytes + // Total: ~123 bytes + return 123; + } + + private static int getDefaultPort(String protocol) { + if (PROTOCOL_ILP_HTTP.equals(protocol) || PROTOCOL_QWP_WEBSOCKET.equals(protocol)) { + return 9000; + } + return 9009; + } + private static void printUsage() { System.out.println("ILP Allocation Test Client"); System.out.println(); @@ -207,17 +258,10 @@ private static void printUsage() { System.out.println(" QwpAllocationTestClient --protocol=ilp-tcp --rows=100000 --no-warmup"); } - private static int getDefaultPort(String protocol) { - if (PROTOCOL_ILP_HTTP.equals(protocol) || PROTOCOL_QWP_WEBSOCKET.equals(protocol)) { - return 9000; - } - return 9009; - } - private static void runTest(String protocol, String host, int port, int totalRows, - int batchSize, int flushBytes, long flushIntervalMs, - int inFlightWindow, int sendQueue, - int warmupRows, int reportInterval) throws IOException { + int batchSize, int flushBytes, long flushIntervalMs, + int inFlightWindow, int sendQueue, + int warmupRows, int reportInterval) throws IOException { System.out.println("Connecting to " + host + ":" + port + "..."); try (Sender sender = createSender(protocol, host, port, batchSize, flushBytes, flushIntervalMs, @@ -289,7 +333,7 @@ private static void runTest(String protocol, String host, int port, int totalRow System.out.println("Batch size: " + String.format("%,d", batchSize)); System.out.println("Total time: " + String.format("%.2f", totalSeconds) + " seconds"); System.out.println("Throughput: " + String.format("%,.0f", rowsPerSecond) + " rows/second"); - System.out.println("Data rate (before compression): " + String.format("%.2f", ((long)totalRows * estimatedRowSize()) / (1024.0 * 1024.0 * totalSeconds)) + " MB/s (estimated)"); + System.out.println("Data rate (before compression): " + String.format("%.2f", ((long) totalRows * estimatedRowSize()) / (1024.0 * 1024.0 * totalSeconds)) + " MB/s (estimated)"); } catch (InterruptedException e) { Thread.currentThread().interrupt(); @@ -297,38 +341,6 @@ private static void runTest(String protocol, String host, int port, int totalRow } } - private static Sender createSender(String protocol, String host, int port, - int batchSize, int flushBytes, long flushIntervalMs, - int inFlightWindow, int sendQueue) { - switch (protocol) { - case PROTOCOL_ILP_TCP: - return Sender.builder(Sender.Transport.TCP) - .address(host) - .port(port) - .build(); - case PROTOCOL_ILP_HTTP: - return Sender.builder(Sender.Transport.HTTP) - .address(host) - .port(port) - .autoFlushRows(batchSize) - .build(); - case PROTOCOL_QWP_WEBSOCKET: - Sender.LineSenderBuilder b = Sender.builder(Sender.Transport.WEBSOCKET) - .address(host) - .port(port) - .asyncMode(true); - if (batchSize > 0) b.autoFlushRows(batchSize); - if (flushBytes > 0) b.autoFlushBytes(flushBytes); - if (flushIntervalMs > 0) b.autoFlushIntervalMillis((int) flushIntervalMs); - if (inFlightWindow > 0) b.inFlightWindowSize(inFlightWindow); - if (sendQueue > 0) b.sendQueueCapacity(sendQueue); - return b.build(); - default: - throw new IllegalArgumentException("Unknown protocol: " + protocol + - ". Use one of: ilp-tcp, ilp-http, qwp-websocket"); - } - } - private static void sendRow(Sender sender, int rowIndex) { // Base timestamp with small variations long baseTimestamp = 1704067200000000L; // 2024-01-01 00:00:00 UTC in micros @@ -360,20 +372,4 @@ private static void sendRow(Sender sender, int rowIndex) { // Designated timestamp .at(timestamp, ChronoUnit.MICROS); } - - /** - * Estimates the size of a single row in bytes for throughput calculation. - */ - private static int estimatedRowSize() { - // Rough estimate (binary protocol): - // - 2 symbols: ~10 bytes each = 20 bytes - // - 3 longs: 8 bytes each = 24 bytes - // - 4 doubles: 8 bytes each = 32 bytes - // - 1 string: ~10 bytes average - // - 1 boolean: 1 byte - // - 2 timestamps: 8 bytes each = 16 bytes - // - Overhead: ~20 bytes - // Total: ~123 bytes - return 123; - } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/StacBenchmarkClient.java b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/StacBenchmarkClient.java index abbf41f..bed59a3 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/StacBenchmarkClient.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/StacBenchmarkClient.java @@ -61,36 +61,36 @@ */ public class StacBenchmarkClient { - private static final String PROTOCOL_ILP_TCP = "ilp-tcp"; - private static final String PROTOCOL_ILP_HTTP = "ilp-http"; - private static final String PROTOCOL_QWP_WEBSOCKET = "qwp-websocket"; - - private static final String DEFAULT_HOST = "localhost"; - private static final int DEFAULT_ROWS = 80_000_000; private static final int DEFAULT_BATCH_SIZE = 10_000; private static final int DEFAULT_FLUSH_BYTES = 0; private static final long DEFAULT_FLUSH_INTERVAL_MS = 0; + private static final String DEFAULT_HOST = "localhost"; private static final int DEFAULT_IN_FLIGHT_WINDOW = 0; - private static final int DEFAULT_SEND_QUEUE = 0; - private static final int DEFAULT_WARMUP_ROWS = 100_000; private static final int DEFAULT_REPORT_INTERVAL = 1_000_000; + private static final int DEFAULT_ROWS = 80_000_000; + private static final int DEFAULT_SEND_QUEUE = 0; private static final String DEFAULT_TABLE = "q"; - - // 8512 unique 4-letter symbols, as per STAC NYSE benchmark - private static final int SYMBOL_COUNT = 8512; - private static final String[] SYMBOLS = generateSymbols(SYMBOL_COUNT); - + private static final int DEFAULT_WARMUP_ROWS = 100_000; + // Estimated row size for throughput calculation: + // - 1 symbol: ~6 bytes (4-char + overhead) + // - 1 char: 2 bytes + // - 2 floats: 4 bytes each = 8 bytes + // - 2 shorts: 2 bytes each = 4 bytes + // - 1 boolean: 1 byte + // - 1 timestamp: 8 bytes + // - overhead: ~10 bytes + // Total: ~39 bytes + private static final int ESTIMATED_ROW_SIZE = 39; // Exchange codes (single characters) private static final char[] EXCHANGES = {'N', 'Q', 'A', 'B', 'C', 'D', 'P', 'Z'}; // Pre-computed single-char strings to avoid allocation private static final String[] EXCHANGE_STRINGS = new String[EXCHANGES.length]; - - static { - for (int i = 0; i < EXCHANGES.length; i++) { - EXCHANGE_STRINGS[i] = String.valueOf(EXCHANGES[i]); - } - } - + private static final String PROTOCOL_ILP_HTTP = "ilp-http"; + private static final String PROTOCOL_ILP_TCP = "ilp-tcp"; + private static final String PROTOCOL_QWP_WEBSOCKET = "qwp-websocket"; + // 8512 unique 4-letter symbols, as per STAC NYSE benchmark + private static final int SYMBOL_COUNT = 8512; + private static final String[] SYMBOLS = generateSymbols(SYMBOL_COUNT); // Pre-computed bid base prices per symbol (to generate realistic spreads) private static final float[] BASE_PRICES = generateBasePrices(SYMBOL_COUNT); @@ -176,6 +176,80 @@ public static void main(String[] args) { } } + private static Sender createSender(String protocol, String host, int port, + int batchSize, int flushBytes, long flushIntervalMs, + int inFlightWindow, int sendQueue) { + switch (protocol) { + case PROTOCOL_ILP_TCP: + return Sender.builder(Sender.Transport.TCP) + .address(host) + .port(port) + .build(); + case PROTOCOL_ILP_HTTP: + return Sender.builder(Sender.Transport.HTTP) + .address(host) + .port(port) + .autoFlushRows(batchSize) + .build(); + case PROTOCOL_QWP_WEBSOCKET: + Sender.LineSenderBuilder b = Sender.builder(Sender.Transport.WEBSOCKET) + .address(host) + .port(port) + .asyncMode(true); + if (batchSize > 0) b.autoFlushRows(batchSize); + if (flushBytes > 0) b.autoFlushBytes(flushBytes); + if (flushIntervalMs > 0) b.autoFlushIntervalMillis((int) flushIntervalMs); + if (inFlightWindow > 0) b.inFlightWindowSize(inFlightWindow); + if (sendQueue > 0) b.sendQueueCapacity(sendQueue); + return b.build(); + default: + throw new IllegalArgumentException("Unknown protocol: " + protocol + + ". Use one of: ilp-tcp, ilp-http, qwp-websocket"); + } + } + + /** + * Generates pseudo-random base prices for each symbol. + * Prices range from $1 to $500 to simulate realistic stock prices. + */ + private static float[] generateBasePrices(int count) { + float[] prices = new float[count]; + Random rng = new Random(42); // fixed seed for reproducibility + for (int i = 0; i < count; i++) { + prices[i] = 1.0f + rng.nextFloat() * 499.0f; + } + return prices; + } + + /** + * Generates N unique 4-letter symbols. + * Uses combinations of uppercase letters to produce predictable, reproducible symbols. + */ + private static String[] generateSymbols(int count) { + String[] symbols = new String[count]; + int idx = 0; + // 26^4 = 456,976 possible 4-letter combinations, far more than 8512 + outer: + for (char a = 'A'; a <= 'Z' && idx < count; a++) { + for (char b = 'A'; b <= 'Z' && idx < count; b++) { + for (char c = 'A'; c <= 'Z' && idx < count; c++) { + for (char d = 'A'; d <= 'Z' && idx < count; d++) { + symbols[idx++] = new String(new char[]{a, b, c, d}); + if (idx >= count) break outer; + } + } + } + } + return symbols; + } + + private static int getDefaultPort(String protocol) { + if (PROTOCOL_ILP_HTTP.equals(protocol) || PROTOCOL_QWP_WEBSOCKET.equals(protocol)) { + return 9000; + } + return 9009; + } + private static void printUsage() { System.out.println("STAC Benchmark Ingestion Client"); System.out.println(); @@ -212,13 +286,6 @@ private static void printUsage() { System.out.println(" ) timestamp(T) PARTITION BY DAY WAL;"); } - private static int getDefaultPort(String protocol) { - if (PROTOCOL_ILP_HTTP.equals(protocol) || PROTOCOL_QWP_WEBSOCKET.equals(protocol)) { - return 9000; - } - return 9009; - } - private static void runTest(String protocol, String host, int port, String table, int totalRows, int batchSize, int flushBytes, long flushIntervalMs, int inFlightWindow, int sendQueue, @@ -305,38 +372,6 @@ private static void runTest(String protocol, String host, int port, String table } } - private static Sender createSender(String protocol, String host, int port, - int batchSize, int flushBytes, long flushIntervalMs, - int inFlightWindow, int sendQueue) { - switch (protocol) { - case PROTOCOL_ILP_TCP: - return Sender.builder(Sender.Transport.TCP) - .address(host) - .port(port) - .build(); - case PROTOCOL_ILP_HTTP: - return Sender.builder(Sender.Transport.HTTP) - .address(host) - .port(port) - .autoFlushRows(batchSize) - .build(); - case PROTOCOL_QWP_WEBSOCKET: - Sender.LineSenderBuilder b = Sender.builder(Sender.Transport.WEBSOCKET) - .address(host) - .port(port) - .asyncMode(true); - if (batchSize > 0) b.autoFlushRows(batchSize); - if (flushBytes > 0) b.autoFlushBytes(flushBytes); - if (flushIntervalMs > 0) b.autoFlushIntervalMillis((int) flushIntervalMs); - if (inFlightWindow > 0) b.inFlightWindowSize(inFlightWindow); - if (sendQueue > 0) b.sendQueueCapacity(sendQueue); - return b.build(); - default: - throw new IllegalArgumentException("Unknown protocol: " + protocol + - ". Use one of: ilp-tcp, ilp-http, qwp-websocket"); - } - } - /** * Sends a single quote row matching the STAC schema. *

@@ -376,49 +411,9 @@ private static void sendQuoteRow(Sender sender, String table, int rowIndex) { .at(timestamp, ChronoUnit.MICROS); } - /** - * Generates N unique 4-letter symbols. - * Uses combinations of uppercase letters to produce predictable, reproducible symbols. - */ - private static String[] generateSymbols(int count) { - String[] symbols = new String[count]; - int idx = 0; - // 26^4 = 456,976 possible 4-letter combinations, far more than 8512 - outer: - for (char a = 'A'; a <= 'Z' && idx < count; a++) { - for (char b = 'A'; b <= 'Z' && idx < count; b++) { - for (char c = 'A'; c <= 'Z' && idx < count; c++) { - for (char d = 'A'; d <= 'Z' && idx < count; d++) { - symbols[idx++] = new String(new char[]{a, b, c, d}); - if (idx >= count) break outer; - } - } - } - } - return symbols; - } - - /** - * Generates pseudo-random base prices for each symbol. - * Prices range from $1 to $500 to simulate realistic stock prices. - */ - private static float[] generateBasePrices(int count) { - float[] prices = new float[count]; - Random rng = new Random(42); // fixed seed for reproducibility - for (int i = 0; i < count; i++) { - prices[i] = 1.0f + rng.nextFloat() * 499.0f; + static { + for (int i = 0; i < EXCHANGES.length; i++) { + EXCHANGE_STRINGS[i] = String.valueOf(EXCHANGES[i]); } - return prices; } - - // Estimated row size for throughput calculation: - // - 1 symbol: ~6 bytes (4-char + overhead) - // - 1 char: 2 bytes - // - 2 floats: 4 bytes each = 8 bytes - // - 2 shorts: 2 bytes each = 4 bytes - // - 1 boolean: 1 byte - // - 1 timestamp: 8 bytes - // - overhead: ~10 bytes - // Total: ~39 bytes - private static final int ESTIMATED_ROW_SIZE = 39; -} \ No newline at end of file +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/InFlightWindowTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/InFlightWindowTest.java index 023cf13..f31956b 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/InFlightWindowTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/InFlightWindowTest.java @@ -46,86 +46,126 @@ public class InFlightWindowTest { @Test - public void testBasicAddAndAcknowledge() { + public void testAcknowledgeAlreadyAcked() { InFlightWindow window = new InFlightWindow(8, 1000); - assertTrue(window.isEmpty()); - assertEquals(0, window.getInFlightCount()); - - // Add a batch (sequential: 0) window.addInFlight(0); - assertFalse(window.isEmpty()); - assertEquals(1, window.getInFlightCount()); + window.addInFlight(1); - // Acknowledge it (cumulative ACK up to 0) + // ACK up to 1 + assertTrue(window.acknowledge(1)); + assertTrue(window.isEmpty()); + + // ACK for already acknowledged sequence returns true (idempotent) assertTrue(window.acknowledge(0)); + assertTrue(window.acknowledge(1)); assertTrue(window.isEmpty()); - assertEquals(0, window.getInFlightCount()); - assertEquals(1, window.getTotalAcked()); } @Test - public void testMultipleBatches() { - InFlightWindow window = new InFlightWindow(8, 1000); + public void testAcknowledgeUpToAllBatches() { + InFlightWindow window = new InFlightWindow(16, 1000); - // Add sequential batches 0-4 - for (long i = 0; i < 5; i++) { + // Add batches + for (int i = 0; i < 10; i++) { window.addInFlight(i); } - assertEquals(5, window.getInFlightCount()); - // Cumulative ACK up to 2 (acknowledges 0, 1, 2) - assertEquals(3, window.acknowledgeUpTo(2)); - assertEquals(2, window.getInFlightCount()); + // ACK all with high sequence + int acked = window.acknowledgeUpTo(Long.MAX_VALUE); + assertEquals(10, acked); + assertTrue(window.isEmpty()); + } - // Cumulative ACK up to 4 (acknowledges 3, 4) - assertEquals(2, window.acknowledgeUpTo(4)); + @Test + public void testAcknowledgeUpToBasic() { + InFlightWindow window = new InFlightWindow(16, 1000); + + // Add batches 0-9 + for (int i = 0; i < 10; i++) { + window.addInFlight(i); + } + assertEquals(10, window.getInFlightCount()); + + // ACK up to 5 (should remove 0-5, leaving 6-9) + int acked = window.acknowledgeUpTo(5); + assertEquals(6, acked); + assertEquals(4, window.getInFlightCount()); + assertEquals(6, window.getTotalAcked()); + } + + @Test + public void testAcknowledgeUpToEmpty() { + InFlightWindow window = new InFlightWindow(16, 1000); + + // ACK on empty window should be no-op + assertEquals(0, window.acknowledgeUpTo(100)); assertTrue(window.isEmpty()); - assertEquals(5, window.getTotalAcked()); } @Test - public void testAcknowledgeAlreadyAcked() { - InFlightWindow window = new InFlightWindow(8, 1000); + public void testAcknowledgeUpToIdempotent() { + InFlightWindow window = new InFlightWindow(16, 1000); window.addInFlight(0); window.addInFlight(1); + window.addInFlight(2); - // ACK up to 1 - assertTrue(window.acknowledge(1)); + // First ACK + assertEquals(3, window.acknowledgeUpTo(2)); assertTrue(window.isEmpty()); - // ACK for already acknowledged sequence returns true (idempotent) - assertTrue(window.acknowledge(0)); - assertTrue(window.acknowledge(1)); + // Duplicate ACK - should be no-op + assertEquals(0, window.acknowledgeUpTo(2)); + assertTrue(window.isEmpty()); + + // ACK with lower sequence - should be no-op + assertEquals(0, window.acknowledgeUpTo(1)); assertTrue(window.isEmpty()); } @Test - public void testWindowFull() { - InFlightWindow window = new InFlightWindow(3, 1000); + public void testAcknowledgeUpToWakesAwaitEmpty() throws Exception { + InFlightWindow window = new InFlightWindow(16, 5000); - // Fill the window window.addInFlight(0); window.addInFlight(1); window.addInFlight(2); - assertTrue(window.isFull()); - assertEquals(3, window.getInFlightCount()); + AtomicBoolean waiting = new AtomicBoolean(true); + CountDownLatch started = new CountDownLatch(1); + CountDownLatch finished = new CountDownLatch(1); - // Free slots by ACKing - window.acknowledgeUpTo(1); - assertFalse(window.isFull()); - assertEquals(1, window.getInFlightCount()); + // Start thread waiting for empty + Thread waitThread = new Thread(() -> { + started.countDown(); + window.awaitEmpty(); + waiting.set(false); + finished.countDown(); + }); + waitThread.start(); + + assertTrue(started.await(1, TimeUnit.SECONDS)); + Thread.sleep(100); + assertTrue(waiting.get()); + + // Single cumulative ACK clears all + window.acknowledgeUpTo(2); + + assertTrue(finished.await(1, TimeUnit.SECONDS)); + assertFalse(waiting.get()); + assertTrue(window.isEmpty()); } @Test - public void testWindowBlocksWhenFull() throws Exception { - InFlightWindow window = new InFlightWindow(2, 5000); + public void testAcknowledgeUpToWakesBlockedAdder() throws Exception { + InFlightWindow window = new InFlightWindow(3, 5000); // Fill the window window.addInFlight(0); window.addInFlight(1); + window.addInFlight(2); + assertTrue(window.isFull()); AtomicBoolean blocked = new AtomicBoolean(true); CountDownLatch started = new CountDownLatch(1); @@ -134,44 +174,23 @@ public void testWindowBlocksWhenFull() throws Exception { // Start thread that will block Thread addThread = new Thread(() -> { started.countDown(); - window.addInFlight(2); + window.addInFlight(3); blocked.set(false); finished.countDown(); }); addThread.start(); - // Wait for thread to start and block assertTrue(started.await(1, TimeUnit.SECONDS)); Thread.sleep(100); // Give time to block assertTrue(blocked.get()); - // Free a slot - window.acknowledge(0); + // Cumulative ACK frees multiple slots + window.acknowledgeUpTo(1); // Removes 0 and 1 // Thread should complete assertTrue(finished.await(1, TimeUnit.SECONDS)); assertFalse(blocked.get()); - assertEquals(2, window.getInFlightCount()); - } - - @Test - public void testWindowBlocksTimeout() { - InFlightWindow window = new InFlightWindow(2, 100); // 100ms timeout - - // Fill the window - window.addInFlight(0); - window.addInFlight(1); - - // Try to add another - should timeout - long start = System.currentTimeMillis(); - try { - window.addInFlight(2); - fail("Expected timeout exception"); - } catch (LineSenderException e) { - assertTrue(e.getMessage().contains("Timeout")); - } - long elapsed = System.currentTimeMillis() - start; - assertTrue("Should have waited at least 100ms", elapsed >= 90); + assertEquals(2, window.getInFlightCount()); // batch 2 and 3 } @Test @@ -205,23 +224,6 @@ public void testAwaitEmpty() throws Exception { assertFalse(waiting.get()); } - @Test - public void testAwaitEmptyTimeout() { - InFlightWindow window = new InFlightWindow(8, 100); // 100ms timeout - - window.addInFlight(0); - - long start = System.currentTimeMillis(); - try { - window.awaitEmpty(); - fail("Expected timeout exception"); - } catch (LineSenderException e) { - assertTrue(e.getMessage().contains("Timeout")); - } - long elapsed = System.currentTimeMillis() - start; - assertTrue("Should have waited at least 100ms", elapsed >= 90); - } - @Test public void testAwaitEmptyAlreadyEmpty() { InFlightWindow window = new InFlightWindow(8, 1000); @@ -232,58 +234,39 @@ public void testAwaitEmptyAlreadyEmpty() { } @Test - public void testFailBatch() { - InFlightWindow window = new InFlightWindow(8, 1000); - - window.addInFlight(0); - window.addInFlight(1); - - // Fail batch 0 - RuntimeException error = new RuntimeException("Test error"); - window.fail(0, error); - - assertEquals(1, window.getTotalFailed()); - assertNotNull(window.getLastError()); - } - - @Test - public void testFailPropagatesError() { - InFlightWindow window = new InFlightWindow(8, 1000); + public void testAwaitEmptyTimeout() { + InFlightWindow window = new InFlightWindow(8, 100); // 100ms timeout window.addInFlight(0); - window.fail(0, new RuntimeException("Test error")); - - // Subsequent operations should throw - try { - window.addInFlight(1); - fail("Expected exception due to error"); - } catch (LineSenderException e) { - assertTrue(e.getMessage().contains("failed")); - } + long start = System.currentTimeMillis(); try { window.awaitEmpty(); - fail("Expected exception due to error"); + fail("Expected timeout exception"); } catch (LineSenderException e) { - assertTrue(e.getMessage().contains("failed")); + assertTrue(e.getMessage().contains("Timeout")); } + long elapsed = System.currentTimeMillis() - start; + assertTrue("Should have waited at least 100ms", elapsed >= 90); } @Test - public void testFailAllPropagatesError() { + public void testBasicAddAndAcknowledge() { InFlightWindow window = new InFlightWindow(8, 1000); + assertTrue(window.isEmpty()); + assertEquals(0, window.getInFlightCount()); + + // Add a batch (sequential: 0) window.addInFlight(0); - window.addInFlight(1); - window.failAll(new RuntimeException("Transport down")); + assertFalse(window.isEmpty()); + assertEquals(1, window.getInFlightCount()); - try { - window.awaitEmpty(); - fail("Expected exception due to failAll"); - } catch (LineSenderException e) { - assertTrue(e.getMessage().contains("failed")); - assertTrue(e.getMessage().contains("Transport down")); - } + // Acknowledge it (cumulative ACK up to 0) + assertTrue(window.acknowledge(0)); + assertTrue(window.isEmpty()); + assertEquals(0, window.getInFlightCount()); + assertEquals(1, window.getTotalAcked()); } @Test @@ -303,21 +286,6 @@ public void testClearError() { assertEquals(2, window.getInFlightCount()); // 0 and 1 both in window (fail doesn't remove) } - @Test - public void testReset() { - InFlightWindow window = new InFlightWindow(8, 1000); - - window.addInFlight(0); - window.addInFlight(1); - window.fail(2, new RuntimeException("Test")); - - window.reset(); - - assertTrue(window.isEmpty()); - assertNull(window.getLastError()); - assertEquals(0, window.getInFlightCount()); - } - @Test public void testConcurrentAddAndAck() throws Exception { InFlightWindow window = new InFlightWindow(4, 5000); @@ -371,37 +339,138 @@ public void testConcurrentAddAndAck() throws Exception { } @Test - public void testFailWakesBlockedAdder() throws Exception { - InFlightWindow window = new InFlightWindow(2, 5000); - - // Fill the window - window.addInFlight(0); - window.addInFlight(1); - - CountDownLatch started = new CountDownLatch(1); - AtomicReference caught = new AtomicReference<>(); + public void testConcurrentAddAndCumulativeAck() throws Exception { + InFlightWindow window = new InFlightWindow(100, 10000); + int numBatches = 500; + CountDownLatch done = new CountDownLatch(2); + AtomicReference error = new AtomicReference<>(); + AtomicInteger highestAdded = new AtomicInteger(-1); - // Thread that will block on add - Thread addThread = new Thread(() -> { - started.countDown(); + // Sender thread + Thread sender = new Thread(() -> { try { - window.addInFlight(2); - } catch (LineSenderException e) { - caught.set(e); + for (int i = 0; i < numBatches; i++) { + window.addInFlight(i); + highestAdded.set(i); + } + } catch (Throwable t) { + error.set(t); + } finally { + done.countDown(); } }); - addThread.start(); - assertTrue(started.await(1, TimeUnit.SECONDS)); - Thread.sleep(100); // Let it block + // ACK thread using cumulative ACKs + Thread acker = new Thread(() -> { + try { + int lastAcked = -1; + while (lastAcked < numBatches - 1) { + int highest = highestAdded.get(); + if (highest > lastAcked) { + window.acknowledgeUpTo(highest); + lastAcked = highest; + } else { + Thread.sleep(1); + } + } + } catch (Throwable t) { + error.set(t); + } finally { + done.countDown(); + } + }); - // Fail a batch - should wake the blocked thread + sender.start(); + acker.start(); + + assertTrue(done.await(30, TimeUnit.SECONDS)); + assertNull(error.get()); + assertTrue(window.isEmpty()); + assertEquals(numBatches, window.getTotalAcked()); + } + + @Test + public void testDefaultWindowSize() { + InFlightWindow window = new InFlightWindow(); + assertEquals(InFlightWindow.DEFAULT_WINDOW_SIZE, window.getMaxWindowSize()); + } + + @Test + public void testFailAllPropagatesError() { + InFlightWindow window = new InFlightWindow(8, 1000); + + window.addInFlight(0); + window.addInFlight(1); + window.failAll(new RuntimeException("Transport down")); + + try { + window.awaitEmpty(); + fail("Expected exception due to failAll"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("failed")); + assertTrue(e.getMessage().contains("Transport down")); + } + } + + @Test + public void testFailBatch() { + InFlightWindow window = new InFlightWindow(8, 1000); + + window.addInFlight(0); + window.addInFlight(1); + + // Fail batch 0 + RuntimeException error = new RuntimeException("Test error"); + window.fail(0, error); + + assertEquals(1, window.getTotalFailed()); + assertNotNull(window.getLastError()); + } + + @Test + public void testFailPropagatesError() { + InFlightWindow window = new InFlightWindow(8, 1000); + + window.addInFlight(0); window.fail(0, new RuntimeException("Test error")); - addThread.join(1000); - assertFalse(addThread.isAlive()); - assertNotNull(caught.get()); - assertTrue(caught.get().getMessage().contains("failed")); + // Subsequent operations should throw + try { + window.addInFlight(1); + fail("Expected exception due to error"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("failed")); + } + + try { + window.awaitEmpty(); + fail("Expected exception due to error"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("failed")); + } + } + + @Test + public void testFailThenClearThenAdd() { + InFlightWindow window = new InFlightWindow(8, 1000); + + window.addInFlight(0); + window.fail(0, new RuntimeException("Error")); + + // Should not be able to add + try { + window.addInFlight(1); + fail("Expected exception"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("failed")); + } + + // Clear error + window.clearError(); + + // Should work now + window.addInFlight(1); + assertEquals(2, window.getInFlightCount()); } @Test @@ -436,29 +505,38 @@ public void testFailWakesAwaitEmpty() throws Exception { assertTrue(caught.get().getMessage().contains("failed")); } - @Test(expected = IllegalArgumentException.class) - public void testInvalidWindowSize() { - new InFlightWindow(0, 1000); - } - @Test - public void testGetMaxWindowSize() { - InFlightWindow window = new InFlightWindow(16, 1000); - assertEquals(16, window.getMaxWindowSize()); - } + public void testFailWakesBlockedAdder() throws Exception { + InFlightWindow window = new InFlightWindow(2, 5000); - @Test - public void testRapidAddAndAck() { - InFlightWindow window = new InFlightWindow(16, 5000); + // Fill the window + window.addInFlight(0); + window.addInFlight(1); - // Rapid add and ack in same thread - for (int i = 0; i < 10000; i++) { - window.addInFlight(i); - assertTrue(window.acknowledge(i)); - } + CountDownLatch started = new CountDownLatch(1); + AtomicReference caught = new AtomicReference<>(); - assertTrue(window.isEmpty()); - assertEquals(10000, window.getTotalAcked()); + // Thread that will block on add + Thread addThread = new Thread(() -> { + started.countDown(); + try { + window.addInFlight(2); + } catch (LineSenderException e) { + caught.set(e); + } + }); + addThread.start(); + + assertTrue(started.await(1, TimeUnit.SECONDS)); + Thread.sleep(100); // Let it block + + // Fail a batch - should wake the blocked thread + window.fail(0, new RuntimeException("Test error")); + + addThread.join(1000); + assertFalse(addThread.isAlive()); + assertNotNull(caught.get()); + assertTrue(caught.get().getMessage().contains("failed")); } @Test @@ -484,267 +562,154 @@ public void testFillAndDrainRepeatedly() { } @Test - public void testMultipleResets() { - InFlightWindow window = new InFlightWindow(8, 1000); - - for (int cycle = 0; cycle < 10; cycle++) { - window.addInFlight(cycle); - window.reset(); - - assertTrue(window.isEmpty()); - assertNull(window.getLastError()); - } + public void testGetMaxWindowSize() { + InFlightWindow window = new InFlightWindow(16, 1000); + assertEquals(16, window.getMaxWindowSize()); } @Test - public void testFailThenClearThenAdd() { - InFlightWindow window = new InFlightWindow(8, 1000); + public void testHasWindowSpace() { + InFlightWindow window = new InFlightWindow(2, 1000); + assertTrue(window.hasWindowSpace()); window.addInFlight(0); - window.fail(0, new RuntimeException("Error")); - - // Should not be able to add - try { - window.addInFlight(1); - fail("Expected exception"); - } catch (LineSenderException e) { - assertTrue(e.getMessage().contains("failed")); - } - - // Clear error - window.clearError(); - - // Should work now + assertTrue(window.hasWindowSpace()); window.addInFlight(1); - assertEquals(2, window.getInFlightCount()); - } - - @Test - public void testDefaultWindowSize() { - InFlightWindow window = new InFlightWindow(); - assertEquals(InFlightWindow.DEFAULT_WINDOW_SIZE, window.getMaxWindowSize()); - } - - @Test - public void testSmallestPossibleWindow() { - InFlightWindow window = new InFlightWindow(1, 1000); - - window.addInFlight(0); - assertTrue(window.isFull()); + assertFalse(window.hasWindowSpace()); window.acknowledge(0); - assertFalse(window.isFull()); + assertTrue(window.hasWindowSpace()); } @Test - public void testVeryLargeWindow() { - InFlightWindow window = new InFlightWindow(10000, 1000); + public void testHighConcurrencyStress() throws Exception { + InFlightWindow window = new InFlightWindow(8, 30000); + int numBatches = 10000; + CountDownLatch done = new CountDownLatch(2); + AtomicReference error = new AtomicReference<>(); + AtomicInteger highestAdded = new AtomicInteger(-1); - // Add many batches - for (int i = 0; i < 5000; i++) { - window.addInFlight(i); - } - assertEquals(5000, window.getInFlightCount()); - assertFalse(window.isFull()); + // Fast sender thread + Thread sender = new Thread(() -> { + try { + for (int i = 0; i < numBatches; i++) { + window.addInFlight(i); + highestAdded.set(i); + } + } catch (Throwable t) { + error.set(t); + } finally { + done.countDown(); + } + }); - // ACK half - window.acknowledgeUpTo(2499); - assertEquals(2500, window.getInFlightCount()); - } - - @Test - public void testZeroBatchId() { - InFlightWindow window = new InFlightWindow(8, 1000); + // Fast ACK thread + Thread acker = new Thread(() -> { + try { + int lastAcked = -1; + while (lastAcked < numBatches - 1) { + int highest = highestAdded.get(); + if (highest > lastAcked) { + window.acknowledgeUpTo(highest); + lastAcked = highest; + } + // No sleep - maximum contention + } + } catch (Throwable t) { + error.set(t); + } finally { + done.countDown(); + } + }); - window.addInFlight(0); - assertEquals(1, window.getInFlightCount()); + sender.start(); + acker.start(); - assertTrue(window.acknowledge(0)); + assertTrue(done.await(60, TimeUnit.SECONDS)); + if (error.get() != null) { + error.get().printStackTrace(); + } + assertNull(error.get()); assertTrue(window.isEmpty()); + assertEquals(numBatches, window.getTotalAcked()); } - // ==================== CUMULATIVE ACK TESTS ==================== + @Test(expected = IllegalArgumentException.class) + public void testInvalidWindowSize() { + new InFlightWindow(0, 1000); + } @Test - public void testAcknowledgeUpToBasic() { - InFlightWindow window = new InFlightWindow(16, 1000); + public void testMultipleBatches() { + InFlightWindow window = new InFlightWindow(8, 1000); - // Add batches 0-9 - for (int i = 0; i < 10; i++) { + // Add sequential batches 0-4 + for (long i = 0; i < 5; i++) { window.addInFlight(i); } - assertEquals(10, window.getInFlightCount()); - - // ACK up to 5 (should remove 0-5, leaving 6-9) - int acked = window.acknowledgeUpTo(5); - assertEquals(6, acked); - assertEquals(4, window.getInFlightCount()); - assertEquals(6, window.getTotalAcked()); - } - - @Test - public void testAcknowledgeUpToIdempotent() { - InFlightWindow window = new InFlightWindow(16, 1000); - - window.addInFlight(0); - window.addInFlight(1); - window.addInFlight(2); + assertEquals(5, window.getInFlightCount()); - // First ACK + // Cumulative ACK up to 2 (acknowledges 0, 1, 2) assertEquals(3, window.acknowledgeUpTo(2)); - assertTrue(window.isEmpty()); - - // Duplicate ACK - should be no-op - assertEquals(0, window.acknowledgeUpTo(2)); - assertTrue(window.isEmpty()); + assertEquals(2, window.getInFlightCount()); - // ACK with lower sequence - should be no-op - assertEquals(0, window.acknowledgeUpTo(1)); + // Cumulative ACK up to 4 (acknowledges 3, 4) + assertEquals(2, window.acknowledgeUpTo(4)); assertTrue(window.isEmpty()); + assertEquals(5, window.getTotalAcked()); } @Test - public void testAcknowledgeUpToWakesBlockedAdder() throws Exception { - InFlightWindow window = new InFlightWindow(3, 5000); - - // Fill the window - window.addInFlight(0); - window.addInFlight(1); - window.addInFlight(2); - assertTrue(window.isFull()); - - AtomicBoolean blocked = new AtomicBoolean(true); - CountDownLatch started = new CountDownLatch(1); - CountDownLatch finished = new CountDownLatch(1); - - // Start thread that will block - Thread addThread = new Thread(() -> { - started.countDown(); - window.addInFlight(3); - blocked.set(false); - finished.countDown(); - }); - addThread.start(); - - assertTrue(started.await(1, TimeUnit.SECONDS)); - Thread.sleep(100); // Give time to block - assertTrue(blocked.get()); + public void testMultipleResets() { + InFlightWindow window = new InFlightWindow(8, 1000); - // Cumulative ACK frees multiple slots - window.acknowledgeUpTo(1); // Removes 0 and 1 + for (int cycle = 0; cycle < 10; cycle++) { + window.addInFlight(cycle); + window.reset(); - // Thread should complete - assertTrue(finished.await(1, TimeUnit.SECONDS)); - assertFalse(blocked.get()); - assertEquals(2, window.getInFlightCount()); // batch 2 and 3 + assertTrue(window.isEmpty()); + assertNull(window.getLastError()); + } } @Test - public void testAcknowledgeUpToWakesAwaitEmpty() throws Exception { + public void testRapidAddAndAck() { InFlightWindow window = new InFlightWindow(16, 5000); - window.addInFlight(0); - window.addInFlight(1); - window.addInFlight(2); - - AtomicBoolean waiting = new AtomicBoolean(true); - CountDownLatch started = new CountDownLatch(1); - CountDownLatch finished = new CountDownLatch(1); - - // Start thread waiting for empty - Thread waitThread = new Thread(() -> { - started.countDown(); - window.awaitEmpty(); - waiting.set(false); - finished.countDown(); - }); - waitThread.start(); - - assertTrue(started.await(1, TimeUnit.SECONDS)); - Thread.sleep(100); - assertTrue(waiting.get()); - - // Single cumulative ACK clears all - window.acknowledgeUpTo(2); + // Rapid add and ack in same thread + for (int i = 0; i < 10000; i++) { + window.addInFlight(i); + assertTrue(window.acknowledge(i)); + } - assertTrue(finished.await(1, TimeUnit.SECONDS)); - assertFalse(waiting.get()); assertTrue(window.isEmpty()); + assertEquals(10000, window.getTotalAcked()); } @Test - public void testAcknowledgeUpToEmpty() { - InFlightWindow window = new InFlightWindow(16, 1000); - - // ACK on empty window should be no-op - assertEquals(0, window.acknowledgeUpTo(100)); - assertTrue(window.isEmpty()); - } + public void testReset() { + InFlightWindow window = new InFlightWindow(8, 1000); - @Test - public void testAcknowledgeUpToAllBatches() { - InFlightWindow window = new InFlightWindow(16, 1000); + window.addInFlight(0); + window.addInFlight(1); + window.fail(2, new RuntimeException("Test")); - // Add batches - for (int i = 0; i < 10; i++) { - window.addInFlight(i); - } + window.reset(); - // ACK all with high sequence - int acked = window.acknowledgeUpTo(Long.MAX_VALUE); - assertEquals(10, acked); assertTrue(window.isEmpty()); + assertNull(window.getLastError()); + assertEquals(0, window.getInFlightCount()); } @Test - public void testConcurrentAddAndCumulativeAck() throws Exception { - InFlightWindow window = new InFlightWindow(100, 10000); - int numBatches = 500; - CountDownLatch done = new CountDownLatch(2); - AtomicReference error = new AtomicReference<>(); - AtomicInteger highestAdded = new AtomicInteger(-1); - - // Sender thread - Thread sender = new Thread(() -> { - try { - for (int i = 0; i < numBatches; i++) { - window.addInFlight(i); - highestAdded.set(i); - } - } catch (Throwable t) { - error.set(t); - } finally { - done.countDown(); - } - }); - - // ACK thread using cumulative ACKs - Thread acker = new Thread(() -> { - try { - int lastAcked = -1; - while (lastAcked < numBatches - 1) { - int highest = highestAdded.get(); - if (highest > lastAcked) { - window.acknowledgeUpTo(highest); - lastAcked = highest; - } else { - Thread.sleep(1); - } - } - } catch (Throwable t) { - error.set(t); - } finally { - done.countDown(); - } - }); + public void testSmallestPossibleWindow() { + InFlightWindow window = new InFlightWindow(1, 1000); - sender.start(); - acker.start(); + window.addInFlight(0); + assertTrue(window.isFull()); - assertTrue(done.await(30, TimeUnit.SECONDS)); - assertNull(error.get()); - assertTrue(window.isEmpty()); - assertEquals(numBatches, window.getTotalAcked()); + window.acknowledge(0); + assertFalse(window.isFull()); } @Test @@ -764,69 +729,102 @@ public void testTryAddInFlight() { } @Test - public void testHasWindowSpace() { - InFlightWindow window = new InFlightWindow(2, 1000); + public void testVeryLargeWindow() { + InFlightWindow window = new InFlightWindow(10000, 1000); - assertTrue(window.hasWindowSpace()); + // Add many batches + for (int i = 0; i < 5000; i++) { + window.addInFlight(i); + } + assertEquals(5000, window.getInFlightCount()); + assertFalse(window.isFull()); + + // ACK half + window.acknowledgeUpTo(2499); + assertEquals(2500, window.getInFlightCount()); + } + + @Test + public void testWindowBlocksTimeout() { + InFlightWindow window = new InFlightWindow(2, 100); // 100ms timeout + + // Fill the window window.addInFlight(0); - assertTrue(window.hasWindowSpace()); window.addInFlight(1); - assertFalse(window.hasWindowSpace()); - window.acknowledge(0); - assertTrue(window.hasWindowSpace()); + // Try to add another - should timeout + long start = System.currentTimeMillis(); + try { + window.addInFlight(2); + fail("Expected timeout exception"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("Timeout")); + } + long elapsed = System.currentTimeMillis() - start; + assertTrue("Should have waited at least 100ms", elapsed >= 90); } @Test - public void testHighConcurrencyStress() throws Exception { - InFlightWindow window = new InFlightWindow(8, 30000); - int numBatches = 10000; - CountDownLatch done = new CountDownLatch(2); - AtomicReference error = new AtomicReference<>(); - AtomicInteger highestAdded = new AtomicInteger(-1); + public void testWindowBlocksWhenFull() throws Exception { + InFlightWindow window = new InFlightWindow(2, 5000); - // Fast sender thread - Thread sender = new Thread(() -> { - try { - for (int i = 0; i < numBatches; i++) { - window.addInFlight(i); - highestAdded.set(i); - } - } catch (Throwable t) { - error.set(t); - } finally { - done.countDown(); - } - }); + // Fill the window + window.addInFlight(0); + window.addInFlight(1); - // Fast ACK thread - Thread acker = new Thread(() -> { - try { - int lastAcked = -1; - while (lastAcked < numBatches - 1) { - int highest = highestAdded.get(); - if (highest > lastAcked) { - window.acknowledgeUpTo(highest); - lastAcked = highest; - } - // No sleep - maximum contention - } - } catch (Throwable t) { - error.set(t); - } finally { - done.countDown(); - } + AtomicBoolean blocked = new AtomicBoolean(true); + CountDownLatch started = new CountDownLatch(1); + CountDownLatch finished = new CountDownLatch(1); + + // Start thread that will block + Thread addThread = new Thread(() -> { + started.countDown(); + window.addInFlight(2); + blocked.set(false); + finished.countDown(); }); + addThread.start(); - sender.start(); - acker.start(); + // Wait for thread to start and block + assertTrue(started.await(1, TimeUnit.SECONDS)); + Thread.sleep(100); // Give time to block + assertTrue(blocked.get()); - assertTrue(done.await(60, TimeUnit.SECONDS)); - if (error.get() != null) { - error.get().printStackTrace(); - } - assertNull(error.get()); + // Free a slot + window.acknowledge(0); + + // Thread should complete + assertTrue(finished.await(1, TimeUnit.SECONDS)); + assertFalse(blocked.get()); + assertEquals(2, window.getInFlightCount()); + } + + @Test + public void testWindowFull() { + InFlightWindow window = new InFlightWindow(3, 1000); + + // Fill the window + window.addInFlight(0); + window.addInFlight(1); + window.addInFlight(2); + + assertTrue(window.isFull()); + assertEquals(3, window.getInFlightCount()); + + // Free slots by ACKing + window.acknowledgeUpTo(1); + assertFalse(window.isFull()); + assertEquals(1, window.getInFlightCount()); + } + + @Test + public void testZeroBatchId() { + InFlightWindow window = new InFlightWindow(8, 1000); + + window.addInFlight(0); + assertEquals(1, window.getInFlightCount()); + + assertTrue(window.acknowledge(0)); assertTrue(window.isEmpty()); - assertEquals(numBatches, window.getTotalAcked()); } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java index 4b31040..ecb2bb7 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java @@ -67,8 +67,6 @@ public void testAddressNull_fails() { () -> Sender.builder(Sender.Transport.WEBSOCKET).address(null)); } - // ==================== Transport Selection Tests ==================== - @Test public void testAddressWithoutPort_usesDefaultPort9000() { Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) @@ -85,8 +83,6 @@ public void testAsyncModeCanBeSetMultipleTimes() { Assert.assertNotNull(builder); } - // ==================== Address Configuration Tests ==================== - @Test public void testAsyncModeDisabled() { Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) @@ -166,8 +162,6 @@ public void testAutoFlushIntervalMillisDoubleSet_fails() { .autoFlushIntervalMillis(200)); } - // ==================== TLS Configuration Tests ==================== - @Test public void testAutoFlushIntervalMillisNegative_fails() { assertThrows("cannot be negative", @@ -209,8 +203,6 @@ public void testAutoFlushRowsNegative_fails() { .autoFlushRows(-1)); } - // ==================== Async Mode Tests ==================== - @Test public void testAutoFlushRowsZero_disablesRowBasedAutoFlush() { Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) @@ -236,8 +228,6 @@ public void testBufferCapacityDoubleSet_fails() { .bufferCapacity(2048)); } - // ==================== Auto Flush Rows Tests ==================== - @Test public void testBufferCapacityNegative_fails() { assertThrows("cannot be negative", @@ -275,8 +265,6 @@ public void testConnectionRefused() throws Exception { ); } - // ==================== Auto Flush Bytes Tests ==================== - @Test public void testCustomTrustStore_butTlsNotEnabled_fails() { assertThrowsAny( @@ -312,8 +300,6 @@ public void testDuplicateAddresses_fails() { .address(LOCALHOST + ":9000")); } - // ==================== Auto Flush Interval Tests ==================== - @Test @Ignore("TCP authentication is not supported for WebSocket protocol") public void testEnableAuth_notSupported() { @@ -361,8 +347,6 @@ public void testHttpPath_mayNotApply() { Assert.assertNotNull(builder); } - // ==================== In-Flight Window Size Tests ==================== - @Test @Ignore("HTTP timeout is HTTP-specific and may not apply to WebSocket") public void testHttpTimeout_mayNotApply() { @@ -410,8 +394,6 @@ public void testInFlightWindowSizeNegative_fails() { .inFlightWindowSize(-1)); } - // ==================== Send Queue Capacity Tests ==================== - @Test public void testInFlightWindowSizeZero_fails() { assertThrows("must be positive", @@ -450,8 +432,6 @@ public void testInvalidSchema_fails() { assertBadConfig("invalid::addr=localhost:9000;", "invalid schema [schema=invalid, supported-schemas=[http, https, tcp, tcps, ws, wss]]"); } - // ==================== Combined Async Configuration Tests ==================== - @Test public void testMalformedPortInAddress_fails() { assertThrows("cannot parse a port from the address", @@ -467,8 +447,6 @@ public void testMaxBackoff_mayNotApply() { Assert.assertNotNull(builder); } - // ==================== Config String Tests (ws:// and wss://) ==================== - @Test public void testMaxNameLength() { Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) @@ -528,8 +506,6 @@ public void testPortMismatch_fails() { "mismatch"); } - // ==================== Buffer Configuration Tests ==================== - @Test @Ignore("Protocol version is for ILP text protocol, WebSocket uses ILP v4 binary protocol") public void testProtocolVersion_notApplicable() { @@ -558,8 +534,6 @@ public void testSendQueueCapacityDoubleSet_fails() { .sendQueueCapacity(32)); } - // ==================== Unsupported Features (TCP Authentication) ==================== - @Test public void testSendQueueCapacityNegative_fails() { assertThrows("must be positive", @@ -578,8 +552,6 @@ public void testSendQueueCapacityZero_fails() { .sendQueueCapacity(0)); } - // ==================== Unsupported Features (HTTP Token Authentication) ==================== - @Test public void testSendQueueCapacity_withAsyncMode() { Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) @@ -598,30 +570,6 @@ public void testSendQueueCapacity_withoutAsyncMode_fails() { "requires async mode"); } - // ==================== Unsupported Features (Username/Password Authentication) ==================== - - @Test - public void testSyncModeDoesNotAllowInFlightWindowSize() { - assertThrowsAny( - Sender.builder(Sender.Transport.WEBSOCKET) - .address(LOCALHOST) - .asyncMode(false) - .inFlightWindowSize(16), - "requires async mode"); - } - - @Test - public void testSyncModeDoesNotAllowSendQueueCapacity() { - assertThrowsAny( - Sender.builder(Sender.Transport.WEBSOCKET) - .address(LOCALHOST) - .asyncMode(false) - .sendQueueCapacity(32), - "requires async mode"); - } - - // ==================== Unsupported Features (HTTP-specific options) ==================== - @Test public void testSyncModeAutoFlushDefaults() throws Exception { // Regression test: sync-mode connect() must not hardcode autoFlush to 0. @@ -648,6 +596,26 @@ public void testSyncModeAutoFlushDefaults() throws Exception { }); } + @Test + public void testSyncModeDoesNotAllowInFlightWindowSize() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .asyncMode(false) + .inFlightWindowSize(16), + "requires async mode"); + } + + @Test + public void testSyncModeDoesNotAllowSendQueueCapacity() { + assertThrowsAny( + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .asyncMode(false) + .sendQueueCapacity(32), + "requires async mode"); + } + @Test public void testSyncModeIsDefault() { Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) @@ -699,8 +667,6 @@ public void testTlsValidationDisabled_butTlsNotEnabled_fails() { "TLS was not enabled"); } - // ==================== Unsupported Features (Protocol Version) ==================== - @Test public void testUsernamePassword_fails() { assertThrowsAny( @@ -710,8 +676,6 @@ public void testUsernamePassword_fails() { "not yet supported"); } - // ==================== Config String Unsupported Options ==================== - @Test @Ignore("Username/password authentication is not yet supported for WebSocket protocol") public void testUsernamePassword_notYetSupported() { @@ -728,8 +692,6 @@ public void testWsConfigString() throws Exception { assertBadConfig("ws::addr=localhost:" + port + ";", "connect", "Failed"); } - // ==================== Edge Cases ==================== - @Test public void testWsConfigString_missingAddr_fails() throws Exception { int port = findUnusedPort(); @@ -764,8 +726,6 @@ public void testWsConfigString_withUsernamePassword_notYetSupported() { assertBadConfig("ws::addr=localhost:9000;username=user;password=pass;", "not yet supported"); } - // ==================== Connection Tests ==================== - @Test public void testWssConfigString() { assertBadConfig("wss::addr=localhost:9000;tls_verify=unsafe_off;", "connect", "Failed", "SSL"); @@ -776,8 +736,6 @@ public void testWssConfigString_uppercaseNotSupported() { assertBadConfig("WSS::addr=localhost:9000;", "invalid schema"); } - // ==================== Sync vs Async Mode Tests ==================== - @SuppressWarnings("resource") private static void assertBadConfig(String config, String... anyOf) { assertThrowsAny(() -> Sender.fromConfig(config), anyOf); @@ -796,12 +754,6 @@ private static void assertThrowsAny(Sender.LineSenderBuilder builder, String... assertThrowsAny(builder::build, anyOf); } - private static int findUnusedPort() throws Exception { - try (java.net.ServerSocket s = new java.net.ServerSocket(0)) { - return s.getLocalPort(); - } - } - private static void assertThrowsAny(Runnable action, String... anyOf) { try { action.run(); @@ -816,4 +768,10 @@ private static void assertThrowsAny(Runnable action, String... anyOf) { Assert.fail("Expected message containing one of [" + String.join(", ", anyOf) + "] but got: " + msg); } } + + private static int findUnusedPort() throws Exception { + try (java.net.ServerSocket s = new java.net.ServerSocket(0)) { + return s.getLocalPort(); + } + } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/MicrobatchBufferTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/MicrobatchBufferTest.java index cbc81ef..a1f1ecf 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/MicrobatchBufferTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/MicrobatchBufferTest.java @@ -32,7 +32,8 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; public class MicrobatchBufferTest { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java index 46eae77..f51aa54 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java @@ -33,6 +33,15 @@ public class NativeBufferWriterTest { + @Test + public void testEnsureCapacityGrowsBuffer() { + try (NativeBufferWriter writer = new NativeBufferWriter(16)) { + assertEquals(16, writer.getCapacity()); + writer.ensureCapacity(32); + assertTrue(writer.getCapacity() >= 32); + } + } + @Test public void testPatchIntAtLastValidOffset() { try (NativeBufferWriter writer = new NativeBufferWriter(16)) { @@ -77,13 +86,4 @@ public void testSkipThenPatchInt() { assertEquals(0xDEAD, Unsafe.getUnsafe().getInt(writer.getBufferPtr() + 4)); } } - - @Test - public void testEnsureCapacityGrowsBuffer() { - try (NativeBufferWriter writer = new NativeBufferWriter(16)) { - assertEquals(16, writer.getCapacity()); - writer.ensureCapacity(32); - assertTrue(writer.getCapacity() >= 32); - } - } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java index 02106f0..0d0d157 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSenderTest.java @@ -26,9 +26,9 @@ import io.questdb.client.cutlass.line.LineSenderException; import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; -import io.questdb.client.std.Decimal64; import io.questdb.client.std.Decimal128; import io.questdb.client.std.Decimal256; +import io.questdb.client.std.Decimal64; import io.questdb.client.test.cutlass.line.AbstractLineSenderTest; import org.junit.Assert; import org.junit.BeforeClass; @@ -54,412 +54,304 @@ public static void setUpStatic() { AbstractLineSenderTest.setUpStatic(); } - // === BYTE coercion tests === - @Test - public void testByteToBooleanCoercionError() throws Exception { - String table = "test_qwp_byte_to_boolean_error"; + public void testBoolToString() throws Exception { + String table = "test_qwp_bool_to_string"; useTable(table); execute("CREATE TABLE " + table + " (" + - "b BOOLEAN, " + + "s STRING, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .byteColumn("b", (byte) 1) + .boolColumn("s", true) .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected error mentioning BYTE and BOOLEAN but got: " + msg, - msg.contains("BYTE") && msg.contains("BOOLEAN") - ); - } - } - - @Test - public void testByteToCharCoercionError() throws Exception { - String table = "test_qwp_byte_to_char_error"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "c CHAR, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .byteColumn("c", (byte) 65) - .at(1_000_000, ChronoUnit.MICROS); + .boolColumn("s", false) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected error mentioning BYTE and CHAR but got: " + msg, - msg.contains("BYTE") && msg.contains("CHAR") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "s\tts\n" + + "true\t1970-01-01T00:00:01.000000000Z\n" + + "false\t1970-01-01T00:00:02.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testByteToDate() throws Exception { - String table = "test_qwp_byte_to_date"; + public void testBoolToVarchar() throws Exception { + String table = "test_qwp_bool_to_varchar"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DATE, " + + "v VARCHAR, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .byteColumn("d", (byte) 100) + .boolColumn("v", true) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .byteColumn("d", (byte) 0) + .boolColumn("v", false) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "1970-01-01T00:00:00.100000000Z\t1970-01-01T00:00:01.000000000Z\n" + - "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + "v\tts\n" + + "true\t1970-01-01T00:00:01.000000000Z\n" + + "false\t1970-01-01T00:00:02.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); } @Test - public void testByteToDecimal() throws Exception { - String table = "test_qwp_byte_to_decimal"; + public void testBoolean() throws Exception { + String table = "test_qwp_boolean"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(6, 2), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .byteColumn("d", (byte) 42) + .boolColumn("b", true) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .byteColumn("d", (byte) -100) + .boolColumn("b", false) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "42.00\t1970-01-01T00:00:01.000000000Z\n" + - "-100.00\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + "b\ttimestamp\n" + + "true\t1970-01-01T00:00:01.000000000Z\n" + + "false\t1970-01-01T00:00:02.000000000Z\n", + "SELECT b, timestamp FROM " + table + " ORDER BY timestamp"); } @Test - public void testByteToDecimal128() throws Exception { - String table = "test_qwp_byte_to_decimal128"; + public void testBooleanToByteCoercionError() throws Exception { + String table = "test_qwp_boolean_to_byte_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(38, 2), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v BYTE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .byteColumn("d", (byte) 42) + .boolColumn("v", true) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .byteColumn("d", (byte) -1) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write BOOLEAN") && msg.contains("BYTE") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "d\tts\n" + - "42.00\t1970-01-01T00:00:01.000000000Z\n" + - "-1.00\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testByteToDecimal16() throws Exception { - String table = "test_qwp_byte_to_decimal16"; + public void testBooleanToCharCoercionError() throws Exception { + String table = "test_qwp_boolean_to_char_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(4, 1), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v CHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .byteColumn("d", (byte) 42) + .boolColumn("v", true) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .byteColumn("d", (byte) -9) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write BOOLEAN") && msg.contains("CHAR") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "d\tts\n" + - "42.0\t1970-01-01T00:00:01.000000000Z\n" + - "-9.0\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testByteToDecimal256() throws Exception { - String table = "test_qwp_byte_to_decimal256"; + public void testBooleanToDateCoercionError() throws Exception { + String table = "test_qwp_boolean_to_date_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(76, 2), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .byteColumn("d", (byte) 42) + .boolColumn("v", true) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .byteColumn("d", (byte) -1) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write BOOLEAN") && msg.contains("DATE") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "d\tts\n" + - "42.00\t1970-01-01T00:00:01.000000000Z\n" + - "-1.00\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testByteToDecimal64() throws Exception { - String table = "test_qwp_byte_to_decimal64"; + public void testBooleanToDecimalCoercionError() throws Exception { + String table = "test_qwp_boolean_to_decimal_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(18, 2), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v DECIMAL(18,2), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .byteColumn("d", (byte) 42) + .boolColumn("v", true) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .byteColumn("d", (byte) -1) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write BOOLEAN") && msg.contains("DECIMAL") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "d\tts\n" + - "42.00\t1970-01-01T00:00:01.000000000Z\n" + - "-1.00\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testByteToDecimal8() throws Exception { - String table = "test_qwp_byte_to_decimal8"; + public void testBooleanToDoubleCoercionError() throws Exception { + String table = "test_qwp_boolean_to_double_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(2, 1), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v DOUBLE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .byteColumn("d", (byte) 5) + .boolColumn("v", true) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .byteColumn("d", (byte) -9) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write BOOLEAN") && msg.contains("DOUBLE") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "d\tts\n" + - "5.0\t1970-01-01T00:00:01.000000000Z\n" + - "-9.0\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testByteToDouble() throws Exception { - String table = "test_qwp_byte_to_double"; + public void testBooleanToFloatCoercionError() throws Exception { + String table = "test_qwp_boolean_to_float_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DOUBLE, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .byteColumn("d", (byte) 42) + .boolColumn("v", true) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .byteColumn("d", (byte) -100) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "d\tts\n" + - "42.0\t1970-01-01T00:00:01.000000000Z\n" + - "-100.0\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write BOOLEAN") && msg.contains("FLOAT") + ); + } } @Test - public void testByteToFloat() throws Exception { - String table = "test_qwp_byte_to_float"; + public void testBooleanToGeoHashCoercionError() throws Exception { + String table = "test_qwp_boolean_to_geohash_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "f FLOAT, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .byteColumn("f", (byte) 42) + .boolColumn("v", true) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .byteColumn("f", (byte) -100) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write BOOLEAN") && msg.contains("GEOHASH") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "f\tts\n" + - "42.0\t1970-01-01T00:00:01.000000000Z\n" + - "-100.0\t1970-01-01T00:00:02.000000000Z\n", - "SELECT f, ts FROM " + table + " ORDER BY ts"); } @Test - public void testByteToGeoHashCoercionError() throws Exception { - String table = "test_qwp_byte_to_geohash_error"; + public void testBooleanToIntCoercionError() throws Exception { + String table = "test_qwp_boolean_to_int_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "g GEOHASH(4c), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .byteColumn("g", (byte) 42) + .boolColumn("v", true) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected coercion error mentioning BYTE but got: " + msg, - msg.contains("type coercion from BYTE to") && msg.contains("is not supported") + "Expected coercion error but got: " + msg, + msg.contains("cannot write BOOLEAN") && msg.contains("INT") ); } } @Test - public void testByteToInt() throws Exception { - String table = "test_qwp_byte_to_int"; + public void testBooleanToLong256CoercionError() throws Exception { + String table = "test_qwp_boolean_to_long256_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "i INT, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .byteColumn("i", (byte) 42) + .boolColumn("v", true) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .byteColumn("i", Byte.MAX_VALUE) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .byteColumn("i", Byte.MIN_VALUE) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write BOOLEAN") && msg.contains("LONG256") + ); } - - assertTableSizeEventually(table, 3); - assertSqlEventually( - "i\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "127\t1970-01-01T00:00:02.000000000Z\n" + - "-128\t1970-01-01T00:00:03.000000000Z\n", - "SELECT i, ts FROM " + table + " ORDER BY ts"); } @Test - public void testByteToLong() throws Exception { - String table = "test_qwp_byte_to_long"; + public void testBooleanToLongCoercionError() throws Exception { + String table = "test_qwp_boolean_to_long_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "l LONG, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v LONG, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .byteColumn("l", (byte) 42) + .boolColumn("v", true) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .byteColumn("l", Byte.MAX_VALUE) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .byteColumn("l", Byte.MIN_VALUE) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write BOOLEAN") && msg.contains("LONG") + ); } - - assertTableSizeEventually(table, 3); - assertSqlEventually( - "l\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "127\t1970-01-01T00:00:02.000000000Z\n" + - "-128\t1970-01-01T00:00:03.000000000Z\n", - "SELECT l, ts FROM " + table + " ORDER BY ts"); } @Test - public void testByteToLong256CoercionError() throws Exception { - String table = "test_qwp_byte_to_long256_error"; + public void testBooleanToShortCoercionError() throws Exception { + String table = "test_qwp_boolean_to_short_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "v LONG256, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v SHORT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .byteColumn("v", (byte) 42) + .boolColumn("v", true) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -467,375 +359,383 @@ public void testByteToLong256CoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("type coercion from BYTE to LONG256 is not supported") + msg.contains("cannot write BOOLEAN") && msg.contains("SHORT") ); } } @Test - public void testByteToShort() throws Exception { - String table = "test_qwp_byte_to_short"; + public void testBooleanToSymbolCoercionError() throws Exception { + String table = "test_qwp_boolean_to_symbol_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "s SHORT, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v SYMBOL, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .byteColumn("s", (byte) 42) + .boolColumn("v", true) .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write BOOLEAN") && msg.contains("SYMBOL") + ); + } + } + + @Test + public void testBooleanToTimestampCoercionError() throws Exception { + String table = "test_qwp_boolean_to_timestamp_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v TIMESTAMP, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .byteColumn("s", Byte.MIN_VALUE) - .at(2_000_000, ChronoUnit.MICROS); + .boolColumn("v", true) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write BOOLEAN") && msg.contains("TIMESTAMP") + ); + } + } + + @Test + public void testBooleanToTimestampNsCoercionError() throws Exception { + String table = "test_qwp_boolean_to_timestamp_ns_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v TIMESTAMP_NS, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .byteColumn("s", Byte.MAX_VALUE) - .at(3_000_000, ChronoUnit.MICROS); + .boolColumn("v", true) + .at(1_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write BOOLEAN") && msg.contains("TIMESTAMP") + ); } + } - assertTableSizeEventually(table, 3); - assertSqlEventually( - "s\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-128\t1970-01-01T00:00:02.000000000Z\n" + - "127\t1970-01-01T00:00:03.000000000Z\n", - "SELECT s, ts FROM " + table + " ORDER BY ts"); + @Test + public void testBooleanToUuidCoercionError() throws Exception { + String table = "test_qwp_boolean_to_uuid_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .boolColumn("v", true) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write BOOLEAN") && msg.contains("UUID") + ); + } } @Test - public void testByteToString() throws Exception { - String table = "test_qwp_byte_to_string"; + public void testByte() throws Exception { + String table = "test_qwp_byte"; useTable(table); execute("CREATE TABLE " + table + " (" + - "s STRING, " + + "b BYTE, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .byteColumn("s", (byte) 42) + .shortColumn("b", (short) -1) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .byteColumn("s", (byte) -100) + .shortColumn("b", (short) 0) .at(2_000_000, ChronoUnit.MICROS); sender.table(table) - .byteColumn("s", (byte) 0) + .shortColumn("b", (short) 127) .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 3); - assertSqlEventually( - "s\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-100\t1970-01-01T00:00:02.000000000Z\n" + - "0\t1970-01-01T00:00:03.000000000Z\n", - "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testByteToSymbol() throws Exception { - String table = "test_qwp_byte_to_symbol"; + public void testByteToBooleanCoercionError() throws Exception { + String table = "test_qwp_byte_to_boolean_error"; useTable(table); execute("CREATE TABLE " + table + " (" + - "s SYMBOL, " + + "b BOOLEAN, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .byteColumn("s", (byte) 42) + .byteColumn("b", (byte) 1) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .byteColumn("s", (byte) -1) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .byteColumn("s", (byte) 0) - .at(3_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 3); - assertSqlEventually( - "s\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-1\t1970-01-01T00:00:02.000000000Z\n" + - "0\t1970-01-01T00:00:03.000000000Z\n", - "SELECT s, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testByteToTimestamp() throws Exception { - String table = "test_qwp_byte_to_timestamp"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "t TIMESTAMP, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .byteColumn("t", (byte) 100) - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .byteColumn("t", (byte) 0) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected error mentioning BYTE and BOOLEAN but got: " + msg, + msg.contains("BYTE") && msg.contains("BOOLEAN") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "t\tts\n" + - "1970-01-01T00:00:00.000100000Z\t1970-01-01T00:00:01.000000000Z\n" + - "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", - "SELECT t, ts FROM " + table + " ORDER BY ts"); } @Test - public void testByteToUuidCoercionError() throws Exception { - String table = "test_qwp_byte_to_uuid_error"; + public void testByteToCharCoercionError() throws Exception { + String table = "test_qwp_byte_to_char_error"; useTable(table); execute("CREATE TABLE " + table + " (" + - "u UUID, " + + "c CHAR, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .byteColumn("u", (byte) 42) + .byteColumn("c", (byte) 65) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from BYTE to UUID is not supported") + "Expected error mentioning BYTE and CHAR but got: " + msg, + msg.contains("BYTE") && msg.contains("CHAR") ); } } @Test - public void testByteToVarchar() throws Exception { - String table = "test_qwp_byte_to_varchar"; + public void testByteToDate() throws Exception { + String table = "test_qwp_byte_to_date"; useTable(table); execute("CREATE TABLE " + table + " (" + - "v VARCHAR, " + + "d DATE, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .byteColumn("v", (byte) 42) + .byteColumn("d", (byte) 100) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .byteColumn("v", (byte) -100) + .byteColumn("d", (byte) 0) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .byteColumn("v", Byte.MAX_VALUE) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 2); assertSqlEventually( - "v\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-100\t1970-01-01T00:00:02.000000000Z\n" + - "127\t1970-01-01T00:00:03.000000000Z\n", - "SELECT v, ts FROM " + table + " ORDER BY ts"); + "d\tts\n" + + "1970-01-01T00:00:00.100000000Z\t1970-01-01T00:00:01.000000000Z\n" + + "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } - // === Exact Type Match Tests === - @Test - public void testBoolean() throws Exception { - String table = "test_qwp_boolean"; + public void testByteToDecimal() throws Exception { + String table = "test_qwp_byte_to_decimal"; useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(6, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .boolColumn("b", true) + .byteColumn("d", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .boolColumn("b", false) + .byteColumn("d", (byte) -100) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "b\ttimestamp\n" + - "true\t1970-01-01T00:00:01.000000000Z\n" + - "false\t1970-01-01T00:00:02.000000000Z\n", - "SELECT b, timestamp FROM " + table + " ORDER BY timestamp"); + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testByte() throws Exception { - String table = "test_qwp_byte"; + public void testByteToDecimal128() throws Exception { + String table = "test_qwp_byte_to_decimal128"; useTable(table); execute("CREATE TABLE " + table + " (" + - "b BYTE, " + + "d DECIMAL(38, 2), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("b", (short) -1) + .byteColumn("d", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .shortColumn("b", (short) 0) + .byteColumn("d", (byte) -1) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("b", (short) 127) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-1.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testChar() throws Exception { - String table = "test_qwp_char"; + public void testByteToDecimal16() throws Exception { + String table = "test_qwp_byte_to_decimal16"; useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(4, 1), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .charColumn("c", 'A') + .byteColumn("d", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .charColumn("c", 'ü') // ü + .byteColumn("d", (byte) -9) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .charColumn("c", '中') // 中 - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 2); assertSqlEventually( - "c\ttimestamp\n" + - "A\t1970-01-01T00:00:01.000000000Z\n" + - "ü\t1970-01-01T00:00:02.000000000Z\n" + - "中\t1970-01-01T00:00:03.000000000Z\n", - "SELECT c, timestamp FROM " + table + " ORDER BY timestamp"); + "d\tts\n" + + "42.0\t1970-01-01T00:00:01.000000000Z\n" + + "-9.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDecimal() throws Exception { - String table = "test_qwp_decimal"; + public void testByteToDecimal256() throws Exception { + String table = "test_qwp_byte_to_decimal256"; useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(76, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .decimalColumn("d", "123.45") + .byteColumn("d", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .decimalColumn("d", "-999.99") + .byteColumn("d", (byte) -1) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .decimalColumn("d", "0.01") - .at(3_000_000, ChronoUnit.MICROS); - sender.table(table) - .decimalColumn("d", Decimal256.fromLong(42_000, 2)) - .at(4_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 4); + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-1.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDouble() throws Exception { - String table = "test_qwp_double"; + public void testByteToDecimal64() throws Exception { + String table = "test_qwp_byte_to_decimal64"; useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(18, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("d", 42.5) + .byteColumn("d", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .doubleColumn("d", -1.0E10) + .byteColumn("d", (byte) -1) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .doubleColumn("d", Double.MAX_VALUE) - .at(3_000_000, ChronoUnit.MICROS); - sender.table(table) - .doubleColumn("d", Double.MIN_VALUE) - .at(4_000_000, ChronoUnit.MICROS); - // NaN and Inf should be stored as null - sender.table(table) - .doubleColumn("d", Double.NaN) - .at(5_000_000, ChronoUnit.MICROS); - sender.table(table) - .doubleColumn("d", Double.POSITIVE_INFINITY) - .at(6_000_000, ChronoUnit.MICROS); - sender.table(table) - .doubleColumn("d", Double.NEGATIVE_INFINITY) - .at(7_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 7); + assertTableSizeEventually(table, 2); assertSqlEventually( - "d\ttimestamp\n" + - "42.5\t1970-01-01T00:00:01.000000000Z\n" + - "-1.0E10\t1970-01-01T00:00:02.000000000Z\n" + - "1.7976931348623157E308\t1970-01-01T00:00:03.000000000Z\n" + - "4.9E-324\t1970-01-01T00:00:04.000000000Z\n" + - "null\t1970-01-01T00:00:05.000000000Z\n" + - "null\t1970-01-01T00:00:06.000000000Z\n" + - "null\t1970-01-01T00:00:07.000000000Z\n", - "SELECT d, timestamp FROM " + table + " ORDER BY timestamp"); + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-1.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDoubleArray() throws Exception { - String table = "test_qwp_double_array"; + public void testByteToDecimal8() throws Exception { + String table = "test_qwp_byte_to_decimal8"; useTable(table); - - double[] arr1d = createDoubleArray(5); - double[][] arr2d = createDoubleArray(2, 3); - double[][][] arr3d = createDoubleArray(1, 2, 3); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(2, 1), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleArray("a1", arr1d) - .doubleArray("a2", arr2d) - .doubleArray("a3", arr3d) + .byteColumn("d", (byte) 5) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .byteColumn("d", (byte) -9) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 1); + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "5.0\t1970-01-01T00:00:01.000000000Z\n" + + "-9.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDoubleToDecimal() throws Exception { - String table = "test_qwp_double_to_decimal"; + public void testByteToDouble() throws Exception { + String table = "test_qwp_byte_to_double"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(10, 2), " + + "d DOUBLE, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("d", 123.45) + .byteColumn("d", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .doubleColumn("d", -42.10) + .byteColumn("d", (byte) -100) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } @@ -843,221 +743,188 @@ public void testDoubleToDecimal() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( "d\tts\n" + - "123.45\t1970-01-01T00:00:01.000000000Z\n" + - "-42.10\t1970-01-01T00:00:02.000000000Z\n", + "42.0\t1970-01-01T00:00:01.000000000Z\n" + + "-100.0\t1970-01-01T00:00:02.000000000Z\n", "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDoubleToDecimalPrecisionLossError() throws Exception { - String table = "test_qwp_double_to_decimal_prec"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(10, 2), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .doubleColumn("d", 123.456) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected precision loss error but got: " + msg, - msg.contains("cannot be converted to") && msg.contains("123.456") && msg.contains("scale=2") - ); - } - } - - @Test - public void testDoubleToByte() throws Exception { - String table = "test_qwp_double_to_byte"; + public void testByteToFloat() throws Exception { + String table = "test_qwp_byte_to_float"; useTable(table); execute("CREATE TABLE " + table + " (" + - "b BYTE, " + + "f FLOAT, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("b", 42.0) + .byteColumn("f", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .doubleColumn("b", -100.0) + .byteColumn("f", (byte) -100) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "b\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-100\t1970-01-01T00:00:02.000000000Z\n", - "SELECT b, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testDoubleToBytePrecisionLossError() throws Exception { - String table = "test_qwp_double_to_byte_prec"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "b BYTE, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .doubleColumn("b", 42.5) - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected precision loss error but got: " + msg, - msg.contains("loses precision") && msg.contains("42.5") - ); - } + "f\tts\n" + + "42.0\t1970-01-01T00:00:01.000000000Z\n" + + "-100.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT f, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDoubleToByteOverflowError() throws Exception { - String table = "test_qwp_double_to_byte_ovf"; + public void testByteToGeoHashCoercionError() throws Exception { + String table = "test_qwp_byte_to_geohash_error"; useTable(table); execute("CREATE TABLE " + table + " (" + - "b BYTE, " + + "g GEOHASH(4c), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("b", 200.0) + .byteColumn("g", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected overflow error but got: " + msg, - msg.contains("integer value 200 out of range for BYTE") + "Expected coercion error mentioning BYTE but got: " + msg, + msg.contains("type coercion from BYTE to") && msg.contains("is not supported") ); } } @Test - public void testDoubleToFloat() throws Exception { - String table = "test_qwp_double_to_float"; + public void testByteToInt() throws Exception { + String table = "test_qwp_byte_to_int"; useTable(table); execute("CREATE TABLE " + table + " (" + - "f FLOAT, " + + "i INT, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("f", 1.5) + .byteColumn("i", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .doubleColumn("f", -42.25) + .byteColumn("i", Byte.MAX_VALUE) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .byteColumn("i", Byte.MIN_VALUE) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); + assertSqlEventually( + "i\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "127\t1970-01-01T00:00:02.000000000Z\n" + + "-128\t1970-01-01T00:00:03.000000000Z\n", + "SELECT i, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDoubleToInt() throws Exception { - String table = "test_qwp_double_to_int"; + public void testByteToLong() throws Exception { + String table = "test_qwp_byte_to_long"; useTable(table); execute("CREATE TABLE " + table + " (" + - "i INT, " + + "l LONG, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("i", 100_000.0) + .byteColumn("l", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .doubleColumn("i", -42.0) + .byteColumn("l", Byte.MAX_VALUE) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .byteColumn("l", Byte.MIN_VALUE) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); assertSqlEventually( - "i\tts\n" + - "100000\t1970-01-01T00:00:01.000000000Z\n" + - "-42\t1970-01-01T00:00:02.000000000Z\n", - "SELECT i, ts FROM " + table + " ORDER BY ts"); + "l\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "127\t1970-01-01T00:00:02.000000000Z\n" + + "-128\t1970-01-01T00:00:03.000000000Z\n", + "SELECT l, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDoubleToIntPrecisionLossError() throws Exception { - String table = "test_qwp_double_to_int_prec"; + public void testByteToLong256CoercionError() throws Exception { + String table = "test_qwp_byte_to_long256_error"; useTable(table); execute("CREATE TABLE " + table + " (" + - "i INT, " + + "v LONG256, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("i", 3.14) + .byteColumn("v", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected precision loss error but got: " + msg, - msg.contains("loses precision") && msg.contains("3.14") + "Expected coercion error but got: " + msg, + msg.contains("type coercion from BYTE to LONG256 is not supported") ); } } @Test - public void testDoubleToLong() throws Exception { - String table = "test_qwp_double_to_long"; + public void testByteToShort() throws Exception { + String table = "test_qwp_byte_to_short"; useTable(table); execute("CREATE TABLE " + table + " (" + - "l LONG, " + + "s SHORT, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("l", 1_000_000.0) + .byteColumn("s", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .doubleColumn("l", -42.0) + .byteColumn("s", Byte.MIN_VALUE) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .byteColumn("s", Byte.MAX_VALUE) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); assertSqlEventually( - "l\tts\n" + - "1000000\t1970-01-01T00:00:01.000000000Z\n" + - "-42\t1970-01-01T00:00:02.000000000Z\n", - "SELECT l, ts FROM " + table + " ORDER BY ts"); + "s\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-128\t1970-01-01T00:00:02.000000000Z\n" + + "127\t1970-01-01T00:00:03.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDoubleToString() throws Exception { - String table = "test_qwp_double_to_string"; + public void testByteToString() throws Exception { + String table = "test_qwp_byte_to_string"; useTable(table); execute("CREATE TABLE " + table + " (" + "s STRING, " + @@ -1067,620 +934,589 @@ public void testDoubleToString() throws Exception { try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("s", 3.14) + .byteColumn("s", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .doubleColumn("s", -42.0) + .byteColumn("s", (byte) -100) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .byteColumn("s", (byte) 0) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); assertSqlEventually( "s\tts\n" + - "3.14\t1970-01-01T00:00:01.000000000Z\n" + - "-42.0\t1970-01-01T00:00:02.000000000Z\n", + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-100\t1970-01-01T00:00:02.000000000Z\n" + + "0\t1970-01-01T00:00:03.000000000Z\n", "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDoubleToSymbol() throws Exception { - String table = "test_qwp_double_to_symbol"; + public void testByteToSymbol() throws Exception { + String table = "test_qwp_byte_to_symbol"; useTable(table); execute("CREATE TABLE " + table + " (" + - "sym SYMBOL, " + + "s SYMBOL, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("sym", 3.14) + .byteColumn("s", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .byteColumn("s", (byte) -1) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .byteColumn("s", (byte) 0) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 1); + assertTableSizeEventually(table, 3); assertSqlEventually( - "sym\tts\n" + - "3.14\t1970-01-01T00:00:01.000000000Z\n", - "SELECT sym, ts FROM " + table + " ORDER BY ts"); + "s\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-1\t1970-01-01T00:00:02.000000000Z\n" + + "0\t1970-01-01T00:00:03.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDoubleToVarchar() throws Exception { - String table = "test_qwp_double_to_varchar"; + public void testByteToTimestamp() throws Exception { + String table = "test_qwp_byte_to_timestamp"; useTable(table); execute("CREATE TABLE " + table + " (" + - "v VARCHAR, " + + "t TIMESTAMP, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("v", 3.14) + .byteColumn("t", (byte) 100) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .doubleColumn("v", -42.0) + .byteColumn("t", (byte) 0) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "v\tts\n" + - "3.14\t1970-01-01T00:00:01.000000000Z\n" + - "-42.0\t1970-01-01T00:00:02.000000000Z\n", - "SELECT v, ts FROM " + table + " ORDER BY ts"); + "t\tts\n" + + "1970-01-01T00:00:00.000100000Z\t1970-01-01T00:00:01.000000000Z\n" + + "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", + "SELECT t, ts FROM " + table + " ORDER BY ts"); } @Test - public void testFloat() throws Exception { - String table = "test_qwp_float"; + public void testByteToUuidCoercionError() throws Exception { + String table = "test_qwp_byte_to_uuid_error"; useTable(table); + execute("CREATE TABLE " + table + " (" + + "u UUID, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .floatColumn("f", 1.5f) + .byteColumn("u", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .floatColumn("f", -42.25f) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .floatColumn("f", 0.0f) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from BYTE to UUID is not supported") + ); } - - assertTableSizeEventually(table, 3); } @Test - public void testFloatToDecimal() throws Exception { - String table = "test_qwp_float_to_decimal"; + public void testByteToVarchar() throws Exception { + String table = "test_qwp_byte_to_varchar"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(10, 2), " + + "v VARCHAR, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .floatColumn("d", 1.5f) + .byteColumn("v", (byte) 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .floatColumn("d", -42.25f) + .byteColumn("v", (byte) -100) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .byteColumn("v", Byte.MAX_VALUE) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); assertSqlEventually( - "d\tts\n" + - "1.50\t1970-01-01T00:00:01.000000000Z\n" + - "-42.25\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + "v\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-100\t1970-01-01T00:00:02.000000000Z\n" + + "127\t1970-01-01T00:00:03.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); } @Test - public void testFloatToDecimalPrecisionLossError() throws Exception { - String table = "test_qwp_float_to_decimal_prec"; + public void testChar() throws Exception { + String table = "test_qwp_char"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(10, 1), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .floatColumn("d", 1.25f) + .charColumn("c", 'A') + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .charColumn("c", 'ü') // ü + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .charColumn("c", '中') // 中 + .at(3_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "c\ttimestamp\n" + + "A\t1970-01-01T00:00:01.000000000Z\n" + + "ü\t1970-01-01T00:00:02.000000000Z\n" + + "中\t1970-01-01T00:00:03.000000000Z\n", + "SELECT c, timestamp FROM " + table + " ORDER BY timestamp"); + } + + @Test + public void testCharToBooleanCoercionError() throws Exception { + String table = "test_qwp_char_to_boolean_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .charColumn("v", 'A') .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected precision loss error but got: " + msg, - msg.contains("cannot be converted to") && msg.contains("scale=1") + "Expected coercion error but got: " + msg, + msg.contains("cannot write") && msg.contains("BOOLEAN") ); } } @Test - public void testFloatToDouble() throws Exception { - String table = "test_qwp_float_to_double"; + public void testCharToByteCoercionError() throws Exception { + String table = "test_qwp_char_to_byte_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DOUBLE, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v BYTE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .floatColumn("d", 1.5f) + .charColumn("v", 'A') .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .floatColumn("d", -42.25f) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("not supported") && msg.contains("BYTE") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "d\tts\n" + - "1.5\t1970-01-01T00:00:01.000000000Z\n" + - "-42.25\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testFloatToInt() throws Exception { - String table = "test_qwp_float_to_int"; + public void testCharToDateCoercionError() throws Exception { + String table = "test_qwp_char_to_date_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "i INT, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .floatColumn("i", 42.0f) + .charColumn("v", 'A') .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .floatColumn("i", -100.0f) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("not supported") && msg.contains("DATE") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "i\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-100\t1970-01-01T00:00:02.000000000Z\n", - "SELECT i, ts FROM " + table + " ORDER BY ts"); } @Test - public void testFloatToIntPrecisionLossError() throws Exception { - String table = "test_qwp_float_to_int_prec"; + public void testCharToDoubleCoercionError() throws Exception { + String table = "test_qwp_char_to_double_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "i INT, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v DOUBLE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .floatColumn("i", 3.14f) + .charColumn("v", 'A') .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected precision loss error but got: " + msg, - msg.contains("loses precision") + "Expected coercion error but got: " + msg, + msg.contains("not supported") && msg.contains("DOUBLE") ); } } @Test - public void testFloatToLong() throws Exception { - String table = "test_qwp_float_to_long"; + public void testCharToFloatCoercionError() throws Exception { + String table = "test_qwp_char_to_float_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "l LONG, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .floatColumn("l", 1000.0f) + .charColumn("v", 'A') .at(1_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("not supported") && msg.contains("FLOAT") + ); } - - assertTableSizeEventually(table, 1); - assertSqlEventually( - "l\tts\n" + - "1000\t1970-01-01T00:00:01.000000000Z\n", - "SELECT l, ts FROM " + table + " ORDER BY ts"); } @Test - public void testFloatToString() throws Exception { - String table = "test_qwp_float_to_string"; + public void testCharToGeoHashCoercionError() throws Exception { + String table = "test_qwp_char_to_geohash_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "s STRING, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .floatColumn("s", 1.5f) + .charColumn("v", 'A') .at(1_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("GEOHASH") + ); } - - assertTableSizeEventually(table, 1); - assertSqlEventually( - "s\tts\n" + - "1.5\t1970-01-01T00:00:01.000000000Z\n", - "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testFloatToSymbol() throws Exception { - String table = "test_qwp_float_to_symbol"; + public void testCharToIntCoercionError() throws Exception { + String table = "test_qwp_char_to_int_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "sym SYMBOL, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .floatColumn("sym", 1.5f) + .charColumn("v", 'A') .at(1_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("not supported") && msg.contains("INT") + ); } - - assertTableSizeEventually(table, 1); - assertSqlEventually( - "sym\tts\n" + - "1.5\t1970-01-01T00:00:01.000000000Z\n", - "SELECT sym, ts FROM " + table + " ORDER BY ts"); } @Test - public void testFloatToVarchar() throws Exception { - String table = "test_qwp_float_to_varchar"; + public void testCharToLong256CoercionError() throws Exception { + String table = "test_qwp_char_to_long256_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "v VARCHAR, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .floatColumn("v", 1.5f) + .charColumn("v", 'A') .at(1_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("not supported") && msg.contains("LONG256") + ); } - - assertTableSizeEventually(table, 1); - assertSqlEventually( - "v\tts\n" + - "1.5\t1970-01-01T00:00:01.000000000Z\n", - "SELECT v, ts FROM " + table + " ORDER BY ts"); } @Test - public void testInt() throws Exception { - String table = "test_qwp_int"; + public void testCharToLongCoercionError() throws Exception { + String table = "test_qwp_char_to_long_error"; useTable(table); - + execute("CREATE TABLE " + table + " (v LONG, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - // Integer.MIN_VALUE is the null sentinel for INT sender.table(table) - .intColumn("i", Integer.MIN_VALUE) + .charColumn("v", 'A') .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("i", 0) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("i", Integer.MAX_VALUE) - .at(3_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("i", -42) - .at(4_000_000, ChronoUnit.MICROS); sender.flush(); - } - - assertTableSizeEventually(table, 4); - assertSqlEventually( - "i\ttimestamp\n" + - "null\t1970-01-01T00:00:01.000000000Z\n" + - "0\t1970-01-01T00:00:02.000000000Z\n" + - "2147483647\t1970-01-01T00:00:03.000000000Z\n" + - "-42\t1970-01-01T00:00:04.000000000Z\n", - "SELECT i, timestamp FROM " + table + " ORDER BY timestamp"); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("not supported") && msg.contains("LONG") + ); + } } @Test - public void testIntToBooleanCoercionError() throws Exception { - String table = "test_qwp_int_to_boolean_error"; + public void testCharToShortCoercionError() throws Exception { + String table = "test_qwp_char_to_short_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "b BOOLEAN, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v SHORT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("b", 1) + .charColumn("v", 'A') .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected error mentioning INT and BOOLEAN but got: " + msg, - msg.contains("INT") && msg.contains("BOOLEAN") + "Expected coercion error but got: " + msg, + msg.contains("not supported") && msg.contains("SHORT") ); } } @Test - public void testIntToByte() throws Exception { - String table = "test_qwp_int_to_byte"; + public void testCharToString() throws Exception { + String table = "test_qwp_char_to_string"; useTable(table); execute("CREATE TABLE " + table + " (" + - "b BYTE, " + + "s STRING, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("b", 42) + .charColumn("s", 'A') .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .intColumn("b", -128) + .charColumn("s", 'Z') .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("b", 127) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 2); assertSqlEventually( - "b\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-128\t1970-01-01T00:00:02.000000000Z\n" + - "127\t1970-01-01T00:00:03.000000000Z\n", - "SELECT b, ts FROM " + table + " ORDER BY ts"); + "s\tts\n" + + "A\t1970-01-01T00:00:01.000000000Z\n" + + "Z\t1970-01-01T00:00:02.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToByteOverflowError() throws Exception { - String table = "test_qwp_int_to_byte_overflow"; + public void testCharToSymbolCoercionError() throws Exception { + String table = "test_qwp_char_to_symbol_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "b BYTE, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v SYMBOL, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("b", 128) + .charColumn("v", 'A') .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected overflow error but got: " + msg, - msg.contains("integer value 128 out of range for BYTE") + "Expected coercion error but got: " + msg, + msg.contains("cannot write") && msg.contains("SYMBOL") ); } } @Test - public void testIntToCharCoercionError() throws Exception { - String table = "test_qwp_int_to_char_error"; + public void testCharToUuidCoercionError() throws Exception { + String table = "test_qwp_char_to_uuid_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "c CHAR, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("c", 65) + .charColumn("v", 'A') .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected error mentioning INT and CHAR but got: " + msg, - msg.contains("INT") && msg.contains("CHAR") + "Expected coercion error but got: " + msg, + msg.contains("not supported") && msg.contains("UUID") ); } } @Test - public void testIntToDate() throws Exception { - String table = "test_qwp_int_to_date"; + public void testCharToVarchar() throws Exception { + String table = "test_qwp_char_to_varchar"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DATE, " + + "v VARCHAR, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - // 86_400_000 millis = 1 day sender.table(table) - .intColumn("d", 86_400_000) + .charColumn("v", 'A') .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .intColumn("d", 0) + .charColumn("v", 'Z') .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "1970-01-02T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z\n" + - "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + "v\tts\n" + + "A\t1970-01-01T00:00:01.000000000Z\n" + + "Z\t1970-01-01T00:00:02.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToDecimal() throws Exception { - String table = "test_qwp_int_to_decimal"; + public void testDecimal() throws Exception { + String table = "test_qwp_decimal"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(6, 2), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("d", 42) + .decimalColumn("d", "123.45") .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .intColumn("d", -100) + .decimalColumn("d", "-999.99") .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .decimalColumn("d", "0.01") + .at(3_000_000, ChronoUnit.MICROS); + sender.table(table) + .decimalColumn("d", Decimal256.fromLong(42_000, 2)) + .at(4_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); - assertSqlEventually( - "d\tts\n" + - "42.00\t1970-01-01T00:00:01.000000000Z\n" + - "-100.00\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + assertTableSizeEventually(table, 4); } @Test - public void testIntToDecimal128() throws Exception { - String table = "test_qwp_int_to_decimal128"; + public void testDecimal128ToDecimal256() throws Exception { + String table = "test_qwp_dec128_to_dec256"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(38, 2), " + + "d DECIMAL(76, 2), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("d", 42) + .decimalColumn("d", Decimal128.fromLong(12345, 2)) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .intColumn("d", -100) + .decimalColumn("d", Decimal128.fromLong(-9999, 2)) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("d", 0) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 2); assertSqlEventually( "d\tts\n" + - "42.00\t1970-01-01T00:00:01.000000000Z\n" + - "-100.00\t1970-01-01T00:00:02.000000000Z\n" + - "0.00\t1970-01-01T00:00:03.000000000Z\n", + "123.45\t1970-01-01T00:00:01.000000000Z\n" + + "-99.99\t1970-01-01T00:00:02.000000000Z\n", "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToDecimal16() throws Exception { - String table = "test_qwp_int_to_decimal16"; + public void testDecimal128ToDecimal64() throws Exception { + String table = "test_qwp_dec128_to_dec64"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(4, 1), " + + "d DECIMAL(18, 2), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("d", 42) + .decimalColumn("d", Decimal128.fromLong(12345, 2)) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .intColumn("d", -100) + .decimalColumn("d", Decimal128.fromLong(-9999, 2)) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("d", 0) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 2); assertSqlEventually( "d\tts\n" + - "42.0\t1970-01-01T00:00:01.000000000Z\n" + - "-100.0\t1970-01-01T00:00:02.000000000Z\n" + - "0.0\t1970-01-01T00:00:03.000000000Z\n", + "123.45\t1970-01-01T00:00:01.000000000Z\n" + + "-99.99\t1970-01-01T00:00:02.000000000Z\n", "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToDecimal256() throws Exception { - String table = "test_qwp_int_to_decimal256"; + public void testDecimal256ToDecimal128() throws Exception { + String table = "test_qwp_dec256_to_dec128"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(76, 2), " + + "d DECIMAL(38, 2), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("d", 42) + .decimalColumn("d", Decimal256.fromLong(12345, 2)) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .intColumn("d", -100) + .decimalColumn("d", Decimal256.fromLong(-9999, 2)) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("d", 0) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 2); assertSqlEventually( "d\tts\n" + - "42.00\t1970-01-01T00:00:01.000000000Z\n" + - "-100.00\t1970-01-01T00:00:02.000000000Z\n" + - "0.00\t1970-01-01T00:00:03.000000000Z\n", + "123.45\t1970-01-01T00:00:01.000000000Z\n" + + "-99.99\t1970-01-01T00:00:02.000000000Z\n", "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToDecimal64() throws Exception { - String table = "test_qwp_int_to_decimal64"; + public void testDecimal256ToDecimal64() throws Exception { + String table = "test_qwp_dec256_to_dec64"; useTable(table); execute("CREATE TABLE " + table + " (" + "d DECIMAL(18, 2), " + @@ -1689,189 +1525,173 @@ public void testIntToDecimal64() throws Exception { assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { + // Send DECIMAL256 wire type to DECIMAL64 column sender.table(table) - .intColumn("d", Integer.MAX_VALUE) + .decimalColumn("d", Decimal256.fromLong(12345, 2)) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .intColumn("d", -100) + .decimalColumn("d", Decimal256.fromLong(-9999, 2)) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("d", 0) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 2); assertSqlEventually( "d\tts\n" + - "2147483647.00\t1970-01-01T00:00:01.000000000Z\n" + - "-100.00\t1970-01-01T00:00:02.000000000Z\n" + - "0.00\t1970-01-01T00:00:03.000000000Z\n", + "123.45\t1970-01-01T00:00:01.000000000Z\n" + + "-99.99\t1970-01-01T00:00:02.000000000Z\n", "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToDecimal8() throws Exception { - String table = "test_qwp_int_to_decimal8"; + public void testDecimal256ToDecimal64OverflowError() throws Exception { + String table = "test_qwp_dec256_to_dec64_overflow"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(2, 1), " + + "d DECIMAL(18, 2), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { + // Create a value that fits in Decimal256 but overflows Decimal64 + // Decimal256 with hi bits set will overflow 64-bit storage + Decimal256 bigValue = Decimal256.fromBigDecimal(new java.math.BigDecimal("99999999999999999999.99")); sender.table(table) - .intColumn("d", 5) + .decimalColumn("d", bigValue) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("d", -9) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("d", 0) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected overflow error but got: " + msg, + msg.contains("decimal value overflows") + ); } - - assertTableSizeEventually(table, 3); - assertSqlEventually( - "d\tts\n" + - "5.0\t1970-01-01T00:00:01.000000000Z\n" + - "-9.0\t1970-01-01T00:00:02.000000000Z\n" + - "0.0\t1970-01-01T00:00:03.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToDouble() throws Exception { - String table = "test_qwp_int_to_double"; + public void testDecimal256ToDecimal8OverflowError() throws Exception { + String table = "test_qwp_dec256_to_dec8_overflow"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DOUBLE, " + + "d DECIMAL(2, 1), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { + // 999.9 with scale=1 → unscaled 9999, which doesn't fit in a byte (-128..127) sender.table(table) - .intColumn("d", 42) + .decimalColumn("d", Decimal256.fromLong(9999, 1)) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("d", -100) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected overflow error but got: " + msg, + msg.contains("decimal value overflows") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "d\tts\n" + - "42.0\t1970-01-01T00:00:01.000000000Z\n" + - "-100.0\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToFloat() throws Exception { - String table = "test_qwp_int_to_float"; + public void testDecimal64ToDecimal128() throws Exception { + String table = "test_qwp_dec64_to_dec128"; useTable(table); execute("CREATE TABLE " + table + " (" + - "f FLOAT, " + + "d DECIMAL(38, 2), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { + // Send DECIMAL64 wire type to DECIMAL128 column (widening) sender.table(table) - .intColumn("f", 42) + .decimalColumn("d", Decimal64.fromLong(12345, 2)) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .intColumn("f", -100) + .decimalColumn("d", Decimal64.fromLong(-9999, 2)) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("f", 0) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 2); assertSqlEventually( - "f\tts\n" + - "42.0\t1970-01-01T00:00:01.000000000Z\n" + - "-100.0\t1970-01-01T00:00:02.000000000Z\n" + - "0.0\t1970-01-01T00:00:03.000000000Z\n", - "SELECT f, ts FROM " + table + " ORDER BY ts"); + "d\tts\n" + + "123.45\t1970-01-01T00:00:01.000000000Z\n" + + "-99.99\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToGeoHashCoercionError() throws Exception { - String table = "test_qwp_int_to_geohash_error"; + public void testDecimal64ToDecimal256() throws Exception { + String table = "test_qwp_dec64_to_dec256"; useTable(table); execute("CREATE TABLE " + table + " (" + - "g GEOHASH(4c), " + + "d DECIMAL(76, 2), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("g", 42) + .decimalColumn("d", Decimal64.fromLong(12345, 2)) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .decimalColumn("d", Decimal64.fromLong(-9999, 2)) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error mentioning INT but got: " + msg, - msg.contains("type coercion from INT to") && msg.contains("is not supported") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "123.45\t1970-01-01T00:00:01.000000000Z\n" + + "-99.99\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToLong() throws Exception { - String table = "test_qwp_int_to_long"; + public void testDecimalRescale() throws Exception { + String table = "test_qwp_decimal_rescale"; useTable(table); execute("CREATE TABLE " + table + " (" + - "l LONG, " + + "d DECIMAL(18, 4), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { + // Send scale=2 wire data to scale=4 column: server should rescale sender.table(table) - .intColumn("l", 42) + .decimalColumn("d", Decimal64.fromLong(12345, 2)) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .intColumn("l", Integer.MAX_VALUE) + .decimalColumn("d", Decimal64.fromLong(-100, 2)) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("l", -1) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 2); assertSqlEventually( - "l\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "2147483647\t1970-01-01T00:00:02.000000000Z\n" + - "-1\t1970-01-01T00:00:03.000000000Z\n", - "SELECT l, ts FROM " + table + " ORDER BY ts"); + "d\tts\n" + + "123.4500\t1970-01-01T00:00:01.000000000Z\n" + + "-1.0000\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToLong256CoercionError() throws Exception { - String table = "test_qwp_int_to_long256_error"; + public void testDecimalToBooleanCoercionError() throws Exception { + String table = "test_qwp_decimal_to_boolean_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "v LONG256, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("v", 42) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -1879,174 +1699,125 @@ public void testIntToLong256CoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("type coercion from INT to LONG256 is not supported") + msg.contains("cannot write DECIMAL64") && msg.contains("BOOLEAN") ); } } @Test - public void testIntToShort() throws Exception { - String table = "test_qwp_int_to_short"; + public void testDecimalToByteCoercionError() throws Exception { + String table = "test_qwp_decimal_to_byte_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "s SHORT, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v BYTE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("s", 1000) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("s", -32768) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("s", 32767) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DECIMAL64") && msg.contains("BYTE") + ); } - - assertTableSizeEventually(table, 3); - assertSqlEventually( - "s\tts\n" + - "1000\t1970-01-01T00:00:01.000000000Z\n" + - "-32768\t1970-01-01T00:00:02.000000000Z\n" + - "32767\t1970-01-01T00:00:03.000000000Z\n", - "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToShortOverflowError() throws Exception { - String table = "test_qwp_int_to_short_overflow"; + public void testDecimalToCharCoercionError() throws Exception { + String table = "test_qwp_decimal_to_char_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "s SHORT, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v CHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("s", 32768) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected overflow error but got: " + msg, - msg.contains("integer value 32768 out of range for SHORT") + "Expected coercion error but got: " + msg, + msg.contains("cannot write DECIMAL64") && msg.contains("CHAR") ); } } @Test - public void testIntToString() throws Exception { - String table = "test_qwp_int_to_string"; + public void testDecimalToDateCoercionError() throws Exception { + String table = "test_qwp_decimal_to_date_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "s STRING, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("s", 42) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("s", -100) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("s", 0) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DECIMAL64") && msg.contains("DATE") + ); } - - assertTableSizeEventually(table, 3); - assertSqlEventually( - "s\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-100\t1970-01-01T00:00:02.000000000Z\n" + - "0\t1970-01-01T00:00:03.000000000Z\n", - "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToSymbol() throws Exception { - String table = "test_qwp_int_to_symbol"; + public void testDecimalToDoubleCoercionError() throws Exception { + String table = "test_qwp_decimal_to_double_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "s SYMBOL, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v DOUBLE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("s", 42) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("s", -1) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("s", 0) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DECIMAL64") && msg.contains("DOUBLE") + ); } - - assertTableSizeEventually(table, 3); - assertSqlEventually( - "s\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-1\t1970-01-01T00:00:02.000000000Z\n" + - "0\t1970-01-01T00:00:03.000000000Z\n", - "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToTimestamp() throws Exception { - String table = "test_qwp_int_to_timestamp"; + public void testDecimalToFloatCoercionError() throws Exception { + String table = "test_qwp_decimal_to_float_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "t TIMESTAMP, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - // 1_000_000 micros = 1 second sender.table(table) - .intColumn("t", 1_000_000) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("t", 0) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DECIMAL64") && msg.contains("FLOAT") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "t\tts\n" + - "1970-01-01T00:00:01.000000000Z\t1970-01-01T00:00:01.000000000Z\n" + - "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", - "SELECT t, ts FROM " + table + " ORDER BY ts"); } @Test - public void testIntToUuidCoercionError() throws Exception { - String table = "test_qwp_int_to_uuid_error"; + public void testDecimalToGeoHashCoercionError() throws Exception { + String table = "test_qwp_decimal_to_geohash_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "u UUID, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("u", 42) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -2054,517 +1825,446 @@ public void testIntToUuidCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("type coercion from INT to UUID is not supported") + msg.contains("cannot write DECIMAL64") && msg.contains("GEOHASH") ); } } @Test - public void testIntToVarchar() throws Exception { - String table = "test_qwp_int_to_varchar"; + public void testDecimalToIntCoercionError() throws Exception { + String table = "test_qwp_decimal_to_int_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "v VARCHAR, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .intColumn("v", 42) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("v", -100) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .intColumn("v", Integer.MAX_VALUE) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DECIMAL64") && msg.contains("INT") + ); } - - assertTableSizeEventually(table, 3); - assertSqlEventually( - "v\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-100\t1970-01-01T00:00:02.000000000Z\n" + - "2147483647\t1970-01-01T00:00:03.000000000Z\n", - "SELECT v, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLong() throws Exception { - String table = "test_qwp_long"; + public void testDecimalToLong256CoercionError() throws Exception { + String table = "test_qwp_decimal_to_long256_error"; useTable(table); - + execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - // Long.MIN_VALUE is the null sentinel for LONG sender.table(table) - .longColumn("l", Long.MIN_VALUE) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .longColumn("l", 0) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .longColumn("l", Long.MAX_VALUE) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DECIMAL64") && msg.contains("LONG256") + ); } - - assertTableSizeEventually(table, 3); - assertSqlEventually( - "l\ttimestamp\n" + - "null\t1970-01-01T00:00:01.000000000Z\n" + - "0\t1970-01-01T00:00:02.000000000Z\n" + - "9223372036854775807\t1970-01-01T00:00:03.000000000Z\n", - "SELECT l, timestamp FROM " + table + " ORDER BY timestamp"); } @Test - public void testLong256() throws Exception { - String table = "test_qwp_long256"; + public void testDecimalToLongCoercionError() throws Exception { + String table = "test_qwp_decimal_to_long_error"; useTable(table); - + execute("CREATE TABLE " + table + " (v LONG, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - // All zeros sender.table(table) - .long256Column("v", 0, 0, 0, 0) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) .at(1_000_000, ChronoUnit.MICROS); - // Mixed values - sender.table(table) - .long256Column("v", 1, 2, 3, 4) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DECIMAL64") && msg.contains("LONG") + ); } - - assertTableSizeEventually(table, 2); } @Test - public void testLongToBooleanCoercionError() throws Exception { - String table = "test_qwp_long_to_boolean_error"; + public void testDecimalToShortCoercionError() throws Exception { + String table = "test_qwp_decimal_to_short_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "b BOOLEAN, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v SHORT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("b", 1) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected error mentioning LONG and BOOLEAN but got: " + msg, - msg.contains("LONG") && msg.contains("BOOLEAN") + "Expected coercion error but got: " + msg, + msg.contains("cannot write DECIMAL64") && msg.contains("SHORT") ); } } @Test - public void testLongToByte() throws Exception { - String table = "test_qwp_long_to_byte"; + public void testDecimalToString() throws Exception { + String table = "test_qwp_decimal_to_string"; useTable(table); execute("CREATE TABLE " + table + " (" + - "b BYTE, " + + "s STRING, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("b", 42) + .decimalColumn("s", Decimal64.fromLong(12345, 2)) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("b", -128) + .decimalColumn("s", Decimal64.fromLong(-9999, 2)) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .longColumn("b", 127) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 2); assertSqlEventually( - "b\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-128\t1970-01-01T00:00:02.000000000Z\n" + - "127\t1970-01-01T00:00:03.000000000Z\n", - "SELECT b, ts FROM " + table + " ORDER BY ts"); + "s\tts\n" + + "123.45\t1970-01-01T00:00:01.000000000Z\n" + + "-99.99\t1970-01-01T00:00:02.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToByteOverflowError() throws Exception { - String table = "test_qwp_long_to_byte_overflow"; + public void testDecimalToSymbolCoercionError() throws Exception { + String table = "test_qwp_decimal_to_symbol_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "b BYTE, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v SYMBOL, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("b", 128) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected overflow error but got: " + msg, - msg.contains("integer value 128 out of range for BYTE") + "Expected coercion error but got: " + msg, + msg.contains("cannot write DECIMAL64") && msg.contains("SYMBOL") ); } } @Test - public void testLongToCharCoercionError() throws Exception { - String table = "test_qwp_long_to_char_error"; + public void testDecimalToTimestampCoercionError() throws Exception { + String table = "test_qwp_decimal_to_timestamp_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "c CHAR, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v TIMESTAMP, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("c", 65) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected error mentioning LONG and CHAR but got: " + msg, - msg.contains("LONG") && msg.contains("CHAR") + "Expected coercion error but got: " + msg, + msg.contains("cannot write DECIMAL64") && msg.contains("TIMESTAMP") ); } } @Test - public void testLongToDate() throws Exception { - String table = "test_qwp_long_to_date"; + public void testDecimalToTimestampNsCoercionError() throws Exception { + String table = "test_qwp_decimal_to_timestamp_ns_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DATE, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v TIMESTAMP_NS, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("d", 86_400_000L) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .longColumn("d", 0L) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DECIMAL64") && msg.contains("TIMESTAMP") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "d\tts\n" + - "1970-01-02T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z\n" + - "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToDecimal() throws Exception { - String table = "test_qwp_long_to_decimal"; + public void testDecimalToUuidCoercionError() throws Exception { + String table = "test_qwp_decimal_to_uuid_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(10, 2), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("d", 42) + .decimalColumn("v", Decimal64.fromLong(12345L, 2)) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .longColumn("d", -100) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DECIMAL64") && msg.contains("UUID") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "d\tts\n" + - "42.00\t1970-01-01T00:00:01.000000000Z\n" + - "-100.00\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToDecimal128() throws Exception { - String table = "test_qwp_long_to_decimal128"; + public void testDecimalToVarchar() throws Exception { + String table = "test_qwp_decimal_to_varchar"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(38, 2), " + + "v VARCHAR, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("d", 1_000_000_000L) + .decimalColumn("v", Decimal64.fromLong(12345, 2)) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("d", -1_000_000_000L) + .decimalColumn("v", Decimal64.fromLong(-9999, 2)) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "1000000000.00\t1970-01-01T00:00:01.000000000Z\n" + - "-1000000000.00\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + "v\tts\n" + + "123.45\t1970-01-01T00:00:01.000000000Z\n" + + "-99.99\t1970-01-01T00:00:02.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToDecimal16() throws Exception { - String table = "test_qwp_long_to_decimal16"; + public void testDouble() throws Exception { + String table = "test_qwp_double"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(4, 1), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("d", 42) + .doubleColumn("d", 42.5) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("d", -100) + .doubleColumn("d", -1.0E10) .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); + sender.table(table) + .doubleColumn("d", Double.MAX_VALUE) + .at(3_000_000, ChronoUnit.MICROS); + sender.table(table) + .doubleColumn("d", Double.MIN_VALUE) + .at(4_000_000, ChronoUnit.MICROS); + // NaN and Inf should be stored as null + sender.table(table) + .doubleColumn("d", Double.NaN) + .at(5_000_000, ChronoUnit.MICROS); + sender.table(table) + .doubleColumn("d", Double.POSITIVE_INFINITY) + .at(6_000_000, ChronoUnit.MICROS); + sender.table(table) + .doubleColumn("d", Double.NEGATIVE_INFINITY) + .at(7_000_000, ChronoUnit.MICROS); + sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 7); assertSqlEventually( - "d\tts\n" + - "42.0\t1970-01-01T00:00:01.000000000Z\n" + - "-100.0\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + "d\ttimestamp\n" + + "42.5\t1970-01-01T00:00:01.000000000Z\n" + + "-1.0E10\t1970-01-01T00:00:02.000000000Z\n" + + "1.7976931348623157E308\t1970-01-01T00:00:03.000000000Z\n" + + "4.9E-324\t1970-01-01T00:00:04.000000000Z\n" + + "null\t1970-01-01T00:00:05.000000000Z\n" + + "null\t1970-01-01T00:00:06.000000000Z\n" + + "null\t1970-01-01T00:00:07.000000000Z\n", + "SELECT d, timestamp FROM " + table + " ORDER BY timestamp"); } @Test - public void testLongToDecimal256() throws Exception { - String table = "test_qwp_long_to_decimal256"; + public void testDoubleArray() throws Exception { + String table = "test_qwp_double_array"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(76, 2), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); + + double[] arr1d = createDoubleArray(5); + double[][] arr2d = createDoubleArray(2, 3); + double[][][] arr3d = createDoubleArray(1, 2, 3); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("d", Long.MAX_VALUE) + .doubleArray("a1", arr1d) + .doubleArray("a2", arr2d) + .doubleArray("a3", arr3d) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .longColumn("d", -1_000_000_000_000L) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); - assertSqlEventually( - "d\tts\n" + - "9223372036854775807.00\t1970-01-01T00:00:01.000000000Z\n" + - "-1000000000000.00\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + assertTableSizeEventually(table, 1); } @Test - public void testLongToDecimal32() throws Exception { - String table = "test_qwp_long_to_decimal32"; + public void testDoubleArrayToIntCoercionError() throws Exception { + String table = "test_qwp_doublearray_to_int_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(6, 2), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("d", 42) + .doubleArray("v", new double[]{1.0, 2.0}) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .longColumn("d", -100) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DOUBLE_ARRAY") && msg.contains("INT") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "d\tts\n" + - "42.00\t1970-01-01T00:00:01.000000000Z\n" + - "-100.00\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToDecimal8() throws Exception { - String table = "test_qwp_long_to_decimal8"; + public void testDoubleArrayToStringCoercionError() throws Exception { + String table = "test_qwp_doublearray_to_string_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(2, 1), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v STRING, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("d", 5) + .doubleArray("v", new double[]{1.0, 2.0}) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .longColumn("d", -9) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DOUBLE_ARRAY") && msg.contains("STRING") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "d\tts\n" + - "5.0\t1970-01-01T00:00:01.000000000Z\n" + - "-9.0\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToDouble() throws Exception { - String table = "test_qwp_long_to_double"; + public void testDoubleArrayToSymbolCoercionError() throws Exception { + String table = "test_qwp_doublearray_to_symbol_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DOUBLE, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v SYMBOL, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("d", 42) + .doubleArray("v", new double[]{1.0, 2.0}) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .longColumn("d", -100) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DOUBLE_ARRAY") && msg.contains("SYMBOL") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "d\tts\n" + - "42.0\t1970-01-01T00:00:01.000000000Z\n" + - "-100.0\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToFloat() throws Exception { - String table = "test_qwp_long_to_float"; + public void testDoubleArrayToTimestampCoercionError() throws Exception { + String table = "test_qwp_doublearray_to_timestamp_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "f FLOAT, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v TIMESTAMP, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("f", 42) + .doubleArray("v", new double[]{1.0, 2.0}) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .longColumn("f", -100) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DOUBLE_ARRAY") && msg.contains("TIMESTAMP") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "f\tts\n" + - "42.0\t1970-01-01T00:00:01.000000000Z\n" + - "-100.0\t1970-01-01T00:00:02.000000000Z\n", - "SELECT f, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToGeoHashCoercionError() throws Exception { - String table = "test_qwp_long_to_geohash_error"; + public void testDoubleToBooleanCoercionError() throws Exception { + String table = "test_qwp_double_to_boolean_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "g GEOHASH(4c), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("g", 42) + .doubleColumn("v", 3.14) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected coercion error mentioning LONG but got: " + msg, - msg.contains("type coercion from LONG to") && msg.contains("is not supported") + "Expected coercion error but got: " + msg, + msg.contains("cannot write DOUBLE") && msg.contains("BOOLEAN") ); } } @Test - public void testLongToInt() throws Exception { - String table = "test_qwp_long_to_int"; + public void testDoubleToByte() throws Exception { + String table = "test_qwp_double_to_byte"; useTable(table); execute("CREATE TABLE " + table + " (" + - "i INT, " + + "b BYTE, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - // Value in INT range should succeed sender.table(table) - .longColumn("i", 42) + .doubleColumn("b", 42.0) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("i", -1) + .doubleColumn("b", -100.0) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "i\tts\n" + + "b\tts\n" + "42\t1970-01-01T00:00:01.000000000Z\n" + - "-1\t1970-01-01T00:00:02.000000000Z\n", - "SELECT i, ts FROM " + table + " ORDER BY ts"); + "-100\t1970-01-01T00:00:02.000000000Z\n", + "SELECT b, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToIntOverflowError() throws Exception { - String table = "test_qwp_long_to_int_overflow"; + public void testDoubleToByteOverflowError() throws Exception { + String table = "test_qwp_double_to_byte_ovf"; useTable(table); execute("CREATE TABLE " + table + " (" + - "i INT, " + + "b BYTE, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("i", (long) Integer.MAX_VALUE + 1) + .doubleColumn("b", 200.0) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -2572,182 +2272,163 @@ public void testLongToIntOverflowError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected overflow error but got: " + msg, - msg.contains("integer value 2147483648 out of range for INT") + msg.contains("integer value 200 out of range for BYTE") ); } } @Test - public void testLongToLong256CoercionError() throws Exception { - String table = "test_qwp_long_to_long256_error"; + public void testDoubleToBytePrecisionLossError() throws Exception { + String table = "test_qwp_double_to_byte_prec"; useTable(table); execute("CREATE TABLE " + table + " (" + - "v LONG256, " + + "b BYTE, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("v", 42) + .doubleColumn("b", 42.5) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from LONG to LONG256 is not supported") + "Expected precision loss error but got: " + msg, + msg.contains("loses precision") && msg.contains("42.5") ); } } @Test - public void testLongToShort() throws Exception { - String table = "test_qwp_long_to_short"; + public void testDoubleToCharCoercionError() throws Exception { + String table = "test_qwp_double_to_char_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "s SHORT, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v CHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - // Value in SHORT range should succeed sender.table(table) - .longColumn("s", 42) + .doubleColumn("v", 3.14) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .longColumn("s", -1) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write DOUBLE") && msg.contains("CHAR") + ); } - - assertTableSizeEventually(table, 2); } @Test - public void testLongToShortOverflowError() throws Exception { - String table = "test_qwp_long_to_short_overflow"; + public void testDoubleToDateCoercionError() throws Exception { + String table = "test_qwp_double_to_date_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "s SHORT, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("s", 32768) + .doubleColumn("v", 3.14) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected overflow error but got: " + msg, - msg.contains("integer value 32768 out of range for SHORT") + "Expected coercion error but got: " + msg, + msg.contains("type coercion from DOUBLE to") && msg.contains("is not supported") ); } } @Test - public void testLongToString() throws Exception { - String table = "test_qwp_long_to_string"; + public void testDoubleToDecimal() throws Exception { + String table = "test_qwp_double_to_decimal"; useTable(table); execute("CREATE TABLE " + table + " (" + - "s STRING, " + + "d DECIMAL(10, 2), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("s", 42) + .doubleColumn("d", 123.45) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("s", Long.MAX_VALUE) + .doubleColumn("d", -42.10) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "s\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "9223372036854775807\t1970-01-01T00:00:02.000000000Z\n", - "SELECT s, ts FROM " + table + " ORDER BY ts"); + "d\tts\n" + + "123.45\t1970-01-01T00:00:01.000000000Z\n" + + "-42.10\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToSymbol() throws Exception { - String table = "test_qwp_long_to_symbol"; + public void testDoubleToDecimalPrecisionLossError() throws Exception { + String table = "test_qwp_double_to_decimal_prec"; useTable(table); execute("CREATE TABLE " + table + " (" + - "s SYMBOL, " + + "d DECIMAL(10, 2), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("s", 42) + .doubleColumn("d", 123.456) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .longColumn("s", -1) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected precision loss error but got: " + msg, + msg.contains("cannot be converted to") && msg.contains("123.456") && msg.contains("scale=2") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "s\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-1\t1970-01-01T00:00:02.000000000Z\n", - "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToTimestamp() throws Exception { - String table = "test_qwp_long_to_timestamp"; + public void testDoubleToFloat() throws Exception { + String table = "test_qwp_double_to_float"; useTable(table); execute("CREATE TABLE " + table + " (" + - "t TIMESTAMP, " + + "f FLOAT, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("t", 1_000_000L) + .doubleColumn("f", 1.5) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("t", 0L) + .doubleColumn("f", -42.25) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); - assertSqlEventually( - "t\tts\n" + - "1970-01-01T00:00:01.000000000Z\t1970-01-01T00:00:01.000000000Z\n" + - "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", - "SELECT t, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLongToUuidCoercionError() throws Exception { - String table = "test_qwp_long_to_uuid_error"; + public void testDoubleToGeoHashCoercionError() throws Exception { + String table = "test_qwp_double_to_geohash_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "u UUID, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("u", 42) + .doubleColumn("v", 3.14) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -2755,429 +2436,370 @@ public void testLongToUuidCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("type coercion from LONG to UUID is not supported") + msg.contains("type coercion from DOUBLE to") && msg.contains("is not supported") ); } } @Test - public void testLongToVarchar() throws Exception { - String table = "test_qwp_long_to_varchar"; + public void testDoubleToInt() throws Exception { + String table = "test_qwp_double_to_int"; useTable(table); execute("CREATE TABLE " + table + " (" + - "v VARCHAR, " + + "i INT, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .longColumn("v", 42) + .doubleColumn("i", 100_000.0) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .longColumn("v", Long.MAX_VALUE) + .doubleColumn("i", -42.0) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "v\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "9223372036854775807\t1970-01-01T00:00:02.000000000Z\n", - "SELECT v, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testMultipleRowsAndBatching() throws Exception { - String table = "test_qwp_multiple_rows"; - useTable(table); - - int rowCount = 1000; - try (QwpWebSocketSender sender = createQwpSender()) { - for (int i = 0; i < rowCount; i++) { - sender.table(table) - .symbol("sym", "s" + (i % 10)) - .longColumn("val", i) - .doubleColumn("dbl", i * 1.5) - .at((long) (i + 1) * 1_000_000, ChronoUnit.MICROS); - } - sender.flush(); - } - - assertTableSizeEventually(table, rowCount); + "i\tts\n" + + "100000\t1970-01-01T00:00:01.000000000Z\n" + + "-42\t1970-01-01T00:00:02.000000000Z\n", + "SELECT i, ts FROM " + table + " ORDER BY ts"); } @Test - public void testShort() throws Exception { - String table = "test_qwp_short"; + public void testDoubleToIntPrecisionLossError() throws Exception { + String table = "test_qwp_double_to_int_prec"; useTable(table); + execute("CREATE TABLE " + table + " (" + + "i INT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - // Short.MIN_VALUE is the null sentinel for SHORT sender.table(table) - .shortColumn("s", Short.MIN_VALUE) + .doubleColumn("i", 3.14) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("s", (short) 0) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("s", Short.MAX_VALUE) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected precision loss error but got: " + msg, + msg.contains("loses precision") && msg.contains("3.14") + ); } - - assertTableSizeEventually(table, 3); } @Test - public void testShortToDecimal128() throws Exception { - String table = "test_qwp_short_to_decimal128"; + public void testDoubleToLong() throws Exception { + String table = "test_qwp_double_to_long"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(38, 2), " + + "l LONG, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("d", Short.MAX_VALUE) + .doubleColumn("l", 1_000_000.0) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .shortColumn("d", Short.MIN_VALUE) + .doubleColumn("l", -42.0) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "32767.00\t1970-01-01T00:00:01.000000000Z\n" + - "-32768.00\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + "l\tts\n" + + "1000000\t1970-01-01T00:00:01.000000000Z\n" + + "-42\t1970-01-01T00:00:02.000000000Z\n", + "SELECT l, ts FROM " + table + " ORDER BY ts"); } @Test - public void testShortToDecimal16() throws Exception { - String table = "test_qwp_short_to_decimal16"; + public void testDoubleToLong256CoercionError() throws Exception { + String table = "test_qwp_double_to_long256_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(4, 1), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("d", (short) 42) + .doubleColumn("v", 3.14) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("d", (short) -100) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from DOUBLE to") && msg.contains("is not supported") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "d\tts\n" + - "42.0\t1970-01-01T00:00:01.000000000Z\n" + - "-100.0\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testShortToDecimal256() throws Exception { - String table = "test_qwp_short_to_decimal256"; + public void testDoubleToShort() throws Exception { + String table = "test_qwp_double_to_short"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(76, 2), " + + "v SHORT, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("d", (short) 42) + .doubleColumn("v", 100.0) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .shortColumn("d", (short) -100) + .doubleColumn("v", -200.0) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "42.00\t1970-01-01T00:00:01.000000000Z\n" + - "-100.00\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + "v\tts\n" + + "100\t1970-01-01T00:00:01.000000000Z\n" + + "-200\t1970-01-01T00:00:02.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); } @Test - public void testShortToDecimal32() throws Exception { - String table = "test_qwp_short_to_decimal32"; + public void testDoubleToString() throws Exception { + String table = "test_qwp_double_to_string"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(6, 2), " + + "s STRING, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("d", (short) 42) + .doubleColumn("s", 3.14) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .shortColumn("d", (short) -100) + .doubleColumn("s", -42.0) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "d\tts\n" + - "42.00\t1970-01-01T00:00:01.000000000Z\n" + - "-100.00\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + "s\tts\n" + + "3.14\t1970-01-01T00:00:01.000000000Z\n" + + "-42.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testShortToDecimal64() throws Exception { - String table = "test_qwp_short_to_decimal64"; + public void testDoubleToSymbol() throws Exception { + String table = "test_qwp_double_to_symbol"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(18, 2), " + + "sym SYMBOL, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("d", (short) 42) + .doubleColumn("sym", 3.14) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("d", (short) -100) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 1); assertSqlEventually( - "d\tts\n" + - "42.00\t1970-01-01T00:00:01.000000000Z\n" + - "-100.00\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + "sym\tts\n" + + "3.14\t1970-01-01T00:00:01.000000000Z\n", + "SELECT sym, ts FROM " + table + " ORDER BY ts"); } @Test - public void testShortToDecimal8() throws Exception { - String table = "test_qwp_short_to_decimal8"; + public void testDoubleToUuidCoercionError() throws Exception { + String table = "test_qwp_double_to_uuid_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(2, 1), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("d", (short) 5) + .doubleColumn("v", 3.14) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("d", (short) -9) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from DOUBLE to") && msg.contains("is not supported") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "d\tts\n" + - "5.0\t1970-01-01T00:00:01.000000000Z\n" + - "-9.0\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testShortToInt() throws Exception { - String table = "test_qwp_short_to_int"; + public void testDoubleToVarchar() throws Exception { + String table = "test_qwp_double_to_varchar"; useTable(table); execute("CREATE TABLE " + table + " (" + - "i INT, " + + "v VARCHAR, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("i", (short) 42) + .doubleColumn("v", 3.14) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .shortColumn("i", Short.MAX_VALUE) + .doubleColumn("v", -42.0) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "i\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "32767\t1970-01-01T00:00:02.000000000Z\n", - "SELECT i, ts FROM " + table + " ORDER BY ts"); + "v\tts\n" + + "3.14\t1970-01-01T00:00:01.000000000Z\n" + + "-42.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); } @Test - public void testShortToLong() throws Exception { - String table = "test_qwp_short_to_long"; + public void testFloat() throws Exception { + String table = "test_qwp_float"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "l LONG, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("l", (short) 42) + .floatColumn("f", 1.5f) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .shortColumn("l", Short.MAX_VALUE) + .floatColumn("f", -42.25f) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .floatColumn("f", 0.0f) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); - assertSqlEventually( - "l\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "32767\t1970-01-01T00:00:02.000000000Z\n", - "SELECT l, ts FROM " + table + " ORDER BY ts"); + assertTableSizeEventually(table, 3); } @Test - public void testShortToBooleanCoercionError() throws Exception { - String table = "test_qwp_short_to_boolean_error"; + public void testFloatToBooleanCoercionError() throws Exception { + String table = "test_qwp_float_to_boolean_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "b BOOLEAN, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("b", (short) 1) + .floatColumn("v", 1.5f) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected error mentioning SHORT and BOOLEAN but got: " + msg, - msg.contains("SHORT") && msg.contains("BOOLEAN") + "Expected coercion error but got: " + msg, + msg.contains("cannot write FLOAT") && msg.contains("BOOLEAN") ); } } @Test - public void testShortToByte() throws Exception { - String table = "test_qwp_short_to_byte"; + public void testFloatToByte() throws Exception { + String table = "test_qwp_float_to_byte"; useTable(table); execute("CREATE TABLE " + table + " (" + - "b BYTE, " + + "v BYTE, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("b", (short) 42) + .floatColumn("v", 7.0f) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .shortColumn("b", (short) -128) + .floatColumn("v", -100.0f) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("b", (short) 127) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 2); assertSqlEventually( - "b\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-128\t1970-01-01T00:00:02.000000000Z\n" + - "127\t1970-01-01T00:00:03.000000000Z\n", - "SELECT b, ts FROM " + table + " ORDER BY ts"); + "v\tts\n" + + "7\t1970-01-01T00:00:01.000000000Z\n" + + "-100\t1970-01-01T00:00:02.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); } @Test - public void testShortToByteOverflowError() throws Exception { - String table = "test_qwp_short_to_byte_overflow"; + public void testFloatToCharCoercionError() throws Exception { + String table = "test_qwp_float_to_char_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "b BYTE, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v CHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("b", (short) 128) + .floatColumn("v", 1.5f) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected overflow error but got: " + msg, - msg.contains("integer value 128 out of range for BYTE") + "Expected coercion error but got: " + msg, + msg.contains("cannot write FLOAT") && msg.contains("CHAR") ); } } @Test - public void testShortToCharCoercionError() throws Exception { - String table = "test_qwp_short_to_char_error"; + public void testFloatToDateCoercionError() throws Exception { + String table = "test_qwp_float_to_date_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "c CHAR, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("c", (short) 65) + .floatColumn("v", 1.5f) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected error mentioning SHORT and CHAR but got: " + msg, - msg.contains("SHORT") && msg.contains("CHAR") + "Expected coercion error but got: " + msg, + msg.contains("type coercion from FLOAT to") && msg.contains("is not supported") ); } } @Test - public void testShortToDate() throws Exception { - String table = "test_qwp_short_to_date"; + public void testFloatToDecimal() throws Exception { + String table = "test_qwp_float_to_decimal"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DATE, " + + "d DECIMAL(10, 2), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - // 1000 millis = 1 second sender.table(table) - .shortColumn("d", (short) 1000) + .floatColumn("d", 1.5f) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .shortColumn("d", (short) 0) + .floatColumn("d", -42.25f) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } @@ -3185,222 +2807,268 @@ public void testShortToDate() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( "d\tts\n" + - "1970-01-01T00:00:01.000000000Z\t1970-01-01T00:00:01.000000000Z\n" + - "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", + "1.50\t1970-01-01T00:00:01.000000000Z\n" + + "-42.25\t1970-01-01T00:00:02.000000000Z\n", "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testShortToDouble() throws Exception { - String table = "test_qwp_short_to_double"; + public void testFloatToDecimalPrecisionLossError() throws Exception { + String table = "test_qwp_float_to_decimal_prec"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DOUBLE, " + + "d DECIMAL(10, 1), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("d", (short) 42) + .floatColumn("d", 1.25f) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("d", (short) -100) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected precision loss error but got: " + msg, + msg.contains("cannot be converted to") && msg.contains("scale=1") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "d\tts\n" + - "42.0\t1970-01-01T00:00:01.000000000Z\n" + - "-100.0\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testShortToFloat() throws Exception { - String table = "test_qwp_short_to_float"; + public void testFloatToDouble() throws Exception { + String table = "test_qwp_float_to_double"; useTable(table); execute("CREATE TABLE " + table + " (" + - "f FLOAT, " + + "d DOUBLE, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("f", (short) 42) + .floatColumn("d", 1.5f) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .shortColumn("f", (short) -100) + .floatColumn("d", -42.25f) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "f\tts\n" + - "42.0\t1970-01-01T00:00:01.000000000Z\n" + - "-100.0\t1970-01-01T00:00:02.000000000Z\n", - "SELECT f, ts FROM " + table + " ORDER BY ts"); + "d\tts\n" + + "1.5\t1970-01-01T00:00:01.000000000Z\n" + + "-42.25\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testShortToGeoHashCoercionError() throws Exception { - String table = "test_qwp_short_to_geohash_error"; + public void testFloatToGeoHashCoercionError() throws Exception { + String table = "test_qwp_float_to_geohash_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "g GEOHASH(4c), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("g", (short) 42) + .floatColumn("v", 1.5f) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected coercion error mentioning SHORT but got: " + msg, - msg.contains("type coercion from SHORT to") && msg.contains("is not supported") + "Expected coercion error but got: " + msg, + msg.contains("type coercion from FLOAT to") && msg.contains("is not supported") ); } } @Test - public void testShortToLong256CoercionError() throws Exception { - String table = "test_qwp_short_to_long256_error"; + public void testFloatToInt() throws Exception { + String table = "test_qwp_float_to_int"; useTable(table); execute("CREATE TABLE " + table + " (" + - "v LONG256, " + + "i INT, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("v", (short) 42) + .floatColumn("i", 42.0f) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .floatColumn("i", -100.0f) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from SHORT to LONG256 is not supported") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "i\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-100\t1970-01-01T00:00:02.000000000Z\n", + "SELECT i, ts FROM " + table + " ORDER BY ts"); } @Test - public void testShortToString() throws Exception { - String table = "test_qwp_short_to_string"; + public void testFloatToIntPrecisionLossError() throws Exception { + String table = "test_qwp_float_to_int_prec"; useTable(table); execute("CREATE TABLE " + table + " (" + - "s STRING, " + + "i INT, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("s", (short) 42) + .floatColumn("i", 3.14f) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("s", (short) -100) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("s", (short) 0) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); - } - - assertTableSizeEventually(table, 3); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected precision loss error but got: " + msg, + msg.contains("loses precision") + ); + } + } + + @Test + public void testFloatToLong() throws Exception { + String table = "test_qwp_float_to_long"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "l LONG, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .floatColumn("l", 1000.0f) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 1); assertSqlEventually( - "s\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-100\t1970-01-01T00:00:02.000000000Z\n" + - "0\t1970-01-01T00:00:03.000000000Z\n", - "SELECT s, ts FROM " + table + " ORDER BY ts"); + "l\tts\n" + + "1000\t1970-01-01T00:00:01.000000000Z\n", + "SELECT l, ts FROM " + table + " ORDER BY ts"); } @Test - public void testShortToSymbol() throws Exception { - String table = "test_qwp_short_to_symbol"; + public void testFloatToLong256CoercionError() throws Exception { + String table = "test_qwp_float_to_long256_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .floatColumn("v", 1.5f) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from FLOAT to") && msg.contains("is not supported") + ); + } + } + + @Test + public void testFloatToShort() throws Exception { + String table = "test_qwp_float_to_short"; useTable(table); execute("CREATE TABLE " + table + " (" + - "s SYMBOL, " + + "v SHORT, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("s", (short) 42) + .floatColumn("v", 42.0f) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .shortColumn("s", (short) -1) + .floatColumn("v", -1000.0f) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("s", (short) 0) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 2); assertSqlEventually( - "s\tts\n" + + "v\tts\n" + "42\t1970-01-01T00:00:01.000000000Z\n" + - "-1\t1970-01-01T00:00:02.000000000Z\n" + - "0\t1970-01-01T00:00:03.000000000Z\n", - "SELECT s, ts FROM " + table + " ORDER BY ts"); + "-1000\t1970-01-01T00:00:02.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); } @Test - public void testShortToTimestamp() throws Exception { - String table = "test_qwp_short_to_timestamp"; + public void testFloatToString() throws Exception { + String table = "test_qwp_float_to_string"; useTable(table); execute("CREATE TABLE " + table + " (" + - "t TIMESTAMP, " + + "s STRING, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("t", (short) 1000) + .floatColumn("s", 1.5f) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("t", (short) 0) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 1); assertSqlEventually( - "t\tts\n" + - "1970-01-01T00:00:00.001000000Z\t1970-01-01T00:00:01.000000000Z\n" + - "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", - "SELECT t, ts FROM " + table + " ORDER BY ts"); + "s\tts\n" + + "1.5\t1970-01-01T00:00:01.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testShortToUuidCoercionError() throws Exception { - String table = "test_qwp_short_to_uuid_error"; + public void testFloatToSymbol() throws Exception { + String table = "test_qwp_float_to_symbol"; useTable(table); execute("CREATE TABLE " + table + " (" + - "u UUID, " + + "sym SYMBOL, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("u", (short) 42) + .floatColumn("sym", 1.5f) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 1); + assertSqlEventually( + "sym\tts\n" + + "1.5\t1970-01-01T00:00:01.000000000Z\n", + "SELECT sym, ts FROM " + table + " ORDER BY ts"); + } + + @Test + public void testFloatToUuidCoercionError() throws Exception { + String table = "test_qwp_float_to_uuid_error"; + useTable(table); + execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .floatColumn("v", 1.5f) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -3408,14 +3076,14 @@ public void testShortToUuidCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("type coercion from SHORT to UUID is not supported") + msg.contains("type coercion from FLOAT to") && msg.contains("is not supported") ); } } @Test - public void testShortToVarchar() throws Exception { - String table = "test_qwp_short_to_varchar"; + public void testFloatToVarchar() throws Exception { + String table = "test_qwp_float_to_varchar"; useTable(table); execute("CREATE TABLE " + table + " (" + "v VARCHAR, " + @@ -3425,342 +3093,313 @@ public void testShortToVarchar() throws Exception { try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .shortColumn("v", (short) 42) + .floatColumn("v", 1.5f) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("v", (short) -100) - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .shortColumn("v", Short.MAX_VALUE) - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 1); assertSqlEventually( "v\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-100\t1970-01-01T00:00:02.000000000Z\n" + - "32767\t1970-01-01T00:00:03.000000000Z\n", + "1.5\t1970-01-01T00:00:01.000000000Z\n", "SELECT v, ts FROM " + table + " ORDER BY ts"); } @Test - public void testString() throws Exception { - String table = "test_qwp_string"; + public void testInt() throws Exception { + String table = "test_qwp_int"; useTable(table); try (QwpWebSocketSender sender = createQwpSender()) { + // Integer.MIN_VALUE is the null sentinel for INT sender.table(table) - .stringColumn("s", "hello world") + .intColumn("i", Integer.MIN_VALUE) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .stringColumn("s", "non-ascii äöü") + .intColumn("i", 0) .at(2_000_000, ChronoUnit.MICROS); sender.table(table) - .stringColumn("s", "") + .intColumn("i", Integer.MAX_VALUE) .at(3_000_000, ChronoUnit.MICROS); sender.table(table) - .stringColumn("s", null) + .intColumn("i", -42) .at(4_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 4); assertSqlEventually( - "s\ttimestamp\n" + - "hello world\t1970-01-01T00:00:01.000000000Z\n" + - "non-ascii äöü\t1970-01-01T00:00:02.000000000Z\n" + - "\t1970-01-01T00:00:03.000000000Z\n" + - "null\t1970-01-01T00:00:04.000000000Z\n", - "SELECT s, timestamp FROM " + table + " ORDER BY timestamp"); + "i\ttimestamp\n" + + "null\t1970-01-01T00:00:01.000000000Z\n" + + "0\t1970-01-01T00:00:02.000000000Z\n" + + "2147483647\t1970-01-01T00:00:03.000000000Z\n" + + "-42\t1970-01-01T00:00:04.000000000Z\n", + "SELECT i, timestamp FROM " + table + " ORDER BY timestamp"); } @Test - public void testStringToChar() throws Exception { - String table = "test_qwp_string_to_char"; + public void testIntToBooleanCoercionError() throws Exception { + String table = "test_qwp_int_to_boolean_error"; useTable(table); execute("CREATE TABLE " + table + " (" + - "c CHAR, " + + "b BOOLEAN, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .stringColumn("c", "A") + .intColumn("b", 1) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("c", "Hello") - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected error mentioning INT and BOOLEAN but got: " + msg, + msg.contains("INT") && msg.contains("BOOLEAN") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "c\tts\n" + - "A\t1970-01-01T00:00:01.000000000Z\n" + - "H\t1970-01-01T00:00:02.000000000Z\n", - "SELECT c, ts FROM " + table + " ORDER BY ts"); } @Test - public void testStringToSymbol() throws Exception { - String table = "test_qwp_string_to_symbol"; + public void testIntToByte() throws Exception { + String table = "test_qwp_int_to_byte"; useTable(table); execute("CREATE TABLE " + table + " (" + - "s SYMBOL, " + + "b BYTE, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .stringColumn("s", "hello") + .intColumn("b", 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .stringColumn("s", "world") + .intColumn("b", -128) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("b", 127) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); assertSqlEventually( - "s\tts\n" + - "hello\t1970-01-01T00:00:01.000000000Z\n" + - "world\t1970-01-01T00:00:02.000000000Z\n", - "SELECT s, ts FROM " + table + " ORDER BY ts"); + "b\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-128\t1970-01-01T00:00:02.000000000Z\n" + + "127\t1970-01-01T00:00:03.000000000Z\n", + "SELECT b, ts FROM " + table + " ORDER BY ts"); } @Test - public void testStringToUuid() throws Exception { - String table = "test_qwp_string_to_uuid"; + public void testIntToByteOverflowError() throws Exception { + String table = "test_qwp_int_to_byte_overflow"; useTable(table); execute("CREATE TABLE " + table + " (" + - "u UUID, " + + "b BYTE, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .stringColumn("u", "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11") + .intColumn("b", 128) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected overflow error but got: " + msg, + msg.contains("integer value 128 out of range for BYTE") + ); } - - assertTableSizeEventually(table, 1); - assertSqlEventually( - "u\tts\n" + - "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11\t1970-01-01T00:00:01.000000000Z\n", - "SELECT u, ts FROM " + table + " ORDER BY ts"); } @Test - public void testSymbol() throws Exception { - String table = "test_qwp_symbol"; + public void testIntToCharCoercionError() throws Exception { + String table = "test_qwp_int_to_char_error"; useTable(table); + execute("CREATE TABLE " + table + " (" + + "c CHAR, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .symbol("s", "alpha") + .intColumn("c", 65) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .symbol("s", "beta") - .at(2_000_000, ChronoUnit.MICROS); - // repeated value reuses dictionary entry - sender.table(table) - .symbol("s", "alpha") - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected error mentioning INT and CHAR but got: " + msg, + msg.contains("INT") && msg.contains("CHAR") + ); } - - assertTableSizeEventually(table, 3); - assertSqlEventually( - "s\ttimestamp\n" + - "alpha\t1970-01-01T00:00:01.000000000Z\n" + - "beta\t1970-01-01T00:00:02.000000000Z\n" + - "alpha\t1970-01-01T00:00:03.000000000Z\n", - "SELECT s, timestamp FROM " + table + " ORDER BY timestamp"); } @Test - public void testTimestampMicros() throws Exception { - String table = "test_qwp_timestamp_micros"; + public void testIntToDate() throws Exception { + String table = "test_qwp_int_to_date"; useTable(table); + execute("CREATE TABLE " + table + " (" + + "d DATE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - long tsMicros = 1_645_747_200_000_000L; // 2022-02-25T00:00:00Z in micros + // 86_400_000 millis = 1 day sender.table(table) - .timestampColumn("ts_col", tsMicros, ChronoUnit.MICROS) + .intColumn("d", 86_400_000) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", 0) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 1); + assertTableSizeEventually(table, 2); assertSqlEventually( - "ts_col\ttimestamp\n" + - "2022-02-25T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z\n", - "SELECT ts_col, timestamp FROM " + table); + "d\tts\n" + + "1970-01-02T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z\n" + + "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testTimestampMicrosToNanos() throws Exception { - String table = "test_qwp_timestamp_micros_to_nanos"; + public void testIntToDecimal() throws Exception { + String table = "test_qwp_int_to_decimal"; useTable(table); execute("CREATE TABLE " + table + " (" + - "ts_col TIMESTAMP_NS, " + + "d DECIMAL(6, 2), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - long tsMicros = 1_645_747_200_111_111L; // 2022-02-25T00:00:00Z sender.table(table) - .timestampColumn("ts_col", tsMicros, ChronoUnit.MICROS) + .intColumn("d", 42) .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 1); - // Microseconds scaled to nanoseconds - assertSqlEventually( - "ts_col\tts\n" + - "2022-02-25T00:00:00.111111000Z\t1970-01-01T00:00:01.000000000Z\n", - "SELECT ts_col, ts FROM " + table); - } - - @Test - public void testTimestampNanos() throws Exception { - String table = "test_qwp_timestamp_nanos"; - useTable(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - long tsNanos = 1_645_747_200_000_000_000L; // 2022-02-25T00:00:00Z in nanos sender.table(table) - .timestampColumn("ts_col", tsNanos, ChronoUnit.NANOS) - .at(tsNanos, ChronoUnit.NANOS); + .intColumn("d", -100) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 1); + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testTimestampNanosToMicros() throws Exception { - String table = "test_qwp_timestamp_nanos_to_micros"; + public void testIntToDecimal128() throws Exception { + String table = "test_qwp_int_to_decimal128"; useTable(table); execute("CREATE TABLE " + table + " (" + - "ts_col TIMESTAMP, " + + "d DECIMAL(38, 2), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - long tsNanos = 1_645_747_200_123_456_789L; sender.table(table) - .timestampColumn("ts_col", tsNanos, ChronoUnit.NANOS) + .intColumn("d", 42) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", 0) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 1); - // Nanoseconds truncated to microseconds + assertTableSizeEventually(table, 3); assertSqlEventually( - "ts_col\tts\n" + - "2022-02-25T00:00:00.123456000Z\t1970-01-01T00:00:01.000000000Z\n", - "SELECT ts_col, ts FROM " + table); + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n" + + "0.00\t1970-01-01T00:00:03.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testUuid() throws Exception { - String table = "test_qwp_uuid"; + public void testIntToDecimal16() throws Exception { + String table = "test_qwp_int_to_decimal16"; useTable(table); - - UUID uuid1 = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); - UUID uuid2 = UUID.fromString("11111111-2222-3333-4444-555555555555"); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(4, 1), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .uuidColumn("u", uuid1.getLeastSignificantBits(), uuid1.getMostSignificantBits()) + .intColumn("d", 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .uuidColumn("u", uuid2.getLeastSignificantBits(), uuid2.getMostSignificantBits()) + .intColumn("d", -100) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", 0) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); assertSqlEventually( - "u\ttimestamp\n" + - "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11\t1970-01-01T00:00:01.000000000Z\n" + - "11111111-2222-3333-4444-555555555555\t1970-01-01T00:00:02.000000000Z\n", - "SELECT u, timestamp FROM " + table + " ORDER BY timestamp"); + "d\tts\n" + + "42.0\t1970-01-01T00:00:01.000000000Z\n" + + "-100.0\t1970-01-01T00:00:02.000000000Z\n" + + "0.0\t1970-01-01T00:00:03.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testUuidToShortCoercionError() throws Exception { - String table = "test_qwp_uuid_to_short_error"; + public void testIntToDecimal256() throws Exception { + String table = "test_qwp_int_to_decimal256"; useTable(table); execute("CREATE TABLE " + table + " (" + - "s SHORT, " + + "d DECIMAL(76, 2), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); sender.table(table) - .uuidColumn("s", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .intColumn("d", 42) .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from UUID to SHORT is not supported") - ); - } - } - - @Test - public void testWriteAllTypesInOneRow() throws Exception { - String table = "test_qwp_all_types"; - useTable(table); - - UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); - double[] arr1d = {1.0, 2.0, 3.0}; - long tsMicros = 1_645_747_200_000_000L; // 2022-02-25T00:00:00Z - - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .symbol("sym", "test_symbol") - .boolColumn("bool_col", true) - .shortColumn("short_col", (short) 42) - .intColumn("int_col", 100_000) - .longColumn("long_col", 1_000_000_000L) - .floatColumn("float_col", 2.5f) - .doubleColumn("double_col", 3.14) - .stringColumn("string_col", "hello") - .charColumn("char_col", 'Z') - .timestampColumn("ts_col", tsMicros, ChronoUnit.MICROS) - .uuidColumn("uuid_col", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) - .long256Column("long256_col", 1, 0, 0, 0) - .doubleArray("arr_col", arr1d) - .decimalColumn("decimal_col", "99.99") - .at(tsMicros, ChronoUnit.MICROS); + .intColumn("d", -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", 0) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 1); + assertTableSizeEventually(table, 3); + assertSqlEventually( + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n" + + "0.00\t1970-01-01T00:00:03.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } - // === Decimal cross-width coercion tests === - @Test - public void testDecimal256ToDecimal64() throws Exception { - String table = "test_qwp_dec256_to_dec64"; + public void testIntToDecimal64() throws Exception { + String table = "test_qwp_int_to_decimal64"; useTable(table); execute("CREATE TABLE " + table + " (" + "d DECIMAL(18, 2), " + @@ -3769,69 +3408,75 @@ public void testDecimal256ToDecimal64() throws Exception { assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - // Send DECIMAL256 wire type to DECIMAL64 column sender.table(table) - .decimalColumn("d", Decimal256.fromLong(12345, 2)) + .intColumn("d", Integer.MAX_VALUE) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .decimalColumn("d", Decimal256.fromLong(-9999, 2)) + .intColumn("d", -100) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", 0) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); assertSqlEventually( "d\tts\n" + - "123.45\t1970-01-01T00:00:01.000000000Z\n" + - "-99.99\t1970-01-01T00:00:02.000000000Z\n", + "2147483647.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n" + + "0.00\t1970-01-01T00:00:03.000000000Z\n", "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDecimal256ToDecimal128() throws Exception { - String table = "test_qwp_dec256_to_dec128"; + public void testIntToDecimal8() throws Exception { + String table = "test_qwp_int_to_decimal8"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(38, 2), " + + "d DECIMAL(2, 1), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .decimalColumn("d", Decimal256.fromLong(12345, 2)) + .intColumn("d", 5) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .decimalColumn("d", Decimal256.fromLong(-9999, 2)) + .intColumn("d", -9) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("d", 0) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); assertSqlEventually( "d\tts\n" + - "123.45\t1970-01-01T00:00:01.000000000Z\n" + - "-99.99\t1970-01-01T00:00:02.000000000Z\n", + "5.0\t1970-01-01T00:00:01.000000000Z\n" + + "-9.0\t1970-01-01T00:00:02.000000000Z\n" + + "0.0\t1970-01-01T00:00:03.000000000Z\n", "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDecimal64ToDecimal128() throws Exception { - String table = "test_qwp_dec64_to_dec128"; + public void testIntToDouble() throws Exception { + String table = "test_qwp_int_to_double"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(38, 2), " + + "d DOUBLE, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - // Send DECIMAL64 wire type to DECIMAL128 column (widening) sender.table(table) - .decimalColumn("d", Decimal64.fromLong(12345, 2)) + .intColumn("d", 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .decimalColumn("d", Decimal64.fromLong(-9999, 2)) + .intColumn("d", -100) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } @@ -3839,166 +3484,170 @@ public void testDecimal64ToDecimal128() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( "d\tts\n" + - "123.45\t1970-01-01T00:00:01.000000000Z\n" + - "-99.99\t1970-01-01T00:00:02.000000000Z\n", + "42.0\t1970-01-01T00:00:01.000000000Z\n" + + "-100.0\t1970-01-01T00:00:02.000000000Z\n", "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDecimal64ToDecimal256() throws Exception { - String table = "test_qwp_dec64_to_dec256"; + public void testIntToFloat() throws Exception { + String table = "test_qwp_int_to_float"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(76, 2), " + + "f FLOAT, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .decimalColumn("d", Decimal64.fromLong(12345, 2)) + .intColumn("f", 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .decimalColumn("d", Decimal64.fromLong(-9999, 2)) + .intColumn("f", -100) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("f", 0) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); assertSqlEventually( - "d\tts\n" + - "123.45\t1970-01-01T00:00:01.000000000Z\n" + - "-99.99\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + "f\tts\n" + + "42.0\t1970-01-01T00:00:01.000000000Z\n" + + "-100.0\t1970-01-01T00:00:02.000000000Z\n" + + "0.0\t1970-01-01T00:00:03.000000000Z\n", + "SELECT f, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDecimal128ToDecimal64() throws Exception { - String table = "test_qwp_dec128_to_dec64"; + public void testIntToGeoHashCoercionError() throws Exception { + String table = "test_qwp_int_to_geohash_error"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(18, 2), " + + "g GEOHASH(4c), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .decimalColumn("d", Decimal128.fromLong(12345, 2)) + .intColumn("g", 42) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .decimalColumn("d", Decimal128.fromLong(-9999, 2)) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error mentioning INT but got: " + msg, + msg.contains("type coercion from INT to") && msg.contains("is not supported") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "d\tts\n" + - "123.45\t1970-01-01T00:00:01.000000000Z\n" + - "-99.99\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDecimal128ToDecimal256() throws Exception { - String table = "test_qwp_dec128_to_dec256"; + public void testIntToLong() throws Exception { + String table = "test_qwp_int_to_long"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(76, 2), " + + "l LONG, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .decimalColumn("d", Decimal128.fromLong(12345, 2)) + .intColumn("l", 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .decimalColumn("d", Decimal128.fromLong(-9999, 2)) + .intColumn("l", Integer.MAX_VALUE) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("l", -1) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); assertSqlEventually( - "d\tts\n" + - "123.45\t1970-01-01T00:00:01.000000000Z\n" + - "-99.99\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + "l\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "2147483647\t1970-01-01T00:00:02.000000000Z\n" + + "-1\t1970-01-01T00:00:03.000000000Z\n", + "SELECT l, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDecimalRescale() throws Exception { - String table = "test_qwp_decimal_rescale"; + public void testIntToLong256CoercionError() throws Exception { + String table = "test_qwp_int_to_long256_error"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(18, 4), " + + "v LONG256, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - // Send scale=2 wire data to scale=4 column: server should rescale sender.table(table) - .decimalColumn("d", Decimal64.fromLong(12345, 2)) + .intColumn("v", 42) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .decimalColumn("d", Decimal64.fromLong(-100, 2)) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from INT to LONG256 is not supported") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "d\tts\n" + - "123.4500\t1970-01-01T00:00:01.000000000Z\n" + - "-1.0000\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDecimal256ToDecimal64OverflowError() throws Exception { - String table = "test_qwp_dec256_to_dec64_overflow"; + public void testIntToShort() throws Exception { + String table = "test_qwp_int_to_short"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(18, 2), " + + "s SHORT, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - // Create a value that fits in Decimal256 but overflows Decimal64 - // Decimal256 with hi bits set will overflow 64-bit storage - Decimal256 bigValue = Decimal256.fromBigDecimal(new java.math.BigDecimal("99999999999999999999.99")); sender.table(table) - .decimalColumn("d", bigValue) + .intColumn("s", 1000) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("s", -32768) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("s", 32767) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected overflow error but got: " + msg, - msg.contains("decimal value overflows") - ); } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "s\tts\n" + + "1000\t1970-01-01T00:00:01.000000000Z\n" + + "-32768\t1970-01-01T00:00:02.000000000Z\n" + + "32767\t1970-01-01T00:00:03.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDecimal256ToDecimal8OverflowError() throws Exception { - String table = "test_qwp_dec256_to_dec8_overflow"; + public void testIntToShortOverflowError() throws Exception { + String table = "test_qwp_int_to_short_overflow"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(2, 1), " + + "s SHORT, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - // 999.9 with scale=1 → unscaled 9999, which doesn't fit in a byte (-128..127) sender.table(table) - .decimalColumn("d", Decimal256.fromLong(9999, 1)) + .intColumn("s", 32768) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -4006,580 +3655,422 @@ public void testDecimal256ToDecimal8OverflowError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected overflow error but got: " + msg, - msg.contains("decimal value overflows") + msg.contains("integer value 32768 out of range for SHORT") ); } } @Test - public void testStringToBoolean() throws Exception { - String table = "test_qwp_string_to_boolean"; + public void testIntToString() throws Exception { + String table = "test_qwp_int_to_string"; useTable(table); execute("CREATE TABLE " + table + " (" + - "b BOOLEAN, " + + "s STRING, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .stringColumn("b", "true") + .intColumn("s", 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .stringColumn("b", "false") + .intColumn("s", -100) .at(2_000_000, ChronoUnit.MICROS); sender.table(table) - .stringColumn("b", "1") + .intColumn("s", 0) .at(3_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("b", "0") - .at(4_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("b", "TRUE") - .at(5_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 5); + assertTableSizeEventually(table, 3); assertSqlEventually( - "b\tts\n" + - "true\t1970-01-01T00:00:01.000000000Z\n" + - "false\t1970-01-01T00:00:02.000000000Z\n" + - "true\t1970-01-01T00:00:03.000000000Z\n" + - "false\t1970-01-01T00:00:04.000000000Z\n" + - "true\t1970-01-01T00:00:05.000000000Z\n", - "SELECT b, ts FROM " + table + " ORDER BY ts"); + "s\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-100\t1970-01-01T00:00:02.000000000Z\n" + + "0\t1970-01-01T00:00:03.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testStringToBooleanParseError() throws Exception { - String table = "test_qwp_string_to_boolean_err"; + public void testIntToSymbol() throws Exception { + String table = "test_qwp_int_to_symbol"; useTable(table); execute("CREATE TABLE " + table + " (" + - "b BOOLEAN, " + + "s SYMBOL, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .stringColumn("b", "yes") + .intColumn("s", 42) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("s", -1) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("s", 0) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected parse error but got: " + msg, - msg.contains("cannot parse boolean from string") - ); } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "s\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-1\t1970-01-01T00:00:02.000000000Z\n" + + "0\t1970-01-01T00:00:03.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testStringToByte() throws Exception { - String table = "test_qwp_string_to_byte"; + public void testIntToTimestamp() throws Exception { + String table = "test_qwp_int_to_timestamp"; useTable(table); execute("CREATE TABLE " + table + " (" + - "b BYTE, " + + "t TIMESTAMP, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { + // 1_000_000 micros = 1 second sender.table(table) - .stringColumn("b", "42") + .intColumn("t", 1_000_000) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .stringColumn("b", "-128") + .intColumn("t", 0) .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("b", "127") - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 3); + assertTableSizeEventually(table, 2); assertSqlEventually( - "b\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-128\t1970-01-01T00:00:02.000000000Z\n" + - "127\t1970-01-01T00:00:03.000000000Z\n", - "SELECT b, ts FROM " + table + " ORDER BY ts"); + "t\tts\n" + + "1970-01-01T00:00:01.000000000Z\t1970-01-01T00:00:01.000000000Z\n" + + "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", + "SELECT t, ts FROM " + table + " ORDER BY ts"); } @Test - public void testStringToByteParseError() throws Exception { - String table = "test_qwp_string_to_byte_err"; + public void testIntToUuidCoercionError() throws Exception { + String table = "test_qwp_int_to_uuid_error"; useTable(table); execute("CREATE TABLE " + table + " (" + - "b BYTE, " + + "u UUID, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .stringColumn("b", "abc") + .intColumn("u", 42) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected parse error but got: " + msg, - msg.contains("cannot parse BYTE from string") + "Expected coercion error but got: " + msg, + msg.contains("type coercion from INT to UUID is not supported") ); } } @Test - public void testStringToDate() throws Exception { - String table = "test_qwp_string_to_date"; + public void testIntToVarchar() throws Exception { + String table = "test_qwp_int_to_varchar"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DATE, " + + "v VARCHAR, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .stringColumn("d", "2022-02-25T00:00:00.000Z") + .intColumn("v", 42) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("v", -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .intColumn("v", Integer.MAX_VALUE) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 1); + assertTableSizeEventually(table, 3); assertSqlEventually( - "d\tts\n" + - "2022-02-25T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testStringToDecimal64() throws Exception { - String table = "test_qwp_string_to_dec64"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(18, 2), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .stringColumn("d", "123.45") - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("d", "-99.99") - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "d\tts\n" + - "123.45\t1970-01-01T00:00:01.000000000Z\n" + - "-99.99\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testStringToDecimal128() throws Exception { - String table = "test_qwp_string_to_dec128"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(38, 2), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .stringColumn("d", "123.45") - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("d", "-99.99") - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "d\tts\n" + - "123.45\t1970-01-01T00:00:01.000000000Z\n" + - "-99.99\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + "v\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-100\t1970-01-01T00:00:02.000000000Z\n" + + "2147483647\t1970-01-01T00:00:03.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); } @Test - public void testStringToDecimal256() throws Exception { - String table = "test_qwp_string_to_dec256"; + public void testLong() throws Exception { + String table = "test_qwp_long"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DECIMAL(76, 2), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { + // Long.MIN_VALUE is the null sentinel for LONG sender.table(table) - .stringColumn("d", "123.45") + .longColumn("l", Long.MIN_VALUE) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .stringColumn("d", "-99.99") + .longColumn("l", 0) .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "d\tts\n" + - "123.45\t1970-01-01T00:00:01.000000000Z\n" + - "-99.99\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); - } - - @Test - public void testStringToDouble() throws Exception { - String table = "test_qwp_string_to_double"; - useTable(table); - execute("CREATE TABLE " + table + " (" + - "d DOUBLE, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .stringColumn("d", "3.14") - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("d", "-2.718") - .at(2_000_000, ChronoUnit.MICROS); + .longColumn("l", Long.MAX_VALUE) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); assertSqlEventually( - "d\tts\n" + - "3.14\t1970-01-01T00:00:01.000000000Z\n" + - "-2.718\t1970-01-01T00:00:02.000000000Z\n", - "SELECT d, ts FROM " + table + " ORDER BY ts"); + "l\ttimestamp\n" + + "null\t1970-01-01T00:00:01.000000000Z\n" + + "0\t1970-01-01T00:00:02.000000000Z\n" + + "9223372036854775807\t1970-01-01T00:00:03.000000000Z\n", + "SELECT l, timestamp FROM " + table + " ORDER BY timestamp"); } @Test - public void testStringToFloat() throws Exception { - String table = "test_qwp_string_to_float"; + public void testLong256() throws Exception { + String table = "test_qwp_long256"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "f FLOAT, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { + // All zeros sender.table(table) - .stringColumn("f", "3.14") + .long256Column("v", 0, 0, 0, 0) .at(1_000_000, ChronoUnit.MICROS); + // Mixed values sender.table(table) - .stringColumn("f", "-2.5") + .long256Column("v", 1, 2, 3, 4) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); - assertSqlEventually( - "f\tts\n" + - "3.14\t1970-01-01T00:00:01.000000000Z\n" + - "-2.5\t1970-01-01T00:00:02.000000000Z\n", - "SELECT f, ts FROM " + table + " ORDER BY ts"); } @Test - public void testStringToGeoHash() throws Exception { - String table = "test_qwp_string_to_geohash"; + public void testLong256ToBooleanCoercionError() throws Exception { + String table = "test_qwp_long256_to_boolean_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "g GEOHASH(5c), " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .stringColumn("g", "s24se") + .long256Column("v", 1L, 0L, 0L, 0L) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("g", "u33dc") - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write LONG256") && msg.contains("BOOLEAN") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "g\tts\n" + - "s24se\t1970-01-01T00:00:01.000000000Z\n" + - "u33dc\t1970-01-01T00:00:02.000000000Z\n", - "SELECT g, ts FROM " + table + " ORDER BY ts"); } @Test - public void testStringToInt() throws Exception { - String table = "test_qwp_string_to_int"; + public void testLong256ToByteCoercionError() throws Exception { + String table = "test_qwp_long256_to_byte_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "i INT, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v BYTE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .stringColumn("i", "42") + .long256Column("v", 1L, 0L, 0L, 0L) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("i", "-100") - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("i", "0") - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") + ); } - - assertTableSizeEventually(table, 3); - assertSqlEventually( - "i\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-100\t1970-01-01T00:00:02.000000000Z\n" + - "0\t1970-01-01T00:00:03.000000000Z\n", - "SELECT i, ts FROM " + table + " ORDER BY ts"); } @Test - public void testStringToLong() throws Exception { - String table = "test_qwp_string_to_long"; + public void testLong256ToCharCoercionError() throws Exception { + String table = "test_qwp_long256_to_char_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "l LONG, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v CHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .stringColumn("l", "1000000000000") + .long256Column("v", 1L, 0L, 0L, 0L) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("l", "-1") - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write LONG256") && msg.contains("CHAR") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "l\tts\n" + - "1000000000000\t1970-01-01T00:00:01.000000000Z\n" + - "-1\t1970-01-01T00:00:02.000000000Z\n", - "SELECT l, ts FROM " + table + " ORDER BY ts"); } @Test - public void testStringToLong256() throws Exception { - String table = "test_qwp_string_to_long256"; + public void testLong256ToDateCoercionError() throws Exception { + String table = "test_qwp_long256_to_date_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "l LONG256, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .stringColumn("l", "0x01") + .long256Column("v", 1L, 0L, 0L, 0L) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") + ); } - - assertTableSizeEventually(table, 1); - assertSqlEventually( - "l\tts\n" + - "0x01\t1970-01-01T00:00:01.000000000Z\n", - "SELECT l, ts FROM " + table + " ORDER BY ts"); } @Test - public void testStringToShort() throws Exception { - String table = "test_qwp_string_to_short"; + public void testLong256ToDoubleCoercionError() throws Exception { + String table = "test_qwp_long256_to_double_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "s SHORT, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v DOUBLE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .stringColumn("s", "1000") + .long256Column("v", 1L, 0L, 0L, 0L) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("s", "-32768") - .at(2_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("s", "32767") - .at(3_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") + ); } - - assertTableSizeEventually(table, 3); - assertSqlEventually( - "s\tts\n" + - "1000\t1970-01-01T00:00:01.000000000Z\n" + - "-32768\t1970-01-01T00:00:02.000000000Z\n" + - "32767\t1970-01-01T00:00:03.000000000Z\n", - "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testStringToTimestamp() throws Exception { - String table = "test_qwp_string_to_timestamp"; + public void testLong256ToFloatCoercionError() throws Exception { + String table = "test_qwp_long256_to_float_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "t TIMESTAMP, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .stringColumn("t", "2022-02-25T00:00:00.000000Z") + .long256Column("v", 1L, 0L, 0L, 0L) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") + ); } - - assertTableSizeEventually(table, 1); - assertSqlEventually( - "t\tts\n" + - "2022-02-25T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z\n", - "SELECT t, ts FROM " + table + " ORDER BY ts"); } @Test - public void testBoolToString() throws Exception { - String table = "test_qwp_bool_to_string"; + public void testLong256ToGeoHashCoercionError() throws Exception { + String table = "test_qwp_long256_to_geohash_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "s STRING, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .boolColumn("s", true) + .long256Column("v", 1L, 0L, 0L, 0L) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .boolColumn("s", false) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "s\tts\n" + - "true\t1970-01-01T00:00:01.000000000Z\n" + - "false\t1970-01-01T00:00:02.000000000Z\n", - "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testBoolToVarchar() throws Exception { - String table = "test_qwp_bool_to_varchar"; + public void testLong256ToIntCoercionError() throws Exception { + String table = "test_qwp_long256_to_int_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "v VARCHAR, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .boolColumn("v", true) + .long256Column("v", 1L, 0L, 0L, 0L) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .boolColumn("v", false) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "v\tts\n" + - "true\t1970-01-01T00:00:01.000000000Z\n" + - "false\t1970-01-01T00:00:02.000000000Z\n", - "SELECT v, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDecimalToString() throws Exception { - String table = "test_qwp_decimal_to_string"; + public void testLong256ToLongCoercionError() throws Exception { + String table = "test_qwp_long256_to_long_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "s STRING, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v LONG, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .decimalColumn("s", Decimal64.fromLong(12345, 2)) + .long256Column("v", 1L, 0L, 0L, 0L) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .decimalColumn("s", Decimal64.fromLong(-9999, 2)) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "s\tts\n" + - "123.45\t1970-01-01T00:00:01.000000000Z\n" + - "-99.99\t1970-01-01T00:00:02.000000000Z\n", - "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDecimalToVarchar() throws Exception { - String table = "test_qwp_decimal_to_varchar"; + public void testLong256ToShortCoercionError() throws Exception { + String table = "test_qwp_long256_to_short_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "v VARCHAR, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v SHORT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .decimalColumn("v", Decimal64.fromLong(12345, 2)) + .long256Column("v", 1L, 0L, 0L, 0L) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .decimalColumn("v", Decimal64.fromLong(-9999, 2)) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "v\tts\n" + - "123.45\t1970-01-01T00:00:01.000000000Z\n" + - "-99.99\t1970-01-01T00:00:02.000000000Z\n", - "SELECT v, ts FROM " + table + " ORDER BY ts"); } @Test - public void testSymbolToString() throws Exception { - String table = "test_qwp_symbol_to_string"; + public void testLong256ToString() throws Exception { + String table = "test_qwp_long256_to_string"; useTable(table); execute("CREATE TABLE " + table + " (" + "s STRING, " + @@ -4589,78 +4080,63 @@ public void testSymbolToString() throws Exception { try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .symbol("s", "hello") + .long256Column("s", 1, 2, 3, 4) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .symbol("s", "world") - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 1); assertSqlEventually( "s\tts\n" + - "hello\t1970-01-01T00:00:01.000000000Z\n" + - "world\t1970-01-01T00:00:02.000000000Z\n", - "SELECT s, ts FROM " + table + " ORDER BY ts"); + "0x04000000000000000300000000000000020000000000000001\t1970-01-01T00:00:01.000000000Z\n", + "SELECT s, ts FROM " + table); } @Test - public void testSymbolToVarchar() throws Exception { - String table = "test_qwp_symbol_to_varchar"; + public void testLong256ToSymbolCoercionError() throws Exception { + String table = "test_qwp_long256_to_symbol_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "v VARCHAR, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v SYMBOL, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .symbol("v", "hello") + .long256Column("v", 1L, 0L, 0L, 0L) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .symbol("v", "world") - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write LONG256") && msg.contains("SYMBOL") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "v\tts\n" + - "hello\t1970-01-01T00:00:01.000000000Z\n" + - "world\t1970-01-01T00:00:02.000000000Z\n", - "SELECT v, ts FROM " + table + " ORDER BY ts"); } @Test - public void testTimestampToString() throws Exception { - String table = "test_qwp_timestamp_to_string"; + public void testLong256ToUuidCoercionError() throws Exception { + String table = "test_qwp_long256_to_uuid_error"; useTable(table); - execute("CREATE TABLE " + table + " (" + - "s STRING, " + - "ts TIMESTAMP" + - ") TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - long tsMicros = 1_645_747_200_000_000L; // 2022-02-25T00:00:00Z in micros sender.table(table) - .timestampColumn("s", tsMicros, ChronoUnit.MICROS) + .long256Column("v", 1L, 0L, 0L, 0L) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") + ); } - - assertTableSizeEventually(table, 1); - assertSqlEventually( - "s\tts\n" + - "2022-02-25T00:00:00.000Z\t1970-01-01T00:00:01.000000000Z\n", - "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testTimestampToVarchar() throws Exception { - String table = "test_qwp_timestamp_to_varchar"; + public void testLong256ToVarchar() throws Exception { + String table = "test_qwp_long256_to_varchar"; useTable(table); execute("CREATE TABLE " + table + " (" + "v VARCHAR, " + @@ -4669,9 +4145,8 @@ public void testTimestampToVarchar() throws Exception { assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - long tsMicros = 1_645_747_200_000_000L; // 2022-02-25T00:00:00Z in micros sender.table(table) - .timestampColumn("v", tsMicros, ChronoUnit.MICROS) + .long256Column("v", 1, 2, 3, 4) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); } @@ -4679,214 +4154,217 @@ public void testTimestampToVarchar() throws Exception { assertTableSizeEventually(table, 1); assertSqlEventually( "v\tts\n" + - "2022-02-25T00:00:00.000Z\t1970-01-01T00:00:01.000000000Z\n", - "SELECT v, ts FROM " + table + " ORDER BY ts"); + "0x04000000000000000300000000000000020000000000000001\t1970-01-01T00:00:01.000000000Z\n", + "SELECT v, ts FROM " + table); } @Test - public void testCharToString() throws Exception { - String table = "test_qwp_char_to_string"; + public void testLongToBooleanCoercionError() throws Exception { + String table = "test_qwp_long_to_boolean_error"; useTable(table); execute("CREATE TABLE " + table + " (" + - "s STRING, " + + "b BOOLEAN, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .charColumn("s", 'A') + .longColumn("b", 1) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .charColumn("s", 'Z') - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected error mentioning LONG and BOOLEAN but got: " + msg, + msg.contains("LONG") && msg.contains("BOOLEAN") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "s\tts\n" + - "A\t1970-01-01T00:00:01.000000000Z\n" + - "Z\t1970-01-01T00:00:02.000000000Z\n", - "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testCharToVarchar() throws Exception { - String table = "test_qwp_char_to_varchar"; + public void testLongToByte() throws Exception { + String table = "test_qwp_long_to_byte"; useTable(table); execute("CREATE TABLE " + table + " (" + - "v VARCHAR, " + + "b BYTE, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .charColumn("v", 'A') + .longColumn("b", 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .charColumn("v", 'Z') + .longColumn("b", -128) .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("b", 127) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + assertTableSizeEventually(table, 3); assertSqlEventually( - "v\tts\n" + - "A\t1970-01-01T00:00:01.000000000Z\n" + - "Z\t1970-01-01T00:00:02.000000000Z\n", - "SELECT v, ts FROM " + table + " ORDER BY ts"); + "b\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-128\t1970-01-01T00:00:02.000000000Z\n" + + "127\t1970-01-01T00:00:03.000000000Z\n", + "SELECT b, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDoubleToShort() throws Exception { - String table = "test_qwp_double_to_short"; + public void testLongToByteOverflowError() throws Exception { + String table = "test_qwp_long_to_byte_overflow"; useTable(table); execute("CREATE TABLE " + table + " (" + - "v SHORT, " + + "b BYTE, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("v", 100.0) + .longColumn("b", 128) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .doubleColumn("v", -200.0) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected overflow error but got: " + msg, + msg.contains("integer value 128 out of range for BYTE") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "v\tts\n" + - "100\t1970-01-01T00:00:01.000000000Z\n" + - "-200\t1970-01-01T00:00:02.000000000Z\n", - "SELECT v, ts FROM " + table + " ORDER BY ts"); } @Test - public void testFloatToByte() throws Exception { - String table = "test_qwp_float_to_byte"; + public void testLongToCharCoercionError() throws Exception { + String table = "test_qwp_long_to_char_error"; useTable(table); execute("CREATE TABLE " + table + " (" + - "v BYTE, " + + "c CHAR, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .floatColumn("v", 7.0f) + .longColumn("c", 65) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .floatColumn("v", -100.0f) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected error mentioning LONG and CHAR but got: " + msg, + msg.contains("LONG") && msg.contains("CHAR") + ); } - - assertTableSizeEventually(table, 2); - assertSqlEventually( - "v\tts\n" + - "7\t1970-01-01T00:00:01.000000000Z\n" + - "-100\t1970-01-01T00:00:02.000000000Z\n", - "SELECT v, ts FROM " + table + " ORDER BY ts"); } @Test - public void testFloatToShort() throws Exception { - String table = "test_qwp_float_to_short"; + public void testLongToDate() throws Exception { + String table = "test_qwp_long_to_date"; useTable(table); execute("CREATE TABLE " + table + " (" + - "v SHORT, " + + "d DATE, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .floatColumn("v", 42.0f) + .longColumn("d", 86_400_000L) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .floatColumn("v", -1000.0f) + .longColumn("d", 0L) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "v\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "-1000\t1970-01-01T00:00:02.000000000Z\n", - "SELECT v, ts FROM " + table + " ORDER BY ts"); + "d\tts\n" + + "1970-01-02T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z\n" + + "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLong256ToString() throws Exception { - String table = "test_qwp_long256_to_string"; + public void testLongToDecimal() throws Exception { + String table = "test_qwp_long_to_decimal"; useTable(table); execute("CREATE TABLE " + table + " (" + - "s STRING, " + + "d DECIMAL(10, 2), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .long256Column("s", 1, 2, 3, 4) + .longColumn("d", 42) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("d", -100) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 1); + assertTableSizeEventually(table, 2); assertSqlEventually( - "s\tts\n" + - "0x04000000000000000300000000000000020000000000000001\t1970-01-01T00:00:01.000000000Z\n", - "SELECT s, ts FROM " + table); + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLong256ToVarchar() throws Exception { - String table = "test_qwp_long256_to_varchar"; + public void testLongToDecimal128() throws Exception { + String table = "test_qwp_long_to_decimal128"; useTable(table); execute("CREATE TABLE " + table + " (" + - "v VARCHAR, " + + "d DECIMAL(38, 2), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .long256Column("v", 1, 2, 3, 4) + .longColumn("d", 1_000_000_000L) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("d", -1_000_000_000L) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 1); + assertTableSizeEventually(table, 2); assertSqlEventually( - "v\tts\n" + - "0x04000000000000000300000000000000020000000000000001\t1970-01-01T00:00:01.000000000Z\n", - "SELECT v, ts FROM " + table); + "d\tts\n" + + "1000000000.00\t1970-01-01T00:00:01.000000000Z\n" + + "-1000000000.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testStringToDecimal8() throws Exception { - String table = "test_qwp_string_to_dec8"; + public void testLongToDecimal16() throws Exception { + String table = "test_qwp_long_to_decimal16"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(2, 1), " + + "d DECIMAL(4, 1), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .stringColumn("d", "1.5") + .longColumn("d", 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .stringColumn("d", "-9.9") + .longColumn("d", -100) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } @@ -4894,27 +4372,27 @@ public void testStringToDecimal8() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( "d\tts\n" + - "1.5\t1970-01-01T00:00:01.000000000Z\n" + - "-9.9\t1970-01-01T00:00:02.000000000Z\n", + "42.0\t1970-01-01T00:00:01.000000000Z\n" + + "-100.0\t1970-01-01T00:00:02.000000000Z\n", "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testStringToDecimal16() throws Exception { - String table = "test_qwp_string_to_dec16"; + public void testLongToDecimal256() throws Exception { + String table = "test_qwp_long_to_decimal256"; useTable(table); execute("CREATE TABLE " + table + " (" + - "d DECIMAL(4, 1), " + + "d DECIMAL(76, 2), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .stringColumn("d", "12.5") + .longColumn("d", Long.MAX_VALUE) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .stringColumn("d", "-99.9") + .longColumn("d", -1_000_000_000_000L) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } @@ -4922,14 +4400,14 @@ public void testStringToDecimal16() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( "d\tts\n" + - "12.5\t1970-01-01T00:00:01.000000000Z\n" + - "-99.9\t1970-01-01T00:00:02.000000000Z\n", + "9223372036854775807.00\t1970-01-01T00:00:01.000000000Z\n" + + "-1000000000000.00\t1970-01-01T00:00:02.000000000Z\n", "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testStringToDecimal32() throws Exception { - String table = "test_qwp_string_to_dec32"; + public void testLongToDecimal32() throws Exception { + String table = "test_qwp_long_to_decimal32"; useTable(table); execute("CREATE TABLE " + table + " (" + "d DECIMAL(6, 2), " + @@ -4939,10 +4417,10 @@ public void testStringToDecimal32() throws Exception { try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .stringColumn("d", "1234.56") + .longColumn("d", 42) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .stringColumn("d", "-999.99") + .longColumn("d", -100) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } @@ -4950,201 +4428,187 @@ public void testStringToDecimal32() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( "d\tts\n" + - "1234.56\t1970-01-01T00:00:01.000000000Z\n" + - "-999.99\t1970-01-01T00:00:02.000000000Z\n", + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n", "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testStringToTimestampNs() throws Exception { - String table = "test_qwp_string_to_timestamp_ns"; + public void testLongToDecimal8() throws Exception { + String table = "test_qwp_long_to_decimal8"; useTable(table); execute("CREATE TABLE " + table + " (" + - "ts_col TIMESTAMP_NS, " + + "d DECIMAL(2, 1), " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .stringColumn("ts_col", "2022-02-25T00:00:00.000000Z") + .longColumn("d", 5) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("d", -9) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 1); + assertTableSizeEventually(table, 2); assertSqlEventually( - "ts_col\tts\n" + - "2022-02-25T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z\n", - "SELECT ts_col, ts FROM " + table); + "d\tts\n" + + "5.0\t1970-01-01T00:00:01.000000000Z\n" + + "-9.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testUuidToString() throws Exception { - String table = "test_qwp_uuid_to_string"; + public void testLongToDouble() throws Exception { + String table = "test_qwp_long_to_double"; useTable(table); execute("CREATE TABLE " + table + " (" + - "s STRING, " + + "d DOUBLE, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .uuidColumn("s", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .longColumn("d", 42) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("d", -100) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 1); + assertTableSizeEventually(table, 2); assertSqlEventually( - "s\tts\n" + - "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11\t1970-01-01T00:00:01.000000000Z\n", - "SELECT s, ts FROM " + table); + "d\tts\n" + + "42.0\t1970-01-01T00:00:01.000000000Z\n" + + "-100.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testUuidToVarchar() throws Exception { - String table = "test_qwp_uuid_to_varchar"; + public void testLongToFloat() throws Exception { + String table = "test_qwp_long_to_float"; useTable(table); execute("CREATE TABLE " + table + " (" + - "v VARCHAR, " + + "f FLOAT, " + "ts TIMESTAMP" + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .longColumn("f", 42) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("f", -100) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 1); + assertTableSizeEventually(table, 2); assertSqlEventually( - "v\tts\n" + - "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11\t1970-01-01T00:00:01.000000000Z\n", - "SELECT v, ts FROM " + table); + "f\tts\n" + + "42.0\t1970-01-01T00:00:01.000000000Z\n" + + "-100.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT f, ts FROM " + table + " ORDER BY ts"); } - // === SYMBOL negative coercion tests === - @Test - public void testSymbolToBooleanCoercionError() throws Exception { - String table = "test_qwp_symbol_to_boolean_error"; + public void testLongToGeoHashCoercionError() throws Exception { + String table = "test_qwp_long_to_geohash_error"; useTable(table); - execute("CREATE TABLE " + table + " (v BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "g GEOHASH(4c), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .symbol("v", "hello") - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write SYMBOL") && msg.contains("BOOLEAN") - ); - } - } - @Test - public void testSymbolToByteCoercionError() throws Exception { - String table = "test_qwp_symbol_to_byte_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v BYTE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .symbol("v", "hello") + .longColumn("g", 42) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write SYMBOL") && msg.contains("BYTE") + "Expected coercion error mentioning LONG but got: " + msg, + msg.contains("type coercion from LONG to") && msg.contains("is not supported") ); } } @Test - public void testSymbolToCharCoercionError() throws Exception { - String table = "test_qwp_symbol_to_char_error"; + public void testLongToInt() throws Exception { + String table = "test_qwp_long_to_int"; useTable(table); - execute("CREATE TABLE " + table + " (v CHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "i INT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .symbol("v", "hello") - .at(1_000_000, ChronoUnit.MICROS); - sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write SYMBOL") && msg.contains("CHAR") - ); - } - } - @Test - public void testSymbolToDateCoercionError() throws Exception { - String table = "test_qwp_symbol_to_date_error"; - useTable(table); - execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { + // Value in INT range should succeed sender.table(table) - .symbol("v", "hello") + .longColumn("i", 42) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("i", -1) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write SYMBOL") && msg.contains("DATE") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "i\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-1\t1970-01-01T00:00:02.000000000Z\n", + "SELECT i, ts FROM " + table + " ORDER BY ts"); } @Test - public void testSymbolToDecimalCoercionError() throws Exception { - String table = "test_qwp_symbol_to_decimal_error"; + public void testLongToIntOverflowError() throws Exception { + String table = "test_qwp_long_to_int_overflow"; useTable(table); - execute("CREATE TABLE " + table + " (v DECIMAL(18,2), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "i INT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .symbol("v", "hello") + .longColumn("i", (long) Integer.MAX_VALUE + 1) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write SYMBOL") && msg.contains("DECIMAL") + "Expected overflow error but got: " + msg, + msg.contains("integer value 2147483648 out of range for INT") ); } } @Test - public void testSymbolToDoubleCoercionError() throws Exception { - String table = "test_qwp_symbol_to_double_error"; + public void testLongToLong256CoercionError() throws Exception { + String table = "test_qwp_long_to_long256_error"; useTable(table); - execute("CREATE TABLE " + table + " (v DOUBLE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "v LONG256, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .symbol("v", "hello") + .longColumn("v", 42) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -5152,125 +4616,157 @@ public void testSymbolToDoubleCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write SYMBOL") && msg.contains("DOUBLE") + msg.contains("type coercion from LONG to LONG256 is not supported") ); } } @Test - public void testSymbolToFloatCoercionError() throws Exception { - String table = "test_qwp_symbol_to_float_error"; + public void testLongToShort() throws Exception { + String table = "test_qwp_long_to_short"; useTable(table); - execute("CREATE TABLE " + table + " (v FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "s SHORT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + // Value in SHORT range should succeed sender.table(table) - .symbol("v", "hello") + .longColumn("s", 42) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("s", -1) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write SYMBOL") && msg.contains("FLOAT") - ); } + + assertTableSizeEventually(table, 2); } @Test - public void testSymbolToGeoHashCoercionError() throws Exception { - String table = "test_qwp_symbol_to_geohash_error"; + public void testLongToShortOverflowError() throws Exception { + String table = "test_qwp_long_to_short_overflow"; useTable(table); - execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "s SHORT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .symbol("v", "hello") + .longColumn("s", 32768) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write SYMBOL") && msg.contains("GEOHASH") + "Expected overflow error but got: " + msg, + msg.contains("integer value 32768 out of range for SHORT") ); } } @Test - public void testSymbolToIntCoercionError() throws Exception { - String table = "test_qwp_symbol_to_int_error"; + public void testLongToString() throws Exception { + String table = "test_qwp_long_to_string"; useTable(table); - execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "s STRING, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .symbol("v", "hello") + .longColumn("s", 42) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("s", Long.MAX_VALUE) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write SYMBOL") && msg.contains("INT") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "s\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "9223372036854775807\t1970-01-01T00:00:02.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testSymbolToLongCoercionError() throws Exception { - String table = "test_qwp_symbol_to_long_error"; + public void testLongToSymbol() throws Exception { + String table = "test_qwp_long_to_symbol"; useTable(table); - execute("CREATE TABLE " + table + " (v LONG, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "s SYMBOL, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .symbol("v", "hello") + .longColumn("s", 42) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("s", -1) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write SYMBOL") && msg.contains("LONG") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "s\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-1\t1970-01-01T00:00:02.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testSymbolToLong256CoercionError() throws Exception { - String table = "test_qwp_symbol_to_long256_error"; + public void testLongToTimestamp() throws Exception { + String table = "test_qwp_long_to_timestamp"; useTable(table); - execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "t TIMESTAMP, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .symbol("v", "hello") + .longColumn("t", 1_000_000L) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("t", 0L) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write SYMBOL") && msg.contains("LONG256") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "t\tts\n" + + "1970-01-01T00:00:01.000000000Z\t1970-01-01T00:00:01.000000000Z\n" + + "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", + "SELECT t, ts FROM " + table + " ORDER BY ts"); } @Test - public void testSymbolToShortCoercionError() throws Exception { - String table = "test_qwp_symbol_to_short_error"; + public void testLongToUuidCoercionError() throws Exception { + String table = "test_qwp_long_to_uuid_error"; useTable(table); - execute("CREATE TABLE " + table + " (v SHORT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "u UUID, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .symbol("v", "hello") + .longColumn("u", 42) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -5278,85 +4774,91 @@ public void testSymbolToShortCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write SYMBOL") && msg.contains("SHORT") + msg.contains("type coercion from LONG to UUID is not supported") ); } } @Test - public void testSymbolToTimestampCoercionError() throws Exception { - String table = "test_qwp_symbol_to_timestamp_error"; + public void testLongToVarchar() throws Exception { + String table = "test_qwp_long_to_varchar"; useTable(table); - execute("CREATE TABLE " + table + " (v TIMESTAMP, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "v VARCHAR, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .symbol("v", "hello") + .longColumn("v", 42) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .longColumn("v", Long.MAX_VALUE) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write SYMBOL") && msg.contains("TIMESTAMP") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "v\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "9223372036854775807\t1970-01-01T00:00:02.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); } @Test - public void testSymbolToTimestampNsCoercionError() throws Exception { - String table = "test_qwp_symbol_to_timestamp_ns_error"; + public void testMultipleRowsAndBatching() throws Exception { + String table = "test_qwp_multiple_rows"; useTable(table); - execute("CREATE TABLE " + table + " (v TIMESTAMP_NS, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); + + int rowCount = 1000; try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .symbol("v", "hello") - .at(1_000_000, ChronoUnit.MICROS); + for (int i = 0; i < rowCount; i++) { + sender.table(table) + .symbol("sym", "s" + (i % 10)) + .longColumn("val", i) + .doubleColumn("dbl", i * 1.5) + .at((long) (i + 1) * 1_000_000, ChronoUnit.MICROS); + } sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write SYMBOL") && msg.contains("TIMESTAMP") - ); } + + assertTableSizeEventually(table, rowCount); } @Test - public void testSymbolToUuidCoercionError() throws Exception { - String table = "test_qwp_symbol_to_uuid_error"; + public void testNullStringToBoolean() throws Exception { + String table = "test_qwp_null_string_to_boolean"; useTable(table); - execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (b BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .symbol("v", "hello") + .stringColumn("b", "true") .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("b", null) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write SYMBOL") && msg.contains("UUID") - ); } + assertTableSizeEventually(table, 2); + assertSqlEventually( + "b\tts\n" + + "true\t1970-01-01T00:00:01.000000000Z\n" + + "false\t1970-01-01T00:00:02.000000000Z\n", + "SELECT b, ts FROM " + table + " ORDER BY ts"); } - // === Null coercion tests === - @Test - public void testNullStringToBoolean() throws Exception { - String table = "test_qwp_null_string_to_boolean"; + public void testNullStringToByte() throws Exception { + String table = "test_qwp_null_string_to_byte"; useTable(table); - execute("CREATE TABLE " + table + " (b BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (b BYTE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .stringColumn("b", "true") + .stringColumn("b", "42") .at(1_000_000, ChronoUnit.MICROS); sender.table(table) .stringColumn("b", null) @@ -5366,8 +4868,8 @@ public void testNullStringToBoolean() throws Exception { assertTableSizeEventually(table, 2); assertSqlEventually( "b\tts\n" + - "true\t1970-01-01T00:00:01.000000000Z\n" + - "false\t1970-01-01T00:00:02.000000000Z\n", + "42\t1970-01-01T00:00:01.000000000Z\n" + + "0\t1970-01-01T00:00:02.000000000Z\n", "SELECT b, ts FROM " + table + " ORDER BY ts"); } @@ -5440,6 +4942,29 @@ public void testNullStringToDecimal() throws Exception { "SELECT d, ts FROM " + table + " ORDER BY ts"); } + @Test + public void testNullStringToFloat() throws Exception { + String table = "test_qwp_null_string_to_float"; + useTable(table); + execute("CREATE TABLE " + table + " (f FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("f", "3.14") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("f", null) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + assertTableSizeEventually(table, 2); + assertSqlEventually( + "f\tts\n" + + "3.14\t1970-01-01T00:00:01.000000000Z\n" + + "null\t1970-01-01T00:00:02.000000000Z\n", + "SELECT f, ts FROM " + table + " ORDER BY ts"); + } + @Test public void testNullStringToGeoHash() throws Exception { String table = "test_qwp_null_string_to_geohash"; @@ -5518,6 +5043,29 @@ public void testNullStringToNumeric() throws Exception { "SELECT i, l, d, ts FROM " + table + " ORDER BY ts"); } + @Test + public void testNullStringToShort() throws Exception { + String table = "test_qwp_null_string_to_short"; + useTable(table); + execute("CREATE TABLE " + table + " (s SHORT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("s", "42") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("s", null) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + assertTableSizeEventually(table, 2); + assertSqlEventually( + "s\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); + } + @Test public void testNullStringToSymbol() throws Exception { String table = "test_qwp_null_string_to_symbol"; @@ -5610,6 +5158,29 @@ public void testNullStringToUuid() throws Exception { "SELECT u, ts FROM " + table + " ORDER BY ts"); } + @Test + public void testNullStringToVarchar() throws Exception { + String table = "test_qwp_null_string_to_varchar"; + useTable(table); + execute("CREATE TABLE " + table + " (v VARCHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .stringColumn("v", "hello") + .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("v", null) + .at(2_000_000, ChronoUnit.MICROS); + sender.flush(); + } + assertTableSizeEventually(table, 2); + assertSqlEventually( + "v\tts\n" + + "hello\t1970-01-01T00:00:01.000000000Z\n" + + "null\t1970-01-01T00:00:02.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); + } + @Test public void testNullSymbolToString() throws Exception { String table = "test_qwp_null_symbol_to_string"; @@ -5634,419 +5205,527 @@ public void testNullSymbolToString() throws Exception { } @Test - public void testNullSymbolToVarchar() throws Exception { - String table = "test_qwp_null_symbol_to_varchar"; + public void testNullSymbolToSymbol() throws Exception { + String table = "test_qwp_null_symbol_to_symbol"; useTable(table); - execute("CREATE TABLE " + table + " (v VARCHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (s SYMBOL, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .symbol("v", "hello") + .symbol("s", "alpha") .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .symbol("v", null) + .symbol("s", null) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } assertTableSizeEventually(table, 2); assertSqlEventually( - "v\tts\n" + - "hello\t1970-01-01T00:00:01.000000000Z\n" + + "s\tts\n" + + "alpha\t1970-01-01T00:00:01.000000000Z\n" + "null\t1970-01-01T00:00:02.000000000Z\n", - "SELECT v, ts FROM " + table + " ORDER BY ts"); + "SELECT s, ts FROM " + table + " ORDER BY ts"); } - // === BOOLEAN negative tests === - @Test - public void testBooleanToByteCoercionError() throws Exception { - String table = "test_qwp_boolean_to_byte_error"; + public void testNullSymbolToVarchar() throws Exception { + String table = "test_qwp_null_symbol_to_varchar"; useTable(table); - execute("CREATE TABLE " + table + " (v BYTE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v VARCHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .boolColumn("v", true) + .symbol("v", "hello") .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .symbol("v", null) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write BOOLEAN") && msg.contains("BYTE") - ); } + assertTableSizeEventually(table, 2); + assertSqlEventually( + "v\tts\n" + + "hello\t1970-01-01T00:00:01.000000000Z\n" + + "null\t1970-01-01T00:00:02.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); } @Test - public void testBooleanToShortCoercionError() throws Exception { - String table = "test_qwp_boolean_to_short_error"; + public void testShort() throws Exception { + String table = "test_qwp_short"; useTable(table); - execute("CREATE TABLE " + table + " (v SHORT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + // Short.MIN_VALUE is the null sentinel for SHORT sender.table(table) - .boolColumn("v", true) + .shortColumn("s", Short.MIN_VALUE) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("s", (short) 0) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("s", Short.MAX_VALUE) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write BOOLEAN") && msg.contains("SHORT") - ); } + + assertTableSizeEventually(table, 3); } @Test - public void testBooleanToIntCoercionError() throws Exception { - String table = "test_qwp_boolean_to_int_error"; + public void testShortToBooleanCoercionError() throws Exception { + String table = "test_qwp_short_to_boolean_error"; useTable(table); - execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "b BOOLEAN, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .boolColumn("v", true) + .shortColumn("b", (short) 1) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write BOOLEAN") && msg.contains("INT") + "Expected error mentioning SHORT and BOOLEAN but got: " + msg, + msg.contains("SHORT") && msg.contains("BOOLEAN") ); } } @Test - public void testBooleanToLongCoercionError() throws Exception { - String table = "test_qwp_boolean_to_long_error"; + public void testShortToByte() throws Exception { + String table = "test_qwp_short_to_byte"; useTable(table); - execute("CREATE TABLE " + table + " (v LONG, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "b BYTE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .boolColumn("v", true) + .shortColumn("b", (short) 42) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("b", (short) -128) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("b", (short) 127) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write BOOLEAN") && msg.contains("LONG") - ); } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "b\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-128\t1970-01-01T00:00:02.000000000Z\n" + + "127\t1970-01-01T00:00:03.000000000Z\n", + "SELECT b, ts FROM " + table + " ORDER BY ts"); } @Test - public void testBooleanToFloatCoercionError() throws Exception { - String table = "test_qwp_boolean_to_float_error"; + public void testShortToByteOverflowError() throws Exception { + String table = "test_qwp_short_to_byte_overflow"; useTable(table); - execute("CREATE TABLE " + table + " (v FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "b BYTE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .boolColumn("v", true) + .shortColumn("b", (short) 128) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write BOOLEAN") && msg.contains("FLOAT") + "Expected overflow error but got: " + msg, + msg.contains("integer value 128 out of range for BYTE") ); } } @Test - public void testBooleanToDoubleCoercionError() throws Exception { - String table = "test_qwp_boolean_to_double_error"; + public void testShortToCharCoercionError() throws Exception { + String table = "test_qwp_short_to_char_error"; useTable(table); - execute("CREATE TABLE " + table + " (v DOUBLE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "c CHAR, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .boolColumn("v", true) + .shortColumn("c", (short) 65) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write BOOLEAN") && msg.contains("DOUBLE") + "Expected error mentioning SHORT and CHAR but got: " + msg, + msg.contains("SHORT") && msg.contains("CHAR") ); } } @Test - public void testBooleanToDateCoercionError() throws Exception { - String table = "test_qwp_boolean_to_date_error"; + public void testShortToDate() throws Exception { + String table = "test_qwp_short_to_date"; useTable(table); - execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "d DATE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + // 1000 millis = 1 second sender.table(table) - .boolColumn("v", true) + .shortColumn("d", (short) 1000) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("d", (short) 0) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write BOOLEAN") && msg.contains("DATE") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "1970-01-01T00:00:01.000000000Z\t1970-01-01T00:00:01.000000000Z\n" + + "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testBooleanToUuidCoercionError() throws Exception { - String table = "test_qwp_boolean_to_uuid_error"; + public void testShortToDecimal128() throws Exception { + String table = "test_qwp_short_to_decimal128"; useTable(table); - execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(38, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .boolColumn("v", true) + .shortColumn("d", Short.MAX_VALUE) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("d", Short.MIN_VALUE) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write BOOLEAN") && msg.contains("UUID") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "32767.00\t1970-01-01T00:00:01.000000000Z\n" + + "-32768.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testBooleanToLong256CoercionError() throws Exception { - String table = "test_qwp_boolean_to_long256_error"; + public void testShortToDecimal16() throws Exception { + String table = "test_qwp_short_to_decimal16"; useTable(table); - execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(4, 1), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .boolColumn("v", true) + .shortColumn("d", (short) 42) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("d", (short) -100) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write BOOLEAN") && msg.contains("LONG256") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "42.0\t1970-01-01T00:00:01.000000000Z\n" + + "-100.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testBooleanToGeoHashCoercionError() throws Exception { - String table = "test_qwp_boolean_to_geohash_error"; + public void testShortToDecimal256() throws Exception { + String table = "test_qwp_short_to_decimal256"; useTable(table); - execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(76, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .boolColumn("v", true) + .shortColumn("d", (short) 42) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("d", (short) -100) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write BOOLEAN") && msg.contains("GEOHASH") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testBooleanToTimestampCoercionError() throws Exception { - String table = "test_qwp_boolean_to_timestamp_error"; + public void testShortToDecimal32() throws Exception { + String table = "test_qwp_short_to_decimal32"; useTable(table); - execute("CREATE TABLE " + table + " (v TIMESTAMP, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(6, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .boolColumn("v", true) + .shortColumn("d", (short) 42) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("d", (short) -100) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write BOOLEAN") && msg.contains("TIMESTAMP") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testBooleanToTimestampNsCoercionError() throws Exception { - String table = "test_qwp_boolean_to_timestamp_ns_error"; + public void testShortToDecimal64() throws Exception { + String table = "test_qwp_short_to_decimal64"; useTable(table); - execute("CREATE TABLE " + table + " (v TIMESTAMP_NS, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(18, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .boolColumn("v", true) + .shortColumn("d", (short) 42) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("d", (short) -100) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write BOOLEAN") && msg.contains("TIMESTAMP") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "42.00\t1970-01-01T00:00:01.000000000Z\n" + + "-100.00\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testBooleanToCharCoercionError() throws Exception { - String table = "test_qwp_boolean_to_char_error"; + public void testShortToDecimal8() throws Exception { + String table = "test_qwp_short_to_decimal8"; useTable(table); - execute("CREATE TABLE " + table + " (v CHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(2, 1), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .boolColumn("v", true) + .shortColumn("d", (short) 5) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("d", (short) -9) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write BOOLEAN") && msg.contains("CHAR") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "5.0\t1970-01-01T00:00:01.000000000Z\n" + + "-9.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testBooleanToSymbolCoercionError() throws Exception { - String table = "test_qwp_boolean_to_symbol_error"; + public void testShortToDouble() throws Exception { + String table = "test_qwp_short_to_double"; useTable(table); - execute("CREATE TABLE " + table + " (v SYMBOL, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "d DOUBLE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .boolColumn("v", true) + .shortColumn("d", (short) 42) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("d", (short) -100) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write BOOLEAN") && msg.contains("SYMBOL") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "42.0\t1970-01-01T00:00:01.000000000Z\n" + + "-100.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testBooleanToDecimalCoercionError() throws Exception { - String table = "test_qwp_boolean_to_decimal_error"; + public void testShortToFloat() throws Exception { + String table = "test_qwp_short_to_float"; useTable(table); - execute("CREATE TABLE " + table + " (v DECIMAL(18,2), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "f FLOAT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .boolColumn("v", true) + .shortColumn("f", (short) 42) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("f", (short) -100) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write BOOLEAN") && msg.contains("DECIMAL") - ); } - } - // === FLOAT negative tests === + assertTableSizeEventually(table, 2); + assertSqlEventually( + "f\tts\n" + + "42.0\t1970-01-01T00:00:01.000000000Z\n" + + "-100.0\t1970-01-01T00:00:02.000000000Z\n", + "SELECT f, ts FROM " + table + " ORDER BY ts"); + } @Test - public void testFloatToBooleanCoercionError() throws Exception { - String table = "test_qwp_float_to_boolean_error"; + public void testShortToGeoHashCoercionError() throws Exception { + String table = "test_qwp_short_to_geohash_error"; useTable(table); - execute("CREATE TABLE " + table + " (v BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "g GEOHASH(4c), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .floatColumn("v", 1.5f) + .shortColumn("g", (short) 42) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write FLOAT") && msg.contains("BOOLEAN") + "Expected coercion error mentioning SHORT but got: " + msg, + msg.contains("type coercion from SHORT to") && msg.contains("is not supported") ); } } @Test - public void testFloatToCharCoercionError() throws Exception { - String table = "test_qwp_float_to_char_error"; + public void testShortToInt() throws Exception { + String table = "test_qwp_short_to_int"; useTable(table); - execute("CREATE TABLE " + table + " (v CHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "i INT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .floatColumn("v", 1.5f) + .shortColumn("i", (short) 42) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("i", Short.MAX_VALUE) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write FLOAT") && msg.contains("CHAR") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "i\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "32767\t1970-01-01T00:00:02.000000000Z\n", + "SELECT i, ts FROM " + table + " ORDER BY ts"); } @Test - public void testFloatToDateCoercionError() throws Exception { - String table = "test_qwp_float_to_date_error"; + public void testShortToLong() throws Exception { + String table = "test_qwp_short_to_long"; useTable(table); - execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "l LONG, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .floatColumn("v", 1.5f) + .shortColumn("l", (short) 42) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("l", Short.MAX_VALUE) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from FLOAT to") && msg.contains("is not supported") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "l\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "32767\t1970-01-01T00:00:02.000000000Z\n", + "SELECT l, ts FROM " + table + " ORDER BY ts"); } @Test - public void testFloatToGeoHashCoercionError() throws Exception { - String table = "test_qwp_float_to_geohash_error"; + public void testShortToLong256CoercionError() throws Exception { + String table = "test_qwp_short_to_long256_error"; useTable(table); - execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "v LONG256, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .floatColumn("v", 1.5f) + .shortColumn("v", (short) 42) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -6054,85 +5733,116 @@ public void testFloatToGeoHashCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("type coercion from FLOAT to") && msg.contains("is not supported") + msg.contains("type coercion from SHORT to LONG256 is not supported") ); } } @Test - public void testFloatToUuidCoercionError() throws Exception { - String table = "test_qwp_float_to_uuid_error"; + public void testShortToString() throws Exception { + String table = "test_qwp_short_to_string"; useTable(table); - execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "s STRING, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .floatColumn("v", 1.5f) + .shortColumn("s", (short) 42) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("s", (short) -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("s", (short) 0) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from FLOAT to") && msg.contains("is not supported") - ); } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "s\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-100\t1970-01-01T00:00:02.000000000Z\n" + + "0\t1970-01-01T00:00:03.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); } - @Test - public void testFloatToLong256CoercionError() throws Exception { - String table = "test_qwp_float_to_long256_error"; + @Test + public void testShortToSymbol() throws Exception { + String table = "test_qwp_short_to_symbol"; useTable(table); - execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "s SYMBOL, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .floatColumn("v", 1.5f) + .shortColumn("s", (short) 42) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("s", (short) -1) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("s", (short) 0) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from FLOAT to") && msg.contains("is not supported") - ); } - } - // === DOUBLE negative tests === + assertTableSizeEventually(table, 3); + assertSqlEventually( + "s\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-1\t1970-01-01T00:00:02.000000000Z\n" + + "0\t1970-01-01T00:00:03.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); + } @Test - public void testDoubleToBooleanCoercionError() throws Exception { - String table = "test_qwp_double_to_boolean_error"; + public void testShortToTimestamp() throws Exception { + String table = "test_qwp_short_to_timestamp"; useTable(table); - execute("CREATE TABLE " + table + " (v BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "t TIMESTAMP, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("v", 3.14) + .shortColumn("t", (short) 1000) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("t", (short) 0) + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write DOUBLE") && msg.contains("BOOLEAN") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "t\tts\n" + + "1970-01-01T00:00:00.001000000Z\t1970-01-01T00:00:01.000000000Z\n" + + "1970-01-01T00:00:00.000000000Z\t1970-01-01T00:00:02.000000000Z\n", + "SELECT t, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDoubleToCharCoercionError() throws Exception { - String table = "test_qwp_double_to_char_error"; + public void testShortToUuidCoercionError() throws Exception { + String table = "test_qwp_short_to_uuid_error"; useTable(table); - execute("CREATE TABLE " + table + " (v CHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "u UUID, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("v", 3.14) + .shortColumn("u", (short) 42) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -6140,813 +5850,986 @@ public void testDoubleToCharCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write DOUBLE") && msg.contains("CHAR") + msg.contains("type coercion from SHORT to UUID is not supported") ); } } @Test - public void testDoubleToDateCoercionError() throws Exception { - String table = "test_qwp_double_to_date_error"; + public void testShortToVarchar() throws Exception { + String table = "test_qwp_short_to_varchar"; useTable(table); - execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "v VARCHAR, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("v", 3.14) + .shortColumn("v", (short) 42) .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("v", (short) -100) + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .shortColumn("v", Short.MAX_VALUE) + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from DOUBLE to") && msg.contains("is not supported") - ); } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "v\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-100\t1970-01-01T00:00:02.000000000Z\n" + + "32767\t1970-01-01T00:00:03.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDoubleToGeoHashCoercionError() throws Exception { - String table = "test_qwp_double_to_geohash_error"; + public void testString() throws Exception { + String table = "test_qwp_string"; useTable(table); - execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("v", 3.14) + .stringColumn("s", "hello world") .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("s", "non-ascii äöü") + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("s", "") + .at(3_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("s", null) + .at(4_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from DOUBLE to") && msg.contains("is not supported") - ); } + + assertTableSizeEventually(table, 4); + assertSqlEventually( + "s\ttimestamp\n" + + "hello world\t1970-01-01T00:00:01.000000000Z\n" + + "non-ascii äöü\t1970-01-01T00:00:02.000000000Z\n" + + "\t1970-01-01T00:00:03.000000000Z\n" + + "null\t1970-01-01T00:00:04.000000000Z\n", + "SELECT s, timestamp FROM " + table + " ORDER BY timestamp"); } @Test - public void testDoubleToUuidCoercionError() throws Exception { - String table = "test_qwp_double_to_uuid_error"; + public void testStringToBoolean() throws Exception { + String table = "test_qwp_string_to_boolean"; useTable(table); - execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "b BOOLEAN, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("v", 3.14) + .stringColumn("b", "true") .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("b", "false") + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("b", "1") + .at(3_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("b", "0") + .at(4_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("b", "TRUE") + .at(5_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from DOUBLE to") && msg.contains("is not supported") - ); } + + assertTableSizeEventually(table, 5); + assertSqlEventually( + "b\tts\n" + + "true\t1970-01-01T00:00:01.000000000Z\n" + + "false\t1970-01-01T00:00:02.000000000Z\n" + + "true\t1970-01-01T00:00:03.000000000Z\n" + + "false\t1970-01-01T00:00:04.000000000Z\n" + + "true\t1970-01-01T00:00:05.000000000Z\n", + "SELECT b, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDoubleToLong256CoercionError() throws Exception { - String table = "test_qwp_double_to_long256_error"; + public void testStringToBooleanParseError() throws Exception { + String table = "test_qwp_string_to_boolean_err"; useTable(table); - execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "b BOOLEAN, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleColumn("v", 3.14) + .stringColumn("b", "yes") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from DOUBLE to") && msg.contains("is not supported") + "Expected parse error but got: " + msg, + msg.contains("cannot parse boolean from string") ); } } - // ==================== CHAR negative tests ==================== - @Test - public void testCharToBooleanCoercionError() throws Exception { - String table = "test_qwp_char_to_boolean_error"; + public void testStringToByte() throws Exception { + String table = "test_qwp_string_to_byte"; useTable(table); - execute("CREATE TABLE " + table + " (v BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "b BYTE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .charColumn("v", 'A') + .stringColumn("b", "42") .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("b", "-128") + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("b", "127") + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write") && msg.contains("BOOLEAN") - ); } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "b\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-128\t1970-01-01T00:00:02.000000000Z\n" + + "127\t1970-01-01T00:00:03.000000000Z\n", + "SELECT b, ts FROM " + table + " ORDER BY ts"); } @Test - public void testCharToSymbolCoercionError() throws Exception { - String table = "test_qwp_char_to_symbol_error"; + public void testStringToByteParseError() throws Exception { + String table = "test_qwp_string_to_byte_err"; useTable(table); - execute("CREATE TABLE " + table + " (v SYMBOL, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "b BYTE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .charColumn("v", 'A') + .stringColumn("b", "abc") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write") && msg.contains("SYMBOL") + "Expected parse error but got: " + msg, + msg.contains("cannot parse BYTE from string") ); } } @Test - public void testCharToByteCoercionError() throws Exception { - String table = "test_qwp_char_to_byte_error"; + public void testStringToChar() throws Exception { + String table = "test_qwp_string_to_char"; useTable(table); - execute("CREATE TABLE " + table + " (v BYTE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "c CHAR, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .charColumn("v", 'A') + .stringColumn("c", "A") .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("c", "Hello") + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("not supported") && msg.contains("BYTE") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "c\tts\n" + + "A\t1970-01-01T00:00:01.000000000Z\n" + + "H\t1970-01-01T00:00:02.000000000Z\n", + "SELECT c, ts FROM " + table + " ORDER BY ts"); } @Test - public void testCharToShortCoercionError() throws Exception { - String table = "test_qwp_char_to_short_error"; + public void testStringToDate() throws Exception { + String table = "test_qwp_string_to_date"; useTable(table); - execute("CREATE TABLE " + table + " (v SHORT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "d DATE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .charColumn("v", 'A') + .stringColumn("d", "2022-02-25T00:00:00.000Z") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("not supported") && msg.contains("SHORT") - ); } + + assertTableSizeEventually(table, 1); + assertSqlEventually( + "d\tts\n" + + "2022-02-25T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testCharToIntCoercionError() throws Exception { - String table = "test_qwp_char_to_int_error"; + public void testStringToDateParseError() throws Exception { + String table = "test_qwp_string_to_date_parse_error"; useTable(table); - execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .charColumn("v", 'A') + .stringColumn("v", "not_a_date") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("not supported") && msg.contains("INT") + "Expected parse error but got: " + msg, + msg.contains("cannot parse DATE from string") && msg.contains("not_a_date") ); } } @Test - public void testCharToLongCoercionError() throws Exception { - String table = "test_qwp_char_to_long_error"; + public void testStringToDecimal128() throws Exception { + String table = "test_qwp_string_to_dec128"; useTable(table); - execute("CREATE TABLE " + table + " (v LONG, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(38, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .charColumn("v", 'A') + .stringColumn("d", "123.45") .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("d", "-99.99") + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("not supported") && msg.contains("LONG") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "123.45\t1970-01-01T00:00:01.000000000Z\n" + + "-99.99\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testCharToFloatCoercionError() throws Exception { - String table = "test_qwp_char_to_float_error"; + public void testStringToDecimal16() throws Exception { + String table = "test_qwp_string_to_dec16"; useTable(table); - execute("CREATE TABLE " + table + " (v FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(4, 1), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .charColumn("v", 'A') + .stringColumn("d", "12.5") .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("d", "-99.9") + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("not supported") && msg.contains("FLOAT") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "12.5\t1970-01-01T00:00:01.000000000Z\n" + + "-99.9\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testCharToDoubleCoercionError() throws Exception { - String table = "test_qwp_char_to_double_error"; + public void testStringToDecimal256() throws Exception { + String table = "test_qwp_string_to_dec256"; useTable(table); - execute("CREATE TABLE " + table + " (v DOUBLE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(76, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .charColumn("v", 'A') + .stringColumn("d", "123.45") .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("d", "-99.99") + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("not supported") && msg.contains("DOUBLE") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "123.45\t1970-01-01T00:00:01.000000000Z\n" + + "-99.99\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testCharToDateCoercionError() throws Exception { - String table = "test_qwp_char_to_date_error"; + public void testStringToDecimal32() throws Exception { + String table = "test_qwp_string_to_dec32"; useTable(table); - execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(6, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .charColumn("v", 'A') + .stringColumn("d", "1234.56") .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("d", "-999.99") + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("not supported") && msg.contains("DATE") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "1234.56\t1970-01-01T00:00:01.000000000Z\n" + + "-999.99\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testCharToUuidCoercionError() throws Exception { - String table = "test_qwp_char_to_uuid_error"; + public void testStringToDecimal64() throws Exception { + String table = "test_qwp_string_to_dec64"; useTable(table); - execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(18, 2), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .charColumn("v", 'A') + .stringColumn("d", "123.45") .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("d", "-99.99") + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("not supported") && msg.contains("UUID") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "123.45\t1970-01-01T00:00:01.000000000Z\n" + + "-99.99\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testCharToLong256CoercionError() throws Exception { - String table = "test_qwp_char_to_long256_error"; + public void testStringToDecimal8() throws Exception { + String table = "test_qwp_string_to_dec8"; useTable(table); - execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "d DECIMAL(2, 1), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .charColumn("v", 'A') + .stringColumn("d", "1.5") .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("d", "-9.9") + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("not supported") && msg.contains("LONG256") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "1.5\t1970-01-01T00:00:01.000000000Z\n" + + "-9.9\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); } @Test - public void testCharToGeoHashCoercionError() throws Exception { - String table = "test_qwp_char_to_geohash_error"; + public void testStringToDouble() throws Exception { + String table = "test_qwp_string_to_double"; useTable(table); - execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "d DOUBLE, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .charColumn("v", 'A') + .stringColumn("d", "3.14") .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("d", "-2.718") + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("GEOHASH") - ); } - } - // ==================== LONG256 negative tests ==================== + assertTableSizeEventually(table, 2); + assertSqlEventually( + "d\tts\n" + + "3.14\t1970-01-01T00:00:01.000000000Z\n" + + "-2.718\t1970-01-01T00:00:02.000000000Z\n", + "SELECT d, ts FROM " + table + " ORDER BY ts"); + } @Test - public void testLong256ToBooleanCoercionError() throws Exception { - String table = "test_qwp_long256_to_boolean_error"; + public void testStringToDoubleParseError() throws Exception { + String table = "test_qwp_string_to_double_parse_error"; useTable(table); - execute("CREATE TABLE " + table + " (v BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v DOUBLE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .long256Column("v", 1L, 0L, 0L, 0L) + .stringColumn("v", "not_a_number") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write LONG256") && msg.contains("BOOLEAN") + "Expected parse error but got: " + msg, + msg.contains("cannot parse DOUBLE from string") && msg.contains("not_a_number") ); } } @Test - public void testLong256ToCharCoercionError() throws Exception { - String table = "test_qwp_long256_to_char_error"; + public void testStringToFloat() throws Exception { + String table = "test_qwp_string_to_float"; useTable(table); - execute("CREATE TABLE " + table + " (v CHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "f FLOAT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .long256Column("v", 1L, 0L, 0L, 0L) + .stringColumn("f", "3.14") .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("f", "-2.5") + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write LONG256") && msg.contains("CHAR") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "f\tts\n" + + "3.14\t1970-01-01T00:00:01.000000000Z\n" + + "-2.5\t1970-01-01T00:00:02.000000000Z\n", + "SELECT f, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLong256ToSymbolCoercionError() throws Exception { - String table = "test_qwp_long256_to_symbol_error"; + public void testStringToFloatParseError() throws Exception { + String table = "test_qwp_string_to_float_parse_error"; useTable(table); - execute("CREATE TABLE " + table + " (v SYMBOL, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .long256Column("v", 1L, 0L, 0L, 0L) + .stringColumn("v", "not_a_number") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write LONG256") && msg.contains("SYMBOL") + "Expected parse error but got: " + msg, + msg.contains("cannot parse FLOAT from string") && msg.contains("not_a_number") ); } } @Test - public void testLong256ToByteCoercionError() throws Exception { - String table = "test_qwp_long256_to_byte_error"; + public void testStringToGeoHash() throws Exception { + String table = "test_qwp_string_to_geohash"; useTable(table); - execute("CREATE TABLE " + table + " (v BYTE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "g GEOHASH(5c), " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .long256Column("v", 1L, 0L, 0L, 0L) + .stringColumn("g", "s24se") .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("g", "u33dc") + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "g\tts\n" + + "s24se\t1970-01-01T00:00:01.000000000Z\n" + + "u33dc\t1970-01-01T00:00:02.000000000Z\n", + "SELECT g, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLong256ToShortCoercionError() throws Exception { - String table = "test_qwp_long256_to_short_error"; + public void testStringToGeoHashParseError() throws Exception { + String table = "test_qwp_string_to_geohash_parse_error"; useTable(table); - execute("CREATE TABLE " + table + " (v SHORT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .long256Column("v", 1L, 0L, 0L, 0L) + .stringColumn("v", "!!!") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") + "Expected parse error but got: " + msg, + msg.contains("cannot parse geohash from string") && msg.contains("!!!") ); } } @Test - public void testLong256ToIntCoercionError() throws Exception { - String table = "test_qwp_long256_to_int_error"; + public void testStringToInt() throws Exception { + String table = "test_qwp_string_to_int"; useTable(table); - execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "i INT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .long256Column("v", 1L, 0L, 0L, 0L) + .stringColumn("i", "42") .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("i", "-100") + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("i", "0") + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") - ); } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "i\tts\n" + + "42\t1970-01-01T00:00:01.000000000Z\n" + + "-100\t1970-01-01T00:00:02.000000000Z\n" + + "0\t1970-01-01T00:00:03.000000000Z\n", + "SELECT i, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLong256ToLongCoercionError() throws Exception { - String table = "test_qwp_long256_to_long_error"; + public void testStringToIntParseError() throws Exception { + String table = "test_qwp_string_to_int_parse_error"; useTable(table); - execute("CREATE TABLE " + table + " (v LONG, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .long256Column("v", 1L, 0L, 0L, 0L) + .stringColumn("v", "not_a_number") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") + "Expected parse error but got: " + msg, + msg.contains("cannot parse INT from string") && msg.contains("not_a_number") ); } } @Test - public void testLong256ToFloatCoercionError() throws Exception { - String table = "test_qwp_long256_to_float_error"; + public void testStringToLong() throws Exception { + String table = "test_qwp_string_to_long"; useTable(table); - execute("CREATE TABLE " + table + " (v FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "l LONG, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .long256Column("v", 1L, 0L, 0L, 0L) + .stringColumn("l", "1000000000000") .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("l", "-1") + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "l\tts\n" + + "1000000000000\t1970-01-01T00:00:01.000000000Z\n" + + "-1\t1970-01-01T00:00:02.000000000Z\n", + "SELECT l, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLong256ToDoubleCoercionError() throws Exception { - String table = "test_qwp_long256_to_double_error"; + public void testStringToLong256() throws Exception { + String table = "test_qwp_string_to_long256"; useTable(table); - execute("CREATE TABLE " + table + " (v DOUBLE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "l LONG256, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .long256Column("v", 1L, 0L, 0L, 0L) + .stringColumn("l", "0x01") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") - ); } + + assertTableSizeEventually(table, 1); + assertSqlEventually( + "l\tts\n" + + "0x01\t1970-01-01T00:00:01.000000000Z\n", + "SELECT l, ts FROM " + table + " ORDER BY ts"); } @Test - public void testLong256ToDateCoercionError() throws Exception { - String table = "test_qwp_long256_to_date_error"; + public void testStringToLong256ParseError() throws Exception { + String table = "test_qwp_string_to_long256_parse_error"; useTable(table); - execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .long256Column("v", 1L, 0L, 0L, 0L) + .stringColumn("v", "not_a_long256") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") + "Expected parse error but got: " + msg, + msg.contains("cannot parse long256 from string") && msg.contains("not_a_long256") ); } } @Test - public void testLong256ToUuidCoercionError() throws Exception { - String table = "test_qwp_long256_to_uuid_error"; + public void testStringToLongParseError() throws Exception { + String table = "test_qwp_string_to_long_parse_error"; useTable(table); - execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v LONG, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .long256Column("v", 1L, 0L, 0L, 0L) + .stringColumn("v", "not_a_number") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") + "Expected parse error but got: " + msg, + msg.contains("cannot parse LONG from string") && msg.contains("not_a_number") ); } } @Test - public void testLong256ToGeoHashCoercionError() throws Exception { - String table = "test_qwp_long256_to_geohash_error"; + public void testStringToShort() throws Exception { + String table = "test_qwp_string_to_short"; useTable(table); - execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "s SHORT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .long256Column("v", 1L, 0L, 0L, 0L) + .stringColumn("s", "1000") .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("s", "-32768") + .at(2_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("s", "32767") + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from LONG256 to") && msg.contains("is not supported") - ); } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "s\tts\n" + + "1000\t1970-01-01T00:00:01.000000000Z\n" + + "-32768\t1970-01-01T00:00:02.000000000Z\n" + + "32767\t1970-01-01T00:00:03.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); } - // ==================== UUID negative tests ==================== - @Test - public void testUuidToBooleanCoercionError() throws Exception { - String table = "test_qwp_uuid_to_boolean_error"; + public void testStringToShortParseError() throws Exception { + String table = "test_qwp_string_to_short_parse_error"; useTable(table); - execute("CREATE TABLE " + table + " (v BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v SHORT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); sender.table(table) - .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .stringColumn("v", "not_a_number") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write UUID") && msg.contains("BOOLEAN") + "Expected parse error but got: " + msg, + msg.contains("cannot parse SHORT from string") && msg.contains("not_a_number") ); } } @Test - public void testUuidToCharCoercionError() throws Exception { - String table = "test_qwp_uuid_to_char_error"; + public void testStringToSymbol() throws Exception { + String table = "test_qwp_string_to_symbol"; useTable(table); - execute("CREATE TABLE " + table + " (v CHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "s SYMBOL, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { - UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); sender.table(table) - .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .stringColumn("s", "hello") .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("s", "world") + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write UUID") && msg.contains("CHAR") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "s\tts\n" + + "hello\t1970-01-01T00:00:01.000000000Z\n" + + "world\t1970-01-01T00:00:02.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testUuidToSymbolCoercionError() throws Exception { - String table = "test_qwp_uuid_to_symbol_error"; + public void testStringToTimestamp() throws Exception { + String table = "test_qwp_string_to_timestamp"; useTable(table); - execute("CREATE TABLE " + table + " (v SYMBOL, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "t TIMESTAMP, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { - UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); sender.table(table) - .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .stringColumn("t", "2022-02-25T00:00:00.000000Z") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write UUID") && msg.contains("SYMBOL") - ); } + + assertTableSizeEventually(table, 1); + assertSqlEventually( + "t\tts\n" + + "2022-02-25T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z\n", + "SELECT t, ts FROM " + table + " ORDER BY ts"); } @Test - public void testUuidToByteCoercionError() throws Exception { - String table = "test_qwp_uuid_to_byte_error"; + public void testStringToTimestampNs() throws Exception { + String table = "test_qwp_string_to_timestamp_ns"; useTable(table); - execute("CREATE TABLE " + table + " (v BYTE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "ts_col TIMESTAMP_NS, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { - UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); sender.table(table) - .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .stringColumn("ts_col", "2022-02-25T00:00:00.000000Z") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from UUID to") && msg.contains("is not supported") - ); } + + assertTableSizeEventually(table, 1); + assertSqlEventually( + "ts_col\tts\n" + + "2022-02-25T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z\n", + "SELECT ts_col, ts FROM " + table); } @Test - public void testUuidToIntCoercionError() throws Exception { - String table = "test_qwp_uuid_to_int_error"; + public void testStringToTimestampParseError() throws Exception { + String table = "test_qwp_string_to_timestamp_parse_error"; useTable(table); - execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v TIMESTAMP, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); sender.table(table) - .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .stringColumn("v", "not_a_timestamp") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from UUID to") && msg.contains("is not supported") + "Expected parse error but got: " + msg, + msg.contains("cannot parse timestamp from string") && msg.contains("not_a_timestamp") ); } } @Test - public void testUuidToLongCoercionError() throws Exception { - String table = "test_qwp_uuid_to_long_error"; + public void testStringToUuid() throws Exception { + String table = "test_qwp_string_to_uuid"; useTable(table); - execute("CREATE TABLE " + table + " (v LONG, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "u UUID, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { - UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); sender.table(table) - .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .stringColumn("u", "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from UUID to") && msg.contains("is not supported") - ); } + + assertTableSizeEventually(table, 1); + assertSqlEventually( + "u\tts\n" + + "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11\t1970-01-01T00:00:01.000000000Z\n", + "SELECT u, ts FROM " + table + " ORDER BY ts"); } @Test - public void testUuidToFloatCoercionError() throws Exception { - String table = "test_qwp_uuid_to_float_error"; + public void testStringToUuidParseError() throws Exception { + String table = "test_qwp_string_to_uuid_parse_error"; useTable(table); - execute("CREATE TABLE " + table + " (v FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); sender.table(table) - .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .stringColumn("v", "not-a-uuid") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from UUID to") && msg.contains("is not supported") + "Expected parse error but got: " + msg, + msg.contains("cannot parse UUID from string") && msg.contains("not-a-uuid") ); } } @Test - public void testUuidToDoubleCoercionError() throws Exception { - String table = "test_qwp_uuid_to_double_error"; + public void testStringToVarchar() throws Exception { + String table = "test_qwp_string_to_varchar"; useTable(table); - execute("CREATE TABLE " + table + " (v DOUBLE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v VARCHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); sender.table(table) - .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .stringColumn("v", "hello") .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .stringColumn("v", "world") + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from UUID to") && msg.contains("is not supported") - ); } + assertTableSizeEventually(table, 2); + assertSqlEventually( + "v\tts\n" + + "hello\t1970-01-01T00:00:01.000000000Z\n" + + "world\t1970-01-01T00:00:02.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); } @Test - public void testUuidToDateCoercionError() throws Exception { - String table = "test_qwp_uuid_to_date_error"; + public void testSymbol() throws Exception { + String table = "test_qwp_symbol"; useTable(table); - execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { - UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); sender.table(table) - .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .symbol("s", "alpha") .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .symbol("s", "beta") + .at(2_000_000, ChronoUnit.MICROS); + // repeated value reuses dictionary entry + sender.table(table) + .symbol("s", "alpha") + .at(3_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("type coercion from UUID to") && msg.contains("is not supported") - ); } + + assertTableSizeEventually(table, 3); + assertSqlEventually( + "s\ttimestamp\n" + + "alpha\t1970-01-01T00:00:01.000000000Z\n" + + "beta\t1970-01-01T00:00:02.000000000Z\n" + + "alpha\t1970-01-01T00:00:03.000000000Z\n", + "SELECT s, timestamp FROM " + table + " ORDER BY timestamp"); } @Test - public void testUuidToLong256CoercionError() throws Exception { - String table = "test_qwp_uuid_to_long256_error"; + public void testSymbolToBooleanCoercionError() throws Exception { + String table = "test_qwp_symbol_to_boolean_error"; useTable(table); - execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); sender.table(table) - .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .symbol("v", "hello") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -6954,21 +6837,20 @@ public void testUuidToLong256CoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("type coercion from UUID to") && msg.contains("is not supported") + msg.contains("cannot write SYMBOL") && msg.contains("BOOLEAN") ); } } @Test - public void testUuidToGeoHashCoercionError() throws Exception { - String table = "test_qwp_uuid_to_geohash_error"; + public void testSymbolToByteCoercionError() throws Exception { + String table = "test_qwp_symbol_to_byte_error"; useTable(table); - execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v BYTE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { - UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); sender.table(table) - .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .symbol("v", "hello") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -6976,22 +6858,20 @@ public void testUuidToGeoHashCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("type coercion from UUID to") && msg.contains("is not supported") + msg.contains("cannot write SYMBOL") && msg.contains("BYTE") ); } } - // === TIMESTAMP negative coercion tests === - @Test - public void testTimestampToBooleanCoercionError() throws Exception { - String table = "test_qwp_timestamp_to_boolean_error"; + public void testSymbolToCharCoercionError() throws Exception { + String table = "test_qwp_symbol_to_char_error"; useTable(table); - execute("CREATE TABLE " + table + " (v BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v CHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) + .symbol("v", "hello") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -6999,20 +6879,20 @@ public void testTimestampToBooleanCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write TIMESTAMP") && msg.contains("BOOLEAN") + msg.contains("cannot write SYMBOL") && msg.contains("CHAR") ); } } @Test - public void testTimestampToByteCoercionError() throws Exception { - String table = "test_qwp_timestamp_to_byte_error"; + public void testSymbolToDateCoercionError() throws Exception { + String table = "test_qwp_symbol_to_date_error"; useTable(table); - execute("CREATE TABLE " + table + " (v BYTE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) + .symbol("v", "hello") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -7020,20 +6900,20 @@ public void testTimestampToByteCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write TIMESTAMP") && msg.contains("BYTE") + msg.contains("cannot write SYMBOL") && msg.contains("DATE") ); } } @Test - public void testTimestampToShortCoercionError() throws Exception { - String table = "test_qwp_timestamp_to_short_error"; + public void testSymbolToDecimalCoercionError() throws Exception { + String table = "test_qwp_symbol_to_decimal_error"; useTable(table); - execute("CREATE TABLE " + table + " (v SHORT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v DECIMAL(18,2), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) + .symbol("v", "hello") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -7041,20 +6921,20 @@ public void testTimestampToShortCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write TIMESTAMP") && msg.contains("SHORT") + msg.contains("cannot write SYMBOL") && msg.contains("DECIMAL") ); } } @Test - public void testTimestampToIntCoercionError() throws Exception { - String table = "test_qwp_timestamp_to_int_error"; + public void testSymbolToDoubleCoercionError() throws Exception { + String table = "test_qwp_symbol_to_double_error"; useTable(table); - execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v DOUBLE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) + .symbol("v", "hello") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -7062,20 +6942,20 @@ public void testTimestampToIntCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write TIMESTAMP") && msg.contains("INT") + msg.contains("cannot write SYMBOL") && msg.contains("DOUBLE") ); } } @Test - public void testTimestampToLongCoercionError() throws Exception { - String table = "test_qwp_timestamp_to_long_error"; + public void testSymbolToFloatCoercionError() throws Exception { + String table = "test_qwp_symbol_to_float_error"; useTable(table); - execute("CREATE TABLE " + table + " (v LONG, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) + .symbol("v", "hello") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -7083,20 +6963,20 @@ public void testTimestampToLongCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write TIMESTAMP") && msg.contains("LONG") + msg.contains("cannot write SYMBOL") && msg.contains("FLOAT") ); } } @Test - public void testTimestampToFloatCoercionError() throws Exception { - String table = "test_qwp_timestamp_to_float_error"; + public void testSymbolToGeoHashCoercionError() throws Exception { + String table = "test_qwp_symbol_to_geohash_error"; useTable(table); - execute("CREATE TABLE " + table + " (v FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) + .symbol("v", "hello") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -7104,20 +6984,20 @@ public void testTimestampToFloatCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write TIMESTAMP") && msg.contains("FLOAT") + msg.contains("cannot write SYMBOL") && msg.contains("GEOHASH") ); } } @Test - public void testTimestampToDoubleCoercionError() throws Exception { - String table = "test_qwp_timestamp_to_double_error"; + public void testSymbolToIntCoercionError() throws Exception { + String table = "test_qwp_symbol_to_int_error"; useTable(table); - execute("CREATE TABLE " + table + " (v DOUBLE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) + .symbol("v", "hello") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -7125,20 +7005,20 @@ public void testTimestampToDoubleCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write TIMESTAMP") && msg.contains("DOUBLE") + msg.contains("cannot write SYMBOL") && msg.contains("INT") ); } } @Test - public void testTimestampToDateCoercionError() throws Exception { - String table = "test_qwp_timestamp_to_date_error"; + public void testSymbolToLong256CoercionError() throws Exception { + String table = "test_qwp_symbol_to_long256_error"; useTable(table); - execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) + .symbol("v", "hello") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -7146,20 +7026,20 @@ public void testTimestampToDateCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write TIMESTAMP") && msg.contains("DATE") + msg.contains("cannot write SYMBOL") && msg.contains("LONG256") ); } } @Test - public void testTimestampToUuidCoercionError() throws Exception { - String table = "test_qwp_timestamp_to_uuid_error"; + public void testSymbolToLongCoercionError() throws Exception { + String table = "test_qwp_symbol_to_long_error"; useTable(table); - execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v LONG, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) + .symbol("v", "hello") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -7167,20 +7047,20 @@ public void testTimestampToUuidCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write TIMESTAMP") && msg.contains("UUID") + msg.contains("cannot write SYMBOL") && msg.contains("LONG") ); } } @Test - public void testTimestampToLong256CoercionError() throws Exception { - String table = "test_qwp_timestamp_to_long256_error"; + public void testSymbolToShortCoercionError() throws Exception { + String table = "test_qwp_symbol_to_short_error"; useTable(table); - execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v SHORT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) + .symbol("v", "hello") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -7188,41 +7068,48 @@ public void testTimestampToLong256CoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write TIMESTAMP") && msg.contains("LONG256") + msg.contains("cannot write SYMBOL") && msg.contains("SHORT") ); } } @Test - public void testTimestampToGeoHashCoercionError() throws Exception { - String table = "test_qwp_timestamp_to_geohash_error"; + public void testSymbolToString() throws Exception { + String table = "test_qwp_symbol_to_string"; useTable(table); - execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "s STRING, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) + .symbol("s", "hello") .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .symbol("s", "world") + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write TIMESTAMP") && msg.contains("GEOHASH") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "s\tts\n" + + "hello\t1970-01-01T00:00:01.000000000Z\n" + + "world\t1970-01-01T00:00:02.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testTimestampToCharCoercionError() throws Exception { - String table = "test_qwp_timestamp_to_char_error"; + public void testSymbolToTimestampCoercionError() throws Exception { + String table = "test_qwp_symbol_to_timestamp_error"; useTable(table); - execute("CREATE TABLE " + table + " (v CHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v TIMESTAMP, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) + .symbol("v", "hello") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -7230,20 +7117,20 @@ public void testTimestampToCharCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write TIMESTAMP") && msg.contains("CHAR") + msg.contains("cannot write SYMBOL") && msg.contains("TIMESTAMP") ); } } @Test - public void testTimestampToSymbolCoercionError() throws Exception { - String table = "test_qwp_timestamp_to_symbol_error"; + public void testSymbolToTimestampNsCoercionError() throws Exception { + String table = "test_qwp_symbol_to_timestamp_ns_error"; useTable(table); - execute("CREATE TABLE " + table + " (v SYMBOL, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v TIMESTAMP_NS, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) + .symbol("v", "hello") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -7251,20 +7138,20 @@ public void testTimestampToSymbolCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write TIMESTAMP") && msg.contains("SYMBOL") + msg.contains("cannot write SYMBOL") && msg.contains("TIMESTAMP") ); } } @Test - public void testTimestampToDecimalCoercionError() throws Exception { - String table = "test_qwp_timestamp_to_decimal_error"; + public void testSymbolToUuidCoercionError() throws Exception { + String table = "test_qwp_symbol_to_uuid_error"; useTable(table); - execute("CREATE TABLE " + table + " (v DECIMAL(18,2), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) + .symbol("v", "hello") .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -7272,106 +7159,136 @@ public void testTimestampToDecimalCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write TIMESTAMP") && msg.contains("DECIMAL") + msg.contains("cannot write SYMBOL") && msg.contains("UUID") ); } } - // === DECIMAL negative coercion tests === - @Test - public void testDecimalToBooleanCoercionError() throws Exception { - String table = "test_qwp_decimal_to_boolean_error"; + public void testSymbolToVarchar() throws Exception { + String table = "test_qwp_symbol_to_varchar"; useTable(table); - execute("CREATE TABLE " + table + " (v BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "v VARCHAR, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .symbol("v", "hello") .at(1_000_000, ChronoUnit.MICROS); + sender.table(table) + .symbol("v", "world") + .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write DECIMAL64") && msg.contains("BOOLEAN") - ); } + + assertTableSizeEventually(table, 2); + assertSqlEventually( + "v\tts\n" + + "hello\t1970-01-01T00:00:01.000000000Z\n" + + "world\t1970-01-01T00:00:02.000000000Z\n", + "SELECT v, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDecimalToByteCoercionError() throws Exception { - String table = "test_qwp_decimal_to_byte_error"; + public void testTimestampMicros() throws Exception { + String table = "test_qwp_timestamp_micros"; useTable(table); - execute("CREATE TABLE " + table + " (v BYTE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + long tsMicros = 1_645_747_200_000_000L; // 2022-02-25T00:00:00Z in micros sender.table(table) - .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .timestampColumn("ts_col", tsMicros, ChronoUnit.MICROS) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write DECIMAL64") && msg.contains("BYTE") - ); } + + assertTableSizeEventually(table, 1); + assertSqlEventually( + "ts_col\ttimestamp\n" + + "2022-02-25T00:00:00.000000000Z\t1970-01-01T00:00:01.000000000Z\n", + "SELECT ts_col, timestamp FROM " + table); } @Test - public void testDecimalToShortCoercionError() throws Exception { - String table = "test_qwp_decimal_to_short_error"; + public void testTimestampMicrosToNanos() throws Exception { + String table = "test_qwp_timestamp_micros_to_nanos"; useTable(table); - execute("CREATE TABLE " + table + " (v SHORT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "ts_col TIMESTAMP_NS, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + long tsMicros = 1_645_747_200_111_111L; // 2022-02-25T00:00:00Z sender.table(table) - .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .timestampColumn("ts_col", tsMicros, ChronoUnit.MICROS) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write DECIMAL64") && msg.contains("SHORT") - ); } + + assertTableSizeEventually(table, 1); + // Microseconds scaled to nanoseconds + assertSqlEventually( + "ts_col\tts\n" + + "2022-02-25T00:00:00.111111000Z\t1970-01-01T00:00:01.000000000Z\n", + "SELECT ts_col, ts FROM " + table); } @Test - public void testDecimalToIntCoercionError() throws Exception { - String table = "test_qwp_decimal_to_int_error"; + public void testTimestampNanos() throws Exception { + String table = "test_qwp_timestamp_nanos"; useTable(table); - execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + + try (QwpWebSocketSender sender = createQwpSender()) { + long tsNanos = 1_645_747_200_000_000_000L; // 2022-02-25T00:00:00Z in nanos + sender.table(table) + .timestampColumn("ts_col", tsNanos, ChronoUnit.NANOS) + .at(tsNanos, ChronoUnit.NANOS); + sender.flush(); + } + + assertTableSizeEventually(table, 1); + } + + @Test + public void testTimestampNanosToMicros() throws Exception { + String table = "test_qwp_timestamp_nanos_to_micros"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "ts_col TIMESTAMP, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + long tsNanos = 1_645_747_200_123_456_789L; sender.table(table) - .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .timestampColumn("ts_col", tsNanos, ChronoUnit.NANOS) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write DECIMAL64") && msg.contains("INT") - ); } + + assertTableSizeEventually(table, 1); + // Nanoseconds truncated to microseconds + assertSqlEventually( + "ts_col\tts\n" + + "2022-02-25T00:00:00.123456000Z\t1970-01-01T00:00:01.000000000Z\n", + "SELECT ts_col, ts FROM " + table); } @Test - public void testDecimalToLongCoercionError() throws Exception { - String table = "test_qwp_decimal_to_long_error"; + public void testTimestampToBooleanCoercionError() throws Exception { + String table = "test_qwp_timestamp_to_boolean_error"; useTable(table); - execute("CREATE TABLE " + table + " (v LONG, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -7379,20 +7296,20 @@ public void testDecimalToLongCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write DECIMAL64") && msg.contains("LONG") + msg.contains("cannot write TIMESTAMP") && msg.contains("BOOLEAN") ); } } @Test - public void testDecimalToFloatCoercionError() throws Exception { - String table = "test_qwp_decimal_to_float_error"; + public void testTimestampToByteCoercionError() throws Exception { + String table = "test_qwp_timestamp_to_byte_error"; useTable(table); - execute("CREATE TABLE " + table + " (v FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v BYTE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -7400,20 +7317,20 @@ public void testDecimalToFloatCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write DECIMAL64") && msg.contains("FLOAT") + msg.contains("cannot write TIMESTAMP") && msg.contains("BYTE") ); } } @Test - public void testDecimalToDoubleCoercionError() throws Exception { - String table = "test_qwp_decimal_to_double_error"; + public void testTimestampToCharCoercionError() throws Exception { + String table = "test_qwp_timestamp_to_char_error"; useTable(table); - execute("CREATE TABLE " + table + " (v DOUBLE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v CHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -7421,20 +7338,20 @@ public void testDecimalToDoubleCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write DECIMAL64") && msg.contains("DOUBLE") + msg.contains("cannot write TIMESTAMP") && msg.contains("CHAR") ); } } @Test - public void testDecimalToDateCoercionError() throws Exception { - String table = "test_qwp_decimal_to_date_error"; + public void testTimestampToDateCoercionError() throws Exception { + String table = "test_qwp_timestamp_to_date_error"; useTable(table); execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -7442,20 +7359,20 @@ public void testDecimalToDateCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write DECIMAL64") && msg.contains("DATE") + msg.contains("cannot write TIMESTAMP") && msg.contains("DATE") ); } } @Test - public void testDecimalToUuidCoercionError() throws Exception { - String table = "test_qwp_decimal_to_uuid_error"; + public void testTimestampToDecimalCoercionError() throws Exception { + String table = "test_qwp_timestamp_to_decimal_error"; useTable(table); - execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v DECIMAL(18,2), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -7463,20 +7380,20 @@ public void testDecimalToUuidCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write DECIMAL64") && msg.contains("UUID") + msg.contains("cannot write TIMESTAMP") && msg.contains("DECIMAL") ); } } @Test - public void testDecimalToLong256CoercionError() throws Exception { - String table = "test_qwp_decimal_to_long256_error"; + public void testTimestampToDoubleCoercionError() throws Exception { + String table = "test_qwp_timestamp_to_double_error"; useTable(table); - execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v DOUBLE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -7484,20 +7401,20 @@ public void testDecimalToLong256CoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write DECIMAL64") && msg.contains("LONG256") + msg.contains("cannot write TIMESTAMP") && msg.contains("DOUBLE") ); } } @Test - public void testDecimalToGeoHashCoercionError() throws Exception { - String table = "test_qwp_decimal_to_geohash_error"; + public void testTimestampToFloatCoercionError() throws Exception { + String table = "test_qwp_timestamp_to_float_error"; useTable(table); - execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -7505,20 +7422,20 @@ public void testDecimalToGeoHashCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write DECIMAL64") && msg.contains("GEOHASH") + msg.contains("cannot write TIMESTAMP") && msg.contains("FLOAT") ); } } @Test - public void testDecimalToTimestampCoercionError() throws Exception { - String table = "test_qwp_decimal_to_timestamp_error"; + public void testTimestampToGeoHashCoercionError() throws Exception { + String table = "test_qwp_timestamp_to_geohash_error"; useTable(table); - execute("CREATE TABLE " + table + " (v TIMESTAMP, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -7526,20 +7443,20 @@ public void testDecimalToTimestampCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write DECIMAL64") && msg.contains("TIMESTAMP") + msg.contains("cannot write TIMESTAMP") && msg.contains("GEOHASH") ); } } @Test - public void testDecimalToTimestampNsCoercionError() throws Exception { - String table = "test_qwp_decimal_to_timestamp_ns_error"; + public void testTimestampToIntCoercionError() throws Exception { + String table = "test_qwp_timestamp_to_int_error"; useTable(table); - execute("CREATE TABLE " + table + " (v TIMESTAMP_NS, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -7547,20 +7464,20 @@ public void testDecimalToTimestampNsCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write DECIMAL64") && msg.contains("TIMESTAMP") + msg.contains("cannot write TIMESTAMP") && msg.contains("INT") ); } } @Test - public void testDecimalToCharCoercionError() throws Exception { - String table = "test_qwp_decimal_to_char_error"; + public void testTimestampToLong256CoercionError() throws Exception { + String table = "test_qwp_timestamp_to_long256_error"; useTable(table); - execute("CREATE TABLE " + table + " (v CHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -7568,20 +7485,20 @@ public void testDecimalToCharCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write DECIMAL64") && msg.contains("CHAR") + msg.contains("cannot write TIMESTAMP") && msg.contains("LONG256") ); } } @Test - public void testDecimalToSymbolCoercionError() throws Exception { - String table = "test_qwp_decimal_to_symbol_error"; + public void testTimestampToLongCoercionError() throws Exception { + String table = "test_qwp_timestamp_to_long_error"; useTable(table); - execute("CREATE TABLE " + table + " (v SYMBOL, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v LONG, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .decimalColumn("v", Decimal64.fromLong(12345L, 2)) + .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -7589,22 +7506,20 @@ public void testDecimalToSymbolCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write DECIMAL64") && msg.contains("SYMBOL") + msg.contains("cannot write TIMESTAMP") && msg.contains("LONG") ); } } - // === DOUBLE_ARRAY negative coercion tests === - @Test - public void testDoubleArrayToIntCoercionError() throws Exception { - String table = "test_qwp_doublearray_to_int_error"; + public void testTimestampToShortCoercionError() throws Exception { + String table = "test_qwp_timestamp_to_short_error"; useTable(table); - execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v SHORT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleArray("v", new double[]{1.0, 2.0}) + .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -7612,41 +7527,45 @@ public void testDoubleArrayToIntCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write DOUBLE_ARRAY") && msg.contains("INT") + msg.contains("cannot write TIMESTAMP") && msg.contains("SHORT") ); } } @Test - public void testDoubleArrayToStringCoercionError() throws Exception { - String table = "test_qwp_doublearray_to_string_error"; + public void testTimestampToString() throws Exception { + String table = "test_qwp_timestamp_to_string"; useTable(table); - execute("CREATE TABLE " + table + " (v STRING, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "s STRING, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + long tsMicros = 1_645_747_200_000_000L; // 2022-02-25T00:00:00Z in micros sender.table(table) - .doubleArray("v", new double[]{1.0, 2.0}) + .timestampColumn("s", tsMicros, ChronoUnit.MICROS) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected coercion error but got: " + msg, - msg.contains("cannot write DOUBLE_ARRAY") && msg.contains("STRING") - ); } + + assertTableSizeEventually(table, 1); + assertSqlEventually( + "s\tts\n" + + "2022-02-25T00:00:00.000Z\t1970-01-01T00:00:01.000000000Z\n", + "SELECT s, ts FROM " + table + " ORDER BY ts"); } @Test - public void testDoubleArrayToSymbolCoercionError() throws Exception { - String table = "test_qwp_doublearray_to_symbol_error"; + public void testTimestampToSymbolCoercionError() throws Exception { + String table = "test_qwp_timestamp_to_symbol_error"; useTable(table); execute("CREATE TABLE " + table + " (v SYMBOL, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleArray("v", new double[]{1.0, 2.0}) + .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -7654,20 +7573,20 @@ public void testDoubleArrayToSymbolCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write DOUBLE_ARRAY") && msg.contains("SYMBOL") + msg.contains("cannot write TIMESTAMP") && msg.contains("SYMBOL") ); } } @Test - public void testDoubleArrayToTimestampCoercionError() throws Exception { - String table = "test_qwp_doublearray_to_timestamp_error"; + public void testTimestampToUuidCoercionError() throws Exception { + String table = "test_qwp_timestamp_to_uuid_error"; useTable(table); - execute("CREATE TABLE " + table + " (v TIMESTAMP, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .doubleArray("v", new double[]{1.0, 2.0}) + .timestampColumn("v", 1_645_747_200_000_000L, ChronoUnit.MICROS) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); @@ -7675,366 +7594,411 @@ public void testDoubleArrayToTimestampCoercionError() throws Exception { String msg = e.getMessage(); Assert.assertTrue( "Expected coercion error but got: " + msg, - msg.contains("cannot write DOUBLE_ARRAY") && msg.contains("TIMESTAMP") + msg.contains("cannot write TIMESTAMP") && msg.contains("UUID") ); } } - // ==================== Additional null coercion tests ==================== - @Test - public void testNullStringToVarchar() throws Exception { - String table = "test_qwp_null_string_to_varchar"; + public void testTimestampToVarchar() throws Exception { + String table = "test_qwp_timestamp_to_varchar"; useTable(table); - execute("CREATE TABLE " + table + " (v VARCHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "v VARCHAR, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + long tsMicros = 1_645_747_200_000_000L; // 2022-02-25T00:00:00Z in micros sender.table(table) - .stringColumn("v", "hello") + .timestampColumn("v", tsMicros, ChronoUnit.MICROS) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("v", null) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } - assertTableSizeEventually(table, 2); + + assertTableSizeEventually(table, 1); assertSqlEventually( "v\tts\n" + - "hello\t1970-01-01T00:00:01.000000000Z\n" + - "null\t1970-01-01T00:00:02.000000000Z\n", + "2022-02-25T00:00:00.000Z\t1970-01-01T00:00:01.000000000Z\n", "SELECT v, ts FROM " + table + " ORDER BY ts"); } @Test - public void testNullSymbolToSymbol() throws Exception { - String table = "test_qwp_null_symbol_to_symbol"; + public void testUuid() throws Exception { + String table = "test_qwp_uuid"; useTable(table); - execute("CREATE TABLE " + table + " (s SYMBOL, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); - try (QwpWebSocketSender sender = createQwpSender()) { - sender.table(table) - .symbol("s", "alpha") - .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .symbol("s", null) - .at(2_000_000, ChronoUnit.MICROS); - sender.flush(); - } - assertTableSizeEventually(table, 2); - assertSqlEventually( - "s\tts\n" + - "alpha\t1970-01-01T00:00:01.000000000Z\n" + - "null\t1970-01-01T00:00:02.000000000Z\n", - "SELECT s, ts FROM " + table + " ORDER BY ts"); - } - @Test - public void testNullStringToByte() throws Exception { - String table = "test_qwp_null_string_to_byte"; - useTable(table); - execute("CREATE TABLE " + table + " (b BYTE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); - assertTableExistsEventually(table); + UUID uuid1 = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); + UUID uuid2 = UUID.fromString("11111111-2222-3333-4444-555555555555"); + try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .stringColumn("b", "42") + .uuidColumn("u", uuid1.getLeastSignificantBits(), uuid1.getMostSignificantBits()) .at(1_000_000, ChronoUnit.MICROS); sender.table(table) - .stringColumn("b", null) + .uuidColumn("u", uuid2.getLeastSignificantBits(), uuid2.getMostSignificantBits()) .at(2_000_000, ChronoUnit.MICROS); sender.flush(); } + assertTableSizeEventually(table, 2); assertSqlEventually( - "b\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "0\t1970-01-01T00:00:02.000000000Z\n", - "SELECT b, ts FROM " + table + " ORDER BY ts"); + "u\ttimestamp\n" + + "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11\t1970-01-01T00:00:01.000000000Z\n" + + "11111111-2222-3333-4444-555555555555\t1970-01-01T00:00:02.000000000Z\n", + "SELECT u, timestamp FROM " + table + " ORDER BY timestamp"); } @Test - public void testNullStringToShort() throws Exception { - String table = "test_qwp_null_string_to_short"; + public void testUuidToBooleanCoercionError() throws Exception { + String table = "test_qwp_uuid_to_boolean_error"; useTable(table); - execute("CREATE TABLE " + table + " (s SHORT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v BOOLEAN, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); sender.table(table) - .stringColumn("s", "42") + .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("s", null) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); - } - assertTableSizeEventually(table, 2); - assertSqlEventually( - "s\tts\n" + - "42\t1970-01-01T00:00:01.000000000Z\n" + - "0\t1970-01-01T00:00:02.000000000Z\n", - "SELECT s, ts FROM " + table + " ORDER BY ts"); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write UUID") && msg.contains("BOOLEAN") + ); + } } @Test - public void testNullStringToFloat() throws Exception { - String table = "test_qwp_null_string_to_float"; + public void testUuidToByteCoercionError() throws Exception { + String table = "test_qwp_uuid_to_byte_error"; useTable(table); - execute("CREATE TABLE " + table + " (f FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v BYTE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); sender.table(table) - .stringColumn("f", "3.14") + .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("f", null) - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("type coercion from UUID to") && msg.contains("is not supported") + ); } - assertTableSizeEventually(table, 2); - assertSqlEventually( - "f\tts\n" + - "3.14\t1970-01-01T00:00:01.000000000Z\n" + - "null\t1970-01-01T00:00:02.000000000Z\n", - "SELECT f, ts FROM " + table + " ORDER BY ts"); } - // ==================== Additional positive coercion test ==================== - @Test - public void testStringToVarchar() throws Exception { - String table = "test_qwp_string_to_varchar"; + public void testUuidToCharCoercionError() throws Exception { + String table = "test_qwp_uuid_to_char_error"; useTable(table); - execute("CREATE TABLE " + table + " (v VARCHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v CHAR, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); sender.table(table) - .stringColumn("v", "hello") + .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) .at(1_000_000, ChronoUnit.MICROS); - sender.table(table) - .stringColumn("v", "world") - .at(2_000_000, ChronoUnit.MICROS); sender.flush(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + String msg = e.getMessage(); + Assert.assertTrue( + "Expected coercion error but got: " + msg, + msg.contains("cannot write UUID") && msg.contains("CHAR") + ); } - assertTableSizeEventually(table, 2); - assertSqlEventually( - "v\tts\n" + - "hello\t1970-01-01T00:00:01.000000000Z\n" + - "world\t1970-01-01T00:00:02.000000000Z\n", - "SELECT v, ts FROM " + table + " ORDER BY ts"); } - // ==================== Additional parse error tests ==================== - @Test - public void testStringToIntParseError() throws Exception { - String table = "test_qwp_string_to_int_parse_error"; + public void testUuidToDateCoercionError() throws Exception { + String table = "test_qwp_uuid_to_date_error"; useTable(table); - execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); sender.table(table) - .stringColumn("v", "not_a_number") + .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected parse error but got: " + msg, - msg.contains("cannot parse INT from string") && msg.contains("not_a_number") + "Expected coercion error but got: " + msg, + msg.contains("type coercion from UUID to") && msg.contains("is not supported") ); } } @Test - public void testStringToLongParseError() throws Exception { - String table = "test_qwp_string_to_long_parse_error"; + public void testUuidToDoubleCoercionError() throws Exception { + String table = "test_qwp_uuid_to_double_error"; useTable(table); - execute("CREATE TABLE " + table + " (v LONG, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v DOUBLE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); sender.table(table) - .stringColumn("v", "not_a_number") + .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected parse error but got: " + msg, - msg.contains("cannot parse LONG from string") && msg.contains("not_a_number") + "Expected coercion error but got: " + msg, + msg.contains("type coercion from UUID to") && msg.contains("is not supported") ); } } @Test - public void testStringToShortParseError() throws Exception { - String table = "test_qwp_string_to_short_parse_error"; + public void testUuidToFloatCoercionError() throws Exception { + String table = "test_qwp_uuid_to_float_error"; useTable(table); - execute("CREATE TABLE " + table + " (v SHORT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); sender.table(table) - .stringColumn("v", "not_a_number") + .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected parse error but got: " + msg, - msg.contains("cannot parse SHORT from string") && msg.contains("not_a_number") + "Expected coercion error but got: " + msg, + msg.contains("type coercion from UUID to") && msg.contains("is not supported") ); } } @Test - public void testStringToFloatParseError() throws Exception { - String table = "test_qwp_string_to_float_parse_error"; + public void testUuidToGeoHashCoercionError() throws Exception { + String table = "test_qwp_uuid_to_geohash_error"; useTable(table); - execute("CREATE TABLE " + table + " (v FLOAT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); sender.table(table) - .stringColumn("v", "not_a_number") + .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected parse error but got: " + msg, - msg.contains("cannot parse FLOAT from string") && msg.contains("not_a_number") + "Expected coercion error but got: " + msg, + msg.contains("type coercion from UUID to") && msg.contains("is not supported") ); } } @Test - public void testStringToDoubleParseError() throws Exception { - String table = "test_qwp_string_to_double_parse_error"; + public void testUuidToIntCoercionError() throws Exception { + String table = "test_qwp_uuid_to_int_error"; useTable(table); - execute("CREATE TABLE " + table + " (v DOUBLE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); sender.table(table) - .stringColumn("v", "not_a_number") + .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected parse error but got: " + msg, - msg.contains("cannot parse DOUBLE from string") && msg.contains("not_a_number") + "Expected coercion error but got: " + msg, + msg.contains("type coercion from UUID to") && msg.contains("is not supported") ); } } @Test - public void testStringToDateParseError() throws Exception { - String table = "test_qwp_string_to_date_parse_error"; + public void testUuidToLong256CoercionError() throws Exception { + String table = "test_qwp_uuid_to_long256_error"; useTable(table); - execute("CREATE TABLE " + table + " (v DATE, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); sender.table(table) - .stringColumn("v", "not_a_date") + .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected parse error but got: " + msg, - msg.contains("cannot parse DATE from string") && msg.contains("not_a_date") + "Expected coercion error but got: " + msg, + msg.contains("type coercion from UUID to") && msg.contains("is not supported") ); } } @Test - public void testStringToTimestampParseError() throws Exception { - String table = "test_qwp_string_to_timestamp_parse_error"; + public void testUuidToLongCoercionError() throws Exception { + String table = "test_qwp_uuid_to_long_error"; useTable(table); - execute("CREATE TABLE " + table + " (v TIMESTAMP, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v LONG, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); sender.table(table) - .stringColumn("v", "not_a_timestamp") + .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected parse error but got: " + msg, - msg.contains("cannot parse timestamp from string") && msg.contains("not_a_timestamp") + "Expected coercion error but got: " + msg, + msg.contains("type coercion from UUID to") && msg.contains("is not supported") ); } } @Test - public void testStringToUuidParseError() throws Exception { - String table = "test_qwp_string_to_uuid_parse_error"; + public void testUuidToShortCoercionError() throws Exception { + String table = "test_qwp_uuid_to_short_error"; useTable(table); - execute("CREATE TABLE " + table + " (v UUID, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "s SHORT, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + try (QwpWebSocketSender sender = createQwpSender()) { + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); sender.table(table) - .stringColumn("v", "not-a-uuid") + .uuidColumn("s", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected parse error but got: " + msg, - msg.contains("cannot parse UUID from string") && msg.contains("not-a-uuid") + "Expected coercion error but got: " + msg, + msg.contains("type coercion from UUID to SHORT is not supported") ); } } @Test - public void testStringToLong256ParseError() throws Exception { - String table = "test_qwp_string_to_long256_parse_error"; + public void testUuidToString() throws Exception { + String table = "test_qwp_uuid_to_string"; useTable(table); - execute("CREATE TABLE " + table + " (v LONG256, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (" + + "s STRING, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); + + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); try (QwpWebSocketSender sender = createQwpSender()) { sender.table(table) - .stringColumn("v", "not_a_long256") + .uuidColumn("s", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); - Assert.fail("Expected LineSenderException"); - } catch (LineSenderException e) { - String msg = e.getMessage(); - Assert.assertTrue( - "Expected parse error but got: " + msg, - msg.contains("cannot parse long256 from string") && msg.contains("not_a_long256") - ); } + + assertTableSizeEventually(table, 1); + assertSqlEventually( + "s\tts\n" + + "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11\t1970-01-01T00:00:01.000000000Z\n", + "SELECT s, ts FROM " + table); } @Test - public void testStringToGeoHashParseError() throws Exception { - String table = "test_qwp_string_to_geohash_parse_error"; + public void testUuidToSymbolCoercionError() throws Exception { + String table = "test_qwp_uuid_to_symbol_error"; useTable(table); - execute("CREATE TABLE " + table + " (v GEOHASH(5c), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); + execute("CREATE TABLE " + table + " (v SYMBOL, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY"); assertTableExistsEventually(table); try (QwpWebSocketSender sender = createQwpSender()) { + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); sender.table(table) - .stringColumn("v", "!!!") + .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) .at(1_000_000, ChronoUnit.MICROS); sender.flush(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { String msg = e.getMessage(); Assert.assertTrue( - "Expected parse error but got: " + msg, - msg.contains("cannot parse geohash from string") && msg.contains("!!!") + "Expected coercion error but got: " + msg, + msg.contains("cannot write UUID") && msg.contains("SYMBOL") ); } } - // === Helper Methods === + @Test + public void testUuidToVarchar() throws Exception { + String table = "test_qwp_uuid_to_varchar"; + useTable(table); + execute("CREATE TABLE " + table + " (" + + "v VARCHAR, " + + "ts TIMESTAMP" + + ") TIMESTAMP(ts) PARTITION BY DAY"); + assertTableExistsEventually(table); + + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .uuidColumn("v", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .at(1_000_000, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 1); + assertSqlEventually( + "v\tts\n" + + "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11\t1970-01-01T00:00:01.000000000Z\n", + "SELECT v, ts FROM " + table); + } + + @Test + public void testWriteAllTypesInOneRow() throws Exception { + String table = "test_qwp_all_types"; + useTable(table); + + UUID uuid = UUID.fromString("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); + double[] arr1d = {1.0, 2.0, 3.0}; + long tsMicros = 1_645_747_200_000_000L; // 2022-02-25T00:00:00Z + + try (QwpWebSocketSender sender = createQwpSender()) { + sender.table(table) + .symbol("sym", "test_symbol") + .boolColumn("bool_col", true) + .shortColumn("short_col", (short) 42) + .intColumn("int_col", 100_000) + .longColumn("long_col", 1_000_000_000L) + .floatColumn("float_col", 2.5f) + .doubleColumn("double_col", 3.14) + .stringColumn("string_col", "hello") + .charColumn("char_col", 'Z') + .timestampColumn("ts_col", tsMicros, ChronoUnit.MICROS) + .uuidColumn("uuid_col", uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()) + .long256Column("long256_col", 1, 0, 0, 0) + .doubleArray("arr_col", arr1d) + .decimalColumn("decimal_col", "99.99") + .at(tsMicros, ChronoUnit.MICROS); + sender.flush(); + } + + assertTableSizeEventually(table, 1); + } private QwpWebSocketSender createQwpSender() { return QwpWebSocketSender.connect(getQuestDbHost(), getHttpPort()); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketChannelTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketChannelTest.java index 607b442..47fe527 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketChannelTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketChannelTest.java @@ -53,13 +53,19 @@ public class WebSocketChannelTest extends AbstractTest { @Test - public void testBinaryRoundTripSmallPayload() throws Exception { - TestUtils.assertMemoryLeak(() -> assertBinaryRoundTrip(13)); - } - - @Test - public void testBinaryRoundTripMediumPayload() throws Exception { - TestUtils.assertMemoryLeak(() -> assertBinaryRoundTrip(4096)); + public void testBinaryRoundTripAllByteValues() throws Exception { + TestUtils.assertMemoryLeak(() -> { + int len = 256; + long sendPtr = Unsafe.malloc(len, MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < len; i++) { + Unsafe.getUnsafe().putByte(sendPtr + i, (byte) i); + } + assertBinaryRoundTrip(sendPtr, len); + } finally { + Unsafe.free(sendPtr, len, MemoryTag.NATIVE_DEFAULT); + } + }); } @Test @@ -74,19 +80,8 @@ public void testBinaryRoundTripLargePayload() throws Exception { } @Test - public void testBinaryRoundTripAllByteValues() throws Exception { - TestUtils.assertMemoryLeak(() -> { - int len = 256; - long sendPtr = Unsafe.malloc(len, MemoryTag.NATIVE_DEFAULT); - try { - for (int i = 0; i < len; i++) { - Unsafe.getUnsafe().putByte(sendPtr + i, (byte) i); - } - assertBinaryRoundTrip(sendPtr, len); - } finally { - Unsafe.free(sendPtr, len, MemoryTag.NATIVE_DEFAULT); - } - }); + public void testBinaryRoundTripMediumPayload() throws Exception { + TestUtils.assertMemoryLeak(() -> assertBinaryRoundTrip(4096)); } @Test @@ -135,6 +130,30 @@ public void testBinaryRoundTripRepeatedFrames() throws Exception { }); } + @Test + public void testBinaryRoundTripSmallPayload() throws Exception { + TestUtils.assertMemoryLeak(() -> assertBinaryRoundTrip(13)); + } + + /** + * Calls receiveFrame in a loop to handle the case where doReceiveFrame + * needs multiple reads to assemble a complete frame (e.g. header and + * payload arrive in separate TCP segments). + */ + private static boolean receiveWithRetry(WebSocketChannel channel, ReceivedPayload handler, int timeoutMs) { + long deadline = System.currentTimeMillis() + timeoutMs; + while (System.currentTimeMillis() < deadline) { + int remaining = (int) (deadline - System.currentTimeMillis()); + if (remaining <= 0) { + break; + } + if (channel.receiveFrame(handler, remaining)) { + return true; + } + } + return false; + } + private void assertBinaryRoundTrip(int payloadLen) throws Exception { long sendPtr = Unsafe.malloc(payloadLen, MemoryTag.NATIVE_DEFAULT); try { @@ -182,40 +201,6 @@ private void assertBinaryRoundTrip(long sendPtr, int payloadLen) throws Exceptio } } - /** - * Calls receiveFrame in a loop to handle the case where doReceiveFrame - * needs multiple reads to assemble a complete frame (e.g. header and - * payload arrive in separate TCP segments). - */ - private static boolean receiveWithRetry(WebSocketChannel channel, ReceivedPayload handler, int timeoutMs) { - long deadline = System.currentTimeMillis() + timeoutMs; - while (System.currentTimeMillis() < deadline) { - int remaining = (int) (deadline - System.currentTimeMillis()); - if (remaining <= 0) { - break; - } - if (channel.receiveFrame(handler, remaining)) { - return true; - } - } - return false; - } - - private static class ReceivedPayload implements WebSocketChannel.ResponseHandler { - long ptr; - int length; - - @Override - public void onBinaryMessage(long payload, int length) { - this.ptr = payload; - this.length = length; - } - - @Override - public void onClose(int code, String reason) { - } - } - /** * Minimal WebSocket echo server. Accepts one connection, completes the * HTTP upgrade handshake, then echoes every binary frame back unmasked. @@ -224,32 +209,14 @@ public void onClose(int code, String reason) { private static class EchoServer implements AutoCloseable { private static final Pattern KEY_PATTERN = Pattern.compile("Sec-WebSocket-Key:\\s*(.+?)\\r\\n"); - - private final ServerSocket serverSocket; private final AtomicReference error = new AtomicReference<>(); + private final ServerSocket serverSocket; private Thread thread; EchoServer() throws IOException { serverSocket = new ServerSocket(0); } - int getPort() { - return serverSocket.getLocalPort(); - } - - void start() { - thread = new Thread(this::run, "ws-echo-server"); - thread.setDaemon(true); - thread.start(); - } - - void assertNoError() { - Throwable t = error.get(); - if (t != null) { - throw new AssertionError("echo server error", t); - } - } - @Override public void close() throws Exception { serverSocket.close(); @@ -258,24 +225,6 @@ public void close() throws Exception { } } - private void run() { - try (Socket client = serverSocket.accept()) { - client.setSoTimeout(10_000); - client.setTcpNoDelay(true); - InputStream in = client.getInputStream(); - OutputStream out = new BufferedOutputStream(client.getOutputStream()); - - completeHandshake(in, out); - echoFrames(in, out); - } catch (IOException e) { - if (!serverSocket.isClosed()) { - error.set(e); - } - } catch (Throwable t) { - error.set(t); - } - } - private void completeHandshake(InputStream in, OutputStream out) throws IOException { byte[] buf = new byte[4096]; int pos = 0; @@ -389,10 +338,18 @@ private void echoFrames(InputStream in, OutputStream out) throws IOException { byte m3 = readBuf[maskKeyOffset + 3]; for (int i = 0; i < (int) payloadLength; i++) { switch (i & 3) { - case 0: readBuf[headerSize + i] ^= m0; break; - case 1: readBuf[headerSize + i] ^= m1; break; - case 2: readBuf[headerSize + i] ^= m2; break; - case 3: readBuf[headerSize + i] ^= m3; break; + case 0: + readBuf[headerSize + i] ^= m0; + break; + case 1: + readBuf[headerSize + i] ^= m1; + break; + case 2: + readBuf[headerSize + i] ^= m2; + break; + case 3: + readBuf[headerSize + i] ^= m3; + break; } } } @@ -426,5 +383,55 @@ private void echoFrames(InputStream in, OutputStream out) throws IOException { out.flush(); } } + + private void run() { + try (Socket client = serverSocket.accept()) { + client.setSoTimeout(10_000); + client.setTcpNoDelay(true); + InputStream in = client.getInputStream(); + OutputStream out = new BufferedOutputStream(client.getOutputStream()); + + completeHandshake(in, out); + echoFrames(in, out); + } catch (IOException e) { + if (!serverSocket.isClosed()) { + error.set(e); + } + } catch (Throwable t) { + error.set(t); + } + } + + void assertNoError() { + Throwable t = error.get(); + if (t != null) { + throw new AssertionError("echo server error", t); + } + } + + int getPort() { + return serverSocket.getLocalPort(); + } + + void start() { + thread = new Thread(this::run, "ws-echo-server"); + thread.setDaemon(true); + thread.start(); + } + } + + private static class ReceivedPayload implements WebSocketChannel.ResponseHandler { + int length; + long ptr; + + @Override + public void onBinaryMessage(long payload, int length) { + this.ptr = payload; + this.length = length; + } + + @Override + public void onClose(int code, String reason) { + } } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/OffHeapAppendMemoryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/OffHeapAppendMemoryTest.java index 073bf27..6034988 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/OffHeapAppendMemoryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/OffHeapAppendMemoryTest.java @@ -29,101 +29,29 @@ import io.questdb.client.std.Unsafe; import org.junit.Test; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; public class OffHeapAppendMemoryTest { @Test - public void testPutAndReadByte() { + public void testCloseFreesMemory() { long before = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); - try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { - mem.putByte((byte) 42); - mem.putByte((byte) -1); - mem.putByte((byte) 0); + OffHeapAppendMemory mem = new OffHeapAppendMemory(1024); + long during = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); + assertTrue(during > before); - assertEquals(3, mem.getAppendOffset()); - assertEquals(42, Unsafe.getUnsafe().getByte(mem.addressOf(0))); - assertEquals(-1, Unsafe.getUnsafe().getByte(mem.addressOf(1))); - assertEquals(0, Unsafe.getUnsafe().getByte(mem.addressOf(2))); - } + mem.close(); long after = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); assertEquals(before, after); } @Test - public void testPutAndReadShort() { - try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { - mem.putShort((short) 12_345); - mem.putShort(Short.MIN_VALUE); - mem.putShort(Short.MAX_VALUE); - - assertEquals(6, mem.getAppendOffset()); - assertEquals(12_345, Unsafe.getUnsafe().getShort(mem.addressOf(0))); - assertEquals(Short.MIN_VALUE, Unsafe.getUnsafe().getShort(mem.addressOf(2))); - assertEquals(Short.MAX_VALUE, Unsafe.getUnsafe().getShort(mem.addressOf(4))); - } - } - - @Test - public void testPutAndReadInt() { - try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { - mem.putInt(100_000); - mem.putInt(Integer.MIN_VALUE); - - assertEquals(8, mem.getAppendOffset()); - assertEquals(100_000, Unsafe.getUnsafe().getInt(mem.addressOf(0))); - assertEquals(Integer.MIN_VALUE, Unsafe.getUnsafe().getInt(mem.addressOf(4))); - } - } - - @Test - public void testPutAndReadLong() { - try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { - mem.putLong(1_000_000_000_000L); - mem.putLong(Long.MIN_VALUE); - - assertEquals(16, mem.getAppendOffset()); - assertEquals(1_000_000_000_000L, Unsafe.getUnsafe().getLong(mem.addressOf(0))); - assertEquals(Long.MIN_VALUE, Unsafe.getUnsafe().getLong(mem.addressOf(8))); - } - } - - @Test - public void testPutAndReadFloat() { - try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { - mem.putFloat(3.14f); - mem.putFloat(Float.NaN); - - assertEquals(8, mem.getAppendOffset()); - assertEquals(3.14f, Unsafe.getUnsafe().getFloat(mem.addressOf(0)), 0.0f); - assertTrue(Float.isNaN(Unsafe.getUnsafe().getFloat(mem.addressOf(4)))); - } - } - - @Test - public void testPutAndReadDouble() { - try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { - mem.putDouble(2.718281828); - mem.putDouble(Double.NaN); - - assertEquals(16, mem.getAppendOffset()); - assertEquals(2.718281828, Unsafe.getUnsafe().getDouble(mem.addressOf(0)), 0.0); - assertTrue(Double.isNaN(Unsafe.getUnsafe().getDouble(mem.addressOf(8)))); - } - } - - @Test - public void testPutBoolean() { - try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { - mem.putBoolean(true); - mem.putBoolean(false); - mem.putBoolean(true); - - assertEquals(3, mem.getAppendOffset()); - assertEquals(1, Unsafe.getUnsafe().getByte(mem.addressOf(0))); - assertEquals(0, Unsafe.getUnsafe().getByte(mem.addressOf(1))); - assertEquals(1, Unsafe.getUnsafe().getByte(mem.addressOf(2))); - } + public void testDoubleCloseIsSafe() { + OffHeapAppendMemory mem = new OffHeapAppendMemory(); + mem.putInt(42); + mem.close(); + mem.close(); // should not throw } @Test @@ -144,24 +72,6 @@ public void testGrowth() { assertEquals(before, after); } - @Test - public void testTruncate() { - try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { - mem.putInt(1); - mem.putInt(2); - mem.putInt(3); - assertEquals(12, mem.getAppendOffset()); - - mem.truncate(); - assertEquals(0, mem.getAppendOffset()); - - // Can write again after truncate - mem.putInt(42); - assertEquals(4, mem.getAppendOffset()); - assertEquals(42, Unsafe.getUnsafe().getInt(mem.addressOf(0))); - } - } - @Test public void testJumpTo() { try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { @@ -183,15 +93,41 @@ public void testJumpTo() { } @Test - public void testSkip() { + public void testLargeGrowth() { + long before = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); + try (OffHeapAppendMemory mem = new OffHeapAppendMemory(8)) { + // Write 10000 doubles to stress growth + for (int i = 0; i < 10_000; i++) { + mem.putDouble(i * 1.1); + } + assertEquals(80_000, mem.getAppendOffset()); + + // Verify first and last values + assertEquals(0.0, Unsafe.getUnsafe().getDouble(mem.addressOf(0)), 0.0); + assertEquals(9999 * 1.1, Unsafe.getUnsafe().getDouble(mem.addressOf(79_992)), 0.001); + } + long after = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); + assertEquals(before, after); + } + + @Test + public void testMixedTypes() { try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { - mem.putInt(1); - mem.skip(8); - mem.putInt(2); + mem.putByte((byte) 1); + mem.putShort((short) 2); + mem.putInt(3); + mem.putLong(4L); + mem.putFloat(5.0f); + mem.putDouble(6.0); - assertEquals(16, mem.getAppendOffset()); - assertEquals(1, Unsafe.getUnsafe().getInt(mem.addressOf(0))); - assertEquals(2, Unsafe.getUnsafe().getInt(mem.addressOf(12))); + long addr = mem.pageAddress(); + assertEquals(1, Unsafe.getUnsafe().getByte(addr)); + assertEquals(2, Unsafe.getUnsafe().getShort(addr + 1)); + assertEquals(3, Unsafe.getUnsafe().getInt(addr + 3)); + assertEquals(4L, Unsafe.getUnsafe().getLong(addr + 7)); + assertEquals(5.0f, Unsafe.getUnsafe().getFloat(addr + 15), 0.0f); + assertEquals(6.0, Unsafe.getUnsafe().getDouble(addr + 19), 0.0); + assertEquals(27, mem.getAppendOffset()); } } @@ -206,62 +142,96 @@ public void testPageAddress() { } @Test - public void testCloseFreesMemory() { + public void testPutAndReadByte() { long before = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); - OffHeapAppendMemory mem = new OffHeapAppendMemory(1024); - long during = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); - assertTrue(during > before); + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putByte((byte) 42); + mem.putByte((byte) -1); + mem.putByte((byte) 0); - mem.close(); + assertEquals(3, mem.getAppendOffset()); + assertEquals(42, Unsafe.getUnsafe().getByte(mem.addressOf(0))); + assertEquals(-1, Unsafe.getUnsafe().getByte(mem.addressOf(1))); + assertEquals(0, Unsafe.getUnsafe().getByte(mem.addressOf(2))); + } long after = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); assertEquals(before, after); } @Test - public void testDoubleCloseIsSafe() { - OffHeapAppendMemory mem = new OffHeapAppendMemory(); - mem.putInt(42); - mem.close(); - mem.close(); // should not throw + public void testPutAndReadDouble() { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putDouble(2.718281828); + mem.putDouble(Double.NaN); + + assertEquals(16, mem.getAppendOffset()); + assertEquals(2.718281828, Unsafe.getUnsafe().getDouble(mem.addressOf(0)), 0.0); + assertTrue(Double.isNaN(Unsafe.getUnsafe().getDouble(mem.addressOf(8)))); + } } @Test - public void testMixedTypes() { + public void testPutAndReadFloat() { try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { - mem.putByte((byte) 1); - mem.putShort((short) 2); - mem.putInt(3); - mem.putLong(4L); - mem.putFloat(5.0f); - mem.putDouble(6.0); + mem.putFloat(3.14f); + mem.putFloat(Float.NaN); - long addr = mem.pageAddress(); - assertEquals(1, Unsafe.getUnsafe().getByte(addr)); - assertEquals(2, Unsafe.getUnsafe().getShort(addr + 1)); - assertEquals(3, Unsafe.getUnsafe().getInt(addr + 3)); - assertEquals(4L, Unsafe.getUnsafe().getLong(addr + 7)); - assertEquals(5.0f, Unsafe.getUnsafe().getFloat(addr + 15), 0.0f); - assertEquals(6.0, Unsafe.getUnsafe().getDouble(addr + 19), 0.0); - assertEquals(27, mem.getAppendOffset()); + assertEquals(8, mem.getAppendOffset()); + assertEquals(3.14f, Unsafe.getUnsafe().getFloat(mem.addressOf(0)), 0.0f); + assertTrue(Float.isNaN(Unsafe.getUnsafe().getFloat(mem.addressOf(4)))); } } @Test - public void testLargeGrowth() { - long before = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); - try (OffHeapAppendMemory mem = new OffHeapAppendMemory(8)) { - // Write 10000 doubles to stress growth - for (int i = 0; i < 10_000; i++) { - mem.putDouble(i * 1.1); - } - assertEquals(80_000, mem.getAppendOffset()); + public void testPutAndReadInt() { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putInt(100_000); + mem.putInt(Integer.MIN_VALUE); - // Verify first and last values - assertEquals(0.0, Unsafe.getUnsafe().getDouble(mem.addressOf(0)), 0.0); - assertEquals(9999 * 1.1, Unsafe.getUnsafe().getDouble(mem.addressOf(79_992)), 0.001); + assertEquals(8, mem.getAppendOffset()); + assertEquals(100_000, Unsafe.getUnsafe().getInt(mem.addressOf(0))); + assertEquals(Integer.MIN_VALUE, Unsafe.getUnsafe().getInt(mem.addressOf(4))); + } + } + + @Test + public void testPutAndReadLong() { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putLong(1_000_000_000_000L); + mem.putLong(Long.MIN_VALUE); + + assertEquals(16, mem.getAppendOffset()); + assertEquals(1_000_000_000_000L, Unsafe.getUnsafe().getLong(mem.addressOf(0))); + assertEquals(Long.MIN_VALUE, Unsafe.getUnsafe().getLong(mem.addressOf(8))); + } + } + + @Test + public void testPutAndReadShort() { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putShort((short) 12_345); + mem.putShort(Short.MIN_VALUE); + mem.putShort(Short.MAX_VALUE); + + assertEquals(6, mem.getAppendOffset()); + assertEquals(12_345, Unsafe.getUnsafe().getShort(mem.addressOf(0))); + assertEquals(Short.MIN_VALUE, Unsafe.getUnsafe().getShort(mem.addressOf(2))); + assertEquals(Short.MAX_VALUE, Unsafe.getUnsafe().getShort(mem.addressOf(4))); + } + } + + @Test + public void testPutBoolean() { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putBoolean(true); + mem.putBoolean(false); + mem.putBoolean(true); + + assertEquals(3, mem.getAppendOffset()); + assertEquals(1, Unsafe.getUnsafe().getByte(mem.addressOf(0))); + assertEquals(0, Unsafe.getUnsafe().getByte(mem.addressOf(1))); + assertEquals(1, Unsafe.getUnsafe().getByte(mem.addressOf(2))); } - long after = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_ILP_RSS); - assertEquals(before, after); } @Test @@ -288,6 +258,15 @@ public void testPutUtf8Empty() { } } + @Test + public void testPutUtf8Mixed() { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + // Mix: ASCII "A" (1 byte) + e-acute (2 bytes) + CJK (3 bytes) + emoji (4 bytes) = 10 bytes + mem.putUtf8("A\u00E9\u4E16\uD83D\uDE00"); + assertEquals(10, mem.getAppendOffset()); + } + } + @Test public void testPutUtf8MultiByte() { try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { @@ -333,11 +312,33 @@ public void testPutUtf8ThreeByte() { } @Test - public void testPutUtf8Mixed() { + public void testSkip() { try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { - // Mix: ASCII "A" (1 byte) + e-acute (2 bytes) + CJK (3 bytes) + emoji (4 bytes) = 10 bytes - mem.putUtf8("A\u00E9\u4E16\uD83D\uDE00"); - assertEquals(10, mem.getAppendOffset()); + mem.putInt(1); + mem.skip(8); + mem.putInt(2); + + assertEquals(16, mem.getAppendOffset()); + assertEquals(1, Unsafe.getUnsafe().getInt(mem.addressOf(0))); + assertEquals(2, Unsafe.getUnsafe().getInt(mem.addressOf(12))); + } + } + + @Test + public void testTruncate() { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + mem.putInt(1); + mem.putInt(2); + mem.putInt(3); + assertEquals(12, mem.getAppendOffset()); + + mem.truncate(); + assertEquals(0, mem.getAppendOffset()); + + // Can write again after truncate + mem.putInt(42); + assertEquals(4, mem.getAppendOffset()); + assertEquals(42, Unsafe.getUnsafe().getInt(mem.addressOf(0))); } } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpBitWriterTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpBitWriterTest.java index 11f4c36..0b516e0 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpBitWriterTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpBitWriterTest.java @@ -36,52 +36,79 @@ public class QwpBitWriterTest { @Test - public void testWriteBitsThrowsOnOverflow() { - long ptr = Unsafe.malloc(4, MemoryTag.NATIVE_ILP_RSS); + public void testFlushThrowsOnOverflow() { + long ptr = Unsafe.malloc(1, MemoryTag.NATIVE_ILP_RSS); try { QwpBitWriter writer = new QwpBitWriter(); - writer.reset(ptr, 4); - // Fill the buffer (32 bits = 4 bytes) - writer.writeBits(0xFFFF_FFFFL, 32); - // Next write should throw — buffer is full + writer.reset(ptr, 1); + // Write 8 bits to fill the single byte + writer.writeBits(0xFF, 8); + // Write a few more bits that sit in the bit buffer + writer.writeBits(0x3, 4); + // Flush should throw because there's no room for the partial byte try { - writer.writeBits(1, 8); + writer.flush(); + fail("expected LineSenderException on buffer overflow during flush"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("buffer overflow")); + } + } finally { + Unsafe.free(ptr, 1, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testGorillaEncoderThrowsOnInsufficientCapacityForFirstTimestamp() { + // Source: 1 timestamp (8 bytes), dest: only 4 bytes + long src = Unsafe.malloc(8, MemoryTag.NATIVE_ILP_RSS); + long dst = Unsafe.malloc(4, MemoryTag.NATIVE_ILP_RSS); + try { + Unsafe.getUnsafe().putLong(src, 1_000_000L); + QwpGorillaEncoder encoder = new QwpGorillaEncoder(); + try { + encoder.encodeTimestamps(dst, 4, src, 1); fail("expected LineSenderException on buffer overflow"); } catch (LineSenderException e) { assertTrue(e.getMessage().contains("buffer overflow")); } } finally { - Unsafe.free(ptr, 4, MemoryTag.NATIVE_ILP_RSS); + Unsafe.free(src, 8, MemoryTag.NATIVE_ILP_RSS); + Unsafe.free(dst, 4, MemoryTag.NATIVE_ILP_RSS); } } @Test - public void testWriteByteThrowsOnOverflow() { - long ptr = Unsafe.malloc(1, MemoryTag.NATIVE_ILP_RSS); + public void testGorillaEncoderThrowsOnInsufficientCapacityForSecondTimestamp() { + // Source: 2 timestamps (16 bytes), dest: only 12 bytes (enough for first, not second) + long src = Unsafe.malloc(16, MemoryTag.NATIVE_ILP_RSS); + long dst = Unsafe.malloc(12, MemoryTag.NATIVE_ILP_RSS); try { - QwpBitWriter writer = new QwpBitWriter(); - writer.reset(ptr, 1); - writer.writeByte(0x42); + Unsafe.getUnsafe().putLong(src, 1_000_000L); + Unsafe.getUnsafe().putLong(src + 8, 2_000_000L); + QwpGorillaEncoder encoder = new QwpGorillaEncoder(); try { - writer.writeByte(0x43); + encoder.encodeTimestamps(dst, 12, src, 2); fail("expected LineSenderException on buffer overflow"); } catch (LineSenderException e) { assertTrue(e.getMessage().contains("buffer overflow")); } } finally { - Unsafe.free(ptr, 1, MemoryTag.NATIVE_ILP_RSS); + Unsafe.free(src, 16, MemoryTag.NATIVE_ILP_RSS); + Unsafe.free(dst, 12, MemoryTag.NATIVE_ILP_RSS); } } @Test - public void testWriteIntThrowsOnOverflow() { + public void testWriteBitsThrowsOnOverflow() { long ptr = Unsafe.malloc(4, MemoryTag.NATIVE_ILP_RSS); try { QwpBitWriter writer = new QwpBitWriter(); writer.reset(ptr, 4); - writer.writeInt(42); + // Fill the buffer (32 bits = 4 bytes) + writer.writeBits(0xFFFF_FFFFL, 32); + // Next write should throw — buffer is full try { - writer.writeInt(99); + writer.writeBits(1, 8); fail("expected LineSenderException on buffer overflow"); } catch (LineSenderException e) { assertTrue(e.getMessage().contains("buffer overflow")); @@ -92,37 +119,30 @@ public void testWriteIntThrowsOnOverflow() { } @Test - public void testWriteLongThrowsOnOverflow() { + public void testWriteBitsWithinCapacitySucceeds() { long ptr = Unsafe.malloc(8, MemoryTag.NATIVE_ILP_RSS); try { QwpBitWriter writer = new QwpBitWriter(); writer.reset(ptr, 8); - writer.writeLong(42L); - try { - writer.writeLong(99L); - fail("expected LineSenderException on buffer overflow"); - } catch (LineSenderException e) { - assertTrue(e.getMessage().contains("buffer overflow")); - } + writer.writeBits(0xDEAD_BEEF_CAFE_BABEL, 64); + writer.flush(); + assertEquals(8, writer.getPosition() - ptr); + assertEquals(0xDEAD_BEEF_CAFE_BABEL, Unsafe.getUnsafe().getLong(ptr)); } finally { Unsafe.free(ptr, 8, MemoryTag.NATIVE_ILP_RSS); } } @Test - public void testFlushThrowsOnOverflow() { + public void testWriteByteThrowsOnOverflow() { long ptr = Unsafe.malloc(1, MemoryTag.NATIVE_ILP_RSS); try { QwpBitWriter writer = new QwpBitWriter(); writer.reset(ptr, 1); - // Write 8 bits to fill the single byte - writer.writeBits(0xFF, 8); - // Write a few more bits that sit in the bit buffer - writer.writeBits(0x3, 4); - // Flush should throw because there's no room for the partial byte + writer.writeByte(0x42); try { - writer.flush(); - fail("expected LineSenderException on buffer overflow during flush"); + writer.writeByte(0x43); + fail("expected LineSenderException on buffer overflow"); } catch (LineSenderException e) { assertTrue(e.getMessage().contains("buffer overflow")); } @@ -131,59 +151,37 @@ public void testFlushThrowsOnOverflow() { } } - // --- QwpGorillaEncoder overflow tests --- - @Test - public void testGorillaEncoderThrowsOnInsufficientCapacityForFirstTimestamp() { - // Source: 1 timestamp (8 bytes), dest: only 4 bytes - long src = Unsafe.malloc(8, MemoryTag.NATIVE_ILP_RSS); - long dst = Unsafe.malloc(4, MemoryTag.NATIVE_ILP_RSS); + public void testWriteIntThrowsOnOverflow() { + long ptr = Unsafe.malloc(4, MemoryTag.NATIVE_ILP_RSS); try { - Unsafe.getUnsafe().putLong(src, 1_000_000L); - QwpGorillaEncoder encoder = new QwpGorillaEncoder(); + QwpBitWriter writer = new QwpBitWriter(); + writer.reset(ptr, 4); + writer.writeInt(42); try { - encoder.encodeTimestamps(dst, 4, src, 1); + writer.writeInt(99); fail("expected LineSenderException on buffer overflow"); } catch (LineSenderException e) { assertTrue(e.getMessage().contains("buffer overflow")); } } finally { - Unsafe.free(src, 8, MemoryTag.NATIVE_ILP_RSS); - Unsafe.free(dst, 4, MemoryTag.NATIVE_ILP_RSS); + Unsafe.free(ptr, 4, MemoryTag.NATIVE_ILP_RSS); } } @Test - public void testGorillaEncoderThrowsOnInsufficientCapacityForSecondTimestamp() { - // Source: 2 timestamps (16 bytes), dest: only 12 bytes (enough for first, not second) - long src = Unsafe.malloc(16, MemoryTag.NATIVE_ILP_RSS); - long dst = Unsafe.malloc(12, MemoryTag.NATIVE_ILP_RSS); + public void testWriteLongThrowsOnOverflow() { + long ptr = Unsafe.malloc(8, MemoryTag.NATIVE_ILP_RSS); try { - Unsafe.getUnsafe().putLong(src, 1_000_000L); - Unsafe.getUnsafe().putLong(src + 8, 2_000_000L); - QwpGorillaEncoder encoder = new QwpGorillaEncoder(); + QwpBitWriter writer = new QwpBitWriter(); + writer.reset(ptr, 8); + writer.writeLong(42L); try { - encoder.encodeTimestamps(dst, 12, src, 2); + writer.writeLong(99L); fail("expected LineSenderException on buffer overflow"); } catch (LineSenderException e) { assertTrue(e.getMessage().contains("buffer overflow")); } - } finally { - Unsafe.free(src, 16, MemoryTag.NATIVE_ILP_RSS); - Unsafe.free(dst, 12, MemoryTag.NATIVE_ILP_RSS); - } - } - - @Test - public void testWriteBitsWithinCapacitySucceeds() { - long ptr = Unsafe.malloc(8, MemoryTag.NATIVE_ILP_RSS); - try { - QwpBitWriter writer = new QwpBitWriter(); - writer.reset(ptr, 8); - writer.writeBits(0xDEAD_BEEF_CAFE_BABEL, 64); - writer.flush(); - assertEquals(8, writer.getPosition() - ptr); - assertEquals(0xDEAD_BEEF_CAFE_BABEL, Unsafe.getUnsafe().getLong(ptr)); } finally { Unsafe.free(ptr, 8, MemoryTag.NATIVE_ILP_RSS); } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpColumnDefTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpColumnDefTest.java index 2bd28b9..2f5a3ac 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpColumnDefTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpColumnDefTest.java @@ -28,7 +28,8 @@ import io.questdb.client.cutlass.qwp.protocol.QwpConstants; import org.junit.Test; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; public class QwpColumnDefTest { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java index 501893e..e4ac0d7 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java @@ -29,88 +29,11 @@ import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; import org.junit.Test; -import static org.junit.Assert.*; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; public class QwpTableBufferTest { - /** - * Simulates the encoder's walk over array data — the same logic as - * QwpWebSocketEncoder.writeDoubleArrayColumn(). Returns the flat - * double values the encoder would serialize for the given column. - */ - private static double[] readDoubleArraysLikeEncoder(QwpTableBuffer.ColumnBuffer col) { - byte[] dims = col.getArrayDims(); - int[] shapes = col.getArrayShapes(); - double[] data = col.getDoubleArrayData(); - int count = col.getValueCount(); - - // First pass: count total elements - int totalElements = 0; - int shapeIdx = 0; - for (int row = 0; row < count; row++) { - int nDims = dims[row]; - int elemCount = 1; - for (int d = 0; d < nDims; d++) { - elemCount *= shapes[shapeIdx++]; - } - totalElements += elemCount; - } - - // Second pass: collect values - double[] result = new double[totalElements]; - shapeIdx = 0; - int dataIdx = 0; - int resultIdx = 0; - for (int row = 0; row < count; row++) { - int nDims = dims[row]; - int elemCount = 1; - for (int d = 0; d < nDims; d++) { - elemCount *= shapes[shapeIdx++]; - } - for (int e = 0; e < elemCount; e++) { - result[resultIdx++] = data[dataIdx++]; - } - } - return result; - } - - /** - * Same as above but for long arrays (mirrors QwpWebSocketEncoder.writeLongArrayColumn()). - */ - private static long[] readLongArraysLikeEncoder(QwpTableBuffer.ColumnBuffer col) { - byte[] dims = col.getArrayDims(); - int[] shapes = col.getArrayShapes(); - long[] data = col.getLongArrayData(); - int count = col.getValueCount(); - - int totalElements = 0; - int shapeIdx = 0; - for (int row = 0; row < count; row++) { - int nDims = dims[row]; - int elemCount = 1; - for (int d = 0; d < nDims; d++) { - elemCount *= shapes[shapeIdx++]; - } - totalElements += elemCount; - } - - long[] result = new long[totalElements]; - shapeIdx = 0; - int dataIdx = 0; - int resultIdx = 0; - for (int row = 0; row < count; row++) { - int nDims = dims[row]; - int elemCount = 1; - for (int d = 0; d < nDims; d++) { - elemCount *= shapes[shapeIdx++]; - } - for (int e = 0; e < elemCount; e++) { - result[resultIdx++] = data[dataIdx++]; - } - } - return result; - } - @Test public void testCancelRowRewindsDoubleArrayOffsets() { try (QwpTableBuffer table = new QwpTableBuffer("test")) { @@ -377,4 +300,82 @@ public void testLongArrayShrinkingSize() { assertEquals(2, shapes[1]); } } + + /** + * Simulates the encoder's walk over array data — the same logic as + * QwpWebSocketEncoder.writeDoubleArrayColumn(). Returns the flat + * double values the encoder would serialize for the given column. + */ + private static double[] readDoubleArraysLikeEncoder(QwpTableBuffer.ColumnBuffer col) { + byte[] dims = col.getArrayDims(); + int[] shapes = col.getArrayShapes(); + double[] data = col.getDoubleArrayData(); + int count = col.getValueCount(); + + // First pass: count total elements + int totalElements = 0; + int shapeIdx = 0; + for (int row = 0; row < count; row++) { + int nDims = dims[row]; + int elemCount = 1; + for (int d = 0; d < nDims; d++) { + elemCount *= shapes[shapeIdx++]; + } + totalElements += elemCount; + } + + // Second pass: collect values + double[] result = new double[totalElements]; + shapeIdx = 0; + int dataIdx = 0; + int resultIdx = 0; + for (int row = 0; row < count; row++) { + int nDims = dims[row]; + int elemCount = 1; + for (int d = 0; d < nDims; d++) { + elemCount *= shapes[shapeIdx++]; + } + for (int e = 0; e < elemCount; e++) { + result[resultIdx++] = data[dataIdx++]; + } + } + return result; + } + + /** + * Same as above but for long arrays (mirrors QwpWebSocketEncoder.writeLongArrayColumn()). + */ + private static long[] readLongArraysLikeEncoder(QwpTableBuffer.ColumnBuffer col) { + byte[] dims = col.getArrayDims(); + int[] shapes = col.getArrayShapes(); + long[] data = col.getLongArrayData(); + int count = col.getValueCount(); + + int totalElements = 0; + int shapeIdx = 0; + for (int row = 0; row < count; row++) { + int nDims = dims[row]; + int elemCount = 1; + for (int d = 0; d < nDims; d++) { + elemCount *= shapes[shapeIdx++]; + } + totalElements += elemCount; + } + + long[] result = new long[totalElements]; + shapeIdx = 0; + int dataIdx = 0; + int resultIdx = 0; + for (int row = 0; row < count; row++) { + int nDims = dims[row]; + int elemCount = 1; + for (int d = 0; d < nDims; d++) { + elemCount *= shapes[shapeIdx++]; + } + for (int e = 0; e < elemCount; e++) { + result[resultIdx++] = data[dataIdx++]; + } + } + return result; + } } From 6e1395948d03cd528d8664847ce5ca54f79d87fb Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Wed, 25 Feb 2026 10:03:40 +0100 Subject: [PATCH 40/89] Fix Gorilla encoder bucket boundaries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bucket boundary constants for two's complement signed ranges were inverted — the min and max magnitudes were swapped. For an N-bit two's complement integer the valid range is [-2^(N-1), 2^(N-1) - 1], so: 7-bit: [-64, 63] not [-63, 64] 9-bit: [-256, 255] not [-255, 256] 12-bit: [-2048, 2047] not [-2047, 2048] With the old boundaries, a value like 64 would be placed in the 7-bit bucket, but 64 in 7-bit two's complement decodes as -64, silently corrupting timestamp data at bucket boundaries. Co-Authored-By: Claude Opus 4.6 --- .../qwp/protocol/QwpGorillaEncoder.java | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java index 082f6b2..a21215a 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java @@ -36,9 +36,9 @@ * DoD = (t[n] - t[n-1]) - (t[n-1] - t[n-2]) * * if DoD == 0: write '0' (1 bit) - * elif DoD in [-63, 64]: write '10' + 7-bit (9 bits) - * elif DoD in [-255, 256]: write '110' + 9-bit (12 bits) - * elif DoD in [-2047, 2048]: write '1110' + 12-bit (16 bits) + * elif DoD in [-64, 63]: write '10' + 7-bit (9 bits) + * elif DoD in [-256, 255]: write '110' + 9-bit (12 bits) + * elif DoD in [-2048, 2047]: write '1110' + 12-bit (16 bits) * else: write '1111' + 32-bit (36 bits) * *

@@ -47,13 +47,13 @@ */ public class QwpGorillaEncoder { - private static final int BUCKET_12BIT_MAX = 2048; - private static final int BUCKET_12BIT_MIN = -2047; - private static final int BUCKET_7BIT_MAX = 64; + private static final int BUCKET_12BIT_MAX = 2047; + private static final int BUCKET_12BIT_MIN = -2048; + private static final int BUCKET_7BIT_MAX = 63; // Bucket boundaries (two's complement signed ranges) - private static final int BUCKET_7BIT_MIN = -63; - private static final int BUCKET_9BIT_MAX = 256; - private static final int BUCKET_9BIT_MIN = -255; + private static final int BUCKET_7BIT_MIN = -64; + private static final int BUCKET_9BIT_MAX = 255; + private static final int BUCKET_9BIT_MIN = -256; private final QwpBitWriter bitWriter = new QwpBitWriter(); /** @@ -202,15 +202,15 @@ public void encodeDoD(long deltaOfDelta) { case 0: // DoD == 0 bitWriter.writeBit(0); break; - case 1: // [-63, 64] -> '10' + 7-bit + case 1: // [-64, 63] -> '10' + 7-bit bitWriter.writeBits(0b01, 2); bitWriter.writeSigned(deltaOfDelta, 7); break; - case 2: // [-255, 256] -> '110' + 9-bit + case 2: // [-256, 255] -> '110' + 9-bit bitWriter.writeBits(0b011, 3); bitWriter.writeSigned(deltaOfDelta, 9); break; - case 3: // [-2047, 2048] -> '1110' + 12-bit + case 3: // [-2048, 2047] -> '1110' + 12-bit bitWriter.writeBits(0b0111, 4); bitWriter.writeSigned(deltaOfDelta, 12); break; From b8514c08d6c880664e00f76ac73b602691645032 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Wed, 25 Feb 2026 10:21:22 +0100 Subject: [PATCH 41/89] Remove unused opcode param from beginFrame() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The opcode parameter in beginFrame(int) was accepted but never stored or used — the opcode only matters when writing the frame header in endFrame(int), where all callers already pass the correct value. Remove the misleading parameter to make the API honest. Also remove beginBinaryFrame() and beginTextFrame() which were just wrappers passing an unused opcode. The single caller in WebSocketClient is updated to call beginFrame() directly. Co-Authored-By: Claude Opus 4.6 --- .../cutlass/http/client/WebSocketClient.java | 2 +- .../http/client/WebSocketSendBuffer.java | 22 +++---------------- 2 files changed, 4 insertions(+), 20 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java index a0cf1a3..32d79dd 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java @@ -294,7 +294,7 @@ public boolean receiveFrame(WebSocketFrameHandler handler) { public void sendBinary(long dataPtr, int length, int timeout) { checkConnected(); sendBuffer.reset(); - sendBuffer.beginBinaryFrame(); + sendBuffer.beginFrame(); sendBuffer.putBlockOfBytes(dataPtr, length); WebSocketSendBuffer.FrameInfo frame = sendBuffer.endBinaryFrame(); doSend(sendBuffer.getBufferPtr() + frame.offset, frame.length, timeout); diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java index 861de15..287bda0 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java @@ -103,19 +103,10 @@ public WebSocketSendBuffer(int initialCapacity, int maxBufferSize) { } /** - * Begins a new binary WebSocket frame. Reserves space for the maximum header size. - * After calling this method, use ArrayBufferAppender methods to write the payload. + * Begins a new WebSocket frame. Reserves space for the maximum header size. + * The opcode is specified later when ending the frame via {@link #endFrame(int)}. */ - public void beginBinaryFrame() { - beginFrame(WebSocketOpcode.BINARY); - } - - /** - * Begins a new WebSocket frame with the specified opcode. - * - * @param opcode the frame opcode - */ - public void beginFrame(int opcode) { + public void beginFrame() { frameStartOffset = writePos; // Reserve maximum header space ensureCapacity(MAX_HEADER_SIZE); @@ -123,13 +114,6 @@ public void beginFrame(int opcode) { payloadStartOffset = writePos; } - /** - * Begins a new text WebSocket frame. Reserves space for the maximum header size. - */ - public void beginTextFrame() { - beginFrame(WebSocketOpcode.TEXT); - } - @Override public void close() { if (bufPtr != 0) { From 7a87c76704c87cc198c914817f10c3ad76303bae Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Wed, 25 Feb 2026 10:25:57 +0100 Subject: [PATCH 42/89] Use correct default port for WebSocket protocol The single-host fallback in configuration string parsing used DEFAULT_HTTP_PORT for all non-TCP protocols. This works by coincidence since DEFAULT_HTTP_PORT and DEFAULT_WEBSOCKET_PORT are both 9000, but is semantically incorrect. Use the proper DEFAULT_WEBSOCKET_PORT constant when the protocol is WebSocket. Also clean up Javadoc: remove a dangling isRetryable() reference from Sender.java, fix grammar ("allows to use" -> "allows using"), and tidy LineSenderException Javadoc. Co-Authored-By: Claude Opus 4.6 --- core/src/main/java/io/questdb/client/Sender.java | 8 +++++--- .../client/cutlass/line/LineSenderException.java | 12 +++--------- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index c998b78..755bd78 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -96,7 +96,7 @@ * 2. Call {@link #reset()} to clear the internal buffers and start building a new row *
* Note: If the underlying error is permanent, retrying {@link #flush()} will fail again. - * Use {@link #reset()} to discard the problematic data and continue with new data. See {@link LineSenderException#isRetryable()} + * Use {@link #reset()} to discard the problematic data and continue with new data. * */ public interface Sender extends Closeable, ArraySender { @@ -109,7 +109,7 @@ public interface Sender extends Closeable, ArraySender { /** * Create a Sender builder instance from a configuration string. *
- * This allows to use the configuration string as a template for creating a Sender builder instance and then + * This allows using the configuration string as a template for creating a Sender builder instance and then * tune options which are not available in the configuration string. Configurations options specified in the * configuration string cannot be overridden via the builder methods. *

@@ -1529,7 +1529,9 @@ private LineSenderBuilder fromConfig(CharSequence configurationString) { address(sink); if (ports.size() == hosts.size() - 1) { // not set - port(protocol == PROTOCOL_TCP ? DEFAULT_TCP_PORT : DEFAULT_HTTP_PORT); + port(protocol == PROTOCOL_TCP ? DEFAULT_TCP_PORT + : protocol == PROTOCOL_WEBSOCKET ? DEFAULT_WEBSOCKET_PORT + : DEFAULT_HTTP_PORT); } } else if (Chars.equals("user", sink)) { // deprecated key: user, new key: username diff --git a/core/src/main/java/io/questdb/client/cutlass/line/LineSenderException.java b/core/src/main/java/io/questdb/client/cutlass/line/LineSenderException.java index 9c6cc16..6fdaf9a 100644 --- a/core/src/main/java/io/questdb/client/cutlass/line/LineSenderException.java +++ b/core/src/main/java/io/questdb/client/cutlass/line/LineSenderException.java @@ -39,15 +39,9 @@ *

  • For permanent errors: Either close and recreate the Sender, or call {@code reset()} to clear * the buffer and continue with new data
    • * + *

    + * @see io.questdb.client.Sender * - *

    Retryability

    - * The {@link #isRetryable()} method provides a best-effort indication of whether the error - * might be resolved by retrying at the application level. This is particularly important - * because this exception is only thrown after the sender has exhausted its own internal - * retry attempts. The retryability flag helps applications decide whether to implement - * additional retry logic with longer delays or different strategies. - * - * @see io.questdb.client.Sender * @see io.questdb.client.Sender#flush() * @see io.questdb.client.Sender#reset() */ @@ -115,4 +109,4 @@ public LineSenderException putAsPrintable(CharSequence nonPrintable) { message.putAsPrintable(nonPrintable); return this; } -} \ No newline at end of file +} From a274be53038f9e02817a483a379379343f411bef Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Wed, 25 Feb 2026 10:35:50 +0100 Subject: [PATCH 43/89] Fix inc() not adding key to list inc() called putAt0() directly without adding the key to the `list` field. This caused keys() to be incomplete and valueQuick() to return wrong results for keys inserted via inc(). Add the missing list.add() call, consistent with putAt() and putIfAbsent(). Co-Authored-By: Claude Opus 4.6 --- .../java/io/questdb/client/std/CharSequenceIntHashMap.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/io/questdb/client/std/CharSequenceIntHashMap.java b/core/src/main/java/io/questdb/client/std/CharSequenceIntHashMap.java index 932d1eb..d56c181 100644 --- a/core/src/main/java/io/questdb/client/std/CharSequenceIntHashMap.java +++ b/core/src/main/java/io/questdb/client/std/CharSequenceIntHashMap.java @@ -67,7 +67,9 @@ public void inc(@NotNull CharSequence key) { if (index < 0) { values[-index - 1]++; } else { - putAt0(index, Chars.toString(key), 1); + String keyString = Chars.toString(key); + putAt0(index, keyString, 1); + list.add(keyString); } } From 7d003e08eb9a08dc38c5630e599ef0106cc8aa2d Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Wed, 25 Feb 2026 11:04:40 +0100 Subject: [PATCH 44/89] Validate low surrogate in UTF-8 encoding When a high surrogate was detected in the UTF-8 encoding paths, the next char was consumed and used as a low surrogate without validating it was actually in the [0xDC00, 0xDFFF] range. This produced garbage 4-byte sequences and silently swallowed the following character. Add Character.isLowSurrogate(c2) checks in all 5 putUtf8/hasher encoding sites and both utf8Length methods. Invalid surrogates now emit '?' and re-process the consumed char on the next iteration, consistent with Utf8s.encodeUtf16Surrogate(). Co-Authored-By: Claude Opus 4.6 --- .../http/client/WebSocketSendBuffer.java | 15 +++-- .../qwp/client/NativeBufferWriter.java | 19 ++++-- .../cutlass/qwp/client/QwpBufferWriter.java | 4 +- .../qwp/protocol/OffHeapAppendMemory.java | 15 +++-- .../cutlass/qwp/protocol/QwpSchemaHash.java | 30 ++++++--- .../http/client/WebSocketSendBufferTest.java | 46 +++++++++++++ .../qwp/client/NativeBufferWriterTest.java | 37 +++++++++++ .../qwp/protocol/OffHeapAppendMemoryTest.java | 12 ++++ .../protocol/QwpSchemaHashSurrogateTest.java | 66 +++++++++++++++++++ 9 files changed, 217 insertions(+), 27 deletions(-) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketSendBufferTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaHashSurrogateTest.java diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java index 287bda0..66cdf13 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java @@ -354,11 +354,16 @@ public void putUtf8(String value) { putByte((byte) (0x80 | (c & 0x3F))); } else if (c >= 0xD800 && c <= 0xDBFF && i + 1 < n) { char c2 = value.charAt(++i); - int codePoint = 0x10000 + ((c - 0xD800) << 10) + (c2 - 0xDC00); - putByte((byte) (0xF0 | (codePoint >> 18))); - putByte((byte) (0x80 | ((codePoint >> 12) & 0x3F))); - putByte((byte) (0x80 | ((codePoint >> 6) & 0x3F))); - putByte((byte) (0x80 | (codePoint & 0x3F))); + if (Character.isLowSurrogate(c2)) { + int codePoint = 0x10000 + ((c - 0xD800) << 10) + (c2 - 0xDC00); + putByte((byte) (0xF0 | (codePoint >> 18))); + putByte((byte) (0x80 | ((codePoint >> 12) & 0x3F))); + putByte((byte) (0x80 | ((codePoint >> 6) & 0x3F))); + putByte((byte) (0x80 | (codePoint & 0x3F))); + } else { + putByte((byte) '?'); + i--; + } } else { putByte((byte) (0xE0 | (c >> 12))); putByte((byte) (0x80 | ((c >> 6) & 0x3F))); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java index a11f70f..1e00e12 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java @@ -66,9 +66,11 @@ public static int utf8Length(String s) { len++; } else if (c < 0x800) { len += 2; - } else if (c >= 0xD800 && c <= 0xDBFF && i + 1 < n) { + } else if (c >= 0xD800 && c <= 0xDBFF && i + 1 < n && Character.isLowSurrogate(s.charAt(i + 1))) { i++; len += 4; + } else if (Character.isSurrogate(c)) { + len++; } else { len += 3; } @@ -243,11 +245,16 @@ public void putUtf8(String value) { putByte((byte) (0x80 | (c & 0x3F))); } else if (c >= 0xD800 && c <= 0xDBFF && i + 1 < n) { char c2 = value.charAt(++i); - int codePoint = 0x10000 + ((c - 0xD800) << 10) + (c2 - 0xDC00); - putByte((byte) (0xF0 | (codePoint >> 18))); - putByte((byte) (0x80 | ((codePoint >> 12) & 0x3F))); - putByte((byte) (0x80 | ((codePoint >> 6) & 0x3F))); - putByte((byte) (0x80 | (codePoint & 0x3F))); + if (Character.isLowSurrogate(c2)) { + int codePoint = 0x10000 + ((c - 0xD800) << 10) + (c2 - 0xDC00); + putByte((byte) (0xF0 | (codePoint >> 18))); + putByte((byte) (0x80 | ((codePoint >> 12) & 0x3F))); + putByte((byte) (0x80 | ((codePoint >> 6) & 0x3F))); + putByte((byte) (0x80 | (codePoint & 0x3F))); + } else { + putByte((byte) '?'); + i--; + } } else { putByte((byte) (0xE0 | (c >> 12))); putByte((byte) (0x80 | ((c >> 6) & 0x3F))); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpBufferWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpBufferWriter.java index c50cdf6..f32f575 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpBufferWriter.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpBufferWriter.java @@ -59,9 +59,11 @@ static int utf8Length(String s) { len++; } else if (c < 0x800) { len += 2; - } else if (c >= 0xD800 && c <= 0xDBFF && i + 1 < n) { + } else if (c >= 0xD800 && c <= 0xDBFF && i + 1 < n && Character.isLowSurrogate(s.charAt(i + 1))) { i++; len += 4; + } else if (Character.isSurrogate(c)) { + len++; } else { len += 3; } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java index b73d88c..e02398f 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java @@ -152,11 +152,16 @@ public void putUtf8(String value) { Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0x80 | (c & 0x3F))); } else if (c >= 0xD800 && c <= 0xDBFF && i + 1 < len) { char c2 = value.charAt(++i); - int codePoint = 0x10000 + ((c - 0xD800) << 10) + (c2 - 0xDC00); - Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0xF0 | (codePoint >> 18))); - Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0x80 | ((codePoint >> 12) & 0x3F))); - Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0x80 | ((codePoint >> 6) & 0x3F))); - Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0x80 | (codePoint & 0x3F))); + if (Character.isLowSurrogate(c2)) { + int codePoint = 0x10000 + ((c - 0xD800) << 10) + (c2 - 0xDC00); + Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0xF0 | (codePoint >> 18))); + Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0x80 | ((codePoint >> 12) & 0x3F))); + Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0x80 | ((codePoint >> 6) & 0x3F))); + Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0x80 | (codePoint & 0x3F))); + } else { + Unsafe.getUnsafe().putByte(appendAddress++, (byte) '?'); + i--; + } } else { Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0xE0 | (c >> 12))); Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0x80 | ((c >> 6) & 0x3F))); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java index dd6388a..b62fc07 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java @@ -112,11 +112,16 @@ public static long computeSchemaHash(String[] columnNames, byte[] columnTypes) { } else if (c >= 0xD800 && c <= 0xDBFF && j + 1 < len) { // Surrogate pair (4 bytes) char c2 = name.charAt(++j); - int codePoint = 0x10000 + ((c - 0xD800) << 10) + (c2 - 0xDC00); - hasher.update((byte) (0xF0 | (codePoint >> 18))); - hasher.update((byte) (0x80 | ((codePoint >> 12) & 0x3F))); - hasher.update((byte) (0x80 | ((codePoint >> 6) & 0x3F))); - hasher.update((byte) (0x80 | (codePoint & 0x3F))); + if (Character.isLowSurrogate(c2)) { + int codePoint = 0x10000 + ((c - 0xD800) << 10) + (c2 - 0xDC00); + hasher.update((byte) (0xF0 | (codePoint >> 18))); + hasher.update((byte) (0x80 | ((codePoint >> 12) & 0x3F))); + hasher.update((byte) (0x80 | ((codePoint >> 6) & 0x3F))); + hasher.update((byte) (0x80 | (codePoint & 0x3F))); + } else { + hasher.update((byte) '?'); + j--; + } } else { // Three bytes hasher.update((byte) (0xE0 | (c >> 12))); @@ -180,11 +185,16 @@ public static long computeSchemaHashDirect(io.questdb.client.std.ObjList= 0xD800 && c <= 0xDBFF && j + 1 < len) { char c2 = name.charAt(++j); - int codePoint = 0x10000 + ((c - 0xD800) << 10) + (c2 - 0xDC00); - hasher.update((byte) (0xF0 | (codePoint >> 18))); - hasher.update((byte) (0x80 | ((codePoint >> 12) & 0x3F))); - hasher.update((byte) (0x80 | ((codePoint >> 6) & 0x3F))); - hasher.update((byte) (0x80 | (codePoint & 0x3F))); + if (Character.isLowSurrogate(c2)) { + int codePoint = 0x10000 + ((c - 0xD800) << 10) + (c2 - 0xDC00); + hasher.update((byte) (0xF0 | (codePoint >> 18))); + hasher.update((byte) (0x80 | ((codePoint >> 12) & 0x3F))); + hasher.update((byte) (0x80 | ((codePoint >> 6) & 0x3F))); + hasher.update((byte) (0x80 | (codePoint & 0x3F))); + } else { + hasher.update((byte) '?'); + j--; + } } else { hasher.update((byte) (0xE0 | (c >> 12))); hasher.update((byte) (0x80 | ((c >> 6) & 0x3F))); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketSendBufferTest.java b/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketSendBufferTest.java new file mode 100644 index 0000000..2218e9d --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketSendBufferTest.java @@ -0,0 +1,46 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.http.client; + +import io.questdb.client.cutlass.http.client.WebSocketSendBuffer; +import io.questdb.client.std.Unsafe; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class WebSocketSendBufferTest { + + @Test + public void testPutUtf8InvalidSurrogatePair() { + try (WebSocketSendBuffer buf = new WebSocketSendBuffer(256)) { + // High surrogate \uD800 followed by non-low-surrogate 'X'. + // Should produce '?' for the lone high surrogate, then 'X'. + buf.putUtf8("\uD800X"); + assertEquals(2, buf.getWritePos()); + assertEquals((byte) '?', Unsafe.getUnsafe().getByte(buf.getBufferPtr())); + assertEquals((byte) 'X', Unsafe.getUnsafe().getByte(buf.getBufferPtr() + 1)); + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java index f51aa54..a22896d 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java @@ -25,6 +25,7 @@ package io.questdb.client.test.cutlass.qwp.client; import io.questdb.client.cutlass.qwp.client.NativeBufferWriter; +import io.questdb.client.cutlass.qwp.client.QwpBufferWriter; import io.questdb.client.std.Unsafe; import org.junit.Test; @@ -74,6 +75,42 @@ public void testSkipAdvancesPosition() { } } + @Test + public void testPutUtf8InvalidSurrogatePair() { + try (NativeBufferWriter writer = new NativeBufferWriter(64)) { + // High surrogate \uD800 followed by non-low-surrogate 'X'. + // Should produce '?' for the lone high surrogate, then 'X'. + writer.putUtf8("\uD800X"); + assertEquals(2, writer.getPosition()); + assertEquals((byte) '?', Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + assertEquals((byte) 'X', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 1)); + } + } + + @Test + public void testNativeBufferWriterUtf8LengthInvalidSurrogatePair() { + // High surrogate followed by non-low-surrogate: '?' (1) + 'X' (1) = 2 + assertEquals(2, NativeBufferWriter.utf8Length("\uD800X")); + // Lone high surrogate at end: '?' (1) + assertEquals(1, NativeBufferWriter.utf8Length("\uD800")); + // Lone low surrogate: '?' (1) + assertEquals(1, NativeBufferWriter.utf8Length("\uDC00")); + // Valid pair still works: 4 bytes + assertEquals(4, NativeBufferWriter.utf8Length("\uD83D\uDE00")); + } + + @Test + public void testQwpBufferWriterUtf8LengthInvalidSurrogatePair() { + // High surrogate followed by non-low-surrogate: '?' (1) + 'X' (1) = 2 + assertEquals(2, QwpBufferWriter.utf8Length("\uD800X")); + // Lone high surrogate at end: '?' (1) + assertEquals(1, QwpBufferWriter.utf8Length("\uD800")); + // Lone low surrogate: '?' (1) + assertEquals(1, QwpBufferWriter.utf8Length("\uDC00")); + // Valid pair still works: 4 bytes + assertEquals(4, QwpBufferWriter.utf8Length("\uD83D\uDE00")); + } + @Test public void testSkipThenPatchInt() { try (NativeBufferWriter writer = new NativeBufferWriter(8)) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/OffHeapAppendMemoryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/OffHeapAppendMemoryTest.java index 6034988..d7755f4 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/OffHeapAppendMemoryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/OffHeapAppendMemoryTest.java @@ -286,6 +286,18 @@ public void testPutUtf8Null() { } } + @Test + public void testPutUtf8InvalidSurrogatePair() { + try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { + // High surrogate \uD800 followed by non-low-surrogate 'X'. + // Should produce '?' for the lone high surrogate, then 'X'. + mem.putUtf8("\uD800X"); + assertEquals(2, mem.getAppendOffset()); + assertEquals((byte) '?', Unsafe.getUnsafe().getByte(mem.addressOf(0))); + assertEquals((byte) 'X', Unsafe.getUnsafe().getByte(mem.addressOf(1))); + } + } + @Test public void testPutUtf8SurrogatePairs() { try (OffHeapAppendMemory mem = new OffHeapAppendMemory()) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaHashSurrogateTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaHashSurrogateTest.java new file mode 100644 index 0000000..76610f7 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaHashSurrogateTest.java @@ -0,0 +1,66 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaHash; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.std.ObjList; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class QwpSchemaHashSurrogateTest { + + private static final byte TYPE_LONG = 0x05; + + @Test + public void testComputeSchemaHashInvalidSurrogatePair() { + byte[] types = {TYPE_LONG}; + + // "\uD800X" has a high surrogate followed by non-low-surrogate 'X'. + // With the fix, the high surrogate becomes '?' and 'X' is preserved, + // so the hash should equal the hash of "?X". + long hashInvalid = QwpSchemaHash.computeSchemaHash( + new String[]{"\uD800X"}, types + ); + long hashExpected = QwpSchemaHash.computeSchemaHash( + new String[]{"?X"}, types + ); + assertEquals(hashExpected, hashInvalid); + } + + @Test + public void testComputeSchemaHashDirectInvalidSurrogatePair() { + ObjList invalidCols = new ObjList<>(); + invalidCols.add(new QwpTableBuffer.ColumnBuffer("\uD800X", TYPE_LONG, false)); + + ObjList expectedCols = new ObjList<>(); + expectedCols.add(new QwpTableBuffer.ColumnBuffer("?X", TYPE_LONG, false)); + + long hashInvalid = QwpSchemaHash.computeSchemaHashDirect(invalidCols); + long hashExpected = QwpSchemaHash.computeSchemaHashDirect(expectedCols); + assertEquals(hashExpected, hashInvalid); + } +} From 11326f1388be6e93f4660319a7d974e3e013b99a Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Wed, 25 Feb 2026 12:08:28 +0100 Subject: [PATCH 45/89] Eliminate hot-path allocations in waitForAck The boolean[] wrapper and anonymous WebSocketFrameHandler were allocated on every loop iteration inside waitForAck(), generating GC pressure on the data ingestion hot path. Hoist both into reusable instance fields: ackResponse (WebSocket response buffer), sawBinaryAck (plain boolean replacing the boolean[] wrapper), and ackHandler (a static nested class AckFrameHandler replacing the per-iteration anonymous class). Co-Authored-By: Claude Opus 4.6 --- .../qwp/client/QwpWebSocketSender.java | 62 +++++++++++-------- 1 file changed, 36 insertions(+), 26 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index b2aa9a3..b597d34 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -133,6 +133,8 @@ public class QwpWebSocketSender implements Sender { private final LongHashSet sentSchemaHashes = new LongHashSet(); private final CharSequenceObjHashMap tableBuffers; private final boolean tlsEnabled; + private final AckFrameHandler ackHandler = new AckFrameHandler(this); + private final WebSocketResponse ackResponse = new WebSocketResponse(); private MicrobatchBuffer activeBuffer; // Double-buffering for async I/O private MicrobatchBuffer buffer0; @@ -160,6 +162,7 @@ public class QwpWebSocketSender implements Sender { private long nextBatchSequence = 0; // Async mode: pending row tracking private int pendingRowCount; + private boolean sawBinaryAck; private WebSocketSendQueue sendQueue; private QwpWebSocketSender( @@ -1386,39 +1389,20 @@ private long toMicros(long value, ChronoUnit unit) { * Waits synchronously for an ACK from the server for the specified batch. */ private void waitForAck(long expectedSequence) { - WebSocketResponse response = new WebSocketResponse(); long deadline = System.currentTimeMillis() + InFlightWindow.DEFAULT_TIMEOUT_MS; while (System.currentTimeMillis() < deadline) { try { - final boolean[] sawBinary = {false}; - boolean received = client.receiveFrame(new WebSocketFrameHandler() { - @Override - public void onBinaryMessage(long payloadPtr, int payloadLen) { - sawBinary[0] = true; - if (!WebSocketResponse.isStructurallyValid(payloadPtr, payloadLen)) { - throw new LineSenderException( - "Invalid ACK response payload [length=" + payloadLen + ']' - ); - } - if (!response.readFrom(payloadPtr, payloadLen)) { - throw new LineSenderException("Failed to parse ACK response"); - } - } - - @Override - public void onClose(int code, String reason) { - throw new LineSenderException("WebSocket closed while waiting for ACK: " + reason); - } - }, 1000); // 1 second timeout per read attempt + sawBinaryAck = false; + boolean received = client.receiveFrame(ackHandler, 1000); // 1 second timeout per read attempt if (received) { // Non-binary frames (e.g. ping/pong/text) are not ACKs. - if (!sawBinary[0]) { + if (!sawBinaryAck) { continue; } - long sequence = response.getSequence(); - if (response.isSuccess()) { + long sequence = ackResponse.getSequence(); + if (ackResponse.isSuccess()) { // Cumulative ACK - acknowledge all batches up to this sequence inFlightWindow.acknowledgeUpTo(sequence); if (sequence >= expectedSequence) { @@ -1426,10 +1410,10 @@ public void onClose(int code, String reason) { } // Got ACK for lower sequence - continue waiting } else { - String errorMessage = response.getErrorMessage(); + String errorMessage = ackResponse.getErrorMessage(); LineSenderException error = new LineSenderException( "Server error for batch " + sequence + ": " + - response.getStatusName() + " - " + errorMessage); + ackResponse.getStatusName() + " - " + errorMessage); inFlightWindow.fail(sequence, error); if (sequence == expectedSequence) { throw error; @@ -1450,4 +1434,30 @@ public void onClose(int code, String reason) { failExpectedIfNeeded(expectedSequence, timeout); throw timeout; } + + private static class AckFrameHandler implements WebSocketFrameHandler { + private final QwpWebSocketSender sender; + + AckFrameHandler(QwpWebSocketSender sender) { + this.sender = sender; + } + + @Override + public void onBinaryMessage(long payloadPtr, int payloadLen) { + sender.sawBinaryAck = true; + if (!WebSocketResponse.isStructurallyValid(payloadPtr, payloadLen)) { + throw new LineSenderException( + "Invalid ACK response payload [length=" + payloadLen + ']' + ); + } + if (!sender.ackResponse.readFrom(payloadPtr, payloadLen)) { + throw new LineSenderException("Failed to parse ACK response"); + } + } + + @Override + public void onClose(int code, String reason) { + throw new LineSenderException("WebSocket closed while waiting for ACK: " + reason); + } + } } From 5f54bc3588a060bbd2b17e621bda546d045e618e Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Wed, 25 Feb 2026 12:21:13 +0100 Subject: [PATCH 46/89] Wait for server ACKs on close() in async mode The close() method in async mode was only waiting for pending batches to be written to the wire (via sendQueue.close()), but did not wait for the server to acknowledge receipt. This caused data loss when close() was called without an explicit flush(), since the connection was torn down before the server finished processing. Add sendQueue.flush() and inFlightWindow.awaitEmpty() before sendQueue.close() to match the behavior of flush(). Co-Authored-By: Claude Opus 4.6 --- .../client/cutlass/qwp/client/QwpWebSocketSender.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index b597d34..13b0206 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -478,6 +478,13 @@ public void close() { if (activeBuffer != null && activeBuffer.hasData()) { sealAndSwapBuffer(); } + // Wait for all batches to be sent and acknowledged before closing + if (sendQueue != null) { + sendQueue.flush(); + } + if (inFlightWindow != null) { + inFlightWindow.awaitEmpty(); + } if (sendQueue != null) { sendQueue.close(); } From b34ba1ecc1d841b1229077f124f3a338b649602b Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Wed, 25 Feb 2026 12:46:07 +0100 Subject: [PATCH 47/89] Add GEOHASH support to QWP table buffers Add TYPE_GEOHASH to elementSize(), allocateStorage(), and addNull() so geohash values are stored as longs (8 bytes) with -1L as the null sentinel. Also add an explicit case in QwpConstants.getFixedTypeSize() to document that GEOHASH is intentionally variable-width on the wire. Co-Authored-By: Claude Opus 4.6 --- .../io/questdb/client/cutlass/qwp/protocol/QwpConstants.java | 2 ++ .../questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java index 34f9b2d..54d6fdd 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java @@ -326,6 +326,8 @@ public static int getFixedTypeSize(byte typeCode) { case TYPE_LONG256: case TYPE_DECIMAL256: return 32; + case TYPE_GEOHASH: + return -1; // Variable width: varint precision + packed values default: return -1; // Variable width } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index 4e47c1b..1999fb6 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -272,6 +272,7 @@ static int elementSize(byte type) { case TYPE_SYMBOL: case TYPE_FLOAT: return 4; + case TYPE_GEOHASH: case TYPE_LONG: case TYPE_TIMESTAMP: case TYPE_TIMESTAMP_NANOS: @@ -760,6 +761,9 @@ public void addNull() { case TYPE_INT: dataBuffer.putInt(0); break; + case TYPE_GEOHASH: + dataBuffer.putLong(-1L); + break; case TYPE_LONG: case TYPE_TIMESTAMP: case TYPE_TIMESTAMP_NANOS: @@ -1142,6 +1146,7 @@ private void allocateStorage(byte type) { case TYPE_INT: dataBuffer = new OffHeapAppendMemory(64); break; + case TYPE_GEOHASH: case TYPE_LONG: case TYPE_TIMESTAMP: case TYPE_TIMESTAMP_NANOS: From cf64a41c190b40a86a79bbf73f6c595b06d1cc46 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Wed, 25 Feb 2026 13:05:13 +0100 Subject: [PATCH 48/89] Use ChaCha20 CSPRNG for WebSocket masking keys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the XorShift128 Rnd PRNG with a new ChaCha20-based SecureRnd for WebSocket frame mask key generation. The previous implementation seeded Rnd with System.nanoTime() and System.currentTimeMillis(), which is predictable and does not meet RFC 6455 Section 5.3's requirement for strong entropy. SecureRnd implements ChaCha20 in counter mode (RFC 7539), seeded once from java.security.SecureRandom at construction time. After initialization there are zero heap allocations — all state lives in two pre-allocated int[16] arrays. Each ChaCha20 block yields 16 mask keys, so the amortized cost is minimal. Includes a known-answer test using the RFC 7539 Section 2.3.2 test vector to verify correctness of the ChaCha20 implementation. Co-Authored-By: Claude Opus 4.6 --- .../http/client/WebSocketSendBuffer.java | 6 +- .../cutlass/qwp/client/WebSocketChannel.java | 10 +- .../java/io/questdb/client/std/SecureRnd.java | 185 ++++++++++++++++++ .../client/test/std/SecureRndTest.java | 120 ++++++++++++ 4 files changed, 313 insertions(+), 8 deletions(-) create mode 100644 core/src/main/java/io/questdb/client/std/SecureRnd.java create mode 100644 core/src/test/java/io/questdb/client/test/std/SecureRndTest.java diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java index 66cdf13..5da1c43 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java @@ -31,7 +31,7 @@ import io.questdb.client.std.MemoryTag; import io.questdb.client.std.Numbers; import io.questdb.client.std.QuietCloseable; -import io.questdb.client.std.Rnd; +import io.questdb.client.std.SecureRnd; import io.questdb.client.std.Unsafe; import io.questdb.client.std.Vect; @@ -63,7 +63,7 @@ public class WebSocketSendBuffer implements QwpBufferWriter, QuietCloseable { private static final int MAX_HEADER_SIZE = 14; private final FrameInfo frameInfo = new FrameInfo(); private final int maxBufferSize; - private final Rnd rnd; + private final SecureRnd rnd; private int bufCapacity; private long bufPtr; private int frameStartOffset; // Where current frame's reserved header starts @@ -99,7 +99,7 @@ public WebSocketSendBuffer(int initialCapacity, int maxBufferSize) { this.writePos = 0; this.frameStartOffset = 0; this.payloadStartOffset = 0; - this.rnd = new Rnd(System.nanoTime(), System.currentTimeMillis()); + this.rnd = new SecureRnd(); } /** diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketChannel.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketChannel.java index da64ca9..d3965fa 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketChannel.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketChannel.java @@ -32,7 +32,7 @@ import io.questdb.client.cutlass.qwp.websocket.WebSocketOpcode; import io.questdb.client.std.MemoryTag; import io.questdb.client.std.QuietCloseable; -import io.questdb.client.std.Rnd; +import io.questdb.client.std.SecureRnd; import io.questdb.client.std.Unsafe; import javax.net.SocketFactory; @@ -76,8 +76,8 @@ public class WebSocketChannel implements QuietCloseable { private final String host; private final String path; private final int port; - // Random for mask key generation - private final Rnd rnd; + // Random for mask key generation (ChaCha20-based CSPRNG, RFC 6455 Section 5.3) + private final SecureRnd rnd; private final boolean tlsEnabled; private final boolean tlsValidationEnabled; private boolean closed; @@ -141,7 +141,7 @@ public WebSocketChannel(String url, boolean tlsEnabled, boolean tlsValidationEna this.tlsValidationEnabled = tlsValidationEnabled; this.frameParser = new WebSocketFrameParser(); - this.rnd = new Rnd(System.nanoTime(), System.currentTimeMillis()); + this.rnd = new SecureRnd(); // Allocate native buffers this.sendBufferSize = DEFAULT_BUFFER_SIZE; @@ -380,7 +380,7 @@ private void performHandshake() throws IOException { // Generate random key (16 bytes, base64 encoded = 24 chars) byte[] keyBytes = new byte[16]; for (int i = 0; i < 16; i++) { - keyBytes[i] = (byte) rnd.nextInt(256); + keyBytes[i] = (byte) rnd.nextInt(); } String key = Base64.getEncoder().encodeToString(keyBytes); diff --git a/core/src/main/java/io/questdb/client/std/SecureRnd.java b/core/src/main/java/io/questdb/client/std/SecureRnd.java new file mode 100644 index 0000000..2ceef88 --- /dev/null +++ b/core/src/main/java/io/questdb/client/std/SecureRnd.java @@ -0,0 +1,185 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.std; + +import java.security.SecureRandom; + +/** + * Zero-GC cryptographically secure random number generator based on ChaCha20 + * in counter mode (RFC 7539). Seeded once from {@link SecureRandom} at + * construction time, then produces unpredictable output with no heap + * allocations. + *

    + * Each {@link #nextInt()} call returns one 32-bit word from the ChaCha20 + * keystream. A single block computation yields 16 words, so the amortized + * cost is one ChaCha20 block per 16 calls. + */ +public class SecureRnd { + + // "expand 32-byte k" in little-endian + private static final int CONSTANT_0 = 0x61707865; + private static final int CONSTANT_1 = 0x3320646e; + private static final int CONSTANT_2 = 0x79622d32; + private static final int CONSTANT_3 = 0x6b206574; + + private final int[] output = new int[16]; + private final int[] state = new int[16]; + private int outputPos = 16; // forces block computation on first call + + /** + * Creates a new instance seeded from {@link SecureRandom}. + */ + public SecureRnd() { + SecureRandom seed = new SecureRandom(); + byte[] seedBytes = new byte[44]; // 32 (key) + 12 (nonce) + seed.nextBytes(seedBytes); + init(seedBytes, 0); + } + + /** + * Creates a new instance with an explicit key, nonce, and initial counter. + * Useful for testing with known RFC 7539 test vectors. + * + * @param key 32-byte key + * @param nonce 12-byte nonce + * @param counter initial block counter value + */ + public SecureRnd(byte[] key, byte[] nonce, int counter) { + byte[] seedBytes = new byte[44]; + System.arraycopy(key, 0, seedBytes, 0, 32); + System.arraycopy(nonce, 0, seedBytes, 32, 12); + init(seedBytes, counter); + } + + /** + * Returns the next cryptographically secure random int. + */ + public int nextInt() { + if (outputPos >= 16) { + computeBlock(); + outputPos = 0; + } + return output[outputPos++]; + } + + private void computeBlock() { + int x0 = state[0], x1 = state[1], x2 = state[2], x3 = state[3]; + int x4 = state[4], x5 = state[5], x6 = state[6], x7 = state[7]; + int x8 = state[8], x9 = state[9], x10 = state[10], x11 = state[11]; + int x12 = state[12], x13 = state[13], x14 = state[14], x15 = state[15]; + + for (int i = 0; i < 10; i++) { + // Column rounds + x0 += x4; x12 ^= x0; x12 = Integer.rotateLeft(x12, 16); + x8 += x12; x4 ^= x8; x4 = Integer.rotateLeft(x4, 12); + x0 += x4; x12 ^= x0; x12 = Integer.rotateLeft(x12, 8); + x8 += x12; x4 ^= x8; x4 = Integer.rotateLeft(x4, 7); + + x1 += x5; x13 ^= x1; x13 = Integer.rotateLeft(x13, 16); + x9 += x13; x5 ^= x9; x5 = Integer.rotateLeft(x5, 12); + x1 += x5; x13 ^= x1; x13 = Integer.rotateLeft(x13, 8); + x9 += x13; x5 ^= x9; x5 = Integer.rotateLeft(x5, 7); + + x2 += x6; x14 ^= x2; x14 = Integer.rotateLeft(x14, 16); + x10 += x14; x6 ^= x10; x6 = Integer.rotateLeft(x6, 12); + x2 += x6; x14 ^= x2; x14 = Integer.rotateLeft(x14, 8); + x10 += x14; x6 ^= x10; x6 = Integer.rotateLeft(x6, 7); + + x3 += x7; x15 ^= x3; x15 = Integer.rotateLeft(x15, 16); + x11 += x15; x7 ^= x11; x7 = Integer.rotateLeft(x7, 12); + x3 += x7; x15 ^= x3; x15 = Integer.rotateLeft(x15, 8); + x11 += x15; x7 ^= x11; x7 = Integer.rotateLeft(x7, 7); + + // Diagonal rounds + x0 += x5; x15 ^= x0; x15 = Integer.rotateLeft(x15, 16); + x10 += x15; x5 ^= x10; x5 = Integer.rotateLeft(x5, 12); + x0 += x5; x15 ^= x0; x15 = Integer.rotateLeft(x15, 8); + x10 += x15; x5 ^= x10; x5 = Integer.rotateLeft(x5, 7); + + x1 += x6; x12 ^= x1; x12 = Integer.rotateLeft(x12, 16); + x11 += x12; x6 ^= x11; x6 = Integer.rotateLeft(x6, 12); + x1 += x6; x12 ^= x1; x12 = Integer.rotateLeft(x12, 8); + x11 += x12; x6 ^= x11; x6 = Integer.rotateLeft(x6, 7); + + x2 += x7; x13 ^= x2; x13 = Integer.rotateLeft(x13, 16); + x8 += x13; x7 ^= x8; x7 = Integer.rotateLeft(x7, 12); + x2 += x7; x13 ^= x2; x13 = Integer.rotateLeft(x13, 8); + x8 += x13; x7 ^= x8; x7 = Integer.rotateLeft(x7, 7); + + x3 += x4; x14 ^= x3; x14 = Integer.rotateLeft(x14, 16); + x9 += x14; x4 ^= x9; x4 = Integer.rotateLeft(x4, 12); + x3 += x4; x14 ^= x3; x14 = Integer.rotateLeft(x14, 8); + x9 += x14; x4 ^= x9; x4 = Integer.rotateLeft(x4, 7); + } + + // Feed-forward: add original state + output[0] = x0 + state[0]; + output[1] = x1 + state[1]; + output[2] = x2 + state[2]; + output[3] = x3 + state[3]; + output[4] = x4 + state[4]; + output[5] = x5 + state[5]; + output[6] = x6 + state[6]; + output[7] = x7 + state[7]; + output[8] = x8 + state[8]; + output[9] = x9 + state[9]; + output[10] = x10 + state[10]; + output[11] = x11 + state[11]; + output[12] = x12 + state[12]; + output[13] = x13 + state[13]; + output[14] = x14 + state[14]; + output[15] = x15 + state[15]; + + // Increment block counter + state[12]++; + } + + private void init(byte[] seedBytes, int counter) { + state[0] = CONSTANT_0; + state[1] = CONSTANT_1; + state[2] = CONSTANT_2; + state[3] = CONSTANT_3; + + // Key: 8 little-endian ints from seedBytes[0..31] + for (int i = 0; i < 8; i++) { + int off = i * 4; + state[4 + i] = (seedBytes[off] & 0xFF) + | ((seedBytes[off + 1] & 0xFF) << 8) + | ((seedBytes[off + 2] & 0xFF) << 16) + | ((seedBytes[off + 3] & 0xFF) << 24); + } + + state[12] = counter; + + // Nonce: 3 little-endian ints from seedBytes[32..43] + for (int i = 0; i < 3; i++) { + int off = 32 + i * 4; + state[13 + i] = (seedBytes[off] & 0xFF) + | ((seedBytes[off + 1] & 0xFF) << 8) + | ((seedBytes[off + 2] & 0xFF) << 16) + | ((seedBytes[off + 3] & 0xFF) << 24); + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/std/SecureRndTest.java b/core/src/test/java/io/questdb/client/test/std/SecureRndTest.java new file mode 100644 index 0000000..6c04837 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/std/SecureRndTest.java @@ -0,0 +1,120 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.std; + +import io.questdb.client.std.SecureRnd; +import org.junit.Assert; +import org.junit.Test; + +public class SecureRndTest { + + @Test + public void testConsecutiveCallsProduceDifferentValues() { + SecureRnd rnd = new SecureRnd(); + int prev = rnd.nextInt(); + boolean foundDifferent = false; + for (int i = 0; i < 100; i++) { + int next = rnd.nextInt(); + if (next != prev) { + foundDifferent = true; + break; + } + prev = next; + } + Assert.assertTrue("Expected different values from consecutive calls", foundDifferent); + } + + @Test + public void testDifferentInstancesProduceDifferentSequences() { + SecureRnd rnd1 = new SecureRnd(); + SecureRnd rnd2 = new SecureRnd(); + boolean foundDifferent = false; + for (int i = 0; i < 16; i++) { + if (rnd1.nextInt() != rnd2.nextInt()) { + foundDifferent = true; + break; + } + } + Assert.assertTrue("Two SecureRnd instances should produce different sequences", foundDifferent); + } + + @Test + public void testMultipleBlocksDoNotRepeat() { + SecureRnd rnd = new SecureRnd(); + // Consume more than one block (16 ints) to trigger block counter increment + int[] first16 = new int[16]; + for (int i = 0; i < 16; i++) { + first16[i] = rnd.nextInt(); + } + // Next 16 should be from a different block + boolean foundDifferent = false; + for (int i = 0; i < 16; i++) { + if (rnd.nextInt() != first16[i]) { + foundDifferent = true; + break; + } + } + Assert.assertTrue("Second block should differ from first", foundDifferent); + } + + // RFC 7539 Section 2.3.2 known-answer test + @Test + public void testRfc7539Section232TestVector() { + // Key: 00:01:02:03:...:1f + byte[] key = new byte[32]; + for (int i = 0; i < 32; i++) { + key[i] = (byte) i; + } + + // Nonce: 00:00:00:09:00:00:00:4a:00:00:00:00 + byte[] nonce = { + 0x00, 0x00, 0x00, 0x09, + 0x00, 0x00, 0x00, 0x4a, + 0x00, 0x00, 0x00, 0x00 + }; + + // Block counter = 1 + SecureRnd rnd = new SecureRnd(key, nonce, 1); + + // Expected output words (ChaCha state after adding original input) + // from RFC 7539 Section 2.3.2 + int[] expected = { + 0xe4e7f110, 0x15593bd1, 0x1fdd0f50, (int) 0xc47120a3, + (int) 0xc7f4d1c7, 0x0368c033, (int) 0x9aaa2204, 0x4e6cd4c3, + 0x466482d2, 0x09aa9f07, 0x05d7c214, (int) 0xa2028bd9, + (int) 0xd19c12b5, (int) 0xb94e16de, (int) 0xe883d0cb, 0x4e3c50a2, + }; + + for (int i = 0; i < 16; i++) { + int actual = rnd.nextInt(); + Assert.assertEquals( + "Mismatch at word " + i + ": expected 0x" + Integer.toHexString(expected[i]) + + " but got 0x" + Integer.toHexString(actual), + expected[i], + actual + ); + } + } +} From b6609cd95bd39197a325144eb7f2e18b61feaa62 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Wed, 25 Feb 2026 13:56:56 +0100 Subject: [PATCH 49/89] Simplify QwpBitReader.alignToByte() Remove the redundant bitsInBuffer % 8 outer guard. Since ensureBits() always loads whole bytes, the invariant (totalBitsRead + bitsInBuffer) % 8 == 0 always holds, making bitsInBuffer % 8 and totalBitsRead % 8 equivalent checks. The simplified version uses only totalBitsRead % 8 which more clearly expresses the intent. Co-Authored-By: Claude Opus 4.6 --- .../client/cutlass/qwp/protocol/QwpBitReader.java | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitReader.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitReader.java index 9241d70..944e4d9 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitReader.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitReader.java @@ -68,15 +68,9 @@ public QwpBitReader() { * @throws IllegalStateException if alignment fails */ public void alignToByte() { - int bitsToSkip = bitsInBuffer % 8; - if (bitsToSkip != 0) { - // We need to skip the remaining bits in the current partial byte - // But since we read in byte chunks, bitsInBuffer should be a multiple of 8 - // minus what we've consumed. The remainder in the conceptual stream is: - int remainder = (int) (totalBitsRead % 8); - if (remainder != 0) { - skipBits(8 - remainder); - } + int remainder = (int) (totalBitsRead % 8); + if (remainder != 0) { + skipBits(8 - remainder); } } From 8d16e0fef7a6744434a25cc3091bd523dc61cef3 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Wed, 25 Feb 2026 14:02:47 +0100 Subject: [PATCH 50/89] Fix putUtf8() lone surrogate encoding mismatch utf8Length() correctly counts lone surrogates as 1 byte ('?' replacement), but putUtf8() let them fall through to the BMP 3-byte encoding path. This 2-byte-per-surrogate discrepancy corrupts varint-prefixed string lengths written by putString(). Add a Character.isSurrogate() check before the 3-byte BMP branch in all three putUtf8() implementations: NativeBufferWriter, WebSocketSendBuffer, and OffHeapAppendMemory. Add tests verifying lone high/low surrogates write 1 byte and that putUtf8() and utf8Length() agree for all surrogate edge cases. Co-Authored-By: Claude Opus 4.6 --- .../http/client/WebSocketSendBuffer.java | 2 ++ .../qwp/client/NativeBufferWriter.java | 2 ++ .../qwp/protocol/OffHeapAppendMemory.java | 2 ++ .../qwp/client/NativeBufferWriterTest.java | 34 +++++++++++++++++++ 4 files changed, 40 insertions(+) diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java index 5da1c43..6bb280c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java @@ -364,6 +364,8 @@ public void putUtf8(String value) { putByte((byte) '?'); i--; } + } else if (Character.isSurrogate(c)) { + putByte((byte) '?'); } else { putByte((byte) (0xE0 | (c >> 12))); putByte((byte) (0x80 | ((c >> 6) & 0x3F))); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java index 1e00e12..3230b4c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java @@ -255,6 +255,8 @@ public void putUtf8(String value) { putByte((byte) '?'); i--; } + } else if (Character.isSurrogate(c)) { + putByte((byte) '?'); } else { putByte((byte) (0xE0 | (c >> 12))); putByte((byte) (0x80 | ((c >> 6) & 0x3F))); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java index e02398f..7388961 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java @@ -162,6 +162,8 @@ public void putUtf8(String value) { Unsafe.getUnsafe().putByte(appendAddress++, (byte) '?'); i--; } + } else if (Character.isSurrogate(c)) { + Unsafe.getUnsafe().putByte(appendAddress++, (byte) '?'); } else { Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0xE0 | (c >> 12))); Unsafe.getUnsafe().putByte(appendAddress++, (byte) (0x80 | ((c >> 6) & 0x3F))); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java index a22896d..67a356d 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java @@ -87,6 +87,40 @@ public void testPutUtf8InvalidSurrogatePair() { } } + @Test + public void testPutUtf8LoneHighSurrogateAtEnd() { + try (NativeBufferWriter writer = new NativeBufferWriter(64)) { + writer.putUtf8("\uD800"); + assertEquals(1, writer.getPosition()); + assertEquals((byte) '?', Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + } + } + + @Test + public void testPutUtf8LoneLowSurrogate() { + try (NativeBufferWriter writer = new NativeBufferWriter(64)) { + writer.putUtf8("\uDC00"); + assertEquals(1, writer.getPosition()); + assertEquals((byte) '?', Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + } + } + + @Test + public void testPutUtf8LoneSurrogateMatchesUtf8Length() { + try (NativeBufferWriter writer = new NativeBufferWriter(64)) { + // Verify putUtf8 and utf8Length agree for all lone surrogate cases + String[] cases = {"\uD800", "\uDBFF", "\uDC00", "\uDFFF", "\uD800X", "A\uDC00B"}; + for (String s : cases) { + writer.reset(); + writer.putUtf8(s); + assertEquals("length mismatch for: " + s.codePoints() + .mapToObj(cp -> String.format("U+%04X", cp)) + .reduce((a, b) -> a + " " + b).orElse(""), + NativeBufferWriter.utf8Length(s), writer.getPosition()); + } + } + } + @Test public void testNativeBufferWriterUtf8LengthInvalidSurrogatePair() { // High surrogate followed by non-low-surrogate: '?' (1) + 'X' (1) = 2 From 9254ebf3e409f541cc93c15356b6c5a65b50a34e Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Wed, 25 Feb 2026 14:48:42 +0100 Subject: [PATCH 51/89] Fix addSymbol() null on non-nullable columns addSymbol() and addSymbolWithGlobalId() incremented size without writing data or incrementing valueCount when value was null and the column was non-nullable. This caused a permanent misalignment between logical row count and physical data. Delegate null values to addNull(), which already handles both nullable (mark in bitmap) and non-nullable (write sentinel) cases correctly. Add a test that verifies size and valueCount stay in sync for non-nullable symbol columns with null values. Co-Authored-By: Claude Opus 4.6 --- .../cutlass/qwp/protocol/QwpTableBuffer.java | 69 +++++++++---------- .../qwp/protocol/QwpTableBufferTest.java | 21 ++++++ 2 files changed, 52 insertions(+), 38 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index 1999fb6..b183958 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -836,53 +836,46 @@ public void addString(String value) { public void addSymbol(String value) { if (value == null) { - if (nullable) { - ensureNullCapacity(size + 1); - markNull(size); - } - } else { - ensureNullBitmapForNonNull(); - int idx = symbolDict.get(value); - if (idx == CharSequenceIntHashMap.NO_ENTRY_VALUE) { - idx = symbolList.size(); - symbolDict.put(value, idx); - symbolList.add(value); - } - dataBuffer.putInt(idx); - valueCount++; + addNull(); + return; } + ensureNullBitmapForNonNull(); + int idx = symbolDict.get(value); + if (idx == CharSequenceIntHashMap.NO_ENTRY_VALUE) { + idx = symbolList.size(); + symbolDict.put(value, idx); + symbolList.add(value); + } + dataBuffer.putInt(idx); + valueCount++; size++; } public void addSymbolWithGlobalId(String value, int globalId) { if (value == null) { - if (nullable) { - ensureNullCapacity(size + 1); - markNull(size); - } - size++; - } else { - ensureNullBitmapForNonNull(); - int localIdx = symbolDict.get(value); - if (localIdx == CharSequenceIntHashMap.NO_ENTRY_VALUE) { - localIdx = symbolList.size(); - symbolDict.put(value, localIdx); - symbolList.add(value); - } - dataBuffer.putInt(localIdx); - - if (auxBuffer == null) { - auxBuffer = new OffHeapAppendMemory(64); - } - auxBuffer.putInt(globalId); + addNull(); + return; + } + ensureNullBitmapForNonNull(); + int localIdx = symbolDict.get(value); + if (localIdx == CharSequenceIntHashMap.NO_ENTRY_VALUE) { + localIdx = symbolList.size(); + symbolDict.put(value, localIdx); + symbolList.add(value); + } + dataBuffer.putInt(localIdx); - if (globalId > maxGlobalSymbolId) { - maxGlobalSymbolId = globalId; - } + if (auxBuffer == null) { + auxBuffer = new OffHeapAppendMemory(64); + } + auxBuffer.putInt(globalId); - valueCount++; - size++; + if (globalId > maxGlobalSymbolId) { + maxGlobalSymbolId = globalId; } + + valueCount++; + size++; } public void addUuid(long high, long low) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java index e4ac0d7..7e74ff7 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java @@ -34,6 +34,27 @@ public class QwpTableBufferTest { + @Test + public void testAddSymbolNullOnNonNullableColumn() { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("sym", QwpConstants.TYPE_SYMBOL, false); + col.addSymbol("server1"); + table.nextRow(); + + // Null on a non-nullable column must write a sentinel value, + // keeping size and valueCount in sync + col.addSymbol(null); + table.nextRow(); + + col.addSymbol("server2"); + table.nextRow(); + + assertEquals(3, table.getRowCount()); + // For non-nullable columns, every row must have a physical value + assertEquals(col.getSize(), col.getValueCount()); + } + } + @Test public void testCancelRowRewindsDoubleArrayOffsets() { try (QwpTableBuffer table = new QwpTableBuffer("test")) { From 391fce46fdf13961a42dfe518d5a57523bd30f9f Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Wed, 25 Feb 2026 14:58:30 +0100 Subject: [PATCH 52/89] Fix addNull() for array types on non-nullable columns QwpTableBuffer.addNull() had no handling for TYPE_DOUBLE_ARRAY and TYPE_LONG_ARRAY in the non-nullable branch. This caused valueCount and size to advance without writing array metadata (dims, shapes), corrupting array index tracking for subsequent rows. The fix writes an empty 1D array (dims=1, shape=0, no data elements) as the sentinel value, keeping dims/shapes/data offsets consistent. Co-Authored-By: Claude Opus 4.6 --- .../cutlass/qwp/protocol/QwpTableBuffer.java | 6 ++ .../qwp/protocol/QwpTableBufferTest.java | 72 +++++++++++++++++++ 2 files changed, 78 insertions(+) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index b183958..119bbbc 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -806,6 +806,12 @@ public void addNull() { dataBuffer.putLong(Decimals.DECIMAL256_LH_NULL); dataBuffer.putLong(Decimals.DECIMAL256_LL_NULL); break; + case TYPE_DOUBLE_ARRAY: + case TYPE_LONG_ARRAY: + ensureArrayCapacity(1, 0); + arrayDims[valueCount] = 1; + arrayShapes[arrayShapeOffset++] = 0; + break; } valueCount++; size++; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java index 7e74ff7..1c294d0 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java @@ -34,6 +34,78 @@ public class QwpTableBufferTest { + @Test + public void testAddDoubleArrayNullOnNonNullableColumn() { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_DOUBLE_ARRAY, false); + + // Row 0: real array + col.addDoubleArray(new double[]{1.0, 2.0}); + table.nextRow(); + + // Row 1: null on non-nullable — must write empty array metadata + col.addDoubleArray((double[]) null); + table.nextRow(); + + // Row 2: real array + col.addDoubleArray(new double[]{3.0, 4.0}); + table.nextRow(); + + assertEquals(3, table.getRowCount()); + assertEquals(3, col.getValueCount()); + assertEquals(col.getSize(), col.getValueCount()); + + // Encoder walk must not corrupt — row 1 is an empty array + double[] encoded = readDoubleArraysLikeEncoder(col); + assertArrayEquals(new double[]{1.0, 2.0, 3.0, 4.0}, encoded, 0.0); + + byte[] dims = col.getArrayDims(); + int[] shapes = col.getArrayShapes(); + assertEquals(1, dims[0]); + assertEquals(2, shapes[0]); + assertEquals(1, dims[1]); // null row: 1D empty + assertEquals(0, shapes[1]); // null row: 0 elements + assertEquals(1, dims[2]); + assertEquals(2, shapes[2]); + } + } + + @Test + public void testAddLongArrayNullOnNonNullableColumn() { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("arr", QwpConstants.TYPE_LONG_ARRAY, false); + + // Row 0: real array + col.addLongArray(new long[]{10, 20}); + table.nextRow(); + + // Row 1: null on non-nullable — must write empty array metadata + col.addLongArray((long[]) null); + table.nextRow(); + + // Row 2: real array + col.addLongArray(new long[]{30, 40}); + table.nextRow(); + + assertEquals(3, table.getRowCount()); + assertEquals(3, col.getValueCount()); + assertEquals(col.getSize(), col.getValueCount()); + + // Encoder walk must not corrupt — row 1 is an empty array + long[] encoded = readLongArraysLikeEncoder(col); + assertArrayEquals(new long[]{10, 20, 30, 40}, encoded); + + byte[] dims = col.getArrayDims(); + int[] shapes = col.getArrayShapes(); + assertEquals(1, dims[0]); + assertEquals(2, shapes[0]); + assertEquals(1, dims[1]); // null row: 1D empty + assertEquals(0, shapes[1]); // null row: 0 elements + assertEquals(1, dims[2]); + assertEquals(2, shapes[2]); + } + } + @Test public void testAddSymbolNullOnNonNullableColumn() { try (QwpTableBuffer table = new QwpTableBuffer("test")) { From a92e7b42ef54f7548c5a0ed0ddb1d2c571315ccb Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Wed, 25 Feb 2026 15:09:51 +0100 Subject: [PATCH 53/89] Fix sendCloseFrame/sendPing clobbering sendBuffer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit sendCloseFrame() and sendPing() used the main sendBuffer to build and send control frames. If the caller had an in-progress data frame in sendBuffer (obtained via getSendBuffer()), these methods would destroy it by calling sendBuffer.reset(). Switch both methods to use controlFrameBuffer, which already exists for exactly this purpose — sendCloseFrameEcho() and sendPongFrame() were already using it correctly. Add unit tests that verify the sendBuffer is preserved across sendCloseFrame() and sendPing() calls. Co-Authored-By: Claude Opus 4.6 --- .../cutlass/http/client/WebSocketClient.java | 16 +- .../http/client/WebSocketClientTest.java | 137 ++++++++++++++++++ 2 files changed, 145 insertions(+), 8 deletions(-) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClientTest.java diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java index 32d79dd..9c6ca38 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java @@ -312,12 +312,12 @@ public void sendBinary(long dataPtr, int length) { * Sends a close frame. */ public void sendCloseFrame(int code, String reason, int timeout) { - sendBuffer.reset(); - WebSocketSendBuffer.FrameInfo frame = sendBuffer.writeCloseFrame(code, reason); + controlFrameBuffer.reset(); + WebSocketSendBuffer.FrameInfo frame = controlFrameBuffer.writeCloseFrame(code, reason); try { - doSend(sendBuffer.getBufferPtr() + frame.offset, frame.length, timeout); + doSend(controlFrameBuffer.getBufferPtr() + frame.offset, frame.length, timeout); } finally { - sendBuffer.reset(); + controlFrameBuffer.reset(); } } @@ -344,10 +344,10 @@ public void sendFrame(WebSocketSendBuffer.FrameInfo frame) { */ public void sendPing(int timeout) { checkConnected(); - sendBuffer.reset(); - WebSocketSendBuffer.FrameInfo frame = sendBuffer.writePingFrame(); - doSend(sendBuffer.getBufferPtr() + frame.offset, frame.length, timeout); - sendBuffer.reset(); + controlFrameBuffer.reset(); + WebSocketSendBuffer.FrameInfo frame = controlFrameBuffer.writePingFrame(); + doSend(controlFrameBuffer.getBufferPtr() + frame.offset, frame.length, timeout); + controlFrameBuffer.reset(); } /** diff --git a/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClientTest.java b/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClientTest.java new file mode 100644 index 0000000..34dd0bd --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClientTest.java @@ -0,0 +1,137 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.http.client; + +import io.questdb.client.DefaultHttpClientConfiguration; +import io.questdb.client.cutlass.http.client.HttpClientException; +import io.questdb.client.cutlass.http.client.WebSocketClient; +import io.questdb.client.cutlass.http.client.WebSocketSendBuffer; +import io.questdb.client.network.PlainSocketFactory; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.lang.reflect.Field; + +public class WebSocketClientTest { + + @Test + public void testSendCloseFrameDoesNotClobberSendBuffer() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (StubWebSocketClient client = new StubWebSocketClient()) { + WebSocketSendBuffer sendBuffer = client.getSendBuffer(); + + // User starts building a data frame + sendBuffer.beginFrame(); + sendBuffer.putLong(0xDEADBEEFL); + int posBeforeClose = sendBuffer.getWritePos(); + Assert.assertTrue("sendBuffer should have data", posBeforeClose > 0); + + // sendCloseFrame() should use controlFrameBuffer, not sendBuffer + try { + client.sendCloseFrame(1000, null, 1000); + } catch (HttpClientException ignored) { + // Expected: doSend() fails because there's no real socket + } + + // Verify sendBuffer was NOT clobbered + Assert.assertEquals( + "sendCloseFrame() must not reset the main sendBuffer", + posBeforeClose, + sendBuffer.getWritePos() + ); + } + }); + } + + @Test + public void testSendPingDoesNotClobberSendBuffer() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (StubWebSocketClient client = new StubWebSocketClient()) { + // Set upgraded=true so checkConnected() passes + setField(client, "upgraded", true); + + WebSocketSendBuffer sendBuffer = client.getSendBuffer(); + + // User starts building a data frame + sendBuffer.beginFrame(); + sendBuffer.putLong(0xCAFEBABEL); + int posBeforePing = sendBuffer.getWritePos(); + Assert.assertTrue("sendBuffer should have data", posBeforePing > 0); + + // sendPing() should use controlFrameBuffer, not sendBuffer + try { + client.sendPing(1000); + } catch (HttpClientException ignored) { + // Expected: doSend() fails because there's no real socket + } + + // Verify sendBuffer was NOT clobbered + Assert.assertEquals( + "sendPing() must not reset the main sendBuffer", + posBeforePing, + sendBuffer.getWritePos() + ); + } + }); + } + + private static void setField(Object obj, String fieldName, Object value) throws Exception { + Class clazz = obj.getClass(); + while (clazz != null) { + try { + Field field = clazz.getDeclaredField(fieldName); + field.setAccessible(true); + field.set(obj, value); + return; + } catch (NoSuchFieldException e) { + clazz = clazz.getSuperclass(); + } + } + throw new NoSuchFieldException(fieldName); + } + + /** + * Minimal concrete WebSocketClient that throws on any I/O, + * allowing us to test buffer management without a real socket. + */ + private static class StubWebSocketClient extends WebSocketClient { + + StubWebSocketClient() { + super(DefaultHttpClientConfiguration.INSTANCE, PlainSocketFactory.INSTANCE); + } + + @Override + protected void ioWait(int timeout, int op) { + throw new HttpClientException("stub: no socket"); + } + + @Override + protected void setupIoWait() { + // no-op + } + } +} From c6278238a8e5f6358eb6796389458af6e6073c65 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Wed, 25 Feb 2026 15:22:06 +0100 Subject: [PATCH 54/89] Fix WebSocketSendBuffer.grow() integer overflow Numbers.ceilPow2(int) returns 2^30 for inputs between 2^30+1 and Integer.MAX_VALUE due to internal overflow handling. This caused grow() to allocate a buffer smaller than the required capacity. Fix by taking the max of ceilPow2's result and the raw required capacity before clamping to maxBufferSize. Co-Authored-By: Claude Opus 4.6 --- .../questdb/client/cutlass/http/client/WebSocketSendBuffer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java index 6bb280c..e4d7ba5 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java @@ -527,7 +527,7 @@ private void grow(long requiredCapacity) { .put(']'); } int newCapacity = Math.min( - Numbers.ceilPow2((int) requiredCapacity), + Math.max(Numbers.ceilPow2((int) requiredCapacity), (int) requiredCapacity), maxBufferSize ); bufPtr = Unsafe.realloc(bufPtr, bufCapacity, newCapacity, MemoryTag.NATIVE_DEFAULT); From 4f4f28b616d78ba6157f2ae036833a5814acb916 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Wed, 25 Feb 2026 15:27:48 +0100 Subject: [PATCH 55/89] Fix readFrom() leaking stale error messages When readFrom() parsed an error response where status != OK and the buffer was large enough to enter the outer branch (length > offset + 2), but msgLen was 0, the inner condition (msgLen > 0) failed without clearing errorMessage. On reused WebSocketResponse objects this left the error message from a previous parse visible to callers. Add an else branch to the inner if that sets errorMessage = null when msgLen is 0 or the message bytes are truncated. Add a regression test that parses an error-with-message followed by an error-with-empty-message on the same object and asserts errorMessage is null. Co-Authored-By: Claude Opus 4.6 --- .../cutlass/qwp/client/WebSocketResponse.java | 2 + .../qwp/client/WebSocketChannelTest.java | 39 +++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java index e1c1e6b..169f419 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java @@ -211,6 +211,8 @@ public boolean readFrom(long ptr, int length) { } errorMessage = new String(msgBytes, StandardCharsets.UTF_8); offset += msgLen; + } else { + errorMessage = null; } } else { errorMessage = null; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketChannelTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketChannelTest.java index 47fe527..4616a7a 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketChannelTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketChannelTest.java @@ -25,6 +25,7 @@ package io.questdb.client.test.cutlass.qwp.client; import io.questdb.client.cutlass.qwp.client.WebSocketChannel; +import io.questdb.client.cutlass.qwp.client.WebSocketResponse; import io.questdb.client.cutlass.qwp.websocket.WebSocketHandshake; import io.questdb.client.cutlass.qwp.websocket.WebSocketOpcode; import io.questdb.client.std.MemoryTag; @@ -135,6 +136,44 @@ public void testBinaryRoundTripSmallPayload() throws Exception { TestUtils.assertMemoryLeak(() -> assertBinaryRoundTrip(13)); } + @Test + public void testResponseReadFromEmptyErrorClearsStaleMessage() { + // First, parse an error response WITH an error message + WebSocketResponse response = new WebSocketResponse(); + WebSocketResponse errorWithMsg = WebSocketResponse.error(42, WebSocketResponse.STATUS_PARSE_ERROR, "bad input"); + int size1 = errorWithMsg.serializedSize(); + long ptr = Unsafe.malloc(size1, MemoryTag.NATIVE_DEFAULT); + try { + errorWithMsg.writeTo(ptr); + Assert.assertTrue(response.readFrom(ptr, size1)); + Assert.assertEquals("bad input", response.getErrorMessage()); + } finally { + Unsafe.free(ptr, size1, MemoryTag.NATIVE_DEFAULT); + } + + // Now, parse an error response with an EMPTY error message (msgLen=0) + // but with a buffer larger than MIN_ERROR_RESPONSE_SIZE. This triggers + // the path where the outer if (length > offset + 2) is true, but the + // inner if (msgLen > 0) is false, leaving errorMessage stale. + int size2 = WebSocketResponse.MIN_ERROR_RESPONSE_SIZE + 1; + ptr = Unsafe.malloc(size2, MemoryTag.NATIVE_DEFAULT); + try { + int offset = 0; + Unsafe.getUnsafe().putByte(ptr + offset, WebSocketResponse.STATUS_WRITE_ERROR); + offset += 1; + Unsafe.getUnsafe().putLong(ptr + offset, 99L); + offset += 8; + Unsafe.getUnsafe().putShort(ptr + offset, (short) 0); // msgLen = 0 + + Assert.assertTrue(response.readFrom(ptr, size2)); + Assert.assertEquals(WebSocketResponse.STATUS_WRITE_ERROR, response.getStatus()); + Assert.assertEquals(99L, response.getSequence()); + Assert.assertNull("errorMessage should be null for empty error message", response.getErrorMessage()); + } finally { + Unsafe.free(ptr, size2, MemoryTag.NATIVE_DEFAULT); + } + } + /** * Calls receiveFrame in a loop to handle the case where doReceiveFrame * needs multiple reads to assemble a complete frame (e.g. header and From c9849192ef80ffed49d28a863e053039bfa88aed Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Wed, 25 Feb 2026 15:44:12 +0100 Subject: [PATCH 56/89] Fix reset() to clear all table buffers QwpWebSocketSender.reset() only reset the current table buffer, leaving other tables' data intact and pendingRowCount nonzero. The Sender.reset() contract requires all pending state to be discarded. Now iterate every table buffer in the map, zero pendingRowCount and firstPendingRowTimeNanos, and clear the current-table and cached-column references. Co-Authored-By: Claude Opus 4.6 --- .../qwp/client/QwpWebSocketSender.java | 23 ++++- .../client/QwpWebSocketSenderResetTest.java | 96 +++++++++++++++++++ 2 files changed, 117 insertions(+), 2 deletions(-) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderResetTest.java diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 13b0206..382107a 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -710,6 +710,14 @@ public int getMaxSentSymbolId() { return maxSentSymbolId; } + /** + * Returns the number of pending rows not yet flushed. + * For testing. + */ + public int getPendingRowCount() { + return pendingRowCount; + } + /** * Registers a symbol in the global dictionary and returns its ID. * For use with fast-path column buffer access. @@ -866,9 +874,20 @@ public QwpWebSocketSender longColumn(CharSequence columnName, long value) { @Override public void reset() { checkNotClosed(); - if (currentTableBuffer != null) { - currentTableBuffer.reset(); + // Reset ALL table buffers, not just the current one + ObjList keys = tableBuffers.keys(); + for (int i = 0, n = keys.size(); i < n; i++) { + QwpTableBuffer buf = tableBuffers.get(keys.getQuick(i)); + if (buf != null) { + buf.reset(); + } } + pendingRowCount = 0; + firstPendingRowTimeNanos = 0; + currentTableBuffer = null; + currentTableName = null; + cachedTimestampColumn = null; + cachedTimestampNanosColumn = null; } /** diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderResetTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderResetTest.java new file mode 100644 index 0000000..ea1cd6c --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderResetTest.java @@ -0,0 +1,96 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.client.InFlightWindow; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.test.AbstractTest; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.time.temporal.ChronoUnit; + +/** + * Verifies that {@link QwpWebSocketSender#reset()} discards all pending state, + * not just the current table buffer. + */ +public class QwpWebSocketSenderResetTest extends AbstractTest { + + @Test + public void testResetClearsAllTableBuffersAndPendingRowCount() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Use high autoFlushRows to prevent auto-flush during the test + QwpWebSocketSender sender = QwpWebSocketSender.createForTesting( + "localhost", 0, 10_000, 10_000_000, 0, 1, 16 + ); + try { + // Bypass ensureConnected() — mark as connected, leave client null + setField(sender, "connected", true); + setField(sender, "inFlightWindow", new InFlightWindow(1, InFlightWindow.DEFAULT_TIMEOUT_MS)); + + // Buffer rows into two different tables via the fluent API + sender.table("t1") + .longColumn("x", 1) + .at(1, ChronoUnit.MICROS); + sender.table("t2") + .longColumn("y", 2) + .at(2, ChronoUnit.MICROS); + + // Verify data is buffered + QwpTableBuffer t1 = sender.getTableBuffer("t1"); + QwpTableBuffer t2 = sender.getTableBuffer("t2"); + Assert.assertEquals("t1 should have 1 row before reset", 1, t1.getRowCount()); + Assert.assertEquals("t2 should have 1 row before reset", 1, t2.getRowCount()); + Assert.assertEquals("pendingRowCount should be 2 before reset", 2, sender.getPendingRowCount()); + + // Select t1 as the current table + sender.table("t1"); + + // Call reset — per the Sender contract this should discard + // ALL pending state, not just the current table + sender.reset(); + + // Both table buffers should be cleared + Assert.assertEquals("t1 row count should be 0 after reset", 0, t1.getRowCount()); + Assert.assertEquals("t2 row count should be 0 after reset", 0, t2.getRowCount()); + + // Pending row count should be zeroed + Assert.assertEquals("pendingRowCount should be 0 after reset", 0, sender.getPendingRowCount()); + } finally { + setField(sender, "connected", false); + sender.close(); + } + }); + } + + private static void setField(Object target, String fieldName, Object value) throws Exception { + Field f = target.getClass().getDeclaredField(fieldName); + f.setAccessible(true); + f.set(target, value); + } +} From 93dce206cfcc1697bf3d4eb3dc10c784380ad2d8 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Wed, 25 Feb 2026 15:56:29 +0100 Subject: [PATCH 57/89] Fix ensureBits() 64-bit buffer overflow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ensureBits() loaded bytes into the 64-bit bitBuffer without checking whether all 8 bits of the incoming byte would actually fit. When bitsInBuffer exceeded 56, the left-shift lost high bits that overflowed position 63, and bitsInBuffer could grow past 64 — silently corrupting the buffer. Add a bitsInBuffer <= 56 guard to the while loop so we never attempt to load a byte when fewer than 8 free bit positions remain. Co-Authored-By: Claude Opus 4.6 --- .../io/questdb/client/cutlass/qwp/protocol/QwpBitReader.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitReader.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitReader.java index 944e4d9..c1e2ec7 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitReader.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitReader.java @@ -316,7 +316,7 @@ public void skipBits(int numBits) { * @return true if sufficient bits available, false otherwise */ private boolean ensureBits(int bitsNeeded) { - while (bitsInBuffer < bitsNeeded && currentAddress < endAddress) { + while (bitsInBuffer < bitsNeeded && bitsInBuffer <= 56 && currentAddress < endAddress) { byte b = Unsafe.getUnsafe().getByte(currentAddress++); bitBuffer |= (long) (b & 0xFF) << bitsInBuffer; bitsInBuffer += 8; From 1180216dd29c1869719cd13701653a3c2f97542a Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Wed, 25 Feb 2026 16:30:15 +0100 Subject: [PATCH 58/89] Add bounds check to NativeBufferWriter.skip() skip() advanced the position without calling ensureCapacity(), so a skip that exceeded the current buffer capacity would let subsequent writes corrupt native memory past the allocation. Add the missing ensureCapacity(bytes) call, matching the existing pattern in WebSocketSendBuffer.skip() and OffHeapAppendMemory.skip(). Add a regression test that skips past the initial capacity and verifies the buffer grows. Co-Authored-By: Claude Opus 4.6 --- .../cutlass/qwp/client/NativeBufferWriter.java | 1 + .../cutlass/qwp/client/NativeBufferWriterTest.java | 14 ++++++++++++++ 2 files changed, 15 insertions(+) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java index 3230b4c..19f0ae3 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java @@ -293,6 +293,7 @@ public void reset() { */ @Override public void skip(int bytes) { + ensureCapacity(bytes); position += bytes; } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java index 67a356d..c3b8991 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java @@ -75,6 +75,20 @@ public void testSkipAdvancesPosition() { } } + @Test + public void testSkipBeyondCapacityGrowsBuffer() { + try (NativeBufferWriter writer = new NativeBufferWriter(16)) { + // skip past the 16-byte buffer — must grow, not corrupt memory + writer.skip(32); + assertEquals(32, writer.getPosition()); + assertTrue(writer.getCapacity() >= 32); + // writing after the skip must also succeed + writer.putInt(0xCAFE); + assertEquals(36, writer.getPosition()); + assertEquals(0xCAFE, Unsafe.getUnsafe().getInt(writer.getBufferPtr() + 32)); + } + } + @Test public void testPutUtf8InvalidSurrogatePair() { try (NativeBufferWriter writer = new NativeBufferWriter(64)) { From c55d9820c9df69d00edf02d9436c31f7633eca2f Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Wed, 25 Feb 2026 17:04:32 +0100 Subject: [PATCH 59/89] Invalidate cached timestamp columns on flush flushPendingRows() and flushSync() reset table buffers but did not clear cachedTimestampColumn and cachedTimestampNanosColumn. If the table buffer's columns were ever recreated rather than just data- reset, the stale references would become dangling. Null both fields at the start of each flush method, consistent with what reset() and table() already do. Add QwpWebSocketSenderFlushCacheTest to verify the invariant for both micros and nanos timestamp paths. Co-Authored-By: Claude Opus 4.6 --- .../qwp/client/QwpWebSocketSender.java | 8 + .../QwpWebSocketSenderFlushCacheTest.java | 140 ++++++++++++++++++ 2 files changed, 148 insertions(+) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderFlushCacheTest.java diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 382107a..50167f6 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -1148,6 +1148,10 @@ private void flushPendingRows() { return; } + // Invalidate cached column references — table buffers will be reset below + cachedTimestampColumn = null; + cachedTimestampNanosColumn = null; + LOG.debug("Flushing pending rows [count={}, tables={}]", pendingRowCount, tableBuffers.size()); // Ensure activeBuffer is ready for writing @@ -1226,6 +1230,10 @@ private void flushSync() { return; } + // Invalidate cached column references — table buffers will be reset below + cachedTimestampColumn = null; + cachedTimestampNanosColumn = null; + LOG.debug("Sync flush [pendingRows={}, tables={}]", pendingRowCount, tableBuffers.size()); // Encode all table buffers that have data into a single message diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderFlushCacheTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderFlushCacheTest.java new file mode 100644 index 0000000..b8ef386 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderFlushCacheTest.java @@ -0,0 +1,140 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.test.AbstractTest; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.time.temporal.ChronoUnit; + +/** + * Verifies that {@link QwpWebSocketSender} invalidates its cached timestamp + * column references ({@code cachedTimestampColumn} and + * {@code cachedTimestampNanosColumn}) during flush operations. + *

    + * These cached references point into a {@code QwpTableBuffer} whose columns + * are reset by {@code flushSync()} / {@code flushPendingRows()}. If the cache + * is not cleared, subsequent rows may write through a stale reference. + *

    + * The test uses {@code autoFlushRows=1} so that every row triggers a flush + * inside {@code sendRow()}. The flush itself fails (no real connection), but + * the cache must be invalidated before the send is attempted. + * After the failed flush the test clears the table buffer, making any + * surviving stale reference point to a freed {@code ColumnBuffer}. A second + * row is then sent: if the cache was properly invalidated, a fresh column is + * created and the row is buffered normally; if stale, {@code addLong()} hits + * an NPE before {@code sendRow()} / {@code nextRow()}, so the row is never + * counted. + */ +public class QwpWebSocketSenderFlushCacheTest extends AbstractTest { + + @Test + public void testCachedTimestampColumnInvalidatedDuringFlush() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpWebSocketSender sender = QwpWebSocketSender.createForTesting( + "localhost", 0, 1, 10_000_000, 0, 1, 16 + ); + try { + setConnected(sender, true); + + // Row 1: caches cachedTimestampColumn, then auto-flush + // triggers and fails (no real connection). + try { + sender.table("t") + .longColumn("x", 1) + .at(1, ChronoUnit.MICROS); + } catch (Exception ignored) { + } + + // Clear the table buffer so a stale cached reference now + // points to a freed ColumnBuffer. + QwpTableBuffer tb = sender.getTableBuffer("t"); + tb.clear(); + + // Row 2: with the fix, atMicros() creates a fresh column + // and the row is buffered. Without, addLong() NPEs before + // sendRow()/nextRow() and the row is never counted. + try { + sender.table("t") + .longColumn("x", 2) + .at(2, ChronoUnit.MICROS); + } catch (Exception ignored) { + } + + Assert.assertEquals("row must be buffered when cache is properly invalidated", + 1, tb.getRowCount()); + } finally { + setConnected(sender, false); + sender.close(); + } + }); + } + + @Test + public void testCachedTimestampNanosColumnInvalidatedDuringFlush() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpWebSocketSender sender = QwpWebSocketSender.createForTesting( + "localhost", 0, 1, 10_000_000, 0, 1, 16 + ); + try { + setConnected(sender, true); + + try { + sender.table("t") + .longColumn("x", 1) + .at(1, ChronoUnit.NANOS); + } catch (Exception ignored) { + } + + QwpTableBuffer tb = sender.getTableBuffer("t"); + tb.clear(); + + try { + sender.table("t") + .longColumn("x", 2) + .at(2, ChronoUnit.NANOS); + } catch (Exception ignored) { + } + + Assert.assertEquals("row must be buffered when cache is properly invalidated", + 1, tb.getRowCount()); + } finally { + setConnected(sender, false); + sender.close(); + } + }); + } + + private static void setConnected(QwpWebSocketSender sender, boolean value) throws Exception { + Field f = QwpWebSocketSender.class.getDeclaredField("connected"); + f.setAccessible(true); + f.set(sender, value); + } +} From ecb595f39aa87d4af9ca01cd567ecb9e337ea3e4 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Wed, 25 Feb 2026 17:09:12 +0100 Subject: [PATCH 60/89] Fix putBlockOfBytes() long-to-int overflow putBlockOfBytes() accepted a long len parameter but cast it to int before calling ensureCapacity(). When len > Integer.MAX_VALUE, the cast wraps to a negative number, so ensureCapacity() skips the buffer grow, but copyMemory() still uses the original long len, causing a buffer overflow. Validate len fits in int range before casting in both NativeBufferWriter and WebSocketSendBuffer. Use the narrowed int value consistently for ensureCapacity(), copyMemory(), and position update. Co-Authored-By: Claude Opus 4.6 --- .../cutlass/http/client/WebSocketSendBuffer.java | 10 +++++++--- .../cutlass/qwp/client/NativeBufferWriter.java | 10 +++++++--- .../cutlass/qwp/client/NativeBufferWriterTest.java | 14 ++++++++++++++ 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java index e4d7ba5..afb0d05 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java @@ -249,9 +249,13 @@ public void putBlockOfBytes(long from, long len) { if (len <= 0) { return; } - ensureCapacity((int) len); - Vect.memcpy(bufPtr + writePos, from, len); - writePos += (int) len; + if (len > Integer.MAX_VALUE) { + throw new IllegalArgumentException("len exceeds int range: " + len); + } + int intLen = (int) len; + ensureCapacity(intLen); + Vect.memcpy(bufPtr + writePos, from, intLen); + writePos += intLen; } @Override diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java index 19f0ae3..5d8eb9e 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java @@ -138,9 +138,13 @@ public void patchInt(int offset, int value) { */ @Override public void putBlockOfBytes(long from, long len) { - ensureCapacity((int) len); - Unsafe.getUnsafe().copyMemory(from, bufferPtr + position, len); - position += (int) len; + if (len < 0 || len > Integer.MAX_VALUE) { + throw new IllegalArgumentException("len exceeds int range: " + len); + } + int intLen = (int) len; + ensureCapacity(intLen); + Unsafe.getUnsafe().copyMemory(from, bufferPtr + position, intLen); + position += intLen; } /** diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java index c3b8991..bfa2bc7 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java @@ -27,10 +27,12 @@ import io.questdb.client.cutlass.qwp.client.NativeBufferWriter; import io.questdb.client.cutlass.qwp.client.QwpBufferWriter; import io.questdb.client.std.Unsafe; +import org.junit.Assert; import org.junit.Test; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; public class NativeBufferWriterTest { @@ -89,6 +91,18 @@ public void testSkipBeyondCapacityGrowsBuffer() { } } + @Test + public void testPutBlockOfBytesRejectsLenExceedingIntMax() { + try (NativeBufferWriter writer = new NativeBufferWriter(16)) { + try { + writer.putBlockOfBytes(0, (long) Integer.MAX_VALUE + 1); + fail("expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { + Assert.assertTrue(e.getMessage().contains("len")); + } + } + } + @Test public void testPutUtf8InvalidSurrogatePair() { try (NativeBufferWriter writer = new NativeBufferWriter(64)) { From 542a667cd0ba199ca5cfc7da1aff6bdfc1087373 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Wed, 25 Feb 2026 17:14:37 +0100 Subject: [PATCH 61/89] Merge sender state tests into one class Merge QwpWebSocketSenderResetTest and QwpWebSocketSenderFlushCacheTest into a single QwpWebSocketSenderStateTest class. Both tested QwpWebSocketSender internal state management with the same reflection pattern and superclass. The merged class unifies the setField/ setConnected helpers into one setField method and keeps all three test methods in alphabetical order. Co-Authored-By: Claude Opus 4.6 --- .../client/QwpWebSocketSenderResetTest.java | 96 ------------------- ....java => QwpWebSocketSenderStateTest.java} | 87 ++++++++++++----- 2 files changed, 62 insertions(+), 121 deletions(-) delete mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderResetTest.java rename core/src/test/java/io/questdb/client/test/cutlass/qwp/client/{QwpWebSocketSenderFlushCacheTest.java => QwpWebSocketSenderStateTest.java} (56%) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderResetTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderResetTest.java deleted file mode 100644 index ea1cd6c..0000000 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderResetTest.java +++ /dev/null @@ -1,96 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.test.cutlass.qwp.client; - -import io.questdb.client.cutlass.qwp.client.InFlightWindow; -import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; -import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; -import io.questdb.client.test.AbstractTest; -import io.questdb.client.test.tools.TestUtils; -import org.junit.Assert; -import org.junit.Test; - -import java.lang.reflect.Field; -import java.time.temporal.ChronoUnit; - -/** - * Verifies that {@link QwpWebSocketSender#reset()} discards all pending state, - * not just the current table buffer. - */ -public class QwpWebSocketSenderResetTest extends AbstractTest { - - @Test - public void testResetClearsAllTableBuffersAndPendingRowCount() throws Exception { - TestUtils.assertMemoryLeak(() -> { - // Use high autoFlushRows to prevent auto-flush during the test - QwpWebSocketSender sender = QwpWebSocketSender.createForTesting( - "localhost", 0, 10_000, 10_000_000, 0, 1, 16 - ); - try { - // Bypass ensureConnected() — mark as connected, leave client null - setField(sender, "connected", true); - setField(sender, "inFlightWindow", new InFlightWindow(1, InFlightWindow.DEFAULT_TIMEOUT_MS)); - - // Buffer rows into two different tables via the fluent API - sender.table("t1") - .longColumn("x", 1) - .at(1, ChronoUnit.MICROS); - sender.table("t2") - .longColumn("y", 2) - .at(2, ChronoUnit.MICROS); - - // Verify data is buffered - QwpTableBuffer t1 = sender.getTableBuffer("t1"); - QwpTableBuffer t2 = sender.getTableBuffer("t2"); - Assert.assertEquals("t1 should have 1 row before reset", 1, t1.getRowCount()); - Assert.assertEquals("t2 should have 1 row before reset", 1, t2.getRowCount()); - Assert.assertEquals("pendingRowCount should be 2 before reset", 2, sender.getPendingRowCount()); - - // Select t1 as the current table - sender.table("t1"); - - // Call reset — per the Sender contract this should discard - // ALL pending state, not just the current table - sender.reset(); - - // Both table buffers should be cleared - Assert.assertEquals("t1 row count should be 0 after reset", 0, t1.getRowCount()); - Assert.assertEquals("t2 row count should be 0 after reset", 0, t2.getRowCount()); - - // Pending row count should be zeroed - Assert.assertEquals("pendingRowCount should be 0 after reset", 0, sender.getPendingRowCount()); - } finally { - setField(sender, "connected", false); - sender.close(); - } - }); - } - - private static void setField(Object target, String fieldName, Object value) throws Exception { - Field f = target.getClass().getDeclaredField(fieldName); - f.setAccessible(true); - f.set(target, value); - } -} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderFlushCacheTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderStateTest.java similarity index 56% rename from core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderFlushCacheTest.java rename to core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderStateTest.java index b8ef386..f67ef61 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderFlushCacheTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderStateTest.java @@ -24,6 +24,7 @@ package io.questdb.client.test.cutlass.qwp.client; +import io.questdb.client.cutlass.qwp.client.InFlightWindow; import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; import io.questdb.client.test.AbstractTest; @@ -35,25 +36,14 @@ import java.time.temporal.ChronoUnit; /** - * Verifies that {@link QwpWebSocketSender} invalidates its cached timestamp - * column references ({@code cachedTimestampColumn} and - * {@code cachedTimestampNanosColumn}) during flush operations. - *

    - * These cached references point into a {@code QwpTableBuffer} whose columns - * are reset by {@code flushSync()} / {@code flushPendingRows()}. If the cache - * is not cleared, subsequent rows may write through a stale reference. - *

    - * The test uses {@code autoFlushRows=1} so that every row triggers a flush - * inside {@code sendRow()}. The flush itself fails (no real connection), but - * the cache must be invalidated before the send is attempted. - * After the failed flush the test clears the table buffer, making any - * surviving stale reference point to a freed {@code ColumnBuffer}. A second - * row is then sent: if the cache was properly invalidated, a fresh column is - * created and the row is buffered normally; if stale, {@code addLong()} hits - * an NPE before {@code sendRow()} / {@code nextRow()}, so the row is never - * counted. + * Verifies {@link QwpWebSocketSender} internal state management: + *

      + *
    • {@code reset()} discards all pending state, not just the current table buffer.
      • + *
    • Cached timestamp column references are invalidated during flush operations, + * preventing stale writes through freed {@code ColumnBuffer} instances.
      • + *
    */ -public class QwpWebSocketSenderFlushCacheTest extends AbstractTest { +public class QwpWebSocketSenderStateTest extends AbstractTest { @Test public void testCachedTimestampColumnInvalidatedDuringFlush() throws Exception { @@ -62,7 +52,7 @@ public void testCachedTimestampColumnInvalidatedDuringFlush() throws Exception { "localhost", 0, 1, 10_000_000, 0, 1, 16 ); try { - setConnected(sender, true); + setField(sender, "connected", true); // Row 1: caches cachedTimestampColumn, then auto-flush // triggers and fails (no real connection). @@ -91,7 +81,7 @@ public void testCachedTimestampColumnInvalidatedDuringFlush() throws Exception { Assert.assertEquals("row must be buffered when cache is properly invalidated", 1, tb.getRowCount()); } finally { - setConnected(sender, false); + setField(sender, "connected", false); sender.close(); } }); @@ -104,7 +94,7 @@ public void testCachedTimestampNanosColumnInvalidatedDuringFlush() throws Except "localhost", 0, 1, 10_000_000, 0, 1, 16 ); try { - setConnected(sender, true); + setField(sender, "connected", true); try { sender.table("t") @@ -126,15 +116,62 @@ public void testCachedTimestampNanosColumnInvalidatedDuringFlush() throws Except Assert.assertEquals("row must be buffered when cache is properly invalidated", 1, tb.getRowCount()); } finally { - setConnected(sender, false); + setField(sender, "connected", false); sender.close(); } }); } - private static void setConnected(QwpWebSocketSender sender, boolean value) throws Exception { - Field f = QwpWebSocketSender.class.getDeclaredField("connected"); + @Test + public void testResetClearsAllTableBuffersAndPendingRowCount() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Use high autoFlushRows to prevent auto-flush during the test + QwpWebSocketSender sender = QwpWebSocketSender.createForTesting( + "localhost", 0, 10_000, 10_000_000, 0, 1, 16 + ); + try { + // Bypass ensureConnected() — mark as connected, leave client null + setField(sender, "connected", true); + setField(sender, "inFlightWindow", new InFlightWindow(1, InFlightWindow.DEFAULT_TIMEOUT_MS)); + + // Buffer rows into two different tables via the fluent API + sender.table("t1") + .longColumn("x", 1) + .at(1, ChronoUnit.MICROS); + sender.table("t2") + .longColumn("y", 2) + .at(2, ChronoUnit.MICROS); + + // Verify data is buffered + QwpTableBuffer t1 = sender.getTableBuffer("t1"); + QwpTableBuffer t2 = sender.getTableBuffer("t2"); + Assert.assertEquals("t1 should have 1 row before reset", 1, t1.getRowCount()); + Assert.assertEquals("t2 should have 1 row before reset", 1, t2.getRowCount()); + Assert.assertEquals("pendingRowCount should be 2 before reset", 2, sender.getPendingRowCount()); + + // Select t1 as the current table + sender.table("t1"); + + // Call reset — per the Sender contract this should discard + // ALL pending state, not just the current table + sender.reset(); + + // Both table buffers should be cleared + Assert.assertEquals("t1 row count should be 0 after reset", 0, t1.getRowCount()); + Assert.assertEquals("t2 row count should be 0 after reset", 0, t2.getRowCount()); + + // Pending row count should be zeroed + Assert.assertEquals("pendingRowCount should be 0 after reset", 0, sender.getPendingRowCount()); + } finally { + setField(sender, "connected", false); + sender.close(); + } + }); + } + + private static void setField(Object target, String fieldName, Object value) throws Exception { + Field f = target.getClass().getDeclaredField(fieldName); f.setAccessible(true); - f.set(sender, value); + f.set(target, value); } } From 243c03e460e2e562fdfb27be064b6cfb03e53f09 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 26 Feb 2026 09:53:53 +0100 Subject: [PATCH 62/89] Remove unused sendQueueCapacity parameter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit WebSocketSendQueue accepted a queueCapacity parameter in its constructor, validated it, and logged it, but never used it. The actual queue is a single volatile slot (pendingBuffer) by design — matching the double-buffering scheme where at most one sealed buffer is pending while the other is being filled. Remove the parameter from the entire chain: WebSocketSendQueue constructor, QwpWebSocketSender field and factory methods, Sender.LineSenderBuilder API, and all tests and benchmark clients that referenced it. Co-Authored-By: Claude Opus 4.6 --- .../main/java/io/questdb/client/Sender.java | 32 +-------- .../qwp/client/QwpWebSocketSender.java | 33 +++------- .../qwp/client/WebSocketSendQueue.java | 22 +++---- .../line/tcp/v4/QwpAllocationTestClient.java | 14 ++-- .../line/tcp/v4/StacBenchmarkClient.java | 14 ++-- .../LineSenderBuilderWebSocketTest.java | 65 +------------------ .../client/QwpWebSocketSenderStateTest.java | 6 +- 7 files changed, 33 insertions(+), 153 deletions(-) diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index 755bd78..705f3fe 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -537,7 +537,6 @@ final class LineSenderBuilder { private static final int DEFAULT_MAX_NAME_LEN = 127; private static final long DEFAULT_MAX_RETRY_NANOS = TimeUnit.SECONDS.toNanos(10); // keep sync with the contract of the configuration method private static final long DEFAULT_MIN_REQUEST_THROUGHPUT = 100 * 1024; // 100KB/s, keep in sync with the contract of the configuration method - private static final int DEFAULT_SEND_QUEUE_CAPACITY = 16; private static final int DEFAULT_TCP_PORT = 9009; private static final int DEFAULT_WEBSOCKET_PORT = 9000; private static final int DEFAULT_WS_AUTO_FLUSH_BYTES = 1024 * 1024; // 1MB @@ -596,7 +595,6 @@ public int getTimeout() { private int protocol = PARAMETER_NOT_SET_EXPLICITLY; private int protocolVersion = PARAMETER_NOT_SET_EXPLICITLY; private int retryTimeoutMillis = PARAMETER_NOT_SET_EXPLICITLY; - private int sendQueueCapacity = PARAMETER_NOT_SET_EXPLICITLY; private boolean shouldDestroyPrivKey; private boolean tlsEnabled; private TlsValidationMode tlsValidationMode; @@ -878,7 +876,6 @@ public Sender build() { ? DEFAULT_WS_AUTO_FLUSH_INTERVAL_NANOS : TimeUnit.MILLISECONDS.toNanos(autoFlushIntervalMillis); int actualInFlightWindowSize = inFlightWindowSize == PARAMETER_NOT_SET_EXPLICITLY ? DEFAULT_IN_FLIGHT_WINDOW_SIZE : inFlightWindowSize; - int actualSendQueueCapacity = sendQueueCapacity == PARAMETER_NOT_SET_EXPLICITLY ? DEFAULT_SEND_QUEUE_CAPACITY : sendQueueCapacity; if (asyncMode) { return QwpWebSocketSender.connectAsync( @@ -888,8 +885,7 @@ public Sender build() { actualAutoFlushRows, actualAutoFlushBytes, actualAutoFlushIntervalNanos, - actualInFlightWindowSize, - actualSendQueueCapacity + actualInFlightWindowSize ); } else { return QwpWebSocketSender.connect( @@ -1374,29 +1370,6 @@ public LineSenderBuilder retryTimeoutMillis(int retryTimeoutMillis) { return this; } - /** - * Set the capacity of the send queue for batches waiting to be sent. - *
    - * This is only used when communicating over WebSocket transport with async mode enabled. - *
    - * Default value is 16. - * - * @param capacity send queue capacity - * @return this instance for method chaining - */ - public LineSenderBuilder sendQueueCapacity(int capacity) { - if (this.sendQueueCapacity != PARAMETER_NOT_SET_EXPLICITLY) { - throw new LineSenderException("send queue capacity was already configured") - .put("[capacity=").put(this.sendQueueCapacity).put("]"); - } - if (capacity < 1) { - throw new LineSenderException("send queue capacity must be positive") - .put("[capacity=").put(capacity).put("]"); - } - this.sendQueueCapacity = capacity; - return this; - } - private static int getValue(CharSequence configurationString, int pos, StringSink sink, String name) { if ((pos = ConfStringParser.value(configurationString, pos, sink)) < 0) { throw new LineSenderException("invalid ").put(name).put(" [error=").put(sink).put("]"); @@ -1802,9 +1775,6 @@ private void validateParameters() { if (inFlightWindowSize != PARAMETER_NOT_SET_EXPLICITLY && !asyncMode) { throw new LineSenderException("in-flight window size requires async mode"); } - if (sendQueueCapacity != PARAMETER_NOT_SET_EXPLICITLY && !asyncMode) { - throw new LineSenderException("send queue capacity requires async mode"); - } } else { throw new LineSenderException("unsupported protocol ") .put("[protocol=").put(protocol).put("]"); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 50167f6..59e890a 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -109,7 +109,6 @@ public class QwpWebSocketSender implements Sender { public static final long DEFAULT_AUTO_FLUSH_INTERVAL_NANOS = 100_000_000L; // 100ms public static final int DEFAULT_AUTO_FLUSH_ROWS = 500; public static final int DEFAULT_IN_FLIGHT_WINDOW_SIZE = InFlightWindow.DEFAULT_WINDOW_SIZE; // 8 - public static final int DEFAULT_SEND_QUEUE_CAPACITY = WebSocketSendQueue.DEFAULT_QUEUE_CAPACITY; // 16 private static final int DEFAULT_BUFFER_SIZE = 8192; private static final int DEFAULT_MICROBATCH_BUFFER_SIZE = 1024 * 1024; // 1MB private static final Logger LOG = LoggerFactory.getLogger(QwpWebSocketSender.class); @@ -126,7 +125,6 @@ public class QwpWebSocketSender implements Sender { // Flow control configuration private final int inFlightWindowSize; private final int port; - private final int sendQueueCapacity; // Track schema hashes that have been sent to the server (for schema reference mode) // First time we send a schema: full schema. Subsequent times: 8-byte hash reference. // Combined key = schemaHash XOR (tableNameHash << 32) to include table name in lookup. @@ -173,8 +171,7 @@ private QwpWebSocketSender( int autoFlushRows, int autoFlushBytes, long autoFlushIntervalNanos, - int inFlightWindowSize, - int sendQueueCapacity + int inFlightWindowSize ) { this.host = host; this.port = port; @@ -189,7 +186,6 @@ private QwpWebSocketSender( this.autoFlushBytes = autoFlushBytes; this.autoFlushIntervalNanos = autoFlushIntervalNanos; this.inFlightWindowSize = inFlightWindowSize; - this.sendQueueCapacity = sendQueueCapacity; // Initialize global symbol dictionary for delta encoding this.globalSymbolDictionary = new GlobalSymbolDictionary(); @@ -247,7 +243,7 @@ public static QwpWebSocketSender connect(String host, int port, boolean tlsEnabl QwpWebSocketSender sender = new QwpWebSocketSender( host, port, tlsEnabled, DEFAULT_BUFFER_SIZE, autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, - 1, 1 // window=1 for sync behavior, queue=1 (not used) + 1 // window=1 for sync behavior ); sender.ensureConnected(); return sender; @@ -268,7 +264,7 @@ public static QwpWebSocketSender connectAsync(String host, int port, boolean tls int autoFlushRows, int autoFlushBytes, long autoFlushIntervalNanos) { return connectAsync(host, port, tlsEnabled, autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, - DEFAULT_IN_FLIGHT_WINDOW_SIZE, DEFAULT_SEND_QUEUE_CAPACITY); + DEFAULT_IN_FLIGHT_WINDOW_SIZE); } /** @@ -281,7 +277,6 @@ public static QwpWebSocketSender connectAsync(String host, int port, boolean tls * @param autoFlushBytes bytes per batch (0 = no limit) * @param autoFlushIntervalNanos age before flush in nanos (0 = no limit) * @param inFlightWindowSize max batches awaiting server ACK (default: 8) - * @param sendQueueCapacity max batches waiting to send (default: 16) * @return connected sender */ public static QwpWebSocketSender connectAsync( @@ -291,13 +286,12 @@ public static QwpWebSocketSender connectAsync( int autoFlushRows, int autoFlushBytes, long autoFlushIntervalNanos, - int inFlightWindowSize, - int sendQueueCapacity + int inFlightWindowSize ) { QwpWebSocketSender sender = new QwpWebSocketSender( host, port, tlsEnabled, DEFAULT_BUFFER_SIZE, autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, - inFlightWindowSize, sendQueueCapacity + inFlightWindowSize ); sender.ensureConnected(); return sender; @@ -331,7 +325,7 @@ public static QwpWebSocketSender create( QwpWebSocketSender sender = new QwpWebSocketSender( host, port, tlsEnabled, bufferSize, 0, 0, 0, - 1, 1 // window=1 for sync behavior + 1 // window=1 for sync behavior ); // TODO: Store auth credentials for connection sender.ensureConnected(); @@ -353,7 +347,7 @@ public static QwpWebSocketSender createForTesting(String host, int port, int inF return new QwpWebSocketSender( host, port, false, DEFAULT_BUFFER_SIZE, DEFAULT_AUTO_FLUSH_ROWS, DEFAULT_AUTO_FLUSH_BYTES, DEFAULT_AUTO_FLUSH_INTERVAL_NANOS, - inFlightWindowSize, DEFAULT_SEND_QUEUE_CAPACITY + inFlightWindowSize ); // Note: does NOT call ensureConnected() } @@ -367,17 +361,16 @@ public static QwpWebSocketSender createForTesting(String host, int port, int inF * @param autoFlushBytes bytes per batch (0 = no limit) * @param autoFlushIntervalNanos age before flush in nanos (0 = no limit) * @param inFlightWindowSize window size: 1 for sync behavior, >1 for async - * @param sendQueueCapacity max batches waiting to send * @return unconnected sender */ public static QwpWebSocketSender createForTesting( String host, int port, int autoFlushRows, int autoFlushBytes, long autoFlushIntervalNanos, - int inFlightWindowSize, int sendQueueCapacity) { + int inFlightWindowSize) { return new QwpWebSocketSender( host, port, false, DEFAULT_BUFFER_SIZE, autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, - inFlightWindowSize, sendQueueCapacity + inFlightWindowSize ); // Note: does NOT call ensureConnected() } @@ -730,13 +723,6 @@ public int getOrAddGlobalSymbol(String value) { return globalId; } - /** - * Returns the send queue capacity. - */ - public int getSendQueueCapacity() { - return sendQueueCapacity; - } - /** * Gets or creates a table buffer for direct access. * For high-throughput generators that want to bypass fluent API overhead. @@ -1119,7 +1105,6 @@ private void ensureConnected() { // The send queue handles both sending AND receiving (single I/O thread) if (inFlightWindowSize > 1) { sendQueue = new WebSocketSendQueue(client, inFlightWindow, - sendQueueCapacity, WebSocketSendQueue.DEFAULT_ENQUEUE_TIMEOUT_MS, WebSocketSendQueue.DEFAULT_SHUTDOWN_TIMEOUT_MS); } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketSendQueue.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketSendQueue.java index 9a8a8bb..f567d38 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketSendQueue.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketSendQueue.java @@ -42,29 +42,29 @@ *

    * This class manages a dedicated I/O thread that handles both: *

      - *
    • Sending batches from a bounded queue
      • + *
    • Sending batches via a single-slot handoff (volatile reference)
      • *
    • Receiving and processing server ACK responses
      • *
    + * The single-slot design matches the double-buffering scheme: at most one + * sealed buffer is pending while the other is being filled. * Using a single thread eliminates concurrency issues with the WebSocket channel. *

    * Thread safety: *

      - *
    • The send queue is thread-safe for concurrent access
      • + *
    • The pending slot is thread-safe for concurrent access
      • *
    • Only the I/O thread interacts with the WebSocket channel
      • *
    • Buffer state transitions ensure safe hand-over
      • *
    *

    * Backpressure: *

      - *
    • When the queue is full, {@link #enqueue} blocks
      • + *
    • When the slot is occupied, {@link #enqueue} blocks
      • *
    • This propagates backpressure to the user thread
      • *
    */ public class WebSocketSendQueue implements QuietCloseable { public static final long DEFAULT_ENQUEUE_TIMEOUT_MS = 30_000; - // Default configuration - public static final int DEFAULT_QUEUE_CAPACITY = 16; public static final long DEFAULT_SHUTDOWN_TIMEOUT_MS = 10_000; private static final Logger LOG = LoggerFactory.getLogger(WebSocketSendQueue.class); // The WebSocket client for I/O (single-threaded access only) @@ -113,7 +113,7 @@ public class WebSocketSendQueue implements QuietCloseable { * @param client the WebSocket client for I/O */ public WebSocketSendQueue(WebSocketClient client) { - this(client, null, DEFAULT_QUEUE_CAPACITY, DEFAULT_ENQUEUE_TIMEOUT_MS, DEFAULT_SHUTDOWN_TIMEOUT_MS); + this(client, null, DEFAULT_ENQUEUE_TIMEOUT_MS, DEFAULT_SHUTDOWN_TIMEOUT_MS); } /** @@ -123,7 +123,7 @@ public WebSocketSendQueue(WebSocketClient client) { * @param inFlightWindow the window to track sent batches awaiting ACK (may be null) */ public WebSocketSendQueue(WebSocketClient client, @Nullable InFlightWindow inFlightWindow) { - this(client, inFlightWindow, DEFAULT_QUEUE_CAPACITY, DEFAULT_ENQUEUE_TIMEOUT_MS, DEFAULT_SHUTDOWN_TIMEOUT_MS); + this(client, inFlightWindow, DEFAULT_ENQUEUE_TIMEOUT_MS, DEFAULT_SHUTDOWN_TIMEOUT_MS); } /** @@ -131,18 +131,14 @@ public WebSocketSendQueue(WebSocketClient client, @Nullable InFlightWindow inFli * * @param client the WebSocket client for I/O * @param inFlightWindow the window to track sent batches awaiting ACK (may be null) - * @param queueCapacity maximum number of pending batches * @param enqueueTimeoutMs timeout for enqueue operations (ms) * @param shutdownTimeoutMs timeout for graceful shutdown (ms) */ public WebSocketSendQueue(WebSocketClient client, @Nullable InFlightWindow inFlightWindow, - int queueCapacity, long enqueueTimeoutMs, long shutdownTimeoutMs) { + long enqueueTimeoutMs, long shutdownTimeoutMs) { if (client == null) { throw new IllegalArgumentException("client cannot be null"); } - if (queueCapacity <= 0) { - throw new IllegalArgumentException("queueCapacity must be positive"); - } this.client = client; this.inFlightWindow = inFlightWindow; @@ -157,7 +153,7 @@ public WebSocketSendQueue(WebSocketClient client, @Nullable InFlightWindow inFli this.ioThread.setDaemon(true); this.ioThread.start(); - LOG.info("WebSocket I/O thread started [capacity={}]", queueCapacity); + LOG.info("WebSocket I/O thread started"); } /** diff --git a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java index 88715be..21f47b0 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java @@ -73,7 +73,6 @@ public class QwpAllocationTestClient { private static final int DEFAULT_IN_FLIGHT_WINDOW = 0; // 0 = use protocol default (8) private static final int DEFAULT_REPORT_INTERVAL = 1_000_000; private static final int DEFAULT_ROWS = 80_000_000; - private static final int DEFAULT_SEND_QUEUE = 0; // 0 = use protocol default (16) private static final int DEFAULT_WARMUP_ROWS = 100_000; private static final String PROTOCOL_ILP_HTTP = "ilp-http"; // Protocol modes @@ -100,7 +99,6 @@ public static void main(String[] args) { int flushBytes = DEFAULT_FLUSH_BYTES; long flushIntervalMs = DEFAULT_FLUSH_INTERVAL_MS; int inFlightWindow = DEFAULT_IN_FLIGHT_WINDOW; - int sendQueue = DEFAULT_SEND_QUEUE; int warmupRows = DEFAULT_WARMUP_ROWS; int reportInterval = DEFAULT_REPORT_INTERVAL; @@ -124,8 +122,6 @@ public static void main(String[] args) { flushIntervalMs = Long.parseLong(arg.substring("--flush-interval-ms=".length())); } else if (arg.startsWith("--in-flight-window=")) { inFlightWindow = Integer.parseInt(arg.substring("--in-flight-window=".length())); - } else if (arg.startsWith("--send-queue=")) { - sendQueue = Integer.parseInt(arg.substring("--send-queue=".length())); } else if (arg.startsWith("--warmup=")) { warmupRows = Integer.parseInt(arg.substring("--warmup=".length())); } else if (arg.startsWith("--report=")) { @@ -157,14 +153,13 @@ public static void main(String[] args) { System.out.println("Flush bytes: " + (flushBytes == 0 ? "(default)" : String.format("%,d", flushBytes))); System.out.println("Flush interval: " + (flushIntervalMs == 0 ? "(default)" : flushIntervalMs + " ms")); System.out.println("In-flight window: " + (inFlightWindow == 0 ? "(default: 8)" : inFlightWindow)); - System.out.println("Send queue: " + (sendQueue == 0 ? "(default: 16)" : sendQueue)); System.out.println("Warmup rows: " + String.format("%,d", warmupRows)); System.out.println("Report interval: " + String.format("%,d", reportInterval)); System.out.println(); try { runTest(protocol, host, port, totalRows, batchSize, flushBytes, flushIntervalMs, - inFlightWindow, sendQueue, warmupRows, reportInterval); + inFlightWindow, warmupRows, reportInterval); } catch (Exception e) { System.err.println("Error: " + e.getMessage()); e.printStackTrace(System.err); @@ -174,7 +169,7 @@ public static void main(String[] args) { private static Sender createSender(String protocol, String host, int port, int batchSize, int flushBytes, long flushIntervalMs, - int inFlightWindow, int sendQueue) { + int inFlightWindow) { switch (protocol) { case PROTOCOL_ILP_TCP: return Sender.builder(Sender.Transport.TCP) @@ -196,7 +191,6 @@ private static Sender createSender(String protocol, String host, int port, if (flushBytes > 0) b.autoFlushBytes(flushBytes); if (flushIntervalMs > 0) b.autoFlushIntervalMillis((int) flushIntervalMs); if (inFlightWindow > 0) b.inFlightWindowSize(inFlightWindow); - if (sendQueue > 0) b.sendQueueCapacity(sendQueue); return b.build(); default: throw new IllegalArgumentException("Unknown protocol: " + protocol + @@ -260,12 +254,12 @@ private static void printUsage() { private static void runTest(String protocol, String host, int port, int totalRows, int batchSize, int flushBytes, long flushIntervalMs, - int inFlightWindow, int sendQueue, + int inFlightWindow, int warmupRows, int reportInterval) throws IOException { System.out.println("Connecting to " + host + ":" + port + "..."); try (Sender sender = createSender(protocol, host, port, batchSize, flushBytes, flushIntervalMs, - inFlightWindow, sendQueue)) { + inFlightWindow)) { System.out.println("Connected! Protocol: " + protocol); System.out.println(); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/StacBenchmarkClient.java b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/StacBenchmarkClient.java index bed59a3..289643c 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/StacBenchmarkClient.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/StacBenchmarkClient.java @@ -68,7 +68,6 @@ public class StacBenchmarkClient { private static final int DEFAULT_IN_FLIGHT_WINDOW = 0; private static final int DEFAULT_REPORT_INTERVAL = 1_000_000; private static final int DEFAULT_ROWS = 80_000_000; - private static final int DEFAULT_SEND_QUEUE = 0; private static final String DEFAULT_TABLE = "q"; private static final int DEFAULT_WARMUP_ROWS = 100_000; // Estimated row size for throughput calculation: @@ -103,7 +102,6 @@ public static void main(String[] args) { int flushBytes = DEFAULT_FLUSH_BYTES; long flushIntervalMs = DEFAULT_FLUSH_INTERVAL_MS; int inFlightWindow = DEFAULT_IN_FLIGHT_WINDOW; - int sendQueue = DEFAULT_SEND_QUEUE; int warmupRows = DEFAULT_WARMUP_ROWS; int reportInterval = DEFAULT_REPORT_INTERVAL; String table = DEFAULT_TABLE; @@ -128,8 +126,6 @@ public static void main(String[] args) { flushIntervalMs = Long.parseLong(arg.substring("--flush-interval-ms=".length())); } else if (arg.startsWith("--in-flight-window=")) { inFlightWindow = Integer.parseInt(arg.substring("--in-flight-window=".length())); - } else if (arg.startsWith("--send-queue=")) { - sendQueue = Integer.parseInt(arg.substring("--send-queue=".length())); } else if (arg.startsWith("--warmup=")) { warmupRows = Integer.parseInt(arg.substring("--warmup=".length())); } else if (arg.startsWith("--report=")) { @@ -160,7 +156,6 @@ public static void main(String[] args) { System.out.println("Flush bytes: " + (flushBytes == 0 ? "(default)" : String.format("%,d", flushBytes))); System.out.println("Flush interval: " + (flushIntervalMs == 0 ? "(default)" : flushIntervalMs + " ms")); System.out.println("In-flight window: " + (inFlightWindow == 0 ? "(default: 8)" : inFlightWindow)); - System.out.println("Send queue: " + (sendQueue == 0 ? "(default: 16)" : sendQueue)); System.out.println("Warmup rows: " + String.format("%,d", warmupRows)); System.out.println("Report interval: " + String.format("%,d", reportInterval)); System.out.println("Symbols: " + String.format("%,d", SYMBOL_COUNT) + " unique 4-letter tickers"); @@ -168,7 +163,7 @@ public static void main(String[] args) { try { runTest(protocol, host, port, table, totalRows, batchSize, flushBytes, flushIntervalMs, - inFlightWindow, sendQueue, warmupRows, reportInterval); + inFlightWindow, warmupRows, reportInterval); } catch (Exception e) { System.err.println("Error: " + e.getMessage()); e.printStackTrace(); @@ -178,7 +173,7 @@ public static void main(String[] args) { private static Sender createSender(String protocol, String host, int port, int batchSize, int flushBytes, long flushIntervalMs, - int inFlightWindow, int sendQueue) { + int inFlightWindow) { switch (protocol) { case PROTOCOL_ILP_TCP: return Sender.builder(Sender.Transport.TCP) @@ -200,7 +195,6 @@ private static Sender createSender(String protocol, String host, int port, if (flushBytes > 0) b.autoFlushBytes(flushBytes); if (flushIntervalMs > 0) b.autoFlushIntervalMillis((int) flushIntervalMs); if (inFlightWindow > 0) b.inFlightWindowSize(inFlightWindow); - if (sendQueue > 0) b.sendQueueCapacity(sendQueue); return b.build(); default: throw new IllegalArgumentException("Unknown protocol: " + protocol + @@ -288,12 +282,12 @@ private static void printUsage() { private static void runTest(String protocol, String host, int port, String table, int totalRows, int batchSize, int flushBytes, long flushIntervalMs, - int inFlightWindow, int sendQueue, + int inFlightWindow, int warmupRows, int reportInterval) throws IOException { System.out.println("Connecting to " + host + ":" + port + "..."); try (Sender sender = createSender(protocol, host, port, batchSize, flushBytes, flushIntervalMs, - inFlightWindow, sendQueue)) { + inFlightWindow)) { System.out.println("Connected! Protocol: " + protocol); System.out.println(); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java index ecb2bb7..0eeccd1 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java @@ -107,8 +107,7 @@ public void testAsyncModeWithAllOptions() { .autoFlushRows(500) .autoFlushBytes(512 * 1024) .autoFlushIntervalMillis(50) - .inFlightWindowSize(8) - .sendQueueCapacity(16); + .inFlightWindowSize(8); Assert.assertNotNull(builder); } @@ -319,8 +318,7 @@ public void testFullAsyncConfiguration() { .autoFlushRows(1000) .autoFlushBytes(1024 * 1024) .autoFlushIntervalMillis(100) - .inFlightWindowSize(16) - .sendQueueCapacity(32); + .inFlightWindowSize(16); Assert.assertNotNull(builder); } @@ -333,8 +331,7 @@ public void testFullAsyncConfigurationWithTls() { .asyncMode(true) .autoFlushRows(1000) .autoFlushBytes(1024 * 1024) - .inFlightWindowSize(16) - .sendQueueCapacity(32); + .inFlightWindowSize(16); Assert.assertNotNull(builder); } @@ -524,52 +521,6 @@ public void testRetryTimeout_mayNotApply() { Assert.assertNotNull(builder); } - @Test - public void testSendQueueCapacityDoubleSet_fails() { - assertThrows("already configured", - () -> Sender.builder(Sender.Transport.WEBSOCKET) - .address(LOCALHOST) - .asyncMode(true) - .sendQueueCapacity(16) - .sendQueueCapacity(32)); - } - - @Test - public void testSendQueueCapacityNegative_fails() { - assertThrows("must be positive", - () -> Sender.builder(Sender.Transport.WEBSOCKET) - .address(LOCALHOST) - .asyncMode(true) - .sendQueueCapacity(-1)); - } - - @Test - public void testSendQueueCapacityZero_fails() { - assertThrows("must be positive", - () -> Sender.builder(Sender.Transport.WEBSOCKET) - .address(LOCALHOST) - .asyncMode(true) - .sendQueueCapacity(0)); - } - - @Test - public void testSendQueueCapacity_withAsyncMode() { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) - .address(LOCALHOST) - .asyncMode(true) - .sendQueueCapacity(32); - Assert.assertNotNull(builder); - } - - @Test - public void testSendQueueCapacity_withoutAsyncMode_fails() { - assertThrowsAny( - Sender.builder(Sender.Transport.WEBSOCKET) - .address(LOCALHOST) - .sendQueueCapacity(32), - "requires async mode"); - } - @Test public void testSyncModeAutoFlushDefaults() throws Exception { // Regression test: sync-mode connect() must not hardcode autoFlush to 0. @@ -606,16 +557,6 @@ public void testSyncModeDoesNotAllowInFlightWindowSize() { "requires async mode"); } - @Test - public void testSyncModeDoesNotAllowSendQueueCapacity() { - assertThrowsAny( - Sender.builder(Sender.Transport.WEBSOCKET) - .address(LOCALHOST) - .asyncMode(false) - .sendQueueCapacity(32), - "requires async mode"); - } - @Test public void testSyncModeIsDefault() { Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderStateTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderStateTest.java index f67ef61..4be668c 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderStateTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderStateTest.java @@ -49,7 +49,7 @@ public class QwpWebSocketSenderStateTest extends AbstractTest { public void testCachedTimestampColumnInvalidatedDuringFlush() throws Exception { TestUtils.assertMemoryLeak(() -> { QwpWebSocketSender sender = QwpWebSocketSender.createForTesting( - "localhost", 0, 1, 10_000_000, 0, 1, 16 + "localhost", 0, 1, 10_000_000, 0, 1 ); try { setField(sender, "connected", true); @@ -91,7 +91,7 @@ public void testCachedTimestampColumnInvalidatedDuringFlush() throws Exception { public void testCachedTimestampNanosColumnInvalidatedDuringFlush() throws Exception { TestUtils.assertMemoryLeak(() -> { QwpWebSocketSender sender = QwpWebSocketSender.createForTesting( - "localhost", 0, 1, 10_000_000, 0, 1, 16 + "localhost", 0, 1, 10_000_000, 0, 1 ); try { setField(sender, "connected", true); @@ -127,7 +127,7 @@ public void testResetClearsAllTableBuffersAndPendingRowCount() throws Exception TestUtils.assertMemoryLeak(() -> { // Use high autoFlushRows to prevent auto-flush during the test QwpWebSocketSender sender = QwpWebSocketSender.createForTesting( - "localhost", 0, 10_000, 10_000_000, 0, 1, 16 + "localhost", 0, 10_000, 10_000_000, 0, 1 ); try { // Bypass ensureConnected() — mark as connected, leave client null From c58b7cd90ef9bd93aeff05da4ea4d720c42d6d25 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 26 Feb 2026 10:10:34 +0100 Subject: [PATCH 63/89] Clarify wire vs buffer type size javadoc Rename elementSize() to elementSizeInBuffer() in QwpTableBuffer to make explicit that it returns the in-memory buffer stride, not the wire-format encoding size. Update javadoc on both elementSizeInBuffer() and QwpConstants.getFixedTypeSize() to document the distinction: getFixedTypeSize() returns wire sizes (0 for bit-packed BOOLEAN, -1 for variable-width GEOHASH), while elementSizeInBuffer() returns the off-heap buffer stride (1 for BOOLEAN, 8 for GEOHASH). Co-Authored-By: Claude Opus 4.6 --- .../cutlass/qwp/protocol/QwpConstants.java | 9 +++++++-- .../cutlass/qwp/protocol/QwpTableBuffer.java | 18 ++++++++++++++---- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java index 54d6fdd..a6142c9 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java @@ -295,10 +295,15 @@ private QwpConstants() { } /** - * Returns the size in bytes for fixed-width types. + * Returns the per-value size in bytes as encoded on the wire. BOOLEAN returns 0 + * because it is bit-packed (1 bit per value). GEOHASH returns -1 because it uses + * variable-width encoding (varint precision + ceil(precision/8) bytes per value). + *

    + * This is distinct from the in-memory buffer stride used by the client's + * {@code QwpTableBuffer.elementSizeInBuffer()}. * * @param typeCode the column type code (without nullable flag) - * @return size in bytes, or -1 for variable-width types + * @return size in bytes, 0 for bit-packed (BOOLEAN), or -1 for variable-width types */ public static int getFixedTypeSize(byte typeCode) { int code = typeCode & TYPE_MASK; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index 119bbbc..5fd3b47 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -257,10 +257,20 @@ public void reset() { } /** - * Returns the element size in bytes for a fixed-width column type. - * Returns 0 for variable-width types (string, arrays). + * Returns the in-memory buffer element stride in bytes. This is the size used + * to store each value in the client's off-heap {@link OffHeapAppendMemory} buffer. + * This is different from element size on the wire. + *

    + * For example, BOOLEAN is stored as 1 byte per value here (for easy indexed access) + * but bit-packed on the wire; GEOHASH is stored as 8-byte longs here but uses + * variable-width encoding on the wire. + *

    + * Returns 0 for variable-width types (string, arrays) that do not use a fixed-stride + * data buffer. + * + * @see QwpConstants#getFixedTypeSize(byte) for wire-format sizes */ - static int elementSize(byte type) { + static int elementSizeInBuffer(byte type) { switch (type) { case TYPE_BOOLEAN: case TYPE_BYTE: @@ -420,7 +430,7 @@ public ColumnBuffer(String name, byte type, boolean nullable) { this.name = name; this.type = type; this.nullable = nullable; - this.elemSize = elementSize(type); + this.elemSize = elementSizeInBuffer(type); this.size = 0; this.valueCount = 0; this.hasNulls = false; From 408c88967219a12bf7cb94684aca9ae0b389cc49 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 26 Feb 2026 11:07:36 +0100 Subject: [PATCH 64/89] Round out GEOHASH support --- .../qwp/client/QwpWebSocketEncoder.java | 30 +++++++ .../cutlass/qwp/protocol/QwpTableBuffer.java | 79 +++++++++++-------- 2 files changed, 75 insertions(+), 34 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java index 0ec22b2..9ee86b5 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java @@ -203,6 +203,9 @@ private void encodeColumn(QwpTableBuffer.ColumnBuffer col, QwpColumnDef colDef, case TYPE_DATE: buffer.putBlockOfBytes(dataAddr, (long) valueCount * 8); break; + case TYPE_GEOHASH: + writeGeoHashColumn(dataAddr, valueCount, col.getGeoHashPrecision()); + break; case TYPE_STRING: case TYPE_VARCHAR: writeStringColumn(col, valueCount); @@ -280,6 +283,9 @@ private void encodeColumnWithGlobalSymbols(QwpTableBuffer.ColumnBuffer col, QwpC case TYPE_DATE: buffer.putBlockOfBytes(dataAddr, (long) valueCount * 8); break; + case TYPE_GEOHASH: + writeGeoHashColumn(dataAddr, valueCount, col.getGeoHashPrecision()); + break; case TYPE_STRING: case TYPE_VARCHAR: writeStringColumn(col, valueCount); @@ -416,6 +422,30 @@ private void writeDecimal64Column(byte scale, long addr, int count) { } } + /** + * Writes a GeoHash column in variable-width wire format. + *

    + * Wire format: [precision varint] [packed values: ceil(precision/8) bytes each] + * Values are stored as 8-byte longs in the off-heap buffer but only the + * lower ceil(precision/8) bytes are written to the wire. + */ + private void writeGeoHashColumn(long addr, int count, int precision) { + if (precision < 1) { + // All values are null: use minimum valid precision. + // The decoder will skip all values via the null bitmap, + // so the precision only needs to be structurally valid. + precision = 1; + } + buffer.putVarint(precision); + int valueSize = (precision + 7) / 8; + for (int i = 0; i < count; i++) { + long value = Unsafe.getUnsafe().getLong(addr + (long) i * 8); + for (int b = 0; b < valueSize; b++) { + buffer.putByte((byte) (value >>> (b * 8))); + } + } + } + private void writeDoubleArrayColumn(QwpTableBuffer.ColumnBuffer col, int count) { byte[] dims = col.getArrayDims(); int[] shapes = col.getArrayShapes(); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index 5fd3b47..b4dc34a 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -411,6 +411,8 @@ public static class ColumnBuffer implements QuietCloseable { // Decimal storage private byte decimalScale = -1; private double[] doubleArrayData; + // GeoHash precision (number of bits, 1-60) + private int geohashPrecision = -1; private boolean hasNulls; private long[] longArrayData; private int maxGlobalSymbolId = -1; @@ -632,6 +634,29 @@ public void addFloat(float value) { size++; } + /** + * Adds a geohash value with the given precision. + * + * @param value the geohash value (bit-packed) + * @param precision number of bits (1-60) + */ + public void addGeoHash(long value, int precision) { + if (precision < 1 || precision > 60) { + throw new LineSenderException("invalid GeoHash precision: " + precision + " (must be 1-60)"); + } + if (geohashPrecision == -1) { + geohashPrecision = precision; + } else if (geohashPrecision != precision) { + throw new LineSenderException( + "GeoHash precision mismatch: column has " + geohashPrecision + " bits, got " + precision + ); + } + ensureNullBitmapForNonNull(); + dataBuffer.putLong(value); + valueCount++; + size++; + } + public void addInt(int value) { ensureNullBitmapForNonNull(); dataBuffer.putInt(value); @@ -754,6 +779,13 @@ public void addNull() { if (nullable) { ensureNullCapacity(size + 1); markNull(size); + // GEOHASH uses dense wire format: all rows (including nulls) + // occupy space in the values array. Write a placeholder value + // so the data buffer stays aligned with the row index. + if (type == TYPE_GEOHASH) { + dataBuffer.putLong(0L); + valueCount++; + } size++; } else { // For non-nullable columns, store a sentinel/default value @@ -928,18 +960,10 @@ public void close() { } } - public int getArrayDataOffset() { - return arrayDataOffset; - } - public byte[] getArrayDims() { return arrayDims; } - public int getArrayShapeOffset() { - return arrayShapeOffset; - } - public int[] getArrayShapes() { return arrayShapes; } @@ -974,6 +998,10 @@ public double[] getDoubleArrayData() { return doubleArrayData; } + public int getGeoHashPrecision() { + return geohashPrecision; + } + public long[] getLongArrayData() { return longArrayData; } @@ -1021,10 +1049,6 @@ public String[] getSymbolDictionary() { return dict; } - public int getSymbolDictionarySize() { - return symbolList == null ? 0 : symbolList.size(); - } - public byte getType() { return type; } @@ -1033,10 +1057,6 @@ public int getValueCount() { return valueCount; } - public boolean hasNulls() { - return hasNulls; - } - public boolean isNull(int index) { if (nullBufPtr == 0) { return false; @@ -1074,6 +1094,7 @@ public void reset() { arrayShapeOffset = 0; arrayDataOffset = 0; decimalScale = -1; + geohashPrecision = -1; } public void truncateTo(int newSize) { @@ -1153,6 +1174,7 @@ private void allocateStorage(byte type) { dataBuffer = new OffHeapAppendMemory(32); break; case TYPE_INT: + case TYPE_FLOAT: dataBuffer = new OffHeapAppendMemory(64); break; case TYPE_GEOHASH: @@ -1160,13 +1182,16 @@ private void allocateStorage(byte type) { case TYPE_TIMESTAMP: case TYPE_TIMESTAMP_NANOS: case TYPE_DATE: + case TYPE_DECIMAL64: + case TYPE_DOUBLE: dataBuffer = new OffHeapAppendMemory(128); break; - case TYPE_FLOAT: - dataBuffer = new OffHeapAppendMemory(64); + case TYPE_DECIMAL128: + dataBuffer = new OffHeapAppendMemory(256); break; - case TYPE_DOUBLE: - dataBuffer = new OffHeapAppendMemory(128); + case TYPE_LONG256: + case TYPE_DECIMAL256: + dataBuffer = new OffHeapAppendMemory(512); break; case TYPE_STRING: case TYPE_VARCHAR: @@ -1180,25 +1205,11 @@ private void allocateStorage(byte type) { symbolList = new ObjList<>(); break; case TYPE_UUID: - dataBuffer = new OffHeapAppendMemory(256); - break; - case TYPE_LONG256: - dataBuffer = new OffHeapAppendMemory(512); - break; case TYPE_DOUBLE_ARRAY: case TYPE_LONG_ARRAY: arrayDims = new byte[16]; arrayCapture = new ArrayCapture(); break; - case TYPE_DECIMAL64: - dataBuffer = new OffHeapAppendMemory(128); - break; - case TYPE_DECIMAL128: - dataBuffer = new OffHeapAppendMemory(256); - break; - case TYPE_DECIMAL256: - dataBuffer = new OffHeapAppendMemory(512); - break; } } From 11a416c8dc1e2abc37ea983c6d5dc9d591e822d6 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 26 Feb 2026 12:43:04 +0100 Subject: [PATCH 65/89] Deduplicate column encoding in QwpWebSocketEncoder encodeColumn() and encodeColumnWithGlobalSymbols() had nearly identical switch statements across 15+ type cases, differing only in the SYMBOL case. Merge them into a single encodeColumn() with a boolean useGlobalSymbols parameter. Similarly merge the duplicate encodeTable() and encodeTableWithGlobalSymbols() into one method. This removes ~100 lines of duplicated code. Co-Authored-By: Claude Opus 4.6 --- .../qwp/client/QwpWebSocketEncoder.java | 115 ++---------------- 1 file changed, 10 insertions(+), 105 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java index 9ee86b5..c62193d 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java @@ -79,7 +79,7 @@ public int encode(QwpTableBuffer tableBuffer, boolean useSchemaRef) { buffer.reset(); writeHeader(1, 0); int payloadStart = buffer.getPosition(); - encodeTable(tableBuffer, useSchemaRef); + encodeTable(tableBuffer, useSchemaRef, false); int payloadLength = buffer.getPosition() - payloadStart; buffer.patchInt(8, payloadLength); return buffer.getPosition(); @@ -105,7 +105,7 @@ public int encodeWithDeltaDict( String symbol = globalDict.getSymbol(id); buffer.putString(symbol); } - encodeTableWithGlobalSymbols(tableBuffer, useSchemaRef); + encodeTable(tableBuffer, useSchemaRef, true); int payloadLength = buffer.getPosition() - payloadStart; buffer.patchInt(8, payloadLength); flags = savedFlags; @@ -165,7 +165,7 @@ public void writeHeader(int tableCount, int payloadLength) { buffer.putInt(payloadLength); } - private void encodeColumn(QwpTableBuffer.ColumnBuffer col, QwpColumnDef colDef, int rowCount, boolean useGorilla) { + private void encodeColumn(QwpTableBuffer.ColumnBuffer col, QwpColumnDef colDef, int rowCount, boolean useGorilla, boolean useGlobalSymbols) { int valueCount = col.getValueCount(); long dataAddr = col.getDataAddress(); @@ -211,7 +211,11 @@ private void encodeColumn(QwpTableBuffer.ColumnBuffer col, QwpColumnDef colDef, writeStringColumn(col, valueCount); break; case TYPE_SYMBOL: - writeSymbolColumn(col, valueCount); + if (useGlobalSymbols) { + writeSymbolColumnWithGlobalIds(col, valueCount); + } else { + writeSymbolColumn(col, valueCount); + } break; case TYPE_UUID: // Stored as lo+hi contiguously, matching wire order @@ -241,106 +245,7 @@ private void encodeColumn(QwpTableBuffer.ColumnBuffer col, QwpColumnDef colDef, } } - private void encodeColumnWithGlobalSymbols(QwpTableBuffer.ColumnBuffer col, QwpColumnDef colDef, int rowCount, boolean useGorilla) { - int valueCount = col.getValueCount(); - - if (colDef.isNullable()) { - writeNullBitmap(col, rowCount); - } - - if (col.getType() == TYPE_SYMBOL) { - writeSymbolColumnWithGlobalIds(col, valueCount); - } else { - // Delegate to standard encoding for all other types - long dataAddr = col.getDataAddress(); - switch (col.getType()) { - case TYPE_BOOLEAN: - writeBooleanColumn(dataAddr, valueCount); - break; - case TYPE_BYTE: - buffer.putBlockOfBytes(dataAddr, valueCount); - break; - case TYPE_SHORT: - case TYPE_CHAR: - buffer.putBlockOfBytes(dataAddr, (long) valueCount * 2); - break; - case TYPE_INT: - buffer.putBlockOfBytes(dataAddr, (long) valueCount * 4); - break; - case TYPE_LONG: - buffer.putBlockOfBytes(dataAddr, (long) valueCount * 8); - break; - case TYPE_FLOAT: - buffer.putBlockOfBytes(dataAddr, (long) valueCount * 4); - break; - case TYPE_DOUBLE: - buffer.putBlockOfBytes(dataAddr, (long) valueCount * 8); - break; - case TYPE_TIMESTAMP: - case TYPE_TIMESTAMP_NANOS: - writeTimestampColumn(dataAddr, valueCount, useGorilla); - break; - case TYPE_DATE: - buffer.putBlockOfBytes(dataAddr, (long) valueCount * 8); - break; - case TYPE_GEOHASH: - writeGeoHashColumn(dataAddr, valueCount, col.getGeoHashPrecision()); - break; - case TYPE_STRING: - case TYPE_VARCHAR: - writeStringColumn(col, valueCount); - break; - case TYPE_UUID: - buffer.putBlockOfBytes(dataAddr, (long) valueCount * 16); - break; - case TYPE_LONG256: - buffer.putBlockOfBytes(dataAddr, (long) valueCount * 32); - break; - case TYPE_DOUBLE_ARRAY: - writeDoubleArrayColumn(col, valueCount); - break; - case TYPE_LONG_ARRAY: - writeLongArrayColumn(col, valueCount); - break; - case TYPE_DECIMAL64: - writeDecimal64Column(col.getDecimalScale(), dataAddr, valueCount); - break; - case TYPE_DECIMAL128: - writeDecimal128Column(col.getDecimalScale(), dataAddr, valueCount); - break; - case TYPE_DECIMAL256: - writeDecimal256Column(col.getDecimalScale(), dataAddr, valueCount); - break; - default: - throw new LineSenderException("Unknown column type: " + col.getType()); - } - } - } - - private void encodeTable(QwpTableBuffer tableBuffer, boolean useSchemaRef) { - QwpColumnDef[] columnDefs = tableBuffer.getColumnDefs(); - int rowCount = tableBuffer.getRowCount(); - - if (useSchemaRef) { - writeTableHeaderWithSchemaRef( - tableBuffer.getTableName(), - rowCount, - tableBuffer.getSchemaHash(), - columnDefs.length - ); - } else { - writeTableHeaderWithSchema(tableBuffer.getTableName(), rowCount, columnDefs); - } - - boolean useGorilla = isGorillaEnabled(); - for (int i = 0; i < tableBuffer.getColumnCount(); i++) { - QwpTableBuffer.ColumnBuffer col = tableBuffer.getColumn(i); - QwpColumnDef colDef = columnDefs[i]; - encodeColumn(col, colDef, rowCount, useGorilla); - } - } - - private void encodeTableWithGlobalSymbols(QwpTableBuffer tableBuffer, boolean useSchemaRef) { + private void encodeTable(QwpTableBuffer tableBuffer, boolean useSchemaRef, boolean useGlobalSymbols) { QwpColumnDef[] columnDefs = tableBuffer.getColumnDefs(); int rowCount = tableBuffer.getRowCount(); @@ -359,7 +264,7 @@ private void encodeTableWithGlobalSymbols(QwpTableBuffer tableBuffer, boolean us for (int i = 0; i < tableBuffer.getColumnCount(); i++) { QwpTableBuffer.ColumnBuffer col = tableBuffer.getColumn(i); QwpColumnDef colDef = columnDefs[i]; - encodeColumnWithGlobalSymbols(col, colDef, rowCount, useGorilla); + encodeColumn(col, colDef, rowCount, useGorilla, useGlobalSymbols); } } From bde1a51913343fbb4123a867564a88cc7e7ef08a Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 26 Feb 2026 12:46:34 +0100 Subject: [PATCH 66/89] Delete WebSocketChannel and ResponseReader MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove WebSocketChannel, ResponseReader, and WebSocketChannelTest. These classes are dead code — the actual sender implementation (QwpWebSocketSender) uses WebSocketClient and WebSocketSendQueue instead. Co-Authored-By: Claude Opus 4.6 --- .../cutlass/qwp/client/ResponseReader.java | 226 ------ .../cutlass/qwp/client/WebSocketChannel.java | 661 ------------------ .../qwp/client/WebSocketChannelTest.java | 476 ------------- 3 files changed, 1363 deletions(-) delete mode 100644 core/src/main/java/io/questdb/client/cutlass/qwp/client/ResponseReader.java delete mode 100644 core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketChannel.java delete mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketChannelTest.java diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/ResponseReader.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/ResponseReader.java deleted file mode 100644 index 552e372..0000000 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/ResponseReader.java +++ /dev/null @@ -1,226 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.cutlass.qwp.client; - -import io.questdb.client.cutlass.line.LineSenderException; -import io.questdb.client.std.QuietCloseable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; - -/** - * Reads server responses from WebSocket channel and updates InFlightWindow. - *

    - * This class runs a dedicated thread that: - *

      - *
    • Reads WebSocket frames from the server
      • - *
    • Parses binary responses containing ACK/error status
      • - *
    • Updates the InFlightWindow with acknowledgments or failures
      • - *
    - *

    - * Thread safety: This class is thread-safe. The reader thread processes - * responses independently of the sender thread. - */ -public class ResponseReader implements QuietCloseable { - - private static final int DEFAULT_READ_TIMEOUT_MS = 100; - private static final long DEFAULT_SHUTDOWN_TIMEOUT_MS = 5_000; - private static final Logger LOG = LoggerFactory.getLogger(ResponseReader.class); - private final WebSocketChannel channel; - private final InFlightWindow inFlightWindow; - private final Thread readerThread; - private final WebSocketResponse response; - private final CountDownLatch shutdownLatch; - // Statistics - private final AtomicLong totalAcks = new AtomicLong(0); - private final AtomicLong totalErrors = new AtomicLong(0); - private volatile Throwable lastError; - // State - private volatile boolean running; - - /** - * Creates a new response reader. - * - * @param channel the WebSocket channel to read from - * @param inFlightWindow the window to update with acknowledgments - */ - public ResponseReader(WebSocketChannel channel, InFlightWindow inFlightWindow) { - if (channel == null) { - throw new IllegalArgumentException("channel cannot be null"); - } - if (inFlightWindow == null) { - throw new IllegalArgumentException("inFlightWindow cannot be null"); - } - - this.channel = channel; - this.inFlightWindow = inFlightWindow; - this.response = new WebSocketResponse(); - - this.running = true; - this.shutdownLatch = new CountDownLatch(1); - - // Start reader thread - this.readerThread = new Thread(this::readLoop, "questdb-websocket-response-reader"); - this.readerThread.setDaemon(true); - this.readerThread.start(); - - LOG.info("Response reader started"); - } - - @Override - public void close() { - if (!running) { - return; - } - - LOG.info("Closing response reader"); - - running = false; - - // Wait for reader thread to finish - try { - shutdownLatch.await(DEFAULT_SHUTDOWN_TIMEOUT_MS, TimeUnit.MILLISECONDS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - - LOG.info("Response reader closed [totalAcks={}, totalErrors={}]", totalAcks.get(), totalErrors.get()); - } - - /** - * Returns the last error that occurred, or null if no error. - */ - public Throwable getLastError() { - return lastError; - } - - /** - * Returns total successful acknowledgments received. - */ - public long getTotalAcks() { - return totalAcks.get(); - } - - /** - * Returns total error responses received. - */ - public long getTotalErrors() { - return totalErrors.get(); - } - - /** - * Returns true if the reader is still running. - */ - public boolean isRunning() { - return running; - } - - /** - * Main read loop that processes incoming WebSocket frames. - */ - private void readLoop() { - LOG.info("Read loop started"); - - try { - while (running && channel.isConnected()) { - try { - // Non-blocking read with short timeout - boolean received = channel.receiveFrame(new ResponseHandlerImpl(), DEFAULT_READ_TIMEOUT_MS); - if (!received) { - // No frame available, continue polling - continue; - } - } catch (LineSenderException e) { - if (running) { - LOG.error("Error reading response: {}", e.getMessage()); - lastError = e; - } - // Continue trying to read unless we're shutting down - } catch (Throwable t) { - if (running) { - LOG.error("Unexpected error in read loop: {}", String.valueOf(t)); - lastError = t; - } - break; - } - } - } finally { - shutdownLatch.countDown(); - LOG.info("Read loop stopped"); - } - } - - /** - * Handler for received WebSocket frames. - */ - private class ResponseHandlerImpl implements WebSocketChannel.ResponseHandler { - - @Override - public void onBinaryMessage(long payload, int length) { - if (length < WebSocketResponse.MIN_RESPONSE_SIZE) { - LOG.error("Response too short [length={}]", length); - return; - } - - // Parse response from binary payload - if (!response.readFrom(payload, length)) { - LOG.error("Failed to parse response"); - return; - } - - long sequence = response.getSequence(); - - if (response.isSuccess()) { - // Cumulative ACK - acknowledge all batches up to this sequence - int acked = inFlightWindow.acknowledgeUpTo(sequence); - if (acked > 0) { - totalAcks.addAndGet(acked); - LOG.debug("Cumulative ACK received [upTo={}, acked={}]", sequence, acked); - } else { - LOG.debug("ACK for already-acknowledged sequences [upTo={}]", sequence); - } - } else { - // Error - fail the batch - String errorMessage = response.getErrorMessage(); - LOG.error("Error response [seq={}, status={}, error={}]", sequence, response.getStatusName(), errorMessage); - - LineSenderException error = new LineSenderException( - "Server error for batch " + sequence + ": " + - response.getStatusName() + " - " + errorMessage); - inFlightWindow.fail(sequence, error); - totalErrors.incrementAndGet(); - } - } - - @Override - public void onClose(int code, String reason) { - LOG.info("WebSocket closed by server [code={}, reason={}]", code, reason); - running = false; - } - } -} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketChannel.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketChannel.java deleted file mode 100644 index d3965fa..0000000 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketChannel.java +++ /dev/null @@ -1,661 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.cutlass.qwp.client; - -import io.questdb.client.cutlass.line.LineSenderException; -import io.questdb.client.cutlass.qwp.websocket.WebSocketCloseCode; -import io.questdb.client.cutlass.qwp.websocket.WebSocketFrameParser; -import io.questdb.client.cutlass.qwp.websocket.WebSocketFrameWriter; -import io.questdb.client.cutlass.qwp.websocket.WebSocketHandshake; -import io.questdb.client.cutlass.qwp.websocket.WebSocketOpcode; -import io.questdb.client.std.MemoryTag; -import io.questdb.client.std.QuietCloseable; -import io.questdb.client.std.SecureRnd; -import io.questdb.client.std.Unsafe; - -import javax.net.SocketFactory; -import javax.net.ssl.SSLContext; -import javax.net.ssl.SSLSocketFactory; -import javax.net.ssl.TrustManager; -import javax.net.ssl.X509TrustManager; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.Socket; -import java.net.SocketTimeoutException; -import java.nio.charset.StandardCharsets; -import java.security.SecureRandom; -import java.security.cert.X509Certificate; -import java.util.Base64; - -/** - * WebSocket client channel for ILP v4 binary streaming. - *

    - * This class handles: - *

      - *
    • HTTP upgrade handshake to establish WebSocket connection
      • - *
    • Binary frame encoding with client-side masking (RFC 6455)
      • - *
    • Ping/pong for connection keepalive
      • - *
    • Close handshake
      • - *
    - *

    - * Thread safety: This class is NOT thread-safe. It should only be accessed - * from a single thread at a time. - */ -public class WebSocketChannel implements QuietCloseable { - - private static final int DEFAULT_BUFFER_SIZE = 65536; - private static final int MAX_FRAME_HEADER_SIZE = 14; // 2 + 8 + 4 (header + extended len + mask) - // Frame parser (reused) - private final WebSocketFrameParser frameParser; - // Temporary byte array for handshake (allocated once) - private final byte[] handshakeBuffer = new byte[4096]; - // Connection state - private final String host; - private final String path; - private final int port; - // Random for mask key generation (ChaCha20-based CSPRNG, RFC 6455 Section 5.3) - private final SecureRnd rnd; - private final boolean tlsEnabled; - private final boolean tlsValidationEnabled; - private boolean closed; - // Timeouts - private int connectTimeoutMs = 10_000; - // State - private boolean connected; - private InputStream in; - private OutputStream out; - private byte[] readTempBuffer; - private int readTimeoutMs = 30_000; - private int recvBufferPos; // Write position - // Pre-allocated receive buffer (native memory) - private long recvBufferPtr; - private int recvBufferReadPos; // Read position - private int recvBufferSize; - // Pre-allocated send buffer (native memory) - private long sendBufferPtr; - private int sendBufferSize; - // Socket I/O - private Socket socket; - // Separate temp buffers for read and write to avoid race conditions - // between send queue thread and response reader thread - private byte[] writeTempBuffer; - - public WebSocketChannel(String url, boolean tlsEnabled) { - this(url, tlsEnabled, true); - } - - public WebSocketChannel(String url, boolean tlsEnabled, boolean tlsValidationEnabled) { - // Parse URL: ws://host:port/path or wss://host:port/path - String remaining = url; - if (remaining.startsWith("wss://")) { - remaining = remaining.substring(6); - this.tlsEnabled = true; - } else if (remaining.startsWith("ws://")) { - remaining = remaining.substring(5); - this.tlsEnabled = tlsEnabled; - } else { - this.tlsEnabled = tlsEnabled; - } - - int slashIdx = remaining.indexOf('/'); - String hostPort; - if (slashIdx >= 0) { - hostPort = remaining.substring(0, slashIdx); - this.path = remaining.substring(slashIdx); - } else { - hostPort = remaining; - this.path = "/"; - } - - int colonIdx = hostPort.lastIndexOf(':'); - if (colonIdx >= 0) { - this.host = hostPort.substring(0, colonIdx); - this.port = Integer.parseInt(hostPort.substring(colonIdx + 1)); - } else { - this.host = hostPort; - this.port = this.tlsEnabled ? 443 : 80; - } - - this.tlsValidationEnabled = tlsValidationEnabled; - this.frameParser = new WebSocketFrameParser(); - this.rnd = new SecureRnd(); - - // Allocate native buffers - this.sendBufferSize = DEFAULT_BUFFER_SIZE; - this.sendBufferPtr = Unsafe.malloc(sendBufferSize, MemoryTag.NATIVE_DEFAULT); - - this.recvBufferSize = DEFAULT_BUFFER_SIZE; - this.recvBufferPtr = Unsafe.malloc(recvBufferSize, MemoryTag.NATIVE_DEFAULT); - this.recvBufferPos = 0; - this.recvBufferReadPos = 0; - - this.connected = false; - this.closed = false; - } - - /** - * Sends a close frame and closes the connection. - */ - @Override - public void close() { - if (closed) { - return; - } - closed = true; - - try { - if (connected) { - // Send close frame - sendCloseFrame(WebSocketCloseCode.NORMAL_CLOSURE, null); - } - } catch (Exception e) { - // Ignore errors during close - } - - closeQuietly(); - - // Free native memory - if (sendBufferPtr != 0) { - Unsafe.free(sendBufferPtr, sendBufferSize, MemoryTag.NATIVE_DEFAULT); - sendBufferPtr = 0; - } - if (recvBufferPtr != 0) { - Unsafe.free(recvBufferPtr, recvBufferSize, MemoryTag.NATIVE_DEFAULT); - recvBufferPtr = 0; - } - } - - /** - * Connects to the WebSocket server. - * Performs TCP connection and HTTP upgrade handshake. - */ - public void connect() { - if (connected) { - return; - } - if (closed) { - throw new LineSenderException("WebSocket channel is closed"); - } - - try { - // Create socket - SocketFactory socketFactory = tlsEnabled ? createSslSocketFactory() : SocketFactory.getDefault(); - socket = socketFactory.createSocket(); - socket.connect(new java.net.InetSocketAddress(host, port), connectTimeoutMs); - socket.setSoTimeout(readTimeoutMs); - socket.setTcpNoDelay(true); - - in = socket.getInputStream(); - out = socket.getOutputStream(); - - // Perform WebSocket handshake - performHandshake(); - - connected = true; - } catch (IOException e) { - closeQuietly(); - throw new LineSenderException("Failed to connect to WebSocket server: " + e.getMessage(), e); - } - } - - public boolean isConnected() { - return connected && !closed; - } - - /** - * Receives and processes incoming frames. - * Handles ping/pong automatically. - * - * @param handler callback for received binary messages (may be null) - * @param timeoutMs read timeout in milliseconds - * @return true if a frame was received, false on timeout - */ - public boolean receiveFrame(ResponseHandler handler, int timeoutMs) { - ensureConnected(); - try { - int oldTimeout = socket.getSoTimeout(); - socket.setSoTimeout(timeoutMs); - try { - return doReceiveFrame(handler); - } finally { - socket.setSoTimeout(oldTimeout); - } - } catch (SocketTimeoutException e) { - return false; - } catch (IOException e) { - throw new LineSenderException("Failed to receive WebSocket frame: " + e.getMessage(), e); - } - } - - /** - * Sends binary data as a WebSocket binary frame. - * The data is read from native memory at the given pointer. - * - * @param dataPtr pointer to the data - * @param length length of data in bytes - */ - public void sendBinary(long dataPtr, int length) { - ensureConnected(); - sendFrame(WebSocketOpcode.BINARY, dataPtr, length); - } - - /** - * Sends a ping frame. - */ - public void sendPing() { - ensureConnected(); - sendFrame(WebSocketOpcode.PING, 0, 0); - } - - /** - * Sets the connection timeout. - */ - public WebSocketChannel setConnectTimeout(int timeoutMs) { - this.connectTimeoutMs = timeoutMs; - return this; - } - - /** - * Sets the read timeout. - */ - public WebSocketChannel setReadTimeout(int timeoutMs) { - this.readTimeoutMs = timeoutMs; - return this; - } - - private void closeQuietly() { - connected = false; - if (socket != null) { - try { - socket.close(); - } catch (IOException e) { - // Ignore - } - socket = null; - } - in = null; - out = null; - } - - private SocketFactory createSslSocketFactory() { - try { - if (!tlsValidationEnabled) { - SSLContext sslContext = SSLContext.getInstance("TLS"); - sslContext.init(null, new TrustManager[]{new X509TrustManager() { - public void checkClientTrusted(X509Certificate[] certs, String t) { - } - - public void checkServerTrusted(X509Certificate[] certs, String t) { - } - - public X509Certificate[] getAcceptedIssuers() { - return null; - } - }}, new SecureRandom()); - return sslContext.getSocketFactory(); - } - return SSLSocketFactory.getDefault(); - } catch (Exception e) { - throw new LineSenderException("Failed to create SSL socket factory: " + e.getMessage(), e); - } - } - - private boolean doReceiveFrame(ResponseHandler handler) throws IOException { - // First, try to parse any data already in the buffer - // This handles the case where multiple frames arrived in a single TCP read - if (recvBufferPos > recvBufferReadPos) { - Boolean result = tryParseFrame(handler); - if (result != null) { - return result; - } - // result == null means we need more data, continue to read - } - - // Read more data into receive buffer - int bytesRead = readFromSocket(); - if (bytesRead <= 0) { - return false; - } - - // Try parsing again with the new data - Boolean result = tryParseFrame(handler); - return result != null && result; - } - - private void ensureConnected() { - if (closed) { - throw new LineSenderException("WebSocket channel is closed"); - } - if (!connected) { - throw new LineSenderException("WebSocket channel is not connected"); - } - } - - private void ensureSendBufferSize(int required) { - if (required > sendBufferSize) { - int newSize = Math.max(required, sendBufferSize * 2); - sendBufferPtr = Unsafe.realloc(sendBufferPtr, sendBufferSize, newSize, MemoryTag.NATIVE_DEFAULT); - sendBufferSize = newSize; - } - } - - private byte[] getReadTempBuffer(int minSize) { - if (readTempBuffer == null || readTempBuffer.length < minSize) { - readTempBuffer = new byte[Math.max(minSize, 8192)]; - } - return readTempBuffer; - } - - private byte[] getWriteTempBuffer(int minSize) { - if (writeTempBuffer == null || writeTempBuffer.length < minSize) { - writeTempBuffer = new byte[Math.max(minSize, 8192)]; - } - return writeTempBuffer; - } - - private void performHandshake() throws IOException { - // Generate random key (16 bytes, base64 encoded = 24 chars) - byte[] keyBytes = new byte[16]; - for (int i = 0; i < 16; i++) { - keyBytes[i] = (byte) rnd.nextInt(); - } - String key = Base64.getEncoder().encodeToString(keyBytes); - - // Build HTTP upgrade request - StringBuilder request = new StringBuilder(); - request.append("GET ").append(path).append(" HTTP/1.1\r\n"); - request.append("Host: ").append(host); - if ((tlsEnabled && port != 443) || (!tlsEnabled && port != 80)) { - request.append(":").append(port); - } - request.append("\r\n"); - request.append("Upgrade: websocket\r\n"); - request.append("Connection: Upgrade\r\n"); - request.append("Sec-WebSocket-Key: ").append(key).append("\r\n"); - request.append("Sec-WebSocket-Version: 13\r\n"); - request.append("\r\n"); - - // Send request - byte[] requestBytes = request.toString().getBytes(StandardCharsets.US_ASCII); - out.write(requestBytes); - out.flush(); - - // Read response - int responseLen = readHttpResponse(); - - // Parse response - String response = new String(handshakeBuffer, 0, responseLen, StandardCharsets.US_ASCII); - - // Check status line - if (!response.startsWith("HTTP/1.1 101")) { - throw new IOException("WebSocket handshake failed: " + response.split("\r\n")[0]); - } - - // Verify Sec-WebSocket-Accept - String expectedAccept = WebSocketHandshake.computeAcceptKey(key); - if (!response.contains("Sec-WebSocket-Accept: " + expectedAccept)) { - throw new IOException("Invalid Sec-WebSocket-Accept in handshake response"); - } - } - - private int readFromSocket() throws IOException { - // Ensure space in receive buffer - int available = recvBufferSize - recvBufferPos; - if (available < 1024) { - // Grow buffer - int newSize = recvBufferSize * 2; - recvBufferPtr = Unsafe.realloc(recvBufferPtr, recvBufferSize, newSize, MemoryTag.NATIVE_DEFAULT); - recvBufferSize = newSize; - available = recvBufferSize - recvBufferPos; - } - - // Read into temp array then copy to native buffer - // Use separate read buffer to avoid race with write thread - byte[] temp = getReadTempBuffer(available); - int bytesRead = in.read(temp, 0, available); - if (bytesRead > 0) { - Unsafe.getUnsafe().copyMemory(temp, Unsafe.BYTE_OFFSET, null, recvBufferPtr + recvBufferPos, bytesRead); - recvBufferPos += bytesRead; - } - return bytesRead; - } - - private int readHttpResponse() throws IOException { - int pos = 0; - int consecutiveCrLf = 0; - - while (pos < handshakeBuffer.length) { - int b = in.read(); - if (b < 0) { - throw new IOException("Connection closed during handshake"); - } - handshakeBuffer[pos++] = (byte) b; - - // Look for \r\n\r\n - if (b == '\r' || b == '\n') { - if ((consecutiveCrLf == 0 && b == '\r') || - (consecutiveCrLf == 1 && b == '\n') || - (consecutiveCrLf == 2 && b == '\r') || - (consecutiveCrLf == 3 && b == '\n')) { - consecutiveCrLf++; - if (consecutiveCrLf == 4) { - return pos; - } - } else { - consecutiveCrLf = (b == '\r') ? 1 : 0; - } - } else { - consecutiveCrLf = 0; - } - } - throw new IOException("HTTP response too large"); - } - - private void sendCloseFrame(int code, String reason) { - int maskKey = rnd.nextInt(); - - // Close payload: 2-byte code + optional reason - // Compute UTF-8 bytes upfront so payload length is correct - byte[] reasonBytes = (reason != null) ? reason.getBytes(StandardCharsets.UTF_8) : null; - int reasonLen = (reasonBytes != null) ? reasonBytes.length : 0; - int payloadLen = 2 + reasonLen; - - int headerSize = WebSocketFrameWriter.headerSize(payloadLen, true); - int frameSize = headerSize + payloadLen; - - ensureSendBufferSize(frameSize); - - // Write header - int headerWritten = WebSocketFrameWriter.writeHeader( - sendBufferPtr, true, WebSocketOpcode.CLOSE, payloadLen, maskKey); - - // Write close code (big-endian) - long payloadStart = sendBufferPtr + headerWritten; - Unsafe.getUnsafe().putByte(payloadStart, (byte) ((code >> 8) & 0xFF)); - Unsafe.getUnsafe().putByte(payloadStart + 1, (byte) (code & 0xFF)); - - // Write reason if present - if (reasonBytes != null) { - for (int i = 0; i < reasonBytes.length; i++) { - Unsafe.getUnsafe().putByte(payloadStart + 2 + i, reasonBytes[i]); - } - } - - // Mask payload - WebSocketFrameWriter.maskPayload(payloadStart, payloadLen, maskKey); - - try { - writeToSocket(sendBufferPtr, frameSize); - } catch (IOException e) { - // Ignore errors during close - } - } - - private void sendFrame(int opcode, long payloadPtr, int payloadLen) { - // Generate mask key - int maskKey = rnd.nextInt(); - - // Calculate required buffer size - int headerSize = WebSocketFrameWriter.headerSize(payloadLen, true); - int frameSize = headerSize + payloadLen; - - // Ensure buffer is large enough - ensureSendBufferSize(frameSize); - - // Write frame header with mask - int headerWritten = WebSocketFrameWriter.writeHeader( - sendBufferPtr, true, opcode, payloadLen, maskKey); - - // Copy payload to buffer after header - if (payloadLen > 0) { - Unsafe.getUnsafe().copyMemory(payloadPtr, sendBufferPtr + headerWritten, payloadLen); - // Mask the payload in place - WebSocketFrameWriter.maskPayload(sendBufferPtr + headerWritten, payloadLen, maskKey); - } - - // Send frame - try { - writeToSocket(sendBufferPtr, frameSize); - } catch (IOException e) { - throw new LineSenderException("Failed to send WebSocket frame: " + e.getMessage(), e); - } - } - - private void sendPongFrame(long pingPayloadPtr, int pingPayloadLen) { - int maskKey = rnd.nextInt(); - int headerSize = WebSocketFrameWriter.headerSize(pingPayloadLen, true); - int frameSize = headerSize + pingPayloadLen; - - ensureSendBufferSize(frameSize); - - int headerWritten = WebSocketFrameWriter.writeHeader( - sendBufferPtr, true, WebSocketOpcode.PONG, pingPayloadLen, maskKey); - - if (pingPayloadLen > 0) { - Unsafe.getUnsafe().copyMemory(pingPayloadPtr, sendBufferPtr + headerWritten, pingPayloadLen); - WebSocketFrameWriter.maskPayload(sendBufferPtr + headerWritten, pingPayloadLen, maskKey); - } - - try { - writeToSocket(sendBufferPtr, frameSize); - } catch (IOException e) { - // Ignore pong send errors - } - } - - /** - * Tries to parse a frame from the receive buffer. - * - * @return true if frame processed, false if error, null if need more data - */ - private Boolean tryParseFrame(ResponseHandler handler) throws IOException { - frameParser.reset(); - int consumed = frameParser.parse( - recvBufferPtr + recvBufferReadPos, - recvBufferPtr + recvBufferPos); - - if (frameParser.getState() == WebSocketFrameParser.STATE_NEED_MORE) { - return null; // Need more data - } - - if (frameParser.getState() == WebSocketFrameParser.STATE_ERROR) { - throw new IOException("WebSocket frame parse error: " + frameParser.getErrorCode()); - } - - if (frameParser.getState() == WebSocketFrameParser.STATE_COMPLETE) { - long payloadPtr = recvBufferPtr + recvBufferReadPos + frameParser.getHeaderSize(); - int payloadLen = (int) frameParser.getPayloadLength(); - - // Handle control frames - int opcode = frameParser.getOpcode(); - switch (opcode) { - case WebSocketOpcode.PING: - sendPongFrame(payloadPtr, payloadLen); - break; - case WebSocketOpcode.PONG: - // Ignore pong - break; - case WebSocketOpcode.CLOSE: - connected = false; - if (handler != null) { - int closeCode = 0; - if (payloadLen >= 2) { - closeCode = ((Unsafe.getUnsafe().getByte(payloadPtr) & 0xFF) << 8) - | (Unsafe.getUnsafe().getByte(payloadPtr + 1) & 0xFF); - } - handler.onClose(closeCode, null); - } - break; - case WebSocketOpcode.BINARY: - if (handler != null) { - handler.onBinaryMessage(payloadPtr, payloadLen); - } - break; - case WebSocketOpcode.TEXT: - // Ignore text frames for now - break; - } - - // Advance read position - recvBufferReadPos += consumed; - - // Compact buffer if needed - if (recvBufferReadPos > 0) { - int remaining = recvBufferPos - recvBufferReadPos; - if (remaining > 0) { - Unsafe.getUnsafe().copyMemory( - recvBufferPtr + recvBufferReadPos, - recvBufferPtr, - remaining); - } - recvBufferPos = remaining; - recvBufferReadPos = 0; - } - - return true; - } - - return false; - } - - private void writeToSocket(long ptr, int len) throws IOException { - // Copy to temp array for socket write (unavoidable with OutputStream) - // Use separate write buffer to avoid race with read thread - byte[] temp = getWriteTempBuffer(len); - Unsafe.getUnsafe().copyMemory(null, ptr, temp, Unsafe.BYTE_OFFSET, len); - out.write(temp, 0, len); - out.flush(); - } - - /** - * Callback interface for received WebSocket messages. - */ - public interface ResponseHandler { - void onBinaryMessage(long payload, int length); - - void onClose(int code, String reason); - } -} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketChannelTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketChannelTest.java deleted file mode 100644 index 4616a7a..0000000 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketChannelTest.java +++ /dev/null @@ -1,476 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.test.cutlass.qwp.client; - -import io.questdb.client.cutlass.qwp.client.WebSocketChannel; -import io.questdb.client.cutlass.qwp.client.WebSocketResponse; -import io.questdb.client.cutlass.qwp.websocket.WebSocketHandshake; -import io.questdb.client.cutlass.qwp.websocket.WebSocketOpcode; -import io.questdb.client.std.MemoryTag; -import io.questdb.client.std.Unsafe; -import io.questdb.client.test.AbstractTest; -import io.questdb.client.test.tools.TestUtils; -import org.junit.Assert; -import org.junit.Test; - -import java.io.BufferedOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.ServerSocket; -import java.net.Socket; -import java.nio.charset.StandardCharsets; -import java.util.concurrent.atomic.AtomicReference; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * Tests for WebSocketChannel's native-heap memory copy paths. - * Exercises writeToSocket (native to heap) and readFromSocket (heap to native) - * through a local echo server. - */ -public class WebSocketChannelTest extends AbstractTest { - - @Test - public void testBinaryRoundTripAllByteValues() throws Exception { - TestUtils.assertMemoryLeak(() -> { - int len = 256; - long sendPtr = Unsafe.malloc(len, MemoryTag.NATIVE_DEFAULT); - try { - for (int i = 0; i < len; i++) { - Unsafe.getUnsafe().putByte(sendPtr + i, (byte) i); - } - assertBinaryRoundTrip(sendPtr, len); - } finally { - Unsafe.free(sendPtr, len, MemoryTag.NATIVE_DEFAULT); - } - }); - } - - @Test - public void testBinaryRoundTripLargePayload() throws Exception { - TestUtils.assertMemoryLeak(() -> { - // Large payload that exercises bulk copyMemory across many cache lines. - // Kept under 32KB so the echo response arrives in a single TCP read - // on loopback (avoids a pre-existing bug in doReceiveFrame with - // partial frame assembly). - assertBinaryRoundTrip(30_000); - }); - } - - @Test - public void testBinaryRoundTripMediumPayload() throws Exception { - TestUtils.assertMemoryLeak(() -> assertBinaryRoundTrip(4096)); - } - - @Test - public void testBinaryRoundTripRepeatedFrames() throws Exception { - TestUtils.assertMemoryLeak(() -> { - int payloadLen = 1000; - int frameCount = 10; - long sendPtr = Unsafe.malloc(payloadLen, MemoryTag.NATIVE_DEFAULT); - try (EchoServer server = new EchoServer()) { - server.start(); - WebSocketChannel channel = new WebSocketChannel( - "localhost:" + server.getPort() + "/", false - ); - try { - channel.setConnectTimeout(5000); - channel.setReadTimeout(5000); - channel.connect(); - - for (int f = 0; f < frameCount; f++) { - for (int i = 0; i < payloadLen; i++) { - Unsafe.getUnsafe().putByte(sendPtr + i, (byte) (i + f)); - } - channel.sendBinary(sendPtr, payloadLen); - - ReceivedPayload received = new ReceivedPayload(); - boolean ok = receiveWithRetry(channel, received, 5000); - server.assertNoError(); - Assert.assertTrue("frame " + f + ": expected response", ok); - Assert.assertEquals("frame " + f + ": length", payloadLen, received.length); - - for (int i = 0; i < payloadLen; i++) { - Assert.assertEquals( - "frame " + f + " byte " + i, - (byte) (i + f), - Unsafe.getUnsafe().getByte(received.ptr + i) - ); - } - } - } finally { - channel.close(); - } - server.assertNoError(); - } finally { - Unsafe.free(sendPtr, payloadLen, MemoryTag.NATIVE_DEFAULT); - } - }); - } - - @Test - public void testBinaryRoundTripSmallPayload() throws Exception { - TestUtils.assertMemoryLeak(() -> assertBinaryRoundTrip(13)); - } - - @Test - public void testResponseReadFromEmptyErrorClearsStaleMessage() { - // First, parse an error response WITH an error message - WebSocketResponse response = new WebSocketResponse(); - WebSocketResponse errorWithMsg = WebSocketResponse.error(42, WebSocketResponse.STATUS_PARSE_ERROR, "bad input"); - int size1 = errorWithMsg.serializedSize(); - long ptr = Unsafe.malloc(size1, MemoryTag.NATIVE_DEFAULT); - try { - errorWithMsg.writeTo(ptr); - Assert.assertTrue(response.readFrom(ptr, size1)); - Assert.assertEquals("bad input", response.getErrorMessage()); - } finally { - Unsafe.free(ptr, size1, MemoryTag.NATIVE_DEFAULT); - } - - // Now, parse an error response with an EMPTY error message (msgLen=0) - // but with a buffer larger than MIN_ERROR_RESPONSE_SIZE. This triggers - // the path where the outer if (length > offset + 2) is true, but the - // inner if (msgLen > 0) is false, leaving errorMessage stale. - int size2 = WebSocketResponse.MIN_ERROR_RESPONSE_SIZE + 1; - ptr = Unsafe.malloc(size2, MemoryTag.NATIVE_DEFAULT); - try { - int offset = 0; - Unsafe.getUnsafe().putByte(ptr + offset, WebSocketResponse.STATUS_WRITE_ERROR); - offset += 1; - Unsafe.getUnsafe().putLong(ptr + offset, 99L); - offset += 8; - Unsafe.getUnsafe().putShort(ptr + offset, (short) 0); // msgLen = 0 - - Assert.assertTrue(response.readFrom(ptr, size2)); - Assert.assertEquals(WebSocketResponse.STATUS_WRITE_ERROR, response.getStatus()); - Assert.assertEquals(99L, response.getSequence()); - Assert.assertNull("errorMessage should be null for empty error message", response.getErrorMessage()); - } finally { - Unsafe.free(ptr, size2, MemoryTag.NATIVE_DEFAULT); - } - } - - /** - * Calls receiveFrame in a loop to handle the case where doReceiveFrame - * needs multiple reads to assemble a complete frame (e.g. header and - * payload arrive in separate TCP segments). - */ - private static boolean receiveWithRetry(WebSocketChannel channel, ReceivedPayload handler, int timeoutMs) { - long deadline = System.currentTimeMillis() + timeoutMs; - while (System.currentTimeMillis() < deadline) { - int remaining = (int) (deadline - System.currentTimeMillis()); - if (remaining <= 0) { - break; - } - if (channel.receiveFrame(handler, remaining)) { - return true; - } - } - return false; - } - - private void assertBinaryRoundTrip(int payloadLen) throws Exception { - long sendPtr = Unsafe.malloc(payloadLen, MemoryTag.NATIVE_DEFAULT); - try { - for (int i = 0; i < payloadLen; i++) { - Unsafe.getUnsafe().putByte(sendPtr + i, (byte) (i & 0xFF)); - } - assertBinaryRoundTrip(sendPtr, payloadLen); - } finally { - Unsafe.free(sendPtr, payloadLen, MemoryTag.NATIVE_DEFAULT); - } - } - - private void assertBinaryRoundTrip(long sendPtr, int payloadLen) throws Exception { - try (EchoServer server = new EchoServer()) { - server.start(); - WebSocketChannel channel = new WebSocketChannel( - "localhost:" + server.getPort() + "/", false - ); - try { - channel.setConnectTimeout(5000); - channel.setReadTimeout(5000); - channel.connect(); - - // Send exercises writeToSocket (native to heap via copyMemory) - channel.sendBinary(sendPtr, payloadLen); - - // Receive exercises readFromSocket (heap to native via copyMemory) - ReceivedPayload received = new ReceivedPayload(); - boolean ok = receiveWithRetry(channel, received, 5000); - - // Check server error before client assertions - server.assertNoError(); - Assert.assertTrue("expected a frame back from echo server", ok); - Assert.assertEquals("payload length mismatch", payloadLen, received.length); - - for (int i = 0; i < payloadLen; i++) { - byte expected = Unsafe.getUnsafe().getByte(sendPtr + i); - byte actual = Unsafe.getUnsafe().getByte(received.ptr + i); - Assert.assertEquals("byte mismatch at offset " + i, expected, actual); - } - } finally { - channel.close(); - } - server.assertNoError(); - } - } - - /** - * Minimal WebSocket echo server. Accepts one connection, completes the - * HTTP upgrade handshake, then echoes every binary frame back unmasked. - * All echo writes use a single byte array to avoid TCP fragmentation. - */ - private static class EchoServer implements AutoCloseable { - private static final Pattern KEY_PATTERN = - Pattern.compile("Sec-WebSocket-Key:\\s*(.+?)\\r\\n"); - private final AtomicReference error = new AtomicReference<>(); - private final ServerSocket serverSocket; - private Thread thread; - - EchoServer() throws IOException { - serverSocket = new ServerSocket(0); - } - - @Override - public void close() throws Exception { - serverSocket.close(); - if (thread != null) { - thread.join(5000); - } - } - - private void completeHandshake(InputStream in, OutputStream out) throws IOException { - byte[] buf = new byte[4096]; - int pos = 0; - - while (pos < buf.length) { - int b = in.read(); - if (b < 0) { - throw new IOException("connection closed during handshake"); - } - buf[pos++] = (byte) b; - if (pos >= 4 - && buf[pos - 4] == '\r' && buf[pos - 3] == '\n' - && buf[pos - 2] == '\r' && buf[pos - 1] == '\n') { - break; - } - } - - String request = new String(buf, 0, pos, StandardCharsets.US_ASCII); - Matcher m = KEY_PATTERN.matcher(request); - if (!m.find()) { - throw new IOException("no Sec-WebSocket-Key in request:\n" + request); - } - String clientKey = m.group(1).trim(); - String acceptKey = WebSocketHandshake.computeAcceptKey(clientKey); - - String response = "HTTP/1.1 101 Switching Protocols\r\n" - + "Upgrade: websocket\r\n" - + "Connection: Upgrade\r\n" - + "Sec-WebSocket-Accept: " + acceptKey + "\r\n" - + "\r\n"; - out.write(response.getBytes(StandardCharsets.US_ASCII)); - out.flush(); - } - - private void echoFrames(InputStream in, OutputStream out) throws IOException { - byte[] readBuf = new byte[256 * 1024]; - - while (true) { - int pos = 0; - while (pos < 2) { - int n = in.read(readBuf, pos, readBuf.length - pos); - if (n < 0) { - return; - } - pos += n; - } - - int byte0 = readBuf[0] & 0xFF; - int byte1 = readBuf[1] & 0xFF; - int opcode = byte0 & 0x0F; - boolean masked = (byte1 & 0x80) != 0; - int lengthField = byte1 & 0x7F; - - int headerSize = 2; - long payloadLength; - if (lengthField <= 125) { - payloadLength = lengthField; - } else if (lengthField == 126) { - while (pos < 4) { - int n = in.read(readBuf, pos, readBuf.length - pos); - if (n < 0) return; - pos += n; - } - payloadLength = ((readBuf[2] & 0xFF) << 8) | (readBuf[3] & 0xFF); - headerSize = 4; - } else { - while (pos < 10) { - int n = in.read(readBuf, pos, readBuf.length - pos); - if (n < 0) return; - pos += n; - } - payloadLength = 0; - for (int i = 0; i < 8; i++) { - payloadLength = (payloadLength << 8) | (readBuf[2 + i] & 0xFF); - } - headerSize = 10; - } - - if (masked) { - headerSize += 4; - } - - int totalFrameSize = (int) (headerSize + payloadLength); - - if (totalFrameSize > readBuf.length) { - byte[] newBuf = new byte[totalFrameSize]; - System.arraycopy(readBuf, 0, newBuf, 0, pos); - readBuf = newBuf; - } - - while (pos < totalFrameSize) { - int n = in.read(readBuf, pos, totalFrameSize - pos); - if (n < 0) return; - pos += n; - } - - if (opcode == WebSocketOpcode.CLOSE) { - return; - } - - if (opcode != WebSocketOpcode.BINARY && opcode != WebSocketOpcode.TEXT) { - continue; - } - - // Unmask payload in place - if (masked) { - int maskKeyOffset = headerSize - 4; - byte m0 = readBuf[maskKeyOffset]; - byte m1 = readBuf[maskKeyOffset + 1]; - byte m2 = readBuf[maskKeyOffset + 2]; - byte m3 = readBuf[maskKeyOffset + 3]; - for (int i = 0; i < (int) payloadLength; i++) { - switch (i & 3) { - case 0: - readBuf[headerSize + i] ^= m0; - break; - case 1: - readBuf[headerSize + i] ^= m1; - break; - case 2: - readBuf[headerSize + i] ^= m2; - break; - case 3: - readBuf[headerSize + i] ^= m3; - break; - } - } - } - - // Build complete unmasked response frame in a single array - byte[] responseHeader; - if (payloadLength <= 125) { - responseHeader = new byte[]{ - (byte) (0x80 | opcode), - (byte) payloadLength - }; - } else if (payloadLength <= 65535) { - responseHeader = new byte[]{ - (byte) (0x80 | opcode), - 126, - (byte) ((payloadLength >> 8) & 0xFF), - (byte) (payloadLength & 0xFF) - }; - } else { - responseHeader = new byte[10]; - responseHeader[0] = (byte) (0x80 | opcode); - responseHeader[1] = 127; - for (int i = 0; i < 8; i++) { - responseHeader[2 + i] = (byte) ((payloadLength >> (56 - i * 8)) & 0xFF); - } - } - - // Single write: header + payload together via BufferedOutputStream - out.write(responseHeader); - out.write(readBuf, headerSize, (int) payloadLength); - out.flush(); - } - } - - private void run() { - try (Socket client = serverSocket.accept()) { - client.setSoTimeout(10_000); - client.setTcpNoDelay(true); - InputStream in = client.getInputStream(); - OutputStream out = new BufferedOutputStream(client.getOutputStream()); - - completeHandshake(in, out); - echoFrames(in, out); - } catch (IOException e) { - if (!serverSocket.isClosed()) { - error.set(e); - } - } catch (Throwable t) { - error.set(t); - } - } - - void assertNoError() { - Throwable t = error.get(); - if (t != null) { - throw new AssertionError("echo server error", t); - } - } - - int getPort() { - return serverSocket.getLocalPort(); - } - - void start() { - thread = new Thread(this::run, "ws-echo-server"); - thread.setDaemon(true); - thread.start(); - } - } - - private static class ReceivedPayload implements WebSocketChannel.ResponseHandler { - int length; - long ptr; - - @Override - public void onBinaryMessage(long payload, int length) { - this.ptr = payload; - this.length = length; - } - - @Override - public void onClose(int code, String reason) { - } - } -} From f304d314643828c770a1fe09068c039b6f5251d6 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 26 Feb 2026 13:04:10 +0100 Subject: [PATCH 67/89] Detect decimal overflow on rescale in QwpTableBuffer addDecimal128 and addDecimal64 rescale values via a Decimal256 temporary, but only read the lower 128 or 64 bits back. If the rescaled value overflows into the upper bits, the data is silently truncated. Add fitsInStorageSizePow2() checks after rescaling and throw LineSenderException when the result no longer fits in the target storage size. Add tests for both Decimal128 and Decimal64 overflow paths. Co-Authored-By: Claude Opus 4.6 --- .../cutlass/qwp/protocol/QwpTableBuffer.java | 8 ++++ .../qwp/protocol/QwpTableBufferTest.java | 41 +++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index b4dc34a..4ba4f0d 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -471,6 +471,10 @@ public void addDecimal128(Decimal128 value) { rescaleTemp.ofRaw(value.getHigh(), value.getLow()); rescaleTemp.setScale(value.getScale()); rescaleTemp.rescale(decimalScale); + if (!rescaleTemp.fitsInStorageSizePow2(4)) { + throw new LineSenderException("Decimal128 overflow: rescaling from scale " + + value.getScale() + " to " + decimalScale + " exceeds 128-bit capacity"); + } dataBuffer.putLong(rescaleTemp.getLh()); dataBuffer.putLong(rescaleTemp.getLl()); valueCount++; @@ -518,6 +522,10 @@ public void addDecimal64(Decimal64 value) { rescaleTemp.ofRaw(value.getValue()); rescaleTemp.setScale(value.getScale()); rescaleTemp.rescale(decimalScale); + if (!rescaleTemp.fitsInStorageSizePow2(3)) { + throw new LineSenderException("Decimal64 overflow: rescaling from scale " + + value.getScale() + " to " + decimalScale + " exceeds 64-bit capacity"); + } dataBuffer.putLong(rescaleTemp.getLl()); } else { dataBuffer.putLong(value.getValue()); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java index 1c294d0..2ed4762 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java @@ -24,16 +24,57 @@ package io.questdb.client.test.cutlass.qwp.protocol; +import io.questdb.client.cutlass.line.LineSenderException; import io.questdb.client.cutlass.line.array.DoubleArray; import io.questdb.client.cutlass.qwp.protocol.QwpConstants; import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.std.Decimal128; +import io.questdb.client.std.Decimal64; import org.junit.Test; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; public class QwpTableBufferTest { + @Test + public void testAddDecimal128RescaleOverflow() { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("d", QwpConstants.TYPE_DECIMAL128, true); + // First row sets decimalScale = 10 + col.addDecimal128(Decimal128.fromLong(1, 10)); + table.nextRow(); + // Second row at scale 0 with a large value — rescaling to scale 10 + // multiplies by 10^10, which exceeds 128-bit capacity + try { + col.addDecimal128(new Decimal128(Long.MAX_VALUE / 2, Long.MAX_VALUE, 0)); + fail("Expected LineSenderException for 128-bit overflow"); + } catch (LineSenderException e) { + assertEquals("Decimal128 overflow: rescaling from scale 0 to 10 exceeds 128-bit capacity", e.getMessage()); + } + } + } + + @Test + public void testAddDecimal64RescaleOverflow() { + try (QwpTableBuffer table = new QwpTableBuffer("test")) { + QwpTableBuffer.ColumnBuffer col = table.getOrCreateColumn("d", QwpConstants.TYPE_DECIMAL64, true); + // First row sets decimalScale = 5 + col.addDecimal64(Decimal64.fromLong(1, 5)); + table.nextRow(); + // Second row at scale 0 with a large value — rescaling to scale 5 + // multiplies by 10^5 = 100_000, which exceeds 64-bit capacity + // Long.MAX_VALUE / 10 ≈ 9.2 * 10^17, * 10^5 ≈ 9.2 * 10^22 >> 2^63 + try { + col.addDecimal64(Decimal64.fromLong(Long.MAX_VALUE / 10, 0)); + fail("Expected LineSenderException for 64-bit overflow"); + } catch (LineSenderException e) { + assertEquals("Decimal64 overflow: rescaling from scale 0 to 5 exceeds 64-bit capacity", e.getMessage()); + } + } + } + @Test public void testAddDoubleArrayNullOnNonNullableColumn() { try (QwpTableBuffer table = new QwpTableBuffer("test")) { From 1e452c699a604a28e942069c68c30b20a575b258 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 26 Feb 2026 13:29:03 +0100 Subject: [PATCH 68/89] Port QwpGorillaEncoder tests from core Port ~30 Gorilla encoder tests from the core module to the java-questdb-client module, adapted for the client's off-heap memory API. The new test file includes helper methods for writing timestamps to off-heap memory and decoding delta-of-delta values for round-trip verification without porting the full decoder. Co-Authored-By: Claude Opus 4.6 --- .../qwp/protocol/QwpGorillaEncoderTest.java | 654 ++++++++++++++++++ 1 file changed, 654 insertions(+) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpGorillaEncoderTest.java diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpGorillaEncoderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpGorillaEncoderTest.java new file mode 100644 index 0000000..0e2e553 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpGorillaEncoderTest.java @@ -0,0 +1,654 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.cutlass.qwp.protocol.QwpBitReader; +import io.questdb.client.cutlass.qwp.protocol.QwpGorillaEncoder; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; +import org.junit.Test; + +public class QwpGorillaEncoderTest { + + @Test + public void testCalculateEncodedSizeConstantDelta() { + long[] timestamps = new long[100]; + for (int i = 0; i < timestamps.length; i++) { + timestamps[i] = 1_000_000_000L + i * 1000L; + } + long src = putTimestamps(timestamps); + try { + int size = QwpGorillaEncoder.calculateEncodedSize(src, timestamps.length); + // 8 (first) + 8 (second) + ceil(98 bits / 8) = 29 + int expectedBits = timestamps.length - 2; + int expectedSize = 8 + 8 + (expectedBits + 7) / 8; + Assert.assertEquals(expectedSize, size); + } finally { + Unsafe.free(src, (long) timestamps.length * 8, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testCalculateEncodedSizeEmpty() { + Assert.assertEquals(0, QwpGorillaEncoder.calculateEncodedSize(0, 0)); + } + + @Test + public void testCalculateEncodedSizeIdenticalDeltas() { + long[] ts = {100L, 200L, 300L}; // delta=100, DoD=0 + long src = putTimestamps(ts); + try { + int size = QwpGorillaEncoder.calculateEncodedSize(src, ts.length); + // 8 + 8 + 1 byte (1 bit padded to byte) = 17 + Assert.assertEquals(17, size); + } finally { + Unsafe.free(src, (long) ts.length * 8, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testCalculateEncodedSizeOneTimestamp() { + long[] ts = {1000L}; + long src = putTimestamps(ts); + try { + Assert.assertEquals(8, QwpGorillaEncoder.calculateEncodedSize(src, 1)); + } finally { + Unsafe.free(src, 8, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testCalculateEncodedSizeSmallDoD() { + long[] ts = {100L, 200L, 350L}; // delta0=100, delta1=150, DoD=50 + long src = putTimestamps(ts); + try { + int size = QwpGorillaEncoder.calculateEncodedSize(src, ts.length); + // 8 + 8 + 2 bytes (9 bits padded to bytes) = 18 + Assert.assertEquals(18, size); + } finally { + Unsafe.free(src, (long) ts.length * 8, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testCalculateEncodedSizeTwoTimestamps() { + long[] ts = {1000L, 2000L}; + long src = putTimestamps(ts); + try { + Assert.assertEquals(16, QwpGorillaEncoder.calculateEncodedSize(src, 2)); + } finally { + Unsafe.free(src, 16, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testCanUseGorillaConstantDelta() { + long[] timestamps = new long[100]; + for (int i = 0; i < timestamps.length; i++) { + timestamps[i] = 1_000_000_000L + i * 1000L; + } + long src = putTimestamps(timestamps); + try { + Assert.assertTrue(QwpGorillaEncoder.canUseGorilla(src, timestamps.length)); + } finally { + Unsafe.free(src, (long) timestamps.length * 8, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testCanUseGorillaEmpty() { + Assert.assertTrue(QwpGorillaEncoder.canUseGorilla(0, 0)); + } + + @Test + public void testCanUseGorillaLargeDoDOutOfRange() { + // DoD = 3_000_000_000 exceeds Integer.MAX_VALUE + long[] timestamps = { + 0L, + 1_000_000_000L, // delta=1_000_000_000 + 5_000_000_000L, // delta=4_000_000_000, DoD=3_000_000_000 + }; + long src = putTimestamps(timestamps); + try { + Assert.assertFalse(QwpGorillaEncoder.canUseGorilla(src, timestamps.length)); + } finally { + Unsafe.free(src, (long) timestamps.length * 8, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testCanUseGorillaNegativeLargeDoDOutOfRange() { + // DoD = -4_000_000_000 is less than Integer.MIN_VALUE + long[] timestamps = { + 10_000_000_000L, + 9_000_000_000L, // delta=-1_000_000_000 + 4_000_000_000L, // delta=-5_000_000_000, DoD=-4_000_000_000 + }; + long src = putTimestamps(timestamps); + try { + Assert.assertFalse(QwpGorillaEncoder.canUseGorilla(src, timestamps.length)); + } finally { + Unsafe.free(src, (long) timestamps.length * 8, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testCanUseGorillaOneTimestamp() { + long[] ts = {1000L}; + long src = putTimestamps(ts); + try { + Assert.assertTrue(QwpGorillaEncoder.canUseGorilla(src, 1)); + } finally { + Unsafe.free(src, 8, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testCanUseGorillaTwoTimestamps() { + long[] ts = {1000L, 2000L}; + long src = putTimestamps(ts); + try { + Assert.assertTrue(QwpGorillaEncoder.canUseGorilla(src, 2)); + } finally { + Unsafe.free(src, 16, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testCanUseGorillaVaryingDelta() { + long[] timestamps = { + 1_000_000_000L, + 1_000_001_000L, // delta=1000 + 1_000_002_100L, // DoD=100 + 1_000_003_500L, // DoD=300 + }; + long src = putTimestamps(timestamps); + try { + Assert.assertTrue(QwpGorillaEncoder.canUseGorilla(src, timestamps.length)); + } finally { + Unsafe.free(src, (long) timestamps.length * 8, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testCompressionRatioConstantInterval() { + long[] timestamps = new long[1000]; + for (int i = 0; i < timestamps.length; i++) { + timestamps[i] = i * 1000L; + } + long src = putTimestamps(timestamps); + try { + int gorillaSize = QwpGorillaEncoder.calculateEncodedSize(src, timestamps.length); + int uncompressedSize = timestamps.length * 8; + double ratio = (double) gorillaSize / uncompressedSize; + Assert.assertTrue("Compression ratio should be < 0.1 for constant interval", ratio < 0.1); + } finally { + Unsafe.free(src, (long) timestamps.length * 8, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testCompressionRatioRandomData() { + long[] timestamps = new long[100]; + timestamps[0] = 1_000_000_000L; + timestamps[1] = 1_000_001_000L; + java.util.Random random = new java.util.Random(42); + for (int i = 2; i < timestamps.length; i++) { + timestamps[i] = timestamps[i - 1] + 1000 + random.nextInt(10_000) - 5000; + } + long src = putTimestamps(timestamps); + try { + int gorillaSize = QwpGorillaEncoder.calculateEncodedSize(src, timestamps.length); + Assert.assertTrue("Size should be positive", gorillaSize > 0); + } finally { + Unsafe.free(src, (long) timestamps.length * 8, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testEncodeDecodeBucketBoundaries() { + QwpGorillaEncoder encoder = new QwpGorillaEncoder(); + QwpBitReader reader = new QwpBitReader(); + + // t0=0, t1=10_000, delta0=10_000 + // For DoD=X: delta1 = 10_000+X, so t2 = 20_000+X + long[][] bucketTests = { + {0L, 10_000L, 20_000L}, // DoD = 0 (bucket 0) + {0L, 10_000L, 20_063L}, // DoD = 63 (bucket 1 max) + {0L, 10_000L, 19_936L}, // DoD = -64 (bucket 1 min) + {0L, 10_000L, 20_064L}, // DoD = 64 (bucket 2 start) + {0L, 10_000L, 19_935L}, // DoD = -65 (bucket 2 start) + {0L, 10_000L, 20_255L}, // DoD = 255 (bucket 2 max) + {0L, 10_000L, 19_744L}, // DoD = -256 (bucket 2 min) + {0L, 10_000L, 20_256L}, // DoD = 256 (bucket 3 start) + {0L, 10_000L, 19_743L}, // DoD = -257 (bucket 3 start) + {0L, 10_000L, 22_047L}, // DoD = 2047 (bucket 3 max) + {0L, 10_000L, 17_952L}, // DoD = -2048 (bucket 3 min) + {0L, 10_000L, 22_048L}, // DoD = 2048 (bucket 4 start) + {0L, 10_000L, 17_951L}, // DoD = -2049 (bucket 4 start) + {0L, 10_000L, 110_000L}, // DoD = 100_000 (bucket 4, large) + {0L, 10_000L, -80_000L}, // DoD = -100_000 (bucket 4, large) + }; + + for (long[] tc : bucketTests) { + long src = putTimestamps(tc); + long dst = Unsafe.malloc(64, MemoryTag.NATIVE_ILP_RSS); + try { + int bytesWritten = encoder.encodeTimestamps(dst, 64, src, tc.length); + Assert.assertTrue("Failed to encode: " + java.util.Arrays.toString(tc), bytesWritten > 0); + + long first = Unsafe.getUnsafe().getLong(dst); + long second = Unsafe.getUnsafe().getLong(dst + 8); + Assert.assertEquals(tc[0], first); + Assert.assertEquals(tc[1], second); + + reader.reset(dst + 16, bytesWritten - 16); + long dod = decodeDoD(reader); + long delta = (second - first) + dod; + long decoded = second + delta; + Assert.assertEquals("Failed for: " + java.util.Arrays.toString(tc), tc[2], decoded); + } finally { + Unsafe.free(src, (long) tc.length * 8, MemoryTag.NATIVE_ILP_RSS); + Unsafe.free(dst, 64, MemoryTag.NATIVE_ILP_RSS); + } + } + } + + @Test + public void testEncodeTimestampsEmpty() { + QwpGorillaEncoder encoder = new QwpGorillaEncoder(); + long dst = Unsafe.malloc(64, MemoryTag.NATIVE_ILP_RSS); + try { + int bytesWritten = encoder.encodeTimestamps(dst, 64, 0, 0); + Assert.assertEquals(0, bytesWritten); + } finally { + Unsafe.free(dst, 64, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testEncodeTimestampsOneTimestamp() { + QwpGorillaEncoder encoder = new QwpGorillaEncoder(); + long[] timestamps = {1_234_567_890L}; + long src = putTimestamps(timestamps); + long dst = Unsafe.malloc(64, MemoryTag.NATIVE_ILP_RSS); + try { + int bytesWritten = encoder.encodeTimestamps(dst, 64, src, 1); + Assert.assertEquals(8, bytesWritten); + Assert.assertEquals(1_234_567_890L, Unsafe.getUnsafe().getLong(dst)); + } finally { + Unsafe.free(src, 8, MemoryTag.NATIVE_ILP_RSS); + Unsafe.free(dst, 64, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testEncodeTimestampsRoundTripAllBuckets() { + QwpGorillaEncoder encoder = new QwpGorillaEncoder(); + QwpBitReader reader = new QwpBitReader(); + + long[] timestamps = new long[10]; + timestamps[0] = 1_000_000_000L; + timestamps[1] = 1_000_001_000L; // delta = 1000 + timestamps[2] = 1_000_002_000L; // DoD=0 (bucket 0) + timestamps[3] = 1_000_003_050L; // DoD=50 (bucket 1) + timestamps[4] = 1_000_003_987L; // DoD=-113 (bucket 2) + timestamps[5] = 1_000_004_687L; // DoD=-237 (bucket 2) + timestamps[6] = 1_000_006_387L; // DoD=1000 (bucket 3) + timestamps[7] = 1_000_020_087L; // DoD=12000 (bucket 4) + timestamps[8] = 1_000_033_787L; // DoD=0 (bucket 0) + timestamps[9] = 1_000_047_487L; // DoD=0 (bucket 0) + + long src = putTimestamps(timestamps); + int capacity = QwpGorillaEncoder.calculateEncodedSize(src, timestamps.length) + 16; + long dst = Unsafe.malloc(capacity, MemoryTag.NATIVE_ILP_RSS); + try { + int bytesWritten = encoder.encodeTimestamps(dst, capacity, src, timestamps.length); + Assert.assertTrue(bytesWritten > 0); + + long first = Unsafe.getUnsafe().getLong(dst); + long second = Unsafe.getUnsafe().getLong(dst + 8); + Assert.assertEquals(timestamps[0], first); + Assert.assertEquals(timestamps[1], second); + + reader.reset(dst + 16, bytesWritten - 16); + long prevTs = second; + long prevDelta = second - first; + for (int i = 2; i < timestamps.length; i++) { + long dod = decodeDoD(reader); + long delta = prevDelta + dod; + long ts = prevTs + delta; + Assert.assertEquals("Mismatch at index " + i, timestamps[i], ts); + prevDelta = delta; + prevTs = ts; + } + } finally { + Unsafe.free(src, (long) timestamps.length * 8, MemoryTag.NATIVE_ILP_RSS); + Unsafe.free(dst, capacity, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testEncodeTimestampsRoundTripConstantDelta() { + QwpGorillaEncoder encoder = new QwpGorillaEncoder(); + QwpBitReader reader = new QwpBitReader(); + + long[] timestamps = new long[100]; + for (int i = 0; i < timestamps.length; i++) { + timestamps[i] = 1_000_000_000L + i * 1000L; + } + + long src = putTimestamps(timestamps); + int capacity = QwpGorillaEncoder.calculateEncodedSize(src, timestamps.length) + 16; + long dst = Unsafe.malloc(capacity, MemoryTag.NATIVE_ILP_RSS); + try { + int bytesWritten = encoder.encodeTimestamps(dst, capacity, src, timestamps.length); + Assert.assertTrue(bytesWritten > 0); + + long first = Unsafe.getUnsafe().getLong(dst); + long second = Unsafe.getUnsafe().getLong(dst + 8); + Assert.assertEquals(timestamps[0], first); + Assert.assertEquals(timestamps[1], second); + + reader.reset(dst + 16, bytesWritten - 16); + long prevTs = second; + long prevDelta = second - first; + for (int i = 2; i < timestamps.length; i++) { + long dod = decodeDoD(reader); + long delta = prevDelta + dod; + long ts = prevTs + delta; + Assert.assertEquals("Mismatch at index " + i, timestamps[i], ts); + prevDelta = delta; + prevTs = ts; + } + } finally { + Unsafe.free(src, (long) timestamps.length * 8, MemoryTag.NATIVE_ILP_RSS); + Unsafe.free(dst, capacity, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testEncodeTimestampsRoundTripLargeDataset() { + QwpGorillaEncoder encoder = new QwpGorillaEncoder(); + QwpBitReader reader = new QwpBitReader(); + + int count = 10_000; + long[] timestamps = new long[count]; + timestamps[0] = 1_000_000_000L; + timestamps[1] = 1_000_001_000L; + + java.util.Random random = new java.util.Random(42); + for (int i = 2; i < count; i++) { + long prevDelta = timestamps[i - 1] - timestamps[i - 2]; + int variation = (i % 10 == 0) ? random.nextInt(100) - 50 : 0; + timestamps[i] = timestamps[i - 1] + prevDelta + variation; + } + + long src = putTimestamps(timestamps); + int capacity = QwpGorillaEncoder.calculateEncodedSize(src, count) + 100; + long dst = Unsafe.malloc(capacity, MemoryTag.NATIVE_ILP_RSS); + try { + int bytesWritten = encoder.encodeTimestamps(dst, capacity, src, count); + Assert.assertTrue(bytesWritten > 0); + Assert.assertTrue("Should compress better than uncompressed", bytesWritten < count * 8); + + long first = Unsafe.getUnsafe().getLong(dst); + long second = Unsafe.getUnsafe().getLong(dst + 8); + Assert.assertEquals(timestamps[0], first); + Assert.assertEquals(timestamps[1], second); + + reader.reset(dst + 16, bytesWritten - 16); + long prevTs = second; + long prevDelta = second - first; + for (int i = 2; i < count; i++) { + long dod = decodeDoD(reader); + long delta = prevDelta + dod; + long ts = prevTs + delta; + Assert.assertEquals("Mismatch at index " + i, timestamps[i], ts); + prevDelta = delta; + prevTs = ts; + } + } finally { + Unsafe.free(src, (long) count * 8, MemoryTag.NATIVE_ILP_RSS); + Unsafe.free(dst, capacity, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testEncodeTimestampsRoundTripNegativeDoD() { + QwpGorillaEncoder encoder = new QwpGorillaEncoder(); + QwpBitReader reader = new QwpBitReader(); + + long[] timestamps = { + 1_000_000_000L, + 1_000_002_000L, // delta=2000 + 1_000_003_000L, // DoD=-1000 (bucket 3) + 1_000_003_500L, // DoD=-500 (bucket 2) + 1_000_003_600L, // DoD=-400 (bucket 2) + }; + + long src = putTimestamps(timestamps); + int capacity = QwpGorillaEncoder.calculateEncodedSize(src, timestamps.length) + 16; + long dst = Unsafe.malloc(capacity, MemoryTag.NATIVE_ILP_RSS); + try { + int bytesWritten = encoder.encodeTimestamps(dst, capacity, src, timestamps.length); + Assert.assertTrue(bytesWritten > 0); + + long first = Unsafe.getUnsafe().getLong(dst); + long second = Unsafe.getUnsafe().getLong(dst + 8); + Assert.assertEquals(timestamps[0], first); + Assert.assertEquals(timestamps[1], second); + + reader.reset(dst + 16, bytesWritten - 16); + long prevTs = second; + long prevDelta = second - first; + for (int i = 2; i < timestamps.length; i++) { + long dod = decodeDoD(reader); + long delta = prevDelta + dod; + long ts = prevTs + delta; + Assert.assertEquals("Mismatch at index " + i, timestamps[i], ts); + prevDelta = delta; + prevTs = ts; + } + } finally { + Unsafe.free(src, (long) timestamps.length * 8, MemoryTag.NATIVE_ILP_RSS); + Unsafe.free(dst, capacity, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testEncodeTimestampsRoundTripVaryingDelta() { + QwpGorillaEncoder encoder = new QwpGorillaEncoder(); + QwpBitReader reader = new QwpBitReader(); + + long[] timestamps = { + 1_000_000_000L, + 1_000_001_000L, // delta=1000 + 1_000_002_000L, // DoD=0 (bucket 0) + 1_000_003_010L, // DoD=10 (bucket 1) + 1_000_004_120L, // DoD=100 (bucket 1) + 1_000_005_420L, // DoD=190 (bucket 2) + 1_000_007_720L, // DoD=1000 (bucket 3) + 1_000_020_020L, // DoD=10000 (bucket 4) + }; + + long src = putTimestamps(timestamps); + int capacity = QwpGorillaEncoder.calculateEncodedSize(src, timestamps.length) + 16; + long dst = Unsafe.malloc(capacity, MemoryTag.NATIVE_ILP_RSS); + try { + int bytesWritten = encoder.encodeTimestamps(dst, capacity, src, timestamps.length); + Assert.assertTrue(bytesWritten > 0); + + long first = Unsafe.getUnsafe().getLong(dst); + long second = Unsafe.getUnsafe().getLong(dst + 8); + Assert.assertEquals(timestamps[0], first); + Assert.assertEquals(timestamps[1], second); + + reader.reset(dst + 16, bytesWritten - 16); + long prevTs = second; + long prevDelta = second - first; + for (int i = 2; i < timestamps.length; i++) { + long dod = decodeDoD(reader); + long delta = prevDelta + dod; + long ts = prevTs + delta; + Assert.assertEquals("Mismatch at index " + i, timestamps[i], ts); + prevDelta = delta; + prevTs = ts; + } + } finally { + Unsafe.free(src, (long) timestamps.length * 8, MemoryTag.NATIVE_ILP_RSS); + Unsafe.free(dst, capacity, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testEncodeTimestampsTwoTimestamps() { + QwpGorillaEncoder encoder = new QwpGorillaEncoder(); + long[] timestamps = {1_000_000_000L, 1_000_001_000L}; + long src = putTimestamps(timestamps); + long dst = Unsafe.malloc(64, MemoryTag.NATIVE_ILP_RSS); + try { + int bytesWritten = encoder.encodeTimestamps(dst, 64, src, 2); + Assert.assertEquals(16, bytesWritten); + Assert.assertEquals(1_000_000_000L, Unsafe.getUnsafe().getLong(dst)); + Assert.assertEquals(1_000_001_000L, Unsafe.getUnsafe().getLong(dst + 8)); + } finally { + Unsafe.free(src, 16, MemoryTag.NATIVE_ILP_RSS); + Unsafe.free(dst, 64, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testGetBitsRequired() { + // Bucket 0: 1 bit + Assert.assertEquals(1, QwpGorillaEncoder.getBitsRequired(0)); + + // Bucket 1: 9 bits (2 prefix + 7 value) + Assert.assertEquals(9, QwpGorillaEncoder.getBitsRequired(1)); + Assert.assertEquals(9, QwpGorillaEncoder.getBitsRequired(-1)); + Assert.assertEquals(9, QwpGorillaEncoder.getBitsRequired(63)); + Assert.assertEquals(9, QwpGorillaEncoder.getBitsRequired(-64)); + + // Bucket 2: 12 bits (3 prefix + 9 value) + Assert.assertEquals(12, QwpGorillaEncoder.getBitsRequired(64)); + Assert.assertEquals(12, QwpGorillaEncoder.getBitsRequired(255)); + Assert.assertEquals(12, QwpGorillaEncoder.getBitsRequired(-256)); + + // Bucket 3: 16 bits (4 prefix + 12 value) + Assert.assertEquals(16, QwpGorillaEncoder.getBitsRequired(256)); + Assert.assertEquals(16, QwpGorillaEncoder.getBitsRequired(2047)); + Assert.assertEquals(16, QwpGorillaEncoder.getBitsRequired(-2048)); + + // Bucket 4: 36 bits (4 prefix + 32 value) + Assert.assertEquals(36, QwpGorillaEncoder.getBitsRequired(2048)); + Assert.assertEquals(36, QwpGorillaEncoder.getBitsRequired(-2049)); + } + + @Test + public void testGetBucket12Bit() { + // DoD in [-2048, 2047] but outside [-256, 255] -> bucket 3 (16 bits) + Assert.assertEquals(3, QwpGorillaEncoder.getBucket(256)); + Assert.assertEquals(3, QwpGorillaEncoder.getBucket(-257)); + Assert.assertEquals(3, QwpGorillaEncoder.getBucket(2047)); + Assert.assertEquals(3, QwpGorillaEncoder.getBucket(-2047)); + Assert.assertEquals(3, QwpGorillaEncoder.getBucket(-2048)); + } + + @Test + public void testGetBucket32Bit() { + // DoD outside [-2048, 2047] -> bucket 4 (36 bits) + Assert.assertEquals(4, QwpGorillaEncoder.getBucket(2048)); + Assert.assertEquals(4, QwpGorillaEncoder.getBucket(-2049)); + Assert.assertEquals(4, QwpGorillaEncoder.getBucket(100_000)); + Assert.assertEquals(4, QwpGorillaEncoder.getBucket(-100_000)); + Assert.assertEquals(4, QwpGorillaEncoder.getBucket(Integer.MAX_VALUE)); + Assert.assertEquals(4, QwpGorillaEncoder.getBucket(Integer.MIN_VALUE)); + } + + @Test + public void testGetBucket7Bit() { + // DoD in [-64, 63] -> bucket 1 (9 bits) + Assert.assertEquals(1, QwpGorillaEncoder.getBucket(1)); + Assert.assertEquals(1, QwpGorillaEncoder.getBucket(-1)); + Assert.assertEquals(1, QwpGorillaEncoder.getBucket(63)); + Assert.assertEquals(1, QwpGorillaEncoder.getBucket(-63)); + Assert.assertEquals(1, QwpGorillaEncoder.getBucket(-64)); + } + + @Test + public void testGetBucket9Bit() { + // DoD in [-256, 255] but outside [-64, 63] -> bucket 2 (12 bits) + Assert.assertEquals(2, QwpGorillaEncoder.getBucket(64)); + Assert.assertEquals(2, QwpGorillaEncoder.getBucket(-65)); + Assert.assertEquals(2, QwpGorillaEncoder.getBucket(255)); + Assert.assertEquals(2, QwpGorillaEncoder.getBucket(-255)); + Assert.assertEquals(2, QwpGorillaEncoder.getBucket(-256)); + } + + @Test + public void testGetBucketZero() { + Assert.assertEquals(0, QwpGorillaEncoder.getBucket(0)); + } + + /** + * Decodes a delta-of-delta value from the bit stream, mirroring the + * core QwpGorillaDecoder.decodeDoD() logic. + */ + private static long decodeDoD(QwpBitReader reader) { + int bit = reader.readBit(); + if (bit == 0) { + return 0; + } + bit = reader.readBit(); + if (bit == 0) { + return reader.readSigned(7); + } + bit = reader.readBit(); + if (bit == 0) { + return reader.readSigned(9); + } + bit = reader.readBit(); + if (bit == 0) { + return reader.readSigned(12); + } + return reader.readSigned(32); + } + + /** + * Writes a Java array of timestamps to off-heap memory. + * + * @param timestamps the timestamps to write + * @return the address of the allocated memory (caller must free) + */ + private static long putTimestamps(long[] timestamps) { + long size = (long) timestamps.length * 8; + long address = Unsafe.malloc(size, MemoryTag.NATIVE_ILP_RSS); + for (int i = 0; i < timestamps.length; i++) { + Unsafe.getUnsafe().putLong(address + (long) i * 8, timestamps[i]); + } + return address; + } +} From ec56390037128f7f5866f5568ff7d2959def003e Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 26 Feb 2026 13:30:38 +0100 Subject: [PATCH 69/89] Explain precondition in gorilla encoder --- .../client/cutlass/qwp/protocol/QwpGorillaEncoder.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java index a21215a..0f57342 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaEncoder.java @@ -231,6 +231,11 @@ public void encodeDoD(long deltaOfDelta) { * - Remaining timestamps: bit-packed delta-of-delta * *

    + * Precondition: the caller must verify that {@link #canUseGorilla(long, int)} + * returns {@code true} before calling this method. The largest delta-of-delta + * bucket uses 32-bit signed encoding, so values outside the {@code int} range + * are silently truncated, producing corrupt output on decode. + *

    * Note: This method does NOT write the encoding flag byte. The caller is * responsible for writing the ENCODING_GORILLA flag before calling this method. * From c81c80f13c964a1ae5f52002f75fcab4c7b2d8a1 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 26 Feb 2026 13:33:59 +0100 Subject: [PATCH 70/89] Add bounds assertion to jumpTo() Add a debug assertion in OffHeapAppendMemory.jumpTo() that validates the offset is non-negative and within the current append position. This is consistent with the assertion pattern used in MemoryPARWImpl.jumpTo() in core. Co-Authored-By: Claude Opus 4.6 --- .../questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java index 7388961..a30cf3c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/OffHeapAppendMemory.java @@ -83,6 +83,7 @@ public long getAppendOffset() { * Used for truncateTo operations on column buffers. */ public void jumpTo(long offset) { + assert offset >= 0 && offset <= getAppendOffset(); appendAddress = pageAddress + offset; } From 99876bc5a6ee9b0acc2af1f75a5ea642b3e8cd7e Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 26 Feb 2026 13:40:57 +0100 Subject: [PATCH 71/89] Avoid BigDecimal allocation in decimalColumn MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit decimalColumn(CharSequence, CharSequence) previously allocated a BigDecimal and a new Decimal256 on every call. Replace this with a reusable Decimal256 field and its ofString() method, which parses the CharSequence directly into the mutable object in-place via DecimalParser — no intermediate objects needed. Co-Authored-By: Claude Opus 4.6 --- .../client/cutlass/qwp/client/QwpWebSocketSender.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 59e890a..1cc8e51 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -146,6 +146,7 @@ public class QwpWebSocketSender implements Sender { private boolean connected; // Track max global symbol ID used in current batch (for delta calculation) private int currentBatchMaxSymbolId = -1; + private final Decimal256 currentDecimal256 = new Decimal256(); private QwpTableBuffer currentTableBuffer; private String currentTableName; private long firstPendingRowTimeNanos; @@ -557,10 +558,9 @@ public Sender decimalColumn(CharSequence name, CharSequence value) { checkNotClosed(); checkTableSelected(); try { - java.math.BigDecimal bd = new java.math.BigDecimal(value.toString()); - Decimal256 decimal = Decimal256.fromBigDecimal(bd); + currentDecimal256.ofString(value); QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name.toString(), TYPE_DECIMAL256, true); - col.addDecimal256(decimal); + col.addDecimal256(currentDecimal256); } catch (Exception e) { throw new LineSenderException("Failed to parse decimal value: " + value, e); } From a999a909f8b6b0954170b6913661604ea3b1b5cc Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 26 Feb 2026 13:41:08 +0100 Subject: [PATCH 72/89] Remove redundant local variable --- .../io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java index e7efe6f..0117c0f 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/MicrobatchBuffer.java @@ -181,8 +181,7 @@ public void ensureCapacity(int requiredCapacity) { } if (requiredCapacity > bufferCapacity) { int newCapacity = Math.max(bufferCapacity * 2, requiredCapacity); - long newPtr = Unsafe.realloc(bufferPtr, bufferCapacity, newCapacity, MemoryTag.NATIVE_ILP_RSS); - bufferPtr = newPtr; + bufferPtr = Unsafe.realloc(bufferPtr, bufferCapacity, newCapacity, MemoryTag.NATIVE_ILP_RSS); bufferCapacity = newCapacity; } } From 19ac1c6ff171708504090a1e0dbfc6026058b7a7 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 26 Feb 2026 14:04:58 +0100 Subject: [PATCH 73/89] Fix UUID storage allocation in QwpTableBuffer allocateStorage() incorrectly grouped TYPE_UUID with the array types (TYPE_DOUBLE_ARRAY, TYPE_LONG_ARRAY), causing it to allocate arrayDims and arrayCapture instead of a dataBuffer. Since addUuid() and addNull() for UUID both write to dataBuffer, any use of a UUID column would NPE. Move TYPE_UUID to the TYPE_DECIMAL128 case, which correctly allocates a 256-byte OffHeapAppendMemory for the 16-byte fixed-width data. Co-Authored-By: Claude Opus 4.6 --- .../io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index 4ba4f0d..7e21881 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -1194,6 +1194,7 @@ private void allocateStorage(byte type) { case TYPE_DOUBLE: dataBuffer = new OffHeapAppendMemory(128); break; + case TYPE_UUID: case TYPE_DECIMAL128: dataBuffer = new OffHeapAppendMemory(256); break; @@ -1212,7 +1213,6 @@ private void allocateStorage(byte type) { symbolDict = new CharSequenceIntHashMap(); symbolList = new ObjList<>(); break; - case TYPE_UUID: case TYPE_DOUBLE_ARRAY: case TYPE_LONG_ARRAY: arrayDims = new byte[16]; From c1500f8e2e37092b517ae38e478471a5730a837b Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 26 Feb 2026 14:23:33 +0100 Subject: [PATCH 74/89] Delete unused getSymbolsInRange() GlobalSymbolDictionary.getSymbolsInRange() had no production callers. Remove the method to reduce dead code. Co-Authored-By: Claude Opus 4.6 --- .../qwp/client/GlobalSymbolDictionary.java | 28 ------------------- 1 file changed, 28 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java index ace342e..b8c1dfe 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java @@ -125,34 +125,6 @@ public String getSymbol(int id) { return idToSymbol.getQuick(id); } - /** - * Gets the symbols in the given ID range [fromId, toId). - *

    - * This is used to extract the delta for sending to the server. - * The range is inclusive of fromId and exclusive of toId. - * - * @param fromId start ID (inclusive) - * @param toId end ID (exclusive) - * @return array of symbols in the range, or empty array if range is invalid/empty - */ - public String[] getSymbolsInRange(int fromId, int toId) { - if (fromId < 0 || toId < fromId || fromId >= idToSymbol.size()) { - return new String[0]; - } - - int actualToId = Math.min(toId, idToSymbol.size()); - int count = actualToId - fromId; - if (count <= 0) { - return new String[0]; - } - - String[] result = new String[count]; - for (int i = 0; i < count; i++) { - result[i] = idToSymbol.getQuick(fromId + i); - } - return result; - } - /** * Checks if the dictionary is empty. * From 2a11d37c39500f08c34ab1d50a083f27ab857671 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 26 Feb 2026 14:28:35 +0100 Subject: [PATCH 75/89] Move GlobalSymbolDictionaryTest to client The test class belongs with the code it tests. Move it from core's test tree into the client module under the matching package io.questdb.client.test.cutlass.qwp.client. Co-Authored-By: Claude Opus 4.6 --- .../client/GlobalSymbolDictionaryTest.java | 249 ++++++++++++++++++ 1 file changed, 249 insertions(+) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/GlobalSymbolDictionaryTest.java diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/GlobalSymbolDictionaryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/GlobalSymbolDictionaryTest.java new file mode 100644 index 0000000..c5eb84e --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/GlobalSymbolDictionaryTest.java @@ -0,0 +1,249 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary; +import org.junit.Test; + +import static org.junit.Assert.*; + +public class GlobalSymbolDictionaryTest { + + @Test + public void testAddSymbol_assignsSequentialIds() { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + + assertEquals(0, dict.getOrAddSymbol("AAPL")); + assertEquals(1, dict.getOrAddSymbol("GOOG")); + assertEquals(2, dict.getOrAddSymbol("MSFT")); + assertEquals(3, dict.getOrAddSymbol("TSLA")); + + assertEquals(4, dict.size()); + } + + @Test + public void testAddSymbol_deduplicatesSameSymbol() { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + + int id1 = dict.getOrAddSymbol("AAPL"); + int id2 = dict.getOrAddSymbol("AAPL"); + int id3 = dict.getOrAddSymbol("AAPL"); + + assertEquals(id1, id2); + assertEquals(id2, id3); + assertEquals(0, id1); + assertEquals(1, dict.size()); + } + + @Test + public void testClear() { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + + dict.getOrAddSymbol("AAPL"); + dict.getOrAddSymbol("GOOG"); + assertEquals(2, dict.size()); + + dict.clear(); + + assertTrue(dict.isEmpty()); + assertEquals(0, dict.size()); + assertFalse(dict.contains("AAPL")); + } + + @Test + public void testClear_thenAddRestartsFromZero() { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + + dict.getOrAddSymbol("AAPL"); + dict.getOrAddSymbol("GOOG"); + dict.clear(); + + // New IDs should start from 0 + assertEquals(0, dict.getOrAddSymbol("MSFT")); + assertEquals(1, dict.getOrAddSymbol("TSLA")); + } + + @Test + public void testContains() { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + + assertFalse(dict.contains("AAPL")); + + dict.getOrAddSymbol("AAPL"); + dict.getOrAddSymbol("GOOG"); + + assertTrue(dict.contains("AAPL")); + assertTrue(dict.contains("GOOG")); + assertFalse(dict.contains("MSFT")); + assertFalse(dict.contains(null)); + } + + @Test + public void testCustomInitialCapacity() { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(1024); + + // Should work normally + for (int i = 0; i < 100; i++) { + assertEquals(i, dict.getOrAddSymbol("SYM_" + i)); + } + assertEquals(100, dict.size()); + } + + @Test + public void testGetId_returnsCorrectId() { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + + dict.getOrAddSymbol("AAPL"); + dict.getOrAddSymbol("GOOG"); + dict.getOrAddSymbol("MSFT"); + + assertEquals(0, dict.getId("AAPL")); + assertEquals(1, dict.getId("GOOG")); + assertEquals(2, dict.getId("MSFT")); + } + + @Test + public void testGetId_returnsMinusOneForNull() { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + assertEquals(-1, dict.getId(null)); + } + + @Test + public void testGetId_returnsMinusOneForUnknown() { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + dict.getOrAddSymbol("AAPL"); + + assertEquals(-1, dict.getId("GOOG")); + assertEquals(-1, dict.getId("UNKNOWN")); + } + + @Test(expected = IllegalArgumentException.class) + public void testGetOrAddSymbol_throwsForNull() { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + dict.getOrAddSymbol(null); + } + + @Test + public void testGetSymbol_returnsCorrectSymbol() { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + + dict.getOrAddSymbol("AAPL"); + dict.getOrAddSymbol("GOOG"); + dict.getOrAddSymbol("MSFT"); + + assertEquals("AAPL", dict.getSymbol(0)); + assertEquals("GOOG", dict.getSymbol(1)); + assertEquals("MSFT", dict.getSymbol(2)); + } + + @Test(expected = IndexOutOfBoundsException.class) + public void testGetSymbol_throwsForInvalidId() { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + dict.getOrAddSymbol("AAPL"); + dict.getSymbol(1); // Only id 0 exists + } + + @Test(expected = IndexOutOfBoundsException.class) + public void testGetSymbol_throwsForNegativeId() { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + dict.getOrAddSymbol("AAPL"); + dict.getSymbol(-1); + } + + @Test + public void testIsEmpty() { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + + assertTrue(dict.isEmpty()); + + dict.getOrAddSymbol("AAPL"); + assertFalse(dict.isEmpty()); + } + + @Test + public void testLargeNumberOfSymbols() { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + + // Add 10000 symbols + for (int i = 0; i < 10000; i++) { + assertEquals(i, dict.getOrAddSymbol("SYMBOL_" + i)); + } + + assertEquals(10000, dict.size()); + + // Verify retrieval + for (int i = 0; i < 10000; i++) { + assertEquals("SYMBOL_" + i, dict.getSymbol(i)); + assertEquals(i, dict.getId("SYMBOL_" + i)); + } + } + + @Test + public void testMixedSymbolsAcrossTables() { + // Simulates symbols from multiple tables sharing the dictionary + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + + // Table "trades": exchange column + int nyse = dict.getOrAddSymbol("NYSE"); // 0 + int nasdaq = dict.getOrAddSymbol("NASDAQ"); // 1 + + // Table "prices": currency column + int usd = dict.getOrAddSymbol("USD"); // 2 + int eur = dict.getOrAddSymbol("EUR"); // 3 + + // Table "orders": exchange column (reuses) + int nyse2 = dict.getOrAddSymbol("NYSE"); // Still 0 + + assertEquals(nyse, nyse2); + assertEquals(4, dict.size()); + + // All symbols accessible + assertEquals("NYSE", dict.getSymbol(nyse)); + assertEquals("NASDAQ", dict.getSymbol(nasdaq)); + assertEquals("USD", dict.getSymbol(usd)); + assertEquals("EUR", dict.getSymbol(eur)); + } + + @Test + public void testSpecialCharactersInSymbols() { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + + dict.getOrAddSymbol(""); // Empty string + dict.getOrAddSymbol(" "); // Space + dict.getOrAddSymbol("a b c"); // With spaces + dict.getOrAddSymbol("AAPL\u0000"); // With null char + dict.getOrAddSymbol("\u00E9"); // Unicode + dict.getOrAddSymbol("\uD83D\uDE00"); // Emoji + + assertEquals(6, dict.size()); + + assertEquals("", dict.getSymbol(0)); + assertEquals(" ", dict.getSymbol(1)); + assertEquals("a b c", dict.getSymbol(2)); + assertEquals("AAPL\u0000", dict.getSymbol(3)); + assertEquals("\u00E9", dict.getSymbol(4)); + assertEquals("\uD83D\uDE00", dict.getSymbol(5)); + } +} From 8630ead11661b6c5747cb8128f13b324317612eb Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 26 Feb 2026 15:05:16 +0100 Subject: [PATCH 76/89] Move DeltaSymbolDictionaryTest to client Move the 18 tests that exercise only client-side classes (GlobalSymbolDictionary, QwpWebSocketEncoder, QwpTableBuffer, QwpBufferWriter) from core into the client submodule. Drop three round-trip tests that depend on server-only classes (QwpStreamingDecoder, QwpMessageCursor, QwpSymbolColumnCursor). Adapt imports from io.questdb.std to io.questdb.client.std and from io.questdb.cutlass.qwp.protocol.QwpConstants to io.questdb.client.cutlass.qwp.protocol.QwpConstants. Remove the unused throws QwpParseException from encodeAndDecode and its helper. Co-Authored-By: Claude Opus 4.6 --- .../qwp/client/DeltaSymbolDictionaryTest.java | 598 ++++++++++++++++++ 1 file changed, 598 insertions(+) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaSymbolDictionaryTest.java diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaSymbolDictionaryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaSymbolDictionaryTest.java new file mode 100644 index 0000000..7001e84 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaSymbolDictionaryTest.java @@ -0,0 +1,598 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary; +import io.questdb.client.cutlass.qwp.client.QwpBufferWriter; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.std.ObjList; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; +import org.junit.Test; + +import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.*; + +/** + * Comprehensive tests for delta symbol dictionary encoding and decoding. + *

    + * Tests cover: + * - Multiple tables sharing the same global dictionary + * - Multiple batches with progressive symbol accumulation + * - Reconnection scenarios where the dictionary resets + * - Multiple symbol columns in the same table + * - Edge cases (empty batches, no symbols, etc.) + */ +public class DeltaSymbolDictionaryTest { + + @Test + public void testEdgeCase_batchWithNoSymbols() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + + // Table with only non-symbol columns + QwpTableBuffer batch = new QwpTableBuffer("metrics"); + QwpTableBuffer.ColumnBuffer valueCol = batch.getOrCreateColumn("value", TYPE_LONG, false); + valueCol.addLong(100L); + batch.nextRow(); + + // MaxId is -1 (no symbols) + int batchMaxId = -1; + + // Can still encode with delta dict (empty delta) + int size = encoder.encodeWithDeltaDict(batch, globalDict, -1, batchMaxId, false); + Assert.assertTrue(size > 0); + + // Verify flag is set + QwpBufferWriter buf = encoder.getBuffer(); + byte flags = Unsafe.getUnsafe().getByte(buf.getBufferPtr() + HEADER_OFFSET_FLAGS); + Assert.assertTrue((flags & FLAG_DELTA_SYMBOL_DICT) != 0); + } + } + + @Test + public void testEdgeCase_duplicateSymbolsInBatch() { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + + QwpTableBuffer batch = new QwpTableBuffer("test"); + QwpTableBuffer.ColumnBuffer col = batch.getOrCreateColumn("sym", TYPE_SYMBOL, false); + + // Same symbol used multiple times + int aaplId = globalDict.getOrAddSymbol("AAPL"); + col.addSymbolWithGlobalId("AAPL", aaplId); + batch.nextRow(); + col.addSymbolWithGlobalId("AAPL", aaplId); + batch.nextRow(); + col.addSymbolWithGlobalId("AAPL", aaplId); + batch.nextRow(); + + Assert.assertEquals(3, batch.getRowCount()); + Assert.assertEquals(1, globalDict.size()); // Only 1 unique symbol + + int maxGlobalId = col.getMaxGlobalSymbolId(); + Assert.assertEquals(0, maxGlobalId); // Max ID is 0 (AAPL) + } + + @Test + public void testEdgeCase_emptyBatch() { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + + // Pre-populate dictionary and send + globalDict.getOrAddSymbol("AAPL"); + int maxSentSymbolId = 0; + + // Empty batch (no rows, no symbols used) + QwpTableBuffer emptyBatch = new QwpTableBuffer("test"); + Assert.assertEquals(0, emptyBatch.getRowCount()); + + // Delta should still work (deltaCount = 0) + int deltaStart = maxSentSymbolId + 1; + int deltaCount = 0; + Assert.assertEquals(1, deltaStart); + Assert.assertEquals(0, deltaCount); + } + + @Test + public void testEdgeCase_gapFill() { + // Client dictionary: AAPL(0), GOOG(1), MSFT(2), TSLA(3) + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + globalDict.getOrAddSymbol("AAPL"); + globalDict.getOrAddSymbol("GOOG"); + globalDict.getOrAddSymbol("MSFT"); + globalDict.getOrAddSymbol("TSLA"); + + // Batch uses AAPL(0) and TSLA(3), skipping GOOG(1) and MSFT(2) + // Delta must include gap-fill: send all symbols from maxSentSymbolId+1 to batchMaxId + int maxSentSymbolId = -1; + int batchMaxId = 3; // TSLA + + int deltaStart = maxSentSymbolId + 1; + int deltaCount = batchMaxId - maxSentSymbolId; + + // Must send symbols 0, 1, 2, 3 (even though 1, 2 aren't used in this batch) + Assert.assertEquals(0, deltaStart); + Assert.assertEquals(4, deltaCount); + + // This ensures server has contiguous dictionary + for (int id = deltaStart; id < deltaStart + deltaCount; id++) { + String symbol = globalDict.getSymbol(id); + Assert.assertNotNull("Symbol " + id + " should exist", symbol); + } + } + + @Test + public void testEdgeCase_largeSymbolDictionary() { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + + // Add 1000 unique symbols + for (int i = 0; i < 1000; i++) { + int id = globalDict.getOrAddSymbol("SYM_" + i); + Assert.assertEquals(i, id); + } + + Assert.assertEquals(1000, globalDict.size()); + + // Send first batch with symbols 0-99 + int maxSentSymbolId = 99; + + // Next batch uses symbols 0-199, delta is 100-199 + int deltaStart = maxSentSymbolId + 1; + int deltaCount = 199 - maxSentSymbolId; + Assert.assertEquals(100, deltaStart); + Assert.assertEquals(100, deltaCount); + } + + @Test + public void testEdgeCase_nullSymbolValues() { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + + QwpTableBuffer batch = new QwpTableBuffer("test"); + QwpTableBuffer.ColumnBuffer col = batch.getOrCreateColumn("sym", TYPE_SYMBOL, true); // nullable + + int aaplId = globalDict.getOrAddSymbol("AAPL"); + col.addSymbolWithGlobalId("AAPL", aaplId); + batch.nextRow(); + + col.addSymbol(null); // NULL value + batch.nextRow(); + + col.addSymbolWithGlobalId("AAPL", aaplId); + batch.nextRow(); + + Assert.assertEquals(3, batch.getRowCount()); + // Dictionary only has 1 symbol (AAPL), NULL doesn't add to dictionary + Assert.assertEquals(1, globalDict.size()); + } + + @Test + public void testEdgeCase_unicodeSymbols() { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + + // Unicode symbols + int id1 = globalDict.getOrAddSymbol("日本"); + int id2 = globalDict.getOrAddSymbol("中国"); + int id3 = globalDict.getOrAddSymbol("한국"); + int id4 = globalDict.getOrAddSymbol("Émoji🚀"); + + Assert.assertEquals(0, id1); + Assert.assertEquals(1, id2); + Assert.assertEquals(2, id3); + Assert.assertEquals(3, id4); + + Assert.assertEquals("日本", globalDict.getSymbol(0)); + Assert.assertEquals("中国", globalDict.getSymbol(1)); + Assert.assertEquals("한국", globalDict.getSymbol(2)); + Assert.assertEquals("Émoji🚀", globalDict.getSymbol(3)); + } + + @Test + public void testEdgeCase_veryLongSymbol() { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + + // Create a very long symbol (1000 chars) + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < 1000; i++) { + sb.append('X'); + } + String longSymbol = sb.toString(); + + int id = globalDict.getOrAddSymbol(longSymbol); + Assert.assertEquals(0, id); + + String retrieved = globalDict.getSymbol(0); + Assert.assertEquals(longSymbol, retrieved); + Assert.assertEquals(1000, retrieved.length()); + } + + @Test + public void testMultipleBatches_encodeAndDecode() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + GlobalSymbolDictionary clientDict = new GlobalSymbolDictionary(); + ObjList serverDict = new ObjList<>(); + int maxSentSymbolId = -1; + QwpTableBuffer batch1 = new QwpTableBuffer("test"); + QwpTableBuffer.ColumnBuffer col1 = batch1.getOrCreateColumn("sym", TYPE_SYMBOL, false); + + int aaplId = clientDict.getOrAddSymbol("AAPL"); + int googId = clientDict.getOrAddSymbol("GOOG"); + col1.addSymbolWithGlobalId("AAPL", aaplId); + batch1.nextRow(); + col1.addSymbolWithGlobalId("GOOG", googId); + batch1.nextRow(); + + int batch1MaxId = 1; + int size1 = encoder.encodeWithDeltaDict(batch1, clientDict, maxSentSymbolId, batch1MaxId, false); + Assert.assertTrue(size1 > 0); + maxSentSymbolId = batch1MaxId; + + // Decode on server side + QwpBufferWriter buf1 = encoder.getBuffer(); + decodeAndAccumulateDict(buf1.getBufferPtr(), size1, serverDict); + + // Verify server dictionary + Assert.assertEquals(2, serverDict.size()); + Assert.assertEquals("AAPL", serverDict.get(0)); + Assert.assertEquals("GOOG", serverDict.get(1)); + QwpTableBuffer batch2 = new QwpTableBuffer("test"); + QwpTableBuffer.ColumnBuffer col2 = batch2.getOrCreateColumn("sym", TYPE_SYMBOL, false); + + int msftId = clientDict.getOrAddSymbol("MSFT"); + col2.addSymbolWithGlobalId("AAPL", aaplId); // Existing + batch2.nextRow(); + col2.addSymbolWithGlobalId("MSFT", msftId); // New + batch2.nextRow(); + + int batch2MaxId = 2; + int size2 = encoder.encodeWithDeltaDict(batch2, clientDict, maxSentSymbolId, batch2MaxId, false); + Assert.assertTrue(size2 > 0); + maxSentSymbolId = batch2MaxId; + + // Decode batch 2 + QwpBufferWriter buf2 = encoder.getBuffer(); + decodeAndAccumulateDict(buf2.getBufferPtr(), size2, serverDict); + + // Server dictionary should now have 3 symbols + Assert.assertEquals(3, serverDict.size()); + Assert.assertEquals("MSFT", serverDict.get(2)); + } + } + + @Test + public void testMultipleBatches_progressiveSymbolAccumulation() { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + + // Batch 1: AAPL, GOOG + int aaplId = globalDict.getOrAddSymbol("AAPL"); + int googId = globalDict.getOrAddSymbol("GOOG"); + int batch1MaxId = Math.max(aaplId, googId); + + // Simulate sending batch 1 - maxSentSymbolId = 1 after send + int maxSentSymbolId = batch1MaxId; // 1 + + // Batch 2: AAPL (existing), MSFT (new), TSLA (new) + globalDict.getOrAddSymbol("AAPL"); // Returns 0, already exists + int msftId = globalDict.getOrAddSymbol("MSFT"); + int tslaId = globalDict.getOrAddSymbol("TSLA"); + int batch2MaxId = Math.max(msftId, tslaId); + + // Delta for batch 2 should be [2, 3] (MSFT, TSLA) + int deltaStart = maxSentSymbolId + 1; + int deltaCount = batch2MaxId - maxSentSymbolId; + Assert.assertEquals(2, deltaStart); + Assert.assertEquals(2, deltaCount); + + // Simulate sending batch 2 + maxSentSymbolId = batch2MaxId; // 3 + + // Batch 3: All existing symbols (no delta needed) + globalDict.getOrAddSymbol("AAPL"); + globalDict.getOrAddSymbol("GOOG"); + int batch3MaxId = 1; // Max used is GOOG(1) + + deltaStart = maxSentSymbolId + 1; + deltaCount = Math.max(0, batch3MaxId - maxSentSymbolId); + Assert.assertEquals(4, deltaStart); + Assert.assertEquals(0, deltaCount); // No new symbols + } + + @Test + public void testMultipleTables_encodedInSameBatch() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + + // Create two tables + QwpTableBuffer table1 = new QwpTableBuffer("trades"); + QwpTableBuffer table2 = new QwpTableBuffer("quotes"); + + // Table 1: ticker column + QwpTableBuffer.ColumnBuffer col1 = table1.getOrCreateColumn("ticker", TYPE_SYMBOL, false); + int aaplId = globalDict.getOrAddSymbol("AAPL"); + int googId = globalDict.getOrAddSymbol("GOOG"); + col1.addSymbolWithGlobalId("AAPL", aaplId); + table1.nextRow(); + col1.addSymbolWithGlobalId("GOOG", googId); + table1.nextRow(); + + // Table 2: symbol column (different name, but shares dictionary) + QwpTableBuffer.ColumnBuffer col2 = table2.getOrCreateColumn("symbol", TYPE_SYMBOL, false); + int msftId = globalDict.getOrAddSymbol("MSFT"); + col2.addSymbolWithGlobalId("AAPL", aaplId); // Reuse AAPL + table2.nextRow(); + col2.addSymbolWithGlobalId("MSFT", msftId); + table2.nextRow(); + + // Encode first table with delta dict + int confirmedMaxId = -1; + int batchMaxId = 2; // AAPL(0), GOOG(1), MSFT(2) + + int size = encoder.encodeWithDeltaDict(table1, globalDict, confirmedMaxId, batchMaxId, false); + Assert.assertTrue(size > 0); + + // Verify delta section contains all 3 symbols + QwpBufferWriter buf = encoder.getBuffer(); + long ptr = buf.getBufferPtr(); + + byte flags = Unsafe.getUnsafe().getByte(ptr + HEADER_OFFSET_FLAGS); + Assert.assertTrue((flags & FLAG_DELTA_SYMBOL_DICT) != 0); + + // After header: deltaStart=0, deltaCount=3 + long pos = ptr + HEADER_SIZE; + int deltaStart = readVarint(pos); + Assert.assertEquals(0, deltaStart); + } + } + + @Test + public void testMultipleTables_multipleSymbolColumns() { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + + QwpTableBuffer table = new QwpTableBuffer("market_data"); + + // Column 1: exchange + QwpTableBuffer.ColumnBuffer exchangeCol = table.getOrCreateColumn("exchange", TYPE_SYMBOL, false); + int nyseId = globalDict.getOrAddSymbol("NYSE"); + int nasdaqId = globalDict.getOrAddSymbol("NASDAQ"); + + // Column 2: currency + QwpTableBuffer.ColumnBuffer currencyCol = table.getOrCreateColumn("currency", TYPE_SYMBOL, false); + int usdId = globalDict.getOrAddSymbol("USD"); + int eurId = globalDict.getOrAddSymbol("EUR"); + + // Column 3: ticker + QwpTableBuffer.ColumnBuffer tickerCol = table.getOrCreateColumn("ticker", TYPE_SYMBOL, false); + int aaplId = globalDict.getOrAddSymbol("AAPL"); + + // Add row with all three columns + exchangeCol.addSymbolWithGlobalId("NYSE", nyseId); + currencyCol.addSymbolWithGlobalId("USD", usdId); + tickerCol.addSymbolWithGlobalId("AAPL", aaplId); + table.nextRow(); + + exchangeCol.addSymbolWithGlobalId("NASDAQ", nasdaqId); + currencyCol.addSymbolWithGlobalId("EUR", eurId); + tickerCol.addSymbolWithGlobalId("AAPL", aaplId); // Reuse AAPL + table.nextRow(); + + // All symbols share the same global dictionary + Assert.assertEquals(5, globalDict.size()); + Assert.assertEquals("NYSE", globalDict.getSymbol(0)); + Assert.assertEquals("NASDAQ", globalDict.getSymbol(1)); + Assert.assertEquals("USD", globalDict.getSymbol(2)); + Assert.assertEquals("EUR", globalDict.getSymbol(3)); + Assert.assertEquals("AAPL", globalDict.getSymbol(4)); + } + + @Test + public void testMultipleTables_sharedGlobalDictionary() { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + + // Table 1 uses symbols AAPL, GOOG + int aaplId = globalDict.getOrAddSymbol("AAPL"); + int googId = globalDict.getOrAddSymbol("GOOG"); + + // Table 2 uses symbols AAPL (reused), MSFT (new) + int aaplId2 = globalDict.getOrAddSymbol("AAPL"); // Should return same ID + int msftId = globalDict.getOrAddSymbol("MSFT"); + + // Verify deduplication + Assert.assertEquals(0, aaplId); + Assert.assertEquals(1, googId); + Assert.assertEquals(0, aaplId2); // Same as aaplId + Assert.assertEquals(2, msftId); + + // Total symbols should be 3 + Assert.assertEquals(3, globalDict.size()); + } + + @Test + public void testReconnection_fullDeltaAfterReconnect() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + GlobalSymbolDictionary clientDict = new GlobalSymbolDictionary(); + + // First connection: add symbols + int aaplId = clientDict.getOrAddSymbol("AAPL"); + clientDict.getOrAddSymbol("GOOG"); + + // Send batch - maxSentSymbolId = 1 + int maxSentSymbolId = 1; + + // Reconnect - reset maxSentSymbolId + maxSentSymbolId = -1; + + // Create new batch using existing symbols + QwpTableBuffer batch = new QwpTableBuffer("test"); + QwpTableBuffer.ColumnBuffer col = batch.getOrCreateColumn("sym", TYPE_SYMBOL, false); + col.addSymbolWithGlobalId("AAPL", aaplId); + batch.nextRow(); + + // Encode - should send full delta (all symbols from 0) + int size = encoder.encodeWithDeltaDict(batch, clientDict, maxSentSymbolId, 1, false); + Assert.assertTrue(size > 0); + + // Verify deltaStart is 0 + QwpBufferWriter buf = encoder.getBuffer(); + long pos = buf.getBufferPtr() + HEADER_SIZE; + int deltaStart = readVarint(pos); + Assert.assertEquals(0, deltaStart); + } + } + + @Test + public void testReconnection_resetsWatermark() { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + + // Build up dictionary and "send" some symbols + globalDict.getOrAddSymbol("AAPL"); + globalDict.getOrAddSymbol("GOOG"); + globalDict.getOrAddSymbol("MSFT"); + + int maxSentSymbolId = 2; + + // Simulate reconnection - reset maxSentSymbolId + maxSentSymbolId = -1; + Assert.assertEquals(-1, maxSentSymbolId); + + // Global dictionary is NOT cleared (it's client-side) + Assert.assertEquals(3, globalDict.size()); + + // Next batch must send full delta from 0 + int deltaStart = maxSentSymbolId + 1; + Assert.assertEquals(0, deltaStart); + } + + @Test + public void testReconnection_serverDictionaryCleared() { + ObjList serverDict = new ObjList<>(); + + // Simulate first connection + serverDict.add("AAPL"); + serverDict.add("GOOG"); + Assert.assertEquals(2, serverDict.size()); + + // Simulate reconnection - server clears dictionary + serverDict.clear(); + Assert.assertEquals(0, serverDict.size()); + + // New connection starts fresh + serverDict.add("MSFT"); + Assert.assertEquals(1, serverDict.size()); + Assert.assertEquals("MSFT", serverDict.get(0)); + } + + @Test + public void testServerSide_accumulateDelta() { + ObjList serverDict = new ObjList<>(); + + // First batch: symbols 0-2 + accumulateDelta(serverDict, 0, new String[]{"AAPL", "GOOG", "MSFT"}); + + Assert.assertEquals(3, serverDict.size()); + Assert.assertEquals("AAPL", serverDict.get(0)); + Assert.assertEquals("GOOG", serverDict.get(1)); + Assert.assertEquals("MSFT", serverDict.get(2)); + + // Second batch: symbols 3-4 + accumulateDelta(serverDict, 3, new String[]{"TSLA", "AMZN"}); + + Assert.assertEquals(5, serverDict.size()); + Assert.assertEquals("TSLA", serverDict.get(3)); + Assert.assertEquals("AMZN", serverDict.get(4)); + + // Third batch: no new symbols (empty delta) + accumulateDelta(serverDict, 5, new String[]{}); + Assert.assertEquals(5, serverDict.size()); + } + + @Test + public void testServerSide_resolveSymbol() { + ObjList serverDict = new ObjList<>(); + serverDict.add("AAPL"); + serverDict.add("GOOG"); + serverDict.add("MSFT"); + + // Resolve by global ID + Assert.assertEquals("AAPL", serverDict.get(0)); + Assert.assertEquals("GOOG", serverDict.get(1)); + Assert.assertEquals("MSFT", serverDict.get(2)); + } + + private void accumulateDelta(ObjList serverDict, int deltaStart, String[] symbols) { + // Ensure capacity + while (serverDict.size() < deltaStart + symbols.length) { + serverDict.add(null); + } + // Add symbols + for (int i = 0; i < symbols.length; i++) { + serverDict.setQuick(deltaStart + i, symbols[i]); + } + } + + private void decodeAndAccumulateDict(long ptr, int size, ObjList serverDict) { + // Parse header + byte flags = Unsafe.getUnsafe().getByte(ptr + HEADER_OFFSET_FLAGS); + if ((flags & FLAG_DELTA_SYMBOL_DICT) == 0) { + return; // No delta dict + } + + // Parse delta section after header + long pos = ptr + HEADER_SIZE; + + // Read deltaStart + int deltaStart = readVarint(pos); + pos += 1; // Assuming single-byte varint + + // Read deltaCount + int deltaCount = readVarint(pos); + pos += 1; + + // Ensure capacity + while (serverDict.size() < deltaStart + deltaCount) { + serverDict.add(null); + } + + // Read symbols + for (int i = 0; i < deltaCount; i++) { + int len = readVarint(pos); + pos += 1; + + byte[] bytes = new byte[len]; + for (int j = 0; j < len; j++) { + bytes[j] = Unsafe.getUnsafe().getByte(pos + j); + } + pos += len; + + serverDict.setQuick(deltaStart + i, new String(bytes, java.nio.charset.StandardCharsets.UTF_8)); + } + } + + private int readVarint(long address) { + byte b = Unsafe.getUnsafe().getByte(address); + if ((b & 0x80) == 0) { + return b & 0x7F; + } + // For simplicity, only handle single-byte varints in tests + return b & 0x7F; + } +} From 793f6f627d0da211ea51f4fd1e1bb39f1dcc09fa Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 26 Feb 2026 15:27:03 +0100 Subject: [PATCH 77/89] Move 5 pure client tests to client submodule Move NativeBufferWriterTest, MicrobatchBufferTest, QwpWebSocketEncoderTest, QwpWebSocketSenderTest, and WebSocketSendQueueTest from core's websocket test directory into the java-questdb-client submodule. These tests only exercise classes in io.questdb.client.* and do not depend on core-module server-side classes. For NativeBufferWriterTest and MicrobatchBufferTest, the client module already had versions with different test methods, so the core methods are merged into the existing files. For the other three, new files are created with package and import paths adjusted to the client module (io.questdb.std -> io.questdb.client.std, etc.). Four integration tests that span both client and core modules remain in the core test tree. Co-Authored-By: Claude Opus 4.6 --- .../qwp/client/MicrobatchBufferTest.java | 639 +++++++++ .../qwp/client/NativeBufferWriterTest.java | 335 ++++- .../qwp/client/QwpWebSocketEncoderTest.java | 1254 +++++++++++++++++ .../qwp/client/QwpWebSocketSenderTest.java | 458 ++++++ .../qwp/client/WebSocketSendQueueTest.java | 339 +++++ 5 files changed, 2991 insertions(+), 34 deletions(-) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketSendQueueTest.java diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/MicrobatchBufferTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/MicrobatchBufferTest.java index a1f1ecf..b5f4355 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/MicrobatchBufferTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/MicrobatchBufferTest.java @@ -25,18 +25,98 @@ package io.questdb.client.test.cutlass.qwp.client; import io.questdb.client.cutlass.qwp.client.MicrobatchBuffer; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; import org.junit.Test; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; public class MicrobatchBufferTest { + @Test + public void testAwaitRecycled() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.seal(); + buffer.markSending(); + + AtomicBoolean recycled = new AtomicBoolean(false); + CountDownLatch started = new CountDownLatch(1); + + Thread waiter = new Thread(() -> { + started.countDown(); + buffer.awaitRecycled(); + recycled.set(true); + }); + waiter.start(); + + started.await(); + Thread.sleep(50); // Give waiter time to start waiting + Assert.assertFalse(recycled.get()); + + buffer.markRecycled(); + waiter.join(1000); + + Assert.assertTrue(recycled.get()); + } + }); + } + + @Test + public void testAwaitRecycledWithTimeout() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.seal(); + buffer.markSending(); + + // Should timeout + boolean result = buffer.awaitRecycled(50, TimeUnit.MILLISECONDS); + Assert.assertFalse(result); + + buffer.markRecycled(); + + // Should succeed immediately now + result = buffer.awaitRecycled(50, TimeUnit.MILLISECONDS); + Assert.assertTrue(result); + } + }); + } + + @Test + public void testBatchIdIncrementsOnReset() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + long id1 = buffer.getBatchId(); + + buffer.seal(); + buffer.markSending(); + buffer.markRecycled(); + buffer.reset(); + + long id2 = buffer.getBatchId(); + Assert.assertNotEquals(id1, id2); + + buffer.seal(); + buffer.markSending(); + buffer.markRecycled(); + buffer.reset(); + + long id3 = buffer.getBatchId(); + Assert.assertNotEquals(id2, id3); + } + }); + } + @Test public void testConcurrentBatchIdUniqueness() throws Exception { int threadCount = 8; @@ -122,4 +202,563 @@ public void testConcurrentResetBatchIdUniqueness() throws Exception { batchIds.size() ); } + + @Test + public void testConcurrentStateTransitions() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + AtomicReference error = new AtomicReference<>(); + CountDownLatch userDone = new CountDownLatch(1); + CountDownLatch ioDone = new CountDownLatch(1); + + // Simulate user thread + Thread userThread = new Thread(() -> { + try { + buffer.writeByte((byte) 1); + buffer.incrementRowCount(); + buffer.seal(); + userDone.countDown(); + + // Wait for I/O thread to recycle + buffer.awaitRecycled(); + + // Reset and write again + buffer.reset(); + buffer.writeByte((byte) 2); + } catch (Throwable t) { + error.set(t); + } + }); + + // Simulate I/O thread + Thread ioThread = new Thread(() -> { + try { + userDone.await(); + buffer.markSending(); + + // Simulate sending + Thread.sleep(10); + + buffer.markRecycled(); + ioDone.countDown(); + } catch (Throwable t) { + error.set(t); + } + }); + + userThread.start(); + ioThread.start(); + + userThread.join(1000); + ioThread.join(1000); + + Assert.assertNull(error.get()); + Assert.assertTrue(buffer.isFilling()); + Assert.assertEquals(1, buffer.getBufferPos()); + } + }); + } + + @Test + public void testConstructionWithCustomThresholds() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024, 100, 4096, 1_000_000_000L)) { + Assert.assertEquals(1024, buffer.getBufferCapacity()); + Assert.assertTrue(buffer.isFilling()); + } + }); + } + + @Test + public void testConstructionWithDefaultThresholds() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + Assert.assertEquals(1024, buffer.getBufferCapacity()); + Assert.assertEquals(0, buffer.getBufferPos()); + Assert.assertEquals(0, buffer.getRowCount()); + Assert.assertTrue(buffer.isFilling()); + Assert.assertFalse(buffer.hasData()); + } + }); + } + + @Test(expected = IllegalArgumentException.class) + public void testConstructionWithNegativeCapacity() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer ignored = new MicrobatchBuffer(-1)) { + Assert.fail("Should have thrown"); + } + }); + } + + @Test(expected = IllegalArgumentException.class) + public void testConstructionWithZeroCapacity() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer ignored = new MicrobatchBuffer(0)) { + Assert.fail("Should have thrown"); + } + }); + } + + @Test + public void testEnsureCapacityGrows() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.ensureCapacity(2000); + Assert.assertTrue(buffer.getBufferCapacity() >= 2000); + } + }); + } + + @Test + public void testEnsureCapacityNoGrowth() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.ensureCapacity(512); + Assert.assertEquals(1024, buffer.getBufferCapacity()); // No change + } + }); + } + + @Test + public void testFirstRowTimeIsRecorded() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + Assert.assertEquals(0, buffer.getAgeNanos()); + + buffer.incrementRowCount(); + long age1 = buffer.getAgeNanos(); + Assert.assertTrue(age1 >= 0); + + Thread.sleep(10); + + long age2 = buffer.getAgeNanos(); + Assert.assertTrue(age2 > age1); + } + }); + } + + @Test + public void testFullStateLifecycle() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + // FILLING + Assert.assertTrue(buffer.isFilling()); + buffer.writeByte((byte) 1); + buffer.incrementRowCount(); + + // FILLING -> SEALED + buffer.seal(); + Assert.assertTrue(buffer.isSealed()); + + // SEALED -> SENDING + buffer.markSending(); + Assert.assertTrue(buffer.isSending()); + + // SENDING -> RECYCLED + buffer.markRecycled(); + Assert.assertTrue(buffer.isRecycled()); + + // RECYCLED -> FILLING (reset) + buffer.reset(); + Assert.assertTrue(buffer.isFilling()); + Assert.assertFalse(buffer.hasData()); + } + }); + } + + @Test + public void testIncrementRowCount() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + Assert.assertEquals(0, buffer.getRowCount()); + buffer.incrementRowCount(); + Assert.assertEquals(1, buffer.getRowCount()); + buffer.incrementRowCount(); + Assert.assertEquals(2, buffer.getRowCount()); + } + }); + } + + @Test(expected = IllegalStateException.class) + public void testIncrementRowCountWhenSealed() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.seal(); + buffer.incrementRowCount(); // Should throw + } + }); + } + + @Test + public void testInitialState() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + Assert.assertEquals(MicrobatchBuffer.STATE_FILLING, buffer.getState()); + Assert.assertTrue(buffer.isFilling()); + Assert.assertFalse(buffer.isSealed()); + Assert.assertFalse(buffer.isSending()); + Assert.assertFalse(buffer.isRecycled()); + Assert.assertFalse(buffer.isInUse()); + } + }); + } + + @Test + public void testMarkRecycledTransition() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.seal(); + buffer.markSending(); + buffer.markRecycled(); + + Assert.assertEquals(MicrobatchBuffer.STATE_RECYCLED, buffer.getState()); + Assert.assertTrue(buffer.isRecycled()); + Assert.assertFalse(buffer.isInUse()); + } + }); + } + + @Test(expected = IllegalStateException.class) + public void testMarkRecycledWhenNotSending() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.seal(); + buffer.markRecycled(); // Should throw - not sending + } + }); + } + + @Test + public void testMarkSendingTransition() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.seal(); + buffer.markSending(); + + Assert.assertEquals(MicrobatchBuffer.STATE_SENDING, buffer.getState()); + Assert.assertTrue(buffer.isSending()); + Assert.assertTrue(buffer.isInUse()); + } + }); + } + + @Test(expected = IllegalStateException.class) + public void testMarkSendingWhenNotSealed() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.markSending(); // Should throw - not sealed + } + }); + } + + @Test + public void testResetFromRecycled() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.writeByte((byte) 1); + buffer.incrementRowCount(); + long oldBatchId = buffer.getBatchId(); + + buffer.seal(); + buffer.markSending(); + buffer.markRecycled(); + buffer.reset(); + + Assert.assertTrue(buffer.isFilling()); + Assert.assertEquals(0, buffer.getBufferPos()); + Assert.assertEquals(0, buffer.getRowCount()); + Assert.assertNotEquals(oldBatchId, buffer.getBatchId()); + } + }); + } + + @Test(expected = IllegalStateException.class) + public void testResetWhenSealed() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.seal(); + buffer.reset(); // Should throw + } + }); + } + + @Test(expected = IllegalStateException.class) + public void testResetWhenSending() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.seal(); + buffer.markSending(); + buffer.reset(); // Should throw + } + }); + } + + @Test + public void testRollbackSealForRetry() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.writeByte((byte) 1); + buffer.incrementRowCount(); + + buffer.seal(); + Assert.assertTrue(buffer.isSealed()); + + buffer.rollbackSealForRetry(); + Assert.assertTrue(buffer.isFilling()); + + // Verify the same batch remains writable after rollback. + buffer.writeByte((byte) 2); + buffer.incrementRowCount(); + Assert.assertEquals(2, buffer.getBufferPos()); + Assert.assertEquals(2, buffer.getRowCount()); + } + }); + } + + @Test(expected = IllegalStateException.class) + public void testRollbackSealWhenNotSealed() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.rollbackSealForRetry(); // Should throw - not sealed + } + }); + } + + @Test + public void testSealTransition() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.writeByte((byte) 1); + buffer.seal(); + + Assert.assertEquals(MicrobatchBuffer.STATE_SEALED, buffer.getState()); + Assert.assertFalse(buffer.isFilling()); + Assert.assertTrue(buffer.isSealed()); + Assert.assertTrue(buffer.isInUse()); + } + }); + } + + @Test(expected = IllegalStateException.class) + public void testSealWhenNotFilling() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.seal(); + buffer.seal(); // Should throw + } + }); + } + + @Test + public void testSetBufferPos() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.setBufferPos(100); + Assert.assertEquals(100, buffer.getBufferPos()); + } + }); + } + + @Test(expected = IllegalArgumentException.class) + public void testSetBufferPosNegative() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.setBufferPos(-1); + } + }); + } + + @Test(expected = IllegalArgumentException.class) + public void testSetBufferPosOutOfBounds() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.setBufferPos(2000); + } + }); + } + + @Test + public void testShouldFlushAgeLimit() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // 50ms timeout + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024, 0, 0, 50_000_000L)) { + buffer.writeByte((byte) 1); + buffer.incrementRowCount(); + Assert.assertFalse(buffer.shouldFlush()); + + Thread.sleep(60); + + Assert.assertTrue(buffer.shouldFlush()); + Assert.assertTrue(buffer.isAgeLimitExceeded()); + } + }); + } + + @Test + public void testShouldFlushByteLimit() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024, 0, 10, 0)) { + for (int i = 0; i < 9; i++) { + buffer.writeByte((byte) i); + buffer.incrementRowCount(); + Assert.assertFalse(buffer.shouldFlush()); + } + buffer.writeByte((byte) 9); + buffer.incrementRowCount(); + Assert.assertTrue(buffer.shouldFlush()); + Assert.assertTrue(buffer.isByteLimitExceeded()); + } + }); + } + + @Test + public void testShouldFlushEmptyBuffer() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024, 1, 1, 1)) { + Assert.assertFalse(buffer.shouldFlush()); // Empty buffer never flushes + } + }); + } + + @Test + public void testShouldFlushRowLimit() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024, 5, 0, 0)) { + for (int i = 0; i < 4; i++) { + buffer.writeByte((byte) i); + buffer.incrementRowCount(); + Assert.assertFalse(buffer.shouldFlush()); + } + buffer.writeByte((byte) 4); + buffer.incrementRowCount(); + Assert.assertTrue(buffer.shouldFlush()); + Assert.assertTrue(buffer.isRowLimitExceeded()); + } + }); + } + + @Test + public void testShouldFlushWithNoThresholds() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.writeByte((byte) 1); + buffer.incrementRowCount(); + Assert.assertFalse(buffer.shouldFlush()); // No thresholds set + } + }); + } + + @Test + public void testStateName() { + Assert.assertEquals("FILLING", MicrobatchBuffer.stateName(MicrobatchBuffer.STATE_FILLING)); + Assert.assertEquals("SEALED", MicrobatchBuffer.stateName(MicrobatchBuffer.STATE_SEALED)); + Assert.assertEquals("SENDING", MicrobatchBuffer.stateName(MicrobatchBuffer.STATE_SENDING)); + Assert.assertEquals("RECYCLED", MicrobatchBuffer.stateName(MicrobatchBuffer.STATE_RECYCLED)); + Assert.assertEquals("UNKNOWN(99)", MicrobatchBuffer.stateName(99)); + } + + @Test + public void testToString() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.writeByte((byte) 1); + buffer.incrementRowCount(); + + String str = buffer.toString(); + Assert.assertTrue(str.contains("MicrobatchBuffer")); + Assert.assertTrue(str.contains("state=FILLING")); + Assert.assertTrue(str.contains("rows=1")); + Assert.assertTrue(str.contains("bytes=1")); + } + }); + } + + @Test + public void testWriteBeyondInitialCapacity() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(16)) { + // Write more than initial capacity + for (int i = 0; i < 100; i++) { + buffer.writeByte((byte) i); + } + Assert.assertEquals(100, buffer.getBufferPos()); + Assert.assertTrue(buffer.getBufferCapacity() >= 100); + + // Verify data integrity after growth + for (int i = 0; i < 100; i++) { + byte read = Unsafe.getUnsafe().getByte(buffer.getBufferPtr() + i); + Assert.assertEquals((byte) i, read); + } + } + }); + } + + @Test + public void testWriteByte() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.writeByte((byte) 0x42); + Assert.assertEquals(1, buffer.getBufferPos()); + Assert.assertTrue(buffer.hasData()); + + byte read = Unsafe.getUnsafe().getByte(buffer.getBufferPtr()); + Assert.assertEquals((byte) 0x42, read); + } + }); + } + + @Test + public void testWriteFromNativeMemory() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long src = Unsafe.malloc(10, MemoryTag.NATIVE_DEFAULT); + try { + // Fill source with test data + for (int i = 0; i < 10; i++) { + Unsafe.getUnsafe().putByte(src + i, (byte) (i + 100)); + } + + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.write(src, 10); + Assert.assertEquals(10, buffer.getBufferPos()); + + // Verify data + for (int i = 0; i < 10; i++) { + byte read = Unsafe.getUnsafe().getByte(buffer.getBufferPtr() + i); + Assert.assertEquals((byte) (i + 100), read); + } + } + } finally { + Unsafe.free(src, 10, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + @Test + public void testWriteMultipleBytes() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + for (int i = 0; i < 100; i++) { + buffer.writeByte((byte) i); + } + Assert.assertEquals(100, buffer.getBufferPos()); + + // Verify data + for (int i = 0; i < 100; i++) { + byte read = Unsafe.getUnsafe().getByte(buffer.getBufferPtr() + i); + Assert.assertEquals((byte) i, read); + } + } + }); + } + + @Test(expected = IllegalStateException.class) + public void testWriteWhenSealed() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (MicrobatchBuffer buffer = new MicrobatchBuffer(1024)) { + buffer.seal(); + buffer.writeByte((byte) 1); // Should throw + } + }); + } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java index bfa2bc7..c77f8d3 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java @@ -30,9 +30,7 @@ import org.junit.Assert; import org.junit.Test; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import static org.junit.Assert.*; public class NativeBufferWriterTest { @@ -45,6 +43,66 @@ public void testEnsureCapacityGrowsBuffer() { } } + @Test + public void testGrowBuffer() { + try (NativeBufferWriter writer = new NativeBufferWriter(16)) { + // Write more than initial capacity + for (int i = 0; i < 100; i++) { + writer.putLong(i); + } + Assert.assertEquals(800, writer.getPosition()); + // Verify data + for (int i = 0; i < 100; i++) { + Assert.assertEquals(i, Unsafe.getUnsafe().getLong(writer.getBufferPtr() + i * 8)); + } + } + } + + @Test + public void testMultipleWrites() { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + writer.putByte((byte) 'I'); + writer.putByte((byte) 'L'); + writer.putByte((byte) 'P'); + writer.putByte((byte) '4'); + writer.putByte((byte) 1); // Version + writer.putByte((byte) 0); // Flags + writer.putShort((short) 1); // Table count + writer.putInt(0); // Payload length placeholder + + Assert.assertEquals(12, writer.getPosition()); + + // Verify ILP4 header + Assert.assertEquals((byte) 'I', Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + Assert.assertEquals((byte) 'L', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 1)); + Assert.assertEquals((byte) 'P', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 2)); + Assert.assertEquals((byte) '4', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 3)); + } + } + + @Test + public void testNativeBufferWriterUtf8LengthInvalidSurrogatePair() { + // High surrogate followed by non-low-surrogate: '?' (1) + 'X' (1) = 2 + assertEquals(2, NativeBufferWriter.utf8Length("\uD800X")); + // Lone high surrogate at end: '?' (1) + assertEquals(1, NativeBufferWriter.utf8Length("\uD800")); + // Lone low surrogate: '?' (1) + assertEquals(1, NativeBufferWriter.utf8Length("\uDC00")); + // Valid pair still works: 4 bytes + assertEquals(4, NativeBufferWriter.utf8Length("\uD83D\uDE00")); + } + + @Test + public void testPatchInt() { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + writer.putInt(0); // Placeholder at offset 0 + writer.putInt(100); // At offset 4 + writer.patchInt(0, 42); // Patch first int + Assert.assertEquals(42, Unsafe.getUnsafe().getInt(writer.getBufferPtr())); + Assert.assertEquals(100, Unsafe.getUnsafe().getInt(writer.getBufferPtr() + 4)); + } + } + @Test public void testPatchIntAtLastValidOffset() { try (NativeBufferWriter writer = new NativeBufferWriter(16)) { @@ -68,26 +126,22 @@ public void testPatchIntAtValidOffset() { } @Test - public void testSkipAdvancesPosition() { - try (NativeBufferWriter writer = new NativeBufferWriter(16)) { - writer.skip(4); - assertEquals(4, writer.getPosition()); - writer.skip(8); - assertEquals(12, writer.getPosition()); - } - } + public void testPutBlockOfBytes() { + try (NativeBufferWriter writer = new NativeBufferWriter(); + NativeBufferWriter source = new NativeBufferWriter()) { + // Prepare source data + source.putByte((byte) 1); + source.putByte((byte) 2); + source.putByte((byte) 3); + source.putByte((byte) 4); - @Test - public void testSkipBeyondCapacityGrowsBuffer() { - try (NativeBufferWriter writer = new NativeBufferWriter(16)) { - // skip past the 16-byte buffer — must grow, not corrupt memory - writer.skip(32); - assertEquals(32, writer.getPosition()); - assertTrue(writer.getCapacity() >= 32); - // writing after the skip must also succeed - writer.putInt(0xCAFE); - assertEquals(36, writer.getPosition()); - assertEquals(0xCAFE, Unsafe.getUnsafe().getInt(writer.getBufferPtr() + 32)); + // Copy to writer + writer.putBlockOfBytes(source.getBufferPtr(), 4); + Assert.assertEquals(4, writer.getPosition()); + Assert.assertEquals((byte) 1, Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + Assert.assertEquals((byte) 2, Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 1)); + Assert.assertEquals((byte) 3, Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 2)); + Assert.assertEquals((byte) 4, Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 3)); } } @@ -149,18 +203,6 @@ public void testPutUtf8LoneSurrogateMatchesUtf8Length() { } } - @Test - public void testNativeBufferWriterUtf8LengthInvalidSurrogatePair() { - // High surrogate followed by non-low-surrogate: '?' (1) + 'X' (1) = 2 - assertEquals(2, NativeBufferWriter.utf8Length("\uD800X")); - // Lone high surrogate at end: '?' (1) - assertEquals(1, NativeBufferWriter.utf8Length("\uD800")); - // Lone low surrogate: '?' (1) - assertEquals(1, NativeBufferWriter.utf8Length("\uDC00")); - // Valid pair still works: 4 bytes - assertEquals(4, NativeBufferWriter.utf8Length("\uD83D\uDE00")); - } - @Test public void testQwpBufferWriterUtf8LengthInvalidSurrogatePair() { // High surrogate followed by non-low-surrogate: '?' (1) + 'X' (1) = 2 @@ -173,6 +215,43 @@ public void testQwpBufferWriterUtf8LengthInvalidSurrogatePair() { assertEquals(4, QwpBufferWriter.utf8Length("\uD83D\uDE00")); } + @Test + public void testReset() { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + writer.putInt(12345); + Assert.assertEquals(4, writer.getPosition()); + writer.reset(); + Assert.assertEquals(0, writer.getPosition()); + // Can write again + writer.putByte((byte) 0xFF); + Assert.assertEquals(1, writer.getPosition()); + } + } + + @Test + public void testSkipAdvancesPosition() { + try (NativeBufferWriter writer = new NativeBufferWriter(16)) { + writer.skip(4); + assertEquals(4, writer.getPosition()); + writer.skip(8); + assertEquals(12, writer.getPosition()); + } + } + + @Test + public void testSkipBeyondCapacityGrowsBuffer() { + try (NativeBufferWriter writer = new NativeBufferWriter(16)) { + // skip past the 16-byte buffer — must grow, not corrupt memory + writer.skip(32); + assertEquals(32, writer.getPosition()); + assertTrue(writer.getCapacity() >= 32); + // writing after the skip must also succeed + writer.putInt(0xCAFE); + assertEquals(36, writer.getPosition()); + assertEquals(0xCAFE, Unsafe.getUnsafe().getInt(writer.getBufferPtr() + 32)); + } + } + @Test public void testSkipThenPatchInt() { try (NativeBufferWriter writer = new NativeBufferWriter(8)) { @@ -185,4 +264,192 @@ public void testSkipThenPatchInt() { assertEquals(0xDEAD, Unsafe.getUnsafe().getInt(writer.getBufferPtr() + 4)); } } + + @Test + public void testUtf8Length() { + Assert.assertEquals(0, NativeBufferWriter.utf8Length(null)); + Assert.assertEquals(0, NativeBufferWriter.utf8Length("")); + Assert.assertEquals(5, NativeBufferWriter.utf8Length("hello")); + Assert.assertEquals(2, NativeBufferWriter.utf8Length("ñ")); + Assert.assertEquals(3, NativeBufferWriter.utf8Length("€")); + } + + @Test + public void testWriteByte() { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + writer.putByte((byte) 0x42); + Assert.assertEquals(1, writer.getPosition()); + Assert.assertEquals((byte) 0x42, Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + } + } + + @Test + public void testWriteDouble() { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + writer.putDouble(3.14159265359); + Assert.assertEquals(8, writer.getPosition()); + Assert.assertEquals(3.14159265359, Unsafe.getUnsafe().getDouble(writer.getBufferPtr()), 0.0000000001); + } + } + + @Test + public void testWriteEmptyString() { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + writer.putString(""); + Assert.assertEquals(1, writer.getPosition()); + Assert.assertEquals((byte) 0, Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + } + } + + @Test + public void testWriteFloat() { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + writer.putFloat(3.14f); + Assert.assertEquals(4, writer.getPosition()); + Assert.assertEquals(3.14f, Unsafe.getUnsafe().getFloat(writer.getBufferPtr()), 0.0001f); + } + } + + @Test + public void testWriteInt() { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + writer.putInt(0x12345678); + Assert.assertEquals(4, writer.getPosition()); + // Little-endian + Assert.assertEquals(0x12345678, Unsafe.getUnsafe().getInt(writer.getBufferPtr())); + } + } + + @Test + public void testWriteLong() { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + writer.putLong(0x123456789ABCDEF0L); + Assert.assertEquals(8, writer.getPosition()); + Assert.assertEquals(0x123456789ABCDEF0L, Unsafe.getUnsafe().getLong(writer.getBufferPtr())); + } + } + + @Test + public void testWriteLongBigEndian() { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + writer.putLongBE(0x0102030405060708L); + Assert.assertEquals(8, writer.getPosition()); + // Check big-endian byte order + long ptr = writer.getBufferPtr(); + Assert.assertEquals((byte) 0x01, Unsafe.getUnsafe().getByte(ptr)); + Assert.assertEquals((byte) 0x02, Unsafe.getUnsafe().getByte(ptr + 1)); + Assert.assertEquals((byte) 0x03, Unsafe.getUnsafe().getByte(ptr + 2)); + Assert.assertEquals((byte) 0x04, Unsafe.getUnsafe().getByte(ptr + 3)); + Assert.assertEquals((byte) 0x05, Unsafe.getUnsafe().getByte(ptr + 4)); + Assert.assertEquals((byte) 0x06, Unsafe.getUnsafe().getByte(ptr + 5)); + Assert.assertEquals((byte) 0x07, Unsafe.getUnsafe().getByte(ptr + 6)); + Assert.assertEquals((byte) 0x08, Unsafe.getUnsafe().getByte(ptr + 7)); + } + } + + @Test + public void testWriteNullString() { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + writer.putString(null); + Assert.assertEquals(1, writer.getPosition()); + Assert.assertEquals((byte) 0, Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + } + } + + @Test + public void testWriteShort() { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + writer.putShort((short) 0x1234); + Assert.assertEquals(2, writer.getPosition()); + // Little-endian + Assert.assertEquals((byte) 0x34, Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + Assert.assertEquals((byte) 0x12, Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 1)); + } + } + + @Test + public void testWriteString() { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + writer.putString("hello"); + // Length (1 byte varint) + 5 bytes + Assert.assertEquals(6, writer.getPosition()); + // Check length + Assert.assertEquals((byte) 5, Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + // Check content + Assert.assertEquals((byte) 'h', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 1)); + Assert.assertEquals((byte) 'e', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 2)); + Assert.assertEquals((byte) 'l', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 3)); + Assert.assertEquals((byte) 'l', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 4)); + Assert.assertEquals((byte) 'o', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 5)); + } + } + + @Test + public void testWriteUtf8Ascii() { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + writer.putUtf8("ABC"); + Assert.assertEquals(3, writer.getPosition()); + Assert.assertEquals((byte) 'A', Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + Assert.assertEquals((byte) 'B', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 1)); + Assert.assertEquals((byte) 'C', Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 2)); + } + } + + @Test + public void testWriteUtf8ThreeByte() { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + // € is 3 bytes in UTF-8 + writer.putUtf8("€"); + Assert.assertEquals(3, writer.getPosition()); + Assert.assertEquals((byte) 0xE2, Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + Assert.assertEquals((byte) 0x82, Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 1)); + Assert.assertEquals((byte) 0xAC, Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 2)); + } + } + + @Test + public void testWriteUtf8TwoByte() { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + // ñ is 2 bytes in UTF-8 + writer.putUtf8("ñ"); + Assert.assertEquals(2, writer.getPosition()); + Assert.assertEquals((byte) 0xC3, Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + Assert.assertEquals((byte) 0xB1, Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 1)); + } + } + + @Test + public void testWriteVarintLarge() { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + // Test larger value + writer.putVarint(16384); + Assert.assertEquals(3, writer.getPosition()); + // LEB128: 16384 = 0x80 0x80 0x01 + Assert.assertEquals((byte) 0x80, Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + Assert.assertEquals((byte) 0x80, Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 1)); + Assert.assertEquals((byte) 0x01, Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 2)); + } + } + + @Test + public void testWriteVarintMedium() { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + // Two bytes for 128 + writer.putVarint(128); + Assert.assertEquals(2, writer.getPosition()); + // LEB128: 128 = 0x80 0x01 + Assert.assertEquals((byte) 0x80, Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + Assert.assertEquals((byte) 0x01, Unsafe.getUnsafe().getByte(writer.getBufferPtr() + 1)); + } + } + + @Test + public void testWriteVarintSmall() { + try (NativeBufferWriter writer = new NativeBufferWriter()) { + // Single byte for values < 128 + writer.putVarint(127); + Assert.assertEquals(1, writer.getPosition()); + Assert.assertEquals((byte) 127, Unsafe.getUnsafe().getByte(writer.getBufferPtr())); + } + } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java new file mode 100644 index 0000000..5a298e9 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java @@ -0,0 +1,1254 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary; +import io.questdb.client.cutlass.qwp.client.QwpBufferWriter; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; +import org.junit.Test; + +import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.*; + +/** + * Unit tests for QwpWebSocketEncoder. + */ +public class QwpWebSocketEncoderTest { + + @Test + public void testBufferResetAndReuse() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test"); + + // First batch + for (int i = 0; i < 100; i++) { + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_LONG, false); + col.addLong(i); + buffer.nextRow(); + } + int size1 = encoder.encode(buffer, false); + + // Reset and second batch + buffer.reset(); + for (int i = 0; i < 50; i++) { + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_LONG, false); + col.addLong(i * 2); + buffer.nextRow(); + } + int size2 = encoder.encode(buffer, false); + + Assert.assertTrue(size1 > size2); // More rows = larger + Assert.assertEquals(50, buffer.getRowCount()); + } + } + + @Test + public void testEncode2DDoubleArray() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("matrix", TYPE_DOUBLE_ARRAY, true); + col.addDoubleArray(new double[][]{{1.0, 2.0}, {3.0, 4.0}}); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + } + + @Test + public void testEncode2DLongArray() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("matrix", TYPE_LONG_ARRAY, true); + col.addLongArray(new long[][]{{1L, 2L}, {3L, 4L}}); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + } + + @Test + public void testEncode3DDoubleArray() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("tensor", TYPE_DOUBLE_ARRAY, true); + col.addDoubleArray(new double[][][]{ + {{1.0, 2.0}, {3.0, 4.0}}, + {{5.0, 6.0}, {7.0, 8.0}} + }); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + } + + @Test + public void testEncodeAllBasicTypesInOneRow() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("all_types"); + + buffer.getOrCreateColumn("b", TYPE_BOOLEAN, false).addBoolean(true); + buffer.getOrCreateColumn("by", TYPE_BYTE, false).addByte((byte) 42); + buffer.getOrCreateColumn("sh", TYPE_SHORT, false).addShort((short) 1000); + buffer.getOrCreateColumn("i", TYPE_INT, false).addInt(100000); + buffer.getOrCreateColumn("l", TYPE_LONG, false).addLong(1000000000L); + buffer.getOrCreateColumn("f", TYPE_FLOAT, false).addFloat(3.14f); + buffer.getOrCreateColumn("d", TYPE_DOUBLE, false).addDouble(3.14159265); + buffer.getOrCreateColumn("s", TYPE_STRING, true).addString("test"); + buffer.getOrCreateColumn("sym", TYPE_SYMBOL, false).addSymbol("AAPL"); + buffer.getOrCreateColumn("", TYPE_TIMESTAMP, true).addLong(1000000L); + + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + Assert.assertEquals(1, buffer.getRowCount()); + } + } + + @Test + public void testEncodeAllBooleanValues() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("flag", TYPE_BOOLEAN, false); + for (int i = 0; i < 100; i++) { + col.addBoolean(i % 2 == 0); + buffer.nextRow(); + } + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + Assert.assertEquals(100, buffer.getRowCount()); + } + } + + @Test + public void testEncodeDecimal128() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("amount", TYPE_DECIMAL128, false); + col.addDecimal128(io.questdb.client.std.Decimal128.fromLong(123456789012345L, 4)); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + } + + @Test + public void testEncodeDecimal256() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("bignum", TYPE_DECIMAL256, false); + col.addDecimal256(io.questdb.client.std.Decimal256.fromLong(Long.MAX_VALUE, 6)); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + } + + @Test + public void testEncodeDecimal64() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("price", TYPE_DECIMAL64, false); + col.addDecimal64(io.questdb.client.std.Decimal64.fromLong(12345L, 2)); // 123.45 + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + } + + @Test + public void testEncodeDoubleArray() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test"); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("values", TYPE_DOUBLE_ARRAY, true); + col.addDoubleArray(new double[]{1.0, 2.0, 3.0}); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + } + + @Test + public void testEncodeEmptyString() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("name", TYPE_STRING, true); + col.addString(""); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + } + + @Test + public void testEncodeEmptyTableName() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + // Edge case: empty table name (probably invalid but let's verify encoding works) + QwpTableBuffer buffer = new QwpTableBuffer(""); + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_LONG, false); + col.addLong(1L); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 0); + } + } + + @Test + public void testEncodeLargeArray() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + // Large 1D array + double[] largeArray = new double[1000]; + for (int i = 0; i < 1000; i++) { + largeArray[i] = i * 1.5; + } + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("values", TYPE_DOUBLE_ARRAY, true); + col.addDoubleArray(largeArray); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 8000); // At least 8 bytes per double + } + } + + @Test + public void testEncodeLargeRowCount() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("metrics"); + + for (int i = 0; i < 10000; i++) { + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_LONG, false); + col.addLong(i); + buffer.nextRow(); + } + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + Assert.assertEquals(10000, buffer.getRowCount()); + } + } + + @Test + public void testEncodeLongArray() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test"); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("values", TYPE_LONG_ARRAY, true); + col.addLongArray(new long[]{1L, 2L, 3L}); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + } + + // ==================== SYMBOL COLUMN TESTS ==================== + + @Test + public void testEncodeLongString() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + String sb = "a".repeat(10000); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("data", TYPE_STRING, true); + col.addString(sb); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 10000); + } + } + + @Test + public void testEncodeMaxMinLong() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_LONG, false); + col.addLong(Long.MAX_VALUE); + buffer.nextRow(); + + col.addLong(Long.MIN_VALUE); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + Assert.assertEquals(2, buffer.getRowCount()); + } + } + + @Test + public void testEncodeMixedColumnTypes() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("events"); + + // Add columns of different types + QwpTableBuffer.ColumnBuffer symbolCol = buffer.getOrCreateColumn("host", TYPE_SYMBOL, false); + symbolCol.addSymbol("server1"); + + QwpTableBuffer.ColumnBuffer longCol = buffer.getOrCreateColumn("count", TYPE_LONG, false); + longCol.addLong(42); + + QwpTableBuffer.ColumnBuffer doubleCol = buffer.getOrCreateColumn("value", TYPE_DOUBLE, false); + doubleCol.addDouble(3.14); + + QwpTableBuffer.ColumnBuffer boolCol = buffer.getOrCreateColumn("active", TYPE_BOOLEAN, false); + boolCol.addBoolean(true); + + QwpTableBuffer.ColumnBuffer stringCol = buffer.getOrCreateColumn("message", TYPE_STRING, true); + stringCol.addString("hello world"); + + QwpTableBuffer.ColumnBuffer tsCol = buffer.getOrCreateColumn("", TYPE_TIMESTAMP, true); + tsCol.addLong(1000000L); + + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + } + + @Test + public void testEncodeMixedColumnsMultipleRows() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("events"); + + for (int i = 0; i < 50; i++) { + QwpTableBuffer.ColumnBuffer symbolCol = buffer.getOrCreateColumn("host", TYPE_SYMBOL, false); + symbolCol.addSymbol("server" + (i % 5)); + + QwpTableBuffer.ColumnBuffer longCol = buffer.getOrCreateColumn("count", TYPE_LONG, false); + longCol.addLong(i * 10); + + QwpTableBuffer.ColumnBuffer doubleCol = buffer.getOrCreateColumn("value", TYPE_DOUBLE, false); + doubleCol.addDouble(i * 1.5); + + QwpTableBuffer.ColumnBuffer tsCol = buffer.getOrCreateColumn("", TYPE_TIMESTAMP, true); + tsCol.addLong(1000000L + i); + + buffer.nextRow(); + } + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + Assert.assertEquals(50, buffer.getRowCount()); + } + } + + // ==================== UUID COLUMN TESTS ==================== + + @Test + public void testEncodeMultipleColumns() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("weather"); + + // Add multiple columns + QwpTableBuffer.ColumnBuffer tempCol = buffer.getOrCreateColumn("temperature", TYPE_DOUBLE, false); + tempCol.addDouble(23.5); + + QwpTableBuffer.ColumnBuffer humCol = buffer.getOrCreateColumn("humidity", TYPE_LONG, false); + humCol.addLong(65); + + QwpTableBuffer.ColumnBuffer tsCol = buffer.getOrCreateColumn("", TYPE_TIMESTAMP, true); + tsCol.addLong(1000000L); + + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + + // Verify header + QwpBufferWriter buf = encoder.getBuffer(); + long ptr = buf.getBufferPtr(); + Assert.assertEquals((byte) 'I', Unsafe.getUnsafe().getByte(ptr)); + Assert.assertEquals((byte) 'L', Unsafe.getUnsafe().getByte(ptr + 1)); + Assert.assertEquals((byte) 'P', Unsafe.getUnsafe().getByte(ptr + 2)); + Assert.assertEquals((byte) '4', Unsafe.getUnsafe().getByte(ptr + 3)); + } + } + + @Test + public void testEncodeMultipleDecimal64() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("price", TYPE_DECIMAL64, false); + col.addDecimal64(io.questdb.client.std.Decimal64.fromLong(12345L, 2)); // 123.45 + buffer.nextRow(); + + col.addDecimal64(io.questdb.client.std.Decimal64.fromLong(67890L, 2)); // 678.90 + buffer.nextRow(); + + col.addDecimal64(io.questdb.client.std.Decimal64.fromLong(11111L, 2)); // 111.11 + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + Assert.assertEquals(3, buffer.getRowCount()); + } + } + + // ==================== DECIMAL COLUMN TESTS ==================== + + @Test + public void testEncodeMultipleRows() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("metrics"); + + for (int i = 0; i < 100; i++) { + QwpTableBuffer.ColumnBuffer valCol = buffer.getOrCreateColumn("value", TYPE_LONG, false); + valCol.addLong(i); + + QwpTableBuffer.ColumnBuffer tsCol = buffer.getOrCreateColumn("", TYPE_TIMESTAMP, true); + tsCol.addLong(1000000L + i); + + buffer.nextRow(); + } + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + Assert.assertEquals(100, buffer.getRowCount()); + } + } + + @Test + public void testEncodeMultipleSymbolsSameDictionary() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("host", TYPE_SYMBOL, false); + col.addSymbol("server1"); + buffer.nextRow(); + + col.addSymbol("server1"); // Same symbol + buffer.nextRow(); + + col.addSymbol("server2"); // Different symbol + buffer.nextRow(); + + col.addSymbol("server1"); // Back to first + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + Assert.assertEquals(4, buffer.getRowCount()); + } + } + + @Test + public void testEncodeMultipleUuids() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("id", TYPE_UUID, false); + for (int i = 0; i < 10; i++) { + col.addUuid(i * 1000L, i * 2000L); + buffer.nextRow(); + } + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + Assert.assertEquals(10, buffer.getRowCount()); + } + } + + @Test + public void testEncodeNaNDouble() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_DOUBLE, false); + col.addDouble(Double.NaN); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + } + + // ==================== ARRAY COLUMN TESTS ==================== + + @Test + public void testEncodeNegativeLong() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_LONG, false); + col.addLong(-123456789L); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + } + + @Test + public void testEncodeNullableColumnWithNull() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test"); + + // Nullable column with null + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("name", TYPE_STRING, true); + col.addString(null); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + } + + @Test + public void testEncodeNullableColumnWithValue() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test"); + + // Nullable column with a value + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("name", TYPE_STRING, true); + col.addString("hello"); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + } + + @Test + public void testEncodeNullableSymbolWithNull() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("host", TYPE_SYMBOL, true); + col.addSymbol("server1"); + buffer.nextRow(); + + col.addSymbol(null); // Null symbol + buffer.nextRow(); + + col.addSymbol("server2"); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + Assert.assertEquals(3, buffer.getRowCount()); + } + } + + // ==================== MULTIPLE ROWS TESTS ==================== + + @Test + public void testEncodeSingleRowWithBoolean() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("active", TYPE_BOOLEAN, false); + col.addBoolean(true); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + } + + @Test + public void testEncodeSingleRowWithDouble() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("temperature", TYPE_DOUBLE, false); + col.addDouble(23.5); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + } + + // ==================== MIXED COLUMN TYPES ==================== + + @Test + public void testEncodeSingleRowWithLong() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + // Add a long column + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("value", TYPE_LONG, false); + col.addLong(12345L); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); // At least header size + + QwpBufferWriter buf = encoder.getBuffer(); + long ptr = buf.getBufferPtr(); + + // Verify header magic + Assert.assertEquals((byte) 'I', Unsafe.getUnsafe().getByte(ptr)); + Assert.assertEquals((byte) 'L', Unsafe.getUnsafe().getByte(ptr + 1)); + Assert.assertEquals((byte) 'P', Unsafe.getUnsafe().getByte(ptr + 2)); + Assert.assertEquals((byte) '4', Unsafe.getUnsafe().getByte(ptr + 3)); + + // Version + Assert.assertEquals(VERSION_1, Unsafe.getUnsafe().getByte(ptr + 4)); + + // Table count (little-endian short) + Assert.assertEquals((short) 1, Unsafe.getUnsafe().getShort(ptr + 6)); + } + } + + @Test + public void testEncodeSingleRowWithString() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("name", TYPE_STRING, true); + col.addString("hello"); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + } + + // ==================== EDGE CASES ==================== + + @Test + public void testEncodeSingleRowWithTimestamp() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + // Add a timestamp column (designated timestamp uses empty name) + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("", TYPE_TIMESTAMP, true); + col.addLong(1000000L); // Micros + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + } + + @Test + public void testEncodeSingleSymbol() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("host", TYPE_SYMBOL, false); + col.addSymbol("server1"); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + } + + @Test + public void testEncodeSpecialDoubles() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_DOUBLE, false); + col.addDouble(Double.MAX_VALUE); + buffer.nextRow(); + + col.addDouble(Double.MIN_VALUE); + buffer.nextRow(); + + col.addDouble(Double.POSITIVE_INFINITY); + buffer.nextRow(); + + col.addDouble(Double.NEGATIVE_INFINITY); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + Assert.assertEquals(4, buffer.getRowCount()); + } + } + + @Test + public void testEncodeSymbolWithManyDistinctValues() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("host", TYPE_SYMBOL, false); + for (int i = 0; i < 100; i++) { + col.addSymbol("server" + i); + buffer.nextRow(); + } + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + Assert.assertEquals(100, buffer.getRowCount()); + } + } + + @Test + public void testEncodeUnicodeString() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("name", TYPE_STRING, true); + col.addString("Hello 世界 🌍"); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + } + + @Test + public void testEncodeUuid() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("id", TYPE_UUID, false); + col.addUuid(0x123456789ABCDEF0L, 0xFEDCBA9876543210L); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + } + + @Test + public void testEncodeWithDeltaDict_freshConnection_sendsAllSymbols() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + // Add symbol column with global IDs + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("ticker", TYPE_SYMBOL, false); + + // Simulate adding symbols via global dictionary + int id1 = globalDict.getOrAddSymbol("AAPL"); // ID 0 + int id2 = globalDict.getOrAddSymbol("GOOG"); // ID 1 + col.addSymbolWithGlobalId("AAPL", id1); + buffer.nextRow(); + col.addSymbolWithGlobalId("GOOG", id2); + buffer.nextRow(); + + // Fresh connection: confirmedMaxId = -1, so delta should include all symbols (0, 1) + int confirmedMaxId = -1; + int batchMaxId = 1; + + int size = encoder.encodeWithDeltaDict(buffer, globalDict, confirmedMaxId, batchMaxId, false); + Assert.assertTrue(size > 12); + + QwpBufferWriter buf = encoder.getBuffer(); + long ptr = buf.getBufferPtr(); + + // Verify header flag has FLAG_DELTA_SYMBOL_DICT set + byte flags = Unsafe.getUnsafe().getByte(ptr + HEADER_OFFSET_FLAGS); + Assert.assertTrue("Delta flag should be set", (flags & FLAG_DELTA_SYMBOL_DICT) != 0); + } + } + + @Test + public void testEncodeWithDeltaDict_noNewSymbols_sendsEmptyDelta() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + // Pre-populate dictionary with all symbols + int id0 = globalDict.getOrAddSymbol("AAPL"); // ID 0 + int id1 = globalDict.getOrAddSymbol("GOOG"); // ID 1 + + // Use only existing symbols + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("ticker", TYPE_SYMBOL, false); + col.addSymbolWithGlobalId("AAPL", id0); + buffer.nextRow(); + col.addSymbolWithGlobalId("GOOG", id1); + buffer.nextRow(); + + // Server has confirmed all symbols (0-1), batchMaxId is 1 + int confirmedMaxId = 1; + int batchMaxId = 1; + + int size = encoder.encodeWithDeltaDict(buffer, globalDict, confirmedMaxId, batchMaxId, false); + Assert.assertTrue(size > 12); + + QwpBufferWriter buf = encoder.getBuffer(); + long ptr = buf.getBufferPtr(); + + // Verify delta flag is set + byte flags = Unsafe.getUnsafe().getByte(ptr + HEADER_OFFSET_FLAGS); + Assert.assertTrue("Delta flag should be set", (flags & FLAG_DELTA_SYMBOL_DICT) != 0); + + // Read delta section after header + long pos = ptr + HEADER_SIZE; + + // Read deltaStart varint (should be 2 = confirmedMaxId + 1) + int deltaStart = Unsafe.getUnsafe().getByte(pos) & 0x7F; + Assert.assertEquals(2, deltaStart); + pos++; + + // Read deltaCount varint (should be 0) + int deltaCount = Unsafe.getUnsafe().getByte(pos) & 0x7F; + Assert.assertEquals(0, deltaCount); + } + } + + @Test + public void testEncodeWithDeltaDict_withConfirmed_sendsOnlyNew() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary(); + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + // Pre-populate dictionary (simulating symbols already sent) + globalDict.getOrAddSymbol("AAPL"); // ID 0 + globalDict.getOrAddSymbol("GOOG"); // ID 1 + + // Now add new symbols + int id2 = globalDict.getOrAddSymbol("MSFT"); // ID 2 + int id3 = globalDict.getOrAddSymbol("TSLA"); // ID 3 + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("ticker", TYPE_SYMBOL, false); + col.addSymbolWithGlobalId("MSFT", id2); + buffer.nextRow(); + col.addSymbolWithGlobalId("TSLA", id3); + buffer.nextRow(); + + // Server has confirmed IDs 0-1, so delta should only include 2-3 + int confirmedMaxId = 1; + int batchMaxId = 3; + + int size = encoder.encodeWithDeltaDict(buffer, globalDict, confirmedMaxId, batchMaxId, false); + Assert.assertTrue(size > 12); + + QwpBufferWriter buf = encoder.getBuffer(); + long ptr = buf.getBufferPtr(); + + // Verify delta flag is set + byte flags = Unsafe.getUnsafe().getByte(ptr + HEADER_OFFSET_FLAGS); + Assert.assertTrue("Delta flag should be set", (flags & FLAG_DELTA_SYMBOL_DICT) != 0); + + // Read delta section after header + long pos = ptr + HEADER_SIZE; + + // Read deltaStart varint (should be 2 = confirmedMaxId + 1) + int deltaStart = Unsafe.getUnsafe().getByte(pos) & 0x7F; + Assert.assertEquals(2, deltaStart); + } + } + + // ==================== SCHEMA REFERENCE TESTS ==================== + + @Test + public void testEncodeWithSchemaRef() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_LONG, false); + col.addLong(42L); + buffer.nextRow(); + + int size = encoder.encode(buffer, true); // Use schema reference + Assert.assertTrue(size > 12); + } + } + + // ==================== BUFFER REUSE TESTS ==================== + + @Test + public void testEncodeZeroLong() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_LONG, false); + col.addLong(0L); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + } + + @Test + public void testEncoderReusability() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + // Encode first message + QwpTableBuffer buffer1 = new QwpTableBuffer("table1"); + QwpTableBuffer.ColumnBuffer col1 = buffer1.getOrCreateColumn("x", TYPE_LONG, false); + col1.addLong(1L); + buffer1.nextRow(); + int size1 = encoder.encode(buffer1, false); + + // Encode second message (encoder should reset internally) + QwpTableBuffer buffer2 = new QwpTableBuffer("table2"); + QwpTableBuffer.ColumnBuffer col2 = buffer2.getOrCreateColumn("y", TYPE_DOUBLE, false); + col2.addDouble(2.0); + buffer2.nextRow(); + int size2 = encoder.encode(buffer2, false); + + // Both should succeed + Assert.assertTrue(size1 > 12); + Assert.assertTrue(size2 > 12); + } + } + + // ==================== ALL BASIC TYPES IN ONE ROW ==================== + + @Test + public void testGlobalSymbolDictionaryBasics() { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + + // Test sequential IDs + Assert.assertEquals(0, dict.getOrAddSymbol("AAPL")); + Assert.assertEquals(1, dict.getOrAddSymbol("GOOG")); + Assert.assertEquals(2, dict.getOrAddSymbol("MSFT")); + + // Test deduplication + Assert.assertEquals(0, dict.getOrAddSymbol("AAPL")); + Assert.assertEquals(1, dict.getOrAddSymbol("GOOG")); + + // Test retrieval + Assert.assertEquals("AAPL", dict.getSymbol(0)); + Assert.assertEquals("GOOG", dict.getSymbol(1)); + Assert.assertEquals("MSFT", dict.getSymbol(2)); + + // Test size + Assert.assertEquals(3, dict.size()); + } + + // ==================== Delta Symbol Dictionary Tests ==================== + + @Test + public void testGorillaEncoding_compressionRatio() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + encoder.setGorillaEnabled(true); + + QwpTableBuffer buffer = new QwpTableBuffer("metrics"); + + // Add many timestamps with constant delta - best case for Gorilla + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("ts", TYPE_TIMESTAMP, true); + for (int i = 0; i < 1000; i++) { + col.addLong(1000000000L + i * 1000L); + buffer.nextRow(); + } + + int sizeWithGorilla = encoder.encode(buffer, false); + + // Calculate theoretical minimum size for Gorilla: + // - Header: 12 bytes + // - Table header, column schema, etc. + // - First timestamp: 8 bytes + // - Second timestamp: 8 bytes + // - Remaining 998 timestamps: 998 bits (1 bit each for DoD=0) = ~125 bytes + + // Calculate size without Gorilla (1000 * 8 = 8000 bytes just for timestamps) + encoder.setGorillaEnabled(false); + buffer.reset(); + col = buffer.getOrCreateColumn("ts", TYPE_TIMESTAMP, true); + for (int i = 0; i < 1000; i++) { + col.addLong(1000000000L + i * 1000L); + buffer.nextRow(); + } + + int sizeWithoutGorilla = encoder.encode(buffer, false); + + // For constant delta, Gorilla should achieve significant compression + double compressionRatio = (double) sizeWithGorilla / sizeWithoutGorilla; + Assert.assertTrue("Compression ratio should be < 0.2 for constant delta", + compressionRatio < 0.2); + } + } + + @Test + public void testGorillaEncoding_multipleTimestampColumns() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + encoder.setGorillaEnabled(true); + + QwpTableBuffer buffer = new QwpTableBuffer("test"); + + // Add multiple timestamp columns + for (int i = 0; i < 50; i++) { + QwpTableBuffer.ColumnBuffer ts1Col = buffer.getOrCreateColumn("ts1", TYPE_TIMESTAMP, true); + ts1Col.addLong(1000000000L + i * 1000L); + + QwpTableBuffer.ColumnBuffer ts2Col = buffer.getOrCreateColumn("ts2", TYPE_TIMESTAMP, true); + ts2Col.addLong(2000000000L + i * 2000L); + + buffer.nextRow(); + } + + int sizeWithGorilla = encoder.encode(buffer, false); + + // Compare with uncompressed + encoder.setGorillaEnabled(false); + buffer.reset(); + for (int i = 0; i < 50; i++) { + QwpTableBuffer.ColumnBuffer ts1Col = buffer.getOrCreateColumn("ts1", TYPE_TIMESTAMP, true); + ts1Col.addLong(1000000000L + i * 1000L); + + QwpTableBuffer.ColumnBuffer ts2Col = buffer.getOrCreateColumn("ts2", TYPE_TIMESTAMP, true); + ts2Col.addLong(2000000000L + i * 2000L); + + buffer.nextRow(); + } + + int sizeWithoutGorilla = encoder.encode(buffer, false); + + Assert.assertTrue("Gorilla should compress multiple timestamp columns", + sizeWithGorilla < sizeWithoutGorilla); + } + } + + @Test + public void testGorillaEncoding_multipleTimestamps_usesGorillaEncoding() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + encoder.setGorillaEnabled(true); + + QwpTableBuffer buffer = new QwpTableBuffer("test"); + + // Add multiple timestamps with constant delta (best compression) + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("", TYPE_TIMESTAMP, true); + for (int i = 0; i < 100; i++) { + col.addLong(1000000000L + i * 1000L); + buffer.nextRow(); + } + + int sizeWithGorilla = encoder.encode(buffer, false); + + // Now encode without Gorilla + encoder.setGorillaEnabled(false); + buffer.reset(); + col = buffer.getOrCreateColumn("", TYPE_TIMESTAMP, true); + for (int i = 0; i < 100; i++) { + col.addLong(1000000000L + i * 1000L); + buffer.nextRow(); + } + + int sizeWithoutGorilla = encoder.encode(buffer, false); + + // Gorilla should produce smaller output for constant-delta timestamps + Assert.assertTrue("Gorilla encoding should be smaller", + sizeWithGorilla < sizeWithoutGorilla); + } + } + + @Test + public void testGorillaEncoding_nanosTimestamps() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + encoder.setGorillaEnabled(true); + + QwpTableBuffer buffer = new QwpTableBuffer("test"); + + // Use TYPE_TIMESTAMP_NANOS + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("ts", TYPE_TIMESTAMP_NANOS, true); + for (int i = 0; i < 100; i++) { + col.addLong(1000000000000000000L + i * 1000000L); // Nanos with millisecond intervals + buffer.nextRow(); + } + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + + // Verify header has Gorilla flag + QwpBufferWriter buf = encoder.getBuffer(); + byte flags = Unsafe.getUnsafe().getByte(buf.getBufferPtr() + HEADER_OFFSET_FLAGS); + Assert.assertEquals(FLAG_GORILLA, (byte) (flags & FLAG_GORILLA)); + } + } + + @Test + public void testGorillaEncoding_singleTimestamp_usesUncompressed() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + encoder.setGorillaEnabled(true); + + QwpTableBuffer buffer = new QwpTableBuffer("test"); + + // Single timestamp - should use uncompressed + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("", TYPE_TIMESTAMP, true); + col.addLong(1000000L); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + } + } + + @Test + public void testGorillaEncoding_twoTimestamps_usesUncompressed() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + encoder.setGorillaEnabled(true); + + QwpTableBuffer buffer = new QwpTableBuffer("test"); + + // Only 2 timestamps - should use uncompressed (Gorilla needs 3+) + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("", TYPE_TIMESTAMP, true); + col.addLong(1000000L); + buffer.nextRow(); + col.addLong(2000000L); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + + // Verify header has Gorilla flag set + QwpBufferWriter buf = encoder.getBuffer(); + byte flags = Unsafe.getUnsafe().getByte(buf.getBufferPtr() + HEADER_OFFSET_FLAGS); + Assert.assertEquals(FLAG_GORILLA, (byte) (flags & FLAG_GORILLA)); + } + } + + @Test + public void testGorillaEncoding_varyingDelta() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + encoder.setGorillaEnabled(true); + + QwpTableBuffer buffer = new QwpTableBuffer("test"); + + // Varying deltas that exercise different buckets + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("ts", TYPE_TIMESTAMP, true); + long[] timestamps = { + 1000000000L, + 1000001000L, // delta=1000 + 1000002000L, // DoD=0 + 1000003050L, // DoD=50 + 1000004200L, // DoD=100 + 1000006200L, // DoD=850 + }; + + for (long ts : timestamps) { + col.addLong(ts); + buffer.nextRow(); + } + + int size = encoder.encode(buffer, false); + Assert.assertTrue(size > 12); + + // Verify header has Gorilla flag + QwpBufferWriter buf = encoder.getBuffer(); + byte flags = Unsafe.getUnsafe().getByte(buf.getBufferPtr() + HEADER_OFFSET_FLAGS); + Assert.assertEquals(FLAG_GORILLA, (byte) (flags & FLAG_GORILLA)); + } + } + + @Test + public void testGorillaFlagDisabled() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + encoder.setGorillaEnabled(false); + Assert.assertFalse(encoder.isGorillaEnabled()); + + QwpTableBuffer buffer = new QwpTableBuffer("test"); + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("ts", TYPE_TIMESTAMP, true); + col.addLong(1000000L); + buffer.nextRow(); + + encoder.encode(buffer, false); + + // Check flags byte doesn't have Gorilla bit set + QwpBufferWriter buf = encoder.getBuffer(); + byte flags = Unsafe.getUnsafe().getByte(buf.getBufferPtr() + 5); + Assert.assertEquals(0, flags & FLAG_GORILLA); + } + } + + @Test + public void testGorillaFlagEnabled() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + encoder.setGorillaEnabled(true); + Assert.assertTrue(encoder.isGorillaEnabled()); + + QwpTableBuffer buffer = new QwpTableBuffer("test"); + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("ts", TYPE_TIMESTAMP, true); + col.addLong(1000000L); + buffer.nextRow(); + + encoder.encode(buffer, false); + + // Check flags byte has Gorilla bit set + QwpBufferWriter buf = encoder.getBuffer(); + byte flags = Unsafe.getUnsafe().getByte(buf.getBufferPtr() + 5); + Assert.assertEquals(FLAG_GORILLA, (byte) (flags & FLAG_GORILLA)); + } + } + + @Test + public void testPayloadLengthPatched() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test_table"); + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_LONG, false); + col.addLong(42L); + buffer.nextRow(); + + int size = encoder.encode(buffer, false); + + // Payload length is at offset 8 (4 magic + 1 version + 1 flags + 2 tablecount) + QwpBufferWriter buf = encoder.getBuffer(); + int payloadLength = Unsafe.getUnsafe().getInt(buf.getBufferPtr() + 8); + + // Payload length should be total size minus header (12 bytes) + Assert.assertEquals(size - 12, payloadLength); + } + } + + @Test + public void testReset() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpTableBuffer buffer = new QwpTableBuffer("test"); + + QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_LONG, false); + col.addLong(1L); + buffer.nextRow(); + + int size1 = encoder.encode(buffer, false); + + // Reset and encode again + buffer.reset(); + col = buffer.getOrCreateColumn("x", TYPE_LONG, false); + col.addLong(2L); + buffer.nextRow(); + + int size2 = encoder.encode(buffer, false); + + // Sizes should be similar (same schema) + Assert.assertEquals(size1, size2); + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderTest.java new file mode 100644 index 0000000..a128ed3 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderTest.java @@ -0,0 +1,458 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.DefaultHttpClientConfiguration; +import io.questdb.client.cutlass.http.client.WebSocketClient; +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.MicrobatchBuffer; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; +import io.questdb.client.cutlass.qwp.client.WebSocketSendQueue; +import io.questdb.client.network.PlainSocketFactory; +import org.junit.Assert; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.time.Instant; +import java.time.temporal.ChronoUnit; + +/** + * Unit tests for QwpWebSocketSender. + * These tests focus on state management and API validation without requiring a live server. + */ +public class QwpWebSocketSenderTest { + + @Test + public void testAtAfterCloseThrows() { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); + + try { + sender.at(1000L, ChronoUnit.MICROS); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + } + + @Test + public void testAtInstantAfterCloseThrows() { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); + + try { + sender.at(Instant.now()); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + } + + @Test + public void testAtNowAfterCloseThrows() { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); + + try { + sender.atNow(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + } + + @Test + public void testBoolColumnAfterCloseThrows() { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); + + try { + sender.boolColumn("x", true); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + } + + @Test + public void testBufferViewNotSupported() { + try (QwpWebSocketSender sender = createUnconnectedSender()) { + sender.bufferView(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("not supported")); + } + } + + @Test + public void testCancelRowAfterCloseThrows() { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); + + try { + sender.cancelRow(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + } + + @Test + public void testCloseIdemponent() { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); + sender.close(); // Should not throw + } + + @Test + public void testConnectToClosedPort() { + try { + QwpWebSocketSender.connect("127.0.0.1", 1); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("Failed to connect")); + } + } + + @Test + public void testDoubleArrayAfterCloseThrows() { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); + + try { + sender.doubleArray("x", new double[]{1.0, 2.0}); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + } + + @Test + public void testDoubleColumnAfterCloseThrows() { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); + + try { + sender.doubleColumn("x", 1.0); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + } + + @Test + public void testGorillaEnabledByDefault() { + try (QwpWebSocketSender sender = createUnconnectedSender()) { + Assert.assertTrue(sender.isGorillaEnabled()); + } + } + + @Test + public void testLongArrayAfterCloseThrows() { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); + + try { + sender.longArray("x", new long[]{1L, 2L}); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + } + + @Test + public void testLongColumnAfterCloseThrows() { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); + + try { + sender.longColumn("x", 1); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + } + + @Test + public void testNullArrayReturnsThis() { + try (QwpWebSocketSender sender = createUnconnectedSender()) { + // Null arrays should be no-ops and return sender + Assert.assertSame(sender, sender.doubleArray("x", (double[]) null)); + Assert.assertSame(sender, sender.longArray("x", (long[]) null)); + } + } + + @Test + public void testOperationsAfterCloseThrow() { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); + + try { + sender.table("test"); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + } + + @Test + public void testResetAfterCloseThrows() { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); + + try { + sender.reset(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + } + + @Test + public void testSealAndSwapRollsBackOnEnqueueFailure() throws Exception { + try (QwpWebSocketSender sender = createUnconnectedAsyncSender(); ThrowingOnceWebSocketSendQueue queue = new ThrowingOnceWebSocketSendQueue()) { + setSendQueue(sender, queue); + + MicrobatchBuffer originalActive = getActiveBuffer(sender); + originalActive.writeByte((byte) 7); + originalActive.incrementRowCount(); + + try { + invokeSealAndSwapBuffer(sender); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("Synthetic enqueue failure")); + } + + // Failed enqueue must not strand the sealed buffer. + Assert.assertSame(originalActive, getActiveBuffer(sender)); + Assert.assertTrue(originalActive.isFilling()); + Assert.assertTrue(originalActive.hasData()); + Assert.assertEquals(1, originalActive.getRowCount()); + + // Retry should be possible on the same sender instance. + invokeSealAndSwapBuffer(sender); + Assert.assertNotSame(originalActive, getActiveBuffer(sender)); + } + } + + @Test + public void testSetGorillaEnabled() { + try (QwpWebSocketSender sender = createUnconnectedSender()) { + sender.setGorillaEnabled(false); + Assert.assertFalse(sender.isGorillaEnabled()); + sender.setGorillaEnabled(true); + Assert.assertTrue(sender.isGorillaEnabled()); + } + } + + @Test + public void testStringColumnAfterCloseThrows() { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); + + try { + sender.stringColumn("x", "test"); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + } + + @Test + public void testSymbolAfterCloseThrows() { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); + + try { + sender.symbol("x", "test"); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + } + + @Test + public void testTableBeforeAtNowRequired() { + try { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.atNow(); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("table()")); + } + } + + @Test + public void testTableBeforeAtRequired() { + try { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.at(1000L, ChronoUnit.MICROS); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("table()")); + } + } + + @Test + public void testTableBeforeColumnsRequired() { + // Create sender without connecting (we'll catch the error earlier) + try { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.longColumn("x", 1); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("table()")); + } + } + + @Test + public void testTimestampColumnAfterCloseThrows() { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); + + try { + sender.timestampColumn("x", 1000L, ChronoUnit.MICROS); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + } + + @Test + public void testTimestampColumnInstantAfterCloseThrows() { + QwpWebSocketSender sender = createUnconnectedSender(); + sender.close(); + + try { + sender.timestampColumn("x", Instant.now()); + Assert.fail("Expected LineSenderException"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } + } + + private static MicrobatchBuffer getActiveBuffer(QwpWebSocketSender sender) throws Exception { + Field field = QwpWebSocketSender.class.getDeclaredField("activeBuffer"); + field.setAccessible(true); + return (MicrobatchBuffer) field.get(sender); + } + + private static void invokeSealAndSwapBuffer(QwpWebSocketSender sender) throws Exception { + Method method = QwpWebSocketSender.class.getDeclaredMethod("sealAndSwapBuffer"); + method.setAccessible(true); + try { + method.invoke(sender); + } catch (InvocationTargetException e) { + Throwable cause = e.getCause(); + if (cause instanceof Exception) { + throw (Exception) cause; + } + if (cause instanceof Error) { + throw (Error) cause; + } + throw new RuntimeException(cause); + } + } + + private static void setSendQueue(QwpWebSocketSender sender, WebSocketSendQueue queue) throws Exception { + Field field = QwpWebSocketSender.class.getDeclaredField("sendQueue"); + field.setAccessible(true); + field.set(sender, queue); + } + + /** + * Creates an async sender without connecting. + */ + private QwpWebSocketSender createUnconnectedAsyncSender() { + return QwpWebSocketSender.createForTesting("localhost", 9000, + 500, 0, 0L, // autoFlushRows, autoFlushBytes, autoFlushIntervalNanos + 8); // inFlightWindowSize + } + + /** + * Creates an async sender with custom flow control settings without connecting. + */ + private QwpWebSocketSender createUnconnectedAsyncSenderWithFlowControl( + int autoFlushRows, int autoFlushBytes, long autoFlushIntervalNanos, + int inFlightWindowSize) { + return QwpWebSocketSender.createForTesting("localhost", 9000, + autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, + inFlightWindowSize); + } + + /** + * Creates a sender without connecting. + * For unit tests that don't need actual connectivity. + */ + private QwpWebSocketSender createUnconnectedSender() { + return QwpWebSocketSender.createForTesting("localhost", 9000, 1); // window=1 for sync + } + + private static class NoOpWebSocketClient extends WebSocketClient { + private NoOpWebSocketClient() { + super(DefaultHttpClientConfiguration.INSTANCE, PlainSocketFactory.INSTANCE); + } + + @Override + public boolean isConnected() { + return false; + } + + @Override + public void sendBinary(long dataPtr, int length) { + // no-op + } + + @Override + protected void ioWait(int timeout, int op) { + // no-op + } + + @Override + protected void setupIoWait() { + // no-op + } + } + + private static class ThrowingOnceWebSocketSendQueue extends WebSocketSendQueue { + private boolean failOnce = true; + + private ThrowingOnceWebSocketSendQueue() { + super(new NoOpWebSocketClient(), null, 50, 50); + } + + @Override + public boolean enqueue(MicrobatchBuffer buffer) { + if (failOnce) { + failOnce = false; + throw new LineSenderException("Synthetic enqueue failure"); + } + return true; + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketSendQueueTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketSendQueueTest.java new file mode 100644 index 0000000..4629272 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketSendQueueTest.java @@ -0,0 +1,339 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.DefaultHttpClientConfiguration; +import io.questdb.client.cutlass.http.client.WebSocketClient; +import io.questdb.client.cutlass.http.client.WebSocketFrameHandler; +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.InFlightWindow; +import io.questdb.client.cutlass.qwp.client.MicrobatchBuffer; +import io.questdb.client.cutlass.qwp.client.WebSocketSendQueue; +import io.questdb.client.network.PlainSocketFactory; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import org.junit.Test; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.Assert.*; + +public class WebSocketSendQueueTest { + + @Test + public void testEnqueueTimeoutWhenPendingSlotOccupied() { + InFlightWindow window = new InFlightWindow(1, 1_000); + FakeWebSocketClient client = new FakeWebSocketClient(); + MicrobatchBuffer batch0 = sealedBuffer((byte) 1); + MicrobatchBuffer batch1 = sealedBuffer((byte) 2); + WebSocketSendQueue queue = null; + + try { + // Keep window full so I/O thread cannot drain pending slot. + window.addInFlight(0); + queue = new WebSocketSendQueue(client, window, 100, 500); + queue.enqueue(batch0); + + try { + queue.enqueue(batch1); + fail("Expected enqueue timeout"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("Enqueue timeout")); + } + } finally { + window.acknowledgeUpTo(Long.MAX_VALUE); + closeQuietly(queue); + batch0.close(); + batch1.close(); + client.close(); + } + } + + @Test + public void testEnqueueWaitsUntilSlotAvailable() throws Exception { + InFlightWindow window = new InFlightWindow(1, 1_000); + FakeWebSocketClient client = new FakeWebSocketClient(); + MicrobatchBuffer batch0 = sealedBuffer((byte) 1); + MicrobatchBuffer batch1 = sealedBuffer((byte) 2); + WebSocketSendQueue queue = null; + + try { + window.addInFlight(0); + queue = new WebSocketSendQueue(client, window, 2_000, 500); + final WebSocketSendQueue finalQueue = queue; + queue.enqueue(batch0); + + CountDownLatch started = new CountDownLatch(1); + CountDownLatch finished = new CountDownLatch(1); + AtomicReference errorRef = new AtomicReference<>(); + + Thread t = new Thread(() -> { + started.countDown(); + try { + finalQueue.enqueue(batch1); + } catch (Throwable t1) { + errorRef.set(t1); + } finally { + finished.countDown(); + } + }); + t.start(); + + assertTrue(started.await(1, TimeUnit.SECONDS)); + Thread.sleep(100); + assertEquals("Second enqueue should still be waiting", 1, finished.getCount()); + + // Free space so I/O thread can poll pending slot. + window.acknowledgeUpTo(0); + + assertTrue("Second enqueue should complete", finished.await(2, TimeUnit.SECONDS)); + assertNull(errorRef.get()); + } finally { + window.acknowledgeUpTo(Long.MAX_VALUE); + closeQuietly(queue); + batch0.close(); + batch1.close(); + client.close(); + } + } + + @Test + public void testFlushFailsOnInvalidAckPayload() throws Exception { + InFlightWindow window = new InFlightWindow(8, 5_000); + FakeWebSocketClient client = new FakeWebSocketClient(); + WebSocketSendQueue queue = null; + CountDownLatch payloadDelivered = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(false); + + try { + window.addInFlight(0); + client.setTryReceiveBehavior(handler -> { + if (fired.compareAndSet(false, true)) { + emitBinary(handler, new byte[]{1, 2, 3}); + payloadDelivered.countDown(); + return true; + } + return false; + }); + + queue = new WebSocketSendQueue(client, window, 1_000, 500); + assertTrue("Expected invalid payload callback", payloadDelivered.await(2, TimeUnit.SECONDS)); + + try { + queue.flush(); + fail("Expected flush failure on invalid payload"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("Invalid ACK response payload")); + } + } finally { + closeQuietly(queue); + client.close(); + } + } + + @Test + public void testFlushFailsOnReceiveIoError() throws Exception { + InFlightWindow window = new InFlightWindow(8, 5_000); + FakeWebSocketClient client = new FakeWebSocketClient(); + WebSocketSendQueue queue = null; + CountDownLatch receiveAttempted = new CountDownLatch(1); + + try { + window.addInFlight(0); + client.setTryReceiveBehavior(handler -> { + receiveAttempted.countDown(); + throw new RuntimeException("recv-fail"); + }); + + queue = new WebSocketSendQueue(client, window, 1_000, 500); + assertTrue("Expected receive attempt", receiveAttempted.await(2, TimeUnit.SECONDS)); + long deadline = System.currentTimeMillis() + 2_000; + while (queue.getLastError() == null && System.currentTimeMillis() < deadline) { + Thread.sleep(5); + } + assertNotNull("Expected queue error after receive failure", queue.getLastError()); + + try { + queue.flush(); + fail("Expected flush failure after receive error"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("Error receiving response")); + } + } finally { + closeQuietly(queue); + client.close(); + } + } + + @Test + public void testFlushFailsOnSendIoError() { + FakeWebSocketClient client = new FakeWebSocketClient(); + MicrobatchBuffer batch = sealedBuffer((byte) 42); + WebSocketSendQueue queue = null; + + try { + client.setSendBehavior((dataPtr, length) -> { + throw new RuntimeException("send-fail"); + }); + queue = new WebSocketSendQueue(client, null, 1_000, 500); + queue.enqueue(batch); + + try { + queue.flush(); + fail("Expected flush failure after send error"); + } catch (LineSenderException e) { + assertTrue( + e.getMessage().contains("Error sending batch") + || e.getMessage().contains("Error in send queue I/O thread") + ); + } + } finally { + closeQuietly(queue); + batch.close(); + client.close(); + } + } + + @Test + public void testFlushFailsWhenServerClosesConnection() throws Exception { + InFlightWindow window = new InFlightWindow(8, 5_000); + FakeWebSocketClient client = new FakeWebSocketClient(); + WebSocketSendQueue queue = null; + CountDownLatch closeDelivered = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(false); + + try { + window.addInFlight(0); + client.setTryReceiveBehavior(handler -> { + if (fired.compareAndSet(false, true)) { + handler.onClose(1006, "boom"); + closeDelivered.countDown(); + return true; + } + return false; + }); + + queue = new WebSocketSendQueue(client, window, 1_000, 500); + assertTrue("Expected close callback", closeDelivered.await(2, TimeUnit.SECONDS)); + + try { + queue.flush(); + fail("Expected flush failure after close"); + } catch (LineSenderException e) { + assertTrue(e.getMessage().contains("closed")); + } + } finally { + closeQuietly(queue); + client.close(); + } + } + + private static void closeQuietly(WebSocketSendQueue queue) { + if (queue != null) { + queue.close(); + } + } + + private static void emitBinary(WebSocketFrameHandler handler, byte[] payload) { + long ptr = Unsafe.malloc(payload.length, MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < payload.length; i++) { + Unsafe.getUnsafe().putByte(ptr + i, payload[i]); + } + handler.onBinaryMessage(ptr, payload.length); + } finally { + Unsafe.free(ptr, payload.length, MemoryTag.NATIVE_DEFAULT); + } + } + + private static MicrobatchBuffer sealedBuffer(byte value) { + MicrobatchBuffer buffer = new MicrobatchBuffer(64); + buffer.writeByte(value); + buffer.incrementRowCount(); + buffer.seal(); + return buffer; + } + + private interface SendBehavior { + void send(long dataPtr, int length); + } + + private interface TryReceiveBehavior { + boolean tryReceive(WebSocketFrameHandler handler); + } + + private static class FakeWebSocketClient extends WebSocketClient { + private volatile TryReceiveBehavior behavior = handler -> false; + private volatile boolean connected = true; + private volatile SendBehavior sendBehavior = (dataPtr, length) -> { + }; + + private FakeWebSocketClient() { + super(DefaultHttpClientConfiguration.INSTANCE, PlainSocketFactory.INSTANCE); + } + + @Override + public void close() { + connected = false; + super.close(); + } + + @Override + public boolean isConnected() { + return connected; + } + + @Override + public void sendBinary(long dataPtr, int length) { + sendBehavior.send(dataPtr, length); + } + + public void setSendBehavior(SendBehavior sendBehavior) { + this.sendBehavior = sendBehavior; + } + + public void setTryReceiveBehavior(TryReceiveBehavior behavior) { + this.behavior = behavior; + } + + @Override + public boolean tryReceiveFrame(WebSocketFrameHandler handler) { + return behavior.tryReceive(handler); + } + + @Override + protected void ioWait(int timeout, int op) { + // no-op + } + + @Override + protected void setupIoWait() { + // no-op + } + } +} From 233f4ed71d6fbe26e996bf37ac222e69a539c43b Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 26 Feb 2026 15:57:39 +0100 Subject: [PATCH 78/89] Port 5 QWP protocol tests from core MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add client-side test coverage for QwpVarint (16 tests), QwpZigZag (10 tests), QwpConstants (14 tests), QwpNullBitmap (13 tests), and QwpSchemaHash (20 tests). Adaptations from core originals: - MemoryTag.NATIVE_DEFAULT → NATIVE_ILP_RSS - QwpParseException → IllegalArgumentException - QwpConstants tests cover newer type codes (0x10–0x16) and FLAG_DELTA_SYMBOL_DICT Co-Authored-By: Claude Opus 4.6 --- .../qwp/protocol/QwpConstantsTest.java | 232 +++++++++++++ .../qwp/protocol/QwpNullBitmapTest.java | 289 +++++++++++++++ .../qwp/protocol/QwpSchemaHashTest.java | 328 ++++++++++++++++++ .../cutlass/qwp/protocol/QwpVarintTest.java | 262 ++++++++++++++ .../cutlass/qwp/protocol/QwpZigZagTest.java | 166 +++++++++ 5 files changed, 1277 insertions(+) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpConstantsTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpNullBitmapTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaHashTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpVarintTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpZigZagTest.java diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpConstantsTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpConstantsTest.java new file mode 100644 index 0000000..2b6ce04 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpConstantsTest.java @@ -0,0 +1,232 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import org.junit.Assert; +import org.junit.Test; + +import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.*; + +public class QwpConstantsTest { + + @Test + public void testDefaultLimits() { + Assert.assertEquals(16 * 1024 * 1024, DEFAULT_MAX_BATCH_SIZE); + Assert.assertEquals(256, DEFAULT_MAX_TABLES_PER_BATCH); + Assert.assertEquals(1_000_000, DEFAULT_MAX_ROWS_PER_TABLE); + Assert.assertEquals(2048, MAX_COLUMNS_PER_TABLE); + Assert.assertEquals(64 * 1024, DEFAULT_INITIAL_RECV_BUFFER_SIZE); + Assert.assertEquals(4, DEFAULT_MAX_IN_FLIGHT_BATCHES); + } + + @Test + public void testFlagBitPositions() { + // Verify flag bits are at correct positions + Assert.assertEquals(0x01, FLAG_LZ4); + Assert.assertEquals(0x02, FLAG_ZSTD); + Assert.assertEquals(0x04, FLAG_GORILLA); + Assert.assertEquals(0x03, FLAG_COMPRESSION_MASK); + Assert.assertEquals(0x08, FLAG_DELTA_SYMBOL_DICT); + } + + @Test + public void testGetFixedTypeSize() { + Assert.assertEquals(0, QwpConstants.getFixedTypeSize(TYPE_BOOLEAN)); // Bit-packed + Assert.assertEquals(1, QwpConstants.getFixedTypeSize(TYPE_BYTE)); + Assert.assertEquals(2, QwpConstants.getFixedTypeSize(TYPE_SHORT)); + Assert.assertEquals(2, QwpConstants.getFixedTypeSize(TYPE_CHAR)); + Assert.assertEquals(4, QwpConstants.getFixedTypeSize(TYPE_INT)); + Assert.assertEquals(8, QwpConstants.getFixedTypeSize(TYPE_LONG)); + Assert.assertEquals(4, QwpConstants.getFixedTypeSize(TYPE_FLOAT)); + Assert.assertEquals(8, QwpConstants.getFixedTypeSize(TYPE_DOUBLE)); + Assert.assertEquals(8, QwpConstants.getFixedTypeSize(TYPE_TIMESTAMP)); + Assert.assertEquals(8, QwpConstants.getFixedTypeSize(TYPE_TIMESTAMP_NANOS)); + Assert.assertEquals(8, QwpConstants.getFixedTypeSize(TYPE_DATE)); + Assert.assertEquals(16, QwpConstants.getFixedTypeSize(TYPE_UUID)); + Assert.assertEquals(32, QwpConstants.getFixedTypeSize(TYPE_LONG256)); + Assert.assertEquals(8, QwpConstants.getFixedTypeSize(TYPE_DECIMAL64)); + Assert.assertEquals(16, QwpConstants.getFixedTypeSize(TYPE_DECIMAL128)); + Assert.assertEquals(32, QwpConstants.getFixedTypeSize(TYPE_DECIMAL256)); + + Assert.assertEquals(-1, QwpConstants.getFixedTypeSize(TYPE_STRING)); + Assert.assertEquals(-1, QwpConstants.getFixedTypeSize(TYPE_SYMBOL)); + Assert.assertEquals(-1, QwpConstants.getFixedTypeSize(TYPE_DOUBLE_ARRAY)); + Assert.assertEquals(-1, QwpConstants.getFixedTypeSize(TYPE_LONG_ARRAY)); + } + + @Test + public void testGetTypeName() { + Assert.assertEquals("BOOLEAN", QwpConstants.getTypeName(TYPE_BOOLEAN)); + Assert.assertEquals("INT", QwpConstants.getTypeName(TYPE_INT)); + Assert.assertEquals("STRING", QwpConstants.getTypeName(TYPE_STRING)); + Assert.assertEquals("TIMESTAMP", QwpConstants.getTypeName(TYPE_TIMESTAMP)); + Assert.assertEquals("TIMESTAMP_NANOS", QwpConstants.getTypeName(TYPE_TIMESTAMP_NANOS)); + Assert.assertEquals("DOUBLE_ARRAY", QwpConstants.getTypeName(TYPE_DOUBLE_ARRAY)); + Assert.assertEquals("LONG_ARRAY", QwpConstants.getTypeName(TYPE_LONG_ARRAY)); + Assert.assertEquals("DECIMAL64", QwpConstants.getTypeName(TYPE_DECIMAL64)); + Assert.assertEquals("DECIMAL128", QwpConstants.getTypeName(TYPE_DECIMAL128)); + Assert.assertEquals("DECIMAL256", QwpConstants.getTypeName(TYPE_DECIMAL256)); + Assert.assertEquals("CHAR", QwpConstants.getTypeName(TYPE_CHAR)); + + // Test nullable types + byte nullableInt = (byte) (TYPE_INT | TYPE_NULLABLE_FLAG); + Assert.assertEquals("INT?", QwpConstants.getTypeName(nullableInt)); + + byte nullableString = (byte) (TYPE_STRING | TYPE_NULLABLE_FLAG); + Assert.assertEquals("STRING?", QwpConstants.getTypeName(nullableString)); + } + + @Test + public void testHeaderSize() { + Assert.assertEquals(12, HEADER_SIZE); + } + + @Test + public void testIsFixedWidthType() { + Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_BOOLEAN)); + Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_BYTE)); + Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_SHORT)); + Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_CHAR)); + Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_INT)); + Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_LONG)); + Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_FLOAT)); + Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_DOUBLE)); + Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_TIMESTAMP)); + Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_TIMESTAMP_NANOS)); + Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_DATE)); + Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_UUID)); + Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_LONG256)); + Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_DECIMAL64)); + Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_DECIMAL128)); + Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_DECIMAL256)); + + Assert.assertFalse(QwpConstants.isFixedWidthType(TYPE_STRING)); + Assert.assertFalse(QwpConstants.isFixedWidthType(TYPE_SYMBOL)); + Assert.assertFalse(QwpConstants.isFixedWidthType(TYPE_GEOHASH)); + Assert.assertFalse(QwpConstants.isFixedWidthType(TYPE_VARCHAR)); + Assert.assertFalse(QwpConstants.isFixedWidthType(TYPE_DOUBLE_ARRAY)); + Assert.assertFalse(QwpConstants.isFixedWidthType(TYPE_LONG_ARRAY)); + } + + @Test + public void testMagicBytesCapabilityRequest() { + // "ILP?" in ASCII + byte[] expected = new byte[]{'I', 'L', 'P', '?'}; + Assert.assertEquals((byte) (MAGIC_CAPABILITY_REQUEST & 0xFF), expected[0]); + Assert.assertEquals((byte) ((MAGIC_CAPABILITY_REQUEST >> 8) & 0xFF), expected[1]); + Assert.assertEquals((byte) ((MAGIC_CAPABILITY_REQUEST >> 16) & 0xFF), expected[2]); + Assert.assertEquals((byte) ((MAGIC_CAPABILITY_REQUEST >> 24) & 0xFF), expected[3]); + } + + @Test + public void testMagicBytesCapabilityResponse() { + // "ILP!" in ASCII + byte[] expected = new byte[]{'I', 'L', 'P', '!'}; + Assert.assertEquals((byte) (MAGIC_CAPABILITY_RESPONSE & 0xFF), expected[0]); + Assert.assertEquals((byte) ((MAGIC_CAPABILITY_RESPONSE >> 8) & 0xFF), expected[1]); + Assert.assertEquals((byte) ((MAGIC_CAPABILITY_RESPONSE >> 16) & 0xFF), expected[2]); + Assert.assertEquals((byte) ((MAGIC_CAPABILITY_RESPONSE >> 24) & 0xFF), expected[3]); + } + + @Test + public void testMagicBytesFallback() { + // "ILP0" in ASCII + byte[] expected = new byte[]{'I', 'L', 'P', '0'}; + Assert.assertEquals((byte) (MAGIC_FALLBACK & 0xFF), expected[0]); + Assert.assertEquals((byte) ((MAGIC_FALLBACK >> 8) & 0xFF), expected[1]); + Assert.assertEquals((byte) ((MAGIC_FALLBACK >> 16) & 0xFF), expected[2]); + Assert.assertEquals((byte) ((MAGIC_FALLBACK >> 24) & 0xFF), expected[3]); + } + + @Test + public void testMagicBytesValue() { + // "ILP4" in ASCII: I=0x49, L=0x4C, P=0x50, 4=0x34 + // Little-endian: 0x34504C49 + Assert.assertEquals(0x34504C49, MAGIC_MESSAGE); + + // Verify ASCII encoding + byte[] expected = new byte[]{'I', 'L', 'P', '4'}; + Assert.assertEquals((byte) (MAGIC_MESSAGE & 0xFF), expected[0]); + Assert.assertEquals((byte) ((MAGIC_MESSAGE >> 8) & 0xFF), expected[1]); + Assert.assertEquals((byte) ((MAGIC_MESSAGE >> 16) & 0xFF), expected[2]); + Assert.assertEquals((byte) ((MAGIC_MESSAGE >> 24) & 0xFF), expected[3]); + } + + @Test + public void testNullableFlag() { + Assert.assertEquals((byte) 0x80, TYPE_NULLABLE_FLAG); + Assert.assertEquals(0x7F, TYPE_MASK); + + // Test nullable type extraction + byte nullableInt = (byte) (TYPE_INT | TYPE_NULLABLE_FLAG); + Assert.assertEquals(TYPE_INT, nullableInt & TYPE_MASK); + } + + @Test + public void testSchemaModes() { + Assert.assertEquals(0x00, SCHEMA_MODE_FULL); + Assert.assertEquals(0x01, SCHEMA_MODE_REFERENCE); + } + + @Test + public void testStatusCodes() { + Assert.assertEquals(0x00, STATUS_OK); + Assert.assertEquals(0x01, STATUS_PARTIAL); + Assert.assertEquals(0x02, STATUS_SCHEMA_REQUIRED); + Assert.assertEquals(0x03, STATUS_SCHEMA_MISMATCH); + Assert.assertEquals(0x04, STATUS_TABLE_NOT_FOUND); + Assert.assertEquals(0x05, STATUS_PARSE_ERROR); + Assert.assertEquals(0x06, STATUS_INTERNAL_ERROR); + Assert.assertEquals(0x07, STATUS_OVERLOADED); + } + + @Test + public void testTypeCodes() { + // Verify type codes match specification + Assert.assertEquals(0x01, TYPE_BOOLEAN); + Assert.assertEquals(0x02, TYPE_BYTE); + Assert.assertEquals(0x03, TYPE_SHORT); + Assert.assertEquals(0x04, TYPE_INT); + Assert.assertEquals(0x05, TYPE_LONG); + Assert.assertEquals(0x06, TYPE_FLOAT); + Assert.assertEquals(0x07, TYPE_DOUBLE); + Assert.assertEquals(0x08, TYPE_STRING); + Assert.assertEquals(0x09, TYPE_SYMBOL); + Assert.assertEquals(0x0A, TYPE_TIMESTAMP); + Assert.assertEquals(0x0B, TYPE_DATE); + Assert.assertEquals(0x0C, TYPE_UUID); + Assert.assertEquals(0x0D, TYPE_LONG256); + Assert.assertEquals(0x0E, TYPE_GEOHASH); + Assert.assertEquals(0x0F, TYPE_VARCHAR); + Assert.assertEquals(0x10, TYPE_TIMESTAMP_NANOS); + Assert.assertEquals(0x11, TYPE_DOUBLE_ARRAY); + Assert.assertEquals(0x12, TYPE_LONG_ARRAY); + Assert.assertEquals(0x13, TYPE_DECIMAL64); + Assert.assertEquals(0x14, TYPE_DECIMAL128); + Assert.assertEquals(0x15, TYPE_DECIMAL256); + Assert.assertEquals(0x16, TYPE_CHAR); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpNullBitmapTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpNullBitmapTest.java new file mode 100644 index 0000000..086d24f --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpNullBitmapTest.java @@ -0,0 +1,289 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.cutlass.qwp.protocol.QwpNullBitmap; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; +import org.junit.Test; + +public class QwpNullBitmapTest { + + @Test + public void testAllNulls() { + int rowCount = 16; + int size = QwpNullBitmap.sizeInBytes(rowCount); + long address = Unsafe.malloc(size, MemoryTag.NATIVE_ILP_RSS); + try { + QwpNullBitmap.fillAllNull(address, rowCount); + Assert.assertTrue(QwpNullBitmap.allNull(address, rowCount)); + Assert.assertEquals(rowCount, QwpNullBitmap.countNulls(address, rowCount)); + + for (int i = 0; i < rowCount; i++) { + Assert.assertTrue(QwpNullBitmap.isNull(address, i)); + } + } finally { + Unsafe.free(address, size, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testAllNullsPartialByte() { + // Test with row count not divisible by 8 + int rowCount = 10; + int size = QwpNullBitmap.sizeInBytes(rowCount); + long address = Unsafe.malloc(size, MemoryTag.NATIVE_ILP_RSS); + try { + QwpNullBitmap.fillAllNull(address, rowCount); + Assert.assertTrue(QwpNullBitmap.allNull(address, rowCount)); + Assert.assertEquals(rowCount, QwpNullBitmap.countNulls(address, rowCount)); + } finally { + Unsafe.free(address, size, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testBitmapBitOrder() { + // Test LSB-first bit ordering + int rowCount = 8; + int size = QwpNullBitmap.sizeInBytes(rowCount); + long address = Unsafe.malloc(size, MemoryTag.NATIVE_ILP_RSS); + try { + QwpNullBitmap.fillNoneNull(address, rowCount); + + // Set bit 0 (LSB) + QwpNullBitmap.setNull(address, 0); + byte b = Unsafe.getUnsafe().getByte(address); + Assert.assertEquals(0b00000001, b & 0xFF); + + // Set bit 7 (MSB of first byte) + QwpNullBitmap.setNull(address, 7); + b = Unsafe.getUnsafe().getByte(address); + Assert.assertEquals(0b10000001, b & 0xFF); + + // Set bit 3 + QwpNullBitmap.setNull(address, 3); + b = Unsafe.getUnsafe().getByte(address); + Assert.assertEquals(0b10001001, b & 0xFF); + } finally { + Unsafe.free(address, size, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testBitmapByteAlignment() { + // Test that bits 8-15 go into second byte + int rowCount = 16; + int size = QwpNullBitmap.sizeInBytes(rowCount); + long address = Unsafe.malloc(size, MemoryTag.NATIVE_ILP_RSS); + try { + QwpNullBitmap.fillNoneNull(address, rowCount); + + // Set bit 8 (first bit of second byte) + QwpNullBitmap.setNull(address, 8); + Assert.assertEquals(0, Unsafe.getUnsafe().getByte(address) & 0xFF); + Assert.assertEquals(0b00000001, Unsafe.getUnsafe().getByte(address + 1) & 0xFF); + + // Set bit 15 (last bit of second byte) + QwpNullBitmap.setNull(address, 15); + Assert.assertEquals(0b10000001, Unsafe.getUnsafe().getByte(address + 1) & 0xFF); + } finally { + Unsafe.free(address, size, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testBitmapSizeCalculation() { + Assert.assertEquals(0, QwpNullBitmap.sizeInBytes(0)); + Assert.assertEquals(1, QwpNullBitmap.sizeInBytes(1)); + Assert.assertEquals(1, QwpNullBitmap.sizeInBytes(7)); + Assert.assertEquals(1, QwpNullBitmap.sizeInBytes(8)); + Assert.assertEquals(2, QwpNullBitmap.sizeInBytes(9)); + Assert.assertEquals(2, QwpNullBitmap.sizeInBytes(16)); + Assert.assertEquals(3, QwpNullBitmap.sizeInBytes(17)); + Assert.assertEquals(125, QwpNullBitmap.sizeInBytes(1000)); + Assert.assertEquals(125000, QwpNullBitmap.sizeInBytes(1000000)); + } + + @Test + public void testBitmapWithPartialLastByte() { + // 10 rows = 2 bytes, but only 2 bits used in second byte + int rowCount = 10; + int size = QwpNullBitmap.sizeInBytes(rowCount); + Assert.assertEquals(2, size); + + long address = Unsafe.malloc(size, MemoryTag.NATIVE_ILP_RSS); + try { + QwpNullBitmap.fillNoneNull(address, rowCount); + + // Set row 9 (bit 1 of second byte) + QwpNullBitmap.setNull(address, 9); + Assert.assertTrue(QwpNullBitmap.isNull(address, 9)); + Assert.assertEquals(0b00000010, Unsafe.getUnsafe().getByte(address + 1) & 0xFF); + + Assert.assertEquals(1, QwpNullBitmap.countNulls(address, rowCount)); + } finally { + Unsafe.free(address, size, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testByteArrayOperations() { + int rowCount = 16; + int size = QwpNullBitmap.sizeInBytes(rowCount); + byte[] bitmap = new byte[size]; + int offset = 0; + + QwpNullBitmap.fillNoneNull(bitmap, offset, rowCount); + + QwpNullBitmap.setNull(bitmap, offset, 0); + QwpNullBitmap.setNull(bitmap, offset, 5); + QwpNullBitmap.setNull(bitmap, offset, 15); + + Assert.assertTrue(QwpNullBitmap.isNull(bitmap, offset, 0)); + Assert.assertFalse(QwpNullBitmap.isNull(bitmap, offset, 1)); + Assert.assertTrue(QwpNullBitmap.isNull(bitmap, offset, 5)); + Assert.assertTrue(QwpNullBitmap.isNull(bitmap, offset, 15)); + + Assert.assertEquals(3, QwpNullBitmap.countNulls(bitmap, offset, rowCount)); + + QwpNullBitmap.clearNull(bitmap, offset, 5); + Assert.assertFalse(QwpNullBitmap.isNull(bitmap, offset, 5)); + Assert.assertEquals(2, QwpNullBitmap.countNulls(bitmap, offset, rowCount)); + } + + @Test + public void testByteArrayWithOffset() { + int rowCount = 8; + int size = QwpNullBitmap.sizeInBytes(rowCount); + byte[] bitmap = new byte[10 + size]; // Extra padding + int offset = 5; // Start at offset 5 + + QwpNullBitmap.fillNoneNull(bitmap, offset, rowCount); + QwpNullBitmap.setNull(bitmap, offset, 3); + + Assert.assertTrue(QwpNullBitmap.isNull(bitmap, offset, 3)); + Assert.assertFalse(QwpNullBitmap.isNull(bitmap, offset, 4)); + } + + @Test + public void testClearNull() { + int rowCount = 8; + int size = QwpNullBitmap.sizeInBytes(rowCount); + long address = Unsafe.malloc(size, MemoryTag.NATIVE_ILP_RSS); + try { + QwpNullBitmap.fillAllNull(address, rowCount); + Assert.assertTrue(QwpNullBitmap.isNull(address, 3)); + + QwpNullBitmap.clearNull(address, 3); + Assert.assertFalse(QwpNullBitmap.isNull(address, 3)); + Assert.assertEquals(7, QwpNullBitmap.countNulls(address, rowCount)); + } finally { + Unsafe.free(address, size, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testEmptyBitmap() { + Assert.assertEquals(0, QwpNullBitmap.sizeInBytes(0)); + } + + @Test + public void testLargeBitmap() { + int rowCount = 100000; + int size = QwpNullBitmap.sizeInBytes(rowCount); + long address = Unsafe.malloc(size, MemoryTag.NATIVE_ILP_RSS); + try { + QwpNullBitmap.fillNoneNull(address, rowCount); + + // Set every 100th row as null + int expectedNulls = 0; + for (int i = 0; i < rowCount; i += 100) { + QwpNullBitmap.setNull(address, i); + expectedNulls++; + } + + Assert.assertEquals(expectedNulls, QwpNullBitmap.countNulls(address, rowCount)); + + // Verify some random positions + Assert.assertTrue(QwpNullBitmap.isNull(address, 0)); + Assert.assertTrue(QwpNullBitmap.isNull(address, 100)); + Assert.assertTrue(QwpNullBitmap.isNull(address, 99900)); + Assert.assertFalse(QwpNullBitmap.isNull(address, 1)); + Assert.assertFalse(QwpNullBitmap.isNull(address, 99)); + } finally { + Unsafe.free(address, size, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testMixedNulls() { + int rowCount = 20; + int size = QwpNullBitmap.sizeInBytes(rowCount); + long address = Unsafe.malloc(size, MemoryTag.NATIVE_ILP_RSS); + try { + QwpNullBitmap.fillNoneNull(address, rowCount); + + // Set specific rows as null: 0, 2, 5, 19 + QwpNullBitmap.setNull(address, 0); + QwpNullBitmap.setNull(address, 2); + QwpNullBitmap.setNull(address, 5); + QwpNullBitmap.setNull(address, 19); + + Assert.assertTrue(QwpNullBitmap.isNull(address, 0)); + Assert.assertFalse(QwpNullBitmap.isNull(address, 1)); + Assert.assertTrue(QwpNullBitmap.isNull(address, 2)); + Assert.assertFalse(QwpNullBitmap.isNull(address, 3)); + Assert.assertFalse(QwpNullBitmap.isNull(address, 4)); + Assert.assertTrue(QwpNullBitmap.isNull(address, 5)); + Assert.assertTrue(QwpNullBitmap.isNull(address, 19)); + + Assert.assertEquals(4, QwpNullBitmap.countNulls(address, rowCount)); + Assert.assertFalse(QwpNullBitmap.allNull(address, rowCount)); + Assert.assertFalse(QwpNullBitmap.noneNull(address, rowCount)); + } finally { + Unsafe.free(address, size, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testNoNulls() { + int rowCount = 16; + int size = QwpNullBitmap.sizeInBytes(rowCount); + long address = Unsafe.malloc(size, MemoryTag.NATIVE_ILP_RSS); + try { + QwpNullBitmap.fillNoneNull(address, rowCount); + Assert.assertTrue(QwpNullBitmap.noneNull(address, rowCount)); + Assert.assertEquals(0, QwpNullBitmap.countNulls(address, rowCount)); + + for (int i = 0; i < rowCount; i++) { + Assert.assertFalse(QwpNullBitmap.isNull(address, i)); + } + } finally { + Unsafe.free(address, size, MemoryTag.NATIVE_ILP_RSS); + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaHashTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaHashTest.java new file mode 100644 index 0000000..75cae68 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaHashTest.java @@ -0,0 +1,328 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaHash; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; +import org.junit.Test; + +import java.nio.charset.StandardCharsets; + +public class QwpSchemaHashTest { + + private static final byte TYPE_LONG = 0x05; + + @Test + public void testColumnOrderMatters() { + // Order 1 + String[] names1 = {"price", "symbol"}; + byte[] types1 = {0x07, 0x09}; + + // Order 2 (different order) + String[] names2 = {"symbol", "price"}; + byte[] types2 = {0x09, 0x07}; + + long hash1 = QwpSchemaHash.computeSchemaHash(names1, types1); + long hash2 = QwpSchemaHash.computeSchemaHash(names2, types2); + + Assert.assertNotEquals("Column order should affect hash", hash1, hash2); + } + + @Test + public void testDeterministic() { + String[] names = {"col1", "col2", "col3"}; + byte[] types = {0x01, 0x02, 0x03}; + + long hash1 = QwpSchemaHash.computeSchemaHash(names, types); + long hash2 = QwpSchemaHash.computeSchemaHash(names, types); + long hash3 = QwpSchemaHash.computeSchemaHash(names, types); + + Assert.assertEquals("Hash should be deterministic", hash1, hash2); + Assert.assertEquals("Hash should be deterministic", hash2, hash3); + } + + @Test + public void testEmptySchema() { + String[] names = {}; + byte[] types = {}; + long hash = QwpSchemaHash.computeSchemaHash(names, types); + // Empty input should produce the same hash consistently + Assert.assertEquals(hash, QwpSchemaHash.computeSchemaHash(names, types)); + } + + @Test + public void testHasherReset() { + QwpSchemaHash.Hasher hasher = new QwpSchemaHash.Hasher(); + + byte[] data1 = "first".getBytes(StandardCharsets.UTF_8); + byte[] data2 = "second".getBytes(StandardCharsets.UTF_8); + + // Hash first data + hasher.reset(0); + hasher.update(data1); + long hash1 = hasher.getValue(); + + // Reset and hash second data + hasher.reset(0); + hasher.update(data2); + long hash2 = hasher.getValue(); + + // Should be different + Assert.assertNotEquals(hash1, hash2); + + // Reset and hash first again - should be same as original + hasher.reset(0); + hasher.update(data1); + Assert.assertEquals(hash1, hasher.getValue()); + } + + @Test + public void testHasherStreaming() { + // Test that streaming hasher produces same result as one-shot + byte[] data = "streaming test data for the hasher".getBytes(StandardCharsets.UTF_8); + + // One-shot + long oneShot = QwpSchemaHash.hash(data); + + // Streaming - byte by byte + QwpSchemaHash.Hasher hasher = new QwpSchemaHash.Hasher(); + hasher.reset(0); + for (byte b : data) { + hasher.update(b); + } + long streaming = hasher.getValue(); + + Assert.assertEquals("Streaming should match one-shot", oneShot, streaming); + } + + @Test + public void testHasherStreamingChunks() { + // Test streaming with various chunk sizes + byte[] data = "This is a longer test string to verify chunked hashing works correctly!".getBytes(StandardCharsets.UTF_8); + + long oneShot = QwpSchemaHash.hash(data); + + // Streaming - in chunks + QwpSchemaHash.Hasher hasher = new QwpSchemaHash.Hasher(); + hasher.reset(0); + + int pos = 0; + int[] chunkSizes = {5, 10, 3, 20, 7, 15}; + for (int chunkSize : chunkSizes) { + int toAdd = Math.min(chunkSize, data.length - pos); + if (toAdd > 0) { + hasher.update(data, pos, toAdd); + pos += toAdd; + } + } + // Add remaining + if (pos < data.length) { + hasher.update(data, pos, data.length - pos); + } + + Assert.assertEquals("Chunked streaming should match one-shot", oneShot, hasher.getValue()); + } + + @Test + public void testLargeSchema() { + // Test with many columns + int columnCount = 100; + String[] names = new String[columnCount]; + byte[] types = new byte[columnCount]; + + for (int i = 0; i < columnCount; i++) { + names[i] = "column_" + i; + types[i] = (byte) ((i % 15) + 1); // Cycle through types 1-15 + } + + long hash1 = QwpSchemaHash.computeSchemaHash(names, types); + long hash2 = QwpSchemaHash.computeSchemaHash(names, types); + + Assert.assertEquals("Large schema should hash consistently", hash1, hash2); + } + + @Test + public void testMultipleColumns() { + String[] names = {"symbol", "price", "timestamp"}; + byte[] types = {0x09, 0x07, 0x0A}; // SYMBOL, DOUBLE, TIMESTAMP + long hash = QwpSchemaHash.computeSchemaHash(names, types); + Assert.assertNotEquals(0, hash); + } + + @Test + public void testNameAffectsHash() { + // Different names, same type + byte[] types = {0x07}; // DOUBLE + + String[] names1 = {"price"}; + String[] names2 = {"value"}; + + long hash1 = QwpSchemaHash.computeSchemaHash(names1, types); + long hash2 = QwpSchemaHash.computeSchemaHash(names2, types); + + Assert.assertNotEquals("Name should affect hash", hash1, hash2); + } + + @Test + public void testNullableFlagAffectsHash() { + String[] names = {"value"}; + + // Non-nullable + byte[] types1 = {0x05}; // LONG + // Nullable (high bit set) + byte[] types2 = {(byte) 0x85}; // LONG | 0x80 + + long hash1 = QwpSchemaHash.computeSchemaHash(names, types1); + long hash2 = QwpSchemaHash.computeSchemaHash(names, types2); + + Assert.assertNotEquals("Nullable flag should affect hash", hash1, hash2); + } + + @Test + public void testSchemaHashWithUtf8Names() { + // Test UTF-8 column names + String[] names = {"prix", "日時", "価格"}; // French, Japanese for datetime, Japanese for price + byte[] types = {0x07, 0x0A, 0x07}; + + long hash1 = QwpSchemaHash.computeSchemaHash(names, types); + long hash2 = QwpSchemaHash.computeSchemaHash(names, types); + + Assert.assertEquals("UTF-8 names should hash consistently", hash1, hash2); + Assert.assertNotEquals(0, hash1); + } + + @Test + public void testSingleColumn() { + String[] names = {"price"}; + byte[] types = {0x07}; // DOUBLE + long hash = QwpSchemaHash.computeSchemaHash(names, types); + Assert.assertNotEquals(0, hash); + } + + @Test + public void testTypeAffectsHash() { + // Same name, different type + String[] names = {"value"}; + + byte[] types1 = {0x04}; // INT + byte[] types2 = {0x05}; // LONG + + long hash1 = QwpSchemaHash.computeSchemaHash(names, types1); + long hash2 = QwpSchemaHash.computeSchemaHash(names, types2); + + Assert.assertNotEquals("Type should affect hash", hash1, hash2); + } + + @Test + public void testXXHash64DirectMemory() { + byte[] data = "test data".getBytes(StandardCharsets.UTF_8); + long addr = Unsafe.malloc(data.length, MemoryTag.NATIVE_ILP_RSS); + try { + for (int i = 0; i < data.length; i++) { + Unsafe.getUnsafe().putByte(addr + i, data[i]); + } + + long hashFromBytes = QwpSchemaHash.hash(data); + long hashFromMem = QwpSchemaHash.hash(addr, data.length); + + Assert.assertEquals("Direct memory hash should match byte array hash", hashFromBytes, hashFromMem); + } finally { + Unsafe.free(addr, data.length, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testXXHash64Empty() { + byte[] data = new byte[0]; + long hash = QwpSchemaHash.hash(data); + // XXH64("", 0) = 0xEF46DB3751D8E999 + Assert.assertEquals(0xEF46DB3751D8E999L, hash); + } + + @Test + public void testXXHash64Exactly32Bytes() { + // Edge case: exactly 32 bytes + byte[] data = new byte[32]; + for (int i = 0; i < data.length; i++) { + data[i] = (byte) (i & 0xFF); + } + + long hash = QwpSchemaHash.hash(data); + Assert.assertNotEquals(0, hash); + Assert.assertEquals(hash, QwpSchemaHash.hash(data)); + } + + @Test + public void testXXHash64KnownValue() { + // Test against a known XXHash64 value + // "abc" with seed 0 should produce a specific value + byte[] data = "abc".getBytes(StandardCharsets.UTF_8); + long hash = QwpSchemaHash.hash(data); + + // XXH64("abc", 0) = 0x44BC2CF5AD770999 + Assert.assertEquals(0x44BC2CF5AD770999L, hash); + } + + @Test + public void testXXHash64LongerString() { + // Test with a longer string to exercise the main loop + byte[] data = "Hello, World! This is a test string for XXHash64.".getBytes(StandardCharsets.UTF_8); + long hash1 = QwpSchemaHash.hash(data); + long hash2 = QwpSchemaHash.hash(data); + Assert.assertEquals(hash1, hash2); + Assert.assertNotEquals(0, hash1); + } + + @Test + public void testXXHash64Over32Bytes() { + // Test data longer than 32 bytes to exercise the main processing loop + byte[] data = new byte[100]; + for (int i = 0; i < data.length; i++) { + data[i] = (byte) (i & 0xFF); + } + + long hash = QwpSchemaHash.hash(data); + Assert.assertNotEquals(0, hash); + + // Verify deterministic + Assert.assertEquals(hash, QwpSchemaHash.hash(data)); + } + + @Test + public void testXXHash64WithSeed() { + byte[] data = "test".getBytes(StandardCharsets.UTF_8); + + long hash0 = QwpSchemaHash.hash(data, 0, data.length, 0); + long hash1 = QwpSchemaHash.hash(data, 0, data.length, 1); + long hash42 = QwpSchemaHash.hash(data, 0, data.length, 42); + + // Different seeds should produce different hashes + Assert.assertNotEquals(hash0, hash1); + Assert.assertNotEquals(hash1, hash42); + Assert.assertNotEquals(hash0, hash42); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpVarintTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpVarintTest.java new file mode 100644 index 0000000..558ecd2 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpVarintTest.java @@ -0,0 +1,262 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.cutlass.qwp.protocol.QwpVarint; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Random; + +public class QwpVarintTest { + + @Test + public void testDecodeFromDirectMemory() { + long addr = Unsafe.malloc(16, MemoryTag.NATIVE_ILP_RSS); + try { + // Encode using byte array, decode from direct memory + byte[] buf = new byte[10]; + int len = QwpVarint.encode(buf, 0, 300); + + for (int i = 0; i < len; i++) { + Unsafe.getUnsafe().putByte(addr + i, buf[i]); + } + + long decoded = QwpVarint.decode(addr, addr + len); + Assert.assertEquals(300, decoded); + } finally { + Unsafe.free(addr, 16, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testDecodeIncompleteVarint() { + // Byte with continuation bit set but no following byte + byte[] buf = new byte[]{(byte) 0x80}; + try { + QwpVarint.decode(buf, 0, 1); + Assert.fail("Should have thrown exception"); + } catch (IllegalArgumentException e) { + Assert.assertTrue(e.getMessage().contains("incomplete varint")); + } + } + + @Test + public void testDecodeOverflow() { + // Create a buffer with too many continuation bytes (>10) + byte[] buf = new byte[12]; + for (int i = 0; i < 11; i++) { + buf[i] = (byte) 0x80; + } + buf[11] = 0x01; + + try { + QwpVarint.decode(buf, 0, 12); + Assert.fail("Should have thrown exception"); + } catch (IllegalArgumentException e) { + Assert.assertTrue(e.getMessage().contains("varint overflow")); + } + } + + @Test + public void testDecodeResult() { + byte[] buf = new byte[10]; + int len = QwpVarint.encode(buf, 0, 300); + + QwpVarint.DecodeResult result = new QwpVarint.DecodeResult(); + QwpVarint.decode(buf, 0, len, result); + + Assert.assertEquals(300, result.value); + Assert.assertEquals(len, result.bytesRead); + } + + @Test + public void testDecodeResultFromDirectMemory() { + long addr = Unsafe.malloc(16, MemoryTag.NATIVE_ILP_RSS); + try { + long endAddr = QwpVarint.encode(addr, 999999); + int expectedLen = (int) (endAddr - addr); + + QwpVarint.DecodeResult result = new QwpVarint.DecodeResult(); + QwpVarint.decode(addr, endAddr, result); + + Assert.assertEquals(999999, result.value); + Assert.assertEquals(expectedLen, result.bytesRead); + } finally { + Unsafe.free(addr, 16, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testDecodeResultReuse() { + byte[] buf = new byte[10]; + QwpVarint.DecodeResult result = new QwpVarint.DecodeResult(); + + // First decode + int len1 = QwpVarint.encode(buf, 0, 100); + QwpVarint.decode(buf, 0, len1, result); + Assert.assertEquals(100, result.value); + + // Reuse for second decode + result.reset(); + int len2 = QwpVarint.encode(buf, 0, 50000); + QwpVarint.decode(buf, 0, len2, result); + Assert.assertEquals(50000, result.value); + } + + @Test + public void testEncodeDecode127() { + // 127 is the maximum 1-byte value + byte[] buf = new byte[10]; + int len = QwpVarint.encode(buf, 0, 127); + Assert.assertEquals(1, len); + Assert.assertEquals(0x7F, buf[0] & 0xFF); + Assert.assertEquals(127, QwpVarint.decode(buf, 0, len)); + } + + @Test + public void testEncodeDecode128() { + // 128 is the minimum 2-byte value + byte[] buf = new byte[10]; + int len = QwpVarint.encode(buf, 0, 128); + Assert.assertEquals(2, len); + Assert.assertEquals(0x80, buf[0] & 0xFF); // 0 + continuation bit + Assert.assertEquals(0x01, buf[1] & 0xFF); // 1 + Assert.assertEquals(128, QwpVarint.decode(buf, 0, len)); + } + + @Test + public void testEncodeDecode16383() { + // 16383 (0x3FFF) is the maximum 2-byte value + byte[] buf = new byte[10]; + int len = QwpVarint.encode(buf, 0, 16383); + Assert.assertEquals(2, len); + Assert.assertEquals(0xFF, buf[0] & 0xFF); // 127 + continuation bit + Assert.assertEquals(0x7F, buf[1] & 0xFF); // 127 + Assert.assertEquals(16383, QwpVarint.decode(buf, 0, len)); + } + + @Test + public void testEncodeDecode16384() { + // 16384 (0x4000) is the minimum 3-byte value + byte[] buf = new byte[10]; + int len = QwpVarint.encode(buf, 0, 16384); + Assert.assertEquals(3, len); + Assert.assertEquals(16384, QwpVarint.decode(buf, 0, len)); + } + + @Test + public void testEncodeDecodeZero() { + byte[] buf = new byte[10]; + int len = QwpVarint.encode(buf, 0, 0); + Assert.assertEquals(1, len); + Assert.assertEquals(0x00, buf[0] & 0xFF); + Assert.assertEquals(0, QwpVarint.decode(buf, 0, len)); + } + + @Test + public void testEncodeLargeValues() { + byte[] buf = new byte[10]; + + // Test various powers of 2 + long[] values = { + 1L << 20, // ~1M + 1L << 30, // ~1B + 1L << 40, // ~1T + 1L << 50, + 1L << 60, + Long.MAX_VALUE + }; + + for (long value : values) { + int len = QwpVarint.encode(buf, 0, value); + Assert.assertTrue(len > 0 && len <= 10); + Assert.assertEquals(value, QwpVarint.decode(buf, 0, len)); + } + } + + @Test + public void testEncodeSpecificValues() { + // Test values from the spec + byte[] buf = new byte[10]; + + // 300 = 0b100101100 + // Should encode as: 0xAC (0b10101100 = 44 + 128), 0x02 (0b00000010) + int len = QwpVarint.encode(buf, 0, 300); + Assert.assertEquals(2, len); + Assert.assertEquals(0xAC, buf[0] & 0xFF); + Assert.assertEquals(0x02, buf[1] & 0xFF); + + // Verify decode + Assert.assertEquals(300, QwpVarint.decode(buf, 0, len)); + } + + @Test + public void testEncodeToDirectMemory() { + long addr = Unsafe.malloc(16, MemoryTag.NATIVE_ILP_RSS); + try { + long endAddr = QwpVarint.encode(addr, 12345); + int len = (int) (endAddr - addr); + Assert.assertTrue(len > 0); + + // Read back and verify + long decoded = QwpVarint.decode(addr, endAddr); + Assert.assertEquals(12345, decoded); + } finally { + Unsafe.free(addr, 16, MemoryTag.NATIVE_ILP_RSS); + } + } + + @Test + public void testEncodedLength() { + Assert.assertEquals(1, QwpVarint.encodedLength(0)); + Assert.assertEquals(1, QwpVarint.encodedLength(1)); + Assert.assertEquals(1, QwpVarint.encodedLength(127)); + Assert.assertEquals(2, QwpVarint.encodedLength(128)); + Assert.assertEquals(2, QwpVarint.encodedLength(16383)); + Assert.assertEquals(3, QwpVarint.encodedLength(16384)); + // Long.MAX_VALUE = 0x7FFFFFFFFFFFFFFF (63 bits) needs ceil(63/7) = 9 bytes + Assert.assertEquals(9, QwpVarint.encodedLength(Long.MAX_VALUE)); + // Test that actual encoding matches + byte[] buf = new byte[10]; + int actualLen = QwpVarint.encode(buf, 0, Long.MAX_VALUE); + Assert.assertEquals(actualLen, QwpVarint.encodedLength(Long.MAX_VALUE)); + } + + @Test + public void testRoundTripRandomValues() { + byte[] buf = new byte[10]; + Random random = new Random(42); // Fixed seed for reproducibility + + for (int i = 0; i < 1000; i++) { + long value = random.nextLong() & Long.MAX_VALUE; // Only positive values + int len = QwpVarint.encode(buf, 0, value); + long decoded = QwpVarint.decode(buf, 0, len); + Assert.assertEquals("Failed for value: " + value, value, decoded); + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpZigZagTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpZigZagTest.java new file mode 100644 index 0000000..6a7ed3b --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpZigZagTest.java @@ -0,0 +1,166 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.cutlass.qwp.protocol.QwpZigZag; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Random; + +public class QwpZigZagTest { + + @Test + public void testEncodeDecodeInt() { + // Test 32-bit version + Assert.assertEquals(0, QwpZigZag.encode(0)); + Assert.assertEquals(0, QwpZigZag.decode(0)); + + Assert.assertEquals(2, QwpZigZag.encode(1)); + Assert.assertEquals(1, QwpZigZag.decode(2)); + + Assert.assertEquals(1, QwpZigZag.encode(-1)); + Assert.assertEquals(-1, QwpZigZag.decode(1)); + + int minInt = Integer.MIN_VALUE; + int encoded = QwpZigZag.encode(minInt); + Assert.assertEquals(minInt, QwpZigZag.decode(encoded)); + + int maxInt = Integer.MAX_VALUE; + encoded = QwpZigZag.encode(maxInt); + Assert.assertEquals(maxInt, QwpZigZag.decode(encoded)); + } + + @Test + public void testEncodeDecodeZero() { + Assert.assertEquals(0, QwpZigZag.encode(0L)); + Assert.assertEquals(0, QwpZigZag.decode(0L)); + } + + @Test + public void testEncodeMaxLong() { + long encoded = QwpZigZag.encode(Long.MAX_VALUE); + Assert.assertEquals(-2L, encoded); // 0xFFFFFFFFFFFFFFFE (all bits except LSB) + Assert.assertEquals(Long.MAX_VALUE, QwpZigZag.decode(encoded)); + } + + @Test + public void testEncodeMinLong() { + long encoded = QwpZigZag.encode(Long.MIN_VALUE); + Assert.assertEquals(-1L, encoded); // All bits set (unsigned max) + Assert.assertEquals(Long.MIN_VALUE, QwpZigZag.decode(encoded)); + } + + @Test + public void testEncodeNegative() { + // ZigZag encoding maps: + // -1 -> 1 + // -2 -> 3 + // -n -> 2n - 1 + Assert.assertEquals(1, QwpZigZag.encode(-1L)); + Assert.assertEquals(3, QwpZigZag.encode(-2L)); + Assert.assertEquals(5, QwpZigZag.encode(-3L)); + Assert.assertEquals(199, QwpZigZag.encode(-100L)); + } + + @Test + public void testEncodePositive() { + // ZigZag encoding maps: + // 0 -> 0 + // 1 -> 2 + // 2 -> 4 + // n -> 2n + Assert.assertEquals(2, QwpZigZag.encode(1L)); + Assert.assertEquals(4, QwpZigZag.encode(2L)); + Assert.assertEquals(6, QwpZigZag.encode(3L)); + Assert.assertEquals(200, QwpZigZag.encode(100L)); + } + + @Test + public void testEncodingPattern() { + // Verify the exact encoding pattern matches the formula: + // zigzag(n) = (n << 1) ^ (n >> 63) + // This means: + // - Non-negative n: zigzag(n) = 2 * n + // - Negative n: zigzag(n) = -2 * n - 1 + + for (int n = -100; n <= 100; n++) { + long encoded = QwpZigZag.encode((long) n); + long expected = (n >= 0) ? (2L * n) : (-2L * n - 1); + Assert.assertEquals("Encoding mismatch for n=" + n, expected, encoded); + } + } + + @Test + public void testRoundTripRandomValues() { + Random random = new Random(42); // Fixed seed for reproducibility + + for (int i = 0; i < 1000; i++) { + long value = random.nextLong(); + long encoded = QwpZigZag.encode(value); + long decoded = QwpZigZag.decode(encoded); + Assert.assertEquals("Failed for value: " + value, value, decoded); + } + } + + @Test + public void testSmallValuesHaveSmallEncodings() { + // The point of ZigZag is that small absolute values produce small encoded values + // which then encode efficiently as varints + + // -1 encodes to 1 (small, 1 byte as varint) + Assert.assertTrue(QwpZigZag.encode(-1L) < 128); + + // Small positive and negative values should encode to small values + // Values in [-63, 63] all encode to values < 128 (1 byte varint) + // 63 encodes to 126, -63 encodes to 125 + for (int n = -63; n <= 63; n++) { + long encoded = QwpZigZag.encode(n); + Assert.assertTrue("Value " + n + " encoded to " + encoded, + encoded < 128); // Fits in 1 byte varint + } + + // 64 encodes to 128, which requires 2 bytes as varint + Assert.assertEquals(128, QwpZigZag.encode(64L)); + } + + @Test + public void testSymmetry() { + // Test that encode then decode returns the original value + long[] testValues = { + 0, 1, -1, 2, -2, + 100, -100, + 1000000, -1000000, + Long.MAX_VALUE, Long.MIN_VALUE, + Long.MAX_VALUE / 2, Long.MIN_VALUE / 2 + }; + + for (long value : testValues) { + long encoded = QwpZigZag.encode(value); + long decoded = QwpZigZag.decode(encoded); + Assert.assertEquals("Failed for value: " + value, value, decoded); + } + } +} From 2ee3ac0ba4adbc186d163ce6f302e83616f11a2a Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 26 Feb 2026 16:15:33 +0100 Subject: [PATCH 79/89] Use VarHandle for InFlightWindow statistics totalAcked and totalFailed used volatile += (read-modify-write) without synchronization, which is racy when the acker thread updates concurrently with readers. Replace with VarHandle getAndAdd() for atomic updates and getOpaque() for tear-free reads, avoiding the overhead of AtomicLong while keeping the fields in-line with the rest of the lock-free design. Co-Authored-By: Claude Opus 4.6 --- .../cutlass/qwp/client/InFlightWindow.java | 30 ++++++++++++++----- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/InFlightWindow.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/InFlightWindow.java index 869d242..6447cb8 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/InFlightWindow.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/InFlightWindow.java @@ -28,6 +28,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.VarHandle; import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.LockSupport; @@ -63,6 +65,18 @@ public class InFlightWindow { private static final long PARK_NANOS = 100_000; // 100 microseconds // Spin parameters private static final int SPIN_TRIES = 100; + private static final VarHandle TOTAL_ACKED; + private static final VarHandle TOTAL_FAILED; + + static { + try { + MethodHandles.Lookup lookup = MethodHandles.lookup(); + TOTAL_ACKED = lookup.findVarHandle(InFlightWindow.class, "totalAcked", long.class); + TOTAL_FAILED = lookup.findVarHandle(InFlightWindow.class, "totalFailed", long.class); + } catch (ReflectiveOperationException e) { + throw new ExceptionInInitializerError(e); + } + } // Error state private final AtomicReference lastError = new AtomicReference<>(); private final int maxWindowSize; @@ -73,9 +87,9 @@ public class InFlightWindow { // Core state // highestSent: the sequence number of the last batch added to the window private volatile long highestSent = -1; - // Statistics (not strictly accurate under contention, but good enough for monitoring) - private volatile long totalAcked = 0; - private volatile long totalFailed = 0; + // Statistics — updated atomically via VarHandle + private long totalAcked = 0; + private long totalFailed = 0; // Thread waiting for empty (flush thread) private volatile Thread waitingForEmpty; // Thread waiting for space (sender thread) @@ -145,7 +159,7 @@ public int acknowledgeUpTo(long sequence) { highestAcked = effectiveSequence; int acknowledged = (int) (effectiveSequence - prevAcked); - totalAcked += acknowledged; + TOTAL_ACKED.getAndAdd(this, (long) acknowledged); LOG.debug("Cumulative ACK [upTo={}, acknowledged={}, remaining={}]", sequence, acknowledged, getInFlightCount()); @@ -298,7 +312,7 @@ public void clearError() { public void fail(long batchId, Throwable error) { this.failedBatchId = batchId; this.lastError.set(error); - totalFailed++; + TOTAL_FAILED.getAndAdd(this, 1L); LOG.error("Batch failed [batchId={}, error={}]", batchId, String.valueOf(error)); @@ -320,7 +334,7 @@ public void failAll(Throwable error) { this.failedBatchId = sent; this.lastError.set(error); - totalFailed += Math.max(1, inFlight); + TOTAL_FAILED.getAndAdd(this, Math.max(1L, inFlight)); LOG.error("All in-flight batches failed [inFlight={}, error={}]", inFlight, String.valueOf(error)); @@ -356,14 +370,14 @@ public int getMaxWindowSize() { * Returns the total number of batches acknowledged. */ public long getTotalAcked() { - return totalAcked; + return (long) TOTAL_ACKED.getOpaque(this); } /** * Returns the total number of batches that failed. */ public long getTotalFailed() { - return totalFailed; + return (long) TOTAL_FAILED.getOpaque(this); } /** From 8a6cbe8db39195b2a8c1727a682b12244b6dbb7a Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 26 Feb 2026 16:26:32 +0100 Subject: [PATCH 80/89] Replace busy-wait with monitor wait in close() WebSocketSendQueue.close() busy-waited with Thread.sleep(10) outside the processingLock monitor to drain pending batches. This burned CPU unnecessarily and read pendingBuffer without holding the lock, risking a missed-update race. Replace the sleep loop with processingLock.wait(), matching the pattern already used by flush() and enqueue(). The I/O thread already calls processingLock.notifyAll() after polling a batch, so close() now wakes up promptly on notification or at the exact shutdown deadline. Co-Authored-By: Claude Opus 4.6 --- .../qwp/client/WebSocketSendQueue.java | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketSendQueue.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketSendQueue.java index f567d38..7b43fd9 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketSendQueue.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketSendQueue.java @@ -179,16 +179,19 @@ public void close() { // Wait for pending batches to be sent long startTime = System.currentTimeMillis(); - while (!isPendingEmpty()) { - if (System.currentTimeMillis() - startTime > shutdownTimeoutMs) { - LOG.error("Shutdown timeout, {} batches not sent", getPendingSize()); - break; - } - try { - Thread.sleep(10); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - break; + synchronized (processingLock) { + while (!isPendingEmpty()) { + long elapsed = System.currentTimeMillis() - startTime; + if (elapsed >= shutdownTimeoutMs) { + LOG.error("Shutdown timeout, {} batches not sent", getPendingSize()); + break; + } + try { + processingLock.wait(shutdownTimeoutMs - elapsed); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } } } From e1538067ac28b76fed01dd6596eb84e28f95f550 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 26 Feb 2026 16:32:57 +0100 Subject: [PATCH 81/89] Validate Upgrade and Connection headers in WS handshake validateUpgradeResponse() previously only checked the HTTP 101 status line and Sec-WebSocket-Accept header. RFC 6455 Section 4.1 also requires the client to verify that the server response contains "Upgrade: websocket" and "Connection: Upgrade" headers. Add both checks with case-insensitive value matching as the RFC requires. The existing containsHeaderValue() helper gains an ignoreValueCase parameter so the Upgrade and Connection checks use equalsIgnoreCase while the Sec-WebSocket-Accept check retains its exact base64 match. Co-Authored-By: Claude Opus 4.6 --- .../cutlass/http/client/WebSocketClient.java | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java index 9c6ca38..e015a04 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java @@ -449,7 +449,7 @@ public void upgrade(CharSequence path) { upgrade(path, defaultTimeout); } - private static boolean containsHeaderValue(String response, String headerName, String expectedValue) { + private static boolean containsHeaderValue(String response, String headerName, String expectedValue, boolean ignoreValueCase) { int headerLen = headerName.length(); int responseLen = response.length(); for (int i = 0; i <= responseLen - headerLen; i++) { @@ -460,7 +460,9 @@ private static boolean containsHeaderValue(String response, String headerName, S lineEnd = responseLen; } String actualValue = response.substring(valueStart, lineEnd).trim(); - return actualValue.equals(expectedValue); + return ignoreValueCase + ? actualValue.equalsIgnoreCase(expectedValue) + : actualValue.equals(expectedValue); } } return false; @@ -842,9 +844,19 @@ private void validateUpgradeResponse(int headerEnd) { throw new HttpClientException("WebSocket upgrade failed: ").put(statusLine); } - // Verify Sec-WebSocket-Accept (case-insensitive per RFC 7230) + // Verify Upgrade: websocket (case-insensitive value per RFC 6455 Section 4.1) + if (!containsHeaderValue(response, "Upgrade:", "websocket", true)) { + throw new HttpClientException("Missing or invalid Upgrade header in WebSocket response"); + } + + // Verify Connection: Upgrade (case-insensitive value per RFC 6455 Section 4.1) + if (!containsHeaderValue(response, "Connection:", "Upgrade", true)) { + throw new HttpClientException("Missing or invalid Connection header in WebSocket response"); + } + + // Verify Sec-WebSocket-Accept (exact value match per RFC 6455 Section 4.1) String expectedAccept = WebSocketHandshake.computeAcceptKey(handshakeKey); - if (!containsHeaderValue(response, "Sec-WebSocket-Accept:", expectedAccept)) { + if (!containsHeaderValue(response, "Sec-WebSocket-Accept:", expectedAccept, false)) { throw new HttpClientException("Invalid Sec-WebSocket-Accept header"); } } From 859c7cb78b7eda9f57cc2395c2bf40c93530b0f7 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 26 Feb 2026 16:38:55 +0100 Subject: [PATCH 82/89] Use SecureRnd for WebSocket handshake key Replace non-cryptographic Rnd (xorshift seeded with nanoTime/currentTimeMillis) with ChaCha20-based SecureRnd for generating the Sec-WebSocket-Key during the upgrade handshake. WebSocketSendBuffer already uses SecureRnd for frame masking, so this aligns the handshake key generation with the same standard. SecureRnd seeds once from SecureRandom at construction time, then produces unpredictable output with no heap allocations. The handshake key needs only 16 nextInt() calls (one ChaCha20 block) and runs once per connection, so the performance impact is negligible. Co-Authored-By: Claude Opus 4.6 --- .../client/cutlass/http/client/WebSocketClient.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java index e015a04..78a3245 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java @@ -37,7 +37,7 @@ import io.questdb.client.std.MemoryTag; import io.questdb.client.std.Misc; import io.questdb.client.std.QuietCloseable; -import io.questdb.client.std.Rnd; +import io.questdb.client.std.SecureRnd; import io.questdb.client.std.Unsafe; import io.questdb.client.std.Vect; import org.slf4j.Logger; @@ -76,7 +76,7 @@ public abstract class WebSocketClient implements QuietCloseable { private final int defaultTimeout; private final WebSocketFrameParser frameParser; private final int maxRecvBufSize; - private final Rnd rnd; + private final SecureRnd rnd; private final WebSocketSendBuffer sendBuffer; private boolean closed; private int fragmentBufPos; @@ -116,7 +116,7 @@ public WebSocketClient(HttpClientConfiguration configuration, SocketFactory sock this.recvReadPos = 0; this.frameParser = new WebSocketFrameParser(); - this.rnd = new Rnd(System.nanoTime(), System.currentTimeMillis()); + this.rnd = new SecureRnd(); this.upgraded = false; this.closed = false; } @@ -405,7 +405,7 @@ public void upgrade(CharSequence path, int timeout) { // Generate random key byte[] keyBytes = new byte[16]; for (int i = 0; i < 16; i++) { - keyBytes[i] = (byte) rnd.nextInt(256); + keyBytes[i] = (byte) rnd.nextInt(); } handshakeKey = Base64.getEncoder().encodeToString(keyBytes); From 9e91498826e310d5a2bf01921f2b57e505ad34be Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 26 Feb 2026 16:47:50 +0100 Subject: [PATCH 83/89] Clean up client QwpConstants Raise javac.target from 11 to 17 in the client module's pom.xml, matching the core module's language level. This enables enhanced switch expressions. Use enhanced switch expressions in getFixedTypeSize() and getTypeName(), matching the core module's style. Remove nine unused constants: CAPABILITY_REQUEST_SIZE, CAPABILITY_RESPONSE_SIZE, DEFAULT_MAX_STRING_LENGTH, HEADER_OFFSET_MAGIC, HEADER_OFFSET_PAYLOAD_LENGTH, HEADER_OFFSET_TABLE_COUNT, HEADER_OFFSET_VERSION, MAX_COLUMN_NAME_LENGTH, MAX_TABLE_NAME_LENGTH. Co-Authored-By: Claude Opus 4.6 --- core/pom.xml | 2 +- .../cutlass/qwp/protocol/QwpConstants.java | 173 ++++-------------- 2 files changed, 37 insertions(+), 138 deletions(-) diff --git a/core/pom.xml b/core/pom.xml index 1ccaa98..a6dfa9f 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -32,7 +32,7 @@ false target none - 11 + 17 -ea -Dfile.encoding=UTF-8 -XX:+UseParallelGC None %regex[.*[^o].class] diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java index a6142c9..a3ad097 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java @@ -29,14 +29,6 @@ */ public final class QwpConstants { - /** - * Size of capability request in bytes. - */ - public static final int CAPABILITY_REQUEST_SIZE = 8; - /** - * Size of capability response in bytes. - */ - public static final int CAPABILITY_RESPONSE_SIZE = 8; /** * Default initial receive buffer size (64 KB). */ @@ -54,10 +46,6 @@ public final class QwpConstants { * Default maximum rows per table in a batch. */ public static final int DEFAULT_MAX_ROWS_PER_TABLE = 1_000_000; - /** - * Default maximum string length in bytes (1 MB). - */ - public static final int DEFAULT_MAX_STRING_LENGTH = 1024 * 1024; /** * Default maximum tables per batch. */ @@ -89,23 +77,6 @@ public final class QwpConstants { * Offset of flags byte in header. */ public static final int HEADER_OFFSET_FLAGS = 5; - /** - * Offset of magic bytes in header (4 bytes). - */ - public static final int HEADER_OFFSET_MAGIC = 0; - /** - * Offset of payload length (uint32, little-endian) in header. - */ - public static final int HEADER_OFFSET_PAYLOAD_LENGTH = 8; - - /** - * Offset of table count (uint16, little-endian) in header. - */ - public static final int HEADER_OFFSET_TABLE_COUNT = 6; - /** - * Offset of version byte in header. - */ - public static final int HEADER_OFFSET_VERSION = 4; /** * Size of the message header in bytes. */ @@ -130,14 +101,6 @@ public final class QwpConstants { * Maximum columns per table (QuestDB limit). */ public static final int MAX_COLUMNS_PER_TABLE = 2048; - /** - * Maximum column name length in bytes. - */ - public static final int MAX_COLUMN_NAME_LENGTH = 127; - /** - * Maximum table name length in bytes. - */ - public static final int MAX_TABLE_NAME_LENGTH = 127; /** * Schema mode: Full schema included. */ @@ -307,35 +270,17 @@ private QwpConstants() { */ public static int getFixedTypeSize(byte typeCode) { int code = typeCode & TYPE_MASK; - switch (code) { - case TYPE_BOOLEAN: - return 0; // Special: bit-packed - case TYPE_BYTE: - return 1; - case TYPE_SHORT: - case TYPE_CHAR: - return 2; - case TYPE_INT: - case TYPE_FLOAT: - return 4; - case TYPE_LONG: - case TYPE_DOUBLE: - case TYPE_TIMESTAMP: - case TYPE_TIMESTAMP_NANOS: - case TYPE_DATE: - case TYPE_DECIMAL64: - return 8; - case TYPE_UUID: - case TYPE_DECIMAL128: - return 16; - case TYPE_LONG256: - case TYPE_DECIMAL256: - return 32; - case TYPE_GEOHASH: - return -1; // Variable width: varint precision + packed values - default: - return -1; // Variable width - } + return switch (code) { + case TYPE_BOOLEAN -> 0; // Special: bit-packed + case TYPE_BYTE -> 1; + case TYPE_SHORT, TYPE_CHAR -> 2; + case TYPE_INT, TYPE_FLOAT -> 4; + case TYPE_LONG, TYPE_DOUBLE, TYPE_TIMESTAMP, TYPE_TIMESTAMP_NANOS, TYPE_DATE, TYPE_DECIMAL64 -> 8; + case TYPE_UUID, TYPE_DECIMAL128 -> 16; + case TYPE_LONG256, TYPE_DECIMAL256 -> 32; + case TYPE_GEOHASH -> -1; // Variable width: varint precision + packed values + default -> -1; // Variable width + }; } /** @@ -347,77 +292,31 @@ public static int getFixedTypeSize(byte typeCode) { public static String getTypeName(byte typeCode) { int code = typeCode & TYPE_MASK; boolean nullable = (typeCode & TYPE_NULLABLE_FLAG) != 0; - String name; - switch (code) { - case TYPE_BOOLEAN: - name = "BOOLEAN"; - break; - case TYPE_BYTE: - name = "BYTE"; - break; - case TYPE_SHORT: - name = "SHORT"; - break; - case TYPE_CHAR: - name = "CHAR"; - break; - case TYPE_INT: - name = "INT"; - break; - case TYPE_LONG: - name = "LONG"; - break; - case TYPE_FLOAT: - name = "FLOAT"; - break; - case TYPE_DOUBLE: - name = "DOUBLE"; - break; - case TYPE_STRING: - name = "STRING"; - break; - case TYPE_SYMBOL: - name = "SYMBOL"; - break; - case TYPE_TIMESTAMP: - name = "TIMESTAMP"; - break; - case TYPE_TIMESTAMP_NANOS: - name = "TIMESTAMP_NANOS"; - break; - case TYPE_DATE: - name = "DATE"; - break; - case TYPE_UUID: - name = "UUID"; - break; - case TYPE_LONG256: - name = "LONG256"; - break; - case TYPE_GEOHASH: - name = "GEOHASH"; - break; - case TYPE_VARCHAR: - name = "VARCHAR"; - break; - case TYPE_DOUBLE_ARRAY: - name = "DOUBLE_ARRAY"; - break; - case TYPE_LONG_ARRAY: - name = "LONG_ARRAY"; - break; - case TYPE_DECIMAL64: - name = "DECIMAL64"; - break; - case TYPE_DECIMAL128: - name = "DECIMAL128"; - break; - case TYPE_DECIMAL256: - name = "DECIMAL256"; - break; - default: - name = "UNKNOWN(" + code + ")"; - } + String name = switch (code) { + case TYPE_BOOLEAN -> "BOOLEAN"; + case TYPE_BYTE -> "BYTE"; + case TYPE_SHORT -> "SHORT"; + case TYPE_CHAR -> "CHAR"; + case TYPE_INT -> "INT"; + case TYPE_LONG -> "LONG"; + case TYPE_FLOAT -> "FLOAT"; + case TYPE_DOUBLE -> "DOUBLE"; + case TYPE_STRING -> "STRING"; + case TYPE_SYMBOL -> "SYMBOL"; + case TYPE_TIMESTAMP -> "TIMESTAMP"; + case TYPE_TIMESTAMP_NANOS -> "TIMESTAMP_NANOS"; + case TYPE_DATE -> "DATE"; + case TYPE_UUID -> "UUID"; + case TYPE_LONG256 -> "LONG256"; + case TYPE_GEOHASH -> "GEOHASH"; + case TYPE_VARCHAR -> "VARCHAR"; + case TYPE_DOUBLE_ARRAY -> "DOUBLE_ARRAY"; + case TYPE_LONG_ARRAY -> "LONG_ARRAY"; + case TYPE_DECIMAL64 -> "DECIMAL64"; + case TYPE_DECIMAL128 -> "DECIMAL128"; + case TYPE_DECIMAL256 -> "DECIMAL256"; + default -> "UNKNOWN(" + code + ")"; + }; return nullable ? name + "?" : name; } From e6ef8e85df96ae902bb9c12a1d5860d9bc7124cf Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 26 Feb 2026 17:15:21 +0100 Subject: [PATCH 84/89] Remove serverMode from client WebSocketFrameParser The client-side parser always operates in client mode (rejects masked frames, accepts unmasked). Remove the serverMode flag and setServerMode()/setStrictMode()/setMaskKey()/getMaskKey() methods that were only needed by the server-side copy. Add WebSocketFrameParserTest with 34 tests covering client-mode frame parsing: opcodes, length encodings, fragmentation, control frames, error cases, and masked-frame rejection. Co-Authored-By: Claude Opus 4.6 --- .../qwp/websocket/WebSocketFrameParser.java | 36 +- .../websocket/WebSocketFrameParserTest.java | 668 ++++++++++++++++++ 2 files changed, 671 insertions(+), 33 deletions(-) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/WebSocketFrameParserTest.java diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameParser.java b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameParser.java index e9d2d54..85603d1 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameParser.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameParser.java @@ -73,8 +73,6 @@ public class WebSocketFrameParser { private boolean masked; private int opcode; private long payloadLength; - // Configuration - private boolean serverMode = false; // If true, expect masked frames from clients // Parser state private int state = STATE_HEADER; private boolean strictMode = false; // If true, reject non-minimal length encodings @@ -87,12 +85,6 @@ public int getHeaderSize() { return headerSize; } - public int getMaskKey() { - return maskKey; - } - - // Getters - public int getOpcode() { return opcode; } @@ -160,13 +152,9 @@ public int parse(long buf, long limit) { int lengthField = byte1 & LENGTH_MASK; // Validate masking based on mode - if (serverMode && !masked) { - // Client frames MUST be masked - state = STATE_ERROR; - errorCode = WebSocketCloseCode.PROTOCOL_ERROR; - return 0; - } - if (!serverMode && masked) { + // Configuration + // If true, expect masked frames from clients + if (masked) { // Server frames MUST NOT be masked state = STATE_ERROR; errorCode = WebSocketCloseCode.PROTOCOL_ERROR; @@ -274,24 +262,6 @@ public void reset() { errorCode = 0; } - /** - * Sets the mask key for unmasking. Used in testing. - */ - public void setMaskKey(int maskKey) { - this.maskKey = maskKey; - this.masked = true; - } - - // Setters for configuration - - public void setServerMode(boolean serverMode) { - this.serverMode = serverMode; - } - - public void setStrictMode(boolean strictMode) { - this.strictMode = strictMode; - } - /** * Unmasks the payload data in place. * diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/WebSocketFrameParserTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/WebSocketFrameParserTest.java new file mode 100644 index 0000000..4bf7eb5 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/WebSocketFrameParserTest.java @@ -0,0 +1,668 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.websocket; + +import io.questdb.client.cutlass.qwp.websocket.WebSocketCloseCode; +import io.questdb.client.cutlass.qwp.websocket.WebSocketFrameParser; +import io.questdb.client.cutlass.qwp.websocket.WebSocketOpcode; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; +import org.junit.Test; + +import java.nio.charset.StandardCharsets; + +/** + * Tests for the client-side WebSocket frame parser. + * The client parser expects unmasked frames (from the server) + * and rejects masked frames. + */ +public class WebSocketFrameParserTest { + + @Test + public void testControlFrameBetweenFragments() { + long buf = allocateBuffer(64); + try { + WebSocketFrameParser parser = new WebSocketFrameParser(); + + // First data fragment + writeBytes(buf, (byte) 0x01, (byte) 0x02, (byte) 'H', (byte) 'i'); + parser.parse(buf, buf + 4); + Assert.assertFalse(parser.isFin()); + Assert.assertEquals(WebSocketOpcode.TEXT, parser.getOpcode()); + + // Ping in the middle (control frame, FIN must be 1) + parser.reset(); + writeBytes(buf, (byte) 0x89, (byte) 0x00); + parser.parse(buf, buf + 2); + Assert.assertTrue(parser.isFin()); + Assert.assertEquals(WebSocketOpcode.PING, parser.getOpcode()); + + // Final data fragment + parser.reset(); + writeBytes(buf, (byte) 0x80, (byte) 0x01, (byte) '!'); + parser.parse(buf, buf + 3); + Assert.assertTrue(parser.isFin()); + Assert.assertEquals(WebSocketOpcode.CONTINUATION, parser.getOpcode()); + } finally { + freeBuffer(buf, 64); + } + } + + @Test + public void testOpcodeIsControlFrame() { + Assert.assertFalse(WebSocketOpcode.isControlFrame(WebSocketOpcode.CONTINUATION)); + Assert.assertFalse(WebSocketOpcode.isControlFrame(WebSocketOpcode.TEXT)); + Assert.assertFalse(WebSocketOpcode.isControlFrame(WebSocketOpcode.BINARY)); + Assert.assertTrue(WebSocketOpcode.isControlFrame(WebSocketOpcode.CLOSE)); + Assert.assertTrue(WebSocketOpcode.isControlFrame(WebSocketOpcode.PING)); + Assert.assertTrue(WebSocketOpcode.isControlFrame(WebSocketOpcode.PONG)); + } + + @Test + public void testOpcodeIsDataFrame() { + Assert.assertTrue(WebSocketOpcode.isDataFrame(WebSocketOpcode.CONTINUATION)); + Assert.assertTrue(WebSocketOpcode.isDataFrame(WebSocketOpcode.TEXT)); + Assert.assertTrue(WebSocketOpcode.isDataFrame(WebSocketOpcode.BINARY)); + Assert.assertFalse(WebSocketOpcode.isDataFrame(WebSocketOpcode.CLOSE)); + Assert.assertFalse(WebSocketOpcode.isDataFrame(WebSocketOpcode.PING)); + Assert.assertFalse(WebSocketOpcode.isDataFrame(WebSocketOpcode.PONG)); + } + + @Test + public void testOpcodeIsValid() { + Assert.assertTrue(WebSocketOpcode.isValid(WebSocketOpcode.CONTINUATION)); + Assert.assertTrue(WebSocketOpcode.isValid(WebSocketOpcode.TEXT)); + Assert.assertTrue(WebSocketOpcode.isValid(WebSocketOpcode.BINARY)); + Assert.assertFalse(WebSocketOpcode.isValid(3)); + Assert.assertFalse(WebSocketOpcode.isValid(4)); + Assert.assertFalse(WebSocketOpcode.isValid(5)); + Assert.assertFalse(WebSocketOpcode.isValid(6)); + Assert.assertFalse(WebSocketOpcode.isValid(7)); + Assert.assertTrue(WebSocketOpcode.isValid(WebSocketOpcode.CLOSE)); + Assert.assertTrue(WebSocketOpcode.isValid(WebSocketOpcode.PING)); + Assert.assertTrue(WebSocketOpcode.isValid(WebSocketOpcode.PONG)); + Assert.assertFalse(WebSocketOpcode.isValid(0xB)); + Assert.assertFalse(WebSocketOpcode.isValid(0xF)); + } + + @Test + public void testParse16BitLength() { + int payloadLen = 1000; + long buf = allocateBuffer(payloadLen + 16); + try { + writeBytes(buf, + (byte) 0x82, // FIN + BINARY + (byte) 126, // 16-bit length follows + (byte) (payloadLen >> 8), // Length high byte + (byte) (payloadLen & 0xFF) // Length low byte + ); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + int consumed = parser.parse(buf, buf + 4 + payloadLen); + + Assert.assertEquals(4 + payloadLen, consumed); + Assert.assertEquals(payloadLen, parser.getPayloadLength()); + } finally { + freeBuffer(buf, payloadLen + 16); + } + } + + @Test + public void testParse64BitLength() { + long payloadLen = 70_000L; + long buf = allocateBuffer((int) payloadLen + 16); + try { + Unsafe.getUnsafe().putByte(buf, (byte) 0x82); + Unsafe.getUnsafe().putByte(buf + 1, (byte) 127); + Unsafe.getUnsafe().putLong(buf + 2, Long.reverseBytes(payloadLen)); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + int consumed = parser.parse(buf, buf + 10 + payloadLen); + + Assert.assertEquals(10 + payloadLen, consumed); + Assert.assertEquals(payloadLen, parser.getPayloadLength()); + } finally { + freeBuffer(buf, (int) payloadLen + 16); + } + } + + @Test + public void testParse7BitLength() { + for (int len = 0; len <= 125; len++) { + long buf = allocateBuffer(256); + try { + writeBytes(buf, (byte) 0x82, (byte) len); + for (int i = 0; i < len; i++) { + Unsafe.getUnsafe().putByte(buf + 2 + i, (byte) i); + } + + WebSocketFrameParser parser = new WebSocketFrameParser(); + int consumed = parser.parse(buf, buf + 2 + len); + + Assert.assertEquals(2 + len, consumed); + Assert.assertEquals(len, parser.getPayloadLength()); + } finally { + freeBuffer(buf, 256); + } + } + } + + @Test + public void testParseBinaryFrame() { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x82, (byte) 0x00); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 2); + + Assert.assertEquals(WebSocketOpcode.BINARY, parser.getOpcode()); + } finally { + freeBuffer(buf, 16); + } + } + + @Test + public void testParseCloseFrame() { + long buf = allocateBuffer(16); + try { + writeBytes(buf, + (byte) 0x88, // FIN + CLOSE + (byte) 0x02, // Length 2 (just the code) + (byte) 0x03, (byte) 0xE8 // 1000 in big-endian + ); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 4); + + Assert.assertEquals(WebSocketOpcode.CLOSE, parser.getOpcode()); + Assert.assertEquals(2, parser.getPayloadLength()); + } finally { + freeBuffer(buf, 16); + } + } + + @Test + public void testParseCloseFrameEmpty() { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x88, (byte) 0x00); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 2); + + Assert.assertEquals(WebSocketOpcode.CLOSE, parser.getOpcode()); + Assert.assertEquals(0, parser.getPayloadLength()); + } finally { + freeBuffer(buf, 16); + } + } + + @Test + public void testParseCloseFrameWithReason() { + long buf = allocateBuffer(64); + try { + String reason = "Normal closure"; + byte[] reasonBytes = reason.getBytes(StandardCharsets.UTF_8); + + Unsafe.getUnsafe().putByte(buf, (byte) 0x88); + Unsafe.getUnsafe().putByte(buf + 1, (byte) (2 + reasonBytes.length)); + Unsafe.getUnsafe().putShort(buf + 2, Short.reverseBytes((short) 1000)); + for (int i = 0; i < reasonBytes.length; i++) { + Unsafe.getUnsafe().putByte(buf + 4 + i, reasonBytes[i]); + } + + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 4 + reasonBytes.length); + + Assert.assertEquals(WebSocketOpcode.CLOSE, parser.getOpcode()); + Assert.assertEquals(2 + reasonBytes.length, parser.getPayloadLength()); + } finally { + freeBuffer(buf, 64); + } + } + + @Test + public void testParseContinuationFrame() { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x00, (byte) 0x05, (byte) 0x01, (byte) 0x02, (byte) 0x03, (byte) 0x04, (byte) 0x05); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 7); + + Assert.assertEquals(WebSocketOpcode.CONTINUATION, parser.getOpcode()); + Assert.assertFalse(parser.isFin()); + } finally { + freeBuffer(buf, 16); + } + } + + @Test + public void testParseEmptyBuffer() { + long buf = allocateBuffer(16); + try { + WebSocketFrameParser parser = new WebSocketFrameParser(); + int consumed = parser.parse(buf, buf); + + Assert.assertEquals(0, consumed); + } finally { + freeBuffer(buf, 16); + } + } + + @Test + public void testParseEmptyPayload() { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x82, (byte) 0x00); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + int consumed = parser.parse(buf, buf + 2); + + Assert.assertEquals(2, consumed); + Assert.assertEquals(0, parser.getPayloadLength()); + } finally { + freeBuffer(buf, 16); + } + } + + @Test + public void testParseFragmentedMessage() { + long buf = allocateBuffer(64); + try { + // First fragment: opcode=TEXT, FIN=0 + writeBytes(buf, (byte) 0x01, (byte) 0x03, (byte) 'H', (byte) 'e', (byte) 'l'); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + int consumed = parser.parse(buf, buf + 5); + + Assert.assertEquals(5, consumed); + Assert.assertFalse(parser.isFin()); + Assert.assertEquals(WebSocketOpcode.TEXT, parser.getOpcode()); + + // Continuation: opcode=CONTINUATION, FIN=0 + parser.reset(); + writeBytes(buf, (byte) 0x00, (byte) 0x02, (byte) 'l', (byte) 'o'); + consumed = parser.parse(buf, buf + 4); + + Assert.assertEquals(4, consumed); + Assert.assertFalse(parser.isFin()); + Assert.assertEquals(WebSocketOpcode.CONTINUATION, parser.getOpcode()); + + // Final fragment: opcode=CONTINUATION, FIN=1 + parser.reset(); + writeBytes(buf, (byte) 0x80, (byte) 0x01, (byte) '!'); + consumed = parser.parse(buf, buf + 3); + + Assert.assertEquals(3, consumed); + Assert.assertTrue(parser.isFin()); + Assert.assertEquals(WebSocketOpcode.CONTINUATION, parser.getOpcode()); + } finally { + freeBuffer(buf, 64); + } + } + + @Test + public void testParseIncompleteHeader16BitLength() { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x82, (byte) 126, (byte) 0x01); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + int consumed = parser.parse(buf, buf + 3); + + Assert.assertEquals(0, consumed); + Assert.assertEquals(WebSocketFrameParser.STATE_NEED_MORE, parser.getState()); + } finally { + freeBuffer(buf, 16); + } + } + + @Test + public void testParseIncompleteHeader1Byte() { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x82); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + int consumed = parser.parse(buf, buf + 1); + + Assert.assertEquals(0, consumed); + Assert.assertEquals(WebSocketFrameParser.STATE_NEED_MORE, parser.getState()); + } finally { + freeBuffer(buf, 16); + } + } + + @Test + public void testParseIncompleteHeader64BitLength() { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x82, (byte) 127, (byte) 0, (byte) 0, (byte) 0, (byte) 0); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + int consumed = parser.parse(buf, buf + 6); + + Assert.assertEquals(0, consumed); + } finally { + freeBuffer(buf, 16); + } + } + + @Test + public void testParseIncompletePayload() { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x82, (byte) 0x05, (byte) 0x01, (byte) 0x02); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + int consumed = parser.parse(buf, buf + 4); + + Assert.assertEquals(2, consumed); + Assert.assertEquals(5, parser.getPayloadLength()); + Assert.assertEquals(WebSocketFrameParser.STATE_NEED_PAYLOAD, parser.getState()); + } finally { + freeBuffer(buf, 16); + } + } + + @Test + public void testParseMaxControlFrameSize() { + long buf = allocateBuffer(256); + try { + Unsafe.getUnsafe().putByte(buf, (byte) 0x89); // PING + Unsafe.getUnsafe().putByte(buf + 1, (byte) 125); + for (int i = 0; i < 125; i++) { + Unsafe.getUnsafe().putByte(buf + 2 + i, (byte) i); + } + + WebSocketFrameParser parser = new WebSocketFrameParser(); + int consumed = parser.parse(buf, buf + 127); + + Assert.assertEquals(127, consumed); + Assert.assertEquals(125, parser.getPayloadLength()); + Assert.assertNotEquals(WebSocketFrameParser.STATE_ERROR, parser.getState()); + } finally { + freeBuffer(buf, 256); + } + } + + @Test + public void testParseMinimalFrame() { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x82, (byte) 0x01, (byte) 0xFF); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + int consumed = parser.parse(buf, buf + 3); + + Assert.assertEquals(3, consumed); + Assert.assertTrue(parser.isFin()); + Assert.assertEquals(WebSocketOpcode.BINARY, parser.getOpcode()); + Assert.assertEquals(1, parser.getPayloadLength()); + Assert.assertFalse(parser.isMasked()); + } finally { + freeBuffer(buf, 16); + } + } + + @Test + public void testParseMultipleFramesInBuffer() { + long buf = allocateBuffer(32); + try { + writeBytes(buf, + (byte) 0x82, (byte) 0x02, (byte) 0x01, (byte) 0x02, + (byte) 0x81, (byte) 0x03, (byte) 'a', (byte) 'b', (byte) 'c' + ); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + + int consumed = parser.parse(buf, buf + 9); + Assert.assertEquals(4, consumed); + Assert.assertEquals(WebSocketOpcode.BINARY, parser.getOpcode()); + Assert.assertEquals(2, parser.getPayloadLength()); + + parser.reset(); + consumed = parser.parse(buf + 4, buf + 9); + Assert.assertEquals(5, consumed); + Assert.assertEquals(WebSocketOpcode.TEXT, parser.getOpcode()); + Assert.assertEquals(3, parser.getPayloadLength()); + } finally { + freeBuffer(buf, 32); + } + } + + @Test + public void testParsePingFrame() { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x89, (byte) 0x04, (byte) 0x01, (byte) 0x02, (byte) 0x03, (byte) 0x04); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 6); + + Assert.assertEquals(WebSocketOpcode.PING, parser.getOpcode()); + Assert.assertEquals(4, parser.getPayloadLength()); + } finally { + freeBuffer(buf, 16); + } + } + + @Test + public void testParsePongFrame() { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x8A, (byte) 0x04, (byte) 0x01, (byte) 0x02, (byte) 0x03, (byte) 0x04); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 6); + + Assert.assertEquals(WebSocketOpcode.PONG, parser.getOpcode()); + } finally { + freeBuffer(buf, 16); + } + } + + @Test + public void testParseTextFrame() { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x81, (byte) 0x00); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 2); + + Assert.assertEquals(WebSocketOpcode.TEXT, parser.getOpcode()); + } finally { + freeBuffer(buf, 16); + } + } + + @Test + public void testRejectCloseFrameWith1BytePayload() { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x88, (byte) 0x01, (byte) 0x00); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 3); + + Assert.assertEquals(WebSocketFrameParser.STATE_ERROR, parser.getState()); + Assert.assertEquals(WebSocketCloseCode.PROTOCOL_ERROR, parser.getErrorCode()); + } finally { + freeBuffer(buf, 16); + } + } + + @Test + public void testRejectFragmentedControlFrame() { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x09, (byte) 0x00); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 2); + + Assert.assertEquals(WebSocketFrameParser.STATE_ERROR, parser.getState()); + Assert.assertEquals(WebSocketCloseCode.PROTOCOL_ERROR, parser.getErrorCode()); + } finally { + freeBuffer(buf, 16); + } + } + + @Test + public void testRejectMaskedFrame() { + // Client-side parser rejects masked frames from the server + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x82, (byte) 0x81, (byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0xFF); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 7); + + Assert.assertEquals(WebSocketFrameParser.STATE_ERROR, parser.getState()); + } finally { + freeBuffer(buf, 16); + } + } + + @Test + public void testRejectOversizeControlFrame() { + long buf = allocateBuffer(256); + try { + writeBytes(buf, (byte) 0x89, (byte) 126, (byte) 0x00, (byte) 0x7E); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 4); + + Assert.assertEquals(WebSocketFrameParser.STATE_ERROR, parser.getState()); + Assert.assertEquals(WebSocketCloseCode.PROTOCOL_ERROR, parser.getErrorCode()); + } finally { + freeBuffer(buf, 256); + } + } + + @Test + public void testRejectRSV2Bit() { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0xA2, (byte) 0x00); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 2); + + Assert.assertEquals(WebSocketFrameParser.STATE_ERROR, parser.getState()); + } finally { + freeBuffer(buf, 16); + } + } + + @Test + public void testRejectRSV3Bit() { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x92, (byte) 0x00); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 2); + + Assert.assertEquals(WebSocketFrameParser.STATE_ERROR, parser.getState()); + } finally { + freeBuffer(buf, 16); + } + } + + @Test + public void testRejectReservedBits() { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0xC2, (byte) 0x00); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 2); + + Assert.assertEquals(WebSocketFrameParser.STATE_ERROR, parser.getState()); + Assert.assertEquals(WebSocketCloseCode.PROTOCOL_ERROR, parser.getErrorCode()); + } finally { + freeBuffer(buf, 16); + } + } + + @Test + public void testRejectUnknownOpcode() { + for (int opcode : new int[]{3, 4, 5, 6, 7, 0xB, 0xC, 0xD, 0xE, 0xF}) { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) (0x80 | opcode), (byte) 0x00); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 2); + + Assert.assertEquals("Opcode " + opcode + " should be rejected", + WebSocketFrameParser.STATE_ERROR, parser.getState()); + } finally { + freeBuffer(buf, 16); + } + } + } + + @Test + public void testReset() { + long buf = allocateBuffer(16); + try { + writeBytes(buf, (byte) 0x82, (byte) 0x02, (byte) 0x01, (byte) 0x02); + + WebSocketFrameParser parser = new WebSocketFrameParser(); + parser.parse(buf, buf + 4); + + Assert.assertEquals(WebSocketOpcode.BINARY, parser.getOpcode()); + Assert.assertEquals(2, parser.getPayloadLength()); + + parser.reset(); + + Assert.assertEquals(0, parser.getOpcode()); + Assert.assertEquals(0, parser.getPayloadLength()); + Assert.assertEquals(WebSocketFrameParser.STATE_HEADER, parser.getState()); + } finally { + freeBuffer(buf, 16); + } + } + + private static long allocateBuffer(int size) { + return Unsafe.malloc(size, MemoryTag.NATIVE_DEFAULT); + } + + private static void freeBuffer(long address, int size) { + Unsafe.free(address, size, MemoryTag.NATIVE_DEFAULT); + } + + private static void writeBytes(long address, byte... bytes) { + for (int i = 0; i < bytes.length; i++) { + Unsafe.getUnsafe().putByte(address + i, bytes[i]); + } + } +} From 44f4db0e90b32e1478a4239cfe2607c0026795c9 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 26 Feb 2026 17:16:01 +0100 Subject: [PATCH 85/89] Style cleanup in Sender --- .../main/java/io/questdb/client/Sender.java | 20 +++++-------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index 705f3fe..33d2df2 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -145,20 +145,11 @@ static LineSenderBuilder builder(CharSequence configurationString) { * @return Builder object to create a new Sender instance. */ static LineSenderBuilder builder(Transport transport) { - int protocol; - switch (transport) { - case HTTP: - protocol = LineSenderBuilder.PROTOCOL_HTTP; - break; - case TCP: - protocol = LineSenderBuilder.PROTOCOL_TCP; - break; - case WEBSOCKET: - protocol = LineSenderBuilder.PROTOCOL_WEBSOCKET; - break; - default: - throw new LineSenderException("unknown transport: " + transport); - } + int protocol = switch (transport) { + case HTTP -> LineSenderBuilder.PROTOCOL_HTTP; + case TCP -> LineSenderBuilder.PROTOCOL_TCP; + case WEBSOCKET -> LineSenderBuilder.PROTOCOL_WEBSOCKET; + }; return new LineSenderBuilder(protocol); } @@ -562,7 +553,6 @@ final class LineSenderBuilder { private String httpSettingsPath; private int httpTimeout = PARAMETER_NOT_SET_EXPLICITLY; private String httpToken; - // WebSocket-specific fields private int inFlightWindowSize = PARAMETER_NOT_SET_EXPLICITLY; private String keyId; private int maxBackoffMillis = PARAMETER_NOT_SET_EXPLICITLY; From 6867c50505e1fa4253814ed803cb75d68e307d50 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 26 Feb 2026 17:19:27 +0100 Subject: [PATCH 86/89] Delete unused code --- .../cutlass/qwp/protocol/QwpBitReader.java | 137 ------------------ 1 file changed, 137 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitReader.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitReader.java index c1e2ec7..cb80a58 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitReader.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitReader.java @@ -62,70 +62,6 @@ public class QwpBitReader { public QwpBitReader() { } - /** - * Aligns the reader to the next byte boundary by skipping any partial bits. - * - * @throws IllegalStateException if alignment fails - */ - public void alignToByte() { - int remainder = (int) (totalBitsRead % 8); - if (remainder != 0) { - skipBits(8 - remainder); - } - } - - /** - * Returns the number of bits remaining to be read. - * - * @return available bits - */ - public long getAvailableBits() { - return totalBitsAvailable - totalBitsRead; - } - - /** - * Returns the current position in bits from the start. - * - * @return bits read since reset - */ - public long getBitPosition() { - return totalBitsRead; - } - - /** - * Returns the current byte address being read. - * Note: This is approximate due to bit buffering. - * - * @return current address - */ - public long getCurrentAddress() { - return currentAddress; - } - - /** - * Returns true if there are more bits to read. - * - * @return true if bits available - */ - public boolean hasMoreBits() { - return totalBitsRead < totalBitsAvailable; - } - - /** - * Peeks at the next bit without consuming it. - * - * @return 0 or 1, or -1 if no more bits - */ - public int peekBit() { - if (totalBitsRead >= totalBitsAvailable) { - return -1; - } - if (!ensureBits(1)) { - return -1; - } - return (int) (bitBuffer & 1); - } - /** * Reads a single bit. * @@ -193,16 +129,6 @@ public long readBits(int numBits) { return result; } - /** - * Reads a complete byte, ensuring byte alignment first. - * - * @return the byte value (0-255) - * @throws IllegalStateException if not enough data - */ - public int readByte() { - return (int) readBits(8) & 0xFF; - } - /** * Reads a complete 32-bit integer in little-endian order. * @@ -213,16 +139,6 @@ public int readInt() { return (int) readBits(32); } - /** - * Reads a complete 64-bit long in little-endian order. - * - * @return the long value - * @throws IllegalStateException if not enough data - */ - public long readLong() { - return readBits(64); - } - /** * Reads multiple bits and interprets them as a signed value using two's complement. * @@ -255,59 +171,6 @@ public void reset(long address, long length) { this.totalBitsRead = 0; } - /** - * Resets the reader to read from the specified byte array. - * - * @param buf the byte array - * @param offset the starting offset - * @param length the number of bytes available - */ - public void reset(byte[] buf, int offset, int length) { - // For byte array, we'll store position info differently - // This is mainly for testing - in production we use direct memory - throw new UnsupportedOperationException("Use direct memory version"); - } - - /** - * Skips the specified number of bits. - * - * @param numBits bits to skip - * @throws IllegalStateException if not enough bits available - */ - public void skipBits(int numBits) { - if (totalBitsRead + numBits > totalBitsAvailable) { - throw new IllegalStateException("bit read overflow"); - } - - // Fast path: skip bits in current buffer - if (numBits <= bitsInBuffer) { - bitBuffer >>>= numBits; - bitsInBuffer -= numBits; - totalBitsRead += numBits; - return; - } - - // Consume all buffered bits - int bitsToSkip = numBits - bitsInBuffer; - totalBitsRead += bitsInBuffer; - bitsInBuffer = 0; - bitBuffer = 0; - - // Skip whole bytes - int bytesToSkip = bitsToSkip / 8; - currentAddress += bytesToSkip; - totalBitsRead += bytesToSkip * 8L; - - // Handle remaining bits - int remainingBits = bitsToSkip % 8; - if (remainingBits > 0) { - ensureBits(remainingBits); - bitBuffer >>>= remainingBits; - bitsInBuffer -= remainingBits; - totalBitsRead += remainingBits; - } - } - /** * Ensures the buffer has at least the requested number of bits. * Loads more bytes from memory if needed. From 099c183184462730d5ab274b3fb5adbc45cedefd Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 26 Feb 2026 17:21:50 +0100 Subject: [PATCH 87/89] Clean up WebSocketClientFactory --- .../http/client/WebSocketClientFactory.java | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientFactory.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientFactory.java index 9284786..c6b36d2 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientFactory.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClientFactory.java @@ -63,6 +63,10 @@ */ public class WebSocketClientFactory { + // Utility class -- no instantiation + private WebSocketClientFactory() { + } + /** * Creates a new WebSocket client with insecure TLS (no certificate validation). *

    @@ -82,17 +86,12 @@ public static WebSocketClient newInsecureTlsInstance() { * @return a new platform-specific WebSocket client */ public static WebSocketClient newInstance(HttpClientConfiguration configuration, SocketFactory socketFactory) { - switch (Os.type) { - case Os.LINUX: - return new WebSocketClientLinux(configuration, socketFactory); - case Os.DARWIN: - case Os.FREEBSD: - return new WebSocketClientOsx(configuration, socketFactory); - case Os.WINDOWS: - return new WebSocketClientWindows(configuration, socketFactory); - default: - throw new UnsupportedOperationException("Unsupported platform: " + Os.type); - } + return switch (Os.type) { + case Os.LINUX -> new WebSocketClientLinux(configuration, socketFactory); + case Os.DARWIN, Os.FREEBSD -> new WebSocketClientOsx(configuration, socketFactory); + case Os.WINDOWS -> new WebSocketClientWindows(configuration, socketFactory); + default -> throw new UnsupportedOperationException("Unsupported platform: " + Os.type); + }; } /** From 254c7c3550dc41995572433fe63715a54d82f29a Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 26 Feb 2026 17:25:34 +0100 Subject: [PATCH 88/89] Fix copyright year --- .../cutlass/qwp/protocol/QwpBitReader.java | 2 +- .../cutlass/qwp/protocol/QwpBitWriter.java | 2 +- .../cutlass/qwp/protocol/QwpColumnDef.java | 2 +- .../cutlass/qwp/protocol/QwpConstants.java | 2 +- .../cutlass/qwp/protocol/QwpNullBitmap.java | 2 +- .../cutlass/qwp/protocol/QwpSchemaHash.java | 2 +- .../cutlass/qwp/protocol/QwpVarint.java | 2 +- .../cutlass/qwp/protocol/QwpZigZag.java | 2 +- .../qwp/websocket/WebSocketFrameWriter.java | 131 ------------------ .../line/tcp/v4/QwpAllocationTestClient.java | 2 +- .../line/tcp/v4/StacBenchmarkClient.java | 2 +- 11 files changed, 10 insertions(+), 141 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitReader.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitReader.java index cb80a58..a253f6e 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitReader.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitReader.java @@ -6,7 +6,7 @@ * \__\_\\__,_|\___||___/\__|____/|____/ * * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2024 QuestDB + * Copyright (c) 2019-2026 QuestDB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitWriter.java index 3f18d2d..30173cb 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitWriter.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitWriter.java @@ -6,7 +6,7 @@ * \__\_\\__,_|\___||___/\__|____/|____/ * * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2024 QuestDB + * Copyright (c) 2019-2026 QuestDB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java index f59a009..8c2c65a 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java @@ -6,7 +6,7 @@ * \__\_\\__,_|\___||___/\__|____/|____/ * * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2024 QuestDB + * Copyright (c) 2019-2026 QuestDB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java index a3ad097..8c36d93 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java @@ -6,7 +6,7 @@ * \__\_\\__,_|\___||___/\__|____/|____/ * * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2024 QuestDB + * Copyright (c) 2019-2026 QuestDB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpNullBitmap.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpNullBitmap.java index dd6020e..f78f65e 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpNullBitmap.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpNullBitmap.java @@ -6,7 +6,7 @@ * \__\_\\__,_|\___||___/\__|____/|____/ * * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2024 QuestDB + * Copyright (c) 2019-2026 QuestDB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java index b62fc07..94b3693 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java @@ -6,7 +6,7 @@ * \__\_\\__,_|\___||___/\__|____/|____/ * * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2024 QuestDB + * Copyright (c) 2019-2026 QuestDB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpVarint.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpVarint.java index ad675f4..f02a4ca 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpVarint.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpVarint.java @@ -6,7 +6,7 @@ * \__\_\\__,_|\___||___/\__|____/|____/ * * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2024 QuestDB + * Copyright (c) 2019-2026 QuestDB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpZigZag.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpZigZag.java index f113460..512de7d 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpZigZag.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpZigZag.java @@ -6,7 +6,7 @@ * \__\_\\__,_|\___||___/\__|____/|____/ * * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2024 QuestDB + * Copyright (c) 2019-2026 QuestDB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameWriter.java index 892fed4..2f3c653 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameWriter.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameWriter.java @@ -26,8 +26,6 @@ import io.questdb.client.std.Unsafe; -import java.nio.charset.StandardCharsets; - /** * Zero-allocation WebSocket frame writer. * Writes WebSocket frames according to RFC 6455. @@ -99,78 +97,6 @@ public static void maskPayload(long buf, long len, int maskKey) { } } - /** - * Writes a binary frame with payload from a memory address. - * - * @param buf the buffer to write to - * @param payloadPtr pointer to the payload data - * @param payloadLen length of payload - * @return the total number of bytes written - */ - public static int writeBinaryFrame(long buf, long payloadPtr, int payloadLen) { - int headerLen = writeHeader(buf, true, WebSocketOpcode.BINARY, payloadLen, false); - - // Copy payload from memory - Unsafe.getUnsafe().copyMemory(payloadPtr, buf + headerLen, payloadLen); - - return headerLen + payloadLen; - } - - /** - * Writes a binary frame header only (for when payload is written separately). - * - * @param buf the buffer to write to - * @param payloadLen length of payload that will follow - * @return the header size in bytes - */ - public static int writeBinaryFrameHeader(long buf, int payloadLen) { - return writeHeader(buf, true, WebSocketOpcode.BINARY, payloadLen, false); - } - - /** - * Writes a complete Close frame to the buffer. - * - * @param buf the buffer to write to - * @param code the close status code - * @param reason the close reason (may be null) - * @return the total number of bytes written (header + payload) - */ - public static int writeCloseFrame(long buf, int code, String reason) { - int payloadLen = 2; // status code - if (reason != null && !reason.isEmpty()) { - payloadLen += reason.getBytes(StandardCharsets.UTF_8).length; - } - - int headerLen = writeHeader(buf, true, WebSocketOpcode.CLOSE, payloadLen, false); - int payloadOffset = writeClosePayload(buf + headerLen, code, reason); - - return headerLen + payloadOffset; - } - - /** - * Writes the payload for a Close frame. - * - * @param buf the buffer to write to (after the header) - * @param code the close status code - * @param reason the close reason (may be null) - * @return the number of bytes written - */ - public static int writeClosePayload(long buf, int code, String reason) { - // Write status code in network byte order (big-endian) - Unsafe.getUnsafe().putShort(buf, Short.reverseBytes((short) code)); - int offset = 2; - - // Write reason if provided - if (reason != null && !reason.isEmpty()) { - byte[] reasonBytes = reason.getBytes(StandardCharsets.UTF_8); - for (byte reasonByte : reasonBytes) { - Unsafe.getUnsafe().putByte(buf + offset++, reasonByte); - } - } - - return offset; - } - /** * Writes a WebSocket frame header to the buffer. * @@ -221,61 +147,4 @@ public static int writeHeader(long buf, boolean fin, int opcode, long payloadLen Unsafe.getUnsafe().putInt(buf + offset, maskKey); return offset + 4; } - - /** - * Writes a complete Ping frame to the buffer. - * - * @param buf the buffer to write to - * @param payload the ping payload - * @param payloadOff offset into payload array - * @param payloadLen length of payload to write - * @return the total number of bytes written - */ - public static int writePingFrame(long buf, byte[] payload, int payloadOff, int payloadLen) { - int headerLen = writeHeader(buf, true, WebSocketOpcode.PING, payloadLen, false); - - // Copy payload - for (int i = 0; i < payloadLen; i++) { - Unsafe.getUnsafe().putByte(buf + headerLen + i, payload[payloadOff + i]); - } - - return headerLen + payloadLen; - } - - /** - * Writes a complete Pong frame to the buffer. - * - * @param buf the buffer to write to - * @param payload the pong payload (should match the received ping) - * @param payloadOff offset into payload array - * @param payloadLen length of payload to write - * @return the total number of bytes written - */ - public static int writePongFrame(long buf, byte[] payload, int payloadOff, int payloadLen) { - int headerLen = writeHeader(buf, true, WebSocketOpcode.PONG, payloadLen, false); - - // Copy payload - for (int i = 0; i < payloadLen; i++) { - Unsafe.getUnsafe().putByte(buf + headerLen + i, payload[payloadOff + i]); - } - - return headerLen + payloadLen; - } - - /** - * Writes a Pong frame with payload from a memory address. - * - * @param buf the buffer to write to - * @param payloadPtr pointer to the ping payload to echo - * @param payloadLen length of payload - * @return the total number of bytes written - */ - public static int writePongFrame(long buf, long payloadPtr, int payloadLen) { - int headerLen = writeHeader(buf, true, WebSocketOpcode.PONG, payloadLen, false); - - // Copy payload from memory - Unsafe.getUnsafe().copyMemory(payloadPtr, buf + headerLen, payloadLen); - - return headerLen + payloadLen; - } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java index 21f47b0..46c0a9c 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java @@ -6,7 +6,7 @@ * \__\_\\__,_|\___||___/\__|____/|____/ * * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2024 QuestDB + * Copyright (c) 2019-2026 QuestDB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/StacBenchmarkClient.java b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/StacBenchmarkClient.java index 289643c..1eb044d 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/StacBenchmarkClient.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/StacBenchmarkClient.java @@ -6,7 +6,7 @@ * \__\_\\__,_|\___||___/\__|____/|____/ * * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2024 QuestDB + * Copyright (c) 2019-2026 QuestDB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From 2a14356060df4fc0ca57ba171e6115377d358f57 Mon Sep 17 00:00:00 2001 From: Marko Topolnik Date: Thu, 26 Feb 2026 17:30:34 +0100 Subject: [PATCH 89/89] Delete duplicate method utf8Length() --- .../http/client/WebSocketSendBuffer.java | 3 ++- .../qwp/client/NativeBufferWriter.java | 3 +++ .../cutlass/qwp/client/QwpBufferWriter.java | 27 ------------------- .../qwp/client/NativeBufferWriterTest.java | 9 +++---- 4 files changed, 9 insertions(+), 33 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java index afb0d05..d31044f 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketSendBuffer.java @@ -25,6 +25,7 @@ package io.questdb.client.cutlass.http.client; import io.questdb.client.cutlass.line.array.ArrayBufferAppender; +import io.questdb.client.cutlass.qwp.client.NativeBufferWriter; import io.questdb.client.cutlass.qwp.client.QwpBufferWriter; import io.questdb.client.cutlass.qwp.websocket.WebSocketFrameWriter; import io.questdb.client.cutlass.qwp.websocket.WebSocketOpcode; @@ -336,7 +337,7 @@ public void putString(String value) { putVarint(0); return; } - int utf8Len = QwpBufferWriter.utf8Length(value); + int utf8Len = NativeBufferWriter.utf8Length(value); putVarint(utf8Len); putUtf8(value); } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java index 5d8eb9e..e538769 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java @@ -56,6 +56,9 @@ public NativeBufferWriter(int initialCapacity) { /** * Returns the UTF-8 encoded length of a string. + * + * @param s the string (may be null) + * @return the number of bytes needed to encode the string as UTF-8 */ public static int utf8Length(String s) { if (s == null) return 0; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpBufferWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpBufferWriter.java index f32f575..644fdf8 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpBufferWriter.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpBufferWriter.java @@ -44,33 +44,6 @@ */ public interface QwpBufferWriter extends ArrayBufferAppender { - /** - * Returns the UTF-8 encoded length of a string. - * - * @param s the string (may be null) - * @return the number of bytes needed to encode the string as UTF-8 - */ - static int utf8Length(String s) { - if (s == null) return 0; - int len = 0; - for (int i = 0, n = s.length(); i < n; i++) { - char c = s.charAt(i); - if (c < 0x80) { - len++; - } else if (c < 0x800) { - len += 2; - } else if (c >= 0xD800 && c <= 0xDBFF && i + 1 < n && Character.isLowSurrogate(s.charAt(i + 1))) { - i++; - len += 4; - } else if (Character.isSurrogate(c)) { - len++; - } else { - len += 3; - } - } - return len; - } - /** * Ensures the buffer has capacity for at least the specified * additional bytes beyond the current position. diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java index c77f8d3..5bf342d 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java @@ -25,7 +25,6 @@ package io.questdb.client.test.cutlass.qwp.client; import io.questdb.client.cutlass.qwp.client.NativeBufferWriter; -import io.questdb.client.cutlass.qwp.client.QwpBufferWriter; import io.questdb.client.std.Unsafe; import org.junit.Assert; import org.junit.Test; @@ -206,13 +205,13 @@ public void testPutUtf8LoneSurrogateMatchesUtf8Length() { @Test public void testQwpBufferWriterUtf8LengthInvalidSurrogatePair() { // High surrogate followed by non-low-surrogate: '?' (1) + 'X' (1) = 2 - assertEquals(2, QwpBufferWriter.utf8Length("\uD800X")); + assertEquals(2, NativeBufferWriter.utf8Length("\uD800X")); // Lone high surrogate at end: '?' (1) - assertEquals(1, QwpBufferWriter.utf8Length("\uD800")); + assertEquals(1, NativeBufferWriter.utf8Length("\uD800")); // Lone low surrogate: '?' (1) - assertEquals(1, QwpBufferWriter.utf8Length("\uDC00")); + assertEquals(1, NativeBufferWriter.utf8Length("\uDC00")); // Valid pair still works: 4 bytes - assertEquals(4, QwpBufferWriter.utf8Length("\uD83D\uDE00")); + assertEquals(4, NativeBufferWriter.utf8Length("\uD83D\uDE00")); } @Test