diff --git a/drmq-broker/src/main/java/com/drmq/broker/AdminHttpServer.java b/drmq-broker/src/main/java/com/drmq/broker/AdminHttpServer.java new file mode 100644 index 0000000..72ef596 --- /dev/null +++ b/drmq-broker/src/main/java/com/drmq/broker/AdminHttpServer.java @@ -0,0 +1,206 @@ +package com.drmq.broker; + +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpHandler; +import com.sun.net.httpserver.HttpServer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.util.List; + +/** + * Lightweight HTTP server for administrative REST APIs. + */ +public class AdminHttpServer { + private static final Logger logger = LoggerFactory.getLogger(AdminHttpServer.class); + private final HttpServer server; + private final MessageStore messageStore; + private final OffsetManager offsetManager; + private final ConsumerGroupCoordinator groupCoordinator; + private final Gson gson = new Gson(); + + public AdminHttpServer(int port, MessageStore messageStore, OffsetManager offsetManager, ConsumerGroupCoordinator groupCoordinator) throws IOException { + this.messageStore = messageStore; + this.offsetManager = offsetManager; + this.groupCoordinator = groupCoordinator; + + this.server = HttpServer.create(new InetSocketAddress(port), 0); + + this.server.createContext("/api/topics", this::handleTopics); + this.server.createContext("/api/consumers", this::handleConsumers); + this.server.createContext("/api/messages", this::handleMessages); + + // CORS and standard executor + this.server.setExecutor(null); + } + + public void start() { + server.start(); + logger.info("Admin HTTP Server started on port {}", server.getAddress().getPort()); + } + + public void stop() { + server.stop(1); + logger.info("Admin HTTP Server stopped."); + } + + private void handleTopics(HttpExchange exchange) throws IOException { + addCorsHeaders(exchange); + if ("OPTIONS".equals(exchange.getRequestMethod())) { + exchange.sendResponseHeaders(204, -1); + return; + } + + JsonArray topicsArray = new JsonArray(); + List topics = messageStore.getTopics(); + + for (String topic : topics) { + JsonObject obj = new JsonObject(); + obj.addProperty("name", topic); + obj.addProperty("messageCount", messageStore.getMessageCount(topic)); + // The global offset is global, not per topic. + // We'll just return the message count for now. + topicsArray.add(obj); + } + + sendJsonResponse(exchange, 200, gson.toJson(topicsArray)); + } + + private void handleConsumers(HttpExchange exchange) throws IOException { + addCorsHeaders(exchange); + if ("OPTIONS".equals(exchange.getRequestMethod())) { + exchange.sendResponseHeaders(204, -1); + return; + } + + JsonArray groupsArray = new JsonArray(); + java.util.Map allOffsets = offsetManager.getAllOffsets(); + + // Group by consumer group name + java.util.Map groupsMap = new java.util.HashMap<>(); + + for (java.util.Map.Entry entry : allOffsets.entrySet()) { + String[] parts = entry.getKey().split("/"); + if (parts.length != 2) continue; + + String groupName = parts[0]; + String topicName = parts[1]; + long committedOffset = entry.getValue(); + + // Calculate lag using true topic head offset rather than message count + // Since DRMQ uses global offsets, messageCount does not correlate to the offset values. + long headOffset = messageStore.getHeadOffset(topicName); + long lag = 0; + if (headOffset >= 0) { + long effectiveCommitted = Math.max(0, committedOffset); // -1 means none committed + lag = Math.max(0, (headOffset + 1) - effectiveCommitted); + } + + JsonObject topicObj = new JsonObject(); + topicObj.addProperty("topic", topicName); + topicObj.addProperty("headOffset", headOffset); + topicObj.addProperty("committedOffset", committedOffset); + topicObj.addProperty("lag", lag); + topicObj.addProperty("activeMembers", groupCoordinator.getConsumerCount(groupName, topicName)); + + groupsMap.computeIfAbsent(groupName, k -> new JsonArray()).add(topicObj); + } + + for (java.util.Map.Entry entry : groupsMap.entrySet()) { + JsonObject groupObj = new JsonObject(); + groupObj.addProperty("groupId", entry.getKey()); + groupObj.add("topics", entry.getValue()); + groupsArray.add(groupObj); + } + + sendJsonResponse(exchange, 200, gson.toJson(groupsArray)); + } + + private void handleMessages(HttpExchange exchange) throws IOException { + addCorsHeaders(exchange); + if ("OPTIONS".equals(exchange.getRequestMethod())) { + exchange.sendResponseHeaders(204, -1); + return; + } + + try { + String query = exchange.getRequestURI().getQuery(); + if (query == null) { + sendJsonResponse(exchange, 400, "{\"error\":\"Missing query parameters\"}"); + return; + } + + String topic = null; + long offset = 0; + int limit = 10; + Long timestamp = null; + + for (String param : query.split("&")) { + String[] pair = param.split("="); + if (pair.length == 2) { + if ("topic".equals(pair[0])) topic = pair[1]; + else if ("offset".equals(pair[0])) offset = Long.parseLong(pair[1]); + else if ("limit".equals(pair[0])) limit = Integer.parseInt(pair[1]); + else if ("timestamp".equals(pair[0])) timestamp = Long.parseLong(pair[1]); + } + } + + if (topic == null) { + sendJsonResponse(exchange, 400, "{\"error\":\"Missing 'topic' parameter\"}"); + return; + } + + // Limit bounds to avoid OOM + limit = Math.min(100, Math.max(1, limit)); + + if (timestamp != null && timestamp > 0) { + offset = messageStore.findOffsetByTimestamp(topic, timestamp); + if (offset == -1) { + sendJsonResponse(exchange, 200, "[]"); + return; + } + } + + List messages = messageStore.getMessages(topic, offset, limit); + JsonArray msgsArray = new JsonArray(); + + for (com.drmq.protocol.DRMQProtocol.StoredMessage msg : messages) { + JsonObject obj = new JsonObject(); + obj.addProperty("offset", msg.getOffset()); + obj.addProperty("timestamp", msg.getTimestamp()); + obj.addProperty("storedAt", msg.getStoredAt()); + if (msg.hasKey()) { + obj.addProperty("key", msg.getKey()); + } + obj.addProperty("payload", msg.getPayload().toStringUtf8()); + msgsArray.add(obj); + } + + sendJsonResponse(exchange, 200, gson.toJson(msgsArray)); + } catch (Exception e) { + logger.error("Error handling messages request", e); + sendJsonResponse(exchange, 500, "{\"error\":\"" + e.getMessage() + "\"}"); + } + } + + private void addCorsHeaders(HttpExchange exchange) { + exchange.getResponseHeaders().add("Access-Control-Allow-Origin", "*"); + exchange.getResponseHeaders().add("Access-Control-Allow-Methods", "GET, POST, DELETE, OPTIONS"); + exchange.getResponseHeaders().add("Access-Control-Allow-Headers", "Content-Type, Authorization"); + } + + private void sendJsonResponse(HttpExchange exchange, int statusCode, String response) throws IOException { + byte[] bytes = response.getBytes("UTF-8"); + exchange.getResponseHeaders().add("Content-Type", "application/json"); + exchange.sendResponseHeaders(statusCode, bytes.length); + try (OutputStream os = exchange.getResponseBody()) { + os.write(bytes); + } + } +} diff --git a/drmq-broker/src/main/java/com/drmq/broker/BrokerConfig.java b/drmq-broker/src/main/java/com/drmq/broker/BrokerConfig.java index 6dafe0a..9f75223 100644 --- a/drmq-broker/src/main/java/com/drmq/broker/BrokerConfig.java +++ b/drmq-broker/src/main/java/com/drmq/broker/BrokerConfig.java @@ -141,7 +141,7 @@ public static BrokerConfig fromArgs(String[] args) { String metricsPath = "/metrics"; long logSegmentBytes = 100 * 1024 * 1024L; // 100MB long logRetentionMs = 7L * 24 * 60 * 60 * 1000; // 7 days - long raftCompactThreshold = 50000L; // Keep 50,000 entries to buffer followers during short outages + long raftCompactThreshold = 1000L; int maxDeliveries = 5; String dlqTopicPrefix = "dlq."; boolean logSegmentFsync = true; @@ -188,12 +188,12 @@ public static BrokerConfig fromArgs(String[] args) { for (int i = 0; i < args.length; i++) { switch (args[i]) { - case "--config" -> i++; // skip the value, already handled + case "--config" -> i++; case "--id", "--node-id" -> nodeId = args[++i]; case "--port" -> port = Integer.parseInt(args[++i]); case "--data-dir" -> dataDir = args[++i]; case "--peers" -> { - peers.clear(); // Override config file peers + peers.clear(); String[] peerStrs = args[++i].split(","); for (String peerStr : peerStrs) { peers.add(PeerAddress.parse(peerStr)); diff --git a/drmq-broker/src/main/java/com/drmq/broker/BrokerServer.java b/drmq-broker/src/main/java/com/drmq/broker/BrokerServer.java index 7534ec9..8a9b627 100644 --- a/drmq-broker/src/main/java/com/drmq/broker/BrokerServer.java +++ b/drmq-broker/src/main/java/com/drmq/broker/BrokerServer.java @@ -46,6 +46,7 @@ public class BrokerServer { private final List raftPeers; private final BrokerMetrics metrics; private TelemetryWebSocketServer telemetryServer; + private AdminHttpServer adminHttpServer; private volatile boolean running = false; @@ -171,6 +172,10 @@ public void initChannel(SocketChannel ch) { telemetryServer = new TelemetryWebSocketServer(wsPort, this); telemetryServer.start(); + int adminPort = config.getPort() + 300; + adminHttpServer = new AdminHttpServer(adminPort, messageStore, offsetManager, groupCoordinator); + adminHttpServer.start(); + logger.info("DRMQ Broker started on port {} with data directory {}", config.getPort(), config.getDataDir()); @@ -214,6 +219,9 @@ public void shutdown() { if (telemetryServer != null) { telemetryServer.shutdown(); } + if (adminHttpServer != null) { + adminHttpServer.stop(); + } if (activeChannels != null) { activeChannels.close().awaitUninterruptibly(); diff --git a/drmq-broker/src/main/java/com/drmq/broker/ClientHandler.java b/drmq-broker/src/main/java/com/drmq/broker/ClientHandler.java index 533f43c..ce2de6d 100644 --- a/drmq-broker/src/main/java/com/drmq/broker/ClientHandler.java +++ b/drmq-broker/src/main/java/com/drmq/broker/ClientHandler.java @@ -90,6 +90,7 @@ private MessageEnvelope handleMessage(MessageEnvelope envelope) throws IOExcepti case APPEND_ENTRIES_REQUEST -> handleAppendEntriesRequest(envelope); case INSTALL_SNAPSHOT_REQUEST -> handleInstallSnapshotRequest(envelope); case SEARCH_OFFSET_BY_TIME_REQUEST -> handleSearchOffsetByTimeRequest(envelope); + case ATOMIC_PRODUCE_REQUEST -> handleAtomicProduceRequest(envelope); default -> createErrorResponse("Unknown message type: " + envelope.getType()); }; } @@ -214,6 +215,74 @@ private MessageEnvelope createProduceBatchErrorResponse(String errorMessage, Err .build(); } + private MessageEnvelope handleAtomicProduceRequest(MessageEnvelope envelope) throws IOException { + long startNanos = System.nanoTime(); + long totalPayloadBytes = 0; + int batchCount = 0; + try { + com.drmq.protocol.DRMQProtocol.AtomicProduceRequest request = com.drmq.protocol.DRMQProtocol.AtomicProduceRequest.parseFrom(envelope.getPayload()); + + for (var slice : request.getSlicesList()) { + batchCount += slice.getEntriesCount(); + for (var entry : slice.getEntriesList()) { + totalPayloadBytes += entry.getPayload().size(); + } + } + + if (batchCount == 0) { + return createAtomicProduceErrorResponse("Atomic batch must contain at least one message", ErrorCode.UNKNOWN_ERROR); + } + if (totalPayloadBytes > MAX_PAYLOAD_BYTES) { + return createAtomicProduceErrorResponse("Batch payload exceeds maximum size of " + MAX_PAYLOAD_BYTES + " bytes", ErrorCode.UNKNOWN_ERROR); + } + + java.util.Map offsets; + if (raftNode != null) { + if (!raftNode.isLeader()) { + String leaderAddr = raftNode.getLeaderAddress(); + return createAtomicProduceErrorResponse("NOT_LEADER:" + + (leaderAddr != null ? leaderAddr : "UNKNOWN"), ErrorCode.NOT_LEADER); + } + offsets = raftNode.proposeAtomicBatch(request.getSlicesList()); + } else { + offsets = messageStore.appendAtomicBatch(request.getSlicesList()); + } + + logger.debug("Produced atomic batch: topics={}, count={}", offsets.keySet(), batchCount); + + com.drmq.protocol.DRMQProtocol.AtomicProduceResponse response = com.drmq.protocol.DRMQProtocol.AtomicProduceResponse.newBuilder() + .setSuccess(true) + .putAllBaseOffsets(offsets) + .build(); + + BrokerMetrics.get().recordRequest("atomic_produce", true, + System.nanoTime() - startNanos, totalPayloadBytes, batchCount); + + return MessageEnvelope.newBuilder() + .setType(MessageType.ATOMIC_PRODUCE_RESPONSE) + .setPayload(response.toByteString()) + .build(); + + } catch (Exception e) { + logger.error("Error processing atomic produce request", e); + BrokerMetrics.get().recordRequest("atomic_produce", false, + System.nanoTime() - startNanos, totalPayloadBytes, batchCount); + return createAtomicProduceErrorResponse(e.getMessage(), ErrorCode.UNKNOWN_ERROR); + } + } + + private MessageEnvelope createAtomicProduceErrorResponse(String errorMessage, ErrorCode errorCode) { + com.drmq.protocol.DRMQProtocol.AtomicProduceResponse response = com.drmq.protocol.DRMQProtocol.AtomicProduceResponse.newBuilder() + .setSuccess(false) + .setErrorMessage(errorMessage != null ? errorMessage : "Unknown error") + .setErrorCode(errorCode) + .build(); + return MessageEnvelope.newBuilder() + .setType(MessageType.ATOMIC_PRODUCE_RESPONSE) + .setPayload(response.toByteString()) + .build(); + } + private MessageEnvelope handleConsumeRequest(MessageEnvelope envelope) throws IOException { long startNanos = System.nanoTime(); try { @@ -532,6 +601,7 @@ private MessageEnvelope createErrorResponse(String errorMessage, MessageType mes case COMMIT_OFFSET_RESPONSE -> createCommitOffsetErrorResponse(errorMessage); case FETCH_OFFSET_RESPONSE -> createFetchOffsetErrorResponse(errorMessage); case NACK_RESPONSE -> createNackErrorResponse(errorMessage); + case ATOMIC_PRODUCE_RESPONSE -> createAtomicProduceErrorResponse(errorMessage, ErrorCode.UNKNOWN_ERROR); default -> createProduceErrorResponse(errorMessage, ErrorCode.UNKNOWN_ERROR); }; } diff --git a/drmq-broker/src/main/java/com/drmq/broker/ConsumerGroupCoordinator.java b/drmq-broker/src/main/java/com/drmq/broker/ConsumerGroupCoordinator.java index 66be5cc..f4aed93 100644 --- a/drmq-broker/src/main/java/com/drmq/broker/ConsumerGroupCoordinator.java +++ b/drmq-broker/src/main/java/com/drmq/broker/ConsumerGroupCoordinator.java @@ -293,7 +293,6 @@ public int getActiveLeasesCount() { return total; } - // ---- Dead-Letter Queue (DLQ) Support ---- /** * Explicitly reject (NACK) a message offset for a consumer within a group. @@ -319,7 +318,6 @@ public boolean nackOffset(String group, String topic, String consumerId, long of state.lock.lock(); try { - // Remove the consumer's active lease Lease lease = state.activeLeases.remove(consumerId); if (lease != null) { state.members.remove(consumerId); @@ -333,7 +331,6 @@ public boolean nackOffset(String group, String topic, String consumerId, long of routeToDlq(state, group, topic, offset); return true; } else { - // Rewind for redelivery using the lease's fromOffset to not skip messages long rewindOffset = (lease != null) ? lease.fromOffset : offset; if (rewindOffset < state.dispatchOffset) { state.dispatchOffset = rewindOffset; @@ -379,7 +376,6 @@ private void routeToDlq(GroupTopicState state, String group, String topic, long } }); - // Advance past the bad offset regardless — don't let a DLQ write failure block progress long nextOffset = badOffset + 1; state.committedRanges.add(new CommittedRange(badOffset, nextOffset)); advanceCommittedOffset(state, group, topic); @@ -387,7 +383,6 @@ private void routeToDlq(GroupTopicState state, String group, String topic, long state.dispatchOffset = nextOffset; } - // Clean up the delivery counter for this offset state.deliveryCounts.remove(badOffset); } diff --git a/drmq-broker/src/main/java/com/drmq/broker/MessageStore.java b/drmq-broker/src/main/java/com/drmq/broker/MessageStore.java index 47d18f1..ddcffbc 100644 --- a/drmq-broker/src/main/java/com/drmq/broker/MessageStore.java +++ b/drmq-broker/src/main/java/com/drmq/broker/MessageStore.java @@ -2,6 +2,7 @@ import com.drmq.broker.persistence.LogManager; import com.drmq.broker.persistence.LogSegment; +import com.drmq.protocol.DRMQProtocol.AtomicBatchTopicSlice; import com.drmq.protocol.DRMQProtocol.ProduceBatchRequest; import com.drmq.protocol.DRMQProtocol.StoredMessage; import org.slf4j.Logger; @@ -48,6 +49,9 @@ public class MessageStore implements Closeable { // Topic -> Total number of messages private final ConcurrentHashMap topicMessageCounts = new ConcurrentHashMap<>(); + + // Topic -> Highest offset appended + private final ConcurrentHashMap topicHeadOffsets = new ConcurrentHashMap<>(); // In-memory cache for recent messages (Topic -> BoundedMessageCache) private final ConcurrentHashMap messageCache = new ConcurrentHashMap<>(); @@ -124,6 +128,11 @@ private void recoverInternal() throws IOException { addToCache(topic, message); topicMessageCounts.computeIfAbsent(topic, k -> new AtomicLong()).incrementAndGet(); + AtomicLong head = topicHeadOffsets.computeIfAbsent(topic, k -> new AtomicLong(-1)); + if (offset > head.get()) { + head.set(offset); + } + if (offset > maxOffset) { maxOffset = offset; } @@ -156,6 +165,7 @@ public void reload() throws IOException { logger.info("Reloading MessageStore state from disk..."); topicIndex.clear(); topicMessageCounts.clear(); + topicHeadOffsets.clear(); messageCache.clear(); topicWriteLocks.clear(); @@ -222,6 +232,11 @@ public long append(String topic, byte[] payload, String key, long clientTimestam addToCache(topic, message); topicMessageCounts.computeIfAbsent(topic, k -> new AtomicLong()).incrementAndGet(); + + AtomicLong head = topicHeadOffsets.computeIfAbsent(topic, k -> new AtomicLong(-1)); + if (offset > head.get()) { + head.set(offset); + } logger.debug("Persisted and indexed message: topic={}, offset={}, position={}, segment={}", topic, offset, position, segment.getFilePath().getFileName()); @@ -292,11 +307,15 @@ public long appendBatch(String topic, List entri } AtomicLong counter = topicMessageCounts.computeIfAbsent(topic, k -> new AtomicLong()); + AtomicLong head = topicHeadOffsets.computeIfAbsent(topic, k -> new AtomicLong(-1)); for (int i = 0; i < messages.size(); i++) { StoredMessage msg = messages.get(i); indexMessage(topic, msg.getOffset(), positions.get(i)); addToCache(topic, msg); counter.incrementAndGet(); + if (msg.getOffset() > head.get()) { + head.set(msg.getOffset()); + } } logger.debug("Batch persisted: topic={}, baseOffset={}, count={}", topic, baseOffset, batchSize); @@ -316,6 +335,94 @@ public long appendBatch(String topic, List entri return baseOffset; } + /** + * Atomically appends messages to multiple topics in a single operation. + * Either all topic writes succeed, or none are visible. + * Returns a map of topic -> base offset for each slice. + */ + public Map appendAtomicBatch(List slices) { + if (slices.isEmpty()) { + return Collections.emptyMap(); + } + + int totalMessages = slices.stream().mapToInt(AtomicBatchTopicSlice::getEntriesCount).sum(); + if (totalMessages == 0) { + return Collections.emptyMap(); + } + + long baseOffset = globalOffset.getAndAdd(totalMessages); + long storedAt = System.currentTimeMillis(); + + Map topicBaseOffsets = new LinkedHashMap<>(); + Map> topicMessages = new LinkedHashMap<>(); + + long currentOffset = baseOffset; + for (var slice : slices) { + String topic = slice.getTopic(); + long topicBase = currentOffset; + topicBaseOffsets.put(topic, topicBase); + + List messages = new ArrayList<>(slice.getEntriesCount()); + for (var entry : slice.getEntriesList()) { + StoredMessage.Builder builder = StoredMessage.newBuilder() + .setOffset(currentOffset++) + .setTopic(topic) + .setPayload(entry.getPayload()) + .setTimestamp(entry.getClientTimestamp()) + .setStoredAt(storedAt); + + if (entry.hasKey()) { + builder.setKey(entry.getKey()); + } + messages.add(builder.build()); + } + topicMessages.put(topic, messages); + } + + globalLock.writeLock().lock(); + try { + for (var entry : topicMessages.entrySet()) { + String topic = entry.getKey(); + List messages = entry.getValue(); + + Object topicLock = topicWriteLocks.computeIfAbsent(topic, k -> new Object()); + List positions; + synchronized (topicLock) { + LogSegment segment = logManager.getOrCreateActiveSegment(topic); + if (segment.getSize() >= config.getLogSegmentBytes()) { + segment = logManager.rollNewSegment(topic, topicBaseOffsets.get(topic)); + } + positions = segment.appendBatch(messages); + } + + AtomicLong counter = topicMessageCounts.computeIfAbsent(topic, k -> new AtomicLong()); + AtomicLong head = topicHeadOffsets.computeIfAbsent(topic, k -> new AtomicLong(-1)); + for (int i = 0; i < messages.size(); i++) { + StoredMessage msg = messages.get(i); + indexMessage(topic, msg.getOffset(), positions.get(i)); + addToCache(topic, msg); + counter.incrementAndGet(); + if (msg.getOffset() > head.get()) { + head.set(msg.getOffset()); + } + } + logger.debug("Atomic batch slice persisted: topic={}, baseOffset={}, count={}", topic, topicBaseOffsets.get(topic), messages.size()); + } + } catch (IOException e) { + logger.error("Failed to persist atomic batch", e); + throw new RuntimeException("Failed to persist atomic batch", e); + } finally { + globalLock.writeLock().unlock(); + } + + synchronized (messageMonitor) { + messageSignal.incrementAndGet(); + messageMonitor.notifyAll(); + } + + return topicBaseOffsets; + } + /** * Lock the store exclusively to safely take a snapshot of the log segments. */ @@ -411,18 +518,14 @@ public List getMessages(String topic, long fromOffset, int maxCou LogSegment segment = logManager.getSegmentForOffset(topic, currentOffset); if (segment != null && segment == lastSegment) { - // The local segment floorEntry returned is the same one we just exhausted. - // This means the segment containing currentOffset is missing locally (archived). segment = null; } if (segment == null) { - // Not found locally? Ask the InfinityLog! if (infinityLogResolver != null) { segment = infinityLogResolver.resolveMissingSegment(topic, currentOffset); } - // If still null (not in S3 or no S3), try to jump to the next available local segment if (segment == null) { ConcurrentSkipListMap allTopicSegments = logManager.getAllSegments().get(topic); if (allTopicSegments != null) { @@ -523,6 +626,11 @@ public int getMessageCount(String topic) { return count == null ? 0 : count.intValue(); } + public long getHeadOffset(String topic) { + AtomicLong head = topicHeadOffsets.get(topic); + return head == null ? -1 : head.get(); + } + public List getTopics() { return new ArrayList<>(topicMessageCounts.keySet()); } @@ -542,6 +650,7 @@ public long getCachedMessageCount() { public void clear() { topicIndex.clear(); topicMessageCounts.clear(); + topicHeadOffsets.clear(); messageCache.clear(); globalOffset.set(0); logger.info("Message store memory state cleared"); @@ -682,9 +791,16 @@ public void removeRange(long fromOffset, long toOffset) { public List getMessagesFrom(long fromOffset, int maxCount) { lock.readLock().lock(); try { + if (!cache.containsKey(fromOffset)) { + return Collections.emptyList(); + } List result = new ArrayList<>(); + boolean found = false; for (Map.Entry entry : cache.entrySet()) { - if (entry.getKey() >= fromOffset) { + if (entry.getKey() == fromOffset) { + found = true; + } + if (found) { result.add(entry.getValue()); if (result.size() >= maxCount) { break; diff --git a/drmq-broker/src/main/java/com/drmq/broker/OffsetManager.java b/drmq-broker/src/main/java/com/drmq/broker/OffsetManager.java index 6c49d01..28b7e77 100644 --- a/drmq-broker/src/main/java/com/drmq/broker/OffsetManager.java +++ b/drmq-broker/src/main/java/com/drmq/broker/OffsetManager.java @@ -74,6 +74,14 @@ public int getOffsetEntryCount() { return offsets.size(); } + /** + * Get a snapshot of all committed offsets across all groups and topics. + * @return map of "group/topic" to offset + */ + public Map getAllOffsets() { + return new java.util.HashMap<>(offsets); + } + /** Load all offsets from disk on startup. */ private void load() throws IOException { if (!Files.exists(offsetsFile)) { diff --git a/drmq-broker/src/main/java/com/drmq/broker/TelemetryWebSocketServer.java b/drmq-broker/src/main/java/com/drmq/broker/TelemetryWebSocketServer.java index 642e786..d9e59e6 100644 --- a/drmq-broker/src/main/java/com/drmq/broker/TelemetryWebSocketServer.java +++ b/drmq-broker/src/main/java/com/drmq/broker/TelemetryWebSocketServer.java @@ -16,8 +16,8 @@ import java.util.List; import java.util.Timer; import java.util.TimerTask; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CopyOnWriteArraySet; -import java.util.concurrent.TimeUnit; public class TelemetryWebSocketServer extends WebSocketServer { private static final Logger logger = LoggerFactory.getLogger(TelemetryWebSocketServer.class); @@ -28,7 +28,10 @@ public class TelemetryWebSocketServer extends WebSocketServer { private final Gson gson; // Throughput history (0-100 scaled for chart) - private final List throughputHistory = new ArrayList<>(); + private final List throughputHistory = new CopyOnWriteArrayList<>(); + private final List produceHistory = new CopyOnWriteArrayList<>(); + private final List consumeHistory = new CopyOnWriteArrayList<>(); + private final List errorHistory = new CopyOnWriteArrayList<>(); // Rolling byte snapshots to compute per-second rates private double lastProduceBytes = 0; @@ -51,16 +54,23 @@ public TelemetryWebSocketServer(int port, BrokerServer brokerServer) { this.broadcastTimer = new Timer("TelemetryBroadcastTimer", true); this.gson = new Gson(); - for (int i = 0; i < 30; i++) { + for (int i = 0; i < 300; i++) { throughputHistory.add(0.0); + produceHistory.add(0.0); + consumeHistory.add(0.0); + errorHistory.add(0.0); } } @Override public void onOpen(WebSocket conn, ClientHandshake handshake) { - connections.add(conn); - logger.info("New telemetry dashboard connected: {}", conn.getRemoteSocketAddress()); - conn.send(buildTelemetryPayload()); + try { + connections.add(conn); + logger.info("New telemetry dashboard connected: {}", conn.getRemoteSocketAddress()); + conn.send(buildTelemetryPayload()); + } catch (Exception e) { + logger.error("Error during onOpen, preventing SelectorThread crash", e); + } } @Override @@ -83,16 +93,24 @@ public void onStart() { broadcastTimer.scheduleAtFixedRate(new TimerTask() { @Override public void run() { - updateRates(); - if (!connections.isEmpty()) { - String payload = buildTelemetryPayload(); - for (WebSocket conn : connections) { - try { - conn.send(payload); - } catch (Exception e) { - logger.debug("Failed to send telemetry to client", e); + try { + updateRates(); + if (!connections.isEmpty()) { + String payload = buildTelemetryPayload(); + for (WebSocket conn : connections) { + try { + conn.send(payload); + } catch (Exception e) { + logger.debug("Failed to send telemetry to client", e); + } } } + } catch (Exception e) { + // CRITICAL: TimerTask kills the schedule permanently on any + // uncaught exception. During leader failovers, concurrent Raft + // state mutations can cause CME/NPE in buildTelemetryPayload(). + // Swallow and retry on the next tick instead of dying forever. + logger.warn("Telemetry broadcast tick failed (will retry): {}", e.getMessage()); } } }, 1000, 1000); @@ -138,14 +156,19 @@ private void updateRates() { lastConsumeRecords = consumeRecords; lastErrorCount = errors; - // Throughput chart history: scale (produce + consume) MB/s to 0-100 double totalMBps = currentProduceMBps + currentConsumeMBps; - // Scale: 50 MB/s = 100 on chart - double chartVal = Math.min(100, (totalMBps / 50.0) * 100); - if (chartVal < 3 && totalMBps > 0) chartVal = 3; // minimum visibility synchronized (throughputHistory) { throughputHistory.remove(0); - throughputHistory.add(chartVal); + throughputHistory.add(totalMBps); + + produceHistory.remove(0); + produceHistory.add(currentProduceMBps); + + consumeHistory.remove(0); + consumeHistory.add(currentConsumeMBps); + + errorHistory.remove(0); + errorHistory.add(currentErrorRate); } } @@ -159,9 +182,9 @@ private String buildTelemetryPayload() { JsonObject metrics = new JsonObject(); double totalMBps = currentProduceMBps + currentConsumeMBps; - metrics.addProperty("totalThroughputMB", round2(totalMBps)); - metrics.addProperty("produceThroughputMB", round2(currentProduceMBps)); - metrics.addProperty("consumeThroughputMB", round2(currentConsumeMBps)); + metrics.addProperty("totalThroughputMB", round4(totalMBps)); + metrics.addProperty("produceThroughputMB", round4(currentProduceMBps)); + metrics.addProperty("consumeThroughputMB", round4(currentConsumeMBps)); metrics.addProperty("produceRate", Math.round(currentProduceRecordRate)); // msgs/s metrics.addProperty("consumeRate", Math.round(currentConsumeRecordRate)); // msgs/s metrics.addProperty("errorRate", Math.round(currentErrorRate)); // errors/s @@ -175,8 +198,8 @@ private String buildTelemetryPayload() { produceLatencyMs = Math.max(singleLatency, batchLatency); // use whichever path is active consumeLatencyMs = bm.getTimerMeanMs("drmq.broker.request.latency", "consume"); } - metrics.addProperty("produceLatencyMs", round2(produceLatencyMs)); - metrics.addProperty("consumeLatencyMs", round2(consumeLatencyMs)); + metrics.addProperty("produceLatencyMs", round4(produceLatencyMs)); + metrics.addProperty("consumeLatencyMs", round4(consumeLatencyMs)); // Active connections: real handler count from Netty int totalHandlers = brokerServer.getActiveChannelsCount(); @@ -265,12 +288,21 @@ private String buildTelemetryPayload() { // Throughput chart history JsonArray history = new JsonArray(); + JsonArray pHistory = new JsonArray(); + JsonArray cHistory = new JsonArray(); + JsonArray eHistory = new JsonArray(); synchronized (throughputHistory) { - for (Double val : throughputHistory) { - history.add(round2(val)); + for (int i = 0; i < throughputHistory.size(); i++) { + history.add(round4(throughputHistory.get(i))); + pHistory.add(round4(produceHistory.get(i))); + cHistory.add(round4(consumeHistory.get(i))); + eHistory.add(round4(errorHistory.get(i))); } } metrics.add("throughputHistory", history); + metrics.add("produceHistory", pHistory); + metrics.add("consumeHistory", cHistory); + metrics.add("errorHistory", eHistory); // Produce-rate history (msgs/s scaled similarly) state.add("metrics", metrics); @@ -286,7 +318,7 @@ private String buildTelemetryPayload() { localNode.addProperty("name", "Broker-" + localId.toUpperCase()); localNode.addProperty("status", localStatus); // throughput in bytes/s for this node (real produce traffic it handled) - localNode.addProperty("throughputMBps", round2(currentProduceMBps + currentConsumeMBps)); + localNode.addProperty("throughputMBps", round4(currentProduceMBps + currentConsumeMBps)); localNode.addProperty("produceRate", Math.round(currentProduceRecordRate)); localNode.addProperty("consumeRate", Math.round(currentConsumeRecordRate)); localNode.addProperty("commitIndex", commitIndex); @@ -366,9 +398,9 @@ private String buildTelemetryPayload() { } JsonObject latencies = new JsonObject(); // Inter-node Raft RPC latency (same for all links since we only know our own outbound) - latencies.addProperty("alphaBeta", round2(raftRpcLatencyMs > 0 ? raftRpcLatencyMs : 0)); - latencies.addProperty("betaGamma", round2(raftRpcLatencyMs > 0 ? raftRpcLatencyMs : 0)); - latencies.addProperty("raftRpcMs", round2(raftRpcLatencyMs)); + latencies.addProperty("alphaBeta", round4(raftRpcLatencyMs > 0 ? raftRpcLatencyMs : 0)); + latencies.addProperty("betaGamma", round4(raftRpcLatencyMs > 0 ? raftRpcLatencyMs : 0)); + latencies.addProperty("raftRpcMs", round4(raftRpcLatencyMs)); state.add("latencies", latencies); // ── EVENTS ──────────────────────────────────────────────────────────── @@ -377,7 +409,7 @@ private String buildTelemetryPayload() { return gson.toJson(state); } - private static double round2(double v) { - return Math.round(v * 100.0) / 100.0; + private static double round4(double v) { + return Math.round(v * 10000.0) / 10000.0; } } diff --git a/drmq-broker/src/main/java/com/drmq/broker/raft/RaftLog.java b/drmq-broker/src/main/java/com/drmq/broker/raft/RaftLog.java index 4af5376..9ced343 100644 --- a/drmq-broker/src/main/java/com/drmq/broker/raft/RaftLog.java +++ b/drmq-broker/src/main/java/com/drmq/broker/raft/RaftLog.java @@ -161,7 +161,21 @@ public synchronized List getEntriesFrom(long fromIndex, int maxEntrie return Collections.emptyList(); } int from = (int) (fromIndex - startIndex); - int to = Math.min(from + maxEntries, entries.size()); + + int to = from; + long currentBytes = 0; + int maxBytes = 2 * 1024 * 1024; // 2 MB limit per RPC + + while (to < entries.size() && to - from < maxEntries) { + RaftEntry entry = entries.get(to); + long entrySize = entry.getSerializedSize(); + if (to > from && currentBytes + entrySize > maxBytes) { + break; // ensure at least one entry is sent if the single entry is > 2MB + } + currentBytes += entrySize; + to++; + } + return new ArrayList<>(entries.subList(from, to)); } diff --git a/drmq-broker/src/main/java/com/drmq/broker/raft/RaftNode.java b/drmq-broker/src/main/java/com/drmq/broker/raft/RaftNode.java index d9329bc..7208f03 100644 --- a/drmq-broker/src/main/java/com/drmq/broker/raft/RaftNode.java +++ b/drmq-broker/src/main/java/com/drmq/broker/raft/RaftNode.java @@ -45,16 +45,16 @@ public class RaftNode { private static final int MAX_PENDING_PROPOSALS = 10000; // Safety limit // Persistent state (survives restart) - private long currentTerm; - private String votedFor; + private volatile long currentTerm; + private volatile String votedFor; private final RaftLog raftLog; // Volatile state - private RaftState state; - private long commitIndex; - private long lastApplied; - private long lastAppliedTerm; - private String leaderId; + private volatile RaftState state; + private volatile long commitIndex; + private volatile long lastApplied; + private volatile long lastAppliedTerm; + private volatile String leaderId; // Leader-only volatile state private final Map nextIndex; @@ -607,7 +607,7 @@ private void replicateTo(PeerAddress peer) { peerNextIndex = nextIndex.getOrDefault(peer.id(), raftLog.getLastIndex() + 1); prevLogIndex = peerNextIndex - 1; - if (prevLogIndex > 0 && prevLogIndex < raftLog.getStartIndex()) { + if (peerNextIndex < raftLog.getStartIndex()) { needsSnapshot = true; return; } @@ -725,6 +725,10 @@ private void sendInstallSnapshotToPeer(PeerAddress peer) { .build(); InstallSnapshotResponse response = handler.apply(request); + if (response.getTerm() == 0) { + throw new IOException("InstallSnapshot RPC failed (peer offline)"); + } + lastContactTime.put(peer.id(), System.currentTimeMillis()); lock.lock(); @@ -829,6 +833,14 @@ private void applyCommitted() { logger.debug("[{}] Applied raft batch entry {} to MessageStore (topic={}, count={})", nodeId, lastApplied, entry.getTopic(), batchRequest.getEntriesCount()); } + case ATOMIC_BATCH -> { + com.drmq.protocol.DRMQProtocol.AtomicBatchRequest req = com.drmq.protocol.DRMQProtocol.AtomicBatchRequest.parseFrom(entry.getPayload()); + Map baseOffsets = messageStore.appendAtomicBatch(req.getSlicesList()); + completionValue = lastApplied; + logger.debug("[{}] Applied ATOMIC_BATCH entry {} to {} topics: {}", + nodeId, lastApplied, req.getSlicesCount(), + baseOffsets.keySet()); + } default -> { long msgOffset = messageStore.append( entry.getTopic(), @@ -1040,6 +1052,77 @@ public long proposeBatch(String topic, List entr throw new IOException("Raft batch proposal interrupted"); } } + + /** + * Propose a cross-topic atomic batch to be replicated via Raft. + * All topic writes are applied atomically in applyCommitted(). + * + * @param slices list of (topic, entries) pairs — must contain ≥2 topics + * @return map of topic -> base offset, populated after commit + */ + public Map proposeAtomicBatch(List slices) throws IOException { + if (slices.size() < 2) { + throw new IllegalArgumentException( + "ATOMIC_BATCH requires at least 2 topics. Use proposeBatch() for single-topic."); + } + + lock.lock(); + long index; + long proposalTerm; + CompletableFuture future; + try { + if (state != RaftState.LEADER) { + throw new IOException("NOT_LEADER:" + + (leaderId != null ? getLeaderAddress() : "UNKNOWN")); + } + + proposalTerm = currentTerm; + index = raftLog.getLastIndex() + 1; + + if (pendingProposals.size() >= MAX_PENDING_PROPOSALS) { + throw new IOException("Too many pending proposals"); + } + + AtomicBatchRequest payload = AtomicBatchRequest.newBuilder() + .addAllSlices(slices) + .build(); + + RaftEntry entry = RaftEntry.newBuilder() + .setTerm(proposalTerm) + .setIndex(index) + .setTopic(slices.stream() + .map(AtomicBatchTopicSlice::getTopic) + .collect(java.util.stream.Collectors.joining(","))) + .setPayload(payload.toByteString()) + .setCommandType(RaftCommandType.ATOMIC_BATCH) + .build(); + + raftLog.append(entry); + future = new CompletableFuture<>(); + pendingProposals.put(index, new ProposalState(proposalTerm, future)); + + } finally { + lock.unlock(); + } + sendHeartbeats(); + try { + future.get(PROPOSAL_TIMEOUT_SECONDS, TimeUnit.SECONDS); + // After commit, fetch offsets from the message store + Map offsets = new java.util.LinkedHashMap<>(); + for (com.drmq.protocol.DRMQProtocol.AtomicBatchTopicSlice s : slices) { + offsets.put(s.getTopic(), + messageStore.getHeadOffset(s.getTopic()) - s.getEntriesCount() + 1); + } + return offsets; + } catch (TimeoutException e) { + throw new IOException("Atomic batch proposal timed out"); + } catch (ExecutionException e) { + throw new IOException("Atomic batch proposal failed: " + e.getCause().getMessage()); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException("Interrupted"); + } + } /** * Propose a consumer offset commit to be replicated via Raft. * Ensures offset durability across leader failover. @@ -1275,10 +1358,11 @@ public AppendEntriesResponse handleAppendEntries(AppendEntriesRequest request) { applyCommitted(); } + long matchIdx = request.getPrevLogIndex() + request.getEntriesCount(); return AppendEntriesResponse.newBuilder() .setTerm(currentTerm) .setSuccess(true) - .setMatchIndex(raftLog.getLastIndex()) + .setMatchIndex(matchIdx) .build(); } finally { @@ -1462,6 +1546,16 @@ private void loadPersistentState() throws IOException { raftLog.setStartIndex(lastApplied + 1); } commitIndex = Math.min(raftLog.getLastIndex(), Math.max(commitIndex, lastApplied)); + + + if (raftLog.getLastIndex() - raftLog.getStartIndex() > raftCompactThreshold) { + long compactUpTo = lastApplied - 100; + if (compactUpTo > raftLog.getStartIndex()) { + raftLog.compact(compactUpTo); + logger.info("[{}] Compacted Raft log on startup up to index {}", nodeId, compactUpTo); + } + } + logger.info("[{}] Loaded persistent state: term={}, votedFor={}, lastApplied={}", nodeId, currentTerm, votedFor, lastApplied); } catch (IOException | NumberFormatException e) { diff --git a/drmq-broker/src/main/java/com/drmq/broker/raft/SnapshotManager.java b/drmq-broker/src/main/java/com/drmq/broker/raft/SnapshotManager.java index 785eee4..ac0367c 100644 --- a/drmq-broker/src/main/java/com/drmq/broker/raft/SnapshotManager.java +++ b/drmq-broker/src/main/java/com/drmq/broker/raft/SnapshotManager.java @@ -48,6 +48,11 @@ public Path createSnapshot(long lastIncludedIndex) throws IOException { Files.createDirectories(snapshotsDir); Path zipFile = snapshotsDir.resolve("snapshot_" + lastIncludedIndex + ".zip"); + if (Files.exists(zipFile)) { + logger.debug("Snapshot for index {} already exists, skipping generation.", lastIncludedIndex); + return zipFile; + } + logger.info("Starting snapshot generation for index {}...", lastIncludedIndex); long startMs = System.currentTimeMillis(); diff --git a/drmq-broker/src/test/java/com/drmq/broker/MessageStoreTest.java b/drmq-broker/src/test/java/com/drmq/broker/MessageStoreTest.java index 149b186..ccbd7f3 100644 --- a/drmq-broker/src/test/java/com/drmq/broker/MessageStoreTest.java +++ b/drmq-broker/src/test/java/com/drmq/broker/MessageStoreTest.java @@ -1,6 +1,9 @@ package com.drmq.broker; import com.drmq.broker.persistence.LogManager; +import com.drmq.protocol.DRMQProtocol.AtomicBatchTopicSlice; +import com.drmq.protocol.DRMQProtocol.ProduceBatchRequest; + import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -8,6 +11,9 @@ import java.io.IOException; import java.nio.file.Path; +import java.util.Arrays; +import java.util.Collections; +import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -250,5 +256,51 @@ void testRetentionPolicy() throws Exception { assertEquals(1, segmentsAfter.size(), "Only the active segment should remain"); } } + + @Test + void appendAtomicBatchStoresMessagesAcrossTopics() { + AtomicBatchTopicSlice slice1 = AtomicBatchTopicSlice.newBuilder() + .setTopic("topic-a") + .addEntries(ProduceBatchRequest.BatchEntry.newBuilder() + .setPayload(com.google.protobuf.ByteString.copyFromUtf8("msgA1")) + .setClientTimestamp(100L).build()) + .addEntries(ProduceBatchRequest.BatchEntry.newBuilder() + .setPayload(com.google.protobuf.ByteString.copyFromUtf8("msgA2")) + .setClientTimestamp(200L).build()) + .build(); + + AtomicBatchTopicSlice slice2 = AtomicBatchTopicSlice.newBuilder() + .setTopic("topic-b") + .addEntries(ProduceBatchRequest.BatchEntry.newBuilder() + .setPayload(com.google.protobuf.ByteString.copyFromUtf8("msgB1")) + .setClientTimestamp(300L).build()) + .build(); + + Map baseOffsets = store.appendAtomicBatch(Arrays.asList(slice1, slice2)); + + assertEquals(2, baseOffsets.size()); + assertEquals(0L, baseOffsets.get("topic-a")); + assertEquals(2L, baseOffsets.get("topic-b")); + + assertEquals(3L, store.getCurrentOffset()); + assertEquals(2L, store.getMessageCount("topic-a")); + assertEquals(1L, store.getMessageCount("topic-b")); + + var msgA1 = store.getMessage("topic-a", 0); + assertEquals("msgA1", msgA1.getPayload().toStringUtf8()); + + var msgA2 = store.getMessage("topic-a", 1); + assertEquals("msgA2", msgA2.getPayload().toStringUtf8()); + + var msgB1 = store.getMessage("topic-b", 2); + assertEquals("msgB1", msgB1.getPayload().toStringUtf8()); + } + + @Test + void appendAtomicBatchWithEmptyListReturnsEmptyMap() { + Map offsets = store.appendAtomicBatch(Collections.emptyList()); + assertTrue(offsets.isEmpty()); + assertEquals(0, store.getCurrentOffset()); + } } diff --git a/drmq-client/src/main/java/com/drmq/client/DRMQProducer.java b/drmq-client/src/main/java/com/drmq/client/DRMQProducer.java index 357161e..75cf193 100644 --- a/drmq-client/src/main/java/com/drmq/client/DRMQProducer.java +++ b/drmq-client/src/main/java/com/drmq/client/DRMQProducer.java @@ -9,6 +9,7 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadLocalRandom; @@ -21,7 +22,7 @@ public class DRMQProducer implements AutoCloseable { private static final Logger logger = LoggerFactory.getLogger(DRMQProducer.class); private static final int MAX_RETRIES = 5; private static final long RECONNECT_DELAY_MS = 500; - private static final int BATCH_SIZE_BYTES = 16384; // 16KB + private static final int BATCH_SIZE_BYTES = 1048576; // 1MB private static final long LINGER_MS = 5; private String host; @@ -36,7 +37,7 @@ public class DRMQProducer implements AutoCloseable { private volatile boolean connected = false; private volatile boolean running = true; - private static final int MAX_ACCUMULATOR_MESSAGES = 100000; + private static final int MAX_ACCUMULATOR_MESSAGES = 10000; private final LinkedBlockingQueue accumulator = new LinkedBlockingQueue<>(MAX_ACCUMULATOR_MESSAGES); private final Thread senderThread; @@ -230,7 +231,7 @@ private void sendBatchWithRetry(String topic, List batch, int re long currentBackoffMs = 100; while (true) { - if (!running || Thread.currentThread().isInterrupted()) { + if (Thread.currentThread().isInterrupted()) { lastException = new IOException("Producer stopped or interrupted during retry"); break; } @@ -319,6 +320,132 @@ private void sendBatchWithRetry(String topic, List batch, int re } } + /** + * Atomically sends messages to multiple topics in a single Raft entry. + * Either all writes are committed, or none. + * + * @param topicMessages map of topic -> payload + * @return CompletableFuture> of topic -> base offset + */ + public CompletableFuture> sendAtomic(java.util.Map topicMessages) { + CompletableFuture> future = new CompletableFuture<>(); + + if (topicMessages.size() < 2) { + future.completeExceptionally(new IllegalArgumentException("sendAtomic() requires at least 2 topics")); + return future; + } + + com.drmq.protocol.DRMQProtocol.AtomicProduceRequest.Builder reqBuilder = com.drmq.protocol.DRMQProtocol.AtomicProduceRequest.newBuilder(); + for (java.util.Map.Entry e : topicMessages.entrySet()) { + reqBuilder.addSlices(com.drmq.protocol.DRMQProtocol.AtomicBatchTopicSlice.newBuilder() + .setTopic(e.getKey()) + .addEntries(com.drmq.protocol.DRMQProtocol.ProduceBatchRequest.BatchEntry.newBuilder() + .setPayload(com.google.protobuf.ByteString.copyFrom(e.getValue())) + .setClientTimestamp(System.currentTimeMillis()) + .build()) + .build()); + } + + AtomicProduceRequest request = reqBuilder.build(); + CompletableFuture.runAsync(() -> sendAtomicWithRetry(request, future)); + return future; + } + + private void sendAtomicWithRetry(AtomicProduceRequest request, CompletableFuture> future) { + MessageEnvelope envelope = MessageEnvelope.newBuilder() + .setType(MessageType.ATOMIC_PRODUCE_REQUEST) + .setPayload(request.toByteString()) + .build(); + + IOException lastException = null; + long deliveryTimeoutMs = 120_000; + long startMs = System.currentTimeMillis(); + long currentBackoffMs = 100; + + while (true) { + if (Thread.currentThread().isInterrupted()) { + lastException = new IOException("Producer stopped or interrupted during retry"); + break; + } + if (System.currentTimeMillis() - startMs > deliveryTimeoutMs) { + break; + } + + try { + ensureConnectedWithRetry(); + + synchronized (sendLock) { + byte[] envelopeBytes = envelope.toByteArray(); + out.writeInt(envelopeBytes.length); + out.write(envelopeBytes); + out.flush(); + + int responseLength = in.readInt(); + byte[] responseBytes = new byte[responseLength]; + in.readFully(responseBytes); + + MessageEnvelope responseEnvelope = MessageEnvelope.parseFrom(responseBytes); + com.drmq.protocol.DRMQProtocol.AtomicProduceResponse response = com.drmq.protocol.DRMQProtocol.AtomicProduceResponse.parseFrom(responseEnvelope.getPayload()); + + if (response.getSuccess()) { + future.complete(response.getBaseOffsetsMap()); + return; // Success + } else { + com.drmq.protocol.DRMQProtocol.ErrorCode errorCode = response.getErrorCode(); + String errorMsg = response.getErrorMessage(); + if (errorCode == com.drmq.protocol.DRMQProtocol.ErrorCode.NOT_LEADER || + (errorMsg != null && errorMsg.startsWith("NOT_LEADER"))) { + String leaderAddr = errorMsg != null && errorMsg.startsWith("NOT_LEADER:") + ? errorMsg.substring("NOT_LEADER:".length()) + : "UNKNOWN"; + if (!leaderAddr.equals("UNKNOWN")) { + try { + redirectToLeader(leaderAddr); + continue; // Try again immediately on new leader + } catch (IOException e) { + lastException = e; + closeConnection(); + rotateToNextServer(); + } + } else { + lastException = new IOException("Leader unknown"); + closeConnection(); + rotateToNextServer(); + } + } else if (errorMsg != null && ( + errorMsg.contains("timed out") || + errorMsg.contains("Lost leadership") || + errorMsg.contains("Raft batch proposal") + )) { + lastException = new IOException("Broker cluster error: " + errorMsg); + closeConnection(); + rotateToNextServer(); + } else { + future.completeExceptionally(new IOException("Failed to send atomic batch: " + errorMsg)); + return; + } + } + } + } catch (IOException e) { + lastException = e; + closeConnection(); + rotateToNextServer(); + } + + // Exponential backoff before retrying + try { + Thread.sleep(currentBackoffMs); + } catch (InterruptedException ignored) { + Thread.currentThread().interrupt(); + } + currentBackoffMs = Math.min(2000, currentBackoffMs * 2); + } + + // Complete exceptionally if we exhausted retries + future.completeExceptionally(new IOException("Failed to send atomic batch: " + + (lastException != null ? lastException.getMessage() : "unknown error"))); + } + private void rotateToNextServer() { if (bootstrapServers.size() <= 1) return; currentServerIndex = (currentServerIndex + 1) % bootstrapServers.size(); diff --git a/drmq-client/src/main/java/com/drmq/client/commandLineExample/AtomicStressTestApp.java b/drmq-client/src/main/java/com/drmq/client/commandLineExample/AtomicStressTestApp.java new file mode 100644 index 0000000..0b16fa5 --- /dev/null +++ b/drmq-client/src/main/java/com/drmq/client/commandLineExample/AtomicStressTestApp.java @@ -0,0 +1,131 @@ +package com.drmq.client.commandLineExample; + +import com.drmq.client.DRMQProducer; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Multi-threaded Atomic Stress Test App + * Hammer the broker with cross-topic atomic batch messages. + */ +public class AtomicStressTestApp { + public static void main(String[] args) throws InterruptedException { + System.out.println("╔═══════════════════════════════════════════════════╗"); + System.out.println("║ DRMQ Atomic Cross-Topic Stress Test ║"); + System.out.println("╚═══════════════════════════════════════════════════╝\n"); + + String bootstrapServers = args.length > 0 ? args[0] : "localhost:9092,localhost:9093,localhost:9094"; + int concurrency; + int msgSize; + int numTopics; + try { + concurrency = args.length > 1 ? Integer.parseInt(args[1]) : 2; + numTopics = args.length > 2 ? Integer.parseInt(args[2]) : 3; + msgSize = args.length > 3 ? Integer.parseInt(args[3]) : 256; + if (concurrency < 1 || msgSize < 1 || numTopics < 2) { + throw new NumberFormatException("Concurrency > 0, msgSize > 0, numTopics >= 2"); + } + } catch (NumberFormatException e) { + System.err.println("Error: " + e.getMessage()); + System.err.println("Usage: AtomicStressTestApp [bootstrapServers] [concurrency] [numTopics] [msgSize]"); + System.exit(1); + return; + } + + System.out.println("Configuration:"); + System.out.println(" Brokers : " + bootstrapServers); + System.out.println(" Concurrency : " + concurrency + " threads"); + System.out.println(" Topics Count : " + numTopics + " topics per atomic batch"); + System.out.println(" Payload Size : " + msgSize + " bytes per topic\n"); + + StringBuilder sb = new StringBuilder(msgSize); + for (int i = 0; i < msgSize; i++) { + sb.append((char) ('a' + (Math.random() * 26))); + } + final String basePayload = sb.toString(); + + AtomicLong batchesSent = new AtomicLong(0); + AtomicLong errors = new AtomicLong(0); + + ExecutorService executor = Executors.newFixedThreadPool(concurrency); + long startTime = System.currentTimeMillis(); + + System.out.println("🚀 Starting atomic load test... Press Ctrl+C to stop."); + + Thread reporter = new Thread(() -> { + try { + long lastSent = 0; + while (!Thread.currentThread().isInterrupted()) { + Thread.sleep(1000); + long currentSent = batchesSent.get(); + long currentErrors = errors.get(); + System.out.printf("[Metrics] %d atomic batches/sec | Total Batches: %d | Total Errors: %d\n", + (currentSent - lastSent), currentSent, currentErrors); + lastSent = currentSent; + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + reporter.start(); + + for (int i = 0; i < concurrency; i++) { + final int threadId = i; + executor.submit(() -> { + try (DRMQProducer producer = new DRMQProducer(bootstrapServers)) { + producer.connect(); + long count = 0; + while (!Thread.currentThread().isInterrupted()) { + java.util.List>> futures = new java.util.ArrayList<>(100); + + // Pipeline 100 atomic requests asynchronously + for (int k = 0; k < 100; k++) { + Map atomicBatch = new HashMap<>(); + String payload = "T" + threadId + "_" + count + "_" + basePayload; + byte[] payloadBytes = payload.getBytes(java.nio.charset.StandardCharsets.UTF_8); + + for (int t = 0; t < numTopics; t++) { + atomicBatch.put("atomic-topic-" + t, payloadBytes); + } + + futures.add(producer.sendAtomic(atomicBatch)); + count++; + } + + // Wait for the pipeline to drain and record results + for (var f : futures) { + try { + f.join(); + batchesSent.incrementAndGet(); + } catch (Exception e) { + errors.incrementAndGet(); + } + } + } + } catch (Exception e) { + errors.incrementAndGet(); + System.err.println("[P" + threadId + "] Producer failed: " + e.getMessage()); + } + }); + } + + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + System.out.println("\n🛑 Shutting down atomic stress test..."); + executor.shutdownNow(); + reporter.interrupt(); + long totalTimeMs = System.currentTimeMillis() - startTime; + System.out.printf("\n📊 Final Results:\n"); + System.out.printf(" Total Batches Sent : %d\n", batchesSent.get()); + System.out.printf(" Total Errors : %d\n", errors.get()); + System.out.printf(" Avg Rate : %.2f batches/sec\n", (batchesSent.get() * 1000.0) / Math.max(1, totalTimeMs)); + })); + + executor.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS); + } +} diff --git a/drmq-client/src/main/java/com/drmq/client/commandLineExample/ProducerApp.java b/drmq-client/src/main/java/com/drmq/client/commandLineExample/ProducerApp.java index 311a978..ee15130 100644 --- a/drmq-client/src/main/java/com/drmq/client/commandLineExample/ProducerApp.java +++ b/drmq-client/src/main/java/com/drmq/client/commandLineExample/ProducerApp.java @@ -28,12 +28,14 @@ public static void main(String[] args) { producer.connect(); System.out.println("✓ Connected to broker (servers: " + bootstrapServers + ")\n"); System.out.println("Commands:"); - System.out.println(" send - Send a message to a topic"); - System.out.println(" exit - Quit the application"); - System.out.println(" help - Show this help\n"); + System.out.println(" send - Send a message to a topic"); + System.out.println(" atomic = | = ... - Atomically send messages to multiple topics"); + System.out.println(" exit - Quit the application"); + System.out.println(" help - Show this help\n"); System.out.println("Examples:"); System.out.println(" send orders Book Order #101"); System.out.println(" send payments Payment of ₦25.50"); + System.out.println(" atomic orders=Order #102 | alerts=New order received"); System.out.println(" send alerts System is ONLINE\n"); System.out.println("─────────────────────────────────────────\n"); @@ -57,9 +59,10 @@ public static void main(String[] args) { case "help" -> { System.out.println("\nCommands:"); - System.out.println(" send - Send a message"); - System.out.println(" exit - Quit"); - System.out.println(" help - Show help\n"); + System.out.println(" send - Send a message"); + System.out.println(" atomic = | = ... - Atomically send messages to multiple topics"); + System.out.println(" exit - Quit"); + System.out.println(" help - Show help\n"); } case "send" -> { @@ -85,6 +88,36 @@ public static void main(String[] args) { } } + case "atomic" -> { + String payloadStr = input.substring(command.length()).trim(); + if (payloadStr.isEmpty() || !payloadStr.contains("=")) { + System.out.println("❌ Usage: atomic = | = ..."); + System.out.println(" Example: atomic orders=Book #102 | alerts=New order\n"); + continue; + } + + String[] pairs = payloadStr.split("\\|"); + java.util.Map batch = new java.util.LinkedHashMap<>(); + for (String pair : pairs) { + String[] kv = pair.split("=", 2); + if (kv.length == 2) { + batch.put(kv[0].trim(), kv[1].trim().getBytes(java.nio.charset.StandardCharsets.UTF_8)); + } + } + + if (batch.size() < 2) { + System.out.println("❌ atomic requires at least 2 topics."); + continue; + } + + try { + java.util.Map result = producer.sendAtomic(batch).join(); + System.out.printf("✓ Atomic commit successful! Base offsets: %s\n\n", result); + } catch (Exception e) { + System.out.printf("❌ Error: %s\n\n", e.getMessage()); + } + } + default -> { System.out.printf("❌ Unknown command: %s\n", command); System.out.println(" Type 'help' for available commands\n"); diff --git a/drmq-dashboard/src/App.tsx b/drmq-dashboard/src/App.tsx index fa43a3c..a7b0c28 100644 --- a/drmq-dashboard/src/App.tsx +++ b/drmq-dashboard/src/App.tsx @@ -1,8 +1,10 @@ import { BrowserRouter, Routes, Route, Link, useLocation } from 'react-router-dom'; -import { LayoutDashboard, Book, Server, ChevronRight } from 'lucide-react'; +import { LayoutDashboard, Book, Server, ChevronRight, Layers, Users } from 'lucide-react'; import { useState, useCallback } from 'react'; import Dashboard from './pages/Dashboard'; import { useClusterTelemetry } from './useClusterTelemetry'; +import Topics from './pages/Topics'; +import Consumers from './pages/Consumers'; function Sidebar({ telemetryState }: { telemetryState: any }) { const location = useLocation(); @@ -11,6 +13,8 @@ function Sidebar({ telemetryState }: { telemetryState: any }) { const navItems = [ { to: '/', icon: LayoutDashboard, label: 'Telemetry' }, + { to: '/topics', icon: Layers, label: 'Topics' }, + { to: '/consumers', icon: Users, label: 'Consumers' }, { to: '/docs', icon: Book, label: 'Documentation' }, ]; @@ -130,6 +134,8 @@ export default function App() { style={{ minHeight: '100vh' }} /> } /> + } /> + } /> diff --git a/drmq-dashboard/src/pages/Consumers.tsx b/drmq-dashboard/src/pages/Consumers.tsx new file mode 100644 index 0000000..b044c78 --- /dev/null +++ b/drmq-dashboard/src/pages/Consumers.tsx @@ -0,0 +1,134 @@ +import React, { useState, useEffect } from 'react'; +import { Panel } from '../components/DashboardWidgets'; +import { RefreshCw, Users, AlertCircle, CheckCircle2 } from 'lucide-react'; +import { DitherButton } from '../components/dither-kit'; + +interface ConsumerTopic { + topic: string; + committedOffset: number; + lag: number; + activeMembers: number; +} + +interface ConsumerGroup { + groupId: string; + topics: ConsumerTopic[]; +} + +export default function Consumers() { + const [groups, setGroups] = useState([]); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + + const fetchConsumers = async () => { + setLoading(true); + setError(null); + try { + const res = await fetch('http://localhost:9392/api/consumers'); + if (!res.ok) throw new Error(`HTTP error! status: ${res.status}`); + const data = await res.json(); + setGroups(data); + } catch (err: any) { + setError(err.message || 'Failed to fetch consumers'); + } finally { + setLoading(false); + } + }; + + useEffect(() => { + fetchConsumers(); + const interval = setInterval(fetchConsumers, 5000); + return () => clearInterval(interval); + }, []); + + const getLagColor = (lag: number) => { + if (lag === 0) return 'text-emerald-400'; + if (lag < 100) return 'text-amber-400'; + return 'text-red-400'; + }; + + return ( +
+
+
+

+ + Consumer Groups & Lag +

+

Monitor message processing lag across all consumer groups

+
+ + + Refresh + +
+ + + {error && ( +
+ {error} +
+ )} + +
+ + + + + + + + + + + + {groups.length === 0 && !loading && !error && ( + + + + )} + {groups.flatMap((group) => + group.topics.map((topic, i) => ( + + {i === 0 ? ( + + ) : null} + + + + + + )) + )} + +
Group IDTopicOffsetConsumer LagMembers
+ No consumer groups registered. +
+ + {group.groupId} + + + {topic.topic} + + {topic.committedOffset.toLocaleString()} + +
+ {topic.lag === 0 ? ( + + ) : ( + 100 ? 'text-red-500/70' : 'text-amber-500/70'}`} /> + )} + + {topic.lag.toLocaleString()} + +
+ 		+ + {topic.activeMembers} + +
+
+ +
+ ); +} diff --git a/drmq-dashboard/src/pages/Dashboard.tsx b/drmq-dashboard/src/pages/Dashboard.tsx index 81ba2a1..1a98175 100644 --- a/drmq-dashboard/src/pages/Dashboard.tsx +++ b/drmq-dashboard/src/pages/Dashboard.tsx @@ -78,6 +78,7 @@ export default function Dashboard({ }) { /* ── Hooks (must come before any early returns) ───────────────── */ + const [historyWindowSeconds, setHistoryWindowSeconds] = useState(30); const prevCommit = useRef(0); const [latencyHistory, setLatencyHistory] = useState([]); @@ -92,7 +93,7 @@ export default function Dashboard({ const { metrics, latencies } = telemetryState; setLatencyHistory(prev => { const next = [...prev, { - t: new Date().toLocaleTimeString([], { second: '2-digit', minute: '2-digit' }), + t: new Date().toLocaleTimeString([], { hour: '2-digit', minute: '2-digit', second: '2-digit' }), produce: metrics.produceLatencyMs, consume: metrics.consumeLatencyMs, rpc: latencies.raftRpcMs @@ -102,8 +103,8 @@ export default function Dashboard({ } }, [telemetryState?.metrics?.produceLatencyMs, telemetryState?.metrics?.consumeLatencyMs, telemetryState?.latencies?.raftRpcMs, paused, telemetryState]); - /* ── Error state ───────────────────────────────────────────────── */ - if (telemetryError) { + /* ── Error state (only block rendering when we have NO data at all) ── */ + if (telemetryError && !telemetryState) { return (
@@ -138,19 +139,30 @@ export default function Dashboard({ const leaderName = nodes.find((n: any) => n.status === 'LEADER')?.name ?? 'No Leader'; /* ── Throughput chart data ─────────────────────────────────────── */ - const totalHist = metrics.throughputHistory ?? []; - const consumeHist = metrics.consumeHistory ?? []; - const errorHist = metrics.errorHistory ?? []; + const totalHistFull = metrics.throughputHistory ?? []; + const produceHistFull = metrics.produceHistory ?? []; + const consumeHistFull = metrics.consumeHistory ?? []; + const errorHistFull = metrics.errorHistory ?? []; + + const totalHist = totalHistFull.slice(-historyWindowSeconds); + const produceHist = produceHistFull.slice(-historyWindowSeconds); + const consumeHist = consumeHistFull.slice(-historyWindowSeconds); + const errorHist = errorHistFull.slice(-historyWindowSeconds); const throughputData = totalHist.map((v: number, i: number) => { const consumeVal = consumeHist[i] ?? 0; - const produceVal = Math.max(0, v - consumeVal); + let produceVal = produceHist[i]; + if (produceVal === undefined) { + produceVal = Math.max(0, v - consumeVal); + } return { t: `-${totalHist.length - i}s`, - produce: parseFloat(((produceVal / 100) * 50).toFixed(2)), - consume: parseFloat(((consumeVal / 100) * 50).toFixed(2)), + produce: parseFloat(produceVal.toFixed(2)), + consume: parseFloat(consumeVal.toFixed(2)), }; }); + + const formatMBps = (val: number) => val > 0 && val < 0.005 ? '< 0.01' : val.toFixed(2); /* ── Radar data for selected/leader node ──────────────────────── */ const leaderNode = nodes.find((n: any) => n.status === 'LEADER'); @@ -219,11 +231,11 @@ export default function Dashboard({
} /> } /> } className="shrink-0">
- {metrics.totalThroughputMB.toFixed(2)} MB/s + {formatMBps(metrics.totalThroughputMB)} MB/s
- ↑ {metrics.produceThroughputMB.toFixed(2)} · ↓ {metrics.consumeThroughputMB.toFixed(2)} + ↑ {formatMBps(metrics.produceThroughputMB)} · ↓ {formatMBps(metrics.consumeThroughputMB)}
+
+ 30s + setHistoryWindowSeconds(Number(e.target.value))} + className="flex-1 h-1.5 bg-white/10 rounded-full appearance-none [&::-webkit-slider-thumb]:appearance-none [&::-webkit-slider-thumb]:w-3 [&::-webkit-slider-thumb]:h-3 [&::-webkit-slider-thumb]:bg-cyan-400 [&::-webkit-slider-thumb]:rounded-full cursor-pointer" + /> + 5min +
+
+ VIEWING LAST {historyWindowSeconds >= 60 ? `${historyWindowSeconds / 60} MIN` : `${historyWindowSeconds} SEC`} +
{/* ── Node Health Radar ──────────────────────────────── */} @@ -345,7 +373,7 @@ export default function Dashboard({ margins={{ top: 24, right: 8, bottom: 20, left: 28 }} animationDuration={400} > - + diff --git a/drmq-dashboard/src/pages/Topics.tsx b/drmq-dashboard/src/pages/Topics.tsx new file mode 100644 index 0000000..a9d628b --- /dev/null +++ b/drmq-dashboard/src/pages/Topics.tsx @@ -0,0 +1,297 @@ +import React, { useState, useEffect } from 'react'; +import { Panel } from '../components/DashboardWidgets'; +import { RefreshCw, Database } from 'lucide-react'; +import { DitherButton } from '../components/dither-kit'; +import { motion, AnimatePresence } from 'framer-motion'; + +interface TopicData { + name: string; + messageCount: number; +} + +export default function Topics() { + const [topics, setTopics] = useState([]); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + + const [inspectorTopic, setInspectorTopic] = useState(null); + const [inspectOffset, setInspectOffset] = useState(0); + const [inspectTimestamp, setInspectTimestamp] = useState(''); + const [inspectLimit, setInspectLimit] = useState(10); + const [messages, setMessages] = useState([]); + const [inspectLoading, setInspectLoading] = useState(false); + const [inspectError, setInspectError] = useState(null); + + const fetchMessages = async () => { + if (!inspectorTopic) return; + setInspectLoading(true); + setInspectError(null); + try { + const offsetParam = inspectOffset === '' ? 0 : inspectOffset; + const limitParam = inspectLimit === '' ? 1 : inspectLimit; + + let url = `http://localhost:9392/api/messages?topic=${inspectorTopic}&limit=${limitParam}`; + if (inspectTimestamp) { + const ts = new Date(inspectTimestamp).getTime(); + if (!isNaN(ts)) { + url += `×tamp=${ts}`; + } else { + url += `&offset=${offsetParam}`; + } + } else { + url += `&offset=${offsetParam}`; + } + + const res = await fetch(url); + if (!res.ok) throw new Error(`HTTP error! status: ${res.status}`); + const data = await res.json(); + if (data.error) throw new Error(data.error); + setMessages(data); + + // Snap the input offset to the actual returned offset to show compaction jumps + if (data && data.length > 0) { + setInspectOffset(data[0].offset); + } + } catch (err: any) { + setInspectError(err.message || 'Failed to fetch messages'); + } finally { + setInspectLoading(false); + } + }; + + // Automatically fetch messages when inspector opens + useEffect(() => { + if (inspectorTopic) { + setInspectOffset(0); // Reset to 0 when opening new topic + fetchMessages(); + } else { + setMessages([]); + } + }, [inspectorTopic]); + + const fetchTopics = async () => { + setLoading(true); + setError(null); + try { + // For now, default to the local admin port of Broker-1 (9392) + const res = await fetch('http://localhost:9392/api/topics'); + if (!res.ok) throw new Error(`HTTP error! status: ${res.status}`); + const data = await res.json(); + setTopics(data); + } catch (err: any) { + setError(err.message || 'Failed to fetch topics'); + } finally { + setLoading(false); + } + }; + + useEffect(() => { + fetchTopics(); + const interval = setInterval(fetchTopics, 5000); + return () => clearInterval(interval); + }, []); + + return ( +
+
+
+

+ + Topic Explorer +

+

Live visibility into DRMQ active topics

+
+ +
+ + + {error && ( +
+ {error} +
+ )} + +
+ + + + + + + + + + + {topics.length === 0 && !loading && !error && ( + + + + )} + {topics.map(topic => ( + + + + + + + ))} + +
Topic NameMessage CountStatusActions
+ No active topics found. +
+
+
+ {topic.name} +
+
+ {topic.messageCount.toLocaleString()} + + + Active + + + +
+
+ + + {/* Live Message Inspector Drawer */} + + {inspectorTopic && ( + +
+
+

+ + Inspect Topic +

+
{inspectorTopic}
+
+ +
+ +
+
+
+ + setInspectOffset(e.target.value === '' ? '' : parseInt(e.target.value))} + className="bg-black/40 border border-white/10 rounded px-3 py-2 text-sm text-white font-mono focus:outline-none focus:border-cyan-500/50 transition-colors disabled:opacity-30" + placeholder="e.g. 100" + /> +
+
+ + setInspectTimestamp(e.target.value)} + className="bg-black/40 border border-white/10 rounded px-3 py-2 text-sm text-white font-mono focus:outline-none focus:border-cyan-500/50 transition-colors [&::-webkit-calendar-picker-indicator]:invert" + /> +
+
+
+
+ + setInspectLimit(e.target.value === '' ? '' : parseInt(e.target.value))} + className="bg-black/40 border border-white/10 rounded px-3 py-2 text-sm text-white font-mono focus:outline-none focus:border-cyan-500/50 transition-colors" + /> +
+
+ +
+
+
+ +
+ {inspectError && ( +
+ ⚠️ +
{inspectError}
+
+ )} + + {messages.length === 0 && !inspectLoading && !inspectError && ( +
+ +
NO MESSAGES FOUND
+
+ )} + +
+ {messages.map((msg, idx) => { + // Attempt to prettify JSON payloads + let formattedPayload = msg.payload; + try { + const parsed = JSON.parse(msg.payload); + formattedPayload = JSON.stringify(parsed, null, 2); + } catch (e) { /* ignore */ } + + return ( +
+
+
+ + OFFSET {msg.offset} + +
+ {new Date(msg.storedAt).toISOString()} +
+ + {msg.key && ( +
+ KEY + {msg.key} +
+ )} + +
+ 
+                          {formattedPayload}
+
+
+
+ ); + })} +
+
+ + )} + +
+ ); +} diff --git a/drmq-dashboard/src/services/telemetry/WebSocketTelemetryProvider.ts b/drmq-dashboard/src/services/telemetry/WebSocketTelemetryProvider.ts index 269a728..ed1c2f2 100644 --- a/drmq-dashboard/src/services/telemetry/WebSocketTelemetryProvider.ts +++ b/drmq-dashboard/src/services/telemetry/WebSocketTelemetryProvider.ts @@ -43,11 +43,29 @@ export class WebSocketTelemetryProvider implements TelemetryProvider { for (const url of this.urls) { this.openSocket(url); } - setTimeout(() => { - if (!this.stopped && this.onErrorCallback && Array.from(this.sockets.values()).every(s => !s || s.readyState !== WebSocket.OPEN)) { + + // Retry-aware connection check: don't show timeout error if sockets are + // still actively handshaking (CONNECTING). Only give up after 15s total. + const startTime = Date.now(); + const checkConnection = () => { + if (this.stopped) return; + const sockets = Array.from(this.sockets.values()); + const anyOpen = sockets.some(s => s && s.readyState === WebSocket.OPEN); + if (anyOpen) return; // At least one connected — all good + + const anyConnecting = sockets.some(s => s && s.readyState === WebSocket.CONNECTING); + if (anyConnecting && Date.now() - startTime < 15000) { + // Still handshaking — check again in 2s instead of giving up + setTimeout(checkConnection, 2000); + return; + } + + // All sockets are closed/null or we've waited 15s — show error + if (this.onErrorCallback) { this.onErrorCallback('Connection timeout. Check if brokers are running.'); } - }, 5000); + }; + setTimeout(checkConnection, 5000); } disconnect(): void { @@ -62,6 +80,14 @@ export class WebSocketTelemetryProvider implements TelemetryProvider { private openSocket(url: string): void { if (this.stopped) return; + + // Cancel any pending reconnect for this URL to prevent duplicate timers + const existingTimer = this.reconnectTimers.get(url); + if (existingTimer) { + clearTimeout(existingTimer); + this.reconnectTimers.delete(url); + } + console.log(`[DRMQ] Connecting to ${url}…`); try { const ws = new WebSocket(url); @@ -84,12 +110,25 @@ export class WebSocketTelemetryProvider implements TelemetryProvider { ws.onclose = () => { console.log(`[DRMQ] Disconnected: ${url}`); - this.sockets.set(url, null); - this.latestFrames.delete(url); - if (!this.stopped && this.onErrorCallback && Array.from(this.sockets.values()).every(s => !s || s.readyState !== WebSocket.OPEN)) { - this.onErrorCallback('Cluster offline. Retrying connection...'); + + // Guard: only clear if THIS socket is still the current one for this URL. + // A reconnect may have already replaced us with a newer WebSocket instance. + if (this.sockets.get(url) === ws) { + this.sockets.set(url, null); + this.latestFrames.delete(url); + + // Immediately re-merge with surviving nodes so the dashboard doesn't + // freeze on the dead leader's last telemetry snapshot. + if (this.onDataCallback && this.latestFrames.size > 0) { + this.onDataCallback(this.merge()); + } + + if (!this.stopped && this.onErrorCallback && Array.from(this.sockets.values()).every(s => !s || s.readyState !== WebSocket.OPEN)) { + this.onErrorCallback('Cluster offline. Retrying connection...'); + } + this.scheduleReconnect(url); } - this.scheduleReconnect(url); + // else: a newer socket already took over — do nothing }; } catch (e) { console.error(`[DRMQ] Cannot create WebSocket for ${url}`, e); diff --git a/drmq-integration-tests/src/test/java/com/drmq/integration/AtomicLatencyBenchmarkTest.java b/drmq-integration-tests/src/test/java/com/drmq/integration/AtomicLatencyBenchmarkTest.java new file mode 100644 index 0000000..9cd1285 --- /dev/null +++ b/drmq-integration-tests/src/test/java/com/drmq/integration/AtomicLatencyBenchmarkTest.java @@ -0,0 +1,148 @@ +package com.drmq.integration; + +import com.drmq.broker.BrokerConfig; +import com.drmq.broker.BrokerConfig.PeerAddress; +import com.drmq.broker.BrokerServer; +import com.drmq.client.DRMQProducer; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Path; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Benchmark for evaluating Cross-Topic Atomicity performance. + * This test spins up a 3-node cluster and compares the latency of writing to N topics + * atomically (1 Raft entry) vs sequentially (N Raft entries). + */ +public class AtomicLatencyBenchmarkTest { + + @TempDir + Path tempDir; + + private static final int PORT_1 = 19300; + private static final int PORT_2 = 19301; + private static final int PORT_3 = 19302; + + private BrokerServer broker1; + private BrokerServer broker2; + private BrokerServer broker3; + + private BrokerConfig clusterConfig(String nodeId, int port, String dataDirName) { + List peers = switch (nodeId) { + case "b1" -> List.of(new PeerAddress("b2", "localhost", PORT_2), new PeerAddress("b3", "localhost", PORT_3)); + case "b2" -> List.of(new PeerAddress("b1", "localhost", PORT_1), new PeerAddress("b3", "localhost", PORT_3)); + case "b3" -> List.of(new PeerAddress("b1", "localhost", PORT_1), new PeerAddress("b2", "localhost", PORT_2)); + default -> throw new IllegalArgumentException("Unknown nodeId: " + nodeId); + }; + return new BrokerConfig(nodeId, port, tempDir.resolve(dataDirName).toString(), peers); + } + + @BeforeEach + void setUp() throws Exception { + broker1 = new BrokerServer(clusterConfig("b1", PORT_1, "data-1")); + broker2 = new BrokerServer(clusterConfig("b2", PORT_2, "data-2")); + broker3 = new BrokerServer(clusterConfig("b3", PORT_3, "data-3")); + + broker1.startAsync(); + broker2.startAsync(); + broker3.startAsync(); + + // Wait for leader election + waitForLeader(3000); + } + + @AfterEach + void tearDown() { + if (broker1 != null) broker1.shutdown(); + if (broker2 != null) broker2.shutdown(); + if (broker3 != null) broker3.shutdown(); + } + + private void waitForLeader(long timeoutMs) throws InterruptedException { + long deadline = System.currentTimeMillis() + timeoutMs; + while (System.currentTimeMillis() < deadline) { + int leaderCount = 0; + if (broker1.getRaftNode() != null && broker1.getRaftNode().isLeader()) leaderCount++; + if (broker2.getRaftNode() != null && broker2.getRaftNode().isLeader()) leaderCount++; + if (broker3.getRaftNode() != null && broker3.getRaftNode().isLeader()) leaderCount++; + if (leaderCount == 1) return; + Thread.sleep(100); + } + } + + private int getLeaderPort() { + if (broker1.getRaftNode() != null && broker1.getRaftNode().isLeader()) return PORT_1; + if (broker2.getRaftNode() != null && broker2.getRaftNode().isLeader()) return PORT_2; + if (broker3.getRaftNode() != null && broker3.getRaftNode().isLeader()) return PORT_3; + throw new IllegalStateException("No leader found"); + } + + @Test + public void benchmarkAtomicVsSequentialWrites() throws Exception { + int leaderPort = getLeaderPort(); + int iterations = 1000; + int warmup = 200; + byte[] payload = new byte[256]; + + System.out.println("================================================="); + System.out.println("BENCHMARK: Cross-Topic Atomicity vs Sequential"); + System.out.println("================================================="); + + String bootstrapServers = "localhost:" + PORT_1 + ",localhost:" + PORT_2 + ",localhost:" + PORT_3; + try (DRMQProducer producer = new DRMQProducer(bootstrapServers)) { + producer.connect(); + + // Run for 2 topics, then 4 topics, then 8 topics + int[] topicCounts = {2, 4, 8}; + + for (int numTopics : topicCounts) { + System.out.println("\nTesting with " + numTopics + " topics:"); + + Map atomicBatch = new LinkedHashMap<>(); + for (int t = 0; t < numTopics; t++) atomicBatch.put("atomic-topic-" + t, payload); + + // --- ATOMIC BENCHMARK --- + // Warmup + for (int i = 0; i < warmup; i++) producer.sendAtomic(atomicBatch).join(); + + long atomicTotalTimeNs = 0; + for (int i = 0; i < iterations; i++) { + long start = System.nanoTime(); + producer.sendAtomic(atomicBatch).join(); + atomicTotalTimeNs += (System.nanoTime() - start); + } + double atomicAvgMs = (atomicTotalTimeNs / (double) iterations) / 1_000_000.0; + double atomicTps = iterations / (atomicTotalTimeNs / 1_000_000_000.0); + + // --- SEQUENTIAL BENCHMARK --- + // Warmup + for (int i = 0; i < warmup; i++) { + for (int t = 0; t < numTopics; t++) { + producer.send("seq-topic-" + t, payload).join(); + } + } + + long seqTotalTimeNs = 0; + for (int i = 0; i < iterations; i++) { + long start = System.nanoTime(); + for (int t = 0; t < numTopics; t++) { + producer.send("seq-topic-" + t, payload).join(); + } + seqTotalTimeNs += (System.nanoTime() - start); + } + double seqAvgMs = (seqTotalTimeNs / (double) iterations) / 1_000_000.0; + double seqTps = iterations / (seqTotalTimeNs / 1_000_000_000.0); + + System.out.printf("Atomic Write (1 Raft roundtrip) : Avg %.2f ms/op | %.2f ops/sec%n", atomicAvgMs, atomicTps); + System.out.printf("Sequential Write (%d Raft trips) : Avg %.2f ms/op | %.2f ops/sec%n", numTopics, seqAvgMs, seqTps); + System.out.printf("SPEEDUP : %.2fx faster%n", seqAvgMs / atomicAvgMs); + } + } + System.out.println("================================================="); + } +} diff --git a/drmq-protocol/src/main/proto/messages.proto b/drmq-protocol/src/main/proto/messages.proto index 82713d0..83c78e9 100644 --- a/drmq-protocol/src/main/proto/messages.proto +++ b/drmq-protocol/src/main/proto/messages.proto @@ -6,23 +6,19 @@ option java_package = "com.drmq.protocol"; option java_outer_classname = "DRMQProtocol"; option java_multiple_files = false; -// Request to produce a message to a topic message ProduceRequest { string topic = 1; bytes payload = 2; - optional string key = 3; // Optional message key for partitioning (future use) - int64 timestamp = 4; // Client-side timestamp + optional string key = 3; + int64 timestamp = 4; } -// Response after producing a message message ProduceResponse { bool success = 1; - int64 offset = 2; // Assigned offset if successful - string error_message = 3; // Error details if failed + int64 offset = 2; + string error_message = 3; ErrorCode error_code = 4; } - -// Request to produce a batch of messages to a topic message ProduceBatchRequest { string topic = 1; @@ -35,179 +31,172 @@ message ProduceBatchRequest { repeated BatchEntry entries = 2; } -// Response after producing a batch of messages message ProduceBatchResponse { bool success = 1; - int64 base_offset = 2; // The offset assigned to the FIRST message in the batch - int32 count = 3; // Number of messages successfully written - string error_message = 4; // Error details if failed + int64 base_offset = 2; + int32 count = 3; + string error_message = 4; ErrorCode error_code = 5; } -// Internal message representation stored in the broker +message AtomicBatchTopicSlice { + string topic = 1; + repeated ProduceBatchRequest.BatchEntry entries = 2; +} +message AtomicBatchRequest { + repeated AtomicBatchTopicSlice slices = 1; +} +message AtomicProduceRequest { + repeated AtomicBatchTopicSlice slices = 1; +} +message AtomicProduceResponse { + bool success = 1; + map base_offsets = 2; + string error_message = 3; + ErrorCode error_code = 4; +} + message StoredMessage { int64 offset = 1; string topic = 2; bytes payload = 3; optional string key = 4; - int64 timestamp = 5; // Original client timestamp - int64 stored_at = 6; // Broker storage timestamp + int64 timestamp = 5; + int64 stored_at = 6; } -// Request to consume messages from a topic message ConsumeRequest { string topic = 1; - int64 from_offset = 2; // Starting offset (inclusive) — ignored when consumer_group is set - int32 max_messages = 3; // Maximum number of messages to fetch - int64 timeout_ms = 4; // Long-poll timeout (0 = return immediately) - optional string consumer_group = 5; // If set, broker manages offset via group coordination - optional string consumer_id = 6; // Unique ID for this consumer instance within the group + int64 from_offset = 2; + int32 max_messages = 3; + int64 timeout_ms = 4; + optional string consumer_group = 5; + optional string consumer_id = 6; } -// Response containing consumed messages message ConsumeResponse { bool success = 1; - repeated StoredMessage messages = 2; // List of messages - string error_message = 3; // Error details if failed + repeated StoredMessage messages = 2; + string error_message = 3; } -// Request to commit a consumer group's offset for a topic message CommitOffsetRequest { string consumer_group = 1; string topic = 2; - int64 offset = 3; // The next offset to read (last processed + 1) - optional string consumer_id = 4; // Consumer instance ID (for group coordination) + int64 offset = 3; + optional string consumer_id = 4; } -// Response to a commit offset request message CommitOffsetResponse { bool success = 1; string error_message = 2; } - -// Request to fetch a consumer group's committed offset for a topic message FetchOffsetRequest { string consumer_group = 1; string topic = 2; } -// Response with the committed offset message FetchOffsetResponse { bool success = 1; - int64 offset = 2; // The committed offset, or -1 if none exists + int64 offset = 2; string error_message = 3; } -// Request to find an offset by a given timestamp message SearchOffsetByTimeRequest { string topic = 1; - int64 timestamp = 2; // The target timestamp + int64 timestamp = 2; } -// Response with the found offset message SearchOffsetByTimeResponse { bool success = 1; - int64 offset = 2; // The found offset, or -1 if none + int64 offset = 2; string error_message = 3; } -// --- Dead-Letter Queue (DLQ) support --- - -// Request to negatively acknowledge (NACK) a message, triggering redelivery or DLQ routing message NackRequest { string consumer_group = 1; string topic = 2; - int64 offset = 3; // The offset of the message being rejected - optional string consumer_id = 4; // Consumer instance ID (for group coordination) + int64 offset = 3; + optional string consumer_id = 4; } -// Response to a NACK request message NackResponse { bool success = 1; - bool routed_to_dlq = 2; // True if the message was sent to the DLQ instead of being requeued + bool routed_to_dlq = 2; string error_message = 3; } -// Envelope for TCP framing - wraps all messages with type info message MessageEnvelope { MessageType type = 1; - bytes payload = 2; // Serialized inner message + bytes payload = 2; } -// --- Raft command types (for multiplexing different operations on the Raft log) --- enum RaftCommandType { - MESSAGE = 0; // Normal message append (default) - OFFSET_COMMIT = 1; // Consumer offset commit (replicated for durability) - BATCH_MESSAGE = 2; // Batch message append (multiple messages in a single Raft entry) + MESSAGE = 0; + OFFSET_COMMIT = 1; + BATCH_MESSAGE = 2; + ATOMIC_BATCH = 3; } -// --- Raft log entry (wraps a user message for Raft replication) --- message RaftEntry { - int64 term = 1; // Term in which the entry was created - int64 index = 2; // Position in the Raft log (1-indexed) - string topic = 3; // Target topic for the message - bytes payload = 4; // Message payload - optional string key = 5; // Optional message key - int64 timestamp = 6; // Client-side timestamp - RaftCommandType command_type = 7; // Type of command (default: MESSAGE) - optional string consumer_group = 8; // Consumer group (for OFFSET_COMMIT) - optional int64 offset_value = 9; // Offset value (for OFFSET_COMMIT) -} - -// --- Raft leader election --- + int64 term = 1; + int64 index = 2; + string topic = 3; + bytes payload = 4; + optional string key = 5; + int64 timestamp = 6; + RaftCommandType command_type = 7; + optional string consumer_group = 8; + optional int64 offset_value = 9; +} + message RequestVoteRequest { - int64 term = 1; // Candidate's term - string candidate_id = 2; // Candidate requesting vote - int64 last_log_index = 3; // Index of candidate's last log entry - int64 last_log_term = 4; // Term of candidate's last log entry + int64 term = 1; + string candidate_id = 2; + int64 last_log_index = 3; + int64 last_log_term = 4; } message RequestVoteResponse { - int64 term = 1; // Current term, for candidate to update itself - bool vote_granted = 2; // True = vote granted + int64 term = 1; + bool vote_granted = 2; } -// --- Raft Pre-Vote (§9.6 disruption prevention) --- -// Sent before a real election to check if the candidate COULD win -// without incrementing the term. Prevents restarting nodes from -// disrupting a healthy leader. message PreVoteRequest { - int64 term = 1; // The term the candidate WOULD use (currentTerm + 1) - string candidate_id = 2; // Candidate requesting pre-vote - int64 last_log_index = 3; // Index of candidate's last log entry - int64 last_log_term = 4; // Term of candidate's last log entry + int64 term = 1; + string candidate_id = 2; + int64 last_log_index = 3; + int64 last_log_term = 4; } message PreVoteResponse { - int64 term = 1; // Current term of the responder - bool vote_granted = 2; // True = would vote for this candidate + int64 term = 1; + bool vote_granted = 2; } -// --- Raft log replication (also used as heartbeat when entries is empty) --- message AppendEntriesRequest { - int64 term = 1; // Leader's term - string leader_id = 2; // So followers know who the leader is - int64 prev_log_index = 3; // Index of log entry immediately preceding new ones - int64 prev_log_term = 4; // Term of prevLogIndex entry - repeated RaftEntry entries = 5; // Log entries to store (empty for heartbeat) - int64 leader_commit = 6; // Leader's commitIndex + int64 term = 1; + string leader_id = 2; + int64 prev_log_index = 3; + int64 prev_log_term = 4; + repeated RaftEntry entries = 5; + int64 leader_commit = 6; } message AppendEntriesResponse { - int64 term = 1; // Current term, for leader to update itself - bool success = 2; // True if follower contained entry matching prevLogIndex/prevLogTerm - int64 match_index = 3; // Follower's last replicated index (for leader's matchIndex tracking) + int64 term = 1; + bool success = 2; + int64 match_index = 3; } -// --- Raft InstallSnapshot (for followers that fell too far behind) --- message InstallSnapshotRequest { int64 term = 1; string leader_id = 2; int64 last_included_index = 3; int64 last_included_term = 4; - int64 offset = 5; // Byte offset of this chunk in the file - bytes data = 6; // Raw zip chunk payload - bool done = 7; // True if this is the final chunk + int64 offset = 5; + bytes data = 6; + bool done = 7; } message InstallSnapshotResponse { @@ -239,6 +228,8 @@ enum MessageType { PRODUCE_BATCH_RESPONSE = 21; SEARCH_OFFSET_BY_TIME_REQUEST = 22; SEARCH_OFFSET_BY_TIME_RESPONSE = 23; + ATOMIC_PRODUCE_REQUEST = 24; + ATOMIC_PRODUCE_RESPONSE = 25; } // --- Standard Error Codes --- diff --git a/drmq-python-client/drmq_client.py b/drmq-python-client/drmq_client.py index baf0460..7281d77 100644 --- a/drmq-python-client/drmq_client.py +++ b/drmq-python-client/drmq_client.py @@ -271,6 +271,87 @@ def _send_batch_with_retry(self, topic: str, batch: List[PendingMessage]): for pm in batch: pm.future.set_exception(Exception("Failed to send batch: timeout exhausted")) + def send_atomic(self, topic_messages: dict) -> concurrent.futures.Future: + """ + Atomically sends messages to multiple topics in a single Raft entry. + topic_messages: a dict mapping topic name to payload bytes. + """ + future = concurrent.futures.Future() + if len(topic_messages) < 2: + future.set_exception(ValueError("send_atomic() requires at least 2 topics")) + return future + + def _do_send(): + try: + req = pb.AtomicProduceRequest() + for topic, payload in topic_messages.items(): + slice = req.slices.add() + slice.topic = topic + entry = slice.entries.add() + entry.payload = payload + entry.client_timestamp = int(time.time() * 1000) + + envelope_bytes = req.SerializeToString() + except Exception as e: + future.set_exception(e) + return + + delivery_timeout_ms = 120000 + start_ms = int(time.time() * 1000) + current_backoff_ms = 100 + + while True: + if not self.running: + future.set_exception(Exception("Producer closed during retry")) + return + + if int(time.time() * 1000) - start_ms > delivery_timeout_ms: + break + + try: + self._ensure_connected() + resp_payload = self._send_envelope(pb.MessageType.ATOMIC_PRODUCE_REQUEST, envelope_bytes) + + resp = pb.AtomicProduceResponse() + resp.ParseFromString(resp_payload) + + if resp.success: + future.set_result(dict(resp.base_offsets)) + return + else: + error_code = resp.error_code + error_msg = resp.error_message + if error_code == pb.ErrorCode.NOT_LEADER or (error_msg and error_msg.startswith("NOT_LEADER")): + leader_addr = error_msg[len("NOT_LEADER:"):] if error_msg and error_msg.startswith("NOT_LEADER:") else "UNKNOWN" + if leader_addr != "UNKNOWN": + parts = leader_addr.split(":") + if len(parts) == 2: + self.host = parts[0] + self.port = int(parts[1]) + super(DRMQProducer, self).close() + self._ensure_connected() + continue + super(DRMQProducer, self).close() + self._rotate_server() + elif error_msg and ("timed out" in error_msg or "Lost leadership" in error_msg or "Raft batch proposal" in error_msg): + super(DRMQProducer, self).close() + self._rotate_server() + else: + future.set_exception(Exception(f"Failed to send atomic batch: {error_msg}")) + return + except Exception as e: + logger.warning("Network error during atomic send: %s", e) + super(DRMQProducer, self).close() + self._rotate_server() + + time.sleep(current_backoff_ms / 1000.0) + current_backoff_ms = min(2000, current_backoff_ms * 2) + + future.set_exception(Exception("Failed to send atomic batch: timeout exhausted")) + + threading.Thread(target=_do_send, daemon=True, name="drmq-producer-atomic-sender").start() + return future + def close(self): self.running = False if self.send_thread and self.send_thread.is_alive(): @@ -483,11 +564,20 @@ def nack(self, topic: str, offset: int) -> bool: producer = DRMQProducer(servers) try: producer.connect() - res = producer.send("python-topic", b"Hello from Python!") + res = producer.send("python-topic", b"Hello from Python!").result() if res.success: print(f"Message sent successfully at offset {res.offset}") else: print(f"Failed to send: {res.error_message}") + + print("\n--- Testing Atomic Producer ---") + batch = { + "python-topic-1": b"Atomic message 1", + "python-topic-2": b"Atomic message 2" + } + atomic_res = producer.send_atomic(batch).result(timeout=10) + print(f"Atomic commit successful! Base offsets: {atomic_res}") + except Exception as e: print(f"Producer error: {e}") finally: diff --git a/drmq-python-client/messages_pb2.py b/drmq-python-client/messages_pb2.py index 310ffe7..8063cc3 100644 --- a/drmq-python-client/messages_pb2.py +++ b/drmq-python-client/messages_pb2.py @@ -24,7 +24,7 @@ -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0emessages.proto\x12\x11\x63om.drmq.protocol\"]\n\x0eProduceRequest\x12\r\n\x05topic\x18\x01 \x01(\t\x12\x0f\n\x07payload\x18\x02 \x01(\x0c\x12\x10\n\x03key\x18\x03 \x01(\tH\x00\x88\x01\x01\x12\x11\n\ttimestamp\x18\x04 \x01(\x03\x42\x06\n\x04_key\"{\n\x0fProduceResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0e\n\x06offset\x18\x02 \x01(\x03\x12\x15\n\rerror_message\x18\x03 \x01(\t\x12\x30\n\nerror_code\x18\x04 \x01(\x0e\x32\x1c.com.drmq.protocol.ErrorCode\"\xbb\x01\n\x13ProduceBatchRequest\x12\r\n\x05topic\x18\x01 \x01(\t\x12\x42\n\x07\x65ntries\x18\x02 \x03(\x0b\x32\x31.com.drmq.protocol.ProduceBatchRequest.BatchEntry\x1aQ\n\nBatchEntry\x12\x0f\n\x07payload\x18\x01 \x01(\x0c\x12\x10\n\x03key\x18\x02 \x01(\tH\x00\x88\x01\x01\x12\x18\n\x10\x63lient_timestamp\x18\x03 \x01(\x03\x42\x06\n\x04_key\"\x94\x01\n\x14ProduceBatchResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x13\n\x0b\x62\x61se_offset\x18\x02 \x01(\x03\x12\r\n\x05\x63ount\x18\x03 \x01(\x05\x12\x15\n\rerror_message\x18\x04 \x01(\t\x12\x30\n\nerror_code\x18\x05 \x01(\x0e\x32\x1c.com.drmq.protocol.ErrorCode\"\x7f\n\rStoredMessage\x12\x0e\n\x06offset\x18\x01 \x01(\x03\x12\r\n\x05topic\x18\x02 \x01(\t\x12\x0f\n\x07payload\x18\x03 \x01(\x0c\x12\x10\n\x03key\x18\x04 \x01(\tH\x00\x88\x01\x01\x12\x11\n\ttimestamp\x18\x05 \x01(\x03\x12\x11\n\tstored_at\x18\x06 \x01(\x03\x42\x06\n\x04_key\"\xb8\x01\n\x0e\x43onsumeRequest\x12\r\n\x05topic\x18\x01 \x01(\t\x12\x13\n\x0b\x66rom_offset\x18\x02 \x01(\x03\x12\x14\n\x0cmax_messages\x18\x03 \x01(\x05\x12\x12\n\ntimeout_ms\x18\x04 \x01(\x03\x12\x1b\n\x0e\x63onsumer_group\x18\x05 \x01(\tH\x00\x88\x01\x01\x12\x18\n\x0b\x63onsumer_id\x18\x06 \x01(\tH\x01\x88\x01\x01\x42\x11\n\x0f_consumer_groupB\x0e\n\x0c_consumer_id\"m\n\x0f\x43onsumeResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x32\n\x08messages\x18\x02 \x03(\x0b\x32 .com.drmq.protocol.StoredMessage\x12\x15\n\rerror_message\x18\x03 \x01(\t\"v\n\x13\x43ommitOffsetRequest\x12\x16\n\x0e\x63onsumer_group\x18\x01 \x01(\t\x12\r\n\x05topic\x18\x02 \x01(\t\x12\x0e\n\x06offset\x18\x03 \x01(\x03\x12\x18\n\x0b\x63onsumer_id\x18\x04 \x01(\tH\x00\x88\x01\x01\x42\x0e\n\x0c_consumer_id\">\n\x14\x43ommitOffsetResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x15\n\rerror_message\x18\x02 \x01(\t\";\n\x12\x46\x65tchOffsetRequest\x12\x16\n\x0e\x63onsumer_group\x18\x01 \x01(\t\x12\r\n\x05topic\x18\x02 \x01(\t\"M\n\x13\x46\x65tchOffsetResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0e\n\x06offset\x18\x02 \x01(\x03\x12\x15\n\rerror_message\x18\x03 \x01(\t\"=\n\x19SearchOffsetByTimeRequest\x12\r\n\x05topic\x18\x01 \x01(\t\x12\x11\n\ttimestamp\x18\x02 \x01(\x03\"T\n\x1aSearchOffsetByTimeResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0e\n\x06offset\x18\x02 \x01(\x03\x12\x15\n\rerror_message\x18\x03 \x01(\t\"n\n\x0bNackRequest\x12\x16\n\x0e\x63onsumer_group\x18\x01 \x01(\t\x12\r\n\x05topic\x18\x02 \x01(\t\x12\x0e\n\x06offset\x18\x03 \x01(\x03\x12\x18\n\x0b\x63onsumer_id\x18\x04 \x01(\tH\x00\x88\x01\x01\x42\x0e\n\x0c_consumer_id\"M\n\x0cNackResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x15\n\rrouted_to_dlq\x18\x02 \x01(\x08\x12\x15\n\rerror_message\x18\x03 \x01(\t\"P\n\x0fMessageEnvelope\x12,\n\x04type\x18\x01 \x01(\x0e\x32\x1e.com.drmq.protocol.MessageType\x12\x0f\n\x07payload\x18\x02 \x01(\x0c\"\x8b\x02\n\tRaftEntry\x12\x0c\n\x04term\x18\x01 \x01(\x03\x12\r\n\x05index\x18\x02 \x01(\x03\x12\r\n\x05topic\x18\x03 \x01(\t\x12\x0f\n\x07payload\x18\x04 \x01(\x0c\x12\x10\n\x03key\x18\x05 \x01(\tH\x00\x88\x01\x01\x12\x11\n\ttimestamp\x18\x06 \x01(\x03\x12\x38\n\x0c\x63ommand_type\x18\x07 \x01(\x0e\x32\".com.drmq.protocol.RaftCommandType\x12\x1b\n\x0e\x63onsumer_group\x18\x08 \x01(\tH\x01\x88\x01\x01\x12\x19\n\x0coffset_value\x18\t \x01(\x03H\x02\x88\x01\x01\x42\x06\n\x04_keyB\x11\n\x0f_consumer_groupB\x0f\n\r_offset_value\"g\n\x12RequestVoteRequest\x12\x0c\n\x04term\x18\x01 \x01(\x03\x12\x14\n\x0c\x63\x61ndidate_id\x18\x02 \x01(\t\x12\x16\n\x0elast_log_index\x18\x03 \x01(\x03\x12\x15\n\rlast_log_term\x18\x04 \x01(\x03\"9\n\x13RequestVoteResponse\x12\x0c\n\x04term\x18\x01 \x01(\x03\x12\x14\n\x0cvote_granted\x18\x02 \x01(\x08\"c\n\x0ePreVoteRequest\x12\x0c\n\x04term\x18\x01 \x01(\x03\x12\x14\n\x0c\x63\x61ndidate_id\x18\x02 \x01(\t\x12\x16\n\x0elast_log_index\x18\x03 \x01(\x03\x12\x15\n\rlast_log_term\x18\x04 \x01(\x03\"5\n\x0fPreVoteResponse\x12\x0c\n\x04term\x18\x01 \x01(\x03\x12\x14\n\x0cvote_granted\x18\x02 \x01(\x08\"\xac\x01\n\x14\x41ppendEntriesRequest\x12\x0c\n\x04term\x18\x01 \x01(\x03\x12\x11\n\tleader_id\x18\x02 \x01(\t\x12\x16\n\x0eprev_log_index\x18\x03 \x01(\x03\x12\x15\n\rprev_log_term\x18\x04 \x01(\x03\x12-\n\x07\x65ntries\x18\x05 \x03(\x0b\x32\x1c.com.drmq.protocol.RaftEntry\x12\x15\n\rleader_commit\x18\x06 \x01(\x03\"K\n\x15\x41ppendEntriesResponse\x12\x0c\n\x04term\x18\x01 \x01(\x03\x12\x0f\n\x07success\x18\x02 \x01(\x08\x12\x13\n\x0bmatch_index\x18\x03 \x01(\x03\"\x9e\x01\n\x16InstallSnapshotRequest\x12\x0c\n\x04term\x18\x01 \x01(\x03\x12\x11\n\tleader_id\x18\x02 \x01(\t\x12\x1b\n\x13last_included_index\x18\x03 \x01(\x03\x12\x1a\n\x12last_included_term\x18\x04 \x01(\x03\x12\x0e\n\x06offset\x18\x05 \x01(\x03\x12\x0c\n\x04\x64\x61ta\x18\x06 \x01(\x0c\x12\x0c\n\x04\x64one\x18\x07 \x01(\x08\"\'\n\x17InstallSnapshotResponse\x12\x0c\n\x04term\x18\x01 \x01(\x03*D\n\x0fRaftCommandType\x12\x0b\n\x07MESSAGE\x10\x00\x12\x11\n\rOFFSET_COMMIT\x10\x01\x12\x11\n\rBATCH_MESSAGE\x10\x02*\xe6\x04\n\x0bMessageType\x12\x0b\n\x07UNKNOWN\x10\x00\x12\x13\n\x0fPRODUCE_REQUEST\x10\x01\x12\x14\n\x10PRODUCE_RESPONSE\x10\x02\x12\x13\n\x0f\x43ONSUME_REQUEST\x10\x03\x12\x14\n\x10\x43ONSUME_RESPONSE\x10\x04\x12\r\n\tHEARTBEAT\x10\x05\x12\x19\n\x15\x43OMMIT_OFFSET_REQUEST\x10\x06\x12\x1a\n\x16\x43OMMIT_OFFSET_RESPONSE\x10\x07\x12\x18\n\x14\x46\x45TCH_OFFSET_REQUEST\x10\x08\x12\x19\n\x15\x46\x45TCH_OFFSET_RESPONSE\x10\t\x12\x18\n\x14REQUEST_VOTE_REQUEST\x10\n\x12\x19\n\x15REQUEST_VOTE_RESPONSE\x10\x0b\x12\x1a\n\x16\x41PPEND_ENTRIES_REQUEST\x10\x0c\x12\x1b\n\x17\x41PPEND_ENTRIES_RESPONSE\x10\r\x12\x14\n\x10PRE_VOTE_REQUEST\x10\x0e\x12\x15\n\x11PRE_VOTE_RESPONSE\x10\x0f\x12\x1c\n\x18INSTALL_SNAPSHOT_REQUEST\x10\x10\x12\x1d\n\x19INSTALL_SNAPSHOT_RESPONSE\x10\x11\x12\x10\n\x0cNACK_REQUEST\x10\x12\x12\x11\n\rNACK_RESPONSE\x10\x13\x12\x19\n\x15PRODUCE_BATCH_REQUEST\x10\x14\x12\x1a\n\x16PRODUCE_BATCH_RESPONSE\x10\x15\x12!\n\x1dSEARCH_OFFSET_BY_TIME_REQUEST\x10\x16\x12\"\n\x1eSEARCH_OFFSET_BY_TIME_RESPONSE\x10\x17*8\n\tErrorCode\x12\x08\n\x04NONE\x10\x00\x12\x0e\n\nNOT_LEADER\x10\x01\x12\x11\n\rUNKNOWN_ERROR\x10\x63\x42#\n\x11\x63om.drmq.protocolB\x0c\x44RMQProtocolP\x00\x62\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0emessages.proto\x12\x11\x63om.drmq.protocol\"]\n\x0eProduceRequest\x12\r\n\x05topic\x18\x01 \x01(\t\x12\x0f\n\x07payload\x18\x02 \x01(\x0c\x12\x10\n\x03key\x18\x03 \x01(\tH\x00\x88\x01\x01\x12\x11\n\ttimestamp\x18\x04 \x01(\x03\x42\x06\n\x04_key\"{\n\x0fProduceResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0e\n\x06offset\x18\x02 \x01(\x03\x12\x15\n\rerror_message\x18\x03 \x01(\t\x12\x30\n\nerror_code\x18\x04 \x01(\x0e\x32\x1c.com.drmq.protocol.ErrorCode\"\xbb\x01\n\x13ProduceBatchRequest\x12\r\n\x05topic\x18\x01 \x01(\t\x12\x42\n\x07\x65ntries\x18\x02 \x03(\x0b\x32\x31.com.drmq.protocol.ProduceBatchRequest.BatchEntry\x1aQ\n\nBatchEntry\x12\x0f\n\x07payload\x18\x01 \x01(\x0c\x12\x10\n\x03key\x18\x02 \x01(\tH\x00\x88\x01\x01\x12\x18\n\x10\x63lient_timestamp\x18\x03 \x01(\x03\x42\x06\n\x04_key\"\x94\x01\n\x14ProduceBatchResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x13\n\x0b\x62\x61se_offset\x18\x02 \x01(\x03\x12\r\n\x05\x63ount\x18\x03 \x01(\x05\x12\x15\n\rerror_message\x18\x04 \x01(\t\x12\x30\n\nerror_code\x18\x05 \x01(\x0e\x32\x1c.com.drmq.protocol.ErrorCode\"j\n\x15\x41tomicBatchTopicSlice\x12\r\n\x05topic\x18\x01 \x01(\t\x12\x42\n\x07\x65ntries\x18\x02 \x03(\x0b\x32\x31.com.drmq.protocol.ProduceBatchRequest.BatchEntry\"N\n\x12\x41tomicBatchRequest\x12\x38\n\x06slices\x18\x01 \x03(\x0b\x32(.com.drmq.protocol.AtomicBatchTopicSlice\"P\n\x14\x41tomicProduceRequest\x12\x38\n\x06slices\x18\x01 \x03(\x0b\x32(.com.drmq.protocol.AtomicBatchTopicSlice\"\xf6\x01\n\x15\x41tomicProduceResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12O\n\x0c\x62\x61se_offsets\x18\x02 \x03(\x0b\x32\x39.com.drmq.protocol.AtomicProduceResponse.BaseOffsetsEntry\x12\x15\n\rerror_message\x18\x03 \x01(\t\x12\x30\n\nerror_code\x18\x04 \x01(\x0e\x32\x1c.com.drmq.protocol.ErrorCode\x1a\x32\n\x10\x42\x61seOffsetsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x03:\x02\x38\x01\"\x7f\n\rStoredMessage\x12\x0e\n\x06offset\x18\x01 \x01(\x03\x12\r\n\x05topic\x18\x02 \x01(\t\x12\x0f\n\x07payload\x18\x03 \x01(\x0c\x12\x10\n\x03key\x18\x04 \x01(\tH\x00\x88\x01\x01\x12\x11\n\ttimestamp\x18\x05 \x01(\x03\x12\x11\n\tstored_at\x18\x06 \x01(\x03\x42\x06\n\x04_key\"\xb8\x01\n\x0e\x43onsumeRequest\x12\r\n\x05topic\x18\x01 \x01(\t\x12\x13\n\x0b\x66rom_offset\x18\x02 \x01(\x03\x12\x14\n\x0cmax_messages\x18\x03 \x01(\x05\x12\x12\n\ntimeout_ms\x18\x04 \x01(\x03\x12\x1b\n\x0e\x63onsumer_group\x18\x05 \x01(\tH\x00\x88\x01\x01\x12\x18\n\x0b\x63onsumer_id\x18\x06 \x01(\tH\x01\x88\x01\x01\x42\x11\n\x0f_consumer_groupB\x0e\n\x0c_consumer_id\"m\n\x0f\x43onsumeResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x32\n\x08messages\x18\x02 \x03(\x0b\x32 .com.drmq.protocol.StoredMessage\x12\x15\n\rerror_message\x18\x03 \x01(\t\"v\n\x13\x43ommitOffsetRequest\x12\x16\n\x0e\x63onsumer_group\x18\x01 \x01(\t\x12\r\n\x05topic\x18\x02 \x01(\t\x12\x0e\n\x06offset\x18\x03 \x01(\x03\x12\x18\n\x0b\x63onsumer_id\x18\x04 \x01(\tH\x00\x88\x01\x01\x42\x0e\n\x0c_consumer_id\">\n\x14\x43ommitOffsetResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x15\n\rerror_message\x18\x02 \x01(\t\";\n\x12\x46\x65tchOffsetRequest\x12\x16\n\x0e\x63onsumer_group\x18\x01 \x01(\t\x12\r\n\x05topic\x18\x02 \x01(\t\"M\n\x13\x46\x65tchOffsetResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0e\n\x06offset\x18\x02 \x01(\x03\x12\x15\n\rerror_message\x18\x03 \x01(\t\"=\n\x19SearchOffsetByTimeRequest\x12\r\n\x05topic\x18\x01 \x01(\t\x12\x11\n\ttimestamp\x18\x02 \x01(\x03\"T\n\x1aSearchOffsetByTimeResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0e\n\x06offset\x18\x02 \x01(\x03\x12\x15\n\rerror_message\x18\x03 \x01(\t\"n\n\x0bNackRequest\x12\x16\n\x0e\x63onsumer_group\x18\x01 \x01(\t\x12\r\n\x05topic\x18\x02 \x01(\t\x12\x0e\n\x06offset\x18\x03 \x01(\x03\x12\x18\n\x0b\x63onsumer_id\x18\x04 \x01(\tH\x00\x88\x01\x01\x42\x0e\n\x0c_consumer_id\"M\n\x0cNackResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x15\n\rrouted_to_dlq\x18\x02 \x01(\x08\x12\x15\n\rerror_message\x18\x03 \x01(\t\"P\n\x0fMessageEnvelope\x12,\n\x04type\x18\x01 \x01(\x0e\x32\x1e.com.drmq.protocol.MessageType\x12\x0f\n\x07payload\x18\x02 \x01(\x0c\"\x8b\x02\n\tRaftEntry\x12\x0c\n\x04term\x18\x01 \x01(\x03\x12\r\n\x05index\x18\x02 \x01(\x03\x12\r\n\x05topic\x18\x03 \x01(\t\x12\x0f\n\x07payload\x18\x04 \x01(\x0c\x12\x10\n\x03key\x18\x05 \x01(\tH\x00\x88\x01\x01\x12\x11\n\ttimestamp\x18\x06 \x01(\x03\x12\x38\n\x0c\x63ommand_type\x18\x07 \x01(\x0e\x32\".com.drmq.protocol.RaftCommandType\x12\x1b\n\x0e\x63onsumer_group\x18\x08 \x01(\tH\x01\x88\x01\x01\x12\x19\n\x0coffset_value\x18\t \x01(\x03H\x02\x88\x01\x01\x42\x06\n\x04_keyB\x11\n\x0f_consumer_groupB\x0f\n\r_offset_value\"g\n\x12RequestVoteRequest\x12\x0c\n\x04term\x18\x01 \x01(\x03\x12\x14\n\x0c\x63\x61ndidate_id\x18\x02 \x01(\t\x12\x16\n\x0elast_log_index\x18\x03 \x01(\x03\x12\x15\n\rlast_log_term\x18\x04 \x01(\x03\"9\n\x13RequestVoteResponse\x12\x0c\n\x04term\x18\x01 \x01(\x03\x12\x14\n\x0cvote_granted\x18\x02 \x01(\x08\"c\n\x0ePreVoteRequest\x12\x0c\n\x04term\x18\x01 \x01(\x03\x12\x14\n\x0c\x63\x61ndidate_id\x18\x02 \x01(\t\x12\x16\n\x0elast_log_index\x18\x03 \x01(\x03\x12\x15\n\rlast_log_term\x18\x04 \x01(\x03\"5\n\x0fPreVoteResponse\x12\x0c\n\x04term\x18\x01 \x01(\x03\x12\x14\n\x0cvote_granted\x18\x02 \x01(\x08\"\xac\x01\n\x14\x41ppendEntriesRequest\x12\x0c\n\x04term\x18\x01 \x01(\x03\x12\x11\n\tleader_id\x18\x02 \x01(\t\x12\x16\n\x0eprev_log_index\x18\x03 \x01(\x03\x12\x15\n\rprev_log_term\x18\x04 \x01(\x03\x12-\n\x07\x65ntries\x18\x05 \x03(\x0b\x32\x1c.com.drmq.protocol.RaftEntry\x12\x15\n\rleader_commit\x18\x06 \x01(\x03\"K\n\x15\x41ppendEntriesResponse\x12\x0c\n\x04term\x18\x01 \x01(\x03\x12\x0f\n\x07success\x18\x02 \x01(\x08\x12\x13\n\x0bmatch_index\x18\x03 \x01(\x03\"\x9e\x01\n\x16InstallSnapshotRequest\x12\x0c\n\x04term\x18\x01 \x01(\x03\x12\x11\n\tleader_id\x18\x02 \x01(\t\x12\x1b\n\x13last_included_index\x18\x03 \x01(\x03\x12\x1a\n\x12last_included_term\x18\x04 \x01(\x03\x12\x0e\n\x06offset\x18\x05 \x01(\x03\x12\x0c\n\x04\x64\x61ta\x18\x06 \x01(\x0c\x12\x0c\n\x04\x64one\x18\x07 \x01(\x08\"\'\n\x17InstallSnapshotResponse\x12\x0c\n\x04term\x18\x01 \x01(\x03*V\n\x0fRaftCommandType\x12\x0b\n\x07MESSAGE\x10\x00\x12\x11\n\rOFFSET_COMMIT\x10\x01\x12\x11\n\rBATCH_MESSAGE\x10\x02\x12\x10\n\x0c\x41TOMIC_BATCH\x10\x03*\x9f\x05\n\x0bMessageType\x12\x0b\n\x07UNKNOWN\x10\x00\x12\x13\n\x0fPRODUCE_REQUEST\x10\x01\x12\x14\n\x10PRODUCE_RESPONSE\x10\x02\x12\x13\n\x0f\x43ONSUME_REQUEST\x10\x03\x12\x14\n\x10\x43ONSUME_RESPONSE\x10\x04\x12\r\n\tHEARTBEAT\x10\x05\x12\x19\n\x15\x43OMMIT_OFFSET_REQUEST\x10\x06\x12\x1a\n\x16\x43OMMIT_OFFSET_RESPONSE\x10\x07\x12\x18\n\x14\x46\x45TCH_OFFSET_REQUEST\x10\x08\x12\x19\n\x15\x46\x45TCH_OFFSET_RESPONSE\x10\t\x12\x18\n\x14REQUEST_VOTE_REQUEST\x10\n\x12\x19\n\x15REQUEST_VOTE_RESPONSE\x10\x0b\x12\x1a\n\x16\x41PPEND_ENTRIES_REQUEST\x10\x0c\x12\x1b\n\x17\x41PPEND_ENTRIES_RESPONSE\x10\r\x12\x14\n\x10PRE_VOTE_REQUEST\x10\x0e\x12\x15\n\x11PRE_VOTE_RESPONSE\x10\x0f\x12\x1c\n\x18INSTALL_SNAPSHOT_REQUEST\x10\x10\x12\x1d\n\x19INSTALL_SNAPSHOT_RESPONSE\x10\x11\x12\x10\n\x0cNACK_REQUEST\x10\x12\x12\x11\n\rNACK_RESPONSE\x10\x13\x12\x19\n\x15PRODUCE_BATCH_REQUEST\x10\x14\x12\x1a\n\x16PRODUCE_BATCH_RESPONSE\x10\x15\x12!\n\x1dSEARCH_OFFSET_BY_TIME_REQUEST\x10\x16\x12\"\n\x1eSEARCH_OFFSET_BY_TIME_RESPONSE\x10\x17\x12\x1a\n\x16\x41TOMIC_PRODUCE_REQUEST\x10\x18\x12\x1b\n\x17\x41TOMIC_PRODUCE_RESPONSE\x10\x19*8\n\tErrorCode\x12\x08\n\x04NONE\x10\x00\x12\x0e\n\nNOT_LEADER\x10\x01\x12\x11\n\rUNKNOWN_ERROR\x10\x63\x42#\n\x11\x63om.drmq.protocolB\x0c\x44RMQProtocolP\x00\x62\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -32,12 +32,14 @@ if not _descriptor._USE_C_DESCRIPTORS: _globals['DESCRIPTOR']._loaded_options = None _globals['DESCRIPTOR']._serialized_options = b'\n\021com.drmq.protocolB\014DRMQProtocolP\000' - _globals['_RAFTCOMMANDTYPE']._serialized_start=2815 - _globals['_RAFTCOMMANDTYPE']._serialized_end=2883 - _globals['_MESSAGETYPE']._serialized_start=2886 - _globals['_MESSAGETYPE']._serialized_end=3500 - _globals['_ERRORCODE']._serialized_start=3502 - _globals['_ERRORCODE']._serialized_end=3558 + _globals['_ATOMICPRODUCERESPONSE_BASEOFFSETSENTRY']._loaded_options = None + _globals['_ATOMICPRODUCERESPONSE_BASEOFFSETSENTRY']._serialized_options = b'8\001' + _globals['_RAFTCOMMANDTYPE']._serialized_start=3334 + _globals['_RAFTCOMMANDTYPE']._serialized_end=3420 + _globals['_MESSAGETYPE']._serialized_start=3423 + _globals['_MESSAGETYPE']._serialized_end=4094 + _globals['_ERRORCODE']._serialized_start=4096 + _globals['_ERRORCODE']._serialized_end=4152 _globals['_PRODUCEREQUEST']._serialized_start=37 _globals['_PRODUCEREQUEST']._serialized_end=130 _globals['_PRODUCERESPONSE']._serialized_start=132 @@ -48,46 +50,56 @@ _globals['_PRODUCEBATCHREQUEST_BATCHENTRY']._serialized_end=445 _globals['_PRODUCEBATCHRESPONSE']._serialized_start=448 _globals['_PRODUCEBATCHRESPONSE']._serialized_end=596 - _globals['_STOREDMESSAGE']._serialized_start=598 - _globals['_STOREDMESSAGE']._serialized_end=725 - _globals['_CONSUMEREQUEST']._serialized_start=728 - _globals['_CONSUMEREQUEST']._serialized_end=912 - _globals['_CONSUMERESPONSE']._serialized_start=914 - _globals['_CONSUMERESPONSE']._serialized_end=1023 - _globals['_COMMITOFFSETREQUEST']._serialized_start=1025 - _globals['_COMMITOFFSETREQUEST']._serialized_end=1143 - _globals['_COMMITOFFSETRESPONSE']._serialized_start=1145 - _globals['_COMMITOFFSETRESPONSE']._serialized_end=1207 - _globals['_FETCHOFFSETREQUEST']._serialized_start=1209 - _globals['_FETCHOFFSETREQUEST']._serialized_end=1268 - _globals['_FETCHOFFSETRESPONSE']._serialized_start=1270 - _globals['_FETCHOFFSETRESPONSE']._serialized_end=1347 - _globals['_SEARCHOFFSETBYTIMEREQUEST']._serialized_start=1349 - _globals['_SEARCHOFFSETBYTIMEREQUEST']._serialized_end=1410 - _globals['_SEARCHOFFSETBYTIMERESPONSE']._serialized_start=1412 - _globals['_SEARCHOFFSETBYTIMERESPONSE']._serialized_end=1496 - _globals['_NACKREQUEST']._serialized_start=1498 - _globals['_NACKREQUEST']._serialized_end=1608 - _globals['_NACKRESPONSE']._serialized_start=1610 - _globals['_NACKRESPONSE']._serialized_end=1687 - _globals['_MESSAGEENVELOPE']._serialized_start=1689 - _globals['_MESSAGEENVELOPE']._serialized_end=1769 - _globals['_RAFTENTRY']._serialized_start=1772 - _globals['_RAFTENTRY']._serialized_end=2039 - _globals['_REQUESTVOTEREQUEST']._serialized_start=2041 - _globals['_REQUESTVOTEREQUEST']._serialized_end=2144 - _globals['_REQUESTVOTERESPONSE']._serialized_start=2146 - _globals['_REQUESTVOTERESPONSE']._serialized_end=2203 - _globals['_PREVOTEREQUEST']._serialized_start=2205 - _globals['_PREVOTEREQUEST']._serialized_end=2304 - _globals['_PREVOTERESPONSE']._serialized_start=2306 - _globals['_PREVOTERESPONSE']._serialized_end=2359 - _globals['_APPENDENTRIESREQUEST']._serialized_start=2362 - _globals['_APPENDENTRIESREQUEST']._serialized_end=2534 - _globals['_APPENDENTRIESRESPONSE']._serialized_start=2536 - _globals['_APPENDENTRIESRESPONSE']._serialized_end=2611 - _globals['_INSTALLSNAPSHOTREQUEST']._serialized_start=2614 - _globals['_INSTALLSNAPSHOTREQUEST']._serialized_end=2772 - _globals['_INSTALLSNAPSHOTRESPONSE']._serialized_start=2774 - _globals['_INSTALLSNAPSHOTRESPONSE']._serialized_end=2813 + _globals['_ATOMICBATCHTOPICSLICE']._serialized_start=598 + _globals['_ATOMICBATCHTOPICSLICE']._serialized_end=704 + _globals['_ATOMICBATCHREQUEST']._serialized_start=706 + _globals['_ATOMICBATCHREQUEST']._serialized_end=784 + _globals['_ATOMICPRODUCEREQUEST']._serialized_start=786 + _globals['_ATOMICPRODUCEREQUEST']._serialized_end=866 + _globals['_ATOMICPRODUCERESPONSE']._serialized_start=869 + _globals['_ATOMICPRODUCERESPONSE']._serialized_end=1115 + _globals['_ATOMICPRODUCERESPONSE_BASEOFFSETSENTRY']._serialized_start=1065 + _globals['_ATOMICPRODUCERESPONSE_BASEOFFSETSENTRY']._serialized_end=1115 + _globals['_STOREDMESSAGE']._serialized_start=1117 + _globals['_STOREDMESSAGE']._serialized_end=1244 + _globals['_CONSUMEREQUEST']._serialized_start=1247 + _globals['_CONSUMEREQUEST']._serialized_end=1431 + _globals['_CONSUMERESPONSE']._serialized_start=1433 + _globals['_CONSUMERESPONSE']._serialized_end=1542 + _globals['_COMMITOFFSETREQUEST']._serialized_start=1544 + _globals['_COMMITOFFSETREQUEST']._serialized_end=1662 + _globals['_COMMITOFFSETRESPONSE']._serialized_start=1664 + _globals['_COMMITOFFSETRESPONSE']._serialized_end=1726 + _globals['_FETCHOFFSETREQUEST']._serialized_start=1728 + _globals['_FETCHOFFSETREQUEST']._serialized_end=1787 + _globals['_FETCHOFFSETRESPONSE']._serialized_start=1789 + _globals['_FETCHOFFSETRESPONSE']._serialized_end=1866 + _globals['_SEARCHOFFSETBYTIMEREQUEST']._serialized_start=1868 + _globals['_SEARCHOFFSETBYTIMEREQUEST']._serialized_end=1929 + _globals['_SEARCHOFFSETBYTIMERESPONSE']._serialized_start=1931 + _globals['_SEARCHOFFSETBYTIMERESPONSE']._serialized_end=2015 + _globals['_NACKREQUEST']._serialized_start=2017 + _globals['_NACKREQUEST']._serialized_end=2127 + _globals['_NACKRESPONSE']._serialized_start=2129 + _globals['_NACKRESPONSE']._serialized_end=2206 + _globals['_MESSAGEENVELOPE']._serialized_start=2208 + _globals['_MESSAGEENVELOPE']._serialized_end=2288 + _globals['_RAFTENTRY']._serialized_start=2291 + _globals['_RAFTENTRY']._serialized_end=2558 + _globals['_REQUESTVOTEREQUEST']._serialized_start=2560 + _globals['_REQUESTVOTEREQUEST']._serialized_end=2663 + _globals['_REQUESTVOTERESPONSE']._serialized_start=2665 + _globals['_REQUESTVOTERESPONSE']._serialized_end=2722 + _globals['_PREVOTEREQUEST']._serialized_start=2724 + _globals['_PREVOTEREQUEST']._serialized_end=2823 + _globals['_PREVOTERESPONSE']._serialized_start=2825 + _globals['_PREVOTERESPONSE']._serialized_end=2878 + _globals['_APPENDENTRIESREQUEST']._serialized_start=2881 + _globals['_APPENDENTRIESREQUEST']._serialized_end=3053 + _globals['_APPENDENTRIESRESPONSE']._serialized_start=3055 + _globals['_APPENDENTRIESRESPONSE']._serialized_end=3130 + _globals['_INSTALLSNAPSHOTREQUEST']._serialized_start=3133 + _globals['_INSTALLSNAPSHOTREQUEST']._serialized_end=3291 + _globals['_INSTALLSNAPSHOTRESPONSE']._serialized_start=3293 + _globals['_INSTALLSNAPSHOTRESPONSE']._serialized_end=3332 # @@protoc_insertion_point(module_scope) diff --git a/drmq-python-client/test_time_seek.py b/drmq-python-client/test_time_seek.py new file mode 100644 index 0000000..1832520 --- /dev/null +++ b/drmq-python-client/test_time_seek.py @@ -0,0 +1,72 @@ +import time +import logging +from drmq_client import DRMQProducer, DRMQConsumer + +logging.basicConfig(level=logging.INFO) + +def run_test(): + servers = "localhost:9092,localhost:9093,localhost:9094" + topic = "time-seek-topic-" + str(int(time.time())) + + print(f"--- Starting Time-Based Seek Test on {topic} ---") + producer = DRMQProducer(servers) + try: + producer.connect() + + # Produce 1st message + producer.send(topic, b"Message 1 - Early").result() + print("Sent Message 1") + + time.sleep(1) + # Capture timestamp before Message 2 + target_time = int(time.time() * 1000) + print(f"Captured target timestamp: {target_time}") + time.sleep(1) + + # Produce 2nd and 3rd messages + producer.send(topic, b"Message 2 - Target").result() + print("Sent Message 2") + producer.send(topic, b"Message 3 - Late").result() + print("Sent Message 3") + + finally: + producer.close() + + print("\n--- Starting Consumer ---") + # Single mode consumer (no group_id) + consumer = DRMQConsumer(servers, group_id=None) + try: + consumer.connect() + + print(f"Seeking to timestamp {target_time}...") + consumer.seek_by_time(topic, target_time) + print(f"Consumer local offsets: {consumer.local_offsets}") + + print("Polling...") + messages = consumer.poll(max_messages=10, timeout_ms=2000) + print(f"Polled {len(messages)} messages") + + if len(messages) == 0: + print("TEST FAILED: No messages received.") + # Let's see what is actually in the broker if we start from 0 + consumer.local_offsets[topic] = 0 + msgs_from_0 = consumer.poll(max_messages=100, timeout_ms=2000) + print(f"Polled from 0: {len(msgs_from_0)} messages") + for m in msgs_from_0: + print(f" - offset={m.offset}, ts={m.timestamp}, payload={m.payload.decode('utf-8')}") + exit(1) + + first_msg = messages[0].payload.decode('utf-8') + print(f"First message received: {first_msg}") + + if "Message 2 - Target" in first_msg: + print("TEST PASSED! Seek successfully skipped Message 1 and started at Message 2.") + else: + print(f"TEST FAILED! Expected Message 2, but got: {first_msg}") + exit(1) + + finally: + consumer.close() + +if __name__ == "__main__": + run_test() diff --git a/drmq-ts-client/src/client.ts b/drmq-ts-client/src/client.ts index 34362fa..ff9126c 100644 --- a/drmq-ts-client/src/client.ts +++ b/drmq-ts-client/src/client.ts @@ -17,6 +17,8 @@ import { ProduceBatchResponse, SearchOffsetByTimeRequest, SearchOffsetByTimeResponse, + AtomicProduceRequest, + AtomicProduceResponse, ErrorCode } from './messages'; @@ -313,6 +315,82 @@ export class DRMQProducer extends DRMQClient { throw new Error("Failed to send batch: timeout exhausted"); } + public async sendAtomic(topicMessages: Record): Promise> { + if (this.isClosed) throw new Error("Producer is closed"); + + const topics = Object.keys(topicMessages); + if (topics.length < 2) { + throw new Error("sendAtomic() requires at least 2 topics"); + } + + const req = AtomicProduceRequest.create({ + slices: topics.map(topic => ({ + topic, + entries: [{ + payload: Buffer.from(topicMessages[topic]), + clientTimestamp: Date.now() + }] + })) + }); + + const envelopeBytes = AtomicProduceRequest.encode(req).finish(); + const deliveryTimeoutMs = 120000; + const startMs = Date.now(); + let currentBackoffMs = 100; + + while (true) { + if (Date.now() - startMs > deliveryTimeoutMs) { + break; + } + + try { + await this.ensureConnected(); + const respBytes = await this.sendEnvelope(MessageType.ATOMIC_PRODUCE_REQUEST, envelopeBytes); + const resp = AtomicProduceResponse.decode(respBytes); + + if (resp.success) { + const result: Record = {}; + for (const [k, v] of Object.entries(resp.baseOffsets)) { + result[k] = Number(v); + } + return result; + } else { + const errorMsg = resp.errorMessage; + if (resp.errorCode === ErrorCode.NOT_LEADER || (errorMsg && errorMsg.startsWith("NOT_LEADER:"))) { + const leaderAddr = errorMsg && errorMsg.startsWith("NOT_LEADER:") + ? errorMsg.substring("NOT_LEADER:".length) + : "UNKNOWN"; + if (leaderAddr !== "UNKNOWN") { + const parts = leaderAddr.split(":"); + if (parts.length === 2) { + this.host = parts[0]; + this.port = parseInt(parts[1], 10); + super.closeConnection(); + await this.ensureConnected(); + continue; + } + } + super.closeConnection(); + this.rotateServer(); + } else if (errorMsg && (errorMsg.includes("timed out") || errorMsg.includes("Lost leadership") || errorMsg.includes("Raft batch proposal"))) { + super.closeConnection(); + this.rotateServer(); + } else { + throw new Error(`Failed to send atomic batch: ${errorMsg}`); + } + } + } catch (e) { + super.closeConnection(); + this.rotateServer(); + } + + await new Promise(res => setTimeout(res, currentBackoffMs)); + currentBackoffMs = Math.min(2000, currentBackoffMs * 2); + } + + throw new Error("Failed to send atomic batch: timeout exhausted"); + } + public close() { this.isClosed = true; if (this.sendTimer) clearTimeout(this.sendTimer); diff --git a/drmq-ts-client/src/messages.ts b/drmq-ts-client/src/messages.ts index 47fabb0..352a235 100644 --- a/drmq-ts-client/src/messages.ts +++ b/drmq-ts-client/src/messages.ts @@ -1,7 +1,7 @@ // Code generated by protoc-gen-ts_proto. DO NOT EDIT. // versions: // protoc-gen-ts_proto v2.11.8 -// protoc v3.21.12 +// protoc v4.25.1 // source: messages.proto /* eslint-disable */ @@ -9,14 +9,11 @@ import { BinaryReader, BinaryWriter } from "@bufbuild/protobuf/wire"; export const protobufPackage = "com.drmq.protocol"; -/** --- Raft command types (for multiplexing different operations on the Raft log) --- */ export enum RaftCommandType { - /** MESSAGE - Normal message append (default) */ MESSAGE = 0, - /** OFFSET_COMMIT - Consumer offset commit (replicated for durability) */ OFFSET_COMMIT = 1, - /** BATCH_MESSAGE - Batch message append (multiple messages in a single Raft entry) */ BATCH_MESSAGE = 2, + ATOMIC_BATCH = 3, UNRECOGNIZED = -1, } @@ -31,6 +28,9 @@ export function raftCommandTypeFromJSON(object: any): RaftCommandType { case 2: case "BATCH_MESSAGE": return RaftCommandType.BATCH_MESSAGE; + case 3: + case "ATOMIC_BATCH": + return RaftCommandType.ATOMIC_BATCH; case -1: case "UNRECOGNIZED": default: @@ -46,6 +46,8 @@ export function raftCommandTypeToJSON(object: RaftCommandType): string { return "OFFSET_COMMIT"; case RaftCommandType.BATCH_MESSAGE: return "BATCH_MESSAGE"; + case RaftCommandType.ATOMIC_BATCH: + return "ATOMIC_BATCH"; case RaftCommandType.UNRECOGNIZED: default: return "UNRECOGNIZED"; @@ -77,6 +79,8 @@ export enum MessageType { PRODUCE_BATCH_RESPONSE = 21, SEARCH_OFFSET_BY_TIME_REQUEST = 22, SEARCH_OFFSET_BY_TIME_RESPONSE = 23, + ATOMIC_PRODUCE_REQUEST = 24, + ATOMIC_PRODUCE_RESPONSE = 25, UNRECOGNIZED = -1, } @@ -154,6 +158,12 @@ export function messageTypeFromJSON(object: any): MessageType { case 23: case "SEARCH_OFFSET_BY_TIME_RESPONSE": return MessageType.SEARCH_OFFSET_BY_TIME_RESPONSE; + case 24: + case "ATOMIC_PRODUCE_REQUEST": + return MessageType.ATOMIC_PRODUCE_REQUEST; + case 25: + case "ATOMIC_PRODUCE_RESPONSE": + return MessageType.ATOMIC_PRODUCE_RESPONSE; case -1: case "UNRECOGNIZED": default: @@ -211,6 +221,10 @@ export function messageTypeToJSON(object: MessageType): string { return "SEARCH_OFFSET_BY_TIME_REQUEST"; case MessageType.SEARCH_OFFSET_BY_TIME_RESPONSE: return "SEARCH_OFFSET_BY_TIME_RESPONSE"; + case MessageType.ATOMIC_PRODUCE_REQUEST: + return "ATOMIC_PRODUCE_REQUEST"; + case MessageType.ATOMIC_PRODUCE_RESPONSE: + return "ATOMIC_PRODUCE_RESPONSE"; case MessageType.UNRECOGNIZED: default: return "UNRECOGNIZED"; @@ -257,29 +271,20 @@ export function errorCodeToJSON(object: ErrorCode): string { } } -/** Request to produce a message to a topic */ export interface ProduceRequest { topic: string; payload: Uint8Array; - /** Optional message key for partitioning (future use) */ - key?: - | string - | undefined; - /** Client-side timestamp */ + key?: string | undefined; timestamp: number; } -/** Response after producing a message */ export interface ProduceResponse { success: boolean; - /** Assigned offset if successful */ offset: number; - /** Error details if failed */ errorMessage: string; errorCode: ErrorCode; } -/** Request to produce a batch of messages to a topic */ export interface ProduceBatchRequest { topic: string; entries: ProduceBatchRequest_BatchEntry[]; @@ -291,233 +296,173 @@ export interface ProduceBatchRequest_BatchEntry { clientTimestamp: number; } -/** Response after producing a batch of messages */ export interface ProduceBatchResponse { success: boolean; - /** The offset assigned to the FIRST message in the batch */ baseOffset: number; - /** Number of messages successfully written */ count: number; - /** Error details if failed */ errorMessage: string; errorCode: ErrorCode; } -/** Internal message representation stored in the broker */ +export interface AtomicBatchTopicSlice { + topic: string; + entries: ProduceBatchRequest_BatchEntry[]; +} + +export interface AtomicBatchRequest { + slices: AtomicBatchTopicSlice[]; +} + +export interface AtomicProduceRequest { + slices: AtomicBatchTopicSlice[]; +} + +export interface AtomicProduceResponse { + success: boolean; + baseOffsets: { [key: string]: number }; + errorMessage: string; + errorCode: ErrorCode; +} + +export interface AtomicProduceResponse_BaseOffsetsEntry { + key: string; + value: number; +} + export interface StoredMessage { offset: number; topic: string; payload: Uint8Array; - key?: - | string - | undefined; - /** Original client timestamp */ + key?: string | undefined; timestamp: number; - /** Broker storage timestamp */ storedAt: number; } -/** Request to consume messages from a topic */ export interface ConsumeRequest { topic: string; - /** Starting offset (inclusive) — ignored when consumer_group is set */ fromOffset: number; - /** Maximum number of messages to fetch */ maxMessages: number; - /** Long-poll timeout (0 = return immediately) */ timeoutMs: number; - /** If set, broker manages offset via group coordination */ - consumerGroup?: - | string - | undefined; - /** Unique ID for this consumer instance within the group */ + consumerGroup?: string | undefined; consumerId?: string | undefined; } -/** Response containing consumed messages */ export interface ConsumeResponse { success: boolean; - /** List of messages */ messages: StoredMessage[]; - /** Error details if failed */ errorMessage: string; } -/** Request to commit a consumer group's offset for a topic */ export interface CommitOffsetRequest { consumerGroup: string; topic: string; - /** The next offset to read (last processed + 1) */ offset: number; - /** Consumer instance ID (for group coordination) */ consumerId?: string | undefined; } -/** Response to a commit offset request */ export interface CommitOffsetResponse { success: boolean; errorMessage: string; } -/** Request to fetch a consumer group's committed offset for a topic */ export interface FetchOffsetRequest { consumerGroup: string; topic: string; } -/** Response with the committed offset */ export interface FetchOffsetResponse { success: boolean; - /** The committed offset, or -1 if none exists */ offset: number; errorMessage: string; } -/** Request to find an offset by a given timestamp */ export interface SearchOffsetByTimeRequest { topic: string; - /** The target timestamp */ timestamp: number; } -/** Response with the found offset */ export interface SearchOffsetByTimeResponse { success: boolean; - /** The found offset, or -1 if none */ offset: number; errorMessage: string; } -/** Request to negatively acknowledge (NACK) a message, triggering redelivery or DLQ routing */ export interface NackRequest { consumerGroup: string; topic: string; - /** The offset of the message being rejected */ offset: number; - /** Consumer instance ID (for group coordination) */ consumerId?: string | undefined; } -/** Response to a NACK request */ export interface NackResponse { success: boolean; - /** True if the message was sent to the DLQ instead of being requeued */ routedToDlq: boolean; errorMessage: string; } -/** Envelope for TCP framing - wraps all messages with type info */ export interface MessageEnvelope { type: MessageType; - /** Serialized inner message */ payload: Uint8Array; } -/** --- Raft log entry (wraps a user message for Raft replication) --- */ export interface RaftEntry { - /** Term in which the entry was created */ term: number; - /** Position in the Raft log (1-indexed) */ index: number; - /** Target topic for the message */ topic: string; - /** Message payload */ payload: Uint8Array; - /** Optional message key */ - key?: - | string - | undefined; - /** Client-side timestamp */ + key?: string | undefined; timestamp: number; - /** Type of command (default: MESSAGE) */ commandType: RaftCommandType; - /** Consumer group (for OFFSET_COMMIT) */ - consumerGroup?: - | string - | undefined; - /** Offset value (for OFFSET_COMMIT) */ + consumerGroup?: string | undefined; offsetValue?: number | undefined; } -/** --- Raft leader election --- */ export interface RequestVoteRequest { - /** Candidate's term */ term: number; - /** Candidate requesting vote */ candidateId: string; - /** Index of candidate's last log entry */ lastLogIndex: number; - /** Term of candidate's last log entry */ lastLogTerm: number; } export interface RequestVoteResponse { - /** Current term, for candidate to update itself */ term: number; - /** True = vote granted */ voteGranted: boolean; } -/** - * --- Raft Pre-Vote (§9.6 disruption prevention) --- - * Sent before a real election to check if the candidate COULD win - * without incrementing the term. Prevents restarting nodes from - * disrupting a healthy leader. - */ export interface PreVoteRequest { - /** The term the candidate WOULD use (currentTerm + 1) */ term: number; - /** Candidate requesting pre-vote */ candidateId: string; - /** Index of candidate's last log entry */ lastLogIndex: number; - /** Term of candidate's last log entry */ lastLogTerm: number; } export interface PreVoteResponse { - /** Current term of the responder */ term: number; - /** True = would vote for this candidate */ voteGranted: boolean; } -/** --- Raft log replication (also used as heartbeat when entries is empty) --- */ export interface AppendEntriesRequest { - /** Leader's term */ term: number; - /** So followers know who the leader is */ leaderId: string; - /** Index of log entry immediately preceding new ones */ prevLogIndex: number; - /** Term of prevLogIndex entry */ prevLogTerm: number; - /** Log entries to store (empty for heartbeat) */ entries: RaftEntry[]; - /** Leader's commitIndex */ leaderCommit: number; } export interface AppendEntriesResponse { - /** Current term, for leader to update itself */ term: number; - /** True if follower contained entry matching prevLogIndex/prevLogTerm */ success: boolean; - /** Follower's last replicated index (for leader's matchIndex tracking) */ matchIndex: number; } -/** --- Raft InstallSnapshot (for followers that fell too far behind) --- */ export interface InstallSnapshotRequest { term: number; leaderId: string; lastIncludedIndex: number; lastIncludedTerm: number; - /** Byte offset of this chunk in the file */ offset: number; - /** Raw zip chunk payload */ data: Uint8Array; - /** True if this is the final chunk */ done: boolean; } @@ -1061,6 +1006,437 @@ export const ProduceBatchResponse: MessageFns = { }, }; +function createBaseAtomicBatchTopicSlice(): AtomicBatchTopicSlice { + return { topic: "", entries: [] }; +} + +export const AtomicBatchTopicSlice: MessageFns = { + encode(message: AtomicBatchTopicSlice, writer: BinaryWriter = new BinaryWriter()): BinaryWriter { + if (message.topic !== "") { + writer.uint32(10).string(message.topic); + } + for (const v of message.entries) { + ProduceBatchRequest_BatchEntry.encode(v!, writer.uint32(18).fork()).join(); + } + return writer; + }, + + decode(input: BinaryReader | Uint8Array, length?: number): AtomicBatchTopicSlice { + const reader = input instanceof BinaryReader ? input : new BinaryReader(input); + const end = length === undefined ? reader.len : reader.pos + length; + const message = createBaseAtomicBatchTopicSlice(); + while (reader.pos < end) { + const tag = reader.uint32(); + switch (tag >>> 3) { + case 1: { + if (tag !== 10) { + break; + } + + message.topic = reader.string(); + continue; + } + case 2: { + if (tag !== 18) { + break; + } + + message.entries.push(ProduceBatchRequest_BatchEntry.decode(reader, reader.uint32())); + continue; + } + } + if ((tag & 7) === 4 || tag === 0) { + break; + } + reader.skip(tag & 7); + } + return message; + }, + + fromJSON(object: any): AtomicBatchTopicSlice { + return { + topic: isSet(object.topic) ? globalThis.String(object.topic) : "", + entries: globalThis.Array.isArray(object?.entries) + ? object.entries.map((e: any) => ProduceBatchRequest_BatchEntry.fromJSON(e)) + : [], + }; + }, + + toJSON(message: AtomicBatchTopicSlice): unknown { + const obj: any = {}; + if (message.topic !== "") { + obj.topic = message.topic; + } + if (message.entries?.length) { + obj.entries = message.entries.map((e) => ProduceBatchRequest_BatchEntry.toJSON(e)); + } + return obj; + }, + + create, I>>(base?: I): AtomicBatchTopicSlice { + return AtomicBatchTopicSlice.fromPartial(base ?? ({} as any)); + }, + fromPartial, I>>(object: I): AtomicBatchTopicSlice { + const message = createBaseAtomicBatchTopicSlice(); + message.topic = object.topic ?? ""; + message.entries = object.entries?.map((e) => ProduceBatchRequest_BatchEntry.fromPartial(e)) || []; + return message; + }, +}; + +function createBaseAtomicBatchRequest(): AtomicBatchRequest { + return { slices: [] }; +} + +export const AtomicBatchRequest: MessageFns = { + encode(message: AtomicBatchRequest, writer: BinaryWriter = new BinaryWriter()): BinaryWriter { + for (const v of message.slices) { + AtomicBatchTopicSlice.encode(v!, writer.uint32(10).fork()).join(); + } + return writer; + }, + + decode(input: BinaryReader | Uint8Array, length?: number): AtomicBatchRequest { + const reader = input instanceof BinaryReader ? input : new BinaryReader(input); + const end = length === undefined ? reader.len : reader.pos + length; + const message = createBaseAtomicBatchRequest(); + while (reader.pos < end) { + const tag = reader.uint32(); + switch (tag >>> 3) { + case 1: { + if (tag !== 10) { + break; + } + + message.slices.push(AtomicBatchTopicSlice.decode(reader, reader.uint32())); + continue; + } + } + if ((tag & 7) === 4 || tag === 0) { + break; + } + reader.skip(tag & 7); + } + return message; + }, + + fromJSON(object: any): AtomicBatchRequest { + return { + slices: globalThis.Array.isArray(object?.slices) + ? object.slices.map((e: any) => AtomicBatchTopicSlice.fromJSON(e)) + : [], + }; + }, + + toJSON(message: AtomicBatchRequest): unknown { + const obj: any = {}; + if (message.slices?.length) { + obj.slices = message.slices.map((e) => AtomicBatchTopicSlice.toJSON(e)); + } + return obj; + }, + + create, I>>(base?: I): AtomicBatchRequest { + return AtomicBatchRequest.fromPartial(base ?? ({} as any)); + }, + fromPartial, I>>(object: I): AtomicBatchRequest { + const message = createBaseAtomicBatchRequest(); + message.slices = object.slices?.map((e) => AtomicBatchTopicSlice.fromPartial(e)) || []; + return message; + }, +}; + +function createBaseAtomicProduceRequest(): AtomicProduceRequest { + return { slices: [] }; +} + +export const AtomicProduceRequest: MessageFns = { + encode(message: AtomicProduceRequest, writer: BinaryWriter = new BinaryWriter()): BinaryWriter { + for (const v of message.slices) { + AtomicBatchTopicSlice.encode(v!, writer.uint32(10).fork()).join(); + } + return writer; + }, + + decode(input: BinaryReader | Uint8Array, length?: number): AtomicProduceRequest { + const reader = input instanceof BinaryReader ? input : new BinaryReader(input); + const end = length === undefined ? reader.len : reader.pos + length; + const message = createBaseAtomicProduceRequest(); + while (reader.pos < end) { + const tag = reader.uint32(); + switch (tag >>> 3) { + case 1: { + if (tag !== 10) { + break; + } + + message.slices.push(AtomicBatchTopicSlice.decode(reader, reader.uint32())); + continue; + } + } + if ((tag & 7) === 4 || tag === 0) { + break; + } + reader.skip(tag & 7); + } + return message; + }, + + fromJSON(object: any): AtomicProduceRequest { + return { + slices: globalThis.Array.isArray(object?.slices) + ? object.slices.map((e: any) => AtomicBatchTopicSlice.fromJSON(e)) + : [], + }; + }, + + toJSON(message: AtomicProduceRequest): unknown { + const obj: any = {}; + if (message.slices?.length) { + obj.slices = message.slices.map((e) => AtomicBatchTopicSlice.toJSON(e)); + } + return obj; + }, + + create, I>>(base?: I): AtomicProduceRequest { + return AtomicProduceRequest.fromPartial(base ?? ({} as any)); + }, + fromPartial, I>>(object: I): AtomicProduceRequest { + const message = createBaseAtomicProduceRequest(); + message.slices = object.slices?.map((e) => AtomicBatchTopicSlice.fromPartial(e)) || []; + return message; + }, +}; + +function createBaseAtomicProduceResponse(): AtomicProduceResponse { + return { success: false, baseOffsets: {}, errorMessage: "", errorCode: 0 }; +} + +export const AtomicProduceResponse: MessageFns = { + encode(message: AtomicProduceResponse, writer: BinaryWriter = new BinaryWriter()): BinaryWriter { + if (message.success !== false) { + writer.uint32(8).bool(message.success); + } + globalThis.Object.entries(message.baseOffsets).forEach(([key, value]: [string, number]) => { + AtomicProduceResponse_BaseOffsetsEntry.encode({ key: key as any, value }, writer.uint32(18).fork()).join(); + }); + if (message.errorMessage !== "") { + writer.uint32(26).string(message.errorMessage); + } + if (message.errorCode !== 0) { + writer.uint32(32).int32(message.errorCode); + } + return writer; + }, + + decode(input: BinaryReader | Uint8Array, length?: number): AtomicProduceResponse { + const reader = input instanceof BinaryReader ? input : new BinaryReader(input); + const end = length === undefined ? reader.len : reader.pos + length; + const message = createBaseAtomicProduceResponse(); + while (reader.pos < end) { + const tag = reader.uint32(); + switch (tag >>> 3) { + case 1: { + if (tag !== 8) { + break; + } + + message.success = reader.bool(); + continue; + } + case 2: { + if (tag !== 18) { + break; + } + + const entry2 = AtomicProduceResponse_BaseOffsetsEntry.decode(reader, reader.uint32()); + if (entry2.value !== undefined) { + message.baseOffsets[entry2.key] = entry2.value; + } + continue; + } + case 3: { + if (tag !== 26) { + break; + } + + message.errorMessage = reader.string(); + continue; + } + case 4: { + if (tag !== 32) { + break; + } + + message.errorCode = reader.int32() as any; + continue; + } + } + if ((tag & 7) === 4 || tag === 0) { + break; + } + reader.skip(tag & 7); + } + return message; + }, + + fromJSON(object: any): AtomicProduceResponse { + return { + success: isSet(object.success) ? globalThis.Boolean(object.success) : false, + baseOffsets: isObject(object.baseOffsets) + ? (globalThis.Object.entries(object.baseOffsets) as [string, any][]).reduce( + (acc: { [key: string]: number }, [key, value]: [string, any]) => { + acc[key] = globalThis.Number(value); + return acc; + }, + {}, + ) + : isObject(object.base_offsets) + ? (globalThis.Object.entries(object.base_offsets) as [string, any][]).reduce( + (acc: { [key: string]: number }, [key, value]: [string, any]) => { + acc[key] = globalThis.Number(value); + return acc; + }, + {}, + ) + : {}, + errorMessage: isSet(object.errorMessage) + ? globalThis.String(object.errorMessage) + : isSet(object.error_message) + ? globalThis.String(object.error_message) + : "", + errorCode: isSet(object.errorCode) + ? errorCodeFromJSON(object.errorCode) + : isSet(object.error_code) + ? errorCodeFromJSON(object.error_code) + : 0, + }; + }, + + toJSON(message: AtomicProduceResponse): unknown { + const obj: any = {}; + if (message.success !== false) { + obj.success = message.success; + } + if (message.baseOffsets) { + const entries = globalThis.Object.entries(message.baseOffsets) as [string, number][]; + if (entries.length > 0) { + obj.baseOffsets = {}; + entries.forEach(([k, v]) => { + obj.baseOffsets[k] = Math.round(v); + }); + } + } + if (message.errorMessage !== "") { + obj.errorMessage = message.errorMessage; + } + if (message.errorCode !== 0) { + obj.errorCode = errorCodeToJSON(message.errorCode); + } + return obj; + }, + + create, I>>(base?: I): AtomicProduceResponse { + return AtomicProduceResponse.fromPartial(base ?? ({} as any)); + }, + fromPartial, I>>(object: I): AtomicProduceResponse { + const message = createBaseAtomicProduceResponse(); + message.success = object.success ?? false; + message.baseOffsets = (globalThis.Object.entries(object.baseOffsets ?? {}) as [string, number][]).reduce( + (acc: { [key: string]: number }, [key, value]: [string, number]) => { + if (value !== undefined) { + acc[key] = globalThis.Number(value); + } + return acc; + }, + {}, + ); + message.errorMessage = object.errorMessage ?? ""; + message.errorCode = object.errorCode ?? 0; + return message; + }, +}; + +function createBaseAtomicProduceResponse_BaseOffsetsEntry(): AtomicProduceResponse_BaseOffsetsEntry { + return { key: "", value: 0 }; +} + +export const AtomicProduceResponse_BaseOffsetsEntry: MessageFns = { + encode(message: AtomicProduceResponse_BaseOffsetsEntry, writer: BinaryWriter = new BinaryWriter()): BinaryWriter { + if (message.key !== "") { + writer.uint32(10).string(message.key); + } + if (message.value !== 0) { + writer.uint32(16).int64(message.value); + } + return writer; + }, + + decode(input: BinaryReader | Uint8Array, length?: number): AtomicProduceResponse_BaseOffsetsEntry { + const reader = input instanceof BinaryReader ? input : new BinaryReader(input); + const end = length === undefined ? reader.len : reader.pos + length; + const message = createBaseAtomicProduceResponse_BaseOffsetsEntry(); + while (reader.pos < end) { + const tag = reader.uint32(); + switch (tag >>> 3) { + case 1: { + if (tag !== 10) { + break; + } + + message.key = reader.string(); + continue; + } + case 2: { + if (tag !== 16) { + break; + } + + message.value = longToNumber(reader.int64()); + continue; + } + } + if ((tag & 7) === 4 || tag === 0) { + break; + } + reader.skip(tag & 7); + } + return message; + }, + + fromJSON(object: any): AtomicProduceResponse_BaseOffsetsEntry { + return { + key: isSet(object.key) ? globalThis.String(object.key) : "", + value: isSet(object.value) ? globalThis.Number(object.value) : 0, + }; + }, + + toJSON(message: AtomicProduceResponse_BaseOffsetsEntry): unknown { + const obj: any = {}; + if (message.key !== "") { + obj.key = message.key; + } + if (message.value !== 0) { + obj.value = Math.round(message.value); + } + return obj; + }, + + create, I>>( + base?: I, + ): AtomicProduceResponse_BaseOffsetsEntry { + return AtomicProduceResponse_BaseOffsetsEntry.fromPartial(base ?? ({} as any)); + }, + fromPartial, I>>( + object: I, + ): AtomicProduceResponse_BaseOffsetsEntry { + const message = createBaseAtomicProduceResponse_BaseOffsetsEntry(); + message.key = object.key ?? ""; + message.value = object.value ?? 0; + return message; + }, +}; + function createBaseStoredMessage(): StoredMessage { return { offset: 0, topic: "", payload: new Uint8Array(0), key: undefined, timestamp: 0, storedAt: 0 }; } @@ -3445,6 +3821,10 @@ function longToNumber(int64: { toString(): string }): number { return num; } +function isObject(value: any): boolean { + return typeof value === "object" && value !== null; +} + function isSet(value: any): boolean { return value !== null && value !== undefined; }