vectordotdev · scMarkus · May 27, 2026 · Aug 14, 2026 · Aug 14, 2026 · Aug 14, 2026
@@ -57,6 +57,7 @@ fn build_file_benchmark_environment(
             truncate: Default::default(),
             base_dir: None,
             confinement: Default::default(),
+            batch: Default::default(),
         },
     );
 
@@ -129,6 +130,7 @@ fn benchmark_files_no_partitions(c: &mut Criterion) {
                         truncate: Default::default(),
                         base_dir: None,
                         confinement: Default::default(),
+                        batch: Default::default(),
                     },
                 );
 

@@ -0,0 +1,23 @@
+The `file` sink now batches events per destination path before writing. Events sharing the
+same rendered path are accumulated into a single buffer and flushed with one write syscall
+per batch, rather than one syscall per event.
+
+This significantly reduces overhead when routing to many partitions — for example, writing
+one file per Kafka topic with a path template like `/data/topics/{{ _topic }}/events.log`.
+Throughput on a single file improves ~10x; high-partition workloads (64 topics) improve ~13%.
+
+Batching is controlled by the new `batch` configuration block:
+
+```yaml
+sinks:
+  file_out:
+    type: file
+    path: /data/topics/{{ _topic }}/events.log
+    batch:
+      max_bytes: 10485760   # 10 MiB (default)
+      timeout_secs: 1       # flush after 1 second of inactivity (default)
+```
+
+Issue: https://github.com/vectordotdev/vector/issues/20394
+
+authors: mbergman
@@ -385,7 +385,7 @@ generated: components: sinks: aws_s3: configuration: {
 			"""
 		required: false
 		type: string: examples: [
-			"gzip",
+			"gzip"
 		]
 	}
 	content_type: {
@@ -884,7 +884,7 @@ generated: components: sinks: aws_s3: configuration: {
 			"""
 		required: false
 		type: string: examples: [
-			"json",
+			"json"
 		]
 	}
 	filename_time_format: {

@@ -37,7 +37,7 @@ generated: components: sinks: blackhole: configuration: {
 		type: uint: {
 			default: 0
 			examples: [
-				10,
+				10
 			]
 			unit: "seconds"
 		}
@@ -50,7 +50,7 @@ generated: components: sinks: blackhole: configuration: {
 			"""
 		required: false
 		type: uint: examples: [
-			1000,
+			1000
 		]
 	}
 }
@@ -870,7 +870,7 @@ generated: components: sinks: doris: configuration: {
 		type: string: {
 			default: "vector"
 			examples: [
-				"vector",
+				"vector"
 			]
 		}
 	}

@@ -41,6 +41,45 @@ generated: components: sinks: file: configuration: {
 		required: false
 		type: string: examples: ["/var/log/vector"]
 	}
+	batch: {
+		description: """
+			Controls how events are batched per destination file before writing.
+
+			Events sharing the same rendered path are accumulated into a single buffer and written
+			with one syscall per batch, reducing overhead when routing to many partitions
+			(for example, one file per Kafka topic). The default timeout is 1 second; raising it
+			increases throughput at the cost of end-to-end latency.
+			"""
+		required: false
+		type: object: options: {
+			max_bytes: {
+				description: """
+					The maximum size of a batch that is processed by a sink.
+
+					This is based on the uncompressed size of the batched events, before they are
+					serialized or compressed.
+					"""
+				required: false
+				type: uint: {
+					default: 10000000
+					unit:    "bytes"
+				}
+			}
+			max_events: {
+				description: "The maximum size of a batch before it is flushed."
+				required:    false
+				type: uint: unit: "events"
+			}
+			timeout_secs: {
+				description: "The maximum age of a batch before it is flushed."
+				required:    false
+				type: float: {
+					default: 1.0
+					unit:    "seconds"
+				}
+			}
+		}
+	}
 	compression: {
 		description: "Compression configuration."
 		required:    false
@@ -596,7 +635,7 @@ generated: components: sinks: file: configuration: {
 		type: uint: {
 			default: 30
 			examples: [
-				600,
+				600
 			]
 			unit: "seconds"
 		}

@@ -75,7 +75,7 @@ generated: components: sinks: greptimedb: configuration: {
 		type: string: {
 			default: "public"
 			examples: [
-				"public",
+				"public"
 			]
 		}
 	}

@@ -124,7 +124,7 @@ generated: components: sinks: greptimedb_logs: configuration: {
 		type: string: {
 			default: "public"
 			examples: [
-				"public",
+				"public"
 			]
 			syntax: "template"
 		}

@@ -75,7 +75,7 @@ generated: components: sinks: greptimedb_metrics: configuration: {
 		type: string: {
 			default: "public"
 			examples: [
-				"public",
+				"public"
 			]
 		}
 	}

@@ -845,7 +845,7 @@ generated: components: sinks: http: configuration: {
 		type: string: {
 			default: ""
 			examples: [
-				"}",
+				"}"
 			]
 		}
 	}

@@ -153,7 +153,7 @@ generated: components: sinks: influxdb_logs: configuration: {
 			"""
 		required: false
 		type: string: examples: [
-			"text",
+			"text"
 		]
 	}
 	org: {
@@ -384,7 +384,7 @@ generated: components: sinks: influxdb_logs: configuration: {
 			"""
 		required: false
 		type: string: examples: [
-			"source",
+			"source"
 		]
 	}
 	tags: {

@@ -70,7 +70,7 @@ generated: components: sinks: mezmo: configuration: {
 		type: string: {
 			default: "vector"
 			examples: [
-				"my-app",
+				"my-app"
 			]
 		}
 	}

@@ -123,7 +123,7 @@ generated: components: sinks: nats: configuration: {
 		type: string: {
 			default: "vector"
 			examples: [
-				"foo",
+				"foo"
 			]
 		}
 	}

@@ -593,7 +593,7 @@ generated: components: sinks: socket: configuration: {
 		required:      false
 		type: uint: {
 			examples: [
-				65536,
+				65536
 			]
 			unit: "bytes"
 		}

@@ -122,7 +122,7 @@ generated: components: sinks: statsd: configuration: {
 		required: false
 		type: uint: {
 			examples: [
-				65536,
+				65536
 			]
 			unit: "bytes"
 		}

@@ -661,7 +661,7 @@ generated: components: sinks: websocket: configuration: {
 		required: false
 		type: uint: {
 			examples: [
-				30,
+				30
 			]
 			unit: "seconds"
 		}
@@ -677,7 +677,7 @@ generated: components: sinks: websocket: configuration: {
 		required: false
 		type: uint: {
 			examples: [
-				5,
+				5
 			]
 			unit: "seconds"
 		}

@@ -616,7 +616,7 @@ generated: components: sources: amqp: configuration: {
 			"""
 		required: false
 		type: uint: examples: [
-			100,
+			100
 		]
 	}
 	queue: {

@@ -83,7 +83,7 @@ generated: components: sources: file: configuration: {
 		type: string: {
 			default: "file"
 			examples: [
-				"path",
+				"path"
 			]
 		}
 	}
@@ -197,7 +197,7 @@ generated: components: sources: file: configuration: {
 		required:    false
 		type: uint: {
 			examples: [
-				600,
+				600
 			]
 			unit: "seconds"
 		}
@@ -227,7 +227,7 @@ generated: components: sources: file: configuration: {
 		type: string: {
 			default: "\n"
 			examples: [
-				"\r\n",
+				"\r\n"
 			]
 		}
 	}
@@ -337,7 +337,7 @@ generated: components: sources: file: configuration: {
 			"""
 		required: false
 		type: string: examples: [
-			"offset",
+			"offset"
 		]
 	}
 	oldest_first: {

@@ -319,7 +319,7 @@ generated: components: sources: file_descriptor: configuration: {
 		description: "The file descriptor number to read from."
 		required:    true
 		type: uint: examples: [
-			10,
+			10
 		]
 	}
 	framing: {

@@ -84,7 +84,7 @@ generated: components: sources: fluent: configuration: {
 		required:      false
 		type: uint: {
 			examples: [
-				65536,
+				65536
 			]
 			unit: "bytes"
 		}

@@ -736,7 +736,7 @@ generated: components: sources: http: configuration: {
 		type: uint: {
 			default: 200
 			examples: [
-				202,
+				202
 			]
 		}
 	}

@@ -736,7 +736,7 @@ generated: components: sources: http_server: configuration: {
 		type: uint: {
 			default: 200
 			examples: [
-				202,
+				202
 			]
 		}
 	}

@@ -701,7 +701,7 @@ generated: components: sources: kafka: configuration: {
 		type: string: {
 			default: "offset"
 			examples: [
-				"offset",
+				"offset"
 			]
 		}
 	}
@@ -891,7 +891,7 @@ generated: components: sources: kafka: configuration: {
 		type: string: {
 			default: "topic"
 			examples: [
-				"topic",
+				"topic"
 			]
 		}
 	}

@@ -128,7 +128,7 @@ generated: components: sources: kubernetes_logs: configuration: {
 		required:    false
 		type: uint: {
 			examples: [
-				600,
+				600
 			]
 			unit: "seconds"
 		}
@@ -138,7 +138,7 @@ generated: components: sources: kubernetes_logs: configuration: {
 		required:    false
 		type: array: {
 			default: [
-				"**/*",
+				"**/*"
 			]
 			items: type: string: examples: ["**/include/**"]
 		}

@@ -56,7 +56,7 @@ generated: components: sources: logstash: configuration: {
 		required:    false
 		type: uint: {
 			examples: [
-				65536,
+				65536
 			]
 			unit: "bytes"
 		}

@@ -700,7 +700,7 @@ generated: components: sources: mqtt: configuration: {
 		type: string: {
 			default: "topic"
 			examples: [
-				"topic",
+				"topic"
 			]
 		}
 	}

@@ -95,7 +95,7 @@ generated: components: sources: nats: configuration: {
 			"""
 		required: true
 		type: string: examples: [
-			"vector",
+			"vector"
 		]
 	}
 	decoding: {
-Original file line number
+Diff line change
@@ Expand Up / @@ -845,7 +845,7 @@ generated: components: sinks: http: configuration: { @@
     		type: string: {
     			default: ""
     			examples: [
-    				"}",
+    				"}"
     			]
     		}
     	}
@@ Expand Down @@