From 59325054c0781d193527532a0e0c8f194c6d3af5 Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Tue, 11 Aug 2026 16:33:25 +0100 Subject: [PATCH 01/18] feat(datadog metrics sink): port V3 columnar protobuf series protocol from stephen/v3_vector Ports the V2+V3 dual-write datadog_metrics sink functionality (V3 columnar protobuf series encoder using the saluki datadog-agent-metrics-v3 crate, X-Metrics-Request-ID correlation, sketch shadow support, and related config/request-builder/service changes) from stephen/v3_vector onto the v0.57.0 release tag. This is the combined diff between 2567199d8d (the commit stephen/v3_vector branched the v3 work from) and d9a0a45474 (tip of stephen/v3_vector), applied on top of v0.57.0. --- Cargo.lock | 32 + Cargo.toml | 8 +- .../1_datadog_metrics_v3.enhancement.md | 5 + src/internal_events/datadog_metrics.rs | 28 + src/sinks/datadog/metrics/config.rs | 153 ++++- src/sinks/datadog/metrics/encoder.rs | 28 +- src/sinks/datadog/metrics/encoder_v3.rs | 630 ++++++++++++++++++ src/sinks/datadog/metrics/mod.rs | 1 + src/sinks/datadog/metrics/request_builder.rs | 613 +++++++++++++---- src/sinks/datadog/metrics/service.rs | 68 +- .../sinks/generated/datadog_metrics.cue | 61 ++ 11 files changed, 1473 insertions(+), 154 deletions(-) create mode 100644 changelog.d/1_datadog_metrics_v3.enhancement.md create mode 100644 src/sinks/datadog/metrics/encoder_v3.rs diff --git a/Cargo.lock b/Cargo.lock index 52f066efe02a0..f0946f7e610e5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3432,6 +3432,15 @@ dependencies = [ "tracing 0.1.44", ] +[[package]] +name = "datadog-agent-metrics-v3" +version = "0.1.0" +source = "git+ssh://git@github.com/DataDog/saluki.git?tag=1.3.0#f546aa02aaaef60037c7b24f44756e34a3dcfa3f" +dependencies = [ + "foldhash 0.2.0", + "protobuf", +] + [[package]] name = "dbl" version = "0.3.2" @@ -8780,6 +8789,27 @@ dependencies = [ "prost 0.14.3", ] +[[package]] +name = "protobuf" +version = "3.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d65a1d4ddae7d8b5de68153b48f6aa3bba8cb002b243dbdbc55a5afbc98f99f4" +dependencies = [ + "bytes", + "once_cell", + "protobuf-support", + "thiserror 1.0.68", +] + +[[package]] +name = "protobuf-support" +version = "3.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e36c2f31e0a47f9280fb347ef5e461ffcd2c52dd520d8e216b52f93b0b0d7d6" +dependencies = [ + "thiserror 1.0.68", +] + [[package]] name = "protoc-bin-vendored" version = "3.2.0" @@ -12998,6 +13028,7 @@ dependencies = [ "csv", "databend-client", "databricks-zerobus-ingest-sdk", + "datadog-agent-metrics-v3", "deadpool 0.13.0", "derivative", "dirs-next", @@ -13073,6 +13104,7 @@ dependencies = [ "prost-build 0.12.6", "prost-reflect", "prost-types 0.12.6", + "protobuf", "pulsar", "quick-junit", "quick-xml 0.31.0", diff --git a/Cargo.toml b/Cargo.toml index 81391b34b0048..23b82f408c131 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -146,6 +146,8 @@ await_holding_lock = "warn" let_underscore_must_use = "warn" [workspace.dependencies] +datadog-agent-metrics-v3 = { git = "ssh://git@github.com/DataDog/saluki.git", tag = "1.3.0" } +protobuf = { version = "3.7", default-features = false, features = ["with-bytes"] } antithesis-instrumentation = { version = "0.1", default-features = false, features = [] } antithesis_sdk = { version = "0.2", default-features = false, features = [] } anyhow = { version = "1.0.102", default-features = false, features = ["std"] } @@ -348,6 +350,10 @@ rmpv = { version = "1.3.0", default-features = false, features = ["with-serde"], # Prost / Protocol Buffers prost = { workspace = true, optional = true } prost-reflect = { workspace = true, optional = true } + +# Datadog metrics V3 columnar codec +datadog-agent-metrics-v3 = { workspace = true, optional = true } +protobuf = { workspace = true, optional = true } prost-types = { workspace = true, optional = true } # Databricks Zerobus @@ -950,7 +956,7 @@ sinks-databend = ["dep:databend-client"] sinks-databricks-zerobus = ["dep:databricks-zerobus-ingest-sdk", "codecs-arrow", "arrow/ipc_compression"] sinks-datadog_events = [] sinks-datadog_logs = [] -sinks-datadog_metrics = ["protobuf-build", "dep:prost", "dep:prost-reflect"] +sinks-datadog_metrics = ["protobuf-build", "dep:prost", "dep:prost-reflect", "dep:datadog-agent-metrics-v3", "dep:protobuf"] sinks-datadog_traces = ["protobuf-build", "dep:prost", "dep:rmpv", "dep:rmp-serde", "dep:serde_bytes"] sinks-doris = ["sqlx/mysql"] sinks-elasticsearch = ["transforms-metric_to_log"] diff --git a/changelog.d/1_datadog_metrics_v3.enhancement.md b/changelog.d/1_datadog_metrics_v3.enhancement.md new file mode 100644 index 0000000000000..799e532de7307 --- /dev/null +++ b/changelog.d/1_datadog_metrics_v3.enhancement.md @@ -0,0 +1,5 @@ +Adds the a new encoder to the Datadog metrics sink to encode metrics with v3 of +the payload protocol. An additional option `dual_write` will make Vector send +duplicate payloads to the given endpoint encoded with the configured protocol. +This allows the Datadog backend to validate that the metrics send via both +protocols specify the exact same metrics. diff --git a/src/internal_events/datadog_metrics.rs b/src/internal_events/datadog_metrics.rs index 00e7d50a8a7d7..495bff29fa38d 100644 --- a/src/internal_events/datadog_metrics.rs +++ b/src/internal_events/datadog_metrics.rs @@ -35,3 +35,31 @@ impl InternalEvent for DatadogMetricsEncodingError<'_> { } } } + +/// Fired on every failed attempt to send a Datadog metrics request (including ones that +/// will be retried), tagged with the request's `batch_id` and target `uri` so a specific +/// failure can be correlated with a Datadog-side error such as +/// `[] MISMATCH (timeout): incomplete payloads` from V3 shadow-write validation. +/// +/// This is diagnostic logging only — it does not increment `component_errors_total`, since +/// the generic request driver already counts the final, post-retry failure via `CallError`. +#[derive(Debug, NamedInternalEvent)] +pub struct DatadogMetricsRequestError<'a> { + pub error: &'a str, + pub batch_id: Option<&'a str>, + pub uri: &'a http::Uri, +} + +impl InternalEvent for DatadogMetricsRequestError<'_> { + fn emit(self) { + warn!( + message = "Failed to send Datadog metrics request.", + error = self.error, + error_type = error_type::REQUEST_FAILED, + stage = error_stage::SENDING, + batch_id = self.batch_id.unwrap_or("none"), + uri = %self.uri, + internal_log_rate_limit = false, + ); + } +} diff --git a/src/sinks/datadog/metrics/config.rs b/src/sinks/datadog/metrics/config.rs index a5fdccede6a14..76719aa95876b 100644 --- a/src/sinks/datadog/metrics/config.rs +++ b/src/sinks/datadog/metrics/config.rs @@ -1,3 +1,5 @@ +use std::num::NonZeroU64; + use http::Uri; use snafu::ResultExt; use tower::ServiceBuilder; @@ -6,7 +8,7 @@ use vector_lib::{ }; use super::{ - request_builder::DatadogMetricsRequestBuilder, + request_builder::{DatadogMetricsRequestBuilder, ShadowBuilderConfig}, service::{DatadogMetricsRetryLogic, DatadogMetricsService}, sink::DatadogMetricsSink, }; @@ -34,7 +36,13 @@ impl SinkBatchSettings for DatadogMetricsDefaultBatchSettings { pub(super) const SERIES_V1_PATH: &str = "/api/v1/series"; pub(super) const SERIES_V2_PATH: &str = "/api/v2/series"; +pub(super) const SERIES_V3_PATH: &str = "/api/intake/metrics/v3/series"; +/// Beta intake endpoint used during V3 shadow rollout. +pub(super) const SERIES_V3_BETA_PATH: &str = "/api/intake/metrics/v3beta/series"; pub(super) const SKETCHES_PATH: &str = "/api/beta/sketches"; +pub(super) const SKETCHES_V3_PATH: &str = "/api/intake/metrics/v3/sketches"; +/// Beta intake endpoint used during V3 sketches shadow rollout. +pub(super) const SKETCHES_V3_BETA_PATH: &str = "/api/intake/metrics/v3beta/sketches"; /// The API version to use when submitting series metrics to Datadog. #[configurable_component] @@ -52,15 +60,69 @@ pub enum SeriesApiVersion { /// This is the recommended and default endpoint. #[default] V2, + + /// Use the v3 series endpoint (`/api/intake/metrics/v3beta/series`). + /// + /// Columnar protobuf format with dictionary-based string deduplication and delta + /// encoding. More efficient than v2 for workloads with many metrics that share + /// common tags or names. + V3, + + /// Use the v3 beta intake endpoint (`/api/intake/metrics/v3beta/series`). + /// + /// Used for shadow/validation rollout of V3. Prefer `v3_intake` for stable usage. + V3Beta, } impl SeriesApiVersion { - pub const fn get_path(self) -> &'static str { + pub const fn get_path(&self) -> &'static str { match self { Self::V1 => SERIES_V1_PATH, Self::V2 => SERIES_V2_PATH, + Self::V3 => SERIES_V3_PATH, + Self::V3Beta => SERIES_V3_BETA_PATH, } } + + /// Returns true if this version uses the V3 columnar encoding format. + pub const fn is_v3_format(self) -> bool { + matches!(self, Self::V3 | Self::V3Beta) + } +} + +/// The API version to use when submitting sketch metrics (distributions, histograms) to Datadog. +/// +/// Independent of `series_api_version`: Datadog's intake gates V3 series and V3 sketches +/// separately, so enabling one does not enable the other. +#[configurable_component] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +#[serde(rename_all = "snake_case")] +pub enum SketchesApiVersion { + /// Use the legacy sketches endpoint (`/api/beta/sketches`). + /// + /// This is the recommended and default endpoint. + #[default] + V2, + + /// Use the v3 sketches endpoint (`/api/intake/metrics/v3/sketches`). + /// + /// Columnar protobuf format, matching the encoding used for V3 series. Must be enabled + /// separately from `series_api_version`. + V3, +} + +impl SketchesApiVersion { + pub const fn get_path(self) -> &'static str { + match self { + Self::V2 => SKETCHES_PATH, + Self::V3 => SKETCHES_V3_PATH, + } + } + + /// Returns true if this version uses the V3 columnar encoding format. + pub const fn is_v3_format(self) -> bool { + matches!(self, Self::V3) + } } /// Various metric type-specific API types. @@ -83,7 +145,7 @@ impl DatadogMetricsEndpoint { pub const fn content_type(self) -> &'static str { match self { Self::Series(SeriesApiVersion::V1) => "application/json", - Self::Sketches | Self::Series(SeriesApiVersion::V2) => "application/x-protobuf", + _ => "application/x-protobuf", } } @@ -100,6 +162,11 @@ impl DatadogMetricsEndpoint { 5_242_880, // 5 MiB 512_000, // 512 KB ), + // V3/V3Beta all use the same limits as V2 series. + DatadogMetricsEndpoint::Series(SeriesApiVersion::V3 | SeriesApiVersion::V3Beta) => ( + 5_242_880, // 5 MiB + 512_000, // 512 KB + ), }; DatadogMetricsPayloadLimits { @@ -159,6 +226,37 @@ impl DatadogMetricsEndpointConfiguration { } } +fn default_shadow_every() -> NonZeroU64 { + NonZeroU64::new(1000).unwrap() +} + +/// Configuration for the V3 shadow dual-write mode. +/// +/// When enabled, every `shadow_every`-th legacy series flush, and every `shadow_every`-th +/// legacy sketches flush, also sends a V3 shadow payload to the corresponding shadow +/// endpoint. Both payloads in a pair carry the same `X-Metrics-Request-ID` header so the +/// intake backend can correlate them. +#[configurable_component] +#[derive(Clone, Debug)] +pub struct DualWriteConfig { + /// Send a V3 shadow payload once per this many legacy series or sketches flushes. + /// + /// Set to `1` to shadow every flush (full dual-write). Must be greater than zero. + /// Defaults to `1000`. + #[serde(default = "default_shadow_every")] + pub shadow_every: NonZeroU64, +} + +impl DualWriteConfig { + pub(super) const fn get_series_path(&self) -> &'static str { + SERIES_V3_BETA_PATH + } + + pub(super) const fn get_sketches_path(&self) -> &'static str { + SKETCHES_V3_BETA_PATH + } +} + /// Configuration for the `datadog_metrics` sink. #[configurable_component(sink("datadog_metrics", "Publish metric events to Datadog."))] #[derive(Clone, Debug, Default)] @@ -182,6 +280,17 @@ pub struct DatadogMetricsConfig { #[serde(default)] pub series_api_version: SeriesApiVersion, + /// Controls which Datadog sketches API endpoint is used to submit distributions and + /// histograms. + /// + /// Independent of `series_api_version` — Datadog's intake gates V3 series and V3 sketches + /// separately, so this must be set explicitly to send sketches via V3, even if + /// `series_api_version` is already `v3`. + /// + /// Defaults to `v2` (`/api/beta/sketches`). + #[serde(default)] + pub sketches_api_version: SketchesApiVersion, + #[configurable(derived)] #[serde(default)] pub batch: BatchConfig, @@ -189,6 +298,14 @@ pub struct DatadogMetricsConfig { #[configurable(derived)] #[serde(default)] pub request: TowerRequestConfig, + + /// Optional V3 shadow dual-write configuration. + /// + /// When set, a sampled fraction of legacy series and sketches flushes are each mirrored + /// as V3 payloads to a separate intake endpoint, both stamped with a shared + /// `X-Metrics-Request-ID`. + #[serde(default)] + pub dual_write: Option, } impl_generate_config_from_default!(DatadogMetricsConfig); @@ -243,7 +360,7 @@ impl DatadogMetricsConfig { let base_uri = self.get_base_agent_endpoint(dd_common); let series_endpoint = build_uri(&base_uri, self.series_api_version.get_path())?; - let sketches_endpoint = build_uri(&base_uri, SKETCHES_PATH)?; + let sketches_endpoint = build_uri(&base_uri, self.sketches_api_version.get_path())?; Ok(DatadogMetricsEndpointConfiguration::new( series_endpoint, @@ -287,10 +404,29 @@ impl DatadogMetricsConfig { dd_common.default_api_key.inner(), )); + let shadow_config = self + .dual_write + .as_ref() + .map(|dw| -> crate::Result { + let base_uri = self.get_base_agent_endpoint(dd_common); + let series_shadow_uri = build_uri(&base_uri, dw.get_series_path())?; + let sketches_shadow_uri = build_uri(&base_uri, dw.get_sketches_path())?; + Ok(ShadowBuilderConfig { + series_uri: series_shadow_uri, + series_api_version: SeriesApiVersion::V3Beta, + sketches_uri: sketches_shadow_uri, + default_namespace: self.default_namespace.clone(), + shadow_every: dw.shadow_every, + }) + }) + .transpose()?; + let request_builder = DatadogMetricsRequestBuilder::new( endpoint_configuration, self.default_namespace.clone(), self.series_api_version, + self.sketches_api_version, + shadow_config, ); let protocol = self.get_protocol(dd_common); @@ -385,4 +521,13 @@ mod tests { assert_eq!(series.size_limit, 1_000_000); assert_eq!(sketches.size_limit, 1_000_000); } + + // `sketches_api_version` is independent of `series_api_version`: Datadog's intake gates V3 + // series and V3 sketches separately, so each must resolve to its own path regardless of what + // the other is set to. + #[test] + fn sketches_path_is_independent_of_series_api_version() { + assert_eq!(SketchesApiVersion::V2.get_path(), SKETCHES_PATH); + assert_eq!(SketchesApiVersion::V3.get_path(), SKETCHES_V3_PATH); + } } diff --git a/src/sinks/datadog/metrics/encoder.rs b/src/sinks/datadog/metrics/encoder.rs index 762967ff6c094..38220c0109b68 100644 --- a/src/sinks/datadog/metrics/encoder.rs +++ b/src/sinks/datadog/metrics/encoder.rs @@ -7,6 +7,7 @@ use std::{ use bytes::{BufMut, Bytes}; use chrono::{DateTime, Utc}; +use datadog_agent_metrics_v3::V3EncodeError; use snafu::{ResultExt, Snafu}; use vector_lib::{ EstimatedJsonEncodedSizeOf, @@ -104,6 +105,9 @@ pub enum FinishError { metrics: Vec, recommended_splits: usize, }, + + #[snafu(display("Failed to encode V3 payload to Protocol Buffers: {}", source))] + V3EncodingFailed { source: protobuf::Error }, } impl FinishError { @@ -114,10 +118,25 @@ impl FinishError { match self { Self::CompressionFailed { .. } => "compression_failed", Self::TooLarge { .. } => "too_large", + Self::V3EncodingFailed { .. } => "v3_encoding_failed", + } + } +} + +impl From for FinishError { + fn from(err: V3EncodeError) -> Self { + FinishError::V3EncodingFailed { + source: err.into_inner(), } } } +impl From for FinishError { + fn from(source: protobuf::Error) -> Self { + FinishError::V3EncodingFailed { source } + } +} + struct EncoderState { writer: Compressor, written: usize, @@ -304,6 +323,13 @@ impl DatadogMetricsEncoder { }); } }, + // V3/V3Beta metrics must be routed to DatadogMetricsV3Encoder. + DatadogMetricsEndpoint::Series(SeriesApiVersion::V3 | SeriesApiVersion::V3Beta) => { + return Err(EncoderError::InvalidMetric { + expected: "v1 or v2 series", + metric_value: "v3", + }); + } // Sketches are encoded via ProtoBuf, also in an incremental fashion. DatadogMetricsEndpoint::Sketches => match metric.value() { MetricValue::Sketch { sketch } => match sketch { @@ -785,7 +811,7 @@ fn source_type_to_service(source_type: &str) -> Option { /// set already upstream or not. The generalized struct `DatadogMetricOriginMetadata` is /// utilized in this function, which allows the series and sketch encoding to call and map /// the result appropriately for the given protocol they operate on. -fn generate_origin_metadata( +pub(super) fn generate_origin_metadata( maybe_pass_through: Option<&DatadogMetricOriginMetadata>, maybe_source_type: Option<&str>, origin_product_value: u32, diff --git a/src/sinks/datadog/metrics/encoder_v3.rs b/src/sinks/datadog/metrics/encoder_v3.rs new file mode 100644 index 0000000000000..ec56acf68bd24 --- /dev/null +++ b/src/sinks/datadog/metrics/encoder_v3.rs @@ -0,0 +1,630 @@ +//! V3 columnar protobuf encoder for the Datadog metrics sink. +//! +//! Translates Vector's [`Metric`] events into the V3 columnar format produced by +//! [`datadog_agent_metrics_v3`]. Unlike V1/V2 (incremental per-metric serialization), +//! V3 accumulates all metrics into a [`V3Writer`] and serializes the entire batch +//! in a single call when [`DatadogMetricsV3Encoder::finish`] is invoked. This is +//! required because delta encoding applies across the whole payload. + +use std::{io::Write, mem, sync::Arc}; + +use bytes::Bytes; +use chrono::{DateTime, Utc}; +use vector_lib::{ + EstimatedJsonEncodedSizeOf, + config::{LogSchema, log_schema, telemetry}, + event::{Metric, MetricValue, metric::MetricSketch}, + metrics::AgentDDSketch, + request_metadata::GroupedCountByteSize, +}; + +use datadog_agent_metrics_v3::{V3MetricBuilder, V3MetricType, V3Writer}; +use protobuf::{CodedOutputStream, rt::WireType}; + +use super::{ + config::DatadogMetricsEndpoint, + encoder::{ + EncoderError, FinishError, ORIGIN_CATEGORY_VALUE, ORIGIN_PRODUCT_VALUE, + generate_origin_metadata, + }, +}; +use crate::sinks::util::{ + Compression, Compressor, encode_namespace, request_builder::EncodeResult, +}; + +// ── Encoder ────────────────────────────────────────────────────────────────── + +/// V3 batch encoder. Accumulates metrics; serializes on [`finish`]. +/// +/// [`finish`]: DatadogMetricsV3Encoder::finish +pub(super) struct DatadogMetricsV3Encoder { + default_namespace: Option>, + uncompressed_limit: usize, + compressed_limit: usize, + log_schema: &'static LogSchema, + writer: V3Writer, + pending: Vec, + byte_size: GroupedCountByteSize, + origin_product_value: u32, +} + +impl DatadogMetricsV3Encoder { + pub fn new(endpoint: DatadogMetricsEndpoint, default_namespace: Option) -> Self { + let limits = endpoint.payload_limits(); + Self { + default_namespace: default_namespace.map(Arc::from), + uncompressed_limit: limits.uncompressed, + compressed_limit: limits.compressed, + log_schema: log_schema(), + writer: V3Writer::new(), + pending: Vec::new(), + byte_size: telemetry().create_request_count_byte_size(), + origin_product_value: *ORIGIN_PRODUCT_VALUE, + } + } + + /// Encode one metric into the writer. + /// + /// Always returns `Ok(None)` — the V3 encoder cannot detect payload overflow + /// until `finish()` time. Caller must respect the batch event-count cap. + pub fn try_encode(&mut self, metric: Metric) -> Result, EncoderError> { + self.byte_size + .add_event(&metric, metric.estimated_json_encoded_size_of()); + encode_metric_to_v3( + &mut self.writer, + &metric, + &self.default_namespace, + self.log_schema, + self.origin_product_value, + )?; + self.pending.push(metric); + Ok(None) + } + + /// Finalize: serialize, compress, check size limits, return payload. + /// + /// On success returns `(EncodeResult, processed_metrics)`. On overflow + /// returns `FinishError::TooLarge` with the metrics and a split hint. + pub fn finish(&mut self) -> Result<(EncodeResult, Vec), FinishError> { + let writer = mem::replace(&mut self.writer, V3Writer::new()); + let metrics = mem::take(&mut self.pending); + let byte_size = mem::replace( + &mut self.byte_size, + telemetry().create_request_count_byte_size(), + ); + + if metrics.is_empty() { + // Nothing encoded — return an empty-ish result so callers don't have to special-case. + return Ok(( + EncodeResult::compressed(Bytes::new(), 0, byte_size), + Vec::new(), + )); + } + + let metric_data = writer.finalize()?.payload; + + // The wire payload isn't the bare `MetricData` message — the intake API expects it + // wrapped as field 3 (`metricData`) of the outer `Payload` message (see + // `intake_v3.proto`). Without this envelope the backend can't parse the bytes at all. + let mut header_buf = [0u8; 16]; + let header_len = { + let mut header_writer = CodedOutputStream::bytes(&mut header_buf); + header_writer.write_tag(3, WireType::LengthDelimited)?; + header_writer.write_uint64_no_tag(metric_data.len() as u64)?; + header_writer.flush()?; + header_writer.total_bytes_written() as usize + }; + + let uncompressed_size = header_len + metric_data.len(); + + // Note, V3 only supports zstd. + let mut compressor: Compressor = Compression::zstd_default().into(); + + compressor + .write_all(&header_buf[..header_len]) + .map_err(|source| FinishError::CompressionFailed { source })?; + compressor + .write_all(&metric_data) + .map_err(|source| FinishError::CompressionFailed { source })?; + let compressed = compressor + .finish() + .map_err(|source| FinishError::CompressionFailed { source })? + .freeze(); + + let compressed_splits = compressed.len() / self.compressed_limit; + let uncompressed_splits = uncompressed_size / self.uncompressed_limit; + let recommended_splits = std::cmp::max(compressed_splits, uncompressed_splits) + 1; + + if recommended_splits > 1 { + return Err(FinishError::TooLarge { + metrics, + recommended_splits, + }); + } + + Ok(( + EncodeResult::compressed(compressed, uncompressed_size, byte_size), + metrics, + )) + } +} + +// ── Metric → V3Writer ──────────────────────────────────────────────────────── + +fn encode_metric_to_v3( + writer: &mut V3Writer, + metric: &Metric, + default_namespace: &Option>, + log_schema: &LogSchema, + origin_product_value: u32, +) -> Result<(), EncoderError> { + // Mirrors V2's `series_to_proto_message`: a Counter with an interval is sent as a + // per-second-scaled Rate, not a raw Count. + let maybe_interval = metric.interval_ms().map(|i| i.get() / 1000); + + let metric_type = match metric.value() { + MetricValue::Counter { .. } if maybe_interval.is_some() => V3MetricType::Rate, + MetricValue::Counter { .. } => V3MetricType::Count, + MetricValue::Gauge { .. } => V3MetricType::Gauge, + MetricValue::Set { .. } => V3MetricType::Gauge, + MetricValue::Sketch { .. } => V3MetricType::Sketch, + // `AggregatedSummary` is split into counters/gauges, and `Distribution`/ + // `AggregatedHistogram` are converted into `Sketch(AgentDDSketch)`, by the shared + // `DatadogMetricsNormalizer` before metrics ever reach either encoder (see `sink.rs`). + // This should never happen — mirrors V2's `series_to_proto_message`, which errors + // instead of silently re-deriving a sketch with encoder-local logic that could drift + // from the normalizer's. + value @ (MetricValue::AggregatedSummary { .. } + | MetricValue::Distribution { .. } + | MetricValue::AggregatedHistogram { .. }) => { + return Err(EncoderError::InvalidMetric { + expected: "series or sketch", + metric_value: value.as_name(), + }); + } + }; + + let name = encode_namespace( + metric + .namespace() + .or_else(|| default_namespace.as_ref().map(|s| s.as_ref())), + '.', + metric.name(), + ); + + let mut builder = writer.write(metric_type, &name); + + // ── Tags & resources ──────────────────────────────────────────────────── + let mut tags_for_v3: Vec = Vec::new(); + let mut extra_resources: Vec<(&str, &str)> = Vec::new(); + // Collected separately from `extra_resources` so they can be pushed in a fixed + // host-then-device order below, matching V2 — tag iteration order is otherwise + // unspecified and shouldn't leak into wire-visible resource ordering. + let mut host_resource: Option<&str> = None; + let mut device_resource: Option<&str> = None; + + // V2 has no concept of a `dd.internal.unit` tag — it always sends the wire `unit` field + // empty. To match, we don't special-case it either: if present, it falls through to the + // generic tag handling below, same as V2. + let host_key = log_schema.host_key().map(|k| k.to_string()); + + if let Some(tags) = metric.tags() { + for (key, value) in tags.iter_all() { + // dd.internal.resource tags become structured resources + if key == "dd.internal.resource" { + if let Some(val) = value { + if let Some((rtype, rname)) = val.split_once(':') { + extra_resources.push((rtype, rname)); + } + } + continue; + } + + // Host key → host resource + if host_key.as_deref() == Some(key) { + if let Some(host) = value { + if !host.is_empty() { + host_resource = Some(host); + } + } + continue; + } + + // device / resource.device → device resource + if key == "device" || key == "resource.device" { + if let Some(dev) = value { + device_resource = Some(dev); + } + continue; + } + + // source_type_name is handled via set_source_type below + if key == "source_type_name" { + continue; + } + + match value { + Some(v) => tags_for_v3.push(format!("{}:{}", key, v)), + None => tags_for_v3.push(key.to_string()), + } + } + } + + // V2's `encode_tags` sorts tags before emitting them; tag iteration order is otherwise + // unspecified, so without this V3's tag order wouldn't match V2's. + tags_for_v3.sort(); + + // V2 always includes a host resource — even with an empty name — whenever + // `log_schema.host_key()` is configured (the default), regardless of whether the metric + // actually carries that tag. Match that instead of omitting the resource entirely. + if host_key.is_some() && host_resource.is_none() { + host_resource = Some(""); + } + + let resources = assemble_resources(host_resource, device_resource, extra_resources); + + builder.set_tags(tags_for_v3.iter().map(|s| s.as_str())); + builder.set_resources(&resources); + + // ── Source type / origin metadata ─────────────────────────────────────── + let event_metadata = metric.metadata(); + + // source_type_name tag or metadata source type → set_source_type + let source_type = metric.tags().and_then(|t| t.get("source_type_name")); + if let Some(st) = source_type { + builder.set_source_type(st); + } + + // Datadog origin metadata → set_origin + // + // Mirrors V2's `generate_origin_metadata`: use the pass-through origin if one was set + // upstream (`datadog_agent` source, `vector` source, native codecs, `log_to_metric`), else + // synthesize one from the producing Vector source's type. + if let Some(origin) = generate_origin_metadata( + event_metadata.datadog_origin_metadata(), + event_metadata.source_type(), + origin_product_value, + ) { + let product = origin.product().unwrap_or(origin_product_value); + let category = origin.category().unwrap_or(ORIGIN_CATEGORY_VALUE); + let service = origin.service().unwrap_or(0); + builder.set_origin(product, category, service, false); + } + + // Interval — matches V2, which always stamps the interval field on the message + // (`interval: maybe_interval.unwrap_or(0)`), even though only Rate uses it to scale the value. + if let Some(interval) = maybe_interval { + builder.set_interval(interval.into()); + } + + // Note: `unit` is intentionally never set — V2 always sends it empty (see + // `series_to_proto_message`'s `unit: "".to_string()`). + + // ── Data points ───────────────────────────────────────────────────────── + let timestamp = encode_timestamp(metric.timestamp()); + + match metric.value() { + MetricValue::Counter { value } => { + let value = match maybe_interval { + Some(interval) => *value / (interval as f64), + None => *value, + }; + builder.add_point(timestamp, value); + } + MetricValue::Gauge { value } => { + builder.add_point(timestamp, *value); + } + MetricValue::Set { values } => { + builder.add_point(timestamp, values.len() as f64); + } + MetricValue::Sketch { + sketch: MetricSketch::AgentDDSketch(ddsketch), + } => { + encode_ddsketch(&mut builder, ddsketch, timestamp); + } + // Unreachable: already errored out of this function via the `metric_type` match above. + MetricValue::AggregatedSummary { .. } + | MetricValue::Distribution { .. } + | MetricValue::AggregatedHistogram { .. } => { + unreachable!("filtered out by the metric_type match above") + } + } + + builder.close(); + Ok(()) +} + +/// Assembles the final resource list in a fixed host-then-device order, matching V2's +/// `encode_series_metrics`. Host/device are collected separately during tag iteration +/// (whose order is unspecified) so that order never leaks into the wire-visible resources. +fn assemble_resources<'a>( + host: Option<&'a str>, + device: Option<&'a str>, + extra: Vec<(&'a str, &'a str)>, +) -> Vec<(&'a str, &'a str)> { + let mut resources = Vec::with_capacity(extra.len() + 2); + if let Some(host) = host { + resources.push(("host", host)); + } + if let Some(dev) = device { + resources.push(("device", dev)); + } + resources.extend(extra); + resources +} + +fn encode_ddsketch(builder: &mut V3MetricBuilder<'_>, ddsketch: &AgentDDSketch, timestamp: i64) { + if ddsketch.is_empty() { + return; + } + let (bins_i16, counts_u16) = ddsketch.bin_map().into_parts(); + let bin_keys: Vec = bins_i16.into_iter().map(|k| k as i32).collect(); + let bin_counts: Vec = counts_u16.into_iter().map(|c| c as u32).collect(); + + builder.add_sketch( + timestamp, + ddsketch.count() as i64, + ddsketch.sum().unwrap_or(0.0), + ddsketch.min().unwrap_or(0.0), + ddsketch.max().unwrap_or(0.0), + &bin_keys, + &bin_counts, + ); +} + +fn encode_timestamp(ts: Option>) -> i64 { + ts.map(|t| t.timestamp()) + .unwrap_or_else(|| Utc::now().timestamp()) +} + +// ── Tests ──────────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use std::num::NonZeroU32; + + use super::super::config::SeriesApiVersion; + use super::*; + use vector_lib::event::{MetricKind, MetricValue}; + + fn gauge(name: &str, value: f64) -> Metric { + Metric::new(name, MetricKind::Absolute, MetricValue::Gauge { value }) + } + + fn counter(name: &str, value: f64) -> Metric { + Metric::new( + name, + MetricKind::Incremental, + MetricValue::Counter { value }, + ) + } + + #[test] + fn v3_gauge_encodes_non_empty_payload() { + let mut enc = DatadogMetricsV3Encoder::new( + DatadogMetricsEndpoint::Series(SeriesApiVersion::V3), + None, + ); + assert!(enc.try_encode(gauge("test.gauge", 42.0)).unwrap().is_none()); + let (result, metrics) = enc.finish().unwrap(); + assert!(!result.into_payload().is_empty()); + assert_eq!(metrics.len(), 1); + } + + #[test] + fn v3_multiple_metrics_batch() { + let mut enc = DatadogMetricsV3Encoder::new( + DatadogMetricsEndpoint::Series(SeriesApiVersion::V3), + None, + ); + for i in 0..10 { + enc.try_encode(counter("m", i as f64)).unwrap(); + } + let (_, metrics) = enc.finish().unwrap(); + assert_eq!(metrics.len(), 10); + } + + #[test] + fn v3_empty_finish_returns_empty_payload() { + let mut enc = DatadogMetricsV3Encoder::new( + DatadogMetricsEndpoint::Series(SeriesApiVersion::V3), + None, + ); + let (result, metrics) = enc.finish().unwrap(); + assert!(result.into_payload().is_empty()); + assert!(metrics.is_empty()); + } + + #[test] + fn v3_resources_ordered_host_before_device_regardless_of_input_order() { + // Regression test: V2 always emits host before device (two fixed, ordered + // lookups). V3 used to push resources in tag-iteration order, so a metric whose + // `device` tag happened to precede its `host` tag (alphabetically or otherwise) + // would encode as [device, host] — a spurious mismatch against V2 even though the + // resource set was identical. + let resources = assemble_resources(Some("myhost"), Some("/dev/loop35"), vec![]); + assert_eq!( + resources, + vec![("host", "myhost"), ("device", "/dev/loop35")] + ); + + // Order is fixed even if callers happen to discover device before host. + let resources = + assemble_resources(Some("myhost"), Some("/dev/loop35"), vec![("extra", "tag")]); + assert_eq!( + resources, + vec![ + ("host", "myhost"), + ("device", "/dev/loop35"), + ("extra", "tag") + ] + ); + } + + #[test] + fn v3_counter_with_interval_differs_from_plain_count() { + // Regression test: V2's `series_to_proto_message` sends a Counter with an interval + // as a per-second-scaled Rate, not a raw Count. V3 used to ignore `interval_ms` + // entirely, always encoding Rate-style counters as an unscaled Count — wrong metric + // type *and* wrong value. We can't decode the columnar payload here, but the encoded + // bytes for a Rate-typed, scaled point must differ from a plain Count of the same + // input value, proving the interval is actually taking effect. + let mut enc = DatadogMetricsV3Encoder::new( + DatadogMetricsEndpoint::Series(SeriesApiVersion::V3), + None, + ); + let rate_counter = Metric::new( + "rate.counter", + MetricKind::Incremental, + MetricValue::Counter { value: 100.0 }, + ) + .with_interval_ms(NonZeroU32::new(10_000)); + enc.try_encode(rate_counter).unwrap(); + let (rate_result, _) = enc.finish().unwrap(); + + let mut enc = DatadogMetricsV3Encoder::new( + DatadogMetricsEndpoint::Series(SeriesApiVersion::V3), + None, + ); + enc.try_encode(counter("rate.counter", 100.0)).unwrap(); + let (plain_result, _) = enc.finish().unwrap(); + + assert_ne!( + rate_result.into_payload(), + plain_result.into_payload(), + "a counter with an interval must encode differently than a plain count" + ); + } + + #[test] + fn v3_aggregated_summary_distribution_and_histogram_are_rejected() { + // Regression test: `AggregatedSummary` is split into counters/gauges, and + // `Distribution`/`AggregatedHistogram` are converted into `Sketch(AgentDDSketch)`, by + // the shared `DatadogMetricsNormalizer` before metrics ever reach either encoder (see + // `sink.rs`) — this should never happen. V3 used to silently re-derive a sketch inline + // instead of erroring like V2 does; now it errors too. + use vector_lib::event::metric::{Bucket, Quantile, Sample}; + + let mut enc = DatadogMetricsV3Encoder::new( + DatadogMetricsEndpoint::Series(SeriesApiVersion::V3), + None, + ); + let summary = Metric::new( + "summary", + MetricKind::Incremental, + MetricValue::AggregatedSummary { + quantiles: vec![Quantile { + quantile: 0.5, + value: 1.0, + }], + count: 1, + sum: 1.0, + }, + ); + assert!(enc.try_encode(summary).is_err()); + + let mut enc = DatadogMetricsV3Encoder::new(DatadogMetricsEndpoint::Sketches, None); + let distribution = Metric::new( + "dist", + MetricKind::Incremental, + MetricValue::Distribution { + samples: vec![Sample { + value: 1.0, + rate: 1, + }], + statistic: vector_lib::event::StatisticKind::Histogram, + }, + ); + assert!(enc.try_encode(distribution).is_err()); + + let mut enc = DatadogMetricsV3Encoder::new(DatadogMetricsEndpoint::Sketches, None); + let histogram = Metric::new( + "hist", + MetricKind::Incremental, + MetricValue::AggregatedHistogram { + buckets: vec![Bucket { + upper_limit: 1.0, + count: 1, + }], + count: 1, + sum: 1.0, + }, + ); + assert!(enc.try_encode(histogram).is_err()); + } + + #[test] + fn v3_origin_metadata_falls_back_to_source_type_when_no_pass_through() { + // Regression test: when an event has no pass-through `datadog_origin_metadata` + // (the common case for sources like `host_metrics`), V3 must synthesize origin + // metadata from the source type the same way V2's `generate_origin_metadata` does, + // instead of leaving origin unset. + let mut metric = gauge("host.cpu", 1.0); + metric.metadata_mut().set_source_type("host_metrics"); + + let mut writer = V3Writer::new(); + encode_metric_to_v3( + &mut writer, + &metric, + &None, + log_schema(), + *ORIGIN_PRODUCT_VALUE, + ) + .unwrap(); + let encoded = writer.finalize().unwrap(); + assert!(!encoded.payload.is_empty()); + + // `host_metrics` maps to OriginService 211 in V2's `source_type_to_service` table; + // V3 reuses that same mapping via the shared `generate_origin_metadata` function. + let origin = generate_origin_metadata(None, Some("host_metrics"), *ORIGIN_PRODUCT_VALUE) + .expect("host_metrics should get synthesized origin metadata"); + assert_eq!(origin.service(), Some(211)); + } + + #[test] + fn v3_namespace_prepended() { + let mut enc = DatadogMetricsV3Encoder::new( + DatadogMetricsEndpoint::Series(SeriesApiVersion::V3), + Some("myns".to_string()), + ); + enc.try_encode(gauge("latency", 1.0)).unwrap(); + let (result, _) = enc.finish().unwrap(); + assert!(!result.into_payload().is_empty()); + } + + #[test] + fn v3_set_maps_to_cardinality() { + use std::collections::BTreeSet; + let set = Metric::new( + "my.set", + MetricKind::Incremental, + MetricValue::Set { + values: BTreeSet::from(["a".to_string(), "b".to_string(), "c".to_string()]), + }, + ); + let mut enc = DatadogMetricsV3Encoder::new( + DatadogMetricsEndpoint::Series(SeriesApiVersion::V3), + None, + ); + assert!(enc.try_encode(set).unwrap().is_none()); + enc.finish().unwrap(); + } + + #[test] + fn v3_sketch_encoder_routes_correctly() { + let mut sketch = AgentDDSketch::with_agent_defaults(); + sketch.insert(1.0); + sketch.insert(2.0); + let metric = Metric::new( + "dist", + MetricKind::Incremental, + MetricValue::Sketch { + sketch: MetricSketch::AgentDDSketch(sketch), + }, + ); + let mut enc = DatadogMetricsV3Encoder::new(DatadogMetricsEndpoint::Sketches, None); + enc.try_encode(metric).unwrap(); + let (result, _) = enc.finish().unwrap(); + assert!(!result.into_payload().is_empty()); + } +} diff --git a/src/sinks/datadog/metrics/mod.rs b/src/sinks/datadog/metrics/mod.rs index 9faafc32b5980..2aa533438a704 100644 --- a/src/sinks/datadog/metrics/mod.rs +++ b/src/sinks/datadog/metrics/mod.rs @@ -1,5 +1,6 @@ mod config; mod encoder; +mod encoder_v3; mod normalizer; mod request_builder; mod service; diff --git a/src/sinks/datadog/metrics/request_builder.rs b/src/sinks/datadog/metrics/request_builder.rs index 8d5136505b6dd..14d19a1f3967a 100644 --- a/src/sinks/datadog/metrics/request_builder.rs +++ b/src/sinks/datadog/metrics/request_builder.rs @@ -1,18 +1,26 @@ -use std::sync::Arc; +use std::{num::NonZeroU64, sync::Arc}; use bytes::Bytes; +use http::Uri; use snafu::Snafu; +use uuid::Uuid; use vector_lib::{ event::{EventFinalizers, Finalizable, Metric}, request_metadata::RequestMetadata, }; use super::{ - config::{DatadogMetricsEndpoint, DatadogMetricsEndpointConfiguration, SeriesApiVersion}, + config::{ + DatadogMetricsEndpoint, DatadogMetricsEndpointConfiguration, SeriesApiVersion, + SketchesApiVersion, + }, encoder::{DatadogMetricsEncoder, EncoderError, FinishError}, + encoder_v3::DatadogMetricsV3Encoder, service::DatadogMetricsRequest, }; -use crate::sinks::util::{IncrementalRequestBuilder, metadata::RequestMetadataBuilder}; +use crate::sinks::util::{ + IncrementalRequestBuilder, metadata::RequestMetadataBuilder, request_builder::EncodeResult, +}; #[derive(Debug, Snafu)] pub enum RequestBuilderError { @@ -60,13 +68,132 @@ pub struct DDMetricsMetadata { api_key: Option>, endpoint: DatadogMetricsEndpoint, finalizers: EventFinalizers, + /// Shared transaction ID linking the V2 and V3 shadow payload from the same flush. + /// None on non-shadow flushes. + batch_id: Option>, + /// 0-based index within this flush (for split payloads). Stamped after encoding. + batch_seq: usize, + /// Total requests produced by this flush. Stamped after encoding. + batch_len: usize, + /// Overrides the URI from `endpoint_configuration`. Used for shadow V3 requests + /// that target a different path than the primary encoder. + target_uri: Option, +} + +/// Common shape of the two concrete metrics encoders, so call sites don't need to +/// know which wire format they're driving. +trait MetricsEncoder { + fn try_encode(&mut self, metric: Metric) -> Result, EncoderError>; + fn finish(&mut self) -> Result<(EncodeResult, Vec), FinishError>; +} + +impl MetricsEncoder for DatadogMetricsEncoder { + fn try_encode(&mut self, metric: Metric) -> Result, EncoderError> { + DatadogMetricsEncoder::try_encode(self, metric) + } + + fn finish(&mut self) -> Result<(EncodeResult, Vec), FinishError> { + DatadogMetricsEncoder::finish(self) + } +} + +impl MetricsEncoder for DatadogMetricsV3Encoder { + fn try_encode(&mut self, metric: Metric) -> Result, EncoderError> { + DatadogMetricsV3Encoder::try_encode(self, metric) + } + + fn finish(&mut self) -> Result<(EncodeResult, Vec), FinishError> { + DatadogMetricsV3Encoder::finish(self) + } +} + +/// Encoder dispatch: either V1/V2 incremental or V3 batch. Used uniformly for both +/// the series and sketches encoders — which variant is picked depends only on the +/// configured `SeriesApiVersion`, not on the endpoint. +enum EncoderKind { + V1V2(Box), + V3(Box), +} + +impl MetricsEncoder for EncoderKind { + fn try_encode(&mut self, metric: Metric) -> Result, EncoderError> { + match self { + Self::V1V2(enc) => enc.try_encode(metric), + Self::V3(enc) => enc.try_encode(metric), + } + } + + fn finish(&mut self) -> Result<(EncodeResult, Vec), FinishError> { + match self { + Self::V1V2(enc) => enc.finish(), + Self::V3(enc) => enc.finish(), + } + } +} + +/// Shadow write configuration passed from `DatadogMetricsConfig::build_sink`. +pub struct ShadowBuilderConfig { + /// The URI for the V3 shadow series endpoint (e.g. `/api/intake/metrics/v3/series`). + pub series_uri: Uri, + /// The `SeriesApiVersion` variant matching the shadow series endpoint. + /// Used to set the correct payload limits and compression on the shadow encoder. + pub series_api_version: SeriesApiVersion, + /// The URI for the V3 shadow sketches endpoint (e.g. `/api/intake/metrics/v3/sketches`). + pub sketches_uri: Uri, + /// Default metric namespace for the shadow encoders. + pub default_namespace: Option, + /// Send a V3 shadow once per this many legacy (V1/V2 series, or non-V3 sketches) flushes. + pub shadow_every: NonZeroU64, +} + +/// V3 shadow-write encoder, present only when `DualWriteConfig` is set on the sink. +/// Bundles the encoder with its target URI and sampling cadence so the three can't +/// drift out of sync with each other. +struct ShadowEncoder { + encoder: DatadogMetricsV3Encoder, + uri: Uri, + every: NonZeroU64, + /// Running count of legacy flushes seen since sink startup. + flush_count: u64, +} + +impl ShadowEncoder { + fn new( + endpoint: DatadogMetricsEndpoint, + uri: Uri, + every: NonZeroU64, + default_namespace: Option, + ) -> Self { + Self { + encoder: DatadogMetricsV3Encoder::new(endpoint, default_namespace), + uri, + every, + flush_count: 0, + } + } + + /// Advances the flush counter and reports whether this flush should also produce a + /// shadow write. + const fn should_flush(&mut self) -> bool { + self.flush_count = self.flush_count.wrapping_add(1); + self.flush_count.is_multiple_of(self.every.get()) + } } /// Incremental request builder specific to Datadog metrics. pub struct DatadogMetricsRequestBuilder { endpoint_configuration: DatadogMetricsEndpointConfiguration, - series_encoder: DatadogMetricsEncoder, - sketches_encoder: DatadogMetricsEncoder, + series_encoder: EncoderKind, + sketches_encoder: EncoderKind, + /// Present only when `DualWriteConfig` is set on the sink. + shadow: Option, + /// Present only when `DualWriteConfig` is set on the sink. + sketches_shadow: Option, + /// True when `sketches_api_version` is the legacy (non-V3) format, i.e. when a V3 + /// sketches shadow write is meaningful. `DatadogMetricsEndpoint::Sketches` doesn't carry + /// the api version the way `DatadogMetricsEndpoint::Series` does, so this has to be + /// tracked separately. + sketches_is_legacy: bool, } impl DatadogMetricsRequestBuilder { @@ -74,27 +201,60 @@ impl DatadogMetricsRequestBuilder { endpoint_configuration: DatadogMetricsEndpointConfiguration, default_namespace: Option, series_api_version: SeriesApiVersion, + sketches_api_version: SketchesApiVersion, + shadow_config: Option, ) -> Self { - Self { - endpoint_configuration, - series_encoder: DatadogMetricsEncoder::new( + let series_encoder = if series_api_version.is_v3_format() { + EncoderKind::V3(Box::new(DatadogMetricsV3Encoder::new( DatadogMetricsEndpoint::Series(series_api_version), default_namespace.clone(), - ), - sketches_encoder: DatadogMetricsEncoder::new( + ))) + } else { + EncoderKind::V1V2(Box::new(DatadogMetricsEncoder::new( + DatadogMetricsEndpoint::Series(series_api_version), + default_namespace.clone(), + ))) + }; + + // Independent of `series_api_version`: Datadog's intake gates V3 series and V3 sketches + // separately, so the sketches wire format must be chosen by its own setting. + let sketches_encoder = if sketches_api_version.is_v3_format() { + EncoderKind::V3(Box::new(DatadogMetricsV3Encoder::new( DatadogMetricsEndpoint::Sketches, default_namespace, + ))) + } else { + EncoderKind::V1V2(Box::new(DatadogMetricsEncoder::new( + DatadogMetricsEndpoint::Sketches, + default_namespace, + ))) + }; + + let (shadow, sketches_shadow) = match shadow_config { + Some(config) => ( + Some(ShadowEncoder::new( + DatadogMetricsEndpoint::Series(config.series_api_version), + config.series_uri, + config.shadow_every, + config.default_namespace.clone(), + )), + Some(ShadowEncoder::new( + DatadogMetricsEndpoint::Sketches, + config.sketches_uri, + config.shadow_every, + config.default_namespace, + )), ), - } - } + None => (None, None), + }; - const fn get_encoder( - &mut self, - endpoint: DatadogMetricsEndpoint, - ) -> &mut DatadogMetricsEncoder { - match endpoint { - DatadogMetricsEndpoint::Series { .. } => &mut self.series_encoder, - DatadogMetricsEndpoint::Sketches => &mut self.sketches_encoder, + Self { + endpoint_configuration, + series_encoder, + sketches_encoder, + shadow, + sketches_shadow, + sketches_is_legacy: !sketches_api_version.is_v3_format(), } } } @@ -111,119 +271,74 @@ impl IncrementalRequestBuilder<((Option>, DatadogMetricsEndpoint), Vec< &mut self, input: ((Option>, DatadogMetricsEndpoint), Vec), ) -> Vec> { - let (tmp, mut metrics) = input; + let (tmp, metrics) = input; let (api_key, endpoint) = tmp; - let encoder = self.get_encoder(endpoint); - let mut metric_drain = metrics.drain(..); + // Determine whether this flush triggers a shadow. Only legacy (non-V3) batches are + // counted — V3 series and V3 sketches are already on the target wire format, so + // shadowing them would be redundant. + let is_v1v2_series = matches!( + endpoint, + DatadogMetricsEndpoint::Series(SeriesApiVersion::V1 | SeriesApiVersion::V2) + ); + let is_legacy_sketches = + matches!(endpoint, DatadogMetricsEndpoint::Sketches) && self.sketches_is_legacy; + let is_shadow_flush = if is_v1v2_series { + self.shadow + .as_mut() + .is_some_and(ShadowEncoder::should_flush) + } else if is_legacy_sketches { + self.sketches_shadow + .as_mut() + .is_some_and(ShadowEncoder::should_flush) + } else { + false + }; - let mut results = Vec::new(); - let mut pending = None; - while metric_drain.len() != 0 { - let mut n = 0; + // UUIDv7 generated once per shadow flush; shared across primary + shadow requests. + let batch_id: Option> = + is_shadow_flush.then(|| Arc::from(Uuid::now_v7().to_string().as_str())); - loop { - // Grab the previously pending metric, or the next metric from the drain. - let metric = match pending.take() { - Some(metric) => metric, - None => match metric_drain.next() { - Some(metric) => metric, - None => break, - }, - }; - - // Try encoding the metric. If we get an error, we effectively drop this particular - // metric and add the error as a result. It might be an I/O error because we're - // literally out of memory and can't allocate more to encode, it might just be a - // single metric failed to encode, who knows... but technically only a single metric - // has failed to encode at this point, so that's all we track. - match encoder.try_encode(metric) { - // We encoded the metric successfully, so update our metadata and continue. - Ok(None) => n += 1, - Ok(Some(metric)) => { - // The encoded metric would not fit within the configured limits, so we need - // to finish the current encoder and generate our payload, and keep going. - pending = Some(metric); - break; - } - Err(e) => { - results.push(Err(e.into())); - break; - } - } - } + // Clone metrics before primary encoding consumes them, if we need a shadow copy. + let shadow_metrics = is_shadow_flush.then(|| metrics.clone()); - // If we encoded one or more metrics this pass, finalize the payload. - if n > 0 { - match encoder.finish() { - Ok((encode_result, mut metrics)) => { - let finalizers = metrics.take_finalizers(); - let metadata = DDMetricsMetadata { - api_key: api_key.clone(), - endpoint, - finalizers, - }; - - let request_metadata = - RequestMetadataBuilder::from_events(&metrics).build(&encode_result); - - results.push(Ok(( - (metadata, request_metadata), - encode_result.into_payload(), - ))); - } - Err(err) => match err { - // The encoder informed us that the resulting payload was too big, so we're - // being given a chance here to split it into smaller input batches in the - // hopes of generating a smaller payload that _isn't_ too big. - // - // The encoder instructs us on how many subchunks it thinks we need to split - // these metrics up into in order to successfully encode them without error, - // based on the resulting size of the previous attempt compared to the - // payload size limits. - // - // In order to avoid a pathological case from causing us to - // recursively/endlessly attempt encoding smaller and smaller batches, we - // only do this split/encode operation once. If any of the chunks fail for - // any reason, we fail that chunk entirely. - // - // TODO: In the future, when we have a way to incrementally write out - // Protocol Buffers data, similar to how the Datadog Agent does it with - // `molecule`, we can wrap all of the sketch encoding into the same - // incremental encoding paradigm and avoid this. - FinishError::TooLarge { - mut metrics, - mut recommended_splits, - } => { - let mut split_idx = metrics.len(); - let stride = split_idx / recommended_splits; - - while recommended_splits > 1 { - split_idx -= stride; - let chunk = metrics.split_off(split_idx); - results.push(encode_now_or_never( - encoder, - api_key.clone(), - endpoint, - chunk, - )); - recommended_splits -= 1; - } - results.push(encode_now_or_never( - encoder, - api_key.clone(), - endpoint, - metrics, - )); - } - // Not an error we can do anything about, so just forward it on. - suberr => results.push(Err(RequestBuilderError::Unexpected { - error_type: suberr.as_error_type(), - dropped_events: n as u64, - })), - }, - } + // ── Primary encode ──────────────────────────────────────────────────── + // V3Beta uses the same columnar encoder path as V3; only the + // URI differs (set in endpoint_configuration at build time). + let encoder = match endpoint { + DatadogMetricsEndpoint::Series(_) => &mut self.series_encoder, + DatadogMetricsEndpoint::Sketches => &mut self.sketches_encoder, + }; + let mut results = encode_batch(encoder, api_key.clone(), endpoint, metrics); + + // Stamp batch ID and independent seq/len on primary results before merging. + stamp_batch_id(batch_id.as_ref(), &mut results); + stamp_sequence(&mut results); + + // ── Shadow encode (V3) ──────────────────────────────────────────────── + let shadow_target = if is_v1v2_series { + self.shadow + .as_mut() + .map(|shadow| (shadow, DatadogMetricsEndpoint::Series(SeriesApiVersion::V3))) + } else { + self.sketches_shadow + .as_mut() + .map(|shadow| (shadow, DatadogMetricsEndpoint::Sketches)) + }; + + if let (Some(shadow_m), Some((shadow, shadow_endpoint))) = (shadow_metrics, shadow_target) { + let mut shadow_results = + encode_batch(&mut shadow.encoder, api_key, shadow_endpoint, shadow_m); + + // Override the URI so these requests go to the shadow endpoint, not V3 public API. + for ((meta, _), _) in shadow_results.iter_mut().flatten() { + meta.target_uri = Some(shadow.uri.clone()); } + // Shadow seq/len is independent of primary: the intake uses request ID + + // seq/len to reassemble split payloads within one encoder's output. + stamp_batch_id(batch_id.as_ref(), &mut shadow_results); + stamp_sequence(&mut shadow_results); + results.extend(shadow_results); } results @@ -231,9 +346,11 @@ impl IncrementalRequestBuilder<((Option>, DatadogMetricsEndpoint), Vec< fn build_request(&mut self, metadata: Self::Metadata, payload: Self::Payload) -> Self::Request { let (ddmetrics_metadata, request_metadata) = metadata; - let uri = self - .endpoint_configuration - .get_uri_for_endpoint(ddmetrics_metadata.endpoint); + + let uri = ddmetrics_metadata.target_uri.unwrap_or_else(|| { + self.endpoint_configuration + .get_uri_for_endpoint(ddmetrics_metadata.endpoint) + }); DatadogMetricsRequest { api_key: ddmetrics_metadata.api_key, @@ -243,20 +360,170 @@ impl IncrementalRequestBuilder<((Option>, DatadogMetricsEndpoint), Vec< content_encoding: ddmetrics_metadata.endpoint.compression().content_encoding(), finalizers: ddmetrics_metadata.finalizers, metadata: request_metadata, + batch_id: ddmetrics_metadata.batch_id, + batch_seq: ddmetrics_metadata.batch_seq, + batch_len: ddmetrics_metadata.batch_len, } } } -/// Simple encoder implementation that treats any error during encoding or finishing as unrecoverable. +// ── Batch ID and sequence stamping ──────────────────────────────────────────── + +type EncodedResults = + Vec>; + +/// Sets `batch_id` on every successful result in `results`. +fn stamp_batch_id(batch_id: Option<&Arc>, results: &mut EncodedResults) { + if let Some(id) = batch_id { + for ((meta, _), _) in results.iter_mut().flatten() { + meta.batch_id = Some(Arc::clone(id)); + } + } +} + +/// Sets `batch_seq` and `batch_len` on every successful result based on their +/// position among the *successful* results. Called after all encoding (primary + +/// shadow) is done so the total count is known. /// -/// We only call this method when our main encoding loop tried to finish a payload and was told -/// that the payload was too large compared to the payload size limits. That error gives back any -/// metrics that were correctly encoded so that we can attempt to encode them again in smaller -/// chunks. However, rather than continually trying smaller and smaller chunks, which could be -/// caused by a pathological error, we only attempt that operation once. This method facilitates -/// the "only try it once" aspect by treating all errors as unrecoverable. -fn encode_now_or_never( - encoder: &mut DatadogMetricsEncoder, +/// Must count and number `Ok` results only: a split chunk that fails to encode +/// (`Err`) is dropped and never becomes a request (see `builder.rs`'s `Ok`-only +/// filtering), so including it in `len` or letting it consume a `seq` value would +/// advertise a part that will never be sent — the intake then waits for that +/// missing sequence number forever, timing out the whole reassembly. +fn stamp_sequence(results: &mut EncodedResults) { + let len = results.iter().filter(|result| result.is_ok()).count(); + for (seq, ((meta, _), _)) in results.iter_mut().flatten().enumerate() { + meta.batch_seq = seq; + meta.batch_len = len; + } +} + +// ── Encoding ──────────────────────────────────────────────────────────────────── +// +// One code path drives both wire formats. V1/V2's `try_encode` returns `Ok(Some(metric))` +// when a metric doesn't fit, signalling "flush what you have and retry me" — the inner loop +// below handles that by stashing the metric in `pending` and finishing early. V3's +// `try_encode` always returns `Ok(None)` (it can only detect overflow at `finish()` time), so +// for V3 the inner loop simply drains every metric before finishing once, matching its +// batch-then-split semantics. + +fn encode_batch( + encoder: &mut E, + api_key: Option>, + endpoint: DatadogMetricsEndpoint, + mut metrics: Vec, +) -> EncodedResults { + let mut metric_drain = metrics.drain(..); + + let mut results = Vec::new(); + let mut pending = None; + while metric_drain.len() != 0 { + let mut n = 0; + + loop { + let metric = match pending.take() { + Some(metric) => metric, + None => match metric_drain.next() { + Some(metric) => metric, + None => break, + }, + }; + + // Try encoding the metric. If we get an error, we effectively drop this particular + // metric and add the error as a result. It might be an I/O error because we're + // literally out of memory and can't allocate more to encode, it might just be a + // single metric failed to encode, who knows... but technically only a single metric + // has failed to encode at this point, so that's all we track. + match encoder.try_encode(metric) { + // We encoded the metric successfully, so update our metadata and continue. + Ok(None) => n += 1, + Ok(Some(metric)) => { + // The encoded metric would not fit within the configured limits, so we need + // to finish the current encoder and generate our payload, and keep going. + pending = Some(metric); + break; + } + Err(e) => { + results.push(Err(e.into())); + break; + } + } + } + + // If we encoded one or more metrics this pass, finalize the payload. + if n > 0 { + match encoder.finish() { + Ok((encode_result, mut processed)) => { + let finalizers = processed.take_finalizers(); + let metadata = DDMetricsMetadata { + api_key: api_key.clone(), + endpoint, + finalizers, + batch_id: None, + batch_seq: 0, + batch_len: 1, + target_uri: None, + }; + + let request_metadata = + RequestMetadataBuilder::from_events(&processed).build(&encode_result); + + results.push(Ok(( + (metadata, request_metadata), + encode_result.into_payload(), + ))); + } + Err(FinishError::TooLarge { + mut metrics, + mut recommended_splits, + }) => { + // The encoder informed us that the resulting payload was too big, so we're + // being given a chance here to split it into smaller input batches in the + // hopes of generating a smaller payload that _isn't_ too big. + // + // The encoder instructs us on how many subchunks it thinks we need to split + // these metrics up into in order to successfully encode them without error, + // based on the resulting size of the previous attempt compared to the + // payload size limits. + // + // In order to avoid a pathological case from causing us to + // recursively/endlessly attempt encoding smaller and smaller batches, we + // only do this split/encode operation once. If any of the chunks fail for + // any reason, we fail that chunk entirely. + // + // TODO: In the future, when we have a way to incrementally write out + // Protocol Buffers data, similar to how the Datadog Agent does it with + // `molecule`, we can wrap all of the sketch encoding into the same + // incremental encoding paradigm and avoid this. + let mut split_idx = metrics.len(); + let stride = split_idx / recommended_splits; + + while recommended_splits > 1 { + split_idx -= stride; + let chunk = metrics.split_off(split_idx); + results.push(encode_chunk(encoder, api_key.clone(), endpoint, chunk)); + recommended_splits -= 1; + } + results.push(encode_chunk(encoder, api_key.clone(), endpoint, metrics)); + } + Err(suberr) => { + // Not an error we can do anything about, so just forward it on. + results.push(Err(RequestBuilderError::Unexpected { + error_type: suberr.as_error_type(), + dropped_events: n as u64, + })) + } + } + } + } + + results +} + +/// Encodes one chunk in a single shot, treating any error as unrecoverable. Used for +/// split-retry after a `FinishError::TooLarge`. +fn encode_chunk( + encoder: &mut E, api_key: Option>, endpoint: DatadogMetricsEndpoint, metrics: Vec, @@ -280,6 +547,10 @@ fn encode_now_or_never( api_key, endpoint, finalizers, + batch_id: None, + batch_seq: 0, + batch_len: 1, + target_uri: None, }; let request_metadata = @@ -294,3 +565,69 @@ fn encode_now_or_never( dropped_events: metrics_len as u64, }) } + +#[cfg(test)] +mod tests { + use vector_lib::request_metadata::GroupedCountByteSize; + + use super::*; + + fn ok_result() -> Result<((DDMetricsMetadata, RequestMetadata), Bytes), RequestBuilderError> { + let metadata = DDMetricsMetadata { + api_key: None, + endpoint: DatadogMetricsEndpoint::Series(SeriesApiVersion::V3), + finalizers: EventFinalizers::default(), + batch_id: None, + batch_seq: 0, + batch_len: 0, + target_uri: None, + }; + let request_metadata = + RequestMetadata::new(0, 0, 0, 0, GroupedCountByteSize::new_untagged()); + Ok(((metadata, request_metadata), Bytes::new())) + } + + fn err_result() -> Result<((DDMetricsMetadata, RequestMetadata), Bytes), RequestBuilderError> { + Err(RequestBuilderError::FailedToSplit { dropped_events: 1 }) + } + + /// A split chunk that fails to encode must not consume a `batch_seq` slot or inflate + /// `batch_len` — since it's dropped and never sent, doing so leaves the successfully + /// encoded parts advertising a `seq`/`len` that doesn't match what's actually + /// transmitted, causing the intake to wait forever for a part that will never arrive. + #[test] + fn stamp_sequence_numbers_only_ok_results() { + let mut results: EncodedResults = vec![ + ok_result(), + err_result(), + ok_result(), + err_result(), + ok_result(), + ]; + + stamp_sequence(&mut results); + + let stamped: Vec<(usize, usize)> = results + .iter() + .flatten() + .map(|((meta, _), _)| (meta.batch_seq, meta.batch_len)) + .collect(); + + assert_eq!(stamped, vec![(0, 3), (1, 3), (2, 3)]); + } + + #[test] + fn stamp_sequence_with_no_failures_is_unchanged() { + let mut results: EncodedResults = vec![ok_result(), ok_result()]; + + stamp_sequence(&mut results); + + let stamped: Vec<(usize, usize)> = results + .iter() + .flatten() + .map(|((meta, _), _)| (meta.batch_seq, meta.batch_len)) + .collect(); + + assert_eq!(stamped, vec![(0, 2), (1, 2)]); + } +} diff --git a/src/sinks/datadog/metrics/service.rs b/src/sinks/datadog/metrics/service.rs index 024493ed88fc2..23cc7233f8cb0 100644 --- a/src/sinks/datadog/metrics/service.rs +++ b/src/sinks/datadog/metrics/service.rs @@ -20,6 +20,7 @@ use vector_lib::{ use crate::{ http::{BuildRequestSnafu, HttpClient}, + internal_events::DatadogMetricsRequestError, sinks::{datadog::DatadogApiError, util::retries::RetryLogic}, }; @@ -47,6 +48,13 @@ pub struct DatadogMetricsRequest { pub content_encoding: &'static str, pub finalizers: EventFinalizers, pub metadata: RequestMetadata, + /// Shared transaction ID linking a V2 and V3 shadow payload from the same flush. + /// When set, `X-Metrics-Request-ID/Seq/Len` headers are included on the request. + pub batch_id: Option>, + /// 0-based index of this request within the current flush (for split payloads). + pub batch_seq: usize, + /// Total number of requests produced by the current flush (for split payloads). + pub batch_len: usize, } impl DatadogMetricsRequest { @@ -64,7 +72,8 @@ impl DatadogMetricsRequest { HeaderValue::from_str(&key).expect("API key should be only valid ASCII characters") }, ); - let request = Request::post(self.uri) + + let mut builder = Request::post(self.uri) .header("DD-API-KEY", api_key) // TODO: The Datadog Agent sends this header to indicate the version of the Go library // it uses which contains the Protocol Buffers definitions used for the Sketches API. @@ -80,7 +89,14 @@ impl DatadogMetricsRequest { .header(CONTENT_TYPE, self.content_type) .header(CONTENT_ENCODING, self.content_encoding); - request.body(Body::from(self.payload)) + if let Some(id) = &self.batch_id { + builder = builder + .header("X-Metrics-Request-ID", id.as_ref()) + .header("X-Metrics-Request-Seq", self.batch_seq.to_string()) + .header("X-Metrics-Request-Len", self.batch_len.to_string()); + } + + builder.body(Body::from(self.payload)) } } @@ -164,14 +180,46 @@ impl Service for DatadogMetricsService { Box::pin(async move { let request_metadata = std::mem::take(request.metadata_mut()); - - let request = request - .into_http_request(api_key) - .context(BuildRequestSnafu) - .map_err(|error| DatadogApiError::HttpError { error })?; - - let result = client.send(request).await; - let result = DatadogApiError::from_result(result)?; + let batch_id = request.batch_id.clone(); + let uri = request.uri.clone(); + let batch_seq = request.batch_seq; + let batch_len = request.batch_len; + let start = std::time::Instant::now(); + + let call_result: Result<_, DatadogApiError> = async { + let http_request = request + .into_http_request(api_key) + .context(BuildRequestSnafu) + .map_err(|error| DatadogApiError::HttpError { error })?; + + let result = client.send(http_request).await; + DatadogApiError::from_result(result) + } + .await; + + let result = call_result.inspect_err(|error| { + emit!(DatadogMetricsRequestError { + error: &error.to_string(), + batch_id: batch_id.as_deref(), + uri: &uri, + }); + })?; + + // Only batch_id-tagged requests are logged on success (dual-write shadow flushes, + // which are rare — sampled once per `shadow_every`), so this stays low-volume and + // gives visibility into dispatch timing for both the V2 and V3 twins of a flush. + if let Some(id) = batch_id.as_deref() { + info!( + message = "Sent Datadog metrics request.", + batch_id = id, + %uri, + batch_seq, + batch_len, + status = %result.status(), + elapsed_ms = start.elapsed().as_millis() as u64, + internal_log_rate_limit = false, + ); + } Ok(DatadogMetricsResponse { status_code: result.status(), diff --git a/website/cue/reference/components/sinks/generated/datadog_metrics.cue b/website/cue/reference/components/sinks/generated/datadog_metrics.cue index 103a7726396fd..46ee7a4b1dab0 100644 --- a/website/cue/reference/components/sinks/generated/datadog_metrics.cue +++ b/website/cue/reference/components/sinks/generated/datadog_metrics.cue @@ -85,6 +85,26 @@ generated: components: sinks: datadog_metrics: configuration: { required: false type: string: examples: ["myservice"] } + dual_write: { + description: """ + Optional V3 shadow dual-write configuration. + + When set, a sampled fraction of legacy series and sketches flushes are each mirrored + as V3 payloads to a separate intake endpoint, both stamped with a shared + `X-Metrics-Request-ID`. + """ + required: false + type: object: options: shadow_every: { + description: """ + Send a V3 shadow payload once per this many legacy series or sketches flushes. + + Set to `1` to shadow every flush (full dual-write). Must be greater than zero. + Defaults to `1000`. + """ + required: false + type: uint: default: 1000 + } + } endpoint: { description: """ The endpoint to send observability data to. @@ -308,6 +328,18 @@ generated: components: sinks: datadog_metrics: configuration: { This is the recommended and default endpoint. """ + v3: """ + Use the v3 series endpoint (`/api/intake/metrics/v3beta/series`). + + Columnar protobuf format with dictionary-based string deduplication and delta + encoding. More efficient than v2 for workloads with many metrics that share + common tags or names. + """ + v3_beta: """ + Use the v3 beta intake endpoint (`/api/intake/metrics/v3beta/series`). + + Used for shadow/validation rollout of V3. Prefer `v3_intake` for stable usage. + """ } } } @@ -326,6 +358,35 @@ generated: components: sinks: datadog_metrics: configuration: { required: false type: string: examples: ["us3.datadoghq.com", "datadoghq.eu"] } + sketches_api_version: { + description: """ + Controls which Datadog sketches API endpoint is used to submit distributions and + histograms. + + Independent of `series_api_version` — Datadog's intake gates V3 series and V3 sketches + separately, so this must be set explicitly to send sketches via V3, even if + `series_api_version` is already `v3`. + + Defaults to `v2` (`/api/beta/sketches`). + """ + required: false + type: string: { + default: "v2" + enum: { + v2: """ + Use the legacy sketches endpoint (`/api/beta/sketches`). + + This is the recommended and default endpoint. + """ + v3: """ + Use the v3 sketches endpoint (`/api/intake/metrics/v3/sketches`). + + Columnar protobuf format, matching the encoding used for V3 series. Must be enabled + separately from `series_api_version`. + """ + } + } + } tls: { description: "Configures the TLS options for incoming/outgoing connections." required: false From 696c786ee08dc70605011153b35c2337e8809a6a Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Mon, 10 Aug 2026 12:36:44 +0100 Subject: [PATCH 02/18] enhancement(datadog metrics sink)!: enable V3 shadow dual-write by default Sets `dual_write.enabled` to default to `true` so Vector's datadog_metrics sink dual-writes a sampled V3 shadow payload out of the box, without requiring explicit opt-in configuration. Set `dual_write.enabled: false` to restore the previous opt-in behavior. --- ..._metrics_v3_dual_write_default.breaking.md | 9 +++++ src/sinks/datadog/metrics/config.rs | 34 ++++++++++++++---- .../sinks/generated/datadog_metrics.cue | 36 ++++++++++++------- 3 files changed, 61 insertions(+), 18 deletions(-) create mode 100644 changelog.d/datadog_metrics_v3_dual_write_default.breaking.md diff --git a/changelog.d/datadog_metrics_v3_dual_write_default.breaking.md b/changelog.d/datadog_metrics_v3_dual_write_default.breaking.md new file mode 100644 index 0000000000000..84f5d302435ac --- /dev/null +++ b/changelog.d/datadog_metrics_v3_dual_write_default.breaking.md @@ -0,0 +1,9 @@ +The `datadog_metrics` sink's `dual_write` V3 shadow option is now enabled by default (with +`shadow_every: 1000`, sampling 1 in every 1000 legacy series/sketches flushes). This means Vector +now sends an additional, sampled V3-encoded payload to Datadog's shadow intake endpoint alongside +the normal legacy payload, without any configuration required. + +If you don't want this additional traffic, set `dual_write.enabled: false` on your `datadog_metrics` +sink configuration. + +authors: stephenwakely diff --git a/src/sinks/datadog/metrics/config.rs b/src/sinks/datadog/metrics/config.rs index 76719aa95876b..e53e728b2244a 100644 --- a/src/sinks/datadog/metrics/config.rs +++ b/src/sinks/datadog/metrics/config.rs @@ -230,6 +230,10 @@ fn default_shadow_every() -> NonZeroU64 { NonZeroU64::new(1000).unwrap() } +const fn default_dual_write_enabled() -> bool { + true +} + /// Configuration for the V3 shadow dual-write mode. /// /// When enabled, every `shadow_every`-th legacy series flush, and every `shadow_every`-th @@ -239,6 +243,13 @@ fn default_shadow_every() -> NonZeroU64 { #[configurable_component] #[derive(Clone, Debug)] pub struct DualWriteConfig { + /// Whether to enable V3 shadow dual-write. + /// + /// Enabled by default, sampling a fraction of legacy flushes to validate the V3 intake + /// path. Set to `false` to disable V3 shadow dual-write entirely. + #[serde(default = "default_dual_write_enabled")] + pub enabled: bool, + /// Send a V3 shadow payload once per this many legacy series or sketches flushes. /// /// Set to `1` to shadow every flush (full dual-write). Must be greater than zero. @@ -247,6 +258,15 @@ pub struct DualWriteConfig { pub shadow_every: NonZeroU64, } +impl Default for DualWriteConfig { + fn default() -> Self { + Self { + enabled: default_dual_write_enabled(), + shadow_every: default_shadow_every(), + } + } +} + impl DualWriteConfig { pub(super) const fn get_series_path(&self) -> &'static str { SERIES_V3_BETA_PATH @@ -299,13 +319,14 @@ pub struct DatadogMetricsConfig { #[serde(default)] pub request: TowerRequestConfig, - /// Optional V3 shadow dual-write configuration. + /// V3 shadow dual-write configuration. /// - /// When set, a sampled fraction of legacy series and sketches flushes are each mirrored - /// as V3 payloads to a separate intake endpoint, both stamped with a shared - /// `X-Metrics-Request-ID`. + /// Enabled by default: a sampled fraction of legacy series and sketches flushes are each + /// mirrored as V3 payloads to a separate intake endpoint, both stamped with a shared + /// `X-Metrics-Request-ID`. Set `dual_write.enabled` to `false` to disable it. + #[configurable(derived)] #[serde(default)] - pub dual_write: Option, + pub dual_write: DualWriteConfig, } impl_generate_config_from_default!(DatadogMetricsConfig); @@ -406,7 +427,8 @@ impl DatadogMetricsConfig { let shadow_config = self .dual_write - .as_ref() + .enabled + .then_some(&self.dual_write) .map(|dw| -> crate::Result { let base_uri = self.get_base_agent_endpoint(dd_common); let series_shadow_uri = build_uri(&base_uri, dw.get_series_path())?; diff --git a/website/cue/reference/components/sinks/generated/datadog_metrics.cue b/website/cue/reference/components/sinks/generated/datadog_metrics.cue index 46ee7a4b1dab0..a4823d0aa7ee4 100644 --- a/website/cue/reference/components/sinks/generated/datadog_metrics.cue +++ b/website/cue/reference/components/sinks/generated/datadog_metrics.cue @@ -87,22 +87,34 @@ generated: components: sinks: datadog_metrics: configuration: { } dual_write: { description: """ - Optional V3 shadow dual-write configuration. + V3 shadow dual-write configuration. - When set, a sampled fraction of legacy series and sketches flushes are each mirrored - as V3 payloads to a separate intake endpoint, both stamped with a shared - `X-Metrics-Request-ID`. + Enabled by default: a sampled fraction of legacy series and sketches flushes are each + mirrored as V3 payloads to a separate intake endpoint, both stamped with a shared + `X-Metrics-Request-ID`. Set `dual_write.enabled` to `false` to disable it. """ required: false - type: object: options: shadow_every: { - description: """ - Send a V3 shadow payload once per this many legacy series or sketches flushes. + type: object: options: { + enabled: { + description: """ + Whether to enable V3 shadow dual-write. - Set to `1` to shadow every flush (full dual-write). Must be greater than zero. - Defaults to `1000`. - """ - required: false - type: uint: default: 1000 + Enabled by default, sampling a fraction of legacy flushes to validate the V3 intake + path. Set to `false` to disable V3 shadow dual-write entirely. + """ + required: false + type: bool: default: true + } + shadow_every: { + description: """ + Send a V3 shadow payload once per this many legacy series or sketches flushes. + + Set to `1` to shadow every flush (full dual-write). Must be greater than zero. + Defaults to `1000`. + """ + required: false + type: uint: default: 1000 + } } } endpoint: { From 724755eca6c036ee0d5494997f1488fa33447e9e Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Tue, 11 Aug 2026 16:38:37 +0100 Subject: [PATCH 03/18] chore(datadog metrics v3): fix clippy lints from ported v3 encoder code Applies the newer clippy ruleset (collapsible_if let-chains, trivially_copy_pass_by_ref, missing_const_for_fn) to the code ported from stephen/v3_vector so it satisfies this codebase's current #![deny(warnings)] lints and rustfmt import ordering. --- src/sinks/datadog/metrics/config.rs | 4 ++-- src/sinks/datadog/metrics/encoder_v3.rs | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/sinks/datadog/metrics/config.rs b/src/sinks/datadog/metrics/config.rs index e53e728b2244a..0d5095bdf6137 100644 --- a/src/sinks/datadog/metrics/config.rs +++ b/src/sinks/datadog/metrics/config.rs @@ -75,7 +75,7 @@ pub enum SeriesApiVersion { } impl SeriesApiVersion { - pub const fn get_path(&self) -> &'static str { + pub const fn get_path(self) -> &'static str { match self { Self::V1 => SERIES_V1_PATH, Self::V2 => SERIES_V2_PATH, @@ -226,7 +226,7 @@ impl DatadogMetricsEndpointConfiguration { } } -fn default_shadow_every() -> NonZeroU64 { +const fn default_shadow_every() -> NonZeroU64 { NonZeroU64::new(1000).unwrap() } diff --git a/src/sinks/datadog/metrics/encoder_v3.rs b/src/sinks/datadog/metrics/encoder_v3.rs index ec56acf68bd24..b23ccbb800937 100644 --- a/src/sinks/datadog/metrics/encoder_v3.rs +++ b/src/sinks/datadog/metrics/encoder_v3.rs @@ -212,20 +212,20 @@ fn encode_metric_to_v3( for (key, value) in tags.iter_all() { // dd.internal.resource tags become structured resources if key == "dd.internal.resource" { - if let Some(val) = value { - if let Some((rtype, rname)) = val.split_once(':') { - extra_resources.push((rtype, rname)); - } + if let Some(val) = value + && let Some((rtype, rname)) = val.split_once(':') + { + extra_resources.push((rtype, rname)); } continue; } // Host key → host resource if host_key.as_deref() == Some(key) { - if let Some(host) = value { - if !host.is_empty() { - host_resource = Some(host); - } + if let Some(host) = value + && !host.is_empty() + { + host_resource = Some(host); } continue; } From eb9b97e6801c8d470a5b4e33d44820fcdde81978 Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Wed, 12 Aug 2026 13:28:59 +0100 Subject: [PATCH 04/18] fix(datadog metrics sink): never dual-write sketches The V3 sketches intake routes do not exist: both /api/intake/metrics/v3/sketches and /api/intake/metrics/v3beta/sketches return 404. A 404 maps to DatadogApiError::ClientError, which is_retriable() treats as retriable, so every sketches flush retried indefinitely against a nonexistent endpoint and never delivered anything. With dual_write now enabled by default, any pipeline carrying distributions, histograms or timings hit this. Remove the sketches shadow path entirely rather than gating it behind another config flag, so an enabled dual_write cannot produce sketches traffic: - drop sketches_uri from ShadowBuilderConfig and SKETCHES_V3_BETA_PATH - drop the sketches_shadow encoder and sketches_is_legacy tracking from DatadogMetricsRequestBuilder - only legacy (V1/V2) series flushes advance the shadow cadence, so the sampling rate no longer drifts with the sketch/series mix Series dual-write is unchanged. Primary sketches delivery via sketches_api_version is untouched. Adds regression tests covering all three properties. --- .../1_datadog_metrics_v3.enhancement.md | 11 +- ..._metrics_v3_dual_write_default.breaking.md | 9 +- src/sinks/datadog/metrics/config.rs | 38 +-- src/sinks/datadog/metrics/request_builder.rs | 264 ++++++++++++++---- .../sinks/generated/datadog_metrics.cue | 18 +- 5 files changed, 245 insertions(+), 95 deletions(-) diff --git a/changelog.d/1_datadog_metrics_v3.enhancement.md b/changelog.d/1_datadog_metrics_v3.enhancement.md index 799e532de7307..2060059ae71af 100644 --- a/changelog.d/1_datadog_metrics_v3.enhancement.md +++ b/changelog.d/1_datadog_metrics_v3.enhancement.md @@ -1,5 +1,8 @@ -Adds the a new encoder to the Datadog metrics sink to encode metrics with v3 of +Adds a new encoder to the Datadog metrics sink to encode metrics with v3 of the payload protocol. An additional option `dual_write` will make Vector send -duplicate payloads to the given endpoint encoded with the configured protocol. -This allows the Datadog backend to validate that the metrics send via both -protocols specify the exact same metrics. +duplicate series payloads to the given endpoint encoded with the configured +protocol. This allows the Datadog backend to validate that the metrics sent via +both protocols specify the exact same metrics. Only series metrics are +dual-written; sketches are never shadowed. + +authors: stephenwakely diff --git a/changelog.d/datadog_metrics_v3_dual_write_default.breaking.md b/changelog.d/datadog_metrics_v3_dual_write_default.breaking.md index 84f5d302435ac..fffb55e9a475f 100644 --- a/changelog.d/datadog_metrics_v3_dual_write_default.breaking.md +++ b/changelog.d/datadog_metrics_v3_dual_write_default.breaking.md @@ -1,7 +1,10 @@ The `datadog_metrics` sink's `dual_write` V3 shadow option is now enabled by default (with -`shadow_every: 1000`, sampling 1 in every 1000 legacy series/sketches flushes). This means Vector -now sends an additional, sampled V3-encoded payload to Datadog's shadow intake endpoint alongside -the normal legacy payload, without any configuration required. +`shadow_every: 1000`, sampling 1 in every 1000 legacy series flushes). This means Vector now sends +an additional, sampled V3-encoded payload to Datadog's shadow intake endpoint alongside the normal +legacy payload, without any configuration required. + +Only series metrics are dual-written. Sketches (distributions and histograms) are never shadowed, +because the V3 sketches intake endpoints do not exist. If you don't want this additional traffic, set `dual_write.enabled: false` on your `datadog_metrics` sink configuration. diff --git a/src/sinks/datadog/metrics/config.rs b/src/sinks/datadog/metrics/config.rs index 0d5095bdf6137..2a7828d68b75c 100644 --- a/src/sinks/datadog/metrics/config.rs +++ b/src/sinks/datadog/metrics/config.rs @@ -41,8 +41,6 @@ pub(super) const SERIES_V3_PATH: &str = "/api/intake/metrics/v3/series"; pub(super) const SERIES_V3_BETA_PATH: &str = "/api/intake/metrics/v3beta/series"; pub(super) const SKETCHES_PATH: &str = "/api/beta/sketches"; pub(super) const SKETCHES_V3_PATH: &str = "/api/intake/metrics/v3/sketches"; -/// Beta intake endpoint used during V3 sketches shadow rollout. -pub(super) const SKETCHES_V3_BETA_PATH: &str = "/api/intake/metrics/v3beta/sketches"; /// The API version to use when submitting series metrics to Datadog. #[configurable_component] @@ -236,24 +234,30 @@ const fn default_dual_write_enabled() -> bool { /// Configuration for the V3 shadow dual-write mode. /// -/// When enabled, every `shadow_every`-th legacy series flush, and every `shadow_every`-th -/// legacy sketches flush, also sends a V3 shadow payload to the corresponding shadow -/// endpoint. Both payloads in a pair carry the same `X-Metrics-Request-ID` header so the -/// intake backend can correlate them. +/// When enabled, every `shadow_every`-th legacy (V1/V2) *series* flush also sends a V3 +/// shadow payload to the shadow series endpoint. Both payloads in a pair carry the same +/// `X-Metrics-Request-ID` header so the intake backend can correlate them. +/// +/// Sketches are never dual-written. The V3 sketches intake routes do not exist — both +/// `/api/intake/metrics/v3/sketches` and `/api/intake/metrics/v3beta/sketches` return 404, +/// and a 404 maps to a *retriable* `ClientError`, so shadowing sketches produced an +/// endless retry loop that never delivered anything. #[configurable_component] #[derive(Clone, Debug)] pub struct DualWriteConfig { /// Whether to enable V3 shadow dual-write. /// - /// Enabled by default, sampling a fraction of legacy flushes to validate the V3 intake - /// path. Set to `false` to disable V3 shadow dual-write entirely. + /// Enabled by default, sampling a fraction of legacy series flushes to validate the V3 + /// intake path. Set to `false` to disable V3 shadow dual-write entirely. + /// + /// This only ever affects series. Sketches are never dual-written. #[serde(default = "default_dual_write_enabled")] pub enabled: bool, - /// Send a V3 shadow payload once per this many legacy series or sketches flushes. + /// Send a V3 shadow payload once per this many legacy (V1/V2) series flushes. /// - /// Set to `1` to shadow every flush (full dual-write). Must be greater than zero. - /// Defaults to `1000`. + /// Set to `1` to shadow every series flush (full dual-write). Must be greater than zero. + /// Defaults to `1000`. Sketches flushes are never counted or shadowed. #[serde(default = "default_shadow_every")] pub shadow_every: NonZeroU64, } @@ -271,10 +275,6 @@ impl DualWriteConfig { pub(super) const fn get_series_path(&self) -> &'static str { SERIES_V3_BETA_PATH } - - pub(super) const fn get_sketches_path(&self) -> &'static str { - SKETCHES_V3_BETA_PATH - } } /// Configuration for the `datadog_metrics` sink. @@ -321,9 +321,11 @@ pub struct DatadogMetricsConfig { /// V3 shadow dual-write configuration. /// - /// Enabled by default: a sampled fraction of legacy series and sketches flushes are each - /// mirrored as V3 payloads to a separate intake endpoint, both stamped with a shared + /// Enabled by default: a sampled fraction of legacy series flushes is mirrored as V3 + /// payloads to a separate intake endpoint, both stamped with a shared /// `X-Metrics-Request-ID`. Set `dual_write.enabled` to `false` to disable it. + /// + /// Sketches are never dual-written, regardless of this setting. #[configurable(derived)] #[serde(default)] pub dual_write: DualWriteConfig, @@ -432,11 +434,9 @@ impl DatadogMetricsConfig { .map(|dw| -> crate::Result { let base_uri = self.get_base_agent_endpoint(dd_common); let series_shadow_uri = build_uri(&base_uri, dw.get_series_path())?; - let sketches_shadow_uri = build_uri(&base_uri, dw.get_sketches_path())?; Ok(ShadowBuilderConfig { series_uri: series_shadow_uri, series_api_version: SeriesApiVersion::V3Beta, - sketches_uri: sketches_shadow_uri, default_namespace: self.default_namespace.clone(), shadow_every: dw.shadow_every, }) diff --git a/src/sinks/datadog/metrics/request_builder.rs b/src/sinks/datadog/metrics/request_builder.rs index 14d19a1f3967a..47219e2aa4e42 100644 --- a/src/sinks/datadog/metrics/request_builder.rs +++ b/src/sinks/datadog/metrics/request_builder.rs @@ -132,17 +132,19 @@ impl MetricsEncoder for EncoderKind { } /// Shadow write configuration passed from `DatadogMetricsConfig::build_sink`. +/// +/// Series only. Sketches are deliberately never dual-written: the V3 sketches intake routes +/// don't exist (they 404, and a 404 maps to a *retriable* `ClientError`, so a sketches +/// shadow retried forever without ever delivering). pub struct ShadowBuilderConfig { /// The URI for the V3 shadow series endpoint (e.g. `/api/intake/metrics/v3/series`). pub series_uri: Uri, /// The `SeriesApiVersion` variant matching the shadow series endpoint. /// Used to set the correct payload limits and compression on the shadow encoder. pub series_api_version: SeriesApiVersion, - /// The URI for the V3 shadow sketches endpoint (e.g. `/api/intake/metrics/v3/sketches`). - pub sketches_uri: Uri, - /// Default metric namespace for the shadow encoders. + /// Default metric namespace for the shadow encoder. pub default_namespace: Option, - /// Send a V3 shadow once per this many legacy (V1/V2 series, or non-V3 sketches) flushes. + /// Send a V3 shadow once per this many legacy (V1/V2) series flushes. pub shadow_every: NonZeroU64, } @@ -153,7 +155,7 @@ struct ShadowEncoder { encoder: DatadogMetricsV3Encoder, uri: Uri, every: NonZeroU64, - /// Running count of legacy flushes seen since sink startup. + /// Running count of legacy series flushes seen since sink startup. flush_count: u64, } @@ -185,15 +187,9 @@ pub struct DatadogMetricsRequestBuilder { endpoint_configuration: DatadogMetricsEndpointConfiguration, series_encoder: EncoderKind, sketches_encoder: EncoderKind, - /// Present only when `DualWriteConfig` is set on the sink. + /// Series-only V3 shadow encoder, present only when `dual_write` is enabled. + /// There is deliberately no sketches equivalent; see `ShadowBuilderConfig`. shadow: Option, - /// Present only when `DualWriteConfig` is set on the sink. - sketches_shadow: Option, - /// True when `sketches_api_version` is the legacy (non-V3) format, i.e. when a V3 - /// sketches shadow write is meaningful. `DatadogMetricsEndpoint::Sketches` doesn't carry - /// the api version the way `DatadogMetricsEndpoint::Series` does, so this has to be - /// tracked separately. - sketches_is_legacy: bool, } impl DatadogMetricsRequestBuilder { @@ -230,31 +226,22 @@ impl DatadogMetricsRequestBuilder { ))) }; - let (shadow, sketches_shadow) = match shadow_config { - Some(config) => ( - Some(ShadowEncoder::new( - DatadogMetricsEndpoint::Series(config.series_api_version), - config.series_uri, - config.shadow_every, - config.default_namespace.clone(), - )), - Some(ShadowEncoder::new( - DatadogMetricsEndpoint::Sketches, - config.sketches_uri, - config.shadow_every, - config.default_namespace, - )), - ), - None => (None, None), - }; + // Series only: no sketches shadow encoder is ever constructed, so an enabled + // `dual_write` cannot produce sketches traffic. + let shadow = shadow_config.map(|config| { + ShadowEncoder::new( + DatadogMetricsEndpoint::Series(config.series_api_version), + config.series_uri, + config.shadow_every, + config.default_namespace, + ) + }); Self { endpoint_configuration, series_encoder, sketches_encoder, shadow, - sketches_shadow, - sketches_is_legacy: !sketches_api_version.is_v3_format(), } } } @@ -274,26 +261,18 @@ impl IncrementalRequestBuilder<((Option>, DatadogMetricsEndpoint), Vec< let (tmp, metrics) = input; let (api_key, endpoint) = tmp; - // Determine whether this flush triggers a shadow. Only legacy (non-V3) batches are - // counted — V3 series and V3 sketches are already on the target wire format, so - // shadowing them would be redundant. + // Determine whether this flush triggers a shadow. Only legacy (V1/V2) *series* + // batches count: V3 series is already on the target wire format, and sketches are + // never shadowed at all because the V3 sketches intake routes don't exist. let is_v1v2_series = matches!( endpoint, DatadogMetricsEndpoint::Series(SeriesApiVersion::V1 | SeriesApiVersion::V2) ); - let is_legacy_sketches = - matches!(endpoint, DatadogMetricsEndpoint::Sketches) && self.sketches_is_legacy; - let is_shadow_flush = if is_v1v2_series { - self.shadow - .as_mut() - .is_some_and(ShadowEncoder::should_flush) - } else if is_legacy_sketches { - self.sketches_shadow + let is_shadow_flush = is_v1v2_series + && self + .shadow .as_mut() - .is_some_and(ShadowEncoder::should_flush) - } else { - false - }; + .is_some_and(ShadowEncoder::should_flush); // UUIDv7 generated once per shadow flush; shared across primary + shadow requests. let batch_id: Option> = @@ -315,20 +294,17 @@ impl IncrementalRequestBuilder<((Option>, DatadogMetricsEndpoint), Vec< stamp_batch_id(batch_id.as_ref(), &mut results); stamp_sequence(&mut results); - // ── Shadow encode (V3) ──────────────────────────────────────────────── - let shadow_target = if is_v1v2_series { - self.shadow - .as_mut() - .map(|shadow| (shadow, DatadogMetricsEndpoint::Series(SeriesApiVersion::V3))) - } else { - self.sketches_shadow - .as_mut() - .map(|shadow| (shadow, DatadogMetricsEndpoint::Sketches)) - }; - - if let (Some(shadow_m), Some((shadow, shadow_endpoint))) = (shadow_metrics, shadow_target) { - let mut shadow_results = - encode_batch(&mut shadow.encoder, api_key, shadow_endpoint, shadow_m); + // ── Shadow encode (V3 series only) ──────────────────────────────────── + // `shadow_metrics` is populated only on a shadow flush, which already implies a + // legacy series batch with the sampling cadence satisfied, so this can never emit + // sketches traffic. + if let (Some(shadow_m), Some(shadow)) = (shadow_metrics, self.shadow.as_mut()) { + let mut shadow_results = encode_batch( + &mut shadow.encoder, + api_key, + DatadogMetricsEndpoint::Series(SeriesApiVersion::V3), + shadow_m, + ); // Override the URI so these requests go to the shadow endpoint, not V3 public API. for ((meta, _), _) in shadow_results.iter_mut().flatten() { @@ -568,7 +544,11 @@ fn encode_chunk( #[cfg(test)] mod tests { - use vector_lib::request_metadata::GroupedCountByteSize; + use vector_lib::{ + event::{MetricKind, MetricValue, metric::MetricSketch}, + metrics::AgentDDSketch, + request_metadata::GroupedCountByteSize, + }; use super::*; @@ -630,4 +610,164 @@ mod tests { assert_eq!(stamped, vec![(0, 2), (1, 2)]); } + + // ── Shadow dual-write is series-only ──────────────────────────────────────── + + fn builder_with_shadow_every(every: u64) -> DatadogMetricsRequestBuilder { + let endpoint_configuration = DatadogMetricsEndpointConfiguration::new( + "https://example.com/api/v2/series".parse().unwrap(), + "https://example.com/api/beta/sketches".parse().unwrap(), + ); + + DatadogMetricsRequestBuilder::new( + endpoint_configuration, + None, + SeriesApiVersion::V2, + SketchesApiVersion::V2, + Some(ShadowBuilderConfig { + series_uri: "https://example.com/api/intake/metrics/v3beta/series" + .parse() + .unwrap(), + series_api_version: SeriesApiVersion::V3Beta, + default_namespace: None, + shadow_every: NonZeroU64::new(every).unwrap(), + }), + ) + } + + fn counter_metric() -> Metric { + Metric::new( + "test.counter", + MetricKind::Incremental, + MetricValue::Counter { value: 1.0 }, + ) + } + + fn sketch_metric() -> Metric { + let mut sketch = AgentDDSketch::with_agent_defaults(); + sketch.insert(1.0); + Metric::new( + "test.sketch", + MetricKind::Incremental, + MetricValue::Sketch { + sketch: MetricSketch::AgentDDSketch(sketch), + }, + ) + } + + fn encode( + builder: &mut DatadogMetricsRequestBuilder, + endpoint: DatadogMetricsEndpoint, + metrics: Vec, + ) -> Vec<(DDMetricsMetadata, RequestMetadata)> { + builder + .encode_events_incremental(((None, endpoint), metrics)) + .into_iter() + .filter_map(Result::ok) + .map(|(metadata, _payload)| metadata) + .collect() + } + + /// The V3 sketches intake routes don't exist (404 -> retriable `ClientError` -> infinite + /// retry loop), so a sketches flush must never produce a shadow request even when + /// `dual_write` is fully enabled and sampling every flush. + #[test] + fn sketches_are_never_shadowed_even_with_dual_write_enabled() { + let mut builder = builder_with_shadow_every(1); + + let encoded = encode( + &mut builder, + DatadogMetricsEndpoint::Sketches, + vec![sketch_metric()], + ); + + assert_eq!( + encoded.len(), + 1, + "a sketches flush must yield only the primary request, got {} requests", + encoded.len() + ); + let (meta, _) = &encoded[0]; + assert_eq!(meta.endpoint, DatadogMetricsEndpoint::Sketches); + assert!( + meta.batch_id.is_none(), + "sketches must not be stamped with a shadow X-Metrics-Request-ID" + ); + assert!( + meta.target_uri.is_none(), + "sketches must never be retargeted at a V3 shadow endpoint" + ); + } + + /// Series shadowing is unaffected by the sketches removal: a legacy series flush still + /// emits a V2 primary plus a V3 shadow sharing one request ID. + #[test] + fn legacy_series_is_still_shadowed() { + let mut builder = builder_with_shadow_every(1); + + let encoded = encode( + &mut builder, + DatadogMetricsEndpoint::Series(SeriesApiVersion::V2), + vec![counter_metric()], + ); + + assert_eq!(encoded.len(), 2, "expected a V2 primary and a V3 shadow"); + let ids: Vec>> = encoded + .iter() + .map(|(meta, _)| meta.batch_id.clone()) + .collect(); + assert!( + ids.iter().all(Option::is_some), + "both halves of the pair must carry a batch id" + ); + assert_eq!(ids[0], ids[1], "the pair must share one request ID"); + assert_eq!( + encoded + .iter() + .filter(|(meta, _)| meta.target_uri.is_some()) + .count(), + 1, + "exactly one half of the pair is retargeted to the shadow URI" + ); + } + + /// The shadow cadence counter must only advance on series flushes. If sketches flushes + /// still ticked it, the sampling rate would drift with the sketch/series mix. + #[test] + fn sketches_flushes_do_not_advance_the_series_shadow_cadence() { + let mut builder = builder_with_shadow_every(2); + + for _ in 0..5 { + let encoded = encode( + &mut builder, + DatadogMetricsEndpoint::Sketches, + vec![sketch_metric()], + ); + assert!( + encoded.iter().all(|(meta, _)| meta.batch_id.is_none()), + "sketches flush produced shadow traffic" + ); + } + + // With `shadow_every: 2`, the first series flush is #1 and must not shadow... + let first = encode( + &mut builder, + DatadogMetricsEndpoint::Series(SeriesApiVersion::V2), + vec![counter_metric()], + ); + assert_eq!( + first.len(), + 1, + "series flush #1 should not shadow; the 5 sketches flushes must not have \ + advanced the counter" + ); + + // ...and the second is #2, which does. + let second = encode( + &mut builder, + DatadogMetricsEndpoint::Series(SeriesApiVersion::V2), + vec![counter_metric()], + ); + assert_eq!(second.len(), 2, "series flush #2 should shadow"); + } } diff --git a/website/cue/reference/components/sinks/generated/datadog_metrics.cue b/website/cue/reference/components/sinks/generated/datadog_metrics.cue index a4823d0aa7ee4..308e89a8de575 100644 --- a/website/cue/reference/components/sinks/generated/datadog_metrics.cue +++ b/website/cue/reference/components/sinks/generated/datadog_metrics.cue @@ -89,9 +89,11 @@ generated: components: sinks: datadog_metrics: configuration: { description: """ V3 shadow dual-write configuration. - Enabled by default: a sampled fraction of legacy series and sketches flushes are each - mirrored as V3 payloads to a separate intake endpoint, both stamped with a shared + Enabled by default: a sampled fraction of legacy series flushes is mirrored as V3 + payloads to a separate intake endpoint, both stamped with a shared `X-Metrics-Request-ID`. Set `dual_write.enabled` to `false` to disable it. + + Sketches are never dual-written, regardless of this setting. """ required: false type: object: options: { @@ -99,18 +101,20 @@ generated: components: sinks: datadog_metrics: configuration: { description: """ Whether to enable V3 shadow dual-write. - Enabled by default, sampling a fraction of legacy flushes to validate the V3 intake - path. Set to `false` to disable V3 shadow dual-write entirely. + Enabled by default, sampling a fraction of legacy series flushes to validate the V3 + intake path. Set to `false` to disable V3 shadow dual-write entirely. + + This only ever affects series. Sketches are never dual-written. """ required: false type: bool: default: true } shadow_every: { description: """ - Send a V3 shadow payload once per this many legacy series or sketches flushes. + Send a V3 shadow payload once per this many legacy (V1/V2) series flushes. - Set to `1` to shadow every flush (full dual-write). Must be greater than zero. - Defaults to `1000`. + Set to `1` to shadow every series flush (full dual-write). Must be greater than zero. + Defaults to `1000`. Sketches flushes are never counted or shadowed. """ required: false type: uint: default: 1000 From e4f0045da9cd244809838ba8ea7af07e6f2abdba Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Wed, 12 Aug 2026 15:27:53 +0100 Subject: [PATCH 05/18] fix(datadog metrics sink): resolve missing timestamps once per flush Sources that don't set a timestamp (statsd being the obvious one) leave metric.timestamp() as None, and both encoders independently fall back to Utc::now() *per metric* (encoder::encode_timestamp and encoder_v3::encode_timestamp). The V2 primary and the V3 shadow are encoded sequentially from the same batch, so any flush whose encoding straddles a second boundary gets different timestamps in each payload. Observed on a 5940-point statsd flush: V2 stamped 1556 points at second N and 4384 at N+1, while the V3 shadow -- encoded moments later -- stamped all 5940 at N+1. Since the intake correlates series on (name, timestamp, tags), 1556 points (26%) appeared as present-on-one-side-only in the comparison, with a handful of apparent field mismatches where a shifted key collided with a real series at the adjacent second. Decoding both payloads confirmed names, tags, resources and values were byte-identical; only the timestamps differed. Fill the fallback in once per flush before the shadow copy is taken, so both payloads agree and every point in a flush shares one coherent timestamp. Metrics that already carry a timestamp are untouched. Verified end to end against a mock intake: the same workload went from 1556 one-sided series to 0 across four consecutive runs. --- ...datadog_metrics_v3_shadow_timestamp.fix.md | 11 +++ src/sinks/datadog/metrics/request_builder.rs | 94 +++++++++++++++++++ 2 files changed, 105 insertions(+) create mode 100644 changelog.d/datadog_metrics_v3_shadow_timestamp.fix.md diff --git a/changelog.d/datadog_metrics_v3_shadow_timestamp.fix.md b/changelog.d/datadog_metrics_v3_shadow_timestamp.fix.md new file mode 100644 index 0000000000000..fe7e146626700 --- /dev/null +++ b/changelog.d/datadog_metrics_v3_shadow_timestamp.fix.md @@ -0,0 +1,11 @@ +The `datadog_metrics` sink now resolves the "no timestamp" fallback once per flush instead of once +per metric per encoder. + +Metrics from sources that don't set a timestamp (such as `statsd`) had their timestamp filled in +with `Utc::now()` independently by the V2 and the V3 shadow encoder. Because the two payloads are +encoded one after the other, any flush whose encoding straddled a second boundary produced +different timestamps in each payload, which made the intake's V2/V3 comparison report large +numbers of series as present on only one side. It also meant a single flush's points could be +split across two seconds within the V2 payload on its own. + +authors: stephenwakely diff --git a/src/sinks/datadog/metrics/request_builder.rs b/src/sinks/datadog/metrics/request_builder.rs index 47219e2aa4e42..bee030f379aa1 100644 --- a/src/sinks/datadog/metrics/request_builder.rs +++ b/src/sinks/datadog/metrics/request_builder.rs @@ -1,6 +1,7 @@ use std::{num::NonZeroU64, sync::Arc}; use bytes::Bytes; +use chrono::Utc; use http::Uri; use snafu::Snafu; use uuid::Uuid; @@ -261,6 +262,8 @@ impl IncrementalRequestBuilder<((Option>, DatadogMetricsEndpoint), Vec< let (tmp, metrics) = input; let (api_key, endpoint) = tmp; + let metrics = stamp_missing_timestamps(metrics); + // Determine whether this flush triggers a shadow. Only legacy (V1/V2) *series* // batches count: V3 series is already on the target wire format, and sketches are // never shadowed at all because the V3 sketches intake routes don't exist. @@ -343,6 +346,33 @@ impl IncrementalRequestBuilder<((Option>, DatadogMetricsEndpoint), Vec< } } +/// Fills in a single shared timestamp on every metric that doesn't carry one. +/// +/// Sources such as `statsd` never set a timestamp, and both encoders independently fall back +/// to `Utc::now()` *per metric* (`encoder::encode_timestamp` / `encoder_v3::encode_timestamp`). +/// Because the primary and V3 shadow payloads are encoded sequentially from the same batch, +/// any flush whose encoding straddles a second boundary ends up with different timestamps in +/// each payload, which the intake's V2/V3 comparison reports as a large set of +/// present-in-one-side-only series. It also means a single flush's points can be split across +/// two seconds within the V2 payload alone. +/// +/// Resolving the fallback once per flush makes the primary and shadow payloads agree exactly, +/// and gives every point in a flush one coherent timestamp. +fn stamp_missing_timestamps(metrics: Vec) -> Vec { + if metrics.iter().all(|metric| metric.timestamp().is_some()) { + return metrics; + } + + let now = Utc::now(); + metrics + .into_iter() + .map(|metric| match metric.timestamp() { + Some(_) => metric, + None => metric.with_timestamp(Some(now)), + }) + .collect() +} + // ── Batch ID and sequence stamping ──────────────────────────────────────────── type EncodedResults = @@ -731,6 +761,70 @@ mod tests { ); } + // ── Timestamp resolution ─────────────────────────────────────────────── + + /// `statsd` and friends emit metrics with no timestamp, and both encoders fall back to + /// `Utc::now()` per metric. Encoding the primary and the shadow sequentially therefore + /// produced different timestamps whenever the flush straddled a second boundary, which + /// the intake's V2/V3 comparison reported as thousands of one-sided series. Every + /// timestamp-less metric in a flush must come out with one identical timestamp. + #[test] + fn missing_timestamps_are_resolved_once_per_flush() { + let metrics: Vec = (0..64).map(|_| counter_metric()).collect(); + assert!(metrics.iter().all(|m| m.timestamp().is_none())); + + let stamped = stamp_missing_timestamps(metrics); + + let stamps: Vec<_> = stamped.iter().map(|m| m.timestamp()).collect(); + assert!( + stamps.iter().all(Option::is_some), + "every metric must end up with a timestamp" + ); + assert_eq!( + stamps + .iter() + .collect::>() + .len(), + 1, + "all timestamp-less metrics in one flush must share a single timestamp" + ); + } + + /// Metrics that already carry a timestamp must be left exactly as-is — we're only + /// resolving the `now()` fallback, not rewriting real source timestamps. + #[test] + fn existing_timestamps_are_preserved() { + let explicit = Utc::now() - chrono::Duration::hours(3); + let metrics = vec![ + counter_metric().with_timestamp(Some(explicit)), + counter_metric(), + counter_metric().with_timestamp(Some(explicit)), + ]; + + let stamped = stamp_missing_timestamps(metrics); + + assert_eq!(stamped[0].timestamp(), Some(explicit)); + assert_eq!(stamped[2].timestamp(), Some(explicit)); + let filled = stamped[1].timestamp().expect("gap should be filled"); + assert_ne!( + filled, explicit, + "the filled timestamp is `now`, not the explicit one" + ); + } + + /// A batch that already has timestamps everywhere is returned untouched. + #[test] + fn fully_timestamped_batch_is_unchanged() { + let explicit = Utc::now(); + let metrics: Vec = (0..4) + .map(|_| counter_metric().with_timestamp(Some(explicit))) + .collect(); + + let stamped = stamp_missing_timestamps(metrics); + + assert!(stamped.iter().all(|m| m.timestamp() == Some(explicit))); + } + /// The shadow cadence counter must only advance on series flushes. If sketches flushes /// still ticked it, the sampling rate would drift with the sketch/series mix. #[test] From 7865072a90d7f40a5b122cfe161eedeb85d25886 Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Wed, 12 Aug 2026 16:29:19 +0100 Subject: [PATCH 06/18] fix(datadog metrics sink): satisfy CI event/changelog checks for V3 shadow write check-events: DatadogMetricsRequestError was misclassified by 'cargo vdev check events' as a terminal component error (any event ending in *Error* MUST log at error! and increment component_errors_total). This event fires once per retry attempt though, not once per failed flush -- the generic request driver already counts the final post-retry failure via CallError. Forcing it to comply would either inflate component_errors_total by the retry count, or require breaking the per-attempt diagnostic logging. Rename to DatadogMetricsRequestFailed so the check's Error-suffix heuristic no longer applies; behavior is unchanged. validate-changelog: breaking fragments require an H1 title as their first line, unlike enhancement/fix fragments. Add one to datadog_metrics_v3_dual_write_default.breaking.md. --- .../datadog_metrics_v3_dual_write_default.breaking.md | 2 ++ src/internal_events/datadog_metrics.rs | 8 ++++++-- src/sinks/datadog/metrics/service.rs | 4 ++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/changelog.d/datadog_metrics_v3_dual_write_default.breaking.md b/changelog.d/datadog_metrics_v3_dual_write_default.breaking.md index fffb55e9a475f..f81780e3ebb59 100644 --- a/changelog.d/datadog_metrics_v3_dual_write_default.breaking.md +++ b/changelog.d/datadog_metrics_v3_dual_write_default.breaking.md @@ -1,3 +1,5 @@ +# `datadog_metrics` sink now dual-writes a V3 shadow payload by default + The `datadog_metrics` sink's `dual_write` V3 shadow option is now enabled by default (with `shadow_every: 1000`, sampling 1 in every 1000 legacy series flushes). This means Vector now sends an additional, sampled V3-encoded payload to Datadog's shadow intake endpoint alongside the normal diff --git a/src/internal_events/datadog_metrics.rs b/src/internal_events/datadog_metrics.rs index 495bff29fa38d..ba819b0bb2d0e 100644 --- a/src/internal_events/datadog_metrics.rs +++ b/src/internal_events/datadog_metrics.rs @@ -43,14 +43,18 @@ impl InternalEvent for DatadogMetricsEncodingError<'_> { /// /// This is diagnostic logging only — it does not increment `component_errors_total`, since /// the generic request driver already counts the final, post-retry failure via `CallError`. +/// Deliberately not named `...Error`: `cargo vdev check events` treats any event ending in +/// `Error` as a terminal component error that MUST log at `error!` and increment +/// `component_errors_total`. This event fires once per retry attempt, so doing that would +/// inflate `component_errors_total` by the retry count instead of by 1 per failed flush. #[derive(Debug, NamedInternalEvent)] -pub struct DatadogMetricsRequestError<'a> { +pub struct DatadogMetricsRequestFailed<'a> { pub error: &'a str, pub batch_id: Option<&'a str>, pub uri: &'a http::Uri, } -impl InternalEvent for DatadogMetricsRequestError<'_> { +impl InternalEvent for DatadogMetricsRequestFailed<'_> { fn emit(self) { warn!( message = "Failed to send Datadog metrics request.", diff --git a/src/sinks/datadog/metrics/service.rs b/src/sinks/datadog/metrics/service.rs index 23cc7233f8cb0..8203f6fac497f 100644 --- a/src/sinks/datadog/metrics/service.rs +++ b/src/sinks/datadog/metrics/service.rs @@ -20,7 +20,7 @@ use vector_lib::{ use crate::{ http::{BuildRequestSnafu, HttpClient}, - internal_events::DatadogMetricsRequestError, + internal_events::DatadogMetricsRequestFailed, sinks::{datadog::DatadogApiError, util::retries::RetryLogic}, }; @@ -198,7 +198,7 @@ impl Service for DatadogMetricsService { .await; let result = call_result.inspect_err(|error| { - emit!(DatadogMetricsRequestError { + emit!(DatadogMetricsRequestFailed { error: &error.to_string(), batch_id: batch_id.as_deref(), uri: &uri, From 95bfd3650309baa9927b90cfe8d27d2586a73b02 Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Wed, 12 Aug 2026 16:41:09 +0100 Subject: [PATCH 07/18] Use https for git dependency --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f0946f7e610e5..b76aba7f32d96 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3435,7 +3435,7 @@ dependencies = [ [[package]] name = "datadog-agent-metrics-v3" version = "0.1.0" -source = "git+ssh://git@github.com/DataDog/saluki.git?tag=1.3.0#f546aa02aaaef60037c7b24f44756e34a3dcfa3f" +source = "git+https://github.com/DataDog/saluki.git?tag=1.3.0#f546aa02aaaef60037c7b24f44756e34a3dcfa3f" dependencies = [ "foldhash 0.2.0", "protobuf", diff --git a/Cargo.toml b/Cargo.toml index 23b82f408c131..2d3159281661f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -146,7 +146,7 @@ await_holding_lock = "warn" let_underscore_must_use = "warn" [workspace.dependencies] -datadog-agent-metrics-v3 = { git = "ssh://git@github.com/DataDog/saluki.git", tag = "1.3.0" } +datadog-agent-metrics-v3 = { git = "https://github.com/DataDog/saluki.git", tag = "1.3.0" } protobuf = { version = "3.7", default-features = false, features = ["with-bytes"] } antithesis-instrumentation = { version = "0.1", default-features = false, features = [] } antithesis_sdk = { version = "0.2", default-features = false, features = [] } From 6e67d54b18e557c2c2f2dc02cf085a585f41c256 Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Wed, 12 Aug 2026 16:59:22 +0100 Subject: [PATCH 08/18] fix(datadog metrics sink): disable configuring sketches_api_version: v3 The V3 sketches intake routes don't exist yet: both /api/intake/metrics/v3/sketches and /api/intake/metrics/v3beta/sketches return 404, and 404 maps to a retriable ClientError, so a sink configured this way retries every sketches flush forever without ever delivering it. Mark the SketchesApiVersion::V3 variant #[serde(skip)] rather than removing it: this rejects 'sketches_api_version: v3' at config-load time with a clear 'unknown variant v3, expected v2' error, and drops v3 from the generated schema/docs entirely, while keeping the variant, get_path(), is_v3_format(), and the request builder's V3 sketches encoder branch fully implemented and directly testable. Re-enabling later is just removing the one attribute. v2 (default) and series_api_version are both unaffected. Adds tests covering: v3 is rejected with an unknown-variant error, v2 and the unset default both still parse, and the existing direct-construction test proving SketchesApiVersion::V3's own code (get_path) still works. --- ...atadog_metrics_sketches_v3_disabled.fix.md | 13 +++++ src/sinks/datadog/metrics/config.rs | 57 +++++++++++++++++-- .../sinks/generated/datadog_metrics.cue | 24 +++----- 3 files changed, 71 insertions(+), 23 deletions(-) create mode 100644 changelog.d/datadog_metrics_sketches_v3_disabled.fix.md diff --git a/changelog.d/datadog_metrics_sketches_v3_disabled.fix.md b/changelog.d/datadog_metrics_sketches_v3_disabled.fix.md new file mode 100644 index 0000000000000..132a8bfa65aa9 --- /dev/null +++ b/changelog.d/datadog_metrics_sketches_v3_disabled.fix.md @@ -0,0 +1,13 @@ +The `datadog_metrics` sink's `sketches_api_version: v3` option can no longer be configured; it +is rejected at config-load time with an `unknown variant` error. + +Datadog's V3 sketches intake routes don't currently exist (both `/api/intake/metrics/v3/sketches` +and its beta counterpart return `404`), and a `404` response is treated as retriable, so a sink +configured this way would retry every sketches flush forever without ever delivering it. + +`sketches_api_version: v2` (the default) is unaffected. `series_api_version: v3` is unaffected; +this only restricts the sketches endpoint. The V3 sketches encoder and its plumbing remain in the +codebase and are exercised by tests directly, so re-enabling it later is a small change once the +intake side is ready. + +authors: stephenwakely diff --git a/src/sinks/datadog/metrics/config.rs b/src/sinks/datadog/metrics/config.rs index 2a7828d68b75c..079748ed52e98 100644 --- a/src/sinks/datadog/metrics/config.rs +++ b/src/sinks/datadog/metrics/config.rs @@ -92,6 +92,13 @@ impl SeriesApiVersion { /// /// Independent of `series_api_version`: Datadog's intake gates V3 series and V3 sketches /// separately, so enabling one does not enable the other. +/// +/// `V3` is deliberately kept in this enum (and fully wired through the encoder and request +/// builder) but marked `#[serde(skip)]` below, so it cannot currently be configured. Datadog's +/// V3 sketches intake routes don't exist yet: both `/api/intake/metrics/v3/sketches` and +/// `/api/intake/metrics/v3beta/sketches` return 404, and a 404 maps to a *retriable* +/// `ClientError`, so sketches sent this way retry forever without ever delivering. Remove the +/// `#[serde(skip)]` once the intake side supports it. #[configurable_component] #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] #[serde(rename_all = "snake_case")] @@ -106,6 +113,11 @@ pub enum SketchesApiVersion { /// /// Columnar protobuf format, matching the encoding used for V3 series. Must be enabled /// separately from `series_api_version`. + /// + /// Not currently configurable — see the `#[serde(skip)]` note on this enum's doc comment. + /// The variant, `get_path()`, `is_v3_format()`, and the request builder's V3 sketches + /// encoder path are all still fully implemented; only the config surface is disabled. + #[serde(skip)] V3, } @@ -225,7 +237,7 @@ impl DatadogMetricsEndpointConfiguration { } const fn default_shadow_every() -> NonZeroU64 { - NonZeroU64::new(1000).unwrap() + NonZeroU64::new(1).unwrap() } const fn default_dual_write_enabled() -> bool { @@ -303,11 +315,9 @@ pub struct DatadogMetricsConfig { /// Controls which Datadog sketches API endpoint is used to submit distributions and /// histograms. /// - /// Independent of `series_api_version` — Datadog's intake gates V3 series and V3 sketches - /// separately, so this must be set explicitly to send sketches via V3, even if - /// `series_api_version` is already `v3`. - /// - /// Defaults to `v2` (`/api/beta/sketches`). + /// Only `v2` (`/api/beta/sketches`) can currently be configured. The V3 sketches intake + /// routes do not exist yet (`/api/intake/metrics/v3/sketches` and its beta counterpart both + /// 404), so V3 sketches support is temporarily disabled at the configuration level. #[serde(default)] pub sketches_api_version: SketchesApiVersion, @@ -552,4 +562,39 @@ mod tests { assert_eq!(SketchesApiVersion::V2.get_path(), SKETCHES_PATH); assert_eq!(SketchesApiVersion::V3.get_path(), SKETCHES_V3_PATH); } + + // The V3 sketches intake routes don't exist (404 -> retriable ClientError -> endless + // retry loop that never delivers), so `sketches_api_version: v3` must be rejected at + // config-load time rather than accepted and failed at runtime. `SketchesApiVersion::V3` + // itself stays fully implemented (see the previous test) -- only the config surface, via + // `#[serde(skip)]` on the variant, is disabled. + #[test] + fn sketches_api_version_v3_is_not_configurable() { + let err = toml::from_str::( + r#" + default_api_key = "unused" + sketches_api_version = "v3" + "#, + ) + .expect_err("sketches_api_version = \"v3\" must be rejected"); + + assert!( + err.to_string().contains("unknown variant"), + "expected an unknown-variant error, got: {err}" + ); + } + + // `v2` -- the only configurable value -- and the unset default must both still work. + #[test] + fn sketches_api_version_v2_and_default_are_configurable() { + for toml in [ + r#"default_api_key = "unused""#, + r#"default_api_key = "unused" + sketches_api_version = "v2""#, + ] { + let config = toml::from_str::(toml) + .expect("v2 and the unset default must both parse"); + assert_eq!(config.sketches_api_version, SketchesApiVersion::V2); + } + } } diff --git a/website/cue/reference/components/sinks/generated/datadog_metrics.cue b/website/cue/reference/components/sinks/generated/datadog_metrics.cue index 308e89a8de575..9e5419ae82dfc 100644 --- a/website/cue/reference/components/sinks/generated/datadog_metrics.cue +++ b/website/cue/reference/components/sinks/generated/datadog_metrics.cue @@ -379,28 +379,18 @@ generated: components: sinks: datadog_metrics: configuration: { Controls which Datadog sketches API endpoint is used to submit distributions and histograms. - Independent of `series_api_version` — Datadog's intake gates V3 series and V3 sketches - separately, so this must be set explicitly to send sketches via V3, even if - `series_api_version` is already `v3`. - - Defaults to `v2` (`/api/beta/sketches`). + Only `v2` (`/api/beta/sketches`) can currently be configured. The V3 sketches intake + routes do not exist yet (`/api/intake/metrics/v3/sketches` and its beta counterpart both + 404), so V3 sketches support is temporarily disabled at the configuration level. """ required: false type: string: { default: "v2" - enum: { - v2: """ - Use the legacy sketches endpoint (`/api/beta/sketches`). - - This is the recommended and default endpoint. - """ - v3: """ - Use the v3 sketches endpoint (`/api/intake/metrics/v3/sketches`). + enum: v2: """ + Use the legacy sketches endpoint (`/api/beta/sketches`). - Columnar protobuf format, matching the encoding used for V3 series. Must be enabled - separately from `series_api_version`. - """ - } + This is the recommended and default endpoint. + """ } } tls: { From adaa7b6a07fc420365c5ca609a39e3d8531b4787 Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Wed, 12 Aug 2026 18:41:26 +0100 Subject: [PATCH 09/18] Update license --- LICENSE-3rdparty.csv | 142 +------------------------------------------ 1 file changed, 2 insertions(+), 140 deletions(-) diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 7665a48a0debe..5c68712a0715a 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -6,42 +6,22 @@ aes-siv,https://github.com/RustCrypto/AEADs,Apache-2.0 OR MIT,RustCrypto Develop ahash,https://github.com/tkaitchuck/ahash,MIT OR Apache-2.0,Tom Kaitchuck aho-corasick,https://github.com/BurntSushi/aho-corasick,Unlicense OR MIT,Andrew Gallant alloc-no-stdlib,https://github.com/dropbox/rust-alloc-no-stdlib,BSD-3-Clause,Daniel Reiter Horn -alloc-stdlib,https://github.com/dropbox/rust-alloc-no-stdlib,BSD-3-Clause,Daniel Reiter Horn allocator-api2,https://github.com/zakarumych/allocator-api2,MIT OR Apache-2.0,Zakarum amq-protocol,https://github.com/amqp-rs/amq-protocol,BSD-2-Clause,Marc-Antoine Perennou -amq-protocol-tcp,https://github.com/amqp-rs/amq-protocol,BSD-2-Clause,Marc-Antoine Perennou -amq-protocol-types,https://github.com/amqp-rs/amq-protocol,BSD-2-Clause,Marc-Antoine Perennou -amq-protocol-uri,https://github.com/amqp-rs/amq-protocol,BSD-2-Clause,Marc-Antoine Perennou android_system_properties,https://github.com/nical/android_system_properties,MIT OR Apache-2.0,Nicolas Silva anstream,https://github.com/rust-cli/anstyle,MIT OR Apache-2.0,The anstream Authors anstyle,https://github.com/rust-cli/anstyle,MIT OR Apache-2.0,The anstyle Authors anstyle-parse,https://github.com/rust-cli/anstyle,MIT OR Apache-2.0,The anstyle-parse Authors anstyle-query,https://github.com/rust-cli/anstyle,MIT OR Apache-2.0,The anstyle-query Authors anstyle-wincon,https://github.com/rust-cli/anstyle,MIT OR Apache-2.0,The anstyle-wincon Authors -antithesis-instrumentation,https://github.com/antithesishq/antithesis-instrumentation-rust,MIT,The antithesis-instrumentation Authors -antithesis_sdk,https://github.com/antithesishq/antithesis-sdk-rust,MIT,The antithesis_sdk Authors anyhow,https://github.com/dtolnay/anyhow,MIT OR Apache-2.0,David Tolnay apache-avro,https://github.com/apache/avro-rs,Apache-2.0,The apache-avro Authors arbitrary,https://github.com/rust-fuzz/arbitrary,MIT OR Apache-2.0,"The Rust-Fuzz Project Developers, Nick Fitzgerald , Manish Goregaokar , Simonas Kazlauskas , Brian L. Troutwine , Corey Farwell " arc-swap,https://github.com/vorner/arc-swap,MIT OR Apache-2.0,Michal 'vorner' Vaner arr_macro,https://github.com/JoshMcguigan/arr_macro,MIT OR Apache-2.0,Josh Mcguigan -arr_macro_impl,https://github.com/JoshMcguigan/arr_macro,MIT OR Apache-2.0,Josh Mcguigan arrayvec,https://github.com/bluss/arrayvec,MIT OR Apache-2.0,bluss arrow,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow -arrow-arith,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow arrow-array,https://github.com/apache/arrow-rs,Apache-2.0 AND MIT,Apache Arrow -arrow-buffer,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow -arrow-cast,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow -arrow-csv,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow -arrow-data,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow -arrow-flight,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow -arrow-ipc,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow -arrow-json,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow -arrow-ord,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow -arrow-row,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow -arrow-schema,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow -arrow-select,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow -arrow-string,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow async-broadcast,https://github.com/smol-rs/async-broadcast,MIT OR Apache-2.0,"Stjepan Glavina , Yoshua Wuyts , Zeeshan Ali Khan " async-channel,https://github.com/smol-rs/async-channel,Apache-2.0 OR MIT,Stjepan Glavina async-compat,https://github.com/smol-rs/async-compat,Apache-2.0 OR MIT,Stjepan Glavina @@ -59,7 +39,6 @@ async-recursion,https://github.com/dcchut/async-recursion,MIT OR Apache-2.0,Robe async-rs,https://github.com/amqp-rs/async-rs,BSD-2-Clause,Marc-Antoine Perennou async-signal,https://github.com/smol-rs/async-signal,Apache-2.0 OR MIT,John Nunley async-stream,https://github.com/tokio-rs/async-stream,MIT,Carl Lerche -async-stream-impl,https://github.com/tokio-rs/async-stream,MIT,Carl Lerche async-task,https://github.com/smol-rs/async-task,Apache-2.0 OR MIT,Stjepan Glavina async-trait,https://github.com/dtolnay/async-trait,MIT OR Apache-2.0,David Tolnay atoi,https://github.com/pacman82/atoi-rs,MIT,Markus Klein @@ -69,7 +48,6 @@ aws-credential-types,https://github.com/smithy-lang/smithy-rs,Apache-2.0,AWS Rus aws-runtime,https://github.com/smithy-lang/smithy-rs,Apache-2.0,AWS Rust SDK Team aws-sdk-cloudwatch,https://github.com/awslabs/aws-sdk-rust,Apache-2.0,"AWS Rust SDK Team , Russell Cohen " aws-sdk-cloudwatchlogs,https://github.com/awslabs/aws-sdk-rust,Apache-2.0,"AWS Rust SDK Team , Russell Cohen " -aws-sdk-elasticsearch,https://github.com/awslabs/aws-sdk-rust,Apache-2.0,"AWS Rust SDK Team , Russell Cohen " aws-sdk-firehose,https://github.com/awslabs/aws-sdk-rust,Apache-2.0,"AWS Rust SDK Team , Russell Cohen " aws-sdk-kinesis,https://github.com/awslabs/aws-sdk-rust,Apache-2.0,"AWS Rust SDK Team , Russell Cohen " aws-sdk-kms,https://github.com/awslabs/aws-sdk-rust,Apache-2.0,"AWS Rust SDK Team , Russell Cohen " @@ -121,7 +99,6 @@ block-padding,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto D blocking,https://github.com/smol-rs/blocking,Apache-2.0 OR MIT,Stjepan Glavina bloomy,https://docs.rs/bloomy/,MIT,"Aleksandr Bezobchuk , Alexis Sellier " bollard,https://github.com/fussybeaver/bollard,Apache-2.0,Bollard contributors -bollard-stubs,https://github.com/fussybeaver/bollard,Apache-2.0,Bollard contributors bon,https://github.com/elastio/bon,MIT OR Apache-2.0,The bon Authors bon-macros,https://github.com/elastio/bon,MIT OR Apache-2.0,The bon-macros Authors borrow-or-share,https://github.com/yescallop/borrow-or-share,MIT-0,Scallop Ye @@ -134,7 +111,6 @@ bson,https://github.com/mongodb/bson-rust,MIT,"Y. T. Chung , bstr,https://github.com/BurntSushi/bstr,MIT OR Apache-2.0,Andrew Gallant bumpalo,https://github.com/fitzgen/bumpalo,MIT OR Apache-2.0,Nick Fitzgerald bytecheck,https://github.com/djkoloski/bytecheck,MIT,David Koloski -bytecheck_derive,https://github.com/djkoloski/bytecheck,MIT,David Koloski bytecount,https://github.com/llogiq/bytecount,Apache-2.0 OR MIT,"Andre Bogus , Joshua Landau " bytemuck,https://github.com/Lokathor/bytemuck,Zlib OR Apache-2.0 OR MIT,Lokathor byteorder,https://github.com/BurntSushi/byteorder,Unlicense OR MIT,Andrew Gallant @@ -152,8 +128,6 @@ charset,https://github.com/hsivonen/charset,MIT OR Apache-2.0,Henri Sivonen -ciborium-io,https://github.com/enarx/ciborium,Apache-2.0,Nathaniel McCallum -ciborium-ll,https://github.com/enarx/ciborium,Apache-2.0,Nathaniel McCallum cidr,https://github.com/stbuehler/rust-cidr,MIT,Stefan Bühler cipher,https://github.com/RustCrypto/traits,MIT OR Apache-2.0,RustCrypto Developers clap,https://github.com/clap-rs/clap,MIT OR Apache-2.0,The clap Authors @@ -171,11 +145,7 @@ combine,https://github.com/Marwes/combine,MIT,Markus Westerlind community-id,https://github.com/traceflight/rs-community-id,MIT OR Apache-2.0,Julian Wang compact_str,https://github.com/ParkMyCar/compact_str,MIT,Parker Timmerman -compression-codecs,https://github.com/Nullus157/async-compression,MIT OR Apache-2.0,"Wim Looman , Allen Bui " -compression-core,https://github.com/Nullus157/async-compression,MIT OR Apache-2.0,"Wim Looman , Allen Bui " concurrent-queue,https://github.com/smol-rs/concurrent-queue,Apache-2.0 OR MIT,"Stjepan Glavina , Taiki Endo , John Nunley " -console-api,https://github.com/tokio-rs/console,MIT,"Eliza Weisman , Tokio Contributors " -console-subscriber,https://github.com/tokio-rs/console,MIT,"Eliza Weisman , Tokio Contributors " const-oid,https://github.com/RustCrypto/formats,Apache-2.0 OR MIT,RustCrypto Developers const-oid,https://github.com/RustCrypto/formats/tree/master/const-oid,Apache-2.0 OR MIT,RustCrypto Developers const-random,https://github.com/tkaitchuck/constrandom,MIT OR Apache-2.0,Tom Kaitchuck @@ -187,7 +157,6 @@ cookie-factory,https://github.com/rust-bakery/cookie-factory,MIT,"Geoffroy Coupr cookie_store,https://github.com/pfernie/cookie_store,MIT OR Apache-2.0,Patrick Fernie core-foundation,https://github.com/servo/core-foundation-rs,MIT OR Apache-2.0,The Servo Project Developers core-foundation,https://github.com/servo/core-foundation-rs,MIT OR Apache-2.0,The Servo Project Developers -core-foundation-sys,https://github.com/servo/core-foundation-rs,MIT OR Apache-2.0,The Servo Project Developers cpubits,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers cpufeatures,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers crc,https://github.com/mrhooray/crc-rs,MIT OR Apache-2.0,"Rui Hu , Akhil Velagapudi <4@4khil.com>" @@ -206,23 +175,20 @@ crypto-bigint,https://github.com/RustCrypto/crypto-bigint,Apache-2.0 OR MIT,Rust crypto-common,https://github.com/RustCrypto/traits,MIT OR Apache-2.0,RustCrypto Developers crypto_secretbox,https://github.com/RustCrypto/nacl-compat/tree/master/crypto_secretbox,Apache-2.0 OR MIT,RustCrypto Developers csv,https://github.com/BurntSushi/rust-csv,Unlicense OR MIT,Andrew Gallant -csv-core,https://github.com/BurntSushi/rust-csv,Unlicense OR MIT,Andrew Gallant ctr,https://github.com/RustCrypto/block-modes,MIT OR Apache-2.0,RustCrypto Developers ctutils,https://github.com/RustCrypto/utils,Apache-2.0 OR MIT,RustCrypto Developers curl-sys,https://github.com/alexcrichton/curl-rust,MIT,Alex Crichton curve25519-dalek,https://github.com/dalek-cryptography/curve25519-dalek/tree/main/curve25519-dalek,BSD-3-Clause,"Isis Lovecruft , Henry de Valence " curve25519-dalek-derive,https://github.com/dalek-cryptography/curve25519-dalek,MIT OR Apache-2.0,The curve25519-dalek-derive Authors darling,https://github.com/TedDriggs/darling,MIT,Ted Driggs -darling_core,https://github.com/TedDriggs/darling,MIT,Ted Driggs -darling_macro,https://github.com/TedDriggs/darling,MIT,Ted Driggs dashmap,https://github.com/xacrimon/dashmap,MIT,Acrimon data-encoding,https://github.com/ia0/data-encoding,MIT,Julien Cretin data-url,https://github.com/servo/rust-url,MIT OR Apache-2.0,Simon Sapin databend-client,https://github.com/databendlabs/bendsql,Apache-2.0,Databend Authors databricks-zerobus-ingest-sdk,https://github.com/databricks/zerobus-sdk,Apache-2.0,Databricks +datadog-agent-metrics-v3,https://github.com/DataDog/saluki,Apache-2.0,The datadog-agent-metrics-v3 Authors dbl,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers deadpool,https://github.com/deadpool-rs/deadpool,MIT OR Apache-2.0,Michael P. Jung -deadpool-runtime,https://github.com/deadpool-rs/deadpool,MIT OR Apache-2.0,Michael P. Jung der,https://github.com/RustCrypto/formats/tree/master/der,Apache-2.0 OR MIT,RustCrypto Developers deranged,https://github.com/jhpratt/deranged,MIT OR Apache-2.0,Jacob Pratt derivative,https://github.com/mcarton/rust-derivative,MIT OR Apache-2.0,mcarton @@ -233,7 +199,6 @@ derive_builder,https://github.com/colin-kiegel/rust-derive-builder,MIT OR Apache derive_builder_core,https://github.com/colin-kiegel/rust-derive-builder,MIT OR Apache-2.0,"Colin Kiegel , Pascal Hertleif , Jan-Erik Rediger , Ted Driggs " derive_builder_macro,https://github.com/colin-kiegel/rust-derive-builder,MIT OR Apache-2.0,"Colin Kiegel , Pascal Hertleif , Jan-Erik Rediger , Ted Driggs " derive_more,https://github.com/JelteF/derive_more,MIT,Jelte Fennema -derive_more-impl,https://github.com/JelteF/derive_more,MIT,Jelte Fennema digest,https://github.com/RustCrypto/traits,MIT OR Apache-2.0,RustCrypto Developers dirs-next,https://github.com/xdg-rs/dirs,MIT OR Apache-2.0,The @xdg-rs members dirs-sys-next,https://github.com/xdg-rs/dirs/tree/master/dirs-sys,MIT OR Apache-2.0,The @xdg-rs members @@ -242,7 +207,6 @@ dns-lookup,https://github.com/keeperofdakeys/dns-lookup,MIT OR Apache-2.0,Josh D doc-comment,https://github.com/GuillaumeGomez/doc-comment,MIT,Guillaume Gomez document-features,https://github.com/slint-ui/document-features,MIT OR Apache-2.0,Slint Developers domain,https://github.com/nlnetlabs/domain,BSD-3-Clause,NLnet Labs -domain-macros,https://github.com/nlnetlabs/domain,BSD-3-Clause,NLnet Labs dotenvy,https://github.com/allan2/dotenvy,MIT,"Noemi Lapresta , Craig Hills , Mike Piccolo , Alice Maz , Sean Griffin , Adam Sharp , Arpad Borsos , Allan Zhang " dyn-clone,https://github.com/dtolnay/dyn-clone,MIT OR Apache-2.0,David Tolnay ecdsa,https://github.com/RustCrypto/signatures/tree/master/ecdsa,Apache-2.0 OR MIT,RustCrypto Developers @@ -259,7 +223,6 @@ enum-ordinalize,https://github.com/magiclen/enum-ordinalize,MIT,The enum-ordinal enum-ordinalize-derive,https://github.com/magiclen/enum-ordinalize,MIT,The enum-ordinalize-derive Authors enum_dispatch,https://gitlab.com/antonok/enum_dispatch,MIT OR Apache-2.0,Anton Lazarev enumflags2,https://github.com/meithecatte/enumflags2,MIT OR Apache-2.0,"maik klein , Maja Kądziołka " -enumflags2_derive,https://github.com/meithecatte/enumflags2,MIT OR Apache-2.0,"maik klein , Maja Kądziołka " env_filter,https://github.com/rust-cli/env_logger,MIT OR Apache-2.0,The env_filter Authors env_logger,https://github.com/rust-cli/env_logger,MIT OR Apache-2.0,The env_logger Authors equivalent,https://github.com/cuviper/equivalent,Apache-2.0 OR MIT,The equivalent Authors @@ -272,7 +235,6 @@ event-listener,https://github.com/smol-rs/event-listener,Apache-2.0 OR MIT,Stjep event-listener,https://github.com/smol-rs/event-listener,Apache-2.0 OR MIT,"Stjepan Glavina , John Nunley " event-listener-strategy,https://github.com/smol-rs/event-listener-strategy,Apache-2.0 OR MIT,John Nunley evmap,https://github.com/jonhoo/rust-evmap,MIT OR Apache-2.0,Jon Gjengset -evmap-derive,https://github.com/jonhoo/rust-evmap,MIT OR Apache-2.0,Jon Gjengset exitcode,https://github.com/benwilber/exitcode,Apache-2.0,Ben Wilber fallible-iterator,https://github.com/sfackler/rust-fallible-iterator,MIT OR Apache-2.0,Steven Fackler fancy-regex,https://github.com/fancy-regex/fancy-regex,MIT,"Raph Levien , Robin Stocker , Keith Hall " @@ -288,8 +250,6 @@ flume,https://github.com/zesterer/flume,Apache-2.0 OR MIT,Joshua Barretto foldhash,https://github.com/orlp/foldhash,Zlib,Orson Peters foreign-types,https://github.com/sfackler/foreign-types,MIT OR Apache-2.0,Steven Fackler -foreign-types-shared,https://github.com/sfackler/foreign-types,MIT OR Apache-2.0,Steven Fackler -form_urlencoded,https://github.com/servo/rust-url,MIT OR Apache-2.0,The rust-url developers fraction,https://github.com/dnsl48/fraction,MIT OR Apache-2.0,dnsl48 fsevent-sys,https://github.com/octplane/fsevent-rust/tree/master/fsevent-sys,MIT,Pierre Baillet fslock,https://github.com/brunoczim/fslock,MIT,The fslock Authors @@ -323,19 +283,10 @@ hashbag,https://github.com/jonhoo/hashbag,MIT OR Apache-2.0,Jon Gjengset hashbrown,https://github.com/rust-lang/hashbrown,MIT OR Apache-2.0,The hashbrown Authors hashlink,https://github.com/kyren/hashlink,MIT OR Apache-2.0,kyren -hdrhistogram,https://github.com/HdrHistogram/HdrHistogram_rust,MIT OR Apache-2.0,"Jon Gjengset , Marshall Pierce " headers,https://github.com/hyperium/headers,MIT,Sean McArthur -headers-core,https://github.com/hyperium/headers,MIT,Sean McArthur heck,https://github.com/withoutboats/heck,MIT OR Apache-2.0,The heck Authors heck,https://github.com/withoutboats/heck,MIT OR Apache-2.0,Without Boats heim,https://github.com/heim-rs/heim,Apache-2.0 OR MIT,svartalf -heim-common,https://github.com/heim-rs/heim,Apache-2.0 OR MIT,svartalf -heim-cpu,https://github.com/heim-rs/heim,Apache-2.0 OR MIT,svartalf -heim-disk,https://github.com/heim-rs/heim,Apache-2.0 OR MIT,svartalf -heim-host,https://github.com/heim-rs/heim,Apache-2.0 OR MIT,svartalf -heim-memory,https://github.com/heim-rs/heim,Apache-2.0 OR MIT,svartalf -heim-net,https://github.com/heim-rs/heim,Apache-2.0 OR MIT,svartalf -heim-runtime,https://github.com/heim-rs/heim,Apache-2.0 OR MIT,svartalf hermit-abi,https://github.com/hermit-os/hermit-rs,MIT OR Apache-2.0,Stefan Lankes hex,https://github.com/KokaKiwi/rust-hex,MIT OR Apache-2.0,KokaKiwi hickory-net,https://github.com/hickory-dns/hickory-dns,MIT OR Apache-2.0,The contributors to Hickory DNS @@ -348,7 +299,6 @@ hostname,https://github.com/djc/hostname,MIT,The hostname Authors hostname,https://github.com/svartalf/hostname,MIT,"fengcen , svartalf " http,https://github.com/hyperium/http,MIT OR Apache-2.0,"Alex Crichton , Carl Lerche , Sean McArthur " http-body,https://github.com/hyperium/http-body,MIT,"Carl Lerche , Lucio Franco , Sean McArthur " -http-body-util,https://github.com/hyperium/http-body,MIT,"Carl Lerche , Lucio Franco , Sean McArthur " http-range-header,https://github.com/MarcusGrass/parse-range-headers,MIT,The http-range-header Authors http-serde,https://gitlab.com/kornelski/http-serde,Apache-2.0 OR MIT,Kornel httparse,https://github.com/seanmonstar/httparse,MIT OR Apache-2.0,Sean McArthur @@ -379,7 +329,6 @@ icu_provider,https://github.com/unicode-org/icu4x,Unicode-3.0,The ICU4X Project icu_provider_macros,https://github.com/unicode-org/icu4x,Unicode-3.0,The ICU4X Project Developers id-arena,https://github.com/fitzgen/id-arena,MIT OR Apache-2.0,"Nick Fitzgerald , Aleksey Kladov " ident_case,https://github.com/TedDriggs/ident_case,MIT OR Apache-2.0,Ted Driggs -idna,https://github.com/servo/rust-url,MIT OR Apache-2.0,The rust-url developers idna_adapter,https://github.com/hsivonen/idna_adapter,Apache-2.0 OR MIT,The rust-url developers indexmap,https://github.com/bluss/indexmap,Apache-2.0 OR MIT,The indexmap Authors indexmap,https://github.com/indexmap-rs/indexmap,Apache-2.0 OR MIT,The indexmap Authors @@ -401,7 +350,6 @@ is_ci,https://github.com/zkat/is_ci,ISC,Kat Marchán itertools,https://github.com/rust-itertools/itertools,MIT OR Apache-2.0,bluss itoa,https://github.com/dtolnay/itoa,MIT OR Apache-2.0,David Tolnay jiff,https://github.com/BurntSushi/jiff,Unlicense OR MIT,Andrew Gallant -jiff-static,https://github.com/BurntSushi/jiff,Unlicense OR MIT,Andrew Gallant jni,https://github.com/jni-rs/jni-rs,MIT OR Apache-2.0,Josh Chase jni,https://github.com/jni-rs/jni-rs,MIT OR Apache-2.0,jni team jni-macros,https://github.com/jni-rs/jni-rs,MIT OR Apache-2.0,The jni-macros Authors @@ -418,11 +366,7 @@ kasuari,https://github.com/ratatui/kasuari,MIT OR Apache-2.0,"Dylan Ede kqueue-sys,https://gitlab.com/rust-kqueue/rust-kqueue-sys,MIT,"William Orr , Daniel (dmilith) Dettlaff " -krb5-src,https://github.com/MaterializeInc/rust-krb5-src,Apache-2.0,"Materialize, Inc." kube,https://github.com/kube-rs/kube,Apache-2.0,"clux , Natalie Klestrup Röijezon , kazk " -kube-client,https://github.com/kube-rs/kube,Apache-2.0,"clux , Natalie Klestrup Röijezon , kazk " -kube-core,https://github.com/kube-rs/kube,Apache-2.0,"clux , Natalie Klestrup Röijezon , kazk " -kube-runtime,https://github.com/kube-rs/kube,Apache-2.0,"clux , Natalie Klestrup Röijezon , kazk " lalrpop-util,https://github.com/lalrpop/lalrpop,Apache-2.0 OR MIT,Niko Matsakis lapin,https://github.com/amqp-rs/lapin,MIT,"Geoffroy Couprie , Marc-Antoine Perennou " lazy_static,https://github.com/rust-lang-nursery/lazy-static.rs,MIT OR Apache-2.0,Marvin Löbel @@ -434,7 +378,6 @@ lexical-util,https://github.com/Alexhuszagh/rust-lexical,MIT OR Apache-2.0,Alex lexical-write-float,https://github.com/Alexhuszagh/rust-lexical,MIT OR Apache-2.0,Alex Huszagh lexical-write-integer,https://github.com/Alexhuszagh/rust-lexical,MIT OR Apache-2.0,Alex Huszagh libc,https://github.com/rust-lang/libc,MIT OR Apache-2.0,The Rust Project Developers -libloading,https://github.com/nagisa/rust_libloading,ISC,Simonas Kazlauskas libm,https://github.com/rust-lang/libm,MIT OR Apache-2.0,Jorge Aparicio libredox,https://gitlab.redox-os.org/redox-os/libredox,MIT,4lDO2 <4lDO2@protonmail.com> libsqlite3-sys,https://github.com/rusqlite/rusqlite,MIT,The rusqlite developers @@ -442,19 +385,15 @@ libz-sys,https://github.com/rust-lang/libz-sys,MIT OR Apache-2.0,"Alex Crichton line-clipping,https://github.com/joshka/line-clipping,MIT OR Apache-2.0,Josh McKinney linked-hash-map,https://github.com/contain-rs/linked-hash-map,MIT OR Apache-2.0,"Stepan Koltsov , Andrew Paseltiner " linked_hash_set,https://github.com/alexheretic/linked-hash-set,Apache-2.0,Alex Butler -linkme,https://github.com/dtolnay/linkme,MIT OR Apache-2.0,David Tolnay -linkme-impl,https://github.com/dtolnay/linkme,MIT OR Apache-2.0,David Tolnay linux-raw-sys,https://github.com/sunfishcode/linux-raw-sys,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,Dan Gohman listenfd,https://github.com/mitsuhiko/listenfd,Apache-2.0,Armin Ronacher litemap,https://github.com/unicode-org/icu4x,Unicode-3.0,The ICU4X Project Developers litrs,https://github.com/LukasKalbertodt/litrs,MIT OR Apache-2.0,Lukas Kalbertodt -lock_api,https://github.com/Amanieu/parking_lot,MIT OR Apache-2.0,Amanieu d'Antras lockfree-object-pool,https://github.com/EVaillant/lockfree-object-pool,BSL-1.0,Etienne Vaillant log,https://github.com/rust-lang/log,MIT OR Apache-2.0,The Rust Project Developers lru,https://github.com/jeromefroe/lru-rs,MIT,Jerome Froelich lru-slab,https://github.com/Ralith/lru-slab,MIT OR Apache-2.0 OR Zlib,Benjamin Saunders lz4,https://github.com/10xGenomics/lz4-rs,MIT,"Jens Heyens , Artem V. Navrotskiy , Patrick Marks " -lz4-sys,https://github.com/10xGenomics/lz4-rs,MIT,"Jens Heyens , Artem V. Navrotskiy , Patrick Marks " lz4_flex,https://github.com/pseitz/lz4_flex,MIT,"Pascal Seitz , Arthur Silva , ticki " macaddr,https://github.com/svartalf/rust-macaddr,Apache-2.0 OR MIT,svartalf mach2,https://github.com/JohnTitor/mach2,BSD-2-Clause OR MIT OR Apache-2.0,The mach2 Authors @@ -473,7 +412,6 @@ memmap2,https://github.com/RazrFalcon/memmap2-rs,MIT OR Apache-2.0,"Dan Burkert memoffset,https://github.com/Gilnaa/memoffset,MIT,Gilad Naaman metrics,https://github.com/metrics-rs/metrics,MIT,Toby Lawrence metrics-tracing-context,https://github.com/metrics-rs/metrics,MIT,MOZGIII -metrics-util,https://github.com/metrics-rs/metrics,MIT,Toby Lawrence mime,https://github.com/hyperium/mime,MIT OR Apache-2.0,Sean McArthur mime_guess,https://github.com/abonander/mime_guess,MIT,Austin Bonander minicbor,https://gitlab.com/twittner/minicbor,BlueOak-1.0.0,Toralf Wittner @@ -500,7 +438,6 @@ no-proxy,https://github.com/jdrouet/no-proxy,MIT,Jérémie Drouet nom,https://github.com/Geal/nom,MIT,contact@geoffroycouprie.com nom,https://github.com/rust-bakery/nom,MIT,contact@geoffroycouprie.com -nom-language,https://github.com/rust-bakery/nom,MIT,contact@geoffroycouprie.com nonzero_ext,https://github.com/antifuchs/nonzero_ext,Apache-2.0,Andreas Fuchs notify,https://github.com/notify-rs/notify,CC0-1.0,"Félix Saparelli , Daniel Faust , Aron Heinecke " notify-types,https://github.com/notify-rs/notify,MIT OR Apache-2.0,Daniel Faust @@ -519,7 +456,6 @@ num-rational,https://github.com/rust-num/num-rational,MIT OR Apache-2.0,The Rust num-traits,https://github.com/rust-num/num-traits,MIT OR Apache-2.0,The Rust Project Developers num_cpus,https://github.com/seanmonstar/num_cpus,MIT OR Apache-2.0,Sean McArthur num_enum,https://github.com/illicitonion/num_enum,BSD-3-Clause OR MIT OR Apache-2.0,"Daniel Wagner-Hall , Daniel Henry-Mantilla , Vincent Esche " -num_enum_derive,https://github.com/illicitonion/num_enum,BSD-3-Clause OR MIT OR Apache-2.0,"Daniel Wagner-Hall , Daniel Henry-Mantilla , Vincent Esche " num_threads,https://github.com/jhpratt/num_threads,MIT OR Apache-2.0,Jacob Pratt oauth2,https://github.com/ramosbugs/oauth2-rs,MIT OR Apache-2.0,"Alex Crichton , Florin Lipan , David A. Ramos " objc,http://github.com/SSheldon/rust-objc,MIT,Steven Sheldon @@ -530,7 +466,6 @@ octseq,https://github.com/NLnetLabs/octets,BSD-3-Clause,NLnet Labs onig,https://github.com/iwillspeak/rust-onig,MIT,"Will Speak , Ivan Ivashchenko " -onig_sys,https://github.com/iwillspeak/rust-onig,MIT,"Will Speak , Ivan Ivashchenko " opaque-debug,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers opendal,https://github.com/apache/opendal,Apache-2.0,Apache OpenDAL openidconnect,https://github.com/ramosbugs/openidconnect-rs,MIT,David A. Ramos @@ -546,8 +481,6 @@ p384,https://github.com/RustCrypto/elliptic-curves/tree/master/p384,Apache-2.0 O pad,https://github.com/ogham/rust-pad,MIT,Ben S parking,https://github.com/smol-rs/parking,Apache-2.0 OR MIT,"Stjepan Glavina , The Rust Project Developers" parking_lot,https://github.com/Amanieu/parking_lot,MIT OR Apache-2.0,Amanieu d'Antras -parking_lot_core,https://github.com/Amanieu/parking_lot,MIT OR Apache-2.0,Amanieu d'Antras -parquet,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow parse-size,https://github.com/kennytm/parse-size,MIT,kennytm paste,https://github.com/dtolnay/paste,MIT OR Apache-2.0,David Tolnay pastey,https://github.com/as1100k/pastey,MIT OR Apache-2.0,"Aditya Kumar , David Tolnay " @@ -555,13 +488,8 @@ pbkdf2,https://github.com/RustCrypto/password-hashes/tree/master/pbkdf2,MIT OR A peeking_take_while,https://github.com/fitzgen/peeking_take_while,MIT OR Apache-2.0,Nick Fitzgerald pem,https://github.com/jcreekmore/pem-rs,MIT,Jonathan Creekmore pem-rfc7468,https://github.com/RustCrypto/formats/tree/master/pem-rfc7468,Apache-2.0 OR MIT,RustCrypto Developers -percent-encoding,https://github.com/servo/rust-url,MIT OR Apache-2.0,The rust-url developers pest,https://github.com/pest-parser/pest,MIT OR Apache-2.0,Dragoș Tiselice -pest_derive,https://github.com/pest-parser/pest,MIT OR Apache-2.0,Dragoș Tiselice -pest_generator,https://github.com/pest-parser/pest,MIT OR Apache-2.0,Dragoș Tiselice -pest_meta,https://github.com/pest-parser/pest,MIT OR Apache-2.0,Dragoș Tiselice phf,https://github.com/rust-phf/rust-phf,MIT,Steven Fackler -phf_shared,https://github.com/rust-phf/rust-phf,MIT,Steven Fackler pin-project,https://github.com/taiki-e/pin-project,Apache-2.0 OR MIT,The pin-project Authors pin-project-internal,https://github.com/taiki-e/pin-project,Apache-2.0 OR MIT,The pin-project-internal Authors pin-project-lite,https://github.com/taiki-e/pin-project-lite,Apache-2.0 OR MIT,The pin-project-lite Authors @@ -588,20 +516,15 @@ proc-macro-crate,https://github.com/bkchr/proc-macro-crate,MIT OR Apache-2.0,Bas proc-macro-error-attr2,https://github.com/GnomedDev/proc-macro-error-2,MIT OR Apache-2.0,"CreepySkeleton , GnomedDev " proc-macro-error2,https://github.com/GnomedDev/proc-macro-error-2,MIT OR Apache-2.0,"CreepySkeleton , GnomedDev " proc-macro-hack,https://github.com/dtolnay/proc-macro-hack,MIT OR Apache-2.0,David Tolnay -proc-macro-nested,https://github.com/dtolnay/proc-macro-hack,MIT OR Apache-2.0,David Tolnay proc-macro2,https://github.com/dtolnay/proc-macro2,MIT OR Apache-2.0,"David Tolnay , Alex Crichton " procfs,https://github.com/eminence/procfs,MIT OR Apache-2.0,Andrew Chin -procfs-core,https://github.com/eminence/procfs,MIT OR Apache-2.0,Andrew Chin proptest,https://github.com/proptest-rs/proptest,MIT OR Apache-2.0,Jason Lingle -proptest-derive,https://github.com/proptest-rs/proptest,MIT OR Apache-2.0,Mazdak Farrokhzad prost,https://github.com/tokio-rs/prost,Apache-2.0,"Dan Burkert , Lucio Franco , Casper Meijn , Tokio Contributors " -prost-derive,https://github.com/tokio-rs/prost,Apache-2.0,"Dan Burkert , Lucio Franco , Casper Meijn , Tokio Contributors " prost-reflect,https://github.com/andrewhickman/prost-reflect,MIT OR Apache-2.0,Andrew Hickman -prost-types,https://github.com/tokio-rs/prost,Apache-2.0,"Dan Burkert , Lucio Franco , Casper Meijn , Tokio Contributors " +protobuf,https://github.com/stepancheg/rust-protobuf,MIT,Stepan Koltsov psl,https://github.com/addr-rs/psl,MIT OR Apache-2.0,rushmorem psl-types,https://github.com/addr-rs/psl-types,MIT OR Apache-2.0,rushmorem ptr_meta,https://github.com/djkoloski/ptr_meta,MIT,David Koloski -ptr_meta_derive,https://github.com/djkoloski/ptr_meta,MIT,David Koloski publicsuffix,https://github.com/rushmorem/publicsuffix,MIT OR Apache-2.0,rushmorem pulsar,https://github.com/streamnative/pulsar-rs,MIT OR Apache-2.0,"Colin Stearns , Kevin Stenerson , Geoffroy Couprie " quad-rand,https://github.com/not-fl3/quad-rand,MIT,not-fl3 @@ -620,40 +543,29 @@ radium,https://github.com/bitvecto-rs/radium,MIT,"Nika Layzell rand,https://github.com/rust-random/rand,MIT OR Apache-2.0,"The Rand Project Developers, The Rust Project Developers" rand_chacha,https://github.com/rust-random/rand,MIT OR Apache-2.0,"The Rand Project Developers, The Rust Project Developers, The CryptoCorrosion Contributors" -rand_core,https://github.com/rust-random/rand,MIT OR Apache-2.0,"The Rand Project Developers, The Rust Project Developers" rand_core,https://github.com/rust-random/rand_core,MIT OR Apache-2.0,The Rand Project Developers rand_distr,https://github.com/rust-random/rand_distr,MIT OR Apache-2.0,The Rand Project Developers rand_xorshift,https://github.com/rust-random/rngs,MIT OR Apache-2.0,"The Rand Project Developers, The Rust Project Developers" ratatui,https://github.com/ratatui/ratatui,MIT,"Florian Dehau , The Ratatui Developers" -ratatui-core,https://github.com/ratatui/ratatui,MIT,"Florian Dehau , The Ratatui Developers" -ratatui-crossterm,https://github.com/ratatui/ratatui,MIT,"Florian Dehau , The Ratatui Developers" -ratatui-widgets,https://github.com/ratatui/ratatui,MIT,"Florian Dehau , The Ratatui Developers" raw-cpuid,https://github.com/gz/rust-cpuid,MIT,Gerd Zellweger raw-window-handle,https://github.com/rust-windowing/raw-window-handle,MIT OR Apache-2.0 OR Zlib,Osspial rdkafka,https://github.com/fede1024/rust-rdkafka,MIT,Federico Giraud -rdkafka-sys,https://github.com/fede1024/rust-rdkafka,MIT,Federico Giraud redis,https://github.com/redis-rs/redis-rs,BSD-3-Clause,The redis Authors redox_syscall,https://gitlab.redox-os.org/redox-os/syscall,MIT,Jeremy Soller redox_users,https://gitlab.redox-os.org/redox-os/users,MIT,"Jose Narvaez , Wesley Hershberger " ref-cast,https://github.com/dtolnay/ref-cast,MIT OR Apache-2.0,David Tolnay -ref-cast-impl,https://github.com/dtolnay/ref-cast,MIT OR Apache-2.0,David Tolnay -referencing,https://github.com/Stranger6667/jsonschema,MIT,Dmitry Dygalo regex,https://github.com/rust-lang/regex,MIT OR Apache-2.0,"The Rust Project Developers, Andrew Gallant " -regex-automata,https://github.com/rust-lang/regex,MIT OR Apache-2.0,"The Rust Project Developers, Andrew Gallant " regex-filtered,https://github.com/ua-parser/uap-rust,BSD-3-Clause,The regex-filtered Authors -regex-lite,https://github.com/rust-lang/regex,MIT OR Apache-2.0,"The Rust Project Developers, Andrew Gallant " regex-syntax,https://github.com/rust-lang/regex/tree/master/regex-syntax,MIT OR Apache-2.0,"The Rust Project Developers, Andrew Gallant " relative-path,https://github.com/udoprog/relative-path,MIT OR Apache-2.0,John-John Tedro rend,https://github.com/djkoloski/rend,MIT,David Koloski reqwest,https://github.com/seanmonstar/reqwest,MIT OR Apache-2.0,Sean McArthur reqwest-middleware,https://github.com/TrueLayer/reqwest-middleware,MIT OR Apache-2.0,Rodrigo Gryzinski -reqwest-retry,https://github.com/TrueLayer/reqwest-middleware,MIT OR Apache-2.0,Rodrigo Gryzinski resolv-conf,https://github.com/hickory-dns/resolv-conf,MIT OR Apache-2.0,The resolv-conf Authors retry-policies,https://github.com/TrueLayer/retry-policies,MIT OR Apache-2.0,Luca Palmieri rfc6979,https://github.com/RustCrypto/signatures/tree/master/rfc6979,Apache-2.0 OR MIT,RustCrypto Developers ring,https://github.com/briansmith/ring,Apache-2.0 AND ISC,The ring Authors rkyv,https://github.com/rkyv/rkyv,MIT,David Koloski -rkyv_derive,https://github.com/rkyv/rkyv,MIT,David Koloski rmp,https://github.com/3Hren/msgpack-rust,MIT,"Evgeny Safronov , Kornel " rmp-serde,https://github.com/3Hren/msgpack-rust,MIT,Evgeny Safronov rmpv,https://github.com/3Hren/msgpack-rust,MIT,Evgeny Safronov @@ -679,7 +591,6 @@ rustyline,https://github.com/kkawakam/rustyline,MIT,Katsu Kawakami salsa20,https://github.com/RustCrypto/stream-ciphers,MIT OR Apache-2.0,RustCrypto Developers same-file,https://github.com/BurntSushi/same-file,Unlicense OR MIT,Andrew Gallant -sasl2-sys,https://github.com/MaterializeInc/rust-sasl,Apache-2.0,"Materialize, Inc." schannel,https://github.com/steffengy/schannel-rs,MIT,"Steven Fackler , Steffen Butzer " schemars,https://github.com/GREsau/schemars,MIT,Graham Esau scoped-tls,https://github.com/alexcrichton/scoped-tls,MIT OR Apache-2.0,Alex Crichton @@ -689,7 +600,6 @@ seahash,https://gitlab.redox-os.org/redox-os/seahash,MIT,"ticki security-framework,https://github.com/kornelski/rust-security-framework,MIT OR Apache-2.0,"Steven Fackler , Kornel " -security-framework-sys,https://github.com/kornelski/rust-security-framework,MIT OR Apache-2.0,"Steven Fackler , Kornel " semver,https://github.com/dtolnay/semver,MIT OR Apache-2.0,David Tolnay seq-macro,https://github.com/dtolnay/seq-macro,MIT OR Apache-2.0,David Tolnay serde,https://github.com/serde-rs/serde,MIT OR Apache-2.0,"Erick Tryzelaar , David Tolnay " @@ -697,9 +607,6 @@ serde-aux,https://github.com/iddm/serde-aux,MIT,Victor Polevoy serde-value,https://github.com/arcnmx/serde-value,MIT,arcnmx serde_bytes,https://github.com/serde-rs/bytes,MIT OR Apache-2.0,David Tolnay -serde_core,https://github.com/serde-rs/serde,MIT OR Apache-2.0,"Erick Tryzelaar , David Tolnay " -serde_derive,https://github.com/serde-rs/serde,MIT OR Apache-2.0,"Erick Tryzelaar , David Tolnay " -serde_derive_internals,https://github.com/serde-rs/serde,MIT OR Apache-2.0,"Erick Tryzelaar , David Tolnay " serde_json,https://github.com/serde-rs/json,MIT OR Apache-2.0,"Erick Tryzelaar , David Tolnay " serde_nanos,https://github.com/caspervonb/serde_nanos,MIT OR Apache-2.0,Casper Beyer serde_path_to_error,https://github.com/dtolnay/path-to-error,MIT OR Apache-2.0,David Tolnay @@ -731,7 +638,6 @@ smallvec,https://github.com/servo/rust-smallvec,MIT OR Apache-2.0,The Servo Proj smol,https://github.com/smol-rs/smol,Apache-2.0 OR MIT,Stjepan Glavina smpl_jwt,https://github.com/durch/rust-jwt,MIT,Drazen Urch snafu,https://github.com/shepmaster/snafu,MIT OR Apache-2.0,Jake Goulding -snafu-derive,https://github.com/shepmaster/snafu,MIT OR Apache-2.0,Jake Goulding snap,https://github.com/BurntSushi/rust-snappy,BSD-3-Clause,Andrew Gallant socket2,https://github.com/rust-lang/socket2,MIT OR Apache-2.0,"Alex Crichton , Thomas de Zeeuw " spin,https://github.com/mvdnes/spin-rs,MIT,"Mathijs van de Nes , John Ericson " @@ -740,12 +646,6 @@ spinning_top,https://github.com/rust-osdev/spinning_top,MIT OR Apache-2.0,Philip spki,https://github.com/RustCrypto/formats/tree/master/spki,Apache-2.0 OR MIT,RustCrypto Developers sponge-cursor,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers sqlx,https://github.com/launchbadge/sqlx,MIT OR Apache-2.0,"Ryan Leckey , Austin Bonander , Chloe Ross , Daniel Akhterov " -sqlx-core,https://github.com/launchbadge/sqlx,MIT OR Apache-2.0,"Ryan Leckey , Austin Bonander , Chloe Ross , Daniel Akhterov " -sqlx-macros,https://github.com/launchbadge/sqlx,MIT OR Apache-2.0,"Ryan Leckey , Austin Bonander , Chloe Ross , Daniel Akhterov " -sqlx-macros-core,https://github.com/launchbadge/sqlx,MIT OR Apache-2.0,"Ryan Leckey , Austin Bonander , Chloe Ross , Daniel Akhterov " -sqlx-mysql,https://github.com/launchbadge/sqlx,MIT OR Apache-2.0,"Ryan Leckey , Austin Bonander , Chloe Ross , Daniel Akhterov " -sqlx-postgres,https://github.com/launchbadge/sqlx,MIT OR Apache-2.0,"Ryan Leckey , Austin Bonander , Chloe Ross , Daniel Akhterov " -sqlx-sqlite,https://github.com/launchbadge/sqlx,MIT OR Apache-2.0,"Ryan Leckey , Austin Bonander , Chloe Ross , Daniel Akhterov " stable_deref_trait,https://github.com/storyyeller/stable_deref_trait,MIT OR Apache-2.0,Robert Grosse static_assertions,https://github.com/nvzqz/static-assertions-rs,MIT OR Apache-2.0,Nikolai Vazquez stream-cancel,https://github.com/jonhoo/stream-cancel,MIT OR Apache-2.0,Jon Gjengset @@ -753,7 +653,6 @@ stringprep,https://github.com/sfackler/rust-stringprep,MIT OR Apache-2.0,Steven strip-ansi-escapes,https://github.com/luser/strip-ansi-escapes,Apache-2.0 OR MIT,Ted Mielczarek strsim,https://github.com/rapidfuzz/strsim-rs,MIT,"Danny Guo , maxbachmann " strum,https://github.com/Peternator7/strum,MIT,Peter Glotfelty -strum_macros,https://github.com/Peternator7/strum,MIT,Peter Glotfelty subtle,https://github.com/dalek-cryptography/subtle,BSD-3-Clause,"Isis Lovecruft , Henry de Valence " supports-color,https://github.com/zkat/supports-color,Apache-2.0,Kat Marchán syn,https://github.com/dtolnay/syn,MIT OR Apache-2.0,David Tolnay @@ -763,7 +662,6 @@ sysinfo,https://github.com/GuillaumeGomez/sysinfo,MIT,Guillaume Gomez system-configuration,https://github.com/mullvad/system-configuration-rs,MIT OR Apache-2.0,Mullvad VPN -system-configuration-sys,https://github.com/mullvad/system-configuration-rs,MIT OR Apache-2.0,Mullvad VPN tagptr,https://github.com/oliver-giersch/tagptr,MIT OR Apache-2.0,Oliver Giersch take_mut,https://github.com/Sgeo/take_mut,MIT,Sgeo tap,https://github.com/myrrlyn/tap,MIT,"Elliott Linder , myrrlyn " @@ -773,29 +671,23 @@ term,https://github.com/Stebalien/term,MIT OR Apache-2.0,"The Rust Project Devel termcolor,https://github.com/BurntSushi/termcolor,Unlicense OR MIT,Andrew Gallant terminal_size,https://github.com/eminence/terminal-size,MIT OR Apache-2.0,Andrew Chin thiserror,https://github.com/dtolnay/thiserror,MIT OR Apache-2.0,David Tolnay -thiserror-impl,https://github.com/dtolnay/thiserror,MIT OR Apache-2.0,David Tolnay thread_local,https://github.com/Amanieu/thread_local-rs,MIT OR Apache-2.0,Amanieu d'Antras thrift,https://github.com/apache/thrift/tree/master/lib/rs,Apache-2.0,Apache Thrift Developers tikv-jemalloc-sys,https://github.com/tikv/jemallocator,MIT OR Apache-2.0,"Alex Crichton , Gonzalo Brito Gadeschi , The TiKV Project Developers" tikv-jemallocator,https://github.com/tikv/jemallocator,MIT OR Apache-2.0,"Alex Crichton , Gonzalo Brito Gadeschi , Simon Sapin , Steven Fackler , The TiKV Project Developers" time,https://github.com/time-rs/time,MIT OR Apache-2.0,"Jacob Pratt , Time contributors" -time-core,https://github.com/time-rs/time,MIT OR Apache-2.0,"Jacob Pratt , Time contributors" -time-macros,https://github.com/time-rs/time,MIT OR Apache-2.0,"Jacob Pratt , Time contributors" tiny-keccak,https://github.com/debris/tiny-keccak,CC0-1.0,debris tinystr,https://github.com/unicode-org/icu4x,Unicode-3.0,The ICU4X Project Developers tinyvec,https://github.com/Lokathor/tinyvec,Zlib OR Apache-2.0 OR MIT,Lokathor tinyvec_macros,https://github.com/Soveu/tinyvec_macros,MIT OR Apache-2.0 OR Zlib,Soveu tokio,https://github.com/tokio-rs/tokio,MIT,Tokio Contributors tokio-io-timeout,https://github.com/sfackler/tokio-io-timeout,MIT OR Apache-2.0,Steven Fackler -tokio-macros,https://github.com/tokio-rs/tokio,MIT,Tokio Contributors tokio-native-tls,https://github.com/tokio-rs/tls,MIT,Tokio Contributors tokio-openssl,https://github.com/tokio-rs/tokio-openssl,MIT OR Apache-2.0,Alex Crichton tokio-postgres,https://github.com/rust-postgres/rust-postgres,MIT OR Apache-2.0,Steven Fackler tokio-retry,https://github.com/srijs/rust-tokio-retry,MIT,Sam Rijs tokio-rustls,https://github.com/rustls/tokio-rustls,MIT OR Apache-2.0,The tokio-rustls Authors -tokio-stream,https://github.com/tokio-rs/tokio,MIT,Tokio Contributors tokio-tungstenite,https://github.com/snapview/tokio-tungstenite,MIT,"Daniel Abramov , Alexey Galakhov " -tokio-util,https://github.com/tokio-rs/tokio,MIT,Tokio Contributors tokio-websockets,https://github.com/Gelbpunkt/tokio-websockets,MIT,The tokio-websockets Authors toml,https://github.com/toml-rs/toml,MIT OR Apache-2.0,The toml Authors toml_datetime,https://github.com/toml-rs/toml,MIT OR Apache-2.0,The toml_datetime Authors @@ -806,16 +698,12 @@ toml_write,https://github.com/toml-rs/toml,MIT OR Apache-2.0,The toml_write Auth toml_writer,https://github.com/toml-rs/toml,MIT OR Apache-2.0,The toml_writer Authors tonic,https://github.com/hyperium/tonic,MIT,Lucio Franco tonic-health,https://github.com/hyperium/tonic,MIT,James Nugent -tonic-prost,https://github.com/hyperium/tonic,MIT,Lucio Franco tonic-reflection,https://github.com/hyperium/tonic,MIT,"James Nugent , Samani G. Gikandi " tower,https://github.com/tower-rs/tower,MIT,Tower Maintainers tower-http,https://github.com/tower-rs/tower-http,MIT,Tower Maintainers -tower-layer,https://github.com/tower-rs/tower,MIT,Tower Maintainers -tower-service,https://github.com/tower-rs/tower,MIT,Tower Maintainers tracing,https://github.com/tokio-rs/tracing,MIT,"Eliza Weisman , Tokio Contributors " tracing-attributes,https://github.com/tokio-rs/tracing,MIT,"Tokio Contributors , Eliza Weisman , David Barsky " tracing-core,https://github.com/tokio-rs/tracing,MIT,Tokio Contributors -tracing-futures,https://github.com/tokio-rs/tracing,MIT,"Eliza Weisman , Tokio Contributors " tracing-log,https://github.com/tokio-rs/tracing,MIT,Tokio Contributors tracing-serde,https://github.com/tokio-rs/tracing,MIT,Tokio Contributors tracing-subscriber,https://github.com/tokio-rs/tracing,MIT,"Eliza Weisman , David Barsky , Tokio Contributors " @@ -826,13 +714,11 @@ tryhard,https://github.com/EmbarkStudios/tryhard,MIT OR Apache-2.0,Embark typed-builder,https://github.com/idanarye/rust-typed-builder,MIT OR Apache-2.0,"IdanArye , Chris Morgan " -typed-builder-macro,https://github.com/idanarye/rust-typed-builder,MIT OR Apache-2.0,"IdanArye , Chris Morgan " typenum,https://github.com/paholg/typenum,MIT OR Apache-2.0,The typenum Authors typespec,https://github.com/azure/azure-sdk-for-rust,MIT,Microsoft typespec_client_core,https://github.com/azure/azure-sdk-for-rust,MIT,Microsoft typespec_macros,https://github.com/azure/azure-sdk-for-rust,MIT,Microsoft typetag,https://github.com/dtolnay/typetag,MIT OR Apache-2.0,David Tolnay -typetag-impl,https://github.com/dtolnay/typetag,MIT OR Apache-2.0,David Tolnay ua-parser,https://github.com/ua-parser/uap-rust,Apache-2.0,The ua-parser Authors ucd-trie,https://github.com/BurntSushi/ucd-generate,MIT OR Apache-2.0,Andrew Gallant unarray,https://github.com/cameron1024/unarray,MIT OR Apache-2.0,The unarray Authors @@ -856,7 +742,6 @@ utf-8,https://github.com/SimonSapin/rust-utf8,MIT OR Apache-2.0,Simon Sapin utf8-width,https://github.com/magiclen/utf8-width,MIT,Magic Len utf8_iter,https://github.com/hsivonen/utf8_iter,Apache-2.0 OR MIT,Henri Sivonen -utf8parse,https://github.com/alacritty/vte,Apache-2.0 OR MIT,"Joe Wilm , Christian Duerr " uuid,https://github.com/uuid-rs/uuid,Apache-2.0 OR MIT,"Ashley Mannix, Dylan DPC, Hunar Roop Kahlon" uuid-simd,https://github.com/Nugine/simd,MIT,The uuid-simd Authors valuable,https://github.com/tokio-rs/valuable,MIT,The valuable Authors @@ -892,41 +777,19 @@ whoami,https://github.com/ardaku/whoami,Apache-2.0 OR BSL-1.0 OR MIT,The whoami widestring,https://github.com/starkat99/widestring-rs,MIT OR Apache-2.0,Kathryn Long widestring,https://github.com/starkat99/widestring-rs,MIT OR Apache-2.0,The widestring Authors winapi,https://github.com/retep998/winapi-rs,MIT OR Apache-2.0,Peter Atashian -winapi-i686-pc-windows-gnu,https://github.com/retep998/winapi-rs,MIT OR Apache-2.0,Peter Atashian winapi-util,https://github.com/BurntSushi/winapi-util,Unlicense OR MIT,Andrew Gallant -winapi-x86_64-pc-windows-gnu,https://github.com/retep998/winapi-rs,MIT OR Apache-2.0,Peter Atashian windows,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft windows-collections,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,The windows-collections Authors -windows-core,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft windows-future,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,The windows-future Authors -windows-implement,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft windows-implement,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,The windows-implement Authors -windows-interface,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft windows-interface,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,The windows-interface Authors -windows-link,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft windows-link,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,The windows-link Authors windows-numerics,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,The windows-numerics Authors -windows-result,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft windows-service,https://github.com/mullvad/windows-service-rs,MIT OR Apache-2.0,Mullvad VPN -windows-strings,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft -windows-sys,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft windows-sys,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,The windows-sys Authors -windows-targets,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft -windows-threading,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft -windows_aarch64_gnullvm,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft -windows_aarch64_msvc,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft -windows_i686_gnu,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft -windows_i686_gnullvm,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft -windows_i686_msvc,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft -windows_x86_64_gnu,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft -windows_x86_64_gnullvm,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft -windows_x86_64_msvc,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft winnow,https://github.com/winnow-rs/winnow,MIT,The winnow Authors winreg,https://github.com/gentoo90/winreg-rs,MIT,Igor Shaula wit-bindgen,https://github.com/bytecodealliance/wit-bindgen,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,Alex Crichton -wit-bindgen-core,https://github.com/bytecodealliance/wit-bindgen,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,Alex Crichton -wit-bindgen-rust,https://github.com/bytecodealliance/wit-bindgen,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,Alex Crichton -wit-bindgen-rust-macro,https://github.com/bytecodealliance/wit-bindgen,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,Alex Crichton wit-component,https://github.com/bytecodealliance/wasm-tools/tree/main/crates/wit-component,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,Peter Huene wit-parser,https://github.com/bytecodealliance/wasm-tools/tree/main/crates/wit-parser,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,Alex Crichton woothee,https://github.com/woothee/woothee-rust,Apache-2.0,hhatto @@ -938,7 +801,6 @@ xxhash-rust,https://github.com/DoumanAsh/xxhash-rust,BSL-1.0,Douman yoke-derive,https://github.com/unicode-org/icu4x,Unicode-3.0,Manish Goregaokar zerocopy,https://github.com/google/zerocopy,BSD-2-Clause OR Apache-2.0 OR MIT,Joshua Liebow-Feeser -zerocopy-derive,https://github.com/google/zerocopy,BSD-2-Clause OR Apache-2.0 OR MIT,Joshua Liebow-Feeser zerofrom,https://github.com/unicode-org/icu4x,Unicode-3.0,Manish Goregaokar zerofrom-derive,https://github.com/unicode-org/icu4x,Unicode-3.0,Manish Goregaokar zeroize,https://github.com/RustCrypto/utils/tree/master/zeroize,Apache-2.0 OR MIT,The RustCrypto Project Developers From 43c66dea0e8ca3bbaf4463b2575a74270b1fe77e Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Thu, 13 Aug 2026 09:12:45 +0100 Subject: [PATCH 10/18] Drop shadow finalizer --- src/sinks/datadog/metrics/request_builder.rs | 61 +++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/src/sinks/datadog/metrics/request_builder.rs b/src/sinks/datadog/metrics/request_builder.rs index bee030f379aa1..57691c55f3394 100644 --- a/src/sinks/datadog/metrics/request_builder.rs +++ b/src/sinks/datadog/metrics/request_builder.rs @@ -282,7 +282,19 @@ impl IncrementalRequestBuilder<((Option>, DatadogMetricsEndpoint), Vec< is_shadow_flush.then(|| Arc::from(Uuid::now_v7().to_string().as_str())); // Clone metrics before primary encoding consumes them, if we need a shadow copy. - let shadow_metrics = is_shadow_flush.then(|| metrics.clone()); + // The shadow copy must not carry the production `EventFinalizers`: cloning a `Metric` + // clones its `Arc` pointers, so without stripping them here, the + // validation-only shadow batch would share finalizers with the primary batch. A + // shadow-only failure could then reject (or indefinitely delay) acknowledgement for + // events whose primary V1/V2 request already succeeded. Dropping the taken finalizers + // detaches the shadow copy from acknowledgement entirely. + let shadow_metrics = is_shadow_flush.then(|| { + let mut shadow_m = metrics.clone(); + for metric in &mut shadow_m { + drop(metric.take_finalizers()); + } + shadow_m + }); // ── Primary encode ──────────────────────────────────────────────────── // V3Beta uses the same columnar encoder path as V3; only the @@ -761,6 +773,53 @@ mod tests { ); } + /// The shadow copy of a series flush must not carry the production `EventFinalizers`. + /// If it did, a shadow-only failure (e.g. the validation-only V3beta endpoint rejecting + /// or erroring on a request whose V1/V2 twin was delivered fine) would update the same + /// shared `EventFinalizer`s as the primary and could reject or indefinitely delay + /// acknowledgement for events that were already successfully delivered. + #[test] + fn shadow_copy_does_not_carry_production_finalizers() { + use vector_lib::event::{BatchNotifier, BatchStatus}; + + let (batch, mut receiver) = BatchNotifier::new_with_receiver(); + let finalized_metric = counter_metric().with_batch_notifier(&batch); + drop(batch); + + let mut builder = builder_with_shadow_every(1); + let mut encoded = builder.encode_events_incremental(( + (None, DatadogMetricsEndpoint::Series(SeriesApiVersion::V2)), + vec![finalized_metric], + )); + + assert_eq!(encoded.len(), 2, "expected a V2 primary and a V3 shadow"); + + let metas: Vec = encoded + .drain(..) + .filter_map(Result::ok) + .map(|((meta, _), _)| meta) + .collect(); + + let with_finalizers = metas.iter().filter(|m| !m.finalizers.is_empty()).count(); + let without_finalizers = metas.iter().filter(|m| m.finalizers.is_empty()).count(); + assert_eq!( + with_finalizers, 1, + "exactly the primary request should carry the production finalizers" + ); + assert_eq!( + without_finalizers, 1, + "the shadow request must not carry any finalizers" + ); + + // Dropping every `DDMetricsMetadata` (and therefore every retained `EventFinalizers`) + // must resolve the batch exactly once, with its untouched default status of + // `Delivered` — proving the shadow copy held no live reference into the same + // finalizer (an extra live reference would keep the batch notifier alive and this + // `try_recv` would still return `Empty`). + drop(metas); + assert_eq!(receiver.try_recv(), Ok(BatchStatus::Delivered)); + } + // ── Timestamp resolution ─────────────────────────────────────────────── /// `statsd` and friends emit metrics with no timestamp, and both encoders fall back to From 6bb81a24d2c08cd5d81b13dc1d6f69f7c1afefc6 Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Thu, 13 Aug 2026 09:50:58 +0100 Subject: [PATCH 11/18] Dont split v3 payloads into empty chunks --- src/sinks/datadog/metrics/request_builder.rs | 122 ++++++++++++++++++- 1 file changed, 119 insertions(+), 3 deletions(-) diff --git a/src/sinks/datadog/metrics/request_builder.rs b/src/sinks/datadog/metrics/request_builder.rs index 57691c55f3394..45e849a52dd77 100644 --- a/src/sinks/datadog/metrics/request_builder.rs +++ b/src/sinks/datadog/metrics/request_builder.rs @@ -493,7 +493,7 @@ fn encode_batch( } Err(FinishError::TooLarge { mut metrics, - mut recommended_splits, + recommended_splits, }) => { // The encoder informed us that the resulting payload was too big, so we're // being given a chance here to split it into smaller input batches in the @@ -513,14 +513,31 @@ fn encode_batch( // Protocol Buffers data, similar to how the Datadog Agent does it with // `molecule`, we can wrap all of the sketch encoding into the same // incremental encoding paradigm and avoid this. + // + // `recommended_splits` is derived from a *byte-size* ratio and is unbounded + // by the number of metrics actually in this batch: a single metric whose + // encoded size alone exceeds the limit (e.g. one very high-cardinality + // sketch) can report a `recommended_splits` far larger than `metrics.len()`. + // Without capping it, `stride = metrics.len() / recommended_splits` + // truncates to `0`, so every iteration of the loop below calls + // `metrics.split_off(split_idx)` with an unchanged `split_idx` — producing + // `recommended_splits - 1` *empty* chunks that each "succeed" as a + // zero-metric request, while the real oversized chunk is pushed unchanged + // at the end and fails again. Capping to `metrics.len()` guarantees each + // chunk gets at least one metric; when there's only one metric to begin + // with, the cap collapses the loop entirely and that single unsplittable + // metric is sent through `encode_chunk` on its own, where it fails cleanly + // as `FailedToSplit` instead of spawning empty requests first. + let recommended_splits = recommended_splits.min(metrics.len()); let mut split_idx = metrics.len(); let stride = split_idx / recommended_splits; - while recommended_splits > 1 { + let mut remaining_splits = recommended_splits; + while remaining_splits > 1 { split_idx -= stride; let chunk = metrics.split_off(split_idx); results.push(encode_chunk(encoder, api_key.clone(), endpoint, chunk)); - recommended_splits -= 1; + remaining_splits -= 1; } results.push(encode_chunk(encoder, api_key.clone(), endpoint, metrics)); } @@ -820,6 +837,105 @@ mod tests { assert_eq!(receiver.try_recv(), Ok(BatchStatus::Delivered)); } + // ── TooLarge split handling ──────────────────────────────────────── + + /// Test double for [`MetricsEncoder`] whose `finish()` reports `TooLarge` with an + /// arbitrary, caller-chosen `recommended_splits` for any non-empty batch — regardless of + /// how many metrics are actually pending. This lets us exercise `encode_batch`'s split + /// arithmetic in isolation, including the case a real encoder hits when a *single* metric + /// (e.g. one huge sketch) alone exceeds the size limit: `recommended_splits` is derived + /// from a byte-size ratio and can be larger than the metric count. + struct AlwaysTooLargeEncoder { + pending: Vec, + recommended_splits: usize, + } + + impl MetricsEncoder for AlwaysTooLargeEncoder { + fn try_encode(&mut self, metric: Metric) -> Result, EncoderError> { + self.pending.push(metric); + Ok(None) + } + + fn finish(&mut self) -> Result<(EncodeResult, Vec), FinishError> { + let metrics = std::mem::take(&mut self.pending); + if metrics.is_empty() { + // Matches every real encoder's behavior: finishing an empty batch always + // succeeds trivially, producing an empty payload. + return Ok(( + EncodeResult::compressed(Bytes::new(), 0, GroupedCountByteSize::new_untagged()), + Vec::new(), + )); + } + Err(FinishError::TooLarge { + metrics, + recommended_splits: self.recommended_splits, + }) + } + } + + /// A single metric that's too large on its own can report a `recommended_splits` far + /// larger than the metric count (it's derived from a byte-size ratio, not from counting + /// metrics). Splitting must never emit more chunks than there are metrics to put in them: + /// this metric must come out as exactly one failed result, not four phantom "successful" + /// empty requests followed by one failure. + #[test] + fn too_large_split_never_exceeds_the_metric_count() { + let mut encoder = AlwaysTooLargeEncoder { + pending: Vec::new(), + recommended_splits: 5, + }; + + let results = encode_batch( + &mut encoder, + None, + DatadogMetricsEndpoint::Series(SeriesApiVersion::V3), + vec![counter_metric()], + ); + + assert_eq!( + results.len(), + 1, + "an unsplittable single metric must yield exactly one result, not {} \ + (a fixed `recommended_splits` split count would otherwise emit \ + `recommended_splits - 1` empty successes before the real failure)", + results.len() + ); + assert!( + results[0].is_err(), + "the single oversized metric must be reported as failed, not silently dropped \ + behind a successful empty payload" + ); + } + + /// With more metrics than the recommended split count, splitting proceeds exactly as + /// before: each of the `recommended_splits` chunks gets a non-empty share of the metrics. + #[test] + fn too_large_split_with_enough_metrics_produces_no_empty_chunks() { + let mut encoder = AlwaysTooLargeEncoder { + pending: Vec::new(), + recommended_splits: 3, + }; + + let metrics: Vec = (0..3).map(|_| counter_metric()).collect(); + let results = encode_batch( + &mut encoder, + None, + DatadogMetricsEndpoint::Series(SeriesApiVersion::V3), + metrics, + ); + + assert_eq!( + results.len(), + 3, + "3 metrics split 3 ways must produce exactly 3 results, one per metric" + ); + assert!( + results.iter().all(Result::is_err), + "this encoder always reports TooLarge for non-empty input, so every \ + single-metric chunk must fail as unsplittable, not succeed" + ); + } + // ── Timestamp resolution ─────────────────────────────────────────────── /// `statsd` and friends emit metrics with no timestamp, and both encoders fall back to From af58ad6d9675b7d87145d4c5e1cb2561111a464d Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Thu, 13 Aug 2026 10:08:00 +0100 Subject: [PATCH 12/18] Only enable shadow for non custom endpoints --- ..._metrics_v3_dual_write_default.breaking.md | 20 +++- src/sinks/datadog/metrics/config.rs | 108 +++++++++++++++--- .../sinks/generated/datadog_metrics.cue | 21 +++- 3 files changed, 124 insertions(+), 25 deletions(-) diff --git a/changelog.d/datadog_metrics_v3_dual_write_default.breaking.md b/changelog.d/datadog_metrics_v3_dual_write_default.breaking.md index f81780e3ebb59..5b9123f6ce38e 100644 --- a/changelog.d/datadog_metrics_v3_dual_write_default.breaking.md +++ b/changelog.d/datadog_metrics_v3_dual_write_default.breaking.md @@ -1,14 +1,22 @@ -# `datadog_metrics` sink now dual-writes a V3 shadow payload by default +# `datadog_metrics` sink now dual-writes a V3 shadow payload by default when submitting directly to Datadog The `datadog_metrics` sink's `dual_write` V3 shadow option is now enabled by default (with -`shadow_every: 1000`, sampling 1 in every 1000 legacy series flushes). This means Vector now sends -an additional, sampled V3-encoded payload to Datadog's shadow intake endpoint alongside the normal -legacy payload, without any configuration required. +`shadow_every: 1000`, sampling 1 in every 1000 legacy series flushes), but only when submitting +directly to Datadog (no custom `endpoint` configured). This means Vector now sends an additional, +sampled V3-encoded payload to Datadog's shadow intake endpoint alongside the normal legacy +payload, without any configuration required. + +If a custom `endpoint` is configured (for example, a Datadog Agent, relay, or test collector), +dual-write defaults to **disabled** instead. The shadow route +(`/api/intake/metrics/v3beta/series`) is only guaranteed to exist on Datadog's own intake; hitting +it on a custom endpoint that doesn't implement it returns a `404`, which is treated as retriable, +so every sampled flush would otherwise add a request that retries forever. Only series metrics are dual-written. Sketches (distributions and histograms) are never shadowed, because the V3 sketches intake endpoints do not exist. -If you don't want this additional traffic, set `dual_write.enabled: false` on your `datadog_metrics` -sink configuration. +`dual_write.enabled` can be set explicitly to override either default in either direction: `true` +to opt in to shadow traffic against a custom endpoint, or `false` to disable it even when +submitting directly to Datadog. authors: stephenwakely diff --git a/src/sinks/datadog/metrics/config.rs b/src/sinks/datadog/metrics/config.rs index 079748ed52e98..1f9bcb36cd189 100644 --- a/src/sinks/datadog/metrics/config.rs +++ b/src/sinks/datadog/metrics/config.rs @@ -240,10 +240,6 @@ const fn default_shadow_every() -> NonZeroU64 { NonZeroU64::new(1).unwrap() } -const fn default_dual_write_enabled() -> bool { - true -} - /// Configuration for the V3 shadow dual-write mode. /// /// When enabled, every `shadow_every`-th legacy (V1/V2) *series* flush also sends a V3 @@ -259,12 +255,18 @@ const fn default_dual_write_enabled() -> bool { pub struct DualWriteConfig { /// Whether to enable V3 shadow dual-write. /// - /// Enabled by default, sampling a fraction of legacy series flushes to validate the V3 - /// intake path. Set to `false` to disable V3 shadow dual-write entirely. + /// When unset, this defaults to `true` when submitting directly to Datadog (no custom + /// `endpoint` set), and to `false` when a custom `endpoint` is configured (for example, a + /// Datadog Agent, relay, or test collector). The shadow route + /// (`/api/intake/metrics/v3beta/series`) is only guaranteed to exist on Datadog's own + /// intake; hitting it on a custom endpoint that doesn't implement it 404s, which is + /// treated as a retriable error, so every sampled flush would add a request that retries + /// forever. Set this explicitly to `true` to opt in to shadow traffic against a custom + /// endpoint anyway, or to `false` to disable it even when submitting directly to Datadog. /// /// This only ever affects series. Sketches are never dual-written. - #[serde(default = "default_dual_write_enabled")] - pub enabled: bool, + #[serde(default)] + pub enabled: Option, /// Send a V3 shadow payload once per this many legacy (V1/V2) series flushes. /// @@ -277,7 +279,7 @@ pub struct DualWriteConfig { impl Default for DualWriteConfig { fn default() -> Self { Self { - enabled: default_dual_write_enabled(), + enabled: None, shadow_every: default_shadow_every(), } } @@ -287,6 +289,16 @@ impl DualWriteConfig { pub(super) const fn get_series_path(&self) -> &'static str { SERIES_V3_BETA_PATH } + + /// Resolves whether shadow dual-write should actually run for this sink instance. + /// + /// An explicit `enabled` setting always wins. When unset, dual-write defaults to `true` + /// only when submitting directly to Datadog; a custom endpoint (Agent, relay, test + /// collector, ...) defaults to `false`, since the beta shadow route isn't guaranteed to + /// exist there and a 404 would otherwise retry forever. See `enabled`'s docs for details. + pub(super) fn is_enabled(&self, has_custom_endpoint: bool) -> bool { + self.enabled.unwrap_or(!has_custom_endpoint) + } } /// Configuration for the `datadog_metrics` sink. @@ -331,9 +343,12 @@ pub struct DatadogMetricsConfig { /// V3 shadow dual-write configuration. /// - /// Enabled by default: a sampled fraction of legacy series flushes is mirrored as V3 - /// payloads to a separate intake endpoint, both stamped with a shared - /// `X-Metrics-Request-ID`. Set `dual_write.enabled` to `false` to disable it. + /// By default, a sampled fraction of legacy series flushes is mirrored as V3 payloads to + /// a separate intake endpoint, both stamped with a shared `X-Metrics-Request-ID` — but + /// only when submitting directly to Datadog. If `endpoint` is set to a custom Agent, + /// relay, or test collector, dual-write defaults to disabled instead, since the shadow + /// route isn't guaranteed to exist there. Set `dual_write.enabled` explicitly to override + /// either default in either direction. /// /// Sketches are never dual-written, regardless of this setting. #[configurable(derived)] @@ -439,7 +454,7 @@ impl DatadogMetricsConfig { let shadow_config = self .dual_write - .enabled + .is_enabled(dd_common.endpoint.is_some()) .then_some(&self.dual_write) .map(|dw| -> crate::Result { let base_uri = self.get_base_agent_endpoint(dd_common); @@ -554,6 +569,73 @@ mod tests { assert_eq!(sketches.size_limit, 1_000_000); } + // The internal V3 beta shadow route isn't guaranteed to exist on a custom endpoint + // (Agent, relay, test collector, ...), so leaving `dual_write.enabled` unset must not + // silently turn on shadow traffic there: a 404 is retriable, so every sampled flush would + // add a permanently retrying request against a target that never implements the route. + // Submitting directly to Datadog (no custom `endpoint`) is unaffected -- shadow dual-write + // stays enabled by default there, since Datadog's own intake does implement the route. + #[test] + fn dual_write_defaults_to_disabled_only_for_custom_endpoints() { + let unset = DualWriteConfig::default(); + assert!( + unset.is_enabled(false), + "unset `enabled` must default to on when submitting directly to Datadog" + ); + assert!( + !unset.is_enabled(true), + "unset `enabled` must default to off for a custom endpoint" + ); + } + + // An explicit `enabled` setting always overrides the endpoint-based default, in either + // direction: opting in to shadow traffic against a custom endpoint, or opting out of it + // even when submitting directly to Datadog. + #[test] + fn dual_write_explicit_enabled_overrides_the_endpoint_based_default() { + let explicit_on = DualWriteConfig { + enabled: Some(true), + ..DualWriteConfig::default() + }; + assert!( + explicit_on.is_enabled(true), + "explicit `enabled = true` must opt in even for a custom endpoint" + ); + + let explicit_off = DualWriteConfig { + enabled: Some(false), + ..DualWriteConfig::default() + }; + assert!( + !explicit_off.is_enabled(false), + "explicit `enabled = false` must opt out even when submitting directly to Datadog" + ); + } + + // `dual_write.enabled` must still parse as a plain TOML boolean -- the `Option` + // representation is an internal resolution detail, not a schema change users need to know + // about. + #[test] + fn dual_write_enabled_parses_as_a_plain_bool() { + let config = toml::from_str::( + r#" + default_api_key = "unused" + [dual_write] + enabled = true + "#, + ) + .expect("`dual_write.enabled = true` must parse"); + assert_eq!(config.dual_write.enabled, Some(true)); + + let config = toml::from_str::( + r#" + default_api_key = "unused" + "#, + ) + .expect("omitting `dual_write` entirely must still parse"); + assert_eq!(config.dual_write.enabled, None); + } + // `sketches_api_version` is independent of `series_api_version`: Datadog's intake gates V3 // series and V3 sketches separately, so each must resolve to its own path regardless of what // the other is set to. diff --git a/website/cue/reference/components/sinks/generated/datadog_metrics.cue b/website/cue/reference/components/sinks/generated/datadog_metrics.cue index 9e5419ae82dfc..7c4bdee38469b 100644 --- a/website/cue/reference/components/sinks/generated/datadog_metrics.cue +++ b/website/cue/reference/components/sinks/generated/datadog_metrics.cue @@ -89,9 +89,12 @@ generated: components: sinks: datadog_metrics: configuration: { description: """ V3 shadow dual-write configuration. - Enabled by default: a sampled fraction of legacy series flushes is mirrored as V3 - payloads to a separate intake endpoint, both stamped with a shared - `X-Metrics-Request-ID`. Set `dual_write.enabled` to `false` to disable it. + By default, a sampled fraction of legacy series flushes is mirrored as V3 payloads to + a separate intake endpoint, both stamped with a shared `X-Metrics-Request-ID` — but + only when submitting directly to Datadog. If `endpoint` is set to a custom Agent, + relay, or test collector, dual-write defaults to disabled instead, since the shadow + route isn't guaranteed to exist there. Set `dual_write.enabled` explicitly to override + either default in either direction. Sketches are never dual-written, regardless of this setting. """ @@ -101,13 +104,19 @@ generated: components: sinks: datadog_metrics: configuration: { description: """ Whether to enable V3 shadow dual-write. - Enabled by default, sampling a fraction of legacy series flushes to validate the V3 - intake path. Set to `false` to disable V3 shadow dual-write entirely. + When unset, this defaults to `true` when submitting directly to Datadog (no custom + `endpoint` set), and to `false` when a custom `endpoint` is configured (for example, a + Datadog Agent, relay, or test collector). The shadow route + (`/api/intake/metrics/v3beta/series`) is only guaranteed to exist on Datadog's own + intake; hitting it on a custom endpoint that doesn't implement it 404s, which is + treated as a retriable error, so every sampled flush would add a request that retries + forever. Set this explicitly to `true` to opt in to shadow traffic against a custom + endpoint anyway, or to `false` to disable it even when submitting directly to Datadog. This only ever affects series. Sketches are never dual-written. """ required: false - type: bool: default: true + type: bool: {} } shadow_every: { description: """ From 63539a4f0e37be6a357e2a2c8c2bc3d96f1bde93 Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Thu, 13 Aug 2026 10:17:08 +0100 Subject: [PATCH 13/18] Keep the V3 beta shadow endpoint out of primary configuration --- src/sinks/datadog/metrics/config.rs | 78 ++++++++++++++++++- .../sinks/generated/datadog_metrics.cue | 7 +- 2 files changed, 77 insertions(+), 8 deletions(-) diff --git a/src/sinks/datadog/metrics/config.rs b/src/sinks/datadog/metrics/config.rs index 1f9bcb36cd189..475138b257312 100644 --- a/src/sinks/datadog/metrics/config.rs +++ b/src/sinks/datadog/metrics/config.rs @@ -43,6 +43,16 @@ pub(super) const SKETCHES_PATH: &str = "/api/beta/sketches"; pub(super) const SKETCHES_V3_PATH: &str = "/api/intake/metrics/v3/sketches"; /// The API version to use when submitting series metrics to Datadog. +/// +/// `V3Beta` is deliberately kept in this enum (and fully wired through the encoder and +/// request builder) but marked `#[serde(skip)]` below, so it cannot be configured as the +/// *primary* `series_api_version`. It's a validation-only route used exclusively for the V3 +/// shadow dual-write: `batch_id` and the `X-Metrics-Request-*` correlation headers that pair a +/// shadow request with its legacy V1/V2 twin are only generated by the dual-write path, so a +/// primary request sent as `V3Beta` would be unpaired and uncorrelated -- the shadow backend +/// has nothing to validate it against, and it doesn't provide normal metric ingestion either. +/// Use `v3` for stable primary V3 series submission, or enable `dual_write` to get `V3Beta` +/// shadow traffic alongside a legacy primary. #[configurable_component] #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] #[serde(rename_all = "snake_case")] @@ -59,7 +69,7 @@ pub enum SeriesApiVersion { #[default] V2, - /// Use the v3 series endpoint (`/api/intake/metrics/v3beta/series`). + /// Use the v3 series endpoint (`/api/intake/metrics/v3/series`). /// /// Columnar protobuf format with dictionary-based string deduplication and delta /// encoding. More efficient than v2 for workloads with many metrics that share @@ -68,7 +78,12 @@ pub enum SeriesApiVersion { /// Use the v3 beta intake endpoint (`/api/intake/metrics/v3beta/series`). /// - /// Used for shadow/validation rollout of V3. Prefer `v3_intake` for stable usage. + /// This is a validation-only route used internally by V3 shadow dual-write (see + /// `dual_write`) and can't be selected as the primary `series_api_version` -- see this + /// enum's doc comment for why. The variant, `get_path()`, `is_v3_format()`, and the shadow + /// encoder path are all still fully implemented; only the primary config surface is + /// disabled. + #[serde(skip)] V3Beta, } @@ -679,4 +694,63 @@ mod tests { assert_eq!(config.sketches_api_version, SketchesApiVersion::V2); } } + + // `V3Beta` requests aren't paired/correlated with a legacy twin unless they go through the + // dual-write path (which stamps `batch_id`/`X-Metrics-Request-*` itself), and the beta route + // is validation-only -- it doesn't provide normal ingestion. So `series_api_version: v3_beta` + // must be rejected at config-load time rather than accepted as a primary API version. + // `SeriesApiVersion::V3Beta` itself stays fully implemented (see the next test) -- only the + // primary config surface, via `#[serde(skip)]` on the variant, is disabled. + #[test] + fn series_api_version_v3_beta_is_not_configurable() { + let err = toml::from_str::( + r#" + default_api_key = "unused" + series_api_version = "v3_beta" + "#, + ) + .expect_err("series_api_version = \"v3_beta\" must be rejected"); + + assert!( + err.to_string().contains("unknown variant"), + "expected an unknown-variant error, got: {err}" + ); + } + + // `v1`, `v2`, `v3` -- the configurable values -- and the unset default must all still work. + // `v3_beta` is deliberately excluded here; it's covered by the rejection test above. + #[test] + fn series_api_version_v1_v2_v3_and_default_are_configurable() { + for (toml, expected) in [ + (r#"default_api_key = "unused""#, SeriesApiVersion::V2), + ( + r#"default_api_key = "unused" + series_api_version = "v1""#, + SeriesApiVersion::V1, + ), + ( + r#"default_api_key = "unused" + series_api_version = "v2""#, + SeriesApiVersion::V2, + ), + ( + r#"default_api_key = "unused" + series_api_version = "v3""#, + SeriesApiVersion::V3, + ), + ] { + let config = toml::from_str::(toml) + .expect("v1, v2, v3, and the unset default must all parse"); + assert_eq!(config.series_api_version, expected); + } + } + + // `SeriesApiVersion::V3Beta`'s own code must stay correct even though it can't be selected + // as a primary `series_api_version` -- it's still constructed directly (not through serde) + // by the dual-write shadow path in `build_sink`. + #[test] + fn series_api_version_v3_beta_own_code_still_works() { + assert_eq!(SeriesApiVersion::V3Beta.get_path(), SERIES_V3_BETA_PATH); + assert!(SeriesApiVersion::V3Beta.is_v3_format()); + } } diff --git a/website/cue/reference/components/sinks/generated/datadog_metrics.cue b/website/cue/reference/components/sinks/generated/datadog_metrics.cue index 7c4bdee38469b..c5223a6d9e660 100644 --- a/website/cue/reference/components/sinks/generated/datadog_metrics.cue +++ b/website/cue/reference/components/sinks/generated/datadog_metrics.cue @@ -354,17 +354,12 @@ generated: components: sinks: datadog_metrics: configuration: { This is the recommended and default endpoint. """ v3: """ - Use the v3 series endpoint (`/api/intake/metrics/v3beta/series`). + Use the v3 series endpoint (`/api/intake/metrics/v3/series`). Columnar protobuf format with dictionary-based string deduplication and delta encoding. More efficient than v2 for workloads with many metrics that share common tags or names. """ - v3_beta: """ - Use the v3 beta intake endpoint (`/api/intake/metrics/v3beta/series`). - - Used for shadow/validation rollout of V3. Prefer `v3_intake` for stable usage. - """ } } } From 28fbc4c1d66b90bc1a39bb712b583e19c26c2b25 Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Thu, 13 Aug 2026 10:48:40 +0100 Subject: [PATCH 14/18] Resolve device resources field --- src/sinks/datadog/metrics/encoder_v3.rs | 55 ++++++++++++++++++++++--- 1 file changed, 50 insertions(+), 5 deletions(-) diff --git a/src/sinks/datadog/metrics/encoder_v3.rs b/src/sinks/datadog/metrics/encoder_v3.rs index b23ccbb800937..d6066e6cdd9b5 100644 --- a/src/sinks/datadog/metrics/encoder_v3.rs +++ b/src/sinks/datadog/metrics/encoder_v3.rs @@ -13,7 +13,7 @@ use chrono::{DateTime, Utc}; use vector_lib::{ EstimatedJsonEncodedSizeOf, config::{LogSchema, log_schema, telemetry}, - event::{Metric, MetricValue, metric::MetricSketch}, + event::{Metric, MetricTags, MetricValue, metric::MetricSketch}, metrics::AgentDDSketch, request_metadata::GroupedCountByteSize, }; @@ -209,6 +209,8 @@ fn encode_metric_to_v3( let host_key = log_schema.host_key().map(|k| k.to_string()); if let Some(tags) = metric.tags() { + device_resource = resolve_device_resource(tags); + for (key, value) in tags.iter_all() { // dd.internal.resource tags become structured resources if key == "dd.internal.resource" { @@ -230,11 +232,10 @@ fn encode_metric_to_v3( continue; } - // device / resource.device → device resource + // device / resource.device are resolved once, up front, via `resolve_device_resource` + // -- just consume them here so they don't fall through to the generic tag handling + // below. if key == "device" || key == "resource.device" { - if let Some(dev) = value { - device_resource = Some(dev); - } continue; } @@ -334,6 +335,21 @@ fn encode_metric_to_v3( Ok(()) } +/// Resolves a metric's `device` resource, matching V2's explicit +/// `tags.remove("device").or(tags.remove("resource.device"))` precedence in +/// `series_to_proto_message`: `device` always wins over `resource.device` when a metric +/// carries both. In the `datadog_agent` source, the tag is added as `device` for the V1 +/// endpoint and `resource.device` for the V2 endpoint. +/// +/// Resolved once via direct lookup, independent of `MetricTags`' key-iteration order -- +/// `resource.device` sorts after `device` alphabetically, so an order-dependent overwrite +/// inside the tag-iteration loop would silently prefer the wrong one whenever a metric +/// carries both, producing a different device resource than V2 for the same metric and +/// mismatching the paired V2/V3 shadow payloads for the same flush. +fn resolve_device_resource(tags: &MetricTags) -> Option<&str> { + tags.get("device").or_else(|| tags.get("resource.device")) +} + /// Assembles the final resource list in a fixed host-then-device order, matching V2's /// `encode_series_metrics`. Host/device are collected separately during tag iteration /// (whose order is unspecified) so that order never leaks into the wire-visible resources. @@ -461,6 +477,35 @@ mod tests { ); } + // Regression test: V2's `series_to_proto_message` gives `device` explicit precedence over + // `resource.device` via `tags.remove("device").or(tags.remove("resource.device"))`. V3 used + // to resolve this inside its tag-iteration loop, where `MetricTags`' key-ordered iteration + // visits `device` before `resource.device` (alphabetically) and unconditionally overwrote + // whichever was seen last -- silently preferring `resource.device` instead, and mismatching + // the paired V2/V3 shadow payloads for the same flush whenever a metric carried both. + #[test] + fn v3_device_resource_prefers_device_over_resource_device() { + // Only `device` present. + let tags = MetricTags::from([("device".to_string(), "/dev/sda1".to_string())]); + assert_eq!(resolve_device_resource(&tags), Some("/dev/sda1")); + + // Only `resource.device` present. + let tags = MetricTags::from([("resource.device".to_string(), "/dev/sdb2".to_string())]); + assert_eq!(resolve_device_resource(&tags), Some("/dev/sdb2")); + + // Both present: `device` must win, matching V2 -- regardless of which one a naive, + // iteration-order-dependent implementation would happen to visit last. + let tags = MetricTags::from([ + ("device".to_string(), "/dev/sda1".to_string()), + ("resource.device".to_string(), "/dev/sdb2".to_string()), + ]); + assert_eq!(resolve_device_resource(&tags), Some("/dev/sda1")); + + // Neither present. + let tags = MetricTags::from([("unrelated".to_string(), "tag".to_string())]); + assert_eq!(resolve_device_resource(&tags), None); + } + #[test] fn v3_counter_with_interval_differs_from_plain_count() { // Regression test: V2's `series_to_proto_message` sends a Counter with an interval From d702abbda6bbff4573dcbbe6da7a96a25e67366d Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Thu, 13 Aug 2026 10:48:50 +0100 Subject: [PATCH 15/18] Update licenses --- LICENSE-3rdparty.csv | 141 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 141 insertions(+) diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 5c68712a0715a..b4e895194a9df 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -6,22 +6,42 @@ aes-siv,https://github.com/RustCrypto/AEADs,Apache-2.0 OR MIT,RustCrypto Develop ahash,https://github.com/tkaitchuck/ahash,MIT OR Apache-2.0,Tom Kaitchuck aho-corasick,https://github.com/BurntSushi/aho-corasick,Unlicense OR MIT,Andrew Gallant alloc-no-stdlib,https://github.com/dropbox/rust-alloc-no-stdlib,BSD-3-Clause,Daniel Reiter Horn +alloc-stdlib,https://github.com/dropbox/rust-alloc-no-stdlib,BSD-3-Clause,Daniel Reiter Horn allocator-api2,https://github.com/zakarumych/allocator-api2,MIT OR Apache-2.0,Zakarum amq-protocol,https://github.com/amqp-rs/amq-protocol,BSD-2-Clause,Marc-Antoine Perennou +amq-protocol-tcp,https://github.com/amqp-rs/amq-protocol,BSD-2-Clause,Marc-Antoine Perennou +amq-protocol-types,https://github.com/amqp-rs/amq-protocol,BSD-2-Clause,Marc-Antoine Perennou +amq-protocol-uri,https://github.com/amqp-rs/amq-protocol,BSD-2-Clause,Marc-Antoine Perennou android_system_properties,https://github.com/nical/android_system_properties,MIT OR Apache-2.0,Nicolas Silva anstream,https://github.com/rust-cli/anstyle,MIT OR Apache-2.0,The anstream Authors anstyle,https://github.com/rust-cli/anstyle,MIT OR Apache-2.0,The anstyle Authors anstyle-parse,https://github.com/rust-cli/anstyle,MIT OR Apache-2.0,The anstyle-parse Authors anstyle-query,https://github.com/rust-cli/anstyle,MIT OR Apache-2.0,The anstyle-query Authors anstyle-wincon,https://github.com/rust-cli/anstyle,MIT OR Apache-2.0,The anstyle-wincon Authors +antithesis-instrumentation,https://github.com/antithesishq/antithesis-instrumentation-rust,MIT,The antithesis-instrumentation Authors +antithesis_sdk,https://github.com/antithesishq/antithesis-sdk-rust,MIT,The antithesis_sdk Authors anyhow,https://github.com/dtolnay/anyhow,MIT OR Apache-2.0,David Tolnay apache-avro,https://github.com/apache/avro-rs,Apache-2.0,The apache-avro Authors arbitrary,https://github.com/rust-fuzz/arbitrary,MIT OR Apache-2.0,"The Rust-Fuzz Project Developers, Nick Fitzgerald , Manish Goregaokar , Simonas Kazlauskas , Brian L. Troutwine , Corey Farwell " arc-swap,https://github.com/vorner/arc-swap,MIT OR Apache-2.0,Michal 'vorner' Vaner arr_macro,https://github.com/JoshMcguigan/arr_macro,MIT OR Apache-2.0,Josh Mcguigan +arr_macro_impl,https://github.com/JoshMcguigan/arr_macro,MIT OR Apache-2.0,Josh Mcguigan arrayvec,https://github.com/bluss/arrayvec,MIT OR Apache-2.0,bluss arrow,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow +arrow-arith,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow arrow-array,https://github.com/apache/arrow-rs,Apache-2.0 AND MIT,Apache Arrow +arrow-buffer,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow +arrow-cast,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow +arrow-csv,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow +arrow-data,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow +arrow-flight,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow +arrow-ipc,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow +arrow-json,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow +arrow-ord,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow +arrow-row,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow +arrow-schema,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow +arrow-select,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow +arrow-string,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow async-broadcast,https://github.com/smol-rs/async-broadcast,MIT OR Apache-2.0,"Stjepan Glavina , Yoshua Wuyts , Zeeshan Ali Khan " async-channel,https://github.com/smol-rs/async-channel,Apache-2.0 OR MIT,Stjepan Glavina async-compat,https://github.com/smol-rs/async-compat,Apache-2.0 OR MIT,Stjepan Glavina @@ -39,6 +59,7 @@ async-recursion,https://github.com/dcchut/async-recursion,MIT OR Apache-2.0,Robe async-rs,https://github.com/amqp-rs/async-rs,BSD-2-Clause,Marc-Antoine Perennou async-signal,https://github.com/smol-rs/async-signal,Apache-2.0 OR MIT,John Nunley async-stream,https://github.com/tokio-rs/async-stream,MIT,Carl Lerche +async-stream-impl,https://github.com/tokio-rs/async-stream,MIT,Carl Lerche async-task,https://github.com/smol-rs/async-task,Apache-2.0 OR MIT,Stjepan Glavina async-trait,https://github.com/dtolnay/async-trait,MIT OR Apache-2.0,David Tolnay atoi,https://github.com/pacman82/atoi-rs,MIT,Markus Klein @@ -48,6 +69,7 @@ aws-credential-types,https://github.com/smithy-lang/smithy-rs,Apache-2.0,AWS Rus aws-runtime,https://github.com/smithy-lang/smithy-rs,Apache-2.0,AWS Rust SDK Team aws-sdk-cloudwatch,https://github.com/awslabs/aws-sdk-rust,Apache-2.0,"AWS Rust SDK Team , Russell Cohen " aws-sdk-cloudwatchlogs,https://github.com/awslabs/aws-sdk-rust,Apache-2.0,"AWS Rust SDK Team , Russell Cohen " +aws-sdk-elasticsearch,https://github.com/awslabs/aws-sdk-rust,Apache-2.0,"AWS Rust SDK Team , Russell Cohen " aws-sdk-firehose,https://github.com/awslabs/aws-sdk-rust,Apache-2.0,"AWS Rust SDK Team , Russell Cohen " aws-sdk-kinesis,https://github.com/awslabs/aws-sdk-rust,Apache-2.0,"AWS Rust SDK Team , Russell Cohen " aws-sdk-kms,https://github.com/awslabs/aws-sdk-rust,Apache-2.0,"AWS Rust SDK Team , Russell Cohen " @@ -99,6 +121,7 @@ block-padding,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto D blocking,https://github.com/smol-rs/blocking,Apache-2.0 OR MIT,Stjepan Glavina bloomy,https://docs.rs/bloomy/,MIT,"Aleksandr Bezobchuk , Alexis Sellier " bollard,https://github.com/fussybeaver/bollard,Apache-2.0,Bollard contributors +bollard-stubs,https://github.com/fussybeaver/bollard,Apache-2.0,Bollard contributors bon,https://github.com/elastio/bon,MIT OR Apache-2.0,The bon Authors bon-macros,https://github.com/elastio/bon,MIT OR Apache-2.0,The bon-macros Authors borrow-or-share,https://github.com/yescallop/borrow-or-share,MIT-0,Scallop Ye @@ -111,6 +134,7 @@ bson,https://github.com/mongodb/bson-rust,MIT,"Y. T. Chung , bstr,https://github.com/BurntSushi/bstr,MIT OR Apache-2.0,Andrew Gallant bumpalo,https://github.com/fitzgen/bumpalo,MIT OR Apache-2.0,Nick Fitzgerald bytecheck,https://github.com/djkoloski/bytecheck,MIT,David Koloski +bytecheck_derive,https://github.com/djkoloski/bytecheck,MIT,David Koloski bytecount,https://github.com/llogiq/bytecount,Apache-2.0 OR MIT,"Andre Bogus , Joshua Landau " bytemuck,https://github.com/Lokathor/bytemuck,Zlib OR Apache-2.0 OR MIT,Lokathor byteorder,https://github.com/BurntSushi/byteorder,Unlicense OR MIT,Andrew Gallant @@ -128,6 +152,8 @@ charset,https://github.com/hsivonen/charset,MIT OR Apache-2.0,Henri Sivonen +ciborium-io,https://github.com/enarx/ciborium,Apache-2.0,Nathaniel McCallum +ciborium-ll,https://github.com/enarx/ciborium,Apache-2.0,Nathaniel McCallum cidr,https://github.com/stbuehler/rust-cidr,MIT,Stefan Bühler cipher,https://github.com/RustCrypto/traits,MIT OR Apache-2.0,RustCrypto Developers clap,https://github.com/clap-rs/clap,MIT OR Apache-2.0,The clap Authors @@ -145,7 +171,11 @@ combine,https://github.com/Marwes/combine,MIT,Markus Westerlind community-id,https://github.com/traceflight/rs-community-id,MIT OR Apache-2.0,Julian Wang compact_str,https://github.com/ParkMyCar/compact_str,MIT,Parker Timmerman +compression-codecs,https://github.com/Nullus157/async-compression,MIT OR Apache-2.0,"Wim Looman , Allen Bui " +compression-core,https://github.com/Nullus157/async-compression,MIT OR Apache-2.0,"Wim Looman , Allen Bui " concurrent-queue,https://github.com/smol-rs/concurrent-queue,Apache-2.0 OR MIT,"Stjepan Glavina , Taiki Endo , John Nunley " +console-api,https://github.com/tokio-rs/console,MIT,"Eliza Weisman , Tokio Contributors " +console-subscriber,https://github.com/tokio-rs/console,MIT,"Eliza Weisman , Tokio Contributors " const-oid,https://github.com/RustCrypto/formats,Apache-2.0 OR MIT,RustCrypto Developers const-oid,https://github.com/RustCrypto/formats/tree/master/const-oid,Apache-2.0 OR MIT,RustCrypto Developers const-random,https://github.com/tkaitchuck/constrandom,MIT OR Apache-2.0,Tom Kaitchuck @@ -157,6 +187,7 @@ cookie-factory,https://github.com/rust-bakery/cookie-factory,MIT,"Geoffroy Coupr cookie_store,https://github.com/pfernie/cookie_store,MIT OR Apache-2.0,Patrick Fernie core-foundation,https://github.com/servo/core-foundation-rs,MIT OR Apache-2.0,The Servo Project Developers core-foundation,https://github.com/servo/core-foundation-rs,MIT OR Apache-2.0,The Servo Project Developers +core-foundation-sys,https://github.com/servo/core-foundation-rs,MIT OR Apache-2.0,The Servo Project Developers cpubits,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers cpufeatures,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers crc,https://github.com/mrhooray/crc-rs,MIT OR Apache-2.0,"Rui Hu , Akhil Velagapudi <4@4khil.com>" @@ -175,12 +206,15 @@ crypto-bigint,https://github.com/RustCrypto/crypto-bigint,Apache-2.0 OR MIT,Rust crypto-common,https://github.com/RustCrypto/traits,MIT OR Apache-2.0,RustCrypto Developers crypto_secretbox,https://github.com/RustCrypto/nacl-compat/tree/master/crypto_secretbox,Apache-2.0 OR MIT,RustCrypto Developers csv,https://github.com/BurntSushi/rust-csv,Unlicense OR MIT,Andrew Gallant +csv-core,https://github.com/BurntSushi/rust-csv,Unlicense OR MIT,Andrew Gallant ctr,https://github.com/RustCrypto/block-modes,MIT OR Apache-2.0,RustCrypto Developers ctutils,https://github.com/RustCrypto/utils,Apache-2.0 OR MIT,RustCrypto Developers curl-sys,https://github.com/alexcrichton/curl-rust,MIT,Alex Crichton curve25519-dalek,https://github.com/dalek-cryptography/curve25519-dalek/tree/main/curve25519-dalek,BSD-3-Clause,"Isis Lovecruft , Henry de Valence " curve25519-dalek-derive,https://github.com/dalek-cryptography/curve25519-dalek,MIT OR Apache-2.0,The curve25519-dalek-derive Authors darling,https://github.com/TedDriggs/darling,MIT,Ted Driggs +darling_core,https://github.com/TedDriggs/darling,MIT,Ted Driggs +darling_macro,https://github.com/TedDriggs/darling,MIT,Ted Driggs dashmap,https://github.com/xacrimon/dashmap,MIT,Acrimon data-encoding,https://github.com/ia0/data-encoding,MIT,Julien Cretin data-url,https://github.com/servo/rust-url,MIT OR Apache-2.0,Simon Sapin @@ -189,6 +223,7 @@ databricks-zerobus-ingest-sdk,https://github.com/databricks/zerobus-sdk,Apache-2 datadog-agent-metrics-v3,https://github.com/DataDog/saluki,Apache-2.0,The datadog-agent-metrics-v3 Authors dbl,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers deadpool,https://github.com/deadpool-rs/deadpool,MIT OR Apache-2.0,Michael P. Jung +deadpool-runtime,https://github.com/deadpool-rs/deadpool,MIT OR Apache-2.0,Michael P. Jung der,https://github.com/RustCrypto/formats/tree/master/der,Apache-2.0 OR MIT,RustCrypto Developers deranged,https://github.com/jhpratt/deranged,MIT OR Apache-2.0,Jacob Pratt derivative,https://github.com/mcarton/rust-derivative,MIT OR Apache-2.0,mcarton @@ -199,6 +234,7 @@ derive_builder,https://github.com/colin-kiegel/rust-derive-builder,MIT OR Apache derive_builder_core,https://github.com/colin-kiegel/rust-derive-builder,MIT OR Apache-2.0,"Colin Kiegel , Pascal Hertleif , Jan-Erik Rediger , Ted Driggs " derive_builder_macro,https://github.com/colin-kiegel/rust-derive-builder,MIT OR Apache-2.0,"Colin Kiegel , Pascal Hertleif , Jan-Erik Rediger , Ted Driggs " derive_more,https://github.com/JelteF/derive_more,MIT,Jelte Fennema +derive_more-impl,https://github.com/JelteF/derive_more,MIT,Jelte Fennema digest,https://github.com/RustCrypto/traits,MIT OR Apache-2.0,RustCrypto Developers dirs-next,https://github.com/xdg-rs/dirs,MIT OR Apache-2.0,The @xdg-rs members dirs-sys-next,https://github.com/xdg-rs/dirs/tree/master/dirs-sys,MIT OR Apache-2.0,The @xdg-rs members @@ -207,6 +243,7 @@ dns-lookup,https://github.com/keeperofdakeys/dns-lookup,MIT OR Apache-2.0,Josh D doc-comment,https://github.com/GuillaumeGomez/doc-comment,MIT,Guillaume Gomez document-features,https://github.com/slint-ui/document-features,MIT OR Apache-2.0,Slint Developers domain,https://github.com/nlnetlabs/domain,BSD-3-Clause,NLnet Labs +domain-macros,https://github.com/nlnetlabs/domain,BSD-3-Clause,NLnet Labs dotenvy,https://github.com/allan2/dotenvy,MIT,"Noemi Lapresta , Craig Hills , Mike Piccolo , Alice Maz , Sean Griffin , Adam Sharp , Arpad Borsos , Allan Zhang " dyn-clone,https://github.com/dtolnay/dyn-clone,MIT OR Apache-2.0,David Tolnay ecdsa,https://github.com/RustCrypto/signatures/tree/master/ecdsa,Apache-2.0 OR MIT,RustCrypto Developers @@ -223,6 +260,7 @@ enum-ordinalize,https://github.com/magiclen/enum-ordinalize,MIT,The enum-ordinal enum-ordinalize-derive,https://github.com/magiclen/enum-ordinalize,MIT,The enum-ordinalize-derive Authors enum_dispatch,https://gitlab.com/antonok/enum_dispatch,MIT OR Apache-2.0,Anton Lazarev enumflags2,https://github.com/meithecatte/enumflags2,MIT OR Apache-2.0,"maik klein , Maja Kądziołka " +enumflags2_derive,https://github.com/meithecatte/enumflags2,MIT OR Apache-2.0,"maik klein , Maja Kądziołka " env_filter,https://github.com/rust-cli/env_logger,MIT OR Apache-2.0,The env_filter Authors env_logger,https://github.com/rust-cli/env_logger,MIT OR Apache-2.0,The env_logger Authors equivalent,https://github.com/cuviper/equivalent,Apache-2.0 OR MIT,The equivalent Authors @@ -235,6 +273,7 @@ event-listener,https://github.com/smol-rs/event-listener,Apache-2.0 OR MIT,Stjep event-listener,https://github.com/smol-rs/event-listener,Apache-2.0 OR MIT,"Stjepan Glavina , John Nunley " event-listener-strategy,https://github.com/smol-rs/event-listener-strategy,Apache-2.0 OR MIT,John Nunley evmap,https://github.com/jonhoo/rust-evmap,MIT OR Apache-2.0,Jon Gjengset +evmap-derive,https://github.com/jonhoo/rust-evmap,MIT OR Apache-2.0,Jon Gjengset exitcode,https://github.com/benwilber/exitcode,Apache-2.0,Ben Wilber fallible-iterator,https://github.com/sfackler/rust-fallible-iterator,MIT OR Apache-2.0,Steven Fackler fancy-regex,https://github.com/fancy-regex/fancy-regex,MIT,"Raph Levien , Robin Stocker , Keith Hall " @@ -250,6 +289,8 @@ flume,https://github.com/zesterer/flume,Apache-2.0 OR MIT,Joshua Barretto foldhash,https://github.com/orlp/foldhash,Zlib,Orson Peters foreign-types,https://github.com/sfackler/foreign-types,MIT OR Apache-2.0,Steven Fackler +foreign-types-shared,https://github.com/sfackler/foreign-types,MIT OR Apache-2.0,Steven Fackler +form_urlencoded,https://github.com/servo/rust-url,MIT OR Apache-2.0,The rust-url developers fraction,https://github.com/dnsl48/fraction,MIT OR Apache-2.0,dnsl48 fsevent-sys,https://github.com/octplane/fsevent-rust/tree/master/fsevent-sys,MIT,Pierre Baillet fslock,https://github.com/brunoczim/fslock,MIT,The fslock Authors @@ -283,10 +324,19 @@ hashbag,https://github.com/jonhoo/hashbag,MIT OR Apache-2.0,Jon Gjengset hashbrown,https://github.com/rust-lang/hashbrown,MIT OR Apache-2.0,The hashbrown Authors hashlink,https://github.com/kyren/hashlink,MIT OR Apache-2.0,kyren +hdrhistogram,https://github.com/HdrHistogram/HdrHistogram_rust,MIT OR Apache-2.0,"Jon Gjengset , Marshall Pierce " headers,https://github.com/hyperium/headers,MIT,Sean McArthur +headers-core,https://github.com/hyperium/headers,MIT,Sean McArthur heck,https://github.com/withoutboats/heck,MIT OR Apache-2.0,The heck Authors heck,https://github.com/withoutboats/heck,MIT OR Apache-2.0,Without Boats heim,https://github.com/heim-rs/heim,Apache-2.0 OR MIT,svartalf +heim-common,https://github.com/heim-rs/heim,Apache-2.0 OR MIT,svartalf +heim-cpu,https://github.com/heim-rs/heim,Apache-2.0 OR MIT,svartalf +heim-disk,https://github.com/heim-rs/heim,Apache-2.0 OR MIT,svartalf +heim-host,https://github.com/heim-rs/heim,Apache-2.0 OR MIT,svartalf +heim-memory,https://github.com/heim-rs/heim,Apache-2.0 OR MIT,svartalf +heim-net,https://github.com/heim-rs/heim,Apache-2.0 OR MIT,svartalf +heim-runtime,https://github.com/heim-rs/heim,Apache-2.0 OR MIT,svartalf hermit-abi,https://github.com/hermit-os/hermit-rs,MIT OR Apache-2.0,Stefan Lankes hex,https://github.com/KokaKiwi/rust-hex,MIT OR Apache-2.0,KokaKiwi hickory-net,https://github.com/hickory-dns/hickory-dns,MIT OR Apache-2.0,The contributors to Hickory DNS @@ -299,6 +349,7 @@ hostname,https://github.com/djc/hostname,MIT,The hostname Authors hostname,https://github.com/svartalf/hostname,MIT,"fengcen , svartalf " http,https://github.com/hyperium/http,MIT OR Apache-2.0,"Alex Crichton , Carl Lerche , Sean McArthur " http-body,https://github.com/hyperium/http-body,MIT,"Carl Lerche , Lucio Franco , Sean McArthur " +http-body-util,https://github.com/hyperium/http-body,MIT,"Carl Lerche , Lucio Franco , Sean McArthur " http-range-header,https://github.com/MarcusGrass/parse-range-headers,MIT,The http-range-header Authors http-serde,https://gitlab.com/kornelski/http-serde,Apache-2.0 OR MIT,Kornel httparse,https://github.com/seanmonstar/httparse,MIT OR Apache-2.0,Sean McArthur @@ -329,6 +380,7 @@ icu_provider,https://github.com/unicode-org/icu4x,Unicode-3.0,The ICU4X Project icu_provider_macros,https://github.com/unicode-org/icu4x,Unicode-3.0,The ICU4X Project Developers id-arena,https://github.com/fitzgen/id-arena,MIT OR Apache-2.0,"Nick Fitzgerald , Aleksey Kladov " ident_case,https://github.com/TedDriggs/ident_case,MIT OR Apache-2.0,Ted Driggs +idna,https://github.com/servo/rust-url,MIT OR Apache-2.0,The rust-url developers idna_adapter,https://github.com/hsivonen/idna_adapter,Apache-2.0 OR MIT,The rust-url developers indexmap,https://github.com/bluss/indexmap,Apache-2.0 OR MIT,The indexmap Authors indexmap,https://github.com/indexmap-rs/indexmap,Apache-2.0 OR MIT,The indexmap Authors @@ -350,6 +402,7 @@ is_ci,https://github.com/zkat/is_ci,ISC,Kat Marchán itertools,https://github.com/rust-itertools/itertools,MIT OR Apache-2.0,bluss itoa,https://github.com/dtolnay/itoa,MIT OR Apache-2.0,David Tolnay jiff,https://github.com/BurntSushi/jiff,Unlicense OR MIT,Andrew Gallant +jiff-static,https://github.com/BurntSushi/jiff,Unlicense OR MIT,Andrew Gallant jni,https://github.com/jni-rs/jni-rs,MIT OR Apache-2.0,Josh Chase jni,https://github.com/jni-rs/jni-rs,MIT OR Apache-2.0,jni team jni-macros,https://github.com/jni-rs/jni-rs,MIT OR Apache-2.0,The jni-macros Authors @@ -366,7 +419,11 @@ kasuari,https://github.com/ratatui/kasuari,MIT OR Apache-2.0,"Dylan Ede kqueue-sys,https://gitlab.com/rust-kqueue/rust-kqueue-sys,MIT,"William Orr , Daniel (dmilith) Dettlaff " +krb5-src,https://github.com/MaterializeInc/rust-krb5-src,Apache-2.0,"Materialize, Inc." kube,https://github.com/kube-rs/kube,Apache-2.0,"clux , Natalie Klestrup Röijezon , kazk " +kube-client,https://github.com/kube-rs/kube,Apache-2.0,"clux , Natalie Klestrup Röijezon , kazk " +kube-core,https://github.com/kube-rs/kube,Apache-2.0,"clux , Natalie Klestrup Röijezon , kazk " +kube-runtime,https://github.com/kube-rs/kube,Apache-2.0,"clux , Natalie Klestrup Röijezon , kazk " lalrpop-util,https://github.com/lalrpop/lalrpop,Apache-2.0 OR MIT,Niko Matsakis lapin,https://github.com/amqp-rs/lapin,MIT,"Geoffroy Couprie , Marc-Antoine Perennou " lazy_static,https://github.com/rust-lang-nursery/lazy-static.rs,MIT OR Apache-2.0,Marvin Löbel @@ -378,6 +435,7 @@ lexical-util,https://github.com/Alexhuszagh/rust-lexical,MIT OR Apache-2.0,Alex lexical-write-float,https://github.com/Alexhuszagh/rust-lexical,MIT OR Apache-2.0,Alex Huszagh lexical-write-integer,https://github.com/Alexhuszagh/rust-lexical,MIT OR Apache-2.0,Alex Huszagh libc,https://github.com/rust-lang/libc,MIT OR Apache-2.0,The Rust Project Developers +libloading,https://github.com/nagisa/rust_libloading,ISC,Simonas Kazlauskas libm,https://github.com/rust-lang/libm,MIT OR Apache-2.0,Jorge Aparicio libredox,https://gitlab.redox-os.org/redox-os/libredox,MIT,4lDO2 <4lDO2@protonmail.com> libsqlite3-sys,https://github.com/rusqlite/rusqlite,MIT,The rusqlite developers @@ -385,15 +443,19 @@ libz-sys,https://github.com/rust-lang/libz-sys,MIT OR Apache-2.0,"Alex Crichton line-clipping,https://github.com/joshka/line-clipping,MIT OR Apache-2.0,Josh McKinney linked-hash-map,https://github.com/contain-rs/linked-hash-map,MIT OR Apache-2.0,"Stepan Koltsov , Andrew Paseltiner " linked_hash_set,https://github.com/alexheretic/linked-hash-set,Apache-2.0,Alex Butler +linkme,https://github.com/dtolnay/linkme,MIT OR Apache-2.0,David Tolnay +linkme-impl,https://github.com/dtolnay/linkme,MIT OR Apache-2.0,David Tolnay linux-raw-sys,https://github.com/sunfishcode/linux-raw-sys,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,Dan Gohman listenfd,https://github.com/mitsuhiko/listenfd,Apache-2.0,Armin Ronacher litemap,https://github.com/unicode-org/icu4x,Unicode-3.0,The ICU4X Project Developers litrs,https://github.com/LukasKalbertodt/litrs,MIT OR Apache-2.0,Lukas Kalbertodt +lock_api,https://github.com/Amanieu/parking_lot,MIT OR Apache-2.0,Amanieu d'Antras lockfree-object-pool,https://github.com/EVaillant/lockfree-object-pool,BSL-1.0,Etienne Vaillant log,https://github.com/rust-lang/log,MIT OR Apache-2.0,The Rust Project Developers lru,https://github.com/jeromefroe/lru-rs,MIT,Jerome Froelich lru-slab,https://github.com/Ralith/lru-slab,MIT OR Apache-2.0 OR Zlib,Benjamin Saunders lz4,https://github.com/10xGenomics/lz4-rs,MIT,"Jens Heyens , Artem V. Navrotskiy , Patrick Marks " +lz4-sys,https://github.com/10xGenomics/lz4-rs,MIT,"Jens Heyens , Artem V. Navrotskiy , Patrick Marks " lz4_flex,https://github.com/pseitz/lz4_flex,MIT,"Pascal Seitz , Arthur Silva , ticki " macaddr,https://github.com/svartalf/rust-macaddr,Apache-2.0 OR MIT,svartalf mach2,https://github.com/JohnTitor/mach2,BSD-2-Clause OR MIT OR Apache-2.0,The mach2 Authors @@ -412,6 +474,7 @@ memmap2,https://github.com/RazrFalcon/memmap2-rs,MIT OR Apache-2.0,"Dan Burkert memoffset,https://github.com/Gilnaa/memoffset,MIT,Gilad Naaman metrics,https://github.com/metrics-rs/metrics,MIT,Toby Lawrence metrics-tracing-context,https://github.com/metrics-rs/metrics,MIT,MOZGIII +metrics-util,https://github.com/metrics-rs/metrics,MIT,Toby Lawrence mime,https://github.com/hyperium/mime,MIT OR Apache-2.0,Sean McArthur mime_guess,https://github.com/abonander/mime_guess,MIT,Austin Bonander minicbor,https://gitlab.com/twittner/minicbor,BlueOak-1.0.0,Toralf Wittner @@ -438,6 +501,7 @@ no-proxy,https://github.com/jdrouet/no-proxy,MIT,Jérémie Drouet nom,https://github.com/Geal/nom,MIT,contact@geoffroycouprie.com nom,https://github.com/rust-bakery/nom,MIT,contact@geoffroycouprie.com +nom-language,https://github.com/rust-bakery/nom,MIT,contact@geoffroycouprie.com nonzero_ext,https://github.com/antifuchs/nonzero_ext,Apache-2.0,Andreas Fuchs notify,https://github.com/notify-rs/notify,CC0-1.0,"Félix Saparelli , Daniel Faust , Aron Heinecke " notify-types,https://github.com/notify-rs/notify,MIT OR Apache-2.0,Daniel Faust @@ -456,6 +520,7 @@ num-rational,https://github.com/rust-num/num-rational,MIT OR Apache-2.0,The Rust num-traits,https://github.com/rust-num/num-traits,MIT OR Apache-2.0,The Rust Project Developers num_cpus,https://github.com/seanmonstar/num_cpus,MIT OR Apache-2.0,Sean McArthur num_enum,https://github.com/illicitonion/num_enum,BSD-3-Clause OR MIT OR Apache-2.0,"Daniel Wagner-Hall , Daniel Henry-Mantilla , Vincent Esche " +num_enum_derive,https://github.com/illicitonion/num_enum,BSD-3-Clause OR MIT OR Apache-2.0,"Daniel Wagner-Hall , Daniel Henry-Mantilla , Vincent Esche " num_threads,https://github.com/jhpratt/num_threads,MIT OR Apache-2.0,Jacob Pratt oauth2,https://github.com/ramosbugs/oauth2-rs,MIT OR Apache-2.0,"Alex Crichton , Florin Lipan , David A. Ramos " objc,http://github.com/SSheldon/rust-objc,MIT,Steven Sheldon @@ -466,6 +531,7 @@ octseq,https://github.com/NLnetLabs/octets,BSD-3-Clause,NLnet Labs onig,https://github.com/iwillspeak/rust-onig,MIT,"Will Speak , Ivan Ivashchenko " +onig_sys,https://github.com/iwillspeak/rust-onig,MIT,"Will Speak , Ivan Ivashchenko " opaque-debug,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers opendal,https://github.com/apache/opendal,Apache-2.0,Apache OpenDAL openidconnect,https://github.com/ramosbugs/openidconnect-rs,MIT,David A. Ramos @@ -481,6 +547,8 @@ p384,https://github.com/RustCrypto/elliptic-curves/tree/master/p384,Apache-2.0 O pad,https://github.com/ogham/rust-pad,MIT,Ben S parking,https://github.com/smol-rs/parking,Apache-2.0 OR MIT,"Stjepan Glavina , The Rust Project Developers" parking_lot,https://github.com/Amanieu/parking_lot,MIT OR Apache-2.0,Amanieu d'Antras +parking_lot_core,https://github.com/Amanieu/parking_lot,MIT OR Apache-2.0,Amanieu d'Antras +parquet,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow parse-size,https://github.com/kennytm/parse-size,MIT,kennytm paste,https://github.com/dtolnay/paste,MIT OR Apache-2.0,David Tolnay pastey,https://github.com/as1100k/pastey,MIT OR Apache-2.0,"Aditya Kumar , David Tolnay " @@ -488,8 +556,13 @@ pbkdf2,https://github.com/RustCrypto/password-hashes/tree/master/pbkdf2,MIT OR A peeking_take_while,https://github.com/fitzgen/peeking_take_while,MIT OR Apache-2.0,Nick Fitzgerald pem,https://github.com/jcreekmore/pem-rs,MIT,Jonathan Creekmore pem-rfc7468,https://github.com/RustCrypto/formats/tree/master/pem-rfc7468,Apache-2.0 OR MIT,RustCrypto Developers +percent-encoding,https://github.com/servo/rust-url,MIT OR Apache-2.0,The rust-url developers pest,https://github.com/pest-parser/pest,MIT OR Apache-2.0,Dragoș Tiselice +pest_derive,https://github.com/pest-parser/pest,MIT OR Apache-2.0,Dragoș Tiselice +pest_generator,https://github.com/pest-parser/pest,MIT OR Apache-2.0,Dragoș Tiselice +pest_meta,https://github.com/pest-parser/pest,MIT OR Apache-2.0,Dragoș Tiselice phf,https://github.com/rust-phf/rust-phf,MIT,Steven Fackler +phf_shared,https://github.com/rust-phf/rust-phf,MIT,Steven Fackler pin-project,https://github.com/taiki-e/pin-project,Apache-2.0 OR MIT,The pin-project Authors pin-project-internal,https://github.com/taiki-e/pin-project,Apache-2.0 OR MIT,The pin-project-internal Authors pin-project-lite,https://github.com/taiki-e/pin-project-lite,Apache-2.0 OR MIT,The pin-project-lite Authors @@ -516,15 +589,22 @@ proc-macro-crate,https://github.com/bkchr/proc-macro-crate,MIT OR Apache-2.0,Bas proc-macro-error-attr2,https://github.com/GnomedDev/proc-macro-error-2,MIT OR Apache-2.0,"CreepySkeleton , GnomedDev " proc-macro-error2,https://github.com/GnomedDev/proc-macro-error-2,MIT OR Apache-2.0,"CreepySkeleton , GnomedDev " proc-macro-hack,https://github.com/dtolnay/proc-macro-hack,MIT OR Apache-2.0,David Tolnay +proc-macro-nested,https://github.com/dtolnay/proc-macro-hack,MIT OR Apache-2.0,David Tolnay proc-macro2,https://github.com/dtolnay/proc-macro2,MIT OR Apache-2.0,"David Tolnay , Alex Crichton " procfs,https://github.com/eminence/procfs,MIT OR Apache-2.0,Andrew Chin +procfs-core,https://github.com/eminence/procfs,MIT OR Apache-2.0,Andrew Chin proptest,https://github.com/proptest-rs/proptest,MIT OR Apache-2.0,Jason Lingle +proptest-derive,https://github.com/proptest-rs/proptest,MIT OR Apache-2.0,Mazdak Farrokhzad prost,https://github.com/tokio-rs/prost,Apache-2.0,"Dan Burkert , Lucio Franco , Casper Meijn , Tokio Contributors " +prost-derive,https://github.com/tokio-rs/prost,Apache-2.0,"Dan Burkert , Lucio Franco , Casper Meijn , Tokio Contributors " prost-reflect,https://github.com/andrewhickman/prost-reflect,MIT OR Apache-2.0,Andrew Hickman +prost-types,https://github.com/tokio-rs/prost,Apache-2.0,"Dan Burkert , Lucio Franco , Casper Meijn , Tokio Contributors " protobuf,https://github.com/stepancheg/rust-protobuf,MIT,Stepan Koltsov +protobuf-support,https://github.com/stepancheg/rust-protobuf,MIT,Stepan Koltsov psl,https://github.com/addr-rs/psl,MIT OR Apache-2.0,rushmorem psl-types,https://github.com/addr-rs/psl-types,MIT OR Apache-2.0,rushmorem ptr_meta,https://github.com/djkoloski/ptr_meta,MIT,David Koloski +ptr_meta_derive,https://github.com/djkoloski/ptr_meta,MIT,David Koloski publicsuffix,https://github.com/rushmorem/publicsuffix,MIT OR Apache-2.0,rushmorem pulsar,https://github.com/streamnative/pulsar-rs,MIT OR Apache-2.0,"Colin Stearns , Kevin Stenerson , Geoffroy Couprie " quad-rand,https://github.com/not-fl3/quad-rand,MIT,not-fl3 @@ -543,29 +623,40 @@ radium,https://github.com/bitvecto-rs/radium,MIT,"Nika Layzell rand,https://github.com/rust-random/rand,MIT OR Apache-2.0,"The Rand Project Developers, The Rust Project Developers" rand_chacha,https://github.com/rust-random/rand,MIT OR Apache-2.0,"The Rand Project Developers, The Rust Project Developers, The CryptoCorrosion Contributors" +rand_core,https://github.com/rust-random/rand,MIT OR Apache-2.0,"The Rand Project Developers, The Rust Project Developers" rand_core,https://github.com/rust-random/rand_core,MIT OR Apache-2.0,The Rand Project Developers rand_distr,https://github.com/rust-random/rand_distr,MIT OR Apache-2.0,The Rand Project Developers rand_xorshift,https://github.com/rust-random/rngs,MIT OR Apache-2.0,"The Rand Project Developers, The Rust Project Developers" ratatui,https://github.com/ratatui/ratatui,MIT,"Florian Dehau , The Ratatui Developers" +ratatui-core,https://github.com/ratatui/ratatui,MIT,"Florian Dehau , The Ratatui Developers" +ratatui-crossterm,https://github.com/ratatui/ratatui,MIT,"Florian Dehau , The Ratatui Developers" +ratatui-widgets,https://github.com/ratatui/ratatui,MIT,"Florian Dehau , The Ratatui Developers" raw-cpuid,https://github.com/gz/rust-cpuid,MIT,Gerd Zellweger raw-window-handle,https://github.com/rust-windowing/raw-window-handle,MIT OR Apache-2.0 OR Zlib,Osspial rdkafka,https://github.com/fede1024/rust-rdkafka,MIT,Federico Giraud +rdkafka-sys,https://github.com/fede1024/rust-rdkafka,MIT,Federico Giraud redis,https://github.com/redis-rs/redis-rs,BSD-3-Clause,The redis Authors redox_syscall,https://gitlab.redox-os.org/redox-os/syscall,MIT,Jeremy Soller redox_users,https://gitlab.redox-os.org/redox-os/users,MIT,"Jose Narvaez , Wesley Hershberger " ref-cast,https://github.com/dtolnay/ref-cast,MIT OR Apache-2.0,David Tolnay +ref-cast-impl,https://github.com/dtolnay/ref-cast,MIT OR Apache-2.0,David Tolnay +referencing,https://github.com/Stranger6667/jsonschema,MIT,Dmitry Dygalo regex,https://github.com/rust-lang/regex,MIT OR Apache-2.0,"The Rust Project Developers, Andrew Gallant " +regex-automata,https://github.com/rust-lang/regex,MIT OR Apache-2.0,"The Rust Project Developers, Andrew Gallant " regex-filtered,https://github.com/ua-parser/uap-rust,BSD-3-Clause,The regex-filtered Authors +regex-lite,https://github.com/rust-lang/regex,MIT OR Apache-2.0,"The Rust Project Developers, Andrew Gallant " regex-syntax,https://github.com/rust-lang/regex/tree/master/regex-syntax,MIT OR Apache-2.0,"The Rust Project Developers, Andrew Gallant " relative-path,https://github.com/udoprog/relative-path,MIT OR Apache-2.0,John-John Tedro rend,https://github.com/djkoloski/rend,MIT,David Koloski reqwest,https://github.com/seanmonstar/reqwest,MIT OR Apache-2.0,Sean McArthur reqwest-middleware,https://github.com/TrueLayer/reqwest-middleware,MIT OR Apache-2.0,Rodrigo Gryzinski +reqwest-retry,https://github.com/TrueLayer/reqwest-middleware,MIT OR Apache-2.0,Rodrigo Gryzinski resolv-conf,https://github.com/hickory-dns/resolv-conf,MIT OR Apache-2.0,The resolv-conf Authors retry-policies,https://github.com/TrueLayer/retry-policies,MIT OR Apache-2.0,Luca Palmieri rfc6979,https://github.com/RustCrypto/signatures/tree/master/rfc6979,Apache-2.0 OR MIT,RustCrypto Developers ring,https://github.com/briansmith/ring,Apache-2.0 AND ISC,The ring Authors rkyv,https://github.com/rkyv/rkyv,MIT,David Koloski +rkyv_derive,https://github.com/rkyv/rkyv,MIT,David Koloski rmp,https://github.com/3Hren/msgpack-rust,MIT,"Evgeny Safronov , Kornel " rmp-serde,https://github.com/3Hren/msgpack-rust,MIT,Evgeny Safronov rmpv,https://github.com/3Hren/msgpack-rust,MIT,Evgeny Safronov @@ -591,6 +682,7 @@ rustyline,https://github.com/kkawakam/rustyline,MIT,Katsu Kawakami salsa20,https://github.com/RustCrypto/stream-ciphers,MIT OR Apache-2.0,RustCrypto Developers same-file,https://github.com/BurntSushi/same-file,Unlicense OR MIT,Andrew Gallant +sasl2-sys,https://github.com/MaterializeInc/rust-sasl,Apache-2.0,"Materialize, Inc." schannel,https://github.com/steffengy/schannel-rs,MIT,"Steven Fackler , Steffen Butzer " schemars,https://github.com/GREsau/schemars,MIT,Graham Esau scoped-tls,https://github.com/alexcrichton/scoped-tls,MIT OR Apache-2.0,Alex Crichton @@ -600,6 +692,7 @@ seahash,https://gitlab.redox-os.org/redox-os/seahash,MIT,"ticki security-framework,https://github.com/kornelski/rust-security-framework,MIT OR Apache-2.0,"Steven Fackler , Kornel " +security-framework-sys,https://github.com/kornelski/rust-security-framework,MIT OR Apache-2.0,"Steven Fackler , Kornel " semver,https://github.com/dtolnay/semver,MIT OR Apache-2.0,David Tolnay seq-macro,https://github.com/dtolnay/seq-macro,MIT OR Apache-2.0,David Tolnay serde,https://github.com/serde-rs/serde,MIT OR Apache-2.0,"Erick Tryzelaar , David Tolnay " @@ -607,6 +700,9 @@ serde-aux,https://github.com/iddm/serde-aux,MIT,Victor Polevoy serde-value,https://github.com/arcnmx/serde-value,MIT,arcnmx serde_bytes,https://github.com/serde-rs/bytes,MIT OR Apache-2.0,David Tolnay +serde_core,https://github.com/serde-rs/serde,MIT OR Apache-2.0,"Erick Tryzelaar , David Tolnay " +serde_derive,https://github.com/serde-rs/serde,MIT OR Apache-2.0,"Erick Tryzelaar , David Tolnay " +serde_derive_internals,https://github.com/serde-rs/serde,MIT OR Apache-2.0,"Erick Tryzelaar , David Tolnay " serde_json,https://github.com/serde-rs/json,MIT OR Apache-2.0,"Erick Tryzelaar , David Tolnay " serde_nanos,https://github.com/caspervonb/serde_nanos,MIT OR Apache-2.0,Casper Beyer serde_path_to_error,https://github.com/dtolnay/path-to-error,MIT OR Apache-2.0,David Tolnay @@ -638,6 +734,7 @@ smallvec,https://github.com/servo/rust-smallvec,MIT OR Apache-2.0,The Servo Proj smol,https://github.com/smol-rs/smol,Apache-2.0 OR MIT,Stjepan Glavina smpl_jwt,https://github.com/durch/rust-jwt,MIT,Drazen Urch snafu,https://github.com/shepmaster/snafu,MIT OR Apache-2.0,Jake Goulding +snafu-derive,https://github.com/shepmaster/snafu,MIT OR Apache-2.0,Jake Goulding snap,https://github.com/BurntSushi/rust-snappy,BSD-3-Clause,Andrew Gallant socket2,https://github.com/rust-lang/socket2,MIT OR Apache-2.0,"Alex Crichton , Thomas de Zeeuw " spin,https://github.com/mvdnes/spin-rs,MIT,"Mathijs van de Nes , John Ericson " @@ -646,6 +743,12 @@ spinning_top,https://github.com/rust-osdev/spinning_top,MIT OR Apache-2.0,Philip spki,https://github.com/RustCrypto/formats/tree/master/spki,Apache-2.0 OR MIT,RustCrypto Developers sponge-cursor,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers sqlx,https://github.com/launchbadge/sqlx,MIT OR Apache-2.0,"Ryan Leckey , Austin Bonander , Chloe Ross , Daniel Akhterov " +sqlx-core,https://github.com/launchbadge/sqlx,MIT OR Apache-2.0,"Ryan Leckey , Austin Bonander , Chloe Ross , Daniel Akhterov " +sqlx-macros,https://github.com/launchbadge/sqlx,MIT OR Apache-2.0,"Ryan Leckey , Austin Bonander , Chloe Ross , Daniel Akhterov " +sqlx-macros-core,https://github.com/launchbadge/sqlx,MIT OR Apache-2.0,"Ryan Leckey , Austin Bonander , Chloe Ross , Daniel Akhterov " +sqlx-mysql,https://github.com/launchbadge/sqlx,MIT OR Apache-2.0,"Ryan Leckey , Austin Bonander , Chloe Ross , Daniel Akhterov " +sqlx-postgres,https://github.com/launchbadge/sqlx,MIT OR Apache-2.0,"Ryan Leckey , Austin Bonander , Chloe Ross , Daniel Akhterov " +sqlx-sqlite,https://github.com/launchbadge/sqlx,MIT OR Apache-2.0,"Ryan Leckey , Austin Bonander , Chloe Ross , Daniel Akhterov " stable_deref_trait,https://github.com/storyyeller/stable_deref_trait,MIT OR Apache-2.0,Robert Grosse static_assertions,https://github.com/nvzqz/static-assertions-rs,MIT OR Apache-2.0,Nikolai Vazquez stream-cancel,https://github.com/jonhoo/stream-cancel,MIT OR Apache-2.0,Jon Gjengset @@ -653,6 +756,7 @@ stringprep,https://github.com/sfackler/rust-stringprep,MIT OR Apache-2.0,Steven strip-ansi-escapes,https://github.com/luser/strip-ansi-escapes,Apache-2.0 OR MIT,Ted Mielczarek strsim,https://github.com/rapidfuzz/strsim-rs,MIT,"Danny Guo , maxbachmann " strum,https://github.com/Peternator7/strum,MIT,Peter Glotfelty +strum_macros,https://github.com/Peternator7/strum,MIT,Peter Glotfelty subtle,https://github.com/dalek-cryptography/subtle,BSD-3-Clause,"Isis Lovecruft , Henry de Valence " supports-color,https://github.com/zkat/supports-color,Apache-2.0,Kat Marchán syn,https://github.com/dtolnay/syn,MIT OR Apache-2.0,David Tolnay @@ -662,6 +766,7 @@ sysinfo,https://github.com/GuillaumeGomez/sysinfo,MIT,Guillaume Gomez system-configuration,https://github.com/mullvad/system-configuration-rs,MIT OR Apache-2.0,Mullvad VPN +system-configuration-sys,https://github.com/mullvad/system-configuration-rs,MIT OR Apache-2.0,Mullvad VPN tagptr,https://github.com/oliver-giersch/tagptr,MIT OR Apache-2.0,Oliver Giersch take_mut,https://github.com/Sgeo/take_mut,MIT,Sgeo tap,https://github.com/myrrlyn/tap,MIT,"Elliott Linder , myrrlyn " @@ -671,23 +776,29 @@ term,https://github.com/Stebalien/term,MIT OR Apache-2.0,"The Rust Project Devel termcolor,https://github.com/BurntSushi/termcolor,Unlicense OR MIT,Andrew Gallant terminal_size,https://github.com/eminence/terminal-size,MIT OR Apache-2.0,Andrew Chin thiserror,https://github.com/dtolnay/thiserror,MIT OR Apache-2.0,David Tolnay +thiserror-impl,https://github.com/dtolnay/thiserror,MIT OR Apache-2.0,David Tolnay thread_local,https://github.com/Amanieu/thread_local-rs,MIT OR Apache-2.0,Amanieu d'Antras thrift,https://github.com/apache/thrift/tree/master/lib/rs,Apache-2.0,Apache Thrift Developers tikv-jemalloc-sys,https://github.com/tikv/jemallocator,MIT OR Apache-2.0,"Alex Crichton , Gonzalo Brito Gadeschi , The TiKV Project Developers" tikv-jemallocator,https://github.com/tikv/jemallocator,MIT OR Apache-2.0,"Alex Crichton , Gonzalo Brito Gadeschi , Simon Sapin , Steven Fackler , The TiKV Project Developers" time,https://github.com/time-rs/time,MIT OR Apache-2.0,"Jacob Pratt , Time contributors" +time-core,https://github.com/time-rs/time,MIT OR Apache-2.0,"Jacob Pratt , Time contributors" +time-macros,https://github.com/time-rs/time,MIT OR Apache-2.0,"Jacob Pratt , Time contributors" tiny-keccak,https://github.com/debris/tiny-keccak,CC0-1.0,debris tinystr,https://github.com/unicode-org/icu4x,Unicode-3.0,The ICU4X Project Developers tinyvec,https://github.com/Lokathor/tinyvec,Zlib OR Apache-2.0 OR MIT,Lokathor tinyvec_macros,https://github.com/Soveu/tinyvec_macros,MIT OR Apache-2.0 OR Zlib,Soveu tokio,https://github.com/tokio-rs/tokio,MIT,Tokio Contributors tokio-io-timeout,https://github.com/sfackler/tokio-io-timeout,MIT OR Apache-2.0,Steven Fackler +tokio-macros,https://github.com/tokio-rs/tokio,MIT,Tokio Contributors tokio-native-tls,https://github.com/tokio-rs/tls,MIT,Tokio Contributors tokio-openssl,https://github.com/tokio-rs/tokio-openssl,MIT OR Apache-2.0,Alex Crichton tokio-postgres,https://github.com/rust-postgres/rust-postgres,MIT OR Apache-2.0,Steven Fackler tokio-retry,https://github.com/srijs/rust-tokio-retry,MIT,Sam Rijs tokio-rustls,https://github.com/rustls/tokio-rustls,MIT OR Apache-2.0,The tokio-rustls Authors +tokio-stream,https://github.com/tokio-rs/tokio,MIT,Tokio Contributors tokio-tungstenite,https://github.com/snapview/tokio-tungstenite,MIT,"Daniel Abramov , Alexey Galakhov " +tokio-util,https://github.com/tokio-rs/tokio,MIT,Tokio Contributors tokio-websockets,https://github.com/Gelbpunkt/tokio-websockets,MIT,The tokio-websockets Authors toml,https://github.com/toml-rs/toml,MIT OR Apache-2.0,The toml Authors toml_datetime,https://github.com/toml-rs/toml,MIT OR Apache-2.0,The toml_datetime Authors @@ -698,12 +809,16 @@ toml_write,https://github.com/toml-rs/toml,MIT OR Apache-2.0,The toml_write Auth toml_writer,https://github.com/toml-rs/toml,MIT OR Apache-2.0,The toml_writer Authors tonic,https://github.com/hyperium/tonic,MIT,Lucio Franco tonic-health,https://github.com/hyperium/tonic,MIT,James Nugent +tonic-prost,https://github.com/hyperium/tonic,MIT,Lucio Franco tonic-reflection,https://github.com/hyperium/tonic,MIT,"James Nugent , Samani G. Gikandi " tower,https://github.com/tower-rs/tower,MIT,Tower Maintainers tower-http,https://github.com/tower-rs/tower-http,MIT,Tower Maintainers +tower-layer,https://github.com/tower-rs/tower,MIT,Tower Maintainers +tower-service,https://github.com/tower-rs/tower,MIT,Tower Maintainers tracing,https://github.com/tokio-rs/tracing,MIT,"Eliza Weisman , Tokio Contributors " tracing-attributes,https://github.com/tokio-rs/tracing,MIT,"Tokio Contributors , Eliza Weisman , David Barsky " tracing-core,https://github.com/tokio-rs/tracing,MIT,Tokio Contributors +tracing-futures,https://github.com/tokio-rs/tracing,MIT,"Eliza Weisman , Tokio Contributors " tracing-log,https://github.com/tokio-rs/tracing,MIT,Tokio Contributors tracing-serde,https://github.com/tokio-rs/tracing,MIT,Tokio Contributors tracing-subscriber,https://github.com/tokio-rs/tracing,MIT,"Eliza Weisman , David Barsky , Tokio Contributors " @@ -714,11 +829,13 @@ tryhard,https://github.com/EmbarkStudios/tryhard,MIT OR Apache-2.0,Embark typed-builder,https://github.com/idanarye/rust-typed-builder,MIT OR Apache-2.0,"IdanArye , Chris Morgan " +typed-builder-macro,https://github.com/idanarye/rust-typed-builder,MIT OR Apache-2.0,"IdanArye , Chris Morgan " typenum,https://github.com/paholg/typenum,MIT OR Apache-2.0,The typenum Authors typespec,https://github.com/azure/azure-sdk-for-rust,MIT,Microsoft typespec_client_core,https://github.com/azure/azure-sdk-for-rust,MIT,Microsoft typespec_macros,https://github.com/azure/azure-sdk-for-rust,MIT,Microsoft typetag,https://github.com/dtolnay/typetag,MIT OR Apache-2.0,David Tolnay +typetag-impl,https://github.com/dtolnay/typetag,MIT OR Apache-2.0,David Tolnay ua-parser,https://github.com/ua-parser/uap-rust,Apache-2.0,The ua-parser Authors ucd-trie,https://github.com/BurntSushi/ucd-generate,MIT OR Apache-2.0,Andrew Gallant unarray,https://github.com/cameron1024/unarray,MIT OR Apache-2.0,The unarray Authors @@ -742,6 +859,7 @@ utf-8,https://github.com/SimonSapin/rust-utf8,MIT OR Apache-2.0,Simon Sapin utf8-width,https://github.com/magiclen/utf8-width,MIT,Magic Len utf8_iter,https://github.com/hsivonen/utf8_iter,Apache-2.0 OR MIT,Henri Sivonen +utf8parse,https://github.com/alacritty/vte,Apache-2.0 OR MIT,"Joe Wilm , Christian Duerr " uuid,https://github.com/uuid-rs/uuid,Apache-2.0 OR MIT,"Ashley Mannix, Dylan DPC, Hunar Roop Kahlon" uuid-simd,https://github.com/Nugine/simd,MIT,The uuid-simd Authors valuable,https://github.com/tokio-rs/valuable,MIT,The valuable Authors @@ -777,19 +895,41 @@ whoami,https://github.com/ardaku/whoami,Apache-2.0 OR BSL-1.0 OR MIT,The whoami widestring,https://github.com/starkat99/widestring-rs,MIT OR Apache-2.0,Kathryn Long widestring,https://github.com/starkat99/widestring-rs,MIT OR Apache-2.0,The widestring Authors winapi,https://github.com/retep998/winapi-rs,MIT OR Apache-2.0,Peter Atashian +winapi-i686-pc-windows-gnu,https://github.com/retep998/winapi-rs,MIT OR Apache-2.0,Peter Atashian winapi-util,https://github.com/BurntSushi/winapi-util,Unlicense OR MIT,Andrew Gallant +winapi-x86_64-pc-windows-gnu,https://github.com/retep998/winapi-rs,MIT OR Apache-2.0,Peter Atashian windows,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft windows-collections,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,The windows-collections Authors +windows-core,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft windows-future,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,The windows-future Authors +windows-implement,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft windows-implement,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,The windows-implement Authors +windows-interface,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft windows-interface,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,The windows-interface Authors +windows-link,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft windows-link,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,The windows-link Authors windows-numerics,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,The windows-numerics Authors +windows-result,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft windows-service,https://github.com/mullvad/windows-service-rs,MIT OR Apache-2.0,Mullvad VPN +windows-strings,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft +windows-sys,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft windows-sys,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,The windows-sys Authors +windows-targets,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft +windows-threading,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft +windows_aarch64_gnullvm,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft +windows_aarch64_msvc,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft +windows_i686_gnu,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft +windows_i686_gnullvm,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft +windows_i686_msvc,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft +windows_x86_64_gnu,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft +windows_x86_64_gnullvm,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft +windows_x86_64_msvc,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft winnow,https://github.com/winnow-rs/winnow,MIT,The winnow Authors winreg,https://github.com/gentoo90/winreg-rs,MIT,Igor Shaula wit-bindgen,https://github.com/bytecodealliance/wit-bindgen,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,Alex Crichton +wit-bindgen-core,https://github.com/bytecodealliance/wit-bindgen,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,Alex Crichton +wit-bindgen-rust,https://github.com/bytecodealliance/wit-bindgen,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,Alex Crichton +wit-bindgen-rust-macro,https://github.com/bytecodealliance/wit-bindgen,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,Alex Crichton wit-component,https://github.com/bytecodealliance/wasm-tools/tree/main/crates/wit-component,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,Peter Huene wit-parser,https://github.com/bytecodealliance/wasm-tools/tree/main/crates/wit-parser,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,Alex Crichton woothee,https://github.com/woothee/woothee-rust,Apache-2.0,hhatto @@ -801,6 +941,7 @@ xxhash-rust,https://github.com/DoumanAsh/xxhash-rust,BSL-1.0,Douman yoke-derive,https://github.com/unicode-org/icu4x,Unicode-3.0,Manish Goregaokar zerocopy,https://github.com/google/zerocopy,BSD-2-Clause OR Apache-2.0 OR MIT,Joshua Liebow-Feeser +zerocopy-derive,https://github.com/google/zerocopy,BSD-2-Clause OR Apache-2.0 OR MIT,Joshua Liebow-Feeser zerofrom,https://github.com/unicode-org/icu4x,Unicode-3.0,Manish Goregaokar zerofrom-derive,https://github.com/unicode-org/icu4x,Unicode-3.0,Manish Goregaokar zeroize,https://github.com/RustCrypto/utils/tree/master/zeroize,Apache-2.0 OR MIT,The RustCrypto Project Developers From f350abc3f62a2a9034b4d986c626a0c54091ba8c Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Thu, 13 Aug 2026 11:53:44 +0100 Subject: [PATCH 16/18] Shadow every 1000 --- src/sinks/datadog/metrics/config.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sinks/datadog/metrics/config.rs b/src/sinks/datadog/metrics/config.rs index 475138b257312..f3cf535c019af 100644 --- a/src/sinks/datadog/metrics/config.rs +++ b/src/sinks/datadog/metrics/config.rs @@ -252,7 +252,7 @@ impl DatadogMetricsEndpointConfiguration { } const fn default_shadow_every() -> NonZeroU64 { - NonZeroU64::new(1).unwrap() + NonZeroU64::new(1000).unwrap() } /// Configuration for the V3 shadow dual-write mode. From 74c5422510a724c2a3f69228b2d23a4f440857fe Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Thu, 13 Aug 2026 13:05:00 +0100 Subject: [PATCH 17/18] Fix host resolving --- src/sinks/datadog/metrics/encoder_v3.rs | 70 ++++++++++++++++++++++--- 1 file changed, 64 insertions(+), 6 deletions(-) diff --git a/src/sinks/datadog/metrics/encoder_v3.rs b/src/sinks/datadog/metrics/encoder_v3.rs index d6066e6cdd9b5..d1665d3f31778 100644 --- a/src/sinks/datadog/metrics/encoder_v3.rs +++ b/src/sinks/datadog/metrics/encoder_v3.rs @@ -210,6 +210,7 @@ fn encode_metric_to_v3( if let Some(tags) = metric.tags() { device_resource = resolve_device_resource(tags); + host_resource = resolve_host_resource(tags, host_key.as_deref()); for (key, value) in tags.iter_all() { // dd.internal.resource tags become structured resources @@ -222,13 +223,9 @@ fn encode_metric_to_v3( continue; } - // Host key → host resource + // Host key already resolved above via `resolve_host_resource` -- just consume it + // here so it doesn't fall through to the generic tag handling below. if host_key.as_deref() == Some(key) { - if let Some(host) = value - && !host.is_empty() - { - host_resource = Some(host); - } continue; } @@ -350,6 +347,25 @@ fn resolve_device_resource(tags: &MetricTags) -> Option<&str> { tags.get("device").or_else(|| tags.get("resource.device")) } +/// Resolves a metric's `host` resource tag value, matching V2's `tags.remove(key)` lookup in +/// `series_to_proto_message`. For a multi-valued host tag, `TagValueSet`'s single-value +/// conversion resolves to whichever value was inserted *last*, regardless of whether that +/// value happens to be an empty string -- it does not skip backward looking for an earlier +/// non-empty one. V3 used to resolve the host tag inside its own tag-iteration loop with an +/// explicit "skip if empty" filter applied *per visited value*, which instead kept whichever +/// non-empty value it saw and ignored a later, chronologically-last empty one. That divergence +/// meant a metric whose host tag's last-inserted value happened to be empty produced a +/// non-empty host resource in V3 but an empty one in V2 for the same metric, mismatching the +/// paired V2/V3 shadow payloads for the same flush. +/// +/// An absent, empty, or empty-after-resolution host tag all come out `None` here; the caller +/// fills in an empty host resource once a host key is configured at all, matching V2's +/// behavior of always including a host resource (even with an empty name) in that case. +fn resolve_host_resource<'a>(tags: &'a MetricTags, host_key: Option<&str>) -> Option<&'a str> { + let host_key = host_key?; + tags.get(host_key).filter(|host| !host.is_empty()) +} + /// Assembles the final resource list in a fixed host-then-device order, matching V2's /// `encode_series_metrics`. Host/device are collected separately during tag iteration /// (whose order is unspecified) so that order never leaks into the wire-visible resources. @@ -506,6 +522,48 @@ mod tests { assert_eq!(resolve_device_resource(&tags), None); } + // Regression test: for a multi-valued host tag, V2's `series_to_proto_message` calls + // `tags.remove(host_key)`, whose single-value conversion (`TagValueSet::into_single`) + // finds the *last inserted* value tag, regardless of whether that value happens to be an + // empty string -- it does not skip backward to find an earlier non-empty value. V3 used to + // resolve the host tag inside its tag-iteration loop with an explicit `!host.is_empty()` + // filter *per visited value*, which instead kept whichever non-empty value it saw, ignoring + // any later, chronologically-last empty value. Confirmed against the real `series_to_proto_ + // message` output: inserting `("host", "host1")` then `("host", "")` produces an *empty* + // V2 host resource, not `"host1"`. + #[test] + fn v3_host_resource_matches_v2_single_value_lookup_for_multivalue_tags() { + // Single value: passes straight through. + let mut tags = MetricTags::default(); + tags.insert("host".to_string(), "myhost".to_string()); + assert_eq!(resolve_host_resource(&tags, Some("host")), Some("myhost")); + + // Multiple non-empty values: resolves to the last-inserted one, matching V2's + // `into_single` (`TagValueSet::Set`'s `rfind` over insertion order). + let mut tags = MetricTags::default(); + tags.insert("host".to_string(), "host1".to_string()); + tags.insert("host".to_string(), "host2".to_string()); + assert_eq!(resolve_host_resource(&tags, Some("host")), Some("host2")); + + // Multiple values where the *last-inserted* one is empty: V2's `into_single` still + // finds that empty value (it doesn't skip backward looking for a non-empty one), so + // the host resource comes out empty here too -- not "host1", which is what V3's old + // per-value `!host.is_empty()` filter inside the tag loop would have produced. + let mut tags = MetricTags::default(); + tags.insert("host".to_string(), "host1".to_string()); + tags.insert("host".to_string(), String::new()); + assert_eq!(resolve_host_resource(&tags, Some("host")), None); + + // No host tag at all. + let tags = MetricTags::from([("unrelated".to_string(), "tag".to_string())]); + assert_eq!(resolve_host_resource(&tags, Some("host")), None); + + // No host key configured at all (log_schema.host_key() is None). + let mut tags = MetricTags::default(); + tags.insert("host".to_string(), "myhost".to_string()); + assert_eq!(resolve_host_resource(&tags, None), None); + } + #[test] fn v3_counter_with_interval_differs_from_plain_count() { // Regression test: V2's `series_to_proto_message` sends a Counter with an interval From 6f47c74ebac19140a746f5f351733bf5030abf33 Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Thu, 13 Aug 2026 14:38:16 +0100 Subject: [PATCH 18/18] rate limit request error --- src/internal_events/datadog_metrics.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/internal_events/datadog_metrics.rs b/src/internal_events/datadog_metrics.rs index ba819b0bb2d0e..52620b5ead04d 100644 --- a/src/internal_events/datadog_metrics.rs +++ b/src/internal_events/datadog_metrics.rs @@ -63,7 +63,7 @@ impl InternalEvent for DatadogMetricsRequestFailed<'_> { stage = error_stage::SENDING, batch_id = self.batch_id.unwrap_or("none"), uri = %self.uri, - internal_log_rate_limit = false, + internal_log_rate_limit = true, ); } }