From bf4fa5c72d772070c0a17e024dcf3cb03040c459 Mon Sep 17 00:00:00 2001 From: Renizmy Date: Thu, 13 Aug 2026 22:28:30 +0200 Subject: [PATCH 1/5] feat(azure_blob source): add Azure Blob Storage source --- Cargo.lock | 15 + Cargo.toml | 7 +- LICENSE-3rdparty.csv | 1 + changelog.d/azure_blob_source.feature.md | 14 + .../src/internal_event/metric_name.rs | 26 + src/internal_events/azure_blob.rs | 294 ++++ src/internal_events/mod.rs | 4 + src/sinks/mod.rs | 6 +- src/sources/azure_blob/integration_tests.rs | 696 +++++++++ src/sources/azure_blob/mod.rs | 1259 ++++++++++++++++ src/sources/azure_blob/queue.rs | 1261 +++++++++++++++++ src/sources/mod.rs | 2 + tests/integration/azure/config/compose.yaml | 2 +- tests/integration/azure/config/test.yaml | 1 + .../configuration/sources/azure_blob.md | 14 + .../components/sources/azure_blob.cue | 149 ++ .../sources/generated/azure_blob.cue | 934 ++++++++++++ .../components/sources/internal_metrics.cue | 57 + website/cue/reference/urls.cue | 2 + 19 files changed, 4740 insertions(+), 4 deletions(-) create mode 100644 changelog.d/azure_blob_source.feature.md create mode 100644 src/internal_events/azure_blob.rs create mode 100644 src/sources/azure_blob/integration_tests.rs create mode 100644 src/sources/azure_blob/mod.rs create mode 100644 src/sources/azure_blob/queue.rs create mode 100644 website/content/en/docs/reference/configuration/sources/azure_blob.md create mode 100644 website/cue/reference/components/sources/azure_blob.cue create mode 100644 website/cue/reference/components/sources/generated/azure_blob.cue diff --git a/Cargo.lock b/Cargo.lock index f8a43799f918f..15e5b98ae3764 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2002,6 +2002,7 @@ dependencies = [ "rustc_version", "serde", "serde_json", + "tokio", "tracing 0.1.44", "typespec", "typespec_client_core", @@ -2056,6 +2057,18 @@ dependencies = [ "time", ] +[[package]] +name = "azure_storage_queue" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e3836053722dfa5aa3461e47aebb96e230a87d8d22eb1f866ca704f691b93ae" +dependencies = [ + "async-trait", + "azure_core", + "serde", + "time", +] + [[package]] name = "backon" version = "1.6.0" @@ -12866,6 +12879,7 @@ dependencies = [ "serde", "serde_json", "time", + "tokio", "tracing 0.1.44", "typespec", "typespec_macros", @@ -13220,6 +13234,7 @@ dependencies = [ "azure_core", "azure_identity", "azure_storage_blob", + "azure_storage_queue", "base64 0.23.0", "bloomy", "bollard", diff --git a/Cargo.toml b/Cargo.toml index 050d818fe2ae6..748c7e526c9bc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -323,11 +323,12 @@ aws-smithy-runtime-api = { version = "1.7.3", default-features = false, optional aws-smithy-types = { version = "1.2.11", default-features = false, features = ["rt-tokio"], optional = true } # Azure -azure_core = { version = "1.0", default-features = false, features = ["reqwest", "hmac_openssl"], optional = true } +azure_core = { version = "1.0", default-features = false, features = ["reqwest", "hmac_openssl", "tokio"], optional = true } azure_identity = { version = "1.0", default-features = false, features = ["client_certificate"], optional = true } # Azure Storage azure_storage_blob = { version = "1.0", default-features = false, optional = true } +azure_storage_queue = { version = "1.0", default-features = false, optional = true } # OpenDAL opendal = { version = "0.54", default-features = false, features = ["services-webhdfs"], optional = true } @@ -686,6 +687,7 @@ sources-logs = [ "sources-aws_kinesis_firehose", "sources-aws_s3", "sources-aws_sqs", + "sources-azure_blob", "sources-datadog_agent", "sources-demo_logs", "sources-docker_logs", @@ -739,6 +741,7 @@ sources-aws_ecs_metrics = ["sources-utils-http-client"] sources-aws_kinesis_firehose = ["dep:base64", "sources-http_server", "sources-utils-http-encoding"] sources-aws_s3 = ["aws-core", "dep:aws-sdk-sqs", "dep:aws-sdk-s3", "dep:async-compression", "sources-aws_sqs", "tokio-util/io"] sources-aws_sqs = ["aws-core", "dep:aws-sdk-sqs"] +sources-azure_blob = ["dep:azure_core", "dep:azure_identity", "dep:azure_storage_blob", "dep:azure_storage_queue", "dep:async-compression", "dep:base64", "tokio-util/io"] sources-datadog_agent = ["sources-utils-http-encoding", "protobuf-build", "dep:prost"] sources-demo_logs = ["dep:fakedata"] sources-dnstap = ["sources-utils-net-tcp", "dep:base64", "dep:hickory-proto", "dep:dnsmsg-parser", "dep:dnstap-parser", "protobuf-build", "dep:prost", "vector-vrl-functions/dnstap"] @@ -1069,7 +1072,7 @@ aws-s3-integration-tests = ["sinks-aws_s3", "sources-aws_s3"] aws-sqs-integration-tests = ["sinks-aws_sqs"] aws-sns-integration-tests = ["sinks-aws_sns"] axiom-integration-tests = ["sinks-axiom"] -azure-blob-integration-tests = ["sinks-azure_blob"] +azure-blob-integration-tests = ["sinks-azure_blob", "sources-azure_blob"] azure-logs-ingestion-integration-tests = ["sinks-azure_logs_ingestion"] clickhouse-integration-tests = ["sinks-clickhouse"] databend-integration-tests = ["sinks-databend"] diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index e3fa1f9f67d90..4ed13a76cf141 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -103,6 +103,7 @@ azure_core,https://github.com/azure/azure-sdk-for-rust,MIT,Microsoft azure_core_macros,https://github.com/azure/azure-sdk-for-rust,MIT,Microsoft azure_identity,https://github.com/azure/azure-sdk-for-rust,MIT,Microsoft azure_storage_blob,https://github.com/azure/azure-sdk-for-rust,MIT,Microsoft +azure_storage_queue,https://github.com/azure/azure-sdk-for-rust,MIT,Microsoft backon,https://github.com/Xuanwo/backon,Apache-2.0,The backon Authors base16,https://github.com/thomcc/rust-base16,CC0-1.0,Thom Chiovoloni base16ct,https://github.com/RustCrypto/formats/tree/master/base16ct,Apache-2.0 OR MIT,RustCrypto Developers diff --git a/changelog.d/azure_blob_source.feature.md b/changelog.d/azure_blob_source.feature.md new file mode 100644 index 0000000000000..8ea9a69582ecd --- /dev/null +++ b/changelog.d/azure_blob_source.feature.md @@ -0,0 +1,14 @@ +Added a new `azure_blob` source that collects logs from blobs in Azure Blob +Storage. Discovery is event-driven: an Event Grid subscription on the storage +account delivers `Microsoft.Storage.BlobCreated` notifications to an Azure +Storage Queue, which Vector polls. Newly created blobs are downloaded, +optionally decompressed (gzip/zstd, auto-detected), decoded with any codec, +and the queue message is deleted once the events are durably accepted by the +pipeline (end-to-end acknowledgements). + +Both the Event Grid and CloudEvents 1.0 notification schemas are supported and +auto-detected. Authentication reuses the same options as the `azure_blob` +sink: connection string (account key or SAS) and all Azure token credentials +(Managed Identity, Service Principal, Workload Identity, Azure CLI). + +authors: Renizmy diff --git a/lib/vector-common/src/internal_event/metric_name.rs b/lib/vector-common/src/internal_event/metric_name.rs index ec052045ed7fd..028be5d2fb25e 100644 --- a/lib/vector-common/src/internal_event/metric_name.rs +++ b/lib/vector-common/src/internal_event/metric_name.rs @@ -26,6 +26,11 @@ pub enum CounterName { AggregateFailedUpdates, AggregateFlushesTotal, ApiStartedTotal, + AzureBlobEventIgnoredTotal, + AzureQueueMessageDeleteSucceededTotal, + AzureQueueMessageProcessingSucceededTotal, + AzureQueueMessageReceiveSucceededTotal, + AzureQueueMessageReceivedMessagesTotal, CheckpointsTotal, ChecksumErrorsTotal, CollectCompletedTotal, @@ -122,6 +127,8 @@ pub enum HistogramName { AdaptiveConcurrencyObservedRtt, AdaptiveConcurrencyPastRttMean, AdaptiveConcurrencyReachedLimit, + AzureBlobProcessingSucceededDurationSeconds, + AzureBlobProcessingFailedDurationSeconds, S3ObjectProcessingSucceededDurationSeconds, S3ObjectProcessingFailedDurationSeconds, CollectDurationSeconds, @@ -153,6 +160,12 @@ impl HistogramName { Self::AdaptiveConcurrencyObservedRtt => "adaptive_concurrency_observed_rtt", Self::AdaptiveConcurrencyPastRttMean => "adaptive_concurrency_past_rtt_mean", Self::AdaptiveConcurrencyReachedLimit => "adaptive_concurrency_reached_limit", + Self::AzureBlobProcessingSucceededDurationSeconds => { + "azure_blob_processing_succeeded_duration_seconds" + } + Self::AzureBlobProcessingFailedDurationSeconds => { + "azure_blob_processing_failed_duration_seconds" + } Self::S3ObjectProcessingSucceededDurationSeconds => { "s3_object_processing_succeeded_duration_seconds" } @@ -287,6 +300,19 @@ impl CounterName { Self::AggregateFailedUpdates => "aggregate_failed_updates", Self::AggregateFlushesTotal => "aggregate_flushes_total", Self::ApiStartedTotal => "api_started_total", + Self::AzureBlobEventIgnoredTotal => "azure_blob_event_ignored_total", + Self::AzureQueueMessageDeleteSucceededTotal => { + "azure_queue_message_delete_succeeded_total" + } + Self::AzureQueueMessageProcessingSucceededTotal => { + "azure_queue_message_processing_succeeded_total" + } + Self::AzureQueueMessageReceiveSucceededTotal => { + "azure_queue_message_receive_succeeded_total" + } + Self::AzureQueueMessageReceivedMessagesTotal => { + "azure_queue_message_received_messages_total" + } Self::CheckpointsTotal => "checkpoints_total", Self::ChecksumErrorsTotal => "checksum_errors_total", Self::CollectCompletedTotal => "collect_completed_total", diff --git a/src/internal_events/azure_blob.rs b/src/internal_events/azure_blob.rs new file mode 100644 index 0000000000000..b24896348e9d2 --- /dev/null +++ b/src/internal_events/azure_blob.rs @@ -0,0 +1,294 @@ +use std::time::Duration; + +use vector_lib::{ + NamedInternalEvent, counter, histogram, + internal_event::{CounterName, HistogramName, InternalEvent, error_stage, error_type}, +}; + +use crate::sources::azure_blob::queue::ProcessingError; + +/// Render an error together with its full `source` chain. +/// +/// `azure_core::Error` only renders its own context through `Display`, so the underlying +/// transport failure never reaches the log. Snafu variants already interpolate their source, so +/// a segment the accumulated text contains is skipped rather than repeated. +fn error_chain(error: &dyn std::error::Error) -> String { + let mut rendered = error.to_string(); + let mut next = error.source(); + while let Some(error) = next { + let segment = error.to_string(); + if !rendered.contains(&segment) { + rendered.push_str(": "); + rendered.push_str(&segment); + } + next = error.source(); + } + rendered +} + +#[derive(Debug, NamedInternalEvent)] +pub struct AzureBlobProcessingSucceeded<'a> { + pub container: &'a str, + pub duration: Duration, +} + +impl InternalEvent for AzureBlobProcessingSucceeded<'_> { + fn emit(self) { + debug!( + message = "Azure blob processing succeeded.", + container = %self.container, + duration_ms = %self.duration.as_millis(), + ); + histogram!( + HistogramName::AzureBlobProcessingSucceededDurationSeconds, + "container" => self.container.to_owned(), + ) + .record(self.duration); + } +} + +#[derive(Debug, NamedInternalEvent)] +pub struct AzureBlobProcessingFailed<'a> { + pub container: &'a str, + pub duration: Duration, +} + +impl InternalEvent for AzureBlobProcessingFailed<'_> { + fn emit(self) { + debug!( + message = "Azure blob processing failed.", + container = %self.container, + duration_ms = %self.duration.as_millis(), + ); + histogram!( + HistogramName::AzureBlobProcessingFailedDurationSeconds, + "container" => self.container.to_owned(), + ) + .record(self.duration); + } +} + +#[derive(Debug, NamedInternalEvent)] +pub struct AzureQueueMessageReceiveSucceeded { + pub count: usize, +} + +impl InternalEvent for AzureQueueMessageReceiveSucceeded { + fn emit(self) { + trace!(message = "Received Azure queue messages.", count = %self.count); + counter!(CounterName::AzureQueueMessageReceiveSucceededTotal).increment(1); + counter!(CounterName::AzureQueueMessageReceivedMessagesTotal).increment(self.count as u64); + } +} + +#[derive(Debug, NamedInternalEvent)] +pub struct AzureQueueMessageReceiveError<'a> { + pub error: &'a azure_core::Error, +} + +impl InternalEvent for AzureQueueMessageReceiveError<'_> { + fn emit(self) { + error!( + message = "Failed to fetch Azure queue messages.", + error = %error_chain(self.error), + error_code = "failed_fetching_azure_queue_messages", + error_type = error_type::REQUEST_FAILED, + stage = error_stage::RECEIVING, + ); + counter!( + CounterName::ComponentErrorsTotal, + "error_code" => "failed_fetching_azure_queue_messages", + "error_type" => error_type::REQUEST_FAILED, + "stage" => error_stage::RECEIVING, + ) + .increment(1); + } +} + +#[derive(Debug, NamedInternalEvent)] +pub struct AzureQueueMessageProcessingSucceeded<'a> { + pub message_id: &'a str, +} + +impl InternalEvent for AzureQueueMessageProcessingSucceeded<'_> { + fn emit(self) { + trace!(message = "Processed Azure queue message successfully.", message_id = %self.message_id); + counter!(CounterName::AzureQueueMessageProcessingSucceededTotal).increment(1); + } +} + +#[derive(Debug, NamedInternalEvent)] +pub struct AzureQueueMessageProcessingError<'a> { + pub message_id: &'a str, + pub error: &'a ProcessingError, + /// With no dead-letter queue, a growing dequeue count is the signal for a poison message. + pub dequeue_count: Option, +} + +const PROCESSING_ERROR_CODE: &str = "failed_processing_azure_queue_message"; + +impl InternalEvent for AzureQueueMessageProcessingError<'_> { + fn emit(self) { + error!( + message = "Failed to process Azure queue message.", + message_id = %self.message_id, + error = %error_chain(self.error), + dequeue_count = self.dequeue_count, + error_code = PROCESSING_ERROR_CODE, + error_type = self.error.error_type(), + stage = error_stage::PROCESSING, + ); + + // Spelled out per error kind rather than driven from `ProcessingError::error_type` + // because `cargo vdev check events` requires a literal `error_type::` constant here. + match self.error { + ProcessingError::InvalidQueueMessage { .. } + | ProcessingError::InvalidBlobPath { .. } => { + counter!( + CounterName::ComponentErrorsTotal, + "error_code" => PROCESSING_ERROR_CODE, + "error_type" => error_type::PARSER_FAILED, + "stage" => error_stage::PROCESSING, + ) + .increment(1); + } + ProcessingError::ContainerClient { .. } => { + counter!( + CounterName::ComponentErrorsTotal, + "error_code" => PROCESSING_ERROR_CODE, + "error_type" => error_type::CONFIGURATION_FAILED, + "stage" => error_stage::PROCESSING, + ) + .increment(1); + } + ProcessingError::GetBlob { .. } => { + counter!( + CounterName::ComponentErrorsTotal, + "error_code" => PROCESSING_ERROR_CODE, + "error_type" => error_type::REQUEST_FAILED, + "stage" => error_stage::PROCESSING, + ) + .increment(1); + } + ProcessingError::ReadBlob { .. } => { + counter!( + CounterName::ComponentErrorsTotal, + "error_code" => PROCESSING_ERROR_CODE, + "error_type" => error_type::READER_FAILED, + "stage" => error_stage::PROCESSING, + ) + .increment(1); + } + ProcessingError::PipelineSend { .. } => { + counter!( + CounterName::ComponentErrorsTotal, + "error_code" => PROCESSING_ERROR_CODE, + "error_type" => error_type::WRITER_FAILED, + "stage" => error_stage::PROCESSING, + ) + .increment(1); + } + ProcessingError::ErrorAcknowledgement { .. } => { + counter!( + CounterName::ComponentErrorsTotal, + "error_code" => PROCESSING_ERROR_CODE, + "error_type" => error_type::ACKNOWLEDGMENT_FAILED, + "stage" => error_stage::PROCESSING, + ) + .increment(1); + } + } + } +} + +#[derive(Debug, NamedInternalEvent)] +pub struct AzureQueueMessageDeleteSucceeded<'a> { + pub message_id: &'a str, +} + +impl InternalEvent for AzureQueueMessageDeleteSucceeded<'_> { + fn emit(self) { + trace!(message = "Deleted Azure queue message.", message_id = %self.message_id); + counter!(CounterName::AzureQueueMessageDeleteSucceededTotal).increment(1); + } +} + +#[derive(Debug, NamedInternalEvent)] +pub struct AzureQueueMessageDeleteError<'a> { + pub message_id: &'a str, + pub error: &'a azure_core::Error, +} + +impl InternalEvent for AzureQueueMessageDeleteError<'_> { + fn emit(self) { + error!( + message = "Deletion of Azure queue message failed.", + message_id = %self.message_id, + error = %error_chain(self.error), + error_code = "failed_deleting_azure_queue_message", + error_type = error_type::ACKNOWLEDGMENT_FAILED, + stage = error_stage::PROCESSING, + ); + counter!( + CounterName::ComponentErrorsTotal, + "error_code" => "failed_deleting_azure_queue_message", + "error_type" => error_type::ACKNOWLEDGMENT_FAILED, + "stage" => error_stage::PROCESSING, + ) + .increment(1); + } +} + +#[derive(Debug, NamedInternalEvent)] +pub struct AzureBlobEventIgnored<'a> { + pub event_type: &'a str, +} + +impl InternalEvent for AzureBlobEventIgnored<'_> { + fn emit(self) { + // Not a warning: an unfiltered Event Grid subscription delivers these as a matter of + // course, and `azure_blob_event_ignored_total` is the signal for operators. + debug!( + message = "Ignored queue message for an event that was not BlobCreated.", + event_type = %self.event_type, + ); + counter!( + CounterName::AzureBlobEventIgnoredTotal, + "event_type" => self.event_type.to_owned(), + ) + .increment(1); + } +} + +#[cfg(test)] +mod tests { + use azure_core::error::ErrorKind; + + use super::error_chain; + + #[test] + fn error_chain_appends_the_hidden_cause() { + let cause = std::io::Error::other("failed to look up address information"); + let error = azure_core::Error::with_error( + ErrorKind::Io, + cause, + "failed to execute `reqwest` request", + ); + + // `Display` alone stops at the outer context. + assert_eq!(error.to_string(), "failed to execute `reqwest` request"); + assert_eq!( + error_chain(&error), + "failed to execute `reqwest` request: failed to look up address information" + ); + } + + #[test] + fn error_chain_does_not_repeat_an_already_interpolated_source() { + let cause = std::io::Error::other("connection refused"); + let error = + azure_core::Error::with_error(ErrorKind::Io, cause, "outer: connection refused"); + + assert_eq!(error_chain(&error), "outer: connection refused"); + } +} diff --git a/src/internal_events/mod.rs b/src/internal_events/mod.rs index 035c909791a5e..9188cb6f6c4bc 100644 --- a/src/internal_events/mod.rs +++ b/src/internal_events/mod.rs @@ -27,6 +27,8 @@ mod aws_kinesis; mod aws_kinesis_firehose; #[cfg(any(feature = "sources-aws_s3", feature = "sources-aws_sqs",))] mod aws_sqs; +#[cfg(feature = "sources-azure_blob")] +mod azure_blob; mod batch; mod common; mod conditions; @@ -190,6 +192,8 @@ pub(crate) use self::aws_kinesis::*; pub(crate) use self::aws_kinesis_firehose::*; #[cfg(any(feature = "sources-aws_s3", feature = "sources-aws_sqs",))] pub(crate) use self::aws_sqs::*; +#[cfg(feature = "sources-azure_blob")] +pub(crate) use self::azure_blob::*; #[cfg(feature = "sources-datadog_agent")] pub(crate) use self::datadog_agent::*; #[cfg(feature = "sinks-datadog_logs")] diff --git a/src/sinks/mod.rs b/src/sinks/mod.rs index 95967c645aa16..f929b716e3717 100644 --- a/src/sinks/mod.rs +++ b/src/sinks/mod.rs @@ -26,7 +26,11 @@ pub mod aws_s_s; pub mod axiom; #[cfg(feature = "sinks-azure_blob")] pub mod azure_blob; -#[cfg(any(feature = "sinks-azure_blob", feature = "sinks-azure_logs_ingestion",))] +#[cfg(any( + feature = "sinks-azure_blob", + feature = "sinks-azure_logs_ingestion", + feature = "sources-azure_blob", +))] pub mod azure_common; #[cfg(feature = "sinks-azure_logs_ingestion")] pub mod azure_logs_ingestion; diff --git a/src/sources/azure_blob/integration_tests.rs b/src/sources/azure_blob/integration_tests.rs new file mode 100644 index 0000000000000..7fdfbd2177cf1 --- /dev/null +++ b/src/sources/azure_blob/integration_tests.rs @@ -0,0 +1,696 @@ +//! Integration tests for the `azure_blob` source, run against Azurite's blob and queue services. +//! Azurite does not run Event Grid, so these tests enqueue synthetic notifications themselves. + +use std::{ + num::{NonZeroU64, NonZeroUsize}, + time::Duration, +}; + +use azure_core::http::{RequestContent, StatusCode}; +use azure_storage_blob::models::BlockBlobClientUploadOptions; +use azure_storage_queue::{QueueClient, models::QueueMessage}; +use base64::prelude::{BASE64_STANDARD, Engine as _}; +use similar_asserts::assert_eq; +use vector_lib::{ + codecs::{JsonDeserializerConfig, decoding::DeserializerConfig}, + lookup::path, +}; +use vrl::value::Value; + +use tokio::time::Instant; + +use super::*; +use crate::{ + SourceSender, + config::{ComponentKey, ProxyConfig, SourceConfig, SourceContext}, + event::EventStatus::{self, *}, + line_agg, + sources::util::MultilineConfig, + test_util::{ + collect_n, + components::{SOURCE_TAGS, assert_source_compliance}, + lines_from_gzip_file, random_lines, trace_init, + }, +}; + +/// The notification wire formats an Event Grid subscription can deliver to a Storage Queue. +#[derive(Clone, Copy, Debug)] +enum NotificationFormat { + /// Event Grid schema, base64-encoded (the format Event Grid itself uses). + EventGridBase64, + /// Event Grid schema, raw JSON (manual or test messages). + EventGridRaw, + /// CloudEvents 1.0 schema, base64-encoded. + CloudEventsBase64, +} + +fn azurite_address() -> String { + std::env::var("AZURITE_ADDRESS").unwrap_or_else(|_| "localhost".into()) +} + +fn connection_string() -> String { + let address = azurite_address(); + format!( + "UseDevelopmentStorage=true;DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://{address}:10000/devstoreaccount1;QueueEndpoint=http://{address}:10001/devstoreaccount1;" + ) +} + +fn config( + queue_name: &str, + multiline: Option, + log_namespace: bool, + decoding: DeserializerConfig, +) -> AzureBlobConfig { + AzureBlobConfig { + connection_string: Some(connection_string().into()), + strategy: Strategy::StorageQueue, + compression: Compression::Auto, + multiline, + queue: Some(queue::Config { + queue_name: queue_name.to_string(), + poll_secs: 1, + // Deliberately short: the assertions below shut the source down and wait for the + // timeout to lapse so that any message left in the queue is visible to `peek_messages`. + visibility_timeout_secs: 2, + max_number_of_messages: 10, + // Serialized on purpose. `collect_n` drops the receiver as soon as it has its events, + // so a concurrent poller can re-send into a closed pipeline and leave the message + // undeleted, failing the queue-depth assertions. + client_concurrency: Some(NonZeroUsize::new(1).expect("nonzero")), + ..Default::default() + }), + acknowledgements: true.into(), + log_namespace: Some(log_namespace), + decoding, + ..Default::default() + } +} + +async fn test_clients( + config: &AzureBlobConfig, + queue_name: &str, +) -> (BlobContainerClient, QueueClient, String) { + let container_name = uuid::Uuid::new_v4().to_string(); + let clients = config + .create_client_source(&ProxyConfig::default()) + .await + .expect("Failed to build client source"); + + let container_client = clients + .container_client(&container_name) + .expect("Failed to build container client"); + match container_client.create(None).await { + Ok(_) => {} + Err(error) if error.http_status() == Some(StatusCode::Conflict) => {} + Err(error) => panic!("Failed to create container: {error}"), + } + + let queue_client = clients + .queue_client(queue_name) + .expect("Failed to build queue client"); + match queue_client.create(None).await { + Ok(_) => {} + Err(error) if error.http_status() == Some(StatusCode::Conflict) => {} + Err(error) => panic!("Failed to create queue: {error}"), + } + + (container_client, queue_client, container_name) +} + +async fn upload_blob( + container_client: &BlobContainerClient, + blob_name: &str, + payload: Vec, + content_type: Option<&str>, + content_encoding: Option<&str>, +) { + let options = BlockBlobClientUploadOptions { + blob_content_type: content_type.map(ToOwned::to_owned), + blob_content_encoding: content_encoding.map(ToOwned::to_owned), + // Force a single-shot PutBlob. The SDK otherwise splits anything over 4 MiB into + // PutBlock/PutBlockList, which Azurite rejects under Shared Key auth. + partition_size: Some(NonZeroU64::new(64 * 1024 * 1024).expect("nonzero")), + ..Default::default() + }; + container_client + .blob_client(blob_name) + .upload(RequestContent::from(payload), Some(options)) + .await + .expect("Failed to upload blob"); +} + +fn notification_body(container: &str, blob: &str, format: NotificationFormat) -> String { + let address = azurite_address(); + let url = format!("http://{address}:10000/devstoreaccount1/{container}/{blob}"); + let subject = format!("/blobServices/default/containers/{container}/blobs/{blob}"); + + let json = match format { + NotificationFormat::EventGridBase64 | NotificationFormat::EventGridRaw => format!( + r#"{{ + "topic": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/rg/providers/Microsoft.Storage/storageAccounts/devstoreaccount1", + "subject": "{subject}", + "eventType": "Microsoft.Storage.BlobCreated", + "eventTime": "2026-06-01T12:00:00.000Z", + "id": "00000000-0000-0000-0000-000000000000", + "data": {{ + "api": "PutBlob", + "blobType": "BlockBlob", + "url": "{url}", + "eTag": "0x8DC0000000000000" + }}, + "dataVersion": "", + "metadataVersion": "1" + }}"# + ), + NotificationFormat::CloudEventsBase64 => format!( + r#"{{ + "specversion": "1.0", + "type": "Microsoft.Storage.BlobCreated", + "source": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/rg/providers/Microsoft.Storage/storageAccounts/devstoreaccount1", + "subject": "{subject}", + "time": "2026-06-01T12:00:00.000Z", + "id": "00000000-0000-0000-0000-000000000000", + "data": {{ + "api": "PutBlob", + "blobType": "BlockBlob", + "url": "{url}", + "eTag": "0x8DC0000000000000" + }} + }}"# + ), + }; + + match format { + NotificationFormat::EventGridRaw => json, + NotificationFormat::EventGridBase64 | NotificationFormat::CloudEventsBase64 => { + BASE64_STANDARD.encode(json) + } + } +} + +async fn enqueue_notification(queue_client: &QueueClient, body: String) { + let message = QueueMessage { + message_text: Some(body), + }; + queue_client + .send_message( + message.try_into().expect("Failed to encode queue message"), + None, + ) + .await + .expect("Failed to enqueue notification"); +} + +/// Count visible messages without altering their visibility. +async fn count_messages(queue_client: &QueueClient) -> usize { + queue_client + .peek_messages(None) + .await + .expect("Failed to peek messages") + .into_model() + .expect("Failed to decode peeked messages") + .items + .map(|items| items.len()) + .unwrap_or(0) +} + +#[allow(clippy::too_many_arguments)] +async fn test_event( + blob_name: Option, + content_encoding: Option<&str>, + content_type: Option<&str>, + multiline: Option, + payload: Vec, + expected_lines: Vec, + status: EventStatus, + log_namespace: bool, + decoding: DeserializerConfig, + format: NotificationFormat, + delete_failed_message: bool, +) { + assert_source_compliance(&SOURCE_TAGS, async move { + let blob_name = blob_name.unwrap_or_else(|| uuid::Uuid::new_v4().to_string()); + let queue_name = uuid::Uuid::new_v4().to_string(); + + let mut config = config(&queue_name, multiline, log_namespace, decoding); + config.queue.as_mut().unwrap().delete_failed_message = delete_failed_message; + + let (container_client, queue_client, container_name) = + test_clients(&config, &queue_name).await; + + upload_blob( + &container_client, + &blob_name, + payload, + content_type, + content_encoding, + ) + .await; + + enqueue_notification( + &queue_client, + notification_body(&container_name, &blob_name, format), + ) + .await; + + let (tx, rx) = SourceSender::new_test_finalize(status); + let key = ComponentKey::from("azure_blob_test"); + let (cx, shutdown) = SourceContext::new_shutdown(&key, tx); + let namespace = cx.log_namespace(Some(log_namespace)); + let source = config.build(cx).await.unwrap(); + tokio::spawn(async move { source.await.unwrap() }); + + let events = collect_n(rx, expected_lines.len()).await; + + assert_eq!(expected_lines.len(), events.len()); + for (i, event) in events.iter().enumerate() { + if let Some(schema_definition) = + config.outputs(namespace).pop().unwrap().schema_definition + { + schema_definition.is_valid_for_event(event).unwrap(); + } + + let message = expected_lines[i].as_str(); + + let log = event.as_log(); + if log_namespace { + assert_eq!(log.value(), &Value::from(message)); + } else { + assert_eq!(log["message"], message.into()); + } + assert_eq!( + namespace + .get_source_metadata( + AzureBlobConfig::NAME, + log, + path!("container"), + path!("container") + ) + .unwrap(), + &container_name.clone().into() + ); + assert_eq!( + namespace + .get_source_metadata(AzureBlobConfig::NAME, log, path!("blob"), path!("blob")) + .unwrap(), + &blob_name.clone().into() + ); + assert_eq!( + namespace + .get_source_metadata( + AzureBlobConfig::NAME, + log, + path!("storage_account"), + path!("storage_account") + ) + .unwrap(), + &"devstoreaccount1".into() + ); + } + + tokio::time::sleep(Duration::from_secs(5)).await; + + shutdown + .shutdown_all(Some(Instant::now() + Duration::from_secs(3))) + .await; + tokio::time::sleep(Duration::from_secs(5)).await; + match status { + Errored => { + assert_eq!(count_messages(&queue_client).await, 1); + } + Rejected if !delete_failed_message => { + assert_eq!(count_messages(&queue_client).await, 1); + } + _ => { + assert_eq!(count_messages(&queue_client).await, 0); + } + }; + }) + .await; +} + +#[tokio::test] +async fn azure_blob_process_message() { + trace_init(); + + let logs: Vec = random_lines(100).take(10).collect(); + + test_event( + None, + None, + None, + None, + logs.join("\n").into_bytes(), + logs, + Delivered, + false, + DeserializerConfig::Bytes, + NotificationFormat::EventGridBase64, + true, + ) + .await; +} + +#[tokio::test] +async fn azure_blob_process_message_raw_json_notification() { + trace_init(); + + let logs: Vec = random_lines(100).take(10).collect(); + + test_event( + None, + None, + None, + None, + logs.join("\n").into_bytes(), + logs, + Delivered, + false, + DeserializerConfig::Bytes, + NotificationFormat::EventGridRaw, + true, + ) + .await; +} + +#[tokio::test] +async fn azure_blob_process_message_cloud_events_notification() { + trace_init(); + + let logs: Vec = random_lines(100).take(10).collect(); + + test_event( + None, + None, + None, + None, + logs.join("\n").into_bytes(), + logs, + Delivered, + false, + DeserializerConfig::Bytes, + NotificationFormat::CloudEventsBase64, + true, + ) + .await; +} + +#[tokio::test] +async fn azure_blob_process_json_message() { + trace_init(); + + let logs: Vec = random_lines(100).take(10).collect(); + + let json_logs: Vec = logs + .iter() + .map(|msg| format!(r#"{{"message": "{msg}"}}"#)) + .collect(); + + test_event( + None, + None, + None, + None, + json_logs.join("\n").into_bytes(), + logs, + Delivered, + false, + DeserializerConfig::Json(JsonDeserializerConfig::default()), + NotificationFormat::EventGridBase64, + true, + ) + .await; +} + +#[tokio::test] +async fn azure_blob_process_message_with_log_namespace() { + trace_init(); + + let logs: Vec = random_lines(100).take(10).collect(); + + test_event( + None, + None, + None, + None, + logs.join("\n").into_bytes(), + logs, + Delivered, + true, + DeserializerConfig::Bytes, + NotificationFormat::EventGridBase64, + true, + ) + .await; +} + +#[tokio::test] +async fn azure_blob_process_message_special_characters() { + trace_init(); + + let blob_name = format!("special blob {}", uuid::Uuid::new_v4()); + let logs: Vec = random_lines(100).take(10).collect(); + + test_event( + Some(blob_name), + None, + None, + None, + logs.join("\n").into_bytes(), + logs, + Delivered, + false, + DeserializerConfig::Bytes, + NotificationFormat::EventGridBase64, + true, + ) + .await; +} + +#[tokio::test] +async fn azure_blob_process_message_larger_than_partition_size() { + trace_init(); + + // Larger than the SDK's 4 MiB `DEFAULT_DOWNLOAD_PARTITION_SIZE`, so `BlobClient::download` + // takes its partitioned path and spawns tasks. Without the `azure_core/tokio` feature those + // run on plain threads and panic on the connect timeout. Every other test here uses a ~1 KB + // blob, which fits one partition and never spawns. + let logs: Vec = random_lines(100).take(60_000).collect(); + + test_event( + None, + None, + None, + None, + logs.join("\n").into_bytes(), + logs, + Delivered, + false, + DeserializerConfig::Bytes, + NotificationFormat::EventGridBase64, + true, + ) + .await; +} + +#[tokio::test] +async fn azure_blob_process_message_gzip() { + use std::io::Read; + + trace_init(); + + let logs: Vec = random_lines(100).take(10).collect(); + + let mut gz = flate2::read::GzEncoder::new( + std::io::Cursor::new(logs.join("\n").into_bytes()), + flate2::Compression::fast(), + ); + let mut buffer = Vec::new(); + gz.read_to_end(&mut buffer).unwrap(); + + test_event( + None, + Some("gzip"), + None, + None, + buffer, + logs, + Delivered, + false, + DeserializerConfig::Bytes, + NotificationFormat::EventGridBase64, + true, + ) + .await; +} + +#[tokio::test] +async fn azure_blob_process_message_multipart_gzip() { + use std::io::Read; + + trace_init(); + + let logs = lines_from_gzip_file("tests/data/multipart-gzip.log.gz"); + + let buffer = { + let mut file = + std::fs::File::open("tests/data/multipart-gzip.log.gz").expect("file can be opened"); + let mut data = Vec::new(); + file.read_to_end(&mut data).expect("file can be read"); + data + }; + + test_event( + None, + Some("gzip"), + None, + None, + buffer, + logs, + Delivered, + false, + DeserializerConfig::Bytes, + NotificationFormat::EventGridBase64, + true, + ) + .await; +} + +#[tokio::test] +async fn azure_blob_process_message_multipart_zstd() { + use std::io::{BufRead, BufReader, Read}; + + trace_init(); + + let logs: Vec = { + let file = std::fs::File::open("tests/data/multipart-zst.log").expect("file can be opened"); + BufReader::new(file).lines().map(|x| x.unwrap()).collect() + }; + + let buffer = { + let mut file = + std::fs::File::open("tests/data/multipart-zst.log.zst").expect("file can be opened"); + let mut data = Vec::new(); + file.read_to_end(&mut data).expect("file can be read"); + data + }; + + test_event( + None, + Some("zstd"), + None, + None, + buffer, + logs, + Delivered, + false, + DeserializerConfig::Bytes, + NotificationFormat::EventGridBase64, + true, + ) + .await; +} + +#[tokio::test] +async fn azure_blob_process_message_multiline() { + trace_init(); + + let logs: Vec = vec!["abc", "def", "geh"] + .into_iter() + .map(ToOwned::to_owned) + .collect(); + + test_event( + None, + None, + None, + Some(MultilineConfig { + start_pattern: "abc".to_owned(), + mode: line_agg::Mode::HaltWith, + condition_pattern: "geh".to_owned(), + timeout_ms: Duration::from_millis(1000), + }), + logs.join("\n").into_bytes(), + vec!["abc\ndef\ngeh".to_owned()], + Delivered, + false, + DeserializerConfig::Bytes, + NotificationFormat::EventGridBase64, + true, + ) + .await; +} + +#[tokio::test] +async fn azure_blob_handles_failed_status() { + trace_init(); + + let logs: Vec = random_lines(100).take(10).collect(); + + test_event( + None, + None, + None, + None, + logs.join("\n").into_bytes(), + logs, + Rejected, + false, + DeserializerConfig::Bytes, + NotificationFormat::EventGridBase64, + true, + ) + .await; +} + +#[tokio::test] +async fn azure_blob_handles_failed_status_without_deletion() { + trace_init(); + + let logs: Vec = random_lines(100).take(10).collect(); + + test_event( + None, + None, + None, + None, + logs.join("\n").into_bytes(), + logs, + Rejected, + false, + DeserializerConfig::Bytes, + NotificationFormat::EventGridBase64, + false, + ) + .await; +} + +#[tokio::test] +async fn azure_blob_ignores_other_event_types() { + trace_init(); + + let queue_name = uuid::Uuid::new_v4().to_string(); + let config = config(&queue_name, None, false, DeserializerConfig::Bytes); + let (_container_client, queue_client, container_name) = + test_clients(&config, &queue_name).await; + + let body = notification_body( + &container_name, + "some.log", + NotificationFormat::EventGridRaw, + ) + .replace( + "Microsoft.Storage.BlobCreated", + "Microsoft.Storage.BlobDeleted", + ); + enqueue_notification(&queue_client, BASE64_STANDARD.encode(body)).await; + + let (tx, _rx) = SourceSender::new_test_finalize(Delivered); + let cx = SourceContext::new_test(tx, None); + let source = config.build(cx).await.unwrap(); + tokio::spawn(async move { source.await.unwrap() }); + + // The ignored message must be deleted from the queue, not redelivered. + tokio::time::sleep(Duration::from_secs(10)).await; + assert_eq!(count_messages(&queue_client).await, 0); +} diff --git a/src/sources/azure_blob/mod.rs b/src/sources/azure_blob/mod.rs new file mode 100644 index 0000000000000..ffe22895fdb54 --- /dev/null +++ b/src/sources/azure_blob/mod.rs @@ -0,0 +1,1259 @@ +use std::{ + collections::HashMap, + fs::File, + io::Read, + sync::{Arc, RwLock}, + time::Duration, +}; + +use async_compression::tokio::bufread; +use azure_core::{ + credentials::TokenCredential, + http::{ + AsyncResponseBody, Context, HttpClient, Request, Transport, Url, + policies::{Policy, PolicyResult}, + }, +}; +use azure_storage_blob::{BlobContainerClient, BlobContainerClientOptions}; +use azure_storage_queue::{QueueClient, QueueClientOptions}; +use futures::{StreamExt, TryStreamExt, stream}; +use snafu::Snafu; +use tokio_util::io::StreamReader; +use vector_common::compression::gzip_multiple_decoder; +use vector_lib::{ + codecs::{ + NewlineDelimitedDecoderConfig, + decoding::{ + DeserializerConfig, FramingConfig, NewlineDelimitedDecoderOptions, OversizedAction, + }, + }, + config::{LegacyKey, LogNamespace}, + configurable::configurable_component, + lookup::owned_value_path, + sensitive_string::SensitiveString, +}; +use vrl::value::{Kind, kind::Collection}; + +use super::util::MultilineConfig; +use crate::{ + codecs::DecodingConfig, + config::{ + ProxyConfig, SourceAcknowledgementsConfig, SourceConfig, SourceContext, SourceOutput, + }, + line_agg, + serde::{bool_or_struct, default_decoding}, + sinks::azure_common::{ + config::{AzureAuthentication, AzureBlobTlsConfig}, + connection_string::{Auth, ParsedConnectionString}, + shared_key_policy::SharedKeyAuthorizationPolicy, + }, +}; + +#[cfg(all(test, feature = "azure-blob-integration-tests"))] +mod integration_tests; +pub mod queue; + +/// The storage service version sent with Shared Key signed requests. Must be a version +/// supported by Azurite for the integration tests to pass. +const STORAGE_SERVICE_VERSION: &str = "2025-11-05"; + +/// Connection timeout for the custom transport, matching the Azure SDK default. Requires the +/// `azure_core/tokio` feature: without it the SDK spawns partitioned downloads (blobs over +/// 4 MiB) onto plain threads, where building the timer panics. +const AZURE_CONNECT_TIMEOUT: Duration = Duration::from_secs(20); + +/// Azurite's queue service port. The blob service uses 10000 and the table service 10002. +const DEV_STORAGE_QUEUE_PORT: u16 = 10001; + +/// Compression scheme for blobs retrieved from Azure Blob Storage. +#[configurable_component] +#[configurable(metadata(docs::advanced))] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum Compression { + /// Automatically attempt to determine the compression scheme. + /// + /// The compression scheme of the blob is determined from its `Content-Encoding` and + /// `Content-Type` metadata, as well as the blob name suffix (for example, `.gz`). + /// + /// It is set to `none` if the compression scheme cannot be determined. + #[default] + Auto, + + /// Uncompressed. + None, + + /// GZIP. + Gzip, + + /// ZSTD. + Zstd, +} + +/// Strategies for consuming blobs from Azure Blob Storage. +#[configurable_component] +#[derive(Clone, Copy, Debug, Default)] +#[serde(rename_all = "snake_case")] +enum Strategy { + /// Consumes blobs by processing `Microsoft.Storage.BlobCreated` notifications delivered by an + /// Event Grid subscription to an [Azure Storage Queue][azure_queue]. + /// + /// [azure_queue]: https://learn.microsoft.com/azure/storage/queues/storage-queues-introduction + #[default] + StorageQueue, +} + +/// Configuration for the `azure_blob` source. +#[configurable_component(source("azure_blob", "Collect logs from Azure Blob Storage."))] +#[derive(Clone, Debug, Derivative)] +#[derivative(Default)] +#[serde(default, deny_unknown_fields)] +pub struct AzureBlobConfig { + /// The Azure Blob Storage Account connection string. + /// + /// Authentication with an access key or shared access signature (SAS) are supported + /// authentication methods. The connection string is also used to derive the blob and + /// queue service endpoints. + #[configurable(metadata( + docs::warnings = "Access keys and SAS tokens can be used to gain unauthorized access to Azure Storage \ + resources. Numerous security breaches have occurred due to leaked connection strings. It is important to keep \ + connection strings secure and not expose them in logs, error messages, or version control systems." + ))] + #[configurable(metadata( + docs::examples = "DefaultEndpointsProtocol=https;AccountName=mylogstorage;AccountKey=storageaccountkeybase64encoded;EndpointSuffix=core.windows.net" + ))] + #[configurable(metadata( + docs::examples = "BlobEndpoint=https://mylogstorage.blob.core.windows.net/;QueueEndpoint=https://mylogstorage.queue.core.windows.net/;SharedAccessSignature=generatedsastoken" + ))] + #[configurable(metadata(docs::examples = "AccountName=mylogstorage"))] + connection_string: Option, + + /// The Azure Blob Storage Account name. + /// + /// If provided, this is used instead of the `connection_string` and requires `auth` to be + /// configured. Both the blob and queue service endpoints are derived from the account name. + #[configurable(metadata(docs::examples = "mylogstorage"))] + account_name: Option, + + /// The Azure Blob Storage service endpoint. + /// + /// Useful for Azurite, sovereign clouds, or private endpoints. Requires `auth` to be + /// configured, and `queue_endpoint` to be provided as well when `account_name` is not set. + #[configurable(metadata(docs::examples = "https://mylogstorage.blob.core.windows.net/"))] + blob_endpoint: Option, + + /// The Azure Queue Storage service endpoint. + /// + /// By default the queue endpoint is derived from `account_name` or the connection string. + #[configurable(metadata(docs::examples = "https://mylogstorage.queue.core.windows.net/"))] + queue_endpoint: Option, + + #[configurable(derived)] + #[serde(default)] + auth: Option, + + /// Configuration options for the Storage Queue. + queue: Option, + + /// The compression scheme used for decompressing blobs retrieved from Azure Blob Storage. + compression: Compression, + + /// The strategy to use to consume blobs from Azure Blob Storage. + #[configurable(metadata(docs::hidden))] + strategy: Strategy, + + /// Multiline aggregation configuration. + /// + /// If not specified, multiline aggregation is disabled. + #[configurable(derived)] + multiline: Option, + + #[configurable(derived)] + #[serde(default, deserialize_with = "bool_or_struct")] + acknowledgements: SourceAcknowledgementsConfig, + + /// The namespace to use for logs. This overrides the global setting. + #[configurable(metadata(docs::hidden))] + #[serde(default)] + log_namespace: Option, + + #[configurable(derived)] + #[serde(default = "default_framing")] + #[derivative(Default(value = "default_framing()"))] + pub framing: FramingConfig, + + #[configurable(derived)] + #[serde(default = "default_decoding")] + #[derivative(Default(value = "default_decoding()"))] + pub decoding: DeserializerConfig, + + #[configurable(derived)] + tls: Option, +} + +const fn default_framing() -> FramingConfig { + // This mirrors the `aws_s3` source's historical default. + FramingConfig::NewlineDelimited(NewlineDelimitedDecoderConfig { + newline_delimited: NewlineDelimitedDecoderOptions { + max_length: None, + oversized_action: OversizedAction::Drop, + }, + }) +} + +impl_generate_config_from_default!(AzureBlobConfig); + +#[async_trait::async_trait] +#[typetag::serde(name = "azure_blob")] +impl SourceConfig for AzureBlobConfig { + async fn build(&self, cx: SourceContext) -> crate::Result { + let log_namespace = cx.log_namespace(self.log_namespace); + + let multiline_config: Option = self + .multiline + .as_ref() + .map(|config| config.try_into()) + .transpose()?; + + match self.strategy { + Strategy::StorageQueue => Ok(Box::pin( + self.create_queue_ingestor(multiline_config, &cx.proxy, log_namespace) + .await? + .run(cx, self.acknowledgements, log_namespace), + )), + } + } + + fn outputs(&self, global_log_namespace: LogNamespace) -> Vec { + let log_namespace = global_log_namespace.merge(self.log_namespace); + let mut schema_definition = self + .decoding + .schema_definition(log_namespace) + .with_source_metadata( + Self::NAME, + Some(LegacyKey::Overwrite(owned_value_path!("container"))), + &owned_value_path!("container"), + Kind::bytes(), + None, + ) + .with_source_metadata( + Self::NAME, + Some(LegacyKey::Overwrite(owned_value_path!("blob"))), + &owned_value_path!("blob"), + Kind::bytes(), + None, + ) + .with_source_metadata( + Self::NAME, + Some(LegacyKey::Overwrite(owned_value_path!("storage_account"))), + &owned_value_path!("storage_account"), + Kind::bytes().or_undefined(), + None, + ) + .with_source_metadata( + Self::NAME, + None, + &owned_value_path!("timestamp"), + Kind::timestamp(), + Some("timestamp"), + ) + .with_standard_vector_source_metadata() + // for metadata that is added to the events dynamically from the blob metadata + .with_source_metadata( + Self::NAME, + None, + &owned_value_path!("metadata"), + Kind::object(Collection::empty().with_unknown(Kind::bytes())).or_undefined(), + None, + ); + + // for metadata that is added to the events dynamically from the blob metadata + if log_namespace == LogNamespace::Legacy { + schema_definition = schema_definition.unknown_fields(Kind::bytes()); + } + + vec![SourceOutput::new_maybe_logs( + self.decoding.output_type(), + schema_definition, + )] + } + + fn can_acknowledge(&self) -> bool { + true + } +} + +impl AzureBlobConfig { + async fn create_client_source( + &self, + proxy: &ProxyConfig, + ) -> crate::Result { + let connection_string: String = match ( + &self.connection_string, + &self.account_name, + &self.blob_endpoint, + ) { + (Some(connstr), None, None) => connstr.inner().into(), + (None, Some(account_name), None) => { + if self.auth.is_none() { + return Err( + "`auth` configuration must be provided when using `account_name`".into(), + ); + } + format!("AccountName={account_name}") + } + (None, None, Some(blob_endpoint)) => { + if self.auth.is_none() { + return Err( + "`auth` configuration must be provided when using `blob_endpoint`".into(), + ); + } + if self.queue_endpoint.is_none() { + return Err("`queue_endpoint` must be provided when using `blob_endpoint` without `account_name`".into()); + } + // BlobEndpoint must always end in a trailing slash + let blob_endpoint = if blob_endpoint.ends_with('/') { + blob_endpoint.clone() + } else { + format!("{blob_endpoint}/") + }; + format!("BlobEndpoint={blob_endpoint}") + } + (None, None, None) => { + return Err("One of `connection_string`, `account_name`, or `blob_endpoint` must be provided".into()); + } + (Some(_), Some(_), _) => { + return Err("Cannot provide both `connection_string` and `account_name`".into()); + } + (Some(_), _, Some(_)) => { + return Err("Cannot provide both `connection_string` and `blob_endpoint`".into()); + } + (_, Some(_), Some(_)) => { + return Err("Cannot provide both `account_name` and `blob_endpoint`".into()); + } + }; + + AzureStorageClientSource::new( + connection_string, + self.queue_endpoint.clone(), + self.auth.clone(), + proxy.clone(), + self.tls.clone(), + ) + .await + } + + async fn create_queue_ingestor( + &self, + multiline: Option, + proxy: &ProxyConfig, + log_namespace: LogNamespace, + ) -> crate::Result { + let clients = self.create_client_source(proxy).await?; + + let decoder = + DecodingConfig::new(self.framing.clone(), self.decoding.clone(), log_namespace) + .build()?; + + match self.queue { + Some(ref queue) => { + let ingestor = queue::Ingestor::new( + clients, + queue.clone(), + self.compression, + multiline, + decoder, + )?; + + Ok(ingestor) + } + None => Err(CreateQueueIngestorError::ConfigMissing {}.into()), + } + } +} + +#[derive(Debug, Snafu)] +enum CreateQueueIngestorError { + #[snafu(display("Configuration for `queue` required when strategy=storage_queue"))] + ConfigMissing, +} + +/// Shared factory for Azure Storage clients, mirroring the sink's `build_client` +/// (`src/sinks/azure_blob/config.rs`) so the same configuration works for both. +pub struct AzureStorageClientSource { + raw_connection_string: String, + parsed: ParsedConnectionString, + queue_endpoint: Option, + credential: Option>, + shared_key: Option<(String, String)>, + proxy: ProxyConfig, + /// The custom root certificate, read once at startup rather than per client build. + ca_pem: Option>, + /// HTTP clients, cached per host. See `build_transport`. + http_clients: RwLock>>, +} + +impl AzureStorageClientSource { + async fn new( + connection_string: String, + queue_endpoint: Option, + auth: Option, + proxy: ProxyConfig, + tls: Option, + ) -> crate::Result { + let parsed = ParsedConnectionString::parse(&connection_string) + .map_err(|e| format!("Invalid connection string: {e}"))?; + + let mut credential: Option> = None; + let mut shared_key: Option<(String, String)> = None; + + match (parsed.auth(), &auth) { + (Auth::None, None) => { + warn!("No authentication method provided, requests will be anonymous."); + } + (Auth::Sas { .. }, None) => { + info!("Using SAS token authentication."); + } + ( + Auth::SharedKey { + account_name, + account_key, + }, + None, + ) => { + info!("Using Shared Key authentication."); + shared_key = Some((account_name, account_key)); + } + (Auth::None, Some(auth_config)) => { + info!("Using Azure Authentication method."); + let credential_result = auth_config + .credential() + .await + .map_err(|e| format!("Failed to configure Azure Authentication: {e}"))?; + credential = Some(credential_result); + } + (Auth::Sas { .. }, Some(_)) => { + return Err( + "Cannot use both SAS token and another Azure Authentication method at the same time".into(), + ); + } + (Auth::SharedKey { .. }, Some(_)) => { + return Err( + "Cannot use both Shared Key and another Azure Authentication method at the same time".into(), + ); + } + } + + let ca_pem = match &tls { + Some(AzureBlobTlsConfig { + ca_file: Some(ca_file), + }) => { + let mut buf = Vec::new(); + File::open(ca_file) + .map_err(|e| format!("Failed to open TLS CA file {}: {e}", ca_file.display()))? + .read_to_end(&mut buf) + .map_err(|e| { + format!("Failed to read TLS CA file {}: {e}", ca_file.display()) + })?; + // Parse eagerly so a malformed certificate fails at startup, not on first request. + reqwest_13::Certificate::from_pem(&buf) + .map_err(|e| format!("Invalid TLS CA file {}: {e}", ca_file.display()))?; + info!("Adding TLS root certificate from {}.", ca_file.display()); + Some(buf) + } + _ => None, + }; + + Ok(Self { + raw_connection_string: connection_string, + parsed, + queue_endpoint, + credential, + shared_key, + proxy, + ca_pem, + http_clients: RwLock::new(HashMap::new()), + }) + } + + /// Resolution order, mirroring `ParsedConnectionString::blob_account_endpoint`: + /// 1. The explicit `queue_endpoint` configuration option. + /// 2. A `QueueEndpoint` key in the connection string, which `ParsedConnectionString` + /// ignores as an unknown key. + /// 3. Development storage: `{proto}://127.0.0.1:10001/{account}`. + /// 4. Public cloud: `{proto}://{account}.queue.{endpoint_suffix}`. + fn queue_account_endpoint(&self) -> crate::Result { + if let Some(explicit) = self.queue_endpoint.as_ref() { + return Ok(explicit.clone()); + } + + if let Some(from_cs) = connection_string_value(&self.raw_connection_string, "QueueEndpoint") + { + return Ok(from_cs); + } + + let proto = self.parsed.default_protocol(); + + let account_name = self.parsed.account_name.as_ref().ok_or( + "Could not determine Queue endpoint: `queue_endpoint` or an account name is required", + )?; + + if self.parsed.use_development_storage { + let base = match self.parsed.development_storage_proxy_uri.as_deref() { + Some(proxy_uri) => dev_storage_queue_base(proxy_uri, &proto), + None => format!("{proto}://127.0.0.1:{DEV_STORAGE_QUEUE_PORT}"), + }; + return Ok(format!("{base}/{account_name}")); + } + + let suffix = self.parsed.endpoint_suffix(); + Ok(format!("{proto}://{account_name}.queue.{suffix}")) + } + + /// The endpoint may already carry a query string (such as a SAS embedded in an explicit + /// `queue_endpoint`), so the queue name is inserted into the path rather than appended. + fn queue_url(&self, queue_name: &str) -> crate::Result { + let base = self.queue_account_endpoint()?; + let (base_path, base_query) = match base.split_once('?') { + Some((path, query)) => (path, Some(query)), + None => (base.as_str(), None), + }; + let url = format!("{}/{queue_name}", base_path.trim_end_matches('/')); + let url = append_query_segment(&url, base_query); + Ok(append_query_segment( + &url, + self.parsed.shared_access_signature.as_deref(), + )) + } + + /// Clients are cached per host because the `no_proxy` bypass is host-specific but a fresh + /// connection pool and TLS root store are expensive to rebuild. `container_client` is called + /// lazily from the async ingestion path, so without the cache every first-sight container + /// would build one on a runtime worker. + fn build_transport(&self, url: &Url) -> crate::Result { + // Keyed on exactly what `build_http_client`'s proxy decision reads, so a cache hit can + // never hand back a client built for a different bypass outcome. + let key = format!( + "{}://{}:{}", + url.scheme(), + url.host_str().unwrap_or_default(), + url.port().map(|port| port.to_string()).unwrap_or_default(), + ); + + if let Some(client) = self.http_clients.read().expect("lock poisoned").get(&key) { + return Ok(Transport::new(Arc::clone(client))); + } + + let client: Arc = Arc::new(self.build_http_client(url)?); + + let mut clients = self.http_clients.write().expect("lock poisoned"); + let entry = clients.entry(key).or_insert(client); + Ok(Transport::new(Arc::clone(entry))) + } + + /// Construct a reqwest client, mirroring the sink's `build_client`: global proxy configuration + /// (with per-host no-proxy bypass) plus an optional custom CA certificate. + fn build_http_client(&self, url: &Url) -> crate::Result { + // Installing a transport skips the SDK's own `automatic_decompression: false` default. + // Ask for the bytes exactly as stored: the source decompresses blob bodies itself from + // `Content-Encoding`, and `BlobClient::download` documents that transparent + // decompression can break partitioned downloads. + let mut default_headers = reqwest_13::header::HeaderMap::new(); + default_headers.insert( + reqwest_13::header::ACCEPT_ENCODING, + reqwest_13::header::HeaderValue::from_static("identity"), + ); + + let mut reqwest_builder = reqwest_13::ClientBuilder::new() + .connect_timeout(AZURE_CONNECT_TIMEOUT) + .default_headers(default_headers) + .redirect(reqwest_13::redirect::Policy::none()); + let bypass_proxy = { + let host = url.host_str().unwrap_or(""); + let port = url.port(); + self.proxy.no_proxy.matches(host) + || port + .map(|p| self.proxy.no_proxy.matches(&format!("{host}:{p}"))) + .unwrap_or(false) + }; + if bypass_proxy || !self.proxy.enabled { + // Ensure no proxy (and disable any potential system proxy auto-detection) + reqwest_builder = reqwest_builder.no_proxy(); + } else { + if let Some(http) = &self.proxy.http { + let p = reqwest_13::Proxy::http(http) + .map_err(|e| format!("Invalid HTTP proxy URL: {e}"))?; + reqwest_builder = reqwest_builder.proxy(p); + } + if let Some(https) = &self.proxy.https { + let p = reqwest_13::Proxy::https(https) + .map_err(|e| format!("Invalid HTTPS proxy URL: {e}"))?; + reqwest_builder = reqwest_builder.proxy(p); + } + } + + if let Some(ca_pem) = &self.ca_pem { + // Already validated in `new`, so this cannot fail in practice. + let cert = reqwest_13::Certificate::from_pem(ca_pem) + .map_err(|e| format!("Invalid TLS root certificate: {e}"))?; + reqwest_builder = reqwest_builder.add_root_certificate(cert); + } + + reqwest_builder + .build() + .map_err(|e| format!("Failed to build reqwest client: {e}").into()) + } + + fn shared_key_policy(&self) -> crate::Result>> { + self.shared_key + .as_ref() + .map(|(account_name, account_key)| { + SharedKeyAuthorizationPolicy::new( + account_name.clone(), + account_key.clone(), + String::from(STORAGE_SERVICE_VERSION), + ) + .map(Arc::new) + .map_err(|e| format!("Failed to create SharedKey policy: {e}").into()) + }) + .transpose() + } + + pub(super) fn queue_client(&self, queue_name: &str) -> crate::Result { + let queue_url = self.queue_url(queue_name)?; + let url = Url::parse(&queue_url).map_err(|e| format!("Invalid queue URL: {e}"))?; + + let mut options = QueueClientOptions::default(); + if let Some(policy) = self.shared_key_policy()? { + options + .client_options + .per_call_policies + .push(Arc::new(ContentLengthPolicy)); + options.client_options.per_call_policies.push(policy); + } + options.client_options.transport = Some(self.build_transport(&url)?); + + let client = QueueClient::new(url, self.credential.clone(), Some(options)) + .map_err(|e| format!("{e}"))?; + Ok(client) + } + + pub(super) fn container_client( + &self, + container_name: &str, + ) -> crate::Result { + let container_url = self + .parsed + .container_url(container_name) + .map_err(|e| format!("Failed to build container URL: {e}"))?; + let url = Url::parse(&container_url).map_err(|e| format!("Invalid container URL: {e}"))?; + + let mut options = BlobContainerClientOptions::default(); + if let Some(policy) = self.shared_key_policy()? { + options.client_options.per_call_policies.push(policy); + } + options.client_options.transport = Some(self.build_transport(&url)?); + + let client = BlobContainerClient::new(url, self.credential.clone(), Some(options)) + .map_err(|e| format!("{e}"))?; + Ok(client) + } +} + +/// Sets the `Content-Length` header from the request body before signing. +/// +/// The generated queue client leaves `Content-Length` to the HTTP transport, which runs after +/// the pipeline policies, so `SharedKeyAuthorizationPolicy` would sign an empty value while the +/// server verifies against the transmitted one. Must be pushed in front of the Shared Key policy. +#[derive(Debug)] +struct ContentLengthPolicy; + +#[async_trait::async_trait] +impl Policy for ContentLengthPolicy { + async fn send( + &self, + ctx: &Context, + request: &mut Request, + next: &[Arc], + ) -> PolicyResult { + if let Some(len) = request.body().len() + && len > 0 + { + request.insert_header("content-length", len.to_string()); + } + next[0].send(ctx, request, &next[1..]).await + } +} + +/// Extract the value of a connection string key (case-insensitive), if present. +fn connection_string_value(connection_string: &str, key: &str) -> Option { + connection_string.split(';').find_map(|seg| { + let (k, v) = seg.trim().split_once('=')?; + k.trim() + .eq_ignore_ascii_case(key) + .then(|| v.trim().to_string()) + }) +} + +/// Rewrite a `DevelopmentStorageProxyUri` so that it addresses the queue service. +/// +/// The proxy URI is the *blob* base, so any port it carries is the blob port. The queue is served +/// on a different one, so the port is replaced rather than reused. +fn dev_storage_queue_base(proxy_uri: &str, proto: &str) -> String { + let trimmed = proxy_uri.trim_end_matches('/'); + let (scheme, authority) = match trimmed.split_once("://") { + Some((scheme, rest)) => (scheme, rest), + None => (proto, trimmed), + }; + let authority = authority.split('/').next().unwrap_or(authority); + // Strip a trailing `:port` while leaving IPv6 literals (`[::1]`) intact. + let host = match authority.rsplit_once(':') { + Some((host, port)) if !port.is_empty() && port.bytes().all(|b| b.is_ascii_digit()) => host, + _ => authority, + }; + format!("{scheme}://{host}:{DEV_STORAGE_QUEUE_PORT}") +} + +fn append_query_segment(base_url: &str, sas: Option<&str>) -> String { + match sas { + None | Some("") => base_url.to_string(), + Some(q) => { + let sep = if base_url.contains('?') { '&' } else { '?' }; + format!("{base_url}{sep}{q}") + } + } +} + +/// Wrap the blob body in a decompressing reader; an empty body yields an empty reader. +async fn blob_decoder( + compression: Compression, + blob_name: &str, + content_encoding: Option<&str>, + content_type: Option<&str>, + body: AsyncResponseBody, +) -> Box { + let mut body = body.map_err(std::io::Error::other); + let first = match body.next().await { + Some(first) => first, + _ => { + return Box::new(tokio::io::empty()); + } + }; + + use Compression::*; + let compression = match compression { + // `first` is borrowed here and handed to the reader below untouched. + Auto => match determine_compression(content_encoding, content_type, blob_name) { + Some((inferred, source)) => verify_inferred_compression( + inferred, + source, + first.as_ref().map(|bytes| bytes.as_ref()).unwrap_or(&[]), + blob_name, + ), + Option::None => Compression::None, + }, + explicit => explicit, + }; + + let r = tokio::io::BufReader::new(StreamReader::new(stream::iter(Some(first)).chain(body))); + + match compression { + Auto => unreachable!(), // is mapped above + None => Box::new(r), + Gzip => Box::new(gzip_multiple_decoder(r)), + Zstd => Box::new({ + let mut decoder = bufread::ZstdDecoder::new(r); + decoder.multiple_members(true); + decoder + }), + } +} + +/// Which piece of metadata selected a compression scheme, reported when it disagrees with the +/// blob's contents. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum CompressionSource { + ContentEncoding, + ContentType, + BlobName, +} + +impl CompressionSource { + const fn as_str(self) -> &'static str { + match self { + Self::ContentEncoding => "Content-Encoding", + Self::ContentType => "Content-Type", + Self::BlobName => "blob name suffix", + } + } +} + +// try to determine the compression given the: +// * content-encoding +// * content-type +// * blob name (for file extension) +// +// It will use this information in this order +fn determine_compression( + content_encoding: Option<&str>, + content_type: Option<&str>, + blob_name: &str, +) -> Option<(Compression, CompressionSource)> { + content_encoding + .and_then(content_encoding_to_compression) + .map(|compression| (compression, CompressionSource::ContentEncoding)) + .or_else(|| { + content_type + .and_then(content_type_to_compression) + .map(|compression| (compression, CompressionSource::ContentType)) + }) + .or_else(|| { + blob_name_to_compression(blob_name) + .map(|compression| (compression, CompressionSource::BlobName)) + }) +} + +/// The leading bytes that identify a compressed stream. +const GZIP_MAGIC: &[u8] = &[0x1f, 0x8b]; +const ZSTD_MAGIC: &[u8] = &[0x28, 0xb5, 0x2f, 0xfd]; + +const fn compression_magic(compression: Compression) -> Option<&'static [u8]> { + match compression { + Compression::Gzip => Some(GZIP_MAGIC), + Compression::Zstd => Some(ZSTD_MAGIC), + Compression::Auto | Compression::None => None, + } +} + +/// Confirm an *inferred* compression scheme against the stream's leading bytes. +/// +/// `compression: auto` picks a codec from metadata alone, so a mismatch downgrades to reading the +/// blob as-is rather than failing it. An explicitly configured codec is never second-guessed: a +/// blob that does not match is an error the operator asked to see. +fn verify_inferred_compression( + inferred: Compression, + source: CompressionSource, + first: &[u8], + blob_name: &str, +) -> Compression { + let Some(magic) = compression_magic(inferred) else { + return inferred; + }; + + // Too few bytes to judge, so trust the metadata rather than guess. + if first.len() < magic.len() || first.starts_with(magic) { + return inferred; + } + + let leading = first + .iter() + .take(magic.len()) + .map(|byte| format!("{byte:02x}")) + .collect::>() + .join(" "); + + warn!( + message = "Blob metadata indicates a compression scheme that its contents do not match. Reading the blob undecompressed.", + blob = %blob_name, + detected = ?inferred, + detected_from = %source.as_str(), + leading_bytes = %leading, + ); + + Compression::None +} + +fn content_encoding_to_compression(content_encoding: &str) -> Option { + match content_encoding { + "gzip" => Some(Compression::Gzip), + "zstd" => Some(Compression::Zstd), + _ => None, + } +} + +fn content_type_to_compression(content_type: &str) -> Option { + match content_type { + "application/gzip" | "application/x-gzip" => Some(Compression::Gzip), + "application/zstd" => Some(Compression::Zstd), + _ => None, + } +} + +fn blob_name_to_compression(blob_name: &str) -> Option { + let extension = std::path::Path::new(blob_name) + .extension() + .and_then(std::ffi::OsStr::to_str); + + use Compression::*; + extension.and_then(|extension| match extension { + "gz" => Some(Gzip), + "zst" => Some(Zstd), + _ => Option::None, + }) +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn determine_compression() { + use super::Compression; + + let cases = vec![ + ("out.log", Some("gzip"), None, Some(Compression::Gzip)), + ( + "out.log", + None, + Some("application/gzip"), + Some(Compression::Gzip), + ), + ("out.log.gz", None, None, Some(Compression::Gzip)), + ("out.log.zst", None, None, Some(Compression::Zstd)), + ("out.txt", None, None, None), + ]; + for case in cases { + let (blob_name, content_encoding, content_type, expected) = case; + assert_eq!( + super::determine_compression(content_encoding, content_type, blob_name) + .map(|(compression, _source)| compression), + expected, + "blob_name={blob_name:?} content_encoding={content_encoding:?} content_type={content_type:?}", + ); + } + } + + #[test] + fn determine_compression_reports_its_source() { + use super::CompressionSource; + + for (blob_name, content_encoding, content_type, expected) in [ + ( + "out.log", + Some("gzip"), + None, + CompressionSource::ContentEncoding, + ), + ( + "out.log", + None, + Some("application/gzip"), + CompressionSource::ContentType, + ), + ("out.log.gz", None, None, CompressionSource::BlobName), + ] { + let (_, source) = + super::determine_compression(content_encoding, content_type, blob_name) + .expect("compression detected"); + assert_eq!(source, expected, "blob_name={blob_name:?}"); + } + } + + #[test] + fn inferred_compression_is_verified_against_the_leading_bytes() { + use super::{CompressionSource, verify_inferred_compression}; + + let verify = |first: &[u8], inferred| { + verify_inferred_compression(inferred, CompressionSource::BlobName, first, "out.log.gz") + }; + + assert_eq!( + verify(&[0x1f, 0x8b, 0x08, 0x00], Compression::Gzip), + Compression::Gzip + ); + assert_eq!( + verify(&[0x28, 0xb5, 0x2f, 0xfd, 0x00], Compression::Zstd), + Compression::Zstd + ); + + // Plain text under a `.gz` name: read it as-is rather than failing the whole blob. + assert_eq!( + verify(b"{\"message\":", Compression::Gzip), + Compression::None + ); + + // A zstd frame is not gzip. + assert_eq!( + verify(&[0x28, 0xb5, 0x2f, 0xfd], Compression::Gzip), + Compression::None + ); + + // Too few bytes to judge: keep trusting the metadata. + assert_eq!(verify(&[0x1f], Compression::Gzip), Compression::Gzip); + assert_eq!(verify(&[], Compression::Gzip), Compression::Gzip); + + assert_eq!(verify(b"plain", Compression::None), Compression::None); + } + + #[test] + fn generate_config() { + crate::test_util::test_generate_config::(); + } + + #[test] + fn connection_string_value_extraction() { + assert_eq!( + connection_string_value( + "AccountName=foo;QueueEndpoint=http://127.0.0.1:10001/foo", + "QueueEndpoint" + ), + Some("http://127.0.0.1:10001/foo".to_string()) + ); + assert_eq!( + connection_string_value("AccountName=foo;queueendpoint=http://q/", "QueueEndpoint"), + Some("http://q/".to_string()) + ); + assert_eq!( + connection_string_value("AccountName=foo", "QueueEndpoint"), + None + ); + } + + async fn clients_for( + connection_string: &str, + queue_endpoint: Option<&str>, + ) -> AzureStorageClientSource { + AzureStorageClientSource::new( + connection_string.to_string(), + queue_endpoint.map(ToOwned::to_owned), + None, + ProxyConfig::default(), + None, + ) + .await + .unwrap() + } + + #[tokio::test] + async fn queue_endpoint_resolution_public_cloud() { + let clients = clients_for( + "DefaultEndpointsProtocol=https;AccountName=myacct;AccountKey=base64==;EndpointSuffix=core.windows.net", + None, + ) + .await; + assert_eq!( + clients.queue_url("my-queue").unwrap(), + "https://myacct.queue.core.windows.net/my-queue" + ); + } + + #[tokio::test] + async fn queue_endpoint_resolution_development_storage() { + let clients = clients_for( + "UseDevelopmentStorage=true;DefaultEndpointsProtocol=http;AccountName=devstoreaccount1", + None, + ) + .await; + assert_eq!( + clients.queue_url("my-queue").unwrap(), + "http://127.0.0.1:10001/devstoreaccount1/my-queue" + ); + } + + #[tokio::test] + async fn queue_endpoint_resolution_development_storage_proxy_uri() { + // `ParsedConnectionString::blob_account_endpoint` would return + // `http://azurite:10000/devstoreaccount1` for the same connection string. + for (proxy_uri, expected) in [ + ("http://azurite:10000", "http://azurite:10001"), + ("http://azurite:10000/", "http://azurite:10001"), + ("http://azurite", "http://azurite:10001"), + ("azurite", "http://azurite:10001"), + ("azurite:10000", "http://azurite:10001"), + ] { + let clients = clients_for( + &format!( + "UseDevelopmentStorage=true;DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;DevelopmentStorageProxyUri={proxy_uri}" + ), + None, + ) + .await; + assert_eq!( + clients.queue_url("my-queue").unwrap(), + format!("{expected}/devstoreaccount1/my-queue"), + "proxy_uri={proxy_uri:?}", + ); + } + } + + #[test] + fn dev_storage_queue_base_leaves_ipv6_literals_intact() { + assert_eq!( + dev_storage_queue_base("http://[::1]:10000", "http"), + "http://[::1]:10001" + ); + assert_eq!( + dev_storage_queue_base("http://[::1]", "http"), + "http://[::1]:10001" + ); + } + + #[tokio::test] + async fn queue_endpoint_resolution_explicit_key() { + let clients = clients_for( + "AccountName=myacct;QueueEndpoint=http://localhost:14431/myacct/", + None, + ) + .await; + assert_eq!( + clients.queue_url("my-queue").unwrap(), + "http://localhost:14431/myacct/my-queue" + ); + } + + #[tokio::test] + async fn queue_endpoint_resolution_explicit_option_takes_precedence() { + let clients = clients_for( + "AccountName=myacct;QueueEndpoint=http://ignored:1/myacct", + Some("http://localhost:14431/myacct"), + ) + .await; + assert_eq!( + clients.queue_url("my-queue").unwrap(), + "http://localhost:14431/myacct/my-queue" + ); + } + + #[tokio::test] + async fn queue_url_appends_sas() { + let clients = clients_for( + "BlobEndpoint=https://myacct.blob.core.windows.net/;QueueEndpoint=https://myacct.queue.core.windows.net/;SharedAccessSignature=sv=2022-11-02&ss=bq", + None, + ) + .await; + assert_eq!( + clients.queue_url("my-queue").unwrap(), + "https://myacct.queue.core.windows.net/my-queue?sv=2022-11-02&ss=bq" + ); + } + + #[tokio::test] + async fn queue_endpoint_with_query_string() { + let clients = clients_for( + "AccountName=myacct", + Some("http://localhost:14431/myacct?sv=2022-11-02&sig=abc"), + ) + .await; + assert_eq!( + clients.queue_url("my-queue").unwrap(), + "http://localhost:14431/myacct/my-queue?sv=2022-11-02&sig=abc" + ); + } + + #[test] + fn append_query_segment_cases() { + assert_eq!(append_query_segment("http://h/p", None), "http://h/p"); + assert_eq!(append_query_segment("http://h/p", Some("")), "http://h/p"); + assert_eq!( + append_query_segment("http://h/p", Some("a=b")), + "http://h/p?a=b" + ); + assert_eq!( + append_query_segment("http://h/p?x=1", Some("a=b")), + "http://h/p?x=1&a=b" + ); + } + + #[tokio::test] + async fn config_validation_blob_endpoint_with_queue_endpoint() { + let config = AzureBlobConfig { + blob_endpoint: Some("http://localhost:10000/myacct".to_string()), + queue_endpoint: Some("http://localhost:10001/myacct".to_string()), + auth: Some(AzureAuthentication::MockCredential), + ..Default::default() + }; + let clients = config + .create_client_source(&ProxyConfig::default()) + .await + .unwrap(); + assert_eq!( + clients.queue_url("my-queue").unwrap(), + "http://localhost:10001/myacct/my-queue" + ); + } + + #[tokio::test] + async fn config_validation_blob_endpoint_requires_queue_endpoint() { + let config = AzureBlobConfig { + blob_endpoint: Some("http://localhost:10000/myacct".to_string()), + auth: Some(AzureAuthentication::MockCredential), + ..Default::default() + }; + let err = config + .create_client_source(&ProxyConfig::default()) + .await + .map(|_| ()) + .unwrap_err(); + assert!(err.to_string().contains("`queue_endpoint`")); + } + + #[tokio::test] + async fn config_validation_requires_queue() { + let config = AzureBlobConfig { + connection_string: Some("AccountName=foo;AccountKey=base64==".to_string().into()), + ..Default::default() + }; + let err = config + .create_queue_ingestor(None, &ProxyConfig::default(), LogNamespace::Legacy) + .await + .map(|_| ()) + .unwrap_err(); + assert!(err.to_string().contains("`queue` required")); + } + + #[tokio::test] + async fn config_validation_rejects_zero_poll_secs() { + let config = AzureBlobConfig { + connection_string: Some("AccountName=foo;AccountKey=base64==".to_string().into()), + queue: Some(queue::Config { + queue_name: "q".to_string(), + poll_secs: 0, + ..Default::default() + }), + ..Default::default() + }; + let err = config + .create_queue_ingestor(None, &ProxyConfig::default(), LogNamespace::Legacy) + .await + .map(|_| ()) + .unwrap_err(); + assert!(err.to_string().contains("poll_secs"), "{err}"); + } + + #[tokio::test] + async fn config_validation_rejects_conflicting_auth() { + let config = AzureBlobConfig { + connection_string: Some("AccountName=foo;AccountKey=base64==".to_string().into()), + auth: Some(AzureAuthentication::MockCredential), + queue: Some(queue::Config { + queue_name: "q".to_string(), + ..Default::default() + }), + ..Default::default() + }; + let err = config + .create_queue_ingestor(None, &ProxyConfig::default(), LogNamespace::Legacy) + .await + .map(|_| ()) + .unwrap_err(); + assert!(err.to_string().contains("Shared Key")); + } + + #[tokio::test] + async fn config_validation_requires_auth_with_account_name() { + let config = AzureBlobConfig { + account_name: Some("foo".to_string()), + queue: Some(queue::Config { + queue_name: "q".to_string(), + ..Default::default() + }), + ..Default::default() + }; + let err = config + .create_queue_ingestor(None, &ProxyConfig::default(), LogNamespace::Legacy) + .await + .map(|_| ()) + .unwrap_err(); + assert!(err.to_string().contains("`auth`")); + } +} diff --git a/src/sources/azure_blob/queue.rs b/src/sources/azure_blob/queue.rs new file mode 100644 index 0000000000000..376abb1e5fd56 --- /dev/null +++ b/src/sources/azure_blob/queue.rs @@ -0,0 +1,1261 @@ +use std::{ + borrow::Cow, + collections::HashMap, + future::ready, + num::NonZeroUsize, + panic, + sync::{Arc, RwLock}, + time::{Duration, Instant}, +}; + +use azure_storage_blob::BlobContainerClient; +use azure_storage_queue::{ + QueueClient, + models::{QueueClientReceiveMessagesOptions, ReceivedMessage}, +}; +use base64::prelude::{BASE64_STANDARD, Engine as _}; +use bytes::Bytes; +use chrono::{DateTime, TimeZone, Utc}; +use futures::{FutureExt, Stream, StreamExt}; +use serde::Deserialize; +use smallvec::SmallVec; +use snafu::{ResultExt, Snafu}; +use tokio::{pin, select}; +use tokio_util::codec::FramedRead; +use vector_lib::{ + codecs::decoding::FramingError, + config::{LegacyKey, LogNamespace, log_schema}, + configurable::configurable_component, + event::MaybeAsLogMut, + internal_event::{ + ByteSize, BytesReceived, CountByteSize, InternalEventHandle as _, Protocol, Registered, + error_type, + }, + lookup::{PathPrefix, metadata_path, path}, + source_sender::SendError, +}; + +use crate::{ + SourceSender, + codecs::Decoder, + common::backoff::ExponentialBackoff, + config::{SourceAcknowledgementsConfig, SourceContext}, + event::{BatchNotifier, BatchStatus, EstimatedJsonEncodedSizeOf, Event, LogEvent}, + internal_events::{ + AzureBlobEventIgnored, AzureBlobProcessingFailed, AzureBlobProcessingSucceeded, + AzureQueueMessageDeleteError, AzureQueueMessageDeleteSucceeded, + AzureQueueMessageProcessingError, AzureQueueMessageProcessingSucceeded, + AzureQueueMessageReceiveError, AzureQueueMessageReceiveSucceeded, EventsReceived, + StreamClosedError, + }, + line_agg::{self, LineAgg}, + shutdown::ShutdownSignal, + sources::azure_blob::{AzureBlobConfig, AzureStorageClientSource}, +}; + +/// The Event Grid event type that triggers ingestion. +const BLOB_CREATED_EVENT_TYPE: &str = "Microsoft.Storage.BlobCreated"; + +/// The prefix of the Event Grid `subject` field for blob events. +const SUBJECT_CONTAINER_PREFIX: &str = "/blobServices/default/containers/"; + +/// The separator between container and blob path in the Event Grid `subject` field. +const SUBJECT_BLOB_SEPARATOR: &str = "/blobs/"; + +/// Azure Storage Queue configuration options. +#[configurable_component] +#[derive(Clone, Debug, Derivative)] +#[derivative(Default)] +#[serde(deny_unknown_fields)] +pub(super) struct Config { + /// The name of the Storage Queue that receives the `Microsoft.Storage.BlobCreated` + /// notifications from the Event Grid subscription. + /// + /// This is a queue name, not a URL; the full URL is derived from the queue service endpoint. + #[configurable(metadata(docs::examples = "vector-blob-events"))] + pub(super) queue_name: String, + + /// Maximum time to wait between polls of the queue when it is empty, in seconds. + /// + /// Azure Storage Queues have no server-side long polling, so an exponential client-side + /// backoff (starting at one second) is applied between empty polls, capped at this value. + /// Polling resumes immediately whenever a poll returns at least one message. + /// + /// Must be at least `1`. + #[serde(default = "default_poll_secs")] + #[derivative(Default(value = "default_poll_secs()"))] + #[configurable(metadata(docs::type_unit = "seconds"))] + pub(super) poll_secs: u32, + + /// The visibility timeout to use for messages, in seconds. + /// + /// This controls how long a message is left unavailable after it is received. If a message + /// is received, and takes longer than `visibility_timeout_secs` to process and delete the + /// message from the queue, it is made available again for another consumer. + /// + /// This can happen if there is an issue between consuming a message and deleting it. + // NOTE: We restrict this to u32 for safe conversion to i32 later. + #[serde(default = "default_visibility_timeout_secs")] + #[derivative(Default(value = "default_visibility_timeout_secs()"))] + #[configurable(metadata(docs::type_unit = "seconds"))] + #[configurable(metadata(docs::human_name = "Visibility Timeout"))] + pub(super) visibility_timeout_secs: u32, + + /// Maximum number of messages to poll from the queue in a batch. + /// + /// Should be set to a smaller value when the blobs are large to help prevent the ingestion + /// of one blob from causing the others to exceed the `visibility_timeout_secs`. Valid + /// values are 1 - 32. + // NOTE: We restrict this to u32 for safe conversion to i32 later. + #[serde(default = "default_max_number_of_messages")] + #[derivative(Default(value = "default_max_number_of_messages()"))] + #[configurable(metadata(docs::human_name = "Max Messages"))] + #[configurable(metadata(docs::examples = 1))] + pub(super) max_number_of_messages: u32, + + /// Number of concurrent tasks to create for polling the queue for messages. + /// + /// Defaults to the number of available CPUs on the system. + /// + /// Should not typically need to be changed, but it can sometimes be beneficial to raise this + /// value when there is a high rate of messages being pushed into the queue and the blobs + /// being fetched are small. In these cases, system resources may not be fully utilized + /// without fetching more messages per second, as the queue message consumption rate affects + /// the blob retrieval rate. + #[configurable(metadata(docs::type_unit = "tasks"))] + #[configurable(metadata(docs::examples = 5))] + pub(super) client_concurrency: Option, + + /// Whether to delete the message once it is processed. + /// + /// It can be useful to set this to `false` for debugging or during the initial setup. + #[serde(default = "default_true")] + #[derivative(Default(value = "default_true()"))] + pub(super) delete_message: bool, + + /// Whether to delete non-retryable messages. + /// + /// If a message is rejected by the sink and not retryable, it is deleted from the queue. + /// With no dead-letter queue support, setting this to `false` means rejected messages are + /// redelivered indefinitely. + #[serde(default = "default_true")] + #[derivative(Default(value = "default_true()"))] + pub(super) delete_failed_message: bool, +} + +const fn default_poll_secs() -> u32 { + 15 +} + +const fn default_visibility_timeout_secs() -> u32 { + 300 +} + +const fn default_max_number_of_messages() -> u32 { + 10 +} + +const fn default_true() -> bool { + true +} + +/// The visibility timeout range permitted by the Queue Storage service: 1 second to 7 days. +const VISIBILITY_TIMEOUT_SECS_RANGE: std::ops::RangeInclusive = 1..=(7 * 24 * 60 * 60); + +#[derive(Debug, Snafu)] +pub(super) enum IngestorNewError { + #[snafu(display( + "Invalid value for max_number_of_messages {}, valid values are 1 - 32", + messages + ))] + InvalidNumberOfMessages { messages: u32 }, + #[snafu(display( + "Invalid value for visibility_timeout_secs {}, valid values are 1 second - 7 days", + seconds + ))] + InvalidVisibilityTimeout { seconds: u32 }, + #[snafu(display("Invalid value for poll_secs 0, must be at least 1 second"))] + ZeroPollSecs, +} + +#[allow(clippy::large_enum_variant)] +#[derive(Debug, Snafu)] +pub enum ProcessingError { + #[snafu(display( + "Could not parse queue message with id {} as a blob notification: {}", + message_id, + source + ))] + InvalidQueueMessage { + source: serde_json::Error, + message_id: String, + }, + #[snafu(display( + "Could not resolve container and blob from notification subject {:?} and url {:?}", + subject, + url + ))] + InvalidBlobPath { + subject: Option, + url: Option, + }, + #[snafu(display("Failed to build client for container {}: {}", container, message))] + ContainerClient { message: String, container: String }, + #[snafu(display("Failed to fetch blob {}/{}: {}", container, blob, source))] + GetBlob { + source: azure_core::Error, + container: String, + blob: String, + }, + #[snafu(display("Failed to read all of blob {}/{}: {}", container, blob, source))] + ReadBlob { + source: Box, + container: String, + blob: String, + }, + #[snafu(display("Failed to flush all of blob {}/{}: {}", container, blob, source))] + PipelineSend { + source: vector_lib::source_sender::SendError, + container: String, + blob: String, + }, + #[snafu(display( + "Sink reported an error sending events for blob {}/{}", + container, + blob + ))] + ErrorAcknowledgement { container: String, blob: String }, +} + +impl ProcessingError { + pub const fn error_type(&self) -> &'static str { + match self { + Self::InvalidQueueMessage { .. } | Self::InvalidBlobPath { .. } => { + error_type::PARSER_FAILED + } + Self::ContainerClient { .. } => error_type::CONFIGURATION_FAILED, + Self::GetBlob { .. } => error_type::REQUEST_FAILED, + Self::ReadBlob { .. } => error_type::READER_FAILED, + Self::PipelineSend { .. } => error_type::WRITER_FAILED, + Self::ErrorAcknowledgement { .. } => error_type::ACKNOWLEDGMENT_FAILED, + } + } +} + +pub struct State { + clients: AzureStorageClientSource, + queue_client: QueueClient, + container_clients: RwLock>>, + + multiline: Option, + compression: super::Compression, + + poll_secs: u32, + visibility_timeout_secs: i32, + max_number_of_messages: i32, + client_concurrency: usize, + delete_message: bool, + delete_failed_message: bool, + decoder: Decoder, +} + +impl State { + /// Event Grid subscriptions are account-scoped, so a single queue can carry notifications + /// for blobs in any container of the storage account. + fn container_client( + &self, + container: &str, + ) -> Result, ProcessingError> { + if let Some(client) = self + .container_clients + .read() + .expect("lock poisoned") + .get(container) + { + return Ok(Arc::clone(client)); + } + + let client = self.clients.container_client(container).map_err(|error| { + ProcessingError::ContainerClient { + message: error.to_string(), + container: container.to_owned(), + } + })?; + + let mut clients = self.container_clients.write().expect("lock poisoned"); + let entry = clients + .entry(container.to_owned()) + .or_insert_with(|| Arc::new(client)); + Ok(Arc::clone(entry)) + } +} + +pub(super) struct Ingestor { + state: Arc, +} + +impl Ingestor { + pub(super) fn new( + clients: AzureStorageClientSource, + config: Config, + compression: super::Compression, + multiline: Option, + decoder: Decoder, + ) -> crate::Result { + if config.max_number_of_messages < 1 || config.max_number_of_messages > 32 { + return Err(IngestorNewError::InvalidNumberOfMessages { + messages: config.max_number_of_messages, + } + .into()); + } + if !VISIBILITY_TIMEOUT_SECS_RANGE.contains(&config.visibility_timeout_secs) { + return Err(IngestorNewError::InvalidVisibilityTimeout { + seconds: config.visibility_timeout_secs, + } + .into()); + } + // A zero cap makes `ExponentialBackoff` yield `Duration::ZERO` forever, turning the + // empty-queue backoff into an unthrottled `GetMessages` loop on every polling task. + if config.poll_secs == 0 { + return Err(IngestorNewError::ZeroPollSecs.into()); + } + + let queue_client = clients.queue_client(&config.queue_name)?; + + let state = Arc::new(State { + clients, + queue_client, + container_clients: RwLock::new(HashMap::new()), + + compression, + multiline, + + poll_secs: config.poll_secs, + visibility_timeout_secs: config.visibility_timeout_secs as i32, + max_number_of_messages: config.max_number_of_messages as i32, + client_concurrency: config + .client_concurrency + .map(|n| n.get()) + .unwrap_or_else(crate::num_threads), + delete_message: config.delete_message, + delete_failed_message: config.delete_failed_message, + decoder, + }); + + Ok(Ingestor { state }) + } + + pub(super) async fn run( + self, + cx: SourceContext, + acknowledgements: SourceAcknowledgementsConfig, + log_namespace: LogNamespace, + ) -> Result<(), ()> { + let acknowledgements = cx.do_acknowledgements(acknowledgements); + let mut handles = Vec::new(); + for _ in 0..self.state.client_concurrency { + let process = IngestorProcess::new( + Arc::clone(&self.state), + cx.out.clone(), + cx.shutdown.clone(), + log_namespace, + acknowledgements, + ); + let fut = process.run(); + let handle = crate::spawn_in_current_span(fut); + handles.push(handle); + } + + for handle in handles.drain(..) { + if let Err(e) = handle.await + && e.is_panic() + { + panic::resume_unwind(e.into_panic()); + } + } + + Ok(()) + } +} + +pub struct IngestorProcess { + state: Arc, + out: SourceSender, + shutdown: ShutdownSignal, + acknowledgements: bool, + log_namespace: LogNamespace, + bytes_received: Registered, + events_received: Registered, + error_backoff: ExponentialBackoff, + empty_backoff: ExponentialBackoff, +} + +impl IngestorProcess { + pub fn new( + state: Arc, + out: SourceSender, + shutdown: ShutdownSignal, + log_namespace: LogNamespace, + acknowledgements: bool, + ) -> Self { + // `GetMessages` has no server-side long poll, so empty polls back off client-side. + let empty_backoff = ExponentialBackoff::from_millis(2) + .factor(500) + .max_delay(Duration::from_secs(state.poll_secs.into())); + + Self { + state, + out, + shutdown, + acknowledgements, + log_namespace, + bytes_received: register!(BytesReceived::from(Protocol::HTTPS)), + events_received: register!(EventsReceived), + error_backoff: ExponentialBackoff::default().max_delay(Duration::from_secs(30)), + empty_backoff, + } + } + + async fn run(mut self) { + let shutdown = self.shutdown.clone().fuse(); + pin!(shutdown); + + loop { + select! { + _ = &mut shutdown => break, + result = self.run_once() => { + let delay = match result { + Ok(received) => { + self.error_backoff.reset(); + if received > 0 { + self.empty_backoff.reset(); + None + } else { + Some(self.empty_backoff.next().expect("backoff never ends")) + } + } + Err(()) => Some(self.error_backoff.next().expect("backoff never ends")), + }; + if let Some(delay) = delay { + trace!( + delay_ms = delay.as_millis(), + "Waiting before polling the queue again.", + ); + select! { + _ = &mut shutdown => break, + _ = tokio::time::sleep(delay) => {}, + } + } + }, + } + } + } + + async fn run_once(&mut self) -> Result { + let messages = match self.receive_messages().await { + Ok(messages) => { + emit!(AzureQueueMessageReceiveSucceeded { + count: messages.len(), + }); + messages + } + Err(err) => { + emit!(AzureQueueMessageReceiveError { error: &err }); + return Err(()); + } + }; + + let count = messages.len(); + for message in messages { + self.handle_message(message).await; + } + + Ok(count) + } + + async fn handle_message(&mut self, message: ReceivedMessage) { + let message_id = message + .message_id + .clone() + .unwrap_or_else(|| "".to_owned()); + let Some(pop_receipt) = message.pop_receipt.clone() else { + warn!( + message = "Refusing to process message with no pop_receipt.", + message_id = %message_id, + ); + return; + }; + let dequeue_count = message.dequeue_count; + + match self.handle_queue_message(message).await { + Ok(()) => { + emit!(AzureQueueMessageProcessingSucceeded { + message_id: &message_id, + }); + if self.state.delete_message { + self.delete_message(&message_id, &pop_receipt).await; + } + } + Err(err) => { + // Left in the queue to redeliver after the visibility timeout. There is no + // dead-letter queue, so a permanently failing message redelivers indefinitely. + emit!(AzureQueueMessageProcessingError { + message_id: &message_id, + error: &err, + dequeue_count, + }); + } + } + } + + async fn handle_queue_message( + &mut self, + message: ReceivedMessage, + ) -> Result<(), ProcessingError> { + let body = message.message_text.unwrap_or_default(); + let body = decode_message_text(&body); + + let event: QueueEvent = + serde_json::from_str(body.as_ref()).context(InvalidQueueMessageSnafu { + message_id: message + .message_id + .clone() + .unwrap_or_else(|| "".to_owned()), + })?; + + for notification in event.into_notifications() { + self.handle_blob_notification(notification).await?; + } + Ok(()) + } + + async fn handle_blob_notification( + &mut self, + notification: BlobNotification, + ) -> Result<(), ProcessingError> { + if notification.event_type != BLOB_CREATED_EVENT_TYPE { + emit!(AzureBlobEventIgnored { + event_type: ¬ification.event_type, + }); + return Ok(()); + } + + let blob_ref = + resolve_blob_ref(¬ification).ok_or_else(|| ProcessingError::InvalidBlobPath { + subject: notification.subject.clone(), + url: notification.url.clone(), + })?; + + let container_client = self.state.container_client(&blob_ref.container)?; + + let download_start = Instant::now(); + + let object = container_client + .blob_client(&blob_ref.blob) + .download(None) + .await + .context(GetBlobSnafu { + container: blob_ref.container.clone(), + blob: blob_ref.blob.clone(), + })?; + + debug!( + message = "Got blob from queue notification.", + container = blob_ref.container, + blob = blob_ref.blob, + ); + + let metadata = object.properties.metadata; + + let timestamp = object + .properties + .last_modified + .and_then(to_chrono_timestamp) + .or(notification.event_time); + + let (batch, receiver) = BatchNotifier::maybe_new_with_receiver(self.acknowledgements); + let object_reader = super::blob_decoder( + self.state.compression, + &blob_ref.blob, + object.properties.content_encoding.as_deref(), + object.properties.content_type.as_deref(), + object.body, + ) + .await; + + // Record the read error seen to propagate up later so we avoid ack'ing the queue + // message + // + // String is used as we cannot clone std::io::Error to take ownership in closure + // + // FramedRead likely stops when it gets an i/o error but I found it more clear to + // show that we `take_while` there hasn't been an error + // + // This can result in blobs being partially processed before an error, but we + // prefer duplicate lines over message loss. Future work could include recording + // the offset of the blob that has been read, but this would only be relevant in + // the case that the same vector instance processes the same message. + let mut read_error = None; + let bytes_received = self.bytes_received.clone(); + let events_received = self.events_received.clone(); + let lines: Box + Send + Unpin> = Box::new( + FramedRead::new(object_reader, self.state.decoder.framer.clone()) + .map(|res| { + res.inspect(|bytes| { + bytes_received.emit(ByteSize(bytes.len())); + }) + .map_err(|err| { + read_error = Some(err); + }) + .ok() + }) + .take_while(|res| ready(res.is_some())) + .map(|r| r.expect("validated by take_while")), + ); + + let lines: Box + Send + Unpin> = match &self.state.multiline { + Some(config) => Box::new( + LineAgg::new( + lines.map(|line| ((), line, ())), + line_agg::Logic::new(config.clone()), + ) + .map(|(_src, line, _context, _lastline_context)| line), + ), + None => lines, + }; + + let log_namespace = self.log_namespace; + let mut stream = lines.flat_map(|line| { + let events = match self.state.decoder.deserializer_parse(line) { + Ok((events, _events_size)) => events, + Err(_error) => { + // Error is handled by `codecs::Decoder`, no further handling + // is needed here. + SmallVec::new() + } + }; + + let events = events + .into_iter() + .map(|mut event: Event| { + event = event.with_batch_notifier_option(&batch); + if let Some(log_event) = event.maybe_as_log_mut() { + handle_single_log( + log_event, + log_namespace, + &blob_ref, + &metadata, + timestamp, + ); + } + events_received.emit(CountByteSize(1, event.estimated_json_encoded_size_of())); + event + }) + .collect::>(); + futures::stream::iter(events) + }); + + let send_error = match self.out.send_event_stream(&mut stream).await { + Ok(_) => None, + Err(SendError::Closed) => { + let (count, _) = stream.size_hint(); + emit!(StreamClosedError { count }); + Some(SendError::Closed) + } + Err(SendError::Timeout) => unreachable!("No timeout is configured here"), + }; + + // Up above, `lines` captures `read_error`, and eventually is captured by `stream`, + // so we explicitly drop it so that we can again utilize `read_error` below. + drop(stream); + + // The BatchNotifier is cloned for each LogEvent in the batch stream, but the last + // reference must be dropped before the status of the batch is sent to the channel. + drop(batch); + + // Deliberately not the same as `result.is_ok()`: a rejected batch is removed from the + // queue when `delete_failed_message` is set, but nothing was delivered. + let mut delivered = false; + let container = blob_ref.container.clone(); + + let result = if let Some(error) = read_error { + Err(ProcessingError::ReadBlob { + source: error, + container: blob_ref.container.clone(), + blob: blob_ref.blob.clone(), + }) + } else if let Some(error) = send_error { + Err(ProcessingError::PipelineSend { + source: error, + container: blob_ref.container.clone(), + blob: blob_ref.blob.clone(), + }) + } else { + match receiver { + None => { + delivered = true; + Ok(()) + } + Some(receiver) => match receiver.await { + BatchStatus::Delivered => { + delivered = true; + debug!( + message = "Blob from queue notification delivered.", + container = blob_ref.container, + blob = blob_ref.blob, + ); + Ok(()) + } + BatchStatus::Errored => Err(ProcessingError::ErrorAcknowledgement { + container: blob_ref.container, + blob: blob_ref.blob, + }), + BatchStatus::Rejected => { + if self.state.delete_failed_message { + warn!( + message = "Blob from queue notification was rejected. Deleting failed message.", + container = blob_ref.container, + blob = blob_ref.blob, + ); + Ok(()) + } else { + Err(ProcessingError::ErrorAcknowledgement { + container: blob_ref.container, + blob: blob_ref.blob, + }) + } + } + }, + } + }; + + // Measured after the acknowledgement, so the outcome is known before a histogram is picked. + let duration = download_start.elapsed(); + if delivered { + emit!(AzureBlobProcessingSucceeded { + container: &container, + duration + }); + } else { + emit!(AzureBlobProcessingFailed { + container: &container, + duration + }); + } + + result + } + + async fn receive_messages(&mut self) -> azure_core::Result> { + let options = QueueClientReceiveMessagesOptions { + number_of_messages: Some(self.state.max_number_of_messages), + visibility_timeout: Some(self.state.visibility_timeout_secs), + ..Default::default() + }; + let response = self + .state + .queue_client + .receive_messages(Some(options)) + .await?; + Ok(response.into_model()?.items.unwrap_or_default()) + } + + /// Delete a single message. There is no batch-delete API in Queue Storage. + /// + /// A failure from a stale pop receipt is benign: the message redelivers and the blob is + /// ingested again, consistent with the source's at-least-once semantics. + async fn delete_message(&mut self, message_id: &str, pop_receipt: &str) { + match self + .state + .queue_client + .delete_message(message_id, pop_receipt, None) + .await + { + Ok(_) => { + emit!(AzureQueueMessageDeleteSucceeded { message_id }); + } + Err(err) => { + emit!(AzureQueueMessageDeleteError { + message_id, + error: &err, + }); + } + } + } +} + +fn handle_single_log( + log: &mut LogEvent, + log_namespace: LogNamespace, + blob_ref: &BlobRef, + metadata: &HashMap, + timestamp: Option>, +) { + log_namespace.insert_source_metadata( + AzureBlobConfig::NAME, + log, + Some(LegacyKey::Overwrite(path!("container"))), + path!("container"), + Bytes::from(blob_ref.container.as_bytes().to_vec()), + ); + + log_namespace.insert_source_metadata( + AzureBlobConfig::NAME, + log, + Some(LegacyKey::Overwrite(path!("blob"))), + path!("blob"), + Bytes::from(blob_ref.blob.as_bytes().to_vec()), + ); + + if let Some(storage_account) = &blob_ref.storage_account { + log_namespace.insert_source_metadata( + AzureBlobConfig::NAME, + log, + Some(LegacyKey::Overwrite(path!("storage_account"))), + path!("storage_account"), + Bytes::from(storage_account.as_bytes().to_vec()), + ); + } + + for (key, value) in metadata { + log_namespace.insert_source_metadata( + AzureBlobConfig::NAME, + log, + Some(LegacyKey::Overwrite(path!(key))), + path!("metadata", key.as_str()), + value.clone(), + ); + } + + log_namespace.insert_vector_metadata( + log, + log_schema().source_type_key(), + path!("source_type"), + Bytes::from_static(AzureBlobConfig::NAME.as_bytes()), + ); + + // The blob's `Last-Modified` time, falling back to the notification's event time, and + // finally (for the Legacy namespace) to `now()`. + match log_namespace { + LogNamespace::Vector => { + if let Some(timestamp) = timestamp { + log.insert( + metadata_path!(AzureBlobConfig::NAME, "timestamp"), + timestamp, + ); + } + + log.insert(metadata_path!("vector", "ingest_timestamp"), Utc::now()); + } + LogNamespace::Legacy => { + if let Some(timestamp_key) = log_schema().timestamp_key() { + log.try_insert( + (PathPrefix::Event, timestamp_key), + timestamp.unwrap_or_else(Utc::now), + ); + } + } + }; +} + +/// Event Grid base64-encodes the JSON event when delivering to a Storage Queue, but manual or +/// test messages may be raw JSON. Trying base64 first is unambiguous because `{`, the first +/// character of any JSON object, is not in the base64 alphabet. +fn decode_message_text(raw: &str) -> Cow<'_, str> { + match BASE64_STANDARD.decode(raw.trim()) { + Ok(bytes) => match String::from_utf8(bytes) { + Ok(s) => Cow::Owned(s), + Err(_) => Cow::Borrowed(raw), + }, + Err(_) => Cow::Borrowed(raw), + } +} + +/// A blob notification in either of the two subscription schemas, auto-detected. +/// +/// The two object variants are mutually exclusive on required fields, so their relative order +/// does not matter. `EventGridBatch` must stay last: it is the only sequence variant, and +/// matching it against an object is what an untagged enum falls through to. +#[derive(Clone, Debug, Deserialize)] +#[serde(untagged)] +enum QueueEvent { + CloudEvent(CloudEventEnvelope), + EventGrid(EventGridEnvelope), + // Defensive: some tooling wraps Event Grid events in a one-element array. + EventGridBatch(Vec), +} + +impl QueueEvent { + fn into_notifications(self) -> Vec { + match self { + QueueEvent::CloudEvent(event) => { + if !event.specversion.starts_with("1.") { + warn!( + message = "Unexpected CloudEvents specversion, processing anyway.", + specversion = %event.specversion, + ); + } + vec![event.into()] + } + QueueEvent::EventGrid(event) => vec![event.into()], + QueueEvent::EventGridBatch(events) => events.into_iter().map(Into::into).collect(), + } + } +} + +// https://learn.microsoft.com/azure/event-grid/event-schema-blob-storage +#[derive(Clone, Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct EventGridEnvelope { + event_type: String, + subject: String, + event_time: Option>, + data: Option, +} + +impl From for BlobNotification { + fn from(event: EventGridEnvelope) -> Self { + BlobNotification { + event_type: event.event_type, + subject: Some(event.subject), + event_time: event.event_time, + url: event.data.and_then(|data| data.url), + } + } +} + +// https://learn.microsoft.com/azure/event-grid/cloud-event-schema +#[derive(Clone, Debug, Deserialize)] +struct CloudEventEnvelope { + specversion: String, + #[serde(rename = "type")] + event_type: String, + subject: Option, + time: Option>, + data: Option, +} + +impl From for BlobNotification { + fn from(event: CloudEventEnvelope) -> Self { + BlobNotification { + event_type: event.event_type, + subject: event.subject, + event_time: event.time, + url: event.data.and_then(|data| data.url), + } + } +} + +/// The `data` payload of a blob storage event; identical in both schemas. +#[derive(Clone, Debug, Default, Deserialize)] +#[serde(rename_all = "camelCase")] +struct BlobEventData { + url: Option, +} + +/// A single blob notification, normalized from either schema. +#[derive(Clone, Debug)] +struct BlobNotification { + event_type: String, + subject: Option, + event_time: Option>, + url: Option, +} + +/// The identity of a blob resolved from a notification. +#[derive(Clone, Debug, PartialEq, Eq)] +struct BlobRef { + storage_account: Option, + container: String, + blob: String, +} + +/// Prefers `subject`, which is stable and endpoint-agnostic, falling back to `data.url`. The +/// storage account name is only available from the URL. +fn resolve_blob_ref(notification: &BlobNotification) -> Option { + if let Some(subject) = notification.subject.as_deref() + && let Some((container, blob)) = parse_subject(subject) + { + return Some(BlobRef { + storage_account: notification.url.as_deref().and_then(account_from_url), + container, + blob, + }); + } + + notification.url.as_deref().and_then(parse_blob_url) +} + +/// Parse an Event Grid subject: `/blobServices/default/containers/{container}/blobs/{path}`. +/// +/// The names are taken verbatim, because only `data.url` is percent-encoded. Decoding here would +/// corrupt any blob whose name contains a literal `%XX` sequence. +fn parse_subject(subject: &str) -> Option<(String, String)> { + let rest = subject.strip_prefix(SUBJECT_CONTAINER_PREFIX)?; + let (container, blob) = rest.split_once(SUBJECT_BLOB_SEPARATOR)?; + if container.is_empty() || blob.is_empty() { + return None; + } + Some((container.to_owned(), blob.to_owned())) +} + +/// Cloud style (`https://{account}.blob.core.windows.net/...`) uses the first host label; +/// path style (Azurite, `http://127.0.0.1:10000/{account}/...`) uses the first path segment. +fn account_from_url(url: &str) -> Option { + let parsed = url::Url::parse(url).ok()?; + let host = parsed.host_str()?; + if host.contains(".blob.") { + return host.split('.').next().map(ToOwned::to_owned); + } + parsed + .path_segments()? + .next() + .filter(|segment| !segment.is_empty()) + .map(percent_decode) +} + +/// Parse a blob URL into its account/container/blob parts. +fn parse_blob_url(url: &str) -> Option { + let parsed = url::Url::parse(url).ok()?; + let host = parsed.host_str()?; + let segments: Vec<&str> = parsed.path_segments()?.collect(); + + let (account, container, blob_segments) = if host.contains(".blob.") { + // Cloud style: the account is the first host label. + let account = host.split('.').next()?; + let (container, blob_segments) = segments.split_first()?; + (account.to_owned(), *container, blob_segments) + } else { + // Path style (Azurite): the account is the first path segment. + let (account, rest) = segments.split_first()?; + let (container, blob_segments) = rest.split_first()?; + ((*account).to_owned(), *container, blob_segments) + }; + + if container.is_empty() || blob_segments.is_empty() { + return None; + } + + let blob = blob_segments + .iter() + .map(|segment| percent_decode(segment)) + .collect::>() + .join("/"); + + Some(BlobRef { + storage_account: Some(percent_decode(&account)), + container: percent_decode(container), + blob, + }) +} + +fn percent_decode(s: &str) -> String { + percent_encoding::percent_decode_str(s) + .decode_utf8_lossy() + .into_owned() +} + +fn to_chrono_timestamp(ts: azure_core::time::OffsetDateTime) -> Option> { + Utc.timestamp_opt(ts.unix_timestamp(), ts.nanosecond()) + .single() +} + +#[cfg(test)] +mod tests { + use super::*; + + const EVENT_GRID_BODY: &str = r#"{ + "topic": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/rg/providers/Microsoft.Storage/storageAccounts/myacct", + "subject": "/blobServices/default/containers/logs/blobs/app/out.log", + "eventType": "Microsoft.Storage.BlobCreated", + "eventTime": "2026-06-01T12:00:00.000Z", + "id": "00000000-0000-0000-0000-000000000000", + "data": { + "api": "PutBlob", + "contentType": "text/plain", + "contentLength": 42, + "blobType": "BlockBlob", + "url": "https://myacct.blob.core.windows.net/logs/app/out.log", + "eTag": "0x8DC0000000000000" + }, + "dataVersion": "", + "metadataVersion": "1" + }"#; + + const CLOUD_EVENT_BODY: &str = r#"{ + "specversion": "1.0", + "type": "Microsoft.Storage.BlobCreated", + "source": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/rg/providers/Microsoft.Storage/storageAccounts/myacct", + "subject": "/blobServices/default/containers/logs/blobs/app/out.log", + "time": "2026-06-01T12:00:00.000Z", + "id": "00000000-0000-0000-0000-000000000000", + "data": { + "api": "PutBlob", + "contentType": "text/plain", + "contentLength": 42, + "blobType": "BlockBlob", + "url": "https://myacct.blob.core.windows.net/logs/app/out.log", + "eTag": "0x8DC0000000000000" + } + }"#; + + fn parse(body: &str) -> Vec { + serde_json::from_str::(body) + .unwrap() + .into_notifications() + } + + #[test] + fn parses_event_grid_schema() { + let notifications = parse(EVENT_GRID_BODY); + assert_eq!(notifications.len(), 1); + let notification = ¬ifications[0]; + assert_eq!(notification.event_type, BLOB_CREATED_EVENT_TYPE); + assert!(matches!( + serde_json::from_str::(EVENT_GRID_BODY).unwrap(), + QueueEvent::EventGrid(_) + )); + assert_eq!( + resolve_blob_ref(notification), + Some(BlobRef { + storage_account: Some("myacct".to_owned()), + container: "logs".to_owned(), + blob: "app/out.log".to_owned(), + }) + ); + } + + #[test] + fn parses_cloud_events_schema() { + let notifications = parse(CLOUD_EVENT_BODY); + assert_eq!(notifications.len(), 1); + assert!(matches!( + serde_json::from_str::(CLOUD_EVENT_BODY).unwrap(), + QueueEvent::CloudEvent(_) + )); + assert_eq!(notifications[0].event_type, BLOB_CREATED_EVENT_TYPE); + } + + #[test] + fn parses_event_grid_array() { + let body = format!("[{EVENT_GRID_BODY}]"); + let notifications = parse(&body); + assert_eq!(notifications.len(), 1); + assert_eq!(notifications[0].event_type, BLOB_CREATED_EVENT_TYPE); + } + + #[test] + fn parses_base64_encoded_body() { + let encoded = BASE64_STANDARD.encode(EVENT_GRID_BODY); + let decoded = decode_message_text(&encoded); + let notifications = parse(&decoded); + assert_eq!(notifications.len(), 1); + } + + #[test] + fn passes_through_raw_body() { + let decoded = decode_message_text(EVENT_GRID_BODY); + assert_eq!(decoded.as_ref(), EVENT_GRID_BODY); + } + + #[test] + fn rejects_garbage_body() { + assert!(serde_json::from_str::("not json").is_err()); + } + + #[test] + fn ignores_other_event_types() { + let body = EVENT_GRID_BODY.replace( + "Microsoft.Storage.BlobCreated", + "Microsoft.Storage.BlobDeleted", + ); + let notifications = parse(&body); + assert_eq!(notifications.len(), 1); + assert_eq!(notifications[0].event_type, "Microsoft.Storage.BlobDeleted"); + } + + #[test] + fn subject_extraction() { + assert_eq!( + parse_subject("/blobServices/default/containers/logs/blobs/app/out.log"), + Some(("logs".to_owned(), "app/out.log".to_owned())) + ); + // The subject carries the raw blob name, so a literal percent sequence is not an escape. + assert_eq!( + parse_subject("/blobServices/default/containers/logs/blobs/file name.log"), + Some(("logs".to_owned(), "file name.log".to_owned())) + ); + assert_eq!( + parse_subject("/blobServices/default/containers/logs/blobs/2026%2Fjan.log"), + Some(("logs".to_owned(), "2026%2Fjan.log".to_owned())) + ); + assert_eq!(parse_subject("/blobServices/default/containers/logs"), None); + assert_eq!(parse_subject("unrelated"), None); + } + + #[test] + fn url_extraction_cloud_style() { + assert_eq!( + parse_blob_url("https://myacct.blob.core.windows.net/logs/app/out.log"), + Some(BlobRef { + storage_account: Some("myacct".to_owned()), + container: "logs".to_owned(), + blob: "app/out.log".to_owned(), + }) + ); + } + + #[test] + fn url_extraction_path_style() { + assert_eq!( + parse_blob_url("http://127.0.0.1:10000/devstoreaccount1/logs/app/out.log"), + Some(BlobRef { + storage_account: Some("devstoreaccount1".to_owned()), + container: "logs".to_owned(), + blob: "app/out.log".to_owned(), + }) + ); + } + + #[test] + fn url_extraction_percent_encoded() { + assert_eq!( + parse_blob_url("https://myacct.blob.core.windows.net/logs/file%20name.log"), + Some(BlobRef { + storage_account: Some("myacct".to_owned()), + container: "logs".to_owned(), + blob: "file name.log".to_owned(), + }) + ); + } + + #[test] + fn subject_takes_precedence_over_url() { + let notification = BlobNotification { + event_type: BLOB_CREATED_EVENT_TYPE.to_owned(), + subject: Some("/blobServices/default/containers/from-subject/blobs/a.log".to_owned()), + event_time: None, + url: Some("https://myacct.blob.core.windows.net/from-url/b.log".to_owned()), + }; + let blob_ref = resolve_blob_ref(¬ification).unwrap(); + assert_eq!(blob_ref.container, "from-subject"); + assert_eq!(blob_ref.blob, "a.log"); + // account still comes from the URL, the only place it is present + assert_eq!(blob_ref.storage_account.as_deref(), Some("myacct")); + } + + #[test] + fn parse_queue_config() { + let config: Config = serde_yaml::from_str( + r#"queue_name: "my-queue" +"#, + ) + .unwrap(); + assert_eq!(config.queue_name, "my-queue"); + assert_eq!(config.poll_secs, 15); + assert_eq!(config.visibility_timeout_secs, 300); + assert_eq!(config.max_number_of_messages, 10); + assert!(config.delete_message); + assert!(config.delete_failed_message); + } +} diff --git a/src/sources/mod.rs b/src/sources/mod.rs index 2384b096f630b..f3ca03c1af6c1 100644 --- a/src/sources/mod.rs +++ b/src/sources/mod.rs @@ -13,6 +13,8 @@ pub mod aws_kinesis_firehose; pub mod aws_s3; #[cfg(feature = "sources-aws_sqs")] pub mod aws_sqs; +#[cfg(feature = "sources-azure_blob")] +pub mod azure_blob; #[cfg(feature = "sources-datadog_agent")] pub mod datadog_agent; #[cfg(feature = "sources-demo_logs")] diff --git a/tests/integration/azure/config/compose.yaml b/tests/integration/azure/config/compose.yaml index f7186a61cb00b..b876b64e26ff9 100644 --- a/tests/integration/azure/config/compose.yaml +++ b/tests/integration/azure/config/compose.yaml @@ -3,7 +3,7 @@ version: "3" services: local-azure-blob: image: mcr.microsoft.com/azure-storage/azurite:${CONFIG_VERSION} - command: azurite --blobHost 0.0.0.0 --loose + command: azurite --blobHost 0.0.0.0 --queueHost 0.0.0.0 --loose volumes: - /var/run:/var/run diff --git a/tests/integration/azure/config/test.yaml b/tests/integration/azure/config/test.yaml index 1b3a0e4cbe151..1fd982ae80203 100644 --- a/tests/integration/azure/config/test.yaml +++ b/tests/integration/azure/config/test.yaml @@ -16,5 +16,6 @@ matrix: # expressions are evaluated using https://github.com/micromatch/picomatch paths: - "src/sinks/azure_**" + - "src/sources/azure_blob/**" - "src/sinks/util/**" - "tests/integration/azure/**" diff --git a/website/content/en/docs/reference/configuration/sources/azure_blob.md b/website/content/en/docs/reference/configuration/sources/azure_blob.md new file mode 100644 index 0000000000000..cd95f7a6212ac --- /dev/null +++ b/website/content/en/docs/reference/configuration/sources/azure_blob.md @@ -0,0 +1,14 @@ +--- +title: Azure Blob Storage +description: Collect logs from [Azure Blob Storage](https://azure.microsoft.com/en-us/products/storage/blobs) +component_kind: source +layout: component +tags: ["azure", "blob", "storage", "component", "source", "logs"] +--- + +{{/* +This doc is generated using: + +1. The template in layouts/docs/component.html +2. The relevant CUE data in cue/reference/components/... +*/}} diff --git a/website/cue/reference/components/sources/azure_blob.cue b/website/cue/reference/components/sources/azure_blob.cue new file mode 100644 index 0000000000000..6ca56a929b9a1 --- /dev/null +++ b/website/cue/reference/components/sources/azure_blob.cue @@ -0,0 +1,149 @@ +package metadata + +components: sources: azure_blob: { + title: "Azure Blob Storage" + + features: { + auto_generated: true + acknowledgements: true + multiline: enabled: true + collect: { + tls: enabled: false + checkpoint: enabled: false + proxy: enabled: true + from: service: services.azure_blob + } + } + + classes: { + deployment_roles: ["aggregator"] + delivery: "at_least_once" + development: "beta" + egress_method: "stream" + stateful: false + } + + support: { + requirements: [ + """ + The Azure Blob Storage source requires an Azure Storage Queue that + receives `Microsoft.Storage.BlobCreated` notifications from an + [Event Grid subscription](\(urls.azure_event_grid_blob)) on the + storage account. + """, + ] + warnings: [] + notices: [] + } + + installation: { + platform_name: null + } + + configuration: generated.components.sources.azure_blob.configuration + + output: { + logs: object: { + description: "A line from a blob in Azure Blob Storage." + fields: { + message: { + description: "A line from the blob." + required: true + type: string: { + examples: ["53.126.150.246 - - [01/Oct/2020:11:25:58 -0400] \"GET /disintermediate HTTP/2.0\" 401 20308"] + } + } + timestamp: fields._current_timestamp & { + description: "The Last-Modified time of the blob, falling back to the notification's event time. Defaults to the current timestamp if this information is missing." + } + source_type: { + description: "The name of the source type." + required: true + type: string: { + examples: ["azure_blob"] + } + } + container: { + description: "The container of the blob the line came from." + required: true + type: string: { + examples: ["insights-logs"] + } + } + blob: { + description: "The blob the line came from." + required: true + type: string: { + examples: ["resourceId=/SUBSCRIPTIONS/.../y=2026/m=06/d=01/h=12/m=00/PT1H.json"] + } + } + storage_account: { + description: "The storage account of the blob the line came from, when it can be determined from the notification." + required: false + common: true + type: string: { + default: null + examples: ["mylogstorage"] + } + } + } + } + } + + how_it_works: { + setup: { + title: "Blob discovery through Event Grid" + body: """ + This source does not scan the storage account for blobs. Instead, it relies + on [Azure Event Grid](\(urls.azure_event_grid_blob)) publishing a + notification to an [Azure Storage Queue](\(urls.azure_storage_queue)) + whenever a blob is created: + + 1. Create a Storage Queue in the storage account (or another account). + 2. Create an Event Grid subscription on the storage account, filtered to + the `Microsoft.Storage.BlobCreated` event type, with the Storage Queue + as its endpoint. + + Vector polls the queue, downloads each newly created blob, and deletes the + queue message once the events have been delivered. Notifications for other + event types are ignored and deleted from the queue. Both the Event Grid + and CloudEvents 1.0 schemas are supported and detected automatically. + + Because a queue message must only be processed (and deleted) once, each + Vector instance must consume its own Storage Queue. To send the same + events to multiple destinations, configure multiple sinks on the same + source instead of multiple sources sharing a queue. + """ + } + events: { + title: "Handling events from the `azure_blob` source" + body: """ + This source behaves very similarly to the `file` source in that + it outputs one event per line (unless the `multiline` + configuration option is used), and you will commonly want to use + [transforms](\(urls.vector_transforms)) to parse the data. + """ + } + failed_messages: { + title: "Failed message handling" + body: """ + When a blob referenced by a queue message cannot be fetched or read, the + message is left in the queue and becomes visible again after + `queue.visibility_timeout_secs`, so ingestion is retried. Azure Storage + Queues have no dead-letter queue: a message that fails permanently is + redelivered indefinitely. The `dequeue_count` field in the error log makes + such poison messages observable. + """ + } + } + + telemetry: metrics: { + azure_blob_event_ignored_total: components.sources.internal_metrics.output.metrics.azure_blob_event_ignored_total + azure_blob_processing_failed_duration_seconds: components.sources.internal_metrics.output.metrics.azure_blob_processing_failed_duration_seconds + azure_blob_processing_succeeded_duration_seconds: components.sources.internal_metrics.output.metrics.azure_blob_processing_succeeded_duration_seconds + azure_queue_message_delete_succeeded_total: components.sources.internal_metrics.output.metrics.azure_queue_message_delete_succeeded_total + azure_queue_message_processing_succeeded_total: components.sources.internal_metrics.output.metrics.azure_queue_message_processing_succeeded_total + azure_queue_message_receive_succeeded_total: components.sources.internal_metrics.output.metrics.azure_queue_message_receive_succeeded_total + azure_queue_message_received_messages_total: components.sources.internal_metrics.output.metrics.azure_queue_message_received_messages_total + } +} diff --git a/website/cue/reference/components/sources/generated/azure_blob.cue b/website/cue/reference/components/sources/generated/azure_blob.cue new file mode 100644 index 0000000000000..1db462f2c24ab --- /dev/null +++ b/website/cue/reference/components/sources/generated/azure_blob.cue @@ -0,0 +1,934 @@ +package metadata + +generated: components: sources: azure_blob: configuration: { + account_name: { + description: """ + The Azure Blob Storage Account name. + + If provided, this is used instead of the `connection_string` and requires `auth` to be + configured. Both the blob and queue service endpoints are derived from the account name. + """ + required: false + type: string: examples: ["mylogstorage"] + } + acknowledgements: { + deprecated: true + description: """ + Controls how acknowledgements are handled by this source. + + This setting is **deprecated** in favor of enabling `acknowledgements` at the [global][global_acks] or sink level. + + Enabling or disabling acknowledgements at the source level has **no effect** on acknowledgement behavior. + + See [End-to-end Acknowledgements][e2e_acks] for more information on how event acknowledgement is handled. + + [global_acks]: https://vector.dev/docs/reference/configuration/global-options/#acknowledgements + [e2e_acks]: https://vector.dev/docs/architecture/end-to-end-acknowledgements/ + """ + required: false + type: object: options: enabled: { + description: "Whether or not end-to-end acknowledgements are enabled for this source." + required: false + type: bool: {} + } + } + auth: { + description: "Azure service principal authentication." + required: false + type: object: options: { + azure_client_id: { + description: """ + The [Azure Client ID][azure_client_id]. + + [azure_client_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + """ + relevant_when: "azure_credential_kind = \"client_certificate_credential\" or azure_credential_kind = \"client_secret_credential\"" + required: true + type: string: examples: ["00000000-0000-0000-0000-000000000000", "${AZURE_CLIENT_ID:?err}"] + } + azure_client_secret: { + description: """ + The [Azure Client Secret][azure_client_secret]. + + [azure_client_secret]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + """ + relevant_when: "azure_credential_kind = \"client_secret_credential\"" + required: true + type: string: examples: ["00-00~000000-0000000~0000000000000000000", "${AZURE_CLIENT_SECRET:?err}"] + } + azure_credential_kind: { + description: "The kind of Azure credential to use." + required: true + type: string: enum: { + azure_cli: "Use Azure CLI credentials" + client_certificate_credential: "Use certificate credentials" + client_secret_credential: "Use client ID/secret credentials" + managed_identity: "Use Managed Identity credentials" + managed_identity_client_assertion: "Use Managed Identity with Client Assertion credentials" + workload_identity: "Use Workload Identity credentials" + } + } + azure_tenant_id: { + description: """ + The [Azure Tenant ID][azure_tenant_id]. + + [azure_tenant_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + """ + relevant_when: "azure_credential_kind = \"client_certificate_credential\" or azure_credential_kind = \"client_secret_credential\"" + required: true + type: string: examples: ["00000000-0000-0000-0000-000000000000", "${AZURE_TENANT_ID:?err}"] + } + certificate_password: { + description: "The password for the client certificate, if applicable." + relevant_when: "azure_credential_kind = \"client_certificate_credential\"" + required: false + type: string: examples: ["${AZURE_CLIENT_CERTIFICATE_PASSWORD}"] + } + certificate_path: { + description: "PKCS12 certificate with RSA private key." + relevant_when: "azure_credential_kind = \"client_certificate_credential\"" + required: true + type: string: examples: ["path/to/certificate.pfx", "${AZURE_CLIENT_CERTIFICATE_PATH:?err}"] + } + client_assertion_client_id: { + description: "The target Client ID to use." + relevant_when: "azure_credential_kind = \"managed_identity_client_assertion\"" + required: true + type: string: examples: ["00000000-0000-0000-0000-000000000000"] + } + client_assertion_tenant_id: { + description: "The target Tenant ID to use." + relevant_when: "azure_credential_kind = \"managed_identity_client_assertion\"" + required: true + type: string: examples: ["00000000-0000-0000-0000-000000000000"] + } + client_id: { + description: """ + The [Azure Client ID][azure_client_id]. Defaults to the value of the environment variable `AZURE_CLIENT_ID`. + + [azure_client_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + """ + relevant_when: "azure_credential_kind = \"workload_identity\"" + required: false + type: string: examples: ["00000000-0000-0000-0000-000000000000", "${AZURE_CLIENT_ID}"] + } + tenant_id: { + description: """ + The [Azure Tenant ID][azure_tenant_id]. Defaults to the value of the environment variable `AZURE_TENANT_ID`. + + [azure_tenant_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + """ + relevant_when: "azure_credential_kind = \"workload_identity\"" + required: false + type: string: examples: ["00000000-0000-0000-0000-000000000000", "${AZURE_TENANT_ID}"] + } + token_file_path: { + description: "Path of a file containing a Kubernetes service account token. Defaults to the value of the environment variable `AZURE_FEDERATED_TOKEN_FILE`." + relevant_when: "azure_credential_kind = \"workload_identity\"" + required: false + type: string: examples: ["/var/run/secrets/azure/tokens/azure-identity-token", "${AZURE_FEDERATED_TOKEN_FILE}"] + } + user_assigned_managed_identity_id: { + description: "The User Assigned Managed Identity to use." + relevant_when: "azure_credential_kind = \"managed_identity\" or azure_credential_kind = \"managed_identity_client_assertion\"" + required: false + type: string: examples: ["00000000-0000-0000-0000-000000000000"] + } + user_assigned_managed_identity_id_type: { + description: """ + The type of the User Assigned Managed Identity ID provided (Client ID, Object ID, + or Resource ID). Defaults to Client ID. + """ + relevant_when: "azure_credential_kind = \"managed_identity\" or azure_credential_kind = \"managed_identity_client_assertion\"" + required: false + type: string: enum: { + client_id: "Client ID" + object_id: "Object ID" + resource_id: "Resource ID" + } + } + } + } + blob_endpoint: { + description: """ + The Azure Blob Storage service endpoint. + + Useful for Azurite, sovereign clouds, or private endpoints. Requires `auth` to be + configured, and `queue_endpoint` to be provided as well when `account_name` is not set. + """ + required: false + type: string: examples: ["https://mylogstorage.blob.core.windows.net/"] + } + compression: { + description: "The compression scheme used for decompressing blobs retrieved from Azure Blob Storage." + required: false + type: string: { + default: "auto" + enum: { + auto: """ + Automatically attempt to determine the compression scheme. + + The compression scheme of the blob is determined from its `Content-Encoding` and + `Content-Type` metadata, as well as the blob name suffix (for example, `.gz`). + + It is set to `none` if the compression scheme cannot be determined. + """ + gzip: "GZIP." + none: "Uncompressed." + zstd: "ZSTD." + } + } + } + connection_string: { + description: """ + The Azure Blob Storage Account connection string. + + Authentication with an access key or shared access signature (SAS) are supported + authentication methods. The connection string is also used to derive the blob and + queue service endpoints. + """ + required: false + type: string: examples: ["DefaultEndpointsProtocol=https;AccountName=mylogstorage;AccountKey=storageaccountkeybase64encoded;EndpointSuffix=core.windows.net", "BlobEndpoint=https://mylogstorage.blob.core.windows.net/;QueueEndpoint=https://mylogstorage.queue.core.windows.net/;SharedAccessSignature=generatedsastoken", "AccountName=mylogstorage"] + warnings: ["Access keys and SAS tokens can be used to gain unauthorized access to Azure Storage resources. Numerous security breaches have occurred due to leaked connection strings. It is important to keep connection strings secure and not expose them in logs, error messages, or version control systems."] + } + decoding: { + description: """ + Configures how events are decoded from raw bytes. Note some decoders can also determine the event output + type (log, metric, trace). + """ + required: false + type: object: options: { + avro: { + description: "Apache Avro-specific encoder options." + relevant_when: "codec = \"avro\"" + required: true + type: object: options: { + schema: { + description: """ + The Avro schema definition. + **Note**: The following [`apache_avro::types::Value`] variants are *not* supported: + * `Date` + * `Decimal` + * `Duration` + * `Fixed` + * `TimeMillis` + """ + required: true + type: string: examples: ["{ \"type\": \"record\", \"name\": \"log\", \"fields\": [{ \"name\": \"message\", \"type\": \"string\" }] }"] + } + strip_schema_id_prefix: { + description: "For Avro datum encoded in Kafka messages, the bytes are prefixed with the schema ID. Set this to `true` to strip the schema ID prefix, as described in [Confluent Kafka's documentation](https://docs.confluent.io/platform/current/schema-registry/fundamentals/serdes-develop/index.html#wire-format)." + required: true + type: bool: {} + } + } + } + codec: { + description: "The codec to use for decoding events." + required: false + type: string: { + default: "bytes" + enum: { + avro: """ + Decodes the raw bytes as an [Apache Avro][apache_avro] message. + + [apache_avro]: https://avro.apache.org/ + """ + bytes: "Uses the raw bytes as-is." + gelf: """ + Decodes the raw bytes as a [GELF][gelf] message. + + This codec is experimental for the following reason: + + The GELF specification is more strict than the actual Graylog receiver. + Vector's decoder adheres more strictly to the GELF spec, with + the exception that some characters such as `@` are allowed in field names. + + Other GELF codecs, such as Loki's, use a [Go SDK][implementation] that is maintained + by Graylog and is much more relaxed than the GELF spec. + + Going forward, Vector will use the [Go SDK][implementation] as the reference implementation, which means + the codec may continue to relax the enforcement of the specification. + + [gelf]: https://docs.graylog.org/docs/gelf + [implementation]: https://github.com/Graylog2/go-gelf/blob/v2/gelf/reader.go + """ + influxdb: """ + Decodes the raw bytes as an [Influxdb Line Protocol][influxdb] message. + + [influxdb]: https://docs.influxdata.com/influxdb/cloud/reference/syntax/line-protocol + """ + json: """ + Decodes the raw bytes as [JSON][json]. + + [json]: https://www.json.org/ + """ + native: """ + Decodes the raw bytes as [native Protocol Buffers format][vector_native_protobuf]. + + This decoder can output all types of events: logs, metrics, and traces. + + This codec is **[experimental][experimental]**. + + [vector_native_protobuf]: https://github.com/vectordotdev/vector/blob/master/lib/vector-core/proto/event.proto + [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs + """ + native_json: """ + Decodes the raw bytes as [native JSON format][vector_native_json]. + + This decoder can output all types of events: logs, metrics, and traces. + + This codec is **[experimental][experimental]**. + + [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue + [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs + """ + otlp: """ + Decodes the raw bytes as [OTLP (OpenTelemetry Protocol)][otlp] protobuf format. + + This decoder handles the three OTLP signal types: logs, metrics, and traces. + It automatically detects which type of OTLP message is being decoded. + + [otlp]: https://opentelemetry.io/docs/specs/otlp/ + """ + protobuf: """ + Decodes the raw bytes as [protobuf][protobuf]. + + [protobuf]: https://protobuf.dev/ + """ + syslog: """ + Decodes the raw bytes as a Syslog message. + + Decodes either as the [RFC 3164][rfc3164]-style format ("old" style) or the + [RFC 5424][rfc5424]-style format ("new" style, includes structured data). + + [rfc3164]: https://www.ietf.org/rfc/rfc3164.txt + [rfc5424]: https://www.ietf.org/rfc/rfc5424.txt + """ + vrl: """ + Decodes the raw bytes as a string and passes them as input to a [VRL][vrl] program. + + [vrl]: https://vector.dev/docs/reference/vrl + """ + } + } + } + gelf: { + description: "GELF-specific decoding options." + relevant_when: "codec = \"gelf\"" + required: false + type: object: options: { + lossy: { + description: """ + Determines whether to replace invalid UTF-8 sequences instead of failing. + + When true, invalid UTF-8 sequences are replaced with the [`U+FFFD REPLACEMENT CHARACTER`][U+FFFD]. + + [U+FFFD]: https://en.wikipedia.org/wiki/Specials_(Unicode_block)#Replacement_character + """ + required: false + type: bool: default: true + } + validation: { + description: "Configures the decoding validation mode." + required: false + type: string: { + default: "strict" + enum: { + relaxed: """ + Uses more relaxed validation that skips strict GELF specification checks. + + This mode does not treat specification violations as errors, allowing the decoder + to accept messages from sources that don't strictly follow the GELF spec. + """ + strict: "Uses strict validation that closely follows the GELF spec." + } + } + } + } + } + influxdb: { + description: "Influxdb-specific decoding options." + relevant_when: "codec = \"influxdb\"" + required: false + type: object: options: lossy: { + description: """ + Determines whether to replace invalid UTF-8 sequences instead of failing. + + When true, invalid UTF-8 sequences are replaced with the [`U+FFFD REPLACEMENT CHARACTER`][U+FFFD]. + + [U+FFFD]: https://en.wikipedia.org/wiki/Specials_(Unicode_block)#Replacement_character + """ + required: false + type: bool: default: true + } + } + json: { + description: "JSON-specific decoding options." + relevant_when: "codec = \"json\"" + required: false + type: object: options: lossy: { + description: """ + Determines whether to replace invalid UTF-8 sequences instead of failing. + + When true, invalid UTF-8 sequences are replaced with the [`U+FFFD REPLACEMENT CHARACTER`][U+FFFD]. + + [U+FFFD]: https://en.wikipedia.org/wiki/Specials_(Unicode_block)#Replacement_character + """ + required: false + type: bool: default: true + } + } + native_json: { + description: "Vector's native JSON-specific decoding options." + relevant_when: "codec = \"native_json\"" + required: false + type: object: options: lossy: { + description: """ + Determines whether to replace invalid UTF-8 sequences instead of failing. + + When true, invalid UTF-8 sequences are replaced with the [`U+FFFD REPLACEMENT CHARACTER`][U+FFFD]. + + [U+FFFD]: https://en.wikipedia.org/wiki/Specials_(Unicode_block)#Replacement_character + """ + required: false + type: bool: default: true + } + } + protobuf: { + description: "Protobuf-specific decoding options." + relevant_when: "codec = \"protobuf\"" + required: false + type: object: options: { + desc_file: { + description: """ + The path to the protobuf descriptor set file. + + This file is the output of `protoc -I -o `. + + For more information, see [How Buf images work](https://buf.build/docs/reference/images/#how-buf-images-work). + """ + required: false + type: string: default: "" + } + message_type: { + description: "The name of the message type to use for serializing." + required: false + type: string: { + default: "" + examples: ["package.Message"] + } + } + use_json_names: { + description: """ + Use JSON field names (camelCase) instead of protobuf field names (snake_case). + + When enabled, the deserializer will output fields using their JSON names as defined + in the `.proto` file (for example, `jobDescription` instead of `job_description`). + + This is useful when working with data that needs to be converted to JSON or + when interfacing with systems that use JSON naming conventions. + """ + required: false + type: bool: default: false + } + } + } + signal_types: { + description: """ + Signal types to attempt parsing, in priority order. + + The deserializer tries to parse signals in the specified order. This allows you to optimize + performance when you know the expected signal types. For example, if you only receive + traces, set this to `["traces"]` to avoid attempting to parse as logs or metrics first. + + If not specified, defaults to trying all types in this order: logs, metrics, traces. + Duplicate signal types are automatically removed while preserving order. + """ + relevant_when: "codec = \"otlp\"" + required: false + type: array: { + default: ["logs", "metrics", "traces"] + items: type: string: enum: { + logs: "OTLP logs signal (ExportLogsServiceRequest)" + metrics: "OTLP metrics signal (ExportMetricsServiceRequest)" + traces: "OTLP traces signal (ExportTraceServiceRequest)" + } + } + } + syslog: { + description: "Syslog-specific decoding options." + relevant_when: "codec = \"syslog\"" + required: false + type: object: options: lossy: { + description: """ + Determines whether to replace invalid UTF-8 sequences instead of failing. + + When true, invalid UTF-8 sequences are replaced with the [`U+FFFD REPLACEMENT CHARACTER`][U+FFFD]. + + [U+FFFD]: https://en.wikipedia.org/wiki/Specials_(Unicode_block)#Replacement_character + """ + required: false + type: bool: default: true + } + } + vrl: { + description: "VRL-specific decoding options." + relevant_when: "codec = \"vrl\"" + required: true + type: object: options: { + source: { + description: """ + The [Vector Remap Language][vrl] (VRL) program to execute for each event. + The final contents of the `.` target are used as the decoding result. + Compilation errors or use of `abort` in the program result in a decoding error. + + [vrl]: https://vector.dev/docs/reference/vrl + """ + required: true + type: string: {} + } + timezone: { + description: """ + The name of the timezone to apply to timestamp conversions that do not contain an explicit + time zone. The time zone name may be any name in the [TZ database][tz_database], or `local` + to indicate system local time. + + If not set, `local` is used. + + [tz_database]: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones + """ + required: false + type: string: examples: ["local", "America/New_York", "EST5EDT"] + } + } + } + } + } + framing: { + description: """ + Framing configuration. + + Framing handles how events are separated when encoded in a raw byte form, where each event is + a frame that must be prefixed, or delimited, in a way that marks where an event begins and + ends within the byte stream. + """ + required: false + type: object: options: { + character_delimited: { + description: "Options for the character delimited decoder." + relevant_when: "method = \"character_delimited\"" + required: true + type: object: options: { + delimiter: { + description: "The character that delimits byte sequences." + required: true + type: ascii_char: {} + } + max_length: { + description: """ + The maximum length of the byte buffer. + + This length does *not* include the trailing delimiter. + + By default, no maximum length is enforced. If events are malformed, this can lead to + additional resource usage as events continue to be buffered in memory, and can potentially + lead to memory exhaustion in extreme cases. + + If there is a risk of processing malformed data, such as logs with user-controlled input, + consider setting the maximum length to a reasonably large value as a safety net. This + prevents processing from being unbounded. + """ + required: false + type: uint: {} + } + oversized_action: { + description: """ + The behavior when a frame exceeds `max_length`. + + When set to `drop` (the default), the entire oversized frame is discarded. + When set to `truncate`, the frame is truncated to `max_length` bytes and the + remainder is discarded up to the next delimiter. + + This option has no effect if `max_length` is not set. + """ + required: false + type: string: { + default: "drop" + enum: { + drop: "Drop the entire oversized frame." + truncate: """ + Truncate the frame to the maximum allowed size and emit the partial content. + + The remainder of the oversized frame is discarded up to the next delimiter. + """ + } + } + } + } + } + chunked_gelf: { + description: "Options for the chunked GELF decoder." + relevant_when: "method = \"chunked_gelf\"" + required: false + type: object: options: { + decompression: { + description: "Decompression configuration for GELF messages." + required: false + type: string: { + default: "Auto" + enum: { + Auto: "Automatically detect the decompression method based on the magic bytes of the message." + Gzip: "Use Gzip decompression." + None: "Do not decompress the message." + Zlib: "Use Zlib decompression." + } + } + } + max_length: { + description: """ + The maximum length of a single GELF message, in bytes. Messages longer than this length are + dropped. If this option is not set, the decoder does not limit the length of messages and + the per-message memory is unbounded. + + **Note**: A message can be composed of multiple chunks, and this limit applies to the whole + message, not to individual chunks. + + This limit takes into account only the message payload. GELF header bytes are excluded from the calculation. + The message payload is the concatenation of all chunk payloads. + """ + required: false + type: uint: {} + } + pending_messages_limit: { + description: """ + The maximum number of pending incomplete messages. If this limit is reached, the decoder starts + dropping chunks of new messages, ensuring the memory usage of the decoder's state is bounded. + If this option is not set, the decoder does not limit the number of pending messages and the memory usage + of its messages buffer can grow unbounded. This matches Graylog Server's behavior. + """ + required: false + type: uint: {} + } + timeout_secs: { + description: """ + The timeout, in seconds, for a message to be fully received. If the timeout is reached, the + decoder drops all received chunks for the timed-out message. + """ + required: false + type: float: default: 5.0 + } + } + } + length_delimited: { + description: "Options for the length delimited decoder." + relevant_when: "method = \"length_delimited\"" + required: true + type: object: options: { + length_field_is_big_endian: { + description: "Length field byte order (little or big endian)" + required: false + type: bool: default: true + } + length_field_length: { + description: "Number of bytes representing the field length" + required: false + type: uint: default: 4 + } + length_field_offset: { + description: "Number of bytes in the header before the length field" + required: false + type: uint: default: 0 + } + max_frame_length: { + description: "Maximum frame length" + required: false + type: uint: default: 8388608 + } + } + } + max_frame_length: { + description: "Maximum frame length" + relevant_when: "method = \"varint_length_delimited\"" + required: false + type: uint: default: 8388608 + } + method: { + description: "The framing method." + required: false + type: string: { + default: "newline_delimited" + enum: { + bytes: "Byte frames are passed through as-is according to the underlying I/O boundaries (for example, split between messages or stream segments)." + character_delimited: "Byte frames which are delimited by a chosen character." + chunked_gelf: """ + Byte frames which are chunked GELF messages. + + [chunked_gelf]: https://go2docs.graylog.org/current/getting_in_log_data/gelf.html + """ + length_delimited: "Byte frames which are prefixed by an unsigned big-endian 32-bit integer indicating the length." + newline_delimited: "Byte frames which are delimited by a newline character." + octet_counting: """ + Byte frames according to the [octet counting][octet_counting] format. + + [octet_counting]: https://tools.ietf.org/html/rfc6587#section-3.4.1 + """ + varint_length_delimited: """ + Byte frames which are prefixed by a varint indicating the length. + This is compatible with protobuf's length-delimited encoding. + """ + } + } + } + newline_delimited: { + description: "Options for the newline delimited decoder." + relevant_when: "method = \"newline_delimited\"" + required: false + type: object: options: { + max_length: { + description: """ + The maximum length of the byte buffer. + + This length does *not* include the trailing delimiter. + + By default, no maximum length is enforced. If events are malformed, this can lead to + additional resource usage as events continue to be buffered in memory, and can potentially + lead to memory exhaustion in extreme cases. + + If there is a risk of processing malformed data, such as logs with user-controlled input, + consider setting the maximum length to a reasonably large value as a safety net. This + prevents processing from being unbounded. + """ + required: false + type: uint: {} + } + oversized_action: { + description: """ + The behavior when a line exceeds `max_length`. + + When set to `drop` (the default), the entire oversized line is discarded. + When set to `truncate`, the line is truncated to `max_length` bytes and the + remainder is discarded up to the next newline. + + This option has no effect if `max_length` is not set. + """ + required: false + type: string: { + default: "drop" + enum: { + drop: "Drop the entire oversized frame." + truncate: """ + Truncate the frame to the maximum allowed size and emit the partial content. + + The remainder of the oversized frame is discarded up to the next delimiter. + """ + } + } + } + } + } + octet_counting: { + description: "Options for the octet counting decoder." + relevant_when: "method = \"octet_counting\"" + required: false + type: object: options: max_length: { + description: "The maximum length of the byte buffer." + required: false + type: uint: {} + } + } + } + } + multiline: { + description: """ + Multiline aggregation configuration. + + If not specified, multiline aggregation is disabled. + """ + required: false + type: object: options: { + condition_pattern: { + description: """ + Regular expression pattern that is used to determine whether or not more lines should be read. + + This setting must be configured in conjunction with `mode`. + """ + required: true + type: string: examples: ["^[\\s]+", "\\\\$", "^(INFO|ERROR) ", ";$"] + } + mode: { + description: """ + Aggregation mode. + + This setting must be configured in conjunction with `condition_pattern`. + """ + required: true + type: string: enum: { + continue_past: """ + All consecutive lines matching this pattern, plus one additional line, are included in the group. + + This is useful in cases where a log message ends with a continuation marker, such as a backslash, indicating + that the following line is part of the same message. + """ + continue_through: """ + All consecutive lines matching this pattern are included in the group. + + The first line (the line that matched the start pattern) does not need to match the `ContinueThrough` pattern. + + This is useful in cases such as a Java stack trace, where some indicator in the line (such as a leading + whitespace) indicates that it is an extension of the proceeding line. + """ + halt_before: """ + All consecutive lines not matching this pattern are included in the group. + + This is useful where a log line contains a marker indicating that it begins a new message. + """ + halt_with: """ + All consecutive lines, up to and including the first line matching this pattern, are included in the group. + + This is useful where a log line ends with a termination marker, such as a semicolon. + """ + } + } + start_pattern: { + description: "Regular expression pattern that is used to match the start of a new message." + required: true + type: string: examples: ["^[\\s]+", "\\\\$", "^(INFO|ERROR) ", ";$"] + } + timeout_ms: { + description: """ + The maximum amount of time to wait for the next additional line, in milliseconds. + + Once this timeout is reached, the buffered message is guaranteed to be flushed, even if incomplete. + """ + required: true + type: uint: { + examples: [1000, 600000] + unit: "milliseconds" + } + } + } + } + queue: { + description: "Configuration options for the Storage Queue." + required: false + type: object: options: { + client_concurrency: { + description: """ + Number of concurrent tasks to create for polling the queue for messages. + + Defaults to the number of available CPUs on the system. + + Should not typically need to be changed, but it can sometimes be beneficial to raise this + value when there is a high rate of messages being pushed into the queue and the blobs + being fetched are small. In these cases, system resources may not be fully utilized + without fetching more messages per second, as the queue message consumption rate affects + the blob retrieval rate. + """ + required: false + type: uint: { + examples: [5] + unit: "tasks" + } + } + delete_failed_message: { + description: """ + Whether to delete non-retryable messages. + + If a message is rejected by the sink and not retryable, it is deleted from the queue. + With no dead-letter queue support, setting this to `false` means rejected messages are + redelivered indefinitely. + """ + required: false + type: bool: default: true + } + delete_message: { + description: """ + Whether to delete the message once it is processed. + + It can be useful to set this to `false` for debugging or during the initial setup. + """ + required: false + type: bool: default: true + } + max_number_of_messages: { + description: """ + Maximum number of messages to poll from the queue in a batch. + + Should be set to a smaller value when the blobs are large to help prevent the ingestion + of one blob from causing the others to exceed the `visibility_timeout_secs`. Valid + values are 1 - 32. + """ + required: false + type: uint: { + default: 10 + examples: [1] + } + } + poll_secs: { + description: """ + Maximum time to wait between polls of the queue when it is empty, in seconds. + + Azure Storage Queues have no server-side long polling, so an exponential client-side + backoff (starting at one second) is applied between empty polls, capped at this value. + Polling resumes immediately whenever a poll returns at least one message. + + Must be at least `1`. + """ + required: false + type: uint: { + default: 15 + unit: "seconds" + } + } + queue_name: { + description: """ + The name of the Storage Queue that receives the `Microsoft.Storage.BlobCreated` + notifications from the Event Grid subscription. + + This is a queue name, not a URL; the full URL is derived from the queue service endpoint. + """ + required: true + type: string: examples: ["vector-blob-events"] + } + visibility_timeout_secs: { + description: """ + The visibility timeout to use for messages, in seconds. + + This controls how long a message is left unavailable after it is received. If a message + is received, and takes longer than `visibility_timeout_secs` to process and delete the + message from the queue, it is made available again for another consumer. + + This can happen if there is an issue between consuming a message and deleting it. + """ + required: false + type: uint: { + default: 300 + unit: "seconds" + } + } + } + } + queue_endpoint: { + description: """ + The Azure Queue Storage service endpoint. + + By default the queue endpoint is derived from `account_name` or the connection string. + """ + required: false + type: string: examples: ["https://mylogstorage.queue.core.windows.net/"] + } + tls: { + description: "TLS configuration." + required: false + type: object: options: ca_file: { + description: """ + Absolute path to an additional CA certificate file. + + The certificate must be in PEM (X.509) format. + """ + required: false + type: string: examples: ["/path/to/certificate_authority.crt"] + } + } +} diff --git a/website/cue/reference/components/sources/internal_metrics.cue b/website/cue/reference/components/sources/internal_metrics.cue index d30f0391b418b..2b2ceafaf1ba6 100644 --- a/website/cue/reference/components/sources/internal_metrics.cue +++ b/website/cue/reference/components/sources/internal_metrics.cue @@ -917,6 +917,63 @@ components: sources: internal_metrics: { default_namespace: "vector" tags: _component_tags } + azure_blob_event_ignored_total: { + description: "The total number of times a blob notification in an Azure queue message was ignored (for an event that was not `Microsoft.Storage.BlobCreated`)." + type: "counter" + default_namespace: "vector" + tags: _component_tags & { + event_type: { + description: "The event type of the ignored notification." + required: true + } + } + } + azure_blob_processing_failed_duration_seconds: { + description: "The time taken to process an Azure blob that failed, in seconds." + type: "histogram" + default_namespace: "vector" + tags: _component_tags & { + container: { + description: "The name of the Azure Blob Storage container." + required: true + } + } + } + azure_blob_processing_succeeded_duration_seconds: { + description: "The time taken to process an Azure blob that succeeded, in seconds." + type: "histogram" + default_namespace: "vector" + tags: _component_tags & { + container: { + description: "The name of the Azure Blob Storage container." + required: true + } + } + } + azure_queue_message_delete_succeeded_total: { + description: "The total number of successful deletions of Azure queue messages." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + azure_queue_message_processing_succeeded_total: { + description: "The total number of Azure queue messages successfully processed." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + azure_queue_message_receive_succeeded_total: { + description: "The total number of times successfully receiving Azure queue messages." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + azure_queue_message_received_messages_total: { + description: "The total number of received Azure queue messages." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } s3_object_processing_failed_duration_seconds: { description: "The time taken to process an S3 object that failed, in seconds." type: "histogram" diff --git a/website/cue/reference/urls.cue b/website/cue/reference/urls.cue index 1c92e72090465..f432f474ae6c4 100644 --- a/website/cue/reference/urls.cue +++ b/website/cue/reference/urls.cue @@ -93,9 +93,11 @@ urls: { axiom_cloud: "https://cloud.axiom.co" azure_blob: "https://azure.microsoft.com/en-us/services/storage/blobs/" azure_blob_endpoints: "https://docs.microsoft.com/en-us/rest/api/storageservices/blob-service-rest-api" + azure_event_grid_blob: "https://learn.microsoft.com/azure/event-grid/event-schema-blob-storage" azure_monitor: "https://azure.microsoft.com/en-us/services/monitor/" azure_monitor_logs_endpoints: "https://docs.microsoft.com/en-us/rest/api/monitor/" azure_monitor_data_collector_deprecation: "https://learn.microsoft.com/previous-versions/azure/azure-monitor/logs/data-collector-api" + azure_storage_queue: "https://learn.microsoft.com/azure/storage/queues/storage-queues-introduction" azure_logs_ingestion_endpoints: "https://learn.microsoft.com/azure/azure-monitor/logs/logs-ingestion-api-overview" base16: "\(wikipedia)/wiki/Hexadecimal" base64: "\(wikipedia)/wiki/Base64" From 305ccd6b3a24db80f3a1dd0525d372bf0e168e09 Mon Sep 17 00:00:00 2001 From: Renizmy Date: Sun, 16 Aug 2026 20:19:54 +0200 Subject: [PATCH 2/5] take comments into consideration --- src/internal_events/azure_blob.rs | 6 +- src/sources/azure_blob/mod.rs | 14 +- src/sources/azure_blob/queue.rs | 207 +++++++++++++++++++++++++----- 3 files changed, 188 insertions(+), 39 deletions(-) diff --git a/src/internal_events/azure_blob.rs b/src/internal_events/azure_blob.rs index b24896348e9d2..b3c0367a97c32 100644 --- a/src/internal_events/azure_blob.rs +++ b/src/internal_events/azure_blob.rs @@ -121,7 +121,6 @@ impl InternalEvent for AzureQueueMessageProcessingSucceeded<'_> { pub struct AzureQueueMessageProcessingError<'a> { pub message_id: &'a str, pub error: &'a ProcessingError, - /// With no dead-letter queue, a growing dequeue count is the signal for a poison message. pub dequeue_count: Option, } @@ -152,7 +151,8 @@ impl InternalEvent for AzureQueueMessageProcessingError<'_> { ) .increment(1); } - ProcessingError::ContainerClient { .. } => { + ProcessingError::ContainerClient { .. } + | ProcessingError::ForeignStorageAccount { .. } => { counter!( CounterName::ComponentErrorsTotal, "error_code" => PROCESSING_ERROR_CODE, @@ -274,8 +274,6 @@ mod tests { cause, "failed to execute `reqwest` request", ); - - // `Display` alone stops at the outer context. assert_eq!(error.to_string(), "failed to execute `reqwest` request"); assert_eq!( error_chain(&error), diff --git a/src/sources/azure_blob/mod.rs b/src/sources/azure_blob/mod.rs index ffe22895fdb54..cb6becdd89a95 100644 --- a/src/sources/azure_blob/mod.rs +++ b/src/sources/azure_blob/mod.rs @@ -387,7 +387,6 @@ pub struct AzureStorageClientSource { credential: Option>, shared_key: Option<(String, String)>, proxy: ProxyConfig, - /// The custom root certificate, read once at startup rather than per client build. ca_pem: Option>, /// HTTP clients, cached per host. See `build_transport`. http_clients: RwLock>>, @@ -455,7 +454,6 @@ impl AzureStorageClientSource { .map_err(|e| { format!("Failed to read TLS CA file {}: {e}", ca_file.display()) })?; - // Parse eagerly so a malformed certificate fails at startup, not on first request. reqwest_13::Certificate::from_pem(&buf) .map_err(|e| format!("Invalid TLS CA file {}: {e}", ca_file.display()))?; info!("Adding TLS root certificate from {}.", ca_file.display()); @@ -476,6 +474,16 @@ impl AzureStorageClientSource { }) } + pub fn account_name(&self) -> Option { + if let Some(account) = self.parsed.account_name.as_ref() { + return Some(account.clone()); + } + if let Some(blob_endpoint) = self.parsed.blob_endpoint.as_deref() { + return queue::account_from_url(blob_endpoint); + } + None + } + /// Resolution order, mirroring `ParsedConnectionString::blob_account_endpoint`: /// 1. The explicit `queue_endpoint` configuration option. /// 2. A `QueueEndpoint` key in the connection string, which `ParsedConnectionString` @@ -593,7 +601,6 @@ impl AzureStorageClientSource { } if let Some(ca_pem) = &self.ca_pem { - // Already validated in `new`, so this cannot fail in practice. let cert = reqwest_13::Certificate::from_pem(ca_pem) .map_err(|e| format!("Invalid TLS root certificate: {e}"))?; reqwest_builder = reqwest_builder.add_root_certificate(cert); @@ -685,7 +692,6 @@ impl Policy for ContentLengthPolicy { } } -/// Extract the value of a connection string key (case-insensitive), if present. fn connection_string_value(connection_string: &str, key: &str) -> Option { connection_string.split(';').find_map(|seg| { let (k, v) = seg.trim().split_once('=')?; diff --git a/src/sources/azure_blob/queue.rs b/src/sources/azure_blob/queue.rs index 376abb1e5fd56..7cb75e15977e4 100644 --- a/src/sources/azure_blob/queue.rs +++ b/src/sources/azure_blob/queue.rs @@ -199,6 +199,13 @@ pub enum ProcessingError { subject: Option, url: Option, }, + #[snafu(display( + "Received notification for storage account '{received}', but source is configured for '{configured}'" + ))] + ForeignStorageAccount { + configured: String, + received: String, + }, #[snafu(display("Failed to build client for container {}: {}", container, message))] ContainerClient { message: String, container: String }, #[snafu(display("Failed to fetch blob {}/{}: {}", container, blob, source))] @@ -233,16 +240,28 @@ impl ProcessingError { Self::InvalidQueueMessage { .. } | Self::InvalidBlobPath { .. } => { error_type::PARSER_FAILED } - Self::ContainerClient { .. } => error_type::CONFIGURATION_FAILED, + Self::ForeignStorageAccount { .. } | Self::ContainerClient { .. } => { + error_type::CONFIGURATION_FAILED + } Self::GetBlob { .. } => error_type::REQUEST_FAILED, Self::ReadBlob { .. } => error_type::READER_FAILED, Self::PipelineSend { .. } => error_type::WRITER_FAILED, Self::ErrorAcknowledgement { .. } => error_type::ACKNOWLEDGMENT_FAILED, } } + + pub const fn is_non_retryable(&self) -> bool { + matches!( + self, + Self::InvalidQueueMessage { .. } + | Self::InvalidBlobPath { .. } + | Self::ForeignStorageAccount { .. } + ) + } } pub struct State { + account_name: Option, clients: AzureStorageClientSource, queue_client: QueueClient, container_clients: RwLock>>, @@ -314,15 +333,15 @@ impl Ingestor { } .into()); } - // A zero cap makes `ExponentialBackoff` yield `Duration::ZERO` forever, turning the - // empty-queue backoff into an unthrottled `GetMessages` loop on every polling task. if config.poll_secs == 0 { return Err(IngestorNewError::ZeroPollSecs.into()); } let queue_client = clients.queue_client(&config.queue_name)?; + let account_name = clients.account_name(); let state = Arc::new(State { + account_name, clients, queue_client, container_clients: RwLock::new(HashMap::new()), @@ -497,13 +516,19 @@ impl IngestorProcess { } } Err(err) => { - // Left in the queue to redeliver after the visibility timeout. There is no - // dead-letter queue, so a permanently failing message redelivers indefinitely. emit!(AzureQueueMessageProcessingError { message_id: &message_id, error: &err, dequeue_count, }); + if self.state.delete_failed_message && err.is_non_retryable() { + warn!( + message = "Deleting non-retryable failed queue message.", + message_id = %message_id, + error = %err, + ); + self.delete_message(&message_id, &pop_receipt).await; + } } } } @@ -546,6 +571,17 @@ impl IngestorProcess { url: notification.url.clone(), })?; + if let (Some(configured), Some(received)) = ( + self.state.account_name.as_deref(), + blob_ref.storage_account.as_deref(), + ) && !configured.eq_ignore_ascii_case(received) + { + return Err(ProcessingError::ForeignStorageAccount { + configured: configured.to_owned(), + received: received.to_owned(), + }); + } + let container_client = self.state.container_client(&blob_ref.container)?; let download_start = Instant::now(); @@ -665,12 +701,8 @@ impl IngestorProcess { Err(SendError::Timeout) => unreachable!("No timeout is configured here"), }; - // Up above, `lines` captures `read_error`, and eventually is captured by `stream`, - // so we explicitly drop it so that we can again utilize `read_error` below. - drop(stream); - // The BatchNotifier is cloned for each LogEvent in the batch stream, but the last - // reference must be dropped before the status of the batch is sent to the channel. + drop(stream); drop(batch); // Deliberately not the same as `result.is_ok()`: a rejected batch is removed from the @@ -729,7 +761,6 @@ impl IngestorProcess { } }; - // Measured after the acknowledgement, so the outcome is known before a histogram is picked. let duration = download_start.elapsed(); if delivered { emit!(AzureBlobProcessingSucceeded { @@ -881,7 +912,6 @@ fn decode_message_text(raw: &str) -> Cow<'_, str> { enum QueueEvent { CloudEvent(CloudEventEnvelope), EventGrid(EventGridEnvelope), - // Defensive: some tooling wraps Event Grid events in a one-element array. EventGridBatch(Vec), } @@ -907,6 +937,7 @@ impl QueueEvent { #[derive(Clone, Debug, Deserialize)] #[serde(rename_all = "camelCase")] struct EventGridEnvelope { + topic: Option, event_type: String, subject: String, event_time: Option>, @@ -915,11 +946,18 @@ struct EventGridEnvelope { impl From for BlobNotification { fn from(event: EventGridEnvelope) -> Self { + let storage_account = event + .data + .as_ref() + .and_then(|data| data.url.as_deref().and_then(account_from_url)) + .or_else(|| event.topic.as_deref().and_then(account_from_resource_id)); + BlobNotification { event_type: event.event_type, subject: Some(event.subject), event_time: event.event_time, url: event.data.and_then(|data| data.url), + storage_account, } } } @@ -930,6 +968,7 @@ struct CloudEventEnvelope { specversion: String, #[serde(rename = "type")] event_type: String, + source: Option, subject: Option, time: Option>, data: Option, @@ -937,29 +976,35 @@ struct CloudEventEnvelope { impl From for BlobNotification { fn from(event: CloudEventEnvelope) -> Self { + let storage_account = event + .data + .as_ref() + .and_then(|data| data.url.as_deref().and_then(account_from_url)) + .or_else(|| event.source.as_deref().and_then(account_from_resource_id)); + BlobNotification { event_type: event.event_type, subject: event.subject, event_time: event.time, url: event.data.and_then(|data| data.url), + storage_account, } } } -/// The `data` payload of a blob storage event; identical in both schemas. #[derive(Clone, Debug, Default, Deserialize)] #[serde(rename_all = "camelCase")] struct BlobEventData { url: Option, } -/// A single blob notification, normalized from either schema. #[derive(Clone, Debug)] struct BlobNotification { event_type: String, subject: Option, event_time: Option>, url: Option, + storage_account: Option, } /// The identity of a blob resolved from a notification. @@ -970,14 +1015,15 @@ struct BlobRef { blob: String, } -/// Prefers `subject`, which is stable and endpoint-agnostic, falling back to `data.url`. The -/// storage account name is only available from the URL. fn resolve_blob_ref(notification: &BlobNotification) -> Option { if let Some(subject) = notification.subject.as_deref() && let Some((container, blob)) = parse_subject(subject) { return Some(BlobRef { - storage_account: notification.url.as_deref().and_then(account_from_url), + storage_account: notification + .storage_account + .clone() + .or_else(|| notification.url.as_deref().and_then(account_from_url)), container, blob, }); @@ -986,10 +1032,6 @@ fn resolve_blob_ref(notification: &BlobNotification) -> Option { notification.url.as_deref().and_then(parse_blob_url) } -/// Parse an Event Grid subject: `/blobServices/default/containers/{container}/blobs/{path}`. -/// -/// The names are taken verbatim, because only `data.url` is percent-encoded. Decoding here would -/// corrupt any blob whose name contains a literal `%XX` sequence. fn parse_subject(subject: &str) -> Option<(String, String)> { let rest = subject.strip_prefix(SUBJECT_CONTAINER_PREFIX)?; let (container, blob) = rest.split_once(SUBJECT_BLOB_SEPARATOR)?; @@ -999,12 +1041,27 @@ fn parse_subject(subject: &str) -> Option<(String, String)> { Some((container.to_owned(), blob.to_owned())) } -/// Cloud style (`https://{account}.blob.core.windows.net/...`) uses the first host label; -/// path style (Azurite, `http://127.0.0.1:10000/{account}/...`) uses the first path segment. -fn account_from_url(url: &str) -> Option { +fn account_from_resource_id(resource_id: &str) -> Option { + let mut segments = resource_id.split('/'); + while let Some(seg) = segments.next() { + if seg.eq_ignore_ascii_case("storageAccounts") { + return segments + .next() + .filter(|account| !account.is_empty()) + .map(ToOwned::to_owned); + } + } + None +} + +fn is_cloud_storage_host(host: &str) -> bool { + host.contains(".blob.") || host.contains(".dfs.") +} + +pub(super) fn account_from_url(url: &str) -> Option { let parsed = url::Url::parse(url).ok()?; let host = parsed.host_str()?; - if host.contains(".blob.") { + if is_cloud_storage_host(host) { return host.split('.').next().map(ToOwned::to_owned); } parsed @@ -1014,19 +1071,16 @@ fn account_from_url(url: &str) -> Option { .map(percent_decode) } -/// Parse a blob URL into its account/container/blob parts. fn parse_blob_url(url: &str) -> Option { let parsed = url::Url::parse(url).ok()?; let host = parsed.host_str()?; let segments: Vec<&str> = parsed.path_segments()?.collect(); - let (account, container, blob_segments) = if host.contains(".blob.") { - // Cloud style: the account is the first host label. + let (account, container, blob_segments) = if is_cloud_storage_host(host) { let account = host.split('.').next()?; let (container, blob_segments) = segments.split_first()?; (account.to_owned(), *container, blob_segments) } else { - // Path style (Azurite): the account is the first path segment. let (account, rest) = segments.split_first()?; let (container, blob_segments) = rest.split_first()?; ((*account).to_owned(), *container, blob_segments) @@ -1180,7 +1234,6 @@ mod tests { parse_subject("/blobServices/default/containers/logs/blobs/app/out.log"), Some(("logs".to_owned(), "app/out.log".to_owned())) ); - // The subject carries the raw blob name, so a literal percent sequence is not an escape. assert_eq!( parse_subject("/blobServices/default/containers/logs/blobs/file name.log"), Some(("logs".to_owned(), "file name.log".to_owned())) @@ -1236,11 +1289,11 @@ mod tests { subject: Some("/blobServices/default/containers/from-subject/blobs/a.log".to_owned()), event_time: None, url: Some("https://myacct.blob.core.windows.net/from-url/b.log".to_owned()), + storage_account: None, }; let blob_ref = resolve_blob_ref(¬ification).unwrap(); assert_eq!(blob_ref.container, "from-subject"); assert_eq!(blob_ref.blob, "a.log"); - // account still comes from the URL, the only place it is present assert_eq!(blob_ref.storage_account.as_deref(), Some("myacct")); } @@ -1258,4 +1311,96 @@ mod tests { assert!(config.delete_message); assert!(config.delete_failed_message); } + + #[test] + fn url_extraction_dfs_style() { + assert_eq!( + account_from_url("https://myacct.dfs.core.windows.net/filesystem/app/out.log"), + Some("myacct".to_owned()) + ); + assert_eq!( + parse_blob_url("https://myacct.dfs.core.windows.net/filesystem/app/out.log"), + Some(BlobRef { + storage_account: Some("myacct".to_owned()), + container: "filesystem".to_owned(), + blob: "app/out.log".to_owned(), + }) + ); + } + + #[test] + fn arm_resource_id_extraction() { + assert_eq!( + account_from_resource_id( + "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/rg/providers/Microsoft.Storage/storageAccounts/myacct" + ), + Some("myacct".to_owned()) + ); + assert_eq!( + account_from_resource_id( + "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/rg/providers/Microsoft.Storage/storageAccounts/myacct/blobServices/default" + ), + Some("myacct".to_owned()) + ); + assert_eq!( + account_from_resource_id("/subscriptions/00000000-0000-0000-0000-000000000000"), + None + ); + } + + #[test] + fn extracts_account_from_topic_or_source_fallback() { + let event_grid_json = r#"{ + "topic": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/rg/providers/Microsoft.Storage/storageAccounts/topicacct", + "subject": "/blobServices/default/containers/logs/blobs/app.log", + "eventType": "Microsoft.Storage.BlobCreated", + "id": "1", + "data": {} + }"#; + let notifications = parse(event_grid_json); + assert_eq!(notifications.len(), 1); + let blob_ref = resolve_blob_ref(¬ifications[0]).unwrap(); + assert_eq!(blob_ref.storage_account.as_deref(), Some("topicacct")); + + let cloud_event_json = r#"{ + "specversion": "1.0", + "type": "Microsoft.Storage.BlobCreated", + "source": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/rg/providers/Microsoft.Storage/storageAccounts/sourceacct", + "subject": "/blobServices/default/containers/logs/blobs/app.log", + "id": "1", + "data": {} + }"#; + let notifications = parse(cloud_event_json); + assert_eq!(notifications.len(), 1); + let blob_ref = resolve_blob_ref(¬ifications[0]).unwrap(); + assert_eq!(blob_ref.storage_account.as_deref(), Some("sourceacct")); + } + + #[test] + fn non_retryable_errors_classification() { + let json_err = serde_json::from_str::("bad json").unwrap_err(); + let err1 = ProcessingError::InvalidQueueMessage { + source: json_err, + message_id: "1".to_owned(), + }; + assert!(err1.is_non_retryable()); + + let err2 = ProcessingError::InvalidBlobPath { + subject: None, + url: None, + }; + assert!(err2.is_non_retryable()); + + let err3 = ProcessingError::ForeignStorageAccount { + configured: "acct_a".to_owned(), + received: "acct_b".to_owned(), + }; + assert!(err3.is_non_retryable()); + + let err4 = ProcessingError::ContainerClient { + message: "err".to_owned(), + container: "c".to_owned(), + }; + assert!(!err4.is_non_retryable()); + } } From 97be5b4f10630dde180c5953ecee90a03a934e21 Mon Sep 17 00:00:00 2001 From: Renizmy Date: Sun, 16 Aug 2026 22:06:31 +0200 Subject: [PATCH 3/5] fix other comments --- src/sources/azure_blob/queue.rs | 134 ++++++++++++++++++++++++++++---- 1 file changed, 118 insertions(+), 16 deletions(-) diff --git a/src/sources/azure_blob/queue.rs b/src/sources/azure_blob/queue.rs index 7cb75e15977e4..e802afeb3747d 100644 --- a/src/sources/azure_blob/queue.rs +++ b/src/sources/azure_blob/queue.rs @@ -8,7 +8,8 @@ use std::{ time::{Duration, Instant}, }; -use azure_storage_blob::BlobContainerClient; +use azure_core::http::{Etag, StatusCode}; +use azure_storage_blob::{BlobContainerClient, models::BlobClientDownloadOptions}; use azure_storage_queue::{ QueueClient, models::{QueueClientReceiveMessagesOptions, ReceivedMessage}, @@ -125,10 +126,7 @@ pub(super) struct Config { #[configurable(metadata(docs::type_unit = "tasks"))] #[configurable(metadata(docs::examples = 5))] pub(super) client_concurrency: Option, - - /// Whether to delete the message once it is processed. - /// - /// It can be useful to set this to `false` for debugging or during the initial setup. + #[serde(default = "default_true")] #[derivative(Default(value = "default_true()"))] pub(super) delete_message: bool, @@ -250,13 +248,28 @@ impl ProcessingError { } } - pub const fn is_non_retryable(&self) -> bool { - matches!( - self, + pub fn is_non_retryable(&self) -> bool { + match self { Self::InvalidQueueMessage { .. } - | Self::InvalidBlobPath { .. } - | Self::ForeignStorageAccount { .. } - ) + | Self::InvalidBlobPath { .. } + | Self::ForeignStorageAccount { .. } => true, + // 404/412 are permanent: the blob is gone, or has changed since the notification. + Self::GetBlob { source, .. } => matches!( + source.http_status(), + Some(StatusCode::NotFound | StatusCode::PreconditionFailed) + ), + _ => false, + } + } +} + +/// Prefers a retryable error over a non-retryable one, so one permanent failure in a batch +/// can't cause the message to be deleted while a sibling failure still needs a retry. +fn merge_batch_errors(existing: ProcessingError, new: ProcessingError) -> ProcessingError { + if existing.is_non_retryable() && !new.is_non_retryable() { + new + } else { + existing } } @@ -548,10 +561,20 @@ impl IngestorProcess { .unwrap_or_else(|| "".to_owned()), })?; + let mut error: Option = None; for notification in event.into_notifications() { - self.handle_blob_notification(notification).await?; + if let Err(err) = self.handle_blob_notification(notification).await { + error = Some(match error { + Some(existing) => merge_batch_errors(existing, err), + None => err, + }); + } + } + + match error { + Some(err) => Err(err), + None => Ok(()), } - Ok(()) } async fn handle_blob_notification( @@ -586,9 +609,19 @@ impl IngestorProcess { let download_start = Instant::now(); + // Condition on the notification's etag so an overwritten blob fails instead of silently + // serving the wrong content. + let download_options = blob_ref + .etag + .as_deref() + .map(|etag| BlobClientDownloadOptions { + if_match: Some(Etag::from(etag)), + ..Default::default() + }); + let object = container_client .blob_client(&blob_ref.blob) - .download(None) + .download(download_options) .await .context(GetBlobSnafu { container: blob_ref.container.clone(), @@ -701,7 +734,6 @@ impl IngestorProcess { Err(SendError::Timeout) => unreachable!("No timeout is configured here"), }; - drop(stream); drop(batch); @@ -951,6 +983,7 @@ impl From for BlobNotification { .as_ref() .and_then(|data| data.url.as_deref().and_then(account_from_url)) .or_else(|| event.topic.as_deref().and_then(account_from_resource_id)); + let etag = event.data.as_ref().and_then(|data| data.e_tag.clone()); BlobNotification { event_type: event.event_type, @@ -958,6 +991,7 @@ impl From for BlobNotification { event_time: event.event_time, url: event.data.and_then(|data| data.url), storage_account, + etag, } } } @@ -981,6 +1015,7 @@ impl From for BlobNotification { .as_ref() .and_then(|data| data.url.as_deref().and_then(account_from_url)) .or_else(|| event.source.as_deref().and_then(account_from_resource_id)); + let etag = event.data.as_ref().and_then(|data| data.e_tag.clone()); BlobNotification { event_type: event.event_type, @@ -988,6 +1023,7 @@ impl From for BlobNotification { event_time: event.time, url: event.data.and_then(|data| data.url), storage_account, + etag, } } } @@ -996,6 +1032,7 @@ impl From for BlobNotification { #[serde(rename_all = "camelCase")] struct BlobEventData { url: Option, + e_tag: Option, } #[derive(Clone, Debug)] @@ -1005,6 +1042,7 @@ struct BlobNotification { event_time: Option>, url: Option, storage_account: Option, + etag: Option, } /// The identity of a blob resolved from a notification. @@ -1013,6 +1051,7 @@ struct BlobRef { storage_account: Option, container: String, blob: String, + etag: Option, } fn resolve_blob_ref(notification: &BlobNotification) -> Option { @@ -1026,10 +1065,18 @@ fn resolve_blob_ref(notification: &BlobNotification) -> Option { .or_else(|| notification.url.as_deref().and_then(account_from_url)), container, blob, + etag: notification.etag.clone(), }); } - notification.url.as_deref().and_then(parse_blob_url) + notification + .url + .as_deref() + .and_then(parse_blob_url) + .map(|blob_ref| BlobRef { + etag: notification.etag.clone(), + ..blob_ref + }) } fn parse_subject(subject: &str) -> Option<(String, String)> { @@ -1100,6 +1147,7 @@ fn parse_blob_url(url: &str) -> Option { storage_account: Some(percent_decode(&account)), container: percent_decode(container), blob, + etag: None, }) } @@ -1175,6 +1223,7 @@ mod tests { storage_account: Some("myacct".to_owned()), container: "logs".to_owned(), blob: "app/out.log".to_owned(), + etag: Some("0x8DC0000000000000".to_owned()), }) ); } @@ -1254,6 +1303,7 @@ mod tests { storage_account: Some("myacct".to_owned()), container: "logs".to_owned(), blob: "app/out.log".to_owned(), + etag: None, }) ); } @@ -1266,6 +1316,7 @@ mod tests { storage_account: Some("devstoreaccount1".to_owned()), container: "logs".to_owned(), blob: "app/out.log".to_owned(), + etag: None, }) ); } @@ -1278,6 +1329,7 @@ mod tests { storage_account: Some("myacct".to_owned()), container: "logs".to_owned(), blob: "file name.log".to_owned(), + etag: None, }) ); } @@ -1290,11 +1342,13 @@ mod tests { event_time: None, url: Some("https://myacct.blob.core.windows.net/from-url/b.log".to_owned()), storage_account: None, + etag: Some("\"etag-value\"".to_owned()), }; let blob_ref = resolve_blob_ref(¬ification).unwrap(); assert_eq!(blob_ref.container, "from-subject"); assert_eq!(blob_ref.blob, "a.log"); assert_eq!(blob_ref.storage_account.as_deref(), Some("myacct")); + assert_eq!(blob_ref.etag.as_deref(), Some("\"etag-value\"")); } #[test] @@ -1324,6 +1378,7 @@ mod tests { storage_account: Some("myacct".to_owned()), container: "filesystem".to_owned(), blob: "app/out.log".to_owned(), + etag: None, }) ); } @@ -1403,4 +1458,51 @@ mod tests { }; assert!(!err4.is_non_retryable()); } + + fn get_blob_error(status: StatusCode) -> ProcessingError { + ProcessingError::GetBlob { + source: azure_core::Error::from(azure_core::error::ErrorKind::HttpResponse { + status, + error_code: None, + raw_response: None, + }), + container: "logs".to_owned(), + blob: "app/out.log".to_owned(), + } + } + + #[test] + fn get_blob_non_retryable_classification() { + assert!(get_blob_error(StatusCode::NotFound).is_non_retryable()); + assert!(get_blob_error(StatusCode::PreconditionFailed).is_non_retryable()); + assert!(!get_blob_error(StatusCode::ServiceUnavailable).is_non_retryable()); + assert!(!get_blob_error(StatusCode::InternalServerError).is_non_retryable()); + } + + #[test] + fn resolve_blob_ref_carries_etag_from_url_fallback() { + let notification = BlobNotification { + event_type: BLOB_CREATED_EVENT_TYPE.to_owned(), + subject: None, + event_time: None, + url: Some("https://myacct.blob.core.windows.net/logs/app/out.log".to_owned()), + storage_account: None, + etag: Some("\"etag-value\"".to_owned()), + }; + let blob_ref = resolve_blob_ref(¬ification).unwrap(); + assert_eq!(blob_ref.etag.as_deref(), Some("\"etag-value\"")); + } + + #[test] + fn batch_error_merging_prefers_retryable() { + let non_retryable = || ProcessingError::InvalidBlobPath { + subject: None, + url: None, + }; + let retryable = || get_blob_error(StatusCode::ServiceUnavailable); + + assert!(!merge_batch_errors(non_retryable(), retryable()).is_non_retryable()); + assert!(!merge_batch_errors(retryable(), non_retryable()).is_non_retryable()); + assert!(merge_batch_errors(non_retryable(), non_retryable()).is_non_retryable()); + } } From e582702785d5224994308fd5a2372e7210da35ae Mon Sep 17 00:00:00 2001 From: Renizmy Date: Mon, 17 Aug 2026 19:29:14 +0200 Subject: [PATCH 4/5] take new coments into consideration --- src/internal_events/azure_blob.rs | 3 +- src/sources/azure_blob/integration_tests.rs | 24 +++-- src/sources/azure_blob/mod.rs | 12 ++- src/sources/azure_blob/queue.rs | 93 ++++++++++++------- .../sources/generated/azure_blob.cue | 5 +- 5 files changed, 90 insertions(+), 47 deletions(-) diff --git a/src/internal_events/azure_blob.rs b/src/internal_events/azure_blob.rs index b3c0367a97c32..06b89a2e5114e 100644 --- a/src/internal_events/azure_blob.rs +++ b/src/internal_events/azure_blob.rs @@ -188,7 +188,8 @@ impl InternalEvent for AzureQueueMessageProcessingError<'_> { ) .increment(1); } - ProcessingError::ErrorAcknowledgement { .. } => { + ProcessingError::ErrorAcknowledgement { .. } + | ProcessingError::RejectedBySink { .. } => { counter!( CounterName::ComponentErrorsTotal, "error_code" => PROCESSING_ERROR_CODE, diff --git a/src/sources/azure_blob/integration_tests.rs b/src/sources/azure_blob/integration_tests.rs index 7fdfbd2177cf1..e591bb875de96 100644 --- a/src/sources/azure_blob/integration_tests.rs +++ b/src/sources/azure_blob/integration_tests.rs @@ -123,7 +123,7 @@ async fn upload_blob( payload: Vec, content_type: Option<&str>, content_encoding: Option<&str>, -) { +) -> String { let options = BlockBlobClientUploadOptions { blob_content_type: content_type.map(ToOwned::to_owned), blob_content_encoding: content_encoding.map(ToOwned::to_owned), @@ -132,17 +132,26 @@ async fn upload_blob( partition_size: Some(NonZeroU64::new(64 * 1024 * 1024).expect("nonzero")), ..Default::default() }; - container_client + let result = container_client .blob_client(blob_name) .upload(RequestContent::from(payload), Some(options)) .await .expect("Failed to upload blob"); + String::from(result.etag.expect("upload response is missing an ETag")) } -fn notification_body(container: &str, blob: &str, format: NotificationFormat) -> String { +fn notification_body( + container: &str, + blob: &str, + etag: &str, + format: NotificationFormat, +) -> String { let address = azurite_address(); let url = format!("http://{address}:10000/devstoreaccount1/{container}/{blob}"); let subject = format!("/blobServices/default/containers/{container}/blobs/{blob}"); + // Real Event Grid delivers `data.eTag` unquoted; strip the quotes Azurite returns so the test + // mirrors that format (the source re-quotes it for `if_match`). + let etag = serde_json::to_string(etag.trim_matches('"')).expect("ETag serializes to JSON"); let json = match format { NotificationFormat::EventGridBase64 | NotificationFormat::EventGridRaw => format!( @@ -156,7 +165,7 @@ fn notification_body(container: &str, blob: &str, format: NotificationFormat) -> "api": "PutBlob", "blobType": "BlockBlob", "url": "{url}", - "eTag": "0x8DC0000000000000" + "eTag": {etag} }}, "dataVersion": "", "metadataVersion": "1" @@ -174,7 +183,7 @@ fn notification_body(container: &str, blob: &str, format: NotificationFormat) -> "api": "PutBlob", "blobType": "BlockBlob", "url": "{url}", - "eTag": "0x8DC0000000000000" + "eTag": {etag} }} }}"# ), @@ -238,7 +247,7 @@ async fn test_event( let (container_client, queue_client, container_name) = test_clients(&config, &queue_name).await; - upload_blob( + let etag = upload_blob( &container_client, &blob_name, payload, @@ -249,7 +258,7 @@ async fn test_event( enqueue_notification( &queue_client, - notification_body(&container_name, &blob_name, format), + notification_body(&container_name, &blob_name, &etag, format), ) .await; @@ -677,6 +686,7 @@ async fn azure_blob_ignores_other_event_types() { let body = notification_body( &container_name, "some.log", + "0x0", NotificationFormat::EventGridRaw, ) .replace( diff --git a/src/sources/azure_blob/mod.rs b/src/sources/azure_blob/mod.rs index cb6becdd89a95..c1b5520a27f99 100644 --- a/src/sources/azure_blob/mod.rs +++ b/src/sources/azure_blob/mod.rs @@ -328,8 +328,16 @@ impl AzureBlobConfig { (Some(_), _, Some(_)) => { return Err("Cannot provide both `connection_string` and `blob_endpoint`".into()); } - (_, Some(_), Some(_)) => { - return Err("Cannot provide both `account_name` and `blob_endpoint`".into()); + (None, Some(account_name), Some(blob_endpoint)) => { + if self.auth.is_none() { + return Err("`auth` configuration must be provided when using `account_name` and `blob_endpoint`".into()); + } + let blob_endpoint = if blob_endpoint.ends_with('/') { + blob_endpoint.clone() + } else { + format!("{blob_endpoint}/") + }; + format!("AccountName={account_name};BlobEndpoint={blob_endpoint}") } }; diff --git a/src/sources/azure_blob/queue.rs b/src/sources/azure_blob/queue.rs index e802afeb3747d..6013e34f247ac 100644 --- a/src/sources/azure_blob/queue.rs +++ b/src/sources/azure_blob/queue.rs @@ -114,19 +114,14 @@ pub(super) struct Config { #[configurable(metadata(docs::examples = 1))] pub(super) max_number_of_messages: u32, - /// Number of concurrent tasks to create for polling the queue for messages. - /// - /// Defaults to the number of available CPUs on the system. - /// - /// Should not typically need to be changed, but it can sometimes be beneficial to raise this - /// value when there is a high rate of messages being pushed into the queue and the blobs - /// being fetched are small. In these cases, system resources may not be fully utilized - /// without fetching more messages per second, as the queue message consumption rate affects - /// the blob retrieval rate. #[configurable(metadata(docs::type_unit = "tasks"))] #[configurable(metadata(docs::examples = 5))] pub(super) client_concurrency: Option, - + + /// Whether to delete a queue message once its blob has been successfully processed. + /// + /// When set to `false`, messages remain in the queue and are redelivered after each + /// visibility timeout, which is useful for debugging but otherwise duplicates data. #[serde(default = "default_true")] #[derivative(Default(value = "default_true()"))] pub(super) delete_message: bool, @@ -174,6 +169,11 @@ pub(super) enum IngestorNewError { InvalidVisibilityTimeout { seconds: u32 }, #[snafu(display("Invalid value for poll_secs 0, must be at least 1 second"))] ZeroPollSecs, + #[snafu(display( + "Could not determine the storage account name. Set `account_name` (it may be combined with \ + a custom `blob_endpoint`) so notifications can be validated against the configured account." + ))] + MissingAccountName, } #[allow(clippy::large_enum_variant)] @@ -230,6 +230,8 @@ pub enum ProcessingError { blob ))] ErrorAcknowledgement { container: String, blob: String }, + #[snafu(display("Sink rejected events for blob {}/{}", container, blob))] + RejectedBySink { container: String, blob: String }, } impl ProcessingError { @@ -244,7 +246,9 @@ impl ProcessingError { Self::GetBlob { .. } => error_type::REQUEST_FAILED, Self::ReadBlob { .. } => error_type::READER_FAILED, Self::PipelineSend { .. } => error_type::WRITER_FAILED, - Self::ErrorAcknowledgement { .. } => error_type::ACKNOWLEDGMENT_FAILED, + Self::ErrorAcknowledgement { .. } | Self::RejectedBySink { .. } => { + error_type::ACKNOWLEDGMENT_FAILED + } } } @@ -252,7 +256,8 @@ impl ProcessingError { match self { Self::InvalidQueueMessage { .. } | Self::InvalidBlobPath { .. } - | Self::ForeignStorageAccount { .. } => true, + | Self::ForeignStorageAccount { .. } + | Self::RejectedBySink { .. } => true, // 404/412 are permanent: the blob is gone, or has changed since the notification. Self::GetBlob { source, .. } => matches!( source.http_status(), @@ -274,7 +279,7 @@ fn merge_batch_errors(existing: ProcessingError, new: ProcessingError) -> Proces } pub struct State { - account_name: Option, + account_name: String, clients: AzureStorageClientSource, queue_client: QueueClient, container_clients: RwLock>>, @@ -351,7 +356,9 @@ impl Ingestor { } let queue_client = clients.queue_client(&config.queue_name)?; - let account_name = clients.account_name(); + let account_name = clients + .account_name() + .ok_or(IngestorNewError::MissingAccountName)?; let state = Arc::new(State { account_name, @@ -594,10 +601,9 @@ impl IngestorProcess { url: notification.url.clone(), })?; - if let (Some(configured), Some(received)) = ( - self.state.account_name.as_deref(), - blob_ref.storage_account.as_deref(), - ) && !configured.eq_ignore_ascii_case(received) + let configured = self.state.account_name.as_str(); + if let Some(received) = blob_ref.storage_account.as_deref() + && !configured.eq_ignore_ascii_case(received) { return Err(ProcessingError::ForeignStorageAccount { configured: configured.to_owned(), @@ -610,12 +616,12 @@ impl IngestorProcess { let download_start = Instant::now(); // Condition on the notification's etag so an overwritten blob fails instead of silently - // serving the wrong content. + // serving stale content. let download_options = blob_ref .etag .as_deref() .map(|etag| BlobClientDownloadOptions { - if_match: Some(Etag::from(etag)), + if_match: Some(Etag::from(quoted_etag(etag).as_str())), ..Default::default() }); @@ -774,21 +780,10 @@ impl IngestorProcess { container: blob_ref.container, blob: blob_ref.blob, }), - BatchStatus::Rejected => { - if self.state.delete_failed_message { - warn!( - message = "Blob from queue notification was rejected. Deleting failed message.", - container = blob_ref.container, - blob = blob_ref.blob, - ); - Ok(()) - } else { - Err(ProcessingError::ErrorAcknowledgement { - container: blob_ref.container, - blob: blob_ref.blob, - }) - } - } + BatchStatus::Rejected => Err(ProcessingError::RejectedBySink { + container: blob_ref.container, + blob: blob_ref.blob, + }), }, } }; @@ -1157,6 +1152,18 @@ fn percent_decode(s: &str) -> String { .into_owned() } +/// Wraps an ETag in the quotes an HTTP `If-Match` entity-tag requires. +/// +/// Event Grid delivers `data.eTag` unquoted (e.g. `0x8D…`), while the Blob service expects a +/// quoted entity-tag (`"0x8D…"`). Values that already carry surrounding quotes are left as-is. +fn quoted_etag(etag: &str) -> String { + if etag.len() >= 2 && etag.starts_with('"') && etag.ends_with('"') { + etag.to_owned() + } else { + format!("\"{etag}\"") + } +} + fn to_chrono_timestamp(ts: azure_core::time::OffsetDateTime) -> Option> { Utc.timestamp_opt(ts.unix_timestamp(), ts.nanosecond()) .single() @@ -1457,6 +1464,12 @@ mod tests { container: "c".to_owned(), }; assert!(!err4.is_non_retryable()); + + let err5 = ProcessingError::RejectedBySink { + container: "c".to_owned(), + blob: "b".to_owned(), + }; + assert!(err5.is_non_retryable()); } fn get_blob_error(status: StatusCode) -> ProcessingError { @@ -1479,6 +1492,16 @@ mod tests { assert!(!get_blob_error(StatusCode::InternalServerError).is_non_retryable()); } + #[test] + fn quoted_etag_wraps_unquoted_and_preserves_quoted() { + assert_eq!(quoted_etag("0x8DC0000000000000"), "\"0x8DC0000000000000\""); + assert_eq!( + quoted_etag("\"0x8DC0000000000000\""), + "\"0x8DC0000000000000\"" + ); + assert_eq!(quoted_etag("\""), "\"\"\""); + } + #[test] fn resolve_blob_ref_carries_etag_from_url_fallback() { let notification = BlobNotification { diff --git a/website/cue/reference/components/sources/generated/azure_blob.cue b/website/cue/reference/components/sources/generated/azure_blob.cue index 1db462f2c24ab..de93d2e445b6f 100644 --- a/website/cue/reference/components/sources/generated/azure_blob.cue +++ b/website/cue/reference/components/sources/generated/azure_blob.cue @@ -844,9 +844,10 @@ generated: components: sources: azure_blob: configuration: { } delete_message: { description: """ - Whether to delete the message once it is processed. + Whether to delete a queue message once its blob has been successfully processed. - It can be useful to set this to `false` for debugging or during the initial setup. + When set to `false`, messages remain in the queue and are redelivered after each + visibility timeout, which is useful for debugging but otherwise duplicates data. """ required: false type: bool: default: true From abe251f29e3c7e9e94723120869a21e9d6123ca5 Mon Sep 17 00:00:00 2001 From: Renizmy Date: Mon, 17 Aug 2026 21:55:18 +0200 Subject: [PATCH 5/5] trigger ci --- src/internal_events/azure_blob.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/internal_events/azure_blob.rs b/src/internal_events/azure_blob.rs index 06b89a2e5114e..90a046915776b 100644 --- a/src/internal_events/azure_blob.rs +++ b/src/internal_events/azure_blob.rs @@ -247,8 +247,6 @@ pub struct AzureBlobEventIgnored<'a> { impl InternalEvent for AzureBlobEventIgnored<'_> { fn emit(self) { - // Not a warning: an unfiltered Event Grid subscription delivers these as a matter of - // course, and `azure_blob_event_ignored_total` is the signal for operators. debug!( message = "Ignored queue message for an event that was not BlobCreated.", event_type = %self.event_type,