feat(azure_blob source): add Azure Blob Storage source

Cargo.toml

@@ -323,11 +323,12 @@ aws-smithy-runtime-api = { version = "1.7.3", default-features = false, optional
 aws-smithy-types = { version = "1.2.11", default-features = false, features = ["rt-tokio"], optional = true }
 
 # Azure
-azure_core = { version = "1.0", default-features = false, features = ["reqwest", "hmac_openssl"], optional = true }
+azure_core = { version = "1.0", default-features = false, features = ["reqwest", "hmac_openssl", "tokio"], optional = true }
 azure_identity = { version = "1.0", default-features = false, features = ["client_certificate"], optional = true }
 
 # Azure Storage
 azure_storage_blob = { version = "1.0", default-features = false, optional = true }
+azure_storage_queue = { version = "1.0", default-features = false, optional = true }
 
 # OpenDAL
 opendal = { version = "0.54", default-features = false, features = ["services-webhdfs"], optional = true }
@@ -686,6 +687,7 @@ sources-logs = [
   "sources-aws_kinesis_firehose",
   "sources-aws_s3",
   "sources-aws_sqs",
+  "sources-azure_blob",
   "sources-datadog_agent",
   "sources-demo_logs",
   "sources-docker_logs",
@@ -739,6 +741,7 @@ sources-aws_ecs_metrics = ["sources-utils-http-client"]
 sources-aws_kinesis_firehose = ["dep:base64", "sources-http_server", "sources-utils-http-encoding"]
 sources-aws_s3 = ["aws-core", "dep:aws-sdk-sqs", "dep:aws-sdk-s3", "dep:async-compression", "sources-aws_sqs", "tokio-util/io"]
 sources-aws_sqs = ["aws-core", "dep:aws-sdk-sqs"]
+sources-azure_blob = ["dep:azure_core", "dep:azure_identity", "dep:azure_storage_blob", "dep:azure_storage_queue", "dep:async-compression", "dep:base64", "tokio-util/io"]
 sources-datadog_agent = ["sources-utils-http-encoding", "protobuf-build", "dep:prost"]
 sources-demo_logs = ["dep:fakedata"]
 sources-dnstap = ["sources-utils-net-tcp", "dep:base64", "dep:hickory-proto", "dep:dnsmsg-parser", "dep:dnstap-parser", "protobuf-build", "dep:prost", "vector-vrl-functions/dnstap"]
@@ -1069,7 +1072,7 @@ aws-s3-integration-tests = ["sinks-aws_s3", "sources-aws_s3"]
 aws-sqs-integration-tests = ["sinks-aws_sqs"]
 aws-sns-integration-tests = ["sinks-aws_sns"]
 axiom-integration-tests = ["sinks-axiom"]
-azure-blob-integration-tests = ["sinks-azure_blob"]
+azure-blob-integration-tests = ["sinks-azure_blob", "sources-azure_blob"]
 azure-logs-ingestion-integration-tests = ["sinks-azure_logs_ingestion"]
 clickhouse-integration-tests = ["sinks-clickhouse"]
 databend-integration-tests = ["sinks-databend"]

LICENSE-3rdparty.csv

@@ -103,6 +103,7 @@ azure_core,https://github.com/azure/azure-sdk-for-rust,MIT,Microsoft
 azure_core_macros,https://github.com/azure/azure-sdk-for-rust,MIT,Microsoft
 azure_identity,https://github.com/azure/azure-sdk-for-rust,MIT,Microsoft
 azure_storage_blob,https://github.com/azure/azure-sdk-for-rust,MIT,Microsoft
+azure_storage_queue,https://github.com/azure/azure-sdk-for-rust,MIT,Microsoft
 backon,https://github.com/Xuanwo/backon,Apache-2.0,The backon Authors
 base16,https://github.com/thomcc/rust-base16,CC0-1.0,Thom Chiovoloni <tchiovoloni@mozilla.com>
 base16ct,https://github.com/RustCrypto/formats/tree/master/base16ct,Apache-2.0 OR MIT,RustCrypto Developers

changelog.d/azure_blob_source.feature.md

@@ -0,0 +1,14 @@
+Added a new `azure_blob` source that collects logs from blobs in Azure Blob
+Storage. Discovery is event-driven: an Event Grid subscription on the storage
+account delivers `Microsoft.Storage.BlobCreated` notifications to an Azure
+Storage Queue, which Vector polls. Newly created blobs are downloaded,
+optionally decompressed (gzip/zstd, auto-detected), decoded with any codec,
+and the queue message is deleted once the events are durably accepted by the
+pipeline (end-to-end acknowledgements).
+
+Both the Event Grid and CloudEvents 1.0 notification schemas are supported and
+auto-detected. Authentication reuses the same options as the `azure_blob`
+sink: connection string (account key or SAS) and all Azure token credentials
+(Managed Identity, Service Principal, Workload Identity, Azure CLI).
+
+authors: Renizmy

lib/vector-common/src/internal_event/metric_name.rs

@@ -26,6 +26,11 @@ pub enum CounterName {
     AggregateFailedUpdates,
     AggregateFlushesTotal,
     ApiStartedTotal,
+    AzureBlobEventIgnoredTotal,
+    AzureQueueMessageDeleteSucceededTotal,
+    AzureQueueMessageProcessingSucceededTotal,
+    AzureQueueMessageReceiveSucceededTotal,
+    AzureQueueMessageReceivedMessagesTotal,
     CheckpointsTotal,
     ChecksumErrorsTotal,
     CollectCompletedTotal,
@@ -122,6 +127,8 @@ pub enum HistogramName {
     AdaptiveConcurrencyObservedRtt,
     AdaptiveConcurrencyPastRttMean,
     AdaptiveConcurrencyReachedLimit,
+    AzureBlobProcessingSucceededDurationSeconds,
+    AzureBlobProcessingFailedDurationSeconds,
     S3ObjectProcessingSucceededDurationSeconds,
     S3ObjectProcessingFailedDurationSeconds,
     CollectDurationSeconds,
@@ -153,6 +160,12 @@ impl HistogramName {
             Self::AdaptiveConcurrencyObservedRtt => "adaptive_concurrency_observed_rtt",
             Self::AdaptiveConcurrencyPastRttMean => "adaptive_concurrency_past_rtt_mean",
             Self::AdaptiveConcurrencyReachedLimit => "adaptive_concurrency_reached_limit",
+            Self::AzureBlobProcessingSucceededDurationSeconds => {
+                "azure_blob_processing_succeeded_duration_seconds"
+            }
+            Self::AzureBlobProcessingFailedDurationSeconds => {
+                "azure_blob_processing_failed_duration_seconds"
+            }
             Self::S3ObjectProcessingSucceededDurationSeconds => {
                 "s3_object_processing_succeeded_duration_seconds"
             }
@@ -287,6 +300,19 @@ impl CounterName {
             Self::AggregateFailedUpdates => "aggregate_failed_updates",
             Self::AggregateFlushesTotal => "aggregate_flushes_total",
             Self::ApiStartedTotal => "api_started_total",
+            Self::AzureBlobEventIgnoredTotal => "azure_blob_event_ignored_total",
+            Self::AzureQueueMessageDeleteSucceededTotal => {
+                "azure_queue_message_delete_succeeded_total"
+            }
+            Self::AzureQueueMessageProcessingSucceededTotal => {
+                "azure_queue_message_processing_succeeded_total"
+            }
+            Self::AzureQueueMessageReceiveSucceededTotal => {
+                "azure_queue_message_receive_succeeded_total"
+            }
+            Self::AzureQueueMessageReceivedMessagesTotal => {
+                "azure_queue_message_received_messages_total"
+            }
             Self::CheckpointsTotal => "checkpoints_total",
             Self::ChecksumErrorsTotal => "checksum_errors_total",
             Self::CollectCompletedTotal => "collect_completed_total",