diff --git a/changelog.d/avro_date_time_millis_encoding.enhancement.md b/changelog.d/avro_date_time_millis_encoding.enhancement.md new file mode 100644 index 0000000000000..b8df0d62de341 --- /dev/null +++ b/changelog.d/avro_date_time_millis_encoding.enhancement.md @@ -0,0 +1,3 @@ +The Avro codec now supports encoding and decoding Date and TimeMillis values. + +authors: omwbennett diff --git a/lib/codecs/src/decoding/format/avro.rs b/lib/codecs/src/decoding/format/avro.rs index 73a703b72ca38..0ab14af0aa88b 100644 --- a/lib/codecs/src/decoding/format/avro.rs +++ b/lib/codecs/src/decoding/format/avro.rs @@ -91,15 +91,13 @@ impl From<&AvroDeserializerOptions> for AvroSerializerOptions { pub struct AvroDeserializerOptions { /// The Avro schema definition. /// **Note**: The following [`apache_avro::types::Value`] variants are *not* supported: - /// * `Date` /// * `Decimal` /// * `Duration` /// * `Fixed` - /// * `TimeMillis` #[configurable(metadata( docs::examples = r#"{ "type": "record", "name": "log", "fields": [{ "name": "message", "type": "string" }] }"#, docs::additional_props_description = r#"Supports most avro data types, unsupported data types includes - ["decimal", "duration", "local-timestamp-millis", "local-timestamp-micros"]"#, + ["decimal", "duration", "fixed"]"#, ))] pub schema: String, @@ -190,9 +188,7 @@ pub fn try_from(value: AvroValue) -> vector_common::Result { } AvroValue::Boolean(boolean) => Ok(VrlValue::from(boolean)), AvroValue::Bytes(bytes) => Ok(VrlValue::from(bytes)), - AvroValue::Date(_) => Err(vector_common::Error::from( - "AvroValue::Date is not supported", - )), + AvroValue::Date(days) => Ok(VrlValue::from(days)), AvroValue::Decimal(_) => Err(vector_common::Error::from( "AvroValue::Decimal is not supported", )), @@ -220,9 +216,7 @@ pub fn try_from(value: AvroValue) -> vector_common::Result { .map(|v| VrlValue::Object(v.into_iter().collect())), AvroValue::String(string) => Ok(VrlValue::from(string)), AvroValue::TimeMicros(time_micros) => Ok(VrlValue::from(time_micros)), - AvroValue::TimeMillis(_) => Err(vector_common::Error::from( - "AvroValue::TimeMillis is not supported", - )), + AvroValue::TimeMillis(millis) => Ok(VrlValue::from(millis)), AvroValue::TimestampMicros(ts_micros) => Ok(VrlValue::from(ts_micros)), AvroValue::TimestampMillis(ts_millis) => Ok(VrlValue::from(ts_millis)), AvroValue::Union(_, v) => try_from(*v), diff --git a/lib/codecs/src/encoding/format/avro.rs b/lib/codecs/src/encoding/format/avro.rs index 304dcc2e7cc90..8cd83b70db501 100644 --- a/lib/codecs/src/encoding/format/avro.rs +++ b/lib/codecs/src/encoding/format/avro.rs @@ -1,3 +1,6 @@ +use std::collections::HashMap; + +use apache_avro::Schema; use bytes::{BufMut, BytesMut}; use serde::{Deserialize, Serialize}; use tokio_util::codec::Encoder; @@ -6,6 +9,129 @@ use vector_core::{config::DataType, event::Event, schema}; use crate::encoding::BuildError; +type AvroValue = apache_avro::types::Value; +type NamedSchemas = HashMap; + +fn resolve_named_schemas(schema: &apache_avro::Schema) -> Result { + let resolved = apache_avro::schema::ResolvedSchema::try_from(schema)?; + Ok(resolved + .get_names() + .iter() + .map(|(name, schema)| (name.clone(), (*schema).clone())) + .collect()) +} + +/// `apache_avro::to_value` serializes VRL values into Avro types which may not +/// resolve against certain logical type schemas +/// (e.g. VRL integer (i64) -> Avro `Long` which cannot resolve to Avro `Date`). +/// This does a recursive pre-pass to coerce such values before resolution. +fn coerce_logical_types( + value: AvroValue, + schema: &apache_avro::Schema, + names: &NamedSchemas, +) -> vector_common::Result { + match (value, schema) { + (AvroValue::Long(days), Schema::Date) => { + i32::try_from(days).map(AvroValue::Date).map_err(|_| { + vector_common::Error::from(format!( + "Avro date value {days} is out of range for i32" + )) + }) + } + (AvroValue::Long(millis), Schema::TimeMillis) => i32::try_from(millis) + .map(AvroValue::TimeMillis) + .map_err(|_| { + vector_common::Error::from(format!( + "Avro time-millis value {millis} is out of range for i32" + )) + }), + (AvroValue::Record(fields), Schema::Record(record_schema)) => { + let fields = fields + .into_iter() + .map(|(name, value)| { + let value = match record_schema.lookup.get(&name) { + Some(index) => { + let field_schema = &record_schema.fields[*index].schema; + coerce_logical_types(value, field_schema, names)? + } + None => value, + }; + Ok((name, value)) + }) + .collect::>>()?; + Ok(AvroValue::Record(fields)) + } + (AvroValue::Map(entries), Schema::Record(record_schema)) => { + let entries = entries + .into_iter() + .map(|(name, value)| { + let value = match record_schema.lookup.get(&name) { + Some(index) => { + let field_schema = &record_schema.fields[*index].schema; + coerce_logical_types(value, field_schema, names)? + } + None => value, + }; + Ok((name, value)) + }) + .collect::>()?; + Ok(AvroValue::Map(entries)) + } + (AvroValue::Array(items), Schema::Array(array_schema)) => items + .into_iter() + .map(|item| coerce_logical_types(item, &array_schema.items, names)) + .collect::, _>>() + .map(AvroValue::Array), + (AvroValue::Map(entries), Schema::Map(map_schema)) => entries + .into_iter() + .map(|(key, value)| { + coerce_logical_types(value, &map_schema.types, names).map(|value| (key, value)) + }) + .collect::>() + .map(AvroValue::Map), + (AvroValue::Union(index, value), Schema::Union(union_schema)) => { + let schema = union_schema + .variants() + .get(index as usize) + .unwrap_or(schema); + coerce_logical_types(*value, schema, names) + .map(|value| AvroValue::Union(index, Box::new(value))) + } + (value, Schema::Union(union_schema)) => { + if let Ok(resolved) = value.clone().resolve(schema) { + return Ok(resolved); + } + + let mut last_err = None; + for (index, variant) in union_schema.variants().iter().enumerate() { + let resolved_variant = match variant { + Schema::Ref { name } => names.get(name).unwrap_or(variant), + other => other, + }; + match coerce_logical_types(value.clone(), variant, names) { + Ok(coerced) if coerced.clone().resolve(resolved_variant).is_ok() => { + return Ok(AvroValue::Union(index as u32, Box::new(coerced))); + } + Ok(_) => {} + Err(err) => last_err = Some(err), + } + } + + match last_err { + Some(err) => Err(err), + None => Ok(value), + } + } + (value, Schema::Ref { name }) => { + let schema = names.get(name).ok_or_else(|| { + vector_common::Error::from(format!("Unknown schema ref: {}", name.fullname(None))) + })?; + coerce_logical_types(value, schema, names) + } + (value, _) => Ok(value), + } +} + /// Config used to build a `AvroSerializer`. #[derive(Debug, Clone, Deserialize, Serialize)] pub struct AvroSerializerConfig { @@ -25,7 +151,12 @@ impl AvroSerializerConfig { pub fn build(&self) -> Result { let schema = apache_avro::Schema::parse_str(&self.avro.schema) .map_err(|error| format!("Failed building Avro serializer: {error}"))?; - Ok(AvroSerializer { schema }) + let named_schemas = resolve_named_schemas(&schema) + .map_err(|error| format!("Failed resolving Avro schema: {error}"))?; + Ok(AvroSerializer { + schema, + named_schemas: Some(named_schemas), + }) } /// The data type of events that are accepted by `AvroSerializer`. @@ -56,12 +187,16 @@ pub struct AvroSerializerOptions { #[derive(Debug, Clone)] pub struct AvroSerializer { schema: apache_avro::Schema, + named_schemas: Option, } impl AvroSerializer { /// Creates a new `AvroSerializer`. pub const fn new(schema: apache_avro::Schema) -> Self { - Self { schema } + Self { + schema, + named_schemas: None, + } } } @@ -71,6 +206,18 @@ impl Encoder for AvroSerializer { fn encode(&mut self, event: Event, buffer: &mut BytesMut) -> Result<(), Self::Error> { let log = event.into_log(); let value = apache_avro::to_value(log)?; + if self.named_schemas.is_none() { + self.named_schemas = Some(resolve_named_schemas(&self.schema).map_err(|error| { + vector_common::Error::from(format!("Failed resolving Avro schema: {error}")) + })?); + } + let value = coerce_logical_types( + value, + &self.schema, + self.named_schemas + .as_ref() + .expect("named schemas are initialized above"), + )?; let value = value.resolve(&self.schema)?; let bytes = apache_avro::to_avro_datum(&self.schema, value)?; buffer.put_slice(&bytes); @@ -113,4 +260,251 @@ mod tests { assert_eq!(bytes.freeze(), b"\0\x06bar".as_slice()); } + + #[test] + fn coerce_date_fields_recursively() { + let schema = apache_avro::Schema::parse_str(indoc! {r#" + { + "type": "record", + "name": "Outer", + "fields": [ + { + "name": "direct_date", + "type": {"type": "int", "logicalType": "date"} + }, + { + "name": "inner", + "type": { + "type": "record", + "name": "Inner", + "fields": [ + { + "name": "date", + "type": {"type": "int", "logicalType": "date"} + } + ] + } + }, + { + "name": "record_as_map", + "type": { + "type": "record", + "name": "MapBackedInner", + "fields": [ + { + "name": "date", + "type": {"type": "int", "logicalType": "date"} + } + ] + } + }, + { + "name": "inner_reference", + "type": "Inner" + }, + { + "name": "date_array", + "type": { + "type": "array", + "items": {"type": "int", "logicalType": "date"} + } + }, + { + "name": "date_map", + "type": { + "type": "map", + "values": {"type": "int", "logicalType": "date"} + } + }, + { + "name": "union_date", + "type": ["null", {"type": "int", "logicalType": "date"}] + }, + { + "name": "fallback_union_date", + "type": [ + "null", + {"type": "int", "logicalType": "date"}, + "long" + ] + }, + { + "name": "logical_only_union_date", + "type": [ + "null", + {"type": "int", "logicalType": "date"} + ] + } + ] + } + "#}) + .unwrap(); + let value = AvroValue::Record(vec![ + ("direct_date".to_owned(), AvroValue::Long(20_000)), + ( + "inner".to_owned(), + AvroValue::Record(vec![("date".to_owned(), AvroValue::Long(20_001))]), + ), + ( + "record_as_map".to_owned(), + AvroValue::Map( + [("date".to_owned(), AvroValue::Long(20_002))] + .into_iter() + .collect(), + ), + ), + ( + "inner_reference".to_owned(), + AvroValue::Record(vec![("date".to_owned(), AvroValue::Long(20_003))]), + ), + ( + "date_array".to_owned(), + AvroValue::Array(vec![AvroValue::Long(20_004), AvroValue::Long(20_005)]), + ), + ( + "date_map".to_owned(), + AvroValue::Map( + [ + ("first".to_owned(), AvroValue::Long(20_006)), + ("second".to_owned(), AvroValue::Long(20_007)), + ] + .into_iter() + .collect(), + ), + ), + ("union_date".to_owned(), AvroValue::Long(20_008)), + ("fallback_union_date".to_owned(), AvroValue::Long(20_010)), + ( + "logical_only_union_date".to_owned(), + AvroValue::Long(20_009), + ), + ]); + + let named_schemas = resolve_named_schemas(&schema).unwrap(); + let value = coerce_logical_types(value, &schema, &named_schemas).unwrap(); + let value = value.resolve(&schema).unwrap(); + + assert!(matches!( + value, + AvroValue::Record(fields) if { + matches!(fields[0].1, AvroValue::Date(20_000)) + && matches!( + &fields[1].1, + AvroValue::Record(inner) if matches!(inner[0].1, AvroValue::Date(20_001)) + ) + && matches!( + &fields[2].1, + AvroValue::Record(inner) if matches!(inner[0].1, AvroValue::Date(20_002)) + ) + && matches!( + &fields[3].1, + AvroValue::Record(inner) if matches!(inner[0].1, AvroValue::Date(20_003)) + ) + && matches!( + &fields[4].1, + AvroValue::Array(items) + if matches!(items.as_slice(), [AvroValue::Date(20_004), AvroValue::Date(20_005)]) + ) + && matches!( + &fields[5].1, + AvroValue::Map(entries) + if matches!(entries.get("first"), Some(AvroValue::Date(20_006))) + && matches!(entries.get("second"), Some(AvroValue::Date(20_007))) + ) + && matches!( + &fields[6].1, + AvroValue::Union(1, value) if matches!(value.as_ref(), AvroValue::Date(20_008)) + ) + && matches!( + &fields[7].1, + AvroValue::Union(2, value) + if matches!(value.as_ref(), AvroValue::Long(20_010)) + ) + && matches!( + &fields[8].1, + AvroValue::Union(1, value) if matches!(value.as_ref(), AvroValue::Date(20_009)) + ) + } + )); + } + + #[test] + fn coerce_nullable_named_record_with_logical_types() { + // Regression: a union branch that is a named schema reference (e.g. ["null", "Inner"]) + // must resolve coerced values against the dereferenced schema, not the bare Schema::Ref. + let schema = apache_avro::Schema::parse_str(indoc! {r#" + { + "type": "record", + "name": "Outer", + "fields": [ + { + "name": "inner", + "type": { + "type": "record", + "name": "Inner", + "fields": [ + { + "name": "date", + "type": {"type": "int", "logicalType": "date"} + }, + { + "name": "time_millis", + "type": {"type": "int", "logicalType": "time-millis"} + } + ] + } + }, + { + "name": "nullable_inner", + "type": ["null", "Inner"] + } + ] + } + "#}) + .unwrap(); + + let value = AvroValue::Record(vec![ + ( + "inner".to_owned(), + AvroValue::Record(vec![ + ("date".to_owned(), AvroValue::Long(20_000)), + ("time_millis".to_owned(), AvroValue::Long(43_200_000)), + ]), + ), + ( + "nullable_inner".to_owned(), + AvroValue::Record(vec![ + ("date".to_owned(), AvroValue::Long(20_001)), + ("time_millis".to_owned(), AvroValue::Long(3_600_000)), + ]), + ), + ]); + + let named_schemas = resolve_named_schemas(&schema).unwrap(); + let value = coerce_logical_types(value, &schema, &named_schemas).unwrap(); + let value = value.resolve(&schema).unwrap(); + + assert!(matches!( + value, + AvroValue::Record(ref fields) if { + matches!( + &fields[0].1, + AvroValue::Record(inner) if { + matches!(inner[0].1, AvroValue::Date(20_000)) + && matches!(inner[1].1, AvroValue::TimeMillis(43_200_000)) + } + ) + && matches!( + &fields[1].1, + AvroValue::Union(1, value) if matches!( + value.as_ref(), + AvroValue::Record(inner) if { + matches!(inner[0].1, AvroValue::Date(20_001)) + && matches!(inner[1].1, AvroValue::TimeMillis(3_600_000)) + } + ) + ) + } + )); + } } diff --git a/lib/codecs/tests/avro.rs b/lib/codecs/tests/avro.rs index fff274706d7d9..8307f1fc45d72 100644 --- a/lib/codecs/tests/avro.rs +++ b/lib/codecs/tests/avro.rs @@ -22,7 +22,7 @@ use vector_core::{config::LogNamespace, event::Event}; #[case(false)] fn roundtrip_avro_fixtures( #[files("tests/data/avro/generated/*.avro")] - #[exclude(".*(date|fixed|time_millis).avro")] + #[exclude(".*fixed.avro")] path: PathBuf, #[case] reserialize: bool, ) { diff --git a/lib/codecs/tests/bin/generate-avro-fixtures.rs b/lib/codecs/tests/bin/generate-avro-fixtures.rs index 9301cb2840bf9..ececa19c6415f 100644 --- a/lib/codecs/tests/bin/generate-avro-fixtures.rs +++ b/lib/codecs/tests/bin/generate-avro-fixtures.rs @@ -280,7 +280,6 @@ fn generate_avro_test_case_record() -> Result<()> { generate_test_case(schema, value, "record") } -#[allow(unused)] fn generate_avro_test_case_date() -> Result<()> { let schema = r#" { @@ -299,6 +298,42 @@ fn generate_avro_test_case_date() -> Result<()> { generate_test_case(schema, value, "date") } +fn generate_avro_test_case_named_record_reference() -> Result<()> { + let schema = r#" + { + "type": "record", + "name": "Outer", + "fields": [ + { + "name": "first", + "type": { + "type": "record", + "name": "Inner", + "fields": [ + {"name": "date", "type": "int", "logicalType": "date"} + ] + } + }, + {"name": "second", "type": "Inner"} + ] + } + "#; + #[derive(Debug, Serialize, Deserialize, Clone)] + struct Inner { + date: i32, + } + #[derive(Debug, Serialize, Deserialize, Clone)] + struct Test { + first: Inner, + second: Inner, + } + let value = Test { + first: Inner { date: 20_000 }, + second: Inner { date: 20_001 }, + }; + generate_test_case(schema, value, "named_record_reference") +} + #[allow(unused)] fn generate_avro_test_case_decimal_var() -> Result<()> { let schema = r#" @@ -320,7 +355,6 @@ fn generate_avro_test_case_decimal_var() -> Result<()> { generate_test_case_from_value(schema, record, "decimal_var") } -#[allow(unused)] fn generate_avro_test_case_time_millis() -> Result<()> { let schema = r#" { @@ -488,15 +522,18 @@ fn main() -> Result<()> { generate_avro_test_case_array()?; generate_avro_test_case_boolean()?; generate_avro_test_case_bytes()?; + generate_avro_test_case_date()?; generate_avro_test_case_double()?; generate_avro_test_case_enum()?; generate_avro_test_case_float()?; generate_avro_test_case_int()?; generate_avro_test_case_long()?; generate_avro_test_case_map()?; + generate_avro_test_case_named_record_reference()?; generate_avro_test_case_record()?; generate_avro_test_case_string()?; generate_avro_test_case_time_micros()?; + generate_avro_test_case_time_millis()?; generate_avro_test_case_timestamp_micros()?; generate_avro_test_case_timestamp_millis()?; generate_avro_test_case_local_timestamp_micros()?; diff --git a/lib/codecs/tests/data/avro/generated/array.avsc b/lib/codecs/tests/data/avro/generated/array.avsc index 3ce68e7b38060..631eb9cae6480 100644 --- a/lib/codecs/tests/data/avro/generated/array.avsc +++ b/lib/codecs/tests/data/avro/generated/array.avsc @@ -1 +1 @@ -{"type":"record","name":"test","fields":[{"name":"array_field","type":{"type":"array","items":"string"},"items":"string"}]} \ No newline at end of file +{"type":"record","name":"test","fields":[{"name":"array_field","type":{"type":"array","items":"string"}}]} \ No newline at end of file diff --git a/lib/codecs/tests/data/avro/generated/date.avro b/lib/codecs/tests/data/avro/generated/date.avro new file mode 100644 index 0000000000000..b067210aeac93 --- /dev/null +++ b/lib/codecs/tests/data/avro/generated/date.avro @@ -0,0 +1 @@ +ü² \ No newline at end of file diff --git a/lib/codecs/tests/data/avro/generated/date.avsc b/lib/codecs/tests/data/avro/generated/date.avsc new file mode 100644 index 0000000000000..c7a70afe9be1a --- /dev/null +++ b/lib/codecs/tests/data/avro/generated/date.avsc @@ -0,0 +1 @@ +{"type":"record","name":"test","fields":[{"name":"date_field","type":{"type":"int","logicalType":"date"}}]} \ No newline at end of file diff --git a/lib/codecs/tests/data/avro/generated/map.avsc b/lib/codecs/tests/data/avro/generated/map.avsc index 69441d98fa538..455a4cd1672ea 100644 --- a/lib/codecs/tests/data/avro/generated/map.avsc +++ b/lib/codecs/tests/data/avro/generated/map.avsc @@ -1 +1 @@ -{"type":"record","name":"test","fields":[{"name":"map_field","type":{"type":"map","values":"long","default":{}},"default":{},"values":"long"}]} \ No newline at end of file +{"type":"record","name":"test","fields":[{"name":"map_field","type":{"type":"map","values":"long","default":{}},"default":{}}]} \ No newline at end of file diff --git a/lib/codecs/tests/data/avro/generated/named_record_reference.avro b/lib/codecs/tests/data/avro/generated/named_record_reference.avro new file mode 100644 index 0000000000000..0fab28d24933d --- /dev/null +++ b/lib/codecs/tests/data/avro/generated/named_record_reference.avro @@ -0,0 +1 @@ +À¸¸ \ No newline at end of file diff --git a/lib/codecs/tests/data/avro/generated/named_record_reference.avsc b/lib/codecs/tests/data/avro/generated/named_record_reference.avsc new file mode 100644 index 0000000000000..f5bf3320c6dcd --- /dev/null +++ b/lib/codecs/tests/data/avro/generated/named_record_reference.avsc @@ -0,0 +1 @@ +{"type":"record","name":"Outer","fields":[{"name":"first","type":{"type":"record","name":"Inner","fields":[{"name":"date","type":{"type":"int","logicalType":"date"}}]}},{"name":"second","type":"Inner"}]} \ No newline at end of file diff --git a/lib/codecs/tests/data/avro/generated/time_millis.avro b/lib/codecs/tests/data/avro/generated/time_millis.avro new file mode 100644 index 0000000000000..c1aee88444369 --- /dev/null +++ b/lib/codecs/tests/data/avro/generated/time_millis.avro @@ -0,0 +1 @@ +¶¡†9 \ No newline at end of file diff --git a/lib/codecs/tests/data/avro/generated/time_millis.avsc b/lib/codecs/tests/data/avro/generated/time_millis.avsc new file mode 100644 index 0000000000000..9a281db2c6e99 --- /dev/null +++ b/lib/codecs/tests/data/avro/generated/time_millis.avsc @@ -0,0 +1 @@ +{"type":"record","name":"test","fields":[{"name":"time_millis_field","type":{"type":"int","logicalType":"time-millis"}}]} \ No newline at end of file diff --git a/website/cue/reference/components/sinks/generated/websocket_server.cue b/website/cue/reference/components/sinks/generated/websocket_server.cue index 98f6f804f6005..98f7f256ae5d1 100644 --- a/website/cue/reference/components/sinks/generated/websocket_server.cue +++ b/website/cue/reference/components/sinks/generated/websocket_server.cue @@ -600,11 +600,9 @@ generated: components: sinks: websocket_server: configuration: { description: """ The Avro schema definition. **Note**: The following [`apache_avro::types::Value`] variants are *not* supported: - * `Date` * `Decimal` * `Duration` * `Fixed` - * `TimeMillis` """ required: true type: string: examples: ["{ \"type\": \"record\", \"name\": \"log\", \"fields\": [{ \"name\": \"message\", \"type\": \"string\" }] }"] diff --git a/website/cue/reference/components/sources/generated/amqp.cue b/website/cue/reference/components/sources/generated/amqp.cue index a028b58a6f370..91b2bf22549b0 100644 --- a/website/cue/reference/components/sources/generated/amqp.cue +++ b/website/cue/reference/components/sources/generated/amqp.cue @@ -62,11 +62,9 @@ generated: components: sources: amqp: configuration: { description: """ The Avro schema definition. **Note**: The following [`apache_avro::types::Value`] variants are *not* supported: - * `Date` * `Decimal` * `Duration` * `Fixed` - * `TimeMillis` """ required: true type: string: examples: ["{ \"type\": \"record\", \"name\": \"log\", \"fields\": [{ \"name\": \"message\", \"type\": \"string\" }] }"] diff --git a/website/cue/reference/components/sources/generated/aws_kinesis_firehose.cue b/website/cue/reference/components/sources/generated/aws_kinesis_firehose.cue index ac0bcbaf799b7..6b2af949ab489 100644 --- a/website/cue/reference/components/sources/generated/aws_kinesis_firehose.cue +++ b/website/cue/reference/components/sources/generated/aws_kinesis_firehose.cue @@ -82,11 +82,9 @@ generated: components: sources: aws_kinesis_firehose: configuration: { description: """ The Avro schema definition. **Note**: The following [`apache_avro::types::Value`] variants are *not* supported: - * `Date` * `Decimal` * `Duration` * `Fixed` - * `TimeMillis` """ required: true type: string: examples: ["{ \"type\": \"record\", \"name\": \"log\", \"fields\": [{ \"name\": \"message\", \"type\": \"string\" }] }"] diff --git a/website/cue/reference/components/sources/generated/aws_s3.cue b/website/cue/reference/components/sources/generated/aws_s3.cue index c3bb9d777fc80..cf0eb042f2f5c 100644 --- a/website/cue/reference/components/sources/generated/aws_s3.cue +++ b/website/cue/reference/components/sources/generated/aws_s3.cue @@ -180,11 +180,9 @@ generated: components: sources: aws_s3: configuration: { description: """ The Avro schema definition. **Note**: The following [`apache_avro::types::Value`] variants are *not* supported: - * `Date` * `Decimal` * `Duration` * `Fixed` - * `TimeMillis` """ required: true type: string: examples: ["{ \"type\": \"record\", \"name\": \"log\", \"fields\": [{ \"name\": \"message\", \"type\": \"string\" }] }"] diff --git a/website/cue/reference/components/sources/generated/aws_sqs.cue b/website/cue/reference/components/sources/generated/aws_sqs.cue index af04280e6783a..ba208aa203287 100644 --- a/website/cue/reference/components/sources/generated/aws_sqs.cue +++ b/website/cue/reference/components/sources/generated/aws_sqs.cue @@ -175,11 +175,9 @@ generated: components: sources: aws_sqs: configuration: { description: """ The Avro schema definition. **Note**: The following [`apache_avro::types::Value`] variants are *not* supported: - * `Date` * `Decimal` * `Duration` * `Fixed` - * `TimeMillis` """ required: true type: string: examples: ["{ \"type\": \"record\", \"name\": \"log\", \"fields\": [{ \"name\": \"message\", \"type\": \"string\" }] }"] diff --git a/website/cue/reference/components/sources/generated/datadog_agent.cue b/website/cue/reference/components/sources/generated/datadog_agent.cue index 4b49969be92e3..39fbd0ef5c559 100644 --- a/website/cue/reference/components/sources/generated/datadog_agent.cue +++ b/website/cue/reference/components/sources/generated/datadog_agent.cue @@ -47,11 +47,9 @@ generated: components: sources: datadog_agent: configuration: { description: """ The Avro schema definition. **Note**: The following [`apache_avro::types::Value`] variants are *not* supported: - * `Date` * `Decimal` * `Duration` * `Fixed` - * `TimeMillis` """ required: true type: string: examples: ["{ \"type\": \"record\", \"name\": \"log\", \"fields\": [{ \"name\": \"message\", \"type\": \"string\" }] }"] diff --git a/website/cue/reference/components/sources/generated/demo_logs.cue b/website/cue/reference/components/sources/generated/demo_logs.cue index 725e59104327e..efa6c09be57b7 100644 --- a/website/cue/reference/components/sources/generated/demo_logs.cue +++ b/website/cue/reference/components/sources/generated/demo_logs.cue @@ -26,11 +26,9 @@ generated: components: sources: demo_logs: configuration: { description: """ The Avro schema definition. **Note**: The following [`apache_avro::types::Value`] variants are *not* supported: - * `Date` * `Decimal` * `Duration` * `Fixed` - * `TimeMillis` """ required: true type: string: examples: ["{ \"type\": \"record\", \"name\": \"log\", \"fields\": [{ \"name\": \"message\", \"type\": \"string\" }] }"] diff --git a/website/cue/reference/components/sources/generated/exec.cue b/website/cue/reference/components/sources/generated/exec.cue index f8032941198cd..bf2bd7286ec0d 100644 --- a/website/cue/reference/components/sources/generated/exec.cue +++ b/website/cue/reference/components/sources/generated/exec.cue @@ -27,11 +27,9 @@ generated: components: sources: exec: configuration: { description: """ The Avro schema definition. **Note**: The following [`apache_avro::types::Value`] variants are *not* supported: - * `Date` * `Decimal` * `Duration` * `Fixed` - * `TimeMillis` """ required: true type: string: examples: ["{ \"type\": \"record\", \"name\": \"log\", \"fields\": [{ \"name\": \"message\", \"type\": \"string\" }] }"] diff --git a/website/cue/reference/components/sources/generated/file_descriptor.cue b/website/cue/reference/components/sources/generated/file_descriptor.cue index 8606fcbfdaeb7..19961c748ded7 100644 --- a/website/cue/reference/components/sources/generated/file_descriptor.cue +++ b/website/cue/reference/components/sources/generated/file_descriptor.cue @@ -17,11 +17,9 @@ generated: components: sources: file_descriptor: configuration: { description: """ The Avro schema definition. **Note**: The following [`apache_avro::types::Value`] variants are *not* supported: - * `Date` * `Decimal` * `Duration` * `Fixed` - * `TimeMillis` """ required: true type: string: examples: ["{ \"type\": \"record\", \"name\": \"log\", \"fields\": [{ \"name\": \"message\", \"type\": \"string\" }] }"] diff --git a/website/cue/reference/components/sources/generated/gcp_pubsub.cue b/website/cue/reference/components/sources/generated/gcp_pubsub.cue index b1dbfa21b4603..08658fbf2e11b 100644 --- a/website/cue/reference/components/sources/generated/gcp_pubsub.cue +++ b/website/cue/reference/components/sources/generated/gcp_pubsub.cue @@ -93,11 +93,9 @@ generated: components: sources: gcp_pubsub: configuration: { description: """ The Avro schema definition. **Note**: The following [`apache_avro::types::Value`] variants are *not* supported: - * `Date` * `Decimal` * `Duration` * `Fixed` - * `TimeMillis` """ required: true type: string: examples: ["{ \"type\": \"record\", \"name\": \"log\", \"fields\": [{ \"name\": \"message\", \"type\": \"string\" }] }"] diff --git a/website/cue/reference/components/sources/generated/heroku_logs.cue b/website/cue/reference/components/sources/generated/heroku_logs.cue index 3ea91413bae32..b842593c66a64 100644 --- a/website/cue/reference/components/sources/generated/heroku_logs.cue +++ b/website/cue/reference/components/sources/generated/heroku_logs.cue @@ -90,11 +90,9 @@ generated: components: sources: heroku_logs: configuration: { description: """ The Avro schema definition. **Note**: The following [`apache_avro::types::Value`] variants are *not* supported: - * `Date` * `Decimal` * `Duration` * `Fixed` - * `TimeMillis` """ required: true type: string: examples: ["{ \"type\": \"record\", \"name\": \"log\", \"fields\": [{ \"name\": \"message\", \"type\": \"string\" }] }"] diff --git a/website/cue/reference/components/sources/generated/http.cue b/website/cue/reference/components/sources/generated/http.cue index 92135d6929eb0..fdd569b0469d0 100644 --- a/website/cue/reference/components/sources/generated/http.cue +++ b/website/cue/reference/components/sources/generated/http.cue @@ -98,11 +98,9 @@ generated: components: sources: http: configuration: { description: """ The Avro schema definition. **Note**: The following [`apache_avro::types::Value`] variants are *not* supported: - * `Date` * `Decimal` * `Duration` * `Fixed` - * `TimeMillis` """ required: true type: string: examples: ["{ \"type\": \"record\", \"name\": \"log\", \"fields\": [{ \"name\": \"message\", \"type\": \"string\" }] }"] diff --git a/website/cue/reference/components/sources/generated/http_client.cue b/website/cue/reference/components/sources/generated/http_client.cue index 1a09deeeb1510..3a26e03d82a41 100644 --- a/website/cue/reference/components/sources/generated/http_client.cue +++ b/website/cue/reference/components/sources/generated/http_client.cue @@ -228,11 +228,9 @@ generated: components: sources: http_client: configuration: { description: """ The Avro schema definition. **Note**: The following [`apache_avro::types::Value`] variants are *not* supported: - * `Date` * `Decimal` * `Duration` * `Fixed` - * `TimeMillis` """ required: true type: string: examples: ["{ \"type\": \"record\", \"name\": \"log\", \"fields\": [{ \"name\": \"message\", \"type\": \"string\" }] }"] diff --git a/website/cue/reference/components/sources/generated/http_server.cue b/website/cue/reference/components/sources/generated/http_server.cue index 94387acb1a436..f70b624643200 100644 --- a/website/cue/reference/components/sources/generated/http_server.cue +++ b/website/cue/reference/components/sources/generated/http_server.cue @@ -98,11 +98,9 @@ generated: components: sources: http_server: configuration: { description: """ The Avro schema definition. **Note**: The following [`apache_avro::types::Value`] variants are *not* supported: - * `Date` * `Decimal` * `Duration` * `Fixed` - * `TimeMillis` """ required: true type: string: examples: ["{ \"type\": \"record\", \"name\": \"log\", \"fields\": [{ \"name\": \"message\", \"type\": \"string\" }] }"] diff --git a/website/cue/reference/components/sources/generated/kafka.cue b/website/cue/reference/components/sources/generated/kafka.cue index 7ad29ba840079..96921f8c1bb28 100644 --- a/website/cue/reference/components/sources/generated/kafka.cue +++ b/website/cue/reference/components/sources/generated/kafka.cue @@ -71,11 +71,9 @@ generated: components: sources: kafka: configuration: { description: """ The Avro schema definition. **Note**: The following [`apache_avro::types::Value`] variants are *not* supported: - * `Date` * `Decimal` * `Duration` * `Fixed` - * `TimeMillis` """ required: true type: string: examples: ["{ \"type\": \"record\", \"name\": \"log\", \"fields\": [{ \"name\": \"message\", \"type\": \"string\" }] }"] diff --git a/website/cue/reference/components/sources/generated/mqtt.cue b/website/cue/reference/components/sources/generated/mqtt.cue index 291df9da30708..51529c7f7fa88 100644 --- a/website/cue/reference/components/sources/generated/mqtt.cue +++ b/website/cue/reference/components/sources/generated/mqtt.cue @@ -22,11 +22,9 @@ generated: components: sources: mqtt: configuration: { description: """ The Avro schema definition. **Note**: The following [`apache_avro::types::Value`] variants are *not* supported: - * `Date` * `Decimal` * `Duration` * `Fixed` - * `TimeMillis` """ required: true type: string: examples: ["{ \"type\": \"record\", \"name\": \"log\", \"fields\": [{ \"name\": \"message\", \"type\": \"string\" }] }"] diff --git a/website/cue/reference/components/sources/generated/nats.cue b/website/cue/reference/components/sources/generated/nats.cue index 806470925f17f..b50f167f90dda 100644 --- a/website/cue/reference/components/sources/generated/nats.cue +++ b/website/cue/reference/components/sources/generated/nats.cue @@ -114,11 +114,9 @@ generated: components: sources: nats: configuration: { description: """ The Avro schema definition. **Note**: The following [`apache_avro::types::Value`] variants are *not* supported: - * `Date` * `Decimal` * `Duration` * `Fixed` - * `TimeMillis` """ required: true type: string: examples: ["{ \"type\": \"record\", \"name\": \"log\", \"fields\": [{ \"name\": \"message\", \"type\": \"string\" }] }"] diff --git a/website/cue/reference/components/sources/generated/pulsar.cue b/website/cue/reference/components/sources/generated/pulsar.cue index 10b93611327a2..3a2faf1c92500 100644 --- a/website/cue/reference/components/sources/generated/pulsar.cue +++ b/website/cue/reference/components/sources/generated/pulsar.cue @@ -120,11 +120,9 @@ generated: components: sources: pulsar: configuration: { description: """ The Avro schema definition. **Note**: The following [`apache_avro::types::Value`] variants are *not* supported: - * `Date` * `Decimal` * `Duration` * `Fixed` - * `TimeMillis` """ required: true type: string: examples: ["{ \"type\": \"record\", \"name\": \"log\", \"fields\": [{ \"name\": \"message\", \"type\": \"string\" }] }"] diff --git a/website/cue/reference/components/sources/generated/redis.cue b/website/cue/reference/components/sources/generated/redis.cue index 20ce8569fd173..693b129e3085f 100644 --- a/website/cue/reference/components/sources/generated/redis.cue +++ b/website/cue/reference/components/sources/generated/redis.cue @@ -32,11 +32,9 @@ generated: components: sources: redis: configuration: { description: """ The Avro schema definition. **Note**: The following [`apache_avro::types::Value`] variants are *not* supported: - * `Date` * `Decimal` * `Duration` * `Fixed` - * `TimeMillis` """ required: true type: string: examples: ["{ \"type\": \"record\", \"name\": \"log\", \"fields\": [{ \"name\": \"message\", \"type\": \"string\" }] }"] diff --git a/website/cue/reference/components/sources/generated/socket.cue b/website/cue/reference/components/sources/generated/socket.cue index a4cd5713a88c9..4f4496bd453db 100644 --- a/website/cue/reference/components/sources/generated/socket.cue +++ b/website/cue/reference/components/sources/generated/socket.cue @@ -34,11 +34,9 @@ generated: components: sources: socket: configuration: { description: """ The Avro schema definition. **Note**: The following [`apache_avro::types::Value`] variants are *not* supported: - * `Date` * `Decimal` * `Duration` * `Fixed` - * `TimeMillis` """ required: true type: string: examples: ["{ \"type\": \"record\", \"name\": \"log\", \"fields\": [{ \"name\": \"message\", \"type\": \"string\" }] }"] diff --git a/website/cue/reference/components/sources/generated/splunk_hec.cue b/website/cue/reference/components/sources/generated/splunk_hec.cue index ac865b14d02ce..0e5c93eb38264 100644 --- a/website/cue/reference/components/sources/generated/splunk_hec.cue +++ b/website/cue/reference/components/sources/generated/splunk_hec.cue @@ -107,11 +107,9 @@ generated: components: sources: splunk_hec: configuration: { description: """ The Avro schema definition. **Note**: The following [`apache_avro::types::Value`] variants are *not* supported: - * `Date` * `Decimal` * `Duration` * `Fixed` - * `TimeMillis` """ required: true type: string: examples: ["{ \"type\": \"record\", \"name\": \"log\", \"fields\": [{ \"name\": \"message\", \"type\": \"string\" }] }"] @@ -699,11 +697,9 @@ generated: components: sources: splunk_hec: configuration: { description: """ The Avro schema definition. **Note**: The following [`apache_avro::types::Value`] variants are *not* supported: - * `Date` * `Decimal` * `Duration` * `Fixed` - * `TimeMillis` """ required: true type: string: examples: ["{ \"type\": \"record\", \"name\": \"log\", \"fields\": [{ \"name\": \"message\", \"type\": \"string\" }] }"] diff --git a/website/cue/reference/components/sources/generated/stdin.cue b/website/cue/reference/components/sources/generated/stdin.cue index 74394b17ae62d..fab6aba88b625 100644 --- a/website/cue/reference/components/sources/generated/stdin.cue +++ b/website/cue/reference/components/sources/generated/stdin.cue @@ -17,11 +17,9 @@ generated: components: sources: stdin: configuration: { description: """ The Avro schema definition. **Note**: The following [`apache_avro::types::Value`] variants are *not* supported: - * `Date` * `Decimal` * `Duration` * `Fixed` - * `TimeMillis` """ required: true type: string: examples: ["{ \"type\": \"record\", \"name\": \"log\", \"fields\": [{ \"name\": \"message\", \"type\": \"string\" }] }"] diff --git a/website/cue/reference/components/sources/generated/websocket.cue b/website/cue/reference/components/sources/generated/websocket.cue index 14418349e88bd..93f981e93797b 100644 --- a/website/cue/reference/components/sources/generated/websocket.cue +++ b/website/cue/reference/components/sources/generated/websocket.cue @@ -204,11 +204,9 @@ generated: components: sources: websocket: configuration: { description: """ The Avro schema definition. **Note**: The following [`apache_avro::types::Value`] variants are *not* supported: - * `Date` * `Decimal` * `Duration` * `Fixed` - * `TimeMillis` """ required: true type: string: examples: ["{ \"type\": \"record\", \"name\": \"log\", \"fields\": [{ \"name\": \"message\", \"type\": \"string\" }] }"]