diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 15135f41be7ce..1cf98b5a82930 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -33,6 +33,10 @@ inputs: required: false default: false description: "Install libsasl2." # Required to fully build Vector + unixodbc: + required: false + default: false + description: "Install unixODBC headers/libs (apt unixodbc-dev, dnf unixODBC-devel, or brew unixodbc). Required when sources-odbc is enabled." # prepare.sh - rust rust: # rustup module @@ -201,10 +205,45 @@ runs: sudo cp "${TEMP}/cue" /usr/bin/cue rm -rf "$TEMP" - - name: Install libsasl2 - if: ${{ inputs.libsasl2 == 'true' }} + - name: Install system libraries (Linux) + if: ${{ runner.os == 'Linux' && (inputs.libsasl2 == 'true' || inputs.unixodbc == 'true') }} shell: bash - run: timeout 30m sudo apt-get update && timeout 30m sudo apt-get install -y libsasl2-dev + run: | + if command -v apt-get >/dev/null 2>&1; then + pkgs=() + [[ "${{ inputs.libsasl2 }}" == "true" ]] && pkgs+=(libsasl2-dev) + [[ "${{ inputs.unixodbc }}" == "true" ]] && pkgs+=(unixodbc-dev) + timeout 30m sudo apt-get update + timeout 30m sudo apt-get install -y "${pkgs[@]}" + elif command -v dnf >/dev/null 2>&1; then + pkgs=() + [[ "${{ inputs.libsasl2 }}" == "true" ]] && pkgs+=(cyrus-sasl-devel) + [[ "${{ inputs.unixodbc }}" == "true" ]] && pkgs+=(unixODBC-devel) + timeout 30m sudo dnf install -y --setopt=install_weak_deps=False "${pkgs[@]}" + else + echo "No supported package manager (apt-get/dnf) found for system libraries" >&2 + exit 1 + fi + + - name: Install unixODBC (macOS) + if: ${{ runner.os == 'macOS' && inputs.unixodbc == 'true' }} + shell: bash + run: | + echo "Installing unixODBC" + brew install unixodbc + # Use the formula prefix so Intel (/usr/local) and Apple Silicon (/opt/homebrew) + # both work. odbc-sys also probes `brew --prefix`, but these keep the linker, + # headers, and pkg-config consistent for the rest of the build. + prefix="$(brew --prefix unixodbc)" + lib_dir="${prefix}/lib" + include_dir="${prefix}/include" + pkgconfig_dir="${prefix}/lib/pkgconfig" + { + echo "LIBRARY_PATH=${lib_dir}${LIBRARY_PATH:+:$LIBRARY_PATH}" + echo "CPATH=${include_dir}${CPATH:+:$CPATH}" + echo "PKG_CONFIG_PATH=${pkgconfig_dir}${PKG_CONFIG_PATH:+:$PKG_CONFIG_PATH}" + echo "DYLD_FALLBACK_LIBRARY_PATH=${lib_dir}${DYLD_FALLBACK_LIBRARY_PATH:+:$DYLD_FALLBACK_LIBRARY_PATH}" + } >> "$GITHUB_ENV" - name: Cache cargo binaries id: cache-cargo-binaries diff --git a/.github/workflows/changes.yml b/.github/workflows/changes.yml index 9b94c72e077e0..5823e24c93069 100644 --- a/.github/workflows/changes.yml +++ b/.github/workflows/changes.yml @@ -126,6 +126,10 @@ on: value: ${{ jobs.int_tests.outputs.nats }} nginx: value: ${{ jobs.int_tests.outputs.nginx }} + odbc-mariadb: + value: ${{ jobs.int_tests.outputs.odbc-mariadb }} + odbc-postgresql: + value: ${{ jobs.int_tests.outputs.odbc-postgresql }} opentelemetry: value: ${{ jobs.int_tests.outputs.opentelemetry }} postgres: @@ -364,6 +368,8 @@ jobs: mqtt: ${{ steps.filter.outputs.mqtt }} nats: ${{ steps.filter.outputs.nats }} nginx: ${{ steps.filter.outputs.nginx }} + odbc-mariadb: ${{ steps.filter.outputs.odbc-mariadb }} + odbc-postgresql: ${{ steps.filter.outputs.odbc-postgresql }} opentelemetry: ${{ steps.filter.outputs.opentelemetry }} postgres: ${{ steps.filter.outputs.postgres }} prometheus: ${{ steps.filter.outputs.prometheus }} @@ -429,6 +435,8 @@ jobs: "mqtt": ${{ steps.filter.outputs.mqtt }}, "nats": ${{ steps.filter.outputs.nats }}, "nginx": ${{ steps.filter.outputs.nginx }}, + "odbc-mariadb": ${{ steps.filter.outputs.odbc-mariadb }}, + "odbc-postgresql": ${{ steps.filter.outputs.odbc-postgresql }}, "opentelemetry": ${{ steps.filter.outputs.opentelemetry }}, "postgres": ${{ steps.filter.outputs.postgres }}, "prometheus": ${{ steps.filter.outputs.prometheus }}, diff --git a/.github/workflows/ci-integration-review.yml b/.github/workflows/ci-integration-review.yml index 7381797a92423..8aa3005df8d2c 100644 --- a/.github/workflows/ci-integration-review.yml +++ b/.github/workflows/ci-integration-review.yml @@ -134,6 +134,8 @@ jobs: "mongodb", "nats", "nginx", + "odbc-mariadb", + "odbc-postgresql", "opentelemetry", "postgres", "prometheus", diff --git a/.github/workflows/component_features.yml b/.github/workflows/component_features.yml index 65f86bfdce128..20cf7eca6b69e 100644 --- a/.github/workflows/component_features.yml +++ b/.github/workflows/component_features.yml @@ -49,6 +49,7 @@ jobs: cargo-nextest: true protoc: true libsasl2: true + unixodbc: true cargo-hack: true cargo-cache: true diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index bc65532f378b7..364d6b36a34c7 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -126,6 +126,8 @@ jobs: "mqtt", "nats", "nginx", + "odbc-mariadb", + "odbc-postgresql", "opentelemetry", "postgres", "prometheus", diff --git a/.github/workflows/msrv.yml b/.github/workflows/msrv.yml index d7497fdadd37c..c21ff8eb9deef 100644 --- a/.github/workflows/msrv.yml +++ b/.github/workflows/msrv.yml @@ -30,6 +30,7 @@ jobs: rust: true protoc: true libsasl2: true + unixodbc: true cargo-msrv: true cargo-cache: true - run: cargo msrv verify diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 1a9cab831dc48..253e5715d96e2 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -91,7 +91,7 @@ jobs: run: | dnf install -y --setopt=install_weak_deps=False \ gcc gcc-c++ make cmake perl perl-IPC-Cmd \ - openssl-devel cyrus-sasl-devel zlib-devel \ + openssl-devel cyrus-sasl-devel zlib-devel unixODBC-devel \ pkgconfig clang \ rpm-build sudo \ git tar gzip xz which findutils \ @@ -126,6 +126,7 @@ jobs: rust: true cargo-deb: true protoc: true + unixodbc: true - name: Build Vector run: make NATIVE=true package-${{ matrix.target }}-all - name: Stage package artifacts for publish diff --git a/.github/workflows/test-make-command.yml b/.github/workflows/test-make-command.yml index e6fd36e01fd75..c7c260bce4d37 100644 --- a/.github/workflows/test-make-command.yml +++ b/.github/workflows/test-make-command.yml @@ -52,6 +52,7 @@ jobs: rust: true protoc: true libsasl2: true + unixodbc: true cargo-nextest: ${{ inputs.cargo_nextest }} datadog-ci: ${{ inputs.upload_test_results }} cargo-cache: true diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4072d8d371330..1c46805a90db1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -53,6 +53,7 @@ jobs: protoc: true libsasl2: true cargo-cache: true + unixodbc: true - run: make check-clippy test: @@ -147,9 +148,9 @@ jobs: rust: true protoc: true cue: true - prettier: true libsasl2: true cargo-cache: true + - run: cd website && yarn install --frozen-lockfile - run: make check-generated-docs check-rust-docs: diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 43aceec986dc0..088fac1bbe940 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -81,6 +81,7 @@ jobs: protoc: true libsasl2: true cargo-cache: true + unixodbc: true - name: Run tests run: | diff --git a/.github/workflows/unit_mac.yml b/.github/workflows/unit_mac.yml index 0358d0eedfe85..9fce9a0865170 100644 --- a/.github/workflows/unit_mac.yml +++ b/.github/workflows/unit_mac.yml @@ -27,6 +27,7 @@ jobs: cargo-nextest: true protoc: true cargo-cache: true + unixodbc: true # Some tests e.g. `reader_exits_cleanly_when_writer_done_and_in_flight_acks` are flaky. - name: Run tests diff --git a/.github/workflows/warm-cache.yml b/.github/workflows/warm-cache.yml index 1821981634b36..ed708a27bee41 100644 --- a/.github/workflows/warm-cache.yml +++ b/.github/workflows/warm-cache.yml @@ -35,6 +35,7 @@ jobs: rust: true protoc: true libsasl2: true + unixodbc: true vdev: true cargo-cache: true cache-save: true diff --git a/Cargo.lock b/Cargo.lock index f8a43799f918f..42c9b0de1c394 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -181,6 +181,31 @@ dependencies = [ "url", ] +[[package]] +name = "android-activity" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f2a1bb052857d5dd49572219344a7332b31b76405648eabac5bc68978251bcd" +dependencies = [ + "android-properties", + "bitflags 2.10.0", + "cc", + "jni", + "libc", + "log", + "ndk", + "ndk-context", + "ndk-sys", + "num_enum 0.7.3", + "thiserror 2.0.18", +] + +[[package]] +name = "android-properties" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7eb209b1518d6bb87b283c20095f5228ecda460da70b44f0802523dea6da04" + [[package]] name = "android_system_properties" version = "0.1.5" @@ -2239,6 +2264,15 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "block2" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c132eebf10f5cad5289222520a4a058514204aed6d791f1cf4fe8088b82d15f" +dependencies = [ + "objc2 0.5.2", +] + [[package]] name = "blocking" version = "1.6.2" @@ -2500,6 +2534,20 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6bd91ee7b2422bcb158d90ef4d14f75ef67f340943fc4149891dcce8f8b972a3" +[[package]] +name = "calloop" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b99da2f8558ca23c71f4fd15dc57c906239752dd27ff3c00a1d56b685b7cbfec" +dependencies = [ + "bitflags 2.10.0", + "log", + "polling", + "rustix 0.38.40", + "slab", + "thiserror 1.0.68", +] + [[package]] name = "cargo-lock" version = "11.0.1" @@ -3142,6 +3190,30 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "core-graphics" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "970a29baf4110c26fedbc7f82107d42c23f7e88e404c4577ed73fe99ff85a212" +dependencies = [ + "bitflags 1.3.2", + "core-foundation 0.9.3", + "core-graphics-types", + "foreign-types 0.5.0", + "libc", +] + +[[package]] +name = "core-graphics-types" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bb142d41022986c1d8ff29103a1411c8a3dfad3552f87a4f8dc50d61d4f4e33" +dependencies = [ + "bitflags 1.3.2", + "core-foundation 0.9.3", + "libc", +] + [[package]] name = "cpubits" version = "0.1.1" @@ -3244,6 +3316,18 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" +[[package]] +name = "cron" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5877d3fbf742507b66bc2a1945106bd30dd8504019d596901ddd012a4dd01740" +dependencies = [ + "chrono", + "once_cell", + "serde", + "winnow 0.6.26", +] + [[package]] name = "crossbeam-channel" version = "0.5.8" @@ -3444,6 +3528,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "cursor-icon" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f27ae1dd37df86211c42e150270f82743308803d90a6f6e6651cd730d5e1732f" + [[package]] name = "curve25519-dalek" version = "4.1.3" @@ -3898,6 +3988,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "dispatch" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd0c93bb4b0c6d9b77f4435b0ae98c24d17f1c45b2ff844c6151a07256ca923b" + [[package]] name = "dispatch2" version = "0.3.1" @@ -3905,7 +4001,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" dependencies = [ "bitflags 2.10.0", - "objc2", + "objc2 0.6.4", ] [[package]] @@ -3919,6 +4015,15 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "dlib" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab8ecd87370524b461f8557c119c405552c396ed91fc0a8eec68679eab26f94a" +dependencies = [ + "libloading", +] + [[package]] name = "dns-lookup" version = "3.0.1" @@ -4030,6 +4135,12 @@ version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" +[[package]] +name = "dpi" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8b14ccef22fc6f5a8f4d7d768562a182c04ce9a3b3157b91390b52ddfdf1a76" + [[package]] name = "duct" version = "0.13.6" @@ -4590,7 +4701,28 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" dependencies = [ - "foreign-types-shared", + "foreign-types-shared 0.1.1", +] + +[[package]] +name = "foreign-types" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965" +dependencies = [ + "foreign-types-macros", + "foreign-types-shared 0.3.1", +] + +[[package]] +name = "foreign-types-macros" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea5190182e6915eb873ddbc16e23b711b6eb1f9c00a0d0a3a91b5f6228475225" +dependencies = [ + "proc-macro2 1.0.106", + "quote 1.0.45", + "syn 3.0.3", ] [[package]] @@ -4599,6 +4731,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" +[[package]] +name = "foreign-types-shared" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa9a19cbb55df58761df49b23516a86d432839add4af60fc256da840f66ed35b" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -6096,7 +6234,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b58db92f96b720de98181bbbe63c831e87005ab460c1bf306eb2622b4707997f" dependencies = [ "socket2 0.5.10", - "widestring 1.0.2", + "widestring 1.2.1", "windows-sys 0.48.0", "winreg", ] @@ -6245,7 +6383,7 @@ dependencies = [ "cfg-if", "combine", "jni-macros", - "jni-sys", + "jni-sys 0.4.1", "log", "simd_cesu8", "thiserror 2.0.18", @@ -6266,6 +6404,15 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "jni-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258" +dependencies = [ + "jni-sys 0.4.1", +] + [[package]] name = "jni-sys" version = "0.4.1" @@ -7409,12 +7556,36 @@ dependencies = [ "rand 0.8.6", ] +[[package]] +name = "ndk" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3f42e7bbe13d351b6bead8286a43aac9534b82bd3cc43e47037f012ebfd62d4" +dependencies = [ + "bitflags 2.10.0", + "jni-sys 0.3.1", + "log", + "ndk-sys", + "num_enum 0.7.3", + "raw-window-handle", + "thiserror 1.0.68", +] + [[package]] name = "ndk-context" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" +[[package]] +name = "ndk-sys" +version = "0.6.0+11769913" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee6cda3051665f1fb8d9e08fc35c96d5a244fb1be711a03b71118828afc9a873" +dependencies = [ + "jni-sys 0.3.1", +] + [[package]] name = "new_debug_unreachable" version = "1.0.4" @@ -7807,6 +7978,22 @@ dependencies = [ "url", ] +[[package]] +name = "objc-sys" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb91bdd390c7ce1a8607f35f3ca7151b65afc0ff5ff3b34fa350f7d7c7e4310" + +[[package]] +name = "objc2" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46a785d4eeff09c14c487497c162e92766fbb3e4059a71840cecc03d9a50b804" +dependencies = [ + "objc-sys", + "objc2-encode", +] + [[package]] name = "objc2" version = "0.6.4" @@ -7816,6 +8003,22 @@ dependencies = [ "objc2-encode", ] +[[package]] +name = "objc2-app-kit" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4e89ad9e3d7d297152b17d39ed92cd50ca8063a89a9fa569046d41568891eff" +dependencies = [ + "bitflags 2.10.0", + "block2", + "libc", + "objc2 0.5.2", + "objc2-core-data", + "objc2-core-image", + "objc2-foundation 0.2.2", + "objc2-quartz-core", +] + [[package]] name = "objc2-app-kit" version = "0.3.2" @@ -7823,8 +8026,44 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d49e936b501e5c5bf01fda3a9452ff86dc3ea98ad5f283e1455153142d97518c" dependencies = [ "bitflags 2.10.0", - "objc2", - "objc2-foundation", + "objc2 0.6.4", + "objc2-foundation 0.3.2", +] + +[[package]] +name = "objc2-cloud-kit" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74dd3b56391c7a0596a295029734d3c1c5e7e510a4cb30245f8221ccea96b009" +dependencies = [ + "bitflags 2.10.0", + "block2", + "objc2 0.5.2", + "objc2-core-location", + "objc2-foundation 0.2.2", +] + +[[package]] +name = "objc2-contacts" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5ff520e9c33812fd374d8deecef01d4a840e7b41862d849513de77e44aa4889" +dependencies = [ + "block2", + "objc2 0.5.2", + "objc2-foundation 0.2.2", +] + +[[package]] +name = "objc2-core-data" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "617fbf49e071c178c0b24c080767db52958f716d9eabdf0890523aeae54773ef" +dependencies = [ + "bitflags 2.10.0", + "block2", + "objc2 0.5.2", + "objc2-foundation 0.2.2", ] [[package]] @@ -7835,7 +8074,31 @@ checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" dependencies = [ "bitflags 2.10.0", "dispatch2", - "objc2", + "objc2 0.6.4", +] + +[[package]] +name = "objc2-core-image" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55260963a527c99f1819c4f8e3b47fe04f9650694ef348ffd2227e8196d34c80" +dependencies = [ + "block2", + "objc2 0.5.2", + "objc2-foundation 0.2.2", + "objc2-metal", +] + +[[package]] +name = "objc2-core-location" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "000cfee34e683244f284252ee206a27953279d370e309649dc3ee317b37e5781" +dependencies = [ + "block2", + "objc2 0.5.2", + "objc2-contacts", + "objc2-foundation 0.2.2", ] [[package]] @@ -7844,6 +8107,19 @@ version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" +[[package]] +name = "objc2-foundation" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ee638a5da3799329310ad4cfa62fbf045d5f56e3ef5ba4149e7452dcf89d5a8" +dependencies = [ + "bitflags 2.10.0", + "block2", + "dispatch", + "libc", + "objc2 0.5.2", +] + [[package]] name = "objc2-foundation" version = "0.3.2" @@ -7851,7 +8127,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" dependencies = [ "bitflags 2.10.0", - "objc2", + "objc2 0.6.4", "objc2-core-foundation", ] @@ -7865,6 +8141,53 @@ dependencies = [ "objc2-core-foundation", ] +[[package]] +name = "objc2-link-presentation" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1a1ae721c5e35be65f01a03b6d2ac13a54cb4fa70d8a5da293d7b0020261398" +dependencies = [ + "block2", + "objc2 0.5.2", + "objc2-app-kit 0.2.2", + "objc2-foundation 0.2.2", +] + +[[package]] +name = "objc2-metal" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd0cba1276f6023976a406a14ffa85e1fdd19df6b0f737b063b95f6c8c7aadd6" +dependencies = [ + "bitflags 2.10.0", + "block2", + "objc2 0.5.2", + "objc2-foundation 0.2.2", +] + +[[package]] +name = "objc2-quartz-core" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e42bee7bff906b14b167da2bac5efe6b6a07e6f7c0a21a7308d40c960242dc7a" +dependencies = [ + "bitflags 2.10.0", + "block2", + "objc2 0.5.2", + "objc2-foundation 0.2.2", + "objc2-metal", +] + +[[package]] +name = "objc2-symbols" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a684efe3dec1b305badae1a28f6555f6ddd3bb2c2267896782858d5a78404dc" +dependencies = [ + "objc2 0.5.2", + "objc2-foundation 0.2.2", +] + [[package]] name = "objc2-system-configuration" version = "0.3.2" @@ -7874,6 +8197,51 @@ dependencies = [ "objc2-core-foundation", ] +[[package]] +name = "objc2-ui-kit" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8bb46798b20cd6b91cbd113524c490f1686f4c4e8f49502431415f3512e2b6f" +dependencies = [ + "bitflags 2.10.0", + "block2", + "objc2 0.5.2", + "objc2-cloud-kit", + "objc2-core-data", + "objc2-core-image", + "objc2-core-location", + "objc2-foundation 0.2.2", + "objc2-link-presentation", + "objc2-quartz-core", + "objc2-symbols", + "objc2-uniform-type-identifiers", + "objc2-user-notifications", +] + +[[package]] +name = "objc2-uniform-type-identifiers" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44fa5f9748dbfe1ca6c0b79ad20725a11eca7c2218bceb4b005cb1be26273bfe" +dependencies = [ + "block2", + "objc2 0.5.2", + "objc2-foundation 0.2.2", +] + +[[package]] +name = "objc2-user-notifications" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76cfcbf642358e8689af64cee815d139339f3ed8ad05103ed5eaf73db8d84cb3" +dependencies = [ + "bitflags 2.10.0", + "block2", + "objc2 0.5.2", + "objc2-core-location", + "objc2-foundation 0.2.2", +] + [[package]] name = "octseq" version = "0.6.1" @@ -7885,6 +8253,26 @@ dependencies = [ "smallvec", ] +[[package]] +name = "odbc-api" +version = "19.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f017d3949731e436bc1bb9a1fbc34197c2f39c588cdcb60d21adb1f8dd3b8514" +dependencies = [ + "atoi", + "log", + "odbc-sys", + "thiserror 2.0.18", + "widestring 1.2.1", + "winit", +] + +[[package]] +name = "odbc-sys" +version = "0.27.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1896e52e97c2f0cf997cc627380f1af1ecb3f6c29ce6175047cd38adaadb46f5" + [[package]] name = "ofb" version = "0.7.1" @@ -8003,7 +8391,7 @@ checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45" dependencies = [ "bitflags 2.10.0", "cfg-if", - "foreign-types", + "foreign-types 0.3.2", "libc", "openssl-macros", "openssl-sys", @@ -8073,6 +8461,16 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "orbclient" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5df339f526ea9a60e371768d50efc2f2508c7203290731565d1f7a6f71d21747" +dependencies = [ + "libc", + "libredox", +] + [[package]] name = "ordered-float" version = "2.10.1" @@ -9568,6 +9966,12 @@ dependencies = [ "bitflags 2.10.0", ] +[[package]] +name = "raw-window-handle" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20675572f6f24e9e76ef639bc5552774ed45f1c30e2951e1e99c59888861c539" + [[package]] name = "rawpointer" version = "0.2.1" @@ -9665,6 +10069,15 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "redox_syscall" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "redox_syscall" version = "0.5.12" @@ -11088,6 +11501,15 @@ dependencies = [ "futures-lite", ] +[[package]] +name = "smol_str" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd538fb6910ac1099850255cf94a94df6551fbdd602454387d0adb2d1ca6dead" +dependencies = [ + "serde", +] + [[package]] name = "smpl_jwt" version = "0.8.0" @@ -13234,6 +13656,7 @@ dependencies = [ "colored", "console-subscriber", "criterion", + "cron", "csv", "cuckoo-clock", "databend-client", @@ -13296,6 +13719,7 @@ dependencies = [ "nkeys", "nom 8.0.0", "notify", + "odbc-api", "opendal", "openssl", "openssl-probe", @@ -14222,9 +14646,9 @@ dependencies = [ "jni", "log", "ndk-context", - "objc2", - "objc2-app-kit", - "objc2-foundation", + "objc2 0.6.4", + "objc2-app-kit 0.3.2", + "objc2-foundation 0.3.2", "url", "web-sys", ] @@ -14296,9 +14720,9 @@ checksum = "c168940144dd21fd8046987c16a46a33d5fc84eec29ef9dcddc2ac9e31526b7c" [[package]] name = "widestring" -version = "1.0.2" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "653f141f39ec16bba3c5abe400a0c60da7468261cc2cbf36805022876bc721a8" +checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" [[package]] name = "winapi" @@ -14500,7 +14924,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "193cae8e647981c35bc947fdd57ba7928b1fa0d4a79305f6dd2dc55221ac35ac" dependencies = [ "bitflags 2.10.0", - "widestring 1.0.2", + "widestring 1.2.1", "windows-sys 0.59.0", ] @@ -14762,6 +15186,46 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" +[[package]] +name = "winit" +version = "0.30.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6755fa58a9f8350bd1e472d4c3fcc25f824ec358933bba33306d0b63df5978d" +dependencies = [ + "android-activity", + "atomic-waker", + "bitflags 2.10.0", + "block2", + "calloop", + "cfg_aliases", + "concurrent-queue", + "core-foundation 0.9.3", + "core-graphics", + "cursor-icon", + "dpi", + "js-sys", + "libc", + "ndk", + "objc2 0.5.2", + "objc2-app-kit 0.2.2", + "objc2-foundation 0.2.2", + "objc2-ui-kit", + "orbclient", + "pin-project", + "raw-window-handle", + "redox_syscall 0.4.1", + "rustix 0.38.40", + "smol_str", + "tracing 0.1.44", + "unicode-segmentation", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "web-time", + "windows-sys 0.52.0", + "xkbcommon-dl", +] + [[package]] name = "winnow" version = "0.5.18" @@ -14771,6 +15235,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "winnow" +version = "0.6.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e90edd2ac1aa278a5c4599b1d89cf03074b610800f866d4026dc199d7929a28" +dependencies = [ + "memchr", +] + [[package]] name = "winnow" version = "0.7.13" @@ -14938,6 +15411,25 @@ dependencies = [ "tap", ] +[[package]] +name = "xkbcommon-dl" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d039de8032a9a8856a6be89cea3e5d12fdd82306ab7c94d74e6deab2460651c5" +dependencies = [ + "bitflags 2.10.0", + "dlib", + "log", + "once_cell", + "xkeysym", +] + +[[package]] +name = "xkeysym" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9cc00251562a284751c9973bace760d86c0276c471b4be569fe6b068ee97a56" + [[package]] name = "xmlparser" version = "0.13.6" diff --git a/Cargo.toml b/Cargo.toml index 050d818fe2ae6..0ae353692347f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -77,6 +77,7 @@ start = false # libc requirements are defined by `cross` # https://github.com/rust-embedded/cross#supported-targets # Though, it seems like aarch64 libc is actually 2.18 and not 2.19 +# libodbc: glibc ODBC builds only; `libodbc1` covers older distros (Debian 11 / Ubuntu 20.04). [package.metadata.deb.variants.arm-unknown-linux-gnueabi] depends = "libc6 (>= 2.15)" @@ -84,13 +85,13 @@ depends = "libc6 (>= 2.15)" depends = "libc6 (>= 2.15)" [package.metadata.deb.variants.x86_64-unknown-linux-gnu] -depends = "libc6 (>= 2.15)" +depends = "libc6 (>= 2.15), libodbc2 | libodbc1" [package.metadata.deb.variants.x86_64-unknown-linux-musl] depends = "" [package.metadata.deb.variants.aarch64-unknown-linux-gnu] -depends = "libc6 (>= 2.18)" +depends = "libc6 (>= 2.18), libodbc2 | libodbc1" [package.metadata.deb.variants.aarch64-unknown-linux-musl] depends = "" @@ -364,6 +365,10 @@ lapin = { version = "4.3.0", default-features = false, features = ["tokio", "nat async-rs = { version = "0.8", default-features = false, features = ["tokio"], optional = true } deadpool = { version = "0.13.0", default-features = false, features = ["managed", "rt_tokio_1"], optional = true } +# ODBC +odbc-api = { version = "19.1.0", optional = true } +cron = { version = "0.15.0", features = ["serde"], optional = true } + # Opentelemetry hex = { version = "0.4.3", default-features = false, optional = true } @@ -548,13 +553,23 @@ ntapi = { git = "https://github.com/MSxDOS/ntapi.git", rev = "24fc1e47677fc9f6e3 # macOS releases pick it up through the `target-aarch64-apple-darwin` feature (see target # table below). Local dev builds skip it so they don't need `libsasl2-dev` / cyrus-sasl # installed, and so rdkafka's build script doesn't link GSSAPI. -default = ["enable-api-client", "sources-dnstap", "tikv-jemallocator"] +# +# Note on odbc: not in `sources-logs` (dynamic libodbc). Enabled on default and the +# *-unknown-linux-gnu targets (x86_64 + aarch64); omitted from musl/arm and Apple +# release targets so those releases stay static / linkable without a sysroot libodbc. +# The gnu cross images install the matching target-arch unixODBC dev files +# (see scripts/cross/bootstrap-ubuntu.sh). macOS links libodbc against the Homebrew +# dylib, so the GA archive (target-aarch64-apple-darwin) omits it to stay self- +# contained and launchable on clean Macs (otherwise: unresolved libodbc.*.dylib). +# Note `default` still includes it, so macOS dev/source builds require a Homebrew +# libodbc at build + run time. +default = ["enable-api-client", "sources-dnstap", "tikv-jemallocator", "sources-odbc"] antithesis-scenario-memory = ["dep:antithesis-instrumentation"] antithesis-scenario-disk = ["dep:antithesis-instrumentation", "vector-lib/antithesis-disk-asserts"] # Default features for `cargo docs`. We're not using `gssapi` which would require installing libsasl2 in our doc environment. -docs = ["enable-api-client", "sources-dnstap"] +docs = ["enable-api-client", "sources-dnstap", "sources-odbc"] # Default features for *-unknown-linux-* which make use of `cmake` for dependencies -default-cmake = ["enable-api-client", "sources-dnstap", "tikv-jemallocator", "vendored", "rdkafka?/cmake_build"] +default-cmake = ["enable-api-client", "sources-dnstap", "tikv-jemallocator", "vendored", "rdkafka?/cmake_build", "sources-odbc"] # Enables Kerberos / GSSAPI SASL support for kafka via dynamic linkage to a # system-provided libsasl2 (and the host's GSS implementation). Requires @@ -578,7 +593,7 @@ vendored = ["gssapi-vendored"] base = ["api", "enrichment-tables", "sinks", "sources", "transforms", "secrets", "vrl/stdlib", "codecs-parquet"] enable-api-client = ["base", "api-client"] default-musl = ["enable-api-client", "sources-dnstap", "tikv-jemallocator", "vendored", "rdkafka?/cmake_build"] -default-no-api-client = ["base", "sources-dnstap", "tikv-jemallocator", "vendored"] +default-no-api-client = ["base", "sources-dnstap", "tikv-jemallocator", "vendored", "sources-odbc"] tokio-console = ["dep:console-subscriber", "tokio/tracing"] @@ -591,20 +606,21 @@ vrl-functions-crypto = ["vrl/enable_crypto_functions"] # Enables the binary secret-backend-example secret-backend-example = ["transforms"] -all-logs = ["sinks-logs", "sources-logs", "sources-dnstap", "transforms-logs"] +all-logs = ["sinks-logs", "sources-logs", "sources-dnstap", "sources-odbc", "transforms-logs"] all-metrics = ["sinks-metrics", "sources-metrics", "transforms-metrics"] # Target specific release features. # The `make` tasks will select this according to the appropriate triple. # Use this section to turn off or on specific features for specific triples. +# sources-odbc: glibc release targets only (x86_64 + aarch64); see note on `default` above. target-base = ["enable-api-client", "rdkafka?/cmake_build", "sources-dnstap"] -target-aarch64-unknown-linux-gnu = ["target-base", "tikv-jemallocator"] +target-aarch64-unknown-linux-gnu = ["target-base", "tikv-jemallocator", "sources-odbc"] target-aarch64-unknown-linux-musl = ["target-base", "tikv-jemallocator"] target-armv7-unknown-linux-gnueabihf = ["target-base", "tikv-jemallocator"] target-armv7-unknown-linux-musleabihf = ["target-base"] target-arm-unknown-linux-gnueabi = ["target-base", "tikv-jemallocator"] target-arm-unknown-linux-musleabi = ["target-base"] -target-x86_64-unknown-linux-gnu = ["target-base", "tikv-jemallocator", "vendored"] +target-x86_64-unknown-linux-gnu = ["target-base", "tikv-jemallocator", "vendored", "sources-odbc"] target-x86_64-unknown-linux-musl = ["target-base", "tikv-jemallocator"] # Apple targets opt into `gssapi` (dynamic linkage to system libsasl2 + Apple's # GSS framework). The `gssapi-vendored` path does not currently link on @@ -764,6 +780,7 @@ sources-mongodb_metrics = ["dep:mongodb"] sources-mqtt = ["dep:rumqttc"] sources-nats = ["dep:async-nats", "dep:nkeys"] sources-nginx_metrics = ["dep:nom"] +sources-odbc = ["dep:odbc-api", "dep:cron", "dep:windows"] sources-okta = ["sources-utils-http-client"] sources-opentelemetry = [ "dep:hex", @@ -1027,6 +1044,7 @@ all-integration-tests = [ "mqtt-integration-tests", "nats-integration-tests", "nginx-integration-tests", + "odbc-integration-tests", "opentelemetry-integration-tests", "postgresql_metrics-integration-tests", "postgres_sink-integration-tests", @@ -1096,6 +1114,7 @@ mongodb_metrics-integration-tests = ["sources-mongodb_metrics"] mqtt-integration-tests = ["sinks-mqtt", "sources-mqtt"] nats-integration-tests = ["sinks-nats", "sources-nats"] nginx-integration-tests = ["sources-nginx_metrics"] +odbc-integration-tests = ["sources-odbc"] opentelemetry-integration-tests = ["sources-opentelemetry", "dep:prost"] postgresql_metrics-integration-tests = ["sources-postgresql_metrics"] postgres_sink-integration-tests = ["sinks-postgres"] diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index e3fa1f9f67d90..d724621ad0b3b 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -12,6 +12,8 @@ amq-protocol,https://github.com/amqp-rs/amq-protocol,BSD-2-Clause,Marc-Antoine P amq-protocol-tcp,https://github.com/amqp-rs/amq-protocol,BSD-2-Clause,Marc-Antoine Perennou amq-protocol-types,https://github.com/amqp-rs/amq-protocol,BSD-2-Clause,Marc-Antoine Perennou amq-protocol-uri,https://github.com/amqp-rs/amq-protocol,BSD-2-Clause,Marc-Antoine Perennou +android-activity,https://github.com/rust-mobile/android-activity,MIT OR Apache-2.0,The android-activity Authors +android-properties,https://github.com/miklelappo/android-properties,MIT,Mikhail Lappo android_system_properties,https://github.com/nical/android_system_properties,MIT OR Apache-2.0,Nicolas Silva anstream,https://github.com/rust-cli/anstyle,MIT OR Apache-2.0,The anstream Authors anstyle,https://github.com/rust-cli/anstyle,MIT OR Apache-2.0,The anstyle Authors @@ -119,6 +121,7 @@ bitmask-enum,https://github.com/Lukas3674/rust-bitmask-enum,MIT OR Apache-2.0,Lu bitvec,https://github.com/bitvecto-rs/bitvec,MIT,The bitvec Authors block-buffer,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers block-padding,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers +block2,https://github.com/madsmtm/objc2,MIT,"Steven Sheldon, Mads Marquart " blocking,https://github.com/smol-rs/blocking,Apache-2.0 OR MIT,Stjepan Glavina bloomy,https://docs.rs/bloomy/,MIT,"Aleksandr Bezobchuk , Alexis Sellier " bollard,https://github.com/fussybeaver/bollard,Apache-2.0,Bollard contributors @@ -142,6 +145,7 @@ byteorder,https://github.com/BurntSushi/byteorder,Unlicense OR MIT,Andrew Gallan bytes,https://github.com/tokio-rs/bytes,MIT,"Carl Lerche , Sean McArthur " bytes-utils,https://github.com/vorner/bytes-utils,Apache-2.0 OR MIT,Michal 'vorner' Vaner bytesize,https://github.com/bytesize-rs/bytesize,Apache-2.0,"Hyunsik Choi , MrCroxx , Rob Ede " +calloop,https://github.com/Smithay/calloop,MIT,Elinor Berger castaway,https://github.com/sagebind/castaway,MIT,Stephen M. Coakley cbc,https://github.com/RustCrypto/block-modes,MIT OR Apache-2.0,RustCrypto Developers cfb-mode,https://github.com/RustCrypto/block-modes,MIT OR Apache-2.0,RustCrypto Developers @@ -188,6 +192,8 @@ cookie_store,https://github.com/pfernie/cookie_store,MIT OR Apache-2.0,Patrick F core-foundation,https://github.com/servo/core-foundation-rs,MIT OR Apache-2.0,The Servo Project Developers core-foundation,https://github.com/servo/core-foundation-rs,MIT OR Apache-2.0,The Servo Project Developers core-foundation-sys,https://github.com/servo/core-foundation-rs,MIT OR Apache-2.0,The Servo Project Developers +core-graphics,https://github.com/servo/core-foundation-rs,MIT OR Apache-2.0,The Servo Project Developers +core-graphics-types,https://github.com/servo/core-foundation-rs,MIT OR Apache-2.0,The Servo Project Developers cpubits,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers cpufeatures,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers crc,https://github.com/mrhooray/crc-rs,MIT OR Apache-2.0,"Rui Hu , Akhil Velagapudi <4@4khil.com>" @@ -195,6 +201,7 @@ crc-catalog,https://github.com/akhilles/crc-catalog,MIT OR Apache-2.0,Akhil Vela crc-fast,https://github.com/awesomized/crc-fast-rust,MIT OR Apache-2.0,Don MacAskill crc32fast,https://github.com/srijs/rust-crc32fast,MIT OR Apache-2.0,"Sam Rijs , Alex Crichton " critical-section,https://github.com/rust-embedded/critical-section,MIT OR Apache-2.0,The critical-section Authors +cron,https://github.com/zslayton/cron,MIT OR Apache-2.0,Zack Slayton crossbeam-channel,https://github.com/crossbeam-rs/crossbeam,MIT OR Apache-2.0,The crossbeam-channel Authors crossbeam-epoch,https://github.com/crossbeam-rs/crossbeam,MIT OR Apache-2.0,The crossbeam-epoch Authors crossbeam-queue,https://github.com/crossbeam-rs/crossbeam,MIT OR Apache-2.0,The crossbeam-queue Authors @@ -211,6 +218,7 @@ ctr,https://github.com/RustCrypto/block-modes,MIT OR Apache-2.0,RustCrypto Devel ctutils,https://github.com/RustCrypto/utils,Apache-2.0 OR MIT,RustCrypto Developers cuckoo-clock,https://github.com/Quad9DNS/cuckoo-clock,MIT,"John Todd , Ensar Sarajčić " curl-sys,https://github.com/alexcrichton/curl-rust,MIT,Alex Crichton +cursor-icon,https://github.com/rust-windowing/cursor-icon,MIT OR Apache-2.0 OR Zlib,Kirill Chibisov curve25519-dalek,https://github.com/dalek-cryptography/curve25519-dalek/tree/main/curve25519-dalek,BSD-3-Clause,"Isis Lovecruft , Henry de Valence " curve25519-dalek-derive,https://github.com/dalek-cryptography/curve25519-dalek,MIT OR Apache-2.0,The curve25519-dalek-derive Authors darling,https://github.com/TedDriggs/darling,MIT,Ted Driggs @@ -238,14 +246,17 @@ derive_more-impl,https://github.com/JelteF/derive_more,MIT,Jelte Fennema , Mary " displaydoc,https://github.com/yaahc/displaydoc,MIT OR Apache-2.0,Jane Lusby +dlib,https://github.com/elinorbgr/dlib,MIT,Elinor Berger dns-lookup,https://github.com/keeperofdakeys/dns-lookup,MIT OR Apache-2.0,Josh Driver doc-comment,https://github.com/GuillaumeGomez/doc-comment,MIT,Guillaume Gomez document-features,https://github.com/slint-ui/document-features,MIT OR Apache-2.0,Slint Developers domain,https://github.com/nlnetlabs/domain,BSD-3-Clause,NLnet Labs domain-macros,https://github.com/nlnetlabs/domain,BSD-3-Clause,NLnet Labs dotenvy,https://github.com/allan2/dotenvy,MIT,"Noemi Lapresta , Craig Hills , Mike Piccolo , Alice Maz , Sean Griffin , Adam Sharp , Arpad Borsos , Allan Zhang " +dpi,https://github.com/rust-windowing/winit,Apache-2.0 AND MIT,The dpi Authors dyn-clone,https://github.com/dtolnay/dyn-clone,MIT OR Apache-2.0,David Tolnay ecdsa,https://github.com/RustCrypto/signatures/tree/master/ecdsa,Apache-2.0 OR MIT,RustCrypto Developers ed25519,https://github.com/RustCrypto/signatures/tree/master/ed25519,Apache-2.0 OR MIT,RustCrypto Developers @@ -290,6 +301,7 @@ flume,https://github.com/zesterer/flume,Apache-2.0 OR MIT,Joshua Barretto foldhash,https://github.com/orlp/foldhash,Zlib,Orson Peters foreign-types,https://github.com/sfackler/foreign-types,MIT OR Apache-2.0,Steven Fackler +foreign-types-macros,https://github.com/sfackler/foreign-types,MIT OR Apache-2.0,Steven Fackler foreign-types-shared,https://github.com/sfackler/foreign-types,MIT OR Apache-2.0,Steven Fackler form_urlencoded,https://github.com/servo/rust-url,MIT OR Apache-2.0,The rust-url developers fraction,https://github.com/dnsl48/fraction,MIT OR Apache-2.0,dnsl48 @@ -405,6 +417,7 @@ jiff,https://github.com/BurntSushi/jiff,Unlicense OR MIT,Andrew Gallant jni,https://github.com/jni-rs/jni-rs,MIT OR Apache-2.0,jni team jni-macros,https://github.com/jni-rs/jni-rs,MIT OR Apache-2.0,The jni-macros Authors +jni-sys,https://github.com/jni-rs/jni-sys,MIT OR Apache-2.0,Steven Fackler jni-sys,https://github.com/jni-rs/jni-sys,MIT OR Apache-2.0,"Steven Fackler , Robert Bragg " jni-sys-macros,https://github.com/jni-rs/jni-sys,MIT OR Apache-2.0,Robert Bragg js-sys,https://github.com/wasm-bindgen/wasm-bindgen/tree/master/crates/js-sys,MIT OR Apache-2.0,The wasm-bindgen Developers @@ -489,7 +502,9 @@ mongodb,https://github.com/mongodb/mongo-rust-driver,Apache-2.0,"Saghm Rossi , Patrick Freed , Isabel Atkinson , Abraham Egnor , Kaitlin Mahar , Patrick Meredith " murmur3,https://github.com/stusmall/murmur3,MIT OR Apache-2.0,Stu Small native-tls,https://github.com/sfackler/rust-native-tls,MIT OR Apache-2.0,Steven Fackler +ndk,https://github.com/rust-mobile/ndk,MIT OR Apache-2.0,The Rust Mobile contributors ndk-context,https://github.com/rust-windowing/android-ndk-rs,MIT OR Apache-2.0,The Rust Windowing contributors +ndk-sys,https://github.com/rust-mobile/ndk,MIT OR Apache-2.0,The Rust Windowing contributors newtype-uuid,https://github.com/oxidecomputer/newtype-uuid,MIT OR Apache-2.0,The newtype-uuid Authors nibble_vec,https://github.com/michaelsproul/rust_nibble_vec,MIT,Michael Sproul nix,https://github.com/nix-rust/nix,MIT,The nix-rust Project Developers @@ -520,14 +535,31 @@ num_enum,https://github.com/illicitonion/num_enum,BSD-3-Clause OR MIT OR Apache- num_enum_derive,https://github.com/illicitonion/num_enum,BSD-3-Clause OR MIT OR Apache-2.0,"Daniel Wagner-Hall , Daniel Henry-Mantilla , Vincent Esche " num_threads,https://github.com/jhpratt/num_threads,MIT OR Apache-2.0,Jacob Pratt oauth2,https://github.com/ramosbugs/oauth2-rs,MIT OR Apache-2.0,"Alex Crichton , Florin Lipan , David A. Ramos " +objc-sys,https://github.com/madsmtm/objc2,MIT,Mads Marquart objc2,https://github.com/madsmtm/objc2,MIT,Mads Marquart +objc2,https://github.com/madsmtm/objc2,MIT,"Steven Sheldon, Mads Marquart " +objc2-app-kit,https://github.com/madsmtm/objc2,MIT,The objc2-app-kit Authors objc2-app-kit,https://github.com/madsmtm/objc2,Zlib OR Apache-2.0 OR MIT,The objc2-app-kit Authors +objc2-cloud-kit,https://github.com/madsmtm/objc2,MIT,The objc2-cloud-kit Authors +objc2-contacts,https://github.com/madsmtm/objc2,MIT,The objc2-contacts Authors +objc2-core-data,https://github.com/madsmtm/objc2,MIT,The objc2-core-data Authors objc2-core-foundation,https://github.com/madsmtm/objc2,Zlib OR Apache-2.0 OR MIT,The objc2-core-foundation Authors +objc2-core-image,https://github.com/madsmtm/objc2,MIT,The objc2-core-image Authors +objc2-core-location,https://github.com/madsmtm/objc2,MIT,The objc2-core-location Authors objc2-encode,https://github.com/madsmtm/objc2,MIT,Mads Marquart objc2-foundation,https://github.com/madsmtm/objc2,MIT,The objc2-foundation Authors objc2-io-kit,https://github.com/madsmtm/objc2,Zlib OR Apache-2.0 OR MIT,The objc2-io-kit Authors +objc2-link-presentation,https://github.com/madsmtm/objc2,MIT,The objc2-link-presentation Authors +objc2-metal,https://github.com/madsmtm/objc2,MIT,The objc2-metal Authors +objc2-quartz-core,https://github.com/madsmtm/objc2,MIT,The objc2-quartz-core Authors +objc2-symbols,https://github.com/madsmtm/objc2,MIT,The objc2-symbols Authors objc2-system-configuration,https://github.com/madsmtm/objc2,Zlib OR Apache-2.0 OR MIT,The objc2-system-configuration Authors +objc2-ui-kit,https://github.com/madsmtm/objc2,MIT,The objc2-ui-kit Authors +objc2-uniform-type-identifiers,https://github.com/madsmtm/objc2,MIT,The objc2-uniform-type-identifiers Authors +objc2-user-notifications,https://github.com/madsmtm/objc2,MIT,The objc2-user-notifications Authors octseq,https://github.com/NLnetLabs/octets,BSD-3-Clause,NLnet Labs +odbc-api,https://github.com/pacman82/odbc-api,MIT,Markus Klein +odbc-sys,https://github.com/pacman82/odbc-sys,MIT,Markus Klein ofb,https://github.com/RustCrypto/block-modes,MIT OR Apache-2.0,RustCrypto Developers once_cell,https://github.com/matklad/once_cell,MIT OR Apache-2.0,Aleksey Kladov onig,https://github.com/iwillspeak/rust-onig,MIT,"Will Speak , Ivan Ivashchenko " @@ -539,6 +571,7 @@ openssl,https://github.com/rust-openssl/rust-openssl,Apache-2.0,Steven Fackler < openssl-macros,https://github.com/sfackler/rust-openssl,MIT OR Apache-2.0,The openssl-macros Authors openssl-probe,https://github.com/alexcrichton/openssl-probe,MIT OR Apache-2.0,Alex Crichton openssl-sys,https://github.com/rust-openssl/rust-openssl,MIT,"Alex Crichton , Steven Fackler " +orbclient,https://gitlab.redox-os.org/redox-os/orbclient,MIT,Jeremy Soller ordered-float,https://github.com/reem/rust-ordered-float,MIT,"Jonathan Reem , Matt Brubeck " outref,https://github.com/Nugine/outref,MIT,The outref Authors owo-colors,https://github.com/owo-colors/owo-colors,MIT,jam1garner <8260240+jam1garner@users.noreply.github.com> @@ -633,6 +666,7 @@ ratatui-core,https://github.com/ratatui/ratatui,MIT,"Florian Dehau , The Ratatui Developers" ratatui-widgets,https://github.com/ratatui/ratatui,MIT,"Florian Dehau , The Ratatui Developers" raw-cpuid,https://github.com/gz/rust-cpuid,MIT,Gerd Zellweger +raw-window-handle,https://github.com/rust-windowing/raw-window-handle,MIT OR Apache-2.0 OR Zlib,Osspial rdkafka,https://github.com/fede1024/rust-rdkafka,MIT,Federico Giraud rdkafka-sys,https://github.com/fede1024/rust-rdkafka,MIT,Federico Giraud redis,https://github.com/redis-rs/redis-rs,BSD-3-Clause,The redis Authors @@ -731,6 +765,7 @@ sketches-ddsketch,https://github.com/mheffner/rust-sketches-ddsketch,Apache-2.0, slab,https://github.com/tokio-rs/slab,MIT,Carl Lerche smallvec,https://github.com/servo/rust-smallvec,MIT OR Apache-2.0,The Servo Project Developers smol,https://github.com/smol-rs/smol,Apache-2.0 OR MIT,Stjepan Glavina +smol_str,https://github.com/rust-analyzer/smol_str,MIT OR Apache-2.0,Aleksey Kladov smpl_jwt,https://github.com/durch/rust-jwt,MIT,Drazen Urch snafu,https://github.com/shepmaster/snafu,MIT OR Apache-2.0,Jake Goulding snafu-derive,https://github.com/shepmaster/snafu,MIT OR Apache-2.0,Jake Goulding @@ -891,8 +926,8 @@ webpki-root-certs,https://github.com/rustls/webpki-roots,CDLA-Permissive-2.0,The webpki-roots,https://github.com/rustls/webpki-roots,CDLA-Permissive-2.0,The webpki-roots Authors webpki-roots,https://github.com/rustls/webpki-roots,MPL-2.0,The webpki-roots Authors whoami,https://github.com/ardaku/whoami,Apache-2.0 OR BSL-1.0 OR MIT,The whoami Authors +widestring,https://github.com/VoidStarKat/widestring-rs,MIT OR Apache-2.0,The widestring Authors widestring,https://github.com/starkat99/widestring-rs,MIT OR Apache-2.0,Kathryn Long -widestring,https://github.com/starkat99/widestring-rs,MIT OR Apache-2.0,The widestring Authors winapi,https://github.com/retep998/winapi-rs,MIT OR Apache-2.0,Peter Atashian winapi-i686-pc-windows-gnu,https://github.com/retep998/winapi-rs,MIT OR Apache-2.0,Peter Atashian winapi-util,https://github.com/BurntSushi/winapi-util,Unlicense OR MIT,Andrew Gallant @@ -923,6 +958,7 @@ windows_i686_msvc,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Micr windows_x86_64_gnu,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft windows_x86_64_gnullvm,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft windows_x86_64_msvc,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft +winit,https://github.com/rust-windowing/winit,Apache-2.0,"The winit contributors, Pierre Krieger " winnow,https://github.com/winnow-rs/winnow,MIT,The winnow Authors winreg,https://github.com/gentoo90/winreg-rs,MIT,Igor Shaula wit-bindgen,https://github.com/bytecodealliance/wit-bindgen,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,Alex Crichton @@ -935,6 +971,8 @@ woothee,https://github.com/woothee/woothee-rust,Apache-2.0,hhatto +xkbcommon-dl,https://github.com/rust-windowing/xkbcommon-dl,MIT,Francesca Frangipane +xkeysym,https://github.com/notgull/xkeysym,MIT OR Apache-2.0 OR Zlib,John Nunley xmlparser,https://github.com/RazrFalcon/xmlparser,MIT OR Apache-2.0,Yevhenii Reizner xxhash-rust,https://github.com/DoumanAsh/xxhash-rust,BSL-1.0,Douman yoke,https://github.com/unicode-org/icu4x,Unicode-3.0,Manish Goregaokar diff --git a/changelog.d/24044_odbc_source.feature.md b/changelog.d/24044_odbc_source.feature.md new file mode 100644 index 0000000000000..c2669e779a6cd --- /dev/null +++ b/changelog.d/24044_odbc_source.feature.md @@ -0,0 +1,7 @@ +Added a new `odbc` source that periodically queries databases through +[ODBC (Open Database Connectivity)](https://en.wikipedia.org/wiki/Open_Database_Connectivity) +and emits each returned row as a structured log event. It supports scheduled and parameterized +queries, batched row fetching, and persisted tracking columns for incremental collection across +runs. A database-specific ODBC driver must be installed separately. + +authors: powerumc diff --git a/distribution/docker/debian/Dockerfile b/distribution/docker/debian/Dockerfile index 33a3338312956..275de55d43128 100644 --- a/distribution/docker/debian/Dockerfile +++ b/distribution/docker/debian/Dockerfile @@ -3,7 +3,7 @@ FROM docker.io/debian:trixie-slim@sha256:3a39a0592364683e6bab97937b72cad5a8fa6dc WORKDIR /vector COPY vector_*.deb ./ -RUN dpkg -i vector_*_"$(dpkg --print-architecture)".deb +RUN dpkg-deb -x vector_*_"$(dpkg --print-architecture)".deb / RUN mkdir -p /var/lib/vector @@ -14,9 +14,9 @@ LABEL org.opencontainers.image.url="https://vector.dev" LABEL org.opencontainers.image.source="https://github.com/vectordotdev/vector" LABEL org.opencontainers.image.documentation="https://vector.dev/docs" -# we want the latest versions of these +# libsasl2-2: rdkafka GSSAPI. libodbc2: unixODBC (sources-odbc). # hadolint ignore=DL3008 -RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates tzdata systemd libsasl2-2 && rm -rf /var/lib/apt/lists/* +RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates tzdata systemd libsasl2-2 libodbc2 && rm -rf /var/lib/apt/lists/* COPY --from=builder /usr/bin/vector /usr/bin/vector COPY --from=builder /usr/share/vector /usr/share/vector diff --git a/distribution/docker/distroless-libc/Dockerfile b/distribution/docker/distroless-libc/Dockerfile index 38ba4b50daf26..03fa85af6ace7 100644 --- a/distribution/docker/distroless-libc/Dockerfile +++ b/distribution/docker/distroless-libc/Dockerfile @@ -2,16 +2,23 @@ FROM docker.io/debian:trixie-slim@sha256:3a39a0592364683e6bab97937b72cad5a8fa6dc WORKDIR /vector +# Shared libs to copy into distroless/cc (not shipped there). +# hadolint ignore=DL3008 +RUN apt-get update \ + && apt-get install -y --no-install-recommends libodbc2 \ + && rm -rf /var/lib/apt/lists/* + COPY vector_*.deb ./ -RUN dpkg -i vector_*_"$(dpkg --print-architecture)".deb +RUN dpkg-deb -x vector_*_"$(dpkg --print-architecture)".deb / RUN mkdir -p /var/lib/vector -# Stage libz at its arch-specific path so it can be copied into the distroless -# runtime. distroless/cc does not include zlib; vector (via rdkafka) links -# against it dynamically and we deliberately keep it as a runtime dependency. -RUN src=$(dpkg -L zlib1g | grep -E '/libz\.so\.1$') && \ - install -D "$(realpath "$src")" "/staging${src}" +RUN src=$(dpkg -L zlib1g | grep -E '/libz\.so\.1$') \ + && install -D "$(realpath "$src")" "/staging${src}" \ + && src=$(dpkg -L libodbc2 | grep -E '/libodbc\.so\.2$') \ + && install -D "$(realpath "$src")" "/staging${src}" \ + && src=$(dpkg -L libltdl7 | grep -E '/libltdl\.so\.7$') \ + && install -D "$(realpath "$src")" "/staging${src}" # distroless doesn't use static tags # hadolint ignore=DL3007 diff --git a/lib/vector-common/src/internal_event/metric_name.rs b/lib/vector-common/src/internal_event/metric_name.rs index ec052045ed7fd..b46414121d83f 100644 --- a/lib/vector-common/src/internal_event/metric_name.rs +++ b/lib/vector-common/src/internal_event/metric_name.rs @@ -12,6 +12,7 @@ pub enum CounterName { ComponentSentBytesTotal, ComponentDiscardedEventsTotal, ComponentErrorsTotal, + ComponentExecutedEventsTotal, ComponentTimedOutEventsTotal, ComponentTimedOutRequestsTotal, BufferReceivedEventsTotal, @@ -274,6 +275,7 @@ impl CounterName { Self::ComponentSentBytesTotal => "component_sent_bytes_total", Self::ComponentDiscardedEventsTotal => "component_discarded_events_total", Self::ComponentErrorsTotal => "component_errors_total", + Self::ComponentExecutedEventsTotal => "component_executed_events_total", Self::ComponentTimedOutEventsTotal => "component_timed_out_events_total", Self::ComponentTimedOutRequestsTotal => "component_timed_out_requests_total", Self::BufferReceivedEventsTotal => "buffer_received_events_total", diff --git a/regression/Dockerfile b/regression/Dockerfile index 01be4e593b546..770b6c8441a3c 100644 --- a/regression/Dockerfile +++ b/regression/Dockerfile @@ -27,7 +27,8 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \ # TARGET # FROM docker.io/debian:trixie-slim@sha256:c85a2732e97694ea77237c61304b3bb410e0e961dd6ee945997a06c788c545bb -RUN apt-get update && apt-get dist-upgrade -y && apt-get -y --no-install-recommends install zlib1g ca-certificates libsasl2-2 && rm -rf /var/lib/apt/lists/* + +RUN apt-get update && apt-get dist-upgrade -y && apt-get -y --no-install-recommends install zlib1g ca-certificates libsasl2-2 libodbc2 && rm -rf /var/lib/apt/lists/* COPY --from=builder /vector/vector /usr/bin/vector RUN mkdir --parents --mode=0777 /var/lib/vector diff --git a/scripts/cross/Dockerfile b/scripts/cross/Dockerfile index 3abe8cd8b2d11..1a3ff5a0b216c 100644 --- a/scripts/cross/Dockerfile +++ b/scripts/cross/Dockerfile @@ -3,10 +3,14 @@ ARG CROSS_DIGEST FROM ghcr.io/cross-rs/${TARGET}@${CROSS_DIGEST} +# Re-declare after FROM so the target triple is available to the build steps +# below (e.g. bootstrap-ubuntu.sh installs target-arch specific dev libraries). +ARG TARGET + # Common steps for all targets COPY scripts/cross/bootstrap-ubuntu.sh / COPY scripts/environment/install-protoc.sh / -RUN /bootstrap-ubuntu.sh && bash /install-protoc.sh +RUN TARGET="${TARGET}" /bootstrap-ubuntu.sh && bash /install-protoc.sh # Allow cmake to find pre-built dependencies (e.g. curl from curl-sys) that # land in CMAKE_PREFIX_PATH rather than the cross sysroot. The cross-rs diff --git a/scripts/cross/bootstrap-ubuntu.sh b/scripts/cross/bootstrap-ubuntu.sh index d89ea85f37ca5..cb56ffb48294c 100755 --- a/scripts/cross/bootstrap-ubuntu.sh +++ b/scripts/cross/bootstrap-ubuntu.sh @@ -24,3 +24,21 @@ apt-get install -y \ unzip \ libsasl2-dev +# unixODBC development files for the `sources-odbc` feature. Only the +# *-unknown-linux-gnu targets enable it (see the `target-*` feature table in +# Cargo.toml); the musl/arm cross targets omit ODBC so they stay linkable +# without a sysroot libodbc. `odbc-sys` links `libodbc.so` dynamically via +# `#[link(name = "odbc")]`, so the dev package must land in the *target* +# sysroot. For the aarch64 GNU cross build that means the arm64 package, not +# the host amd64 one, so the cross linker can resolve `-lodbc`. +case "${TARGET:-}" in + x86_64-unknown-linux-gnu) + apt-get install -y unixodbc-dev + ;; + aarch64-unknown-linux-gnu) + dpkg --add-architecture arm64 + apt-get update + apt-get install -y unixodbc-dev:arm64 + ;; +esac + diff --git a/scripts/environment/install-debian-build-deps.sh b/scripts/environment/install-debian-build-deps.sh index 4de7028a4fbb5..6d019a6b4bfaf 100755 --- a/scripts/environment/install-debian-build-deps.sh +++ b/scripts/environment/install-debian-build-deps.sh @@ -15,8 +15,13 @@ apt-get install -y --no-install-recommends \ libssl-dev \ libxxhash-dev \ mold \ + odbcinst \ + odbc-mariadb \ + odbc-postgresql \ perl \ pkg-config \ + unixodbc \ + unixodbc-dev \ unzip \ zlib1g-dev rm -rf /var/lib/apt/lists/* diff --git a/scripts/verify-install.sh b/scripts/verify-install.sh index ab9e6039f3f14..38756753f9e11 100755 --- a/scripts/verify-install.sh +++ b/scripts/verify-install.sh @@ -9,6 +9,28 @@ set -euo pipefail package="${1:?must pass package as argument}" +# Resolve shared-library deps that plain `dpkg -i` / `rpm -i` do not install +# (glibc ODBC builds need libodbc / unixODBC for `vector --version`). +ensure_odbc_runtime () { + case "$1" in + *.deb) + export DEBIAN_FRONTEND=noninteractive + apt-get update -qq + apt-get install -y libodbc2 || apt-get install -y libodbc1 + ;; + *.rpm) + if command -v dnf >/dev/null 2>&1; then + dnf install -y unixODBC + elif command -v yum >/dev/null 2>&1; then + yum install -y unixODBC + else + echo "No dnf/yum available to install unixODBC" >&2 + exit 1 + fi + ;; + esac +} + install_package () { case "$1" in *.deb) @@ -20,6 +42,7 @@ install_package () { esac } +ensure_odbc_runtime "$package" install_package "$package" getent passwd vector || (echo "vector user missing" && exit 1) diff --git a/src/internal_events/mod.rs b/src/internal_events/mod.rs index 035c909791a5e..a3e1874727b4a 100644 --- a/src/internal_events/mod.rs +++ b/src/internal_events/mod.rs @@ -160,6 +160,9 @@ mod windows_event_log; #[cfg(windows)] mod windows; +#[cfg(feature = "sources-odbc")] +mod odbc_metrics; + #[cfg(any(feature = "transforms-log_to_metric", feature = "sinks-loki"))] mod expansion; #[cfg(feature = "sources-mongodb_metrics")] @@ -258,6 +261,8 @@ pub(crate) use self::metric_to_log::*; pub(crate) use self::mqtt::*; #[cfg(feature = "sources-nginx_metrics")] pub(crate) use self::nginx_metrics::*; +#[cfg(feature = "sources-odbc")] +pub(crate) use self::odbc_metrics::*; #[cfg(any( feature = "sources-kubernetes_logs", feature = "transforms-log_to_metric", diff --git a/src/internal_events/odbc_metrics.rs b/src/internal_events/odbc_metrics.rs new file mode 100644 index 0000000000000..e7f5cf1cfd91d --- /dev/null +++ b/src/internal_events/odbc_metrics.rs @@ -0,0 +1,148 @@ +use vector_common::internal_event::{CounterName, InternalEvent, error_stage, error_type}; +use vector_lib::source_sender::SendError; +use vector_lib::{NamedInternalEvent, counter}; + +use crate::sources::odbc::OdbcError; + +#[derive(Debug, NamedInternalEvent)] +pub struct OdbcFailedError<'a> { + pub statement: &'a str, + pub error: OdbcError, +} + +impl InternalEvent for OdbcFailedError<'_> { + fn emit(self) { + match self.error { + OdbcError::Db { .. } | OdbcError::BlockingTask { .. } => { + error!( + message = "Unable to execute statement.", + statement = %self.statement, + error = %self.error, + error_type = error_type::REQUEST_FAILED, + stage = error_stage::RECEIVING, + ); + counter!( + CounterName::ComponentErrorsTotal, + "error_type" => error_type::REQUEST_FAILED, + "stage" => error_stage::RECEIVING, + ) + .increment(1); + } + OdbcError::Io { .. } => { + error!( + message = "Unable to execute statement.", + statement = %self.statement, + error = %self.error, + error_type = error_type::IO_FAILED, + stage = error_stage::RECEIVING, + ); + counter!( + CounterName::ComponentErrorsTotal, + "error_type" => error_type::IO_FAILED, + "stage" => error_stage::RECEIVING, + ) + .increment(1); + } + // `StreamClosedError` already incremented ComponentErrorsTotal and recorded + // ComponentEventsDropped for the failed chunk. Keep the ODBC-context log only. + OdbcError::SendError { + source: SendError::Closed, + } + | OdbcError::SendFailedAfterCheckpoint { + source: SendError::Closed, + .. + } => { + error!( + message = "Unable to execute statement.", + statement = %self.statement, + error = %self.error, + error_type = error_type::WRITER_FAILED, + stage = error_stage::SENDING, + ); + } + OdbcError::SendError { .. } | OdbcError::SendFailedAfterCheckpoint { .. } => { + error!( + message = "Unable to execute statement.", + statement = %self.statement, + error = %self.error, + error_type = error_type::WRITER_FAILED, + stage = error_stage::SENDING, + ); + counter!( + CounterName::ComponentErrorsTotal, + "error_type" => error_type::WRITER_FAILED, + "stage" => error_stage::SENDING, + ) + .increment(1); + } + OdbcError::Json { .. } | OdbcError::InvalidResultRow => { + error!( + message = "Unable to execute statement.", + statement = %self.statement, + error = %self.error, + error_type = error_type::PARSER_FAILED, + stage = error_stage::PROCESSING, + ); + counter!( + CounterName::ComponentErrorsTotal, + "error_type" => error_type::PARSER_FAILED, + "stage" => error_stage::PROCESSING, + ) + .increment(1); + } + OdbcError::MissingTrackingColumn { .. } + | OdbcError::InvalidTrackingValue { .. } + | OdbcError::InvalidTrackingRow => { + error!( + message = "Invalid ODBC tracking state.", + statement = %self.statement, + error = %self.error, + error_type = error_type::CONFIGURATION_FAILED, + stage = error_stage::PROCESSING, + ); + counter!( + CounterName::ComponentErrorsTotal, + "error_type" => error_type::CONFIGURATION_FAILED, + "stage" => error_stage::PROCESSING, + ) + .increment(1); + } + OdbcError::DuplicateColumnNames { .. } => { + error!( + message = "Query returned duplicate column names.", + statement = %self.statement, + error = %self.error, + error_type = error_type::CONFIGURATION_FAILED, + stage = error_stage::PROCESSING, + ); + counter!( + CounterName::ComponentErrorsTotal, + "error_type" => error_type::CONFIGURATION_FAILED, + "stage" => error_stage::PROCESSING, + ) + .increment(1); + } + OdbcError::Shutdown | OdbcError::ShutdownAfterCheckpoint { .. } => { + // Handled by the scheduler as a clean exit, not as a failure metric. + // ComponentEventsDropped for ShutdownAfterCheckpoint is emitted by the client. + } + } + } +} + +#[derive(Debug, NamedInternalEvent)] +pub struct OdbcQueryExecuted<'a> { + pub statement: &'a str, + pub elapsed: u128, +} + +impl InternalEvent for OdbcQueryExecuted<'_> { + fn emit(self) { + trace!( + message = "Executed statement.", + statement = %self.statement, + elapsedMs = %self.elapsed + ); + counter!(CounterName::CollectCompletedTotal).increment(1); + } +} diff --git a/src/sources/mod.rs b/src/sources/mod.rs index 2384b096f630b..f721c1069ff8d 100644 --- a/src/sources/mod.rs +++ b/src/sources/mod.rs @@ -64,6 +64,8 @@ pub mod mqtt; pub mod nats; #[cfg(feature = "sources-nginx_metrics")] pub mod nginx_metrics; +#[cfg(feature = "sources-odbc")] +pub mod odbc; #[cfg(feature = "sources-okta")] pub mod okta; #[cfg(feature = "sources-opentelemetry")] diff --git a/src/sources/odbc/README.md b/src/sources/odbc/README.md new file mode 100644 index 0000000000000..b1a48f0b5b998 --- /dev/null +++ b/src/sources/odbc/README.md @@ -0,0 +1,48 @@ +# ODBC Development + +## Setup + +#### MacOS + +```shell +brew install unixodbc + +brew install mariadb-connector-odbc@3.2.6 # Install MariaDB ODBC, and you need to configure odbcinst.ini. +``` + +Refs + +- MySQL Connector/ODBC: +- MariaDB Connector/ODBC: + - Homebrew mariadb-connector-odbc: + - ODBC Configuration: + + ```shell + cat << EOF >> /opt/homebrew/etc/odbcinst.ini + + [MariaDB ODBC 3.0 Driver] + Description = MariaDB Connector/ODBC v.3.0 + Driver = /opt/homebrew/Cellar/mariadb-connector-odbc/3.2.6/lib/mariadb/libmaodbc.dylib + EOF + ``` + +- MSSQL + Connector/ODBC: + +## ODBC Tips + +Show ODBC configuration + +```shell +odbcinst -j + +### Output Example ### +# unixODBC 2.3.12 +# DRIVERS............: /opt/homebrew/etc/odbcinst.ini +# SYSTEM DATA SOURCES: /opt/homebrew/etc/odbc.ini +# FILE DATA SOURCES..: /opt/homebrew/etc/ODBCDataSources +# USER DATA SOURCES..: /Users//.odbc.ini +# SQLULEN Size.......: 8 +# SQLLEN Size........: 8 +# SQLSETPOSIROW Size.: 8 +``` diff --git a/src/sources/odbc/client.rs b/src/sources/odbc/client.rs new file mode 100644 index 0000000000000..ae2e654ef5244 --- /dev/null +++ b/src/sources/odbc/client.rs @@ -0,0 +1,2291 @@ +use crate::config::{LogNamespace, SourceContext}; +use crate::event::{Event, LogEvent}; +use crate::internal_events::{ + EventsReceived, OdbcFailedError, OdbcQueryExecuted, StreamClosedError, +}; +use crate::shutdown::ShutdownSignal; +use crate::sinks::prelude::*; +use crate::sources::odbc::config::{OdbcConfig, OdbcStatementParam}; +use chrono::{DateTime, NaiveDateTime, Timelike, Utc}; +use chrono_tz::Tz; +use futures::pin_mut; +use futures_util::StreamExt; +use odbc_api::buffers::{AnySlice, BufferDesc, ColumnarAnyBuffer}; +use odbc_api::parameter::VarCharBox; +use odbc_api::{ + ConnectionOptions, Cursor, CursorRow, DataType, Environment, IntoParameter, ResultSetMetadata, + environment, +}; +use snafu::{ResultExt, Snafu}; +use std::collections::HashSet; +use std::fs::{self, File}; +use std::io::{BufReader, Write}; +use std::mem; +use std::num::NonZeroUsize; +use std::path::{Path, PathBuf}; +use std::str::FromStr; +use std::time::{Duration, Instant}; +use tokio::select; +use vector_common::internal_event::{ + ByteSize, BytesReceived, ComponentEventsDropped, CountByteSize, InternalEventHandle as _, + Protocol, Registered, UNINTENTIONAL, +}; +use vector_common::json_size::JsonSize; +use vector_lib::EstimatedJsonEncodedSizeOf; +use vector_lib::emit; +use vector_lib::source_sender::{SendError, chunk_size_events}; +use vrl::prelude::*; + +const TIMESTAMP_FORMATS: &[&str] = &[ + "%Y-%m-%d %H:%M:%S", + "%Y-%m-%dT%H:%M:%S", + "%Y/%m/%d %H:%M:%S", + "%Y/%m/%dT%H:%M:%S", + "%Y-%m-%d %H:%M:%S%.f", + "%Y-%m-%dT%H:%M:%S%.f", + "%Y/%m/%d %H:%M:%S%.f", + "%Y/%m/%dT%H:%M:%S%.f", +]; + +struct Column { + column_name: String, + column_type: DataType, +} + +/// Columns of the query result. +type Columns = Vec; +/// Rows of the query result. +type Rows = Vec; + +#[derive(Debug, Snafu)] +pub enum OdbcError { + #[snafu(display("ODBC database error: {source}"))] + Db { source: odbc_api::Error }, + + #[snafu(display("File IO error: {source}"))] + Io { source: std::io::Error }, + + #[snafu(display("Send error: {source}"))] + SendError { source: SendError }, + + #[snafu(display("JSON error: {source}"))] + Json { source: serde_json::Error }, + + #[snafu(display("Blocking ODBC task failed: {source}"))] + BlockingTask { source: tokio::task::JoinError }, + + #[snafu(display("Missing tracking column `{column}`"))] + MissingTrackingColumn { column: String }, + + #[snafu(display( + "Tracking column `{column}` has a value that cannot be converted to an ODBC parameter" + ))] + InvalidTrackingValue { column: String }, + + #[snafu(display("Last query result row is not an object; cannot extract tracking columns"))] + InvalidTrackingRow, + + #[snafu(display("Query result row is not an object; cannot convert to a log event"))] + InvalidResultRow, + + #[snafu(display( + "Query returned duplicate column names: {columns:?}; alias columns in the SQL statement" + ))] + DuplicateColumnNames { columns: Vec }, + + /// Pipeline send failed after the tracking checkpoint was already committed. + /// + /// The schedule must still advance `prev_params` with `next_params` so in-memory + /// tracking (no `last_run_metadata_path`) does not re-emit rows already handed off. + /// `SendError::Closed` is terminal for the schedule loop; `SendError::Timeout` remains + /// retryable on the next tick. + #[snafu(display("Send failed after tracking checkpoint was committed: {source}"))] + SendFailedAfterCheckpoint { + source: SendError, + next_params: Vec, + /// Events permanently skipped after the checkpoint was committed. + dropped_events: usize, + }, + + /// Shutdown interrupted delivery after an on-disk tracking checkpoint was committed. + #[snafu(display( + "Shutdown interrupted delivery after tracking checkpoint was committed ({dropped_events} events dropped)" + ))] + ShutdownAfterCheckpoint { dropped_events: usize }, + + #[snafu(display("ODBC source shutting down"))] + Shutdown, +} + +pub(crate) struct Context { + cfg: OdbcConfig, + env: &'static Environment, + cx: SourceContext, + log_namespace: LogNamespace, +} + +impl Context { + pub(crate) fn new( + cfg: OdbcConfig, + cx: SourceContext, + log_namespace: LogNamespace, + ) -> Result { + let env = environment().context(DbSnafu)?; + + Ok(Self { + cfg, + env, + cx, + log_namespace, + }) + } + + pub(crate) async fn run_schedule(self: Box) -> Result<(), ()> { + let shutdown = self.cx.shutdown.clone(); + + let schedule = self.cfg.schedule.clone().stream(self.cfg.schedule_timezone); + pin_mut!(schedule); + + let bytes_received = register!(BytesReceived::from(Protocol::from("odbc"))); + let events_received = register!(EventsReceived); + + #[cfg(test)] + let mut count = 0; + + let mut prev_params = self.cfg.statement_init_params.clone(); + + loop { + select! { + _ = shutdown.clone() => { + debug!(message = "Shutdown signal received. Shutting down ODBC source."); + break; + } + next = schedule.next() => { + if next.is_none() { + debug!(message = "Schedule exhausted. Shutting down ODBC source."); + break; + } + + let instant = Instant::now(); + match self + .process( + prev_params.clone(), + &bytes_received, + &events_received, + shutdown.clone(), + ) + .await + { + Ok(result) => { + // Cache the overlaid param list for runs without an on-disk + // checkpoint. When `last_run_metadata_path` exists, later ticks + // reload tracking from disk onto the config template instead. + if result.is_some() { + prev_params = result; + } + + emit!(OdbcQueryExecuted { + statement: &self.cfg.statement.clone().unwrap_or_default(), + elapsed: instant.elapsed().as_millis(), + }); + } + Err(OdbcError::Shutdown) => { + debug!( + message = + "Shutdown signal received during ODBC query. Shutting down ODBC source." + ); + break; + } + Err(OdbcError::SendFailedAfterCheckpoint { + source, + next_params, + dropped_events, + }) => { + // Checkpoint was committed before emit; advance in-memory overlay + // so the next tick does not replay rows already handed to the pipeline. + prev_params = Some(next_params); + if dropped_events > 0 { + emit!(ComponentEventsDropped:: { + count: dropped_events, + reason: "ODBC tracking checkpoint was committed before downstream delivery failed.", + }); + } + // Closed is terminal: SourceSender never reopens. Timeout stays + // retryable on the next schedule tick. + let closed = matches!(source, SendError::Closed); + emit!(OdbcFailedError { + statement: &self.cfg.statement.clone().unwrap_or_default(), + error: OdbcError::SendError { source }, + }); + if closed { + break; + } + } + Err(OdbcError::ShutdownAfterCheckpoint { dropped_events }) => { + if dropped_events > 0 { + emit!(ComponentEventsDropped:: { + count: dropped_events, + reason: "ODBC tracking checkpoint was committed before shutdown completed downstream delivery.", + }); + } + debug!( + message = + "Shutdown signal received after ODBC tracking checkpoint was committed. Shutting down ODBC source." + ); + break; + } + Err(error) => { + // Closed is terminal for the same reason as above; other errors + // (including send timeout) remain retryable. + let closed = matches!( + error, + OdbcError::SendError { + source: SendError::Closed + } + ); + emit!(OdbcFailedError { + statement: &self.cfg.statement.clone().unwrap_or_default(), + error, + }); + if closed { + break; + } + } + } + + #[cfg(test)] + { + count += 1; + if let Some(iterations) = self.cfg.iterations + && count >= iterations { + debug!(message = "No additional schedule configured. Shutting down ODBC source."); + break; + } + } + } + } + } + + Ok(()) + } + + /// Executes the scheduled ODBC query and sends the result as events in bounded batches. + /// + /// When `tracking_columns` is set, batches are buffered until the query finishes, the + /// final-row checkpoint is validated, overlaid onto `statement_init_params`, persisted, + /// and only then are events sent downstream. That preserves at-most-once tracking + /// semantics: a missing/unbindable tracking value fails the poll before any pipeline + /// emit (avoiding infinite replay), while a send failure after a successful checkpoint + /// save may skip those rows on the next run. When `last_run_metadata_path` is unset, the + /// in-memory overlay is still advanced after a post-checkpoint send failure so + /// already-sent rows are not replayed. + /// + /// Without tracking, batches are streamed to the pipeline as they arrive. + /// + /// Shutdown closes the batch channel so the blocking fetch stops on the next send, then + /// waits for the blocking task. Connect/execute still depend on `login_timeout` / + /// `statement_timeout`; with either set to `0`, that wait can block until the driver returns. + async fn process( + &self, + params: Option>, + bytes_received: &Registered, + events_received: &Registered, + mut shutdown: ShutdownSignal, + ) -> Result>, OdbcError> { + let conn_str = self.cfg.connection_string_or_file().context(IoSnafu)?; + let stmt_str = self.cfg.statement_or_file().context(IoSnafu)?; + if stmt_str.trim().is_empty() { + return Err(OdbcError::Io { + source: std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "either a non-empty `statement` or a readable `statement_filepath` must be provided", + ), + }); + } + let out = self.cx.out.clone(); + let env = self.env; + + // Prefer on-disk tracking overlays when available. Otherwise bind the in-memory + // parameter list (config template, then prior run with tracking values overlaid). + // Unreadable or corrupt metadata is treated as an error to avoid replaying old rows. + // + // When a checkpoint exists it is the tracking SSOT and is overlaid onto the config + // template (`prev_params` unused). Without a checkpoint, `current` is bound as-is. + let tz = self.cfg.odbc_default_timezone; + let template = self + .cfg + .statement_init_params + .as_deref() + .unwrap_or_default(); + let current = params.as_deref().unwrap_or(template); + let tracking_columns = self.cfg.tracking_columns.as_deref(); + let tracking_enabled = tracking_columns.is_some_and(|columns| !columns.is_empty()); + let overlay = self + .cfg + .last_run_metadata_path + .as_deref() + .map(load_tracking_map) + .transpose()? + .flatten(); + let (base, overlay) = match overlay.as_ref() { + // Checkpoint overlays keep static template values intact. + Some(overlay) => (template, Some(overlay)), + None => (current, None), + }; + let stmt_params = order_params(base, overlay, tracking_columns, tz)?; + let cfg = self.cfg.clone(); + let login_timeout = cfg.login_timeout; + let statement_timeout = cfg.statement_timeout; + let batch_size = cfg.odbc_batch_size; + let max_str_limit = (cfg.odbc_max_str_limit > 0).then_some(cfg.odbc_max_str_limit); + + let (tx, mut rx) = tokio::sync::mpsc::channel(1); + let blocking = tokio::task::spawn_blocking(move || { + let result = execute_query( + env, + &conn_str, + &stmt_str, + stmt_params, + login_timeout, + statement_timeout, + tz, + batch_size, + max_str_limit, + |batch| { + if tx.blocking_send(batch).is_err() { + return Ok(false); + } + Ok(true) + }, + ); + drop(tx); + result + }); + + // With tracking enabled, hold converted batches and the final row until the query + // completes so checkpoint validation can run before any pipeline emit. + // Received-event metrics are recorded at conversion time (before enrichment / + // buffering / send) so checkpoint, shutdown, or downstream failures cannot hide + // already-created input. + let mut pending_batches = Vec::new(); + let mut final_row = None; + let mut stream_error = None; + + loop { + let batch_rows = select! { + _ = &mut shutdown => { + return shutdown_query(rx, blocking).await; + } + batch = rx.recv() => batch, + }; + + let Some(batch_rows) = batch_rows else { + break; + }; + if batch_rows.is_empty() { + continue; + } + + // Keep the last raw row for tracking before rows are moved into events. + if tracking_enabled { + final_row = batch_rows.last().cloned(); + } + let mut events = match rows_to_events(batch_rows) { + Ok(result) => result, + Err(error) => { + stream_error = Some(error); + break; + } + }; + + if events.is_empty() { + continue; + } + + // Count/size unenriched events immediately after creation, matching + // ComponentEventsReceived (docs/specs/component.md). + let event_count = events.len(); + let byte_size = events.estimated_json_encoded_size_of(); + record_received(bytes_received, events_received, event_count, byte_size); + + self.enrich_events(&mut events); + + if tracking_enabled { + pending_batches.push(events); + continue; + } + + match send_enriched_batch(&out, events, &mut shutdown).await { + Ok(()) => {} + Err(BatchSendError::Shutdown { .. }) => { + // Remaining chunks are re-read after restart; in-flight is via UnsentEventCount. + return shutdown_query(rx, blocking).await; + } + Err(BatchSendError::Send { + source, + dropped_events, + }) => { + // Closed: emit chunks never handed to SourceSender (in-flight via StreamClosedError). + // Timeout is retryable — do not discard. + if dropped_events > 0 && matches!(source, SendError::Closed) { + emit!(ComponentEventsDropped:: { + count: dropped_events, + reason: "Source stream closed before remaining ODBC batch chunks were sent.", + }); + } + stream_error = Some(OdbcError::SendError { source }); + break; + } + } + } + + drop(rx); + let join_result = blocking.await.context(BlockingTaskSnafu); + + if let Some(error) = stream_error { + // Prefer the stream conversion/send error over a later join outcome. + drop(join_result); + return Err(error); + } + + match join_result { + Ok(Ok(())) => {} + Ok(Err(error)) | Err(error) => { + return Err(error); + } + } + + // Tracking path: validate + overlay + persist the final-row checkpoint before any + // pipeline emit so a missing/null tracking value cannot replay forever, and an + // overlay failure cannot leave a persisted checkpoint that would skip unsent rows. + let next_params = match (final_row, cfg.tracking_columns.as_ref()) { + (Some(last), Some(tracking_columns)) => Some(prepare_tracking_checkpoint( + cfg.last_run_metadata_path.as_deref(), + last, + template, + tracking_columns, + tz, + )?), + _ => None, + }; + + let mut pending_batches = pending_batches.into_iter(); + while let Some(events) = pending_batches.next() { + match send_enriched_batch(&out, events, &mut shutdown).await { + Ok(()) => {} + Err(BatchSendError::Shutdown { dropped_events }) => { + let dropped_events = dropped_events + + pending_batches + .as_slice() + .iter() + .map(Vec::len) + .sum::(); + + // An on-disk checkpoint prevents these rows from being replayed after + // restart, so they must be accounted for as dropped. Without one, a + // restart re-reads the rows and this shutdown does not lose them. + return if cfg.last_run_metadata_path.is_some() { + Err(OdbcError::ShutdownAfterCheckpoint { dropped_events }) + } else { + Err(OdbcError::Shutdown) + }; + } + Err(BatchSendError::Send { + source, + dropped_events, + }) => { + // Checkpoint is already committed. Advance in-memory tracking on send + // failure so the next tick cannot replay rows already emitted. + return match next_params { + Some(next_params) => Err(OdbcError::SendFailedAfterCheckpoint { + source, + next_params, + dropped_events: dropped_events + + pending_batches + .as_slice() + .iter() + .map(Vec::len) + .sum::(), + }), + None => Err(OdbcError::SendError { source }), + }; + } + } + } + + Ok(next_params) + } + + fn enrich_events(&self, events: &mut [Event]) { + let now = Utc::now(); + + for event in events { + let Event::Log(log) = event else { + continue; + }; + + self.log_namespace + .insert_standard_vector_source_metadata(log, OdbcConfig::NAME, now); + } + } +} + +/// Closes the batch channel, waits for the blocking ODBC task, then returns +/// `OdbcError::Shutdown`. Received metrics for converted batches were already emitted +/// at conversion time. +async fn shutdown_query( + rx: tokio::sync::mpsc::Receiver, + blocking: tokio::task::JoinHandle>, +) -> Result>, OdbcError> { + drop(rx); + let join_result = blocking.await.context(BlockingTaskSnafu); + + // Join errors are fatal. The query outcome is ignored on shutdown because the poll is + // ending; converted rows were already counted at creation time. + match join_result { + Ok(_) => Err(OdbcError::Shutdown), + Err(error) => Err(error), + } +} + +/// Sends an enriched batch to the pipeline in source-sender-sized chunks, racing against +/// shutdown between chunks so a large batch can stop promptly. +/// +/// On `SendError::Closed`, emits `StreamClosedError` for the failed chunk so +/// `ComponentEventsDropped` is recorded (SourceSender discards its unsent count in that +/// case and expects the callee to emit). A timeout is reported by SourceSender as timed out, but +/// must also be reported as dropped when a tracking checkpoint makes retry impossible. Therefore, +/// the returned count excludes a closed failed chunk but includes a timed-out failed chunk. +/// +/// On shutdown while `send_batch` is in flight, cancelling that future already makes +/// `UnsentEventCount` emit `ComponentEventsDropped` for the current chunk, so the returned +/// shutdown count excludes that chunk and only covers events not yet handed to SourceSender. +enum BatchSendError { + Shutdown { + dropped_events: usize, + }, + Send { + source: SendError, + dropped_events: usize, + }, +} + +async fn send_enriched_batch( + out: &crate::SourceSender, + events: Vec, + shutdown: &mut ShutdownSignal, +) -> Result<(), BatchSendError> { + let mut events = events.into_iter(); + let mut unsent_events = events.len(); + loop { + let events: Vec<_> = events.by_ref().take(chunk_size_events()).collect(); + if events.is_empty() { + break; + } + let count = events.len(); + let mut out = out.clone(); + let send_result = select! { + _ = &mut *shutdown => { + // SourceSender already emits ComponentEventsDropped for this chunk via + // UnsentEventCount::drop when the cancelled send_batch future is dropped. + return Err(BatchSendError::Shutdown { + dropped_events: unsent_events - count, + }); + } + send_result = out.send_batch(events) => send_result, + }; + match send_result { + Ok(()) => unsent_events -= count, + Err(SendError::Closed) => { + emit!(StreamClosedError { count }); + return Err(BatchSendError::Send { + source: SendError::Closed, + dropped_events: unsent_events - count, + }); + } + Err(SendError::Timeout) => { + return Err(BatchSendError::Send { + source: SendError::Timeout, + dropped_events: unsent_events, + }); + } + } + } + Ok(()) +} + +/// Records `BytesReceived` / `EventsReceived` for a converted batch before enrichment. +fn record_received( + bytes_received: &Registered, + events_received: &Registered, + event_count: usize, + byte_size: JsonSize, +) { + if event_count == 0 { + return; + } + bytes_received.emit(ByteSize(byte_size.get())); + events_received.emit(CountByteSize(event_count, byte_size)); +} + +/// Converts ODBC result rows into log events without a JSON round-trip so typed +/// values such as timestamps and integers are preserved for downstream transforms. +fn rows_to_events(rows: Rows) -> Result, OdbcError> { + let mut events = Vec::with_capacity(rows.len()); + + for row in rows { + let Value::Object(obj) = row else { + return Err(OdbcError::InvalidResultRow); + }; + + events.push(LogEvent::from(obj).into()); + } + + Ok(events) +} + +/// Extracts declared tracking columns from the final result row as SQL bind text. +/// +/// Checkpoint values are stored as the exact SQL parameter text used for ODBC binding +/// so JSON roundtrips do not lose timestamp timezone formatting. +fn extract_tracking( + obj: Value, + tracking_columns: &[String], + tz: Tz, +) -> Result { + let Value::Object(obj) = obj else { + return Err(OdbcError::InvalidTrackingRow); + }; + + let mut save_obj = ObjectMap::new(); + for column in tracking_columns { + let (_, param) = resolve_tracking_column_parameter(&obj, column.as_str(), tz)?; + save_obj.insert( + KeyString::from(column.as_str()), + Value::Bytes(Bytes::from(param)), + ); + } + Ok(save_obj) +} + +/// Validates the final-row checkpoint, builds the next in-memory parameter list, then +/// persists tracking state. Persistence runs only after overlay succeeds so a bind-list +/// failure cannot advance an on-disk checkpoint that would skip unsent rows. +fn prepare_tracking_checkpoint( + path: Option<&str>, + last_row: Value, + template: &[OdbcStatementParam], + tracking_columns: &[String], + tz: Tz, +) -> Result, OdbcError> { + let tracking = extract_tracking(last_row, tracking_columns, tz)?; + let next_params = overlay_params(template, &tracking, tracking_columns, tz)?; + if let Some(path) = path { + save_params(path, &tracking)?; + } + Ok(next_params) +} + +/// Returns an error when the query result contains duplicate column labels. +fn ensure_unique_column_names(names: &[String]) -> Result<(), OdbcError> { + let mut seen = HashSet::with_capacity(names.len()); + let mut duplicates = Vec::new(); + + for name in names { + if !seen.insert(name.as_str()) { + duplicates.push(name.clone()); + } + } + + if duplicates.is_empty() { + Ok(()) + } else { + duplicates.sort_unstable(); + duplicates.dedup(); + Err(OdbcError::DuplicateColumnNames { + columns: duplicates, + }) + } +} + +/// Returns true for ODBC binary column types that must be fetched with a binary buffer. +const fn is_binary_data_type(data_type: &DataType) -> bool { + matches!( + data_type, + DataType::Varbinary { .. } | DataType::Binary { .. } | DataType::LongVarbinary { .. } + ) +} + +/// Caps a driver-reported cell size the same way `TextRowSet::for_cursor` does. +/// +/// When `max_str_limit` is set, missing reports fall back to that upper bound. +/// When unset, a missing report cannot allocate a buffer. +fn capped_buffer_length( + reported: Option, + max_str_limit: Option, + buffer_index: u16, + batch_size: usize, +) -> Result { + match max_str_limit { + Some(upper_bound) => Ok(reported.unwrap_or(upper_bound).min(upper_bound)), + None => reported.ok_or(OdbcError::Db { + source: odbc_api::Error::TooLargeColumnBufferSize { + buffer_index, + num_elements: batch_size, + element_size: usize::MAX, + }, + }), + } +} + +/// Chooses a fetch buffer for one column. +/// +/// Binary SQL types use `BufferDesc::Binary` with the octet length (not the hex +/// display size). All other types stay text so existing timestamp/decimal/tracking +/// text round-trips are unchanged. +/// +/// `reported_fallback` is used when the SQL type does not carry a length: binary +/// columns fall back to `col_octet_length`, text columns to `col_display_size`. +fn buffer_desc_for_data_type( + data_type: &DataType, + reported_fallback: Option, + max_str_limit: Option, + buffer_index: u16, + batch_size: usize, +) -> Result { + if is_binary_data_type(data_type) { + let reported = match data_type { + DataType::Varbinary { length } + | DataType::Binary { length } + | DataType::LongVarbinary { length } => { + length.map(NonZeroUsize::get).or(reported_fallback) + } + _ => reported_fallback, + }; + let length = capped_buffer_length(reported, max_str_limit, buffer_index, batch_size)?; + Ok(BufferDesc::Binary { length }) + } else { + // Match `TextRowSet::for_cursor` / `utf8_display_sizes`: prefer UTF-8 length + // from the SQL type, otherwise use the driver-reported fallback. + let reported = data_type + .utf8_len() + .map(NonZeroUsize::get) + .or(reported_fallback); + let max_str_len = capped_buffer_length(reported, max_str_limit, buffer_index, batch_size)?; + Ok(BufferDesc::Text { max_str_len }) + } +} + +/// Builds per-column fetch buffers for a result set cursor. +fn buffer_descs_for_columns( + cursor: &mut impl ResultSetMetadata, + columns: &[Column], + max_str_limit: Option, + batch_size: usize, +) -> Result, OdbcError> { + columns + .iter() + .enumerate() + .map(|(index, column)| { + let col_index = (index + 1) as u16; + let buffer_index = index as u16; + + let reported_fallback = if is_binary_data_type(&column.column_type) { + match &column.column_type { + DataType::Varbinary { length: None } + | DataType::Binary { length: None } + | DataType::LongVarbinary { length: None } => cursor + .col_octet_length(col_index) + .context(DbSnafu)? + .map(NonZeroUsize::get), + _ => None, + } + } else if column.column_type.utf8_len().is_none() { + cursor + .col_display_size(col_index) + .context(DbSnafu)? + .map(NonZeroUsize::get) + } else { + None + }; + + buffer_desc_for_data_type( + &column.column_type, + reported_fallback, + max_str_limit, + buffer_index, + batch_size, + ) + }) + .collect() +} + +/// Returns whether a column needs the row-by-row `SQLGetData` path when fetch buffers are capped. +/// +/// A row-set buffer cannot be enlarged after a forward-only cursor has fetched a truncated row. +/// Variable-width values therefore use `CursorRow::{get_text,get_binary}`, which grow their +/// destination buffer until the complete value has been read. +fn requires_streaming_fetch(data_type: &DataType, max_str_limit: usize) -> bool { + match data_type { + DataType::Char { .. } + | DataType::WChar { .. } + | DataType::Varchar { .. } + | DataType::WVarchar { .. } => data_type + .utf8_len() + .map(NonZeroUsize::get) + .is_none_or(|length| length > max_str_limit), + DataType::Varbinary { length } | DataType::Binary { length } => length + .map(NonZeroUsize::get) + .is_none_or(|length| length > max_str_limit), + // Long data types may have an imprecise driver-reported size, so never bind them to a + // capped row-set buffer. + DataType::LongVarchar { .. } + | DataType::WLongVarchar { .. } + | DataType::LongVarbinary { .. } + | DataType::Unknown + | DataType::Other { .. } => true, + _ => false, + } +} + +/// Reads one cell through `SQLGetData`, growing the vector until the full cell is available. +fn read_streamed_cell( + row: &mut CursorRow<'_>, + column_index: u16, + data_type: &DataType, + initial_capacity: usize, +) -> Result>, OdbcError> { + let mut value = Vec::with_capacity(initial_capacity); + let is_not_null = if is_binary_data_type(data_type) { + row.get_binary(column_index, &mut value).context(DbSnafu)? + } else { + row.get_text(column_index, &mut value).context(DbSnafu)? + }; + Ok(is_not_null.then_some(value)) +} + +/// Executes a result set one row at a time when any bound row-set buffer could truncate a cell. +fn execute_streaming_query( + mut cursor: impl Cursor, + columns: &Columns, + tz: Tz, + batch_size: usize, + initial_cell_capacity: usize, + mut on_batch: F, +) -> Result<(), OdbcError> +where + F: FnMut(Rows) -> Result, +{ + let mut batch_rows = Rows::with_capacity(batch_size); + + while let Some(mut row) = cursor.next_row().context(DbSnafu)? { + let mut cols = ObjectMap::new(); + for (index, column) in columns.iter().enumerate() { + let value = read_streamed_cell( + &mut row, + (index + 1) as u16, + &column.column_type, + initial_cell_capacity, + )?; + cols.insert( + KeyString::from(column.column_name.as_str()), + map_value(&column.column_type, value.as_deref(), tz), + ); + } + batch_rows.push(Value::Object(cols)); + + if batch_rows.len() == batch_size { + if !on_batch(mem::take(&mut batch_rows))? { + return Ok(()); + } + batch_rows.reserve(batch_size); + } + } + + if !batch_rows.is_empty() { + on_batch(batch_rows)?; + } + Ok(()) +} + +/// Reads one cell from a columnar batch as optional bytes. +/// +/// Only text and binary column buffers are allocated by [`buffer_descs_for_columns`]. +fn cell_bytes<'a>(column: AnySlice<'a>, row_index: usize) -> Option<&'a [u8]> { + if let Some(view) = column.as_bin_view() { + view.get(row_index) + } else if let Some(view) = column.as_text_view() { + view.get(row_index) + } else { + unreachable!("ODBC fetch buffers are only text or binary") + } +} + +/// Executes an ODBC SQL query with optional parameters and invokes `on_batch` for each +/// fetched batch instead of accumulating the full result set in memory. +/// +/// The callback returns `Ok(true)` to continue fetching or `Ok(false)` to stop early. +#[allow(clippy::too_many_arguments)] +pub(crate) fn execute_query( + env: &Environment, + conn_str: &str, + stmt_str: &str, + stmt_params: Vec, + login_timeout: Duration, + statement_timeout: Duration, + tz: Tz, + batch_size: usize, + max_str_limit: Option, + mut on_batch: F, +) -> Result<(), OdbcError> +where + F: FnMut(Rows) -> Result, +{ + let conn_options = ConnectionOptions { + login_timeout_sec: Some(login_timeout.as_secs() as u32), + packet_size: None, + }; + let conn = env + .connect_with_connection_string(conn_str, conn_options) + .context(DbSnafu)?; + let mut statement = conn.preallocate().context(DbSnafu)?; + statement + .set_query_timeout_sec(statement_timeout.as_secs() as usize) + .context(DbSnafu)?; + + let result = if stmt_params.is_empty() { + statement.execute(stmt_str, ()) + } else { + statement.execute(stmt_str, &stmt_params[..]) + } + .context(DbSnafu)?; + + let Some(mut cursor) = result else { + return Ok(()); + }; + + let names = cursor + .column_names() + .context(DbSnafu)? + .collect::, _>>() + .context(DbSnafu)?; + + ensure_unique_column_names(&names)?; + + let types = (1..=names.len()) + .map(|col_index| cursor.col_data_type(col_index as u16).context(DbSnafu)) + .collect::, _>>()?; + let columns = names + .into_iter() + .zip(types) + .map(|(column_name, column_type)| Column { + column_name, + column_type, + }) + .collect::(); + + // A capped row-set buffer is safe only if no result column can outgrow it. For + // variable-width/unknown columns, read cells through SQLGetData instead of accepting a + // truncation error after the forward-only cursor has already advanced. + if let Some(limit) = max_str_limit + && columns + .iter() + .any(|column| requires_streaming_fetch(&column.column_type, limit)) + { + return execute_streaming_query(cursor, &columns, tz, batch_size, limit.min(256), on_batch); + } + + let descs = buffer_descs_for_columns(&mut cursor, &columns, max_str_limit, batch_size)?; + let buffer = ColumnarAnyBuffer::try_from_descs(batch_size, descs).context(DbSnafu)?; + let mut row_set_cursor = cursor.bind_buffer(buffer).context(DbSnafu)?; + let mut batch_rows = Rows::with_capacity(batch_size); + + while let Some(batch) = row_set_cursor + .fetch_with_truncation_check(true) + .context(DbSnafu)? + { + let num_rows = batch.num_rows(); + + for row_index in 0..num_rows { + let mut cols = ObjectMap::new(); + + for (index, column) in columns.iter().enumerate() { + let data_name = &column.column_name; + let data_type = &column.column_type; + let data_value = cell_bytes(batch.column(index), row_index); + let key = KeyString::from(data_name.as_str()); + let value = map_value(data_type, data_value, tz); + cols.insert(key, value); + } + + batch_rows.push(Value::Object(cols)); + } + + if !batch_rows.is_empty() { + if !on_batch(mem::take(&mut batch_rows))? { + break; + } + batch_rows.reserve(batch_size); + } + } + + Ok(()) +} + +/// Loads tracked column overlays from disk. +/// +/// Returns `Ok(None)` only when the metadata file does not exist. +fn load_tracking_map(path: &str) -> Result, OdbcError> { + let file = match File::open(path) { + Ok(file) => file, + Err(source) if source.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(source) => return Err(OdbcError::Io { source }), + }; + let reader = BufReader::new(file); + let map: ObjectMap = serde_json::from_reader(reader).context(JsonSnafu)?; + Ok(Some(map)) +} + +/// Resolves a single statement parameter value. +/// +/// Tracking columns use `overlay` when present; otherwise the configured value is kept. +fn resolve_param_value( + param: &OdbcStatementParam, + overlay: Option<&ObjectMap>, + tracking: &HashSet<&str>, + tz: Tz, +) -> Result { + match (tracking.contains(param.name.as_str()), overlay) { + (true, Some(overlay)) => { + Ok(resolve_tracking_column_parameter(overlay, param.name.as_str(), tz)?.1) + } + _ => Ok(param.value.clone()), + } +} + +/// Builds ODBC bind parameters from the ordered `statement_init_params` list. +/// +/// Array order is the bind order. When `overlay` is set, tracking-column values are +/// taken from it; non-tracking values always keep the template entry. +fn order_params( + params: &[OdbcStatementParam], + overlay: Option<&ObjectMap>, + tracking_columns: Option<&[String]>, + tz: Tz, +) -> Result, OdbcError> { + let tracking: HashSet<&str> = tracking_columns + .unwrap_or_default() + .iter() + .map(String::as_str) + .collect(); + + params + .iter() + .map(|param| { + resolve_param_value(param, overlay, &tracking, tz).map(|value| value.into_parameter()) + }) + .collect() +} + +/// Returns `params` with tracking-column values overlaid from `overlay`. +/// +/// Non-tracking entries and overall array order are preserved. +fn overlay_params( + params: &[OdbcStatementParam], + overlay: &ObjectMap, + tracking_columns: &[String], + tz: Tz, +) -> Result, OdbcError> { + let tracking: HashSet<&str> = tracking_columns.iter().map(String::as_str).collect(); + + params + .iter() + .map(|param| { + Ok(OdbcStatementParam { + name: param.name.clone(), + value: resolve_param_value(param, Some(overlay), &tracking, tz)?, + }) + }) + .collect() +} + +/// Creates parent directories for the metadata path when needed. +pub(crate) fn prepare_metadata_path(path: &str) -> Result<(), OdbcError> { + if path.is_empty() { + return Err(OdbcError::Io { + source: std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "`last_run_metadata_path` must not be empty", + ), + }); + } + + let path = Path::new(path); + let parent = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")); + + fs::create_dir_all(parent).context(IoSnafu) +} + +/// Returns a sibling temp path that is always distinct from `path`. +/// +/// Appending `.tmp` to the full filename avoids `Path::with_extension("tmp")` +/// returning the destination path when it already ends in `.tmp`. +fn checkpoint_temp_path(path: &Path) -> PathBuf { + let file_name = path + .file_name() + .map(|name| format!("{}.tmp", name.to_string_lossy())) + .unwrap_or_else(|| "checkpoint.tmp".into()); + path.with_file_name(file_name) +} + +/// Replaces `path` with the contents of `tmp_path` atomically. +/// +/// On Windows, `rename` cannot replace an existing destination file, so use +/// `ReplaceFileW` and fall back to `rename` when the destination does not exist yet. +fn replace_checkpoint_file(tmp_path: &Path, path: &Path) -> Result<(), OdbcError> { + #[cfg(windows)] + { + use windows::Win32::Storage::FileSystem::ReplaceFileW; + use windows::core::HSTRING; + + let dst = HSTRING::from(path.to_string_lossy().as_ref()); + let src = HSTRING::from(tmp_path.to_string_lossy().as_ref()); + let replaced = unsafe { + ReplaceFileW( + &dst, + &src, + None, + windows::Win32::Storage::FileSystem::REPLACE_FILE_FLAGS(0), + None, + None, + ) + }; + if replaced.is_ok() { + return Ok(()); + } + // Destination may not exist yet — fall back to rename. + } + + fs::rename(tmp_path, path).context(IoSnafu) +} + +/// Serializes and persists the latest tracked values for reuse as SQL parameters. +/// +/// Writes to a sibling `.tmp` file and atomically replaces the checkpoint on success. +fn save_params(path: &str, obj: &ObjectMap) -> Result<(), OdbcError> { + prepare_metadata_path(path)?; + let path = Path::new(path); + let tmp_path = checkpoint_temp_path(path); + let json = serde_json::to_string(obj).context(JsonSnafu)?; + + { + let mut file = fs::File::create(&tmp_path).context(IoSnafu)?; + file.write_all(json.as_bytes()).context(IoSnafu)?; + file.sync_all().context(IoSnafu)?; + } + + replace_checkpoint_file(&tmp_path, path) +} + +/// Localizes a naive datetime with `tz`, using `.latest()` for DST ambiguity and +/// preserving `fallback_text` when the local time does not exist. +fn naive_local_to_timestamp_value(ndt: NaiveDateTime, tz: Tz, fallback_text: &str) -> Value { + if let Some(dt) = ndt.and_local_timezone(tz).latest() { + Value::Timestamp(dt.with_timezone(&Utc)) + } else { + Value::Bytes(Bytes::copy_from_slice(fallback_text.as_bytes())) + } +} + +/// Returns true for SQL-style timestamps with a space date/time separator and offset suffix, +/// such as `YYYY-MM-DD HH:MM:SS+00` or `YYYY-MM-DD HH:MM:SS-05:00`. Returns false for +/// RFC3339 forms that use a `T` separator or a `Z` suffix; those are detected with +/// `DateTime::parse_from_rfc3339` instead. +fn sql_timestamp_text_has_offset(text: &str) -> bool { + if text.len() <= 10 { + return false; + } + + // SQL form uses a space between date and time; RFC3339 uses `T`. + if text.as_bytes().get(10) != Some(&b' ') { + return false; + } + + let Some(tail) = text.get(10..) else { + return false; + }; + let Some(sign_idx) = tail.rfind(['+', '-']) else { + return false; + }; + + let Some(offset) = text.get(10 + sign_idx..) else { + return false; + }; + offset.len() >= 2 + && offset + .get(1..) + .is_some_and(|rest| rest.chars().all(|c| c.is_ascii_digit() || c == ':')) +} + +/// Returns true when timestamp text carries an explicit zone that must be preserved +/// for tracking parameter round-trips. +/// +/// Covers SQL-style offsets (`YYYY-MM-DD HH:MM:SS+02:00`) and any valid RFC3339 value +/// (which always includes an offset or `Z`). Converting those to `Value::Timestamp` +/// would drop the original offset and later rebind a naive local datetime in +/// `odbc_default_timezone`. +fn timestamp_text_has_preserved_offset(text: &str) -> bool { + sql_timestamp_text_has_offset(text) || DateTime::parse_from_rfc3339(text).is_ok() +} + +/// Maps ODBC timestamp bytes to a Vector value. +/// +/// Offset-bearing SQL and RFC3339 forms are preserved as bytes so tracking parameters +/// round-trip the exact ODBC text. Naive timestamps are parsed to `Value::Timestamp` +/// using `tz`. +fn map_timestamp_value(value: &[u8], tz: Tz) -> Value { + let Ok(text) = std::str::from_utf8(value) else { + return Value::Bytes(Bytes::copy_from_slice(value)); + }; + + if timestamp_text_has_preserved_offset(text) { + return Value::Bytes(Bytes::copy_from_slice(value)); + } + + TIMESTAMP_FORMATS + .iter() + .find_map(|fmt| NaiveDateTime::parse_from_str(text, fmt).ok()) + .map(|ndt| naive_local_to_timestamp_value(ndt, tz, text)) + .unwrap_or_else(|| Value::Bytes(Bytes::copy_from_slice(value))) +} + +/// Converts ODBC data types to Vector values. +/// +/// # Arguments +/// * `data_type`: The ODBC data type. +/// * `value`: The ODBC value to convert. Binary columns are raw bytes from a binary +/// buffer; character and other text-fetched columns are driver text bytes. +/// * `tz`: The timezone to use for date/time conversions. +/// +/// # Returns +/// A `Value` compatible with Vector events. +fn map_value(data_type: &DataType, value: Option<&[u8]>, tz: Tz) -> Value { + match data_type { + // Character / unknown text-fetched columns. + DataType::Unknown + | DataType::Char { .. } + | DataType::WChar { .. } + | DataType::Varchar { .. } + | DataType::WVarchar { .. } + | DataType::LongVarchar { .. } + | DataType::WLongVarchar { .. } + | DataType::Other { .. } => { + let Some(value) = value else { + return Value::Null; + }; + + Value::Bytes(Bytes::copy_from_slice(value)) + } + + // Binary columns are fetched with `BufferDesc::Binary` so these bytes are the + // original octet sequence, not ODBC's hex text conversion of binary values. + DataType::Varbinary { .. } | DataType::Binary { .. } | DataType::LongVarbinary { .. } => { + let Some(value) = value else { + return Value::Null; + }; + + Value::Bytes(Bytes::copy_from_slice(value)) + } + + // Convert to integer. + DataType::TinyInt | DataType::SmallInt | DataType::BigInt | DataType::Integer => { + let Some(value) = value else { + return Value::Null; + }; + + // Preserve unrepresentable integers as bytes so tracking metadata is not lost. + match std::str::from_utf8(value).map(|s| s.parse::()) { + Ok(Ok(i)) => Value::Integer(i), + _ => Value::Bytes(Bytes::copy_from_slice(value)), + } + } + + // Convert to float. + DataType::Float { .. } | DataType::Real | DataType::Double => { + let Some(value) = value else { + return Value::Null; + }; + + // Preserve unrepresentable floats (for example NaN) as bytes so tracking metadata is not lost. + // Downstream consumers may see `Value::Bytes` instead of `Value::Float` for NaN and other + // values that `NotNan` cannot represent. + match std::str::from_utf8(value).map(NotNan::from_str) { + Ok(Ok(f)) => Value::Float(f), + _ => Value::Bytes(Bytes::copy_from_slice(value)), + } + } + + // Preserve exact decimal values from the database. + DataType::Decimal { .. } | DataType::Numeric { .. } => { + let Some(value) = value else { + return Value::Null; + }; + + Value::Bytes(Bytes::copy_from_slice(value)) + } + + // Convert to timestamp. + DataType::Timestamp { .. } => { + let Some(value) = value else { + return Value::Null; + }; + + map_timestamp_value(value, tz) + } + + // Preserve the original time text so tracking parameters bind as `HH:MM:SS` + // instead of a full timestamp such as `1970-01-01 15:30:00`. + // MariaDB TIME can represent durations outside a clock-of-day range (for example + // `25:00:00`), so keep the ODBC text even when chrono cannot parse it. + DataType::Time { .. } => { + let Some(value) = value else { + return Value::Null; + }; + + Value::Bytes(Bytes::copy_from_slice(value)) + } + + // Preserve the original date text so tracking parameters bind as `YYYY-MM-DD` + // instead of a full timestamp such as `2025-10-04 00:00:00`. + // MariaDB/MySQL zero dates such as `0000-00-00` are not chrono-compatible but + // remain valid for SQL comparison and tracking parameter binding. + DataType::Date => { + let Some(value) = value else { + return Value::Null; + }; + + Value::Bytes(Bytes::copy_from_slice(value)) + } + + // Convert to boolean. + // Some ODBC drivers return a non-NULL BIT with an empty buffer; treat that as null + // instead of panicking on an empty slice. + DataType::Bit => { + let Some(value) = value else { + return Value::Null; + }; + + match value.first().copied() { + Some(b) => Value::Boolean(b == 1 || b == b'1'), + None => Value::Null, + } + } + } +} + +/// Formats a UTC timestamp as a naive local datetime string for ODBC parameter binding. +fn format_timestamp_for_sql_parameter(timestamp: DateTime, tz: Tz) -> String { + let local = timestamp.with_timezone(&tz); + if local.nanosecond() != 0 { + local.format("%Y-%m-%d %H:%M:%S%.f").to_string() + } else { + local.format("%Y-%m-%d %H:%M:%S").to_string() + } +} + +/// Validates that `map` contains every declared tracking column with a value that can +/// be converted to an ODBC parameter. +pub(crate) fn validate_tracking_state( + map: &ObjectMap, + tracking_columns: &[String], + tz: Tz, +) -> Result<(), String> { + for column in tracking_columns { + resolve_tracking_column_parameter(map, column.as_str(), tz) + .map_err(|error| error.to_string())?; + } + + Ok(()) +} + +/// Resolves a single tracking column to its source value and the text used for ODBC +/// parameter binding. +fn resolve_tracking_column_parameter( + map: &ObjectMap, + column: &str, + tz: Tz, +) -> Result<(Value, String), OdbcError> { + let value = map + .get(column) + .ok_or_else(|| OdbcError::MissingTrackingColumn { + column: column.to_owned(), + })?; + let param = + value_to_sql_parameter(value, tz).ok_or_else(|| OdbcError::InvalidTrackingValue { + column: column.to_owned(), + })?; + Ok((value.clone(), param)) +} + +/// Converts a scalar VRL value to raw text for ODBC parameter binding. +/// +/// Unlike `Value::to_string()`, this does not use VRL literal syntax (e.g. quoted +/// strings or `t'…'` timestamps). +/// +/// Only `Value::Timestamp` is reformatted in `tz` as a naive local datetime. +/// Byte/string values are preserved as-is so VARCHAR tracking columns that +/// happen to look like RFC3339 are not coerced into timestamp predicates. +/// Non-UTF-8 bytes (for example raw `VARBINARY` payloads) cannot be bound as +/// text parameters and return `None`. +fn value_to_sql_parameter(value: &Value, tz: Tz) -> Option { + match value { + Value::Integer(i) => Some(i.to_string()), + Value::Float(f) => Some(f.to_string()), + Value::Boolean(b) => Some(boolean_to_sql_parameter(*b)), + Value::Bytes(b) => std::str::from_utf8(b).ok().map(str::to_owned), + Value::Timestamp(t) => Some(format_timestamp_for_sql_parameter(*t, tz)), + Value::Null => None, + other => serde_json::to_value(other).ok().and_then(|v| match v { + serde_json::Value::String(s) => Some(s), + serde_json::Value::Number(n) => Some(n.to_string()), + serde_json::Value::Bool(b) => Some(boolean_to_sql_parameter(b)), + _ => None, + }), + } +} + +/// Formats a boolean as `1`/`0` for ODBC parameter binding. +/// +/// Numeric/bit columns (for example MariaDB `BIT`) coerce string parameters to +/// numbers; `"true"` and `"false"` both become `0`, so use bit literals instead. +fn boolean_to_sql_parameter(value: bool) -> String { + if value { "1" } else { "0" }.to_owned() +} + +#[cfg(test)] +mod tests { + use super::*; + use bytes::Bytes; + use chrono::TimeZone; + use vrl::event_path; + + #[test] + fn rows_to_events_preserves_typed_values() { + let timestamp = chrono::Utc + .with_ymd_and_hms(2025, 10, 4, 12, 34, 56) + .unwrap(); + let mut row = ObjectMap::new(); + row.insert(KeyString::from("id"), Value::Integer(1)); + row.insert(KeyString::from("active"), Value::Boolean(true)); + row.insert(KeyString::from("datetime_col"), Value::Timestamp(timestamp)); + row.insert( + KeyString::from("date_col"), + Value::Bytes(Bytes::from_static(b"2025-10-04")), + ); + + let rows = vec![Value::Object(row)]; + let events = rows_to_events(rows).expect("events"); + + assert_eq!(events.len(), 1); + + let Event::Log(log) = &events[0] else { + panic!("expected log event"); + }; + + assert_eq!(log.get(event_path!("id")).unwrap(), &Value::Integer(1)); + assert_eq!( + log.get(event_path!("active")).unwrap(), + &Value::Boolean(true) + ); + assert_eq!( + log.get(event_path!("datetime_col")).unwrap(), + &Value::Timestamp(timestamp) + ); + assert_eq!( + log.get(event_path!("date_col")).unwrap(), + &Value::Bytes(Bytes::from_static(b"2025-10-04")) + ); + } + + #[test] + fn rows_to_events_errors_on_non_object_row() { + let rows = vec![Value::Integer(1)]; + let error = rows_to_events(rows).expect_err("expected error"); + assert!(matches!(error, OdbcError::InvalidResultRow)); + } + + #[tokio::test] + async fn send_enriched_batch_excludes_in_flight_chunk_from_shutdown_drops() { + let (mut out, _recv) = crate::SourceSender::new_test_sender_with_options(1, None); + out.send_batch(vec![Event::Log(LogEvent::from("already buffered"))]) + .await + .expect("first batch should fill the output buffer"); + + let (trigger_shutdown, mut shutdown, _) = ShutdownSignal::new_wired(); + drop(trigger_shutdown); + + // Both events fit in one SourceSender chunk. Cancelling the blocked send_batch + // already drops that in-flight chunk via UnsentEventCount, so the shutdown error + // must report 0 additional drops. + let result = send_enriched_batch( + &out, + vec![ + Event::Log(LogEvent::from("unsent one")), + Event::Log(LogEvent::from("unsent two")), + ], + &mut shutdown, + ) + .await; + + assert!(matches!( + result, + Err(BatchSendError::Shutdown { dropped_events: 0 }) + )); + } + + #[test] + fn map_value_bit_binary_and_text() { + assert_eq!( + map_value(&odbc_api::DataType::Bit, Some(&[1]), chrono_tz::UTC), + Value::Boolean(true) + ); + assert_eq!( + map_value(&odbc_api::DataType::Bit, Some(&[0]), chrono_tz::UTC), + Value::Boolean(false) + ); + assert_eq!( + map_value(&odbc_api::DataType::Bit, Some(b"1"), chrono_tz::UTC), + Value::Boolean(true) + ); + assert_eq!( + map_value(&odbc_api::DataType::Bit, Some(b"0"), chrono_tz::UTC), + Value::Boolean(false) + ); + } + + #[test] + fn map_value_bit_empty_buffer_maps_to_null() { + assert_eq!( + map_value(&odbc_api::DataType::Bit, Some(&[]), chrono_tz::UTC), + Value::Null + ); + } + + #[test] + fn map_value_integer_in_range() { + let value = map_value( + &odbc_api::DataType::BigInt, + Some(b"9223372036854775807"), + chrono_tz::UTC, + ); + assert_eq!(value, Value::Integer(9223372036854775807)); + } + + #[test] + fn map_value_integer_out_of_range_preserved_as_bytes() { + let raw = b"18446744073709551615"; + let value = map_value(&odbc_api::DataType::BigInt, Some(raw), chrono_tz::UTC); + assert_eq!(value, Value::Bytes(Bytes::from_static(raw))); + assert_eq!( + value_to_sql_parameter(&value, chrono_tz::UTC), + Some("18446744073709551615".to_owned()) + ); + } + + #[test] + fn map_value_binary_preserves_raw_bytes_not_hex_text() { + // ODBC text conversion would turn 0x00FF into ASCII "00FF"; binary fetch must keep + // the original octets so event payloads and any future binary binding stay correct. + let raw = &[0x00, 0xFF, 0x10]; + for data_type in [ + DataType::Varbinary { + length: NonZeroUsize::new(3), + }, + DataType::Binary { + length: NonZeroUsize::new(3), + }, + DataType::LongVarbinary { + length: NonZeroUsize::new(3), + }, + ] { + let value = map_value(&data_type, Some(raw), chrono_tz::UTC); + assert_eq!(value, Value::Bytes(Bytes::copy_from_slice(raw))); + assert_eq!( + value_to_sql_parameter(&value, chrono_tz::UTC), + None, + "non-UTF-8 binary payloads must not bind as text tracking parameters" + ); + } + } + + #[test] + fn capped_buffer_length_matches_text_row_set_rules() { + assert_eq!( + capped_buffer_length(Some(8192), Some(4096), 0, 100).unwrap(), + 4096 + ); + assert_eq!( + capped_buffer_length(None, Some(4096), 0, 100).unwrap(), + 4096 + ); + assert_eq!( + capped_buffer_length(Some(128), Some(4096), 0, 100).unwrap(), + 128 + ); + assert!(matches!( + capped_buffer_length(None, None, 2, 50), + Err(OdbcError::Db { + source: odbc_api::Error::TooLargeColumnBufferSize { + buffer_index: 2, + num_elements: 50, + element_size: usize::MAX, + }, + }) + )); + } + + #[test] + fn binary_buffer_desc_uses_octet_length_not_hex_display_size() { + // ODBC display_size for VARBINARY(3) is 6 (two hex chars per byte). Fetching with a + // text buffer of that size is what caused the hex-text bug; binary buffers must use 3. + assert_eq!( + DataType::Varbinary { + length: NonZeroUsize::new(3), + } + .display_size() + .map(NonZeroUsize::get), + Some(6) + ); + + let desc = buffer_desc_for_data_type( + &DataType::Varbinary { + length: NonZeroUsize::new(3), + }, + None, + Some(4096), + 0, + 100, + ) + .unwrap(); + assert_eq!(desc, BufferDesc::Binary { length: 3 }); + + let text_desc = buffer_desc_for_data_type( + &DataType::Varchar { + length: NonZeroUsize::new(3), + }, + None, + Some(4096), + 0, + 100, + ) + .unwrap(); + // VARCHAR(3) UTF-8 buffer is 3 * 4 = 12, matching TextRowSet::for_cursor. + assert_eq!(text_desc, BufferDesc::Text { max_str_len: 12 }); + } + + #[test] + fn is_binary_data_type_detects_binary_sql_types() { + assert!(is_binary_data_type(&DataType::Varbinary { + length: NonZeroUsize::new(16) + })); + assert!(is_binary_data_type(&DataType::Binary { + length: NonZeroUsize::new(16) + })); + assert!(is_binary_data_type(&DataType::LongVarbinary { + length: None + })); + assert!(!is_binary_data_type(&DataType::Varchar { + length: NonZeroUsize::new(16) + })); + assert!(!is_binary_data_type(&DataType::Unknown)); + assert!(!is_binary_data_type(&DataType::Integer)); + } + + #[test] + fn variable_width_columns_use_streaming_fetch_with_a_buffer_limit() { + assert!(requires_streaming_fetch( + &DataType::LongVarchar { length: None }, + 4096 + )); + assert!(!requires_streaming_fetch( + &DataType::Varbinary { + length: NonZeroUsize::new(16), + }, + 4096 + )); + assert!(requires_streaming_fetch( + &DataType::Varbinary { + length: NonZeroUsize::new(4097), + }, + 4096 + )); + assert!(!requires_streaming_fetch(&DataType::Integer, 4096)); + assert!(!requires_streaming_fetch( + &DataType::Char { + length: NonZeroUsize::new(16), + }, + 4096 + )); + assert!(requires_streaming_fetch( + &DataType::Char { + length: NonZeroUsize::new(2048), + }, + 4096 + )); + } + + #[test] + fn map_value_time_preserved_as_bytes_for_tracking_bind() { + let raw = b"15:30:00"; + let value = map_value( + &odbc_api::DataType::Time { precision: 0 }, + Some(raw), + chrono_tz::UTC, + ); + assert_eq!(value, Value::Bytes(Bytes::from_static(raw))); + assert_eq!( + value_to_sql_parameter(&value, chrono_tz::UTC), + Some("15:30:00".to_owned()) + ); + } + + #[test] + fn map_value_date_preserved_as_bytes_for_tracking_bind() { + let raw = b"2025-10-04"; + let value = map_value(&odbc_api::DataType::Date, Some(raw), chrono_tz::UTC); + assert_eq!(value, Value::Bytes(Bytes::from_static(raw))); + assert_eq!( + value_to_sql_parameter(&value, chrono_tz::UTC), + Some("2025-10-04".to_owned()) + ); + } + + #[test] + fn map_value_zero_date_preserved_as_bytes_for_tracking_bind() { + let raw = b"0000-00-00"; + let value = map_value(&odbc_api::DataType::Date, Some(raw), chrono_tz::UTC); + assert_eq!(value, Value::Bytes(Bytes::from_static(raw))); + assert_eq!( + value_to_sql_parameter(&value, chrono_tz::UTC), + Some("0000-00-00".to_owned()) + ); + } + + #[test] + fn map_value_duration_time_preserved_as_bytes_for_tracking_bind() { + let raw = b"25:00:00"; + let value = map_value( + &odbc_api::DataType::Time { precision: 0 }, + Some(raw), + chrono_tz::UTC, + ); + assert_eq!(value, Value::Bytes(Bytes::from_static(raw))); + assert_eq!( + value_to_sql_parameter(&value, chrono_tz::UTC), + Some("25:00:00".to_owned()) + ); + } + + #[test] + fn map_value_timestamp_with_offset_preserved_as_bytes_for_tracking_bind() { + let raw = b"2025-04-28 01:20:04+00"; + let value = map_value( + &odbc_api::DataType::Timestamp { precision: 0 }, + Some(raw), + chrono_tz::UTC, + ); + assert_eq!(value, Value::Bytes(Bytes::from_static(raw))); + assert_eq!( + value_to_sql_parameter(&value, chrono_tz::UTC), + Some("2025-04-28 01:20:04+00".to_owned()) + ); + } + + #[test] + fn map_value_timestamp_with_offset_and_colon_preserved_as_bytes_for_tracking_bind() { + let raw = b"2025-04-28 01:20:04+00:00"; + let value = map_value( + &odbc_api::DataType::Timestamp { precision: 0 }, + Some(raw), + chrono_tz::UTC, + ); + assert_eq!(value, Value::Bytes(Bytes::from_static(raw))); + assert_eq!( + value_to_sql_parameter(&value, chrono_tz::UTC), + Some("2025-04-28 01:20:04+00:00".to_owned()) + ); + } + + #[test] + fn map_value_timestamp_with_negative_offset_preserved_as_bytes_for_tracking_bind() { + let raw = b"2025-04-28 01:20:04-05:00"; + let value = map_value( + &odbc_api::DataType::Timestamp { precision: 0 }, + Some(raw), + chrono_tz::UTC, + ); + assert_eq!(value, Value::Bytes(Bytes::from_static(raw))); + assert_eq!( + value_to_sql_parameter(&value, chrono_tz::UTC), + Some("2025-04-28 01:20:04-05:00".to_owned()) + ); + } + + #[test] + fn map_value_timestamp_rfc3339_z_preserved_as_bytes_for_tracking_bind() { + let raw = b"2025-04-28T01:20:04Z"; + let value = map_value( + &odbc_api::DataType::Timestamp { precision: 0 }, + Some(raw), + chrono_tz::UTC, + ); + assert_eq!(value, Value::Bytes(Bytes::from_static(raw))); + assert_eq!( + value_to_sql_parameter(&value, chrono_tz::UTC), + Some("2025-04-28T01:20:04Z".to_owned()) + ); + } + + #[test] + fn map_value_timestamp_rfc3339_t_offset_preserved_as_bytes_for_tracking_bind() { + let raw = b"2025-04-28T01:20:04+00:00"; + let value = map_value( + &odbc_api::DataType::Timestamp { precision: 0 }, + Some(raw), + chrono_tz::UTC, + ); + assert_eq!(value, Value::Bytes(Bytes::from_static(raw))); + assert_eq!( + value_to_sql_parameter(&value, chrono_tz::UTC), + Some("2025-04-28T01:20:04+00:00".to_owned()) + ); + } + + #[test] + fn map_value_timestamp_rfc3339_non_utc_offset_preserved_as_bytes_for_tracking_bind() { + // Rebinding through Value::Timestamp would yield a naive local time in + // odbc_default_timezone and can skip/replay rows when the DB offset differs. + let raw = b"2025-04-28T01:20:04+02:00"; + let value = map_value( + &odbc_api::DataType::Timestamp { precision: 0 }, + Some(raw), + chrono_tz::Asia::Seoul, + ); + assert_eq!(value, Value::Bytes(Bytes::from_static(raw))); + assert_eq!( + value_to_sql_parameter(&value, chrono_tz::Asia::Seoul), + Some("2025-04-28T01:20:04+02:00".to_owned()) + ); + } + + #[test] + fn map_value_timestamp_rfc3339_fractional_offset_preserved_as_bytes_for_tracking_bind() { + let raw = b"2025-04-28T01:20:04.123456+02:00"; + let value = map_value( + &odbc_api::DataType::Timestamp { precision: 6 }, + Some(raw), + chrono_tz::UTC, + ); + assert_eq!(value, Value::Bytes(Bytes::from_static(raw))); + assert_eq!( + value_to_sql_parameter(&value, chrono_tz::UTC), + Some("2025-04-28T01:20:04.123456+02:00".to_owned()) + ); + } + + #[test] + fn map_value_timestamp_unparseable_preserved_as_bytes() { + let raw = b"not-a-timestamp"; + let value = map_value( + &odbc_api::DataType::Timestamp { precision: 0 }, + Some(raw), + chrono_tz::UTC, + ); + assert_eq!(value, Value::Bytes(Bytes::from_static(raw))); + assert_eq!( + value_to_sql_parameter(&value, chrono_tz::UTC), + Some("not-a-timestamp".to_owned()) + ); + } + + #[test] + fn map_value_timestamp_naive_parsed_to_timestamp() { + let raw = b"2025-10-04 12:34:56"; + let value = map_value( + &odbc_api::DataType::Timestamp { precision: 0 }, + Some(raw), + chrono_tz::UTC, + ); + assert_eq!( + value, + Value::Timestamp( + chrono::Utc + .with_ymd_and_hms(2025, 10, 4, 12, 34, 56) + .unwrap() + ) + ); + } + + #[test] + fn value_to_sql_parameter_preserves_rfc3339_looking_string_bytes() { + // VARCHAR/TEXT tracking values must round-trip unchanged even when they + // parse as RFC3339; only Value::Timestamp is reformatted. + let raw = "2024-06-01T00:00:00Z"; + let value = Value::Bytes(Bytes::from_static(raw.as_bytes())); + assert_eq!( + value_to_sql_parameter(&value, chrono_tz::Asia::Seoul), + Some(raw.to_owned()) + ); + } + + #[test] + fn value_to_sql_parameter_formats_timestamp_in_odbc_timezone() { + let value = Value::Timestamp(chrono::Utc.with_ymd_and_hms(2024, 6, 1, 0, 0, 0).unwrap()); + assert_eq!( + value_to_sql_parameter(&value, chrono_tz::Asia::Seoul), + Some("2024-06-01 09:00:00".to_owned()) + ); + } + + #[test] + fn order_params_preserves_static_and_array_order() { + let params = vec![ + OdbcStatementParam { + name: "tenant_id".to_owned(), + value: "acme".to_owned(), + }, + OdbcStatementParam { + name: "id".to_owned(), + value: "0".to_owned(), + }, + OdbcStatementParam { + name: "region".to_owned(), + value: "us-east".to_owned(), + }, + ]; + let mut overlay = ObjectMap::new(); + overlay.insert(KeyString::from("id"), Value::Integer(42)); + let tracking = vec!["id".to_owned()]; + + let bound = order_params(¶ms, Some(&overlay), Some(&tracking), chrono_tz::UTC) + .expect("order params"); + + assert_eq!(bound.len(), 3); + // Array order is the bind order even when static params surround tracking ones. + let expected = ["acme", "42", "us-east"]; + for (param, expected) in bound.iter().zip(expected) { + assert_eq!( + std::str::from_utf8(param.as_bytes().expect("bound bytes")).expect("utf-8"), + expected + ); + } + } + + #[test] + fn order_params_errors_on_missing_tracking_overlay() { + let params = vec![ + OdbcStatementParam { + name: "tenant_id".to_owned(), + value: "acme".to_owned(), + }, + OdbcStatementParam { + name: "id".to_owned(), + value: "0".to_owned(), + }, + ]; + let overlay = ObjectMap::new(); + let tracking = vec!["id".to_owned()]; + + let error = match order_params(¶ms, Some(&overlay), Some(&tracking), chrono_tz::UTC) { + Err(error) => error, + Ok(_) => panic!("expected missing tracking column error"), + }; + + assert!(matches!( + error, + OdbcError::MissingTrackingColumn { column } if column == "id" + )); + } + + #[test] + fn overlay_params_keeps_static_values() { + let params = vec![ + OdbcStatementParam { + name: "tenant_id".to_owned(), + value: "acme".to_owned(), + }, + OdbcStatementParam { + name: "id".to_owned(), + value: "0".to_owned(), + }, + ]; + let mut overlay = ObjectMap::new(); + overlay.insert( + KeyString::from("id"), + Value::Bytes(Bytes::from_static(b"42")), + ); + let tracking = vec!["id".to_owned()]; + + let next = overlay_params(¶ms, &overlay, &tracking, chrono_tz::UTC).expect("overlay"); + + assert_eq!( + next, + vec![ + OdbcStatementParam { + name: "tenant_id".to_owned(), + value: "acme".to_owned(), + }, + OdbcStatementParam { + name: "id".to_owned(), + value: "42".to_owned(), + }, + ] + ); + } + + #[test] + fn extract_tracking_errors_on_missing_column() { + let mut obj = ObjectMap::new(); + obj.insert(KeyString::from("id"), Value::Integer(1)); + + let error = extract_tracking( + Value::Object(obj), + &["id".to_owned(), "name".to_owned()], + chrono_tz::UTC, + ) + .expect_err("expected missing tracking column error"); + + assert!(matches!( + error, + OdbcError::MissingTrackingColumn { column } if column == "name" + )); + } + + #[test] + fn extract_tracking_errors_on_null_tracking_value() { + let mut obj = ObjectMap::new(); + obj.insert(KeyString::from("id"), Value::Null); + + let error = extract_tracking(Value::Object(obj), &["id".to_owned()], chrono_tz::UTC) + .expect_err("expected invalid tracking value error"); + + assert!(matches!( + error, + OdbcError::InvalidTrackingValue { column } if column == "id" + )); + } + + #[test] + fn extract_tracking_errors_on_invalid_tracking_row() { + let error = extract_tracking(Value::Integer(1), &["id".to_owned()], chrono_tz::UTC) + .expect_err("expected invalid tracking row error"); + + assert!(matches!(error, OdbcError::InvalidTrackingRow)); + } + + #[test] + fn prepare_tracking_checkpoint_does_not_persist_when_extract_fails() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let path = temp_dir.path().join("tracking.json"); + let path = path.to_str().expect("utf-8 path"); + let mut obj = ObjectMap::new(); + obj.insert(KeyString::from("id"), Value::Null); + let template = vec![OdbcStatementParam { + name: "id".to_owned(), + value: "0".to_owned(), + }]; + + let error = prepare_tracking_checkpoint( + Some(path), + Value::Object(obj), + &template, + &["id".to_owned()], + chrono_tz::UTC, + ) + .expect_err("expected invalid tracking value"); + + assert!(matches!( + error, + OdbcError::InvalidTrackingValue { column } if column == "id" + )); + assert!( + !std::path::Path::new(path).exists(), + "checkpoint must not be written when extract fails" + ); + } + + #[test] + fn prepare_tracking_checkpoint_persists_after_overlay() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let path = temp_dir.path().join("tracking.json"); + let path = path.to_str().expect("utf-8 path"); + let mut obj = ObjectMap::new(); + obj.insert(KeyString::from("id"), Value::Integer(42)); + obj.insert(KeyString::from("name"), Value::from("vector")); + let template = vec![ + OdbcStatementParam { + name: "tenant_id".to_owned(), + value: "acme".to_owned(), + }, + OdbcStatementParam { + name: "id".to_owned(), + value: "0".to_owned(), + }, + OdbcStatementParam { + name: "name".to_owned(), + value: "init".to_owned(), + }, + ]; + + let next = prepare_tracking_checkpoint( + Some(path), + Value::Object(obj), + &template, + &["id".to_owned(), "name".to_owned()], + chrono_tz::UTC, + ) + .expect("prepared tracking checkpoint"); + + assert_eq!( + next, + vec![ + OdbcStatementParam { + name: "tenant_id".to_owned(), + value: "acme".to_owned(), + }, + OdbcStatementParam { + name: "id".to_owned(), + value: "42".to_owned(), + }, + OdbcStatementParam { + name: "name".to_owned(), + value: "vector".to_owned(), + }, + ] + ); + + let saved = load_tracking_map(path) + .expect("load checkpoint") + .expect("checkpoint exists"); + assert_eq!( + saved.get("id"), + Some(&Value::Bytes(Bytes::from_static(b"42"))) + ); + assert_eq!( + saved.get("name"), + Some(&Value::Bytes(Bytes::from_static(b"vector"))) + ); + } + + #[test] + fn load_tracking_map_reads_valid_tracking_metadata() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let path = temp_dir.path().join("tracking.json"); + fs::write(&path, r#"{"id":1,"name":"vector"}"#).expect("write metadata"); + let path = path.to_str().expect("utf-8 path"); + + let map = load_tracking_map(path) + .expect("load tracking map") + .expect("metadata map"); + + assert_eq!(map.get("id"), Some(&Value::Integer(1))); + assert_eq!(map.get("name"), Some(&Value::from("vector"))); + } + + #[test] + fn validate_tracking_state_errors_on_missing_column() { + let mut map = ObjectMap::new(); + map.insert(KeyString::from("id"), Value::Integer(1)); + + let error = + validate_tracking_state(&map, &["id".to_owned(), "name".to_owned()], chrono_tz::UTC) + .expect_err("validation error"); + + assert!(error.contains("name")); + } + + #[test] + fn prepare_metadata_path_creates_parent_directory() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let path = temp_dir.path().join("nested").join("tracking.json"); + let path = path.to_str().expect("utf-8 path"); + + prepare_metadata_path(path).expect("prepare metadata path"); + + assert!(temp_dir.path().join("nested").is_dir()); + } + + #[test] + fn prepare_metadata_path_rejects_empty_path() { + let error = match prepare_metadata_path("") { + Err(error) => error, + Ok(_) => panic!("expected empty path error"), + }; + + assert!(matches!(error, OdbcError::Io { .. })); + } + + #[test] + fn ensure_unique_column_names_accepts_distinct_names() { + ensure_unique_column_names(&["id".to_owned(), "name".to_owned()]) + .expect("distinct column names"); + } + + #[test] + fn ensure_unique_column_names_errors_on_duplicates() { + let error = match ensure_unique_column_names(&[ + "id".to_owned(), + "name".to_owned(), + "id".to_owned(), + ]) { + Err(error) => error, + Ok(_) => panic!("expected duplicate column names error"), + }; + + assert!(matches!( + error, + OdbcError::DuplicateColumnNames { columns } if columns == vec!["id".to_owned()] + )); + } + + #[test] + fn save_params_overwrites_existing_checkpoint() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let path = temp_dir.path().join("tracking.json"); + let path = path.to_str().expect("utf-8 path"); + + let mut first = ObjectMap::new(); + first.insert(KeyString::from("id"), Value::Integer(1)); + save_params(path, &first).expect("first save"); + + let mut second = ObjectMap::new(); + second.insert(KeyString::from("id"), Value::Integer(2)); + save_params(path, &second).expect("second save should overwrite"); + + let saved: ObjectMap = + serde_json::from_reader(File::open(path).expect("open checkpoint")).expect("parse"); + assert_eq!(saved.get("id"), Some(&Value::Integer(2))); + } + + #[test] + fn ensure_unique_column_names_errors_on_multiple_duplicates() { + let error = match ensure_unique_column_names(&[ + "id".to_owned(), + "name".to_owned(), + "id".to_owned(), + "name".to_owned(), + ]) { + Err(error) => error, + Ok(_) => panic!("expected duplicate column names error"), + }; + + assert!(matches!( + error, + OdbcError::DuplicateColumnNames { columns } + if columns == vec!["id".to_owned(), "name".to_owned()] + )); + } +} diff --git a/src/sources/odbc/config.rs b/src/sources/odbc/config.rs new file mode 100644 index 0000000000000..5885bded53925 --- /dev/null +++ b/src/sources/odbc/config.rs @@ -0,0 +1,638 @@ +use crate::config::{LogNamespace, SourceConfig, SourceContext, SourceOutput, log_schema}; +use crate::sources::Source; +use crate::sources::odbc::client::{Context, prepare_metadata_path, validate_tracking_state}; +use crate::sources::odbc::schedule::OdbcSchedule; +use chrono_tz::Tz; +use futures_util::FutureExt; +use serde_with::DurationSeconds; +use serde_with::serde_as; +use std::collections::HashSet; +use std::fs; +use std::io::BufReader; +use std::time::Duration; +use vector_config_macros::configurable_component; +use vector_lib::config::DataType; +use vector_lib::schema; +use vector_lib::sensitive_string::SensitiveString; +use vrl::prelude::ObjectMap; +use vrl::value::{KeyString, Kind, Value, kind::Collection}; + +/// A positional SQL parameter for an ODBC statement placeholder (`?`). +/// +/// Array order is the single source of truth for bind order and is independent of +/// configuration format key sorting. +#[configurable_component] +#[derive(Clone, Debug, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct OdbcStatementParam { + /// Parameter name. + /// + /// When the same name appears in `tracking_columns`, later runs overlay the + /// checkpointed or last-row value onto this entry while preserving bind order. + #[configurable(metadata(docs::examples = "id"))] + #[configurable(metadata(docs::examples = "tenant_id"))] + pub name: String, + + /// Initial value bound for this placeholder. + /// + /// For non-tracking parameters this value is reused on every scheduled run. + /// For tracking parameters it is used until a checkpoint or previous result + /// provides an updated value. + #[configurable(metadata(docs::examples = "0"))] + #[configurable(metadata(docs::examples = "acme"))] + pub value: String, +} + +/// Configuration for the `odbc` source. +#[serde_as] +#[configurable_component(source( + "odbc", + "Periodically pulls observability data from an ODBC interface by running a scheduled query." +))] +#[derive(Clone, Debug)] +#[serde(deny_unknown_fields)] +pub struct OdbcConfig { + /// The connection string to use for ODBC. + /// If the `connection_string_filepath` is set, this value is ignored. + #[configurable(metadata( + docs::examples = "driver={MariaDB Unicode};server=;port=;database=;uid=;pwd=" + ))] + #[serde(default)] + pub connection_string: SensitiveString, + + /// The path to the file that contains the connection string. + /// If this is not set or the file at that path does not exist, the `connection_string` field is used instead. + #[configurable(metadata(docs::examples = "/path/to/connection_string.txt"))] + pub connection_string_filepath: Option, + + /// The SQL statement to execute. + /// This SQL statement is executed periodically according to the `schedule`. + /// Defaults to `None`. If no SQL statement is provided, the source returns an error. + /// If the `statement_filepath` is set, this value is ignored. + #[configurable(metadata(docs::examples = "SELECT * FROM users WHERE id = ?"))] + pub statement: Option, + + /// The path to the file that contains the SQL statement. + /// If this is set, the `statement` field is ignored and the file must exist and be readable. + pub statement_filepath: Option, + + /// Maximum time to allow the SQL statement to run. + /// If the query does not finish within this window, it is canceled and retried at the next scheduled run. + /// Set to 0 to disable the timeout and wait indefinitely. + /// Prefer a positive timeout: Vector shutdown waits for any in-flight connect/execute, and + /// `0` can delay exit until the ODBC driver returns. + /// The default is 3 seconds. + #[configurable(metadata(docs::examples = 3))] + #[configurable(metadata( + docs::additional_props_description = "Maximum time to wait for the SQL statement to execute" + ))] + #[serde(default = "default_statement_timeout_sec")] + #[serde_as(as = "DurationSeconds")] + pub statement_timeout: Duration, + + /// Maximum time to wait for the ODBC connection/login to complete. + /// If the connection does not succeed within this window, the attempt fails + /// and is retried at the next scheduled run. + /// Set to 0 to disable the timeout and wait indefinitely. + /// Prefer a positive timeout: Vector shutdown waits for any in-flight connect/execute, and + /// `0` can delay exit until the ODBC driver returns. + /// The default is 3 seconds. + #[configurable(metadata(docs::examples = 3))] + #[configurable(metadata( + docs::additional_props_description = "Maximum time to wait for the ODBC connection/login to complete" + ))] + #[serde(default = "default_login_timeout_sec")] + #[serde_as(as = "DurationSeconds")] + pub login_timeout: Duration, + + /// Positional parameters for SQL statement placeholders (`?`). + /// + /// Array order is the bind order. Static filter values and tracking bootstrap + /// values can be mixed; only names listed in `tracking_columns` are overlaid + /// from checkpoints or the previous result. + /// + /// # Examples + /// + /// Incremental query with a static tenant filter: + /// + /// ```yaml + /// sources: + /// odbc: + /// statement: "SELECT * FROM users WHERE tenant_id = ? AND id > ? ORDER BY id ASC" + /// statement_init_params: + /// - name: tenant_id + /// value: "acme" + /// - name: id + /// value: "0" + /// tracking_columns: + /// - id + /// last_run_metadata_path: /path/to/tracking.json + /// # The rest of the fields are omitted + /// ``` + /// + /// Static-only filter without tracking: + /// + /// ```yaml + /// sources: + /// odbc: + /// statement: "SELECT * FROM users WHERE tenant_id = ?" + /// statement_init_params: + /// - name: tenant_id + /// value: "acme" + /// # The rest of the fields are omitted + /// ``` + pub statement_init_params: Option>, + + /// Cron expression used to schedule database queries. This field is required. + #[configurable(derived)] + pub schedule: OdbcSchedule, + + /// The timezone to use for the `schedule`. + /// Typically the timezone used when evaluating the cron expression. + /// The default is UTC. + /// + /// [Wikipedia]: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones + #[configurable(metadata(docs::examples = "UTC"))] + #[configurable(metadata( + docs::additional_props_description = "Timezone to use for the schedule" + ))] + #[serde(default = "default_schedule_timezone")] + pub schedule_timezone: Tz, + + /// Number of rows to fetch, convert, and send per batch. + /// This bounds ODBC driver fetch buffers and in-memory processing for each batch. + /// Must be greater than 0. + /// The default is 100. + #[configurable(metadata(docs::examples = 100))] + #[serde(default = "default_odbc_batch_size")] + pub odbc_batch_size: usize, + + /// Maximum bytes per cell when sizing ODBC text and binary row-set fetch buffers. + /// Columns that may exceed this limit are fetched individually and grow to their + /// full value size. Set to `0` to omit the upper bound and use driver-reported + /// sizes for row-set buffers instead. + /// The default is 4096. + #[configurable(metadata(docs::examples = 4096))] + #[serde(default = "default_odbc_max_str_limit")] + pub odbc_max_str_limit: usize, + + /// Timezone applied to database date/time columns that lack timezone information. + /// Ambiguous DST times use the latest matching instant; nonexistent times are kept as text. + /// Offset-bearing SQL or RFC3339 timestamp text is preserved as bytes and is not rewritten + /// with this timezone, so tracking parameters can round-trip the exact ODBC text. + /// The default is UTC. + #[configurable(metadata(docs::examples = "UTC"))] + #[configurable(metadata( + docs::additional_props_description = "Timezone to use for the database date/time type without a timezone" + ))] + #[serde(default = "default_odbc_default_timezone")] + pub odbc_default_timezone: Tz, + + /// Specifies the columns to track from the last row of the statement result set. + /// Their values overlay matching entries in `statement_init_params` on later runs while + /// preserving the declared bind order. + /// + /// When set, result batches are buffered until the query finishes; the final-row + /// checkpoint is validated (and persisted when `last_run_metadata_path` is set) before + /// any events are emitted. That avoids replaying the same rows when the last row is + /// missing a tracking column or has an unbindable value such as null. + /// Prefer incremental/`WHERE` bounded queries so buffering stays memory-safe. + /// + /// Requires `statement_init_params` entries whose names cover every tracking column. + /// Optional `last_run_metadata_path` overlays checkpointed values onto those entries. + /// Prefer non-binary tracking columns: checkpoints bind text parameters, so raw + /// `BINARY`/`VARBINARY`/`BYTEA` values that are not valid UTF-8 fail validation. + /// + /// # Examples + /// + /// ```yaml + /// sources: + /// odbc: + /// statement: "SELECT * FROM users WHERE id > ? ORDER BY id ASC" + /// statement_init_params: + /// - name: id + /// value: "0" + /// tracking_columns: + /// - id + /// # The rest of the fields are omitted + /// ``` + #[configurable(metadata(docs::examples = "id"))] + pub tracking_columns: Option>, + + /// The path to the file where tracked column values will be saved. + /// The tracked values are saved in JSON format and overlaid onto `statement_init_params` + /// for the next scheduled run. + /// If the file does not exist or the path is not specified, the initial values from + /// `statement_init_params` are used. + /// + /// When tracking is enabled, the full query result is buffered, the final-row checkpoint + /// is validated and written here, and only then are events emitted. A missing or + /// unbindable tracking value fails the poll before any emit (avoiding infinite replay). + /// A send failure after a successful checkpoint write may skip those rows on the next + /// run (at-most-once). The in-memory overlay is also advanced in that case so tracking + /// without `last_run_metadata_path` does not replay already-emitted rows. Prefer + /// incremental queries so the buffered result stays bounded. + /// + /// Parent directories are created automatically if they do not exist. + /// + /// # Examples + /// + /// If `tracking_columns = ["id", "name"]`, it is saved as the following JSON data. + /// + /// ```json + /// {"id":"42","name":"vector"} + /// ``` + #[configurable(metadata(docs::examples = "/path/to/tracking.json"))] + pub last_run_metadata_path: Option, + + /// The namespace to use for logs. This overrides the global setting. + #[configurable(metadata(docs::hidden))] + #[serde(default)] + pub log_namespace: Option, + + #[cfg(test)] + #[configurable(derived)] + #[serde(default)] + pub iterations: Option, +} + +impl OdbcConfig { + /// Returns the connection string to use for ODBC. + /// If the `connection_string_filepath` is set, read the file and return its content. + /// Trailing `\r`/`\n` from the file contents are stripped. + pub fn connection_string_or_file(&self) -> Result { + if let Some(path) = &self.connection_string_filepath { + match fs::read_to_string(path) { + Ok(content) => Ok(content.trim_end_matches(['\r', '\n']).to_string()), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + Ok(self.connection_string.inner().to_string()) + } + Err(err) => Err(err), + } + } else { + Ok(self.connection_string.inner().to_string()) + } + } + + /// Returns the SQL statement to execute. + /// If the `statement_filepath` is set, read the file and return its content. + /// When a filepath is configured, read failures are returned instead of falling back to `statement`. + pub fn statement_or_file(&self) -> Result { + if let Some(path) = &self.statement_filepath { + fs::read_to_string(path) + } else if let Some(statement) = &self.statement { + Ok(statement.clone()) + } else { + Ok(String::new()) + } + } + + fn validate_statement_init_params(&self) -> Result<(), String> { + let Some(params) = &self.statement_init_params else { + return Ok(()); + }; + + if params.is_empty() { + return Err("`statement_init_params` must not be empty when set".to_owned()); + } + + let mut seen = HashSet::with_capacity(params.len()); + for param in params { + if param.name.trim().is_empty() { + return Err("`statement_init_params[].name` must not be empty".to_owned()); + } + if !seen.insert(param.name.as_str()) { + return Err(format!( + "duplicate `statement_init_params` name `{}`; parameter names must be unique", + param.name + )); + } + } + + Ok(()) + } + + fn validate_tracking_columns(&self) -> Result<(), String> { + let has_metadata = self.last_run_metadata_path.is_some(); + let has_statement_init_params = self + .statement_init_params + .as_ref() + .is_some_and(|params| !params.is_empty()); + let has_tracking_columns = self + .tracking_columns + .as_ref() + .is_some_and(|columns| !columns.is_empty()); + + // Checkpoint files store tracking overlays only; static-only `statement_init_params` + // do not require tracking columns. Bind order always comes from `statement_init_params`. + if has_metadata && !has_tracking_columns { + return Err( + "`tracking_columns` must be set when using `last_run_metadata_path`".to_owned(), + ); + } + + if has_tracking_columns && !has_statement_init_params { + return Err( + "`statement_init_params` must be set when using `tracking_columns` so bind order is explicit" + .to_owned(), + ); + } + + if let (Some(tracking_columns), Some(params)) = ( + self.tracking_columns.as_ref(), + self.statement_init_params.as_ref(), + ) { + let param_names: HashSet<&str> = + params.iter().map(|param| param.name.as_str()).collect(); + for column in tracking_columns { + if !param_names.contains(column.as_str()) { + return Err(format!( + "`tracking_columns` entry `{column}` must also appear in `statement_init_params`" + )); + } + } + } + + Ok(()) + } + + fn statement_init_params_as_object_map(&self) -> Option { + self.statement_init_params.as_ref().map(|params| { + params + .iter() + .map(|param| { + ( + KeyString::from(param.name.as_str()), + Value::from(param.value.as_str()), + ) + }) + .collect() + }) + } + + fn validate_tracking_bootstrap(&self) -> Result<(), String> { + let Some(tracking_columns) = self + .tracking_columns + .as_ref() + .filter(|columns| !columns.is_empty()) + else { + return Ok(()); + }; + + // `validate_tracking_columns` already requires matching `statement_init_params`. + let init_params = self.statement_init_params_as_object_map().ok_or_else(|| { + "`statement_init_params` must be set when using `tracking_columns`".to_owned() + })?; + let tz = self.odbc_default_timezone; + + if let Some(path) = &self.last_run_metadata_path { + prepare_metadata_path(path).map_err(|error| error.to_string())?; + + return match fs::metadata(path) { + Ok(_) => { + let file = fs::File::open(path).map_err(|source| { + format!("unable to read `last_run_metadata_path` `{path}`: {source}") + })?; + let map: ObjectMap = + serde_json::from_reader(BufReader::new(file)).map_err(|source| { + format!("unable to parse `last_run_metadata_path` `{path}`: {source}") + })?; + validate_tracking_state(&map, tracking_columns, tz) + } + Err(source) if source.kind() == std::io::ErrorKind::NotFound => { + validate_tracking_state(&init_params, tracking_columns, tz) + } + Err(source) => Err(format!( + "unable to access `last_run_metadata_path` `{path}`: {source}" + )), + }; + } + + validate_tracking_state(&init_params, tracking_columns, tz) + } +} + +impl_generate_config_from_default!(OdbcConfig); + +const fn default_statement_timeout_sec() -> Duration { + Duration::from_secs(3) +} + +const fn default_login_timeout_sec() -> Duration { + Duration::from_secs(3) +} + +const fn default_schedule_timezone() -> Tz { + Tz::UTC +} + +const fn default_odbc_default_timezone() -> Tz { + default_schedule_timezone() +} + +const fn default_odbc_batch_size() -> usize { + 100 +} + +const fn default_odbc_max_str_limit() -> usize { + 4096 +} + +fn default_schedule() -> OdbcSchedule { + "0 * * * * *".into() +} + +fn odbc_schema_definition(log_namespace: LogNamespace) -> schema::Definition { + // Query rows are always objects whose columns become top-level fields. Column + // types vary by SQL query, so unknown fields allow any VRL value including timestamps. + let row_fields = Collection::empty().with_unknown(Kind::json().or_timestamp()); + + match log_namespace { + LogNamespace::Legacy => { + let mut definition = schema::Definition::empty_legacy_namespace() + .unknown_fields(Kind::json().or_timestamp()); + + if let Some(timestamp_key) = log_schema().timestamp_key() { + definition = + definition.try_with_field(timestamp_key, Kind::timestamp(), Some("timestamp")); + } + + definition + } + LogNamespace::Vector => { + schema::Definition::new_with_default_metadata(Kind::object(row_fields), [log_namespace]) + } + } +} + +impl Default for OdbcConfig { + fn default() -> Self { + Self { + connection_string: SensitiveString::default(), + connection_string_filepath: None, + schedule: default_schedule(), + schedule_timezone: Tz::UTC, + statement: None, + statement_timeout: default_statement_timeout_sec(), + login_timeout: default_login_timeout_sec(), + statement_init_params: None, + odbc_batch_size: default_odbc_batch_size(), + odbc_max_str_limit: default_odbc_max_str_limit(), + odbc_default_timezone: Tz::UTC, + tracking_columns: None, + last_run_metadata_path: None, + log_namespace: None, + statement_filepath: None, + #[cfg(test)] + iterations: None, + } + } +} + +#[async_trait::async_trait] +#[typetag::serde(name = "odbc")] +impl SourceConfig for OdbcConfig { + async fn build(&self, cx: SourceContext) -> crate::Result { + if self.connection_string_or_file()?.trim().is_empty() { + return Err( + "either a non-empty `connection_string` or a readable `connection_string_filepath` must be provided".into(), + ); + } + + if self.statement_or_file()?.trim().is_empty() { + return Err( + "either a non-empty `statement` or a readable `statement_filepath` must be provided" + .into(), + ); + } + + if self.odbc_batch_size == 0 { + return Err("`odbc_batch_size` must be greater than 0".into()); + } + + self.validate_statement_init_params()?; + self.validate_tracking_columns()?; + self.validate_tracking_bootstrap()?; + + let log_namespace = cx.log_namespace(self.log_namespace); + let guard = Context::new(self.clone(), cx, log_namespace)?; + let context = Box::new(guard); + Ok(context.run_schedule().boxed()) + } + + fn outputs(&self, global_log_namespace: LogNamespace) -> Vec { + let log_namespace = global_log_namespace.merge(self.log_namespace); + + let schema_definition = + odbc_schema_definition(log_namespace).with_standard_vector_source_metadata(); + + vec![SourceOutput::new_maybe_logs( + DataType::Log, + schema_definition, + )] + } + + // At-most-once when `tracking_columns` is set: the final-row checkpoint is validated and + // persisted before events are emitted. A later send failure advances the in-memory overlay + // (and keeps any on-disk checkpoint) so already-emitted rows are not replayed; unsent rows + // from that poll may be skipped. + fn can_acknowledge(&self) -> bool { + false + } +} + +#[cfg(test)] +mod tests { + use super::{OdbcConfig, OdbcStatementParam}; + + #[test] + fn parses_statement_init_params_array_preserving_order() { + let config: OdbcConfig = toml::from_str( + r#" + connection_string = "driver={MariaDB Unicode};server=localhost;database=db;uid=u;pwd=p;" + statement = "SELECT * FROM t WHERE tenant_id = ? AND id > ?" + schedule = "*/5 * * * * *" + statement_init_params = [ + { name = "tenant_id", value = "acme" }, + { name = "id", value = "0" }, + ] + tracking_columns = ["id"] + last_run_metadata_path = "tracking.json" + "#, + ) + .expect("parse config"); + + assert_eq!( + config.statement_init_params, + Some(vec![ + OdbcStatementParam { + name: "tenant_id".to_owned(), + value: "acme".to_owned(), + }, + OdbcStatementParam { + name: "id".to_owned(), + value: "0".to_owned(), + }, + ]) + ); + } + + #[test] + fn allows_static_statement_init_params_without_tracking() { + let config = OdbcConfig { + statement_init_params: Some(vec![OdbcStatementParam { + name: "tenant_id".to_owned(), + value: "acme".to_owned(), + }]), + tracking_columns: None, + last_run_metadata_path: None, + ..Default::default() + }; + + config + .validate_statement_init_params() + .expect("statement params"); + config + .validate_tracking_columns() + .expect("static-only params allowed"); + } + + #[test] + fn rejects_tracking_columns_missing_from_statement_init_params() { + let config = OdbcConfig { + statement_init_params: Some(vec![OdbcStatementParam { + name: "tenant_id".to_owned(), + value: "acme".to_owned(), + }]), + tracking_columns: Some(vec!["id".to_owned()]), + ..Default::default() + }; + + let error = config + .validate_tracking_columns() + .expect_err("tracking name must exist in statement_init_params"); + assert!(error.contains("`id`")); + } + + #[test] + fn rejects_duplicate_statement_param_names() { + let config = OdbcConfig { + statement_init_params: Some(vec![ + OdbcStatementParam { + name: "id".to_owned(), + value: "0".to_owned(), + }, + OdbcStatementParam { + name: "id".to_owned(), + value: "1".to_owned(), + }, + ]), + ..Default::default() + }; + + let error = config + .validate_statement_init_params() + .expect_err("duplicate names"); + assert!(error.contains("duplicate")); + } +} diff --git a/src/sources/odbc/integration_tests.rs b/src/sources/odbc/integration_tests.rs new file mode 100644 index 0000000000000..9286ceb25d571 --- /dev/null +++ b/src/sources/odbc/integration_tests.rs @@ -0,0 +1,970 @@ +use crate::sources::odbc::client::execute_query; +use crate::sources::odbc::config::{OdbcConfig, OdbcStatementParam}; +use crate::test_util::components::SOURCE_TAGS; +use crate::test_util::components::run_and_assert_source_compliance; +use bytes::Bytes; +use chrono::TimeZone; +use chrono_tz::Tz; +use odbc_api::{ConnectionOptions, IntoParameter}; +use ordered_float::NotNan; +use std::borrow::Cow; +use std::fs; +use std::time::{Duration, Instant}; +use vector_lib::event::Event; +use vector_lib::sensitive_string::SensitiveString; +use vrl::value::Value; + +enum DbType { + MariaDb, + Postgres, +} + +fn get_db_type() -> DbType { + match std::env::var("ODBC_DB_TYPE").as_deref() { + Ok("mariadb") => DbType::MariaDb, + Ok("postgresql") => DbType::Postgres, + _ => panic!("Required environment variable 'ODBC_DB_TYPE'"), + } +} + +#[allow(clippy::too_many_arguments)] +fn collect_query_rows( + env: &odbc_api::Environment, + conn_str: &str, + stmt_str: &str, + stmt_params: Vec, + login_timeout: Duration, + statement_timeout: Duration, + tz: Tz, + batch_size: usize, + max_str_limit: Option, +) -> Result, crate::sources::odbc::OdbcError> { + let mut rows = Vec::new(); + execute_query( + env, + conn_str, + stmt_str, + stmt_params, + login_timeout, + statement_timeout, + tz, + batch_size, + max_str_limit, + |batch| { + rows.extend(batch); + Ok(true) + }, + )?; + Ok(rows) +} + +fn get_conn_str() -> String { + std::env::var("ODBC_CONN_STRING").expect("Required environment variable 'ODBC_CONN_STRING'") +} + +const fn get_conn_opt() -> ConnectionOptions { + ConnectionOptions { + login_timeout_sec: Some(3), + packet_size: None, + } +} + +fn connect_when_ready<'a>( + env: &'a odbc_api::Environment, + conn_str: &str, +) -> odbc_api::Connection<'a> { + const TIMEOUT: Duration = Duration::from_secs(30); + const INTERVAL: Duration = Duration::from_millis(500); + + let start = Instant::now(); + loop { + match env.connect_with_connection_string(conn_str, get_conn_opt()) { + Ok(conn) => { + debug!( + elapsed = ?start.elapsed(), + "ODBC database accepted connection." + ); + return conn; + } + Err(err) => { + let elapsed = start.elapsed(); + if elapsed >= TIMEOUT { + error!( + %err, + ?elapsed, + ?TIMEOUT, + "ODBC database not ready before timeout." + ); + panic!("ODBC database not ready after {TIMEOUT:?}: {err}"); + } + warn!( + %err, + ?elapsed, + ?INTERVAL, + "ODBC database not ready, retrying." + ); + std::thread::sleep(INTERVAL); + } + } + } +} + +fn get_value_from_event<'a>(event: &'a Event, key: &str) -> Option> { + event.as_log().value().as_object()?.get(key)?.as_str() +} + +#[tokio::test] +async fn parse_odbc_config() { + let conn_str = get_conn_str(); + let config_str = format!( + r#" + connection_string = "{conn_str}" + statement = "SELECT * FROM odbc_table WHERE id > ? ORDER BY id ASC LIMIT 1;" + schedule = "*/5 * * * * *" + schedule_timezone = "UTC" + last_run_metadata_path = "odbc_tracking.json" + tracking_columns = ["id"] + statement_init_params = [ + {{ name = "id", value = "0" }}, + ] + iterations = 1 + "# + ); + let config = toml::from_str::(&config_str).expect("parse ODBC config"); + assert_eq!(config.tracking_columns, Some(vec!["id".to_owned()])); + assert_eq!( + config.statement_init_params, + Some(vec![OdbcStatementParam { + name: "id".to_owned(), + value: "0".to_owned(), + }]) + ); +} + +#[tokio::test] +async fn scheduled_query_executed() { + let conn_str = get_conn_str(); + let env = odbc_api::environment().unwrap(); + drop(connect_when_ready(env, &conn_str)); + + let events = run_and_assert_source_compliance( + OdbcConfig { + connection_string: SensitiveString::from(conn_str), + schedule: "*/1 * * * * *".into(), + statement: Some("SELECT 1".to_string()), + iterations: Some(1), + ..Default::default() + }, + Duration::from_secs(3), + &SOURCE_TAGS, + ) + .await; + + assert!( + !events.is_empty(), + "expected ODBC source to emit events from SELECT 1" + ); +} + +#[tokio::test] +async fn query_executed_with_init_params() { + const LAST_RUN_METADATA_PATH: &str = "odbc_tracking-integration-tests.json"; + + let conn_str = get_conn_str(); + let env = odbc_api::environment().unwrap(); + let conn = connect_when_ready(env, &conn_str); + let _ = conn + .execute("DROP TABLE IF EXISTS odbc_table;", (), Some(3)) + .unwrap(); + let _ = conn + .execute( + match get_db_type() { + DbType::MariaDb => { + r#" +CREATE TABLE odbc_table +( + id int auto_increment primary key, + name varchar(255) null, + `datetime` datetime null +); + "# + } + DbType::Postgres => { + r#" +CREATE TABLE odbc_table +( + id SERIAL PRIMARY KEY, + name VARCHAR(255), + "datetime" TIMESTAMP NULL +); +"# + } + }, + (), + Some(3), + ) + .unwrap(); + let _ = conn + .execute( + r#" +INSERT INTO odbc_table (name, datetime) VALUES +('test1', now()), +('test2', now()), +('test3', now()), +('test4', now()), +('test5', now()); + "#, + (), + Some(3), + ) + .unwrap(); + let params = vec![OdbcStatementParam { + name: "id".to_string(), + value: "0".to_string(), + }]; + + fs::remove_file(LAST_RUN_METADATA_PATH).ok(); + + let events = run_and_assert_source_compliance( + OdbcConfig { + connection_string: SensitiveString::from(conn_str), + schedule: "*/1 * * * * *".into(), + statement: Some( + "SELECT * FROM odbc_table WHERE id > ? ORDER BY id ASC LIMIT 1;".to_string(), + ), + statement_init_params: Some(params), + tracking_columns: Some(vec!["id".to_string()]), + last_run_metadata_path: Some(LAST_RUN_METADATA_PATH.to_string()), + iterations: Some(5), + ..Default::default() + }, + Duration::from_secs(10), + &SOURCE_TAGS, + ) + .await; + + debug!("{}", serde_json::to_string_pretty(&events).unwrap()); + assert_eq!( + get_value_from_event(&events[0], "name"), + Some("test1".into()) + ); + assert_eq!( + get_value_from_event(&events[1], "name"), + Some("test2".into()) + ); + assert_eq!( + get_value_from_event(&events[2], "name"), + Some("test3".into()) + ); + assert_eq!( + get_value_from_event(&events[3], "name"), + Some("test4".into()) + ); + assert_eq!( + get_value_from_event(&events[4], "name"), + Some("test5".into()) + ); +} + +#[tokio::test] +async fn query_executed_with_filepath() { + const CONNECTION_STRING_FILE_PATH: &str = "odbc_connection_string.txt"; + const STATEMENT_FILE_PATH: &str = "odbc_statement.sql"; + const LAST_RUN_METADATA_PATH: &str = "odbc_tracking-integration-tests.json"; + + let conn_str = get_conn_str(); + let env = odbc_api::environment().unwrap(); + let conn = connect_when_ready(env, &conn_str); + let _ = conn + .execute("DROP TABLE IF EXISTS odbc_table;", (), Some(3)) + .unwrap(); + let _ = conn + .execute( + match get_db_type() { + DbType::MariaDb => { + r#" +CREATE TABLE odbc_table +( + id int auto_increment primary key, + name varchar(255) null, + `datetime` datetime null +);"# + } + DbType::Postgres => { + r#" +CREATE TABLE odbc_table +( + id SERIAL PRIMARY KEY, + name VARCHAR(255), + "datetime" TIMESTAMP NULL +);"# + } + }, + (), + Some(3), + ) + .unwrap(); + let _ = conn + .execute( + r#" +INSERT INTO odbc_table (name, datetime) VALUES +('test1', now()), +('test2', now()), +('test3', now()), +('test4', now()), +('test5', now()); + "#, + (), + Some(3), + ) + .unwrap(); + let params = vec![OdbcStatementParam { + name: "id".to_string(), + value: "0".to_string(), + }]; + + fs::write(CONNECTION_STRING_FILE_PATH, conn_str).unwrap(); + fs::write( + STATEMENT_FILE_PATH, + "SELECT * FROM odbc_table WHERE id > ? ORDER BY id ASC LIMIT 1;", + ) + .unwrap(); + fs::remove_file(LAST_RUN_METADATA_PATH).ok(); + + let events = run_and_assert_source_compliance( + OdbcConfig { + connection_string_filepath: Some(CONNECTION_STRING_FILE_PATH.to_string()), + schedule: "*/1 * * * * *".into(), + statement_filepath: Some(STATEMENT_FILE_PATH.to_string()), + statement_init_params: Some(params), + tracking_columns: Some(vec!["id".to_string()]), + last_run_metadata_path: Some(LAST_RUN_METADATA_PATH.to_string()), + iterations: Some(5), + ..Default::default() + }, + Duration::from_secs(10), + &SOURCE_TAGS, + ) + .await; + + debug!("{}", serde_json::to_string_pretty(&events).unwrap()); + assert_eq!( + get_value_from_event(&events[0], "name"), + Some("test1".into()) + ); + assert_eq!( + get_value_from_event(&events[1], "name"), + Some("test2".into()) + ); + assert_eq!( + get_value_from_event(&events[2], "name"), + Some("test3".into()) + ); + assert_eq!( + get_value_from_event(&events[3], "name"), + Some("test4".into()) + ); + assert_eq!( + get_value_from_event(&events[4], "name"), + Some("test5".into()) + ); +} + +#[tokio::test] +async fn query_fetches_text_larger_than_rowset_buffer() { + let conn_str = get_conn_str(); + let env = odbc_api::environment().unwrap(); + let conn = connect_when_ready(env, &conn_str); + let _ = conn + .execute("DROP TABLE IF EXISTS large_text_column;", (), Some(3)) + .unwrap(); + let _ = conn + .execute( + "CREATE TABLE large_text_column (content VARCHAR(8192) NOT NULL);", + (), + Some(3), + ) + .unwrap(); + + let content = "x".repeat(8192); + let _ = conn + .execute( + "INSERT INTO large_text_column (content) VALUES (?);", + &content.as_str().into_parameter(), + Some(3), + ) + .unwrap(); + + let rows = collect_query_rows( + env, + &conn_str, + "SELECT content FROM large_text_column;", + Vec::new(), + Duration::from_secs(3), + Duration::from_secs(3), + chrono_tz::UTC, + 100, + Some(4096), + ) + .unwrap(); + + assert_eq!(rows.len(), 1); + assert_eq!( + rows[0].as_object().unwrap().get("content"), + Some(&Value::Bytes(Bytes::from(content))) + ); +} + +#[tokio::test] +async fn query_number_types() { + let conn_str = get_conn_str(); + let env = odbc_api::environment().unwrap(); + let conn = connect_when_ready(env, &conn_str); + let _ = conn + .execute("DROP TABLE IF EXISTS number_columns;", (), Some(3)) + .unwrap(); + let _ = conn + .execute( + match get_db_type() { + DbType::MariaDb => { + r#" +create table number_columns +( + int_col int(10) null, + bit_col bit null, + mediumint_col mediumint null, + middleint_col mediumint null, + smallint_col smallint null, + tinyint_col tinyint null, + bigint_col bigint null, + boolean_col tinyint(1) null, + double_col double null, + float_col float null, + decimal_col decimal(10, 2) null +); + "# + } + DbType::Postgres => { + r#" +CREATE TABLE number_columns +( + int_col INTEGER, -- integer + bit_col BIT, -- single bit (use BIT(n) to specify multiple bits) + mediumint_col INTEGER, -- no MEDIUMINT in PostgreSQL, mapped to INTEGER + middleint_col INTEGER, -- same as MEDIUMINT, mapped to INTEGER + smallint_col SMALLINT, -- small integer + tinyint_col SMALLINT, -- no TINYINT in PostgreSQL, mapped to SMALLINT + bigint_col BIGINT, -- big integer (64-bit) + boolean_col BOOLEAN, -- MySQL tinyint(1) mapped to BOOLEAN + double_col DOUBLE PRECISION, -- MySQL DOUBLE mapped to PostgreSQL DOUBLE PRECISION + float_col REAL, -- MySQL FLOAT mapped to PostgreSQL REAL (4-byte float) + decimal_col NUMERIC(10,2) -- MySQL DECIMAL mapped to PostgreSQL NUMERIC(p,s) +); + "# + } + }, + (), + Some(3), + ) + .unwrap(); + + let _ = conn + .execute( + r#" +INSERT INTO number_columns ( + int_col, + bit_col, + mediumint_col, + middleint_col, + smallint_col, + tinyint_col, + bigint_col, + boolean_col, + double_col, + float_col, + decimal_col +) VALUES ( + -2147483648, + b'0', + -8388608, + -8388608, + -32768, + -128, + -9223372036854775808, + FALSE, + -1.7976931348623157e308, + -3.402823466e38, + -99999999.99 +); + "#, + (), + Some(3), + ) + .unwrap(); + + let _ = conn + .execute( + r#" +INSERT INTO number_columns ( + int_col, + bit_col, + mediumint_col, + middleint_col, + smallint_col, + tinyint_col, + bigint_col, + boolean_col, + double_col, + float_col, + decimal_col +) VALUES ( + 2147483647, + b'1', + 8388607, + 8388607, + 32767, + 127, + 9223372036854775807, + TRUE, + 1.7976931348623157e308, + 3.402823466e38, + 99999999.99 +); + "#, + (), + Some(3), + ) + .unwrap(); + + let rows = collect_query_rows( + env, + &conn_str, + "SELECT * FROM number_columns ORDER BY int_col ASC;", + vec![], + Duration::from_secs(3), + Duration::from_secs(3), + Tz::UTC, + 10, + Some(1000), + ) + .unwrap(); + debug!("Rows Count: {}", rows.len()); + for row in &rows { + if let Value::Object(map) = row { + for (key, value) in map { + debug!("{key}: {value:?}"); + } + } + } + + let Value::Object(row) = &rows[0] else { + panic!("No rows returned") + }; + assert_eq!(*row.get("int_col").unwrap(), Value::Integer(-2147483648)); + match get_db_type() { + DbType::MariaDb => assert_eq!(*row.get("bit_col").unwrap(), Value::Boolean(false)), + DbType::Postgres => assert_eq!( + *row.get("bit_col").unwrap(), + Value::Bytes(Bytes::from_static(b"0")) + ), + } + assert_eq!(*row.get("mediumint_col").unwrap(), Value::Integer(-8388608)); + assert_eq!(*row.get("middleint_col").unwrap(), Value::Integer(-8388608)); + assert_eq!(*row.get("smallint_col").unwrap(), Value::Integer(-32768)); + assert_eq!(*row.get("tinyint_col").unwrap(), Value::Integer(-128)); + assert_eq!( + *row.get("bigint_col").unwrap(), + Value::Integer(-9223372036854775808) + ); + match get_db_type() { + DbType::MariaDb => assert_eq!(*row.get("boolean_col").unwrap(), Value::Integer(0)), + DbType::Postgres => assert_eq!( + *row.get("boolean_col").unwrap(), + Value::Bytes(Bytes::from_static(b"0")) + ), + } + assert_eq!( + *row.get("double_col").unwrap(), + Value::Float(NotNan::new(-1.7976931348623157e308).unwrap()) + ); + match get_db_type() { + DbType::MariaDb => assert_eq!( + *row.get("float_col").unwrap(), + Value::Float(NotNan::new(-3.40282e38).unwrap()) + ), + DbType::Postgres => assert_eq!( + *row.get("float_col").unwrap(), + Value::Float(NotNan::new(-3.4028235e38).unwrap()) + ), + } + assert_eq!( + *row.get("decimal_col").unwrap(), + Value::Bytes(Bytes::from_static(b"-99999999.99")) + ); + + let Value::Object(row) = &rows[1] else { + panic!("No second row returned") + }; + assert_eq!(*row.get("int_col").unwrap(), Value::Integer(2147483647)); + match get_db_type() { + DbType::MariaDb => assert_eq!(*row.get("bit_col").unwrap(), Value::Boolean(true)), + DbType::Postgres => assert_eq!( + *row.get("bit_col").unwrap(), + Value::Bytes(Bytes::from_static(b"1")) + ), + } + assert_eq!(*row.get("mediumint_col").unwrap(), Value::Integer(8388607)); + assert_eq!(*row.get("middleint_col").unwrap(), Value::Integer(8388607)); + assert_eq!(*row.get("smallint_col").unwrap(), Value::Integer(32767)); + assert_eq!(*row.get("tinyint_col").unwrap(), Value::Integer(127)); + assert_eq!( + *row.get("bigint_col").unwrap(), + Value::Integer(9223372036854775807) + ); + match get_db_type() { + DbType::MariaDb => assert_eq!(*row.get("boolean_col").unwrap(), Value::Integer(1)), + DbType::Postgres => assert_eq!( + *row.get("boolean_col").unwrap(), + Value::Bytes(Bytes::from_static(b"1")) + ), + } + assert_eq!( + *row.get("double_col").unwrap(), + Value::Float(NotNan::new(1.7976931348623157e308).unwrap()) + ); + match get_db_type() { + DbType::MariaDb => assert_eq!( + *row.get("float_col").unwrap(), + Value::Float(NotNan::new(3.40282e38).unwrap()) + ), + DbType::Postgres => assert_eq!( + *row.get("float_col").unwrap(), + Value::Float(NotNan::new(3.4028235e38).unwrap()) + ), + } + assert_eq!( + *row.get("decimal_col").unwrap(), + Value::Bytes(Bytes::from_static(b"99999999.99")) + ); + + debug!("{rows:#?}"); +} + +#[tokio::test] +async fn query_string_types() { + let conn_str = get_conn_str(); + let env = odbc_api::environment().unwrap(); + let conn = connect_when_ready(env, &conn_str); + let _ = conn + .execute("DROP TABLE IF EXISTS string_columns;", (), Some(3)) + .unwrap(); + let _ = conn + .execute( + match get_db_type() { + DbType::MariaDb => { + r#" +CREATE TABLE string_columns ( + char10_col CHAR(10) NULL, + nchar10_col NCHAR(10) NULL, + nvarchar10_col NVARCHAR(10) NULL, + text_col TEXT NULL, + tinytext_col TINYTEXT NULL, + mediumtext_col MEDIUMTEXT NULL, + longtext_col LONGTEXT NULL +) DEFAULT CHARSET = utf8mb3 COLLATE = utf8mb3_general_ci; + "# + } + DbType::Postgres => { + r#" +CREATE TABLE string_columns ( + char10_col CHAR(10), -- fixed-length character column (10) + nchar10_col CHAR(10), -- PostgreSQL has no NCHAR; use CHAR with UTF-8 encoding + nvarchar10_col VARCHAR(10), -- PostgreSQL has no NVARCHAR; use VARCHAR with UTF-8 encoding + text_col TEXT, -- unlimited length text + tinytext_col TEXT, -- PostgreSQL has no TINYTEXT; use TEXT + mediumtext_col TEXT, -- PostgreSQL has no MEDIUMTEXT; use TEXT + longtext_col TEXT -- PostgreSQL has no LONGTEXT; use TEXT +); + "# + } + }, + (), + Some(3), + ) + .unwrap(); + + let _ = conn + .execute( + r#" +INSERT INTO string_columns ( + char10_col, + nchar10_col, + nvarchar10_col, + text_col, + tinytext_col, + mediumtext_col, + longtext_col +) VALUES ( + '0123456789', + '0123456789', + '0123456789', + 'text', + 'tinytext', + 'mediumtext', + 'longtext' +); + "#, + (), + Some(3), + ) + .unwrap(); + + let rows = collect_query_rows( + env, + &conn_str, + "SELECT * FROM string_columns;", + vec![], + Duration::from_secs(3), + Duration::from_secs(3), + Tz::UTC, + 10, + Some(1000), + ) + .unwrap(); + + let Value::Object(row) = &rows[0] else { + panic!("No rows returned") + }; + + assert_eq!( + *row.get("char10_col").unwrap(), + Value::Bytes(Bytes::from_static(b"0123456789")) + ); + assert_eq!( + *row.get("nchar10_col").unwrap(), + Value::Bytes(Bytes::from_static(b"0123456789")) + ); + assert_eq!( + *row.get("nvarchar10_col").unwrap(), + Value::Bytes(Bytes::from_static(b"0123456789")) + ); + assert_eq!( + *row.get("text_col").unwrap(), + Value::Bytes(Bytes::from_static(b"text")) + ); + assert_eq!( + *row.get("tinytext_col").unwrap(), + Value::Bytes(Bytes::from_static(b"tinytext")) + ); + assert_eq!( + *row.get("mediumtext_col").unwrap(), + Value::Bytes(Bytes::from_static(b"mediumtext")) + ); + assert_eq!( + *row.get("longtext_col").unwrap(), + Value::Bytes(Bytes::from_static(b"longtext")) + ); +} + +#[tokio::test] +async fn query_binary_columns_emit_raw_bytes_not_hex_text() { + let conn_str = get_conn_str(); + let env = odbc_api::environment().unwrap(); + let conn = connect_when_ready(env, &conn_str); + let _ = conn + .execute("DROP TABLE IF EXISTS binary_columns;", (), Some(3)) + .unwrap(); + let _ = conn + .execute( + match get_db_type() { + DbType::MariaDb => { + r#" +CREATE TABLE binary_columns ( + id INT PRIMARY KEY, + bin_col BINARY(3) NULL, + varbin_col VARBINARY(16) NULL, + blob_col BLOB NULL +); + "# + } + DbType::Postgres => { + r#" +CREATE TABLE binary_columns ( + id INT PRIMARY KEY, + bin_col BYTEA, + varbin_col BYTEA, + blob_col BYTEA +); + "# + } + }, + (), + Some(3), + ) + .unwrap(); + + let _ = conn + .execute( + match get_db_type() { + // 0x00FF10 must arrive as three octets, not the ASCII hex text "00FF10". + DbType::MariaDb => { + r#"INSERT INTO binary_columns (id, bin_col, varbin_col, blob_col) + VALUES (1, X'00FF10', X'00FF10', X'00FF10');"# + } + DbType::Postgres => { + r#"INSERT INTO binary_columns (id, bin_col, varbin_col, blob_col) + VALUES (1, '\x00ff10', '\x00ff10', '\x00ff10');"# + } + }, + (), + Some(3), + ) + .unwrap(); + + let rows = collect_query_rows( + env, + &conn_str, + "SELECT * FROM binary_columns;", + vec![], + Duration::from_secs(3), + Duration::from_secs(3), + Tz::UTC, + 10, + Some(1000), + ) + .unwrap(); + + let Value::Object(row) = &rows[0] else { + panic!("No rows returned") + }; + + let expected = Bytes::from_static(&[0x00, 0xFF, 0x10]); + let hex_text_regression = Bytes::from_static(b"00FF10"); + + for column in ["bin_col", "varbin_col", "blob_col"] { + let value = row + .get(column) + .unwrap_or_else(|| panic!("missing {column}")); + assert_eq!( + value, + &Value::Bytes(expected.clone()), + "{column} must be raw binary octets" + ); + assert_ne!( + value, + &Value::Bytes(hex_text_regression.clone()), + "{column} must not be ODBC hex text" + ); + } +} + +#[tokio::test] +async fn query_timestamp_columns() { + let conn_str = get_conn_str(); + let env = odbc_api::environment().unwrap(); + let conn = connect_when_ready(env, &conn_str); + let _ = conn + .execute("DROP TABLE IF EXISTS timestamp_columns;", (), Some(3)) + .unwrap(); + let _ = conn + .execute( + match get_db_type() { + DbType::MariaDb => r#" +CREATE TABLE timestamp_columns ( + date_col DATE NULL, + datetime_col DATETIME NULL, + time_col TIME NULL, + timestamp_col TIMESTAMP NULL, + year_col YEAR NULL +) DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_general_ci; + "#, + DbType::Postgres => r#" +CREATE TABLE timestamp_columns ( + date_col DATE, -- MySQL DATE → PostgreSQL DATE + datetime_col TIMESTAMP, -- MySQL DATETIME → PostgreSQL TIMESTAMP + time_col TIME, -- Same in both + timestamp_col TIMESTAMP, -- Same type (use TIMESTAMPTZ if timezone is needed) + year_col SMALLINT -- MySQL YEAR → PostgreSQL SMALLINT +); + "#, + }, + (), + Some(3), + ) + .unwrap(); + + let _ = conn + .execute( + r#" +INSERT INTO timestamp_columns ( + date_col, + datetime_col, + time_col, + timestamp_col, + year_col +) +VALUES ( + '2025-10-04', + '2025-10-04 12:34:56', + '15:30:00', + '2025-10-04 12:34:56', + 2025 +); + "#, + (), + Some(3), + ) + .unwrap(); + + let rows = collect_query_rows( + env, + &conn_str, + "SELECT * FROM timestamp_columns;", + vec![], + Duration::from_secs(3), + Duration::from_secs(3), + Tz::UTC, + 10, + Some(1000), + ) + .unwrap(); + + debug!("Rows Count: {}", rows.len()); + for row in &rows { + if let Value::Object(map) = row { + for (key, value) in map { + debug!("{key}: {value:?}"); + } + } + } + + let Value::Object(row) = &rows[0] else { + panic!("No rows returned") + }; + + assert_eq!( + *row.get("date_col").unwrap(), + Value::Bytes(bytes::Bytes::from_static(b"2025-10-04")) + ); + assert_eq!( + *row.get("datetime_col").unwrap(), + Value::Timestamp( + chrono::Utc + .with_ymd_and_hms(2025, 10, 4, 12, 34, 56) + .unwrap() + ) + ); + assert_eq!( + *row.get("time_col").unwrap(), + Value::Bytes(bytes::Bytes::from_static(b"15:30:00")) + ); + assert_eq!( + *row.get("timestamp_col").unwrap(), + Value::Timestamp( + chrono::Utc + .with_ymd_and_hms(2025, 10, 4, 12, 34, 56) + .unwrap() + ) + ); + assert_eq!(*row.get("year_col").unwrap(), Value::Integer(2025)); +} diff --git a/src/sources/odbc/mod.rs b/src/sources/odbc/mod.rs new file mode 100644 index 0000000000000..32bb35bf84db9 --- /dev/null +++ b/src/sources/odbc/mod.rs @@ -0,0 +1,81 @@ +//! ODBC Data Source +//! +//! This data source runs a database query through the ODBC interface on the required `schedule` (cron expression). +//! Query results are fetched, converted to log events, and sent in batches bounded by `odbc_batch_size`. +//! Each row is emitted as a log event. When tracking is enabled, the final row of the result set is +//! validated and checkpointed before events are emitted, then used as a parameter for the next +//! scheduled SQL query. +//! +//! The ODBC data source offers functionality similar to the [Logstash JDBC plugin](https://www.elastic.co/docs/reference/logstash/plugins/plugins-inputs-jdbc). +//! +//! # Example +//! +//! Given the following MariaDB table and sample data: +//! +//! ```sql +//! create table odbc_table +//! ( +//! id int auto_increment primary key, +//! name varchar(255) null, +//! `datetime` datetime null +//! ); +//! +//! INSERT INTO odbc_table (name, datetime) VALUES +//! ('test1', now()), +//! ('test2', now()), +//! ('test3', now()), +//! ('test4', now()), +//! ('test5', now()); +//! ``` +//! +//! The example below shows how to connect to a MariaDB database with the ODBC driver, +//! run a query periodically, and send the results to Vector. +//! Provide a database connection string. +//! +//! ```yaml +//! sources: +//! odbc: +//! type: odbc +//! connection_string: "driver={MariaDB Unicode};server=;port=;database=;uid=;pwd=;" +//! statement: "SELECT * FROM odbc_table WHERE id > ? ORDER BY id ASC LIMIT 1;" +//! statement_init_params: +//! - name: id +//! value: "0" +//! schedule: "*/5 * * * * *" +//! schedule_timezone: UTC +//! odbc_batch_size: 100 +//! last_run_metadata_path: /path/to/odbc_tracking.json +//! tracking_columns: +//! - id +//! +//! sinks: +//! console: +//! type: console +//! inputs: +//! - odbc +//! encoding: +//! codec: json +//! ``` +//! +//! Every five seconds, the source emits one log event per result row. Column values +//! keep their Vector types where possible (for example naive `datetime` values as +//! timestamps via `odbc_default_timezone`, and `id` as an integer). Offset-bearing +//! SQL or RFC3339 timestamp text is kept as bytes so tracking parameters round-trip +//! the exact ODBC text. When encoded to JSON by a sink, the events look similar to: +//! +//! ```json +//! {"datetime":"2025-04-28T01:20:04Z","id":1,"name":"test1","source_type":"odbc","timestamp":"2025-04-28T01:50:45.075484Z"} +//! {"datetime":"2025-04-28T01:20:04Z","id":2,"name":"test2","source_type":"odbc","timestamp":"2025-04-28T01:50:50.017276Z"} +//! {"datetime":"2025-04-28T01:20:04Z","id":3,"name":"test3","source_type":"odbc","timestamp":"2025-04-28T01:50:55.016432Z"} +//! {"datetime":"2025-04-28T01:20:04Z","id":4,"name":"test4","source_type":"odbc","timestamp":"2025-04-28T01:51:00.016328Z"} +//! {"datetime":"2025-04-28T01:20:04Z","id":5,"name":"test5","source_type":"odbc","timestamp":"2025-04-28T01:51:05.010063Z"} +//! ``` + +#[cfg(feature = "sources-odbc")] +mod client; +#[cfg(feature = "sources-odbc")] +pub(crate) use client::OdbcError; +mod config; +#[cfg(all(test, feature = "odbc-integration-tests"))] +mod integration_tests; +mod schedule; diff --git a/src/sources/odbc/schedule.rs b/src/sources/odbc/schedule.rs new file mode 100644 index 0000000000000..4fa8ef34532e6 --- /dev/null +++ b/src/sources/odbc/schedule.rs @@ -0,0 +1,73 @@ +use chrono::{DateTime, Utc}; +use chrono_tz::Tz; +use cron::Schedule; +use futures::Stream; +use futures_util::stream; +use serde::{Deserialize, Serialize}; +use std::cell::RefCell; +use std::fmt; +use std::fmt::{Debug, Formatter}; +use std::str::FromStr; +use tokio::time::sleep; +use vector_config::schema::generate_string_schema; +use vector_config::{Configurable, GenerateError, Metadata, ToValue}; +use vector_config_common::schema::{SchemaGenerator, SchemaObject}; + +/// Newtype around `cron::Schedule` that enables a `Configurable` implementation. +#[derive(Clone, Serialize, Deserialize)] +#[serde(transparent)] +pub struct OdbcSchedule { + inner: Schedule, +} + +impl ToValue for OdbcSchedule { + fn to_value(&self) -> serde_json::Value { + serde_json::to_value(&self.inner) + .expect("Could not convert schedule(cron expression) to JSON") + } +} + +impl Configurable for OdbcSchedule { + fn referenceable_name() -> Option<&'static str> { + Some("cron::Schedule") + } + + fn metadata() -> Metadata { + let mut metadata = Metadata::default(); + metadata.set_description("Cron expression in seconds."); + metadata + } + + fn generate_schema(_: &RefCell) -> Result { + Ok(generate_string_schema()) + } +} + +impl From<&str> for OdbcSchedule { + fn from(s: &str) -> Self { + let schedule = Schedule::from_str(s).expect("Invalid cron expression"); + Self { inner: schedule } + } +} + +impl Debug for OdbcSchedule { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.write_str(&self.inner.to_string()) + } +} + +impl OdbcSchedule { + /// Creates a stream that asynchronously waits for each scheduled cron time. + pub(crate) fn stream(self, tz: Tz) -> impl Stream> { + let schedule = self.inner.clone(); + stream::unfold(schedule, move |schedule| async move { + let now = Utc::now().with_timezone(&tz); + let mut upcoming = schedule.upcoming(tz); + let next = upcoming.next()?; + let delay = (next - now).abs(); + + sleep(delay.to_std().unwrap_or_default()).await; + Some((next, schedule)) + }) + } +} diff --git a/tests/data/odbc/odbc-init.sh b/tests/data/odbc/odbc-init.sh new file mode 100755 index 0000000000000..403df3f8ac74a --- /dev/null +++ b/tests/data/odbc/odbc-init.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +apt update +apt install -y unixodbc odbcinst odbc-mariadb \ No newline at end of file diff --git a/tests/integration/odbc-mariadb/config/compose.yaml b/tests/integration/odbc-mariadb/config/compose.yaml new file mode 100644 index 0000000000000..211b41af825d0 --- /dev/null +++ b/tests/integration/odbc-mariadb/config/compose.yaml @@ -0,0 +1,24 @@ +version: "3" + +services: + mariadb: + image: docker.io/mariadb:${CONFIG_VERSION} + hostname: mariadb + ports: + - "3306:3306" + environment: + - MARIADB_ROOT_PASSWORD=vector + - MARIADB_USER=vector + - MARIADB_PASSWORD=vector + - MARIADB_DATABASE=vector_db + healthcheck: + test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"] + interval: 5s + timeout: 5s + retries: 12 + start_period: 30s + +networks: + default: + name: ${VECTOR_NETWORK} + external: true diff --git a/tests/integration/odbc-mariadb/config/test.yaml b/tests/integration/odbc-mariadb/config/test.yaml new file mode 100644 index 0000000000000..fdab09b630dfb --- /dev/null +++ b/tests/integration/odbc-mariadb/config/test.yaml @@ -0,0 +1,21 @@ +args: + - --test-threads + - "1" + +features: + - odbc-integration-tests + +test_filter: "::odbc::" + +env: + ODBC_DB_TYPE: "mariadb" + ODBC_CONN_STRING: "driver={MariaDB Unicode};server=mariadb;port=3306;database=vector_db;uid=vector;pwd=vector;" + +matrix: + version: ["11-jammy"] + +# changes to these files/paths will invoke the integration test in CI +# expressions are evaluated using https://github.com/micromatch/picomatch +paths: + - "src/sources/odbc/**" + - "tests/integration/odbc-mariadb/**" diff --git a/tests/integration/odbc-postgresql/config/compose.yaml b/tests/integration/odbc-postgresql/config/compose.yaml new file mode 100644 index 0000000000000..37b384b9702eb --- /dev/null +++ b/tests/integration/odbc-postgresql/config/compose.yaml @@ -0,0 +1,23 @@ +version: "3" + +services: + postgresql: + image: docker.io/postgres:${CONFIG_VERSION} + hostname: postgresql + ports: + - "5432:5432" + environment: + - POSTGRES_USER=vector + - POSTGRES_PASSWORD=vector + - POSTGRES_DB=vector_db + healthcheck: + test: ["CMD-SHELL", "pg_isready -U vector -d vector_db"] + interval: 5s + timeout: 5s + retries: 12 + start_period: 10s + +networks: + default: + name: ${VECTOR_NETWORK} + external: true diff --git a/tests/integration/odbc-postgresql/config/test.yaml b/tests/integration/odbc-postgresql/config/test.yaml new file mode 100644 index 0000000000000..3b0a6508faddd --- /dev/null +++ b/tests/integration/odbc-postgresql/config/test.yaml @@ -0,0 +1,21 @@ +args: + - --test-threads + - "1" + +features: + - odbc-integration-tests + +test_filter: "::odbc::" + +env: + ODBC_DB_TYPE: "postgresql" + ODBC_CONN_STRING: "driver={PostgreSQL Unicode};server=postgresql;port=5432;database=vector_db;uid=vector;pwd=vector;" + +matrix: + version: ["16"] + +# changes to these files/paths will invoke the integration test in CI +# expressions are evaluated using https://github.com/micromatch/picomatch +paths: + - "src/sources/odbc/**" + - "tests/integration/odbc-postgresql/**" diff --git a/website/content/en/docs/reference/configuration/sources/odbc.md b/website/content/en/docs/reference/configuration/sources/odbc.md new file mode 100644 index 0000000000000..795b401387ac0 --- /dev/null +++ b/website/content/en/docs/reference/configuration/sources/odbc.md @@ -0,0 +1,14 @@ +--- +title: ODBC +description: ODBC(Open Database Connectivity) Data Source. +component_kind: source +layout: component +tags: ["odbc", "component", "source", "database"] +--- + +{{/* +This doc is generated using: + +1. The template in layouts/docs/component.html +2. The relevant CUE data in cue/reference/components/... +*/}} diff --git a/website/cue/reference/components/sources/generated/odbc.cue b/website/cue/reference/components/sources/generated/odbc.cue new file mode 100644 index 0000000000000..acbe1c69627e8 --- /dev/null +++ b/website/cue/reference/components/sources/generated/odbc.cue @@ -0,0 +1,273 @@ +package metadata + +generated: components: sources: odbc: configuration: { + connection_string: { + description: """ + The connection string to use for ODBC. + If the `connection_string_filepath` is set, this value is ignored. + """ + required: false + type: string: { + default: "" + examples: ["driver={MariaDB Unicode};server=;port=;database=;uid=;pwd="] + } + } + connection_string_filepath: { + description: """ + The path to the file that contains the connection string. + If this is not set or the file at that path does not exist, the `connection_string` field is used instead. + """ + required: false + type: string: examples: ["/path/to/connection_string.txt"] + } + last_run_metadata_path: { + description: """ + The path to the file where tracked column values will be saved. + The tracked values are saved in JSON format and overlaid onto `statement_init_params` + for the next scheduled run. + If the file does not exist or the path is not specified, the initial values from + `statement_init_params` are used. + + When tracking is enabled, the full query result is buffered, the final-row checkpoint + is validated and written here, and only then are events emitted. A missing or + unbindable tracking value fails the poll before any emit (avoiding infinite replay). + A send failure after a successful checkpoint write may skip those rows on the next + run (at-most-once). The in-memory overlay is also advanced in that case so tracking + without `last_run_metadata_path` does not replay already-emitted rows. Prefer + incremental queries so the buffered result stays bounded. + + Parent directories are created automatically if they do not exist. + + # Examples + + If `tracking_columns = ["id", "name"]`, it is saved as the following JSON data. + + ```json + {"id":"42","name":"vector"} + ``` + """ + required: false + type: string: examples: ["/path/to/tracking.json"] + } + login_timeout: { + description: """ + Maximum time to wait for the ODBC connection/login to complete. + If the connection does not succeed within this window, the attempt fails + and is retried at the next scheduled run. + Set to 0 to disable the timeout and wait indefinitely. + Prefer a positive timeout: Vector shutdown waits for any in-flight connect/execute, and + `0` can delay exit until the ODBC driver returns. + The default is 3 seconds. + """ + required: false + type: uint: { + default: 3 + examples: [ + 3, + ] + unit: "seconds" + } + } + odbc_batch_size: { + description: """ + Number of rows to fetch, convert, and send per batch. + This bounds ODBC driver fetch buffers and in-memory processing for each batch. + Must be greater than 0. + The default is 100. + """ + required: false + type: uint: { + default: 100 + examples: [ + 100, + ] + } + } + odbc_default_timezone: { + description: """ + Timezone applied to database date/time columns that lack timezone information. + Ambiguous DST times use the latest matching instant; nonexistent times are kept as text. + Offset-bearing SQL or RFC3339 timestamp text is preserved as bytes and is not rewritten + with this timezone, so tracking parameters can round-trip the exact ODBC text. + The default is UTC. + """ + required: false + type: string: { + default: "UTC" + examples: [ + "UTC", + ] + } + } + odbc_max_str_limit: { + description: """ + Maximum bytes per cell when sizing ODBC text and binary row-set fetch buffers. + Columns that may exceed this limit are fetched individually and grow to their + full value size. Set to `0` to omit the upper bound and use driver-reported + sizes for row-set buffers instead. + The default is 4096. + """ + required: false + type: uint: { + default: 4096 + examples: [ + 4096, + ] + } + } + schedule: { + description: "Cron expression used to schedule database queries. This field is required." + required: true + type: string: {} + } + schedule_timezone: { + description: """ + The timezone to use for the `schedule`. + Typically the timezone used when evaluating the cron expression. + The default is UTC. + + [Wikipedia]: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones + """ + required: false + type: string: { + default: "UTC" + examples: [ + "UTC", + ] + } + } + statement: { + description: """ + The SQL statement to execute. + This SQL statement is executed periodically according to the `schedule`. + Defaults to `None`. If no SQL statement is provided, the source returns an error. + If the `statement_filepath` is set, this value is ignored. + """ + required: false + type: string: examples: ["SELECT * FROM users WHERE id = ?"] + } + statement_filepath: { + description: """ + The path to the file that contains the SQL statement. + If this is set, the `statement` field is ignored and the file must exist and be readable. + """ + required: false + type: string: {} + } + statement_init_params: { + description: """ + Positional parameters for SQL statement placeholders (`?`). + + Array order is the bind order. Static filter values and tracking bootstrap + values can be mixed; only names listed in `tracking_columns` are overlaid + from checkpoints or the previous result. + + # Examples + + Incremental query with a static tenant filter: + + ```yaml + sources: + odbc: + statement: "SELECT * FROM users WHERE tenant_id = ? AND id > ? ORDER BY id ASC" + statement_init_params: + - name: tenant_id + value: "acme" + - name: id + value: "0" + tracking_columns: + - id + last_run_metadata_path: /path/to/tracking.json + # The rest of the fields are omitted + ``` + + Static-only filter without tracking: + + ```yaml + sources: + odbc: + statement: "SELECT * FROM users WHERE tenant_id = ?" + statement_init_params: + - name: tenant_id + value: "acme" + # The rest of the fields are omitted + ``` + """ + required: false + type: array: items: type: object: options: { + name: { + description: """ + Parameter name. + + When the same name appears in `tracking_columns`, later runs overlay the + checkpointed or last-row value onto this entry while preserving bind order. + """ + required: true + type: string: examples: ["id", "tenant_id"] + } + value: { + description: """ + Initial value bound for this placeholder. + + For non-tracking parameters this value is reused on every scheduled run. + For tracking parameters it is used until a checkpoint or previous result + provides an updated value. + """ + required: true + type: string: examples: ["0", "acme"] + } + } + } + statement_timeout: { + description: """ + Maximum time to allow the SQL statement to run. + If the query does not finish within this window, it is canceled and retried at the next scheduled run. + Set to 0 to disable the timeout and wait indefinitely. + Prefer a positive timeout: Vector shutdown waits for any in-flight connect/execute, and + `0` can delay exit until the ODBC driver returns. + The default is 3 seconds. + """ + required: false + type: uint: { + default: 3 + examples: [ + 3, + ] + unit: "seconds" + } + } + tracking_columns: { + description: """ + Specifies the columns to track from the last row of the statement result set. + Their values overlay matching entries in `statement_init_params` on later runs while + preserving the declared bind order. + + When set, result batches are buffered until the query finishes; the final-row + checkpoint is validated (and persisted when `last_run_metadata_path` is set) before + any events are emitted. That avoids replaying the same rows when the last row is + missing a tracking column or has an unbindable value such as null. + Prefer incremental/`WHERE` bounded queries so buffering stays memory-safe. + + Requires `statement_init_params` entries whose names cover every tracking column. + Optional `last_run_metadata_path` overlays checkpointed values onto those entries. + Prefer non-binary tracking columns: checkpoints bind text parameters, so raw + `BINARY`/`VARBINARY`/`BYTEA` values that are not valid UTF-8 fail validation. + + # Examples + + ```yaml + sources: + odbc: + statement: "SELECT * FROM users WHERE id > ? ORDER BY id ASC" + statement_init_params: + - name: id + value: "0" + tracking_columns: + - id + # The rest of the fields are omitted + ``` + """ + required: false + type: array: items: type: string: examples: ["id"] + } +} diff --git a/website/cue/reference/components/sources/odbc.cue b/website/cue/reference/components/sources/odbc.cue new file mode 100644 index 0000000000000..8511db8d4ae18 --- /dev/null +++ b/website/cue/reference/components/sources/odbc.cue @@ -0,0 +1,239 @@ +package metadata + +components: sources: odbc: { + title: "ODBC" + + classes: { + delivery: "best_effort" + deployment_roles: ["daemon", "sidecar", "aggregator"] + development: "beta" + egress_method: "batch" + stateful: true + } + + features: { + auto_generated: true + acknowledgements: false + collect: { + checkpoint: enabled: true + from: { + service: services.odbc + } + } + multiline: enabled: false + } + + support: { + requirements: [ + """ + Only included in official 64-bit glibc Linux builds + (`x86_64-unknown-linux-gnu`, `aarch64-unknown-linux-gnu`) and the + official Windows archive (`x86_64-pc-windows-msvc`). It is not + included in musl builds (Alpine, distroless-static, + `*-unknown-linux-musl`), 32-bit ARM GNU builds + (`armv7-unknown-linux-gnueabihf`, `arm-unknown-linux-gnueabi`), or + official macOS archives (`aarch64-apple-darwin`). For those + targets, use a custom build with `sources-odbc`. + """, + """ + Linux glibc builds link the unixODBC driver manager (`libodbc`). + Official Debian and distroless-libc images include it; `.deb` + packages depend on `libodbc2` or `libodbc1`. Windows builds use the + system ODBC Driver Manager. A database ODBC driver must still be + installed and configured separately. + """, + ] + warnings: [ + """ + When `last_run_metadata_path` is set, the query result is buffered and the + final-row tracking checkpoint is validated and saved before any batches are + sent. If the checkpoint save succeeds but downstream delivery then fails (or + Vector restarts before delivery is complete), those rows may be skipped on the + next run (at-most-once). If the checkpoint save itself fails, previous tracking + values are kept and the next scheduled run may re-emit the same rows. This + source does not provide acknowledgements. + """, + """ + Setting `login_timeout` or `statement_timeout` to `0` disables that bound. + Shutdown still waits for any in-flight ODBC connect or execute, so `0` can + delay Vector exit until the driver returns. Prefer positive timeouts. + """, + ] + notices: [] + } + + installation: { + platform_name: null + } + + configuration: generated.components.sources.odbc.configuration + + output: { + logs: record: { + description: """ + A single row returned by the ODBC query. Each column becomes a + top-level log field and retains its Vector typed value when possible + (for example naive timestamps via `odbc_default_timezone`, integers, + booleans, and floats). Offset-bearing SQL or RFC3339 timestamp text + is emitted as bytes so tracking parameters can reuse the exact ODBC + text. Other columns that cannot be represented as a native Vector + type are also emitted as bytes. + """ + fields: { + "*": { + common: false + description: "A column from the query result set." + required: false + type: "*": {} + } + timestamp: fields._current_timestamp + } + } + } + + how_it_works: { + requirement: { + title: "Requirement for unixODBC" + body: """ + To connect to a database and execute queries via ODBC, you must have the unixODBC + driver manager available, then install and configure the appropriate ODBC driver. + See Requirements above for which official Vector builds include this source and + the driver manager. + + For example, on Debian-based Linux: + ```bash + # apt-get install unixodbc odbcinst odbc-mariadb + ``` + + You can use the `odbcinst -j` command to check the installation path and configuration files for unixODBC. + ```bash + $ odbcinst -j + unixODBC 2.3.12 + DRIVERS............: /etc/odbcinst.ini + SYSTEM DATA SOURCES: /etc/odbc.ini + FILE DATA SOURCES..: /etc/ODBCDataSources + USER DATA SOURCES..: /root/.odbc.ini + SQLULEN Size.......: 8 + SQLLEN Size........: 8 + SQLSETPOSIROW Size.: 8 + ``` + + Review the `/etc/odbcinst.ini` file in the output to ensure the ODBC driver is properly configured. + If you installed the ODBC driver via a package manager, it is usually configured automatically. + When you install the `odbc-mariadb` package, the `odbcinst.ini` file will be configured as follows: + ```bash + $ cat /etc/odbcinst.ini + + [MariaDB Unicode] + Driver=libmaodbc.so + Description=MariaDB Connector/ODBC(Unicode) + Threading=0 + UsageCount=1 + ``` + """ + } + + examples: { + title: "Example ODBC Source Configuration" + body: """ + This section walks through a simple example of configuring an ODBC data source and scheduling it. + """ + sub_sections: [ + { + title: "Step 1: Configure Test Data" + body: """ + Given the following MariaDB table and sample data: + + ```sql + create table odbc_table + ( + id int auto_increment primary key, + name varchar(255) null, + `datetime` datetime null + ); + + INSERT INTO odbc_table (name, datetime) VALUES + ('test1', now()), + ('test2', now()), + ('test3', now()), + ('test4', now()), + ('test5', now()); + ``` + """ + }, + { + title: "Step 2: Configure ODBC Source" + body: """ + The example below shows how to connect to a MariaDB database with the ODBC driver, + run a query periodically, and send the results to Vector. + Start by providing a database connection string. + + ```yaml + sources: + odbc: + type: odbc + connection_string: "driver={MariaDB Unicode};server=;port=;database=;uid=;pwd=;" + statement: "SELECT * FROM odbc_table WHERE id > ? ORDER BY id ASC LIMIT 1;" + statement_init_params: + - name: id + value: "0" + schedule: "*/5 * * * * *" + schedule_timezone: UTC + last_run_metadata_path: /path/to/odbc_tracking.json + tracking_columns: + - id + + sinks: + console: + type: console + inputs: + - odbc + encoding: + codec: json + ``` + + Every five seconds, the source emits one log event per result row. + Column values keep their Vector types when possible (for example + naive `datetime` values as timestamps via `odbc_default_timezone`, + and `id` as an integer). Offset-bearing SQL or RFC3339 timestamp + text is kept as bytes so tracking parameters round-trip the exact + ODBC text. When a sink encodes events as JSON, the output looks + similar to the following. + + ```json + {"datetime":"2025-04-28T01:20:04Z","id":1,"name":"test1","source_type":"odbc","timestamp":"2025-04-28T01:50:45.075484Z"} + {"datetime":"2025-04-28T01:20:04Z","id":2,"name":"test2","source_type":"odbc","timestamp":"2025-04-28T01:50:50.017276Z"} + {"datetime":"2025-04-28T01:20:04Z","id":3,"name":"test3","source_type":"odbc","timestamp":"2025-04-28T01:50:55.016432Z"} + {"datetime":"2025-04-28T01:20:04Z","id":4,"name":"test4","source_type":"odbc","timestamp":"2025-04-28T01:51:00.016328Z"} + {"datetime":"2025-04-28T01:20:04Z","id":5,"name":"test5","source_type":"odbc","timestamp":"2025-04-28T01:51:05.010063Z"} + ``` + """ + }, + ] + } + + timestamp_mapping: { + title: "Timestamp mapping" + body: """ + Naive date/time text from the driver is parsed to a Vector timestamp + using `odbc_default_timezone`. Timestamp text that already includes a + zone, such as SQL-style `YYYY-MM-DD HH:MM:SS+02:00` or RFC3339 values + like `2025-04-28T01:20:04Z` / `2025-04-28T01:20:04+02:00`, is preserved + as bytes. That keeps tracking-column round-trips faithful to the + original ODBC text instead of rebinding a naive local datetime that can + skip or replay rows when the database offset differs from + `odbc_default_timezone`. + """ + } + + check_license: { + title: "Check ODBC Driver License" + body: """ + Review the license information on [the official unixODBC website](\(urls.unixodbc)). + + Because ODBC drivers are supplied by various vendors, each with different license terms, + be sure to review and comply with the terms for the driver you plan to use. + """ + } + } +} diff --git a/website/cue/reference/services/odbc.cue b/website/cue/reference/services/odbc.cue new file mode 100644 index 0000000000000..45cce15567c57 --- /dev/null +++ b/website/cue/reference/services/odbc.cue @@ -0,0 +1,8 @@ +package metadata + +services: odbc: { + name: "ODBC" + thing: "an \(name) datasource" + url: urls.odbc + versions: null +} diff --git a/website/cue/reference/urls.cue b/website/cue/reference/urls.cue index 1c92e72090465..c7111b01f79c1 100644 --- a/website/cue/reference/urls.cue +++ b/website/cue/reference/urls.cue @@ -402,6 +402,7 @@ urls: { nix: "https://nixos.org/nix/" nixos: "https://nixos.org/" nixpkgs_9682: "\(github)/NixOS/nixpkgs/issues/9682" + odbc: "\(wikipedia)/wiki/Open_Database_Connectivity" openssl: "https://www.openssl.org/" openssl_conf: "https://www.openssl.org/docs/man3.1/man5/config.html" opentelemetry: "https://opentelemetry.io" @@ -548,6 +549,7 @@ urls: { uds: "\(wikipedia)/wiki/Unix_domain_socket" unicode_replacement_character: "\(wikipedia)/wiki/Specials_(Unicode_block)#Replacement_character" unicode_whitespace: "\(wikipedia)/wiki/Unicode_character_property#Whitespace" + unixodbc: "https://www.unixodbc.org/" unix_timestamp: "\(wikipedia)/wiki/Unix_time" utf8: "\(wikipedia)/wiki/UTF-8" uuidv4: "\(wikipedia)/wiki/Universally_unique_identifier#Version_4_(random)"