Add Zvec Agent

.gitfiles

@@ -64,6 +64,7 @@
 .github/helm/values/values-mirror-02.yaml
 .github/helm/values/values-profile.yaml
 .github/helm/values/values-readreplica.yaml
+.github/helm/values/values-zvec.yaml
 .github/issue_label_bot.yaml
 .github/kubelinter.yaml
 .github/labeler.yaml
@@ -460,6 +461,7 @@ charts/vald/templates/agent/sidecar/configmap.yaml
 charts/vald/templates/agent/sidecar/svc.yaml
 charts/vald/templates/agent/statefulset.yaml
 charts/vald/templates/agent/svc.yaml
+charts/vald/templates/agent/zvec/configmap.yaml
 charts/vald/templates/discoverer/clusterrole.yaml
 charts/vald/templates/discoverer/clusterrolebinding.yaml
 charts/vald/templates/discoverer/configmap.yaml
@@ -2295,6 +2297,8 @@ rust/libs/algorithms/qbg/build.rs
 rust/libs/algorithms/qbg/src/input.cpp
 rust/libs/algorithms/qbg/src/input.h
 rust/libs/algorithms/qbg/src/lib.rs
+rust/libs/algorithms/zvec/Cargo.toml
+rust/libs/algorithms/zvec/src/lib.rs
 rust/libs/build-utils/Cargo.toml
 rust/libs/build-utils/src/lib.rs
 rust/libs/kvs/Cargo.toml
@@ -2380,6 +2384,7 @@ tests/v2/e2e/assets/readreplica.yaml
 tests/v2/e2e/assets/rollout.yaml
 tests/v2/e2e/assets/stream_crud.yaml
 tests/v2/e2e/assets/unary_crud.yaml
+tests/v2/e2e/assets/zvec_crud.yaml
 tests/v2/e2e/config/config.go
 tests/v2/e2e/config/enums.go
 tests/v2/e2e/config/enums_test.go

.github/actions/detect-docker-image-tags/action.yaml

@@ -23,7 +23,7 @@ inputs:
   images:
     description: "Image names"
     required: false
-    default: "vdaas/vald-agent-ngt vdaas/vald-agent-faiss vdaas/vald-agent-sidecar vdaas/vald-discoverer-k8s vdaas/vald-lb-gateway vdaas/vald-filter-gateway vdaas/vald-mirror-gateway vdaas/vald-manager-index vdaas/vald-index-operator vdaas/vald-readreplica-rotate"
+    default: "vdaas/vald-agent vdaas/vald-agent-ngt vdaas/vald-agent-faiss vdaas/vald-agent-sidecar vdaas/vald-discoverer-k8s vdaas/vald-lb-gateway vdaas/vald-filter-gateway vdaas/vald-mirror-gateway vdaas/vald-manager-index vdaas/vald-index-operator vdaas/vald-readreplica-rotate"
 outputs:
   HELM_EXTRA_OPTIONS:
     description: "Helm extra options that specifies image tags"
@@ -39,6 +39,7 @@ runs:
       id: specify_container_versions
       run: |
         declare -A m=(
+          ["vdaas/vald-agent"]="agent.image.tag"
           ["vdaas/vald-agent-ngt"]="agent.image.tag"
           ["vdaas/vald-agent-faiss"]="agent.image.tag"
           ["vdaas/vald-agent-sidecar"]="agent.sidecar.image.tag"

.github/actions/setup-e2e/action.yaml

@@ -51,7 +51,7 @@ inputs:
   target_images:
     description: "Image names"
     required: false
-    default: "vdaas/vald-agent-ngt vdaas/vald-agent-faiss vdaas/vald-agent-sidecar vdaas/vald-discoverer-k8s vdaas/vald-lb-gateway vdaas/vald-filter-gateway vdaas/vald-mirror-gateway vdaas/vald-manager-index vdaas/vald-index-operator vdaas/vald-readreplica-rotate"
+    default: "vdaas/vald-agent vdaas/vald-agent-ngt vdaas/vald-agent-faiss vdaas/vald-agent-sidecar vdaas/vald-discoverer-k8s vdaas/vald-lb-gateway vdaas/vald-filter-gateway vdaas/vald-mirror-gateway vdaas/vald-manager-index vdaas/vald-index-operator vdaas/vald-readreplica-rotate"
 outputs:
   HELM_EXTRA_OPTIONS:
     description: "Helm extra options that specifies E2E target image tags"

.github/actions/wait-for-docker-image/action.yaml

@@ -19,7 +19,7 @@ inputs:
   images:
     description: "image names"
     required: false
-    default: "vdaas/vald-agent-ngt vdaas/vald-agent-faiss vdaas/vald-agent-sidecar vdaas/vald-discoverer-k8s vdaas/vald-lb-gateway vdaas/vald-filter-gateway vdaas/vald-mirror-gateway vdaas/vald-manager-index vdaas/vald-index-operator vdaas/vald-readreplica-rotate"
+    default: "vdaas/vald-agent vdaas/vald-agent-ngt vdaas/vald-agent-faiss vdaas/vald-agent-sidecar vdaas/vald-discoverer-k8s vdaas/vald-lb-gateway vdaas/vald-filter-gateway vdaas/vald-mirror-gateway vdaas/vald-manager-index vdaas/vald-index-operator vdaas/vald-readreplica-rotate"
 outputs: {}
 runs:
   using: "composite"

.github/helm/values/values-zvec.yaml

@@ -0,0 +1,66 @@
+#
+# Copyright (C) 2019-2026 vdaas.org vald team <vald@vdaas.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+defaults:
+  logging:
+    level: info
+  networkPolicy:
+    enabled: true
+gateway:
+  lb:
+    enabled: true
+    minReplicas: 1
+    hpa:
+      enabled: false
+    resources:
+      requests:
+        cpu: 100m
+        memory: 50Mi
+    gateway_config:
+      index_replica: 3
+agent:
+  algorithm: zvec
+  minReplicas: 3
+  maxReplicas: 3
+  podManagementPolicy: Parallel
+  hpa:
+    enabled: false
+  image:
+    repository: vdaas/vald-agent
+    pullPolicy: IfNotPresent
+  resources:
+    requests:
+      cpu: 100m
+      memory: 50Mi
+    limits:
+      memory: 50Mi
+  zvec:
+    dimension: 784
+    index_path: /var/lib/vald/zvec
+    fields:
+      - category
+    fts_fields:
+      - content
+discoverer:
+  minReplicas: 1
+  hpa:
+    enabled: false
+  resources:
+    requests:
+      cpu: 100m
+      memory: 50Mi
+manager:
+  index:
+    enabled: false

.github/workflows/dockers-agent-image.yaml

@@ -56,6 +56,7 @@ on:
       - rust/Cargo.lock
       - rust/Cargo.toml
       - rust/bin/agent
+      - rust/libs/algorithms/zvec/**
       - rust/libs/ngt-rs/**
       - rust/libs/ngt/**
       - rust/libs/proto/**
@@ -81,6 +82,7 @@ on:
       - rust/Cargo.lock
       - rust/Cargo.toml
       - rust/bin/agent
+      - rust/libs/algorithms/zvec/**
       - rust/libs/ngt-rs/**
       - rust/libs/ngt/**
       - rust/libs/proto/**

.github/workflows/e2e.v2.yaml

@@ -73,6 +73,7 @@ jobs:
               { scenario: "stream_crud", deployment: "helm-chart", cluster: "k3d", environment: "management" },
               { scenario: "stream_crud", deployment: "helm-chart", cluster: "k3d", environment: "mirror" },
               { scenario: "stream_crud", deployment: "helm-chart", cluster: "k3d", environment: "profile" },
+              { scenario: "zvec_crud", deployment: "helm-chart", cluster: "k3d", environment: "zvec", values: "zvec", insert_count: "1000", expected_index: "3000" },
               { scenario: "stream_crud", deployment: "helm-chart", cluster: "k3d", environment: "failure" },
               { scenario: "stream_crud", deployment: "helm-chart", cluster: "k3d", environment: "kill" },
               { scenario: "stream_crud", deployment: "helm-chart", cluster: "k3d", environment: "partition" },
@@ -110,6 +111,15 @@ jobs:
               core.info('Matched ' + filtered.length + ' job(s).');
             }
 
+            filtered = filtered.map(function(o) {
+              const values = o.values || ((o.environment === 'profile' || o.environment === 'faiss') ? o.environment : 'lb');
+              return Object.assign({
+                values: values,
+                insert_count: '10000',
+                expected_index: '30000',
+              }, o);
+            });
+
             // Set output "result" as a JSON string
             const matrix = { include: filtered };
             core.info('Generated matrix:\n' + JSON.stringify(matrix, null, 2));
@@ -223,6 +233,12 @@ jobs:
         if: ${{ matrix.environment == 'management' }}
         run: |
           echo "HELM_EXTRA_OPTIONS=\"--values .github/helm/values/values-index-management-jobs.yaml\"" >> $GITHUB_ENV
+      - name: Build local Rust agent image for Zvec
+        if: ${{ matrix.environment == 'zvec' && matrix.cluster == 'k3d' }}
+        shell: bash
+        run: |
+          make TAG=nightly docker/build/agent
+          k3d image import vdaas/vald-agent:nightly -c vald
       - name: Deploy Vald by Helm Chart
         if: ${{ matrix.deployment == 'helm-chart' && matrix.environment != 'mirror' && matrix.scenario != 'readreplica' }}
         uses: ./.github/actions/e2e-deploy-vald
@@ -232,8 +248,8 @@ jobs:
           # that override last so it wins over k8s/vald/manifests' own
           # `--set agent.image.repository=...-agent-ngt` (see Makefile.d/e2e.mk
           # e2e/v2/actions/run/faiss for the equivalent local dev target).
-          helm_extra_options: "${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }}${{ matrix.environment == 'faiss' && ' --set agent.image.repository=vdaas/vald-agent-faiss' || '' }}"
-          values: .github/helm/values/values-${{ (matrix.environment == 'profile' || matrix.environment == 'faiss') && matrix.environment || 'lb' }}.yaml
+          helm_extra_options: "${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }}${{ matrix.environment == 'faiss' && ' --set agent.image.repository=vdaas/vald-agent-faiss' || matrix.environment == 'zvec' && ' --set agent.image.repository=vdaas/vald-agent' || '' }}"
+          values: .github/helm/values/values-${{ matrix.values }}.yaml
           wait_for_selector: "app=vald-lb-gateway"
       - name: Deploy Vald Read Replica
         if: ${{ 'readreplica' == matrix.scenario }}
@@ -327,8 +343,8 @@ jobs:
           kubectl get pods
           make e2e/v2 E2E_CONFIG=$(pwd)/tests/v2/e2e/assets/${{ matrix.scenario }}.yaml \
             E2E_PARALLELISM="4" \
-            E2E_INSERT_COUNT="10000" \
-            E2E_EXPECTED_INDEX="30000" \
+            E2E_INSERT_COUNT="${{ matrix.insert_count }}" \
+            E2E_EXPECTED_INDEX="${{ matrix.expected_index }}" \
             E2E_QPS="30" \
             E2E_SEARCH_COUNT="10" \
             E2E_UPDATE_COUNT="100" \

Makefile.d/build.mk

@@ -132,20 +132,22 @@ example/client/client:
 	$(eval CGO_ENABLED = 1)
 	$(call go-example-build,example/client,-linkmode 'external',$(HDF5_LDFLAGS), cgo,$(HDF5_VERSION),$@)
 
+RUST_AGENT_FEATURES ?= qbg,zvec
+
 rust/target/release/agent:
 	pushd rust && \
 	$(CC_ENV_VARS) \
 	OPENSSL_STATIC=1 \
 	PKG_CONFIG_ALL_STATIC=1 \
-	cargo build -p agent --release && \
+	cargo build -p agent --no-default-features --features "$(RUST_AGENT_FEATURES)" --release && \
 	popd
 
 rust/target/debug/agent:
 	pushd rust && \
 	$(CC_ENV_VARS) \
 	OPENSSL_STATIC=1 \
 	PKG_CONFIG_ALL_STATIC=1 \
-	cargo build -p agent && \
+	cargo build -p agent --no-default-features --features "$(RUST_AGENT_FEATURES)" && \
 	popd
 
 tests/v2/e2e/e2e:

Makefile.d/test.mk

@@ -203,6 +203,7 @@ test/cmd: \
 ## run tests for rust
 test/rust: \
 	test/rust/qbg \
+	test/rust/zvec \
 	test/rust/agent
 
 .PHONY: test/rust/qbg
@@ -219,11 +220,17 @@ test/rust/qbg:
 	cargo test --manifest-path rust/Cargo.toml --package qbg --lib -- tests::test_index --exact --show-output
 	rm -rf rust/libs/algorithms/qbg/index/
 
+.PHONY: test/rust/zvec
+## run tests for zvec
+test/rust/zvec:
+	$(CC_ENV_VARS) \
+	cargo test --manifest-path rust/Cargo.toml --package zvec --lib -- --show-output
+
 .PHONY: test/rust/agent
 ## run tests for agent
 test/rust/agent:
 	$(CC_ENV_VARS) \
-	cargo test --manifest-path rust/Cargo.toml --package agent -- handler::common::tests --show-output
+	cargo test --manifest-path rust/Cargo.toml --package agent --no-default-features --features "$(RUST_AGENT_FEATURES)" -- handler::common::tests --show-output
 
 .PHONY: test/hack
 ## run tests for hack

charts/vald/templates/agent/zvec/configmap.yaml

@@ -0,0 +1,46 @@
+#
+# Copyright (C) 2019-2026 vdaas.org vald team <vald@vdaas.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+{{- $agent := .Values.agent -}}
+{{- if and ($agent.enabled) (eq (lower $agent.algorithm) "zvec")}}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ $agent.name }}-config
+  labels:
+    app.kubernetes.io/name: {{ include "vald.name" . }}
+    helm.sh/chart: {{ include "vald.chart" . }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/version: {{ .Chart.Version }}
+    app.kubernetes.io/component: agent
+data:
+  config.yaml: |
+    ---
+    algorithm: zvec
+    version: {{ $agent.version }}
+    time_zone: {{ default .Values.defaults.time_zone $agent.time_zone }}
+    logging:
+      {{- $logging := dict "Values" $agent.logging "default" .Values.defaults.logging }}
+      {{- include "vald.logging" $logging | nindent 6 }}
+    server_config:
+      {{- $servers := dict "Values" $agent.server_config "default" .Values.defaults.server_config }}
+      {{- include "vald.servers" $servers | nindent 6 }}
+    observability:
+      {{- $observability := dict "Values" $agent.observability "default" .Values.defaults.observability }}
+      {{- include "vald.observability" $observability | nindent 6 }}
+    zvec:
+      {{- toYaml $agent.zvec | nindent 6 }}
+{{- end }}

charts/vald/values.schema.json

@@ -80,8 +80,8 @@
         },
         "algorithm": {
           "type": "string",
-          "description": "agent algorithm type. it should be `ngt` or `faiss`.",
-          "enum": ["ngt", "faiss"]
+          "description": "agent algorithm type. it should be `ngt`, `faiss` or `zvec`.",
+          "enum": ["ngt", "faiss", "zvec"]
         },
         "annotations": {
           "type": "object",
@@ -640,6 +640,34 @@
             "name": "config"
           }
         },
+        "zvec": {
+          "type": "object",
+          "properties": {
+            "dimension": {
+              "type": "integer",
+              "description": "vector dimension",
+              "minimum": 1
+            },
+            "index_path": {
+              "type": "string",
+              "description": "Zvec collection storage path"
+            },
+            "fields": {
+              "type": "array",
+              "description": "string fields stored in the Zvec collection",
+              "items": {
+                "type": "string"
+              }
+            },
+            "fts_fields": {
+              "type": "array",
+              "description": "full-text indexed string fields stored in the Zvec collection",
+              "items": {
+                "type": "string"
+              }
+            }
+          }
+        },
         "persistentVolume": {
           "type": "object",
           "properties": {

charts/vald/values.yaml

@@ -1998,9 +1998,9 @@ agent:
   # @schema {"name": "agent.version", "alias": "version"}
   # agent.version -- version of agent config
   version: v0.0.0
-  # @schema {"name": "agent.algorithm", "type": "string", "enum": ["ngt", "faiss"]}
+  # @schema {"name": "agent.algorithm", "type": "string", "enum": ["ngt", "faiss", "zvec"]}
   # agent.algorithm -- agent algorithm type.
-  # it should be `ngt` or `faiss`.
+  # it should be `ngt`, `faiss` or `zvec`.
   algorithm: ngt
   # @schema {"name": "agent.time_zone", "type": "string"}
   # agent.time_zone -- Time zone
@@ -2468,6 +2468,22 @@ agent:
       # @schema {"name": "agent.faiss.kvsdb.concurrency", "type": "integer"}
       # agent.faiss.kvsdb.concurrency -- kvsdb processing concurrency
       concurrency: 6
+  # @schema {"name": "agent.zvec", "type": "object"}
+  zvec:
+    # @schema {"name": "agent.zvec.dimension", "type": "integer", "minimum": 1}
+    # agent.zvec.dimension -- vector dimension
+    dimension: 4096
+    # @schema {"name": "agent.zvec.index_path", "type": "string"}
+    # agent.zvec.index_path -- Zvec collection storage path
+    index_path: /var/lib/vald/zvec
+    # @schema {"name": "agent.zvec.fields", "type": "array", "items": {"type": "string"}}
+    # agent.zvec.fields -- string fields stored in the Zvec collection
+    fields:
+      - category
+    # @schema {"name": "agent.zvec.fts_fields", "type": "array", "items": {"type": "string"}}
+    # agent.zvec.fts_fields -- full-text indexed string fields stored in the Zvec collection
+    fts_fields:
+      - content
   # @schema {"name": "agent.sidecar", "type": "object"}
   sidecar:
     # @schema {"name": "agent.sidecar.enabled", "type": "boolean"}