diff --git a/ramenctl/ramenctl/config.py b/ramenctl/ramenctl/config.py index f3b0b2f917..8f84257ee3 100644 --- a/ramenctl/ramenctl/config.py +++ b/ramenctl/ramenctl/config.py @@ -25,7 +25,7 @@ def run(args): cloud_secret = generate_cloud_credentials_secret(env["clusters"][0], args) if env["hub"]: - hub_cm = generate_config_map("hub", env["clusters"], args) + hub_cm = generate_config_map("hub", env, args) wait_for_ramen_hub_operator(env["hub"], args) @@ -38,7 +38,7 @@ def run(args): wait_for_dr_clusters(env["hub"], env["clusters"], args) wait_for_dr_policy(env["hub"], args) else: - dr_cluster_cm = generate_config_map("dr-cluster", env["clusters"], args) + dr_cluster_cm = generate_config_map("dr-cluster", env, args) for cluster in env["clusters"]: create_ramen_s3_secrets(cluster, s3_secrets) @@ -89,7 +89,9 @@ def create_cloud_credentials_secret(cluster, yaml): kubectl.apply("--filename=-", input=yaml, context=cluster, log=command.debug) -def generate_config_map(controller, clusters, args): +def generate_config_map(controller, env, args): + clusters = env["clusters"] + volsync = env["features"].get("volsync", True) template = drenv.template(command.resource("configmap.yaml")) return template.substitute( name=f"ramen-{controller}-operator-config", @@ -98,6 +100,7 @@ def generate_config_map(controller, clusters, args): cluster2=clusters[1], minio_url_cluster1=minio.service_url(clusters[0]), minio_url_cluster2=minio.service_url(clusters[1]), + volsync_disabled="false" if volsync else "true", namespace=args.ramen_namespace, ) @@ -145,7 +148,6 @@ def wait_for_dr_clusters(hub, clusters, args): drenv.wait_for( f"drcluster/{name}", output="jsonpath={.status.phase}", - namespace=args.ramen_namespace, timeout=180, profile=hub, log=command.debug, @@ -156,7 +158,6 @@ def wait_for_dr_clusters(hub, clusters, args): "drcluster", "--all", "--for=jsonpath={.status.phase}=Available", - f"--namespace={args.ramen_namespace}", context=hub, log=command.debug, ) @@ -167,7 +168,6 @@ def wait_for_dr_policy(hub, args): kubectl.wait( "drpolicy/dr-policy", "--for=condition=Validated", - f"--namespace={args.ramen_namespace}", context=hub, log=command.debug, ) diff --git a/ramenctl/ramenctl/resources/configmap.yaml b/ramenctl/ramenctl/resources/configmap.yaml index c82136f1f7..53b2b7fb44 100644 --- a/ramenctl/ramenctl/resources/configmap.yaml +++ b/ramenctl/ramenctl/resources/configmap.yaml @@ -34,6 +34,8 @@ data: clusterServiceVersionName: ramen-dr-cluster-operator.v0.0.1 kubeObjectProtection: veleroNamespaceName: velero + volSync: + disabled: $volsync_disabled s3StoreProfiles: - s3ProfileName: minio-on-$cluster1 s3Bucket: bucket diff --git a/test/Makefile b/test/Makefile index 95cb69b563..75ea8d9a4f 100644 --- a/test/Makefile +++ b/test/Makefile @@ -15,6 +15,8 @@ sources := $(wildcard \ addons/*/start \ addons/*/test \ addons/*/stop \ + */enable-dr \ + */disable-dr \ */deploy \ */undeploy \ */failover \ diff --git a/test/README.md b/test/README.md index 7f52c62e47..2d13d797f9 100644 --- a/test/README.md +++ b/test/README.md @@ -466,6 +466,10 @@ $ drenv delete envs/example.yaml - `extra_config`: List of extra config key=value. Each item adds `--extra-config` minikube option. See `minikube start --help` to see the possible keys and values. + - `feature_gates`: List of Kubernetes feature gates key=value. Each + item adds `--feature-gates` minikube option. See + [Feature Gates](https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/) + for possible keys and values. - `containerd`: Optional containerd configuration object. See `containerd config default` for available options. - `workers`: Optional list of workers to run when starting a diff --git a/test/addons/cdi/cr/kustomization.yaml b/test/addons/cdi/cr/kustomization.yaml index 3fd3d08eb5..4aa6bdfe56 100644 --- a/test/addons/cdi/cr/kustomization.yaml +++ b/test/addons/cdi/cr/kustomization.yaml @@ -4,4 +4,4 @@ # yamllint disable rule:line-length --- resources: -- https://github.com/kubevirt/containerized-data-importer/releases/download/v1.57.0/cdi-cr.yaml +- https://github.com/kubevirt/containerized-data-importer/releases/download/v1.58.0/cdi-cr.yaml diff --git a/test/addons/cdi/disk/source.yaml b/test/addons/cdi/disk/source.yaml index cdd40c0646..c320d64541 100644 --- a/test/addons/cdi/disk/source.yaml +++ b/test/addons/cdi/disk/source.yaml @@ -9,4 +9,4 @@ metadata: spec: source: registry: - url: "docker://quay.io/alitke/cirros:latest" + url: "docker://quay.io/nirsof/cirros:0.6.2-1" diff --git a/test/addons/cdi/operator/kustomization.yaml b/test/addons/cdi/operator/kustomization.yaml index b894cb9790..208e01b464 100644 --- a/test/addons/cdi/operator/kustomization.yaml +++ b/test/addons/cdi/operator/kustomization.yaml @@ -4,4 +4,4 @@ # yamllint disable rule:line-length --- resources: -- https://github.com/kubevirt/containerized-data-importer/releases/download/v1.57.0/cdi-operator.yaml +- https://github.com/kubevirt/containerized-data-importer/releases/download/v1.58.0/cdi-operator.yaml diff --git a/test/addons/cdi/start b/test/addons/cdi/start index 38193dafdb..3ff72595ef 100755 --- a/test/addons/cdi/start +++ b/test/addons/cdi/start @@ -3,6 +3,7 @@ # SPDX-FileCopyrightText: The RamenDR authors # SPDX-License-Identifier: Apache-2.0 +import json import os import sys @@ -27,6 +28,19 @@ def deploy(cluster): print("Deploying cdi cr") kubectl.apply("--kustomize=cr", context=cluster) + print("Configuring insecure local registry") + patch = { + "spec": {"config": {"insecureRegistries": ["host.minikube.internal:5000"]}} + } + kubectl.patch( + "cdi", + "cdi", + "--patch", + json.dumps(patch), + "--type=merge", + context=cluster, + ) + def wait(cluster): print("Waiting until cdi cr is available") @@ -37,6 +51,14 @@ def wait(cluster): "--timeout=300s", context=cluster, ) + print("Waiting until cdi cr finished progressing") + kubectl.wait( + "cdi.cdi.kubevirt.io/cdi", + "--for=condition=progressing=False", + f"--namespace={NAMESPACE}", + "--timeout=300s", + context=cluster, + ) if len(sys.argv) != 2: diff --git a/test/addons/kubevirt/cr/kustomization.yaml b/test/addons/kubevirt/cr/kustomization.yaml index 2f02b94a99..2927d5293e 100644 --- a/test/addons/kubevirt/cr/kustomization.yaml +++ b/test/addons/kubevirt/cr/kustomization.yaml @@ -4,4 +4,4 @@ # yamllint disable rule:line-length --- resources: -- https://github.com/kubevirt/kubevirt/releases/download/v1.0.1/kubevirt-cr.yaml +- https://github.com/kubevirt/kubevirt/releases/download/v1.1.1/kubevirt-cr.yaml diff --git a/test/addons/kubevirt/operator/kustomization.yaml b/test/addons/kubevirt/operator/kustomization.yaml index 2846de42f8..0be84e4902 100644 --- a/test/addons/kubevirt/operator/kustomization.yaml +++ b/test/addons/kubevirt/operator/kustomization.yaml @@ -4,4 +4,4 @@ # yamllint disable rule:line-length --- resources: -- https://github.com/kubevirt/kubevirt/releases/download/v1.0.1/kubevirt-operator.yaml +- https://github.com/kubevirt/kubevirt/releases/download/v1.1.1/kubevirt-operator.yaml diff --git a/test/basic-test/config.yaml b/test/basic-test/config.yaml index ca31c93a75..e56ff0f00b 100644 --- a/test/basic-test/config.yaml +++ b/test/basic-test/config.yaml @@ -2,7 +2,10 @@ # SPDX-License-Identifier: Apache-2.0 --- -repo: https://github.com/ramendr/ocm-ramen-samples.git -branch: main -name: busybox-sample -namespace: busybox-sample +repo: https://github.com/nirs/ocm-ramen-samples.git +path: k8s/busybox-regional-rbd-deploy/sub +branch: test +name: busybox-regional-rbd-deploy +namespace: busybox-regional-rbd-deploy +dr_policy: busybox-regional-rbd-deploy +pvc_label: busybox diff --git a/test/basic-test/undeploy b/test/basic-test/undeploy index 7fa04aca12..950bf4768c 100755 --- a/test/basic-test/undeploy +++ b/test/basic-test/undeploy @@ -8,6 +8,6 @@ from drenv import test test.start("undeploy", __file__) args = test.parse_args() -test.info("Deleting busybox example application") +test.info("Deleting application") test.undeploy() -test.info("Application was undeployed successfully") +test.info("Application was deleted") diff --git a/test/configs/k8s/busybox-regional-rbd-deploy.yaml b/test/configs/k8s/busybox-regional-rbd-deploy.yaml new file mode 100644 index 0000000000..e56ff0f00b --- /dev/null +++ b/test/configs/k8s/busybox-regional-rbd-deploy.yaml @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +repo: https://github.com/nirs/ocm-ramen-samples.git +path: k8s/busybox-regional-rbd-deploy/sub +branch: test +name: busybox-regional-rbd-deploy +namespace: busybox-regional-rbd-deploy +dr_policy: busybox-regional-rbd-deploy +pvc_label: busybox diff --git a/test/configs/k8s/busybox-regional-rbd-ds.yaml b/test/configs/k8s/busybox-regional-rbd-ds.yaml new file mode 100644 index 0000000000..9fa5e58b6a --- /dev/null +++ b/test/configs/k8s/busybox-regional-rbd-ds.yaml @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +repo: https://github.com/nirs/ocm-ramen-samples.git +path: k8s/busybox-regional-rbd-ds/sub +branch: test +name: busybox-regional-rbd-ds +namespace: busybox-regional-rbd-ds +dr_policy: busybox-regional-rbd-ds +pvc_label: busybox diff --git a/test/configs/k8s/busybox-regional-rbd-sts.yaml b/test/configs/k8s/busybox-regional-rbd-sts.yaml new file mode 100644 index 0000000000..ec600300a1 --- /dev/null +++ b/test/configs/k8s/busybox-regional-rbd-sts.yaml @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +repo: https://github.com/nirs/ocm-ramen-samples.git +path: k8s/busybox-regional-rbd-sts/sub +branch: test +name: busybox-regional-rbd-sts +namespace: busybox-regional-rbd-sts +dr_policy: busybox-regional-rbd-sts +pvc_label: busybox diff --git a/test/configs/k8s/kubevirt-pvc.yaml b/test/configs/k8s/kubevirt-pvc.yaml new file mode 100644 index 0000000000..f6d71ef550 --- /dev/null +++ b/test/configs/k8s/kubevirt-pvc.yaml @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +repo: https://github.com/nirs/ocm-kubevirt-samples.git +path: subscription +branch: test +name: kubevirt-sample +namespace: kubevirt-sample +dr_policy: kubevirt-sample +pvc_label: vm-standalone-dv-odr-regional diff --git a/test/configs/odr/busybox-metro-rbd-deploy.yaml b/test/configs/odr/busybox-metro-rbd-deploy.yaml new file mode 100644 index 0000000000..3c815190ba --- /dev/null +++ b/test/configs/odr/busybox-metro-rbd-deploy.yaml @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +repo: https://github.com/nirs/ocm-ramen-samples.git +path: odr/busybox-metro-rbd-deploy/sub +branch: test-odr +name: busybox-metro-rbd-deploy +namespace: busybox-metro-rbd-deploy +dr_policy: busybox-metro-rbd-deploy +pvc_label: busybox diff --git a/test/configs/odr/busybox-regional-cephfs-deploy.yaml b/test/configs/odr/busybox-regional-cephfs-deploy.yaml new file mode 100644 index 0000000000..a45e949d5f --- /dev/null +++ b/test/configs/odr/busybox-regional-cephfs-deploy.yaml @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +repo: https://github.com/nirs/ocm-ramen-samples.git +path: odr/busybox-regional-cephfs-deploy/sub +branch: test-odr +name: busybox-regional-cephfs-deploy +namespace: busybox-regional-cephfs-deploy +dr_policy: busybox-regional-cephfs-deploy +pvc_label: busybox diff --git a/test/configs/odr/busybox-regional-rbd-deploy.yaml b/test/configs/odr/busybox-regional-rbd-deploy.yaml new file mode 100644 index 0000000000..686c50e9ef --- /dev/null +++ b/test/configs/odr/busybox-regional-rbd-deploy.yaml @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +repo: https://github.com/nirs/ocm-ramen-samples.git +path: odr/busybox-regional-rbd-deploy/sub +branch: test-odr +name: busybox-regional-rbd-deploy +namespace: busybox-regional-rbd-deploy +dr_policy: busybox-regional-rbd-deploy +pvc_label: busybox diff --git a/test/drenv/__main__.py b/test/drenv/__main__.py index 779cae63c4..08a993ba11 100644 --- a/test/drenv/__main__.py +++ b/test/drenv/__main__.py @@ -30,7 +30,16 @@ def main(): p = argparse.ArgumentParser(prog="drenv") p.add_argument("-v", "--verbose", action="store_true", help="Be more verbose") p.add_argument( - "--skip-tests", dest="run_tests", action="store_false", help="Skip self tests" + "--skip-tests", + dest="run_tests", + action="store_false", + help="Skip addons 'test' hooks", + ) + p.add_argument( + "--skip-addons", + dest="run_addons", + action="store_false", + help="Skip addons 'start' and 'stop' hooks", ) p.add_argument("command", choices=commands, help="Command to run") p.add_argument("--name-prefix", help="Prefix profile names") @@ -52,7 +61,12 @@ def main(): def cmd_start(env, args): start = time.monotonic() logging.info("[%s] Starting environment", env["name"]) - hooks = ["start", "test"] if args.run_tests else ["start"] + + hooks = [] + if args.run_addons: + hooks.append("start") + if args.run_tests: + hooks.append("test") # Delaying `minikube start` ensures cluster start order. execute( @@ -62,7 +76,9 @@ def cmd_start(env, args): hooks=hooks, verbose=args.verbose, ) - execute(run_worker, env["workers"], hooks=hooks) + + if hooks: + execute(run_worker, env["workers"], hooks=hooks) if "ramen" in env: ramen.dump_e2e_config(env) @@ -77,7 +93,8 @@ def cmd_start(env, args): def cmd_stop(env, args): start = time.monotonic() logging.info("[%s] Stopping environment", env["name"]) - execute(stop_cluster, env["profiles"]) + hooks = ["stop"] if args.run_addons else [] + execute(stop_cluster, env["profiles"], hooks=hooks) logging.info( "[%s] Environment stopped in %.2f seconds", env["name"], @@ -147,17 +164,18 @@ def start_cluster(profile, hooks=(), verbose=False, **options): if is_restart: wait_for_deployments(profile) - execute(run_worker, profile["workers"], hooks=hooks) + if hooks: + execute(run_worker, profile["workers"], hooks=hooks) -def stop_cluster(profile, **options): +def stop_cluster(profile, hooks=(), **options): cluster_status = cluster.status(profile["name"]) - if cluster_status == cluster.READY: + if cluster_status == cluster.READY and hooks: execute( run_worker, profile["workers"], - hooks=["stop"], + hooks=hooks, reverse=True, allow_failure=True, ) @@ -207,6 +225,7 @@ def start_minikube_cluster(profile, verbose=False): addons=profile["addons"], service_cluster_ip_range=profile["service_cluster_ip_range"], extra_config=profile["extra_config"], + feature_gates=profile["feature_gates"], alsologtostderr=verbose, ) diff --git a/test/drenv/envfile.py b/test/drenv/envfile.py index e132fc5998..6e9c39ccc5 100644 --- a/test/drenv/envfile.py +++ b/test/drenv/envfile.py @@ -106,6 +106,7 @@ def _validate_profile(profile, addons_root): profile.setdefault("ser", []) profile.setdefault("service_cluster_ip_range", None) profile.setdefault("extra_config", []) + profile.setdefault("feature_gates", []) profile.setdefault("containerd", None) profile.setdefault("workers", []) diff --git a/test/drenv/minikube.py b/test/drenv/minikube.py index 411bab751c..ee95ef5bdd 100644 --- a/test/drenv/minikube.py +++ b/test/drenv/minikube.py @@ -43,6 +43,7 @@ def start( addons=(), service_cluster_ip_range=None, extra_config=None, + feature_gates=None, alsologtostderr=False, ): args = [] @@ -77,6 +78,10 @@ def start( for pair in extra_config: args.extend(("--extra-config", pair)) + if feature_gates: + # Unlike --extra-config this requires one comma separated value. + args.extend(("--feature-gates", ",".join(feature_gates))) + if alsologtostderr: args.append("--alsologtostderr") diff --git a/test/drenv/test.py b/test/drenv/test.py index 34be0c6766..b6f674a231 100644 --- a/test/drenv/test.py +++ b/test/drenv/test.py @@ -27,8 +27,8 @@ ) -def start(name, file, config_file="config.yaml"): - global workdir, config, parser, log +def start(name, file): + global workdir, parser, log # Setting up logging and sys.excepthook must be first so any failure will # be reported using the logger. @@ -42,10 +42,6 @@ def start(name, file, config_file="config.yaml"): # Working directory for runing the test. workdir = os.path.abspath(os.path.dirname(file)) - config_path = os.path.join(workdir, config_file) - with open(config_path) as f: - config = yaml.safe_load(f) - parser = argparse.ArgumentParser(name) parser.add_argument( "-v", @@ -57,6 +53,12 @@ def start(name, file, config_file="config.yaml"): "--name-prefix", help="Prefix profile names", ) + parser.add_argument( + "-c", + "--config", + default=os.path.join(workdir, "config.yaml"), + help="Test configuration file", + ) parser.add_argument( "filename", help="Environment filename", @@ -68,13 +70,16 @@ def add_argument(*args, **kw): def parse_args(): - global env + global config, env args = parser.parse_args() if args.verbose: log.setLevel(logging.DEBUG) debug("Parsed arguments: %s", args) + with open(args.config) as f: + config = yaml.safe_load(f) + env = ramen.env_info(args.filename, name_prefix=args.name_prefix) debug("Using environment: %s", env) @@ -96,19 +101,50 @@ def _excepthook(t, v, tb): log.exception("test failed", exc_info=(t, v, tb)) +def _deploy_dr_policy(): + dr_policy = f""" +apiVersion: ramendr.openshift.io/v1alpha1 +kind: DRPolicy +metadata: + name: {config['dr_policy']} +spec: + drClusters: + - {env['clusters'][0]} + - {env['clusters'][1]} + schedulingInterval: 1m +""" + info("Deploying drpolicy '%s'", config["dr_policy"]) + kubectl.apply("--filename=-", input=dr_policy, context=env["hub"], log=debug) + + info("Waiting until drpolicy '%s' is validated", config["dr_policy"]) + kubectl.wait( + "drpolicy", + config["dr_policy"], + "--for=condition=Validated", + context=env["hub"], + log=debug, + ) + + +def _delete_dr_policy(): + info("Deleting drpolicy '%s'", config["dr_policy"]) + kubectl.delete( + "drpolicy", + config["dr_policy"], + "--ignore-not-found", + context=env["hub"], + log=debug, + ) + + def deploy(): """ Deploy application on cluster. """ - info("Deploying channel") - kubectl.apply( - f"--kustomize={config['repo']}/channel?ref={config['branch']}", - context=env["hub"], - log=debug, - ) - info("Deploying subscription based application") + _deploy_dr_policy() + info("Deploying application '%s'", config["name"]) kubectl.apply( - f"--kustomize={config['repo']}/subscription?ref={config['branch']}", + f"--kustomize={config['repo']}/{config['path']}?ref={config['branch']}", context=env["hub"], log=debug, ) @@ -118,20 +154,14 @@ def undeploy(): """ Undeploy an application. """ - info("Undeploying channel") - kubectl.delete( - f"--kustomize={config['repo']}/channel?ref={config['branch']}", - "--ignore-not-found", - context=env["hub"], - log=debug, - ) - info("Undeploying subscription based application") + info("Undeploying application '%s'", config["name"]) kubectl.delete( - f"--kustomize={config['repo']}/subscription?ref={config['branch']}", + f"--kustomize={config['repo']}/{config['path']}?ref={config['branch']}", "--ignore-not-found", context=env["hub"], log=debug, ) + _delete_dr_policy() def enable_dr(): @@ -158,20 +188,20 @@ def enable_dr(): apiVersion: ramendr.openshift.io/v1alpha1 kind: DRPlacementControl metadata: - name: busybox-drpc + name: {config['name']}-drpc namespace: {config['namespace']} labels: app: {config['name']} spec: preferredCluster: {cluster} drPolicyRef: - name: dr-policy + name: {config['dr_policy']} placementRef: kind: Placement name: {placement_name} pvcSelector: matchLabels: - appname: busybox + appname: {config['pvc_label']} """ kubectl.apply("--filename=-", input=drpc, context=env["hub"], log=debug) diff --git a/test/drtest b/test/drtest new file mode 100755 index 0000000000..51ecfdd85e --- /dev/null +++ b/test/drtest @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +import argparse +import collections +import concurrent.futures +import logging +import os +import subprocess + +import yaml + + +def main(): + p = argparse.ArgumentParser("drtest") + p.add_argument("-k", "--kubeconfig", help="If set, use speified kubeconfig path") + p.add_argument("-o", "--outdir", default=".", help="Output directory") + p.add_argument("-v", "--verbose", action="store_true", help="Be more verbose") + p.add_argument("suite", help="Test suite file") + p.add_argument("env", help="Environemnt file") + args = p.parse_args() + + logging.basicConfig( + level=logging.DEBUG if args.verbose else logging.INFO, + format="%(asctime)s %(levelname)-7s %(message)s", + ) + + with open(args.suite) as f: + suite = yaml.safe_load(f) + + os.makedirs(args.outdir, exist_ok=True) + + logging.info("Running '%s'", suite["name"]) + logging.info("Storing output to '%s'", args.outdir) + + stats = collections.Counter({"pass": 0, "fail": 0}) + + with concurrent.futures.ThreadPoolExecutor() as e: + futures = {} + + for test in suite["tests"]: + logging.info("Starting test '%s'", test["name"]) + f = e.submit(run_test, test, args) + futures[f] = test["name"] + + for f in concurrent.futures.as_completed(futures): + try: + f.result() + except Exception: + logging.error("Test '%s' FAIL", futures[f]) + stats["fail"] += 1 + else: + logging.info("Test '%s' PASS", futures[f]) + stats["pass"] += 1 + + logging.info( + "%s (%s pass, %s fail)", + "PASS" if stats["fail"] == 0 else "FAIL", + stats["pass"], + stats["fail"], + ) + + +def run_test(test, args): + log = os.path.join(args.outdir, test["name"] + ".log") + cmd = [test["command"], "--config", test["config"], args.env] + if args.kubeconfig: + env = dict(os.environ) + env["KUBECONFIG"] = args.kubeconfig + else: + env = None + with open(log, "w") as f: + subprocess.run(cmd, stderr=f, check=True, env=env) + + +if __name__ == "__main__": + main() diff --git a/test/envs/regional-dr-hubless.yaml b/test/envs/regional-dr-hubless.yaml index fff2a57f74..2928f7747a 100644 --- a/test/envs/regional-dr-hubless.yaml +++ b/test/envs/regional-dr-hubless.yaml @@ -9,6 +9,8 @@ ramen: hub: null clusters: [dr1, dr2] topology: regional-dr + features: + volsync: true templates: - name: "dr-cluster" diff --git a/test/envs/regional-dr-kubevirt.yaml b/test/envs/regional-dr-kubevirt.yaml index eeb7fdbced..a45eca833e 100644 --- a/test/envs/regional-dr-kubevirt.yaml +++ b/test/envs/regional-dr-kubevirt.yaml @@ -9,6 +9,8 @@ ramen: hub: hub clusters: [dr1, dr2] topology: regional-dr + features: + volsync: false templates: - name: "dr-cluster" @@ -25,7 +27,6 @@ templates: extra_disks: 1 disk_size: "50g" addons: - - volumesnapshots - csi-hostpath-driver workers: - addons: @@ -57,8 +58,6 @@ templates: - name: ocm-controller - name: cert-manager - name: olm - - name: submariner - args: ["hub", "dr1", "dr2"] profiles: - name: "dr1" @@ -72,6 +71,3 @@ workers: - addons: - name: rbd-mirror args: ["dr1", "dr2"] - - addons: - - name: volsync - args: ["dr1", "dr2"] diff --git a/test/envs/regional-dr.yaml b/test/envs/regional-dr.yaml index 66ea1bfd13..74c1646621 100644 --- a/test/envs/regional-dr.yaml +++ b/test/envs/regional-dr.yaml @@ -9,6 +9,8 @@ ramen: hub: hub clusters: [dr1, dr2] topology: regional-dr + features: + volsync: true templates: - name: "dr-cluster" @@ -21,6 +23,8 @@ templates: addons: - volumesnapshots - csi-hostpath-driver + feature_gates: + - StatefulSetAutoDeletePVC=true workers: - addons: - name: cert-manager diff --git a/test/gitlap/README.md b/test/gitlap/README.md new file mode 100644 index 0000000000..d71460c68e --- /dev/null +++ b/test/gitlap/README.md @@ -0,0 +1,88 @@ +# Setting up a local git server + +## Initial setup + +1. Install lighttpd + + ``` + sudo dnf install lighttpd + ``` + +1. Create the git repo + + Create a directory where the git repositories will be served: + + ``` + sudo mkdir /var/www/gitlap + cd /var/www/gitlap + sudo git clone --bare https://github.com/nirs/ocm-kubevirt-samples.git + ``` + + Set git repo permissions so you can push changes, and the web server + can serve the repo. + + ``` + sudo chown -R $USER:lighttpd /var/www/gitlap + ``` + +1. Copy the vhost configuration + + ``` + sudo cp gitlap.conf /etc/lighttpd/vhosts.d/ + ``` + +1. Uncomment the vhost include in /etc/lighttpd/lighttpd.conf + + ``` + include conf_dir + "/vhosts.d/*.conf" + ``` + +1. Enable and start the service + + ``` + sudo systemctl enable --now lighttpd + ``` + +1. Allow http access in the libvirt zone + + ``` + sudo firewall-cmd --zone=libvirt --add-service=http --permanent + sudo firewall-cmd --reload + ``` + +## Testing the server + +1. Add entry in /etc/hosts for testing locally + + ``` + 192.168.122.1 host.minikube.internal + ``` + +1. Check that git clone works + + ``` + git clone http://host.minikube.internal/ocm-kubevirt-samples.git + rm -rf ocm-kubevirt-samples + ``` + +1. Check git clone in a minikube cluster + + ``` + minikube ssh -p dr1 + git clone http://host.minikube.internal/ocm-kubevirt-samples.git + rm -rf ocm-kubevirt-samples + ``` + +## Updating the git repo + +1. Add a remote to your working repo + + ``` + git remote add gitlap file:///var/www/gitlap/ocm-kubevirt-samples.git + ``` + +1. Push changes to the remote + + ``` + git push -f gitlap main + ``` diff --git a/test/gitlap/gitlap.conf b/test/gitlap/gitlap.conf new file mode 100644 index 0000000000..bfe98fc706 --- /dev/null +++ b/test/gitlap/gitlap.conf @@ -0,0 +1,17 @@ +# Minimal configuration for local git server for minikube clusters. +# +# For more options see: +# https://redmine.lighttpd.net/projects/lighttpd/wiki/How_to_set_up_a_git_server_over_http(s) + +server.modules += ("mod_setenv", "mod_cgi", "mod_alias") + +# `host.minikube.internal` is a special DNS name injected by minikube to all clusters. +# https://minikube.sigs.k8s.io/docs/handbook/host-access/ +$HTTP["host"] == "host.minikube.internal" { + alias.url = ( "" => "/usr/libexec/git-core/git-http-backend" ) + setenv.set-environment = ( + "GIT_PROJECT_ROOT" => "/var/www/gitlap/", + "GIT_HTTP_EXPORT_ALL" => "1" + ) + cgi.assign = ( "" => "" ) +} diff --git a/test/registry/README.md b/test/registry/README.md new file mode 100644 index 0000000000..f172558e80 --- /dev/null +++ b/test/registry/README.md @@ -0,0 +1,73 @@ +# Using local registry for minikube clusters + +## Initial setup + +1. Install podman + + ``` + sudo dnf install podman + ``` + +1. Run the registry container + + ``` + podman run --name registry \ + --publish 5000:5000 \ + --volume registry:/var/lib/registry:Z \ + --detach \ + --replace \ + docker.io/library/registry:latest + ``` + + Use `--replace` to replace an existing container, typically left + after reboot the host. + +1. Allow access to port 5000 in the libvirt zone + + ``` + sudo firewall-cmd --zone=libvirt --add-port=5000/tcp --permanent + sudo firewall-cmd --reload + ``` + +1. Configure podman to allow insecure access + + ``` + sudo cp host.minikube.internal.conf /etc/containers/registries.conf.d/ + ``` + +1. Testing the registry + + ``` + $ curl host.minikube.internal:5000/v2/_catalog + {} + ``` + +## Pushing to the local registry + +1. Pull the image from a remote registry + + ``` + podman pull quay.io/nirsof/cirros:0.6.2-1 + ``` + +1. Push to the local registry + + ``` + podman push quay.io/nirsof/cirros:0.6.2-1 host.minikube.internal:5000/nirsof/cirros:0.6.2-1 + ``` + +## Using images from the local registry + +Example source.yaml: + +``` +--- +apiVersion: cdi.kubevirt.io/v1beta1 +kind: VolumeImportSource +metadata: + name: cirros-source +spec: + source: + registry: + url: "docker://host.minikube.internal:5000/nirsof/cirros:0.6.2-1" +``` diff --git a/test/registry/host.minikube.internal.conf b/test/registry/host.minikube.internal.conf new file mode 100644 index 0000000000..03bc08aa31 --- /dev/null +++ b/test/registry/host.minikube.internal.conf @@ -0,0 +1,4 @@ +# Local registry for serving minikube clusters +[[registry]] +location = 'host.minikube.internal:5000' +insecure = true diff --git a/test/suites/basic-k8s.yaml b/test/suites/basic-k8s.yaml new file mode 100644 index 0000000000..c62b0e942c --- /dev/null +++ b/test/suites/basic-k8s.yaml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +name: Basic Kubernetes Regional DR tests +tests: + - name: deploymnet + command: basic-test/run + config: configs/k8s/busybox-regional-rbd-deploy.yaml + - name: statefulset + command: basic-test/run + config: configs/k8s/busybox-regional-rbd-sts.yaml + - name: daemonset + command: basic-test/run + config: configs/k8s/busybox-regional-rbd-ds.yaml diff --git a/test/suites/basic-odr.yaml b/test/suites/basic-odr.yaml new file mode 100644 index 0000000000..b05d118dc7 --- /dev/null +++ b/test/suites/basic-odr.yaml @@ -0,0 +1,12 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +name: Basic OpenShift Regional DR tests +tests: + - name: rbd + command: basic-test/run + config: configs/odr/busybox-regional-rbd-deploy.yaml + - name: cephfs + command: basic-test/run + config: configs/odr/busybox-regional-cephfs-deploy.yaml