From 987d3e1f93c4f10f3ded6abff1de636e5d4f849c Mon Sep 17 00:00:00 2001 From: Nir Soffer Date: Wed, 10 Jan 2024 19:06:29 +0200 Subject: [PATCH 01/22] Lint also enable-dr and disable-dr scripts The script are tiny but it is nice to have verify them with flake8, pylint and black. Signed-off-by: Nir Soffer --- test/Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/Makefile b/test/Makefile index 95cb69b563..75ea8d9a4f 100644 --- a/test/Makefile +++ b/test/Makefile @@ -15,6 +15,8 @@ sources := $(wildcard \ addons/*/start \ addons/*/test \ addons/*/stop \ + */enable-dr \ + */disable-dr \ */deploy \ */undeploy \ */failover \ From 08c8817db2a5f75d0e9baaabe5967343e96d13f3 Mon Sep 17 00:00:00 2001 From: Nir Soffer Date: Mon, 13 Nov 2023 21:57:33 +0200 Subject: [PATCH 02/22] Make application path configurable Add "path" config, so test config looks like OpenShift UI: repo: https://github.com/RamenDR/ocm-ramen-samples.git path: subscription branch: main name: busybox-sample namespace: busybox-sample With this we can use basic test to test any subscription based application in ocm-ramen-samples[1] and ocm-kubevirt-samples[2]. [1] https://github.com/RamenDR/ocm-ramen-samples [2] https://github.com/aglitke/ocm-kubevirt-samples Signed-off-by: Nir Soffer --- test/basic-test/config.yaml | 1 + test/drenv/test.py | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/test/basic-test/config.yaml b/test/basic-test/config.yaml index ca31c93a75..8413dca4af 100644 --- a/test/basic-test/config.yaml +++ b/test/basic-test/config.yaml @@ -3,6 +3,7 @@ --- repo: https://github.com/ramendr/ocm-ramen-samples.git +path: subscription branch: main name: busybox-sample namespace: busybox-sample diff --git a/test/drenv/test.py b/test/drenv/test.py index 34be0c6766..9c28b18cf8 100644 --- a/test/drenv/test.py +++ b/test/drenv/test.py @@ -106,9 +106,9 @@ def deploy(): context=env["hub"], log=debug, ) - info("Deploying subscription based application") + info("Deploying application") kubectl.apply( - f"--kustomize={config['repo']}/subscription?ref={config['branch']}", + f"--kustomize={config['repo']}/{config['path']}?ref={config['branch']}", context=env["hub"], log=debug, ) @@ -125,9 +125,9 @@ def undeploy(): context=env["hub"], log=debug, ) - info("Undeploying subscription based application") + info("Undeploying application") kubectl.delete( - f"--kustomize={config['repo']}/subscription?ref={config['branch']}", + f"--kustomize={config['repo']}/{config['path']}?ref={config['branch']}", "--ignore-not-found", context=env["hub"], log=debug, From f5178bc6c68c3fa35f0ba4d9aa5d72dcc6f6425c Mon Sep 17 00:00:00 2001 From: Nir Soffer Date: Mon, 13 Nov 2023 23:17:24 +0200 Subject: [PATCH 03/22] Add feature_gates profile option It is possible now to enable Kubernetes feature gates[1] using minikube --feature-gates option[2]. We will use this to enable StatefulSetAutoDeletePVC feature gate. Example config: profiles: - name: featured feature_gates: - StatefulSetAutoDeletePVC=true [1] https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/ [2] https://minikube.sigs.k8s.io/docs/handbook/config/#enabling-feature-gates Signed-off-by: Nir Soffer --- test/README.md | 4 ++++ test/drenv/__main__.py | 1 + test/drenv/envfile.py | 1 + test/drenv/minikube.py | 5 +++++ 4 files changed, 11 insertions(+) diff --git a/test/README.md b/test/README.md index 7f52c62e47..2d13d797f9 100644 --- a/test/README.md +++ b/test/README.md @@ -466,6 +466,10 @@ $ drenv delete envs/example.yaml - `extra_config`: List of extra config key=value. Each item adds `--extra-config` minikube option. See `minikube start --help` to see the possible keys and values. + - `feature_gates`: List of Kubernetes feature gates key=value. Each + item adds `--feature-gates` minikube option. See + [Feature Gates](https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/) + for possible keys and values. - `containerd`: Optional containerd configuration object. See `containerd config default` for available options. - `workers`: Optional list of workers to run when starting a diff --git a/test/drenv/__main__.py b/test/drenv/__main__.py index 779cae63c4..f8c48852a6 100644 --- a/test/drenv/__main__.py +++ b/test/drenv/__main__.py @@ -207,6 +207,7 @@ def start_minikube_cluster(profile, verbose=False): addons=profile["addons"], service_cluster_ip_range=profile["service_cluster_ip_range"], extra_config=profile["extra_config"], + feature_gates=profile["feature_gates"], alsologtostderr=verbose, ) diff --git a/test/drenv/envfile.py b/test/drenv/envfile.py index e132fc5998..6e9c39ccc5 100644 --- a/test/drenv/envfile.py +++ b/test/drenv/envfile.py @@ -106,6 +106,7 @@ def _validate_profile(profile, addons_root): profile.setdefault("ser", []) profile.setdefault("service_cluster_ip_range", None) profile.setdefault("extra_config", []) + profile.setdefault("feature_gates", []) profile.setdefault("containerd", None) profile.setdefault("workers", []) diff --git a/test/drenv/minikube.py b/test/drenv/minikube.py index 411bab751c..ee95ef5bdd 100644 --- a/test/drenv/minikube.py +++ b/test/drenv/minikube.py @@ -43,6 +43,7 @@ def start( addons=(), service_cluster_ip_range=None, extra_config=None, + feature_gates=None, alsologtostderr=False, ): args = [] @@ -77,6 +78,10 @@ def start( for pair in extra_config: args.extend(("--extra-config", pair)) + if feature_gates: + # Unlike --extra-config this requires one comma separated value. + args.extend(("--feature-gates", ",".join(feature_gates))) + if alsologtostderr: args.append("--alsologtostderr") From 2b26772d7db48663244a87353189e116792a30cf Mon Sep 17 00:00:00 2001 From: Nir Soffer Date: Mon, 13 Nov 2023 23:22:17 +0200 Subject: [PATCH 04/22] Enable StatefulSetAutoDeletePVC feature gate With this we can create a statesfulset with persistentVolumeClaimRetentionPolicy[1] to have the PVCs deleted when a stateful set is deleted. This policy is required for relocate, otherwise ramen get stuck waiting for vrs to become secondary. [1] https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#persistentvolumeclaim-retention Signed-off-by: Nir Soffer --- test/envs/regional-dr.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/envs/regional-dr.yaml b/test/envs/regional-dr.yaml index 66ea1bfd13..516ae921cc 100644 --- a/test/envs/regional-dr.yaml +++ b/test/envs/regional-dr.yaml @@ -21,6 +21,8 @@ templates: addons: - volumesnapshots - csi-hostpath-driver + feature_gates: + - StatefulSetAutoDeletePVC=true workers: - addons: - name: cert-manager From 22c5e660a9877287b9c10ea8aea123c614321c53 Mon Sep 17 00:00:00 2001 From: Nir Soffer Date: Thu, 16 Nov 2023 17:45:57 +0200 Subject: [PATCH 05/22] Fix waiting for drcluster and drpolicy We use `--namespace ramen-system` but these are deployed in cluster scope. I guess the `--namespace` is ignored in this case since this code works as is. Signed-off-by: Nir Soffer --- ramenctl/ramenctl/config.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/ramenctl/ramenctl/config.py b/ramenctl/ramenctl/config.py index f3b0b2f917..977d286bba 100644 --- a/ramenctl/ramenctl/config.py +++ b/ramenctl/ramenctl/config.py @@ -145,7 +145,6 @@ def wait_for_dr_clusters(hub, clusters, args): drenv.wait_for( f"drcluster/{name}", output="jsonpath={.status.phase}", - namespace=args.ramen_namespace, timeout=180, profile=hub, log=command.debug, @@ -156,7 +155,6 @@ def wait_for_dr_clusters(hub, clusters, args): "drcluster", "--all", "--for=jsonpath={.status.phase}=Available", - f"--namespace={args.ramen_namespace}", context=hub, log=command.debug, ) @@ -167,7 +165,6 @@ def wait_for_dr_policy(hub, args): kubectl.wait( "drpolicy/dr-policy", "--for=condition=Validated", - f"--namespace={args.ramen_namespace}", context=hub, log=command.debug, ) From bddd404320b4ec4a19a35e7f358ec6bae18f378c Mon Sep 17 00:00:00 2001 From: Nir Soffer Date: Thu, 16 Nov 2023 21:35:45 +0200 Subject: [PATCH 06/22] Fix hard coded pvc selector label Add `pvc_label` configuration so we can test any application. With this we can run basic-test with vms from ocm-kubeivrt-samples[1]. [1] https://github.com/aglitke/ocm-kubevirt-samples Signed-off-by: Nir Soffer --- test/basic-test/config.yaml | 1 + test/drenv/test.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/test/basic-test/config.yaml b/test/basic-test/config.yaml index 8413dca4af..ac2081d327 100644 --- a/test/basic-test/config.yaml +++ b/test/basic-test/config.yaml @@ -7,3 +7,4 @@ path: subscription branch: main name: busybox-sample namespace: busybox-sample +pvc_label: busybox diff --git a/test/drenv/test.py b/test/drenv/test.py index 9c28b18cf8..b2cf8d3343 100644 --- a/test/drenv/test.py +++ b/test/drenv/test.py @@ -171,7 +171,7 @@ def enable_dr(): name: {placement_name} pvcSelector: matchLabels: - appname: busybox + appname: {config['pvc_label']} """ kubectl.apply("--filename=-", input=drpc, context=env["hub"], log=debug) From 75f623b39fccd549e08aa3b0f5da24c885757376 Mon Sep 17 00:00:00 2001 From: Nir Soffer Date: Wed, 10 Jan 2024 19:32:33 +0200 Subject: [PATCH 07/22] Use dr policy owned by the test The basic-test can be used now with OpenShift clusters (using new ocm-ramen-sampels providing subscription and dr kustomizations). The only issue is the dr policy - basic-test is hard coded to use `dr-policy` installed by ramenctl, which is not available in our OpenShift test clusters. Fix by using a dr policy owned by the test, created by the tests when deploying the application, and removed when undeploying the application. The name of the policy must be configured in the test `config.yaml`. To be able to run concurrent tests, each test config must have its own dr policy. Signed-off-by: Nir Soffer --- test/basic-test/config.yaml | 1 + test/drenv/test.py | 42 +++++++++++++++++++++++++++++++++++-- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/test/basic-test/config.yaml b/test/basic-test/config.yaml index ac2081d327..eb98582f35 100644 --- a/test/basic-test/config.yaml +++ b/test/basic-test/config.yaml @@ -7,4 +7,5 @@ path: subscription branch: main name: busybox-sample namespace: busybox-sample +dr_policy: busybox-sample pvc_label: busybox diff --git a/test/drenv/test.py b/test/drenv/test.py index b2cf8d3343..280918afa7 100644 --- a/test/drenv/test.py +++ b/test/drenv/test.py @@ -96,6 +96,42 @@ def _excepthook(t, v, tb): log.exception("test failed", exc_info=(t, v, tb)) +def _deploy_dr_policy(): + dr_policy = f""" +apiVersion: ramendr.openshift.io/v1alpha1 +kind: DRPolicy +metadata: + name: {config['dr_policy']} +spec: + drClusters: + - {env['clusters'][0]} + - {env['clusters'][1]} + schedulingInterval: 1m +""" + info("Deploying drpolicy '%s'", config["dr_policy"]) + kubectl.apply("--filename=-", input=dr_policy, context=env["hub"], log=debug) + + info("Waiting until drpolicy '%s' is validated", config["dr_policy"]) + kubectl.wait( + "drpolicy", + config["dr_policy"], + "--for=condition=Validated", + context=env["hub"], + log=debug, + ) + + +def _delete_dr_policy(): + info("Deleting drpolicy '%s'", config["dr_policy"]) + kubectl.delete( + "drpolicy", + config["dr_policy"], + "--ignore-not-found", + context=env["hub"], + log=debug, + ) + + def deploy(): """ Deploy application on cluster. @@ -106,6 +142,7 @@ def deploy(): context=env["hub"], log=debug, ) + _deploy_dr_policy() info("Deploying application") kubectl.apply( f"--kustomize={config['repo']}/{config['path']}?ref={config['branch']}", @@ -132,6 +169,7 @@ def undeploy(): context=env["hub"], log=debug, ) + _delete_dr_policy() def enable_dr(): @@ -158,14 +196,14 @@ def enable_dr(): apiVersion: ramendr.openshift.io/v1alpha1 kind: DRPlacementControl metadata: - name: busybox-drpc + name: {config['name']}-drpc namespace: {config['namespace']} labels: app: {config['name']} spec: preferredCluster: {cluster} drPolicyRef: - name: dr-policy + name: {config['dr_policy']} placementRef: kind: Placement name: {placement_name} From 063ccc76e917c253f84b45103fdf79b99d99477f Mon Sep 17 00:00:00 2001 From: Nir Soffer Date: Wed, 10 Jan 2024 19:32:51 +0200 Subject: [PATCH 08/22] Move generic logs Since we can test any application now (e.g. busybox, kubvirt), we don't want to mention busybox in the logs. Use config['name'] when we can to make the logs more clear. Signed-off-by: Nir Soffer --- test/basic-test/undeploy | 4 ++-- test/drenv/test.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/basic-test/undeploy b/test/basic-test/undeploy index 7fa04aca12..950bf4768c 100755 --- a/test/basic-test/undeploy +++ b/test/basic-test/undeploy @@ -8,6 +8,6 @@ from drenv import test test.start("undeploy", __file__) args = test.parse_args() -test.info("Deleting busybox example application") +test.info("Deleting application") test.undeploy() -test.info("Application was undeployed successfully") +test.info("Application was deleted") diff --git a/test/drenv/test.py b/test/drenv/test.py index 280918afa7..6bc5494e86 100644 --- a/test/drenv/test.py +++ b/test/drenv/test.py @@ -143,7 +143,7 @@ def deploy(): log=debug, ) _deploy_dr_policy() - info("Deploying application") + info("Deploying application '%s'", config["name"]) kubectl.apply( f"--kustomize={config['repo']}/{config['path']}?ref={config['branch']}", context=env["hub"], @@ -162,7 +162,7 @@ def undeploy(): context=env["hub"], log=debug, ) - info("Undeploying application") + info("Undeploying application '%s'", config["name"]) kubectl.delete( f"--kustomize={config['repo']}/{config['path']}?ref={config['branch']}", "--ignore-not-found", From 35eb4eaffa9d44aa9d4ad769124c53ac900720d9 Mon Sep 17 00:00:00 2001 From: Nir Soffer Date: Wed, 10 Jan 2024 19:20:34 +0200 Subject: [PATCH 09/22] Remove channel deployment The channel is part of the subscription kustomization in ocm-ramen-samples, so we don't need to deploy or undeploy it. The basic config uses now the new deployment from my repo. We will update the repo when the ocm-ramen-samples PR is merged. Signed-off-by: Nir Soffer --- test/basic-test/config.yaml | 12 ++++++------ test/drenv/test.py | 13 ------------- 2 files changed, 6 insertions(+), 19 deletions(-) diff --git a/test/basic-test/config.yaml b/test/basic-test/config.yaml index eb98582f35..e56ff0f00b 100644 --- a/test/basic-test/config.yaml +++ b/test/basic-test/config.yaml @@ -2,10 +2,10 @@ # SPDX-License-Identifier: Apache-2.0 --- -repo: https://github.com/ramendr/ocm-ramen-samples.git -path: subscription -branch: main -name: busybox-sample -namespace: busybox-sample -dr_policy: busybox-sample +repo: https://github.com/nirs/ocm-ramen-samples.git +path: k8s/busybox-regional-rbd-deploy/sub +branch: test +name: busybox-regional-rbd-deploy +namespace: busybox-regional-rbd-deploy +dr_policy: busybox-regional-rbd-deploy pvc_label: busybox diff --git a/test/drenv/test.py b/test/drenv/test.py index 6bc5494e86..fc6e36c5af 100644 --- a/test/drenv/test.py +++ b/test/drenv/test.py @@ -136,12 +136,6 @@ def deploy(): """ Deploy application on cluster. """ - info("Deploying channel") - kubectl.apply( - f"--kustomize={config['repo']}/channel?ref={config['branch']}", - context=env["hub"], - log=debug, - ) _deploy_dr_policy() info("Deploying application '%s'", config["name"]) kubectl.apply( @@ -155,13 +149,6 @@ def undeploy(): """ Undeploy an application. """ - info("Undeploying channel") - kubectl.delete( - f"--kustomize={config['repo']}/channel?ref={config['branch']}", - "--ignore-not-found", - context=env["hub"], - log=debug, - ) info("Undeploying application '%s'", config["name"]) kubectl.delete( f"--kustomize={config['repo']}/{config['path']}?ref={config['branch']}", From 73c51c4f1ec860c9e2f115d42e364f95290b117e Mon Sep 17 00:00:00 2001 From: Nir Soffer Date: Mon, 27 Nov 2023 04:11:28 +0200 Subject: [PATCH 10/22] Support multiple configurations basic-test can be run now with custom configuration file. This can be used to run multiple tests concurrently. test/basic-test/run --config rbd-deploy.yaml $env 2>rbd.log & test/basic-test/run --config cephfs-deploy.yaml $env 2>cephfs.log & wait Signed-off-by: Nir Soffer --- test/drenv/test.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/test/drenv/test.py b/test/drenv/test.py index fc6e36c5af..b6f674a231 100644 --- a/test/drenv/test.py +++ b/test/drenv/test.py @@ -27,8 +27,8 @@ ) -def start(name, file, config_file="config.yaml"): - global workdir, config, parser, log +def start(name, file): + global workdir, parser, log # Setting up logging and sys.excepthook must be first so any failure will # be reported using the logger. @@ -42,10 +42,6 @@ def start(name, file, config_file="config.yaml"): # Working directory for runing the test. workdir = os.path.abspath(os.path.dirname(file)) - config_path = os.path.join(workdir, config_file) - with open(config_path) as f: - config = yaml.safe_load(f) - parser = argparse.ArgumentParser(name) parser.add_argument( "-v", @@ -57,6 +53,12 @@ def start(name, file, config_file="config.yaml"): "--name-prefix", help="Prefix profile names", ) + parser.add_argument( + "-c", + "--config", + default=os.path.join(workdir, "config.yaml"), + help="Test configuration file", + ) parser.add_argument( "filename", help="Environment filename", @@ -68,13 +70,16 @@ def add_argument(*args, **kw): def parse_args(): - global env + global config, env args = parser.parse_args() if args.verbose: log.setLevel(logging.DEBUG) debug("Parsed arguments: %s", args) + with open(args.config) as f: + config = yaml.safe_load(f) + env = ramen.env_info(args.filename, name_prefix=args.name_prefix) debug("Using environment: %s", env) From df47aff07018f3ba231b66e4bd13c8073577c2e3 Mon Sep 17 00:00:00 2001 From: Nir Soffer Date: Mon, 27 Nov 2023 04:31:29 +0200 Subject: [PATCH 11/22] Add test configurations Add tests configurations for multiple applications for OpenShift and Kubernetes using ocm-ramen-samples repo. Currently using my own repo until the new applications are merged. To run test using a custom configuration use: basic-test/run \ --config configs/odr/busybox-regional-rbd-deploy.yaml \ env.yaml Signed-off-by: Nir Soffer --- test/configs/k8s/busybox-regional-rbd-deploy.yaml | 11 +++++++++++ test/configs/k8s/busybox-regional-rbd-ds.yaml | 11 +++++++++++ test/configs/k8s/busybox-regional-rbd-sts.yaml | 11 +++++++++++ test/configs/odr/busybox-metro-rbd-deploy.yaml | 11 +++++++++++ test/configs/odr/busybox-regional-cephfs-deploy.yaml | 11 +++++++++++ test/configs/odr/busybox-regional-rbd-deploy.yaml | 11 +++++++++++ 6 files changed, 66 insertions(+) create mode 100644 test/configs/k8s/busybox-regional-rbd-deploy.yaml create mode 100644 test/configs/k8s/busybox-regional-rbd-ds.yaml create mode 100644 test/configs/k8s/busybox-regional-rbd-sts.yaml create mode 100644 test/configs/odr/busybox-metro-rbd-deploy.yaml create mode 100644 test/configs/odr/busybox-regional-cephfs-deploy.yaml create mode 100644 test/configs/odr/busybox-regional-rbd-deploy.yaml diff --git a/test/configs/k8s/busybox-regional-rbd-deploy.yaml b/test/configs/k8s/busybox-regional-rbd-deploy.yaml new file mode 100644 index 0000000000..e56ff0f00b --- /dev/null +++ b/test/configs/k8s/busybox-regional-rbd-deploy.yaml @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +repo: https://github.com/nirs/ocm-ramen-samples.git +path: k8s/busybox-regional-rbd-deploy/sub +branch: test +name: busybox-regional-rbd-deploy +namespace: busybox-regional-rbd-deploy +dr_policy: busybox-regional-rbd-deploy +pvc_label: busybox diff --git a/test/configs/k8s/busybox-regional-rbd-ds.yaml b/test/configs/k8s/busybox-regional-rbd-ds.yaml new file mode 100644 index 0000000000..9fa5e58b6a --- /dev/null +++ b/test/configs/k8s/busybox-regional-rbd-ds.yaml @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +repo: https://github.com/nirs/ocm-ramen-samples.git +path: k8s/busybox-regional-rbd-ds/sub +branch: test +name: busybox-regional-rbd-ds +namespace: busybox-regional-rbd-ds +dr_policy: busybox-regional-rbd-ds +pvc_label: busybox diff --git a/test/configs/k8s/busybox-regional-rbd-sts.yaml b/test/configs/k8s/busybox-regional-rbd-sts.yaml new file mode 100644 index 0000000000..ec600300a1 --- /dev/null +++ b/test/configs/k8s/busybox-regional-rbd-sts.yaml @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +repo: https://github.com/nirs/ocm-ramen-samples.git +path: k8s/busybox-regional-rbd-sts/sub +branch: test +name: busybox-regional-rbd-sts +namespace: busybox-regional-rbd-sts +dr_policy: busybox-regional-rbd-sts +pvc_label: busybox diff --git a/test/configs/odr/busybox-metro-rbd-deploy.yaml b/test/configs/odr/busybox-metro-rbd-deploy.yaml new file mode 100644 index 0000000000..3c815190ba --- /dev/null +++ b/test/configs/odr/busybox-metro-rbd-deploy.yaml @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +repo: https://github.com/nirs/ocm-ramen-samples.git +path: odr/busybox-metro-rbd-deploy/sub +branch: test-odr +name: busybox-metro-rbd-deploy +namespace: busybox-metro-rbd-deploy +dr_policy: busybox-metro-rbd-deploy +pvc_label: busybox diff --git a/test/configs/odr/busybox-regional-cephfs-deploy.yaml b/test/configs/odr/busybox-regional-cephfs-deploy.yaml new file mode 100644 index 0000000000..a45e949d5f --- /dev/null +++ b/test/configs/odr/busybox-regional-cephfs-deploy.yaml @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +repo: https://github.com/nirs/ocm-ramen-samples.git +path: odr/busybox-regional-cephfs-deploy/sub +branch: test-odr +name: busybox-regional-cephfs-deploy +namespace: busybox-regional-cephfs-deploy +dr_policy: busybox-regional-cephfs-deploy +pvc_label: busybox diff --git a/test/configs/odr/busybox-regional-rbd-deploy.yaml b/test/configs/odr/busybox-regional-rbd-deploy.yaml new file mode 100644 index 0000000000..686c50e9ef --- /dev/null +++ b/test/configs/odr/busybox-regional-rbd-deploy.yaml @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +repo: https://github.com/nirs/ocm-ramen-samples.git +path: odr/busybox-regional-rbd-deploy/sub +branch: test-odr +name: busybox-regional-rbd-deploy +namespace: busybox-regional-rbd-deploy +dr_policy: busybox-regional-rbd-deploy +pvc_label: busybox From 51f4c4483af55d86bb807c3d739988394b541813 Mon Sep 17 00:00:00 2001 From: Nir Soffer Date: Thu, 30 Nov 2023 02:45:40 +0200 Subject: [PATCH 12/22] Add kubvirt configuration Using my repo until the relevant PR[1] is merged. [1] https://github.com/aglitke/ocm-kubevirt-samples/pull/6 Signed-off-by: Nir Soffer --- test/configs/k8s/kubevirt-pvc.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 test/configs/k8s/kubevirt-pvc.yaml diff --git a/test/configs/k8s/kubevirt-pvc.yaml b/test/configs/k8s/kubevirt-pvc.yaml new file mode 100644 index 0000000000..f6d71ef550 --- /dev/null +++ b/test/configs/k8s/kubevirt-pvc.yaml @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +repo: https://github.com/nirs/ocm-kubevirt-samples.git +path: subscription +branch: test +name: kubevirt-sample +namespace: kubevirt-sample +dr_policy: kubevirt-sample +pvc_label: vm-standalone-dv-odr-regional From c62becc8c533e6747e8500f23bd0d11a1021c6d3 Mon Sep 17 00:00:00 2001 From: Nir Soffer Date: Mon, 27 Nov 2023 23:45:15 +0200 Subject: [PATCH 13/22] Add drtest demo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This tiny tool reads test suites yaml and run the tests in parallel logging test logs to separate files. A test suite binds tests (e.g. basic-test) to application configurations (e.g. busybox-regional-rbd-deploy). We have 2 test suites: $ tree suites/ suites/ ├── basic-k8s.yaml └── basic-odr.yaml Example run with drenv created environment: $ ./drtest --outdir /tmp/k8s-logs suites/basic-k8s.yaml envs/regional-dr.yaml 2023-11-28 00:55:54,099 INFO Running 'Basic Kubernetes Regional DR tests' 2023-11-28 00:55:54,099 INFO Storing output to '/tmp/k8s-logs' 2023-11-28 00:55:54,101 INFO Starting test 'deploymnet' 2023-11-28 00:55:54,101 INFO Starting test 'statefulset' 2023-11-28 00:55:54,102 INFO Starting test 'daemonset' 2023-11-28 01:04:23,274 INFO Test 'daemonset' PASS 2023-11-28 01:04:24,161 INFO Test 'deploymnet' PASS 2023-11-28 01:04:53,600 INFO Test 'statefulset' PASS 2023-11-28 01:04:53,600 INFO PASS (3 pass, 0 fail) The test logs to separate file: $ tree /tmp/k8s-logs /tmp/k8s-logs ├── daemonset.log ├── deploymnet.log └── statefulset.log To test with OpenShift we need to create a tiny environment file: $ cat env.yaml ramen: hub: hub clusters: [cluster1, cluster2] topology: regional-dr And use a kubeconfig file with the clusters. The file can be created with `oc login` and some `oc config` commands, or using the oc-clusterset plugin: $ cat config.yaml clusters: - name: cluster1 url: perf1.example.com:6443 username: kubeadmin password: PeSkM-R6YcH-LyPZa-oTOO1 - name: cluster2 url: perf2.example.com:6443 username: kubeadmin password: ZjIZn-SFUyR-aE4gI-fJcfL - name: hub url: perf3.example.com:6443 username: kubeadmin password: 7C700-oVS3Q-25rtx-YMew5 current-context: hub $ oc clusterset login --config config.yaml --kubeconfig kubeconfig $ oc config get-contexts --kubeconfig kubeconfig CURRENT NAME CLUSTER AUTHINFO NAMESPACE cluster1 perf1-example-com:6443 kube:admin/perf1-example-com:6443 default cluster2 perf2-example-com:6443 kube:admin/perf2-example-com:6443 default * hub perf3-example-com:6443 kube:admin/perf3-example-com:6443 default Example run with the OpenShift environment: $ ./drtest --kubeconfig kubeconfig --outdir /tmp/odr-logs suites/basic-odr.yaml env.yaml 2023-11-29 23:45:14,849 INFO Running 'Basic OpenShift Regional DR tests' 2023-11-29 23:45:14,849 INFO Storing output to '/tmp/odr-logs' 2023-11-29 23:45:14,850 INFO Starting test 'rbd' 2023-11-29 23:45:14,850 INFO Starting test 'cephfs' 2023-11-29 23:54:24,599 INFO Test 'rbd' PASS 2023-11-29 23:54:51,461 INFO Test 'cephfs' PASS 2023-11-29 23:54:51,461 INFO PASS (2 pass, 0 fail) Signed-off-by: Nir Soffer --- test/drtest | 79 ++++++++++++++++++++++++++++++++++++++ test/suites/basic-k8s.yaml | 15 ++++++++ test/suites/basic-odr.yaml | 12 ++++++ 3 files changed, 106 insertions(+) create mode 100755 test/drtest create mode 100644 test/suites/basic-k8s.yaml create mode 100644 test/suites/basic-odr.yaml diff --git a/test/drtest b/test/drtest new file mode 100755 index 0000000000..51ecfdd85e --- /dev/null +++ b/test/drtest @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +import argparse +import collections +import concurrent.futures +import logging +import os +import subprocess + +import yaml + + +def main(): + p = argparse.ArgumentParser("drtest") + p.add_argument("-k", "--kubeconfig", help="If set, use speified kubeconfig path") + p.add_argument("-o", "--outdir", default=".", help="Output directory") + p.add_argument("-v", "--verbose", action="store_true", help="Be more verbose") + p.add_argument("suite", help="Test suite file") + p.add_argument("env", help="Environemnt file") + args = p.parse_args() + + logging.basicConfig( + level=logging.DEBUG if args.verbose else logging.INFO, + format="%(asctime)s %(levelname)-7s %(message)s", + ) + + with open(args.suite) as f: + suite = yaml.safe_load(f) + + os.makedirs(args.outdir, exist_ok=True) + + logging.info("Running '%s'", suite["name"]) + logging.info("Storing output to '%s'", args.outdir) + + stats = collections.Counter({"pass": 0, "fail": 0}) + + with concurrent.futures.ThreadPoolExecutor() as e: + futures = {} + + for test in suite["tests"]: + logging.info("Starting test '%s'", test["name"]) + f = e.submit(run_test, test, args) + futures[f] = test["name"] + + for f in concurrent.futures.as_completed(futures): + try: + f.result() + except Exception: + logging.error("Test '%s' FAIL", futures[f]) + stats["fail"] += 1 + else: + logging.info("Test '%s' PASS", futures[f]) + stats["pass"] += 1 + + logging.info( + "%s (%s pass, %s fail)", + "PASS" if stats["fail"] == 0 else "FAIL", + stats["pass"], + stats["fail"], + ) + + +def run_test(test, args): + log = os.path.join(args.outdir, test["name"] + ".log") + cmd = [test["command"], "--config", test["config"], args.env] + if args.kubeconfig: + env = dict(os.environ) + env["KUBECONFIG"] = args.kubeconfig + else: + env = None + with open(log, "w") as f: + subprocess.run(cmd, stderr=f, check=True, env=env) + + +if __name__ == "__main__": + main() diff --git a/test/suites/basic-k8s.yaml b/test/suites/basic-k8s.yaml new file mode 100644 index 0000000000..c62b0e942c --- /dev/null +++ b/test/suites/basic-k8s.yaml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +name: Basic Kubernetes Regional DR tests +tests: + - name: deploymnet + command: basic-test/run + config: configs/k8s/busybox-regional-rbd-deploy.yaml + - name: statefulset + command: basic-test/run + config: configs/k8s/busybox-regional-rbd-sts.yaml + - name: daemonset + command: basic-test/run + config: configs/k8s/busybox-regional-rbd-ds.yaml diff --git a/test/suites/basic-odr.yaml b/test/suites/basic-odr.yaml new file mode 100644 index 0000000000..b05d118dc7 --- /dev/null +++ b/test/suites/basic-odr.yaml @@ -0,0 +1,12 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +name: Basic OpenShift Regional DR tests +tests: + - name: rbd + command: basic-test/run + config: configs/odr/busybox-regional-rbd-deploy.yaml + - name: cephfs + command: basic-test/run + config: configs/odr/busybox-regional-cephfs-deploy.yaml From db20da2b037ecbad72b511982c72734ef25b0959 Mon Sep 17 00:00:00 2001 From: Nir Soffer Date: Wed, 10 Jan 2024 20:36:36 +0200 Subject: [PATCH 14/22] Upgrade CDI to latest release https: //github.com/kubevirt/containerized-data-importer/releases/tag/v1.58.0 Signed-off-by: Nir Soffer --- test/addons/cdi/cr/kustomization.yaml | 2 +- test/addons/cdi/operator/kustomization.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/addons/cdi/cr/kustomization.yaml b/test/addons/cdi/cr/kustomization.yaml index 3fd3d08eb5..4aa6bdfe56 100644 --- a/test/addons/cdi/cr/kustomization.yaml +++ b/test/addons/cdi/cr/kustomization.yaml @@ -4,4 +4,4 @@ # yamllint disable rule:line-length --- resources: -- https://github.com/kubevirt/containerized-data-importer/releases/download/v1.57.0/cdi-cr.yaml +- https://github.com/kubevirt/containerized-data-importer/releases/download/v1.58.0/cdi-cr.yaml diff --git a/test/addons/cdi/operator/kustomization.yaml b/test/addons/cdi/operator/kustomization.yaml index b894cb9790..208e01b464 100644 --- a/test/addons/cdi/operator/kustomization.yaml +++ b/test/addons/cdi/operator/kustomization.yaml @@ -4,4 +4,4 @@ # yamllint disable rule:line-length --- resources: -- https://github.com/kubevirt/containerized-data-importer/releases/download/v1.57.0/cdi-operator.yaml +- https://github.com/kubevirt/containerized-data-importer/releases/download/v1.58.0/cdi-operator.yaml From fe09eb1c3a01ed950dffb3df4c5809ff8d59dabc Mon Sep 17 00:00:00 2001 From: Nir Soffer Date: Wed, 10 Jan 2024 20:39:05 +0200 Subject: [PATCH 15/22] Update kubvirt to latest release https: //github.com/kubevirt/kubevirt/releases/tag/v1.1.1 Signed-off-by: Nir Soffer --- test/addons/kubevirt/cr/kustomization.yaml | 2 +- test/addons/kubevirt/operator/kustomization.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/addons/kubevirt/cr/kustomization.yaml b/test/addons/kubevirt/cr/kustomization.yaml index 2f02b94a99..2927d5293e 100644 --- a/test/addons/kubevirt/cr/kustomization.yaml +++ b/test/addons/kubevirt/cr/kustomization.yaml @@ -4,4 +4,4 @@ # yamllint disable rule:line-length --- resources: -- https://github.com/kubevirt/kubevirt/releases/download/v1.0.1/kubevirt-cr.yaml +- https://github.com/kubevirt/kubevirt/releases/download/v1.1.1/kubevirt-cr.yaml diff --git a/test/addons/kubevirt/operator/kustomization.yaml b/test/addons/kubevirt/operator/kustomization.yaml index 2846de42f8..0be84e4902 100644 --- a/test/addons/kubevirt/operator/kustomization.yaml +++ b/test/addons/kubevirt/operator/kustomization.yaml @@ -4,4 +4,4 @@ # yamllint disable rule:line-length --- resources: -- https://github.com/kubevirt/kubevirt/releases/download/v1.0.1/kubevirt-operator.yaml +- https://github.com/kubevirt/kubevirt/releases/download/v1.1.1/kubevirt-operator.yaml From 47324832f21e5457b0c53112307240e383c60f6d Mon Sep 17 00:00:00 2001 From: Nir Soffer Date: Wed, 10 Jan 2024 20:26:44 +0200 Subject: [PATCH 16/22] Use the standard cirros image in CDR test There is no point in using tow versions of the same image. Using this image in the CDI test can save time in the kubvirt tests later, using the cached image. Signed-off-by: Nir Soffer --- test/addons/cdi/disk/source.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/addons/cdi/disk/source.yaml b/test/addons/cdi/disk/source.yaml index cdd40c0646..c320d64541 100644 --- a/test/addons/cdi/disk/source.yaml +++ b/test/addons/cdi/disk/source.yaml @@ -9,4 +9,4 @@ metadata: spec: source: registry: - url: "docker://quay.io/alitke/cirros:latest" + url: "docker://quay.io/nirsof/cirros:0.6.2-1" From 32e75d96b36a242ddf9313da4c9559da1198f4a7 Mon Sep 17 00:00:00 2001 From: Nir Soffer Date: Wed, 10 Jan 2024 21:36:20 +0200 Subject: [PATCH 17/22] Fix waiting for cdi CDI may become available before it is ready to use. If we try to use it while it is progressing we may fail with errors about missing CRDS. Wait until the progressing condition becomes false. Example run showing the issue: 2024-01-10 21:42:24,080 DEBUG [kubevirt/1] Deploying cdi cr 2024-01-10 21:42:25,674 DEBUG [kubevirt/1] Waiting until cdi cr is available 2024-01-10 21:42:26,005 DEBUG [kubevirt/1] cdi.cdi.kubevirt.io/cdi condition met We stopped waiting here... 2024-01-10 21:42:26,007 DEBUG [kubevirt/1] Waiting until cdi cr finished progressing 2024-01-10 21:42:39,472 DEBUG [kubevirt/1] cdi.cdi.kubevirt.io/cdi condition met But CDI finished progressing 13 seconds later. Signed-off-by: Nir Soffer --- test/addons/cdi/start | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/addons/cdi/start b/test/addons/cdi/start index 38193dafdb..cedb78efd0 100755 --- a/test/addons/cdi/start +++ b/test/addons/cdi/start @@ -37,6 +37,14 @@ def wait(cluster): "--timeout=300s", context=cluster, ) + print("Waiting until cdi cr finished progressing") + kubectl.wait( + "cdi.cdi.kubevirt.io/cdi", + "--for=condition=progressing=False", + f"--namespace={NAMESPACE}", + "--timeout=300s", + context=cluster, + ) if len(sys.argv) != 2: From a5f451f4de45d7ddefba4c830fbc0d6156a0e1c8 Mon Sep 17 00:00:00 2001 From: Nir Soffer Date: Sun, 19 Nov 2023 13:18:47 +0200 Subject: [PATCH 18/22] Trim kubevirt environment We cannot use volsync with ramen yet, and the kubevirt environment is already too big. Without volsync we can remove the volumesnapshot addon and submariner, which does not handle well suspending of the machine running the minikube VMs. With this change we should be able to start an environment, suspend the laptop, and resume it in an environment with unreliable network or no network access. This will be useful for live demo in conferences. Keep volsync enabled in `regional-dr` and `regional-dr-hubless` to keep the submariner and volsync addons functional. Signed-off-by: Nir Soffer --- ramenctl/ramenctl/config.py | 9 ++++++--- ramenctl/ramenctl/resources/configmap.yaml | 2 ++ test/envs/regional-dr-hubless.yaml | 2 ++ test/envs/regional-dr-kubevirt.yaml | 8 ++------ test/envs/regional-dr.yaml | 2 ++ 5 files changed, 14 insertions(+), 9 deletions(-) diff --git a/ramenctl/ramenctl/config.py b/ramenctl/ramenctl/config.py index 977d286bba..8f84257ee3 100644 --- a/ramenctl/ramenctl/config.py +++ b/ramenctl/ramenctl/config.py @@ -25,7 +25,7 @@ def run(args): cloud_secret = generate_cloud_credentials_secret(env["clusters"][0], args) if env["hub"]: - hub_cm = generate_config_map("hub", env["clusters"], args) + hub_cm = generate_config_map("hub", env, args) wait_for_ramen_hub_operator(env["hub"], args) @@ -38,7 +38,7 @@ def run(args): wait_for_dr_clusters(env["hub"], env["clusters"], args) wait_for_dr_policy(env["hub"], args) else: - dr_cluster_cm = generate_config_map("dr-cluster", env["clusters"], args) + dr_cluster_cm = generate_config_map("dr-cluster", env, args) for cluster in env["clusters"]: create_ramen_s3_secrets(cluster, s3_secrets) @@ -89,7 +89,9 @@ def create_cloud_credentials_secret(cluster, yaml): kubectl.apply("--filename=-", input=yaml, context=cluster, log=command.debug) -def generate_config_map(controller, clusters, args): +def generate_config_map(controller, env, args): + clusters = env["clusters"] + volsync = env["features"].get("volsync", True) template = drenv.template(command.resource("configmap.yaml")) return template.substitute( name=f"ramen-{controller}-operator-config", @@ -98,6 +100,7 @@ def generate_config_map(controller, clusters, args): cluster2=clusters[1], minio_url_cluster1=minio.service_url(clusters[0]), minio_url_cluster2=minio.service_url(clusters[1]), + volsync_disabled="false" if volsync else "true", namespace=args.ramen_namespace, ) diff --git a/ramenctl/ramenctl/resources/configmap.yaml b/ramenctl/ramenctl/resources/configmap.yaml index c82136f1f7..53b2b7fb44 100644 --- a/ramenctl/ramenctl/resources/configmap.yaml +++ b/ramenctl/ramenctl/resources/configmap.yaml @@ -34,6 +34,8 @@ data: clusterServiceVersionName: ramen-dr-cluster-operator.v0.0.1 kubeObjectProtection: veleroNamespaceName: velero + volSync: + disabled: $volsync_disabled s3StoreProfiles: - s3ProfileName: minio-on-$cluster1 s3Bucket: bucket diff --git a/test/envs/regional-dr-hubless.yaml b/test/envs/regional-dr-hubless.yaml index fff2a57f74..2928f7747a 100644 --- a/test/envs/regional-dr-hubless.yaml +++ b/test/envs/regional-dr-hubless.yaml @@ -9,6 +9,8 @@ ramen: hub: null clusters: [dr1, dr2] topology: regional-dr + features: + volsync: true templates: - name: "dr-cluster" diff --git a/test/envs/regional-dr-kubevirt.yaml b/test/envs/regional-dr-kubevirt.yaml index eeb7fdbced..a45eca833e 100644 --- a/test/envs/regional-dr-kubevirt.yaml +++ b/test/envs/regional-dr-kubevirt.yaml @@ -9,6 +9,8 @@ ramen: hub: hub clusters: [dr1, dr2] topology: regional-dr + features: + volsync: false templates: - name: "dr-cluster" @@ -25,7 +27,6 @@ templates: extra_disks: 1 disk_size: "50g" addons: - - volumesnapshots - csi-hostpath-driver workers: - addons: @@ -57,8 +58,6 @@ templates: - name: ocm-controller - name: cert-manager - name: olm - - name: submariner - args: ["hub", "dr1", "dr2"] profiles: - name: "dr1" @@ -72,6 +71,3 @@ workers: - addons: - name: rbd-mirror args: ["dr1", "dr2"] - - addons: - - name: volsync - args: ["dr1", "dr2"] diff --git a/test/envs/regional-dr.yaml b/test/envs/regional-dr.yaml index 516ae921cc..74c1646621 100644 --- a/test/envs/regional-dr.yaml +++ b/test/envs/regional-dr.yaml @@ -9,6 +9,8 @@ ramen: hub: hub clusters: [dr1, dr2] topology: regional-dr + features: + volsync: true templates: - name: "dr-cluster" From 4f76e17036d2377483ce9ba6b4dee06f35acaa64 Mon Sep 17 00:00:00 2001 From: Nir Soffer Date: Mon, 20 Nov 2023 01:02:06 +0200 Subject: [PATCH 19/22] Add --skip-addons option This is useful for starting a stopped working environment quickly without trying to redeploy everything. The main motivation is using a pre created environment in location with weak network like a conference. Other use cases are working around bugs in addons that do not work well when starting a stopped cluster, for example clusteradm. With `--skip-addons` we skip the `start` and `stop` hooks, but we do run the `test` hooks. This is useful for starting a stopped environment faster but testing that the environment works. To skip all hooks run with both `--skip-addons` and `--skip-tests`. Example run: $ drenv start --skip-addons --skip-tests $env 2023-11-20 00:59:25,341 INFO [rdr-kubevirt] Starting environment 2023-11-20 00:59:25,464 INFO [dr1] Starting minikube cluster 2023-11-20 00:59:29,566 INFO [hub] Starting minikube cluster 2023-11-20 00:59:29,578 INFO [dr2] Starting minikube cluster 2023-11-20 01:00:23,402 INFO [dr1] Cluster started in 57.94 seconds 2023-11-20 01:00:23,402 INFO [dr1] Configuring containerd 2023-11-20 01:00:24,936 INFO [dr1] Waiting until all deployments are available 2023-11-20 01:00:28,749 INFO [hub] Cluster started in 59.18 seconds 2023-11-20 01:00:28,750 INFO [hub] Waiting until all deployments are available 2023-11-20 01:00:53,834 INFO [dr2] Cluster started in 84.26 seconds 2023-11-20 01:00:53,834 INFO [dr2] Configuring containerd 2023-11-20 01:00:55,042 INFO [dr2] Waiting until all deployments are available 2023-11-20 01:01:01,063 INFO [hub] Deployments are available in 32.31 seconds 2023-11-20 01:01:09,482 INFO [dr1] Deployments are available in 44.55 seconds 2023-11-20 01:01:34,661 INFO [dr2] Deployments are available in 39.62 seconds 2023-11-20 01:01:34,661 INFO [rdr-kubevirt] Dumping ramen e2e config to '/home/nsoffer/.config/drenv/rdr-kubevirt' 2023-11-20 01:01:34,827 INFO [rdr-kubevirt] Environment started in 129.49 seconds Signed-off-by: Nir Soffer --- test/drenv/__main__.py | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/test/drenv/__main__.py b/test/drenv/__main__.py index f8c48852a6..08a993ba11 100644 --- a/test/drenv/__main__.py +++ b/test/drenv/__main__.py @@ -30,7 +30,16 @@ def main(): p = argparse.ArgumentParser(prog="drenv") p.add_argument("-v", "--verbose", action="store_true", help="Be more verbose") p.add_argument( - "--skip-tests", dest="run_tests", action="store_false", help="Skip self tests" + "--skip-tests", + dest="run_tests", + action="store_false", + help="Skip addons 'test' hooks", + ) + p.add_argument( + "--skip-addons", + dest="run_addons", + action="store_false", + help="Skip addons 'start' and 'stop' hooks", ) p.add_argument("command", choices=commands, help="Command to run") p.add_argument("--name-prefix", help="Prefix profile names") @@ -52,7 +61,12 @@ def main(): def cmd_start(env, args): start = time.monotonic() logging.info("[%s] Starting environment", env["name"]) - hooks = ["start", "test"] if args.run_tests else ["start"] + + hooks = [] + if args.run_addons: + hooks.append("start") + if args.run_tests: + hooks.append("test") # Delaying `minikube start` ensures cluster start order. execute( @@ -62,7 +76,9 @@ def cmd_start(env, args): hooks=hooks, verbose=args.verbose, ) - execute(run_worker, env["workers"], hooks=hooks) + + if hooks: + execute(run_worker, env["workers"], hooks=hooks) if "ramen" in env: ramen.dump_e2e_config(env) @@ -77,7 +93,8 @@ def cmd_start(env, args): def cmd_stop(env, args): start = time.monotonic() logging.info("[%s] Stopping environment", env["name"]) - execute(stop_cluster, env["profiles"]) + hooks = ["stop"] if args.run_addons else [] + execute(stop_cluster, env["profiles"], hooks=hooks) logging.info( "[%s] Environment stopped in %.2f seconds", env["name"], @@ -147,17 +164,18 @@ def start_cluster(profile, hooks=(), verbose=False, **options): if is_restart: wait_for_deployments(profile) - execute(run_worker, profile["workers"], hooks=hooks) + if hooks: + execute(run_worker, profile["workers"], hooks=hooks) -def stop_cluster(profile, **options): +def stop_cluster(profile, hooks=(), **options): cluster_status = cluster.status(profile["name"]) - if cluster_status == cluster.READY: + if cluster_status == cluster.READY and hooks: execute( run_worker, profile["workers"], - hooks=["stop"], + hooks=hooks, reverse=True, allow_failure=True, ) From 71b6e0deaae81a454bf07abbf75742c75895850b Mon Sep 17 00:00:00 2001 From: Nir Soffer Date: Wed, 10 Jan 2024 15:07:37 +0200 Subject: [PATCH 20/22] Allow pulling from local insecure registry Configure CDI to allow pulling from a local insecure registry. This is useful for demos in an environment with unreliable network, or for CI environment when we want to avoid random failures due to flaky network. The image must be pushed to the local registry, this is easy using standard podman push command. Signed-off-by: Nir Soffer --- test/addons/cdi/start | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/test/addons/cdi/start b/test/addons/cdi/start index cedb78efd0..3ff72595ef 100755 --- a/test/addons/cdi/start +++ b/test/addons/cdi/start @@ -3,6 +3,7 @@ # SPDX-FileCopyrightText: The RamenDR authors # SPDX-License-Identifier: Apache-2.0 +import json import os import sys @@ -27,6 +28,19 @@ def deploy(cluster): print("Deploying cdi cr") kubectl.apply("--kustomize=cr", context=cluster) + print("Configuring insecure local registry") + patch = { + "spec": {"config": {"insecureRegistries": ["host.minikube.internal:5000"]}} + } + kubectl.patch( + "cdi", + "cdi", + "--patch", + json.dumps(patch), + "--type=merge", + context=cluster, + ) + def wait(cluster): print("Waiting until cdi cr is available") From 74c7dce326bd3d805c14a5ca605abe0c5b19afcc Mon Sep 17 00:00:00 2001 From: Nir Soffer Date: Wed, 10 Jan 2024 23:44:18 +0200 Subject: [PATCH 21/22] How to configure local git server Using a local git server we can deploy ocm applications without network access to github. This is useful for demos when the network is unreliable, for example in a conference. Signed-off-by: Nir Soffer --- test/gitlap/README.md | 88 +++++++++++++++++++++++++++++++++++++++++ test/gitlap/gitlap.conf | 17 ++++++++ 2 files changed, 105 insertions(+) create mode 100644 test/gitlap/README.md create mode 100644 test/gitlap/gitlap.conf diff --git a/test/gitlap/README.md b/test/gitlap/README.md new file mode 100644 index 0000000000..d71460c68e --- /dev/null +++ b/test/gitlap/README.md @@ -0,0 +1,88 @@ +# Setting up a local git server + +## Initial setup + +1. Install lighttpd + + ``` + sudo dnf install lighttpd + ``` + +1. Create the git repo + + Create a directory where the git repositories will be served: + + ``` + sudo mkdir /var/www/gitlap + cd /var/www/gitlap + sudo git clone --bare https://github.com/nirs/ocm-kubevirt-samples.git + ``` + + Set git repo permissions so you can push changes, and the web server + can serve the repo. + + ``` + sudo chown -R $USER:lighttpd /var/www/gitlap + ``` + +1. Copy the vhost configuration + + ``` + sudo cp gitlap.conf /etc/lighttpd/vhosts.d/ + ``` + +1. Uncomment the vhost include in /etc/lighttpd/lighttpd.conf + + ``` + include conf_dir + "/vhosts.d/*.conf" + ``` + +1. Enable and start the service + + ``` + sudo systemctl enable --now lighttpd + ``` + +1. Allow http access in the libvirt zone + + ``` + sudo firewall-cmd --zone=libvirt --add-service=http --permanent + sudo firewall-cmd --reload + ``` + +## Testing the server + +1. Add entry in /etc/hosts for testing locally + + ``` + 192.168.122.1 host.minikube.internal + ``` + +1. Check that git clone works + + ``` + git clone http://host.minikube.internal/ocm-kubevirt-samples.git + rm -rf ocm-kubevirt-samples + ``` + +1. Check git clone in a minikube cluster + + ``` + minikube ssh -p dr1 + git clone http://host.minikube.internal/ocm-kubevirt-samples.git + rm -rf ocm-kubevirt-samples + ``` + +## Updating the git repo + +1. Add a remote to your working repo + + ``` + git remote add gitlap file:///var/www/gitlap/ocm-kubevirt-samples.git + ``` + +1. Push changes to the remote + + ``` + git push -f gitlap main + ``` diff --git a/test/gitlap/gitlap.conf b/test/gitlap/gitlap.conf new file mode 100644 index 0000000000..bfe98fc706 --- /dev/null +++ b/test/gitlap/gitlap.conf @@ -0,0 +1,17 @@ +# Minimal configuration for local git server for minikube clusters. +# +# For more options see: +# https://redmine.lighttpd.net/projects/lighttpd/wiki/How_to_set_up_a_git_server_over_http(s) + +server.modules += ("mod_setenv", "mod_cgi", "mod_alias") + +# `host.minikube.internal` is a special DNS name injected by minikube to all clusters. +# https://minikube.sigs.k8s.io/docs/handbook/host-access/ +$HTTP["host"] == "host.minikube.internal" { + alias.url = ( "" => "/usr/libexec/git-core/git-http-backend" ) + setenv.set-environment = ( + "GIT_PROJECT_ROOT" => "/var/www/gitlap/", + "GIT_HTTP_EXPORT_ALL" => "1" + ) + cgi.assign = ( "" => "" ) +} From bda6fcb2c429f2219292b9e3c9f80e96f7f81536 Mon Sep 17 00:00:00 2001 From: Nir Soffer Date: Thu, 11 Jan 2024 00:16:13 +0200 Subject: [PATCH 22/22] How to use a local registry for minikube clusters Using local registry is useful for demos when network is unreliable, for example in a conference. It can also be used to avoid random failures when the network is flaky, by caching remove images locally. Signed-off-by: Nir Soffer --- test/registry/README.md | 73 +++++++++++++++++++++++ test/registry/host.minikube.internal.conf | 4 ++ 2 files changed, 77 insertions(+) create mode 100644 test/registry/README.md create mode 100644 test/registry/host.minikube.internal.conf diff --git a/test/registry/README.md b/test/registry/README.md new file mode 100644 index 0000000000..f172558e80 --- /dev/null +++ b/test/registry/README.md @@ -0,0 +1,73 @@ +# Using local registry for minikube clusters + +## Initial setup + +1. Install podman + + ``` + sudo dnf install podman + ``` + +1. Run the registry container + + ``` + podman run --name registry \ + --publish 5000:5000 \ + --volume registry:/var/lib/registry:Z \ + --detach \ + --replace \ + docker.io/library/registry:latest + ``` + + Use `--replace` to replace an existing container, typically left + after reboot the host. + +1. Allow access to port 5000 in the libvirt zone + + ``` + sudo firewall-cmd --zone=libvirt --add-port=5000/tcp --permanent + sudo firewall-cmd --reload + ``` + +1. Configure podman to allow insecure access + + ``` + sudo cp host.minikube.internal.conf /etc/containers/registries.conf.d/ + ``` + +1. Testing the registry + + ``` + $ curl host.minikube.internal:5000/v2/_catalog + {} + ``` + +## Pushing to the local registry + +1. Pull the image from a remote registry + + ``` + podman pull quay.io/nirsof/cirros:0.6.2-1 + ``` + +1. Push to the local registry + + ``` + podman push quay.io/nirsof/cirros:0.6.2-1 host.minikube.internal:5000/nirsof/cirros:0.6.2-1 + ``` + +## Using images from the local registry + +Example source.yaml: + +``` +--- +apiVersion: cdi.kubevirt.io/v1beta1 +kind: VolumeImportSource +metadata: + name: cirros-source +spec: + source: + registry: + url: "docker://host.minikube.internal:5000/nirsof/cirros:0.6.2-1" +``` diff --git a/test/registry/host.minikube.internal.conf b/test/registry/host.minikube.internal.conf new file mode 100644 index 0000000000..03bc08aa31 --- /dev/null +++ b/test/registry/host.minikube.internal.conf @@ -0,0 +1,4 @@ +# Local registry for serving minikube clusters +[[registry]] +location = 'host.minikube.internal:5000' +insecure = true