diff --git a/cmd/openshift-install/testdata/agent/image/configurations/tnf_with_fencing_credentials_by_mac.txt b/cmd/openshift-install/testdata/agent/image/configurations/tnf_with_fencing_credentials_by_mac.txt new file mode 100644 index 00000000000..658270fd12d --- /dev/null +++ b/cmd/openshift-install/testdata/agent/image/configurations/tnf_with_fencing_credentials_by_mac.txt @@ -0,0 +1,92 @@ +# Verify TNF configuration with MAC-keyed fencing credentials placed in per-host directories + +exec openshift-install agent create image --dir $WORK + +stderr 'The rendezvous host IP \(node0 IP\) is 192.168.111.20' + +exists $WORK/agent.x86_64.iso +exists $WORK/auth/kubeconfig +exists $WORK/auth/kubeadmin-password + +# Verify per-host fencing credentials are placed in each host's directory +isoIgnitionContains agent.x86_64.iso /etc/assisted/hostconfig/master-0/fencing-credentials.yaml +isocmp agent.x86_64.iso /etc/assisted/hostconfig/master-0/fencing-credentials.yaml expected/master-0-fencing-credentials.yaml + +isoIgnitionContains agent.x86_64.iso /etc/assisted/hostconfig/master-1/fencing-credentials.yaml +isocmp agent.x86_64.iso /etc/assisted/hostconfig/master-1/fencing-credentials.yaml expected/master-1-fencing-credentials.yaml + +-- install-config.yaml -- +apiVersion: v1 +baseDomain: test.metalkube.org +controlPlane: + name: master + replicas: 2 + fencing: + credentials: + - macAddress: "00:aa:bb:cc:dd:01" + address: redfish+https://192.168.111.1:8000/redfish/v1/Systems/abc123 + username: admin + password: password123 + certificateVerification: Disabled + - macAddress: "00:aa:bb:cc:dd:02" + address: redfish+https://192.168.111.2:8000/redfish/v1/Systems/def456 + username: admin + password: password456 + certificateVerification: Enabled +compute: +- name: worker + replicas: 0 +metadata: + namespace: cluster0 + name: ostest +networking: + clusterNetwork: + - cidr: 10.128.0.0/14 + hostPrefix: 23 + networkType: OVNKubernetes + machineNetwork: + - cidr: 192.168.111.0/24 + serviceNetwork: + - 172.30.0.0/16 +platform: + baremetal: + apiVips: + - 192.168.111.5 + ingressVips: + - 192.168.111.4 +sshKey: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDK6UTEydcEKzuNdPaofn8Z2DwgHqdcionLZBiPf/zIRNco++etLsat7Avv7yt04DINQd5zjxIFgG8jblaUB5E5C9ClUcMwb52GO0ay2Y9v1uBv1a4WhI3peKktAzYNk0EBMQlJtXPjRMrC9ylBPh+DsBHMu+KmDnfk7PIwyN4efC8k5kSRuPWoNdme1rz2+umU8FSmaWTHIajrbspf4GQbsntA5kuKEtDbfoNCU97o2KrRnUbeg3a8hwSjfh3u6MhlnGcg5K2Ij+zivEsWGCLKYUtE1ErqwfIzwWmJ6jnV66XCQGHf4Q1iIxqF7s2a1q24cgG2Z/iDXfqXrCIfy4P7b/Ztak3bdT9jfAdVZtdO5/r7I+O5hYhF86ayFlDWzZWP/ByiSb+q4CQbfVgK3BMmiAv2MqLHdhesmD/SmIcoOWUF6rFmRKZVFFpKpt5ATNTgUJ3JRowoXrrDruVXClUGRiCS6Zabd1rZ3VmTchaPJwtzQMdfIWISXj+Ig+C4UK0= +pullSecret: '{"auths": {"quay.io": {"auth": "c3VwZXItc2VjcmV0Cg=="}}}' +featureSet: TechPreviewNoUpgrade + +-- agent-config.yaml -- +apiVersion: v1alpha1 +metadata: + name: ostest + namespace: cluster0 +rendezvousIP: 192.168.111.20 +hosts: + - hostname: master-0 + role: master + interfaces: + - name: eth0 + macAddress: "00:aa:bb:cc:dd:01" + - hostname: master-1 + role: master + interfaces: + - name: eth0 + macAddress: "00:aa:bb:cc:dd:02" + +-- expected/master-0-fencing-credentials.yaml -- +credentials: +- macaddress: 00:aa:bb:cc:dd:01 + username: admin + password: password123 + address: redfish+https://192.168.111.1:8000/redfish/v1/Systems/abc123 + certificateVerification: Disabled +-- expected/master-1-fencing-credentials.yaml -- +credentials: +- macaddress: 00:aa:bb:cc:dd:02 + username: admin + password: password456 + address: redfish+https://192.168.111.2:8000/redfish/v1/Systems/def456 + certificateVerification: Enabled diff --git a/pkg/asset/agent/agentconfig/agenthosts.go b/pkg/asset/agent/agentconfig/agenthosts.go index 652d6ead436..9265518edde 100644 --- a/pkg/asset/agent/agentconfig/agenthosts.go +++ b/pkg/asset/agent/agentconfig/agenthosts.go @@ -8,6 +8,7 @@ import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" + goyaml "gopkg.in/yaml.v2" "k8s.io/apimachinery/pkg/util/validation/field" "sigs.k8s.io/yaml" @@ -16,6 +17,7 @@ import ( agentAsset "github.com/openshift/installer/pkg/asset/agent" "github.com/openshift/installer/pkg/asset/agent/joiner" "github.com/openshift/installer/pkg/asset/agent/workflow" + "github.com/openshift/installer/pkg/types" "github.com/openshift/installer/pkg/types/agent" "github.com/openshift/installer/pkg/types/baremetal/validation" "github.com/openshift/installer/pkg/validate" @@ -41,8 +43,9 @@ type nmStateInterface struct { // AgentHosts generates the hosts information from the AgentConfig and // OptionalInstallConfig assets. type AgentHosts struct { - Hosts []agent.Host - rendezvousIP string + Hosts []agent.Host + FencingCredentialsHost []types.Credential + rendezvousIP string } // Name returns a human friendly name. @@ -93,6 +96,9 @@ func (a *AgentHosts) Generate(_ context.Context, dependencies asset.Parents) err } } + // store per host fencing-credentials only when MAC address is used + a.populateFencingCredentialHosts(installConfig) + case workflow.AgentWorkflowTypeAddNodes: a.Hosts = append(a.Hosts, addNodesConfig.Config.Hosts...) for i, host := range a.Hosts { @@ -367,5 +373,52 @@ func (a *AgentHosts) HostConfigFiles() (HostConfigFileMap, error) { files[filepath.Join(name, "role")] = []byte(host.Role) } } + + maxNewDirs := len(a.Hosts) + for i := range a.FencingCredentialsHost { + cred := &a.FencingCredentialsHost[i] + dirName := findHostDirForMAC(files, cred.MACAddress) + if dirName == "" { + dirName = fmt.Sprintf("host-%d", maxNewDirs) + maxNewDirs++ + files[filepath.Join(dirName, "mac_addresses")] = []byte(strings.ToLower(cred.MACAddress) + "\n") + } + cfg := &FencingCredentialsConfig{Credentials: []*types.Credential{cred}} + data, err := goyaml.Marshal(cfg) + if err != nil { + return nil, err + } + files[filepath.Join(dirName, "fencing-credentials.yaml")] = data + } + return files, nil } + +func (a *AgentHosts) populateFencingCredentialHosts(installConfig *agentAsset.OptionalInstallConfig) { + if installConfig.Config == nil || installConfig.Config.ControlPlane == nil || + installConfig.Config.ControlPlane.Fencing == nil { + return + } + + for _, cred := range installConfig.Config.ControlPlane.Fencing.Credentials { + if cred.HostName == "" && cred.MACAddress != "" { + a.FencingCredentialsHost = append(a.FencingCredentialsHost, *cred) + } + } +} + +func findHostDirForMAC(files HostConfigFileMap, macAddress string) string { + normalizedMAC := strings.ToLower(macAddress) + for key, content := range files { + if !strings.HasSuffix(key, "/mac_addresses") { + continue + } + dirName := strings.TrimSuffix(key, "/mac_addresses") + for _, mac := range strings.Split(strings.TrimSpace(string(content)), "\n") { + if strings.TrimSpace(mac) == normalizedMAC { + return dirName + } + } + } + return "" +} diff --git a/pkg/asset/agent/agentconfig/agenthosts_test.go b/pkg/asset/agent/agentconfig/agenthosts_test.go index 36727af3926..ea8488c3736 100644 --- a/pkg/asset/agent/agentconfig/agenthosts_test.go +++ b/pkg/asset/agent/agentconfig/agenthosts_test.go @@ -955,6 +955,200 @@ func (ib *InterfacetBuilder) build() *aiv1beta1.Interface { return &ib.Interface } +func TestAgentHosts_FencingCredentialsHost(t *testing.T) { + cases := []struct { + name string + dependencies []asset.Asset + expectedCredentials []types.Credential + }{ + { + name: "mac-keyed credentials are collected", + dependencies: []asset.Asset{ + &workflow.AgentWorkflow{Workflow: workflow.AgentWorkflowTypeInstall}, + &joiner.AddNodesConfig{}, + getInstallConfigWithFencing([]*types.Credential{ + {MACAddress: "28:d2:44:d2:b2:1a", Username: "admin", Password: "pass", Address: "redfish+https://10.0.0.1/redfish/v1/Systems/1"}, + }), + getAgentConfigMultiHost("worker"), + }, + expectedCredentials: []types.Credential{ + {MACAddress: "28:d2:44:d2:b2:1a", Username: "admin", Password: "pass", Address: "redfish+https://10.0.0.1/redfish/v1/Systems/1"}, + }, + }, + { + name: "hostname-keyed credentials are excluded", + dependencies: []asset.Asset{ + &workflow.AgentWorkflow{Workflow: workflow.AgentWorkflowTypeInstall}, + &joiner.AddNodesConfig{}, + getInstallConfigWithFencing([]*types.Credential{ + {HostName: "master-0", Username: "admin", Password: "pass", Address: "redfish+https://10.0.0.1/redfish/v1/Systems/1"}, + }), + getAgentConfigMultiHost("worker"), + }, + expectedCredentials: nil, + }, + { + name: "mixed credentials only collect mac-keyed", + dependencies: []asset.Asset{ + &workflow.AgentWorkflow{Workflow: workflow.AgentWorkflowTypeInstall}, + &joiner.AddNodesConfig{}, + getInstallConfigWithFencing([]*types.Credential{ + {HostName: "master-0", Username: "admin", Password: "pass", Address: "redfish+https://10.0.0.1/redfish/v1/Systems/1"}, + {MACAddress: "28:d2:44:d2:b2:1b", Username: "admin2", Password: "pass2", Address: "redfish+https://10.0.0.2/redfish/v1/Systems/1"}, + }), + getAgentConfigMultiHost("worker"), + }, + expectedCredentials: []types.Credential{ + {MACAddress: "28:d2:44:d2:b2:1b", Username: "admin2", Password: "pass2", Address: "redfish+https://10.0.0.2/redfish/v1/Systems/1"}, + }, + }, + { + name: "credential with both hostname and mac is treated as hostname-keyed", + dependencies: []asset.Asset{ + &workflow.AgentWorkflow{Workflow: workflow.AgentWorkflowTypeInstall}, + &joiner.AddNodesConfig{}, + getInstallConfigWithFencing([]*types.Credential{ + {HostName: "master-0", MACAddress: "28:d2:44:d2:b2:1a", Username: "admin", Password: "pass", Address: "redfish+https://10.0.0.1/redfish/v1/Systems/1"}, + }), + getAgentConfigMultiHost("worker"), + }, + expectedCredentials: nil, + }, + { + name: "no fencing config produces nil", + dependencies: []asset.Asset{ + &workflow.AgentWorkflow{Workflow: workflow.AgentWorkflowTypeInstall}, + &joiner.AddNodesConfig{}, + getInstallConfigSingleHost(), + getAgentConfigSingleHost(), + }, + expectedCredentials: nil, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + parents := asset.Parents{} + parents.Add(tc.dependencies...) + + ah := &AgentHosts{} + err := ah.Generate(context.Background(), parents) + assert.NoError(t, err) + assert.Equal(t, tc.expectedCredentials, ah.FencingCredentialsHost) + }) + } +} + +func TestHostConfigFiles_FencingCredentials(t *testing.T) { + cases := []struct { + name string + hosts []agent.Host + credentials []types.Credential + expectedFiles map[string]string + }{ + { + name: "mac matches existing host directory", + hosts: []agent.Host{ + { + Hostname: "master-0", + Interfaces: []*aiv1beta1.Interface{{MacAddress: "aa:bb:cc:dd:ee:01"}}, + }, + }, + credentials: []types.Credential{ + {MACAddress: "AA:BB:CC:DD:EE:01", Username: "admin", Password: "pass", Address: "redfish+https://10.0.0.1/redfish/v1/Systems/1"}, + }, + expectedFiles: map[string]string{ + "master-0/fencing-credentials.yaml": "username: admin", + }, + }, + { + name: "mac with no matching host creates new directory", + hosts: []agent.Host{ + { + Hostname: "master-0", + Interfaces: []*aiv1beta1.Interface{{MacAddress: "aa:bb:cc:dd:ee:01"}}, + }, + }, + credentials: []types.Credential{ + {MACAddress: "FF:FF:FF:FF:FF:FF", Username: "admin", Password: "pass", Address: "redfish+https://10.0.0.1/redfish/v1/Systems/1"}, + }, + expectedFiles: map[string]string{ + "host-1/fencing-credentials.yaml": "username: admin", + "host-1/mac_addresses": "ff:ff:ff:ff:ff:ff", + }, + }, + { + name: "case-insensitive mac matching", + hosts: []agent.Host{ + { + Hostname: "master-0", + Interfaces: []*aiv1beta1.Interface{{MacAddress: "aa:bb:cc:dd:ee:01"}}, + }, + }, + credentials: []types.Credential{ + {MACAddress: "AA:BB:CC:DD:EE:01", Username: "admin", Password: "pass", Address: "redfish+https://10.0.0.1/redfish/v1/Systems/1"}, + }, + expectedFiles: map[string]string{ + "master-0/fencing-credentials.yaml": "username: admin", + }, + }, + { + name: "new dir avoids collision with existing host-0", + hosts: []agent.Host{ + { + Interfaces: []*aiv1beta1.Interface{{MacAddress: "aa:bb:cc:dd:ee:01"}}, + }, + }, + credentials: []types.Credential{ + {MACAddress: "FF:FF:FF:FF:FF:FF", Username: "admin", Password: "pass", Address: "redfish+https://10.0.0.1/redfish/v1/Systems/1"}, + }, + expectedFiles: map[string]string{ + "host-1/fencing-credentials.yaml": "username: admin", + "host-1/mac_addresses": "ff:ff:ff:ff:ff:ff", + }, + }, + { + name: "new dir avoids collision with empty host that has no files", + hosts: []agent.Host{ + {}, + }, + credentials: []types.Credential{ + {MACAddress: "FF:FF:FF:FF:FF:FF", Username: "admin", Password: "pass", Address: "redfish+https://10.0.0.1/redfish/v1/Systems/1"}, + }, + expectedFiles: map[string]string{ + "host-1/fencing-credentials.yaml": "username: admin", + "host-1/mac_addresses": "ff:ff:ff:ff:ff:ff", + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ah := &AgentHosts{ + Hosts: tc.hosts, + FencingCredentialsHost: tc.credentials, + } + + files, err := ah.HostConfigFiles() + assert.NoError(t, err) + + for path, expectedContent := range tc.expectedFiles { + data, ok := files[path] + assert.True(t, ok, "expected file at %s", path) + assert.Contains(t, string(data), expectedContent) + } + }) + } +} + +func getInstallConfigWithFencing(credentials []*types.Credential) *agentAsset.OptionalInstallConfig { + ic := getNoHostsInstallConfig() + ic.Config.ControlPlane.Fencing = &types.Fencing{ + Credentials: credentials, + } + return ic +} + func getInstallConfigWithMismatchedNetworkConfig() *agentAsset.OptionalInstallConfig { a := getInstallConfigSingleHost() a.Config.Platform.BareMetal.Hosts[0].NetworkConfig = &apiextv1.JSON{ diff --git a/pkg/asset/agent/agentconfig/fencingcredentials_test.go b/pkg/asset/agent/agentconfig/fencingcredentials_test.go index a18f4095f6d..d3b4f80a9f1 100644 --- a/pkg/asset/agent/agentconfig/fencingcredentials_test.go +++ b/pkg/asset/agent/agentconfig/fencingcredentials_test.go @@ -442,6 +442,8 @@ func TestFencingCredentials_YAMLFormat(t *testing.T) { assert.NotContains(t, yamlContent, "macaddress:") }) + // This subtest verifies YAML format only. Placement into per-host directories + // for MAC-keyed credentials is handled by addFencingCredentials() in ignition.go. t.Run("with macaddress", func(t *testing.T) { parents := asset.Parents{} parents.Add( diff --git a/pkg/asset/agent/image/ignition.go b/pkg/asset/agent/image/ignition.go index 1727b672636..7770a194abc 100644 --- a/pkg/asset/agent/image/ignition.go +++ b/pkg/asset/agent/image/ignition.go @@ -353,7 +353,9 @@ func (a *Ignition) Generate(ctx context.Context, dependencies asset.Parents) err return err } - addFencingCredentials(&config, fencingCredentials) + if err = addFencingCredentialsByHostname(&config, fencingCredentials); err != nil { + return err + } err = addExtraManifests(&config, extraManifests) if err != nil { @@ -638,20 +640,35 @@ func addHostConfig(config *igntypes.Config, agentHosts *agentconfig.AgentHosts) return nil } -// addFencingCredentials adds the fencing credentials file to the ignition config. -// Fencing credentials are host-scoped, so they go under /etc/assisted/hostconfig/ -// rather than /etc/assisted/manifests/ which is for cluster-scoped manifests. -func addFencingCredentials(config *igntypes.Config, fencingCredentials *agentconfig.FencingCredentials) { - if fencingCredentials == nil || fencingCredentials.File == nil { - return +// addFencingCredentialsByHostname adds hostname-keyed fencing credentials to the ignition config +// at /etc/assisted/hostconfig/fencing-credentials.yaml. MAC-keyed credentials are handled +// separately through AgentHosts.HostConfigFiles() and addHostConfig(). +func addFencingCredentialsByHostname(config *igntypes.Config, fencingCredentials *agentconfig.FencingCredentials) error { + if fencingCredentials == nil || fencingCredentials.Config == nil || len(fencingCredentials.Config.Credentials) == 0 { + return nil + } + + var hostnameCredentials []*types.Credential + for _, cred := range fencingCredentials.Config.Credentials { + if cred.HostName != "" { + hostnameCredentials = append(hostnameCredentials, cred) + } } - fencingFile := ignition.FileFromBytes( + if len(hostnameCredentials) == 0 { + return nil + } + + cfg := &agentconfig.FencingCredentialsConfig{Credentials: hostnameCredentials} + data, err := yaml.Marshal(cfg) + if err != nil { + return errors.Wrap(err, "failed to marshal hostname fencing credentials") + } + config.Storage.Files = append(config.Storage.Files, ignition.FileFromBytes( path.Join("/etc/assisted/hostconfig", "fencing-credentials.yaml"), - "root", 0644, - fencingCredentials.File.Data, - ) - config.Storage.Files = append(config.Storage.Files, fencingFile) + "root", 0644, data, + )) + return nil } func addDay2ClusterConfigFiles(config *igntypes.Config, clusterInfo joiner.ClusterInfo, importClusterConfig joiner.ImportClusterConfig) error { diff --git a/pkg/asset/agent/image/ignition_test.go b/pkg/asset/agent/image/ignition_test.go index a6534245d09..f8399605090 100644 --- a/pkg/asset/agent/image/ignition_test.go +++ b/pkg/asset/agent/image/ignition_test.go @@ -988,3 +988,72 @@ func TestIgnition_getPublicContainerRegistries(t *testing.T) { }) } } + +func TestAddFencingCredentialsByHostname(t *testing.T) { + cases := []struct { + name string + credentials []*types.Credential + expectedFiles int + }{ + { + name: "hostname-only credentials go to top-level file", + credentials: []*types.Credential{ + {HostName: "master-0", Username: "admin", Password: "pass", Address: "redfish+https://10.0.0.1/redfish/v1/Systems/1"}, + {HostName: "master-1", Username: "admin", Password: "pass", Address: "redfish+https://10.0.0.2/redfish/v1/Systems/1"}, + }, + expectedFiles: 1, + }, + { + name: "mac-only credentials are excluded", + credentials: []*types.Credential{ + {MACAddress: "AA:BB:CC:DD:EE:01", Username: "admin", Password: "pass", Address: "redfish+https://10.0.0.1/redfish/v1/Systems/1"}, + }, + expectedFiles: 0, + }, + { + name: "hostname takes precedence when both set", + credentials: []*types.Credential{ + {HostName: "master-0", MACAddress: "AA:BB:CC:DD:EE:01", Username: "admin", Password: "pass", Address: "redfish+https://10.0.0.1/redfish/v1/Systems/1"}, + }, + expectedFiles: 1, + }, + { + name: "mixed credentials only includes hostname-keyed", + credentials: []*types.Credential{ + {HostName: "master-0", Username: "admin", Password: "pass", Address: "redfish+https://10.0.0.1/redfish/v1/Systems/1"}, + {MACAddress: "AA:BB:CC:DD:EE:02", Username: "admin", Password: "pass", Address: "redfish+https://10.0.0.2/redfish/v1/Systems/1"}, + }, + expectedFiles: 1, + }, + { + name: "nil credentials produces no files", + credentials: nil, + expectedFiles: 0, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + config := &igntypes.Config{} + var fencingCreds *agentconfig.FencingCredentials + if tc.credentials != nil { + fencingCreds = &agentconfig.FencingCredentials{ + Config: &agentconfig.FencingCredentialsConfig{ + Credentials: tc.credentials, + }, + } + } + + err := addFencingCredentialsByHostname(config, fencingCreds) + assert.NoError(t, err) + assert.Equal(t, tc.expectedFiles, len(config.Storage.Files)) + + for _, f := range config.Storage.Files { + assert.Equal(t, "/etc/assisted/hostconfig/fencing-credentials.yaml", f.Path) + actualData, err := dataurl.DecodeString(*f.FileEmbedded1.Contents.Source) + assert.NoError(t, err) + assert.Contains(t, string(actualData.Data), "credentials:") + } + }) + } +}