OCPBUGS-77056: Make external cert validation asynchronous (v3 — fix re-admission after secret deletion)

hack/Makefile.debug

@@ -1,6 +1,7 @@
 # -*- mode: makefile -*-
 
 export GOOS=linux
+export GOARCH ?= amd64
 
 REGISTRY ?= quay.io
 IMAGE ?= openshift/openshift-router
@@ -11,8 +12,8 @@ REMOTE_IMAGE ?= $(REGISTRY)/$(IMAGE):$(TAG)
 OPENSHIFT_ENDPOINT ?= $(shell oc config view --minify --template '{{(index .clusters 0).cluster.server}}' | grep -o '//[^ :]*' | sed 's/^..//')
 
 new-openshift-router-image:
-	GO111MODULE=on CGO_ENABLED=0 GOFLAGS=-mod=vendor go build -o openshift-router -gcflags=all="-N -l" ./cmd/openshift-router
-	$(IMAGEBUILDER) build -t $(LOCAL_IMAGE) -f hack/Dockerfile.debug .
+	GO111MODULE=on CGO_ENABLED=0 GOOS=$(GOOS) GOARCH=$(GOARCH) GOFLAGS=-mod=vendor go build -o openshift-router -gcflags=all="-N -l" ./cmd/openshift-router
+	$(IMAGEBUILDER) build --arch $(GOARCH) -t $(LOCAL_IMAGE) -f hack/Dockerfile.debug .
 
 push:
 	$(IMAGEBUILDER) tag $(LOCAL_IMAGE) $(REMOTE_IMAGE)

pkg/cmd/infra/router/clientcmd.go

@@ -60,6 +60,13 @@ func (cfg *Config) KubeConfig() (*restclient.Config, string, error) {
 	if err != nil {
 		return nil, "", err
 	}
+
+	// Increase client-side rate limiting to support higher throughput during
+	// router startup, especially when many external certificate routes are
+	// present.
+	clientConfig.QPS = 50
+	clientConfig.Burst = 100
+
 	return clientConfig, namespace, nil
 }
 

pkg/cmd/infra/router/template.go

@@ -37,13 +37,13 @@ import (
 	routelisters "github.com/openshift/client-go/route/listers/route/v1"
 	"github.com/openshift/library-go/pkg/crypto"
 	"github.com/openshift/library-go/pkg/proc"
-	"github.com/openshift/library-go/pkg/route/secretmanager"
 
 	"github.com/openshift/router/pkg/router"
 	"github.com/openshift/router/pkg/router/client"
 	"github.com/openshift/router/pkg/router/controller"
 	"github.com/openshift/router/pkg/router/metrics"
 	"github.com/openshift/router/pkg/router/metrics/haproxy"
+
 	"github.com/openshift/router/pkg/router/shutdown"
 	templateplugin "github.com/openshift/router/pkg/router/template"
 	haproxyconfigmanager "github.com/openshift/router/pkg/router/template/configmanager/haproxy"
@@ -770,7 +770,7 @@ func (o *TemplateRouterOptions) Run(stopCh <-chan struct{}) error {
 		return err
 	}
 
-	secretManager := secretmanager.NewManager(kc, nil)
+	secretManager := controller.NewSharedSecretManager(kc, nil)
 
 	pluginCfg := templateplugin.TemplatePluginConfig{
 		AppCtx:                        ctx,
@@ -816,7 +816,7 @@ func (o *TemplateRouterOptions) Run(stopCh <-chan struct{}) error {
 	informer := factory.CreateRoutesSharedInformer()
 	routeLister := routelisters.NewRouteLister(informer.GetIndexer())
 	if o.UpdateStatus {
-		lease := writerlease.New(time.Minute, 3*time.Second)
+		lease := writerlease.New(time.Minute, 3*time.Second, 1)
 		go lease.Run(stopCh)
 		tracker := controller.NewSimpleContentionTracker(informer, o.RouterName, o.ResyncInterval/10)
 		tracker.SetConflictMessage(fmt.Sprintf("The router detected another process is writing conflicting updates to route status with name %q. Please ensure that the configuration of all routers is consistent. Route status will not be updated as long as conflicts are detected.", o.RouterName))

pkg/router/controller/contention.go

@@ -46,6 +46,7 @@ var (
 		ExtCrtStatusReasonSecretRecreated,
 		ExtCrtStatusReasonSecretUpdated,
 		ExtCrtStatusReasonSecretDeleted,
+		ExtCrtStatusReasonSARCompleted,
 	)
 )
 
@@ -294,8 +295,10 @@ func ingressConditionsEqual(a, b []routev1.RouteIngressCondition) bool {
 
 // conditionsEqual compares two RouteIngressConditions, ignoring LastTransitionTime and any reason in ignoreIngressConditionReason.
 func conditionsEqual(a, b *routev1.RouteIngressCondition) bool {
-	if ignoreIngressConditionReason.Has(a.Reason) || ignoreIngressConditionReason.Has(b.Reason) {
-		return true
+	if a.Type == b.Type && a.Status == b.Status {
+		if ignoreIngressConditionReason.Has(a.Reason) || ignoreIngressConditionReason.Has(b.Reason) {
+			return true
+		}
 	}
 	return a.Type == b.Type &&
 		a.Status == b.Status &&

pkg/router/controller/factory/factory_endpointslices_test.go

@@ -2,7 +2,6 @@ package factory_test
 
 import (
 	"context"
-	"os"
 	"testing"
 	"time"
 
@@ -77,15 +76,15 @@ func protocolPtr(p kapi.Protocol) *kapi.Protocol {
 	return &p
 }
 
-func newEndpointSliceTestSetup(plugin router.Plugin, initialObjects ...runtime.Object) (*fakekubeclient.Clientset, chan struct{}) {
+func newEndpointSliceTestSetup(t *testing.T, plugin router.Plugin, initialObjects ...runtime.Object) (*fakekubeclient.Clientset, chan struct{}) {
 	stopCh := make(chan struct{})
 	client := fakekubeclient.NewSimpleClientset(initialObjects...)
 	fakeProject := &fakeproject.FakeProjectV1{}
 
 	// WatchListClient featuregate is enabled by default since v0.35. Fake client does not support
 	// initializing its cache from Watch, so falling back to use List instead. The envvar below
 	// configures the featuregate state.
-	os.Setenv("KUBE_FEATURE_"+string(features.WatchListClient), "False")
+	t.Setenv("KUBE_FEATURE_"+string(features.WatchListClient), "False")
 
 	factory.NewDefaultRouterControllerFactory(
 		fakerouterclient.NewSimpleClientset(),
@@ -104,7 +103,7 @@ func TestEndpointSlicesAdd(t *testing.T) {
 		handleEndpointsCh: make(chan handleEndpointsEvent),
 	}
 
-	client, stopCh := newEndpointSliceTestSetup(plugin)
+	client, stopCh := newEndpointSliceTestSetup(t, plugin)
 	defer close(stopCh)
 
 	type testCase struct {
@@ -339,7 +338,7 @@ func TestEndpointSlicesDelete(t *testing.T) {
 		handleEndpointsCh: make(chan handleEndpointsEvent),
 	}
 
-	client, stopCh := newEndpointSliceTestSetup(plugin)
+	client, stopCh := newEndpointSliceTestSetup(t, plugin)
 	defer close(stopCh)
 
 	for _, eps := range []discoveryv1.EndpointSlice{eps1, eps2} {