crio: add retry to ContainerInfo call

lib/container/crio/client.go

@@ -17,6 +17,7 @@ package crio
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"flag"
 	"fmt"
 	"io"
@@ -27,6 +28,11 @@ import (
 	"time"
 )
 
+// ErrContainerNotFound indicates that CRI-O does not yet know about the
+// requested container. This is expected during a short window after the
+// container cgroup is created but before CRI-O finishes registration.
+var ErrContainerNotFound = errors.New("container not found")
+
 var crioClientTimeout = flag.Duration("crio_client_timeout", time.Duration(0), "CRI-O client timeout. Default is no timeout.")
 
 const (
@@ -152,6 +158,9 @@ func (c *crioClientImpl) ContainerInfo(id string) (*ContainerInfo, error) {
 		if err != nil {
 			return nil, fmt.Errorf("error finding container %s: status %d", id, resp.StatusCode)
 		}
+		if resp.StatusCode == http.StatusNotFound {
+			return nil, fmt.Errorf("error finding container %s: %s: %w", id, string(respBody), ErrContainerNotFound)
+		}
 		return nil, fmt.Errorf("error finding container %s: status %d returned error %s", id, resp.StatusCode, string(respBody))
 	}
 

lib/container/crio/handler.go

@@ -18,10 +18,12 @@
 package crio
 
 import (
+	"errors"
 	"fmt"
 	"path/filepath"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/opencontainers/cgroups"
 
@@ -30,6 +32,8 @@ import (
 	containerlibcontainer "github.com/google/cadvisor/lib/container/libcontainer"
 	"github.com/google/cadvisor/lib/fs"
 	info "github.com/google/cadvisor/lib/model"
+
+	"k8s.io/klog/v2"
 )
 
 type crioContainerHandler struct {
@@ -108,9 +112,27 @@ func newCrioContainerHandler(
 	id := ContainerNameToCrioId(name)
 	pidKnown := true
 
-	cInfo, err := client.ContainerInfo(id)
-	if err != nil {
-		return nil, err
+	// Cgroup is created during container setup. When cadvisor sees the cgroup
+	// via inotify, the container may not be fully registered in CRI-O yet.
+	// Use retry+backoff to tolerate the race condition, mirroring the
+	// containerd handler's approach for the same issue.
+	var cInfo *ContainerInfo
+	backoff := 100 * time.Millisecond
+	retry := 5
+	for {
+		cInfo, err = client.ContainerInfo(id)
+		if err == nil && cInfo != nil {
+			break
+		}
+
+		if !errors.Is(err, ErrContainerNotFound) || retry == 0 {
+			return nil, err
+		}
+
+		klog.V(4).Infof("Container %s not yet registered in CRI-O, retrying (%d retries left): %v", id, retry, err)
+		retry--
+		time.Sleep(backoff)
+		backoff *= 2
 	}
 	if cInfo.Pid == 0 {
 		// If pid is not known yet, network related stats can not be retrieved by the