diff --git a/lib/container/crio/client.go b/lib/container/crio/client.go index 8e271d1413..ee867412d7 100644 --- a/lib/container/crio/client.go +++ b/lib/container/crio/client.go @@ -17,6 +17,7 @@ package crio import ( "context" "encoding/json" + "errors" "flag" "fmt" "io" @@ -27,6 +28,11 @@ import ( "time" ) +// ErrContainerNotFound indicates that CRI-O does not yet know about the +// requested container. This is expected during a short window after the +// container cgroup is created but before CRI-O finishes registration. +var ErrContainerNotFound = errors.New("container not found") + var crioClientTimeout = flag.Duration("crio_client_timeout", time.Duration(0), "CRI-O client timeout. Default is no timeout.") const ( @@ -152,6 +158,9 @@ func (c *crioClientImpl) ContainerInfo(id string) (*ContainerInfo, error) { if err != nil { return nil, fmt.Errorf("error finding container %s: status %d", id, resp.StatusCode) } + if resp.StatusCode == http.StatusNotFound { + return nil, fmt.Errorf("error finding container %s: %s: %w", id, string(respBody), ErrContainerNotFound) + } return nil, fmt.Errorf("error finding container %s: status %d returned error %s", id, resp.StatusCode, string(respBody)) } diff --git a/lib/container/crio/handler.go b/lib/container/crio/handler.go index a7d9290dd7..1c6513b751 100644 --- a/lib/container/crio/handler.go +++ b/lib/container/crio/handler.go @@ -18,10 +18,12 @@ package crio import ( + "errors" "fmt" "path/filepath" "strconv" "strings" + "time" "github.com/opencontainers/cgroups" @@ -30,6 +32,8 @@ import ( containerlibcontainer "github.com/google/cadvisor/lib/container/libcontainer" "github.com/google/cadvisor/lib/fs" info "github.com/google/cadvisor/lib/model" + + "k8s.io/klog/v2" ) type crioContainerHandler struct { @@ -108,9 +112,27 @@ func newCrioContainerHandler( id := ContainerNameToCrioId(name) pidKnown := true - cInfo, err := client.ContainerInfo(id) - if err != nil { - return nil, err + // Cgroup is created during container setup. When cadvisor sees the cgroup + // via inotify, the container may not be fully registered in CRI-O yet. + // Use retry+backoff to tolerate the race condition, mirroring the + // containerd handler's approach for the same issue. + var cInfo *ContainerInfo + backoff := 100 * time.Millisecond + retry := 5 + for { + cInfo, err = client.ContainerInfo(id) + if err == nil && cInfo != nil { + break + } + + if !errors.Is(err, ErrContainerNotFound) || retry == 0 { + return nil, err + } + + klog.V(4).Infof("Container %s not yet registered in CRI-O, retrying (%d retries left): %v", id, retry, err) + retry-- + time.Sleep(backoff) + backoff *= 2 } if cInfo.Pid == 0 { // If pid is not known yet, network related stats can not be retrieved by the