in clusterloader2/pkg/prometheus/util.go [61:106]
func CheckTargetsReady(k8sClient kubernetes.Interface, selector func(Target) bool, minActiveTargets, minReadyTargets int) (bool, error) {
raw, err := k8sClient.CoreV1().
Services(namespace).
ProxyGet("http", "prometheus-k8s", "9090", "api/v1/targets", nil /*params*/).
DoRaw(context.TODO())
if err != nil {
response := "(empty)"
if raw != nil {
response = string(raw)
}
// This might happen if prometheus server is temporary down, log error but don't return it.
klog.Warningf("error while calling prometheus api: %v, response: %v", err, response)
return false, nil
}
var response targetsResponse
if err := json.Unmarshal(raw, &response); err != nil {
return false, err // This shouldn't happen, return error.
}
nReady, nTotal := 0, 0
var exampleNotReadyTarget Target
for _, t := range response.Data.ActiveTargets {
if !selector(t) {
continue
}
nTotal++
if t.Health == "up" {
nReady++
continue
}
exampleNotReadyTarget = t
}
if nTotal < minActiveTargets {
klog.V(2).Infof("Not enough active targets (%d), expected at least (%d), waiting for more to become active...",
nTotal, minActiveTargets)
return false, nil
}
if minReadyTargets == allTargets {
minReadyTargets = nTotal
}
if nReady < minReadyTargets {
klog.V(2).Infof("%d/%d targets are ready, example not ready target: %v", nReady, minReadyTargets, exampleNotReadyTarget)
return false, nil
}
klog.V(2).Infof("All %d expected targets are ready", minReadyTargets)
return true, nil
}