|
@@ -86,6 +86,7 @@ type QueryOpts struct {
|
|
|
StartRange uint `schema:"startrange"`
|
|
StartRange uint `schema:"startrange"`
|
|
|
EndRange uint `schema:"endrange"`
|
|
EndRange uint `schema:"endrange"`
|
|
|
Resolution string `schema:"resolution"`
|
|
Resolution string `schema:"resolution"`
|
|
|
|
|
+ Percentile float64 `schema:"percentile"`
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
func QueryPrometheus(
|
|
func QueryPrometheus(
|
|
@@ -97,7 +98,7 @@ func QueryPrometheus(
|
|
|
return nil, fmt.Errorf("prometheus service has no exposed ports to query")
|
|
return nil, fmt.Errorf("prometheus service has no exposed ports to query")
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
- podSelectionRegex, err := getPodSelectionRegex(opts.Kind, opts.Name)
|
|
|
|
|
|
|
+ selectionRegex, err := getSelectionRegex(opts.Kind, opts.Name)
|
|
|
|
|
|
|
|
if err != nil {
|
|
if err != nil {
|
|
|
return nil, err
|
|
return nil, err
|
|
@@ -108,7 +109,7 @@ func QueryPrometheus(
|
|
|
if len(opts.PodList) > 0 {
|
|
if len(opts.PodList) > 0 {
|
|
|
podSelector = fmt.Sprintf(`namespace="%s",pod=~"%s",container!="POD",container!=""`, opts.Namespace, strings.Join(opts.PodList, "|"))
|
|
podSelector = fmt.Sprintf(`namespace="%s",pod=~"%s",container!="POD",container!=""`, opts.Namespace, strings.Join(opts.PodList, "|"))
|
|
|
} else {
|
|
} else {
|
|
|
- podSelector = fmt.Sprintf(`namespace="%s",pod=~"%s",container!="POD",container!=""`, opts.Namespace, podSelectionRegex)
|
|
|
|
|
|
|
+ podSelector = fmt.Sprintf(`namespace="%s",pod=~"%s",container!="POD",container!=""`, opts.Namespace, selectionRegex)
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
query := ""
|
|
query := ""
|
|
@@ -118,12 +119,18 @@ func QueryPrometheus(
|
|
|
} else if opts.Metric == "memory" {
|
|
} else if opts.Metric == "memory" {
|
|
|
query = fmt.Sprintf("container_memory_usage_bytes{%s}", podSelector)
|
|
query = fmt.Sprintf("container_memory_usage_bytes{%s}", podSelector)
|
|
|
} else if opts.Metric == "network" {
|
|
} else if opts.Metric == "network" {
|
|
|
- netPodSelector := fmt.Sprintf(`namespace="%s",pod=~"%s",container="POD"`, opts.Namespace, podSelectionRegex)
|
|
|
|
|
|
|
+ netPodSelector := fmt.Sprintf(`namespace="%s",pod=~"%s",container="POD"`, opts.Namespace, selectionRegex)
|
|
|
query = fmt.Sprintf("rate(container_network_receive_bytes_total{%s}[5m])", netPodSelector)
|
|
query = fmt.Sprintf("rate(container_network_receive_bytes_total{%s}[5m])", netPodSelector)
|
|
|
} else if opts.Metric == "nginx:errors" {
|
|
} else if opts.Metric == "nginx:errors" {
|
|
|
- num := fmt.Sprintf(`sum(rate(nginx_ingress_controller_requests{status=~"5.*",namespace="%s",ingress=~"%s"}[5m]) OR on() vector(0))`, opts.Namespace, podSelectionRegex)
|
|
|
|
|
- denom := fmt.Sprintf(`sum(rate(nginx_ingress_controller_requests{namespace="%s",ingress=~"%s"}[5m]) > 0)`, opts.Namespace, podSelectionRegex)
|
|
|
|
|
|
|
+ num := fmt.Sprintf(`sum(rate(nginx_ingress_controller_requests{status=~"5.*",namespace="%s",ingress=~"%s"}[5m]) OR on() vector(0))`, opts.Namespace, selectionRegex)
|
|
|
|
|
+ denom := fmt.Sprintf(`sum(rate(nginx_ingress_controller_requests{namespace="%s",ingress=~"%s"}[5m]) > 0)`, opts.Namespace, selectionRegex)
|
|
|
query = fmt.Sprintf(`%s / %s * 100 OR on() vector(0)`, num, denom)
|
|
query = fmt.Sprintf(`%s / %s * 100 OR on() vector(0)`, num, denom)
|
|
|
|
|
+ } else if opts.Metric == "nginx:latency" {
|
|
|
|
|
+ num := fmt.Sprintf(`sum(rate(nginx_ingress_controller_request_duration_seconds_sum{namespace=~"%s",ingress=~"%s"}[5m]) OR on() vector(0))`, opts.Namespace, selectionRegex)
|
|
|
|
|
+ denom := fmt.Sprintf(`sum(rate(nginx_ingress_controller_request_duration_seconds_count{namespace=~"%s",ingress=~"%s"}[5m]))`, opts.Namespace, selectionRegex)
|
|
|
|
|
+ query = fmt.Sprintf(`%s / %s OR on() vector(0)`, num, denom)
|
|
|
|
|
+ } else if opts.Metric == "nginx:latency-histogram" {
|
|
|
|
|
+ query = fmt.Sprintf(`histogram_quantile(%f, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{status!="404",status!="500",namespace=~"%s",ingress=~"%s"}[5m])) by (le, ingress))`, opts.Percentile, opts.Namespace, selectionRegex)
|
|
|
} else if opts.Metric == "cpu_hpa_threshold" {
|
|
} else if opts.Metric == "cpu_hpa_threshold" {
|
|
|
// get the name of the kube hpa metric
|
|
// get the name of the kube hpa metric
|
|
|
metricName, hpaMetricName := getKubeHPAMetricName(clientset, service, opts, "spec_target_metric")
|
|
metricName, hpaMetricName := getKubeHPAMetricName(clientset, service, opts, "spec_target_metric")
|
|
@@ -135,7 +142,7 @@ func QueryPrometheus(
|
|
|
appLabel = ksmSvc.ObjectMeta.Labels["app.kubernetes.io/instance"]
|
|
appLabel = ksmSvc.ObjectMeta.Labels["app.kubernetes.io/instance"]
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
- query = createHPAAbsoluteCPUThresholdQuery(cpuMetricName, metricName, podSelectionRegex, opts.Name, opts.Namespace, appLabel, hpaMetricName)
|
|
|
|
|
|
|
+ query = createHPAAbsoluteCPUThresholdQuery(cpuMetricName, metricName, selectionRegex, opts.Name, opts.Namespace, appLabel, hpaMetricName)
|
|
|
} else if opts.Metric == "memory_hpa_threshold" {
|
|
} else if opts.Metric == "memory_hpa_threshold" {
|
|
|
metricName, hpaMetricName := getKubeHPAMetricName(clientset, service, opts, "spec_target_metric")
|
|
metricName, hpaMetricName := getKubeHPAMetricName(clientset, service, opts, "spec_target_metric")
|
|
|
memMetricName := getKubeMemoryMetricName(clientset, service, opts)
|
|
memMetricName := getKubeMemoryMetricName(clientset, service, opts)
|
|
@@ -146,7 +153,7 @@ func QueryPrometheus(
|
|
|
appLabel = ksmSvc.ObjectMeta.Labels["app.kubernetes.io/instance"]
|
|
appLabel = ksmSvc.ObjectMeta.Labels["app.kubernetes.io/instance"]
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
- query = createHPAAbsoluteMemoryThresholdQuery(memMetricName, metricName, podSelectionRegex, opts.Name, opts.Namespace, appLabel, hpaMetricName)
|
|
|
|
|
|
|
+ query = createHPAAbsoluteMemoryThresholdQuery(memMetricName, metricName, selectionRegex, opts.Name, opts.Namespace, appLabel, hpaMetricName)
|
|
|
} else if opts.Metric == "hpa_replicas" {
|
|
} else if opts.Metric == "hpa_replicas" {
|
|
|
metricName, hpaMetricName := getKubeHPAMetricName(clientset, service, opts, "status_current_replicas")
|
|
metricName, hpaMetricName := getKubeHPAMetricName(clientset, service, opts, "status_current_replicas")
|
|
|
ksmSvc, found, _ := getKubeStateMetricsService(clientset)
|
|
ksmSvc, found, _ := getKubeStateMetricsService(clientset)
|
|
@@ -208,6 +215,7 @@ type promParsedSingletonQueryResult struct {
|
|
|
Memory interface{} `json:"memory,omitempty"`
|
|
Memory interface{} `json:"memory,omitempty"`
|
|
|
Bytes interface{} `json:"bytes,omitempty"`
|
|
Bytes interface{} `json:"bytes,omitempty"`
|
|
|
ErrorPct interface{} `json:"error_pct,omitempty"`
|
|
ErrorPct interface{} `json:"error_pct,omitempty"`
|
|
|
|
|
+ Latency interface{} `json:"latency,omitempty"`
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
type promParsedSingletonQuery struct {
|
|
type promParsedSingletonQuery struct {
|
|
@@ -248,6 +256,8 @@ func parseQuery(rawQuery []byte, metric string) ([]byte, error) {
|
|
|
singletonResult.Memory = values[1]
|
|
singletonResult.Memory = values[1]
|
|
|
} else if metric == "hpa_replicas" {
|
|
} else if metric == "hpa_replicas" {
|
|
|
singletonResult.Replicas = values[1]
|
|
singletonResult.Replicas = values[1]
|
|
|
|
|
+ } else if metric == "nginx:latency" || metric == "nginx:latency-histogram" {
|
|
|
|
|
+ singletonResult.Latency = values[1]
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
singletonResults = append(singletonResults, *singletonResult)
|
|
singletonResults = append(singletonResults, *singletonResult)
|
|
@@ -261,7 +271,7 @@ func parseQuery(rawQuery []byte, metric string) ([]byte, error) {
|
|
|
return json.Marshal(res)
|
|
return json.Marshal(res)
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
-func getPodSelectionRegex(kind, name string) (string, error) {
|
|
|
|
|
|
|
+func getSelectionRegex(kind, name string) (string, error) {
|
|
|
var suffix string
|
|
var suffix string
|
|
|
|
|
|
|
|
switch strings.ToLower(kind) {
|
|
switch strings.ToLower(kind) {
|
|
@@ -273,6 +283,8 @@ func getPodSelectionRegex(kind, name string) (string, error) {
|
|
|
suffix = "[a-z0-9]+"
|
|
suffix = "[a-z0-9]+"
|
|
|
case "cronjob":
|
|
case "cronjob":
|
|
|
suffix = "[a-z0-9]+-[a-z0-9]+"
|
|
suffix = "[a-z0-9]+-[a-z0-9]+"
|
|
|
|
|
+ case "ingress":
|
|
|
|
|
+ return name, nil
|
|
|
default:
|
|
default:
|
|
|
return "", fmt.Errorf("not a supported controller to query for metrics")
|
|
return "", fmt.Errorf("not a supported controller to query for metrics")
|
|
|
}
|
|
}
|