Merge branch 'master' of https://github.com/porter-dev/porter into 0.7.0-form-fix-validation

docs/developing/test-autoscaling.md

@@ -0,0 +1,18 @@
+# Test Cluster and HPA Autoscaling
+
+Prerequisites: 
+- [`metrics-server`](https://artifacthub.io/packages/helm/bitnami/metrics-server) must be installed (installed by default on all Porter clusters). 
+- [`cluster-autoscaler`](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler) must be enabled (enabled by default on all Porter clusters). 
+- Have `kubectl` access to the cluster, as it is easiest to port-forward to the exposed service. 
+- Download [`hey`](https://github.com/rakyll/hey) or an equivalent. 
+
+# Steps
+
+1. Launch a Docker template with image URL `k8s.gcr.io/hpa-example`. Enable autoscaling under the **Resources** tab -- suggested params are 100 Mi RAM, 100m CPU, 1-100 replicas, 50% CPU & RAM util. Do not **Expose to external traffic** to run load tests.
+
+2. Confirm that the horizontal pod autoscaler was created via kubectl (`kubectl get hpa`) and that the metrics server is reporting metrics via `kubectl top pods`. Current utilization may initially show up as `<undefined>`, but this information should load after a minute or so.
+
+3. Port-forward to the service via `kubectl port-forward svc/<svc-name> 10000:80`. 
+4. In a different terminal window, run `hey http://localhost:10000`. Vary the `hey` parameters to test autoscaling under different loads (`hey -h` for options). 
+
+Check that replicas scale when passing the threshold from the metrics tab. Use kubectl to probe the nodes and confirm cluster-level upscaling has also occurred. Delete the chart and confirm node downscaling when done.

internal/kubernetes/prometheus/metrics.go

@@ -126,7 +126,8 @@ func QueryPrometheus(
 		query = fmt.Sprintf(`%s / %s * 100 OR on() vector(0)`, num, denom)
 	} else if opts.Metric == "cpu_hpa_threshold" {
 		// get the name of the kube hpa metric
-		metricName := getKubeHPAMetricName(clientset, service, opts, "spec_target_metric")
+		metricName, hpaMetricName := getKubeHPAMetricName(clientset, service, opts, "spec_target_metric")
+		cpuMetricName := getKubeCPUMetricName(clientset, service, opts)
 		ksmSvc, found, _ := getKubeStateMetricsService(clientset)
 		appLabel := ""
 
@@ -134,9 +135,10 @@ func QueryPrometheus(
 			appLabel = ksmSvc.ObjectMeta.Labels["app.kubernetes.io/instance"]
 		}
 
-		query = createHPAAbsoluteCPUThresholdQuery(metricName, podSelectionRegex, opts.Name, opts.Namespace, appLabel)
+		query = createHPAAbsoluteCPUThresholdQuery(cpuMetricName, metricName, podSelectionRegex, opts.Name, opts.Namespace, appLabel, hpaMetricName)
 	} else if opts.Metric == "memory_hpa_threshold" {
-		metricName := getKubeHPAMetricName(clientset, service, opts, "spec_target_metric")
+		metricName, hpaMetricName := getKubeHPAMetricName(clientset, service, opts, "spec_target_metric")
+		memMetricName := getKubeMemoryMetricName(clientset, service, opts)
 		ksmSvc, found, _ := getKubeStateMetricsService(clientset)
 		appLabel := ""
 
@@ -144,9 +146,9 @@ func QueryPrometheus(
 			appLabel = ksmSvc.ObjectMeta.Labels["app.kubernetes.io/instance"]
 		}
 
-		query = createHPAAbsoluteMemoryThresholdQuery(metricName, podSelectionRegex, opts.Name, opts.Namespace, appLabel)
+		query = createHPAAbsoluteMemoryThresholdQuery(memMetricName, metricName, podSelectionRegex, opts.Name, opts.Namespace, appLabel, hpaMetricName)
 	} else if opts.Metric == "hpa_replicas" {
-		metricName := getKubeHPAMetricName(clientset, service, opts, "status_current_replicas")
+		metricName, hpaMetricName := getKubeHPAMetricName(clientset, service, opts, "status_current_replicas")
 		ksmSvc, found, _ := getKubeStateMetricsService(clientset)
 		appLabel := ""
 
@@ -154,13 +156,15 @@ func QueryPrometheus(
 			appLabel = ksmSvc.ObjectMeta.Labels["app.kubernetes.io/instance"]
 		}
 
-		query = createHPACurrentReplicasQuery(metricName, opts.Name, opts.Namespace, appLabel)
+		query = createHPACurrentReplicasQuery(metricName, opts.Name, opts.Namespace, appLabel, hpaMetricName)
 	}
 
 	if opts.ShouldSum {
 		query = fmt.Sprintf("sum(%s)", query)
 	}
 
+	fmt.Println("QUERY IS", query)
+
 	queryParams := map[string]string{
 		"query": query,
 		"start": fmt.Sprintf("%d", opts.StartRange),
@@ -276,15 +280,20 @@ func getPodSelectionRegex(kind, name string) (string, error) {
 	return fmt.Sprintf("%s-%s", name, suffix), nil
 }
 
-func createHPAAbsoluteCPUThresholdQuery(metricName, podSelectionRegex, hpaName, namespace, appLabel string) string {
+func createHPAAbsoluteCPUThresholdQuery(cpuMetricName, metricName, podSelectionRegex, hpaName, namespace, appLabel, hpaMetricName string) string {
 	kubeMetricsPodSelector := getKubeMetricsPodSelector(podSelectionRegex, namespace)
 
 	kubeMetricsHPASelector := fmt.Sprintf(
-		`hpa="%s",namespace="%s",metric_name="cpu",metric_target_type="utilization"`,
+		`%s="%s",namespace="%s",metric_name="cpu",metric_target_type="utilization"`,
+		hpaMetricName,
 		hpaName,
 		namespace,
 	)
 
+	if cpuMetricName == "kube_pod_container_resource_requests" {
+		kubeMetricsPodSelector += `,resource="cpu",unit="core"`
+	}
+
 	// the kube-state-metrics queries are less prone to error if the field app_kubernetes_io_instance is matched
 	// as well
 	if appLabel != "" {
@@ -293,8 +302,11 @@ func createHPAAbsoluteCPUThresholdQuery(metricName, podSelectionRegex, hpaName,
 	}
 
 	requestCPU := fmt.Sprintf(
-		`sum by (hpa) (label_replace(kube_pod_container_resource_requests_cpu_cores{%s},"hpa", "%s", "", ""))`,
+		`sum by (%s) (label_replace(%s{%s},"%s", "%s", "", ""))`,
+		hpaMetricName,
+		cpuMetricName,
 		kubeMetricsPodSelector,
+		hpaMetricName,
 		hpaName,
 	)
 
@@ -304,18 +316,23 @@ func createHPAAbsoluteCPUThresholdQuery(metricName, podSelectionRegex, hpaName,
 		kubeMetricsHPASelector,
 	)
 
-	return fmt.Sprintf(`%s * on(hpa) %s`, requestCPU, targetCPUUtilThreshold)
+	return fmt.Sprintf(`%s * on(%s) %s`, requestCPU, hpaMetricName, targetCPUUtilThreshold)
 }
 
-func createHPAAbsoluteMemoryThresholdQuery(metricName, podSelectionRegex, hpaName, namespace, appLabel string) string {
+func createHPAAbsoluteMemoryThresholdQuery(memMetricName, metricName, podSelectionRegex, hpaName, namespace, appLabel, hpaMetricName string) string {
 	kubeMetricsPodSelector := getKubeMetricsPodSelector(podSelectionRegex, namespace)
 
 	kubeMetricsHPASelector := fmt.Sprintf(
-		`hpa="%s",namespace="%s",metric_name="memory",metric_target_type="utilization"`,
+		`%s="%s",namespace="%s",metric_name="memory",metric_target_type="utilization"`,
+		hpaMetricName,
 		hpaName,
 		namespace,
 	)
 
+	if memMetricName == "kube_pod_container_resource_requests" {
+		kubeMetricsPodSelector += `,resource="memory",unit="byte"`
+	}
+
 	// the kube-state-metrics queries are less prone to error if the field app_kubernetes_io_instance is matched
 	// as well
 	if appLabel != "" {
@@ -324,8 +341,11 @@ func createHPAAbsoluteMemoryThresholdQuery(metricName, podSelectionRegex, hpaNam
 	}
 
 	requestMem := fmt.Sprintf(
-		`sum by (hpa) (label_replace(kube_pod_container_resource_requests_memory_bytes{%s},"hpa", "%s", "", ""))`,
+		`sum by (%s) (label_replace(%s{%s},"%s", "%s", "", ""))`,
+		hpaMetricName,
+		memMetricName,
 		kubeMetricsPodSelector,
+		hpaMetricName,
 		hpaName,
 	)
 
@@ -335,7 +355,7 @@ func createHPAAbsoluteMemoryThresholdQuery(metricName, podSelectionRegex, hpaNam
 		kubeMetricsHPASelector,
 	)
 
-	return fmt.Sprintf(`%s * on(hpa) %s`, requestMem, targetMemUtilThreshold)
+	return fmt.Sprintf(`%s * on(%s) %s`, requestMem, hpaMetricName, targetMemUtilThreshold)
 }
 
 func getKubeMetricsPodSelector(podSelectionRegex, namespace string) string {
@@ -346,9 +366,10 @@ func getKubeMetricsPodSelector(podSelectionRegex, namespace string) string {
 	)
 }
 
-func createHPACurrentReplicasQuery(metricName, hpaName, namespace, appLabel string) string {
+func createHPACurrentReplicasQuery(metricName, hpaName, namespace, appLabel, hpaMetricName string) string {
 	kubeMetricsHPASelector := fmt.Sprintf(
-		`hpa="%s",namespace="%s"`,
+		`%s="%s",namespace="%s"`,
+		hpaMetricName,
 		hpaName,
 		namespace,
 	)
@@ -372,7 +393,7 @@ type promRawValuesQuery struct {
 }
 
 // getKubeHPAMetricName performs a "best guess" for the name of the kube HPA metric,
-// which was renamed to kube_horizontal_pod_autoscaler... in later versions of kube-state-metrics.
+// which was renamed to kube_horizontalpodautoscaler... in later versions of kube-state-metrics.
 // we query Prometheus for a list of metric names to see if any match the new query
 // value, otherwise we return the deprecated name.
 func getKubeHPAMetricName(
@@ -380,9 +401,81 @@ func getKubeHPAMetricName(
 	service *v1.Service,
 	opts *QueryOpts,
 	suffix string,
+) (string, string) {
+	queryParams := map[string]string{
+		"match[]": fmt.Sprintf("kube_horizontalpodautoscaler_%s", suffix),
+		"start":   fmt.Sprintf("%d", opts.StartRange),
+		"end":     fmt.Sprintf("%d", opts.EndRange),
+	}
+
+	resp := clientset.CoreV1().Services(service.Namespace).ProxyGet(
+		"http",
+		service.Name,
+		fmt.Sprintf("%d", service.Spec.Ports[0].Port),
+		"/api/v1/label/__name__/values",
+		queryParams,
+	)
+
+	rawQuery, err := resp.DoRaw(context.TODO())
+
+	if err != nil {
+		return fmt.Sprintf("kube_hpa_%s", suffix), "hpa"
+	}
+
+	rawQueryObj := &promRawValuesQuery{}
+
+	json.Unmarshal(rawQuery, rawQueryObj)
+
+	if rawQueryObj.Status == "success" && len(rawQueryObj.Data) == 1 {
+		return fmt.Sprintf("kube_horizontalpodautoscaler_%s", suffix), "horizontalpodautoscaler"
+	}
+
+	return fmt.Sprintf("kube_hpa_%s", suffix), "hpa"
+}
+
+func getKubeCPUMetricName(
+	clientset kubernetes.Interface,
+	service *v1.Service,
+	opts *QueryOpts,
+) string {
+	queryParams := map[string]string{
+		"match[]": "kube_pod_container_resource_requests",
+		"start":   fmt.Sprintf("%d", opts.StartRange),
+		"end":     fmt.Sprintf("%d", opts.EndRange),
+	}
+
+	resp := clientset.CoreV1().Services(service.Namespace).ProxyGet(
+		"http",
+		service.Name,
+		fmt.Sprintf("%d", service.Spec.Ports[0].Port),
+		"/api/v1/label/__name__/values",
+		queryParams,
+	)
+
+	rawQuery, err := resp.DoRaw(context.TODO())
+
+	if err != nil {
+		return "kube_pod_container_resource_requests_cpu_cores"
+	}
+
+	rawQueryObj := &promRawValuesQuery{}
+
+	json.Unmarshal(rawQuery, rawQueryObj)
+
+	if rawQueryObj.Status == "success" && len(rawQueryObj.Data) == 1 {
+		return "kube_pod_container_resource_requests"
+	}
+
+	return "kube_pod_container_resource_requests_cpu_cores"
+}
+
+func getKubeMemoryMetricName(
+	clientset kubernetes.Interface,
+	service *v1.Service,
+	opts *QueryOpts,
 ) string {
 	queryParams := map[string]string{
-		"match[]": fmt.Sprintf("kube_horizontal_pod_autoscaler_%s", suffix),
+		"match[]": "kube_pod_container_resource_requests",
 		"start":   fmt.Sprintf("%d", opts.StartRange),
 		"end":     fmt.Sprintf("%d", opts.EndRange),
 	}
@@ -398,7 +491,7 @@ func getKubeHPAMetricName(
 	rawQuery, err := resp.DoRaw(context.TODO())
 
 	if err != nil {
-		return fmt.Sprintf("kube_hpa_%s", suffix)
+		return "kube_pod_container_resource_requests_memory_bytes"
 	}
 
 	rawQueryObj := &promRawValuesQuery{}
@@ -406,8 +499,8 @@ func getKubeHPAMetricName(
 	json.Unmarshal(rawQuery, rawQueryObj)
 
 	if rawQueryObj.Status == "success" && len(rawQueryObj.Data) == 1 {
-		return fmt.Sprintf("kube_horizontal_pod_autoscaler_%s", suffix)
+		return "kube_pod_container_resource_requests"
 	}
 
-	return fmt.Sprintf("kube_hpa_%s", suffix)
+	return "kube_pod_container_resource_requests_memory_bytes"
 }