Enable the use of mTLS with the prometheus backend (#3459)

Signed-off-by: Eric Mrak <eric.mrak@fastly.com>
Signed-off-by: Eric Mrak <enmrak@gmail.com>
Co-authored-by: Alex Meijer <ameijer@users.noreply.github.com>

.gitignore

@@ -19,3 +19,7 @@ pkg/cloud/oracle/cloud-integration.json
 
 # tilt
 tilt_config.json
+
+# Test reports
+coverage.html
+coverage.out

modules/prometheus-source/pkg/env/promenv.go

@@ -34,6 +34,10 @@ const (
 	DBBasicAuthPassword = "DB_BASIC_AUTH_PW"
 	DBBearerToken       = "DB_BEARER_TOKEN"
 
+	PromMtlsAuthCAFile  = "PROM_MTLS_AUTH_CA_FILE"
+	PromMtlsAuthCrtFile = "PROM_MTLS_AUTH_CRT_FILE"
+	PromMtlsAuthKeyFile = "PROM_MTLS_AUTH_KEY_FILE"
+
 	CurrentClusterIdFilterEnabledVar = "CURRENT_CLUSTER_ID_FILTER_ENABLED"
 
 	KubecostJobNameEnvVar = "KUBECOST_JOB_NAME"
@@ -136,6 +140,31 @@ func GetDBBearerToken() string {
 	return env.Get(DBBearerToken, "")
 }
 
+func IsPromMtlsAuthEnabled() bool {
+	if GetPromMtlsAuthCAFile() == "" {
+		return false
+	}
+	if GetPromMtlsAuthCrtFile() == "" {
+		return false
+	}
+	if GetPromMtlsAuthKeyFile() == "" {
+		return false
+	}
+	return true
+}
+
+func GetPromMtlsAuthCAFile() string {
+	return env.Get(PromMtlsAuthCAFile, "")
+}
+
+func GetPromMtlsAuthCrtFile() string {
+	return env.Get(PromMtlsAuthCrtFile, "")
+}
+
+func GetPromMtlsAuthKeyFile() string {
+	return env.Get(PromMtlsAuthKeyFile, "")
+}
+
 func GetPrometheusMaxQueryDuration() time.Duration {
 	dayMins := 60 * 24
 	mins := time.Duration(env.GetInt64(PrometheusMaxQueryDurationMinutesEnvVar, int64(dayMins)))

modules/prometheus-source/pkg/env/promenv_test.go

@@ -0,0 +1,30 @@
+package env
+
+import "testing"
+
+func TestIsPromMtlsAuthEnabled(t *testing.T) {
+	t.Run("IsDBmTLSAuthEnabled returns false if all mTLS env vars are not set", func(t *testing.T) {
+		got := IsPromMtlsAuthEnabled()
+		if got == true {
+			t.Errorf("IsDBmTLSAuthEnabled() = %v, want %v", got, false)
+		}
+
+		t.Setenv("PROM_MTLS_AUTH_CA_FILE", "some/client.ca")
+		got = IsPromMtlsAuthEnabled()
+		if got == true {
+			t.Errorf("IsDBmTLSAuthEnabled() = %v, want %v", got, false)
+		}
+
+		t.Setenv("PROM_MTLS_AUTH_CRT_FILE", "some/client.crt")
+		got = IsPromMtlsAuthEnabled()
+		if got == true {
+			t.Errorf("IsDBmTLSAuthEnabled() = %v, want %v", got, false)
+		}
+
+		t.Setenv("PROM_MTLS_AUTH_KEY_FILE", "some/client.key")
+		got = IsPromMtlsAuthEnabled()
+		if got == false {
+			t.Errorf("IsDBmTLSAuthEnabled() = %v, want %v", got, true)
+		}
+	})
+}

modules/prometheus-source/pkg/prom/config.go

@@ -1,8 +1,10 @@
 package prom
 
 import (
+	"crypto/tls"
 	"crypto/x509"
 	"fmt"
+	"os"
 	"time"
 
 	coreenv "github.com/opencost/opencost/core/pkg/env"
@@ -78,6 +80,7 @@ func NewOpenCostPrometheusConfigFromEnv() (*OpenCostPrometheusConfig, error) {
 	// We will use the service account token and service-ca.crt to authenticate with the Prometheus server via kube-rbac-proxy.
 	// We need to ensure that the service account has the necessary permissions to access the Prometheus server by binding it to the appropriate role.
 	var tlsCaCert *x509.CertPool
+	var tlsClientCertificates []tls.Certificate
 	if env.IsKubeRbacProxyEnabled() {
 		restConfig, err := restclient.InClusterConfig()
 		if err != nil {
@@ -88,6 +91,27 @@ func NewOpenCostPrometheusConfigFromEnv() (*OpenCostPrometheusConfig, error) {
 		if err != nil {
 			log.Errorf("%s was set to true but failed to load service-ca.crt: %s", env.KubeRbacProxyEnabledEnvVar, err)
 		}
+	} else if env.IsPromMtlsAuthEnabled() {
+		tlsCaCert = x509.NewCertPool()
+		// The /etc/ssl/cert.pem location is correct for Alpine Linux, the container base used here
+		systemCa, err := os.ReadFile("/etc/ssl/cert.pem")
+		if err != nil {
+			log.Errorf("mTLS options were set but failed to load system CAs: %s", err)
+		} else {
+			tlsCaCert.AppendCertsFromPEM(systemCa)
+		}
+		mTlsCa, err := os.ReadFile(env.GetPromMtlsAuthCAFile())
+		if err != nil {
+			log.Errorf("mTLS options were set but failed to load PROM_MTLS_AUTH_CA_FILE: %s", err)
+		} else {
+			tlsCaCert.AppendCertsFromPEM(mTlsCa)
+		}
+		mTlsKeyPair, err := tls.LoadX509KeyPair(env.GetPromMtlsAuthCrtFile(), env.GetPromMtlsAuthKeyFile())
+		if err != nil {
+			log.Errorf("mTLS options were set but failed to load PROM_MTLS_AUTH_CRT_FILE or PROM_MTLS_AUTH_KEY_FILE: %s", err)
+		} else {
+			tlsClientCertificates = []tls.Certificate{mTlsKeyPair}
+		}
 	}
 
 	dataResolution := env.GetPrometheusQueryResolution()
@@ -104,6 +128,7 @@ func NewOpenCostPrometheusConfigFromEnv() (*OpenCostPrometheusConfig, error) {
 		TLSHandshakeTimeout:   tlsHandshakeTimeout,
 		TLSInsecureSkipVerify: env.IsInsecureSkipVerify(),
 		RootCAs:               tlsCaCert,
+		ClientCertificates:    tlsClientCertificates,
 		RateLimitRetryOpts:    rateLimitRetryOpts,
 		Auth:                  auth,
 		QueryConcurrency:      queryConcurrency,

modules/prometheus-source/pkg/prom/prom.go

@@ -379,6 +379,7 @@ type PrometheusClientConfig struct {
 	QueryLogFile          string
 	HeaderXScopeOrgId     string
 	RootCAs               *x509.CertPool
+	ClientCertificates    []tls.Certificate
 }
 
 // NewPrometheusClient creates a new rate limited client which limits by outbound concurrent requests.
@@ -395,6 +396,7 @@ func NewPrometheusClient(address string, config *PrometheusClientConfig) (promet
 		TLSClientConfig: &tls.Config{
 			InsecureSkipVerify: config.TLSInsecureSkipVerify,
 			RootCAs:            config.RootCAs,
+			Certificates:       config.ClientCertificates,
 			MinVersion:         tls.VersionTLS12,
 		},
 	})