Implment new authorizer: Connection String for azure cloud integration (#3434)

pkg/cloud/azure/storageauthorizer.go

@@ -3,12 +3,28 @@ package azure
 import (
 	"encoding/json"
 	"fmt"
+	"net/http"
+	"time"
 
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
+	"github.com/opencost/opencost/core/pkg/storage"
 	"github.com/opencost/opencost/pkg/cloud"
 )
 
 const SharedKeyAuthorizerType = "AzureAccessKey"
+const StorageConnectionStringAuthorizerType = "AzureStorageConnectionString"
+
+var defaultHTTPConfig = storage.HTTPConfig{
+	IdleConnTimeout:       90 * time.Second,
+	ResponseHeaderTimeout: 2 * time.Minute,
+	TLSHandshakeTimeout:   10 * time.Second,
+	ExpectContinueTimeout: 1 * time.Second,
+	MaxIdleConns:          100,
+	MaxIdleConnsPerHost:   100,
+	MaxConnsPerHost:       0,
+	DisableCompression:    false,
+}
 
 // StorageAuthorizer is a service specific Authorizer for Azure Storage, it exists so that we can support existing Shared
 // Key configurations while allowing the Authorizer to have a service agnostic api
@@ -22,6 +38,10 @@ func SelectStorageAuthorizerByType(typeStr string) (StorageAuthorizer, error) {
 	switch typeStr {
 	case SharedKeyAuthorizerType:
 		return &SharedKeyCredential{}, nil
+	case StorageConnectionStringAuthorizerType:
+		return &StorageConnectionStringCredential{
+			HTTPConfig: defaultHTTPConfig,
+		}, nil
 	default:
 		authorizer, err := SelectAuthorizerByType(typeStr)
 		if err != nil {
@@ -127,3 +147,65 @@ func (ah *AuthorizerHolder) GetBlobClient(serviceURL string) (*azblob.Client, er
 func (ah *AuthorizerHolder) UnmarshalJSON(b []byte) error {
 	return json.Unmarshal(b, ah.Authorizer)
 }
+
+type StorageConnectionStringCredential struct {
+	StorageConnectionString string             `json:"storageConnectionString"`
+	HTTPConfig              storage.HTTPConfig `json:"httpConfig"`
+}
+
+func (s *StorageConnectionStringCredential) MarshalJSON() ([]byte, error) {
+	fmap := make(map[string]any, 3)
+	fmap[cloud.AuthorizerTypeProperty] = StorageConnectionStringAuthorizerType
+	fmap["storageConnectionString"] = s.StorageConnectionString
+	fmap["httpConfig"] = s.HTTPConfig
+	return json.Marshal(fmap)
+}
+
+func (s *StorageConnectionStringCredential) Validate() error {
+	if s.StorageConnectionString == "" {
+		return fmt.Errorf("StorageConnectionStringCredential: missing storage connection string")
+	}
+	return nil
+}
+
+func (s *StorageConnectionStringCredential) Equals(config cloud.Config) bool {
+	if config == nil {
+		return false
+	}
+
+	thatConfig, ok := config.(*StorageConnectionStringCredential)
+	if !ok {
+		return false
+	}
+
+	if s.HTTPConfig != thatConfig.HTTPConfig {
+		return false
+	}
+
+	if s.StorageConnectionString != thatConfig.StorageConnectionString {
+		return false
+	}
+
+	return true
+}
+
+func (s *StorageConnectionStringCredential) Sanitize() cloud.Config {
+	return &StorageConnectionStringCredential{
+		StorageConnectionString: cloud.Redacted,
+		HTTPConfig:              s.HTTPConfig,
+	}
+}
+
+func (s *StorageConnectionStringCredential) GetBlobClient(serviceURL string) (*azblob.Client, error) {
+	dt, err := s.HTTPConfig.GetHTTPTransport()
+	if err != nil {
+		return nil, fmt.Errorf("error creating transport: %w", err)
+	}
+	options := &azblob.ClientOptions{
+		ClientOptions: azcore.ClientOptions{
+			Transport: &http.Client{Transport: dt},
+		},
+	}
+	client, err := azblob.NewClientFromConnectionString(s.StorageConnectionString, options)
+	return client, err
+}

pkg/cloud/azure/storageconfiguration_test.go

@@ -5,6 +5,7 @@ import (
 	"testing"
 
 	"github.com/opencost/opencost/core/pkg/log"
+	"github.com/opencost/opencost/core/pkg/storage"
 	"github.com/opencost/opencost/core/pkg/util/json"
 	"github.com/opencost/opencost/pkg/cloud"
 )
@@ -122,6 +123,30 @@ func TestStorageConfiguration_Validate(t *testing.T) {
 			},
 			expected: nil,
 		},
+		"valid config StorageConnectionStringCredential": {
+			config: StorageConfiguration{
+				SubscriptionID: "subscriptionID",
+				Account:        "account",
+				Container:      "container",
+				Path:           "path",
+				Cloud:          "cloud",
+				Authorizer: &StorageConnectionStringCredential{
+					StorageConnectionString: "storageConnectionString",
+				},
+			},
+			expected: nil,
+		},
+		"missing storage connection string": {
+			config: StorageConfiguration{
+				SubscriptionID: "subscriptionID",
+				Account:        "account",
+				Container:      "container",
+				Path:           "path",
+				Cloud:          "cloud",
+				Authorizer:     &StorageConnectionStringCredential{},
+			},
+			expected: fmt.Errorf("StorageConnectionStringCredential: missing storage connection string"),
+		},
 	}
 
 	for name, testCase := range testCases {
@@ -426,6 +451,79 @@ func TestStorageConfiguration_Equals(t *testing.T) {
 			},
 			expected: false,
 		},
+		"matching config StorageConnectionStringCredential": {
+			left: StorageConfiguration{
+				SubscriptionID: "subscriptionID",
+				Account:        "account",
+				Container:      "container",
+				Path:           "path",
+				Cloud:          "cloud",
+				Authorizer: &StorageConnectionStringCredential{
+					StorageConnectionString: "storageConnectionString",
+				},
+			},
+			right: &StorageConfiguration{
+				SubscriptionID: "subscriptionID",
+				Account:        "account",
+				Container:      "container",
+				Path:           "path",
+				Cloud:          "cloud",
+				Authorizer: &StorageConnectionStringCredential{
+					StorageConnectionString: "storageConnectionString",
+				},
+			},
+			expected: true,
+		},
+		"different StorageConnectionString in StorageConnectionStringCredential": {
+			left: StorageConfiguration{
+				SubscriptionID: "subscriptionID",
+				Account:        "account",
+				Container:      "container",
+				Path:           "path",
+				Cloud:          "cloud",
+				Authorizer: &StorageConnectionStringCredential{
+					StorageConnectionString: "storageConnectionString1",
+				},
+			},
+			right: &StorageConfiguration{
+				SubscriptionID: "subscriptionID",
+				Account:        "account",
+				Container:      "container",
+				Path:           "path",
+				Cloud:          "cloud",
+				Authorizer: &StorageConnectionStringCredential{
+					StorageConnectionString: "storageConnectionString2",
+				},
+			},
+			expected: false,
+		},
+		"different HTTPConfig in StorageConnectionStringCredential": {
+			left: StorageConfiguration{
+				SubscriptionID: "subscriptionID",
+				Account:        "account",
+				Container:      "container",
+				Path:           "path",
+				Cloud:          "cloud",
+				Authorizer: &StorageConnectionStringCredential{
+					StorageConnectionString: "storageConnectionString",
+					HTTPConfig:              defaultHTTPConfig,
+				},
+			},
+			right: &StorageConfiguration{
+				SubscriptionID: "subscriptionID",
+				Account:        "account",
+				Container:      "container",
+				Path:           "path",
+				Cloud:          "cloud",
+				Authorizer: &StorageConnectionStringCredential{
+					StorageConnectionString: "storageConnectionString",
+					HTTPConfig: storage.HTTPConfig{
+						InsecureSkipVerify: true,
+					},
+				},
+			},
+			expected: false,
+		},
 	}
 
 	for name, testCase := range testCases {
@@ -440,13 +538,19 @@ func TestStorageConfiguration_Equals(t *testing.T) {
 
 func TestStorageConfiguration_JSON(t *testing.T) {
 	testCases := map[string]struct {
-		config StorageConfiguration
+		input          map[string]interface{}
+		afterUnmarshal StorageConfiguration
 	}{
-		"Empty Config": {
-			config: StorageConfiguration{},
-		},
 		"Nil Authorizer": {
-			config: StorageConfiguration{
+			input: map[string]interface{}{
+				"subscriptionID": "subscriptionID",
+				"account":        "account",
+				"container":      "container",
+				"path":           "path",
+				"cloud":          "cloud",
+				"authorizer":     nil,
+			},
+			afterUnmarshal: StorageConfiguration{
 				SubscriptionID: "subscriptionID",
 				Account:        "account",
 				Container:      "container",
@@ -456,7 +560,19 @@ func TestStorageConfiguration_JSON(t *testing.T) {
 			},
 		},
 		"SharedKeyCredential Authorizer": {
-			config: StorageConfiguration{
+			input: map[string]interface{}{
+				"subscriptionID": "subscriptionID",
+				"account":        "account",
+				"container":      "container",
+				"path":           "path",
+				"cloud":          "cloud",
+				"authorizer": map[string]interface{}{
+					"authorizerType": "AzureAccessKey",
+					"accessKey":      "accessKey",
+					"account":        "account",
+				},
+			},
+			afterUnmarshal: StorageConfiguration{
 				SubscriptionID: "subscriptionID",
 				Account:        "account",
 				Container:      "container",
@@ -469,7 +585,17 @@ func TestStorageConfiguration_JSON(t *testing.T) {
 			},
 		},
 		"Default AuthorizerHolder Authorizer": {
-			config: StorageConfiguration{
+			input: map[string]interface{}{
+				"subscriptionID": "subscriptionID",
+				"account":        "account",
+				"container":      "container",
+				"path":           "path",
+				"cloud":          "cloud",
+				"authorizer": map[string]interface{}{
+					"authorizerType": "AzureDefaultCredential",
+				},
+			},
+			afterUnmarshal: StorageConfiguration{
 				SubscriptionID: "subscriptionID",
 				Account:        "account",
 				Container:      "container",
@@ -481,7 +607,20 @@ func TestStorageConfiguration_JSON(t *testing.T) {
 			},
 		},
 		"ClientSecretCredential Authorizer": {
-			config: StorageConfiguration{
+			input: map[string]interface{}{
+				"subscriptionID": "subscriptionID",
+				"account":        "account",
+				"container":      "container",
+				"path":           "path",
+				"cloud":          "cloud",
+				"authorizer": map[string]interface{}{
+					"authorizerType": "AzureClientSecretCredential",
+					"tenantID":       "tenantID",
+					"clientID":       "clientID",
+					"clientSecret":   "clientSecret",
+				},
+			},
+			afterUnmarshal: StorageConfiguration{
 				SubscriptionID: "subscriptionID",
 				Account:        "account",
 				Container:      "container",
@@ -496,12 +635,43 @@ func TestStorageConfiguration_JSON(t *testing.T) {
 				},
 			},
 		},
+		"StorageConnectionStringCredential Authorizer": {
+			input: map[string]interface{}{
+				"subscriptionID": "subscriptionID",
+				"account":        "account",
+				"container":      "container",
+				"path":           "path",
+				"cloud":          "cloud",
+				"authorizer": map[string]interface{}{
+					"authorizerType":          "AzureStorageConnectionString",
+					"storageConnectionString": "storageConnectionString",
+					"httpConfig": map[string]interface{}{
+						"insecureSkipVerify": true,
+					},
+				},
+			},
+			afterUnmarshal: StorageConfiguration{
+				SubscriptionID: "subscriptionID",
+				Account:        "account",
+				Container:      "container",
+				Path:           "path",
+				Cloud:          "cloud",
+				Authorizer: &StorageConnectionStringCredential{
+					StorageConnectionString: "storageConnectionString",
+					HTTPConfig: func() storage.HTTPConfig {
+						cfg := defaultHTTPConfig
+						cfg.InsecureSkipVerify = true
+						return cfg
+					}(),
+				},
+			},
+		},
 	}
 
 	for name, testCase := range testCases {
 		t.Run(name, func(t *testing.T) {
 			// test JSON Marshalling
-			configJSON, err := json.Marshal(testCase.config)
+			configJSON, err := json.Marshal(testCase.input)
 			if err != nil {
 				t.Errorf("failed to marshal configuration: %s", err.Error())
 			}
@@ -512,9 +682,75 @@ func TestStorageConfiguration_JSON(t *testing.T) {
 				t.Errorf("failed to unmarshal configuration: %s", err.Error())
 			}
 
-			if !testCase.config.Equals(unmarshalledConfig) {
+			if !testCase.afterUnmarshal.Equals(unmarshalledConfig) {
 				t.Error("config does not equal unmarshalled config")
 			}
 		})
 	}
 }
+
+func TestStorageConfiguration_Sanitize(t *testing.T) {
+	testCases := map[string]struct {
+		config   StorageConfiguration
+		expected StorageConfiguration
+	}{
+		"Sanitize StorageConnectionStringCredential": {
+			config: StorageConfiguration{
+				SubscriptionID: "subscriptionID",
+				Account:        "account",
+				Container:      "container",
+				Path:           "path",
+				Cloud:          "cloud",
+				Authorizer: &StorageConnectionStringCredential{
+					StorageConnectionString: "storageConnectionString",
+					HTTPConfig:              defaultHTTPConfig,
+				},
+			},
+			expected: StorageConfiguration{
+				SubscriptionID: "subscriptionID",
+				Account:        "account",
+				Container:      "container",
+				Path:           "path",
+				Cloud:          "cloud",
+				Authorizer: &StorageConnectionStringCredential{
+					StorageConnectionString: cloud.Redacted,
+					HTTPConfig:              defaultHTTPConfig,
+				},
+			},
+		},
+		"Sanitize SharedKeyCredential": {
+			config: StorageConfiguration{
+				SubscriptionID: "subscriptionID",
+				Account:        "account",
+				Container:      "container",
+				Path:           "path",
+				Cloud:          "cloud",
+				Authorizer: &SharedKeyCredential{
+					AccessKey: "accessKey",
+					Account:   "account",
+				},
+			},
+			expected: StorageConfiguration{
+				SubscriptionID: "subscriptionID",
+				Account:        "account",
+				Container:      "container",
+				Path:           "path",
+				Cloud:          "cloud",
+				Authorizer: &SharedKeyCredential{
+					AccessKey: cloud.Redacted,
+					Account:   "account",
+				},
+			},
+		},
+	}
+
+	for name, testCase := range testCases {
+		t.Run(name, func(t *testing.T) {
+			actual := testCase.config.Sanitize()
+
+			if !testCase.expected.Equals(actual) {
+				t.Errorf("incorrect result: got %#v, want %#v", actual, testCase.expected)
+			}
+		})
+	}
+}