implement gzip encoding for agent

core/pkg/env/core.go

@@ -13,7 +13,8 @@ const (
 	AppNameEnvVar    = "APP_NAME"
 	ConfigPathEnvVar = "CONFIG_PATH"
 
-	PProfEnabledEnvVar = "PPROF_ENABLED"
+	PProfEnabledEnvVar       = "PPROF_ENABLED"
+	CompressionEnabledEnvVar = "EXPORT_COMPRESSION_ENABLED"
 
 	InstallNamespaceEnvVar = "INSTALL_NAMESPACE"
 
@@ -58,6 +59,10 @@ func IsPProfEnabled() bool {
 	return GetBool(PProfEnabledEnvVar, false)
 }
 
+func IsCompressionEnabled() bool {
+	return GetBool(CompressionEnabledEnvVar, false)
+}
+
 func GetInstallNamespace(def string) string {
 	return Get(InstallNamespaceEnvVar, def)
 }

core/pkg/exporter/exporter.go

@@ -1,9 +1,12 @@
 package exporter
 
 import (
+	"bytes"
+	"compress/gzip"
 	"fmt"
 	"time"
 
+	coreenv "github.com/opencost/opencost/core/pkg/env"
 	"github.com/opencost/opencost/core/pkg/exporter/pathing"
 	"github.com/opencost/opencost/core/pkg/exporter/validator"
 	"github.com/opencost/opencost/core/pkg/log"
@@ -120,6 +123,20 @@ func (se *ComputeStorageExporter[T]) Export(window opencost.Window, data *T) err
 		return fmt.Errorf("failed to encode data: %w", err)
 	}
 
+	compressionEnabled := coreenv.IsCompressionEnabled()
+	if compressionEnabled {
+		log.Debugf("compressing data with gzip, compression enabled")
+		log.Debugf("original data size: %d bytes", len(bin))
+		var buf bytes.Buffer
+		gzWriter := gzip.NewWriter(&buf)
+		defer gzWriter.Close()
+		_, err := gzWriter.Write(bin)
+		if err != nil {
+			return fmt.Errorf("failed to write compressed data: %w", err)
+		}
+		bin = buf.Bytes()
+		log.Debugf("compressed data size: %d bytes", len(bin))
+	}
 	log.Debugf("writing new binary data to storage %s", path)
 	err = se.storage.Write(path, bin)
 	if err != nil {