Update HTTP Handling for Public Pricing (#3982)

.github/workflows/compare-pricing-data.yaml

@@ -2,7 +2,7 @@ name: Compare Pricing Data
 
 on:
   schedule:
-    - cron: '0 10 * * *'
+    - cron: '0 18 * * *'
 
 permissions:
   contents: read

modules/pricing/public/aws/pricelistapi.go

@@ -9,8 +9,11 @@ import (
 	"github.com/opencost/opencost/core/pkg/log"
 	"github.com/opencost/opencost/core/pkg/util/json"
 	"github.com/opencost/opencost/modules/pricing/public/env"
+	"github.com/opencost/opencost/modules/pricing/public/httpclient"
 )
 
+var awsHTTPClient = httpclient.NewClient(0) // no timeout
+
 const (
 	awsPricingBaseURL      = "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/"
 	awsChinaPricingBaseURL = "https://pricing.cn-north-1.amazonaws.com.cn/offers/v1.0/cn/"
@@ -62,7 +65,7 @@ func QueryEC2PriceList(
 	pricingURL := getListPriceURL("AmazonEC2", region)
 
 	log.Infof("starting download of \"%s\", which is quite large ...", pricingURL)
-	resp, err := http.Get(pricingURL)
+	resp, err := awsHTTPClient.Get(pricingURL)
 	if err != nil {
 		return fmt.Errorf("bogus fetch of \"%s\": %w", pricingURL, err)
 	}

modules/pricing/public/azure/azurepricingsource.go

@@ -13,6 +13,7 @@ import (
 	"github.com/opencost/opencost/core/pkg/log"
 	"github.com/opencost/opencost/core/pkg/pricing"
 	"github.com/opencost/opencost/core/pkg/unit"
+	"github.com/opencost/opencost/modules/pricing/public/httpclient"
 )
 
 const (
@@ -26,7 +27,7 @@ type AzurePricingSourceConfig struct {
 	CurrencyCode string
 }
 
-var azureHTTPClient = &http.Client{Timeout: 60 * time.Second}
+var azureHTTPClient = httpclient.NewClient(120 * time.Second)
 
 // AzurePricingSource implements the PricingSource interface using the
 // Azure Retail Prices API (no auth required).
@@ -232,11 +233,7 @@ func (a *AzurePricingSource) includeItem(item AzurePricingAttributes) bool {
 	}
 
 	skuLower := strings.ToLower(item.SkuName)
-	if strings.Contains(skuLower, "low priority") {
-		return false
-	}
-
-	return true
+	return !strings.Contains(skuLower, "low priority")
 }
 
 // includeDiskItem filters disk items to include only managed disks.

modules/pricing/public/gcp/gcppricingsource.go

@@ -15,11 +15,12 @@ import (
 	"github.com/opencost/opencost/core/pkg/log"
 	"github.com/opencost/opencost/core/pkg/pricing"
 	"github.com/opencost/opencost/core/pkg/unit"
+	"github.com/opencost/opencost/modules/pricing/public/httpclient"
 )
 
 var BillingAPIBaseURL = "https://cloudbilling.googleapis.com/v1/services/6F81-5844-456A/skus"
 
-var gcpHTTPClient = &http.Client{Timeout: 120 * time.Second}
+var gcpHTTPClient = httpclient.NewClient(120 * time.Second)
 
 type GCPPricingSourceConfig struct {
 	APIKey       string

modules/pricing/public/httpclient/httpclient.go

@@ -0,0 +1,79 @@
+package httpclient
+
+import (
+	"io"
+	"net/http"
+	"strconv"
+	"time"
+
+	"github.com/opencost/opencost/core/pkg/log"
+)
+
+const (
+	defaultMaxRetries    = 5
+	defaultRetryBaseWait = 2 * time.Second
+	defaultRetryMaxWait  = 60 * time.Second
+)
+
+// retryTransport is an http.RoundTripper that retries requests on 429 and 503s
+type retryTransport struct {
+	wrapped     http.RoundTripper
+	maxRetries  int
+	baseWait    time.Duration
+	maxWait     time.Duration
+}
+
+func (t *retryTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	wait := t.baseWait
+	for attempt := 0; attempt <= t.maxRetries; attempt++ {
+		resp, err := t.wrapped.RoundTrip(req)
+		if err != nil {
+			return nil, err
+		}
+
+		if resp.StatusCode != http.StatusTooManyRequests && resp.StatusCode != http.StatusServiceUnavailable {
+			return resp, nil
+		}
+
+		if attempt == t.maxRetries {
+			// Return the final error response untouched so the caller can
+			// read the body and status code.
+			return resp, nil
+		}
+
+		// Consume and discard the error body so the connection can be reused,
+		// then close it before sleeping.
+		_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 4096))
+		_ = resp.Body.Close()
+
+		delay := wait
+		if ra := resp.Header.Get("Retry-After"); ra != "" {
+			if secs, err := strconv.Atoi(ra); err == nil {
+				delay = time.Duration(secs) * time.Second
+			}
+		}
+		if delay > t.maxWait {
+			delay = t.maxWait
+		}
+
+		log.Warnf("pricing httpclient: HTTP %d, retrying in %s (attempt %d/%d)",
+			resp.StatusCode, delay, attempt+1, t.maxRetries)
+		time.Sleep(delay)
+		wait *= 2
+	}
+	return nil, nil
+}
+
+// NewClient returns an *http.Client whose transport automatically retries
+// on HTTP 429 / 503 with exponential backoff
+func NewClient(timeout time.Duration) *http.Client {
+	return &http.Client{
+		Timeout: timeout,
+		Transport: &retryTransport{
+			wrapped:    http.DefaultTransport,
+			maxRetries: defaultMaxRetries,
+			baseWait:   defaultRetryBaseWait,
+			maxWait:    defaultRetryMaxWait,
+		},
+	}
+}