Merge branch 'nico/fixes-for-stack-source-config-name' of github.com:porter-dev/porter into dev

api/types/stacks.go

@@ -22,6 +22,12 @@ type CreateStackRequest struct {
 	EnvGroups []*CreateStackEnvGroupRequest `json:"env_groups,omitempty" form:"required,dive,required"`
 }
 
+type PutStackSourceConfigPaylod struct {
+	*CreateStackSourceConfigRequest
+
+	StableSourceConfigID string `json:"stable_source_config_id" form:"required"`
+}
+
 // swagger:model
 type PutStackSourceConfigRequest struct {
 	SourceConfigs []*CreateStackSourceConfigRequest `json:"source_configs,omitempty" form:"required,dive,required"`
@@ -262,7 +268,7 @@ type CreateStackSourceConfigRequest struct {
 	// required: true
 	ImageTag string `json:"image_tag" form:"required"`
 
-	StableSourceConfigID string `json:"source_config_id,omitempty"`
+	StableSourceConfigID string `json:"stable_source_config_id,omitempty"`
 
 	// If this field is empty, the resource is deployed directly from the image repo uri
 	StackSourceConfigBuild *StackSourceConfigBuild `json:"build,omitempty"`

cmd/migrate/main.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/porter-dev/porter/api/server/shared/config/envloader"
 	"github.com/porter-dev/porter/cmd/migrate/keyrotate"
+	"github.com/porter-dev/porter/cmd/migrate/stable_source_config_id_population"
 
 	adapter "github.com/porter-dev/porter/internal/adapter"
 	"github.com/porter-dev/porter/internal/repository/gorm"
@@ -61,6 +62,10 @@ func main() {
 		}
 	}
 
+	if shouldPopulateStableSourceConfigId() {
+		stable_source_config_id_population.PopulateStableSourceConfigId(db)
+	}
+
 	if err := InstanceMigrate(db, envConf.DBConf); err != nil {
 		logger.Fatal().Err(err).Msg("vault migration failed")
 	}
@@ -83,3 +88,19 @@ func shouldKeyRotate() (bool, string, string) {
 
 	return c.OldEncryptionKey != "" && c.NewEncryptionKey != "", c.OldEncryptionKey, c.NewEncryptionKey
 }
+
+type StableSourcePopulateConf struct {
+	// Simple env variable that will let us know if we should populate the stable_source_config_id column
+	POPULATE_SOURCE_CONFIG_ID string `env:"POPULATE_SOURCE_CONFIG_ID"`
+}
+
+func shouldPopulateStableSourceConfigId() bool {
+	var c StableSourcePopulateConf
+
+	if err := envdecode.StrictDecode(&c); err != nil {
+		log.Fatalf("Failed to decode migration conf: %s", err)
+		return false
+	}
+
+	return c.POPULATE_SOURCE_CONFIG_ID == "true"
+}

cmd/migrate/stable_source_config_id_population/helpers_test.go

@@ -0,0 +1,323 @@
+package stable_source_config_id_population_test
+
+import (
+	"fmt"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/porter-dev/porter/api/server/shared/config/env"
+	"github.com/porter-dev/porter/api/types"
+	"github.com/porter-dev/porter/internal/adapter"
+	"github.com/porter-dev/porter/internal/encryption"
+	"github.com/porter-dev/porter/internal/models"
+	ints "github.com/porter-dev/porter/internal/models/integrations"
+	"github.com/porter-dev/porter/internal/repository"
+	"github.com/porter-dev/porter/internal/repository/gorm"
+	_gorm "gorm.io/gorm"
+)
+
+type tester struct {
+	Key *[32]byte
+	DB  *_gorm.DB
+
+	repo         repository.Repository
+	dbFileName   string
+	key          *[32]byte
+	initUsers    []*models.User
+	initProjects []*models.Project
+	initClusters []*models.Cluster
+	initKIs      []*ints.KubeIntegration
+	initStacks   []*models.Stack
+}
+
+func setupTestEnv(tester *tester, t *testing.T) {
+	t.Helper()
+
+	db, err := adapter.New(&env.DBConf{
+		EncryptionKey: "__random_strong_encryption_key__",
+		SQLLite:       true,
+		SQLLitePath:   tester.dbFileName,
+	})
+
+	if err != nil {
+		t.Fatalf("%\n", err)
+	}
+
+	err = db.AutoMigrate(
+		&models.Project{},
+		&models.User{},
+		&models.Cluster{},
+		&models.Stack{},
+		&models.StackEnvGroup{},
+		&models.StackSourceConfig{},
+		&models.StackRevision{},
+		&models.StackResource{},
+		&ints.KubeIntegration{},
+		&ints.ClusterTokenCache{},
+	)
+
+	if err != nil {
+		t.Fatalf("%v\n", err)
+	}
+
+	var key [32]byte
+
+	for i, b := range []byte("__random_strong_encryption_key__") {
+		key[i] = b
+	}
+
+	tester.key = &key
+	tester.Key = &key
+	tester.DB = db
+
+	tester.repo = gorm.NewRepository(db, &key, nil)
+}
+
+func cleanup(tester *tester, t *testing.T) {
+	t.Helper()
+
+	// remove the created file file
+	os.Remove(tester.dbFileName)
+}
+
+func initUser(tester *tester, t *testing.T) {
+	t.Helper()
+
+	user := &models.User{
+		Email:    "example@example.com",
+		Password: "hello1234",
+	}
+
+	user, err := tester.repo.User().CreateUser(user)
+
+	if err != nil {
+		t.Fatalf("%v\n", err)
+	}
+
+	tester.initUsers = append(tester.initUsers, user)
+}
+
+func initCluster(tester *tester, t *testing.T) {
+	t.Helper()
+
+	if len(tester.initProjects) == 0 {
+		initProject(tester, t)
+	}
+
+	if len(tester.initKIs) == 0 {
+		initKubeIntegration(tester, t)
+	}
+
+	cluster := &models.Cluster{
+		ProjectID:                tester.initProjects[0].ID,
+		Name:                     "cluster-test",
+		Server:                   "https://localhost",
+		KubeIntegrationID:        tester.initKIs[0].ID,
+		CertificateAuthorityData: []byte("-----BEGIN"),
+		TokenCache: ints.ClusterTokenCache{
+			TokenCache: ints.TokenCache{
+				Token:  []byte("token-1"),
+				Expiry: time.Now().Add(-1 * time.Hour),
+			},
+		},
+	}
+
+	cluster, err := tester.repo.Cluster().CreateCluster(cluster)
+
+	if err != nil {
+		t.Fatalf("%v\n", err)
+	}
+
+	tester.initClusters = append(tester.initClusters, cluster)
+}
+
+func initProject(tester *tester, t *testing.T) {
+	t.Helper()
+
+	proj := &models.Project{
+		Name: "project-test",
+	}
+
+	proj, err := tester.repo.Project().CreateProject(proj)
+
+	if err != nil {
+		t.Fatalf("%v\n", err)
+	}
+
+	tester.initProjects = append(tester.initProjects, proj)
+}
+
+func initKubeIntegration(tester *tester, t *testing.T) {
+	t.Helper()
+
+	if len(tester.initProjects) == 0 {
+		initProject(tester, t)
+	}
+
+	if len(tester.initUsers) == 0 {
+		initUser(tester, t)
+	}
+
+	ki := &ints.KubeIntegration{
+		Mechanism:             ints.KubeLocal,
+		ProjectID:             tester.initProjects[0].ID,
+		UserID:                tester.initUsers[0].ID,
+		Kubeconfig:            []byte("current-context: testing\n"),
+		ClientCertificateData: []byte("clientcertdata"),
+		ClientKeyData:         []byte("clientkeydata"),
+		Token:                 []byte("token"),
+		Username:              []byte("username"),
+		Password:              []byte("password"),
+	}
+
+	ki, err := tester.repo.KubeIntegration().CreateKubeIntegration(ki)
+
+	if err != nil {
+		t.Fatalf("%v\n", err)
+	}
+
+	tester.initKIs = append(tester.initKIs, ki)
+}
+
+func initStack(tester *tester, t *testing.T, stackName string) {
+	t.Helper()
+
+	if len(tester.initProjects) == 0 {
+		initProject(tester, t)
+	}
+
+	if len(tester.initUsers) == 0 {
+		initUser(tester, t)
+	}
+
+	if len(tester.initClusters) == 0 {
+		initCluster(tester, t)
+	}
+
+	uid, _ := encryption.GenerateRandomBytes(16)
+
+	sourceConfigs := []models.StackSourceConfig{
+		{
+			Name:         "source-config-1",
+			ImageRepoURI: "some-repo",
+			ImageTag:     "some-tag",
+			UID:          uid,
+		},
+	}
+
+	// write stack to the database with creating status
+	stack := &models.Stack{
+		ProjectID: tester.initProjects[0].ID,
+		ClusterID: tester.initClusters[0].ID,
+		Namespace: "test-namespace",
+		Name:      stackName,
+		UID:       uid,
+		Revisions: []models.StackRevision{
+			{
+				RevisionNumber: 1,
+				Status:         string(types.StackRevisionStatusDeploying),
+				SourceConfigs:  sourceConfigs,
+			},
+		},
+	}
+
+	newStack, err := tester.repo.Stack().CreateStack(stack)
+
+	if err != nil {
+		t.Fatalf("%v\n", err)
+	}
+
+	tester.initStacks = append(tester.initStacks, newStack)
+}
+
+func createNewStackRevision(tester *tester, t *testing.T, stackName string) {
+	t.Helper()
+
+	if len(tester.initProjects) == 0 {
+		initProject(tester, t)
+	}
+	if len(tester.initUsers) == 0 {
+		initUser(tester, t)
+	}
+	if len(tester.initClusters) == 0 {
+		initCluster(tester, t)
+	}
+	if len(tester.initStacks) == 0 {
+		initStack(tester, t, stackName)
+	}
+
+	stack := tester.initStacks[0]
+
+	for _, s := range tester.initStacks {
+		if s.Name == stackName {
+			stack = s
+			break
+		}
+	}
+
+	prevRevision := findLatestRevisionByRevisionNumber(t, stack.Revisions)
+
+	oldSourceConfig := prevRevision.SourceConfigs[0]
+
+	newUid, _ := encryption.GenerateRandomBytes(16)
+	sourceConfigs := []models.StackSourceConfig{
+		{
+			Name:         oldSourceConfig.Name,
+			ImageRepoURI: "some-repo-" + fmt.Sprint(prevRevision.RevisionNumber+1),
+			ImageTag:     "some-tag-" + fmt.Sprint(prevRevision.RevisionNumber+1),
+			UID:          newUid,
+		},
+	}
+
+	newRevision := models.StackRevision{
+		RevisionNumber: prevRevision.RevisionNumber + 1,
+		Status:         string(types.StackRevisionStatusDeploying),
+		SourceConfigs:  sourceConfigs,
+		StackID:        stack.ID,
+	}
+
+	tester.repo.Stack().AppendNewRevision(&newRevision)
+}
+
+func findLatestRevisionByRevisionNumber(t *testing.T, revisions []models.StackRevision) *models.StackRevision {
+	t.Helper()
+
+	latestRevision := revisions[0]
+	for _, revision := range revisions {
+		if revision.RevisionNumber > latestRevision.RevisionNumber {
+			latestRevision = revision
+		}
+	}
+
+	return &latestRevision
+}
+
+func appendNewSourceConfig(t *testing.T, tester *tester, stack *models.Stack, sourceConfig models.StackSourceConfig) {
+	t.Helper()
+
+	prevRevision := findLatestRevisionByRevisionNumber(t, stack.Revisions)
+
+	previousSourceConfigs := []models.StackSourceConfig{}
+
+	for _, sourceConfig := range prevRevision.SourceConfigs {
+		newUid, _ := encryption.GenerateRandomBytes(16)
+
+		sc := models.StackSourceConfig{
+			Name:         sourceConfig.Name,
+			ImageRepoURI: sourceConfig.ImageRepoURI,
+			ImageTag:     sourceConfig.ImageTag,
+			UID:          newUid,
+		}
+		previousSourceConfigs = append(previousSourceConfigs, sc)
+	}
+
+	newRevision := models.StackRevision{
+		RevisionNumber: prevRevision.RevisionNumber + 1,
+		Status:         string(types.StackRevisionStatusDeploying),
+		SourceConfigs:  append(prevRevision.SourceConfigs, sourceConfig),
+		StackID:        stack.ID,
+	}
+
+	tester.repo.Stack().AppendNewRevision(&newRevision)
+}

cmd/migrate/stable_source_config_id_population/populate.go

@@ -0,0 +1,65 @@
+package stable_source_config_id_population
+
+import (
+	"github.com/porter-dev/porter/internal/models"
+	_gorm "gorm.io/gorm"
+)
+
+func PopulateStableSourceConfigId(db *_gorm.DB) {
+	var dest []*models.StackRevision
+	db.Model(&models.StackRevision{}).Preload("SourceConfigs").Find(&dest)
+
+	// Create a map that will separate source configs based on stack and name
+	// this will allow us to check all the source configs revisions that correspond
+	// to the same source config object.
+	sourceConfigsPerStack := make(map[uint]map[string][]models.StackSourceConfig)
+	for _, revision := range dest {
+		if sourceConfigsPerStack[revision.StackID] == nil {
+			sourceConfigsPerStack[revision.StackID] = make(map[string][]models.StackSourceConfig)
+		}
+
+		for _, sc := range revision.SourceConfigs {
+			sourceConfigsPerStack[revision.StackID][sc.Name] = append(sourceConfigsPerStack[revision.StackID][sc.Name], sc)
+		}
+	}
+
+	// Populate the stable source config id for each revision of the source config
+	for _, sourceConfigsWithSameNameMap := range sourceConfigsPerStack {
+		for _, sc := range sourceConfigsWithSameNameMap {
+			sortedSourceConfigs := sortSourceConfigsByCreationDate(sc)
+
+			stableSourceConfigId := findSourceConfigWithStableSourceConfigID(sortedSourceConfigs)
+
+			if stableSourceConfigId == "" {
+				stableSourceConfigId = sortedSourceConfigs[0].UID
+			}
+
+			for _, sourceConfig := range sortedSourceConfigs {
+				sourceConfig.StableSourceConfigID = stableSourceConfigId
+				db.Save(sourceConfig)
+			}
+		}
+	}
+}
+
+func findSourceConfigWithStableSourceConfigID(sourceConfigs []models.StackSourceConfig) string {
+	for _, sc := range sourceConfigs {
+		if sc.StableSourceConfigID != "" {
+			return sc.StableSourceConfigID
+		}
+	}
+	return ""
+}
+
+// sort source configs by creation date
+func sortSourceConfigsByCreationDate(sourceConfigs []models.StackSourceConfig) []models.StackSourceConfig {
+	for i := 0; i < len(sourceConfigs); i++ {
+		for j := i + 1; j < len(sourceConfigs); j++ {
+			if sourceConfigs[i].CreatedAt.After(sourceConfigs[j].CreatedAt) {
+				sourceConfigs[i], sourceConfigs[j] = sourceConfigs[j], sourceConfigs[i]
+			}
+		}
+	}
+
+	return sourceConfigs
+}

cmd/migrate/stable_source_config_id_population/populate_test.go

@@ -0,0 +1,207 @@
+package stable_source_config_id_population_test
+
+import (
+	"testing"
+
+	"github.com/porter-dev/porter/cmd/migrate/stable_source_config_id_population"
+	"github.com/porter-dev/porter/internal/encryption"
+	"github.com/porter-dev/porter/internal/models"
+)
+
+func TestAllSourceConfigHaveSameStableSourceConfigID(t *testing.T) {
+
+	tester := &tester{
+		dbFileName: "./porter_stable_source_config_id_population.db",
+	}
+
+	setupTestEnv(tester, t)
+
+	defer cleanup(tester, t)
+
+	stackName := "first-stack"
+
+	initStack(tester, t, stackName)
+
+	createNewStackRevision(tester, t, stackName)
+
+	createNewStackRevision(tester, t, stackName)
+
+	createNewStackRevision(tester, t, stackName)
+
+	stable_source_config_id_population.PopulateStableSourceConfigId(tester.DB)
+
+	sourceConfigs := []*models.StackSourceConfig{}
+
+	if err := tester.DB.Find(&sourceConfigs).Error; err != nil {
+		t.Fatalf("failed to find source configs: %s", err)
+	}
+
+	if len(sourceConfigs) != 4 {
+		t.Fatalf("expected 4 source configs, got %d", len(sourceConfigs))
+	}
+
+	for _, sc := range sourceConfigs {
+		if sc.StableSourceConfigID == "" {
+			t.Fatalf("expected stable source config id to be populated, got empty string")
+		}
+	}
+
+	// check if all StableSourceConfigID are equal
+	for _, sc := range sourceConfigs {
+		if sc.StableSourceConfigID != sourceConfigs[0].StableSourceConfigID {
+			t.Fatalf("expected all StableSourceConfigID to be equal, got %s", sc.StableSourceConfigID)
+		}
+	}
+
+}
+
+func TestSourceConfigWithDifferentNamesShouldHaveDifferentStableSourceConfigID(t *testing.T) {
+	tester := &tester{
+		dbFileName: "./porter_stable_source_config_id_population.db",
+	}
+
+	setupTestEnv(tester, t)
+
+	initStack(tester, t, "first-stack")
+
+	defer cleanup(tester, t)
+
+	createNewStackRevision(tester, t, "first-stack")
+
+	uid, _ := encryption.GenerateRandomBytes(16)
+
+	newSourceConfig := &models.StackSourceConfig{
+		Name:         "second-source-config",
+		ImageRepoURI: "docker.io/porter-dev/porter-test-image",
+		ImageTag:     "latest",
+		UID:          uid,
+	}
+
+	appendNewSourceConfig(t, tester, tester.initStacks[0], *newSourceConfig)
+
+	stable_source_config_id_population.PopulateStableSourceConfigId(tester.DB)
+
+	sourceConfigs := []*models.StackSourceConfig{}
+
+	if err := tester.DB.Find(&sourceConfigs).Error; err != nil {
+		t.Fatalf("failed to find source configs: %s", err)
+	}
+
+	if len(sourceConfigs) != 3 {
+		t.Fatalf("expected 3 source configs, got %d", len(sourceConfigs))
+	}
+
+	for _, sc := range sourceConfigs {
+		if sc.StableSourceConfigID == "" {
+			t.Fatalf("expected stable source config id to be populated, got empty string on source config %s", sc.Name)
+		}
+	}
+
+	// map source configs into a map of StableSourceConfigID to SourceConfig
+	sourceConfigMap := make(map[string][]*models.StackSourceConfig)
+
+	for _, sc := range sourceConfigs {
+		sourceConfigMap[sc.Name] = append(sourceConfigMap[sc.Name], sc)
+	}
+
+	// check if all source configs that share name have the same StableSourceConfigID
+	for sourceConfigName, _ := range sourceConfigMap {
+		for _, sc := range sourceConfigMap[sourceConfigName] {
+			if sc.StableSourceConfigID != sourceConfigMap[sourceConfigName][0].StableSourceConfigID {
+				t.Fatalf("expected all StableSourceConfigID to be equal, got %s", sc.StableSourceConfigID)
+			}
+		}
+	}
+}
+
+func TestSourceConfigsFromDifferentStacksShouldHaveDifferentStableSourceConfigId(t *testing.T) {
+	tester := &tester{
+		dbFileName: "./porter_stable_source_config_id_population.db",
+	}
+
+	setupTestEnv(tester, t)
+
+	initStack(tester, t, "first-stack")
+	initStack(tester, t, "second-stack")
+
+	defer cleanup(tester, t)
+
+	createNewStackRevision(tester, t, "first-stack")
+	createNewStackRevision(tester, t, "second-stack")
+	createNewStackRevision(tester, t, "first-stack")
+	createNewStackRevision(tester, t, "second-stack")
+
+	stable_source_config_id_population.PopulateStableSourceConfigId(tester.DB)
+
+	sourceConfigs := []*models.StackSourceConfig{}
+
+	if err := tester.DB.Find(&sourceConfigs).Error; err != nil {
+		t.Fatalf("failed to find source configs: %s", err)
+	}
+
+	if len(sourceConfigs) != 6 {
+		t.Fatalf("expected 6 source configs, got %d", len(sourceConfigs))
+	}
+
+	for _, sc := range sourceConfigs {
+		if sc.StableSourceConfigID == "" {
+			t.Fatalf("expected stable source config id to be populated, got empty string on source config %s", sc.Name)
+		}
+	}
+
+	var firstStack *models.Stack
+	var secondStack *models.Stack
+
+	stacks := []*models.Stack{}
+
+	if err := tester.DB.Model(&models.Stack{}).Preload("Revisions").Preload("Revisions.SourceConfigs").Find(&stacks).Error; err != nil {
+		t.Fatalf("failed to find stacks: %s", err)
+	}
+
+	for _, stack := range stacks {
+		if stack.Name == "first-stack" {
+			firstStack = stack
+		} else if stack.Name == "second-stack" {
+			secondStack = stack
+		}
+	}
+
+	firstStackSourceConfigs := []models.StackSourceConfig{}
+	// Get source configs from revisions on firstStack
+	for _, revision := range firstStack.Revisions {
+		for _, sourceConfig := range revision.SourceConfigs {
+			firstStackSourceConfigs = append(firstStackSourceConfigs, sourceConfig)
+		}
+	}
+
+	secondStackSourceConfigs := []models.StackSourceConfig{}
+	// Get source configs from revisions on secondStack
+	for _, revision := range secondStack.Revisions {
+		for _, sourceConfig := range revision.SourceConfigs {
+			secondStackSourceConfigs = append(secondStackSourceConfigs, sourceConfig)
+		}
+	}
+
+	// Check that all the source configs from the stacks have the same StableSourceConfigID
+	for _, sc := range firstStackSourceConfigs {
+		if sc.StableSourceConfigID != firstStackSourceConfigs[0].StableSourceConfigID {
+			t.Fatalf("expected all StableSourceConfigID to be equal, got %s", sc.StableSourceConfigID)
+		}
+	}
+
+	for _, sc := range secondStackSourceConfigs {
+		if sc.StableSourceConfigID != secondStackSourceConfigs[0].StableSourceConfigID {
+			t.Fatalf("expected all StableSourceConfigID to be equal, got %s", sc.StableSourceConfigID)
+		}
+	}
+
+	// check that all source configs from first stack have different StableSourceConfigID than source configs from second stack
+	for _, sc := range firstStackSourceConfigs {
+		for _, sc2 := range secondStackSourceConfigs {
+			if sc.StableSourceConfigID == sc2.StableSourceConfigID {
+				t.Fatalf("expected all StableSourceConfigID to be different, got %s", sc.StableSourceConfigID)
+			}
+		}
+	}
+
+}

internal/stacks/helpers.go

@@ -19,12 +19,20 @@ func CloneSourceConfigs(sourceConfigs []models.StackSourceConfig) ([]models.Stac
 			return nil, err
 		}
 
-		res = append(res, models.StackSourceConfig{
+		newSourceConfig := &models.StackSourceConfig{
 			UID:          uid,
 			Name:         sourceConfig.Name,
 			ImageRepoURI: sourceConfig.ImageRepoURI,
 			ImageTag:     sourceConfig.ImageTag,
-		})
+		}
+
+		if sourceConfig.StableSourceConfigID != "" {
+			newSourceConfig.StableSourceConfigID = sourceConfig.StableSourceConfigID
+		} else {
+			newSourceConfig.StableSourceConfigID = string(uid)
+		}
+
+		res = append(res, *newSourceConfig)
 	}
 
 	return res, nil