Merge pull request #2231 from porter-dev/master

Namespace termination race condition -> staging

cli/cmd/apply.go

@@ -265,7 +265,7 @@ func (d *Driver) applyAddon(resource *models.Resource, client *api.Client, shoul
 	}
 
 	if shouldCreate {
-		err = client.DeployAddon(
+		err := client.DeployAddon(
 			context.Background(),
 			d.target.Project,
 			d.target.Cluster,
@@ -280,6 +280,10 @@ func (d *Driver) applyAddon(resource *models.Resource, client *api.Client, shoul
 				},
 			},
 		)
+
+		if err != nil {
+			return nil, err
+		}
 	} else {
 		bytes, err := json.Marshal(addonConfig)
 
@@ -297,17 +301,17 @@ func (d *Driver) applyAddon(resource *models.Resource, client *api.Client, shoul
 				Values: string(bytes),
 			},
 		)
-	}
 
-	if err != nil {
-		return nil, err
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	if err = d.assignOutput(resource, client); err != nil {
 		return nil, err
 	}
 
-	return resource, err
+	return resource, nil
 }
 
 func (d *Driver) applyApplication(resource *models.Resource, client *api.Client, shouldCreate bool) (*models.Resource, error) {

internal/kubernetes/agent.go

@@ -624,7 +624,39 @@ func (a *Agent) CreateNamespace(name string) (*v1.Namespace, error) {
 	)
 
 	if err == nil && checkNS != nil {
-		return checkNS, nil
+		if checkNS.Status.Phase == v1.NamespaceTerminating {
+			// edge case for when the same namespace was previously created
+			// but was deleted and is currently in the "Terminating" phase
+
+			// let us wait for a maximum of 10 seconds
+			timeNow := time.Now().Add(10 * time.Second)
+			stillTerminating := true
+			for {
+				_, err := a.Clientset.CoreV1().Namespaces().Get(
+					context.TODO(),
+					name,
+					metav1.GetOptions{},
+				)
+
+				if err != nil && errors.IsNotFound(err) {
+					stillTerminating = false
+					break
+				}
+
+				time.Sleep(time.Second)
+
+				if time.Now().After(timeNow) {
+					break
+				}
+			}
+
+			if stillTerminating {
+				// the namespace has been in the "Terminating" phase
+				return nil, fmt.Errorf("cannot create namespace %s, stuck in \"Terminating\" phase", name)
+			}
+		} else {
+			return checkNS, nil
+		}
 	}
 
 	namespace := v1.Namespace{
@@ -658,7 +690,7 @@ func (a *Agent) GetNamespace(name string) (*v1.Namespace, error) {
 // DeleteNamespace deletes the namespace given the name.
 func (a *Agent) DeleteNamespace(name string) error {
 	// check if namespace exists
-	_, err := a.Clientset.CoreV1().Namespaces().Get(
+	checkNS, err := a.Clientset.CoreV1().Namespaces().Get(
 		context.TODO(),
 		name,
 		metav1.GetOptions{},
@@ -669,6 +701,12 @@ func (a *Agent) DeleteNamespace(name string) error {
 		return nil
 	}
 
+	// if the namespace was found but is in the "Terminating" phase
+	// we should ignore it and not return an error
+	if checkNS != nil && checkNS.Status.Phase == v1.NamespaceTerminating {
+		return nil
+	}
+
 	return a.Clientset.CoreV1().Namespaces().Delete(
 		context.TODO(),
 		name,