cost-model/.github/workflows/integration-testing.yaml

name: Run OpenCost Integration Tests

on:
    schedule:
      - cron: '0 14 * * *'
    push:
      branches:
        - develop
    pull_request_target:
      branches:
        - develop
    merge_group:
      types: [checks_requested]

concurrency:
    group: ${{ github.event.merge_group.head.sha || github.event.pull_request.head.sha || github.ref }}-intg-tests
    cancel-in-progress: false

permissions: {}
jobs:
    check_actor_permissions:
      runs-on: ubuntu-latest
      if: ${{ github.event_name == 'pull_request_target' }}
      outputs:
          ismaintainer: ${{ steps.determine-maintainer.outputs.ismaintainer }}
      steps:
        - name: Check team membership
          uses: tspascoal/get-user-teams-membership@v3
          if: ${{ github.actor != 'dependabot[bot]' }}
          id: teamAffiliation
          with:
            GITHUB_TOKEN: ${{ secrets.ORG_READER_PAT }}
            username: ${{ github.actor }}
            organization: opencost
        - name: determine if actor is a maintainer
          id: determine-maintainer
          env:
            TEAMS: ${{ join(steps.teamAffiliation.outputs.teams, ',') }}
            ACTOR: ${{ github.actor }}
            IS_MAINTAINER: ${{ contains(join(steps.teamAffiliation.outputs.teams, ','), 'OpenCost Maintainers') || github.actor == 'dependabot[bot]' }}
          run: |
              echo "Actor: $ACTOR"
              echo "Is maintainer: $IS_MAINTAINER"
              echo "ismaintainer=$IS_MAINTAINER" >> $GITHUB_OUTPUT
  
    noop-tests:
        needs: check_actor_permissions
        permissions: {}
        runs-on: ubuntu-latest
        if: ${{ (always() && !cancelled()) && github.event_name == 'pull_request_target'  && needs.check_actor_permissions.outputs.ismaintainer == 'false' }}
        outputs:
            is_noop: ${{ steps.noop-tests.outputs.is_noop }}
        steps:
          - name: Tests Not Needed
            id: noop-tests
            run: |
              echo "integration tests not running because you are not a maintainer. they will run automatically when a PR is merged."
              echo "is_noop=true" >> $GITHUB_OUTPUT
    wait_for_image_ready:
        runs-on: ubuntu-latest
        permissions: {}
        needs: check_actor_permissions
        if: ${{ (always() && !cancelled()) && ( github.event.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'merge_group' || (github.event_name == 'pull_request_target'  && needs.check_actor_permissions.outputs.ismaintainer == 'true')) }}
        outputs:
            IMAGE_TAG: ${{ steps.set_image_tags.outputs.IMAGE_TAG }}
            NAMESPACE: ${{ steps.set_image_tags.outputs.NAMESPACE }}
            MAINBRANCH: ${{ steps.set_image_tags.outputs.mainbranch }}
            passed: ${{ steps.wait_for_image_ready.outputs.passed }}
        steps:
          - uses: actions/checkout@v4
            with:
              ref: ${{ github.event.merge_group.head.sha || github.event.pull_request.head.sha || github.ref }}
          - name: Set OC SHA
            id: sha
            run: |
                 echo "OC_SHORTHASH=$(git rev-parse --short HEAD)"
                 echo "OC_SHORTHASH=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
          - name: Set image tags
            id: set_image_tags
            env:
              OC_SHORTHASH: ${{ steps.sha.outputs.OC_SHORTHASH }}
              REPO_OWNER: ${{ github.repository_owner }}
              EVENT_NAME: ${{ github.event_name }}
              PR_NUMBER: ${{ github.event.pull_request.number }}
            run: |
                    echo "github.event_name: $EVENT_NAME"
                    if [[ "$EVENT_NAME" == "merge_group" ]]; then
                      echo "IMAGE_TAG=ghcr.io/$REPO_OWNER/opencost:test-$OC_SHORTHASH" >> $GITHUB_OUTPUT
                      echo "NAMESPACE=merge-queue-oc-$OC_SHORTHASH" >> $GITHUB_OUTPUT
                      echo "mainbranch=false" >> $GITHUB_OUTPUT
                    elif [[ "$EVENT_NAME" == "pull_request_target" ]]; then
                      echo "building on maintainer pull request branch"
                      echo "IMAGE_TAG=ghcr.io/$REPO_OWNER/opencost:test-$OC_SHORTHASH" >> $GITHUB_OUTPUT
                      echo "NAMESPACE=pr-$PR_NUMBER-oc-$OC_SHORTHASH" >> $GITHUB_OUTPUT
                      echo "mainbranch=false" >> $GITHUB_OUTPUT
                    else
                      echo "building on develop branch"
                      echo "IMAGE_TAG=ghcr.io/$REPO_OWNER/opencost:develop-$OC_SHORTHASH" >> $GITHUB_OUTPUT
                      echo "NAMESPACE=develop-oc-$OC_SHORTHASH" >> $GITHUB_OUTPUT
                      echo "mainbranch=true" >> $GITHUB_OUTPUT
                    fi

          - name: Log into ghcr.io
            uses: docker/login-action@v3
            with:
              registry: ghcr.io
              username: ${{ github.actor }}
              password: ${{ secrets.GITHUB_TOKEN }}

          - name: wait for docker image to be ready
            id: wait_for_image_ready
            env:
              IMAGE_TAG: ${{ steps.set_image_tags.outputs.IMAGE_TAG }}
            run: |
                max_attempts=100
                # Loop until the Docker image can be pulled
                until docker manifest inspect $IMAGE_TAG; do
                    echo "Waiting for Docker image $IMAGE_TAG to be available, $max_attempts tries remain..."
                    sleep 6
                    max_attempts=$((max_attempts - 1))
                    if [[ $max_attempts -eq 0 ]]; then
                        echo "Docker image $IMAGE_TAG is not available after 10 minutes. Exiting..."
                        exit 1
                    fi
                done

                echo "Docker image $IMAGE_TAG is ready!"
                
                echo "passed=true" >> $GITHUB_OUTPUT
                
    build-test-stack:
        needs: wait_for_image_ready
        if: ${{ (always() && !cancelled()) && ( github.event.event_name == 'schedule'  || github.event_name == 'push' || github.event_name == 'merge_group' || (github.event_name == 'pull_request_target'  && needs.check_actor_permissions.outputs.ismaintainer == 'true')) }}
        uses: opencost/opencost-infra/.github/workflows/build-stack.yaml@main
        secrets: inherit
        with:
            oc-container-version: "${{ needs.wait_for_image_ready.outputs.IMAGE_TAG }}"
            namespace: "${{ needs.wait_for_image_ready.outputs.NAMESPACE }}"
            
    build-test-stack-promless:
        needs: wait_for_image_ready
        if: ${{ (always() && !cancelled()) && ( github.event.event_name == 'schedule'  || github.event_name == 'push' || github.event_name == 'merge_group' || (github.event_name == 'pull_request_target'  && needs.check_actor_permissions.outputs.ismaintainer == 'true')) }}
        uses: opencost/opencost-infra/.github/workflows/build-stack.yaml@main
        secrets: inherit
        with:
            oc-container-version: "${{ needs.wait_for_image_ready.outputs.IMAGE_TAG }}"
            namespace: "${{ needs.wait_for_image_ready.outputs.NAMESPACE }}-promless"
            prometheus: false

    wait-for-dns:
        needs: [wait_for_image_ready, build-test-stack]
        runs-on: ubuntu-latest
        if: ${{ (always() && !cancelled()) && ( github.event.event_name == 'schedule'  || github.event_name == 'push' || github.event_name == 'merge_group' || (github.event_name == 'pull_request_target'  && needs.check_actor_permissions.outputs.ismaintainer == 'true')) }}
        permissions: {}
        steps:
          - name: Wait for DNS to resolve
            id: wait-for-dns
            env:
              NAMESPACE: ${{ needs.wait_for_image_ready.outputs.NAMESPACE }}
            run: |
              echo "Waiting for $NAMESPACE.infra.opencost.io to resolve in DNS..."
              
              max_attempts=60
              until host $NAMESPACE.infra.opencost.io; do
                echo "DNS not yet resolved for $NAMESPACE.infra.opencost.io, $max_attempts tries remain..."
                sleep 10
                max_attempts=$((max_attempts - 1))
                if [[ $max_attempts -eq 0 ]]; then
                  echo "DNS resolution failed for $NAMESPACE.infra.opencost.io after 10 minutes. Exiting..."
                  exit 1
                fi
              done
              
              echo "DNS resolved successfully for $NAMESPACE.infra.opencost.io!"

    run-tests:
        needs: [wait_for_image_ready, build-test-stack, wait-for-dns]
        if: ${{ (always() && !cancelled()) && ( github.event.event_name == 'schedule'  || github.event_name == 'push' || github.event_name == 'merge_group' || (github.event_name == 'pull_request_target'  && needs.check_actor_permissions.outputs.ismaintainer == 'true')) }}
        permissions: {}
        uses: opencost/opencost-infra/.github/workflows/test-stack.yaml@main
        secrets: inherit
        with:
            namespace: "${{ needs.wait_for_image_ready.outputs.NAMESPACE }}"
            target_branch: "${{ github.event.pull_request.head.ref || 'main' }}"
  
    wait-for-data-collection:
      needs: [wait_for_image_ready, build-test-stack, build-test-stack-promless]
      runs-on: ubuntu-latest
      if: ${{ (always() && !cancelled()) && ( github.event.event_name == 'schedule'  || github.event_name == 'push' || github.event_name == 'merge_group' || (github.event_name == 'pull_request_target'  && needs.check_actor_permissions.outputs.ismaintainer == 'true')) }}
      permissions: {}
      steps:
        - name: Wait 22 minutes for promless data collection
          run: |
            sleep 1320 # 22 minutes
    
    run-comparison-tests:
      needs: [wait_for_image_ready, build-test-stack, build-test-stack-promless, wait-for-data-collection]
      if: ${{ (always() && !cancelled()) && ( github.event.event_name == 'schedule'  || github.event_name == 'push' || github.event_name == 'merge_group' || (github.event_name == 'pull_request_target'  && needs.check_actor_permissions.outputs.ismaintainer == 'true')) }}
      permissions: {}
      uses: opencost/opencost-infra/.github/workflows/test-stack.yaml@main
      secrets: inherit
      with:
        namespace: "${{ needs.wait_for_image_ready.outputs.NAMESPACE }}"
        comparison_namespace: "${{ needs.wait_for_image_ready.outputs.NAMESPACE }}-promless"
        target_branch: "${{ github.event.pull_request.head.ref || 'main' }}"
        comparison: true
    
    print-outputs:
      needs: [run-comparison-tests, run-tests]
      if: ${{ (always() && !cancelled()) && ( github.event.event_name == 'schedule'  || github.event_name == 'push' || github.event_name == 'merge_group' || (github.event_name == 'pull_request_target'  && needs.check_actor_permissions.outputs.ismaintainer == 'true')) }}
      runs-on: ubuntu-latest
      permissions: {}
      steps:
        - name: Print outputs
          run: |
            echo "NAMESPACE: ${{ needs.wait_for_image_ready.outputs.NAMESPACE }}"
            echo "COMPARISON_NAMESPACE: ${{ needs.wait_for_image_ready.outputs.NAMESPACE }}-promless"
            echo "TARGET_BRANCH: ${{ github.event.pull_request.head.ref || 'main' }}"
            echo "PASSED COMPARISON: ${{ needs.run-comparison-tests.outputs.passed }}"
            echo "PASSED INTEGRATION: ${{ needs.run-tests.outputs.passed }}"
           
    hold-on-failure:
      needs: [wait_for_image_ready, run-tests, run-comparison-tests]
      if: ${{ always() && (needs.run-tests.outputs.passed == 'false' || needs.run-comparison-tests.outputs.passed == 'false') && github.event_name != 'merge_group'}} 
      runs-on: ubuntu-latest
      permissions: {}
      steps:
        - name: Hold stack for investigation
          env:
            NAMESPACE: ${{ needs.wait_for_image_ready.outputs.NAMESPACE }}
          run: |
            echo "Tests failed. Holding stacks up for 1 hour for investigation..."
            echo "Stack namespace: $NAMESPACE"
            echo "Stack will be automatically torn down after 1 hour"
            sleep 3600

    teardown-test-stack:
        needs: [wait_for_image_ready, run-tests, run-comparison-tests, hold-on-failure]
        if: ${{ (always() && !cancelled()) && ( github.event.event_name == 'schedule'  || github.event_name == 'push' || github.event_name == 'merge_group' || (github.event_name == 'pull_request_target'  && needs.check_actor_permissions.outputs.ismaintainer == 'true')) }}
        uses: opencost/opencost-infra/.github/workflows/destroy-stack.yaml@main
        secrets: inherit 
        permissions: {}
        with:
            namespace: "${{ needs.wait_for_image_ready.outputs.NAMESPACE }}"
    
    teardown-test-stack-comparison:
        needs: [wait_for_image_ready, run-comparison-tests, hold-on-failure]
        if: ${{ (always() && !cancelled()) && ( github.event.event_name == 'schedule'  || github.event_name == 'push' || github.event_name == 'merge_group' || (github.event_name == 'pull_request_target'  && needs.check_actor_permissions.outputs.ismaintainer == 'true')) }}
        uses: opencost/opencost-infra/.github/workflows/destroy-stack.yaml@main
        secrets: inherit 
        permissions: {}
        with:
            namespace: "${{ needs.wait_for_image_ready.outputs.NAMESPACE }}-promless"

    check-success:
        needs: [noop-tests, run-tests, run-comparison-tests]
        permissions: {}
        runs-on: ubuntu-latest
        if: ${{ always() }}
        steps:
          - name: Check success
            id: check-success
            env:
              IS_NOOP: ${{ needs.noop-tests.outputs.is_noop }}
              PASSED: ${{ needs.run-tests.outputs.passed }}
            run: |
              echo "IS_NOOP: $IS_NOOP"
              echo "PASSED: $PASSED"
              if [[ "$IS_NOOP" == "true" ]]; then
                echo "No-op tests, skipping success check"
                exit 0
              fi
              
              if [[ "$PASSED" != "true" ]]; then
                echo "One or more integration tests failed"
                exit 1
              fi
              
              echo "All integration tests passed"
              exit 0

    set-labels:
      needs: [noop-tests, run-tests, run-comparison-tests]
      if: ${{ (always() && !cancelled()) && ( github.event_name == 'pull_request_target' && needs.check_actor_permissions.outputs.ismaintainer == 'true') }}
      runs-on: ubuntu-latest
      permissions: {}
      steps:
        - name: label integration tests failing
          if: ${{ always()  && contains(needs.*.result, 'failure') && !cancelled()}}
          uses: andymckay/labeler@1.0.4
          with:
            add-labels: "integration tests failed"
        - uses: mondeja/remove-labels-gh-action@v2
          if: ${{ always()  && contains(needs.*.result, 'failure') && !cancelled()}}
          with:
                token: ${{ secrets.GITHUB_TOKEN }}
                labels: |
                  integration tests passed
        - name: Label integration tests passing
          if: ${{ always()  && !contains(needs.*.result, 'failure') && !cancelled()}}
          uses: andymckay/labeler@1.0.4
          with:
            add-labels: "integration tests passed"
        - uses: mondeja/remove-labels-gh-action@v2
          if: ${{ always()  && !contains(needs.*.result, 'failure') && !cancelled()}}
          with:
              token: ${{ secrets.GITHUB_TOKEN }}
              labels: |
                integration tests failed