cost-model/.github/workflows/integration-testing.yaml

name: Run OpenCost Integration Tests

# Privileged trigger: pull_request_target is required to run integration stacks for maintainer PRs. Only PRs authored by maintainers run; see check_actor_permissions.
on: # zizmor: ignore[dangerous-triggers]
    schedule:
      - cron: '0 14 * * *'
    push:
      branches:
        - develop
    pull_request_target:
      branches:
        - develop
    merge_group:
      types: [checks_requested]

concurrency:
    group: ${{ github.event.merge_group.head.sha || github.event.pull_request.head.sha || github.ref }}-intg-tests
    cancel-in-progress: false

permissions: {}
jobs:
    # pull_request_target runs with repo secrets, and the test stacks deploy an
    # image built from the PR head. Only run for PRs *authored* by a maintainer
    # (or by dependabot from this repo). Check the PR author, not github.actor:
    # actor is whoever triggered the event, e.g. a maintainer reopening a PR.
    check_actor_permissions:
      runs-on: ubuntu-latest
      if: ${{ github.event_name == 'pull_request_target' || github.event_name == 'merge_group' }}
      outputs:
          ismaintainer: ${{ steps.determine-maintainer.outputs.ismaintainer }}
      steps:
        - name: Check team membership
          uses: tspascoal/get-user-teams-membership@57e9f42acd78f4d0f496b3be4368fc5f62696662 # v3.0.0
          if: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.user.login != 'dependabot[bot]' }}
          id: teamAffiliation
          with:
            GITHUB_TOKEN: ${{ secrets.ORG_READER_PAT }}
            username: ${{ github.event.pull_request.user.login }}
            organization: opencost
        - name: determine if PR author is a maintainer
          id: determine-maintainer
          env:
            TEAMS: ${{ join(steps.teamAffiliation.outputs.teams, ',') }}
            AUTHOR: ${{ github.event.pull_request.user.login }}
            IS_MAINTAINER: ${{ contains(join(steps.teamAffiliation.outputs.teams, ','), 'OpenCost Maintainers') || (github.event.pull_request.user.login == 'dependabot[bot]' && github.event.pull_request.head.repo.full_name == 'opencost/opencost') }}
          run: |
              echo "PR author: $AUTHOR"
              echo "Is maintainer: $IS_MAINTAINER"
              echo "ismaintainer=$IS_MAINTAINER" >> "$GITHUB_OUTPUT"
  
    noop-tests:
        needs: check_actor_permissions
        permissions: {}
        runs-on: ubuntu-latest
        if: ${{ (always() && !cancelled()) && github.event_name == 'pull_request_target'  && needs.check_actor_permissions.outputs.ismaintainer == 'false' }}
        outputs:
            is_noop: ${{ steps.noop-tests.outputs.is_noop }}
        steps:
          - name: Tests Not Needed
            id: noop-tests
            run: |
              echo "integration tests not running because you are not a maintainer. they will run automatically when a PR is merged."
              echo "is_noop=true" >> $GITHUB_OUTPUT
    wait_for_image_ready:
        runs-on: ubuntu-latest
        permissions: {}
        needs: check_actor_permissions
        if: ${{ (always() && !cancelled()) && ( github.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'merge_group' || (github.event_name == 'pull_request_target'  && needs.check_actor_permissions.outputs.ismaintainer == 'true')) }}
        outputs:
            IMAGE_TAG: ${{ steps.set_image_tags.outputs.IMAGE_TAG }}
            NAMESPACE: ${{ steps.set_image_tags.outputs.NAMESPACE }}
            MAINBRANCH: ${{ steps.set_image_tags.outputs.mainbranch }}
            passed: ${{ steps.wait_for_image_ready.outputs.passed }}
        steps:
          - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
            with:
              ref: ${{ github.event.merge_group.head.sha || github.event.pull_request.head.sha || github.ref }}
              persist-credentials: false
          - name: Set OC SHA
            id: sha
            run: |
                 echo "OC_SHORTHASH=$(git rev-parse --short HEAD)"
                 echo "OC_SHORTHASH=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
          - name: Set image tags
            id: set_image_tags
            env:
              OC_SHORTHASH: ${{ steps.sha.outputs.OC_SHORTHASH }}
              REPO_OWNER: ${{ github.repository_owner }}
              EVENT_NAME: ${{ github.event_name }}
              PR_NUMBER: ${{ github.event.pull_request.number }}
            run: |
                    echo "github.event_name: $EVENT_NAME"
                    if [[ "$EVENT_NAME" == "merge_group" ]]; then
                      echo "IMAGE_TAG=ghcr.io/$REPO_OWNER/opencost:test-$OC_SHORTHASH" >> $GITHUB_OUTPUT
                      echo "NAMESPACE=merge-queue-oc-$OC_SHORTHASH" >> $GITHUB_OUTPUT
                      echo "mainbranch=false" >> $GITHUB_OUTPUT
                    elif [[ "$EVENT_NAME" == "pull_request_target" ]]; then
                      echo "building on maintainer pull request branch"
                      echo "IMAGE_TAG=ghcr.io/$REPO_OWNER/opencost:test-$OC_SHORTHASH" >> $GITHUB_OUTPUT
                      echo "NAMESPACE=pr-$PR_NUMBER-oc-$OC_SHORTHASH" >> $GITHUB_OUTPUT
                      echo "mainbranch=false" >> $GITHUB_OUTPUT
                    else
                      echo "building on develop branch"
                      echo "IMAGE_TAG=ghcr.io/$REPO_OWNER/opencost:develop-$OC_SHORTHASH" >> $GITHUB_OUTPUT
                      echo "NAMESPACE=develop-oc-$OC_SHORTHASH" >> $GITHUB_OUTPUT
                      echo "mainbranch=true" >> $GITHUB_OUTPUT
                    fi

          - name: Log into ghcr.io
            uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
            with:
              registry: ghcr.io
              username: ${{ github.actor }}
              password: ${{ secrets.GITHUB_TOKEN }}

          - name: wait for docker image to be ready
            id: wait_for_image_ready
            env:
              IMAGE_TAG: ${{ steps.set_image_tags.outputs.IMAGE_TAG }}
            run: |
                max_attempts=100
                # Loop until the Docker image can be pulled
                until docker manifest inspect $IMAGE_TAG; do
                    echo "Waiting for Docker image $IMAGE_TAG to be available, $max_attempts tries remain..."
                    sleep 6
                    max_attempts=$((max_attempts - 1))
                    if [[ $max_attempts -eq 0 ]]; then
                        echo "Docker image $IMAGE_TAG is not available after 10 minutes. Exiting..."
                        exit 1
                    fi
                done

                echo "Docker image $IMAGE_TAG is ready!"
                
                echo "passed=true" >> $GITHUB_OUTPUT
                
    build-test-stack:
        needs: wait_for_image_ready
        if: ${{ (always() && !cancelled()) && ( github.event_name == 'schedule'  || github.event_name == 'push' || github.event_name == 'merge_group' || (github.event_name == 'pull_request_target'  && needs.check_actor_permissions.outputs.ismaintainer == 'true')) }}
        uses: opencost/opencost-infra/.github/workflows/build-stack.yaml@main # zizmor: ignore[secrets-inherit] opencost-infra workflows do not declare their secrets yet
        secrets: inherit
        with:
            oc-container-version: "${{ needs.wait_for_image_ready.outputs.IMAGE_TAG }}"
            namespace: "${{ needs.wait_for_image_ready.outputs.NAMESPACE }}"
            
    build-test-stack-promless:
        needs: wait_for_image_ready
        if: ${{ (always() && !cancelled()) && ( github.event_name == 'schedule'  || github.event_name == 'push' || github.event_name == 'merge_group' || (github.event_name == 'pull_request_target'  && needs.check_actor_permissions.outputs.ismaintainer == 'true')) }}
        uses: opencost/opencost-infra/.github/workflows/build-stack.yaml@main # zizmor: ignore[secrets-inherit] opencost-infra workflows do not declare their secrets yet
        secrets: inherit
        with:
            oc-container-version: "${{ needs.wait_for_image_ready.outputs.IMAGE_TAG }}"
            namespace: "${{ needs.wait_for_image_ready.outputs.NAMESPACE }}-promless"
            prometheus: false

    wait-for-dns:
        needs: [wait_for_image_ready, build-test-stack]
        runs-on: ubuntu-latest
        if: ${{ (always() && !cancelled()) && ( github.event_name == 'schedule'  || github.event_name == 'push' || github.event_name == 'merge_group' || (github.event_name == 'pull_request_target'  && needs.check_actor_permissions.outputs.ismaintainer == 'true')) }}
        permissions: {}
        steps:
          - name: Wait for DNS to resolve
            id: wait-for-dns
            env:
              NAMESPACE: ${{ needs.wait_for_image_ready.outputs.NAMESPACE }}
            run: |
              echo "Waiting for $NAMESPACE.infra.opencost.io to resolve in DNS..."
              
              max_attempts=60
              until host $NAMESPACE.infra.opencost.io; do
                echo "DNS not yet resolved for $NAMESPACE.infra.opencost.io, $max_attempts tries remain..."
                sleep 10
                max_attempts=$((max_attempts - 1))
                if [[ $max_attempts -eq 0 ]]; then
                  echo "DNS resolution failed for $NAMESPACE.infra.opencost.io after 10 minutes. Exiting..."
                  exit 1
                fi
              done
              
              echo "DNS resolved successfully for $NAMESPACE.infra.opencost.io!"

    run-tests:
        needs: [wait_for_image_ready, build-test-stack, wait-for-dns]
        if: ${{ (always() && !cancelled()) && ( github.event_name == 'schedule'  || github.event_name == 'push' || github.event_name == 'merge_group' || (github.event_name == 'pull_request_target'  && needs.check_actor_permissions.outputs.ismaintainer == 'true')) }}
        permissions: {}
        uses: opencost/opencost-infra/.github/workflows/test-stack.yaml@main
        with:
            namespace: "${{ needs.wait_for_image_ready.outputs.NAMESPACE }}"
            target_branch: "${{ github.event.pull_request.head.ref || 'main' }}"
  
    wait-for-data-collection:
      needs: [wait_for_image_ready, build-test-stack, build-test-stack-promless]
      runs-on: ubuntu-latest
      if: ${{ (always() && !cancelled()) && ( github.event_name == 'schedule'  || github.event_name == 'push' || github.event_name == 'merge_group' || (github.event_name == 'pull_request_target'  && needs.check_actor_permissions.outputs.ismaintainer == 'true')) }}
      permissions: {}
      steps:
        - name: Wait 22 minutes for promless data collection
          run: |
            sleep 1320 # 22 minutes
    
    run-comparison-tests:
      needs: [wait_for_image_ready, build-test-stack, build-test-stack-promless, wait-for-data-collection]
      if: ${{ (always() && !cancelled()) && ( github.event_name == 'schedule'  || github.event_name == 'push' || github.event_name == 'merge_group' || (github.event_name == 'pull_request_target'  && needs.check_actor_permissions.outputs.ismaintainer == 'true')) }}
      permissions: {}
      uses: opencost/opencost-infra/.github/workflows/test-stack.yaml@main
      with:
        namespace: "${{ needs.wait_for_image_ready.outputs.NAMESPACE }}"
        comparison_namespace: "${{ needs.wait_for_image_ready.outputs.NAMESPACE }}-promless"
        target_branch: "${{ github.event.pull_request.head.ref || 'main' }}"
        comparison: true
    
    print-outputs:
      needs: [run-comparison-tests, run-tests]
      if: ${{ (always() && !cancelled()) && ( github.event_name == 'schedule'  || github.event_name == 'push' || github.event_name == 'merge_group' || (github.event_name == 'pull_request_target'  && needs.check_actor_permissions.outputs.ismaintainer == 'true')) }}
      runs-on: ubuntu-latest
      permissions: {}
      steps:
        - name: Print outputs
          env:
            NAMESPACE: ${{ needs.wait_for_image_ready.outputs.NAMESPACE }}
            TARGET_BRANCH: ${{ github.event.pull_request.head.ref || 'main' }}
            PASSED_COMPARISON: ${{ needs.run-comparison-tests.outputs.passed }}
            PASSED_INTEGRATION: ${{ needs.run-tests.outputs.passed }}
          run: |
            echo "NAMESPACE: $NAMESPACE"
            echo "COMPARISON_NAMESPACE: $NAMESPACE-promless"
            echo "TARGET_BRANCH: $TARGET_BRANCH"
            echo "PASSED COMPARISON: $PASSED_COMPARISON"
            echo "PASSED INTEGRATION: $PASSED_INTEGRATION"
           
    hold-on-failure:
      needs: [wait_for_image_ready, run-tests, run-comparison-tests]
      if: ${{ always() && (needs.run-tests.outputs.passed == 'false' || needs.run-comparison-tests.outputs.passed == 'false') && github.event_name != 'merge_group'}} 
      runs-on: ubuntu-latest
      permissions: {}
      steps:
        - name: Hold stack for investigation
          env:
            NAMESPACE: ${{ needs.wait_for_image_ready.outputs.NAMESPACE }}
          run: |
            echo "Tests failed. Holding stacks up for 1 hour for investigation..."
            echo "Stack namespace: $NAMESPACE"
            echo "Stack will be automatically torn down after 1 hour"
            sleep 3600

    teardown-test-stack:
        needs: [wait_for_image_ready, run-tests, run-comparison-tests, hold-on-failure]
        if: ${{ (always() && !cancelled()) && ( github.event_name == 'schedule'  || github.event_name == 'push' || github.event_name == 'merge_group' || (github.event_name == 'pull_request_target'  && needs.check_actor_permissions.outputs.ismaintainer == 'true')) }}
        uses: opencost/opencost-infra/.github/workflows/destroy-stack.yaml@main # zizmor: ignore[secrets-inherit] opencost-infra workflows do not declare their secrets yet
        secrets: inherit 
        permissions: {}
        with:
            namespace: "${{ needs.wait_for_image_ready.outputs.NAMESPACE }}"
    
    teardown-test-stack-comparison:
        needs: [wait_for_image_ready, run-comparison-tests, hold-on-failure]
        if: ${{ (always() && !cancelled()) && ( github.event_name == 'schedule'  || github.event_name == 'push' || github.event_name == 'merge_group' || (github.event_name == 'pull_request_target'  && needs.check_actor_permissions.outputs.ismaintainer == 'true')) }}
        uses: opencost/opencost-infra/.github/workflows/destroy-stack.yaml@main # zizmor: ignore[secrets-inherit] opencost-infra workflows do not declare their secrets yet
        secrets: inherit 
        permissions: {}
        with:
            namespace: "${{ needs.wait_for_image_ready.outputs.NAMESPACE }}-promless"

    check-success:
        needs: [noop-tests, run-tests, run-comparison-tests]
        permissions: {}
        runs-on: ubuntu-latest
        if: ${{ always() }}
        steps:
          - name: Check success
            id: check-success
            env:
              IS_NOOP: ${{ needs.noop-tests.outputs.is_noop }}
              PASSED: ${{ needs.run-tests.outputs.passed }}
            run: |
              echo "IS_NOOP: $IS_NOOP"
              echo "PASSED: $PASSED"
              if [[ "$IS_NOOP" == "true" ]]; then
                echo "No-op tests, skipping success check"
                exit 0
              fi
              
              if [[ "$PASSED" != "true" ]]; then
                echo "One or more integration tests failed"
                exit 1
              fi
              
              echo "All integration tests passed"
              exit 0

    set-labels:
      needs: [check_actor_permissions, noop-tests, run-tests, run-comparison-tests]
      if: ${{ (always() && !cancelled()) && ( github.event_name == 'pull_request_target' && needs.check_actor_permissions.outputs.ismaintainer == 'true') }}
      runs-on: ubuntu-latest
      permissions:
        issues: write # create the labels on first use
        pull-requests: write
      steps:
        - name: Label integration test result
          env:
            GH_TOKEN: ${{ github.token }}
            GH_REPO: ${{ github.repository }}
            PR_NUMBER: ${{ github.event.pull_request.number }}
            FAILED: ${{ contains(needs.*.result, 'failure') }}
          run: |
            if [[ "$FAILED" == "true" ]]; then
              add="integration tests failed"; remove="integration tests passed"
            else
              add="integration tests passed"; remove="integration tests failed"
            fi
            gh label create "integration tests passed" --color 0e8a16 2>/dev/null || true
            gh label create "integration tests failed" --color d93f0b 2>/dev/null || true
            gh pr edit "$PR_NUMBER" --add-label "$add" --remove-label "$remove"