name: Integration tests

on: pull_request

permissions:
  contents: read

env:
  CARGO_INCREMENTAL: 0
  CARGO_NET_RETRY: 10
  DOCKER_REGISTRY: ghcr.io/linkerd
  GH_ANNOTATION: true
  K3D_VERSION: v5.4.4
  RUST_BACKTRACE: short
  RUSTUP_MAX_RETRIES: 10
  YQ_VERSION: v4.25.1
  LINKERD2_PROXY_REPO: ${{ vars.LINKERD2_PROXY_REPO || 'linkerd/linkerd2-proxy' }}
  LINKERD2_PROXY_RELEASE_PREFIX: ${{ vars.LINKERD2_PROXY_RELEASE_PREFIX || 'release/' }}

concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref }}
  cancel-in-progress: true

jobs:
  meta:
    runs-on: ubuntu-22.04
    steps:
      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
      - id: tag
        run: echo "tag=$(CI_FORCE_CLEAN=1 bin/root-tag)" >> "$GITHUB_OUTPUT"
      - uses: tj-actions/changed-files@d6babd6899969df1a11d14c368283ea4436bca78
        id: core
        with:
          files: |
            .github/workflows/integration.yml
            .proxy-version
            go.sum
            **/*.go
            **/Dockerfile*
            charts/**
            justfile
            bin/fetch-proxy
            bin/_test-helper.sh
          files_ignore: |
            .devcontainer/**
            **/Chart.yaml
            **/README*
    outputs:
      tag: ${{ steps.tag.outputs.tag }}
      changed: ${{ steps.core.outputs.any_changed }}

  info:
    needs: meta
    runs-on: ubuntu-22.04
    timeout-minutes: 2
    steps:
      - name: Info
        run: |
          echo "tag=${{ needs.meta.outputs.tag }}"
          echo "changed=${{ needs.meta.outputs.changed }}"

  build-cli:
    needs: meta
    if: needs.meta.outputs.changed == 'true'
    runs-on: ubuntu-22.04
    timeout-minutes: 15
    steps:
      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
      - uses: ./.github/actions/docker-build
        id: build
        with:
          docker-registry: ${{ env.DOCKER_REGISTRY }}
          docker-target: linux-amd64
          component: cli-bin
          tag: ${{ needs.meta.outputs.tag }}
      - name: Extract CLI binary
        run: |
          mkdir -p /home/runner/archives
          id=$(docker create '${{ steps.build.outputs.image }}')
          docker cp "$id:/out/linkerd-linux-amd64" /home/runner/archives/linkerd
          v=$(/home/runner/archives/linkerd version --short --client)
          [[ "$v" == '${{ needs.meta.outputs.tag }}' ]] || exit 1
      - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808
        with:
          name: image-archives-cli
          path: /home/runner/archives

  ##
  ## Core: Test the core control plane
  ##
  ## TODO(ver) CNI configurations should be tested separately.
  ##

  build-core:
    needs: meta
    if: needs.meta.outputs.changed == 'true'
    runs-on: ubuntu-22.04
    strategy:
      matrix:
        component:
          - controller
          - policy-controller
          - proxy
    timeout-minutes: 20
    steps:
      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
      - uses: ./.github/actions/docker-build
        id: build
        env:
          LINKERD2_PROXY_GITHUB_TOKEN: ${{ secrets.LINKERD2_PROXY_GITHUB_TOKEN || github.token }}
        with:
          docker-registry: ${{ env.DOCKER_REGISTRY }}
          docker-target: linux-amd64
          component: ${{ matrix.component }}
          tag: ${{ needs.meta.outputs.tag }}
      - name: Run docker save
        run: |
          mkdir -p /home/runner/archives
          docker save '${{ steps.build.outputs.image }}' >'/home/runner/archives/${{ matrix.component }}.tar'
      - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808
        with:
          name: image-archives-${{ matrix.component }}
          path: /home/runner/archives

  test-core:
    needs: [meta, build-cli, build-core]
    if: needs.meta.outputs.changed == 'true'
    strategy:
      matrix:
        test:
          - cni-calico-deep
          - deep
          - deep-native-sidecar
    runs-on: ubuntu-22.04
    timeout-minutes: 15
    steps:
      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
      - uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7
        with:
          go-version: "1.22"
      - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e
        with:
          pattern: image-archives-*
          path: image-archives
          merge-multiple: true
      - run: cp image-archives/linkerd "$HOME" && chmod 755 "$HOME/linkerd"
      - run: find image-archives -ls
      - run: bin/tests --images archive --cleanup-docker --name ${{ matrix.test }} "$HOME/linkerd"
        env:
          LINKERD_DOCKER_REGISTRY: ${{ env.DOCKER_REGISTRY }}
          TAG: ${{ needs.meta.outputs.tag }}

  ##
  ## Policy: Only run policy tests when the policy controller or proxy changes
  ##

  test-policy:
    needs: [meta, build-cli, build-core]
    if: needs.meta.outputs.changed == 'true'
    runs-on: ubuntu-22.04
    timeout-minutes: 20
    strategy:
      matrix:
        k8s:
          - v1.22
          - v1.29
    steps:
      - uses: extractions/setup-just@dd310ad5a97d8e7b41793f8ef055398d51ad4de6
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - uses: olix0r/cargo-action-fmt/setup@9269f3aa1ff01775d95efc97037e2cbdb41d9684
      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
      - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e
        with:
          pattern: image-archives-*
          path: image-archives
          merge-multiple: true
      - run: find image-archives -ls
      - run: cp image-archives/linkerd "$HOME" && chmod 755 "$HOME/linkerd"
      - name: Setup deps
        shell: bash
        run: |
          rm -rf "$HOME/.cargo"
          bin/scurl -v https://sh.rustup.rs | sh -s -- -y --default-toolchain "$(./bin/rust-toolchain-version)"
          # shellcheck disable=SC1090
          source ~/.cargo/env
          echo "PATH=$PATH" >> "$GITHUB_ENV"
          bin/scurl -v "https://raw.githubusercontent.com/k3d-io/k3d/${K3D_VERSION}/install.sh" | bash
          bin/scurl -vo /usr/local/bin/yq "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64" && chmod +x /usr/local/bin/yq
      - uses: Swatinem/rust-cache@23bce251a8cd2ffc3c1075eaa2367cf899916d84
      - run: cargo install cargo-nextest
      - run: just policy-test-build
      - run: just k3d-k8s='${{ matrix.k8s }}' k3d-create
      - run: docker load <image-archives/controller.tar
      - run: docker load <image-archives/policy-controller.tar
      - run: docker load <image-archives/proxy.tar
      - run: docker image ls
      - run: just linkerd-tag='${{ needs.meta.outputs.tag }}' linkerd-exec="$HOME/linkerd" linkerd-install
      - name: Load images
        run: |
          # Image loading is flakey in CI, so retry!
          for _ in {1..6} ; do
            if just linkerd-tag='${{ needs.meta.outputs.tag }}' policy-test-deps-load ; then exit 0 ; fi
            sleep 10
            echo retrying...
          done
          exit 1
      - run: just policy-test-run --jobs=1
        env:
          # https://nexte.st/book/retries.html
          NEXTEST_RETRIES: 3

  ##
  ## Ext: Run tests that require non-core components.
  ##

  build-ext:
    needs: meta
    if: needs.meta.outputs.changed == 'true'
    runs-on: ubuntu-22.04
    strategy:
      matrix:
        component:
          - jaeger-webhook
          - metrics-api
          - tap
          - web
    timeout-minutes: 15
    steps:
      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
      - uses: ./.github/actions/docker-build
        id: build
        with:
          docker-registry: ${{ env.DOCKER_REGISTRY }}
          docker-target: linux-amd64
          component: ${{ matrix.component }}
          tag: ${{ needs.meta.outputs.tag }}
      - name: Run docker save
        run: |
          mkdir -p /home/runner/archives
          docker save '${{ steps.build.outputs.image }}' >'/home/runner/archives/${{ matrix.component }}.tar'
      - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808
        with:
          name: image-archives-${{ matrix.component }}
          path: /home/runner/archives

  # These tests exercise core functionality, but need the viz extension.
  test-ext:
    needs: [meta, build-cli, build-core, build-ext]
    if: needs.meta.outputs.changed == 'true'
    strategy:
      matrix:
        integration_test:
          - cluster-domain
          - default-policy-deny
          - external
          - rsa-ca
          - helm-upgrade
          - uninstall
          - upgrade-edge
    runs-on: ubuntu-22.04
    timeout-minutes: 15
    steps:
      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
      - uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7
        with:
          go-version: "1.22"
      - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e
        with:
          pattern: image-archives-*
          path: image-archives
          merge-multiple: true
      - run: cp image-archives/linkerd "$HOME" && chmod 755 "$HOME/linkerd"
      - run: ls -l image-archives/linkerd
      - run: bin/tests --images archive --cleanup-docker --name '${{ matrix.integration_test }}' "$HOME/linkerd"
        env:
          LINKERD_DOCKER_REGISTRY: ${{ env.DOCKER_REGISTRY }}

  ##
  ## Viz: Run the (flakey) `viz` suite only when the `viz` extension is updated.
  ##

  test-viz:
    needs: [meta, build-cli, build-core, build-ext]
    if: needs.meta.outputs.changed == 'true'
    runs-on: ubuntu-22.04
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
      - uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7
        with:
          go-version: "1.22"
      - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e
        with:
          pattern: image-archives-*
          path: image-archives
          merge-multiple: true
      - run: cp image-archives/linkerd "$HOME" && chmod 755 "$HOME/linkerd"
      - run: ls -l image-archives/linkerd
      - run: bin/tests --images archive --cleanup-docker --name viz "$HOME/linkerd"
        env:
          LINKERD_DOCKER_REGISTRY: ${{ env.DOCKER_REGISTRY }}

  ##
  ## Multicluster: Run 'multicluster' suite only when the 'multicluster' extension is updated.
  ##               Tests are run on min and max k8s versions
  ##

  test-multicluster:
    needs: [meta, build-cli, build-core, build-ext]
    if: needs.meta.outputs.changed == 'true'
    runs-on: ubuntu-22.04
    timeout-minutes: 20
    strategy:
      matrix:
        k8s:
          - v1.22
          - v1.29
    steps:
      - uses: extractions/setup-just@dd310ad5a97d8e7b41793f8ef055398d51ad4de6
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
      - uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7
        with:
          go-version: "1.22"
      - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e
        with:
          pattern: image-archives-*
          path: image-archives
          merge-multiple: true
      - run: cp image-archives/linkerd "$HOME" && chmod 755 "$HOME/linkerd"
      - run: ls -l image-archives/linkerd
      - name: Setup deps
        shell: bash
        run: |
          echo "PATH=$PATH" >> "$GITHUB_ENV"
          bin/scurl -v "https://raw.githubusercontent.com/k3d-io/k3d/${K3D_VERSION}/install.sh" | bash
      - name: Load docker images
        run: |
          for img in controller policy-controller proxy; do
            docker load <"image-archives/${img}.tar"
          done
      - run: docker image ls
      - run: just mc-test-build
      - name: Run just mc-test-load
        run: |
          just linkerd-tag='${{ needs.meta.outputs.tag }}' \
              k3d-k8s='${{ matrix.k8s }}' \
              mc-test-load
      - name: Run just mc-test-run
        run: |
          just linkerd-tag='${{ needs.meta.outputs.tag }}' \
              k3d-k8s='${{ matrix.k8s }}' \
              mc-test-run

  build-ok:
    needs: [build-cli, build-core, build-ext]
    if: always()
    runs-on: ubuntu-22.04
    steps:
      - name: Results
        run: |
          echo 'needs.build-cli.result: ${{ needs.build-cli.result }}'
          echo 'needs.build-core.result: ${{ needs.build-core.result }}'
          echo 'needs.build-ext.result: ${{ needs.build-ext.result }}'
      - name: Verify jobs
        # All jobs must succeed or be skipped.
        if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')
        run: exit 1

  # Try to re-run the integration tests if they fail, but only up to 3 times.
  integrations-retry:
    needs:
      [build-ok, test-core, test-policy, test-ext, test-viz, test-multicluster]
    if: failure() && fromJSON(github.run_attempt) < 3 && needs.build-ok.result == 'success'
    runs-on: ubuntu-22.04
    permissions:
      actions: write
    env:
      GH_REPO: ${{ github.repository }}
      GH_TOKEN: ${{ github.token }}
      GH_DEBUG: api
      REF: ${{ github.head_ref }}
    steps:
      - run: gh workflow run rerun.yml -F 'run_id=${{ github.run_id }}' --ref "$REF"

  integrations-ok:
    needs:
      [build-ok, test-core, test-policy, test-ext, test-viz, test-multicluster]
    if: always()
    runs-on: ubuntu-22.04
    steps:
      - name: Results
        run: |
          echo 'needs.build-ok.result: ${{ needs.build-ok.result }}'
          echo 'needs.test-core.result: ${{ needs.test-core.result }}'
          echo 'needs.test-policy.result: ${{ needs.test-policy.result }}'
          echo 'needs.test-ext.result: ${{ needs.test-ext.result }}'
          echo 'needs.test-viz.result: ${{ needs.test-viz.result }}'
          echo 'needs.test-multicluster.result: ${{ needs.test-multicluster.result }}'
      - name: Verify jobs
        # All jobs must succeed or be skipped.
        if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')
        run: exit 1