name: CI
on:
  - push
  - pull_request

env:
  GO_VERSION: "1.19.x"
  GOTESTSUM_VERSION: "latest"

jobs:

  lint:
    runs-on: "windows-2022"
    strategy:
      fail-fast: false
      matrix:
        goos: [windows, linux]
        root: ["", test] # cannot specify "./... ./test/..." unless in go workspace
        include:
          - goos: linux
            root: ""
            dirs: >-
              ./cmd/gcs/...
              ./cmd/gcstools/...
              ./internal/guest...
              ./internal/tools/...
              ./pkg/...
              ./ext4/...

    steps:
      - name: Checkout
        uses: actions/checkout@v3

      - name: Install go
        uses: actions/setup-go@v4
        with:
          go-version: ${{ env.GO_VERSION }}
          # sometimes go cache causes issues with lint
          cache: false

      - uses: golangci/golangci-lint-action@v3
        with:
          version: v1.52
          args: >-
            --verbose
            --max-issues-per-linter=0
            --max-same-issues=0
            --modules-download-mode=readonly
            --timeout=10m
            ${{ matrix.dirs }}
          working-directory: ${{ matrix.root }}
        env:
          GOOS: ${{ matrix.goos }}

  protos:
    runs-on: "windows-2022"

    env:
      # translating from github.com/Microsoft/hcsshim/<path> (via `go list`) to <path> is easier if hcsshim is in GOPATH/src
      GOPATH: '${{ github.workspace }}\go'

    steps:
      - name: Checkout hcsshim
        uses: actions/checkout@v3
        with:
          path: go/src/github.com/Microsoft/hcsshim

      - name: Install go
        uses: actions/setup-go@v4
        with:
          go-version: ${{ env.GO_VERSION }}
          cache-dependency-path: go/src/github.com/Microsoft/hcsshim/go.sum

      - name: Get containerd ref
        shell: powershell
        run: |
          $v = go list -m -f '{{ .Version }}' 'github.com/containerd/containerd' 2>&1
          if ( $LASTEXITCODE ) {
            Write-Output '::error::Could not retrieve containerd version.'
            exit $LASTEXITCODE
          }
          Write-Output "containerd ref is: $v"
          "containerd_ref=$v" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
        working-directory: go/src/github.com/Microsoft/hcsshim

      - name: Checkout containerd
        uses: actions/checkout@v3
        with:
          repository: containerd/containerd
          path: "containerd"
          ref: "${{ env.containerd_ref }}"

      - name: Install protobuild and protoc-gen-gogoctrd
        shell: powershell
        run: |
          # not actually GOBIN
          $goBin = Join-Path (go env GOPATH) 'bin'
          mkdir -f $goBin
          $goBin | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append

          go install github.com/containerd/protobuild@v0.2.0

          cd containerd
          go build -o $goBin ./cmd/protoc-gen-gogoctrd

      - name: Install protoc
        shell: powershell
        run: |
          gh release download -R protocolbuffers/protobuf -p 'protoc-*-win32.zip' -O protoc.zip 'v23.2'
          if ( $LASTEXITCODE ) {
            Write-Output '::error::Could not download protoc.'
            exit $LASTEXITCODE
          }

          tar.exe xf protoc.zip
          if ( $LASTEXITCODE ) {
            Write-Output '::error::Could not install protoc.'
            exit $LASTEXITCODE
          }

          mkdir -f ${{ github.workspace }}/go/src/github.com/Microsoft/hcsshim/protobuf
          mv include/* ${{ github.workspace }}/go/src/github.com/Microsoft/hcsshim/protobuf

          # put protoc in GOPATH to make things easier
          mv bin\protoc.exe (Join-Path (go env GOPATH) 'bin')
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Run Protobuild
        shell: powershell
        run: |
          Write-Output "::group::protobuild"
          protobuild $(go list ./... | grep -v /vendor/)
          Write-Output "::endgroup::"
          if ( $LASTEXITCODE ) {
            Write-Output '::error::Failed to run protobuild.'
            exit $LASTEXITCODE
          }

          Write-Output "::group::git diff"
          # look for any new files not previously tracked
          git add --all --intent-to-add .
          git diff --exit-code
          Write-Output "::endgroup::"
        working-directory: "${{ github.workspace }}/go/src/github.com/Microsoft/hcsshim"

  verify-vendor:
    runs-on: "windows-2022"
    env:
      GOPROXY: "https://proxy.golang.org,direct"
    steps:
      - name: Checkout
        uses: actions/checkout@v3

      - name: Install go
        uses: actions/setup-go@v4
        with:
          go-version: ${{ env.GO_VERSION }}

      - name: Validate go.mod and vendoring
        shell: powershell
        run: |
          $currentPath = (Get-Location).Path
          $process = Start-Process powershell.exe -PassThru -Verb runAs -Wait -ArgumentList $currentPath/scripts/Verify-GoModules.ps1, $currentPath
          if ($process.ExitCode -ne 0) {
            Write-Error "Main modules are not up to date. Please validate your go version >= this job's and run `go mod vendor` followed by `go mod tidy` in the repo root path."
          }
          exit $process.ExitCode

      - name: Validate test/go.mod
        shell: powershell
        working-directory: test
        run: |
          Write-Output "::group::go mod tidy"
          go mod tidy
          Write-Output "::endgroup::"

          git add --all --intent-to-add .
          Write-Output "::group::git diff"
          git diff --stat --exit-code
          Write-Output "::endgroup::"

          if ($LASTEXITCODE -ne 0) {
            Write-Output "::error ::./test/go.mod is not up to date. Please run ``go mod tidy`` from within ``./test``"
            exit $LASTEXITCODE
          }

  go-gen:
    name: Go Generate
    runs-on: "windows-2022"
    steps:
      - name: Checkout
        uses: actions/checkout@v3

      - name: Install go
        uses: actions/setup-go@v4
        with:
          go-version: ${{ env.GO_VERSION }}

      - name: Validate go generate
        shell: powershell
        run: |
          Write-Output "::group::go generate"
          go generate -x .\...
          Write-Output "::endgroup::"
          if ($LASTEXITCODE -ne 0) {
            Write-Output "::error title=Go Generate::Error running go generate."
            exit $LASTEXITCODE
          }

          git add --all --intent-to-add .
          Write-Output "::group::git diff"
          git diff --stat --exit-code
          Write-Output "::endgroup::"
          if ($LASTEXITCODE -ne 0) {
            Write-Output "::error ::Generated files are not up to date. Please run ``go generate .\...``."
            exit $LASTEXITCODE
          }

  test-linux:
    needs: [lint, protos, verify-vendor, go-gen]
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v3

      - name: Install go
        uses: actions/setup-go@v4
        with:
          go-version: ${{ env.GO_VERSION }}

      - name: Install gotestsum
        run: go install gotest.tools/gotestsum@${{ env.GOTESTSUM_VERSION }}

      - name: Test standard security policy
        run: gotestsum --format standard-verbose --debug -- -timeout=30m -mod=mod -gcflags=all=-d=checkptr ./pkg/securitypolicy

      - name: Test rego security policy
        run: gotestsum --format standard-verbose --debug -- -tags=rego -timeout=30m -mod=mod -gcflags=all=-d=checkptr ./pkg/securitypolicy

      - name: Test rego policy interpreter
        run: gotestsum --format standard-verbose --debug -- -mod=mod -gcflags=all=-d=checkptr ./internal/regopolicyinterpreter

      - name: Run guest code unit tests
        run: gotestsum --format standard-verbose --debug -- -mod=mod -gcflags=all=-d=checkptr ./internal/guest/...
      
      - name: Build gcs Testing Binary
        run: go test -mod=mod -gcflags=all=-d=checkptr -c -tags functional ./gcs
        working-directory: test

  test-windows:
    needs: [lint, protos, verify-vendor, go-gen]
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [windows-2019, windows-2022]
    steps:
      - name: Checkout
        uses: actions/checkout@v3

      - name: Install go
        uses: actions/setup-go@v4
        with:
          go-version: ${{ env.GO_VERSION }}

      - name: Install gotestsum
        run: go install gotest.tools/gotestsum@${{ env.GOTESTSUM_VERSION }}

      # run tests
      - name: Test repo
        run: gotestsum --format standard-verbose --debug -- -gcflags=all=-d=checkptr -tags admin ./...
      - name: Test schema version
        run: gotestsum --format standard-verbose --debug -- -mod=mod -gcflags=all=-d=checkptr -tags admin ./internal
        working-directory: test
      - name: Test rego policy interpreter
        run: gotestsum --format standard-verbose --debug -- -mod=mod -gcflags=all=-d=checkptr ./internal/regopolicyinterpreter

      # build testing binaries
      - name: Build containerd-shim-runhcs-v1 Testing Binary
        run: go test -mod=mod -gcflags=all=-d=checkptr -c -tags functional ./containerd-shim-runhcs-v1
        working-directory: test
      - name: Build cri-containerd Testing Binary
        run: go test -mod=mod -gcflags=all=-d=checkptr -c -tags functional ./cri-containerd
        working-directory: test
      - name: Build functional Testing Binary
        run: go test -mod=mod -gcflags=all=-d=checkptr -c -tags functional ./functional
        working-directory: test
      - name: Build runhcs Testing Binary
        run: go test -mod=mod -gcflags=all=-d=checkptr -c -tags functional ./runhcs
        working-directory: test
      - name: Build logging-driver Binary
        run: go build -mod=mod -o sample-logging-driver.exe ./cri-containerd/helpers/log.go
        working-directory: test

      - uses: actions/upload-artifact@v3
        if: ${{ github.event_name == 'pull_request' }}
        with:
          name: test_binaries_${{ matrix.os }}
          path: |
            test/containerd-shim-runhcs-v1.test.exe
            test/cri-containerd.test.exe
            test/functional.test.exe
            test/runhcs.test.exe
            test/sample-logging-driver.exe

  integration-tests:
    needs: [lint, protos, verify-vendor, go-gen]
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: [windows-2019, windows-2022]

    steps:
      - name: Checkout hcsshim
        uses: actions/checkout@v3
        with:
          path: src/github.com/Microsoft/hcsshim

      - name: Install go
        uses: actions/setup-go@v4
        with:
          go-version: ${{ env.GO_VERSION }}
          check-latest: true
          cache-dependency-path: src/github.com/Microsoft/hcsshim/go.sum

      - name: Set env
        shell: bash
        run: |
          mkdir -p "${{ github.workspace }}/bin"
          echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
          echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
          echo "${{ github.workspace }}/src/github.com/containerd/containerd/bin" >> $GITHUB_PATH

      - name: Get containerd ref
        shell: powershell
        run: |
          $v = go list -m -f '{{ .Version }}' 'github.com/containerd/containerd' 2>&1
          if ( $LASTEXITCODE ) {
            Write-Output '::error::Could not retrieve containerd version.'
            exit $LASTEXITCODE
          }
          Write-Output "containerd ref is: $v"
          "containerd_ref=$v" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
        working-directory: src/github.com/Microsoft/hcsshim

      - uses: actions/checkout@v3
        with:
          path: src/github.com/containerd/containerd
          repository: "containerd/containerd"
          ref: "${{ env.containerd_ref }}"
        name: Checkout containerd

      - name: Install crictl
        shell: powershell
        run: |
          gh release download -R kubernetes-sigs/cri-tools -p 'crictl-*-windows-amd64.tar.gz' -O c:\crictl.tar.gz 'v1.24.2'
          tar.exe xf c:\crictl.tar.gz -C '${{ github.workspace }}/bin'

          if ( $LASTEXITCODE ) {
            Write-Output '::error::Could not install crictl.'
            exit $LASTEXITCODE
          }
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      # needs to be a separate step since terminal reload is required to bring in new env variables and PATH
      - name: Upgrade Chocolaty
        shell: powershell
        run: |
          choco upgrade -y chocolatey 2>&1

      - name: Install mingw
        shell: powershell
        run: |
          $VerbosePreference = 'Continue'
          # dont set $ErrorActionPreference since we want to allow choco install to fail later on

          Write-Output 'Install mingw'
          # Install sometimes fails when downloading mingw zip from source-forge with:
          #  "ERROR: The remote file either doesn't exist, is unauthorized, or is forbidden for url"
          # Issue is with accessing from source-forge, which version 10.3+ do not use, but cannot upgrade versions.

          # Add retry and backoff
          foreach ( $i in 1..3 ) {
            Write-Output "::group::Attempt $i"
            if ( $i -gt 1 ) {
              # remove any left-over state
              choco uninstall -y --no-progress --force mingw

              Write-Output 'Sleeping for 60 seconds'
              Sleep -Seconds 60
            }

            choco install -y --no-progress --stop-on-first-failure --force mingw --allow-downgrade --version 10.3.0
            Write-Output '::endgroup::'
            if ( -not $LASTEXITCODE ) {
              Write-Output "Attempt $i succeeded (exit code: $LASTEXITCODE)"
              break
            }
            Write-Output "::warning title=mingw::Attempt $i failed (exit code: $LASTEXITCODE)"
          }

          if ( $LASTEXITCODE ) {
            Write-Output "::error::Could not install mingw after $i attempts."
            exit $LASTEXITCODE
          }

          # verify mingw32-make was installed
          Get-Command -CommandType Application -ErrorAction Stop mingw32-make.exe

      - name: Build binaries
        shell: bash
        working-directory: src/github.com/containerd/containerd
        run: |
          set -o xtrace
          mingw32-make.exe binaries
          script/setup/install-cni-windows

      - name: Build the shim
        working-directory: src/github.com/Microsoft/hcsshim
        shell: powershell
        run: |
          go build -mod vendor -o "${{ github.workspace }}/src/github.com/containerd/containerd/bin/containerd-shim-runhcs-v1.exe" .\cmd\containerd-shim-runhcs-v1

      - name: Install gotestsum
        run: go install gotest.tools/gotestsum@${{ env.GOTESTSUM_VERSION }}

      - name: Run containerd integration tests
        shell: bash
        working-directory: src/github.com/containerd/containerd
        run: |
          export EXTRA_TESTFLAGS='-timeout=20m'
          export GOTEST='gotestsum --format=standard-verbose --debug --'
          make integration

      - name: Run containerd CRI integration tests
        shell: bash
        working-directory: src/github.com/containerd/containerd
        env:
          TEST_IMAGE_LIST: ${{github.workspace}}/repolist.toml
          BUSYBOX_TESTING_IMAGE_REF: "k8s.gcr.io/e2e-test-images/busybox:1.29-2"
          RESOURCE_CONSUMER_TESTING_IMAGE_REF: "k8s.gcr.io/e2e-test-images/resource-consumer:1.10"
          CGO_ENABLED: 1
        run: |
          cat > "${{ env.TEST_IMAGE_LIST }}" << EOF
          busybox = "${{ env.BUSYBOX_TESTING_IMAGE_REF }}"
          ResourceConsumer = "${{ env.RESOURCE_CONSUMER_TESTING_IMAGE_REF }}"
          EOF
          # In the stable version of hcsshim that is used in containerd, killing a task
          # that has already exited or a task that has not yet been started, yields a
          # ErrNotFound. The master version of hcsshim returns nil, which is in line with
          # how the linux runtime behaves. See:
          # https://github.com/containerd/containerd/blob/f4f41296c2b0ac7d60aae3dd9c219a7636b0a07e/integration/restart_test.go#L152-L160
          #
          # We skip this test here, until a new release of hcsshim is cut and the one in
          # containerd is updated. When the shim is updated in containerd, this test will
          # also need to be updated and the special case for windows, removed.
          FOCUS="[^(TestContainerdRestart|TestContainerSymlinkVolumes)]" make cri-integration

  #      Enable these tests once the required JobContainer images are updated.
  #
  #      - name: Install containerd service
  #        shell: powershell
  #        run: |
  #          mkdir C:\containerd
  #          Set-Content C:/containerd/containerd.toml @"
  #          version = 2
  #          [plugins]
  #              [plugins."io.containerd.grpc.v1.cri".containerd]
  #                default_runtime_name = "runhcs-wcow-process"
  #                disable_snapshot_annotations = false
  #                discard_unpacked_layers = false
  #                ignore_blockio_not_enabled_errors = false
  #                ignore_rdt_not_enabled_errors = false
  #                no_pivot = false
  #                snapshotter = "windows"
  #
  #                [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
  #
  #                [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runhcs-wcow-hypervisor]
  #                  runtime_type = "io.containerd.runhcs.v1"
  #                  [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runhcs-wcow-hypervisor.options]
  #                    Debug = true
  #                    DebugType = 2
  #                    SandboxPlatform = "windows/amd64"
  #                    SandboxIsolation = 1
  #
  #                  [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runhcs-wcow-process]
  #                    runtime_type = "io.containerd.runhcs.v1"
  #                    pod_annotations = ["microsoft.com/*", "io.microsoft.*" ]
  #                    container_annotations = ["microsoft.com/*", "io.microsoft.*" ]
  #
  #                    [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runhcs-wcow-process.options]
  #          "@
  #
  #          containerd.exe --register-service --log-level=debug --config C:/containerd/containerd.toml --service-name containerd --address //./pipe/containerd-containerd --state C:/ProgramData/containerd/state --root C:/ProgramData/containerd/root --log-file C:/containerd/containerd.log
  #          Set-Service containerd -StartupType Automatic
  #          Start-Service containerd
  #
  #      - name: Build test binary
  #        working-directory: src/github.com/Microsoft/hcsshim/test
  #        shell: powershell
  #        run: |
  #          go test -mod=mod -o "${{ github.workspace }}/bin/cri-containerd.test.exe" -gcflags=all=-d=checkptr -c ./cri-containerd/ -tags functional
  #
  #      - name: Run hcsshim integration tests
  #        shell: powershell
  #        run: |
  #          cri-containerd.test.exe -cri-endpoint="npipe://./pipe/containerd-containerd" -feature="WCOWProcess" -feature="HostProcess"

  build:
    needs: [test-windows, test-linux]
    runs-on: "windows-2022"
    steps:
      - name: Checkout
        uses: actions/checkout@v3

      - name: Install go
        uses: actions/setup-go@v4
        with:
          go-version: ${{ env.GO_VERSION }}

      - run: go build ./cmd/containerd-shim-runhcs-v1
      - run: go build ./cmd/runhcs
      - run: go build ./cmd/tar2ext4
      - run: go build ./cmd/wclayer
      - run: go build ./cmd/device-util
      - run: go build ./cmd/ncproxy
      - run: go build ./cmd/dmverity-vhd
      - run: go build ./cmd/dmverity-vhd
        env:
          GOOS: linux
          GOARCH: amd64
      - run: go build ./internal/tools/grantvmgroupaccess
      - run: go build ./internal/tools/networkagent
      - run: go build ./internal/tools/securitypolicy
      - run: go build ./internal/tools/uvmboot
      - run: go build ./internal/tools/zapdir

      - uses: actions/upload-artifact@v3
        if: ${{ github.event_name == 'pull_request' }}
        with:
          name: binaries
          path: |
            containerd-shim-runhcs-v1.exe
            runhcs.exe
            tar2ext4.exe
            wclayer.exe
            device-util.exe
            ncproxy.exe
            dmverity-vhd.exe
            dmverity-vhd
            grantvmgroupaccess.exe
            networkagent.exe
            securitypolicy.exe
            uvmboot.exe
            zapdir.exe

  build_gcs:
    needs: test-linux
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v3

      - name: Install go
        uses: actions/setup-go@v4
        with:
          go-version: ${{ env.GO_VERSION }}

      - name: Test
        run: make test

      - name: Pull busybox image
        run: docker pull busybox

      - name: Run Busybox Container
        run: docker run --name base_image_container busybox

      - name: Export container to tar file
        run: |
          docker export base_image_container | gzip > base.tar.gz

      - name: Build
        run: make BASE=./base.tar.gz all