diff --git a/.forgejo/cascading-forgejo b/.forgejo/cascading-forgejo new file mode 100755 index 00000000..7c305127 --- /dev/null +++ b/.forgejo/cascading-forgejo @@ -0,0 +1,46 @@ +#!/bin/bash + +set -ex + +forgejo=$1 +forgejo_pr=$2 +runner=$3 +runner_pr_or_ref=$4 + +# +# Get information from the runner +# +cd $runner +# +# code.forgejo.org/forgejo/runner/vN may be +# upgraded to code.forgejo.org/forgejo/runner/vN+1 +# +module=$(cat go.mod | head -1 | cut -d' ' -f2) +test "$module" +sha=$(git -C $runner show --no-patch --format=%H) +test "$sha" + +# +# Update Forgejo to use the runner at $runner_pr_or_ref +# +cd $forgejo +# +# Update the runner major version if needed +# +find * -name '*.go' -o -name 'go.mod' | xargs sed -i -E -e "s|code.forgejo.org/forgejo/runner/v[0-9]+|$module|" +# +# If it is a pull request, change the module to reference the forked repository so +# go mod tidy can find the SHA from a known branch or tag +# +if test -f "$runner_pr_or_ref"; then + repository=$(jq --raw-output .head.repo.full_name <$runner_pr_or_ref) + test "$repository" + module=$(echo $module | sed -e "s|code.forgejo.org/forgejo/runner|code.forgejo.org/$repository|") +fi +# +# add a "replace code.forgejo.org/forgejo/runner/v?? $sha" line to the forgejo go.mod +# so that it uses the branch or pull request from which the cascade is run. +# +sed -i -e "\|replace $module|d" go.mod +echo "replace $module => $module $sha" >>go.mod +go mod tidy diff --git a/.forgejo/cascading-pr-setup-forgejo b/.forgejo/cascading-pr-setup-forgejo deleted file mode 100755 index 06472a7b..00000000 --- a/.forgejo/cascading-pr-setup-forgejo +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -set -ex - -setup_forgejo=$1 -setup_forgejo_pr=$2 -runner=$3 -runner_pr=$4 - -url=$(jq --raw-output .head.repo.html_url < $runner_pr) -test "$url" != null -branch=$(jq --raw-output .head.ref < $runner_pr) -test "$branch" != null -cd $setup_forgejo -./utils/upgrade-runner.sh $url @$branch -date > last-upgrade diff --git a/.forgejo/cascading-setup-forgejo b/.forgejo/cascading-setup-forgejo new file mode 100755 index 00000000..1fc70ed9 --- /dev/null +++ b/.forgejo/cascading-setup-forgejo @@ -0,0 +1,24 @@ +#!/bin/bash + +set -ex + +setup_forgejo=$1 +setup_forgejo_pr=$2 +runner=$3 +runner_pr_or_ref=$4 + +if test -f "$runner_pr_or_ref"; then + url=$(jq --raw-output .head.repo.html_url <$runner_pr_or_ref) + test "$url" != null + branch=$(jq --raw-output .head.ref <$runner_pr_or_ref) +else + url=https://code.forgejo.org/forgejo/runner + branch=${runner_pr_or_ref#refs/heads/} +fi +test "$url" +test "$branch" +test "$branch" != null +cd $setup_forgejo +./utils/upgrade-runner.sh $url @$branch +rm -f .forgejo/workflows/integration*.yml +date >last-upgrade diff --git a/.forgejo/labelscompare.py b/.forgejo/labelscompare.py index 2274d384..9580b747 100644 --- a/.forgejo/labelscompare.py +++ b/.forgejo/labelscompare.py @@ -8,7 +8,7 @@ expectedLabels = { "org.opencontainers.image.source": "https://code.forgejo.org/forgejo/runner", "org.opencontainers.image.version": "1.2.3", "org.opencontainers.image.vendor": "Forgejo", - "org.opencontainers.image.licenses": "MIT", + "org.opencontainers.image.licenses": "GPL-3.0-or-later", "org.opencontainers.image.title": "Forgejo Runner", "org.opencontainers.image.description": "A runner for Forgejo Actions.", } diff --git a/.forgejo/workflows/act.yml b/.forgejo/workflows/act.yml deleted file mode 100644 index 46175b3b..00000000 --- a/.forgejo/workflows/act.yml +++ /dev/null @@ -1,80 +0,0 @@ -name: act -on: - push: - branches: - - 'main' - pull_request: - -env: - GOPROXY: https://goproxy.io,direct - GOPATH: /go_path - GOCACHE: /go_cache - -jobs: - unit: - runs-on: docker - if: vars.ROLE == 'forgejo-coding' - container: - image: 'code.forgejo.org/oci/node:22-bookworm' - steps: - - name: cache go path - id: cache-go-path - uses: https://code.forgejo.org/actions/cache@v4 - with: - path: /go_path - key: go_path-${{ forge.repository }}-${{ forge.ref_name }} - restore-keys: | - go_path-${{ forge.repository }}- - go_path- - - name: cache go cache - id: cache-go-cache - uses: https://code.forgejo.org/actions/cache@v4 - with: - path: /go_cache - key: go_cache-${{ forge.repository }}-${{ forge.ref_name }} - restore-keys: | - go_cache-${{ forge.repository }}- - go_cache- - - - uses: https://code.forgejo.org/actions/checkout@v4 - - - uses: https://code.forgejo.org/actions/setup-go@v5 - with: - go-version-file: go.mod - - - name: validate go version - run: | - set -ex - toolchain=$(grep -oP '(?<=toolchain ).+' go.mod) - version=$(go version | cut -d' ' -f3) - if dpkg --compare-versions ${version#go} lt ${toolchain#go}; then - echo "go version too low: $toolchain >= $version" - exit 1 - fi - - - name: unit test - run: | - go test -short ./act/container - go test ./act/artifactcache/... ./act/workflowpattern/... ./act/filecollector/... ./act/common/... ./act/jobparser ./act/model ./act/exprparser ./act/schema - - integration: - runs-on: lxc-bookworm - if: vars.ROLE == 'forgejo-coding' - needs: [unit] - steps: - - uses: https://code.forgejo.org/actions/checkout@v4 - - - uses: https://code.forgejo.org/actions/setup-go@v5 - with: - go-version-file: go.mod - - - name: apt install docker.io - run: | - export DEBIAN_FRONTEND=noninteractive - apt-get update -qq - apt-get -q install -qq -y docker.io - - - name: integration test - run: | - go test ./act/container - go test ./act/runner/... diff --git a/.forgejo/workflows/build-ipcei.yml b/.forgejo/workflows/build-ipcei.yml new file mode 100644 index 00000000..d2a754ba --- /dev/null +++ b/.forgejo/workflows/build-ipcei.yml @@ -0,0 +1,27 @@ +name: ci + +on: + push: + tags: + - v* + +jobs: + goreleaser: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version: ">=1.25.1" + - name: Test code + run: make test + - name: Run GoReleaser + uses: https://github.com/goreleaser/goreleaser-action@v6 + env: + GITEA_TOKEN: ${{ secrets.PACKAGES_TOKEN }} + with: + args: release --clean diff --git a/.forgejo/workflows/build-release-integration.yml b/.forgejo/workflows/build-release-integration.yml index 4adba81c..4f387998 100644 --- a/.forgejo/workflows/build-release-integration.yml +++ b/.forgejo/workflows/build-release-integration.yml @@ -18,19 +18,22 @@ on: enable-email-notifications: true +env: + FORGEJO_VERSION: 11.0.7 # renovate: datasource=docker depName=code.forgejo.org/forgejo/forgejo + jobs: release-simulation: runs-on: lxc-bookworm if: vars.ROLE == 'forgejo-coding' steps: - - uses: actions/checkout@v4 + - uses: https://data.forgejo.org/actions/checkout@v4 - id: forgejo - uses: https://data.forgejo.org/actions/setup-forgejo@v3.0.1 + uses: https://data.forgejo.org/actions/setup-forgejo@v3.0.4 with: user: root password: admin1234 - image-version: 1.20 + image-version: ${{ env.FORGEJO_VERSION }} lxc-ip-prefix: 10.0.9 - name: publish diff --git a/.forgejo/workflows/build-release.yml b/.forgejo/workflows/build-release.yml index 6bb6c4b4..12d790ab 100644 --- a/.forgejo/workflows/build-release.yml +++ b/.forgejo/workflows/build-release.yml @@ -23,7 +23,7 @@ jobs: # root is used for testing, allow it if: vars.ROLE == 'forgejo-integration' || forge.repository_owner == 'root' steps: - - uses: actions/checkout@v4 + - uses: https://data.forgejo.org/actions/checkout@v4 - name: Increase the verbosity when there are no secrets id: verbose diff --git a/.forgejo/workflows/cascade-forgejo.yml b/.forgejo/workflows/cascade-forgejo.yml new file mode 100644 index 00000000..36bc899d --- /dev/null +++ b/.forgejo/workflows/cascade-forgejo.yml @@ -0,0 +1,110 @@ +# Copyright 2025 The Forgejo Authors +# SPDX-License-Identifier: MIT +# +# FORGEJO_CASCADING_PR_ORIGIN_TOKEN is a token from the https://code.forgejo.org/cascading-pr user +# with scope write:issue read:repository read:user +# FORGEJO_CASCADING_PR_DESTINATION_TOKEN is a token from the https://codeberg.org/forgejo-cascading-pr user +# with scope write:issue write:repository read:user +# +# To modify this workflow: +# +# - push it to the wip-cascade branch on the repository +# otherwise it will not have access to the secrets required to push +# the cascading PR +# +# - once it works, open a pull request for the sake of keeping track +# of the change even if the PR won't run it because it will use +# whatever is in the default branch instead +# +# - after it is merged, double check it works by setting the +# label on a pull request (any pull request will do) +# +name: cascade + +on: + push: + branches: + - 'main' + - 'wip-cascade' + pull_request_target: + types: + - synchronize + - labeled + - closed + +enable-email-notifications: true + +jobs: + debug: + if: > + vars.DEBUG == 'yes' + runs-on: docker + container: + image: data.forgejo.org/oci/node:22-bookworm + steps: + - name: event + run: | + cat <<'EOF' + ${{ toJSON(forge.event.pull_request.labels.*.name) }} + EOF + cat <<'EOF' + push => ${{ forge.event_name == 'push' && ( forge.ref_name == 'main' || forge.ref_name == 'wip-cascade') }} + pull_request_target synchornized => ${{ ( forge.event.action == 'synchronized' && contains(forge.event.pull_request.labels.*.name, 'run-forgejo-tests') ) }} + pull_request_target label_updated => ${{ ( forge.event.action == 'label_updated' && forge.event.label.name == 'run-forgejo-tests' ) }} + contains => ${{ contains(forge.event.pull_request.labels.*.name, 'run-forgejo-tests') }} + contains boolean => ${{ contains(forge.event.pull_request.labels.*.name, 'run-forgejo-tests') == true }} + EOF + cat <<'EOF' + ${{ toJSON(forge) }} + EOF + + forgejo: + # + # Always run when a commit is pushed to the main or wip-cascade branch + # If this is a pull request, run + # - when the `run-forgejo-tests` label is set (label_updated) (but not if another label is set or if a label is removed) + # - when a new commit is pushed to the pull request (synchronized) if the `run-forgejo-tests` is already present + # - when the pull request is closed, which also happens when it is merged, so that the Forgejo pull request is closed + # + if: > + vars.ROLE == 'forgejo-coding' && ( + ( + forge.event_name == 'push' && ( forge.ref_name == 'main' || forge.ref_name == 'wip-cascade') + ) || ( + forge.event_name == 'pull_request_target' && ( + forge.event.action == 'closed' || + ( forge.event.action == 'synchronized' && contains(forge.event.pull_request.labels.*.name, 'run-forgejo-tests') ) || + ( forge.event.action == 'label_updated' && forge.event.label.name == 'run-forgejo-tests' ) + ) + ) + ) + + runs-on: docker + container: + image: data.forgejo.org/oci/node:22-bookworm + steps: + - uses: https://data.forgejo.org/actions/checkout@v4 + with: + fetch-depth: '0' + show-progress: 'false' + - uses: https://data.forgejo.org/actions/setup-go@v5 + with: + go-version-file: go.mod + - uses: https://data.forgejo.org/actions/cascading-pr@v2.3.0 + with: + origin-url: ${{ forge.server_url }} + origin-repo: ${{ forge.repository }} + origin-token: ${{ secrets.FORGEJO_CASCADING_PR_ORIGIN_TOKEN }} + origin-pr: ${{ forge.event.pull_request.number }} + origin-ref: ${{ forge.event_name == 'push' && forge.event.ref || '' }} + destination-url: https://codeberg.org + destination-fork-repo: forgejo-cascading-pr/forgejo + destination-repo: forgejo/forgejo + destination-branch: forgejo + destination-token: ${{ secrets.FORGEJO_CASCADING_PR_DESTINATION_TOKEN }} + prefix: runner + close: true + verbose: ${{ vars.VERBOSE == 'yes' }} + debug: ${{ vars.DEBUG == 'yes' }} + wait-iteration: 120 + update: .forgejo/cascading-forgejo diff --git a/.forgejo/workflows/cascade-setup-forgejo.yml b/.forgejo/workflows/cascade-setup-forgejo.yml index bd3684ba..9292d936 100644 --- a/.forgejo/workflows/cascade-setup-forgejo.yml +++ b/.forgejo/workflows/cascade-setup-forgejo.yml @@ -1,31 +1,99 @@ +# Copyright 2025 The Forgejo Authors # SPDX-License-Identifier: MIT +# +# CASCADING_PR_ORIGIN is a token from the https://code.forgejo.org/cascading-pr user +# with scope write:issue read:repository read:user +# CASCADING_PR_DESTINATION is a token from the https://code.forgejo.org/cascading-pr user +# with scope write:issue write:repository read:user +# +# To modify this workflow: +# +# - push it to the wip-cascade branch on the repository +# otherwise it will not have access to the secrets required to push +# the cascading PR +# +# - once it works, open a pull request for the sake of keeping track +# of the change even if the PR won't run it because it will use +# whatever is in the default branch instead +# +# - after it is merged, double check it works by setting the +# label on a pull request (any pull request will do) +# +name: cascade + on: + push: + branches: + - 'wip-cascade' pull_request_target: types: - - opened - synchronize + - labeled - closed enable-email-notifications: true jobs: - cascade: + debug: + if: > + vars.DEBUG == 'yes' + runs-on: docker + container: + image: data.forgejo.org/oci/node:22-bookworm + steps: + - name: event + run: | + cat <<'EOF' + ${{ toJSON(forge.event.pull_request.labels.*.name) }} + EOF + cat <<'EOF' + push => ${{ forge.event_name == 'push' && ( forge.ref_name == 'wip-cascade') }} + pull_request_target synchornized => ${{ ( forge.event.action == 'synchronized' && contains(forge.event.pull_request.labels.*.name, 'run-end-to-end-tests') ) }} + pull_request_target label_updated => ${{ ( forge.event.action == 'label_updated' && forge.event.label.name == 'run-end-to-end-tests' ) }} + contains => ${{ contains(forge.event.pull_request.labels.*.name, 'run-end-to-end-tests') }} + contains boolean => ${{ contains(forge.event.pull_request.labels.*.name, 'run-end-to-end-tests') == true }} + EOF + cat <<'EOF' + ${{ toJSON(forge) }} + EOF + + end-to-end: + # + # Always run when a commit is pushed to the wip-cascade branch + # If this is a pull request, run + # - when the `run-end-to-end-tests` label is set (label_updated) (but not if another label is set or if a label is removed) + # - when a new commit is pushed to the pull request (synchronized) if the `run-end-to-end-tests` is already present + # - when the pull request is closed, which also happens when it is merged, so that the setup-forgejo & end-to-end pull requests are closed + # + if: > + vars.ROLE == 'forgejo-coding' && ( + ( + forge.event_name == 'push' && ( forge.ref_name == 'wip-cascade' ) + ) || ( + forge.event_name == 'pull_request_target' && ( + forge.event.action == 'closed' || + ( forge.event.action == 'synchronized' && contains(forge.event.pull_request.labels.*.name, 'run-end-to-end-tests') ) || + ( forge.event.action == 'label_updated' && forge.event.label.name == 'run-end-to-end-tests' ) + ) + ) + ) runs-on: docker container: image: 'code.forgejo.org/oci/node:22-bookworm' - if: > - ! contains(forge.event.pull_request.title, '[skip cascade]') steps: - - uses: https://code.forgejo.org/actions/cascading-pr@v2.2.1 + - uses: https://data.forgejo.org/actions/cascading-pr@v2.3.0 with: - origin-url: ${{ env.FORGEJO_SERVER_URL }} - origin-repo: forgejo/runner + origin-url: ${{ forge.server_url }} + origin-repo: ${{ forge.repository }} origin-token: ${{ secrets.CASCADING_PR_ORIGIN }} origin-pr: ${{ forge.event.pull_request.number }} - destination-url: ${{ env.FORGEJO_SERVER_URL }} + origin-ref: ${{ forge.event_name == 'push' && forge.event.ref || '' }} + destination-url: ${{ forge.server_url }} destination-repo: actions/setup-forgejo destination-fork-repo: cascading-pr/setup-forgejo destination-branch: main destination-token: ${{ secrets.CASCADING_PR_DESTINATION }} - close-merge: true - update: .forgejo/cascading-pr-setup-forgejo + close: true + verbose: ${{ vars.VERBOSE == 'yes' }} + debug: ${{ vars.DEBUG == 'yes' }} + update: .forgejo/cascading-setup-forgejo diff --git a/.forgejo/workflows/docker-build-push-action-in-lxc.yml b/.forgejo/workflows/docker-build-push-action-in-lxc.yml index 25786011..dc1fe14b 100644 --- a/.forgejo/workflows/docker-build-push-action-in-lxc.yml +++ b/.forgejo/workflows/docker-build-push-action-in-lxc.yml @@ -21,7 +21,7 @@ on: enable-email-notifications: true env: - FORGEJO_VERSION: 11.0.3 # renovate: datasource=docker depName=code.forgejo.org/forgejo/forgejo + FORGEJO_VERSION: 11.0.7 # renovate: datasource=docker depName=code.forgejo.org/forgejo/forgejo FORGEJO_USER: root FORGEJO_PASSWORD: admin1234 @@ -34,7 +34,7 @@ jobs: - name: install Forgejo so it can be used as a container registry id: registry - uses: https://data.forgejo.org/actions/setup-forgejo@v3.0.1 + uses: https://data.forgejo.org/actions/setup-forgejo@v3.0.4 with: user: ${{ env.FORGEJO_USER }} password: ${{ env.FORGEJO_PASSWORD }} diff --git a/.forgejo/workflows/example-docker-compose.yml b/.forgejo/workflows/example-docker-compose.yml index dce57a39..c39802ad 100644 --- a/.forgejo/workflows/example-docker-compose.yml +++ b/.forgejo/workflows/example-docker-compose.yml @@ -15,7 +15,7 @@ jobs: if: vars.ROLE == 'forgejo-coding' runs-on: lxc-bookworm steps: - - uses: actions/checkout@v4 + - uses: https://data.forgejo.org/actions/checkout@v4 - name: Install docker run: | diff --git a/.forgejo/workflows/example-lxc-systemd.yml b/.forgejo/workflows/example-lxc-systemd.yml index cca4f934..ba3c6a5d 100644 --- a/.forgejo/workflows/example-lxc-systemd.yml +++ b/.forgejo/workflows/example-lxc-systemd.yml @@ -14,11 +14,12 @@ env: SERIAL: "30" LIFETIME: "60" SYSTEMD_OPTIONS: "--no-pager --full" + USE_VERSION: 11.0.7 # renovate: datasource=docker depName=code.forgejo.org/forgejo/forgejo jobs: example-lxc-systemd: if: vars.ROLE == 'forgejo-coding' - runs-on: lxc-bookworm + runs-on: lxc-trixie steps: - uses: https://data.forgejo.org/actions/checkout@v4 @@ -53,11 +54,11 @@ jobs: done - id: forgejo - uses: https://data.forgejo.org/actions/setup-forgejo@v3.0.1 + uses: https://data.forgejo.org/actions/setup-forgejo@v3.0.4 with: user: root password: admin1234 - binary: https://code.forgejo.org/forgejo/forgejo/releases/download/v7.0.12/forgejo-7.0.12-linux-amd64 + binary: https://code.forgejo.org/forgejo/forgejo/releases/download/v${{ env.USE_VERSION }}/forgejo-${{ env.USE_VERSION }}-linux-amd64 # must be the same as LXC_IPV4_PREFIX in examples/lxc-systemd/forgejo-runner-service.sh lxc-ip-prefix: 10.105.7 @@ -123,8 +124,8 @@ jobs: started_running=/etc/forgejo-runner/$serial/started-running killed_gracefully=/etc/forgejo-runner/$serial/killed-gracefully stopped_gracefully=/etc/forgejo-runner/$serial/stopped-gracefully - retry --delay 5 --times 20 cp -a $started_running /tmp/first-run - retry --delay 1 --times 30 grep --quiet 'Starting runner daemon' /var/log/forgejo-runner/$serial.log + retry --delay 10 --times 20 cp -a $started_running /tmp/first-run + retry --delay 2 --times 30 grep --quiet 'Starting runner daemon' /var/log/forgejo-runner/$serial.log systemctl stop forgejo-runner@$serial ! systemctl $all status forgejo-runner@$serial ls -l /etc/forgejo-runner/$serial @@ -136,7 +137,7 @@ jobs: ! test -f $killed_gracefully ! test -f $stopped_gracefully lifetime=${{ env.LIFETIME }} - # give it time to restart at least once + : give it time to restart at least once ls -l /etc/forgejo-runner/$serial sleep $lifetime ; sleep $lifetime ls -l /etc/forgejo-runner/$serial diff --git a/.forgejo/workflows/publish-release.yml b/.forgejo/workflows/publish-release.yml index 247ec929..5555996a 100644 --- a/.forgejo/workflows/publish-release.yml +++ b/.forgejo/workflows/publish-release.yml @@ -38,6 +38,10 @@ jobs: to-owner: ${{ vars.TO_OWNER }} repo: "runner" release-notes: | + - [User guide](https://forgejo.org/docs/next/user/actions/overview/) + - [Administrator guide](https://forgejo.org/docs/next/admin/actions/) + - [Container images](https://code.forgejo.org/forgejo/-/packages/container/runner/versions) + Release Notes --- diff --git a/.forgejo/workflows/release-notes-assistant.yml b/.forgejo/workflows/release-notes-assistant.yml index bf63f433..9dc12abe 100644 --- a/.forgejo/workflows/release-notes-assistant.yml +++ b/.forgejo/workflows/release-notes-assistant.yml @@ -6,16 +6,17 @@ name: issue-labels on: pull_request_target: types: + - opened - edited - synchronize - labeled env: - RNA_VERSION: v1.4.0 # renovate: datasource=forgejo-releases depName=forgejo/release-notes-assistant registryUrl=https://code.forgejo.org + RNA_VERSION: v1.4.1 # renovate: datasource=forgejo-releases depName=forgejo/release-notes-assistant jobs: release-notes: - if: vars.ROLE == 'forgejo-coding' + if: vars.ROLE == 'forgejo-coding' && !contains(forge.event.pull_request.labels.*.name, 'Kind/DependencyUpdate') runs-on: docker container: image: 'data.forgejo.org/oci/ci:1' diff --git a/.forgejo/workflows/test.yml b/.forgejo/workflows/test.yml index 9ce07101..773e0ae2 100644 --- a/.forgejo/workflows/test.yml +++ b/.forgejo/workflows/test.yml @@ -14,7 +14,6 @@ env: FORGEJO_RUNNER_SECRET: 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA' FORGEJO_SCRIPT: | /usr/bin/s6-svscan /etc/s6 & sleep 10 ; su -c "forgejo admin user create --admin --username $FORGEJO_ADMIN_USER --password $FORGEJO_ADMIN_PASSWORD --email root@example.com" git && su -c "forgejo forgejo-cli actions register --labels docker --name therunner --secret $FORGEJO_RUNNER_SECRET" git && sleep infinity - GOPROXY: https://goproxy.io,direct jobs: build-and-tests: @@ -40,9 +39,9 @@ jobs: - '/usr/bin/s6-svscan /etc/s6 & sleep 10 ; su -c "forgejo admin user create --admin --username $FORGEJO_ADMIN_USER --password $FORGEJO_ADMIN_PASSWORD --email root@example.com" git && su -c "forgejo forgejo-cli actions register --labels docker --name therunner --secret $FORGEJO_RUNNER_SECRET" git && sleep infinity' steps: - - uses: actions/checkout@v4 + - uses: https://data.forgejo.org/actions/checkout@v4 - - uses: actions/setup-go@v5 + - uses: https://data.forgejo.org/actions/setup-go@v5 with: go-version-file: go.mod @@ -60,7 +59,7 @@ jobs: - run: make build - - uses: https://code.forgejo.org/actions/upload-artifact@v3 + - uses: https://data.forgejo.org/actions/upload-artifact@v3 with: name: forgejo-runner path: forgejo-runner @@ -74,16 +73,15 @@ jobs: - run: make FORGEJO_URL=http://$FORGEJO_HOST_PORT test runner-exec-tests: - needs: [build-and-tests] name: runner exec tests if: vars.ROLE == 'forgejo-coding' runs-on: lxc-bookworm - + needs: [build-and-tests] steps: - - uses: actions/checkout@v4 + - uses: https://data.forgejo.org/actions/checkout@v4 - - uses: https://code.forgejo.org/actions/download-artifact@v3 + - uses: https://data.forgejo.org/actions/download-artifact@v3 with: name: forgejo-runner @@ -127,3 +125,150 @@ jobs: set -x ./forgejo-runner exec --var MY_VAR=testvariable --workflows .forgejo/testdata/var.yml |& tee /tmp/var.out grep --quiet 'Success - Main echo "VAR -> testvariable"' /tmp/var.out + + integration-tests: + name: integration tests + if: vars.ROLE == 'forgejo-coding' + runs-on: lxc-bookworm + steps: + - uses: https://data.forgejo.org/actions/checkout@v4 + + - uses: https://data.forgejo.org/actions/setup-go@v5 + with: + go-version-file: go.mod + + - name: apt install docker.io + run: | + export DEBIAN_FRONTEND=noninteractive + apt-get update -qq + apt-get -q install -qq -y docker.io + + - run: apt-get -q install -qq -y gcc # required for `-race` + - name: integration test + run: | + go test -race ./act/container + go test -race -timeout 30m ./act/runner/... + + runner-integration-tests: + name: runner integration tests + if: vars.ROLE == 'forgejo-coding' + runs-on: lxc-bookworm + needs: [build-and-tests] + + steps: + - uses: https://data.forgejo.org/actions/checkout@v4 + + - uses: https://data.forgejo.org/actions/setup-go@v5 + with: + go-version-file: go.mod + + - name: install docker + run: | + mkdir /etc/docker + cat > /etc/docker/daemon.json <=3.2.0' + + - name: validate .pre-commit-hooks.yaml + run: pre-commit validate-manifest .pre-commit-hooks.yaml + + # Will fail due to `act/runner/testdata/local-action-fails-schema-validation/action/action.yml` + - name: check pre-commit hook against local action files (should fail) + continue-on-error: true + run: | + pre-commit try-repo --all-files --verbose . forgejo-runner-validate + + - name: check that a bad workflow file doesn’t validate (should fail) + continue-on-error: true + run: | + mkdir -p test-repo + cd test-repo + git config set advice.defaultBranchName false + git init --quiet + mkdir -p .forgejo/workflows + cp ../act/runner/testdata/local-action-fails-schema-validation/action/action.yml ./ + touch .forgejo/workflows/bad-workflow.yml + cat > .pre-commit-config.yaml <- + {{ .ProjectName }}_ + {{- title .Os }}_ + {{- if eq .Arch "amd64" }}x86_64 + {{- else if eq .Arch "386" }}i386 + {{- else }}{{ .Arch }}{{ end }} + {{- if .Arm }}v{{ .Arm }}{{ end }} + +changelog: + abbrev: 10 + filters: + exclude: + - "^docs:" + - "^test:" + format: "{{.SHA}}: {{.Message}}" + groups: + - title: Features + regexp: '^.*?feat(\([[:word:]]+\))??!?:.+$' + order: 0 + - title: "Bug fixes" + regexp: '^.*?fix(\([[:word:]]+\))??!?:.+$' + order: 1 + - title: "Chores" + regexp: '^.*?chore(\([[:word:]]+\))??!?:.+$' + order: 2 + - title: Others + order: 999 + sort: asc + +release: + gitea: + owner: DevFW-CICD + name: runner + +force_token: gitea +gitea_urls: + api: https://edp.buildth.ing/api/v1 + download: https://edp.buildth.ing + # set to true if you use a self-signed certificate + skip_tls_verify: false diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml new file mode 100644 index 00000000..315c03e1 --- /dev/null +++ b/.pre-commit-hooks.yaml @@ -0,0 +1,13 @@ +- id: forgejo-runner-validate + name: Validate Forgejo Actions files + description: This hook validates Forgejo Actions action and workflow files. + language: golang + entry: runner validate + args: ['--directory', '.'] + pass_filenames: false + files: (?:(?:^|/)action|^\.(?:forgejo|github|gitea)/workflows/[^/\n]+)\.ya?ml$ + types: [yaml] + # 3.2.0 is when the pre-* `stages` used here were added. + # Old names (without the pre- prefix) are deprecated since 4.0.0. + minimum_pre_commit_version: '3.2.0' + stages: [pre-commit, pre-merge-commit, pre-push, manual] diff --git a/Dockerfile b/Dockerfile index d1ab9e93..2c654211 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM --platform=$BUILDPLATFORM data.forgejo.org/oci/xx AS xx -FROM --platform=$BUILDPLATFORM data.forgejo.org/oci/golang:1.24-alpine3.22 AS build-env +FROM --platform=$BUILDPLATFORM data.forgejo.org/oci/golang:1.25-alpine3.22 AS build-env # # Transparently cross compile for the target platform @@ -32,7 +32,7 @@ LABEL maintainer="contact@forgejo.org" \ org.opencontainers.image.source="https://code.forgejo.org/forgejo/runner" \ org.opencontainers.image.version="${RELEASE_VERSION}" \ org.opencontainers.image.vendor="Forgejo" \ - org.opencontainers.image.licenses="MIT" \ + org.opencontainers.image.licenses="GPL-3.0-or-later" \ org.opencontainers.image.title="Forgejo Runner" \ org.opencontainers.image.description="A runner for Forgejo Actions." diff --git a/LICENSE b/LICENSE index 76462726..f288702d 100644 --- a/LICENSE +++ b/LICENSE @@ -1,20 +1,674 @@ -Copyright (c) 2023-2025 The Forgejo Authors -Copyright (c) 2022 The Gitea Authors + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. + Preamble -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/Makefile b/Makefile index 1995f027..07ab0134 100644 --- a/Makefile +++ b/Makefile @@ -13,8 +13,8 @@ WINDOWS_ARCHS ?= windows/amd64 GO_FMT_FILES := $(shell find . -type f -name "*.go" ! -name "generated.*") GOFILES := $(shell find . -type f -name "*.go" -o -name "go.mod" ! -name "generated.*") -MOCKERY_PACKAGE ?= github.com/vektra/mockery/v2@v2.53.4 # renovate: datasource=go -GOLANGCI_LINT_PACKAGE ?= github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.3.1 # renovate: datasource=go +MOCKERY_PACKAGE ?= github.com/vektra/mockery/v2@v2.53.5 # renovate: datasource=go +GOLANGCI_LINT_PACKAGE ?= github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.5.0 # renovate: datasource=go DOCKER_IMAGE ?= gitea/act_runner DOCKER_TAG ?= nightly @@ -62,18 +62,18 @@ else endif endif -GO_PACKAGES_TO_VET ?= $(filter-out code.forgejo.org/forgejo/runner/v9/internal/pkg/client/mocks,$(shell $(GO) list ./...)) +GO_PACKAGES_TO_VET ?= $(filter-out code.forgejo.org/forgejo/runner/v11/internal/pkg/client/mocks code.forgejo.org/forgejo/runner/v11/act/artifactcache/mock_caches.go,$(shell $(GO) list ./...)) TAGS ?= -LDFLAGS ?= -X "code.forgejo.org/forgejo/runner/v9/internal/pkg/ver.version=v$(RELEASE_VERSION)" +LDFLAGS ?= -X "code.forgejo.org/forgejo/runner/v11/internal/pkg/ver.version=v$(RELEASE_VERSION)" all: build -.PHONY: lint +.PHONY: lint-check lint-check: $(GO) run $(GOLANGCI_LINT_PACKAGE) run $(GOLANGCI_LINT_ARGS) -.PHONY: lint-fix +.PHONY: lint lint: $(GO) run $(GOLANGCI_LINT_PACKAGE) run $(GOLANGCI_LINT_ARGS) --fix @@ -106,7 +106,12 @@ fmt-check: fi; test: lint-check fmt-check - @$(GO) test -v -cover -coverprofile coverage.txt ./internal/... && echo "\n==>\033[32m Ok\033[m\n" || exit 1 + $(GO) test -v -race -short -cover -coverprofile coverage.txt ./internal/... + $(GO) test -race -short ./act/container + $(GO) test -race ./act/artifactcache/... ./act/workflowpattern/... ./act/filecollector/... ./act/common/... ./act/jobparser ./act/model ./act/exprparser ./act/schema + +integration-test: + @$(GO) test -race -v ./internal/app/run/... .PHONY: vet vet: @@ -115,7 +120,7 @@ vet: .PHONY: generate generate: - $(GO) generate ./internal/... + $(GO) generate ./... install: $(GOFILES) $(GO) install -v -tags '$(TAGS)' -ldflags '$(EXTLDFLAGS)-s -w $(LDFLAGS)' diff --git a/README.md b/README.md index 043f6952..36a67a8a 100644 --- a/README.md +++ b/README.md @@ -1,22 +1,14 @@ # Forgejo Runner -**WARNING:** this is [alpha release quality](https://en.wikipedia.org/wiki/Software_release_life_cycle#Alpha) code and should not be considered secure enough to deploy in production. - A daemon that connects to a Forgejo instance and runs jobs for continuous integration. The [installation and usage instructions](https://forgejo.org/docs/next/admin/actions/) are part of the Forgejo documentation. -# Reporting bugs - -When filing a bug in [the issue tracker](https://code.forgejo.org/forgejo/runner/issues), it is very helpful to propose a pull request [in the end-to-end tests](https://code.forgejo.org/forgejo/end-to-end/src/branch/main/actions) repository that adds a reproducer. It will fail the CI and unambiguously demonstrate that the problem exists. In most cases it is enough to add a workflow ([see the echo example](https://code.forgejo.org/forgejo/end-to-end/src/branch/main/actions/example-echo)). For more complicated cases it is also possible to add a runner config file as well as shell scripts to setup and teardown the test case ([see the service example](https://code.forgejo.org/forgejo/end-to-end/src/branch/main/actions/example-service)). +# Reporting security-related issues Sensitive security-related issues should be reported to [security@forgejo.org](mailto:security@forgejo.org) using [encryption](https://keyoxide.org/security@forgejo.org). - ## License -The Forgejo runner source code is distributed under the terms of the following licenses: - -- [MIT](LICENSE) for the most part. -- [Apache 2](act/container/DOCKER_LICENSE) for parts found in the [act/container](act/container) directory. +The Forgejo runner is distributed under the terms of the [GPL version 3.0](LICENSE) or any later version. # Architectures & OS @@ -38,77 +30,44 @@ The Forgejo runner is a dependency of the [setup-forgejo action](https://code.fo - Install [Go](https://go.dev/doc/install) and `make(1)` - `make build` -The [test workflow](.forgejo/workflows/test.yml) is a full example that builds the binary, runs the tests and launches the runner binary against a live Forgejo instance. +## Linting -## Generate mocks +- `make lint-check` +- `make lint` # will fix some lint errors -- `make deps-tools` -- `make generate` +## Testing -If there are changes, commit them to the repository. +The [workflow](.forgejo/workflows/test.yml) that runs in the CI uses similar commands. -## Local debug +### Without a Forgejo instance -The repositories are checked out in the same directory: +- Install [Docker](https://docs.docker.com/engine/install/) +- `make test integration-test` -- **runner**: [Forgejo runner](https://code.forgejo.org/forgejo/runner) -- **setup-forgejo**: [setup-forgejo](https://code.forgejo.org/actions/setup-forgejo) +The `TestRunner_RunEvent` test suite contains most integration tests +with real-world workflows and is time-consuming to run. During +development, it is helpful to run a specific test through a targeted +command such as this: -### Install dependencies +- `go test -count=1 -run='TestRunner_RunEvent$/local-action-dockerfile$' ./act/runner` -The dependencies are installed manually or with: +### With a Forgejo instance -```shell -setup-forgejo/forgejo-dependencies.sh +- Run a Forgejo instance locally (for instance at http://0.0.0.0:8080) and create as shared secret +```sh +export FORGEJO_RUNNER_SECRET='AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA' +export FORGEJO_URL=http://0.0.0.0:8080 +forgejo forgejo-cli actions register --labels docker --name therunner --secret $FORGEJO_RUNNER_SECRET ``` +- `make test integration-test` # which will run addional tests because FORGEJO_URL is set -### Build the Forgejo runner +### end-to-end -```shell -cd runner ; rm -f forgejo-runner ; make forgejo-runner -``` - -### Launch Forgejo and the runner - -A Forgejo instance is launched with: - -```shell -cd setup-forgejo -./forgejo.sh setup -firefox $(cat forgejo-url) -``` - -The user is `root` with password `admin1234`. The runner is registered with: - -``` -cd setup-forgejo -docker exec --user 1000 forgejo forgejo actions generate-runner-token > forgejo-runner-token -../runner/forgejo-runner register --no-interactive --instance "$(cat forgejo-url)" --name runner --token $(cat forgejo-runner-token) --labels docker:docker://node:22-bookworm,self-hosted:host,lxc:lxc://debian:bookworm -``` - -And launched with: - -```shell -cd setup-forgejo ; ../runner/forgejo-runner --config runner-config.yml daemon -``` - -Note that the `runner-config.yml` is required in that particular case -to configure the network in `bridge` mode, otherwise the runner will -create a network that cannot reach the forgejo instance. - -### Try a sample workflow - -From the Forgejo web interface, create a repository and add the -following to `.forgejo/workflows/try.yaml`. It will launch the job and -the result can be observed from the `actions` tab. - -```yaml -on: [push] -jobs: - ls: - runs-on: docker - steps: - - uses: actions/checkout@v4 - - run: | - ls ${{ github.workspace }} -``` +- Follow the instructions from the end-to-end tests to [run actions tests locally](https://code.forgejo.org/forgejo/end-to-end#running-from-locally-built-binary). +- `./end-to-end.sh actions_teardown` # stop the Forgejo and runner daemons running in the end-to-end environment +- `( cd ~/clone-of-the-runner-repo ; make build ; cp forgejo-runner /tmp/forgejo-end-to-end/forgejo-runner )` # install the runner built from sources +- `./end-to-end.sh actions_setup 13.0` # start Forgejo v13.0 and the runner daemon in the end-to-end environment +- `./end-to-end.sh actions_verify_example echo` # run the [echo workflow](https://code.forgejo.org/forgejo/end-to-end/src/branch/main/actions/example-echo/.forgejo/workflows/test.yml) +- `xdg-open http://127.0.0.1:3000/root/example-echo/actions/runs/1` # see the logs workflow +- `less /tmp/forgejo-end-to-end/forgejo-runner.log` # analyze the runner logs +- `less /tmp/forgejo-end-to-end/forgejo-work-path/log/forgejo.log` # analyze the Forgejo logs diff --git a/act/artifactcache/caches.go b/act/artifactcache/caches.go new file mode 100644 index 00000000..b4d9bd17 --- /dev/null +++ b/act/artifactcache/caches.go @@ -0,0 +1,324 @@ +package artifactcache + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "regexp" + "sync/atomic" + "time" + + "github.com/sirupsen/logrus" + "github.com/timshannon/bolthold" + "go.etcd.io/bbolt" +) + +//go:generate mockery --inpackage --name caches +type caches interface { + getDB() *bolthold.Store + validateMac(rundata RunData) (string, error) + readCache(id uint64, repo string) (*Cache, error) + useCache(id uint64) error + setgcAt(at time.Time) + gcCache() + close() + + serve(w http.ResponseWriter, r *http.Request, id uint64) + commit(id uint64, size int64) (int64, error) + exist(id uint64) (bool, error) + write(id, offset uint64, reader io.Reader) error +} + +type cachesImpl struct { + dir string + storage *Storage + logger logrus.FieldLogger + secret string + + db *bolthold.Store + + gcing atomic.Bool + gcAt time.Time +} + +func newCaches(dir, secret string, logger logrus.FieldLogger) (caches, error) { + c := &cachesImpl{ + secret: secret, + } + + c.logger = logger + + if dir == "" { + home, err := os.UserHomeDir() + if err != nil { + return nil, err + } + dir = filepath.Join(home, ".cache", "actcache") + } + if err := os.MkdirAll(dir, 0o755); err != nil { + return nil, err + } + + c.dir = dir + + storage, err := NewStorage(filepath.Join(dir, "cache")) + if err != nil { + return nil, err + } + c.storage = storage + + file := filepath.Join(c.dir, "bolt.db") + db, err := bolthold.Open(file, 0o644, &bolthold.Options{ + Encoder: json.Marshal, + Decoder: json.Unmarshal, + Options: &bbolt.Options{ + Timeout: 5 * time.Second, + NoGrowSync: bbolt.DefaultOptions.NoGrowSync, + FreelistType: bbolt.DefaultOptions.FreelistType, + }, + }) + if err != nil { + return nil, fmt.Errorf("Open(%s): %w", file, err) + } + c.db = db + + c.gcCache() + + return c, nil +} + +func (c *cachesImpl) close() { + if c.db != nil { + c.db.Close() + c.db = nil + } +} + +func (c *cachesImpl) getDB() *bolthold.Store { + return c.db +} + +var findCacheWithIsolationKeyFallback = func(db *bolthold.Store, repo string, keys []string, version, writeIsolationKey string) (*Cache, error) { + cache, err := findCache(db, repo, keys, version, writeIsolationKey) + if err != nil { + return nil, err + } + // If read was scoped to WriteIsolationKey and didn't find anything, we can fallback to the non-isolated cache read + if cache == nil && writeIsolationKey != "" { + cache, err = findCache(db, repo, keys, version, "") + if err != nil { + return nil, err + } + } + return cache, nil +} + +// if not found, return (nil, nil) instead of an error. +func findCache(db *bolthold.Store, repo string, keys []string, version, writeIsolationKey string) (*Cache, error) { + cache := &Cache{} + for _, prefix := range keys { + // if a key in the list matches exactly, don't return partial matches + if err := db.FindOne(cache, + bolthold.Where("Repo").Eq(repo).Index("Repo"). + And("Key").Eq(prefix). + And("Version").Eq(version). + And("WriteIsolationKey").Eq(writeIsolationKey). + And("Complete").Eq(true). + SortBy("CreatedAt").Reverse()); err == nil || !errors.Is(err, bolthold.ErrNotFound) { + if err != nil { + return nil, fmt.Errorf("find cache entry equal to %s: %w", prefix, err) + } + return cache, nil + } + prefixPattern := fmt.Sprintf("^%s", regexp.QuoteMeta(prefix)) + re, err := regexp.Compile(prefixPattern) + if err != nil { + continue + } + if err := db.FindOne(cache, + bolthold.Where("Repo").Eq(repo).Index("Repo"). + And("Key").RegExp(re). + And("Version").Eq(version). + And("WriteIsolationKey").Eq(writeIsolationKey). + And("Complete").Eq(true). + SortBy("CreatedAt").Reverse()); err != nil { + if errors.Is(err, bolthold.ErrNotFound) { + continue + } + return nil, fmt.Errorf("find cache entry starting with %s: %w", prefix, err) + } + return cache, nil + } + return nil, nil +} + +func insertCache(db *bolthold.Store, cache *Cache) error { + if err := db.Insert(bolthold.NextSequence(), cache); err != nil { + return fmt.Errorf("insert cache: %w", err) + } + // write back id to db + if err := db.Update(cache.ID, cache); err != nil { + return fmt.Errorf("write back id to db: %w", err) + } + return nil +} + +func (c *cachesImpl) readCache(id uint64, repo string) (*Cache, error) { + db := c.getDB() + cache := &Cache{} + if err := db.Get(id, cache); err != nil { + return nil, fmt.Errorf("readCache: Get(%v): %w", id, err) + } + if cache.Repo != repo { + return nil, fmt.Errorf("readCache: Get(%v): cache.Repo %s != repo %s", id, cache.Repo, repo) + } + + return cache, nil +} + +func (c *cachesImpl) useCache(id uint64) error { + db := c.getDB() + cache := &Cache{} + if err := db.Get(id, cache); err != nil { + return fmt.Errorf("useCache: Get(%v): %w", id, err) + } + cache.UsedAt = time.Now().Unix() + if err := db.Update(cache.ID, cache); err != nil { + return fmt.Errorf("useCache: Update(%v): %v", cache.ID, err) + } + return nil +} + +func (c *cachesImpl) serve(w http.ResponseWriter, r *http.Request, id uint64) { + c.storage.Serve(w, r, id) +} + +func (c *cachesImpl) commit(id uint64, size int64) (int64, error) { + return c.storage.Commit(id, size) +} + +func (c *cachesImpl) exist(id uint64) (bool, error) { + return c.storage.Exist(id) +} + +func (c *cachesImpl) write(id, offset uint64, reader io.Reader) error { + return c.storage.Write(id, offset, reader) +} + +const ( + keepUsed = 30 * 24 * time.Hour + keepUnused = 7 * 24 * time.Hour + keepTemp = 5 * time.Minute + keepOld = 5 * time.Minute +) + +func (c *cachesImpl) setgcAt(at time.Time) { + c.gcAt = at +} + +func (c *cachesImpl) gcCache() { + if c.gcing.Load() { + return + } + if !c.gcing.CompareAndSwap(false, true) { + return + } + defer c.gcing.Store(false) + + if time.Since(c.gcAt) < time.Hour { + c.logger.Debugf("skip gc: %v", c.gcAt.String()) + return + } + c.gcAt = time.Now() + c.logger.Debugf("gc: %v", c.gcAt.String()) + + db := c.getDB() + + // Remove the caches which are not completed for a while, they are most likely to be broken. + var caches []*Cache + if err := db.Find(&caches, bolthold. + Where("UsedAt").Lt(time.Now().Add(-keepTemp).Unix()). + And("Complete").Eq(false), + ); err != nil { + fatal(c.logger, fmt.Errorf("gc caches not completed: %v", err)) + } else { + for _, cache := range caches { + c.storage.Remove(cache.ID) + if err := db.Delete(cache.ID, cache); err != nil { + c.logger.Errorf("delete cache: %v", err) + continue + } + c.logger.Infof("deleted cache: %+v", cache) + } + } + + // Remove the old caches which have not been used recently. + caches = caches[:0] + if err := db.Find(&caches, bolthold. + Where("UsedAt").Lt(time.Now().Add(-keepUnused).Unix()), + ); err != nil { + fatal(c.logger, fmt.Errorf("gc caches old not used: %v", err)) + } else { + for _, cache := range caches { + c.storage.Remove(cache.ID) + if err := db.Delete(cache.ID, cache); err != nil { + c.logger.Warnf("delete cache: %v", err) + continue + } + c.logger.Infof("deleted cache: %+v", cache) + } + } + + // Remove the old caches which are too old. + caches = caches[:0] + if err := db.Find(&caches, bolthold. + Where("CreatedAt").Lt(time.Now().Add(-keepUsed).Unix()), + ); err != nil { + fatal(c.logger, fmt.Errorf("gc caches too old: %v", err)) + } else { + for _, cache := range caches { + c.storage.Remove(cache.ID) + if err := db.Delete(cache.ID, cache); err != nil { + c.logger.Warnf("delete cache: %v", err) + continue + } + c.logger.Infof("deleted cache: %+v", cache) + } + } + + // Remove the old caches with the same key and version, keep the latest one. + // Also keep the olds which have been used recently for a while in case of the cache is still in use. + if results, err := db.FindAggregate( + &Cache{}, + bolthold.Where("Complete").Eq(true), + "Key", "Version", + ); err != nil { + fatal(c.logger, fmt.Errorf("gc aggregate caches: %v", err)) + } else { + for _, result := range results { + if result.Count() <= 1 { + continue + } + result.Sort("CreatedAt") + caches = caches[:0] + result.Reduction(&caches) + for _, cache := range caches[:len(caches)-1] { + if time.Since(time.Unix(cache.UsedAt, 0)) < keepOld { + // Keep it since it has been used recently, even if it's old. + // Or it could break downloading in process. + continue + } + c.storage.Remove(cache.ID) + if err := db.Delete(cache.ID, cache); err != nil { + c.logger.Warnf("delete cache: %v", err) + continue + } + c.logger.Infof("deleted cache: %+v", cache) + } + } + } +} diff --git a/act/artifactcache/caches_test.go b/act/artifactcache/caches_test.go new file mode 100644 index 00000000..34eab331 --- /dev/null +++ b/act/artifactcache/caches_test.go @@ -0,0 +1,53 @@ +package artifactcache + +import ( + "testing" + "time" + + "github.com/sirupsen/logrus" + "github.com/timshannon/bolthold" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCacheReadWrite(t *testing.T) { + caches, err := newCaches(t.TempDir(), "secret", logrus.New()) + require.NoError(t, err) + defer caches.close() + t.Run("NotFound", func(t *testing.T) { + found, err := caches.readCache(456, "repo") + assert.Nil(t, found) + assert.ErrorIs(t, err, bolthold.ErrNotFound) + }) + + repo := "repository" + cache := &Cache{ + Repo: repo, + Key: "key", + Version: "version", + Size: 444, + } + now := time.Now().Unix() + cache.CreatedAt = now + cache.UsedAt = now + cache.Repo = repo + + t.Run("Insert", func(t *testing.T) { + db := caches.getDB() + assert.NoError(t, insertCache(db, cache)) + }) + + t.Run("Found", func(t *testing.T) { + found, err := caches.readCache(cache.ID, cache.Repo) + require.NoError(t, err) + assert.Equal(t, cache.ID, found.ID) + }) + + t.Run("InvalidRepo", func(t *testing.T) { + invalidRepo := "INVALID REPO" + found, err := caches.readCache(cache.ID, invalidRepo) + assert.Nil(t, found) + assert.ErrorContains(t, err, invalidRepo) + }) +} diff --git a/act/artifactcache/handler.go b/act/artifactcache/handler.go index b50b6b11..cbc58d8f 100644 --- a/act/artifactcache/handler.go +++ b/act/artifactcache/handler.go @@ -7,46 +7,56 @@ import ( "io" "net" "net/http" - "os" - "path/filepath" - "regexp" "strconv" "strings" - "sync/atomic" "time" "github.com/julienschmidt/httprouter" "github.com/sirupsen/logrus" "github.com/timshannon/bolthold" - "go.etcd.io/bbolt" - "code.forgejo.org/forgejo/runner/v9/act/cacheproxy" - "code.forgejo.org/forgejo/runner/v9/act/common" + "code.forgejo.org/forgejo/runner/v11/act/common" ) const ( urlBase = "/_apis/artifactcache" ) -type Handler struct { - dir string - storage *Storage +var fatal = func(logger logrus.FieldLogger, err error) { + logger.Errorf("unrecoverable error in the cache: %v", err) + if err := suicide(); err != nil { + logger.Errorf("unrecoverable error in the cache: failed to send the TERM signal to shutdown the daemon %v", err) + } +} + +type Handler interface { + ExternalURL() string + Close() error + isClosed() bool + getCaches() caches + setCaches(caches caches) + find(w http.ResponseWriter, r *http.Request, params httprouter.Params) + reserve(w http.ResponseWriter, r *http.Request, params httprouter.Params) + upload(w http.ResponseWriter, r *http.Request, params httprouter.Params) + commit(w http.ResponseWriter, r *http.Request, params httprouter.Params) + get(w http.ResponseWriter, r *http.Request, params httprouter.Params) + clean(w http.ResponseWriter, r *http.Request, _ httprouter.Params) + middleware(handler httprouter.Handle) httprouter.Handle + responseJSON(w http.ResponseWriter, r *http.Request, code int, v ...any) +} + +type handler struct { + caches caches router *httprouter.Router listener net.Listener server *http.Server logger logrus.FieldLogger - secret string - - gcing atomic.Bool - gcAt time.Time outboundIP string } -func StartHandler(dir, outboundIP string, port uint16, secret string, logger logrus.FieldLogger) (*Handler, error) { - h := &Handler{ - secret: secret, - } +func StartHandler(dir, outboundIP string, port uint16, secret string, logger logrus.FieldLogger) (Handler, error) { + h := &handler{} if logger == nil { discard := logrus.New() @@ -56,24 +66,11 @@ func StartHandler(dir, outboundIP string, port uint16, secret string, logger log logger = logger.WithField("module", "artifactcache") h.logger = logger - if dir == "" { - home, err := os.UserHomeDir() - if err != nil { - return nil, err - } - dir = filepath.Join(home, ".cache", "actcache") - } - if err := os.MkdirAll(dir, 0o755); err != nil { - return nil, err - } - - h.dir = dir - - storage, err := NewStorage(filepath.Join(dir, "cache")) + caches, err := newCaches(dir, secret, logger) if err != nil { return nil, err } - h.storage = storage + h.caches = caches if outboundIP != "" { h.outboundIP = outboundIP @@ -93,8 +90,6 @@ func StartHandler(dir, outboundIP string, port uint16, secret string, logger log h.router = router - h.gcCache() - listener, err := net.Listen("tcp", fmt.Sprintf(":%d", port)) // listen on all interfaces if err != nil { return nil, err @@ -114,18 +109,22 @@ func StartHandler(dir, outboundIP string, port uint16, secret string, logger log return h, nil } -func (h *Handler) ExternalURL() string { +func (h *handler) ExternalURL() string { port := strconv.Itoa(h.listener.Addr().(*net.TCPAddr).Port) // TODO: make the external url configurable if necessary return fmt.Sprintf("http://%s", net.JoinHostPort(h.outboundIP, port)) } -func (h *Handler) Close() error { +func (h *handler) Close() error { if h == nil { return nil } var retErr error + if h.caches != nil { + h.caches.close() + h.caches = nil + } if h.server != nil { err := h.server.Close() if err != nil { @@ -146,22 +145,25 @@ func (h *Handler) Close() error { return retErr } -func (h *Handler) openDB() (*bolthold.Store, error) { - return bolthold.Open(filepath.Join(h.dir, "bolt.db"), 0o644, &bolthold.Options{ - Encoder: json.Marshal, - Decoder: json.Unmarshal, - Options: &bbolt.Options{ - Timeout: 5 * time.Second, - NoGrowSync: bbolt.DefaultOptions.NoGrowSync, - FreelistType: bbolt.DefaultOptions.FreelistType, - }, - }) +func (h *handler) isClosed() bool { + return h.listener == nil && h.server == nil +} + +func (h *handler) getCaches() caches { + return h.caches +} + +func (h *handler) setCaches(caches caches) { + if h.caches != nil { + h.caches.close() + } + h.caches = caches } // GET /_apis/artifactcache/cache -func (h *Handler) find(w http.ResponseWriter, r *http.Request, params httprouter.Params) { +func (h *handler) find(w http.ResponseWriter, r *http.Request, params httprouter.Params) { rundata := runDataFromHeaders(r) - repo, err := h.validateMac(rundata) + repo, err := h.caches.validateMac(rundata) if err != nil { h.responseJSON(w, r, 403, err) return @@ -174,16 +176,11 @@ func (h *Handler) find(w http.ResponseWriter, r *http.Request, params httprouter } version := r.URL.Query().Get("version") - db, err := h.openDB() - if err != nil { - h.responseJSON(w, r, 500, err) - return - } - defer db.Close() + db := h.caches.getDB() - cache, err := findCache(db, repo, keys, version) + cache, err := findCacheWithIsolationKeyFallback(db, repo, keys, version, rundata.WriteIsolationKey) if err != nil { - h.responseJSON(w, r, 500, err) + h.responseFatalJSON(w, r, err) return } if cache == nil { @@ -191,7 +188,7 @@ func (h *Handler) find(w http.ResponseWriter, r *http.Request, params httprouter return } - if ok, err := h.storage.Exist(cache.ID); err != nil { + if ok, err := h.caches.exist(cache.ID); err != nil { h.responseJSON(w, r, 500, err) return } else if !ok { @@ -208,9 +205,9 @@ func (h *Handler) find(w http.ResponseWriter, r *http.Request, params httprouter } // POST /_apis/artifactcache/caches -func (h *Handler) reserve(w http.ResponseWriter, r *http.Request, params httprouter.Params) { +func (h *handler) reserve(w http.ResponseWriter, r *http.Request, params httprouter.Params) { rundata := runDataFromHeaders(r) - repo, err := h.validateMac(rundata) + repo, err := h.caches.validateMac(rundata) if err != nil { h.responseJSON(w, r, 403, err) return @@ -225,17 +222,13 @@ func (h *Handler) reserve(w http.ResponseWriter, r *http.Request, params httprou api.Key = strings.ToLower(api.Key) cache := api.ToCache() - db, err := h.openDB() - if err != nil { - h.responseJSON(w, r, 500, err) - return - } - defer db.Close() + db := h.caches.getDB() now := time.Now().Unix() cache.CreatedAt = now cache.UsedAt = now cache.Repo = repo + cache.WriteIsolationKey = rundata.WriteIsolationKey if err := insertCache(db, cache); err != nil { h.responseJSON(w, r, 500, err) return @@ -246,9 +239,9 @@ func (h *Handler) reserve(w http.ResponseWriter, r *http.Request, params httprou } // PATCH /_apis/artifactcache/caches/:id -func (h *Handler) upload(w http.ResponseWriter, r *http.Request, params httprouter.Params) { +func (h *handler) upload(w http.ResponseWriter, r *http.Request, params httprouter.Params) { rundata := runDataFromHeaders(r) - repo, err := h.validateMac(rundata) + repo, err := h.caches.validateMac(rundata) if err != nil { h.responseJSON(w, r, 403, err) return @@ -260,19 +253,18 @@ func (h *Handler) upload(w http.ResponseWriter, r *http.Request, params httprout return } - cache, err := h.readCache(id) + cache, err := h.caches.readCache(id, repo) if err != nil { if errors.Is(err, bolthold.ErrNotFound) { h.responseJSON(w, r, 404, fmt.Errorf("cache %d: not reserved", id)) return } - h.responseJSON(w, r, 500, fmt.Errorf("cache Get: %w", err)) + h.responseFatalJSON(w, r, fmt.Errorf("cache Get: %w", err)) return } - // Should not happen - if cache.Repo != repo { - h.responseJSON(w, r, 500, fmt.Errorf("cache repo is not valid")) + if cache.WriteIsolationKey != rundata.WriteIsolationKey { + h.responseJSON(w, r, 403, fmt.Errorf("cache authorized for write isolation %q, but attempting to operate on %q", rundata.WriteIsolationKey, cache.WriteIsolationKey)) return } @@ -285,11 +277,11 @@ func (h *Handler) upload(w http.ResponseWriter, r *http.Request, params httprout h.responseJSON(w, r, 400, fmt.Errorf("cache parseContentRange(%s): %w", r.Header.Get("Content-Range"), err)) return } - if err := h.storage.Write(cache.ID, start, r.Body); err != nil { + if err := h.caches.write(cache.ID, start, r.Body); err != nil { h.responseJSON(w, r, 500, fmt.Errorf("cache storage.Write: %w", err)) return } - if err := h.useCache(id); err != nil { + if err := h.caches.useCache(id); err != nil { h.responseJSON(w, r, 500, fmt.Errorf("cache useCache: %w", err)) return } @@ -297,9 +289,9 @@ func (h *Handler) upload(w http.ResponseWriter, r *http.Request, params httprout } // POST /_apis/artifactcache/caches/:id -func (h *Handler) commit(w http.ResponseWriter, r *http.Request, params httprouter.Params) { +func (h *handler) commit(w http.ResponseWriter, r *http.Request, params httprouter.Params) { rundata := runDataFromHeaders(r) - repo, err := h.validateMac(rundata) + repo, err := h.caches.validateMac(rundata) if err != nil { h.responseJSON(w, r, 403, err) return @@ -311,19 +303,18 @@ func (h *Handler) commit(w http.ResponseWriter, r *http.Request, params httprout return } - cache, err := h.readCache(id) + cache, err := h.caches.readCache(id, repo) if err != nil { if errors.Is(err, bolthold.ErrNotFound) { h.responseJSON(w, r, 404, fmt.Errorf("cache %d: not reserved", id)) return } - h.responseJSON(w, r, 500, fmt.Errorf("cache Get: %w", err)) + h.responseFatalJSON(w, r, fmt.Errorf("cache Get: %w", err)) return } - // Should not happen - if cache.Repo != repo { - h.responseJSON(w, r, 500, fmt.Errorf("cache repo is not valid")) + if cache.WriteIsolationKey != rundata.WriteIsolationKey { + h.responseJSON(w, r, 403, fmt.Errorf("cache authorized for write isolation %q, but attempting to operate on %q", rundata.WriteIsolationKey, cache.WriteIsolationKey)) return } @@ -332,20 +323,15 @@ func (h *Handler) commit(w http.ResponseWriter, r *http.Request, params httprout return } - size, err := h.storage.Commit(cache.ID, cache.Size) + size, err := h.caches.commit(cache.ID, cache.Size) if err != nil { - h.responseJSON(w, r, 500, err) + h.responseJSON(w, r, 500, fmt.Errorf("commit(%v): %w", cache.ID, err)) return } // write real size back to cache, it may be different from the current value when the request doesn't specify it. cache.Size = size - db, err := h.openDB() - if err != nil { - h.responseJSON(w, r, 500, err) - return - } - defer db.Close() + db := h.caches.getDB() cache.Complete = true if err := db.Update(cache.ID, cache); err != nil { @@ -357,9 +343,9 @@ func (h *Handler) commit(w http.ResponseWriter, r *http.Request, params httprout } // GET /_apis/artifactcache/artifacts/:id -func (h *Handler) get(w http.ResponseWriter, r *http.Request, params httprouter.Params) { +func (h *handler) get(w http.ResponseWriter, r *http.Request, params httprouter.Params) { rundata := runDataFromHeaders(r) - repo, err := h.validateMac(rundata) + repo, err := h.caches.validateMac(rundata) if err != nil { h.responseJSON(w, r, 403, err) return @@ -371,33 +357,33 @@ func (h *Handler) get(w http.ResponseWriter, r *http.Request, params httprouter. return } - cache, err := h.readCache(id) + cache, err := h.caches.readCache(id, repo) if err != nil { if errors.Is(err, bolthold.ErrNotFound) { h.responseJSON(w, r, 404, fmt.Errorf("cache %d: not reserved", id)) return } - h.responseJSON(w, r, 500, fmt.Errorf("cache Get: %w", err)) + h.responseFatalJSON(w, r, fmt.Errorf("cache Get: %w", err)) return } - // Should not happen - if cache.Repo != repo { - h.responseJSON(w, r, 500, fmt.Errorf("cache repo is not valid")) + // reads permitted against caches w/ the same isolation key, or no isolation key + if cache.WriteIsolationKey != rundata.WriteIsolationKey && cache.WriteIsolationKey != "" { + h.responseJSON(w, r, 403, fmt.Errorf("cache authorized for write isolation %q, but attempting to operate on %q", rundata.WriteIsolationKey, cache.WriteIsolationKey)) return } - if err := h.useCache(id); err != nil { + if err := h.caches.useCache(id); err != nil { h.responseJSON(w, r, 500, fmt.Errorf("cache useCache: %w", err)) return } - h.storage.Serve(w, r, id) + h.caches.serve(w, r, id) } // POST /_apis/artifactcache/clean -func (h *Handler) clean(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { +func (h *handler) clean(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { rundata := runDataFromHeaders(r) - _, err := h.validateMac(rundata) + _, err := h.caches.validateMac(rundata) if err != nil { h.responseJSON(w, r, 403, err) return @@ -408,204 +394,20 @@ func (h *Handler) clean(w http.ResponseWriter, r *http.Request, _ httprouter.Par h.responseJSON(w, r, 200) } -func (h *Handler) middleware(handler httprouter.Handle) httprouter.Handle { +func (h *handler) middleware(handler httprouter.Handle) httprouter.Handle { return func(w http.ResponseWriter, r *http.Request, params httprouter.Params) { h.logger.Debugf("%s %s", r.Method, r.RequestURI) handler(w, r, params) - go h.gcCache() + go h.caches.gcCache() } } -// if not found, return (nil, nil) instead of an error. -func findCache(db *bolthold.Store, repo string, keys []string, version string) (*Cache, error) { - cache := &Cache{} - for _, prefix := range keys { - // if a key in the list matches exactly, don't return partial matches - if err := db.FindOne(cache, - bolthold.Where("Repo").Eq(repo). - And("Key").Eq(prefix). - And("Version").Eq(version). - And("Complete").Eq(true). - SortBy("CreatedAt").Reverse()); err == nil || !errors.Is(err, bolthold.ErrNotFound) { - if err != nil { - return nil, fmt.Errorf("find cache: %w", err) - } - return cache, nil - } - prefixPattern := fmt.Sprintf("^%s", regexp.QuoteMeta(prefix)) - re, err := regexp.Compile(prefixPattern) - if err != nil { - continue - } - if err := db.FindOne(cache, - bolthold.Where("Repo").Eq(repo). - And("Key").RegExp(re). - And("Version").Eq(version). - And("Complete").Eq(true). - SortBy("CreatedAt").Reverse()); err != nil { - if errors.Is(err, bolthold.ErrNotFound) { - continue - } - return nil, fmt.Errorf("find cache: %w", err) - } - return cache, nil - } - return nil, nil +func (h *handler) responseFatalJSON(w http.ResponseWriter, r *http.Request, err error) { + h.responseJSON(w, r, 500, err) + fatal(h.logger, err) } -func insertCache(db *bolthold.Store, cache *Cache) error { - if err := db.Insert(bolthold.NextSequence(), cache); err != nil { - return fmt.Errorf("insert cache: %w", err) - } - // write back id to db - if err := db.Update(cache.ID, cache); err != nil { - return fmt.Errorf("write back id to db: %w", err) - } - return nil -} - -func (h *Handler) readCache(id uint64) (*Cache, error) { - db, err := h.openDB() - if err != nil { - return nil, err - } - defer db.Close() - cache := &Cache{} - if err := db.Get(id, cache); err != nil { - return nil, err - } - return cache, nil -} - -func (h *Handler) useCache(id uint64) error { - db, err := h.openDB() - if err != nil { - return err - } - defer db.Close() - cache := &Cache{} - if err := db.Get(id, cache); err != nil { - return err - } - cache.UsedAt = time.Now().Unix() - return db.Update(cache.ID, cache) -} - -const ( - keepUsed = 30 * 24 * time.Hour - keepUnused = 7 * 24 * time.Hour - keepTemp = 5 * time.Minute - keepOld = 5 * time.Minute -) - -func (h *Handler) gcCache() { - if h.gcing.Load() { - return - } - if !h.gcing.CompareAndSwap(false, true) { - return - } - defer h.gcing.Store(false) - - if time.Since(h.gcAt) < time.Hour { - h.logger.Debugf("skip gc: %v", h.gcAt.String()) - return - } - h.gcAt = time.Now() - h.logger.Debugf("gc: %v", h.gcAt.String()) - - db, err := h.openDB() - if err != nil { - return - } - defer db.Close() - - // Remove the caches which are not completed for a while, they are most likely to be broken. - var caches []*Cache - if err := db.Find(&caches, bolthold. - Where("UsedAt").Lt(time.Now().Add(-keepTemp).Unix()). - And("Complete").Eq(false), - ); err != nil { - h.logger.Warnf("find caches: %v", err) - } else { - for _, cache := range caches { - h.storage.Remove(cache.ID) - if err := db.Delete(cache.ID, cache); err != nil { - h.logger.Warnf("delete cache: %v", err) - continue - } - h.logger.Infof("deleted cache: %+v", cache) - } - } - - // Remove the old caches which have not been used recently. - caches = caches[:0] - if err := db.Find(&caches, bolthold. - Where("UsedAt").Lt(time.Now().Add(-keepUnused).Unix()), - ); err != nil { - h.logger.Warnf("find caches: %v", err) - } else { - for _, cache := range caches { - h.storage.Remove(cache.ID) - if err := db.Delete(cache.ID, cache); err != nil { - h.logger.Warnf("delete cache: %v", err) - continue - } - h.logger.Infof("deleted cache: %+v", cache) - } - } - - // Remove the old caches which are too old. - caches = caches[:0] - if err := db.Find(&caches, bolthold. - Where("CreatedAt").Lt(time.Now().Add(-keepUsed).Unix()), - ); err != nil { - h.logger.Warnf("find caches: %v", err) - } else { - for _, cache := range caches { - h.storage.Remove(cache.ID) - if err := db.Delete(cache.ID, cache); err != nil { - h.logger.Warnf("delete cache: %v", err) - continue - } - h.logger.Infof("deleted cache: %+v", cache) - } - } - - // Remove the old caches with the same key and version, keep the latest one. - // Also keep the olds which have been used recently for a while in case of the cache is still in use. - if results, err := db.FindAggregate( - &Cache{}, - bolthold.Where("Complete").Eq(true), - "Key", "Version", - ); err != nil { - h.logger.Warnf("find aggregate caches: %v", err) - } else { - for _, result := range results { - if result.Count() <= 1 { - continue - } - result.Sort("CreatedAt") - caches = caches[:0] - result.Reduction(&caches) - for _, cache := range caches[:len(caches)-1] { - if time.Since(time.Unix(cache.UsedAt, 0)) < keepOld { - // Keep it since it has been used recently, even if it's old. - // Or it could break downloading in process. - continue - } - h.storage.Remove(cache.ID) - if err := db.Delete(cache.ID, cache); err != nil { - h.logger.Warnf("delete cache: %v", err) - continue - } - h.logger.Infof("deleted cache: %+v", cache) - } - } - } -} - -func (h *Handler) responseJSON(w http.ResponseWriter, r *http.Request, code int, v ...any) { +func (h *handler) responseJSON(w http.ResponseWriter, r *http.Request, code int, v ...any) { w.Header().Set("Content-Type", "application/json; charset=utf-8") var data []byte if len(v) == 0 || v[0] == nil { @@ -638,11 +440,20 @@ func parseContentRange(s string) (uint64, uint64, error) { return start, stop, nil } -func runDataFromHeaders(r *http.Request) cacheproxy.RunData { - return cacheproxy.RunData{ +type RunData struct { + RepositoryFullName string + RunNumber string + Timestamp string + RepositoryMAC string + WriteIsolationKey string +} + +func runDataFromHeaders(r *http.Request) RunData { + return RunData{ RepositoryFullName: r.Header.Get("Forgejo-Cache-Repo"), RunNumber: r.Header.Get("Forgejo-Cache-RunNumber"), Timestamp: r.Header.Get("Forgejo-Cache-Timestamp"), RepositoryMAC: r.Header.Get("Forgejo-Cache-MAC"), + WriteIsolationKey: r.Header.Get("Forgejo-Cache-WriteIsolationKey"), } } diff --git a/act/artifactcache/handler_test.go b/act/artifactcache/handler_test.go index b8ff6769..91754f77 100644 --- a/act/artifactcache/handler_test.go +++ b/act/artifactcache/handler_test.go @@ -4,48 +4,71 @@ import ( "bytes" "crypto/rand" "encoding/json" + "errors" "fmt" "io" "net/http" + "net/http/httptest" + "os" + "os/signal" "path/filepath" "strings" + "syscall" "testing" "time" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" + "code.forgejo.org/forgejo/runner/v11/testutils" + "github.com/julienschmidt/httprouter" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/mock" "github.com/timshannon/bolthold" "go.etcd.io/bbolt" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) const ( cacheRepo = "testuser/repo" cacheRunnum = "1" cacheTimestamp = "0" - cacheMac = "c13854dd1ac599d1d61680cd93c26b77ba0ee10f374a3408bcaea82f38ca1865" + cacheMac = "bc2e9167f9e310baebcead390937264e4c0b21d2fdd49f5b9470d54406099360" ) var handlerExternalURL string type AuthHeaderTransport struct { - T http.RoundTripper + T http.RoundTripper + WriteIsolationKey string + OverrideDefaultMac string } func (t *AuthHeaderTransport) RoundTrip(req *http.Request) (*http.Response, error) { req.Header.Set("Forgejo-Cache-Repo", cacheRepo) req.Header.Set("Forgejo-Cache-RunNumber", cacheRunnum) req.Header.Set("Forgejo-Cache-Timestamp", cacheTimestamp) - req.Header.Set("Forgejo-Cache-MAC", cacheMac) + if t.OverrideDefaultMac != "" { + req.Header.Set("Forgejo-Cache-MAC", t.OverrideDefaultMac) + } else { + req.Header.Set("Forgejo-Cache-MAC", cacheMac) + } req.Header.Set("Forgejo-Cache-Host", handlerExternalURL) + if t.WriteIsolationKey != "" { + req.Header.Set("Forgejo-Cache-WriteIsolationKey", t.WriteIsolationKey) + } return t.T.RoundTrip(req) } var ( - httpClientTransport = AuthHeaderTransport{http.DefaultTransport} + httpClientTransport = AuthHeaderTransport{T: http.DefaultTransport} httpClient = http.Client{Transport: &httpClientTransport} ) func TestHandler(t *testing.T) { + defer testutils.MockVariable(&fatal, func(_ logrus.FieldLogger, err error) { + t.Fatalf("unexpected call to fatal(%v)", err) + })() + dir := filepath.Join(t.TempDir(), "artifactcache") handler, err := StartHandler(dir, "", 0, "secret", nil) require.NoError(t, err) @@ -54,10 +77,8 @@ func TestHandler(t *testing.T) { base := fmt.Sprintf("%s%s", handler.ExternalURL(), urlBase) defer func() { - t.Run("inpect db", func(t *testing.T) { - db, err := handler.openDB() - require.NoError(t, err) - defer db.Close() + t.Run("inspect db", func(t *testing.T) { + db := handler.getCaches().getDB() require.NoError(t, db.Bolt().View(func(tx *bbolt.Tx) error { return tx.Bucket([]byte("Cache")).ForEach(func(k, v []byte) error { t.Logf("%s: %s", k, v) @@ -67,8 +88,7 @@ func TestHandler(t *testing.T) { }) t.Run("close", func(t *testing.T) { require.NoError(t, handler.Close()) - assert.Nil(t, handler.server) - assert.Nil(t, handler.listener) + assert.True(t, handler.isClosed()) _, err := httpClient.Post(fmt.Sprintf("%s/caches/%d", base, 1), "", nil) assert.Error(t, err) }) @@ -88,7 +108,7 @@ func TestHandler(t *testing.T) { content := make([]byte, 100) _, err := rand.Read(content) require.NoError(t, err) - uploadCacheNormally(t, base, key, version, content) + uploadCacheNormally(t, base, key, version, "", content) }) t.Run("clean", func(t *testing.T) { @@ -380,7 +400,7 @@ func TestHandler(t *testing.T) { _, err := rand.Read(content) require.NoError(t, err) - uploadCacheNormally(t, base, key, version, content) + uploadCacheNormally(t, base, key, version, "", content) // Perform the request with the custom `httpClient` which will send correct MAC data resp, err := httpClient.Get(fmt.Sprintf("%s/cache?keys=%s&version=%s", base, key, version)) @@ -416,7 +436,7 @@ func TestHandler(t *testing.T) { for i := range contents { _, err := rand.Read(contents[i]) require.NoError(t, err) - uploadCacheNormally(t, base, keys[i], version, contents[i]) + uploadCacheNormally(t, base, keys[i], version, "", contents[i]) time.Sleep(time.Second) // ensure CreatedAt of caches are different } @@ -454,13 +474,98 @@ func TestHandler(t *testing.T) { assert.Equal(t, contents[except], content) }) + t.Run("find can't match without WriteIsolationKey match", func(t *testing.T) { + defer func() { httpClientTransport.WriteIsolationKey = "" }() + + version := "c19da02a2bd7e77277f1ac29ab45c09b7d46a4ee758284e26bb3045ad11d9d20" + key := strings.ToLower(t.Name()) + + uploadCacheNormally(t, base, key, version, "TestWriteKey", make([]byte, 64)) + + func() { + defer overrideWriteIsolationKey("AnotherTestWriteKey")() + resp, err := httpClient.Get(fmt.Sprintf("%s/cache?keys=%s&version=%s", base, key, version)) + require.NoError(t, err) + require.Equal(t, 204, resp.StatusCode) + }() + + { + resp, err := httpClient.Get(fmt.Sprintf("%s/cache?keys=%s&version=%s", base, key, version)) + require.NoError(t, err) + require.Equal(t, 204, resp.StatusCode) + } + + func() { + defer overrideWriteIsolationKey("TestWriteKey")() + resp, err := httpClient.Get(fmt.Sprintf("%s/cache?keys=%s&version=%s", base, key, version)) + require.NoError(t, err) + require.Equal(t, 200, resp.StatusCode) + }() + }) + + t.Run("find prefers WriteIsolationKey match", func(t *testing.T) { + version := "c19da02a2bd7e77277f1ac29ab45c09b7d46a4ee758284e26bb3045ad11d9d21" + key := strings.ToLower(t.Name()) + + // Between two values with the same `key`... + uploadCacheNormally(t, base, key, version, "TestWriteKey", make([]byte, 64)) + uploadCacheNormally(t, base, key, version, "", make([]byte, 128)) + + // We should read the value with the matching WriteIsolationKey from the cache... + func() { + defer overrideWriteIsolationKey("TestWriteKey")() + + resp, err := httpClient.Get(fmt.Sprintf("%s/cache?keys=%s&version=%s", base, key, version)) + require.NoError(t, err) + require.Equal(t, 200, resp.StatusCode) + + got := struct { + ArchiveLocation string `json:"archiveLocation"` + }{} + require.NoError(t, json.NewDecoder(resp.Body).Decode(&got)) + contentResp, err := httpClient.Get(got.ArchiveLocation) + require.NoError(t, err) + require.Equal(t, 200, contentResp.StatusCode) + content, err := io.ReadAll(contentResp.Body) + require.NoError(t, err) + // Which we finally check matches the correct WriteIsolationKey's content here. + assert.Equal(t, make([]byte, 64), content) + }() + }) + + t.Run("find falls back if matching WriteIsolationKey not available", func(t *testing.T) { + version := "c19da02a2bd7e77277f1ac29ab45c09b7d46a4ee758284e26bb3045ad11d9d21" + key := strings.ToLower(t.Name()) + + uploadCacheNormally(t, base, key, version, "", make([]byte, 128)) + + func() { + defer overrideWriteIsolationKey("TestWriteKey")() + + resp, err := httpClient.Get(fmt.Sprintf("%s/cache?keys=%s&version=%s", base, key, version)) + require.NoError(t, err) + require.Equal(t, 200, resp.StatusCode) + + got := struct { + ArchiveLocation string `json:"archiveLocation"` + }{} + require.NoError(t, json.NewDecoder(resp.Body).Decode(&got)) + contentResp, err := httpClient.Get(got.ArchiveLocation) + require.NoError(t, err) + require.Equal(t, 200, contentResp.StatusCode) + content, err := io.ReadAll(contentResp.Body) + require.NoError(t, err) + assert.Equal(t, make([]byte, 128), content) + }() + }) + t.Run("case insensitive", func(t *testing.T) { version := "c19da02a2bd7e77277f1ac29ab45c09b7d46a4ee758284e26bb3045ad11d9d20" key := strings.ToLower(t.Name()) content := make([]byte, 100) _, err := rand.Read(content) require.NoError(t, err) - uploadCacheNormally(t, base, key+"_ABC", version, content) + uploadCacheNormally(t, base, key+"_ABC", version, "", content) { reqKey := key + "_aBc" @@ -494,7 +599,7 @@ func TestHandler(t *testing.T) { for i := range contents { _, err := rand.Read(contents[i]) require.NoError(t, err) - uploadCacheNormally(t, base, keys[i], version, contents[i]) + uploadCacheNormally(t, base, keys[i], version, "", contents[i]) time.Sleep(time.Second) // ensure CreatedAt of caches are different } @@ -545,7 +650,7 @@ func TestHandler(t *testing.T) { for i := range contents { _, err := rand.Read(contents[i]) require.NoError(t, err) - uploadCacheNormally(t, base, keys[i], version, contents[i]) + uploadCacheNormally(t, base, keys[i], version, "", contents[i]) time.Sleep(time.Second) // ensure CreatedAt of caches are different } @@ -581,9 +686,166 @@ func TestHandler(t *testing.T) { require.NoError(t, err) assert.Equal(t, contents[expect], content) }) + + t.Run("upload across WriteIsolationKey", func(t *testing.T) { + defer overrideWriteIsolationKey("CorrectKey")() + + key := strings.ToLower(t.Name()) + version := "c19da02a2bd7e77277f1ac29ab45c09b7d46a4ee758284e26bb3045ad11d9d20" + content := make([]byte, 256) + + var id uint64 + // reserve + { + body, err := json.Marshal(&Request{ + Key: key, + Version: version, + Size: int64(len(content)), + }) + require.NoError(t, err) + resp, err := httpClient.Post(fmt.Sprintf("%s/caches", base), "application/json", bytes.NewReader(body)) + require.NoError(t, err) + assert.Equal(t, 200, resp.StatusCode) + + got := struct { + CacheID uint64 `json:"cacheId"` + }{} + require.NoError(t, json.NewDecoder(resp.Body).Decode(&got)) + id = got.CacheID + } + // upload, but with the incorrect write isolation key relative to the cache obj created + func() { + defer overrideWriteIsolationKey("WrongKey")() + req, err := http.NewRequest(http.MethodPatch, + fmt.Sprintf("%s/caches/%d", base, id), bytes.NewReader(content)) + require.NoError(t, err) + req.Header.Set("Content-Type", "application/octet-stream") + req.Header.Set("Content-Range", "bytes 0-99/*") + resp, err := httpClient.Do(req) + require.NoError(t, err) + assert.Equal(t, 403, resp.StatusCode) + }() + }) + + t.Run("commit across WriteIsolationKey", func(t *testing.T) { + defer overrideWriteIsolationKey("CorrectKey")() + + key := strings.ToLower(t.Name()) + version := "c19da02a2bd7e77277f1ac29ab45c09b7d46a4ee758284e26bb3045ad11d9d20" + content := make([]byte, 256) + + var id uint64 + // reserve + { + body, err := json.Marshal(&Request{ + Key: key, + Version: version, + Size: int64(len(content)), + }) + require.NoError(t, err) + resp, err := httpClient.Post(fmt.Sprintf("%s/caches", base), "application/json", bytes.NewReader(body)) + require.NoError(t, err) + assert.Equal(t, 200, resp.StatusCode) + + got := struct { + CacheID uint64 `json:"cacheId"` + }{} + require.NoError(t, json.NewDecoder(resp.Body).Decode(&got)) + id = got.CacheID + } + // upload + { + req, err := http.NewRequest(http.MethodPatch, + fmt.Sprintf("%s/caches/%d", base, id), bytes.NewReader(content)) + require.NoError(t, err) + req.Header.Set("Content-Type", "application/octet-stream") + req.Header.Set("Content-Range", "bytes 0-99/*") + resp, err := httpClient.Do(req) + require.NoError(t, err) + assert.Equal(t, 200, resp.StatusCode) + } + // commit, but with the incorrect write isolation key relative to the cache obj created + func() { + defer overrideWriteIsolationKey("WrongKey")() + resp, err := httpClient.Post(fmt.Sprintf("%s/caches/%d", base, id), "", nil) + require.NoError(t, err) + assert.Equal(t, 403, resp.StatusCode) + }() + }) + + t.Run("get across WriteIsolationKey", func(t *testing.T) { + defer func() { httpClientTransport.WriteIsolationKey = "" }() + + version := "c19da02a2bd7e77277f1ac29ab45c09b7d46a4ee758284e26bb3045ad11d9d21" + key := strings.ToLower(t.Name()) + uploadCacheNormally(t, base, key, version, "", make([]byte, 128)) + keyIsolated := strings.ToLower(t.Name()) + "_isolated" + uploadCacheNormally(t, base, keyIsolated, version, "CorrectKey", make([]byte, 128)) + + // Perform the 'get' without the right WriteIsolationKey for the cache entry... should be OK for `key` since it + // was written with WriteIsolationKey "" meaning it is available for non-isolated access + func() { + defer overrideWriteIsolationKey("WhoopsWrongKey")() + + resp, err := httpClient.Get(fmt.Sprintf("%s/cache?keys=%s&version=%s", base, key, version)) + require.NoError(t, err) + require.Equal(t, 200, resp.StatusCode) + got := struct { + ArchiveLocation string `json:"archiveLocation"` + }{} + require.NoError(t, json.NewDecoder(resp.Body).Decode(&got)) + + contentResp, err := httpClient.Get(got.ArchiveLocation) + require.NoError(t, err) + require.Equal(t, 200, contentResp.StatusCode) + httpClientTransport.WriteIsolationKey = "CorrectKey" // reset for next find + }() + + // Perform the 'get' without the right WriteIsolationKey for the cache entry... should be 403 for `keyIsolated` + // because it was written with a different WriteIsolationKey. + { + got := func() struct { + ArchiveLocation string `json:"archiveLocation"` + } { + defer overrideWriteIsolationKey("CorrectKey")() // for test purposes make the `find` successful... + resp, err := httpClient.Get(fmt.Sprintf("%s/cache?keys=%s&version=%s", base, keyIsolated, version)) + require.NoError(t, err) + require.Equal(t, 200, resp.StatusCode) + got := struct { + ArchiveLocation string `json:"archiveLocation"` + }{} + require.NoError(t, json.NewDecoder(resp.Body).Decode(&got)) + return got + }() + + func() { + defer overrideWriteIsolationKey("WhoopsWrongKey")() // but then access w/ the wrong key for `get` + contentResp, err := httpClient.Get(got.ArchiveLocation) + require.NoError(t, err) + require.Equal(t, 403, contentResp.StatusCode) + }() + } + }) } -func uploadCacheNormally(t *testing.T, base, key, version string, content []byte) { +func overrideWriteIsolationKey(writeIsolationKey string) func() { + originalWriteIsolationKey := httpClientTransport.WriteIsolationKey + originalMac := httpClientTransport.OverrideDefaultMac + + httpClientTransport.WriteIsolationKey = writeIsolationKey + httpClientTransport.OverrideDefaultMac = ComputeMac("secret", cacheRepo, cacheRunnum, cacheTimestamp, httpClientTransport.WriteIsolationKey) + + return func() { + httpClientTransport.WriteIsolationKey = originalWriteIsolationKey + httpClientTransport.OverrideDefaultMac = originalMac + } +} + +func uploadCacheNormally(t *testing.T, base, key, version, writeIsolationKey string, content []byte) { + if writeIsolationKey != "" { + defer overrideWriteIsolationKey(writeIsolationKey)() + } + var id uint64 { body, err := json.Marshal(&Request{ @@ -642,6 +904,130 @@ func uploadCacheNormally(t *testing.T, base, key, version string, content []byte } } +func TestHandlerAPIFatalErrors(t *testing.T) { + for _, testCase := range []struct { + name string + prepare func(message string) func() + caches func(t *testing.T, message string) caches + call func(t *testing.T, handler Handler, w http.ResponseWriter) + }{ + { + name: "find", + prepare: func(message string) func() { + return testutils.MockVariable(&findCacheWithIsolationKeyFallback, func(db *bolthold.Store, repo string, keys []string, version, writeIsolationKey string) (*Cache, error) { + return nil, errors.New(message) + }) + }, + caches: func(t *testing.T, message string) caches { + caches, err := newCaches(t.TempDir(), "secret", logrus.New()) + require.NoError(t, err) + return caches + }, + call: func(t *testing.T, handler Handler, w http.ResponseWriter) { + keyOne := "ONE" + req, err := http.NewRequest("GET", fmt.Sprintf("http://example.com/cache?keys=%s", keyOne), nil) + require.NoError(t, err) + req.Header.Set("Forgejo-Cache-Repo", cacheRepo) + req.Header.Set("Forgejo-Cache-RunNumber", cacheRunnum) + req.Header.Set("Forgejo-Cache-Timestamp", cacheTimestamp) + req.Header.Set("Forgejo-Cache-MAC", cacheMac) + req.Header.Set("Forgejo-Cache-Host", "http://example.com") + handler.find(w, req, nil) + }, + }, + { + name: "upload", + caches: func(t *testing.T, message string) caches { + caches := newMockCaches(t) + caches.On("close").Return() + caches.On("validateMac", RunData{}).Return(cacheRepo, nil) + caches.On("readCache", mock.Anything, mock.Anything).Return(nil, errors.New(message)) + return caches + }, + call: func(t *testing.T, handler Handler, w http.ResponseWriter) { + id := 1234 + req, err := http.NewRequest("PATCH", fmt.Sprintf("http://example.com/caches/%d", id), nil) + require.NoError(t, err) + handler.upload(w, req, httprouter.Params{ + httprouter.Param{Key: "id", Value: fmt.Sprintf("%d", id)}, + }) + }, + }, + { + name: "commit", + caches: func(t *testing.T, message string) caches { + caches := newMockCaches(t) + caches.On("close").Return() + caches.On("validateMac", RunData{}).Return(cacheRepo, nil) + caches.On("readCache", mock.Anything, mock.Anything).Return(nil, errors.New(message)) + return caches + }, + call: func(t *testing.T, handler Handler, w http.ResponseWriter) { + id := 1234 + req, err := http.NewRequest("POST", fmt.Sprintf("http://example.com/caches/%d", id), nil) + require.NoError(t, err) + handler.commit(w, req, httprouter.Params{ + httprouter.Param{Key: "id", Value: fmt.Sprintf("%d", id)}, + }) + }, + }, + { + name: "get", + caches: func(t *testing.T, message string) caches { + caches := newMockCaches(t) + caches.On("close").Return() + caches.On("validateMac", RunData{}).Return(cacheRepo, nil) + caches.On("readCache", mock.Anything, mock.Anything).Return(nil, errors.New(message)) + return caches + }, + call: func(t *testing.T, handler Handler, w http.ResponseWriter) { + id := 1234 + req, err := http.NewRequest("GET", fmt.Sprintf("http://example.com/artifacts/%d", id), nil) + require.NoError(t, err) + handler.get(w, req, httprouter.Params{ + httprouter.Param{Key: "id", Value: fmt.Sprintf("%d", id)}, + }) + }, + }, + // it currently is a noop + //{ + //name: "clean", + //} + } { + t.Run(testCase.name, func(t *testing.T) { + message := "ERROR MESSAGE" + if testCase.prepare != nil { + defer testCase.prepare(message)() + } + + fatalMessage := "" + defer testutils.MockVariable(&fatal, func(_ logrus.FieldLogger, err error) { + fatalMessage = err.Error() + })() + + assertFatalMessage := func(t *testing.T, expected string) { + t.Helper() + assert.Contains(t, fatalMessage, expected) + } + + dir := filepath.Join(t.TempDir(), "artifactcache") + handler, err := StartHandler(dir, "", 0, "secret", nil) + require.NoError(t, err) + defer handler.Close() + + fatalMessage = "" + + caches := testCase.caches(t, message) // doesn't need to be closed because it will be given to handler + handler.setCaches(caches) + + w := httptest.NewRecorder() + testCase.call(t, handler, w) + require.Equal(t, 500, w.Code) + assertFatalMessage(t, message) + }) + } +} + func TestHandler_gcCache(t *testing.T) { dir := filepath.Join(t.TempDir(), "artifactcache") handler, err := StartHandler(dir, "", 0, "", nil) @@ -725,18 +1111,15 @@ func TestHandler_gcCache(t *testing.T) { }, } - db, err := handler.openDB() - require.NoError(t, err) + db := handler.getCaches().getDB() for _, c := range cases { require.NoError(t, insertCache(db, c.Cache)) } - require.NoError(t, db.Close()) - handler.gcAt = time.Time{} // ensure gcCache will not skip - handler.gcCache() + handler.getCaches().setgcAt(time.Time{}) // ensure gcCache will not skip + handler.getCaches().gcCache() - db, err = handler.openDB() - require.NoError(t, err) + db = handler.getCaches().getDB() for i, v := range cases { t.Run(fmt.Sprintf("%d_%s", i, v.Cache.Key), func(t *testing.T) { cache := &Cache{} @@ -748,7 +1131,6 @@ func TestHandler_gcCache(t *testing.T) { } }) } - require.NoError(t, db.Close()) } func TestHandler_ExternalURL(t *testing.T) { @@ -759,8 +1141,7 @@ func TestHandler_ExternalURL(t *testing.T) { assert.Equal(t, handler.ExternalURL(), "http://127.0.0.1:34567") require.NoError(t, handler.Close()) - assert.Nil(t, handler.server) - assert.Nil(t, handler.listener) + assert.True(t, handler.isClosed()) }) t.Run("reports correct URL on IPv6 zero host", func(t *testing.T) { @@ -770,8 +1151,7 @@ func TestHandler_ExternalURL(t *testing.T) { assert.Equal(t, handler.ExternalURL(), "http://[2001:db8::]:34567") require.NoError(t, handler.Close()) - assert.Nil(t, handler.server) - assert.Nil(t, handler.listener) + assert.True(t, handler.isClosed()) }) t.Run("reports correct URL on IPv6", func(t *testing.T) { @@ -781,7 +1161,45 @@ func TestHandler_ExternalURL(t *testing.T) { assert.Equal(t, handler.ExternalURL(), "http://[2001:db8::1:2:3:4]:34567") require.NoError(t, handler.Close()) - assert.Nil(t, handler.server) - assert.Nil(t, handler.listener) + assert.True(t, handler.isClosed()) }) } + +var ( + settleTime = 100 * time.Millisecond + fatalWaitingTime = 30 * time.Second +) + +func waitSig(t *testing.T, c <-chan os.Signal, sig os.Signal) { + t.Helper() + + // Sleep multiple times to give the kernel more tries to + // deliver the signal. + start := time.Now() + timer := time.NewTimer(settleTime / 10) + defer timer.Stop() + for time.Since(start) < fatalWaitingTime { + select { + case s := <-c: + if s == sig { + return + } + t.Fatalf("signal was %v, want %v", s, sig) + case <-timer.C: + timer.Reset(settleTime / 10) + } + } + t.Fatalf("timeout after %v waiting for %v", fatalWaitingTime, sig) +} + +func TestHandler_fatal(t *testing.T) { + c := make(chan os.Signal, 1) + signal.Notify(c, syscall.SIGTERM) + defer signal.Stop(c) + + discard := logrus.New() + discard.Out = io.Discard + fatal(discard, errors.New("fatal error")) + + waitSig(t, c, syscall.SIGTERM) +} diff --git a/act/artifactcache/mac.go b/act/artifactcache/mac.go index 6ed20b01..aa9a7c54 100644 --- a/act/artifactcache/mac.go +++ b/act/artifactcache/mac.go @@ -10,19 +10,17 @@ import ( "errors" "strconv" "time" - - "code.forgejo.org/forgejo/runner/v9/act/cacheproxy" ) var ErrValidation = errors.New("validation error") -func (h *Handler) validateMac(rundata cacheproxy.RunData) (string, error) { +func (c *cachesImpl) validateMac(rundata RunData) (string, error) { // TODO: allow configurable max age if !validateAge(rundata.Timestamp) { return "", ErrValidation } - expectedMAC := computeMac(h.secret, rundata.RepositoryFullName, rundata.RunNumber, rundata.Timestamp) + expectedMAC := ComputeMac(c.secret, rundata.RepositoryFullName, rundata.RunNumber, rundata.Timestamp, rundata.WriteIsolationKey) if hmac.Equal([]byte(expectedMAC), []byte(rundata.RepositoryMAC)) { return rundata.RepositoryFullName, nil } @@ -40,12 +38,14 @@ func validateAge(ts string) bool { return true } -func computeMac(secret, repo, run, ts string) string { +func ComputeMac(secret, repo, run, ts, writeIsolationKey string) string { mac := hmac.New(sha256.New, []byte(secret)) mac.Write([]byte(repo)) mac.Write([]byte(">")) mac.Write([]byte(run)) mac.Write([]byte(">")) mac.Write([]byte(ts)) + mac.Write([]byte(">")) + mac.Write([]byte(writeIsolationKey)) return hex.EncodeToString(mac.Sum(nil)) } diff --git a/act/artifactcache/mac_test.go b/act/artifactcache/mac_test.go index a4b801a9..91607386 100644 --- a/act/artifactcache/mac_test.go +++ b/act/artifactcache/mac_test.go @@ -5,12 +5,11 @@ import ( "testing" "time" - "code.forgejo.org/forgejo/runner/v9/act/cacheproxy" "github.com/stretchr/testify/require" ) func TestMac(t *testing.T) { - handler := &Handler{ + cache := &cachesImpl{ secret: "secret for testing", } @@ -19,15 +18,15 @@ func TestMac(t *testing.T) { run := "1" ts := strconv.FormatInt(time.Now().Unix(), 10) - mac := computeMac(handler.secret, name, run, ts) - rundata := cacheproxy.RunData{ + mac := ComputeMac(cache.secret, name, run, ts, "") + rundata := RunData{ RepositoryFullName: name, RunNumber: run, Timestamp: ts, RepositoryMAC: mac, } - repoName, err := handler.validateMac(rundata) + repoName, err := cache.validateMac(rundata) require.NoError(t, err) require.Equal(t, name, repoName) }) @@ -37,15 +36,15 @@ func TestMac(t *testing.T) { run := "1" ts := "9223372036854775807" // This should last us for a while... - mac := computeMac(handler.secret, name, run, ts) - rundata := cacheproxy.RunData{ + mac := ComputeMac(cache.secret, name, run, ts, "") + rundata := RunData{ RepositoryFullName: name, RunNumber: run, Timestamp: ts, RepositoryMAC: mac, } - _, err := handler.validateMac(rundata) + _, err := cache.validateMac(rundata) require.Error(t, err) }) @@ -54,14 +53,14 @@ func TestMac(t *testing.T) { run := "1" ts := strconv.FormatInt(time.Now().Unix(), 10) - rundata := cacheproxy.RunData{ + rundata := RunData{ RepositoryFullName: name, RunNumber: run, Timestamp: ts, RepositoryMAC: "this is not the right mac :D", } - repoName, err := handler.validateMac(rundata) + repoName, err := cache.validateMac(rundata) require.Error(t, err) require.Equal(t, "", repoName) }) @@ -72,9 +71,12 @@ func TestMac(t *testing.T) { run := "42" ts := "1337" - mac := computeMac(secret, name, run, ts) - expectedMac := "f666f06f917acb7186e152195b2a8c8d36d068ce683454be0878806e08e04f2b" // * Precomputed, anytime the computeMac function changes this needs to be recalculated + mac := ComputeMac(secret, name, run, ts, "") + expectedMac := "4754474b21329e8beadd2b4054aa4be803965d66e710fa1fee091334ed804f29" // * Precomputed, anytime the ComputeMac function changes this needs to be recalculated + require.Equal(t, expectedMac, mac) - require.Equal(t, mac, expectedMac) + mac = ComputeMac(secret, name, run, ts, "refs/pull/12/head") + expectedMac = "9ca8f4cb5e1b083ee8cd215215bc00f379b28511d3ef7930bf054767de34766d" // * Precomputed, anytime the ComputeMac function changes this needs to be recalculated + require.Equal(t, expectedMac, mac) }) } diff --git a/act/artifactcache/mock_caches.go b/act/artifactcache/mock_caches.go new file mode 100644 index 00000000..cadf0b95 --- /dev/null +++ b/act/artifactcache/mock_caches.go @@ -0,0 +1,225 @@ +// Code generated by mockery v2.53.5. DO NOT EDIT. + +package artifactcache + +import ( + http "net/http" + + bolthold "github.com/timshannon/bolthold" + + io "io" + + mock "github.com/stretchr/testify/mock" + + time "time" +) + +// mockCaches is an autogenerated mock type for the caches type +type mockCaches struct { + mock.Mock +} + +// close provides a mock function with no fields +func (_m *mockCaches) close() { + _m.Called() +} + +// commit provides a mock function with given fields: id, size +func (_m *mockCaches) commit(id uint64, size int64) (int64, error) { + ret := _m.Called(id, size) + + if len(ret) == 0 { + panic("no return value specified for commit") + } + + var r0 int64 + var r1 error + if rf, ok := ret.Get(0).(func(uint64, int64) (int64, error)); ok { + return rf(id, size) + } + if rf, ok := ret.Get(0).(func(uint64, int64) int64); ok { + r0 = rf(id, size) + } else { + r0 = ret.Get(0).(int64) + } + + if rf, ok := ret.Get(1).(func(uint64, int64) error); ok { + r1 = rf(id, size) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// exist provides a mock function with given fields: id +func (_m *mockCaches) exist(id uint64) (bool, error) { + ret := _m.Called(id) + + if len(ret) == 0 { + panic("no return value specified for exist") + } + + var r0 bool + var r1 error + if rf, ok := ret.Get(0).(func(uint64) (bool, error)); ok { + return rf(id) + } + if rf, ok := ret.Get(0).(func(uint64) bool); ok { + r0 = rf(id) + } else { + r0 = ret.Get(0).(bool) + } + + if rf, ok := ret.Get(1).(func(uint64) error); ok { + r1 = rf(id) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// gcCache provides a mock function with no fields +func (_m *mockCaches) gcCache() { + _m.Called() +} + +// getDB provides a mock function with no fields +func (_m *mockCaches) getDB() *bolthold.Store { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for getDB") + } + + var r0 *bolthold.Store + if rf, ok := ret.Get(0).(func() *bolthold.Store); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*bolthold.Store) + } + } + + return r0 +} + +// readCache provides a mock function with given fields: id, repo +func (_m *mockCaches) readCache(id uint64, repo string) (*Cache, error) { + ret := _m.Called(id, repo) + + if len(ret) == 0 { + panic("no return value specified for readCache") + } + + var r0 *Cache + var r1 error + if rf, ok := ret.Get(0).(func(uint64, string) (*Cache, error)); ok { + return rf(id, repo) + } + if rf, ok := ret.Get(0).(func(uint64, string) *Cache); ok { + r0 = rf(id, repo) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*Cache) + } + } + + if rf, ok := ret.Get(1).(func(uint64, string) error); ok { + r1 = rf(id, repo) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// serve provides a mock function with given fields: w, r, id +func (_m *mockCaches) serve(w http.ResponseWriter, r *http.Request, id uint64) { + _m.Called(w, r, id) +} + +// setgcAt provides a mock function with given fields: at +func (_m *mockCaches) setgcAt(at time.Time) { + _m.Called(at) +} + +// useCache provides a mock function with given fields: id +func (_m *mockCaches) useCache(id uint64) error { + ret := _m.Called(id) + + if len(ret) == 0 { + panic("no return value specified for useCache") + } + + var r0 error + if rf, ok := ret.Get(0).(func(uint64) error); ok { + r0 = rf(id) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// validateMac provides a mock function with given fields: rundata +func (_m *mockCaches) validateMac(rundata RunData) (string, error) { + ret := _m.Called(rundata) + + if len(ret) == 0 { + panic("no return value specified for validateMac") + } + + var r0 string + var r1 error + if rf, ok := ret.Get(0).(func(RunData) (string, error)); ok { + return rf(rundata) + } + if rf, ok := ret.Get(0).(func(RunData) string); ok { + r0 = rf(rundata) + } else { + r0 = ret.Get(0).(string) + } + + if rf, ok := ret.Get(1).(func(RunData) error); ok { + r1 = rf(rundata) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// write provides a mock function with given fields: id, offset, reader +func (_m *mockCaches) write(id uint64, offset uint64, reader io.Reader) error { + ret := _m.Called(id, offset, reader) + + if len(ret) == 0 { + panic("no return value specified for write") + } + + var r0 error + if rf, ok := ret.Get(0).(func(uint64, uint64, io.Reader) error); ok { + r0 = rf(id, offset, reader) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// newMockCaches creates a new instance of mockCaches. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func newMockCaches(t interface { + mock.TestingT + Cleanup(func()) +}, +) *mockCaches { + mock := &mockCaches{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/act/artifactcache/model.go b/act/artifactcache/model.go index 1c0f855d..cb1af860 100644 --- a/act/artifactcache/model.go +++ b/act/artifactcache/model.go @@ -24,12 +24,13 @@ func (c *Request) ToCache() *Cache { } type Cache struct { - ID uint64 `json:"id" boltholdKey:"ID"` - Repo string `json:"repo" boltholdIndex:"Repo"` - Key string `json:"key" boltholdIndex:"Key"` - Version string `json:"version" boltholdIndex:"Version"` - Size int64 `json:"cacheSize"` - Complete bool `json:"complete" boltholdIndex:"Complete"` - UsedAt int64 `json:"usedAt" boltholdIndex:"UsedAt"` - CreatedAt int64 `json:"createdAt" boltholdIndex:"CreatedAt"` + ID uint64 `json:"id" boltholdKey:"ID"` + Repo string `json:"repo" boltholdIndex:"Repo"` + Key string `json:"key"` + Version string `json:"version"` + Size int64 `json:"cacheSize"` + Complete bool `json:"complete"` + UsedAt int64 `json:"usedAt"` + CreatedAt int64 `json:"createdAt"` + WriteIsolationKey string `json:"writeIsolationKey"` } diff --git a/act/artifactcache/utils.go b/act/artifactcache/utils.go new file mode 100644 index 00000000..37274a4e --- /dev/null +++ b/act/artifactcache/utils.go @@ -0,0 +1,9 @@ +//go:build !windows + +package artifactcache + +import "syscall" + +func suicide() error { + return syscall.Kill(syscall.Getpid(), syscall.SIGTERM) +} diff --git a/act/artifactcache/utils_windows.go b/act/artifactcache/utils_windows.go new file mode 100644 index 00000000..90b0f112 --- /dev/null +++ b/act/artifactcache/utils_windows.go @@ -0,0 +1,14 @@ +//go:build windows + +package artifactcache + +import "syscall" + +func suicide() error { + handle, err := syscall.GetCurrentProcess() + if err != nil { + return err + } + + return syscall.TerminateProcess(handle, uint32(syscall.SIGTERM)) +} diff --git a/act/cacheproxy/handler.go b/act/cacheproxy/handler.go index 79d3f14b..ebaf65e2 100644 --- a/act/cacheproxy/handler.go +++ b/act/cacheproxy/handler.go @@ -4,9 +4,6 @@ package cacheproxy import ( - "crypto/hmac" - "crypto/sha256" - "encoding/hex" "errors" "fmt" "io" @@ -22,7 +19,8 @@ import ( "github.com/julienschmidt/httprouter" "github.com/sirupsen/logrus" - "code.forgejo.org/forgejo/runner/v9/act/common" + "code.forgejo.org/forgejo/runner/v11/act/artifactcache" + "code.forgejo.org/forgejo/runner/v11/act/common" ) const ( @@ -39,31 +37,26 @@ type Handler struct { outboundIP string - cacheServerHost string + cacheServerHost string + cacheProxyHostOverride string cacheSecret string runs sync.Map } -type RunData struct { - RepositoryFullName string - RunNumber string - Timestamp string - RepositoryMAC string -} - -func (h *Handler) CreateRunData(fullName, runNumber, timestamp string) RunData { - mac := computeMac(h.cacheSecret, fullName, runNumber, timestamp) - return RunData{ +func (h *Handler) CreateRunData(fullName, runNumber, timestamp, writeIsolationKey string) artifactcache.RunData { + mac := artifactcache.ComputeMac(h.cacheSecret, fullName, runNumber, timestamp, writeIsolationKey) + return artifactcache.RunData{ RepositoryFullName: fullName, RunNumber: runNumber, Timestamp: timestamp, RepositoryMAC: mac, + WriteIsolationKey: writeIsolationKey, } } -func StartHandler(targetHost, outboundIP string, port uint16, cacheSecret string, logger logrus.FieldLogger) (*Handler, error) { +func StartHandler(targetHost, outboundIP string, port uint16, cacheProxyHostOverride, cacheSecret string, logger logrus.FieldLogger) (*Handler, error) { h := &Handler{} if logger == nil { @@ -71,7 +64,7 @@ func StartHandler(targetHost, outboundIP string, port uint16, cacheSecret string discard.Out = io.Discard logger = discard } - logger = logger.WithField("module", "artifactcache") + logger = logger.WithField("module", "cacheproxy") h.logger = logger h.cacheSecret = cacheSecret @@ -85,10 +78,11 @@ func StartHandler(targetHost, outboundIP string, port uint16, cacheSecret string } h.cacheServerHost = targetHost + h.cacheProxyHostOverride = cacheProxyHostOverride proxy, err := h.newReverseProxy(targetHost) if err != nil { - return nil, fmt.Errorf("unable to set up proxy to target host") + return nil, fmt.Errorf("unable to set up proxy to target host: %v", err) } router := httprouter.New() @@ -140,11 +134,12 @@ func (h *Handler) newReverseProxy(targetHost string) (*httputil.ReverseProxy, er h.logger.Warn(fmt.Sprintf("Tried starting a cache proxy with id %s, which does not exist.", id)) return } - runData := data.(RunData) + runData := data.(artifactcache.RunData) uri := matches[2] r.SetURL(targetURL) r.Out.URL.Path = uri + h.logger.Debugf("proxy req %s %q to %q", r.In.Method, r.In.URL, r.Out.URL) r.Out.Header.Set("Forgejo-Cache-Repo", runData.RepositoryFullName) r.Out.Header.Set("Forgejo-Cache-RunNumber", runData.RunNumber) @@ -152,26 +147,31 @@ func (h *Handler) newReverseProxy(targetHost string) (*httputil.ReverseProxy, er r.Out.Header.Set("Forgejo-Cache-Timestamp", runData.Timestamp) r.Out.Header.Set("Forgejo-Cache-MAC", runData.RepositoryMAC) r.Out.Header.Set("Forgejo-Cache-Host", h.ExternalURL()) + if runData.WriteIsolationKey != "" { + r.Out.Header.Set("Forgejo-Cache-WriteIsolationKey", runData.WriteIsolationKey) + } + }, + ModifyResponse: func(r *http.Response) error { + h.logger.Debugf("proxy resp %s w/ %d bytes", r.Status, r.ContentLength) + return nil }, } return proxy, nil } func (h *Handler) ExternalURL() string { - // TODO: make the external url configurable if necessary + if h.cacheProxyHostOverride != "" { + return h.cacheProxyHostOverride + } return fmt.Sprintf("http://%s", net.JoinHostPort(h.outboundIP, strconv.Itoa(h.listener.Addr().(*net.TCPAddr).Port))) } // Informs the proxy of a workflow run that can make cache requests. // The RunData contains the information about the repository. // The function returns the 32-bit random key which the run will use to identify itself. -func (h *Handler) AddRun(data RunData) (string, error) { - for retries := 0; retries < 3; retries++ { - key, err := common.RandName(4) - if err != nil { - return "", errors.New("Could not generate the run id") - } - +func (h *Handler) AddRun(data artifactcache.RunData) (string, error) { + for range 3 { + key := common.MustRandName(4) _, loaded := h.runs.LoadOrStore(key, data) if !loaded { // The key was unique and added successfully @@ -210,13 +210,3 @@ func (h *Handler) Close() error { } return retErr } - -func computeMac(secret, repo, run, ts string) string { - mac := hmac.New(sha256.New, []byte(secret)) - mac.Write([]byte(repo)) - mac.Write([]byte(">")) - mac.Write([]byte(run)) - mac.Write([]byte(">")) - mac.Write([]byte(ts)) - return hex.EncodeToString(mac.Sum(nil)) -} diff --git a/act/common/cartesian.go b/act/common/cartesian.go index 9cd60655..f7467c90 100644 --- a/act/common/cartesian.go +++ b/act/common/cartesian.go @@ -1,9 +1,9 @@ package common // CartesianProduct takes map of lists and returns list of unique tuples -func CartesianProduct(mapOfLists map[string][]interface{}) []map[string]interface{} { +func CartesianProduct(mapOfLists map[string][]any) []map[string]any { listNames := make([]string, 0) - lists := make([][]interface{}, 0) + lists := make([][]any, 0) for k, v := range mapOfLists { listNames = append(listNames, k) lists = append(lists, v) @@ -11,9 +11,9 @@ func CartesianProduct(mapOfLists map[string][]interface{}) []map[string]interfac listCart := cartN(lists...) - rtn := make([]map[string]interface{}, 0) + rtn := make([]map[string]any, 0) for _, list := range listCart { - vMap := make(map[string]interface{}) + vMap := make(map[string]any) for i, v := range list { vMap[listNames[i]] = v } @@ -22,7 +22,7 @@ func CartesianProduct(mapOfLists map[string][]interface{}) []map[string]interfac return rtn } -func cartN(a ...[]interface{}) [][]interface{} { +func cartN(a ...[]any) [][]any { c := 1 for _, a := range a { c *= len(a) @@ -30,8 +30,8 @@ func cartN(a ...[]interface{}) [][]interface{} { if c == 0 || len(a) == 0 { return nil } - p := make([][]interface{}, c) - b := make([]interface{}, c*len(a)) + p := make([][]any, c) + b := make([]any, c*len(a)) n := make([]int, len(a)) s := 0 for i := range p { diff --git a/act/common/cartesian_test.go b/act/common/cartesian_test.go index c49de06c..a7f4c02e 100644 --- a/act/common/cartesian_test.go +++ b/act/common/cartesian_test.go @@ -8,7 +8,7 @@ import ( func TestCartesianProduct(t *testing.T) { assert := assert.New(t) - input := map[string][]interface{}{ + input := map[string][]any{ "foo": {1, 2, 3, 4}, "bar": {"a", "b", "c"}, "baz": {false, true}, @@ -25,7 +25,7 @@ func TestCartesianProduct(t *testing.T) { assert.Contains(v, "baz") } - input = map[string][]interface{}{ + input = map[string][]any{ "foo": {1, 2, 3, 4}, "bar": {}, "baz": {false, true}, @@ -33,7 +33,7 @@ func TestCartesianProduct(t *testing.T) { output = CartesianProduct(input) assert.Len(output, 0) - input = map[string][]interface{}{} + input = map[string][]any{} output = CartesianProduct(input) assert.Len(output, 0) } diff --git a/act/common/draw.go b/act/common/draw.go index 30bd3dd5..998a539d 100644 --- a/act/common/draw.go +++ b/act/common/draw.go @@ -127,11 +127,8 @@ func (p *Pen) DrawBoxes(labels ...string) *Drawing { // Draw to writer func (d *Drawing) Draw(writer io.Writer, centerOnWidth int) { - padSize := (centerOnWidth - d.GetWidth()) / 2 - if padSize < 0 { - padSize = 0 - } - for _, l := range strings.Split(d.buf.String(), "\n") { + padSize := max((centerOnWidth-d.GetWidth())/2, 0) + for l := range strings.SplitSeq(d.buf.String(), "\n") { if len(l) > 0 { padding := strings.Repeat(" ", padSize) fmt.Fprintf(writer, "%s%s\n", padding, l) diff --git a/act/common/executor.go b/act/common/executor.go index 3033ca09..fb97085a 100644 --- a/act/common/executor.go +++ b/act/common/executor.go @@ -18,7 +18,7 @@ func (w Warning) Error() string { } // Warningf create a warning -func Warningf(format string, args ...interface{}) Warning { +func Warningf(format string, args ...any) Warning { w := Warning{ Message: fmt.Sprintf(format, args...), } @@ -32,7 +32,7 @@ type Executor func(ctx context.Context) error type Conditional func(ctx context.Context) bool // NewInfoExecutor is an executor that logs messages -func NewInfoExecutor(format string, args ...interface{}) Executor { +func NewInfoExecutor(format string, args ...any) Executor { return func(ctx context.Context) error { logger := Logger(ctx) logger.Infof(format, args...) @@ -41,7 +41,7 @@ func NewInfoExecutor(format string, args ...interface{}) Executor { } // NewDebugExecutor is an executor that logs messages -func NewDebugExecutor(format string, args ...interface{}) Executor { +func NewDebugExecutor(format string, args ...any) Executor { return func(ctx context.Context) error { logger := Logger(ctx) logger.Debugf(format, args...) @@ -109,14 +109,14 @@ func NewParallelExecutor(parallel int, executors ...Executor) Executor { }(work, errs) } - for i := 0; i < len(executors); i++ { + for i := range executors { work <- executors[i] } close(work) // Executor waits all executors to cleanup these resources. var firstErr error - for i := 0; i < len(executors); i++ { + for range executors { err := <-errs if firstErr == nil { firstErr = err diff --git a/act/common/executor_test.go b/act/common/executor_test.go index e70c638e..721d487f 100644 --- a/act/common/executor_test.go +++ b/act/common/executor_test.go @@ -3,6 +3,7 @@ package common import ( "context" "fmt" + "sync/atomic" "testing" "time" @@ -12,7 +13,7 @@ import ( func TestNewWorkflow(t *testing.T) { assert := assert.New(t) - ctx := context.Background() + ctx := t.Context() // empty emptyWorkflow := NewPipelineExecutor() @@ -40,7 +41,7 @@ func TestNewWorkflow(t *testing.T) { func TestNewConditionalExecutor(t *testing.T) { assert := assert.New(t) - ctx := context.Background() + ctx := t.Context() trueCount := 0 falseCount := 0 @@ -77,39 +78,50 @@ func TestNewConditionalExecutor(t *testing.T) { func TestNewParallelExecutor(t *testing.T) { assert := assert.New(t) - ctx := context.Background() + ctx := t.Context() + + var count atomic.Int32 + var activeCount atomic.Int32 + var maxCount atomic.Int32 - count := 0 - activeCount := 0 - maxCount := 0 emptyWorkflow := NewPipelineExecutor(func(ctx context.Context) error { - count++ + count.Add(1) - activeCount++ - if activeCount > maxCount { - maxCount = activeCount + currentActive := activeCount.Add(1) + + // maxCount = max(maxCount, currentActive) -- but concurrent-safe by using CompareAndSwap. + for { + currentMax := maxCount.Load() + if currentActive <= currentMax { + break + } + if maxCount.CompareAndSwap(currentMax, currentActive) { + break + } + // If CompareAndSwap failed, retry due to concurrent update by another goroutine. } + time.Sleep(2 * time.Second) - activeCount-- + activeCount.Add(-1) return nil }) err := NewParallelExecutor(2, emptyWorkflow, emptyWorkflow, emptyWorkflow)(ctx) - assert.Equal(3, count, "should run all 3 executors") - assert.Equal(2, maxCount, "should run at most 2 executors in parallel") + assert.Equal(int32(3), count.Load(), "should run all 3 executors") + assert.Equal(int32(2), maxCount.Load(), "should run at most 2 executors in parallel") assert.Nil(err) // Reset to test running the executor with 0 parallelism - count = 0 - activeCount = 0 - maxCount = 0 + count.Store(0) + activeCount.Store(0) + maxCount.Store(0) errSingle := NewParallelExecutor(0, emptyWorkflow, emptyWorkflow, emptyWorkflow)(ctx) - assert.Equal(3, count, "should run all 3 executors") - assert.Equal(1, maxCount, "should run at most 1 executors in parallel") + assert.Equal(int32(3), count.Load(), "should run all 3 executors") + assert.Equal(int32(1), maxCount.Load(), "should run at most 1 executors in parallel") assert.Nil(errSingle) } @@ -119,13 +131,13 @@ func TestNewParallelExecutorFailed(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - count := 0 + var count atomic.Int32 errorWorkflow := NewPipelineExecutor(func(ctx context.Context) error { - count++ + count.Add(1) return fmt.Errorf("fake error") }) err := NewParallelExecutor(1, errorWorkflow)(ctx) - assert.Equal(1, count) + assert.Equal(int32(1), count.Load()) assert.ErrorIs(context.Canceled, err) } @@ -137,16 +149,16 @@ func TestNewParallelExecutorCanceled(t *testing.T) { errExpected := fmt.Errorf("fake error") - count := 0 + var count atomic.Int32 successWorkflow := NewPipelineExecutor(func(ctx context.Context) error { - count++ + count.Add(1) return nil }) errorWorkflow := NewPipelineExecutor(func(ctx context.Context) error { - count++ + count.Add(1) return errExpected }) err := NewParallelExecutor(3, errorWorkflow, successWorkflow, successWorkflow)(ctx) - assert.Equal(3, count) + assert.Equal(int32(3), count.Load()) assert.Error(errExpected, err) } diff --git a/act/common/git/git.go b/act/common/git/git.go index 1748e797..2c47b5d2 100644 --- a/act/common/git/git.go +++ b/act/common/git/git.go @@ -19,7 +19,7 @@ import ( "github.com/mattn/go-isatty" log "github.com/sirupsen/logrus" - "code.forgejo.org/forgejo/runner/v9/act/common" + "code.forgejo.org/forgejo/runner/v11/act/common" ) var ( @@ -305,7 +305,7 @@ func gitOptions(token string) (fetchOptions git.FetchOptions, pullOptions git.Pu func NewGitCloneExecutor(input NewGitCloneExecutorInput) common.Executor { return func(ctx context.Context) error { logger := common.Logger(ctx) - logger.Infof(" \u2601 git clone '%s' # ref=%s", input.URL, input.Ref) + logger.Infof(" \u2601\ufe0f git clone '%s' # ref=%s", input.URL, input.Ref) logger.Debugf(" cloning %s to %s", input.URL, input.Dir) cloneLock.Lock() @@ -340,7 +340,7 @@ func NewGitCloneExecutor(input NewGitCloneExecutorInput) common.Executor { logger.Errorf("Unable to resolve %s: %v", input.Ref, err) } - if hash.String() != input.Ref && strings.HasPrefix(hash.String(), input.Ref) { + if hash.String() != input.Ref && len(input.Ref) >= 4 && strings.HasPrefix(hash.String(), input.Ref) { return &Error{ err: ErrShortRef, commit: hash.String(), diff --git a/act/common/git/git_test.go b/act/common/git/git_test.go index f03720e9..8bb5ec64 100644 --- a/act/common/git/git_test.go +++ b/act/common/git/git_test.go @@ -1,7 +1,6 @@ package git import ( - "context" "fmt" "os" "os/exec" @@ -13,7 +12,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "code.forgejo.org/forgejo/runner/v9/act/common" + "code.forgejo.org/forgejo/runner/v11/act/common" ) func TestFindGitSlug(t *testing.T) { @@ -54,13 +53,6 @@ func TestFindGitSlug(t *testing.T) { } } -func testDir(t *testing.T) string { - basedir, err := os.MkdirTemp("", "act-test") - require.NoError(t, err) - t.Cleanup(func() { _ = os.RemoveAll(basedir) }) - return basedir -} - func cleanGitHooks(dir string) error { hooksDir := filepath.Join(dir, ".git", "hooks") files, err := os.ReadDir(hooksDir) @@ -85,7 +77,7 @@ func cleanGitHooks(dir string) error { func TestFindGitRemoteURL(t *testing.T) { assert := assert.New(t) - basedir := testDir(t) + basedir := t.TempDir() gitConfig() err := gitCmd("init", basedir) assert.NoError(err) @@ -96,20 +88,20 @@ func TestFindGitRemoteURL(t *testing.T) { err = gitCmd("-C", basedir, "remote", "add", "origin", remoteURL) assert.NoError(err) - u, err := findGitRemoteURL(context.Background(), basedir, "origin") + u, err := findGitRemoteURL(t.Context(), basedir, "origin") assert.NoError(err) assert.Equal(remoteURL, u) remoteURL = "git@github.com/AwesomeOwner/MyAwesomeRepo.git" err = gitCmd("-C", basedir, "remote", "add", "upstream", remoteURL) assert.NoError(err) - u, err = findGitRemoteURL(context.Background(), basedir, "upstream") + u, err = findGitRemoteURL(t.Context(), basedir, "upstream") assert.NoError(err) assert.Equal(remoteURL, u) } func TestGitFindRef(t *testing.T) { - basedir := testDir(t) + basedir := t.TempDir() gitConfig() for name, tt := range map[string]struct { @@ -174,8 +166,6 @@ func TestGitFindRef(t *testing.T) { }, }, } { - tt := tt - name := name t.Run(name, func(t *testing.T) { dir := filepath.Join(basedir, name) require.NoError(t, os.MkdirAll(dir, 0o755)) @@ -184,7 +174,7 @@ func TestGitFindRef(t *testing.T) { require.NoError(t, gitCmd("-C", dir, "config", "user.email", "user@example.com")) require.NoError(t, cleanGitHooks(dir)) tt.Prepare(t, dir) - ref, err := FindGitRef(context.Background(), dir) + ref, err := FindGitRef(t.Context(), dir) tt.Assert(t, ref, err) }) } @@ -220,10 +210,10 @@ func TestGitCloneExecutor(t *testing.T) { clone := NewGitCloneExecutor(NewGitCloneExecutorInput{ URL: tt.URL, Ref: tt.Ref, - Dir: testDir(t), + Dir: t.TempDir(), }) - err := clone(context.Background()) + err := clone(t.Context()) if tt.Err != nil { assert.Error(t, err) assert.Equal(t, tt.Err, err) @@ -263,7 +253,7 @@ func gitCmd(args ...string) error { func TestCloneIfRequired(t *testing.T) { tempDir := t.TempDir() - ctx := context.Background() + ctx := t.Context() t.Run("clone", func(t *testing.T) { repo, err := CloneIfRequired(ctx, "refs/heads/main", NewGitCloneExecutorInput{ diff --git a/act/common/randname.go b/act/common/randname.go index a9350106..2010680e 100644 --- a/act/common/randname.go +++ b/act/common/randname.go @@ -5,12 +5,21 @@ package common import ( "crypto/rand" "encoding/hex" + "fmt" ) -func RandName(size int) (string, error) { +func randName(size int) (string, error) { randBytes := make([]byte, size) if _, err := rand.Read(randBytes); err != nil { return "", err } return hex.EncodeToString(randBytes), nil } + +func MustRandName(size int) string { + name, err := randName(size) + if err != nil { + panic(fmt.Errorf("RandName(%d): %v", size, err)) + } + return name +} diff --git a/act/container/container_types.go b/act/container/container_types.go index 3d3a5162..aade7799 100644 --- a/act/container/container_types.go +++ b/act/container/container_types.go @@ -3,8 +3,9 @@ package container import ( "context" "io" + "time" - "code.forgejo.org/forgejo/runner/v9/act/common" + "code.forgejo.org/forgejo/runner/v11/act/common" "github.com/docker/go-connections/nat" ) @@ -34,9 +35,6 @@ type NewContainerInput struct { ConfigOptions string JobOptions string - // Gitea specific - AutoRemove bool - ValidVolumes []string } @@ -63,6 +61,7 @@ type Container interface { Remove() common.Executor Close() common.Executor ReplaceLogWriter(io.Writer, io.Writer) (io.Writer, io.Writer) + IsHealthy(ctx context.Context) (time.Duration, error) } // NewDockerBuildExecutorInput the input for the NewDockerBuildExecutor function diff --git a/act/container/docker_auth.go b/act/container/docker_auth.go index 01b0c816..003a0f14 100644 --- a/act/container/docker_auth.go +++ b/act/container/docker_auth.go @@ -1,4 +1,4 @@ -//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd)) +//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd)) package container @@ -6,7 +6,7 @@ import ( "context" "strings" - "code.forgejo.org/forgejo/runner/v9/act/common" + "code.forgejo.org/forgejo/runner/v11/act/common" "github.com/docker/cli/cli/config" "github.com/docker/cli/cli/config/credentials" "github.com/docker/docker/api/types/registry" diff --git a/act/container/docker_build.go b/act/container/docker_build.go index 7b8dd906..258a9363 100644 --- a/act/container/docker_build.go +++ b/act/container/docker_build.go @@ -1,4 +1,4 @@ -//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd)) +//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd)) package container @@ -14,7 +14,7 @@ import ( "github.com/moby/patternmatcher" "github.com/moby/patternmatcher/ignorefile" - "code.forgejo.org/forgejo/runner/v9/act/common" + "code.forgejo.org/forgejo/runner/v11/act/common" ) // NewDockerBuildExecutor function to create a run executor for the container diff --git a/act/container/docker_cli.go b/act/container/docker_cli.go index 1a1ffe6b..adf0bb0e 100644 --- a/act/container/docker_cli.go +++ b/act/container/docker_cli.go @@ -1,4 +1,4 @@ -//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd)) +//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd)) // This file is exact copy of https://github.com/docker/cli/blob/9ac8584acfd501c3f4da0e845e3a40ed15c85041/cli/command/container/opts.go // appended with license information. @@ -13,6 +13,7 @@ package container import ( "bytes" "encoding/json" + "errors" "fmt" "os" "path" @@ -32,7 +33,6 @@ import ( "github.com/docker/docker/api/types/versions" "github.com/docker/docker/errdefs" "github.com/docker/go-connections/nat" - "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/spf13/pflag" ) @@ -331,7 +331,7 @@ func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*con // Validate the input mac address if copts.macAddress != "" { if _, err := opts.ValidateMACAddress(copts.macAddress); err != nil { - return nil, errors.Errorf("%s is not a valid mac address", copts.macAddress) + return nil, fmt.Errorf("%s is not a valid mac address", copts.macAddress) } } if copts.stdin { @@ -347,7 +347,7 @@ func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*con swappiness := copts.swappiness if swappiness != -1 && (swappiness < 0 || swappiness > 100) { - return nil, errors.Errorf("invalid value: %d. Valid memory swappiness range is 0-100", swappiness) + return nil, fmt.Errorf("invalid value: %d. Valid memory swappiness range is 0-100", swappiness) } mounts := copts.mounts.Value() @@ -430,7 +430,7 @@ func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*con // Merge in exposed ports to the map of published ports for _, e := range copts.expose.GetSlice() { if strings.Contains(e, ":") { - return nil, errors.Errorf("invalid port format for --expose: %s", e) + return nil, fmt.Errorf("invalid port format for --expose: %s", e) } // support two formats for expose, original format /[] // or /[] @@ -439,7 +439,7 @@ func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*con // if expose a port, the start and end port are the same start, end, err := nat.ParsePortRange(port) if err != nil { - return nil, errors.Errorf("invalid range format for --expose: %s, error: %s", e, err) + return nil, fmt.Errorf("invalid range format for --expose: %s, error: %s", e, err) } for i := start; i <= end; i++ { p, err := nat.NewPort(proto, strconv.FormatUint(i, 10)) @@ -488,22 +488,22 @@ func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*con pidMode := container.PidMode(copts.pidMode) if !pidMode.Valid() { - return nil, errors.Errorf("--pid: invalid PID mode") + return nil, errors.New("--pid: invalid PID mode") } utsMode := container.UTSMode(copts.utsMode) if !utsMode.Valid() { - return nil, errors.Errorf("--uts: invalid UTS mode") + return nil, errors.New("--uts: invalid UTS mode") } usernsMode := container.UsernsMode(copts.usernsMode) if !usernsMode.Valid() { - return nil, errors.Errorf("--userns: invalid USER mode") + return nil, errors.New("--userns: invalid USER mode") } cgroupnsMode := container.CgroupnsMode(copts.cgroupnsMode) if !cgroupnsMode.Valid() { - return nil, errors.Errorf("--cgroupns: invalid CGROUP mode") + return nil, errors.New("--cgroupns: invalid CGROUP mode") } restartPolicy, err := opts.ParseRestartPolicy(copts.restartPolicy) @@ -537,7 +537,7 @@ func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*con copts.healthRetries != 0 if copts.noHealthcheck { if haveHealthSettings { - return nil, errors.Errorf("--no-healthcheck conflicts with --health-* options") + return nil, errors.New("--no-healthcheck conflicts with --health-* options") } test := strslice.StrSlice{"NONE"} healthConfig = &container.HealthConfig{Test: test} @@ -548,16 +548,16 @@ func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*con probe = strslice.StrSlice(args) } if copts.healthInterval < 0 { - return nil, errors.Errorf("--health-interval cannot be negative") + return nil, errors.New("--health-interval cannot be negative") } if copts.healthTimeout < 0 { - return nil, errors.Errorf("--health-timeout cannot be negative") + return nil, errors.New("--health-timeout cannot be negative") } if copts.healthRetries < 0 { - return nil, errors.Errorf("--health-retries cannot be negative") + return nil, errors.New("--health-retries cannot be negative") } if copts.healthStartPeriod < 0 { - return nil, fmt.Errorf("--health-start-period cannot be negative") + return nil, errors.New("--health-start-period cannot be negative") } healthConfig = &container.HealthConfig{ @@ -677,7 +677,7 @@ func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*con } if copts.autoRemove && !hostConfig.RestartPolicy.IsNone() { - return nil, errors.Errorf("Conflicting options: --restart and --rm") + return nil, errors.New("Conflicting options: --restart and --rm") } // only set this value if the user provided the flag, else it should default to nil @@ -741,7 +741,7 @@ func parseNetworkOpts(copts *containerOptions) (map[string]*networktypes.Endpoin return nil, err } if _, ok := endpoints[n.Target]; ok { - return nil, errdefs.InvalidParameter(errors.Errorf("network %q is specified multiple times", n.Target)) + return nil, errdefs.InvalidParameter(fmt.Errorf("network %q is specified multiple times", n.Target)) } // For backward compatibility: if no custom options are provided for the network, @@ -838,7 +838,7 @@ func convertToStandardNotation(ports []string) ([]string, error) { for _, param := range strings.Split(publish, ",") { opt := strings.Split(param, "=") if len(opt) < 2 { - return optsList, errors.Errorf("invalid publish opts format (should be name=value but got '%s')", param) + return optsList, fmt.Errorf("invalid publish opts format (should be name=value but got '%s')", param) } params[opt[0]] = opt[1] @@ -854,7 +854,7 @@ func convertToStandardNotation(ports []string) ([]string, error) { func parseLoggingOpts(loggingDriver string, loggingOpts []string) (map[string]string, error) { loggingOptsMap := opts.ConvertKVStringsToMap(loggingOpts) if loggingDriver == "none" && len(loggingOpts) > 0 { - return map[string]string{}, errors.Errorf("invalid logging opts for driver %s", loggingDriver) + return map[string]string{}, fmt.Errorf("invalid logging opts for driver %s", loggingDriver) } return loggingOptsMap, nil } @@ -867,17 +867,17 @@ func parseSecurityOpts(securityOpts []string) ([]string, error) { if strings.Contains(opt, ":") { con = strings.SplitN(opt, ":", 2) } else { - return securityOpts, errors.Errorf("Invalid --security-opt: %q", opt) + return securityOpts, fmt.Errorf("Invalid --security-opt: %q", opt) } } if con[0] == "seccomp" && con[1] != "unconfined" { f, err := os.ReadFile(con[1]) if err != nil { - return securityOpts, errors.Errorf("opening seccomp profile (%s) failed: %v", con[1], err) + return securityOpts, fmt.Errorf("opening seccomp profile (%s) failed: %v", con[1], err) } b := bytes.NewBuffer(nil) if err := json.Compact(b, f); err != nil { - return securityOpts, errors.Errorf("compacting json for seccomp profile (%s) failed: %v", con[1], err) + return securityOpts, fmt.Errorf("compacting json for seccomp profile (%s) failed: %v", con[1], err) } securityOpts[key] = fmt.Sprintf("seccomp=%s", b.Bytes()) } @@ -913,7 +913,7 @@ func parseStorageOpts(storageOpts []string) (map[string]string, error) { opt := strings.SplitN(option, "=", 2) m[opt[0]] = opt[1] } else { - return nil, errors.Errorf("invalid storage option") + return nil, errors.New("invalid storage option") } } return m, nil @@ -927,7 +927,7 @@ func parseDevice(device, serverOS string) (container.DeviceMapping, error) { case "windows": return parseWindowsDevice(device) } - return container.DeviceMapping{}, errors.Errorf("unknown server OS: %s", serverOS) + return container.DeviceMapping{}, fmt.Errorf("unknown server OS: %s", serverOS) } // parseLinuxDevice parses a device mapping string to a container.DeviceMapping struct @@ -950,7 +950,7 @@ func parseLinuxDevice(device string) (container.DeviceMapping, error) { case 1: src = arr[0] default: - return container.DeviceMapping{}, errors.Errorf("invalid device specification: %s", device) + return container.DeviceMapping{}, fmt.Errorf("invalid device specification: %s", device) } if dst == "" { @@ -980,7 +980,7 @@ func validateDeviceCgroupRule(val string) (string, error) { return val, nil } - return val, errors.Errorf("invalid device cgroup format '%s'", val) + return val, fmt.Errorf("invalid device cgroup format '%s'", val) } // validDeviceMode checks if the mode for device is valid or not. @@ -1012,7 +1012,7 @@ func validateDevice(val, serverOS string) (string, error) { // Windows does validation entirely server-side return val, nil } - return "", errors.Errorf("unknown server OS: %s", serverOS) + return "", fmt.Errorf("unknown server OS: %s", serverOS) } // validateLinuxPath is the implementation of validateDevice knowing that the @@ -1027,12 +1027,12 @@ func validateLinuxPath(val string, validator func(string) bool) (string, error) var mode string if strings.Count(val, ":") > 2 { - return val, errors.Errorf("bad format for path: %s", val) + return val, fmt.Errorf("bad format for path: %s", val) } split := strings.SplitN(val, ":", 3) if split[0] == "" { - return val, errors.Errorf("bad format for path: %s", val) + return val, fmt.Errorf("bad format for path: %s", val) } switch len(split) { case 1: @@ -1051,13 +1051,13 @@ func validateLinuxPath(val string, validator func(string) bool) (string, error) containerPath = split[1] mode = split[2] if isValid := validator(split[2]); !isValid { - return val, errors.Errorf("bad mode specified: %s", mode) + return val, fmt.Errorf("bad mode specified: %s", mode) } val = fmt.Sprintf("%s:%s:%s", split[0], containerPath, mode) } if !path.IsAbs(containerPath) { - return val, errors.Errorf("%s is not an absolute path", containerPath) + return val, fmt.Errorf("%s is not an absolute path", containerPath) } return val, nil } @@ -1070,13 +1070,13 @@ func validateAttach(val string) (string, error) { return s, nil } } - return val, errors.Errorf("valid streams are STDIN, STDOUT and STDERR") + return val, errors.New("valid streams are STDIN, STDOUT and STDERR") } func validateAPIVersion(c *containerConfig, serverAPIVersion string) error { for _, m := range c.HostConfig.Mounts { if m.BindOptions != nil && m.BindOptions.NonRecursive && versions.LessThan(serverAPIVersion, "1.40") { - return errors.Errorf("bind-nonrecursive requires API v1.40 or later") + return errors.New("bind-nonrecursive requires API v1.40 or later") } } return nil diff --git a/act/container/docker_cli_test.go b/act/container/docker_cli_test.go index 7d41ed15..f885bfb6 100644 --- a/act/container/docker_cli_test.go +++ b/act/container/docker_cli_test.go @@ -10,6 +10,7 @@ package container import ( + "errors" "fmt" "io" "os" @@ -21,7 +22,6 @@ import ( "github.com/docker/docker/api/types/container" networktypes "github.com/docker/docker/api/types/network" "github.com/docker/go-connections/nat" - "github.com/pkg/errors" "github.com/spf13/pflag" "gotest.tools/v3/assert" is "gotest.tools/v3/assert/cmp" @@ -288,7 +288,7 @@ func compareRandomizedStrings(a, b, c, d string) error { if a == d && b == c { return nil } - return errors.Errorf("strings don't match") + return errors.New("strings don't match") } // Simple parse with MacAddress validation diff --git a/act/container/docker_images.go b/act/container/docker_images.go index 484cc834..70b99b51 100644 --- a/act/container/docker_images.go +++ b/act/container/docker_images.go @@ -1,4 +1,4 @@ -//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd)) +//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd)) package container @@ -6,6 +6,7 @@ import ( "context" "fmt" + "code.forgejo.org/forgejo/runner/v11/act/common" cerrdefs "github.com/containerd/errdefs" "github.com/docker/docker/api/types/image" ) @@ -26,10 +27,15 @@ func ImageExistsLocally(ctx context.Context, imageName, platform string) (bool, return false, err } - if platform == "" || platform == "any" || fmt.Sprintf("%s/%s", inspectImage.Os, inspectImage.Architecture) == platform { + imagePlatform := fmt.Sprintf("%s/%s", inspectImage.Os, inspectImage.Architecture) + + if platform == "" || platform == "any" || imagePlatform == platform { return true, nil } + logger := common.Logger(ctx) + logger.Infof("image found but platform does not match: %s (image) != %s (platform)\n", imagePlatform, platform) + return false, nil } diff --git a/act/container/docker_images_test.go b/act/container/docker_images_test.go index 3344120d..a7b9babb 100644 --- a/act/container/docker_images_test.go +++ b/act/container/docker_images_test.go @@ -1,7 +1,6 @@ package container import ( - "context" "io" "testing" @@ -19,7 +18,7 @@ func TestImageExistsLocally(t *testing.T) { if testing.Short() { t.Skip("skipping integration test") } - ctx := context.Background() + ctx := t.Context() // to help make this test reliable and not flaky, we need to have // an image that will exist, and onew that won't exist @@ -36,7 +35,7 @@ func TestImageExistsLocally(t *testing.T) { // pull an image cli, err := client.NewClientWithOpts(client.FromEnv) assert.Nil(t, err) - cli.NegotiateAPIVersion(context.Background()) + cli.NegotiateAPIVersion(t.Context()) // Chose alpine latest because it's so small // maybe we should build an image instead so that tests aren't reliable on dockerhub diff --git a/act/container/docker_logger.go b/act/container/docker_logger.go index b9eb503e..615801c4 100644 --- a/act/container/docker_logger.go +++ b/act/container/docker_logger.go @@ -1,4 +1,4 @@ -//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd)) +//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd)) package container @@ -74,7 +74,7 @@ func logDockerResponse(logger logrus.FieldLogger, dockerResponse io.ReadCloser, return nil } -func writeLog(logger logrus.FieldLogger, isError bool, format string, args ...interface{}) { +func writeLog(logger logrus.FieldLogger, isError bool, format string, args ...any) { if isError { logger.Errorf(format, args...) } else { diff --git a/act/container/docker_network.go b/act/container/docker_network.go index 23d884f6..100d1b0f 100644 --- a/act/container/docker_network.go +++ b/act/container/docker_network.go @@ -1,11 +1,11 @@ -//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd)) +//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd)) package container import ( "context" - "code.forgejo.org/forgejo/runner/v9/act/common" + "code.forgejo.org/forgejo/runner/v11/act/common" "github.com/docker/docker/api/types/network" ) diff --git a/act/container/docker_pull.go b/act/container/docker_pull.go index 3b1ad5c7..f0fb1485 100644 --- a/act/container/docker_pull.go +++ b/act/container/docker_pull.go @@ -1,4 +1,4 @@ -//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd)) +//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd)) package container @@ -13,7 +13,7 @@ import ( "github.com/docker/docker/api/types/image" "github.com/docker/docker/api/types/registry" - "code.forgejo.org/forgejo/runner/v9/act/common" + "code.forgejo.org/forgejo/runner/v11/act/common" ) // NewDockerPullExecutor function to create a run executor for the container diff --git a/act/container/docker_pull_test.go b/act/container/docker_pull_test.go index bfbe89dd..c0ff1d6f 100644 --- a/act/container/docker_pull_test.go +++ b/act/container/docker_pull_test.go @@ -1,7 +1,6 @@ package container import ( - "context" "testing" "github.com/docker/cli/cli/config" @@ -29,13 +28,13 @@ func TestCleanImage(t *testing.T) { } for _, table := range tables { - imageOut := cleanImage(context.Background(), table.imageIn) + imageOut := cleanImage(t.Context(), table.imageIn) assert.Equal(t, table.imageOut, imageOut) } } func TestGetImagePullOptions(t *testing.T) { - ctx := context.Background() + ctx := t.Context() config.SetDir("/non-existent/docker") diff --git a/act/container/docker_run.go b/act/container/docker_run.go index 1c790d6a..cf15887c 100644 --- a/act/container/docker_run.go +++ b/act/container/docker_run.go @@ -1,4 +1,4 @@ -//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd)) +//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd)) package container @@ -15,7 +15,9 @@ import ( "runtime" "strconv" "strings" + "time" + "dario.cat/mergo" "github.com/Masterminds/semver" "github.com/docker/cli/cli/compose/loader" "github.com/docker/cli/cli/connhelper" @@ -30,15 +32,14 @@ import ( "github.com/go-git/go-billy/v5/osfs" "github.com/go-git/go-git/v5/plumbing/format/gitignore" "github.com/gobwas/glob" - "github.com/imdario/mergo" "github.com/joho/godotenv" "github.com/kballard/go-shellquote" specs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/spf13/pflag" "golang.org/x/term" - "code.forgejo.org/forgejo/runner/v9/act/common" - "code.forgejo.org/forgejo/runner/v9/act/filecollector" + "code.forgejo.org/forgejo/runner/v11/act/common" + "code.forgejo.org/forgejo/runner/v11/act/filecollector" ) // NewContainer creates a reference to a container @@ -191,6 +192,47 @@ func (cr *containerReference) Remove() common.Executor { ).IfNot(common.Dryrun) } +func (cr *containerReference) inspect(ctx context.Context) (container.InspectResponse, error) { + resp, err := cr.cli.ContainerInspect(ctx, cr.id) + if err != nil { + err = fmt.Errorf("service %v: %s", cr.input.NetworkAliases, err) + } + return resp, err +} + +func (cr *containerReference) IsHealthy(ctx context.Context) (time.Duration, error) { + resp, err := cr.inspect(ctx) + if err != nil { + return 0, err + } + return cr.isHealthy(ctx, resp) +} + +func (cr *containerReference) isHealthy(ctx context.Context, resp container.InspectResponse) (time.Duration, error) { + logger := common.Logger(ctx) + if resp.Config == nil || resp.Config.Healthcheck == nil || resp.State == nil || resp.State.Health == nil || len(resp.Config.Healthcheck.Test) == 1 && strings.EqualFold(resp.Config.Healthcheck.Test[0], "NONE") { + logger.Debugf("no container health check defined, hope for the best") + return 0, nil + } + + switch resp.State.Health.Status { + case container.Starting: + wait := resp.Config.Healthcheck.Interval + if wait <= 0 { + wait = time.Second + } + logger.Infof("service %v: container health check %s (%s) is starting, waiting %v", cr.input.NetworkAliases, cr.id, resp.Config.Image, wait) + return wait, nil + case container.Healthy: + logger.Infof("service %v: container health check %s (%s) is healthy", cr.input.NetworkAliases, cr.id, resp.Config.Image) + return 0, nil + case container.Unhealthy: + return 0, fmt.Errorf("service %v: container health check %s (%s) is not healthy", cr.input.NetworkAliases, cr.id, resp.Config.Image) + default: + return 0, fmt.Errorf("service %v: unexpected health status %s (%s) %v", cr.input.NetworkAliases, cr.id, resp.Config.Image, resp.State.Health.Status) + } +} + func (cr *containerReference) ReplaceLogWriter(stdout, stderr io.Writer) (io.Writer, io.Writer) { out := cr.input.Stdout err := cr.input.Stderr @@ -441,6 +483,14 @@ func (cr *containerReference) mergeJobOptions(ctx context.Context, config *conta } } + if jobConfig.HostConfig.Memory > 0 { + logger.Debugf("--memory %v", jobConfig.HostConfig.Memory) + if hostConfig.Memory > 0 && jobConfig.HostConfig.Memory > hostConfig.Memory { + return nil, nil, fmt.Errorf("the --memory %v option found in the workflow cannot be greater than the --memory %v option from the runner configuration file", jobConfig.HostConfig.Memory, hostConfig.Memory) + } + hostConfig.Memory = jobConfig.HostConfig.Memory + } + if len(jobConfig.Config.Hostname) > 0 { logger.Debugf("--hostname %v", jobConfig.Config.Hostname) config.Hostname = jobConfig.Config.Hostname @@ -535,7 +585,6 @@ func (cr *containerReference) create(capAdd, capDrop []string) common.Executor { Privileged: input.Privileged, UsernsMode: container.UsernsMode(input.UsernsMode), PortBindings: input.PortBindings, - AutoRemove: input.AutoRemove, } logger.Debugf("Common container.HostConfig ==> %+v", hostConfig) @@ -793,7 +842,7 @@ func (cr *containerReference) CopyTarStream(ctx context.Context, destPath string // Copy Content err = cr.cli.CopyToContainer(ctx, cr.id, destPath, tarStream, container.CopyToContainerOptions{}) if err != nil { - return fmt.Errorf("failed to copy content to container: %w", err) + return fmt.Errorf("copyTarStream: failed to copy content to container: %w", err) } // If this fails, then folders have wrong permissions on non root container if cr.UID != 0 || cr.GID != 0 { @@ -869,7 +918,7 @@ func (cr *containerReference) copyDir(dstPath, srcPath string, useGitIgnore bool } err = cr.cli.CopyToContainer(ctx, cr.id, "/", tarFile, container.CopyToContainerOptions{}) if err != nil { - return fmt.Errorf("failed to copy content to container: %w", err) + return fmt.Errorf("copyDir: failed to copy content to container: %w", err) } return nil } @@ -903,7 +952,7 @@ func (cr *containerReference) copyContent(dstPath string, files ...*FileEntry) c logger.Debugf("Extracting content to '%s'", dstPath) err := cr.cli.CopyToContainer(ctx, cr.id, dstPath, &buf, container.CopyToContainerOptions{}) if err != nil { - return fmt.Errorf("failed to copy content to container: %w", err) + return fmt.Errorf("copyContent: failed to copy content to container: %w", err) } return nil } diff --git a/act/container/docker_run_test.go b/act/container/docker_run_test.go index 1069f895..b70350b4 100644 --- a/act/container/docker_run_test.go +++ b/act/container/docker_run_test.go @@ -11,7 +11,7 @@ import ( "testing" "time" - "code.forgejo.org/forgejo/runner/v9/act/common" + "code.forgejo.org/forgejo/runner/v11/act/common" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" @@ -26,7 +26,7 @@ func TestDocker(t *testing.T) { if testing.Short() { t.Skip("skipping integration test") } - ctx := context.Background() + ctx := t.Context() client, err := GetDockerClient(ctx) assert.NoError(t, err) defer client.Close() @@ -152,7 +152,7 @@ func TestDockerExecAbort(t *testing.T) { } func TestDockerExecFailure(t *testing.T) { - ctx := context.Background() + ctx := t.Context() conn := &mockConn{} @@ -183,7 +183,7 @@ func TestDockerExecFailure(t *testing.T) { } func TestDockerCopyTarStream(t *testing.T) { - ctx := context.Background() + ctx := t.Context() conn := &mockConn{} @@ -205,7 +205,7 @@ func TestDockerCopyTarStream(t *testing.T) { } func TestDockerCopyTarStreamErrorInCopyFiles(t *testing.T) { - ctx := context.Background() + ctx := t.Context() conn := &mockConn{} @@ -230,7 +230,7 @@ func TestDockerCopyTarStreamErrorInCopyFiles(t *testing.T) { } func TestDockerCopyTarStreamErrorInMkdir(t *testing.T) { - ctx := context.Background() + ctx := t.Context() conn := &mockConn{} @@ -318,7 +318,7 @@ func TestCheckVolumes(t *testing.T) { for _, tc := range testCases { t.Run(tc.desc, func(t *testing.T) { logger, _ := test.NewNullLogger() - ctx := common.WithLogger(context.Background(), logger) + ctx := common.WithLogger(t.Context(), logger) cr := &containerReference{ input: &NewContainerInput{ ValidVolumes: tc.validVolumes, @@ -379,10 +379,86 @@ func TestMergeJobOptions(t *testing.T) { JobOptions: testCase.options, }, } - config, hostConfig, err := cr.mergeJobOptions(context.Background(), &container.Config{}, &container.HostConfig{}) + config, hostConfig, err := cr.mergeJobOptions(t.Context(), &container.Config{}, &container.HostConfig{}) require.NoError(t, err) assert.EqualValues(t, testCase.config, config) assert.EqualValues(t, testCase.hostConfig, hostConfig) }) } } + +func TestDockerRun_isHealthy(t *testing.T) { + cr := containerReference{ + id: "containerid", + input: &NewContainerInput{ + NetworkAliases: []string{"servicename"}, + }, + } + ctx := t.Context() + makeInspectResponse := func(interval time.Duration, status container.HealthStatus, test []string) container.InspectResponse { + return container.InspectResponse{ + Config: &container.Config{ + Image: "example.com/some/image", + Healthcheck: &container.HealthConfig{ + Interval: interval, + Test: test, + }, + }, + ContainerJSONBase: &container.ContainerJSONBase{ + State: &container.State{ + Health: &container.Health{ + Status: status, + }, + }, + }, + } + } + + t.Run("IncompleteResponseOrNoHealthCheck", func(t *testing.T) { + wait, err := cr.isHealthy(ctx, container.InspectResponse{}) + assert.Zero(t, wait) + assert.NoError(t, err) + + // --no-healthcheck translates into a NONE test command + resp := makeInspectResponse(0, container.NoHealthcheck, []string{"NONE"}) + wait, err = cr.isHealthy(ctx, resp) + assert.Zero(t, wait) + assert.NoError(t, err) + }) + + t.Run("StartingUndefinedIntervalIsNotZero", func(t *testing.T) { + resp := makeInspectResponse(0, container.Starting, nil) + wait, err := cr.isHealthy(ctx, resp) + assert.NotZero(t, wait) + assert.NoError(t, err) + }) + + t.Run("StartingWithInterval", func(t *testing.T) { + expectedWait := time.Duration(42) + resp := makeInspectResponse(expectedWait, container.Starting, nil) + actualWait, err := cr.isHealthy(ctx, resp) + assert.Equal(t, expectedWait, actualWait) + assert.NoError(t, err) + }) + + t.Run("Unhealthy", func(t *testing.T) { + resp := makeInspectResponse(0, container.Unhealthy, nil) + wait, err := cr.isHealthy(ctx, resp) + assert.Zero(t, wait) + assert.ErrorContains(t, err, "is not healthy") + }) + + t.Run("Healthy", func(t *testing.T) { + resp := makeInspectResponse(0, container.Healthy, nil) + wait, err := cr.isHealthy(ctx, resp) + assert.Zero(t, wait) + assert.NoError(t, err) + }) + + t.Run("UnknownStatus", func(t *testing.T) { + resp := makeInspectResponse(0, container.NoHealthcheck, nil) + wait, err := cr.isHealthy(ctx, resp) + assert.Zero(t, wait) + assert.ErrorContains(t, err, "unexpected") + }) +} diff --git a/act/container/docker_stub.go b/act/container/docker_stub.go index 2f3624fd..34cc094b 100644 --- a/act/container/docker_stub.go +++ b/act/container/docker_stub.go @@ -1,15 +1,15 @@ -//go:build WITHOUT_DOCKER || !(linux || darwin || windows || netbsd) +//go:build WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd) package container import ( "context" + "errors" "runtime" - "code.forgejo.org/forgejo/runner/v9/act/common" + "code.forgejo.org/forgejo/runner/v11/act/common" "github.com/docker/docker/api/types/network" "github.com/docker/docker/api/types/system" - "github.com/pkg/errors" ) // ImageExistsLocally returns a boolean indicating if an image with the diff --git a/act/container/docker_volume.go b/act/container/docker_volume.go index 94e4036b..e15262c4 100644 --- a/act/container/docker_volume.go +++ b/act/container/docker_volume.go @@ -1,16 +1,17 @@ -//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd)) +//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd || freebsd)) package container import ( "context" + "slices" - "code.forgejo.org/forgejo/runner/v9/act/common" + "code.forgejo.org/forgejo/runner/v11/act/common" "github.com/docker/docker/api/types/filters" "github.com/docker/docker/api/types/volume" ) -func NewDockerVolumeRemoveExecutor(volumeName string, force bool) common.Executor { +func NewDockerVolumesRemoveExecutor(volumeNames []string) common.Executor { return func(ctx context.Context) error { cli, err := GetDockerClient(ctx) if err != nil { @@ -24,17 +25,18 @@ func NewDockerVolumeRemoveExecutor(volumeName string, force bool) common.Executo } for _, vol := range list.Volumes { - if vol.Name == volumeName { - return removeExecutor(volumeName, force)(ctx) + if slices.Contains(volumeNames, vol.Name) { + if err := removeExecutor(vol.Name)(ctx); err != nil { + return err + } } } - // Volume not found - do nothing return nil } } -func removeExecutor(volume string, force bool) common.Executor { +func removeExecutor(volume string) common.Executor { return func(ctx context.Context) error { logger := common.Logger(ctx) logger.Debugf("%sdocker volume rm %s", logPrefix, volume) @@ -49,6 +51,7 @@ func removeExecutor(volume string, force bool) common.Executor { } defer cli.Close() + force := false return cli.VolumeRemove(ctx, volume, force) } } diff --git a/act/container/executions_environment.go b/act/container/executions_environment.go index e334107d..7808096f 100644 --- a/act/container/executions_environment.go +++ b/act/container/executions_environment.go @@ -12,7 +12,7 @@ type ExecutionsEnvironment interface { GetPathVariableName() string DefaultPathVariable() string JoinPathVariable(...string) string - GetRunnerContext(ctx context.Context) map[string]interface{} + GetRunnerContext(ctx context.Context) map[string]any // On windows PATH and Path are the same key IsEnvironmentCaseInsensitive() bool } diff --git a/act/container/host_environment.go b/act/container/host_environment.go index 045d7774..34d24b9b 100644 --- a/act/container/host_environment.go +++ b/act/container/host_environment.go @@ -13,6 +13,7 @@ import ( "path/filepath" "runtime" "strings" + "sync/atomic" "time" "github.com/go-git/go-billy/v5/helper/polyfill" @@ -20,9 +21,9 @@ import ( "github.com/go-git/go-git/v5/plumbing/format/gitignore" "golang.org/x/term" - "code.forgejo.org/forgejo/runner/v9/act/common" - "code.forgejo.org/forgejo/runner/v9/act/filecollector" - "code.forgejo.org/forgejo/runner/v9/act/lookpath" + "code.forgejo.org/forgejo/runner/v11/act/common" + "code.forgejo.org/forgejo/runner/v11/act/filecollector" + "code.forgejo.org/forgejo/runner/v11/act/lookpath" ) type HostEnvironment struct { @@ -33,7 +34,6 @@ type HostEnvironment struct { Workdir string ActPath string Root string - CleanUp func() StdOut io.Writer LXC bool } @@ -191,12 +191,12 @@ func (e *HostEnvironment) Start(_ bool) common.Executor { type ptyWriter struct { Out io.Writer - AutoStop bool + AutoStop atomic.Bool dirtyLine bool } func (w *ptyWriter) Write(buf []byte) (int, error) { - if w.AutoStop && len(buf) > 0 && buf[len(buf)-1] == 4 { + if w.AutoStop.Load() && len(buf) > 0 && buf[len(buf)-1] == 4 { n, err := w.Out.Write(buf[:len(buf)-1]) if err != nil { return n, err @@ -365,7 +365,7 @@ func (e *HostEnvironment) exec(ctx context.Context, commandparam []string, cmdli return fmt.Errorf("RUN %w", err) } if tty != nil { - writer.AutoStop = true + writer.AutoStop.Store(true) if _, err := tty.Write([]byte("\x04")); err != nil { common.Logger(ctx).Debug("Failed to write EOT") } @@ -388,7 +388,7 @@ func (e *HostEnvironment) ExecWithCmdLine(command []string, cmdline string, env if err := e.exec(ctx, command, cmdline, env, user, workdir); err != nil { select { case <-ctx.Done(): - return fmt.Errorf("this step has been cancelled: %w", err) + return fmt.Errorf("this step has been cancelled: ctx: %w, exec: %w", ctx.Err(), err) default: return err } @@ -403,10 +403,12 @@ func (e *HostEnvironment) UpdateFromEnv(srcPath string, env *map[string]string) func (e *HostEnvironment) Remove() common.Executor { return func(ctx context.Context) error { - if e.CleanUp != nil { - e.CleanUp() + if e.GetLXC() { + // there may be files owned by root: removal + // is the responsibility of the LXC backend + return nil } - return os.RemoveAll(e.Path) + return os.RemoveAll(e.Root) } } @@ -462,6 +464,7 @@ func (*HostEnvironment) JoinPathVariable(paths ...string) string { // https://docs.github.com/en/actions/learn-github-actions/contexts#runner-context func goArchToActionArch(arch string) string { archMapper := map[string]string{ + "amd64": "X64", "x86_64": "X64", "386": "X86", "aarch64": "ARM64", @@ -484,8 +487,8 @@ func goOsToActionOs(os string) string { return os } -func (e *HostEnvironment) GetRunnerContext(_ context.Context) map[string]interface{} { - return map[string]interface{}{ +func (e *HostEnvironment) GetRunnerContext(_ context.Context) map[string]any { + return map[string]any{ "os": goOsToActionOs(runtime.GOOS), "arch": goArchToActionArch(runtime.GOARCH), "temp": e.TmpDir, @@ -493,6 +496,10 @@ func (e *HostEnvironment) GetRunnerContext(_ context.Context) map[string]interfa } } +func (e *HostEnvironment) IsHealthy(ctx context.Context) (time.Duration, error) { + return 0, nil +} + func (e *HostEnvironment) ReplaceLogWriter(stdout, _ io.Writer) (io.Writer, io.Writer) { org := e.StdOut e.StdOut = stdout diff --git a/act/container/host_environment_test.go b/act/container/host_environment_test.go index 85456288..6efb8560 100644 --- a/act/container/host_environment_test.go +++ b/act/container/host_environment_test.go @@ -2,7 +2,6 @@ package container import ( "archive/tar" - "context" "io" "os" "path" @@ -16,10 +15,8 @@ import ( var _ ExecutionsEnvironment = &HostEnvironment{} func TestCopyDir(t *testing.T) { - dir, err := os.MkdirTemp("", "test-host-env-*") - assert.NoError(t, err) - defer os.RemoveAll(dir) - ctx := context.Background() + dir := t.TempDir() + ctx := t.Context() e := &HostEnvironment{ Path: filepath.Join(dir, "path"), TmpDir: filepath.Join(dir, "tmp"), @@ -32,15 +29,13 @@ func TestCopyDir(t *testing.T) { _ = os.MkdirAll(e.TmpDir, 0o700) _ = os.MkdirAll(e.ToolCache, 0o700) _ = os.MkdirAll(e.ActPath, 0o700) - err = e.CopyDir(e.Workdir, e.Path, true)(ctx) + err := e.CopyDir(e.Workdir, e.Path, true)(ctx) assert.NoError(t, err) } func TestGetContainerArchive(t *testing.T) { - dir, err := os.MkdirTemp("", "test-host-env-*") - assert.NoError(t, err) - defer os.RemoveAll(dir) - ctx := context.Background() + dir := t.TempDir() + ctx := t.Context() e := &HostEnvironment{ Path: filepath.Join(dir, "path"), TmpDir: filepath.Join(dir, "tmp"), @@ -54,7 +49,7 @@ func TestGetContainerArchive(t *testing.T) { _ = os.MkdirAll(e.ToolCache, 0o700) _ = os.MkdirAll(e.ActPath, 0o700) expectedContent := []byte("sdde/7sh") - err = os.WriteFile(filepath.Join(e.Path, "action.yml"), expectedContent, 0o600) + err := os.WriteFile(filepath.Join(e.Path, "action.yml"), expectedContent, 0o600) assert.NoError(t, err) archive, err := e.GetContainerArchive(ctx, e.Path) assert.NoError(t, err) diff --git a/act/container/linux_container_environment_extensions.go b/act/container/linux_container_environment_extensions.go index 6ba30f1d..6bf4c2b1 100644 --- a/act/container/linux_container_environment_extensions.go +++ b/act/container/linux_container_environment_extensions.go @@ -76,8 +76,8 @@ func (*LinuxContainerEnvironmentExtensions) JoinPathVariable(paths ...string) st return strings.Join(paths, ":") } -func (l *LinuxContainerEnvironmentExtensions) GetRunnerContext(ctx context.Context) map[string]interface{} { - return map[string]interface{}{ +func (l *LinuxContainerEnvironmentExtensions) GetRunnerContext(ctx context.Context) map[string]any { + return map[string]any{ "os": "Linux", "arch": RunnerArch(ctx), "temp": "/tmp", diff --git a/act/container/linux_container_environment_extensions_test.go b/act/container/linux_container_environment_extensions_test.go index 38111714..32034874 100644 --- a/act/container/linux_container_environment_extensions_test.go +++ b/act/container/linux_container_environment_extensions_test.go @@ -35,18 +35,13 @@ func TestContainerPath(t *testing.T) { {fmt.Sprintf("/mnt/%v/act", rootDriveLetter), "act", fmt.Sprintf("%s\\", rootDrive)}, } { if v.workDir != "" { - if err := os.Chdir(v.workDir); err != nil { - log.Error(err) - t.Fail() - } + t.Chdir(v.workDir) } assert.Equal(t, v.destinationPath, linuxcontainerext.ToContainerPath(v.sourcePath)) } - if err := os.Chdir(cwd); err != nil { - log.Error(err) - } + t.Chdir(cwd) } else { cwd, err := os.Getwd() if err != nil { diff --git a/act/container/parse_env_file.go b/act/container/parse_env_file.go index 227facb4..e65ae54a 100644 --- a/act/container/parse_env_file.go +++ b/act/container/parse_env_file.go @@ -8,7 +8,7 @@ import ( "io" "strings" - "code.forgejo.org/forgejo/runner/v9/act/common" + "code.forgejo.org/forgejo/runner/v11/act/common" ) func parseEnvFile(e Container, srcPath string, env *map[string]string) common.Executor { diff --git a/act/exprparser/functions.go b/act/exprparser/functions.go index 297df879..03bde85e 100644 --- a/act/exprparser/functions.go +++ b/act/exprparser/functions.go @@ -15,7 +15,7 @@ import ( "github.com/go-git/go-git/v5/plumbing/format/gitignore" - "code.forgejo.org/forgejo/runner/v9/act/model" + "code.forgejo.org/forgejo/runner/v11/act/model" "github.com/rhysd/actionlint" ) @@ -165,12 +165,12 @@ func (impl *interperterImpl) toJSON(value reflect.Value) (string, error) { return string(json), nil } -func (impl *interperterImpl) fromJSON(value reflect.Value) (interface{}, error) { +func (impl *interperterImpl) fromJSON(value reflect.Value) (any, error) { if value.Kind() != reflect.String { return nil, fmt.Errorf("Cannot parse non-string type %v as JSON", value.Kind()) } - var data interface{} + var data any err := json.Unmarshal([]byte(value.String()), &data) if err != nil { diff --git a/act/exprparser/functions_test.go b/act/exprparser/functions_test.go index e8525efa..6073ac3b 100644 --- a/act/exprparser/functions_test.go +++ b/act/exprparser/functions_test.go @@ -4,14 +4,14 @@ import ( "path/filepath" "testing" - "code.forgejo.org/forgejo/runner/v9/act/model" + "code.forgejo.org/forgejo/runner/v11/act/model" "github.com/stretchr/testify/assert" ) func TestFunctionContains(t *testing.T) { table := []struct { input string - expected interface{} + expected any name string }{ {"contains('search', 'item') }}", false, "contains-str-str"}, @@ -31,6 +31,13 @@ func TestFunctionContains(t *testing.T) { {`contains(fromJSON('[3.14,"second"]'), 3.14) }}`, true, "contains-item-number-number"}, {`contains(fromJSON('["","second"]'), fromJSON('[]')) }}`, false, "contains-item-str-arr"}, {`contains(fromJSON('["","second"]'), fromJSON('{}')) }}`, false, "contains-item-str-obj"}, + {`contains(fromJSON('[{ "first": { "result": "success" }},{ "second": { "result": "success" }}]').first.result, 'success') }}`, true, "multiple-contains-item"}, + {`contains(fromJSON('[{ "result": "success" },{ "result": "failure" }]').*.result, 'failure') }}`, true, "multiple-contains-dereferenced-failure-item"}, + {`contains(fromJSON('[{ "result": "failure" },{ "result": "success" }]').*.result, 'success') }}`, true, "multiple-contains-dereferenced-success-item"}, + {`contains(fromJSON('[{ "result": "failure" },{ "result": "success" }]').*.result, 'notthere') }}`, false, "multiple-contains-dereferenced-missing-item"}, + {`contains(fromJSON('[{ "result": "failure", "outputs": { "key": "val1" } },{ "result": "success", "outputs": { "key": "val2" } }]').*.outputs.key, 'val1') }}`, true, "multiple-contains-dereferenced-output-item"}, + {`contains(fromJSON('[{ "result": "failure", "outputs": { "key": "val1" } },{ "result": "success", "outputs": { "key": "val2" } }]').*.outputs.key, 'val2') }}`, true, "multiple-contains-dereferenced-output-item-2"}, + {`contains(fromJSON('[{ "result": "failure", "outputs": { "key": "val1" } },{ "result": "success", "outputs": { "key": "val2" } }]').*.outputs.key, 'missing') }}`, false, "multiple-contains-dereferenced-output-misssing-item"}, } env := &EvaluationEnvironment{} @@ -51,7 +58,7 @@ func TestFunctionContains(t *testing.T) { func TestFunctionStartsWith(t *testing.T) { table := []struct { input string - expected interface{} + expected any name string }{ {"startsWith('search', 'se') }}", true, "startswith-string"}, @@ -83,7 +90,7 @@ func TestFunctionStartsWith(t *testing.T) { func TestFunctionEndsWith(t *testing.T) { table := []struct { input string - expected interface{} + expected any name string }{ {"endsWith('search', 'ch') }}", true, "endsWith-string"}, @@ -115,7 +122,7 @@ func TestFunctionEndsWith(t *testing.T) { func TestFunctionJoin(t *testing.T) { table := []struct { input string - expected interface{} + expected any name string }{ {"join(fromJSON('[\"a\", \"b\"]'), ',')", "a,b", "join-arr"}, @@ -145,7 +152,7 @@ func TestFunctionJoin(t *testing.T) { func TestFunctionToJSON(t *testing.T) { table := []struct { input string - expected interface{} + expected any name string }{ {"toJSON(env) }}", "{\n \"key\": \"value\"\n}", "toJSON"}, @@ -174,10 +181,10 @@ func TestFunctionToJSON(t *testing.T) { func TestFunctionFromJSON(t *testing.T) { table := []struct { input string - expected interface{} + expected any name string }{ - {"fromJSON('{\"foo\":\"bar\"}') }}", map[string]interface{}{ + {"fromJSON('{\"foo\":\"bar\"}') }}", map[string]any{ "foo": "bar", }, "fromJSON"}, } @@ -200,7 +207,7 @@ func TestFunctionFromJSON(t *testing.T) { func TestFunctionHashFiles(t *testing.T) { table := []struct { input string - expected interface{} + expected any name string }{ {"hashFiles('**/non-extant-files') }}", "", "hash-non-existing-file"}, @@ -230,8 +237,8 @@ func TestFunctionHashFiles(t *testing.T) { func TestFunctionFormat(t *testing.T) { table := []struct { input string - expected interface{} - error interface{} + expected any + error any name string }{ {"format('text')", "text", nil, "format-plain-string"}, @@ -270,3 +277,23 @@ func TestFunctionFormat(t *testing.T) { _, err := NewInterpeter(env, Config{}).Evaluate("format()", DefaultStatusCheckNone) assert.Error(t, err) } + +func TestMapContains(t *testing.T) { + env := &EvaluationEnvironment{ + Needs: map[string]Needs{ + "first-job": { + Outputs: map[string]string{}, + Result: "success", + }, + "second-job": { + Outputs: map[string]string{}, + Result: "failure", + }, + }, + } + + output, err := NewInterpeter(env, Config{}).Evaluate("contains(needs.*.result, 'failure')", DefaultStatusCheckNone) + assert.NoError(t, err) + + assert.Equal(t, true, output) +} diff --git a/act/exprparser/interpreter.go b/act/exprparser/interpreter.go index 202ade22..26ad9d0e 100644 --- a/act/exprparser/interpreter.go +++ b/act/exprparser/interpreter.go @@ -7,7 +7,7 @@ import ( "reflect" "strings" - "code.forgejo.org/forgejo/runner/v9/act/model" + "code.forgejo.org/forgejo/runner/v11/act/model" "github.com/rhysd/actionlint" ) @@ -17,14 +17,14 @@ type EvaluationEnvironment struct { Job *model.JobContext Jobs *map[string]*model.WorkflowCallResult Steps map[string]*model.StepResult - Runner map[string]interface{} + Runner map[string]any Secrets map[string]string Vars map[string]string - Strategy map[string]interface{} - Matrix map[string]interface{} + Strategy map[string]any + Matrix map[string]any Needs map[string]Needs - Inputs map[string]interface{} - HashFiles func([]reflect.Value) (interface{}, error) + Inputs map[string]any + HashFiles func([]reflect.Value) (any, error) } type Needs struct { @@ -63,7 +63,7 @@ func (dsc DefaultStatusCheck) String() string { } type Interpreter interface { - Evaluate(input string, defaultStatusCheck DefaultStatusCheck) (interface{}, error) + Evaluate(input string, defaultStatusCheck DefaultStatusCheck) (any, error) } type interperterImpl struct { @@ -78,7 +78,7 @@ func NewInterpeter(env *EvaluationEnvironment, config Config) Interpreter { } } -func (impl *interperterImpl) Evaluate(input string, defaultStatusCheck DefaultStatusCheck) (interface{}, error) { +func (impl *interperterImpl) Evaluate(input string, defaultStatusCheck DefaultStatusCheck) (any, error) { input = strings.TrimPrefix(input, "${{") if defaultStatusCheck != DefaultStatusCheckNone && input == "" { input = "success()" @@ -117,7 +117,7 @@ func (impl *interperterImpl) Evaluate(input string, defaultStatusCheck DefaultSt return result, err2 } -func (impl *interperterImpl) evaluateNode(exprNode actionlint.ExprNode) (interface{}, error) { +func (impl *interperterImpl) evaluateNode(exprNode actionlint.ExprNode) (any, error) { switch node := exprNode.(type) { case *actionlint.VariableNode: return impl.evaluateVariable(node) @@ -150,7 +150,7 @@ func (impl *interperterImpl) evaluateNode(exprNode actionlint.ExprNode) (interfa } } -func (impl *interperterImpl) evaluateVariable(variableNode *actionlint.VariableNode) (interface{}, error) { +func (impl *interperterImpl) evaluateVariable(variableNode *actionlint.VariableNode) (any, error) { switch strings.ToLower(variableNode.Name) { case "github": return impl.env.Github, nil @@ -158,6 +158,8 @@ func (impl *interperterImpl) evaluateVariable(variableNode *actionlint.VariableN return impl.env.Github, nil case "forge": return impl.env.Github, nil + case "forgejo": + return impl.env.Github, nil case "env": return impl.env.Env, nil case "job": @@ -192,7 +194,7 @@ func (impl *interperterImpl) evaluateVariable(variableNode *actionlint.VariableN } } -func (impl *interperterImpl) evaluateIndexAccess(indexAccessNode *actionlint.IndexAccessNode) (interface{}, error) { +func (impl *interperterImpl) evaluateIndexAccess(indexAccessNode *actionlint.IndexAccessNode) (any, error) { left, err := impl.evaluateNode(indexAccessNode.Operand) if err != nil { return nil, err @@ -227,16 +229,20 @@ func (impl *interperterImpl) evaluateIndexAccess(indexAccessNode *actionlint.Ind } } -func (impl *interperterImpl) evaluateObjectDeref(objectDerefNode *actionlint.ObjectDerefNode) (interface{}, error) { +func (impl *interperterImpl) evaluateObjectDeref(objectDerefNode *actionlint.ObjectDerefNode) (any, error) { left, err := impl.evaluateNode(objectDerefNode.Receiver) if err != nil { return nil, err } + _, receiverIsDeref := objectDerefNode.Receiver.(*actionlint.ArrayDerefNode) + if receiverIsDeref { + return impl.getPropertyValueDereferenced(reflect.ValueOf(left), objectDerefNode.Property) + } return impl.getPropertyValue(reflect.ValueOf(left), objectDerefNode.Property) } -func (impl *interperterImpl) evaluateArrayDeref(arrayDerefNode *actionlint.ArrayDerefNode) (interface{}, error) { +func (impl *interperterImpl) evaluateArrayDeref(arrayDerefNode *actionlint.ArrayDerefNode) (any, error) { left, err := impl.evaluateNode(arrayDerefNode.Receiver) if err != nil { return nil, err @@ -245,7 +251,7 @@ func (impl *interperterImpl) evaluateArrayDeref(arrayDerefNode *actionlint.Array return impl.getSafeValue(reflect.ValueOf(left)), nil } -func (impl *interperterImpl) getPropertyValue(left reflect.Value, property string) (value interface{}, err error) { +func (impl *interperterImpl) getPropertyValue(left reflect.Value, property string) (value any, err error) { switch left.Kind() { case reflect.Ptr: return impl.getPropertyValue(left.Elem(), property) @@ -299,7 +305,7 @@ func (impl *interperterImpl) getPropertyValue(left reflect.Value, property strin return nil, nil case reflect.Slice: - var values []interface{} + var values []any for i := 0; i < left.Len(); i++ { value, err := impl.getPropertyValue(left.Index(i).Elem(), property) @@ -316,7 +322,30 @@ func (impl *interperterImpl) getPropertyValue(left reflect.Value, property strin return nil, nil } -func (impl *interperterImpl) getMapValue(value reflect.Value) (interface{}, error) { +func (impl *interperterImpl) getPropertyValueDereferenced(left reflect.Value, property string) (value any, err error) { + switch left.Kind() { + case reflect.Map: + iter := left.MapRange() + + var values []any + for iter.Next() { + value, err := impl.getPropertyValue(iter.Value(), property) + if err != nil { + return nil, err + } + + values = append(values, value) + } + + return values, nil + case reflect.Ptr, reflect.Struct, reflect.Slice: + return impl.getPropertyValue(left, property) + } + + return nil, nil +} + +func (impl *interperterImpl) getMapValue(value reflect.Value) (any, error) { if value.Kind() == reflect.Ptr { return impl.getMapValue(value.Elem()) } @@ -324,7 +353,7 @@ func (impl *interperterImpl) getMapValue(value reflect.Value) (interface{}, erro return value.Interface(), nil } -func (impl *interperterImpl) evaluateNot(notNode *actionlint.NotOpNode) (interface{}, error) { +func (impl *interperterImpl) evaluateNot(notNode *actionlint.NotOpNode) (any, error) { operand, err := impl.evaluateNode(notNode.Operand) if err != nil { return nil, err @@ -333,7 +362,7 @@ func (impl *interperterImpl) evaluateNot(notNode *actionlint.NotOpNode) (interfa return !IsTruthy(operand), nil } -func (impl *interperterImpl) evaluateCompare(compareNode *actionlint.CompareOpNode) (interface{}, error) { +func (impl *interperterImpl) evaluateCompare(compareNode *actionlint.CompareOpNode) (any, error) { left, err := impl.evaluateNode(compareNode.Left) if err != nil { return nil, err @@ -350,7 +379,7 @@ func (impl *interperterImpl) evaluateCompare(compareNode *actionlint.CompareOpNo return impl.compareValues(leftValue, rightValue, compareNode.Kind) } -func (impl *interperterImpl) compareValues(leftValue, rightValue reflect.Value, kind actionlint.CompareOpNodeKind) (interface{}, error) { +func (impl *interperterImpl) compareValues(leftValue, rightValue reflect.Value, kind actionlint.CompareOpNodeKind) (any, error) { if leftValue.Kind() != rightValue.Kind() { if !impl.isNumber(leftValue) { leftValue = impl.coerceToNumber(leftValue) @@ -500,7 +529,7 @@ func (impl *interperterImpl) compareNumber(left, right float64, kind actionlint. } } -func IsTruthy(input interface{}) bool { +func IsTruthy(input any) bool { value := reflect.ValueOf(input) switch value.Kind() { case reflect.Bool: @@ -536,7 +565,7 @@ func (impl *interperterImpl) isNumber(value reflect.Value) bool { } } -func (impl *interperterImpl) getSafeValue(value reflect.Value) interface{} { +func (impl *interperterImpl) getSafeValue(value reflect.Value) any { switch value.Kind() { case reflect.Invalid: return nil @@ -550,7 +579,7 @@ func (impl *interperterImpl) getSafeValue(value reflect.Value) interface{} { return value.Interface() } -func (impl *interperterImpl) evaluateLogicalCompare(compareNode *actionlint.LogicalOpNode) (interface{}, error) { +func (impl *interperterImpl) evaluateLogicalCompare(compareNode *actionlint.LogicalOpNode) (any, error) { left, err := impl.evaluateNode(compareNode.Left) if err != nil { return nil, err @@ -579,7 +608,7 @@ func (impl *interperterImpl) evaluateLogicalCompare(compareNode *actionlint.Logi return nil, fmt.Errorf("Unable to compare incompatibles types '%s' and '%s'", leftValue.Kind(), rightValue.Kind()) } -func (impl *interperterImpl) evaluateFuncCall(funcCallNode *actionlint.FuncCallNode) (interface{}, error) { +func (impl *interperterImpl) evaluateFuncCall(funcCallNode *actionlint.FuncCallNode) (any, error) { args := make([]reflect.Value, 0) for _, arg := range funcCallNode.Args { diff --git a/act/exprparser/interpreter_test.go b/act/exprparser/interpreter_test.go index 4a7741fa..883ef84a 100644 --- a/act/exprparser/interpreter_test.go +++ b/act/exprparser/interpreter_test.go @@ -4,14 +4,14 @@ import ( "math" "testing" - "code.forgejo.org/forgejo/runner/v9/act/model" + "code.forgejo.org/forgejo/runner/v11/act/model" "github.com/stretchr/testify/assert" ) func TestLiterals(t *testing.T) { table := []struct { input string - expected interface{} + expected any name string }{ {"true", true, "true"}, @@ -40,7 +40,7 @@ func TestLiterals(t *testing.T) { func TestOperators(t *testing.T) { table := []struct { input string - expected interface{} + expected any name string error string }{ @@ -68,7 +68,7 @@ func TestOperators(t *testing.T) { {`true && false`, false, "and", ""}, {`true || false`, true, "or", ""}, {`fromJSON('{}') && true`, true, "and-boolean-object", ""}, - {`fromJSON('{}') || false`, make(map[string]interface{}), "or-boolean-object", ""}, + {`fromJSON('{}') || false`, make(map[string]any), "or-boolean-object", ""}, {"github.event.commits[0].author.username != github.event.commits[1].author.username", true, "property-comparison1", ""}, {"github.event.commits[0].author.username1 != github.event.commits[1].author.username", true, "property-comparison2", ""}, {"github.event.commits[0].author.username != github.event.commits[1].author.username1", true, "property-comparison3", ""}, @@ -79,15 +79,15 @@ func TestOperators(t *testing.T) { env := &EvaluationEnvironment{ Github: &model.GithubContext{ Action: "push", - Event: map[string]interface{}{ - "commits": []interface{}{ - map[string]interface{}{ - "author": map[string]interface{}{ + Event: map[string]any{ + "commits": []any{ + map[string]any{ + "author": map[string]any{ "username": "someone", }, }, - map[string]interface{}{ - "author": map[string]interface{}{ + map[string]any{ + "author": map[string]any{ "username": "someone-else", }, }, @@ -114,7 +114,7 @@ func TestOperators(t *testing.T) { func TestOperatorsCompare(t *testing.T) { table := []struct { input string - expected interface{} + expected any name string }{ {"!null", true, "not-null"}, @@ -162,7 +162,7 @@ func TestOperatorsCompare(t *testing.T) { func TestOperatorsBooleanEvaluation(t *testing.T) { table := []struct { input string - expected interface{} + expected any name string }{ // true && @@ -529,7 +529,7 @@ func TestOperatorsBooleanEvaluation(t *testing.T) { func TestContexts(t *testing.T) { table := []struct { input string - expected interface{} + expected any name string }{ {"github.action", "push", "github-context"}, @@ -562,6 +562,8 @@ func TestContexts(t *testing.T) { {"matrix.os", "Linux", "matrix-context"}, {"needs.job-id.outputs.output-name", "value", "needs-context"}, {"needs.job-id.result", "success", "needs-context"}, + {"contains(needs.*.result, 'success')", true, "needs-wildcard-context-contains-success"}, + {"contains(needs.*.result, 'failure')", false, "needs-wildcard-context-contains-failure"}, {"inputs.name", "value", "inputs-context"}, } @@ -586,7 +588,7 @@ func TestContexts(t *testing.T) { Conclusion: model.StepStatusSkipped, }, }, - Runner: map[string]interface{}{ + Runner: map[string]any{ "os": "Linux", "temp": "/tmp", "tool_cache": "/opt/hostedtoolcache", @@ -597,10 +599,10 @@ func TestContexts(t *testing.T) { Vars: map[string]string{ "name": "value", }, - Strategy: map[string]interface{}{ + Strategy: map[string]any{ "fail-fast": true, }, - Matrix: map[string]interface{}{ + Matrix: map[string]any{ "os": "Linux", }, Needs: map[string]Needs{ @@ -610,8 +612,14 @@ func TestContexts(t *testing.T) { }, Result: "success", }, + "another-job-id": { + Outputs: map[string]string{ + "output-name": "value", + }, + Result: "success", + }, }, - Inputs: map[string]interface{}{ + Inputs: map[string]any{ "name": "value", }, } diff --git a/act/filecollector/file_collector_test.go b/act/filecollector/file_collector_test.go index 45ec6594..0437c034 100644 --- a/act/filecollector/file_collector_test.go +++ b/act/filecollector/file_collector_test.go @@ -2,7 +2,6 @@ package filecollector import ( "archive/tar" - "context" "io" "path/filepath" "strings" @@ -27,7 +26,7 @@ func (mfs *memoryFs) walk(root string, fn filepath.WalkFunc) error { if err != nil { return err } - for i := 0; i < len(dir); i++ { + for i := range dir { filename := filepath.Join(root, dir[i].Name()) err = fn(filename, dir[i], nil) if dir[i].IsDir() { @@ -104,7 +103,7 @@ func TestIgnoredTrackedfile(t *testing.T) { TarWriter: tw, }, } - err := fc.Fs.Walk("mygitrepo", fc.CollectFiles(context.Background(), []string{})) + err := fc.Fs.Walk("mygitrepo", fc.CollectFiles(t.Context(), []string{})) assert.NoError(t, err, "successfully collect files") tw.Close() _, _ = tmpTar.Seek(0, io.SeekStart) @@ -153,7 +152,7 @@ func TestSymlinks(t *testing.T) { TarWriter: tw, }, } - err = fc.Fs.Walk("mygitrepo", fc.CollectFiles(context.Background(), []string{})) + err = fc.Fs.Walk("mygitrepo", fc.CollectFiles(t.Context(), []string{})) assert.NoError(t, err, "successfully collect files") tw.Close() _, _ = tmpTar.Seek(0, io.SeekStart) diff --git a/act/jobparser/evaluator.go b/act/jobparser/evaluator.go index 16f520b1..99230256 100644 --- a/act/jobparser/evaluator.go +++ b/act/jobparser/evaluator.go @@ -5,8 +5,8 @@ import ( "regexp" "strings" - "code.forgejo.org/forgejo/runner/v9/act/exprparser" - "gopkg.in/yaml.v3" + "code.forgejo.org/forgejo/runner/v11/act/exprparser" + "go.yaml.in/yaml/v3" ) // ExpressionEvaluator is copied from runner.expressionEvaluator, @@ -19,7 +19,7 @@ func NewExpressionEvaluator(interpreter exprparser.Interpreter) *ExpressionEvalu return &ExpressionEvaluator{interpreter: interpreter} } -func (ee ExpressionEvaluator) evaluate(in string, defaultStatusCheck exprparser.DefaultStatusCheck) (interface{}, error) { +func (ee ExpressionEvaluator) evaluate(in string, defaultStatusCheck exprparser.DefaultStatusCheck) (any, error) { evaluated, err := ee.interpreter.Evaluate(in, defaultStatusCheck) return evaluated, err diff --git a/act/jobparser/interpeter.go b/act/jobparser/interpeter.go index f798121a..362e18cc 100644 --- a/act/jobparser/interpeter.go +++ b/act/jobparser/interpeter.go @@ -1,9 +1,9 @@ package jobparser import ( - "code.forgejo.org/forgejo/runner/v9/act/exprparser" - "code.forgejo.org/forgejo/runner/v9/act/model" - "gopkg.in/yaml.v3" + "code.forgejo.org/forgejo/runner/v11/act/exprparser" + "code.forgejo.org/forgejo/runner/v11/act/model" + "go.yaml.in/yaml/v3" ) // NewInterpeter returns an interpeter used in the server, @@ -12,12 +12,13 @@ import ( func NewInterpeter( jobID string, job *model.Job, - matrix map[string]interface{}, + matrix map[string]any, gitCtx *model.GithubContext, results map[string]*JobResult, vars map[string]string, + inputs map[string]any, ) exprparser.Interpreter { - strategy := make(map[string]interface{}) + strategy := make(map[string]any) if job.Strategy != nil { strategy["fail-fast"] = job.Strategy.FailFast strategy["max-parallel"] = job.Strategy.MaxParallel @@ -62,7 +63,7 @@ func NewInterpeter( Strategy: strategy, Matrix: matrix, Needs: using, - Inputs: nil, // not supported yet + Inputs: inputs, Vars: vars, } @@ -75,6 +76,36 @@ func NewInterpeter( return exprparser.NewInterpeter(ee, config) } +// Returns an interpeter used in the server in the context of workflow-level templates. Needs github, inputs, and vars +// context only. +func NewWorkflowInterpeter( + gitCtx *model.GithubContext, + vars map[string]string, + inputs map[string]any, +) exprparser.Interpreter { + ee := &exprparser.EvaluationEnvironment{ + Github: gitCtx, + Env: nil, // no need + Job: nil, // no need + Steps: nil, // no need + Runner: nil, // no need + Secrets: nil, // no need + Strategy: nil, // no need + Matrix: nil, // no need + Needs: nil, // no need + Inputs: inputs, + Vars: vars, + } + + config := exprparser.Config{ + Run: nil, + WorkingDir: "", // WorkingDir is used for the function hashFiles, but it's not needed in the server + Context: "workflow", + } + + return exprparser.NewInterpeter(ee, config) +} + // JobResult is the minimum requirement of job results for Interpeter type JobResult struct { Needs []string diff --git a/act/jobparser/jobparser.go b/act/jobparser/jobparser.go index 0fd1768e..3542236f 100644 --- a/act/jobparser/jobparser.go +++ b/act/jobparser/jobparser.go @@ -6,9 +6,9 @@ import ( "sort" "strings" - "gopkg.in/yaml.v3" + "go.yaml.in/yaml/v3" - "code.forgejo.org/forgejo/runner/v9/act/model" + "code.forgejo.org/forgejo/runner/v11/act/model" ) func Parse(content []byte, validate bool, options ...ParseOption) ([]*SingleWorkflow, error) { @@ -48,7 +48,7 @@ func Parse(content []byte, validate bool, options ...ParseOption) ([]*SingleWork } for _, matrix := range matricxes { job := job.Clone() - evaluator := NewExpressionEvaluator(NewInterpeter(id, origin.GetJob(id), matrix, pc.gitContext, results, pc.vars)) + evaluator := NewExpressionEvaluator(NewInterpeter(id, origin.GetJob(id), matrix, pc.gitContext, results, pc.vars, nil)) if job.Name == "" { job.Name = nameWithMatrix(id, matrix) } else { @@ -103,7 +103,7 @@ type parseContext struct { type ParseOption func(c *parseContext) -func getMatrixes(job *model.Job) ([]map[string]interface{}, error) { +func getMatrixes(job *model.Job) ([]map[string]any, error) { ret, err := job.GetMatrixes() if err != nil { return nil, fmt.Errorf("GetMatrixes: %w", err) @@ -114,13 +114,13 @@ func getMatrixes(job *model.Job) ([]map[string]interface{}, error) { return ret, nil } -func encodeMatrix(matrix map[string]interface{}) yaml.Node { +func encodeMatrix(matrix map[string]any) yaml.Node { if len(matrix) == 0 { return yaml.Node{} } - value := map[string][]interface{}{} + value := map[string][]any{} for k, v := range matrix { - value[k] = []interface{}{v} + value[k] = []any{v} } node := yaml.Node{} _ = node.Encode(value) @@ -137,7 +137,7 @@ func encodeRunsOn(runsOn []string) yaml.Node { return node } -func nameWithMatrix(name string, m map[string]interface{}) string { +func nameWithMatrix(name string, m map[string]any) string { if len(m) == 0 { return name } @@ -145,7 +145,7 @@ func nameWithMatrix(name string, m map[string]interface{}) string { return name + " " + matrixName(m) } -func matrixName(m map[string]interface{}) string { +func matrixName(m map[string]any) string { ks := make([]string, 0, len(m)) for k := range m { ks = append(ks, k) diff --git a/act/jobparser/jobparser_test.go b/act/jobparser/jobparser_test.go index 02b58def..4ae0a190 100644 --- a/act/jobparser/jobparser_test.go +++ b/act/jobparser/jobparser_test.go @@ -8,7 +8,7 @@ import ( "github.com/stretchr/testify/require" - "gopkg.in/yaml.v3" + "go.yaml.in/yaml/v3" ) func TestParse(t *testing.T) { @@ -42,6 +42,16 @@ func TestParse(t *testing.T) { options: nil, wantErr: false, }, + { + name: "job_concurrency", + options: nil, + wantErr: false, + }, + { + name: "job_concurrency_eval", + options: nil, + wantErr: false, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/act/jobparser/model.go b/act/jobparser/model.go index ac1d871d..74de0450 100644 --- a/act/jobparser/model.go +++ b/act/jobparser/model.go @@ -2,9 +2,10 @@ package jobparser import ( "fmt" + "strings" - "code.forgejo.org/forgejo/runner/v9/act/model" - "gopkg.in/yaml.v3" + "code.forgejo.org/forgejo/runner/v11/act/model" + "go.yaml.in/yaml/v3" ) // SingleWorkflow is a workflow with single job and single matrix @@ -80,8 +81,9 @@ type Job struct { Defaults Defaults `yaml:"defaults,omitempty"` Outputs map[string]string `yaml:"outputs,omitempty"` Uses string `yaml:"uses,omitempty"` - With map[string]interface{} `yaml:"with,omitempty"` + With map[string]any `yaml:"with,omitempty"` RawSecrets yaml.Node `yaml:"secrets,omitempty"` + RawConcurrency *model.RawConcurrency `yaml:"concurrency,omitempty"` } func (j *Job) Clone() *Job { @@ -104,6 +106,7 @@ func (j *Job) Clone() *Job { Uses: j.Uses, With: j.With, RawSecrets: j.RawSecrets, + RawConcurrency: j.RawConcurrency, } } @@ -190,35 +193,63 @@ func (evt *Event) Schedules() []map[string]string { return evt.schedules } +// Convert the raw YAML from the `concurrency` block on a workflow into the evaluated concurrency group and +// cancel-in-progress value. This implementation only supports workflow-level concurrency definition, where we expect +// expressions to be able to access only the github, inputs and vars contexts. If RawConcurrency is empty, then the +// returned concurrency group will be "" and cancel-in-progress will be nil -- this can be used to distinguish from an +// explicit cancel-in-progress choice even if a group isn't specified. +func EvaluateWorkflowConcurrency(rc *model.RawConcurrency, gitCtx *model.GithubContext, vars map[string]string, inputs map[string]any) (string, *bool, error) { + evaluator := NewExpressionEvaluator(NewWorkflowInterpeter(gitCtx, vars, inputs)) + var node yaml.Node + if err := node.Encode(rc); err != nil { + return "", nil, fmt.Errorf("failed to encode concurrency: %w", err) + } + if err := evaluator.EvaluateYamlNode(&node); err != nil { + return "", nil, fmt.Errorf("failed to evaluate concurrency: %w", err) + } + var evaluated model.RawConcurrency + if err := node.Decode(&evaluated); err != nil { + return "", nil, fmt.Errorf("failed to unmarshal evaluated concurrency: %w", err) + } + if evaluated.RawExpression != "" { + return evaluated.RawExpression, nil, nil + } + if evaluated.CancelInProgress == "" { + return evaluated.Group, nil, nil + } + cancelInProgress := evaluated.CancelInProgress == "true" + return evaluated.Group, &cancelInProgress, nil +} + func ParseRawOn(rawOn *yaml.Node) ([]*Event, error) { switch rawOn.Kind { case yaml.ScalarNode: var val string err := rawOn.Decode(&val) if err != nil { - return nil, err + return nil, fmt.Errorf("unable to interpret scalar value into a string: %w", err) } return []*Event{ {Name: val}, }, nil case yaml.SequenceNode: - var val []interface{} + var val []any err := rawOn.Decode(&val) if err != nil { return nil, err } res := make([]*Event, 0, len(val)) - for _, v := range val { + for i, v := range val { switch t := v.(type) { case string: res = append(res, &Event{Name: t}) default: - return nil, fmt.Errorf("invalid type %T", t) + return nil, fmt.Errorf("value at index %d was unexpected type %[2]T; must be a string but was %#[2]v", i, v) } } return res, nil case yaml.MappingNode: - events, triggers, err := parseMappingNode[interface{}](rawOn) + events, triggers, err := parseMappingNode[any](rawOn) if err != nil { return nil, err } @@ -233,17 +264,7 @@ func ParseRawOn(rawOn *yaml.Node) ([]*Event, error) { continue } switch t := v.(type) { - case string: - res = append(res, &Event{ - Name: k, - acts: map[string][]string{}, - }) - case []string: - res = append(res, &Event{ - Name: k, - acts: map[string][]string{}, - }) - case map[string]interface{}: + case map[string]any: acts := make(map[string][]string, len(t)) for act, branches := range t { switch b := branches.(type) { @@ -251,20 +272,20 @@ func ParseRawOn(rawOn *yaml.Node) ([]*Event, error) { acts[act] = []string{b} case []string: acts[act] = b - case []interface{}: + case []any: acts[act] = make([]string, len(b)) for i, v := range b { var ok bool if acts[act][i], ok = v.(string); !ok { - return nil, fmt.Errorf("unknown on type: %#v", branches) + return nil, fmt.Errorf("key %q.%q index %d had unexpected type %[4]T; a string was expected but was %#[4]v", k, act, i, v) } } - case map[string]interface{}: - if isInvalidOnType(k, act) { - return nil, fmt.Errorf("unknown on type: %#v", v) + case map[string]any: + if err := isInvalidOnType(k, act); err != nil { + return nil, fmt.Errorf("invalid value on key %q: %w", k, err) } default: - return nil, fmt.Errorf("unknown on type: %#v", branches) + return nil, fmt.Errorf("key %q.%q had unexpected type %T; was %#v", k, act, branches, branches) } } if k == "workflow_dispatch" || k == "workflow_call" { @@ -274,21 +295,24 @@ func ParseRawOn(rawOn *yaml.Node) ([]*Event, error) { Name: k, acts: acts, }) - case []interface{}: + case []any: if k != "schedule" { - return nil, fmt.Errorf("unknown on type: %#v", v) + return nil, fmt.Errorf("key %q had an type %T; only the 'schedule' key is expected with this type", k, v) } schedules := make([]map[string]string, len(t)) for i, tt := range t { - vv, ok := tt.(map[string]interface{}) + vv, ok := tt.(map[string]any) if !ok { - return nil, fmt.Errorf("unknown on type: %#v", v) + return nil, fmt.Errorf("key %q[%d] had unexpected type %[3]T; a map with a key \"cron\" was expected, but value was %#[3]v", k, i, tt) } schedules[i] = make(map[string]string, len(vv)) - for k, vvv := range vv { + for kk, vvv := range vv { + if strings.ToLower(kk) != "cron" { + return nil, fmt.Errorf("key %q[%d] had unexpected key %q; \"cron\" was expected", k, i, kk) + } var ok bool - if schedules[i][k], ok = vvv.(string); !ok { - return nil, fmt.Errorf("unknown on type: %#v", v) + if schedules[i][kk], ok = vvv.(string); !ok { + return nil, fmt.Errorf("key %q[%d].%q had unexpected type %[4]T; a string was expected by was %#[4]v", k, i, kk, vvv) } } } @@ -297,23 +321,29 @@ func ParseRawOn(rawOn *yaml.Node) ([]*Event, error) { schedules: schedules, }) default: - return nil, fmt.Errorf("unknown on type: %#v", v) + return nil, fmt.Errorf("key %q had unexpected type %[2]T; expected a map or array but was %#[2]v", k, v) } } return res, nil default: - return nil, fmt.Errorf("unknown on type: %v", rawOn.Kind) + return nil, fmt.Errorf("unexpected yaml node in `on`: %v", rawOn.Kind) } } -func isInvalidOnType(onType, subKey string) bool { - if onType == "workflow_dispatch" && subKey == "inputs" { - return false +func isInvalidOnType(onType, subKey string) error { + if onType == "workflow_dispatch" { + if subKey == "inputs" { + return nil + } + return fmt.Errorf("workflow_dispatch only supports key \"inputs\", but key %q was found", subKey) } - if onType == "workflow_call" && (subKey == "inputs" || subKey == "outputs") { - return false + if onType == "workflow_call" { + if subKey == "inputs" || subKey == "outputs" { + return nil + } + return fmt.Errorf("workflow_call only supports keys \"inputs\" and \"outputs\", but key %q was found", subKey) } - return true + return fmt.Errorf("unexpected key %q.%q", onType, subKey) } // parseMappingNode parse a mapping node and preserve order. diff --git a/act/jobparser/model_test.go b/act/jobparser/model_test.go index 30dc4890..454de71c 100644 --- a/act/jobparser/model_test.go +++ b/act/jobparser/model_test.go @@ -5,17 +5,18 @@ import ( "strings" "testing" - "code.forgejo.org/forgejo/runner/v9/act/model" + "code.forgejo.org/forgejo/runner/v11/act/model" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "gopkg.in/yaml.v3" + "go.yaml.in/yaml/v3" ) func TestParseRawOn(t *testing.T) { kases := []struct { input string result []*Event + err string }{ { input: "on: issue_comment", @@ -33,7 +34,10 @@ func TestParseRawOn(t *testing.T) { }, }, }, - + { + input: "on: [123]", + err: "value at index 0 was unexpected type int; must be a string but was 123", + }, { input: "on:\n - push\n - pull_request", result: []*Event{ @@ -45,6 +49,19 @@ func TestParseRawOn(t *testing.T) { }, }, }, + { + input: "on: { push: null }", + result: []*Event{ + { + Name: "push", + acts: map[string][]string{}, + }, + }, + }, + { + input: "on: { push: 'abc' }", + err: "key \"push\" had unexpected type string; expected a map or array but was \"abc\"", + }, { input: "on:\n push:\n branches:\n - master", result: []*Event{ @@ -72,6 +89,10 @@ func TestParseRawOn(t *testing.T) { }, }, }, + { + input: "on:\n branch_protection_rule:\n types: [123, deleted]", + err: "key \"branch_protection_rule\".\"types\" index 0 had unexpected type int; a string was expected but was 123", + }, { input: "on:\n project:\n types: [created, deleted]\n milestone:\n types: [opened, deleted]", result: []*Event{ @@ -189,6 +210,22 @@ func TestParseRawOn(t *testing.T) { }, }, }, + { + input: "on:\n schedule2:\n - cron: '20 6 * * *'", + err: "key \"schedule2\" had an type []interface {}; only the 'schedule' key is expected with this type", + }, + { + input: "on:\n schedule:\n - 123", + err: "key \"schedule\"[0] had unexpected type int; a map with a key \"cron\" was expected, but value was 123", + }, + { + input: "on:\n schedule:\n - corn: '20 6 * * *'", + err: "key \"schedule\"[0] had unexpected key \"corn\"; \"cron\" was expected", + }, + { + input: "on:\n schedule:\n - cron: 123", + err: "key \"schedule\"[0].\"cron\" had unexpected type int; a string was expected by was 123", + }, { input: ` on: @@ -222,15 +259,37 @@ on: }, }, }, + { + input: ` +on: + workflow_call: + mistake: + access-token: + description: 'A token passed from the caller workflow' + required: false +`, + err: "invalid value on key \"workflow_call\": workflow_call only supports keys \"inputs\" and \"outputs\", but key \"mistake\" was found", + }, + { + input: ` +on: + workflow_call: { map: 123 } +`, + err: "key \"workflow_call\".\"map\" had unexpected type int; was 123", + }, } for _, kase := range kases { t.Run(kase.input, func(t *testing.T) { origin, err := model.ReadWorkflow(strings.NewReader(kase.input), false) - assert.NoError(t, err) + require.NoError(t, err) events, err := ParseRawOn(&origin.RawOn) - assert.NoError(t, err) - assert.EqualValues(t, kase.result, events, fmt.Sprintf("%#v", events)) + if kase.err != "" { + assert.ErrorContains(t, err, kase.err) + } else { + assert.NoError(t, err) + assert.EqualValues(t, kase.result, events, fmt.Sprintf("%#v", events)) + } }) } } @@ -261,66 +320,66 @@ func TestParseMappingNode(t *testing.T) { tests := []struct { input string scalars []string - datas []interface{} + datas []any }{ { input: "on:\n push:\n branches:\n - master", scalars: []string{"push"}, - datas: []interface{}{ - map[string]interface{}{ - "branches": []interface{}{"master"}, + datas: []any{ + map[string]any{ + "branches": []any{"master"}, }, }, }, { input: "on:\n branch_protection_rule:\n types: [created, deleted]", scalars: []string{"branch_protection_rule"}, - datas: []interface{}{ - map[string]interface{}{ - "types": []interface{}{"created", "deleted"}, + datas: []any{ + map[string]any{ + "types": []any{"created", "deleted"}, }, }, }, { input: "on:\n project:\n types: [created, deleted]\n milestone:\n types: [opened, deleted]", scalars: []string{"project", "milestone"}, - datas: []interface{}{ - map[string]interface{}{ - "types": []interface{}{"created", "deleted"}, + datas: []any{ + map[string]any{ + "types": []any{"created", "deleted"}, }, - map[string]interface{}{ - "types": []interface{}{"opened", "deleted"}, + map[string]any{ + "types": []any{"opened", "deleted"}, }, }, }, { input: "on:\n pull_request:\n types:\n - opened\n branches:\n - 'releases/**'", scalars: []string{"pull_request"}, - datas: []interface{}{ - map[string]interface{}{ - "types": []interface{}{"opened"}, - "branches": []interface{}{"releases/**"}, + datas: []any{ + map[string]any{ + "types": []any{"opened"}, + "branches": []any{"releases/**"}, }, }, }, { input: "on:\n push:\n branches:\n - main\n pull_request:\n types:\n - opened\n branches:\n - '**'", scalars: []string{"push", "pull_request"}, - datas: []interface{}{ - map[string]interface{}{ - "branches": []interface{}{"main"}, + datas: []any{ + map[string]any{ + "branches": []any{"main"}, }, - map[string]interface{}{ - "types": []interface{}{"opened"}, - "branches": []interface{}{"**"}, + map[string]any{ + "types": []any{"opened"}, + "branches": []any{"**"}, }, }, }, { input: "on:\n schedule:\n - cron: '20 6 * * *'", scalars: []string{"schedule"}, - datas: []interface{}{ - []interface{}{map[string]interface{}{ + datas: []any{ + []any{map[string]any{ "cron": "20 6 * * *", }}, }, @@ -332,10 +391,137 @@ func TestParseMappingNode(t *testing.T) { workflow, err := model.ReadWorkflow(strings.NewReader(test.input), false) assert.NoError(t, err) - scalars, datas, err := parseMappingNode[interface{}](&workflow.RawOn) + scalars, datas, err := parseMappingNode[any](&workflow.RawOn) assert.NoError(t, err) assert.EqualValues(t, test.scalars, scalars, fmt.Sprintf("%#v", scalars)) assert.EqualValues(t, test.datas, datas, fmt.Sprintf("%#v", datas)) }) } } + +func TestEvaluateConcurrency(t *testing.T) { + tests := []struct { + name string + input model.RawConcurrency + group string + cancelInProgressNil bool + cancelInProgress bool + }{ + { + name: "basic", + input: model.RawConcurrency{ + Group: "group-name", + CancelInProgress: "true", + }, + group: "group-name", + cancelInProgress: true, + }, + { + name: "undefined", + input: model.RawConcurrency{}, + group: "", + cancelInProgressNil: true, + }, + { + name: "group-evaluation", + input: model.RawConcurrency{ + Group: "${{ github.workflow }}-${{ github.ref }}", + }, + group: "test_workflow-main", + cancelInProgressNil: true, + }, + { + name: "cancel-evaluation-true", + input: model.RawConcurrency{ + Group: "group-name", + CancelInProgress: "${{ !contains(github.ref, 'release/')}}", + }, + group: "group-name", + cancelInProgress: true, + }, + { + name: "cancel-evaluation-false", + input: model.RawConcurrency{ + Group: "group-name", + CancelInProgress: "${{ contains(github.ref, 'release/')}}", + }, + group: "group-name", + cancelInProgress: false, + }, + { + name: "event-evaluation", + input: model.RawConcurrency{ + Group: "user-${{ github.event.commits[0].author.username }}", + }, + group: "user-someone", + cancelInProgressNil: true, + }, + { + name: "arbitrary-var", + input: model.RawConcurrency{ + Group: "${{ vars.eval_arbitrary_var }}", + }, + group: "123", + cancelInProgressNil: true, + }, + { + name: "arbitrary-input", + input: model.RawConcurrency{ + Group: "${{ inputs.eval_arbitrary_input }}", + }, + group: "456", + cancelInProgressNil: true, + }, + { + name: "cancel-in-progress-only", + input: model.RawConcurrency{ + CancelInProgress: "true", + }, + group: "", + cancelInProgress: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + group, cancelInProgress, err := EvaluateWorkflowConcurrency( + &test.input, + // gitCtx + &model.GithubContext{ + Workflow: "test_workflow", + Ref: "main", + Event: map[string]any{ + "commits": []any{ + map[string]any{ + "author": map[string]any{ + "username": "someone", + }, + }, + map[string]any{ + "author": map[string]any{ + "username": "someone-else", + }, + }, + }, + }, + }, + // vars + map[string]string{ + "eval_arbitrary_var": "123", + }, + // inputs + map[string]any{ + "eval_arbitrary_input": "456", + }, + ) + assert.NoError(t, err) + assert.EqualValues(t, test.group, group) + if test.cancelInProgressNil { + assert.Nil(t, cancelInProgress) + } else { + require.NotNil(t, cancelInProgress) + assert.EqualValues(t, test.cancelInProgress, *cancelInProgress) + } + }) + } +} diff --git a/act/jobparser/testdata/job_concurrency.in.yaml b/act/jobparser/testdata/job_concurrency.in.yaml new file mode 100644 index 00000000..a2e8e1d9 --- /dev/null +++ b/act/jobparser/testdata/job_concurrency.in.yaml @@ -0,0 +1,9 @@ +name: test +jobs: + job1: + runs-on: linux + concurrency: + group: major-tests + cancel-in-progress: true + steps: + - run: uname -a diff --git a/act/jobparser/testdata/job_concurrency.out.yaml b/act/jobparser/testdata/job_concurrency.out.yaml new file mode 100644 index 00000000..a7e1e118 --- /dev/null +++ b/act/jobparser/testdata/job_concurrency.out.yaml @@ -0,0 +1,10 @@ +name: test +jobs: + job1: + name: job1 + runs-on: linux + steps: + - run: uname -a + concurrency: + group: major-tests + cancel-in-progress: "true" diff --git a/act/jobparser/testdata/job_concurrency_eval.in.yaml b/act/jobparser/testdata/job_concurrency_eval.in.yaml new file mode 100644 index 00000000..23d06b53 --- /dev/null +++ b/act/jobparser/testdata/job_concurrency_eval.in.yaml @@ -0,0 +1,9 @@ +name: test +jobs: + job1: + runs-on: linux + concurrency: + group: ${{ github.workflow }} + cancel-in-progress: ${{ !contains(github.ref, 'release/')}} + steps: + - run: uname -a diff --git a/act/jobparser/testdata/job_concurrency_eval.out.yaml b/act/jobparser/testdata/job_concurrency_eval.out.yaml new file mode 100644 index 00000000..40cbb529 --- /dev/null +++ b/act/jobparser/testdata/job_concurrency_eval.out.yaml @@ -0,0 +1,10 @@ +name: test +jobs: + job1: + name: job1 + runs-on: linux + steps: + - run: uname -a + concurrency: + group: ${{ github.workflow }} + cancel-in-progress: ${{ !contains(github.ref, 'release/')}} diff --git a/act/jobparser/testdata_test.go b/act/jobparser/testdata_test.go index dfbf2f76..a918f353 100644 --- a/act/jobparser/testdata_test.go +++ b/act/jobparser/testdata_test.go @@ -6,7 +6,7 @@ import ( "path/filepath" "testing" - "code.forgejo.org/forgejo/runner/v9/act/model" + "code.forgejo.org/forgejo/runner/v11/act/model" "github.com/stretchr/testify/require" ) diff --git a/act/model/action.go b/act/model/action.go index 62b1d025..72b986f0 100644 --- a/act/model/action.go +++ b/act/model/action.go @@ -5,14 +5,14 @@ import ( "io" "strings" - "code.forgejo.org/forgejo/runner/v9/act/schema" - "gopkg.in/yaml.v3" + "code.forgejo.org/forgejo/runner/v11/act/schema" + "go.yaml.in/yaml/v3" ) // ActionRunsUsing is the type of runner for the action type ActionRunsUsing string -func (a *ActionRunsUsing) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (a *ActionRunsUsing) UnmarshalYAML(unmarshal func(any) error) error { var using string if err := unmarshal(&using); err != nil { return err @@ -21,7 +21,7 @@ func (a *ActionRunsUsing) UnmarshalYAML(unmarshal func(interface{}) error) error // Force input to lowercase for case insensitive comparison format := ActionRunsUsing(strings.ToLower(using)) switch format { - case ActionRunsUsingNode20, ActionRunsUsingNode16, ActionRunsUsingNode12, ActionRunsUsingDocker, ActionRunsUsingComposite, ActionRunsUsingGo, ActionRunsUsingSh: + case ActionRunsUsingNode24, ActionRunsUsingNode20, ActionRunsUsingNode16, ActionRunsUsingNode12, ActionRunsUsingDocker, ActionRunsUsingComposite, ActionRunsUsingGo, ActionRunsUsingSh: *a = format default: return fmt.Errorf("The runs.using key in action.yml must be one of: %v, got %s", []string{ @@ -30,6 +30,7 @@ func (a *ActionRunsUsing) UnmarshalYAML(unmarshal func(interface{}) error) error ActionRunsUsingNode12, ActionRunsUsingNode16, ActionRunsUsingNode20, + ActionRunsUsingNode24, ActionRunsUsingGo, ActionRunsUsingSh, }, format) @@ -44,6 +45,8 @@ const ( ActionRunsUsingNode16 = "node16" // ActionRunsUsingNode20 for running with node20 ActionRunsUsingNode20 = "node20" + // ActionRunsUsingNode24 for running with node24 + ActionRunsUsingNode24 = "node24" // ActionRunsUsingDocker for running with docker ActionRunsUsingDocker = "docker" // ActionRunsUsingComposite for running composite @@ -69,6 +72,22 @@ type ActionRuns struct { Steps []Step `yaml:"steps"` } +func (actionRuns *ActionRuns) UnmarshalYAML(value *yaml.Node) error { + type ActionRunsDefault ActionRuns + if err := value.Decode((*ActionRunsDefault)(actionRuns)); err != nil { + return err + } + for i := range actionRuns.Steps { + step := &actionRuns.Steps[i] + // Set `Number` and `ID` on each step based upon their position in the steps array: + if step.ID == "" { + step.ID = fmt.Sprintf("%d", i) + } + step.Number = i + } + return nil +} + // Action describes a metadata file for GitHub actions. The metadata filename must be either action.yml or action.yaml. The data in the metadata file defines the inputs, outputs and main entrypoint for your action. type Action struct { Name string `yaml:"name"` diff --git a/act/model/github_context.go b/act/model/github_context.go index e7c8809e..4a3d0bd9 100644 --- a/act/model/github_context.go +++ b/act/model/github_context.go @@ -5,43 +5,44 @@ import ( "fmt" "strings" - "code.forgejo.org/forgejo/runner/v9/act/common" - "code.forgejo.org/forgejo/runner/v9/act/common/git" + "code.forgejo.org/forgejo/runner/v11/act/common" + "code.forgejo.org/forgejo/runner/v11/act/common/git" ) type GithubContext struct { - Event map[string]interface{} `json:"event"` - EventPath string `json:"event_path"` - Workflow string `json:"workflow"` - RunID string `json:"run_id"` - RunNumber string `json:"run_number"` - Actor string `json:"actor"` - Repository string `json:"repository"` - EventName string `json:"event_name"` - Sha string `json:"sha"` - Ref string `json:"ref"` - RefName string `json:"ref_name"` - RefType string `json:"ref_type"` - HeadRef string `json:"head_ref"` - BaseRef string `json:"base_ref"` - Token string `json:"token"` - Workspace string `json:"workspace"` - Action string `json:"action"` - ActionPath string `json:"action_path"` - ActionRef string `json:"action_ref"` - ActionRepository string `json:"action_repository"` - Job string `json:"job"` - JobName string `json:"job_name"` - RepositoryOwner string `json:"repository_owner"` - RetentionDays string `json:"retention_days"` - RunnerPerflog string `json:"runner_perflog"` - RunnerTrackingID string `json:"runner_tracking_id"` - ServerURL string `json:"server_url"` - APIURL string `json:"api_url"` - GraphQLURL string `json:"graphql_url"` + Event map[string]any `json:"event"` + EventPath string `json:"event_path"` + Workflow string `json:"workflow"` + RunAttempt string `json:"run_attempt"` + RunID string `json:"run_id"` + RunNumber string `json:"run_number"` + Actor string `json:"actor"` + Repository string `json:"repository"` + EventName string `json:"event_name"` + Sha string `json:"sha"` + Ref string `json:"ref"` + RefName string `json:"ref_name"` + RefType string `json:"ref_type"` + HeadRef string `json:"head_ref"` + BaseRef string `json:"base_ref"` + Token string `json:"token"` + Workspace string `json:"workspace"` + Action string `json:"action"` + ActionPath string `json:"action_path"` + ActionRef string `json:"action_ref"` + ActionRepository string `json:"action_repository"` + Job string `json:"job"` + JobName string `json:"job_name"` + RepositoryOwner string `json:"repository_owner"` + RetentionDays string `json:"retention_days"` + RunnerPerflog string `json:"runner_perflog"` + RunnerTrackingID string `json:"runner_tracking_id"` + ServerURL string `json:"server_url"` + APIURL string `json:"api_url"` + GraphQLURL string `json:"graphql_url"` } -func asString(v interface{}) string { +func asString(v any) string { if v == nil { return "" } else if s, ok := v.(string); ok { @@ -50,7 +51,7 @@ func asString(v interface{}) string { return "" } -func nestedMapLookup(m map[string]interface{}, ks ...string) (rval interface{}) { +func nestedMapLookup(m map[string]any, ks ...string) (rval any) { var ok bool if len(ks) == 0 { // degenerate input @@ -60,20 +61,20 @@ func nestedMapLookup(m map[string]interface{}, ks ...string) (rval interface{}) return nil } else if len(ks) == 1 { // we've reached the final key return rval - } else if m, ok = rval.(map[string]interface{}); !ok { + } else if m, ok = rval.(map[string]any); !ok { return nil } // 1+ more keys return nestedMapLookup(m, ks[1:]...) } -func withDefaultBranch(ctx context.Context, b string, event map[string]interface{}) map[string]interface{} { +func withDefaultBranch(ctx context.Context, b string, event map[string]any) map[string]any { repoI, ok := event["repository"] if !ok { - repoI = make(map[string]interface{}) + repoI = make(map[string]any) } - repo, ok := repoI.(map[string]interface{}) + repo, ok := repoI.(map[string]any) if !ok { common.Logger(ctx).Warnf("unable to set default branch to %v", b) return event @@ -170,7 +171,7 @@ func (ghc *GithubContext) SetRepositoryAndOwner(ctx context.Context, githubInsta if ghc.Repository == "" { repo, err := git.FindGithubRepo(ctx, repoPath, githubInstance, remoteName) if err != nil { - common.Logger(ctx).Warningf("unable to get git repo (githubInstance: %v; remoteName: %v, repoPath: %v): %v", githubInstance, remoteName, repoPath, err) + common.Logger(ctx).Debugf("unable to get git repo (githubInstance: %v; remoteName: %v, repoPath: %v): %v", githubInstance, remoteName, repoPath, err) return } ghc.Repository = repo diff --git a/act/model/github_context_test.go b/act/model/github_context_test.go index ed08e231..bf3ecad9 100644 --- a/act/model/github_context_test.go +++ b/act/model/github_context_test.go @@ -27,19 +27,19 @@ func TestSetRef(t *testing.T) { tables := []struct { eventName string - event map[string]interface{} + event map[string]any ref string refName string }{ { eventName: "pull_request_target", - event: map[string]interface{}{}, + event: map[string]any{}, ref: "refs/heads/master", refName: "master", }, { eventName: "pull_request", - event: map[string]interface{}{ + event: map[string]any{ "number": 1234., }, ref: "refs/pull/1234/merge", @@ -47,8 +47,8 @@ func TestSetRef(t *testing.T) { }, { eventName: "deployment", - event: map[string]interface{}{ - "deployment": map[string]interface{}{ + event: map[string]any{ + "deployment": map[string]any{ "ref": "refs/heads/somebranch", }, }, @@ -57,8 +57,8 @@ func TestSetRef(t *testing.T) { }, { eventName: "release", - event: map[string]interface{}{ - "release": map[string]interface{}{ + event: map[string]any{ + "release": map[string]any{ "tag_name": "v1.0.0", }, }, @@ -67,7 +67,7 @@ func TestSetRef(t *testing.T) { }, { eventName: "push", - event: map[string]interface{}{ + event: map[string]any{ "ref": "refs/heads/somebranch", }, ref: "refs/heads/somebranch", @@ -75,8 +75,8 @@ func TestSetRef(t *testing.T) { }, { eventName: "unknown", - event: map[string]interface{}{ - "repository": map[string]interface{}{ + event: map[string]any{ + "repository": map[string]any{ "default_branch": "main", }, }, @@ -85,7 +85,7 @@ func TestSetRef(t *testing.T) { }, { eventName: "no-event", - event: map[string]interface{}{}, + event: map[string]any{}, ref: "refs/heads/master", refName: "master", }, @@ -99,7 +99,7 @@ func TestSetRef(t *testing.T) { Event: table.event, } - ghc.SetRef(context.Background(), "main", "/some/dir") + ghc.SetRef(t.Context(), "main", "/some/dir") ghc.SetRefTypeAndName() assert.Equal(t, table.ref, ghc.Ref) @@ -114,10 +114,10 @@ func TestSetRef(t *testing.T) { ghc := &GithubContext{ EventName: "no-default-branch", - Event: map[string]interface{}{}, + Event: map[string]any{}, } - ghc.SetRef(context.Background(), "", "/some/dir") + ghc.SetRef(t.Context(), "", "/some/dir") assert.Equal(t, "refs/heads/master", ghc.Ref) }) @@ -141,14 +141,14 @@ func TestSetSha(t *testing.T) { tables := []struct { eventName string - event map[string]interface{} + event map[string]any sha string }{ { eventName: "pull_request_target", - event: map[string]interface{}{ - "pull_request": map[string]interface{}{ - "base": map[string]interface{}{ + event: map[string]any{ + "pull_request": map[string]any{ + "base": map[string]any{ "sha": "pr-base-sha", }, }, @@ -157,15 +157,15 @@ func TestSetSha(t *testing.T) { }, { eventName: "pull_request", - event: map[string]interface{}{ + event: map[string]any{ "number": 1234., }, sha: "1234fakesha", }, { eventName: "deployment", - event: map[string]interface{}{ - "deployment": map[string]interface{}{ + event: map[string]any{ + "deployment": map[string]any{ "sha": "deployment-sha", }, }, @@ -173,12 +173,12 @@ func TestSetSha(t *testing.T) { }, { eventName: "release", - event: map[string]interface{}{}, + event: map[string]any{}, sha: "1234fakesha", }, { eventName: "push", - event: map[string]interface{}{ + event: map[string]any{ "after": "push-sha", "deleted": false, }, @@ -186,12 +186,12 @@ func TestSetSha(t *testing.T) { }, { eventName: "unknown", - event: map[string]interface{}{}, + event: map[string]any{}, sha: "1234fakesha", }, { eventName: "no-event", - event: map[string]interface{}{}, + event: map[string]any{}, sha: "1234fakesha", }, } @@ -204,7 +204,7 @@ func TestSetSha(t *testing.T) { Event: table.event, } - ghc.SetSha(context.Background(), "/some/dir") + ghc.SetSha(t.Context(), "/some/dir") assert.Equal(t, table.sha, ghc.Sha) }) diff --git a/act/model/planner.go b/act/model/planner.go index 6ea5f753..05f78bbe 100644 --- a/act/model/planner.go +++ b/act/model/planner.go @@ -8,7 +8,7 @@ import ( "os" "path/filepath" "regexp" - "sort" + "slices" log "github.com/sirupsen/logrus" ) @@ -286,11 +286,8 @@ func (wp *workflowPlanner) GetEvents() []string { for _, w := range wp.workflows { found := false for _, e := range events { - for _, we := range w.On() { - if e == we { - found = true - break - } + if slices.Contains(w.On(), e) { + found = true } if found { break @@ -303,9 +300,7 @@ func (wp *workflowPlanner) GetEvents() []string { } // sort the list based on depth of dependencies - sort.Slice(events, func(i, j int) bool { - return events[i] < events[j] - }) + slices.Sort(events) return events } @@ -336,7 +331,7 @@ func (s *Stage) GetJobIDs() []string { // Merge stages with existing stages in plan func (p *Plan) mergeStages(stages []*Stage) { newStages := make([]*Stage, int(math.Max(float64(len(p.Stages)), float64(len(stages))))) - for i := 0; i < len(newStages); i++ { + for i := range newStages { newStages[i] = new(Stage) if i >= len(p.Stages) { newStages[i].Runs = append(newStages[i].Runs, stages[i].Runs...) diff --git a/act/model/workflow.go b/act/model/workflow.go index ab6ad296..c1bc4097 100644 --- a/act/model/workflow.go +++ b/act/model/workflow.go @@ -4,16 +4,19 @@ import ( "errors" "fmt" "io" + "maps" "path/filepath" "reflect" "regexp" + "slices" "strconv" "strings" + "sync" - "code.forgejo.org/forgejo/runner/v9/act/common" - "code.forgejo.org/forgejo/runner/v9/act/schema" + "code.forgejo.org/forgejo/runner/v11/act/common" + "code.forgejo.org/forgejo/runner/v11/act/schema" log "github.com/sirupsen/logrus" - "gopkg.in/yaml.v3" + "go.yaml.in/yaml/v3" ) // Workflow is the structure of the files in .github/workflows @@ -25,7 +28,8 @@ type Workflow struct { Jobs map[string]*Job `yaml:"jobs"` Defaults Defaults `yaml:"defaults"` - RawNotifications yaml.Node `yaml:"enable-email-notifications"` + RawNotifications yaml.Node `yaml:"enable-email-notifications"` + RawConcurrency *RawConcurrency `yaml:"concurrency"` // For Gitea } // On events for the workflow @@ -46,7 +50,7 @@ func (w *Workflow) On() []string { } return val case yaml.MappingNode: - var val map[string]interface{} + var val map[string]any err := w.RawOn.Decode(&val) if err != nil { log.Fatal(err) @@ -60,9 +64,9 @@ func (w *Workflow) On() []string { return nil } -func (w *Workflow) OnEvent(event string) interface{} { +func (w *Workflow) OnEvent(event string) any { if w.RawOn.Kind == yaml.MappingNode { - var val map[string]interface{} + var val map[string]any if !decodeNode(w.RawOn, &val) { return nil } @@ -78,10 +82,10 @@ func (w *Workflow) OnSchedule() []string { } switch val := schedules.(type) { - case []interface{}: + case []any: allSchedules := []string{} for _, v := range val { - for k, cron := range v.(map[string]interface{}) { + for k, cron := range v.(map[string]any) { if k != "cron" { continue } @@ -136,10 +140,8 @@ func (w *Workflow) WorkflowDispatchConfig() *WorkflowDispatch { if !decodeNode(w.RawOn, &val) { return nil } - for _, v := range val { - if v == "workflow_dispatch" { - return &WorkflowDispatch{} - } + if slices.Contains(val, "workflow_dispatch") { + return &WorkflowDispatch{} } case yaml.MappingNode: var val map[string]yaml.Node @@ -159,10 +161,10 @@ func (w *Workflow) WorkflowDispatchConfig() *WorkflowDispatch { } type WorkflowCallInput struct { - Description string `yaml:"description"` - Required bool `yaml:"required"` - Default string `yaml:"default"` - Type string `yaml:"type"` + Description string `yaml:"description"` + Required bool `yaml:"required"` + Default yaml.Node `yaml:"default"` + Type string `yaml:"type"` } type WorkflowCallOutput struct { @@ -205,7 +207,7 @@ type Job struct { RawNeeds yaml.Node `yaml:"needs"` RawRunsOn yaml.Node `yaml:"runs-on"` Env yaml.Node `yaml:"env"` - If yaml.Node `yaml:"if"` + RawIf yaml.Node `yaml:"if"` Steps []*Step `yaml:"steps"` TimeoutMinutes string `yaml:"timeout-minutes"` Services map[string]*ContainerSpec `yaml:"services"` @@ -214,9 +216,11 @@ type Job struct { Defaults Defaults `yaml:"defaults"` Outputs map[string]string `yaml:"outputs"` Uses string `yaml:"uses"` - With map[string]interface{} `yaml:"with"` + With map[string]any `yaml:"with"` RawSecrets yaml.Node `yaml:"secrets"` - Result string + + Result string + ResultMutex sync.Mutex } // Strategy for the job @@ -335,14 +339,21 @@ func (j *Job) Needs() []string { // RunsOn list for Job func (j *Job) RunsOn() []string { - switch j.RawRunsOn.Kind { + return FlattenRunsOnNode(j.RawRunsOn) +} + +// Given an already expression-evaluated `runs-on` yaml node from a job, compute all the labels that will be run for a +// job. Can be a single string label, an array of labels, or an object { group: "...", labels: [...] }; +// FlattenRunsOnNode will flatten all the options down to a []string. +func FlattenRunsOnNode(runsOn yaml.Node) []string { + switch runsOn.Kind { case yaml.MappingNode: var val struct { Group string Labels yaml.Node } - if !decodeNode(j.RawRunsOn, &val) { + if !decodeNode(runsOn, &val) { return nil } @@ -354,10 +365,17 @@ func (j *Job) RunsOn() []string { return labels default: - return nodeAsStringSlice(j.RawRunsOn) + return nodeAsStringSlice(runsOn) } } +func (j *Job) IfClause() string { + if j.RawIf.Value == "" { + return "success()" + } + return j.RawIf.Value +} + func nodeAsStringSlice(node yaml.Node) []string { switch node.Kind { case yaml.ScalarNode: @@ -392,9 +410,9 @@ func (j *Job) Environment() map[string]string { } // Matrix decodes RawMatrix YAML node -func (j *Job) Matrix() map[string][]interface{} { +func (j *Job) Matrix() map[string][]any { if j.Strategy.RawMatrix.Kind == yaml.MappingNode { - var val map[string][]interface{} + var val map[string][]any if !decodeNode(j.Strategy.RawMatrix, &val) { return nil } @@ -405,20 +423,20 @@ func (j *Job) Matrix() map[string][]interface{} { // GetMatrixes returns the matrix cross product // It skips includes and hard fails excludes for non-existing keys -func (j *Job) GetMatrixes() ([]map[string]interface{}, error) { - matrixes := make([]map[string]interface{}, 0) +func (j *Job) GetMatrixes() ([]map[string]any, error) { + matrixes := make([]map[string]any, 0) if j.Strategy != nil { j.Strategy.FailFast = j.Strategy.GetFailFast() j.Strategy.MaxParallel = j.Strategy.GetMaxParallel() if m := j.Matrix(); m != nil { - includes := make([]map[string]interface{}, 0) - extraIncludes := make([]map[string]interface{}, 0) + includes := make([]map[string]any, 0) + extraIncludes := make([]map[string]any, 0) for _, v := range m["include"] { switch t := v.(type) { - case []interface{}: + case []any: for _, i := range t { - i := i.(map[string]interface{}) + i := i.(map[string]any) extraInclude := true for k := range i { if _, ok := m[k]; ok { @@ -431,8 +449,8 @@ func (j *Job) GetMatrixes() ([]map[string]interface{}, error) { extraIncludes = append(extraIncludes, i) } } - case interface{}: - v := v.(map[string]interface{}) + case any: + v := v.(map[string]any) extraInclude := true for k := range v { if _, ok := m[k]; ok { @@ -448,9 +466,9 @@ func (j *Job) GetMatrixes() ([]map[string]interface{}, error) { } delete(m, "include") - excludes := make([]map[string]interface{}, 0) + excludes := make([]map[string]any, 0) for _, e := range m["exclude"] { - e := e.(map[string]interface{}) + e := e.(map[string]any) for k := range e { if _, ok := m[k]; ok { excludes = append(excludes, e) @@ -479,9 +497,7 @@ func (j *Job) GetMatrixes() ([]map[string]interface{}, error) { if commonKeysMatch2(matrix, include, m) { matched = true log.Debugf("Adding include values '%v' to existing entry", include) - for k, v := range include { - matrix[k] = v - } + maps.Copy(matrix, include) } } if !matched { @@ -493,19 +509,19 @@ func (j *Job) GetMatrixes() ([]map[string]interface{}, error) { matrixes = append(matrixes, include) } if len(matrixes) == 0 { - matrixes = append(matrixes, make(map[string]interface{})) + matrixes = append(matrixes, make(map[string]any)) } } else { - matrixes = append(matrixes, make(map[string]interface{})) + matrixes = append(matrixes, make(map[string]any)) } } else { - matrixes = append(matrixes, make(map[string]interface{})) + matrixes = append(matrixes, make(map[string]any)) log.Debugf("Empty Strategy, matrixes=%v", matrixes) } return matrixes, nil } -func commonKeysMatch(a, b map[string]interface{}) bool { +func commonKeysMatch(a, b map[string]any) bool { for aKey, aVal := range a { if bVal, ok := b[aKey]; ok && !reflect.DeepEqual(aVal, bVal) { return false @@ -514,7 +530,7 @@ func commonKeysMatch(a, b map[string]interface{}) bool { return true } -func commonKeysMatch2(a, b map[string]interface{}, m map[string][]interface{}) bool { +func commonKeysMatch2(a, b map[string]any, m map[string][]any) bool { for aKey, aVal := range a { _, useKey := m[aKey] if bVal, ok := b[aKey]; useKey && ok && !reflect.DeepEqual(aVal, bVal) { @@ -578,6 +594,24 @@ func (j *Job) Type() (JobType, error) { return JobTypeDefault, nil } +func (j *Job) UnmarshalYAML(value *yaml.Node) error { + type JobDefault Job + if err := value.Decode((*JobDefault)(j)); err != nil { + return err + } + for i, step := range j.Steps { + if step == nil { + continue + } + // Set `Number` and `ID` on each step based upon their position in the steps array: + if step.ID == "" { + step.ID = fmt.Sprintf("%d", i) + } + step.Number = i + } + return nil +} + // ContainerSpec is the specification of the container to use for the job type ContainerSpec struct { Image string `yaml:"image"` @@ -604,7 +638,7 @@ type Step struct { Uses string `yaml:"uses"` Run string `yaml:"run"` WorkingDirectory string `yaml:"working-directory"` - Shell string `yaml:"shell"` + RawShell string `yaml:"shell"` Env yaml.Node `yaml:"env"` With map[string]string `yaml:"with"` RawContinueOnError string `yaml:"continue-on-error"` @@ -640,32 +674,6 @@ func (s *Step) GetEnv() map[string]string { return env } -// ShellCommand returns the command for the shell -func (s *Step) ShellCommand() string { - var shellCommand string - - // Reference: https://github.com/actions/runner/blob/8109c962f09d9acc473d92c595ff43afceddb347/src/Runner.Worker/Handlers/ScriptHandlerHelpers.cs#L9-L17 - switch s.Shell { - case "", "bash": - shellCommand = "bash --noprofile --norc -e -o pipefail {0}" - case "pwsh": - shellCommand = "pwsh -command . '{0}'" - case "python": - shellCommand = "python {0}" - case "sh": - shellCommand = "sh -e {0}" - case "cmd": - shellCommand = "cmd /D /E:ON /V:OFF /S /C \"CALL \"{0}\"\"" - case "powershell": - shellCommand = "powershell -command . '{0}'" - case "node": - shellCommand = "node {0}" - default: - shellCommand = s.Shell - } - return shellCommand -} - // StepType describes what type of step we are about to run type StepType int @@ -759,9 +767,6 @@ func (w *Workflow) GetJob(jobID string) *Job { if j.Name == "" { j.Name = id } - if j.If.Value == "" { - j.If.Value = "success()" - } return j } } @@ -777,11 +782,11 @@ func (w *Workflow) GetJobIDs() []string { return ids } -var OnDecodeNodeError = func(node yaml.Node, out interface{}, err error) { +var OnDecodeNodeError = func(node yaml.Node, out any, err error) { log.Errorf("Failed to decode node %v into %T: %v", node, out, err) } -func decodeNode(node yaml.Node, out interface{}) bool { +func decodeNode(node yaml.Node, out any) bool { if err := node.Decode(out); err != nil { if OnDecodeNodeError != nil { OnDecodeNodeError(node, out, err) @@ -806,3 +811,28 @@ func (w *Workflow) Notifications() (bool, error) { return false, fmt.Errorf("enable-email-notifications: unknown type: %v", w.RawNotifications.Kind) } } + +// For Gitea +// RawConcurrency represents a workflow concurrency or a job concurrency with uninterpolated options +type RawConcurrency struct { + Group string `yaml:"group,omitempty"` + CancelInProgress string `yaml:"cancel-in-progress,omitempty"` + RawExpression string `yaml:"-,omitempty"` +} + +type objectConcurrency RawConcurrency + +func (r *RawConcurrency) UnmarshalYAML(n *yaml.Node) error { + if err := n.Decode(&r.RawExpression); err == nil { + return nil + } + return n.Decode((*objectConcurrency)(r)) +} + +func (r *RawConcurrency) MarshalYAML() (any, error) { + if r.RawExpression != "" { + return r.RawExpression, nil + } + + return (*objectConcurrency)(r), nil +} diff --git a/act/model/workflow_test.go b/act/model/workflow_test.go index d3ea7234..d1a237e4 100644 --- a/act/model/workflow_test.go +++ b/act/model/workflow_test.go @@ -493,24 +493,24 @@ func TestReadWorkflow_Strategy(t *testing.T) { job := wf.Jobs["strategy-only-max-parallel"] matrixes, err := job.GetMatrixes() assert.NoError(t, err) - assert.Equal(t, matrixes, []map[string]interface{}{{}}) - assert.Equal(t, job.Matrix(), map[string][]interface{}(nil)) + assert.Equal(t, matrixes, []map[string]any{{}}) + assert.Equal(t, job.Matrix(), map[string][]any(nil)) assert.Equal(t, job.Strategy.MaxParallel, 2) assert.Equal(t, job.Strategy.FailFast, true) job = wf.Jobs["strategy-only-fail-fast"] matrixes, err = job.GetMatrixes() assert.NoError(t, err) - assert.Equal(t, matrixes, []map[string]interface{}{{}}) - assert.Equal(t, job.Matrix(), map[string][]interface{}(nil)) + assert.Equal(t, matrixes, []map[string]any{{}}) + assert.Equal(t, job.Matrix(), map[string][]any(nil)) assert.Equal(t, job.Strategy.MaxParallel, 4) assert.Equal(t, job.Strategy.FailFast, false) job = wf.Jobs["strategy-no-matrix"] matrixes, err = job.GetMatrixes() assert.NoError(t, err) - assert.Equal(t, matrixes, []map[string]interface{}{{}}) - assert.Equal(t, job.Matrix(), map[string][]interface{}(nil)) + assert.Equal(t, matrixes, []map[string]any{{}}) + assert.Equal(t, job.Matrix(), map[string][]any(nil)) assert.Equal(t, job.Strategy.MaxParallel, 2) assert.Equal(t, job.Strategy.FailFast, false) @@ -518,7 +518,7 @@ func TestReadWorkflow_Strategy(t *testing.T) { matrixes, err = job.GetMatrixes() assert.NoError(t, err) assert.Equal(t, matrixes, - []map[string]interface{}{ + []map[string]any{ {"datacenter": "site-c", "node-version": "14.x", "site": "staging"}, {"datacenter": "site-c", "node-version": "16.x", "site": "staging"}, {"datacenter": "site-d", "node-version": "16.x", "site": "staging"}, @@ -528,15 +528,15 @@ func TestReadWorkflow_Strategy(t *testing.T) { }, ) assert.Equal(t, job.Matrix(), - map[string][]interface{}{ + map[string][]any{ "datacenter": {"site-c", "site-d"}, "exclude": { - map[string]interface{}{"datacenter": "site-d", "node-version": "14.x", "site": "staging"}, + map[string]any{"datacenter": "site-d", "node-version": "14.x", "site": "staging"}, }, "include": { - map[string]interface{}{"php-version": 5.4}, - map[string]interface{}{"datacenter": "site-a", "node-version": "10.x", "site": "prod"}, - map[string]interface{}{"datacenter": "site-b", "node-version": "12.x", "site": "dev"}, + map[string]any{"php-version": 5.4}, + map[string]any{"datacenter": "site-a", "node-version": "10.x", "site": "prod"}, + map[string]any{"datacenter": "site-b", "node-version": "12.x", "site": "dev"}, }, "node-version": {"14.x", "16.x"}, "site": {"staging"}, @@ -546,25 +546,6 @@ func TestReadWorkflow_Strategy(t *testing.T) { assert.Equal(t, job.Strategy.FailFast, false) } -func TestStep_ShellCommand(t *testing.T) { - tests := []struct { - shell string - want string - }{ - {"pwsh -v '. {0}'", "pwsh -v '. {0}'"}, - {"pwsh", "pwsh -command . '{0}'"}, - {"powershell", "powershell -command . '{0}'"}, - {"node", "node {0}"}, - {"python", "python {0}"}, - } - for _, tt := range tests { - t.Run(tt.shell, func(t *testing.T) { - got := (&Step{Shell: tt.shell}).ShellCommand() - assert.Equal(t, got, tt.want) - }) - } -} - func TestReadWorkflow_WorkflowDispatchConfig(t *testing.T) { yaml := ` name: local-action-docker-url @@ -702,3 +683,48 @@ func TestStepUsesHash(t *testing.T) { }) } } + +func TestReadWorkflow_Concurrency(t *testing.T) { + for _, testCase := range []struct { + expected *RawConcurrency + err string + snippet string + }{ + { + expected: nil, + snippet: "# nothing", + }, + { + expected: &RawConcurrency{Group: "${{ github.workflow }}-${{ github.ref }}", CancelInProgress: ""}, + snippet: "concurrency: { group: \"${{ github.workflow }}-${{ github.ref }}\" }", + }, + { + expected: &RawConcurrency{Group: "example-group", CancelInProgress: "true"}, + snippet: "concurrency: { group: example-group, cancel-in-progress: true }", + }, + } { + t.Run(testCase.snippet, func(t *testing.T) { + yaml := fmt.Sprintf(` +name: name-455 +on: push +%s +jobs: + valid-JOB-Name-455: + runs-on: docker + steps: + - run: echo hi +`, testCase.snippet) + + workflow, err := ReadWorkflow(strings.NewReader(yaml), true) + if testCase.err != "" { + assert.ErrorContains(t, err, testCase.err) + } else { + assert.NoError(t, err, "read workflow should succeed") + + concurrency := workflow.RawConcurrency + // assert.NoError(t, err) + assert.Equal(t, testCase.expected, concurrency) + } + }) + } +} diff --git a/act/runner/action.go b/act/runner/action.go index c777573d..1ecd51c1 100644 --- a/act/runner/action.go +++ b/act/runner/action.go @@ -16,9 +16,9 @@ import ( "github.com/kballard/go-shellquote" - "code.forgejo.org/forgejo/runner/v9/act/common" - "code.forgejo.org/forgejo/runner/v9/act/container" - "code.forgejo.org/forgejo/runner/v9/act/model" + "code.forgejo.org/forgejo/runner/v11/act/common" + "code.forgejo.org/forgejo/runner/v11/act/container" + "code.forgejo.org/forgejo/runner/v11/act/model" ) type actionStep interface { @@ -182,7 +182,7 @@ func runActionImpl(step actionStep, actionDir string, remoteAction *remoteAction logger.Debugf("type=%v actionDir=%s actionPath=%s workdir=%s actionCacheDir=%s actionName=%s containerActionDir=%s", stepModel.Type(), actionDir, actionPath, rc.Config.Workdir, rc.ActionCacheDir(), actionName, containerActionDir) switch action.Runs.Using { - case model.ActionRunsUsingNode12, model.ActionRunsUsingNode16, model.ActionRunsUsingNode20: + case model.ActionRunsUsingNode12, model.ActionRunsUsingNode16, model.ActionRunsUsingNode20, model.ActionRunsUsingNode24: if err := maybeCopyToActionDir(ctx, step, actionDir, actionPath, containerActionDir); err != nil { return err } @@ -237,6 +237,7 @@ func runActionImpl(step actionStep, actionDir string, remoteAction *remoteAction model.ActionRunsUsingNode12, model.ActionRunsUsingNode16, model.ActionRunsUsingNode20, + model.ActionRunsUsingNode24, model.ActionRunsUsingComposite, model.ActionRunsUsingGo, model.ActionRunsUsingSh, @@ -276,7 +277,6 @@ func removeGitIgnore(ctx context.Context, directory string) error { return nil } -// TODO: break out parts of function to reduce complexicity func execAsDocker(ctx context.Context, step actionStep, actionName, basedir string, localAction bool) error { logger := common.Logger(ctx) rc := step.getRunContext() @@ -285,15 +285,16 @@ func execAsDocker(ctx context.Context, step actionStep, actionName, basedir stri var prepImage common.Executor var image string forcePull := false - if strings.HasPrefix(action.Runs.Image, "docker://") { - image = strings.TrimPrefix(action.Runs.Image, "docker://") + if after, ok := strings.CutPrefix(action.Runs.Image, "docker://"); ok { + image = after // Apply forcePull only for prebuild docker images forcePull = rc.Config.ForcePull } else { - // "-dockeraction" enshures that "./", "./test " won't get converted to "act-:latest", "act-test-:latest" which are invalid docker image names - image = fmt.Sprintf("%s-dockeraction:%s", regexp.MustCompile("[^a-zA-Z0-9]").ReplaceAllString(actionName, "-"), "latest") - image = fmt.Sprintf("act-%s", strings.TrimLeft(image, "-")) - image = strings.ToLower(image) + if localAction { + image = fmt.Sprintf("runner-local-docker-action-%s:latest", common.MustRandName(16)) + } else { + image = fmt.Sprintf("runner-remote-docker-action-%s:latest", common.Sha256(step.getStepModel().Uses)) + } contextDir, fileName := filepath.Split(filepath.Join(basedir, action.Runs.Image)) anyArchExists, err := container.ImageExistsLocally(ctx, image, "any") @@ -427,7 +428,7 @@ func newStepContainer(ctx context.Context, step step, image string, cmd, entrypo envList = append(envList, fmt.Sprintf("%s=%s", "RUNNER_ARCH", container.RunnerArch(ctx))) envList = append(envList, fmt.Sprintf("%s=%s", "RUNNER_TEMP", "/tmp")) - binds, mounts := rc.GetBindsAndMounts() + binds, mounts, validVolumes := rc.GetBindsAndMounts(ctx) networkMode := fmt.Sprintf("container:%s", rc.jobContainerName()) if rc.IsHostEnv(ctx) { networkMode = "default" @@ -450,8 +451,7 @@ func newStepContainer(ctx context.Context, step step, image string, cmd, entrypo Privileged: rc.Config.Privileged, UsernsMode: rc.Config.UsernsMode, Platform: rc.Config.ContainerArchitecture, - AutoRemove: rc.Config.AutoRemove, - ValidVolumes: rc.Config.ValidVolumes, + ValidVolumes: validVolumes, ConfigOptions: rc.Config.ContainerOptions, }) @@ -530,6 +530,7 @@ func hasPreStep(step actionStep) common.Conditional { ((action.Runs.Using == model.ActionRunsUsingNode12 || action.Runs.Using == model.ActionRunsUsingNode16 || action.Runs.Using == model.ActionRunsUsingNode20 || + action.Runs.Using == model.ActionRunsUsingNode24 || action.Runs.Using == model.ActionRunsUsingGo || action.Runs.Using == model.ActionRunsUsingSh) && action.Runs.Pre != "") @@ -546,7 +547,7 @@ func runPreStep(step actionStep) common.Executor { action := step.getActionModel() switch action.Runs.Using { - case model.ActionRunsUsingNode12, model.ActionRunsUsingNode16, model.ActionRunsUsingNode20, model.ActionRunsUsingSh: + case model.ActionRunsUsingNode12, model.ActionRunsUsingNode16, model.ActionRunsUsingNode20, model.ActionRunsUsingNode24, model.ActionRunsUsingSh: // defaults in pre steps were missing, however provided inputs are available populateEnvsFromInput(ctx, step.getEnv(), action, rc) // todo: refactor into step @@ -672,6 +673,7 @@ func hasPostStep(step actionStep) common.Conditional { ((action.Runs.Using == model.ActionRunsUsingNode12 || action.Runs.Using == model.ActionRunsUsingNode16 || action.Runs.Using == model.ActionRunsUsingNode20 || + action.Runs.Using == model.ActionRunsUsingNode24 || action.Runs.Using == model.ActionRunsUsingGo || action.Runs.Using == model.ActionRunsUsingSh) && action.Runs.Post != "") @@ -708,9 +710,10 @@ func runPostStep(step actionStep) common.Executor { _, containerActionDir := getContainerActionPaths(stepModel, actionLocation, rc) switch action.Runs.Using { - case model.ActionRunsUsingNode12, model.ActionRunsUsingNode16, model.ActionRunsUsingNode20: + case model.ActionRunsUsingNode12, model.ActionRunsUsingNode16, model.ActionRunsUsingNode20, model.ActionRunsUsingNode24: populateEnvsFromSavedState(step.getEnv(), step, rc) + populateEnvsFromInput(ctx, step.getEnv(), step.getActionModel(), rc) containerArgs := []string{"node", path.Join(containerActionDir, action.Runs.Post)} logger.Debugf("executing remote job container: %s", containerArgs) diff --git a/act/runner/action_cache.go b/act/runner/action_cache.go index 219656cf..2650c700 100644 --- a/act/runner/action_cache.go +++ b/act/runner/action_cache.go @@ -18,7 +18,7 @@ import ( "github.com/go-git/go-git/v5/plumbing/transport" "github.com/go-git/go-git/v5/plumbing/transport/http" - "code.forgejo.org/forgejo/runner/v9/act/common" + "code.forgejo.org/forgejo/runner/v11/act/common" ) type ActionCache interface { @@ -39,10 +39,7 @@ func (c GoGitActionCache) Fetch(ctx context.Context, cacheDir, url, ref, token s if err != nil { return "", err } - branchName, err := common.RandName(12) - if err != nil { - return "", err - } + branchName := common.MustRandName(12) var auth transport.AuthMethod if token != "" { diff --git a/act/runner/action_cache_test.go b/act/runner/action_cache_test.go index 286336e9..16518856 100644 --- a/act/runner/action_cache_test.go +++ b/act/runner/action_cache_test.go @@ -3,7 +3,6 @@ package runner import ( "archive/tar" "bytes" - "context" "io" "os" "testing" @@ -17,7 +16,7 @@ func TestActionCache(t *testing.T) { cache := &GoGitActionCache{ Path: os.TempDir(), } - ctx := context.Background() + ctx := t.Context() cacheDir := "nektos/act-test-actions" repo := "https://code.forgejo.org/forgejo/act-test-actions" refs := []struct { @@ -58,9 +57,10 @@ func TestActionCache(t *testing.T) { return } atar, err := cache.GetTarArchive(ctx, c.CacheDir, sha, "js") - if !a.NoError(err) || !a.NotEmpty(atar) { + if !a.NoError(err) { return } + defer atar.Close() mytar := tar.NewReader(atar) th, err := mytar.Next() if !a.NoError(err) || !a.NotEqual(0, th.Size) { diff --git a/act/runner/action_composite.go b/act/runner/action_composite.go index fdee793d..8992f5bc 100644 --- a/act/runner/action_composite.go +++ b/act/runner/action_composite.go @@ -6,8 +6,8 @@ import ( "regexp" "strings" - "code.forgejo.org/forgejo/runner/v9/act/common" - "code.forgejo.org/forgejo/runner/v9/act/model" + "code.forgejo.org/forgejo/runner/v11/act/common" + "code.forgejo.org/forgejo/runner/v11/act/model" ) func evaluateCompositeInputAndEnv(ctx context.Context, parent *RunContext, step actionStep) map[string]string { @@ -136,17 +136,16 @@ func (rc *RunContext) compositeExecutor(action *model.Action) *compositeSteps { sf := &stepFactoryImpl{} - for i, step := range action.Runs.Steps { - if step.ID == "" { - step.ID = fmt.Sprintf("%d", i) + for i, stepModel := range action.Runs.Steps { + if stepModel.Number != i { + return &compositeSteps{ + pre: func(ctx context.Context) error { return nil }, + main: common.NewErrorExecutor(fmt.Errorf("internal error: invalid Step: Number expected %v, was actually %v", i, stepModel.Number)), + post: func(ctx context.Context) error { return nil }, + } } - step.Number = i - // create a copy of the step, since this composite action could - // run multiple times and we might modify the instance - stepcopy := step - - step, err := sf.newStep(&stepcopy, rc) + step, err := sf.newStep(&stepModel, rc) if err != nil { return &compositeSteps{ main: common.NewErrorExecutor(err), diff --git a/act/runner/action_test.go b/act/runner/action_test.go index a5d9e1d7..f549f7e9 100644 --- a/act/runner/action_test.go +++ b/act/runner/action_test.go @@ -7,7 +7,7 @@ import ( "strings" "testing" - "code.forgejo.org/forgejo/runner/v9/act/model" + "code.forgejo.org/forgejo/runner/v11/act/model" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" ) @@ -130,7 +130,7 @@ runs: closerMock.On("Close") } - action, err := readActionImpl(context.Background(), tt.step, "actionDir", "actionPath", readFile, writeFile) + action, err := readActionImpl(t.Context(), tt.step, "actionDir", "actionPath", readFile, writeFile) assert.Nil(t, err) assert.Equal(t, tt.expected, action) @@ -222,7 +222,7 @@ func TestActionRunner(t *testing.T) { for _, tt := range table { t.Run(tt.name, func(t *testing.T) { - ctx := context.Background() + ctx := t.Context() cm := &containerMock{} cm.On("CopyDir", "/var/run/act/actions/dir/", "dir/", false).Return(func(ctx context.Context) error { return nil }) diff --git a/act/runner/command.go b/act/runner/command.go index 98942322..3250a3f2 100644 --- a/act/runner/command.go +++ b/act/runner/command.go @@ -5,7 +5,9 @@ import ( "regexp" "strings" - "code.forgejo.org/forgejo/runner/v9/act/common" + "code.forgejo.org/forgejo/runner/v11/act/common" + + "github.com/sirupsen/logrus" ) var ( @@ -43,11 +45,12 @@ func (rc *RunContext) commandHandler(ctx context.Context) common.LineHandler { } if resumeCommand != "" && command != resumeCommand { - logger.Infof(" \U00002699 %s", line) + logger.WithFields(logrus.Fields{"command": "ignored", "raw": line}).Infof(" \U00002699 %s", line) return false } arg = unescapeCommandData(arg) kvPairs = unescapeKvPairs(kvPairs) + defCommandLogger := logger.WithFields(logrus.Fields{"command": command, "kvPairs": kvPairs, "arg": arg, "raw": line}) switch command { case "set-env": rc.setEnv(ctx, kvPairs, arg) @@ -56,27 +59,27 @@ func (rc *RunContext) commandHandler(ctx context.Context) common.LineHandler { case "add-path": rc.addPath(ctx, arg) case "debug": - logger.Infof(" \U0001F4AC %s", line) + defCommandLogger.Debugf(" \U0001F4AC %s", line) case "warning": - logger.Infof(" \U0001F6A7 %s", line) + defCommandLogger.Warnf(" \U0001F6A7 %s", line) case "error": - logger.Infof(" \U00002757 %s", line) + defCommandLogger.Errorf(" \U00002757 %s", line) case "add-mask": rc.AddMask(arg) - logger.Infof(" \U00002699 %s", "***") + defCommandLogger.Infof(" \U00002699 %s", "***") case "stop-commands": resumeCommand = arg - logger.Infof(" \U00002699 %s", line) + defCommandLogger.Infof(" \U00002699 %s", line) case resumeCommand: resumeCommand = "" - logger.Infof(" \U00002699 %s", line) + defCommandLogger.Infof(" \U00002699 %s", line) case "save-state": - logger.Infof(" \U0001f4be %s", line) + defCommandLogger.Infof(" \U0001f4be %s", line) rc.saveState(ctx, kvPairs, arg) case "add-matcher": - logger.Infof(" \U00002753 add-matcher %s", arg) + defCommandLogger.Infof(" \U00002753 add-matcher %s", arg) default: - logger.Infof(" \U00002753 %s", line) + defCommandLogger.Infof(" \U00002753 %s", line) } // return true to let gitea's logger handle these special outputs also @@ -86,7 +89,7 @@ func (rc *RunContext) commandHandler(ctx context.Context) common.LineHandler { func (rc *RunContext) setEnv(ctx context.Context, kvPairs map[string]string, arg string) { name := kvPairs["name"] - common.Logger(ctx).Infof(" \U00002699 ::set-env:: %s=%s", name, arg) + common.Logger(ctx).WithFields(logrus.Fields{"command": "set-env", "name": name, "arg": arg}).Infof(" \U00002699 ::set-env:: %s=%s", name, arg) if rc.Env == nil { rc.Env = make(map[string]string) } @@ -119,12 +122,12 @@ func (rc *RunContext) setOutput(ctx context.Context, kvPairs map[string]string, return } - logger.Infof(" \U00002699 ::set-output:: %s=%s", outputName, arg) + logger.WithFields(logrus.Fields{"command": "set-output", "name": outputName, "arg": arg}).Infof(" \U00002699 ::set-output:: %s=%s", outputName, arg) result.Outputs[outputName] = arg } func (rc *RunContext) addPath(ctx context.Context, arg string) { - common.Logger(ctx).Infof(" \U00002699 ::add-path:: %s", arg) + common.Logger(ctx).WithFields(logrus.Fields{"command": "add-path", "arg": arg}).Infof(" \U00002699 ::add-path:: %s", arg) extraPath := []string{arg} for _, v := range rc.ExtraPath { if v != arg { @@ -136,8 +139,8 @@ func (rc *RunContext) addPath(ctx context.Context, arg string) { func parseKeyValuePairs(kvPairs, separator string) map[string]string { rtn := make(map[string]string) - kvPairList := strings.Split(kvPairs, separator) - for _, kvPair := range kvPairList { + kvPairList := strings.SplitSeq(kvPairs, separator) + for kvPair := range kvPairList { kv := strings.Split(kvPair, "=") if len(kv) == 2 { rtn[kv[0]] = kv[1] diff --git a/act/runner/command_test.go b/act/runner/command_test.go index b2c3824a..4db11089 100644 --- a/act/runner/command_test.go +++ b/act/runner/command_test.go @@ -2,7 +2,6 @@ package runner import ( "bytes" - "context" "io" "os" "testing" @@ -10,13 +9,13 @@ import ( "github.com/sirupsen/logrus/hooks/test" "github.com/stretchr/testify/assert" - "code.forgejo.org/forgejo/runner/v9/act/common" - "code.forgejo.org/forgejo/runner/v9/act/model" + "code.forgejo.org/forgejo/runner/v11/act/common" + "code.forgejo.org/forgejo/runner/v11/act/model" ) func TestCommandSetEnv(t *testing.T) { a := assert.New(t) - ctx := context.Background() + ctx := t.Context() rc := new(RunContext) handler := rc.commandHandler(ctx) @@ -26,7 +25,7 @@ func TestCommandSetEnv(t *testing.T) { func TestCommandSetOutput(t *testing.T) { a := assert.New(t) - ctx := context.Background() + ctx := t.Context() rc := new(RunContext) rc.StepResults = make(map[string]*model.StepResult) handler := rc.commandHandler(ctx) @@ -56,7 +55,7 @@ func TestCommandSetOutput(t *testing.T) { func TestCommandAddpath(t *testing.T) { a := assert.New(t) - ctx := context.Background() + ctx := t.Context() rc := new(RunContext) handler := rc.commandHandler(ctx) @@ -71,7 +70,7 @@ func TestCommandStopCommands(t *testing.T) { logger, hook := test.NewNullLogger() a := assert.New(t) - ctx := common.WithLogger(context.Background(), logger) + ctx := common.WithLogger(t.Context(), logger) rc := new(RunContext) handler := rc.commandHandler(ctx) @@ -94,7 +93,7 @@ func TestCommandStopCommands(t *testing.T) { func TestCommandAddpathADO(t *testing.T) { a := assert.New(t) - ctx := context.Background() + ctx := t.Context() rc := new(RunContext) handler := rc.commandHandler(ctx) @@ -109,7 +108,7 @@ func TestCommandAddmask(t *testing.T) { logger, hook := test.NewNullLogger() a := assert.New(t) - ctx := context.Background() + ctx := t.Context() loggerCtx := common.WithLogger(ctx, logger) rc := new(RunContext) @@ -163,8 +162,8 @@ func TestCommandAddmaskUsemask(t *testing.T) { } re := captureOutput(t, func() { - ctx := context.Background() - ctx = WithJobLogger(ctx, "0", "testjob", config, &rc.Masks, map[string]interface{}{}) + ctx := t.Context() + ctx = WithJobLogger(ctx, "0", "testjob", config, &rc.Masks, map[string]any{}) handler := rc.commandHandler(ctx) handler("::add-mask::secret\n") @@ -180,7 +179,7 @@ func TestCommandSaveState(t *testing.T) { StepResults: map[string]*model.StepResult{}, } - ctx := context.Background() + ctx := t.Context() handler := rc.commandHandler(ctx) handler("::save-state name=state-name::state-value\n") diff --git a/act/runner/container_mock_test.go b/act/runner/container_mock_test.go index 5d884a21..ce5603f7 100644 --- a/act/runner/container_mock_test.go +++ b/act/runner/container_mock_test.go @@ -4,8 +4,8 @@ import ( "context" "io" - "code.forgejo.org/forgejo/runner/v9/act/common" - "code.forgejo.org/forgejo/runner/v9/act/container" + "code.forgejo.org/forgejo/runner/v11/act/common" + "code.forgejo.org/forgejo/runner/v11/act/container" "github.com/stretchr/testify/mock" ) diff --git a/act/runner/expression.go b/act/runner/expression.go index a81c7d39..152a2f8e 100644 --- a/act/runner/expression.go +++ b/act/runner/expression.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "fmt" + "maps" "path" "reflect" "regexp" @@ -12,16 +13,16 @@ import ( _ "embed" - "code.forgejo.org/forgejo/runner/v9/act/common" - "code.forgejo.org/forgejo/runner/v9/act/container" - "code.forgejo.org/forgejo/runner/v9/act/exprparser" - "code.forgejo.org/forgejo/runner/v9/act/model" - "gopkg.in/yaml.v3" + "code.forgejo.org/forgejo/runner/v11/act/common" + "code.forgejo.org/forgejo/runner/v11/act/container" + "code.forgejo.org/forgejo/runner/v11/act/exprparser" + "code.forgejo.org/forgejo/runner/v11/act/model" + "go.yaml.in/yaml/v3" ) // ExpressionEvaluator is the interface for evaluating expressions type ExpressionEvaluator interface { - evaluate(context.Context, string, exprparser.DefaultStatusCheck) (interface{}, error) + evaluate(context.Context, string, exprparser.DefaultStatusCheck) (any, error) EvaluateYamlNode(context.Context, *yaml.Node) error Interpolate(context.Context, string) string } @@ -36,7 +37,7 @@ func (rc *RunContext) NewExpressionEvaluatorWithEnv(ctx context.Context, env map // todo: cleanup EvaluationEnvironment creation using := make(map[string]exprparser.Needs) - strategy := make(map[string]interface{}) + strategy := make(map[string]any) if rc.Run != nil { job := rc.Run.Job() if job != nil && job.Strategy != nil { @@ -64,9 +65,7 @@ func (rc *RunContext) NewExpressionEvaluatorWithEnv(ctx context.Context, env map result := model.WorkflowCallResult{ Outputs: map[string]string{}, } - for k, v := range job.Outputs { - result.Outputs[k] = v - } + maps.Copy(result.Outputs, job.Outputs) workflowCallResult[jobName] = &result } } @@ -108,9 +107,22 @@ var hashfiles string // NewStepExpressionEvaluator creates a new evaluator func (rc *RunContext) NewStepExpressionEvaluator(ctx context.Context, step step) ExpressionEvaluator { + return rc.NewStepExpressionEvaluatorExt(ctx, step, false) +} + +// NewStepExpressionEvaluatorExt creates a new evaluator +func (rc *RunContext) NewStepExpressionEvaluatorExt(ctx context.Context, step step, rcInputs bool) ExpressionEvaluator { + ghc := rc.getGithubContext(ctx) + if rcInputs { + return rc.newStepExpressionEvaluator(ctx, step, ghc, getEvaluatorInputs(ctx, rc, nil, ghc)) + } + return rc.newStepExpressionEvaluator(ctx, step, ghc, getEvaluatorInputs(ctx, rc, step, ghc)) +} + +func (rc *RunContext) newStepExpressionEvaluator(ctx context.Context, step step, ghc *model.GithubContext, inputs map[string]any) ExpressionEvaluator { // todo: cleanup EvaluationEnvironment creation job := rc.Run.Job() - strategy := make(map[string]interface{}) + strategy := make(map[string]any) if job.Strategy != nil { strategy["fail-fast"] = job.Strategy.FailFast strategy["max-parallel"] = job.Strategy.MaxParallel @@ -127,9 +139,6 @@ func (rc *RunContext) NewStepExpressionEvaluator(ctx context.Context, step step) } } - ghc := rc.getGithubContext(ctx) - inputs := getEvaluatorInputs(ctx, rc, step, ghc) - ee := &exprparser.EvaluationEnvironment{ Github: step.getGithubContext(ctx), Env: *step.getEnv(), @@ -157,8 +166,8 @@ func (rc *RunContext) NewStepExpressionEvaluator(ctx context.Context, step step) } } -func getHashFilesFunction(ctx context.Context, rc *RunContext) func(v []reflect.Value) (interface{}, error) { - hashFiles := func(v []reflect.Value) (interface{}, error) { +func getHashFilesFunction(ctx context.Context, rc *RunContext) func(v []reflect.Value) (any, error) { + hashFiles := func(v []reflect.Value) (any, error) { if rc.JobContainer != nil { timeed, cancel := context.WithTimeout(ctx, time.Minute) defer cancel() @@ -182,9 +191,7 @@ func getHashFilesFunction(ctx context.Context, rc *RunContext) func(v []reflect. patterns = append(patterns, s) } env := map[string]string{} - for k, v := range rc.Env { - env[k] = v - } + maps.Copy(env, rc.Env) env["patterns"] = strings.Join(patterns, "\n") if followSymlink { env["followSymbolicLinks"] = "true" @@ -222,7 +229,7 @@ type expressionEvaluator struct { interpreter exprparser.Interpreter } -func (ee expressionEvaluator) evaluate(ctx context.Context, in string, defaultStatusCheck exprparser.DefaultStatusCheck) (interface{}, error) { +func (ee expressionEvaluator) evaluate(ctx context.Context, in string, defaultStatusCheck exprparser.DefaultStatusCheck) (any, error) { logger := common.Logger(ctx) logger.Debugf("evaluating expression '%s'", in) evaluated, err := ee.interpreter.Evaluate(in, defaultStatusCheck) @@ -469,10 +476,8 @@ func rewriteSubExpression(ctx context.Context, in string, forceFormat bool) (str return out, nil } -func getEvaluatorInputs(ctx context.Context, rc *RunContext, step step, ghc *model.GithubContext) map[string]interface{} { - inputs := map[string]interface{}{} - - setupWorkflowInputs(ctx, &inputs, rc) +func getEvaluatorInputs(ctx context.Context, rc *RunContext, step step, ghc *model.GithubContext) map[string]any { + inputs := map[string]any{} var env map[string]string if step != nil { @@ -482,12 +487,14 @@ func getEvaluatorInputs(ctx context.Context, rc *RunContext, step step, ghc *mod } for k, v := range env { - if strings.HasPrefix(k, "INPUT_") { - inputs[strings.ToLower(strings.TrimPrefix(k, "INPUT_"))] = v + if after, ok := strings.CutPrefix(k, "INPUT_"); ok { + inputs[strings.ToLower(after)] = v } } - if ghc.EventName == "workflow_dispatch" { + setupWorkflowInputs(ctx, &inputs, rc) + + if rc.caller == nil && ghc.EventName == "workflow_dispatch" { config := rc.Run.Workflow.WorkflowDispatchConfig() if config != nil && config.Inputs != nil { for k, v := range config.Inputs { @@ -510,7 +517,7 @@ func getEvaluatorInputs(ctx context.Context, rc *RunContext, step step, ghc *mod for k, v := range config.Inputs { value := nestedMapLookup(ghc.Event, "inputs", k) if value == nil { - value = v.Default + _ = v.Default.Decode(&value) } if v.Type == "boolean" { inputs[k] = value == "true" @@ -523,27 +530,30 @@ func getEvaluatorInputs(ctx context.Context, rc *RunContext, step step, ghc *mod return inputs } -func setupWorkflowInputs(ctx context.Context, inputs *map[string]interface{}, rc *RunContext) { +func setupWorkflowInputs(ctx context.Context, inputs *map[string]any, rc *RunContext) { if rc.caller != nil { config := rc.Run.Workflow.WorkflowCallConfig() for name, input := range config.Inputs { value := rc.caller.runContext.Run.Job().With[name] + if value != nil { - if str, ok := value.(string); ok { + node := yaml.Node{} + _ = node.Encode(value) + if rc.caller.runContext.ExprEval != nil { // evaluate using the calling RunContext (outside) - value = rc.caller.runContext.ExprEval.Interpolate(ctx, str) + _ = rc.caller.runContext.ExprEval.EvaluateYamlNode(ctx, &node) } + _ = node.Decode(&value) } if value == nil && config != nil && config.Inputs != nil { - value = input.Default + def := input.Default if rc.ExprEval != nil { - if str, ok := value.(string); ok { - // evaluate using the called RunContext (inside) - value = rc.ExprEval.Interpolate(ctx, str) - } + // evaluate using the called RunContext (inside) + _ = rc.ExprEval.EvaluateYamlNode(ctx, &def) } + _ = def.Decode(&value) } (*inputs)[name] = value @@ -554,21 +564,21 @@ func setupWorkflowInputs(ctx context.Context, inputs *map[string]interface{}, rc func getWorkflowSecrets(ctx context.Context, rc *RunContext) map[string]string { if rc.caller != nil { job := rc.caller.runContext.Run.Job() - secrets := job.Secrets() + rawSecrets := job.Secrets() - if secrets == nil && job.InheritSecrets() { - secrets = rc.caller.runContext.Config.Secrets + if rawSecrets == nil && job.InheritSecrets() { + rawSecrets = rc.caller.runContext.Config.Secrets } - if secrets == nil { - secrets = map[string]string{} + if rawSecrets == nil { + return map[string]string{} } - for k, v := range secrets { - secrets[k] = rc.caller.runContext.ExprEval.Interpolate(ctx, v) + interpolatedSecrets := make(map[string]string, len(rawSecrets)) + for k, v := range rawSecrets { + interpolatedSecrets[k] = rc.caller.runContext.ExprEval.Interpolate(ctx, v) } - - return secrets + return interpolatedSecrets } return rc.Config.Secrets diff --git a/act/runner/expression_test.go b/act/runner/expression_test.go index 3bebc301..12eecd9f 100644 --- a/act/runner/expression_test.go +++ b/act/runner/expression_test.go @@ -1,18 +1,17 @@ package runner import ( - "context" "testing" - "code.forgejo.org/forgejo/runner/v9/act/exprparser" - "code.forgejo.org/forgejo/runner/v9/act/model" + "code.forgejo.org/forgejo/runner/v11/act/exprparser" + "code.forgejo.org/forgejo/runner/v11/act/model" assert "github.com/stretchr/testify/assert" - yaml "gopkg.in/yaml.v3" + yaml "go.yaml.in/yaml/v3" ) func createRunContext(t *testing.T) *RunContext { var yml yaml.Node - err := yml.Encode(map[string][]interface{}{ + err := yml.Encode(map[string][]any{ "os": {"Linux", "Windows"}, "foo": {"bar", "baz"}, }) @@ -44,7 +43,7 @@ func createRunContext(t *testing.T) *RunContext { }, }, }, - Matrix: map[string]interface{}{ + Matrix: map[string]any{ "os": "Linux", "foo": "bar", }, @@ -76,11 +75,11 @@ func createRunContext(t *testing.T) *RunContext { func TestExpressionEvaluateRunContext(t *testing.T) { rc := createRunContext(t) - ee := rc.NewExpressionEvaluator(context.Background()) + ee := rc.NewExpressionEvaluator(t.Context()) tables := []struct { in string - out interface{} + out any errMesg string }{ {" 1 ", 1, ""}, @@ -134,10 +133,9 @@ func TestExpressionEvaluateRunContext(t *testing.T) { } for _, table := range tables { - table := table t.Run(table.in, func(t *testing.T) { assertObject := assert.New(t) - out, err := ee.evaluate(context.Background(), table.in, exprparser.DefaultStatusCheckNone) + out, err := ee.evaluate(t.Context(), table.in, exprparser.DefaultStatusCheckNone) if table.errMesg == "" { assertObject.NoError(err, table.in) assertObject.Equal(table.out, out, table.in) @@ -155,11 +153,11 @@ func TestExpressionEvaluateStep(t *testing.T) { RunContext: rc, } - ee := rc.NewStepExpressionEvaluator(context.Background(), step) + ee := rc.NewStepExpressionEvaluator(t.Context(), step) tables := []struct { in string - out interface{} + out any errMesg string }{ {"steps.idwithnothing.conclusion", model.StepStatusSuccess.String(), ""}, @@ -174,10 +172,9 @@ func TestExpressionEvaluateStep(t *testing.T) { } for _, table := range tables { - table := table t.Run(table.in, func(t *testing.T) { assertObject := assert.New(t) - out, err := ee.evaluate(context.Background(), table.in, exprparser.DefaultStatusCheckNone) + out, err := ee.evaluate(t.Context(), table.in, exprparser.DefaultStatusCheckNone) if table.errMesg == "" { assertObject.NoError(err, table.in) assertObject.Equal(table.out, out, table.in) @@ -217,7 +214,7 @@ func TestExpressionInterpolate(t *testing.T) { }, }, } - ee := rc.NewExpressionEvaluator(context.Background()) + ee := rc.NewExpressionEvaluator(t.Context()) tables := []struct { in string out string @@ -257,10 +254,9 @@ func TestExpressionInterpolate(t *testing.T) { } for _, table := range tables { - table := table t.Run("interpolate", func(t *testing.T) { assertObject := assert.New(t) - out := ee.Interpolate(context.Background(), table.in) + out := ee.Interpolate(t.Context(), table.in) assertObject.Equal(table.out, out, table.in) }) } @@ -287,7 +283,7 @@ func TestExpressionRewriteSubExpression(t *testing.T) { for _, table := range table { t.Run("TestRewriteSubExpression", func(t *testing.T) { assertObject := assert.New(t) - out, err := rewriteSubExpression(context.Background(), table.in, false) + out, err := rewriteSubExpression(t.Context(), table.in, false) if err != nil { t.Fatal(err) } @@ -311,7 +307,7 @@ func TestExpressionRewriteSubExpressionForceFormat(t *testing.T) { for _, table := range table { t.Run("TestRewriteSubExpressionForceFormat", func(t *testing.T) { assertObject := assert.New(t) - out, err := rewriteSubExpression(context.Background(), table.in, true) + out, err := rewriteSubExpression(t.Context(), table.in, true) if err != nil { t.Fatal(err) } diff --git a/act/runner/job_executor.go b/act/runner/job_executor.go index f9ab36a0..f1ef21c5 100644 --- a/act/runner/job_executor.go +++ b/act/runner/job_executor.go @@ -5,13 +5,14 @@ import ( "fmt" "time" - "code.forgejo.org/forgejo/runner/v9/act/common" - "code.forgejo.org/forgejo/runner/v9/act/container" - "code.forgejo.org/forgejo/runner/v9/act/model" + "code.forgejo.org/forgejo/runner/v11/act/common" + "code.forgejo.org/forgejo/runner/v11/act/container" + "code.forgejo.org/forgejo/runner/v11/act/model" + "github.com/sirupsen/logrus" ) type jobInfo interface { - matrix() map[string]interface{} + matrix() map[string]any steps() []*model.Step startContainer() common.Executor stopContainer() common.Executor @@ -56,16 +57,11 @@ func newJobExecutor(info jobInfo, sf stepFactory, rc *RunContext) common.Executo }) for i, stepModel := range infoSteps { - stepModel := stepModel if stepModel == nil { - return func(ctx context.Context) error { - return fmt.Errorf("invalid Step %v: missing run or uses key", i) - } + return common.NewErrorExecutor(fmt.Errorf("invalid Step %v: missing run or uses key", i)) + } else if stepModel.Number != i { + return common.NewErrorExecutor(fmt.Errorf("internal error: invalid Step: Number expected %v, was actually %v", i, stepModel.Number)) } - if stepModel.ID == "" { - stepModel.ID = fmt.Sprintf("%d", i) - } - stepModel.Number = i step, err := sf.newStep(stepModel, rc) if err != nil { @@ -109,41 +105,40 @@ func newJobExecutor(info jobInfo, sf stepFactory, rc *RunContext) common.Executo } } - postExecutor = postExecutor.Finally(func(ctx context.Context) error { + setJobResults := func(ctx context.Context) error { jobError := common.JobError(ctx) // Fresh context to ensure job result output works even if prev. context was a cancelled job ctx, cancel := context.WithTimeout(common.WithLogger(context.Background(), common.Logger(ctx)), time.Minute) defer cancel() setJobResult(ctx, info, rc, jobError == nil) - setJobOutputs(ctx, rc) + return nil + } + + cleanupJob := func(_ context.Context) error { var err error - if rc.Config.AutoRemove || jobError == nil { - // Separate timeout for cleanup tasks; logger is cleared so that cleanup logs go to runner, not job - ctx, cancel := context.WithTimeout(context.Background(), cleanupTimeout) - defer cancel() - logger := common.Logger(ctx) - logger.Debugf("Cleaning up container for job %s", rc.jobContainerName()) - if err = info.stopContainer()(ctx); err != nil { - logger.Errorf("Error while stop job container %s: %v", rc.jobContainerName(), err) - } + // Separate timeout for cleanup tasks; logger is cleared so that cleanup logs go to runner, not job + ctx, cancel := context.WithTimeout(context.Background(), cleanupTimeout) + defer cancel() - if !rc.IsHostEnv(ctx) && rc.Config.ContainerNetworkMode == "" { - // clean network in docker mode only - // if the value of `ContainerNetworkMode` is empty string, - // it means that the network to which containers are connecting is created by `act_runner`, - // so, we should remove the network at last. - networkName, _ := rc.networkName() - logger.Debugf("Cleaning up network %s for job %s", networkName, rc.jobContainerName()) - if err := container.NewDockerNetworkRemoveExecutor(networkName)(ctx); err != nil { - logger.Errorf("Error while cleaning network %s: %v", networkName, err) - } + logger := common.Logger(ctx) + logger.Debugf("Cleaning up container for job %s", rc.jobContainerName()) + if err = info.stopContainer()(ctx); err != nil { + logger.Errorf("Error while stop job container %s: %v", rc.jobContainerName(), err) + } + + if !rc.IsHostEnv(ctx) && rc.getNetworkCreated(ctx) { + networkName := rc.getNetworkName(ctx) + logger.Debugf("Cleaning up network %s for job %s", networkName, rc.jobContainerName()) + if err := container.NewDockerNetworkRemoveExecutor(networkName)(ctx); err != nil { + logger.Errorf("Error while cleaning network %s: %v", networkName, err) } } + return err - }) + } pipeline := make([]common.Executor, 0) pipeline = append(pipeline, preSteps...) @@ -161,12 +156,19 @@ func newJobExecutor(info jobInfo, sf stepFactory, rc *RunContext) common.Executo return postExecutor(ctx) }). Finally(info.interpolateOutputs()). + Finally(setJobResults). + Finally(cleanupJob). Finally(info.closeContainer())) } func setJobResult(ctx context.Context, info jobInfo, rc *RunContext, success bool) { logger := common.Logger(ctx) + // As we're reading the matrix build's status (`rc.Run.Job().Result`), it's possible for it change in another + // goroutine running `setJobResult` and invoking `.result(...)`. Prevent concurrent execution of `setJobResult`... + rc.Run.Job().ResultMutex.Lock() + defer rc.Run.Job().ResultMutex.Unlock() + jobResult := "success" // we have only one result for a whole matrix build, so we need // to keep an existing result state if we run a matrix @@ -178,33 +180,42 @@ func setJobResult(ctx context.Context, info jobInfo, rc *RunContext, success boo jobResult = "failure" } + // Set local result on current job (child or parent) info.result(jobResult) + if rc.caller != nil { - // set reusable workflow job result + // Child reusable workflow: + // 1) propagate result to parent job state rc.caller.runContext.result(jobResult) + + // 2) copy workflow_call outputs from child to parent (as in upstream) + jobOutputs := make(map[string]string) + ee := rc.NewExpressionEvaluator(ctx) + if wfcc := rc.Run.Workflow.WorkflowCallConfig(); wfcc != nil { + for k, v := range wfcc.Outputs { + jobOutputs[k] = ee.Interpolate(ctx, ee.Interpolate(ctx, v.Value)) + } + } + rc.caller.runContext.Run.Job().Outputs = jobOutputs + + // 3) DO NOT print banner in child job (prevents premature token revocation) + logger.Debugf("Reusable job result=%s (parent will finalize, no banner)", jobResult) + return } + // Parent job: print the final banner ONCE (job-level) jobResultMessage := "succeeded" if jobResult != "success" { jobResultMessage = "failed" } + jobOutputs := rc.Run.Job().Outputs - logger.WithField("jobResult", jobResult).Infof("\U0001F3C1 Job %s", jobResultMessage) -} - -func setJobOutputs(ctx context.Context, rc *RunContext) { - if rc.caller != nil { - // map outputs for reusable workflows - callerOutputs := make(map[string]string) - - ee := rc.NewExpressionEvaluator(ctx) - - for k, v := range rc.Run.Workflow.WorkflowCallConfig().Outputs { - callerOutputs[k] = ee.Interpolate(ctx, ee.Interpolate(ctx, v.Value)) - } - - rc.caller.runContext.Run.Job().Outputs = callerOutputs - } + logger. + WithFields(logrus.Fields{ + "jobResult": jobResult, + "jobOutputs": jobOutputs, + }). + Infof("\U0001F3C1 Job %s", jobResultMessage) } func useStepLogger(rc *RunContext, stepModel *model.Step, stage stepStage, executor common.Executor) common.Executor { diff --git a/act/runner/job_executor_test.go b/act/runner/job_executor_test.go index f5c6e110..96ace803 100644 --- a/act/runner/job_executor_test.go +++ b/act/runner/job_executor_test.go @@ -4,15 +4,22 @@ import ( "context" "fmt" "io" + "slices" + "sync" "testing" + "time" - "code.forgejo.org/forgejo/runner/v9/act/common" - "code.forgejo.org/forgejo/runner/v9/act/container" - "code.forgejo.org/forgejo/runner/v9/act/model" + "code.forgejo.org/forgejo/runner/v11/act/common" + "code.forgejo.org/forgejo/runner/v11/act/container" + "code.forgejo.org/forgejo/runner/v11/act/model" + "code.forgejo.org/forgejo/runner/v11/act/runner/mocks" + "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" ) +//go:generate mockery --srcpkg=github.com/sirupsen/logrus --name=FieldLogger + func TestJobExecutor(t *testing.T) { tables := []TestJobFileInfo{ {workdir, "uses-and-run-in-one-step", "push", "Invalid run/uses syntax for job:test step:Test", platforms, secrets}, @@ -26,7 +33,7 @@ func TestJobExecutor(t *testing.T) { {workdir, "job-nil-step", "push", "invalid Step 0: missing run or uses key", platforms, secrets}, } // These tests are sufficient to only check syntax. - ctx := common.WithDryrun(context.Background(), true) + ctx := common.WithDryrun(t.Context(), true) for _, table := range tables { t.Run(table.workflowPath, func(t *testing.T) { table.runTest(ctx, t, &Config{}) @@ -38,9 +45,9 @@ type jobInfoMock struct { mock.Mock } -func (jim *jobInfoMock) matrix() map[string]interface{} { +func (jim *jobInfoMock) matrix() map[string]any { args := jim.Called() - return args.Get(0).(map[string]interface{}) + return args.Get(0).(map[string]any) } func (jim *jobInfoMock) steps() []*model.Step { @@ -124,8 +131,9 @@ func TestJobExecutorNewJobExecutor(t *testing.T) { executedSteps: []string{ "startContainer", "step1", - "stopContainer", "interpolateOutputs", + "setJobResults", + "stopContainer", "closeContainer", }, result: "success", @@ -142,6 +150,8 @@ func TestJobExecutorNewJobExecutor(t *testing.T) { "startContainer", "step1", "interpolateOutputs", + "setJobResults", + "stopContainer", "closeContainer", }, result: "failure", @@ -158,8 +168,9 @@ func TestJobExecutorNewJobExecutor(t *testing.T) { "startContainer", "pre1", "step1", - "stopContainer", "interpolateOutputs", + "setJobResults", + "stopContainer", "closeContainer", }, result: "success", @@ -176,8 +187,9 @@ func TestJobExecutorNewJobExecutor(t *testing.T) { "startContainer", "step1", "post1", - "stopContainer", "interpolateOutputs", + "setJobResults", + "stopContainer", "closeContainer", }, result: "success", @@ -195,8 +207,9 @@ func TestJobExecutorNewJobExecutor(t *testing.T) { "pre1", "step1", "post1", - "stopContainer", "interpolateOutputs", + "setJobResults", + "stopContainer", "closeContainer", }, result: "success", @@ -205,11 +218,14 @@ func TestJobExecutorNewJobExecutor(t *testing.T) { { name: "stepsWithPreAndPost", steps: []*model.Step{{ - ID: "1", + ID: "1", + Number: 0, }, { - ID: "2", + ID: "2", + Number: 1, }, { - ID: "3", + ID: "3", + Number: 2, }}, preSteps: []bool{true, false, true}, postSteps: []bool{false, true, true}, @@ -222,8 +238,9 @@ func TestJobExecutorNewJobExecutor(t *testing.T) { "step3", "post3", "post2", - "stopContainer", "interpolateOutputs", + "setJobResults", + "stopContainer", "closeContainer", }, result: "success", @@ -232,19 +249,34 @@ func TestJobExecutorNewJobExecutor(t *testing.T) { } contains := func(needle string, haystack []string) bool { - for _, item := range haystack { - if item == needle { - return true - } - } - return false + return slices.Contains(haystack, needle) } for _, tt := range table { t.Run(tt.name, func(t *testing.T) { fmt.Printf("::group::%s\n", tt.name) - ctx := common.WithJobErrorContainer(context.Background()) + executorOrder := make([]string, 0) + + mockLogger := mocks.NewFieldLogger(t) + mockLogger.On("Debugf", mock.Anything, mock.Anything).Return(0).Maybe() + mockLogger.On("Warningf", mock.Anything, mock.Anything).Return(0).Maybe() + mockLogger.On("WithField", mock.Anything, mock.Anything).Return(&logrus.Entry{Logger: &logrus.Logger{}}).Maybe() + // When `WithFields()` is called with jobResult & jobOutputs field, add `setJobResults` to executorOrder. + mockLogger.On("WithFields", + mock.MatchedBy(func(fields logrus.Fields) bool { + _, okJobResult := fields["jobResult"] + _, okJobOutput := fields["jobOutputs"] + return okJobOutput && okJobResult + })). + Run(func(args mock.Arguments) { + executorOrder = append(executorOrder, "setJobResults") + }). + Return(&logrus.Entry{Logger: &logrus.Logger{}}).Maybe() + + mockLogger.On("WithFields", mock.Anything).Return(&logrus.Entry{Logger: &logrus.Logger{}}).Maybe() + + ctx := common.WithLogger(common.WithJobErrorContainer(t.Context()), mockLogger) jim := &jobInfoMock{} sfm := &stepFactoryMock{} rc := &RunContext{ @@ -260,7 +292,6 @@ func TestJobExecutorNewJobExecutor(t *testing.T) { Config: &Config{}, } rc.ExprEval = rc.NewExpressionEvaluator(ctx) - executorOrder := make([]string, 0) jim.On("steps").Return(tt.steps) @@ -272,9 +303,6 @@ func TestJobExecutorNewJobExecutor(t *testing.T) { } for i, stepModel := range tt.steps { - i := i - stepModel := stepModel - sm := &stepMock{} sfm.On("newStep", stepModel, rc).Return(sm, nil) @@ -305,7 +333,7 @@ func TestJobExecutorNewJobExecutor(t *testing.T) { } if len(tt.steps) > 0 { - jim.On("matrix").Return(map[string]interface{}{}) + jim.On("matrix").Return(map[string]any{}) jim.On("interpolateOutputs").Return(func(ctx context.Context) error { executorOrder = append(executorOrder, "interpolateOutputs") @@ -339,3 +367,153 @@ func TestJobExecutorNewJobExecutor(t *testing.T) { }) } } + +func TestSetJobResultConcurrency(t *testing.T) { + jim := &jobInfoMock{} + job := model.Job{ + Result: "success", + } + // Distinct RunContext objects are used to replicate realistic setJobResult in matrix build + rc1 := &RunContext{ + Run: &model.Run{ + JobID: "test", + Workflow: &model.Workflow{ + Jobs: map[string]*model.Job{ + "test": &job, + }, + }, + }, + } + rc2 := &RunContext{ + Run: &model.Run{ + JobID: "test", + Workflow: &model.Workflow{ + Jobs: map[string]*model.Job{ + "test": &job, + }, + }, + }, + } + // Hack: Job() invokes GetJob() which can mutate the job name, this will trip the data race detector if it is + // encountered later when `setJobResult()` is being tested. This is a false-positive caused by this test invoking + // setJobResult outside of the regular RunContext, so it's invoked here before the goroutines are spawned to prevent + // the false positive. + rc1.Run.Job() + rc2.Run.Job() + + jim.On("matrix").Return(map[string]interface{}{ + "python": []string{"3.10", "3.11", "3.12"}, + }) + + // Synthesize a race condition in setJobResult where, by reading data from the job matrix earlier and then + // performing unsynchronzied writes to the same shared data structure, it can overwrite a failure status. + // + // Goroutine 1: Start marking job as success + // (artificially suspended + // by result() mock) + // Goroutine 2: Mark job as failure + // Goroutine 1: Finish marking job as success + // + // Correct behavior: Job is marked as a failure + // Bug behavior: Job is marked as a success + + var lastResult string + jim.On("result", mock.Anything).Run(func(args mock.Arguments) { + result := args.String(0) + // Artificially suspend the "success" case so that the failure case races past it. + if result == "success" { + time.Sleep(1 * time.Second) + } + job.Result = result + lastResult = result + }) + + var wg sync.WaitGroup + wg.Add(2) + // Goroutine 1, mark as success: + go func() { + defer wg.Done() + setJobResult(t.Context(), jim, rc1, true) + }() + // Goroutine 2, mark as failure: + go func() { + defer wg.Done() + setJobResult(t.Context(), jim, rc2, false) + }() + wg.Wait() + + assert.Equal(t, "failure", lastResult) +} + +func TestSetJobResult_SkipsBannerInChildReusableWorkflow(t *testing.T) { + // Test that child reusable workflow does not print final banner + // to prevent premature token revocation + + mockLogger := mocks.NewFieldLogger(t) + // Allow all variants of Debugf (git operations can call with 1-3 args) + mockLogger.On("Debugf", mock.Anything).Return(0).Maybe() + mockLogger.On("Debugf", mock.Anything, mock.Anything).Return(0).Maybe() + mockLogger.On("Debugf", mock.Anything, mock.Anything, mock.Anything).Return(0).Maybe() + // CRITICAL: In CI, git ref detection may fail and call Warningf + mockLogger.On("Warningf", mock.Anything, mock.Anything).Return(0).Maybe() + mockLogger.On("WithField", mock.Anything, mock.Anything).Return(&logrus.Entry{Logger: &logrus.Logger{}}).Maybe() + mockLogger.On("WithFields", mock.Anything).Return(&logrus.Entry{Logger: &logrus.Logger{}}).Maybe() + + ctx := common.WithLogger(common.WithJobErrorContainer(t.Context()), mockLogger) + + // Setup parent job + parentJob := &model.Job{ + Result: "success", + } + parentRC := &RunContext{ + Config: &Config{Env: map[string]string{}}, // Must have Config + Run: &model.Run{ + JobID: "parent", + Workflow: &model.Workflow{ + Jobs: map[string]*model.Job{ + "parent": parentJob, + }, + }, + }, + } + + // Setup child job with caller reference + childJob := &model.Job{ + Result: "success", + } + childRC := &RunContext{ + Config: &Config{Env: map[string]string{}}, // Must have Config + Run: &model.Run{ + JobID: "child", + Workflow: &model.Workflow{ + Jobs: map[string]*model.Job{ + "child": childJob, + }, + }, + }, + caller: &caller{ + runContext: parentRC, + }, + } + + jim := &jobInfoMock{} + jim.On("matrix").Return(map[string]any{}) // REQUIRED: setJobResult always calls matrix() + jim.On("result", "success") + + // Call setJobResult for child workflow + setJobResult(ctx, jim, childRC, true) + + // Verify: + // 1. Child result is set + jim.AssertCalled(t, "result", "success") + + // 2. Parent result is propagated + assert.Equal(t, "success", parentJob.Result) + + // 3. Final banner was NOT printed by child (critical for token security) + mockLogger.AssertNotCalled(t, "WithFields", mock.MatchedBy(func(fields logrus.Fields) bool { + _, okJobResult := fields["jobResult"] + _, okJobOutput := fields["jobOutputs"] + return okJobOutput && okJobResult + })) +} diff --git a/act/runner/local_repository_cache.go b/act/runner/local_repository_cache.go index 984ee2cd..17b60c92 100644 --- a/act/runner/local_repository_cache.go +++ b/act/runner/local_repository_cache.go @@ -12,7 +12,7 @@ import ( "path/filepath" "strings" - "code.forgejo.org/forgejo/runner/v9/act/filecollector" + "code.forgejo.org/forgejo/runner/v11/act/filecollector" ) type LocalRepositoryCache struct { diff --git a/act/runner/logger.go b/act/runner/logger.go index f8d4f000..5f4dae1d 100644 --- a/act/runner/logger.go +++ b/act/runner/logger.go @@ -5,11 +5,12 @@ import ( "context" "fmt" "io" + "maps" "os" "strings" "sync" - "code.forgejo.org/forgejo/runner/v9/act/common" + "code.forgejo.org/forgejo/runner/v11/act/common" "github.com/sirupsen/logrus" "golang.org/x/term" @@ -72,7 +73,7 @@ func WithJobLoggerFactory(ctx context.Context, factory JobLoggerFactory) context } // WithJobLogger attaches a new logger to context that is aware of steps -func WithJobLogger(ctx context.Context, jobID, jobName string, config *Config, masks *[]string, matrix map[string]interface{}) context.Context { +func WithJobLogger(ctx context.Context, jobID, jobName string, config *Config, masks *[]string, matrix map[string]any) context.Context { ctx = WithMasks(ctx, masks) var logger *logrus.Logger @@ -145,6 +146,26 @@ func WithCompositeStepLogger(ctx context.Context, stepID string) context.Context }).WithContext(ctx)) } +func GetOuterStepResult(entry *logrus.Entry) any { + r, ok := entry.Data["stepResult"] + if !ok { + return nil + } + + // composite actions steps log with a list of stepID + if s, ok := entry.Data["stepID"]; ok { + if stepIDs, ok := s.([]string); ok { + if len(stepIDs) > 1 { + return nil + } + } + } else { + return nil + } + + return r +} + func withStepLogger(ctx context.Context, stepNumber int, stepID, stepName, stageName string) context.Context { rtn := common.Logger(ctx).WithFields(logrus.Fields{ "stepNumber": stepNumber, @@ -158,6 +179,7 @@ func withStepLogger(ctx context.Context, stepNumber int, stepID, stepName, stage type entryProcessor func(entry *logrus.Entry) *logrus.Entry func valueMasker(insecureSecrets bool, secrets map[string]string) entryProcessor { + secrets = maps.Clone(secrets) return func(entry *logrus.Entry) *logrus.Entry { if insecureSecrets { return entry diff --git a/act/runner/logger_test.go b/act/runner/logger_test.go new file mode 100644 index 00000000..4df2db12 --- /dev/null +++ b/act/runner/logger_test.go @@ -0,0 +1,63 @@ +package runner + +import ( + "testing" + + "code.forgejo.org/forgejo/runner/v11/act/common" + + "github.com/sirupsen/logrus/hooks/test" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRunner_GetOuterStepResult(t *testing.T) { + nullLogger, hook := test.NewNullLogger() + ctx := common.WithLogger(t.Context(), nullLogger) + + t.Run("no stepResult", func(t *testing.T) { + hook.Reset() + common.Logger(ctx).Info("✅ Success") + entry := hook.LastEntry() + require.NotNil(t, entry) + assert.Nil(t, GetOuterStepResult(entry)) + }) + + t.Run("stepResult and no stepID", func(t *testing.T) { + hook.Reset() + common.Logger(ctx).WithField("stepResult", "success").Info("✅ Success") + entry := hook.LastEntry() + require.NotNil(t, entry) + assert.Nil(t, GetOuterStepResult(entry)) + }) + + stepNumber := 123 + stepID := "step id" + stepName := "readable name" + stageName := "Main" + ctx = withStepLogger(ctx, stepNumber, stepID, stepName, stageName) + + t.Run("stepResult and stepID", func(t *testing.T) { + hook.Reset() + common.Logger(ctx).WithField("stepResult", "success").Info("✅ Success") + entry := hook.LastEntry() + actualStepIDs, ok := entry.Data["stepID"] + require.True(t, ok) + require.Equal(t, []string{stepID}, actualStepIDs) + require.NotNil(t, entry) + assert.Equal(t, "success", GetOuterStepResult(entry)) + }) + + compositeStepID := "composite step id" + ctx = WithCompositeStepLogger(ctx, compositeStepID) + + t.Run("stepResult and composite stepID", func(t *testing.T) { + hook.Reset() + common.Logger(ctx).WithField("stepResult", "success").Info("✅ Success") + entry := hook.LastEntry() + actualStepIDs, ok := entry.Data["stepID"] + require.True(t, ok) + require.Equal(t, []string{stepID, compositeStepID}, actualStepIDs) + require.NotNil(t, entry) + assert.Nil(t, GetOuterStepResult(entry)) + }) +} diff --git a/act/runner/lxc-helpers-lib.sh b/act/runner/lxc-helpers-lib.sh index 2f36541f..4d3b959a 100755 --- a/act/runner/lxc-helpers-lib.sh +++ b/act/runner/lxc-helpers-lib.sh @@ -11,6 +11,8 @@ LXC_IPV6_PREFIX_DEFAULT="fd15" LXC_DOCKER_PREFIX_DEFAULT="172.17" LXC_IPV6_DOCKER_PREFIX_DEFAULT="fd00:d0ca" LXC_APT_TOO_OLD='1 week ago' +: ${LXC_TRANSACTION_TIMEOUT:=600} +LXC_TRANSACTION_LOCK_FILE=/tmp/lxc-helper.lock : ${LXC_SUDO:=} : ${LXC_CONTAINER_RELEASE:=bookworm} @@ -28,16 +30,22 @@ function lxc_template_release() { echo lxc-helpers-$LXC_CONTAINER_RELEASE } +function lxc_directory() { + local name="$1" + + echo /var/lib/lxc/$name +} + function lxc_root() { local name="$1" - echo /var/lib/lxc/$name/rootfs + echo $(lxc_directory $name)/rootfs } function lxc_config() { local name="$1" - echo /var/lib/lxc/$name/config + echo $(lxc_directory $name)/config } function lxc_container_run() { @@ -47,6 +55,42 @@ function lxc_container_run() { $LXC_SUDO lxc-attach --clear-env --name $name -- "$@" } +function lxc_transaction_lock() { + exec 7>$LXC_TRANSACTION_LOCK_FILE + flock --timeout $LXC_TRANSACTION_TIMEOUT 7 +} + +function lxc_transaction_unlock() { + exec 7>&- +} + +function lxc_transaction_draft_name() { + echo "lxc-helper-draft" +} + +function lxc_transaction_begin() { + local name=$1 # not actually used but it helps when reading in the caller + local draft=$(lxc_transaction_draft_name) + + lxc_transaction_lock + lxc_container_destroy $draft +} + +function lxc_transaction_commit() { + local name=$1 + local draft=$(lxc_transaction_draft_name) + + # do not use lxc-copy because it is not atomic if lxc-copy is + # interrupted it may leave the $name container half populated + $LXC_SUDO sed -i -e "s/$draft/$name/g" \ + $(lxc_config $draft) \ + $(lxc_root $draft)/etc/hosts \ + $(lxc_root $draft)/etc/hostname + $LXC_SUDO rm -f $(lxc_root $draft)/var/lib/dhcp/dhclient.* + $LXC_SUDO mv $(lxc_directory $draft) $(lxc_directory $name) + lxc_transaction_unlock +} + function lxc_container_run_script_as() { local name="$1" local user="$2" @@ -242,7 +286,7 @@ function lxc_container_configure() { function lxc_container_install_lxc_helpers() { local name="$1" - $LXC_SUDO cp -a $LXC_SELF_DIR/lxc-helpers*.sh $root/$LXC_BIN + $LXC_SUDO cp -a $LXC_SELF_DIR/lxc-helpers*.sh $(lxc_root $name)/$LXC_BIN # # Wait for the network to come up # @@ -304,10 +348,9 @@ function lxc_container_stop() { function lxc_container_destroy() { local name="$1" - local root="$2" if lxc_exists "$name"; then - lxc_container_stop $name $root + lxc_container_stop $name $LXC_SUDO lxc-destroy --force --name="$name" fi } @@ -342,36 +385,44 @@ function lxc_running() { function lxc_build_template_release() { local name="$(lxc_template_release)" + lxc_transaction_begin $name + if lxc_exists_and_apt_not_old $name; then + lxc_transaction_unlock return fi - local root=$(lxc_root $name) - $LXC_SUDO lxc-create --name $name --template debian -- --release=$LXC_CONTAINER_RELEASE - echo 'lxc.apparmor.profile = unconfined' | $LXC_SUDO tee -a $(lxc_config $name) - lxc_container_install_lxc_helpers $name - lxc_container_start $name - lxc_container_run $name apt-get update -qq - lxc_apt_install $name sudo git python3 - lxc_container_stop $name + local draft=$(lxc_transaction_draft_name) + $LXC_SUDO lxc-create --name $draft --template debian -- --release=$LXC_CONTAINER_RELEASE + echo 'lxc.apparmor.profile = unconfined' | $LXC_SUDO tee -a $(lxc_config $draft) + lxc_container_install_lxc_helpers $draft + lxc_container_start $draft + lxc_container_run $draft apt-get update -qq + lxc_apt_install $draft sudo git python3 + lxc_container_stop $draft + lxc_transaction_commit $name } function lxc_build_template() { local name="$1" local newname="$2" - if lxc_exists_and_apt_not_old $newname; then - return - fi - if test "$name" = "$(lxc_template_release)"; then lxc_build_template_release fi - if ! $LXC_SUDO lxc-copy --name=$name --newname=$newname; then - echo lxc-copy --name=$name --newname=$newname failed + lxc_transaction_begin $name + if lxc_exists_and_apt_not_old $newname; then + lxc_transaction_unlock + return + fi + + local draft=$(lxc_transaction_draft_name) + if ! $LXC_SUDO lxc-copy --name=$name --newname=$draft; then + echo lxc-copy --name=$name --newname=$draft failed return 1 fi + lxc_transaction_commit $newname lxc_container_configure $newname } @@ -413,7 +464,7 @@ function lxc_install_lxc_inside() { local prefixv6="${2:-$LXC_IPV6_PREFIX_DEFAULT}" local packages="make git libvirt0 libpam-cgfs bridge-utils uidmap dnsmasq-base dnsmasq dnsmasq-utils qemu-user-static lxc-templates debootstrap" - if test "$(lxc_release)" = bookworm; then + if test "$(lxc_release)" != bullseye; then packages="$packages distro-info" fi diff --git a/act/runner/lxc-helpers.sh b/act/runner/lxc-helpers.sh index 705c5aab..769e238a 100755 --- a/act/runner/lxc-helpers.sh +++ b/act/runner/lxc-helpers.sh @@ -18,11 +18,11 @@ lxc-helpers.sh - LXC container management helpers SYNOPSIS lxc-helpers.sh [-v|--verbose] [-h|--help] - [-o|--os {bookworm|bullseye} (default bookworm)] + [-o|--os {trixie|bookworm|bullseye} (default bookworm)] command [arguments] lxc-helpers.sh [-v|--verbose] [-h|--help] - [-o|--os {bookworm|bullseye} (default bookworm)] + [-o|--os {trixie|bookworm|bullseye} (default bookworm)] [-c|--config {unprivileged lxc libvirt docker k8s} (default "lxc libvirt docker")] lxc_container_create [arguments] diff --git a/act/runner/mocks/FieldLogger.go b/act/runner/mocks/FieldLogger.go new file mode 100644 index 00000000..5cfa5ad2 --- /dev/null +++ b/act/runner/mocks/FieldLogger.go @@ -0,0 +1,264 @@ +// Code generated by mockery v2.53.5. DO NOT EDIT. + +package mocks + +import ( + logrus "github.com/sirupsen/logrus" + mock "github.com/stretchr/testify/mock" +) + +// FieldLogger is an autogenerated mock type for the FieldLogger type +type FieldLogger struct { + mock.Mock +} + +// Debug provides a mock function with given fields: args +func (_m *FieldLogger) Debug(args ...interface{}) { + var _ca []interface{} + _ca = append(_ca, args...) + _m.Called(_ca...) +} + +// Debugf provides a mock function with given fields: format, args +func (_m *FieldLogger) Debugf(format string, args ...interface{}) { + var _ca []interface{} + _ca = append(_ca, format) + _ca = append(_ca, args...) + _m.Called(_ca...) +} + +// Debugln provides a mock function with given fields: args +func (_m *FieldLogger) Debugln(args ...interface{}) { + var _ca []interface{} + _ca = append(_ca, args...) + _m.Called(_ca...) +} + +// Error provides a mock function with given fields: args +func (_m *FieldLogger) Error(args ...interface{}) { + var _ca []interface{} + _ca = append(_ca, args...) + _m.Called(_ca...) +} + +// Errorf provides a mock function with given fields: format, args +func (_m *FieldLogger) Errorf(format string, args ...interface{}) { + var _ca []interface{} + _ca = append(_ca, format) + _ca = append(_ca, args...) + _m.Called(_ca...) +} + +// Errorln provides a mock function with given fields: args +func (_m *FieldLogger) Errorln(args ...interface{}) { + var _ca []interface{} + _ca = append(_ca, args...) + _m.Called(_ca...) +} + +// Fatal provides a mock function with given fields: args +func (_m *FieldLogger) Fatal(args ...interface{}) { + var _ca []interface{} + _ca = append(_ca, args...) + _m.Called(_ca...) +} + +// Fatalf provides a mock function with given fields: format, args +func (_m *FieldLogger) Fatalf(format string, args ...interface{}) { + var _ca []interface{} + _ca = append(_ca, format) + _ca = append(_ca, args...) + _m.Called(_ca...) +} + +// Fatalln provides a mock function with given fields: args +func (_m *FieldLogger) Fatalln(args ...interface{}) { + var _ca []interface{} + _ca = append(_ca, args...) + _m.Called(_ca...) +} + +// Info provides a mock function with given fields: args +func (_m *FieldLogger) Info(args ...interface{}) { + var _ca []interface{} + _ca = append(_ca, args...) + _m.Called(_ca...) +} + +// Infof provides a mock function with given fields: format, args +func (_m *FieldLogger) Infof(format string, args ...interface{}) { + var _ca []interface{} + _ca = append(_ca, format) + _ca = append(_ca, args...) + _m.Called(_ca...) +} + +// Infoln provides a mock function with given fields: args +func (_m *FieldLogger) Infoln(args ...interface{}) { + var _ca []interface{} + _ca = append(_ca, args...) + _m.Called(_ca...) +} + +// Panic provides a mock function with given fields: args +func (_m *FieldLogger) Panic(args ...interface{}) { + var _ca []interface{} + _ca = append(_ca, args...) + _m.Called(_ca...) +} + +// Panicf provides a mock function with given fields: format, args +func (_m *FieldLogger) Panicf(format string, args ...interface{}) { + var _ca []interface{} + _ca = append(_ca, format) + _ca = append(_ca, args...) + _m.Called(_ca...) +} + +// Panicln provides a mock function with given fields: args +func (_m *FieldLogger) Panicln(args ...interface{}) { + var _ca []interface{} + _ca = append(_ca, args...) + _m.Called(_ca...) +} + +// Print provides a mock function with given fields: args +func (_m *FieldLogger) Print(args ...interface{}) { + var _ca []interface{} + _ca = append(_ca, args...) + _m.Called(_ca...) +} + +// Printf provides a mock function with given fields: format, args +func (_m *FieldLogger) Printf(format string, args ...interface{}) { + var _ca []interface{} + _ca = append(_ca, format) + _ca = append(_ca, args...) + _m.Called(_ca...) +} + +// Println provides a mock function with given fields: args +func (_m *FieldLogger) Println(args ...interface{}) { + var _ca []interface{} + _ca = append(_ca, args...) + _m.Called(_ca...) +} + +// Warn provides a mock function with given fields: args +func (_m *FieldLogger) Warn(args ...interface{}) { + var _ca []interface{} + _ca = append(_ca, args...) + _m.Called(_ca...) +} + +// Warnf provides a mock function with given fields: format, args +func (_m *FieldLogger) Warnf(format string, args ...interface{}) { + var _ca []interface{} + _ca = append(_ca, format) + _ca = append(_ca, args...) + _m.Called(_ca...) +} + +// Warning provides a mock function with given fields: args +func (_m *FieldLogger) Warning(args ...interface{}) { + var _ca []interface{} + _ca = append(_ca, args...) + _m.Called(_ca...) +} + +// Warningf provides a mock function with given fields: format, args +func (_m *FieldLogger) Warningf(format string, args ...interface{}) { + var _ca []interface{} + _ca = append(_ca, format) + _ca = append(_ca, args...) + _m.Called(_ca...) +} + +// Warningln provides a mock function with given fields: args +func (_m *FieldLogger) Warningln(args ...interface{}) { + var _ca []interface{} + _ca = append(_ca, args...) + _m.Called(_ca...) +} + +// Warnln provides a mock function with given fields: args +func (_m *FieldLogger) Warnln(args ...interface{}) { + var _ca []interface{} + _ca = append(_ca, args...) + _m.Called(_ca...) +} + +// WithError provides a mock function with given fields: err +func (_m *FieldLogger) WithError(err error) *logrus.Entry { + ret := _m.Called(err) + + if len(ret) == 0 { + panic("no return value specified for WithError") + } + + var r0 *logrus.Entry + if rf, ok := ret.Get(0).(func(error) *logrus.Entry); ok { + r0 = rf(err) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*logrus.Entry) + } + } + + return r0 +} + +// WithField provides a mock function with given fields: key, value +func (_m *FieldLogger) WithField(key string, value interface{}) *logrus.Entry { + ret := _m.Called(key, value) + + if len(ret) == 0 { + panic("no return value specified for WithField") + } + + var r0 *logrus.Entry + if rf, ok := ret.Get(0).(func(string, interface{}) *logrus.Entry); ok { + r0 = rf(key, value) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*logrus.Entry) + } + } + + return r0 +} + +// WithFields provides a mock function with given fields: fields +func (_m *FieldLogger) WithFields(fields logrus.Fields) *logrus.Entry { + ret := _m.Called(fields) + + if len(ret) == 0 { + panic("no return value specified for WithFields") + } + + var r0 *logrus.Entry + if rf, ok := ret.Get(0).(func(logrus.Fields) *logrus.Entry); ok { + r0 = rf(fields) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*logrus.Entry) + } + } + + return r0 +} + +// NewFieldLogger creates a new instance of FieldLogger. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewFieldLogger(t interface { + mock.TestingT + Cleanup(func()) +}, +) *FieldLogger { + mock := &FieldLogger{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/act/runner/reusable_workflow.go b/act/runner/reusable_workflow.go index 3a805b0a..c117c4a7 100644 --- a/act/runner/reusable_workflow.go +++ b/act/runner/reusable_workflow.go @@ -6,15 +6,17 @@ import ( "errors" "fmt" "io/fs" + "net/url" "os" "path" "regexp" "strings" "sync" - "code.forgejo.org/forgejo/runner/v9/act/common" - "code.forgejo.org/forgejo/runner/v9/act/common/git" - "code.forgejo.org/forgejo/runner/v9/act/model" + "code.forgejo.org/forgejo/runner/v11/act/common" + "code.forgejo.org/forgejo/runner/v11/act/common/git" + "code.forgejo.org/forgejo/runner/v11/act/model" + "github.com/sirupsen/logrus" ) func newLocalReusableWorkflowExecutor(rc *RunContext) common.Executor { @@ -54,9 +56,14 @@ func newLocalReusableWorkflowExecutor(rc *RunContext) common.Executor { func newRemoteReusableWorkflowExecutor(rc *RunContext) common.Executor { uses := rc.Run.Job().Uses - remoteReusableWorkflow := newRemoteReusableWorkflowWithPlat(rc.Config.GitHubInstance, uses) + url, err := url.Parse(uses) + if err != nil { + return common.NewErrorExecutor(fmt.Errorf("'%s' cannot be parsed as a URL: %v", uses, err)) + } + + remoteReusableWorkflow := newRemoteReusableWorkflowWithPlat(url.Host, strings.TrimPrefix(url.Path, "/")) if remoteReusableWorkflow == nil { - return common.NewErrorExecutor(fmt.Errorf("expected format {owner}/{repo}/.{git_platform}/workflows/{filename}@{ref}. Actual '%s' Input string was not in a correct format", uses)) + return common.NewErrorExecutor(fmt.Errorf("expected format {owner}/{repo}/.{git_platform}/workflows/{filename}@{ref}. Actual '%s' Input string was not in a correct format", url.Path)) } // uses with safe filename makes the target directory look something like this {owner}-{repo}-.github-workflows-{filename}@{ref} @@ -109,7 +116,10 @@ func newActionCacheReusableWorkflowExecutor(rc *RunContext, filename string, rem return err } - return runner.NewPlanExecutor(plan)(ctx) + planErr := runner.NewPlanExecutor(plan)(ctx) + + // Finalize from parent context: one job-level banner + return finalizeReusableWorkflow(ctx, rc, planErr) } } @@ -165,7 +175,10 @@ func newReusableWorkflowExecutor(rc *RunContext, directory, workflow string) com return err } - return runner.NewPlanExecutor(plan)(ctx) + planErr := runner.NewPlanExecutor(plan)(ctx) + + // Finalize from parent context: one job-level banner + return finalizeReusableWorkflow(ctx, rc, planErr) } } @@ -223,3 +236,29 @@ func newRemoteReusableWorkflowWithPlat(url, uses string) *remoteReusableWorkflow URL: url, } } + +// finalizeReusableWorkflow prints the final job banner from the parent job context. +// +// The Forgejo reporter waits for this banner (log entry with "jobResult" +// field and without stage="Main") before marking the job as complete and revoking +// tokens. Printing this banner from the child reusable workflow would cause +// premature token revocation, breaking subsequent steps in the parent workflow. +func finalizeReusableWorkflow(ctx context.Context, rc *RunContext, planErr error) error { + jobResult := "success" + jobResultMessage := "succeeded" + if planErr != nil { + jobResult = "failure" + jobResultMessage = "failed" + } + + // Outputs should already be present in the parent context: + // - copied by child's setJobResult branch (rc.caller != nil) + jobOutputs := rc.Run.Job().Outputs + + common.Logger(ctx).WithFields(logrus.Fields{ + "jobResult": jobResult, + "jobOutputs": jobOutputs, + }).Infof("\U0001F3C1 Job %s", jobResultMessage) + + return planErr +} diff --git a/act/runner/reusable_workflow_test.go b/act/runner/reusable_workflow_test.go new file mode 100644 index 00000000..f4aab893 --- /dev/null +++ b/act/runner/reusable_workflow_test.go @@ -0,0 +1,247 @@ +package runner + +import ( + "errors" + "testing" + + "code.forgejo.org/forgejo/runner/v11/act/common" + "code.forgejo.org/forgejo/runner/v11/act/model" + "code.forgejo.org/forgejo/runner/v11/act/runner/mocks" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +func TestConfig_GetToken(t *testing.T) { + t.Run("returns GITEA_TOKEN when both GITEA_TOKEN and GITHUB_TOKEN present", func(t *testing.T) { + c := &Config{ + Secrets: map[string]string{ + "GITHUB_TOKEN": "github-token", + "GITEA_TOKEN": "gitea-token", + }, + } + assert.Equal(t, "gitea-token", c.GetToken()) + }) + + t.Run("returns GITHUB_TOKEN when only GITHUB_TOKEN present", func(t *testing.T) { + c := &Config{ + Secrets: map[string]string{ + "GITHUB_TOKEN": "github-token", + }, + } + assert.Equal(t, "github-token", c.GetToken()) + }) + + t.Run("returns empty string when no tokens present", func(t *testing.T) { + c := &Config{ + Secrets: map[string]string{}, + } + assert.Equal(t, "", c.GetToken()) + }) + + t.Run("returns empty string when Secrets is nil", func(t *testing.T) { + c := &Config{} + assert.Equal(t, "", c.GetToken()) + }) +} + +func TestRemoteReusableWorkflow_CloneURL(t *testing.T) { + t.Run("adds https prefix when missing", func(t *testing.T) { + rw := &remoteReusableWorkflow{ + URL: "code.forgejo.org", + Org: "owner", + Repo: "repo", + } + assert.Equal(t, "https://code.forgejo.org/owner/repo", rw.CloneURL()) + }) + + t.Run("preserves https prefix", func(t *testing.T) { + rw := &remoteReusableWorkflow{ + URL: "https://code.forgejo.org", + Org: "owner", + Repo: "repo", + } + assert.Equal(t, "https://code.forgejo.org/owner/repo", rw.CloneURL()) + }) + + t.Run("preserves http prefix", func(t *testing.T) { + rw := &remoteReusableWorkflow{ + URL: "http://localhost:3000", + Org: "owner", + Repo: "repo", + } + assert.Equal(t, "http://localhost:3000/owner/repo", rw.CloneURL()) + }) +} + +func TestRemoteReusableWorkflow_FilePath(t *testing.T) { + tests := []struct { + name string + gitPlatform string + filename string + expectedPath string + }{ + { + name: "github platform", + gitPlatform: "github", + filename: "test.yml", + expectedPath: "./.github/workflows/test.yml", + }, + { + name: "gitea platform", + gitPlatform: "gitea", + filename: "build.yaml", + expectedPath: "./.gitea/workflows/build.yaml", + }, + { + name: "forgejo platform", + gitPlatform: "forgejo", + filename: "deploy.yml", + expectedPath: "./.forgejo/workflows/deploy.yml", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + rw := &remoteReusableWorkflow{ + GitPlatform: tt.gitPlatform, + Filename: tt.filename, + } + assert.Equal(t, tt.expectedPath, rw.FilePath()) + }) + } +} + +func TestNewRemoteReusableWorkflowWithPlat(t *testing.T) { + tests := []struct { + name string + url string + uses string + expectedOrg string + expectedRepo string + expectedPlatform string + expectedFilename string + expectedRef string + shouldFail bool + }{ + { + name: "valid github workflow", + url: "github.com", + uses: "owner/repo/.github/workflows/test.yml@main", + expectedOrg: "owner", + expectedRepo: "repo", + expectedPlatform: "github", + expectedFilename: "test.yml", + expectedRef: "main", + shouldFail: false, + }, + { + name: "valid gitea workflow", + url: "code.forgejo.org", + uses: "forgejo/runner/.gitea/workflows/build.yaml@v1.0.0", + expectedOrg: "forgejo", + expectedRepo: "runner", + expectedPlatform: "gitea", + expectedFilename: "build.yaml", + expectedRef: "v1.0.0", + shouldFail: false, + }, + { + name: "invalid format - missing platform", + url: "github.com", + uses: "owner/repo/workflows/test.yml@main", + shouldFail: true, + }, + { + name: "invalid format - no ref", + url: "github.com", + uses: "owner/repo/.github/workflows/test.yml", + shouldFail: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := newRemoteReusableWorkflowWithPlat(tt.url, tt.uses) + + if tt.shouldFail { + assert.Nil(t, result) + } else { + assert.NotNil(t, result) + assert.Equal(t, tt.expectedOrg, result.Org) + assert.Equal(t, tt.expectedRepo, result.Repo) + assert.Equal(t, tt.expectedPlatform, result.GitPlatform) + assert.Equal(t, tt.expectedFilename, result.Filename) + assert.Equal(t, tt.expectedRef, result.Ref) + assert.Equal(t, tt.url, result.URL) + } + }) + } +} + +func TestFinalizeReusableWorkflow_PrintsBannerSuccess(t *testing.T) { + mockLogger := mocks.NewFieldLogger(t) + + bannerCalled := false + mockLogger.On("WithFields", + mock.MatchedBy(func(fields logrus.Fields) bool { + result, ok := fields["jobResult"].(string) + if !ok || result != "success" { + return false + } + outs, ok := fields["jobOutputs"].(map[string]string) + return ok && outs["foo"] == "bar" + }), + ).Run(func(args mock.Arguments) { + bannerCalled = true + }).Return(&logrus.Entry{Logger: &logrus.Logger{}}).Once() + + ctx := common.WithLogger(t.Context(), mockLogger) + rc := &RunContext{ + Run: &model.Run{ + JobID: "parent", + Workflow: &model.Workflow{ + Jobs: map[string]*model.Job{ + "parent": { + Outputs: map[string]string{"foo": "bar"}, + }, + }, + }, + }, + } + + err := finalizeReusableWorkflow(ctx, rc, nil) + assert.NoError(t, err) + assert.True(t, bannerCalled, "final banner should be printed from parent") +} + +func TestFinalizeReusableWorkflow_PrintsBannerFailure(t *testing.T) { + mockLogger := mocks.NewFieldLogger(t) + + bannerCalled := false + mockLogger.On("WithFields", + mock.MatchedBy(func(fields logrus.Fields) bool { + result, ok := fields["jobResult"].(string) + return ok && result == "failure" + }), + ).Run(func(args mock.Arguments) { + bannerCalled = true + }).Return(&logrus.Entry{Logger: &logrus.Logger{}}).Once() + + ctx := common.WithLogger(t.Context(), mockLogger) + rc := &RunContext{ + Run: &model.Run{ + JobID: "parent", + Workflow: &model.Workflow{ + Jobs: map[string]*model.Job{ + "parent": {}, + }, + }, + }, + } + + planErr := errors.New("workflow failed") + err := finalizeReusableWorkflow(ctx, rc, planErr) + assert.EqualError(t, err, "workflow failed") + assert.True(t, bannerCalled, "banner should be printed even on failure") +} diff --git a/act/runner/run_context.go b/act/runner/run_context.go index d1fc1cd8..dd72ff09 100644 --- a/act/runner/run_context.go +++ b/act/runner/run_context.go @@ -11,6 +11,7 @@ import ( "errors" "fmt" "io" + "maps" "os" "path" "path/filepath" @@ -20,10 +21,10 @@ import ( "text/template" "time" - "code.forgejo.org/forgejo/runner/v9/act/common" - "code.forgejo.org/forgejo/runner/v9/act/container" - "code.forgejo.org/forgejo/runner/v9/act/exprparser" - "code.forgejo.org/forgejo/runner/v9/act/model" + "code.forgejo.org/forgejo/runner/v11/act/common" + "code.forgejo.org/forgejo/runner/v11/act/container" + "code.forgejo.org/forgejo/runner/v11/act/exprparser" + "code.forgejo.org/forgejo/runner/v11/act/model" "github.com/docker/docker/api/types/network" "github.com/docker/go-connections/nat" "github.com/opencontainers/selinux/go-selinux" @@ -33,7 +34,7 @@ import ( type RunContext struct { Name string Config *Config - Matrix map[string]interface{} + Matrix map[string]any Run *model.Run EventJSON string Env map[string]string @@ -52,6 +53,9 @@ type RunContext struct { Masks []string cleanUpJobContainer common.Executor caller *caller // job calling this RunContext (reusable workflows) + randomName string + networkName string + networkCreated bool } func (rc *RunContext) AddMask(mask string) { @@ -94,15 +98,7 @@ func (rc *RunContext) GetEnv() map[string]string { } func (rc *RunContext) jobContainerName() string { - return createSimpleContainerName(rc.Config.ContainerNamePrefix, "WORKFLOW-"+common.Sha256(rc.Run.Workflow.Name), "JOB-"+rc.Name) -} - -// networkName return the name of the network which will be created by `act` automatically for job, -func (rc *RunContext) networkName() (string, bool) { - if len(rc.Run.Job().Services) > 0 || rc.Config.ContainerNetworkMode == "" { - return fmt.Sprintf("%s-%s-network", rc.jobContainerName(), rc.Run.JobID), true - } - return string(rc.Config.ContainerNetworkMode), false + return createSimpleContainerName(rc.Config.ContainerNamePrefix, "WORKFLOW-"+common.Sha256(rc.String()), "JOB-"+rc.Name) } func getDockerDaemonSocketMountPath(daemonPath string) string { @@ -123,24 +119,37 @@ func getDockerDaemonSocketMountPath(daemonPath string) string { return daemonPath } -// Returns the binds and mounts for the container, resolving paths as appopriate -func (rc *RunContext) GetBindsAndMounts() ([]string, map[string]string) { - name := rc.jobContainerName() - - if rc.Config.ContainerDaemonSocket == "" { - rc.Config.ContainerDaemonSocket = "/var/run/docker.sock" +func (rc *RunContext) getInternalVolumeNames(ctx context.Context) []string { + return []string{ + rc.getInternalVolumeWorkdir(ctx), + rc.getInternalVolumeEnv(ctx), } +} +func (rc *RunContext) getInternalVolumeWorkdir(ctx context.Context) string { + rc.ensureRandomName(ctx) + return rc.randomName +} + +func (rc *RunContext) getInternalVolumeEnv(ctx context.Context) string { + rc.ensureRandomName(ctx) + return fmt.Sprintf("%s-env", rc.randomName) +} + +// Returns the binds and mounts for the container, resolving paths as appopriate +func (rc *RunContext) GetBindsAndMounts(ctx context.Context) ([]string, map[string]string, []string) { binds := []string{} - if rc.Config.ContainerDaemonSocket != "-" { - daemonPath := getDockerDaemonSocketMountPath(rc.Config.ContainerDaemonSocket) + + containerDaemonSocket := rc.Config.GetContainerDaemonSocket() + if containerDaemonSocket != "-" { + daemonPath := getDockerDaemonSocketMountPath(containerDaemonSocket) binds = append(binds, fmt.Sprintf("%s:%s", daemonPath, "/var/run/docker.sock")) } ext := container.LinuxContainerEnvironmentExtensions{} mounts := map[string]string{ - name + "-env": ext.GetActPath(), + rc.getInternalVolumeEnv(ctx): ext.GetActPath(), } if job := rc.Run.Job(); job != nil { @@ -168,16 +177,12 @@ func (rc *RunContext) GetBindsAndMounts() ([]string, map[string]string) { } binds = append(binds, fmt.Sprintf("%s:%s%s", rc.Config.Workdir, ext.ToContainerPath(rc.Config.Workdir), bindModifiers)) } else { - mounts[name] = ext.ToContainerPath(rc.Config.Workdir) + mounts[rc.getInternalVolumeWorkdir(ctx)] = ext.ToContainerPath(rc.Config.Workdir) } - // add some default binds and mounts to ValidVolumes - rc.Config.ValidVolumes = append(rc.Config.ValidVolumes, name) - rc.Config.ValidVolumes = append(rc.Config.ValidVolumes, name+"-env") - // TODO: add a new configuration to control whether the docker daemon can be mounted - rc.Config.ValidVolumes = append(rc.Config.ValidVolumes, getDockerDaemonSocketMountPath(rc.Config.ContainerDaemonSocket)) - - return binds, mounts + validVolumes := append(rc.getInternalVolumeNames(ctx), getDockerDaemonSocketMountPath(containerDaemonSocket)) + validVolumes = append(validVolumes, rc.Config.ValidVolumes...) + return binds, mounts, validVolumes } //go:embed lxc-helpers-lib.sh @@ -243,12 +248,18 @@ var stopTemplate = template.Must(template.New("stop").Parse(`#!/bin/bash source $(dirname $0)/lxc-helpers-lib.sh lxc_container_destroy "{{.Name}}" +lxc_maybe_sudo +$LXC_SUDO rm -fr "{{ .Root }}" `)) func (rc *RunContext) stopHostEnvironment(ctx context.Context) error { logger := common.Logger(ctx) logger.Debugf("stopHostEnvironment") + if !rc.IsLXCHostEnv(ctx) { + return nil + } + var stopScript bytes.Buffer if err := stopTemplate.Execute(&stopScript, struct { Name string @@ -283,10 +294,7 @@ func (rc *RunContext) startHostEnvironment() common.Executor { return true }) cacheDir := rc.ActionCacheDir() - randName, err := common.RandName(8) - if err != nil { - return err - } + randName := common.MustRandName(8) miscpath := filepath.Join(cacheDir, randName) actPath := filepath.Join(miscpath, "act") if err := os.MkdirAll(actPath, 0o777); err != nil { @@ -308,13 +316,18 @@ func (rc *RunContext) startHostEnvironment() common.Executor { ToolCache: rc.getToolCache(ctx), Workdir: rc.Config.Workdir, ActPath: actPath, - CleanUp: func() { - os.RemoveAll(miscpath) - }, - StdOut: logWriter, - LXC: rc.IsLXCHostEnv(ctx), + StdOut: logWriter, + LXC: rc.IsLXCHostEnv(ctx), + } + rc.cleanUpJobContainer = func(ctx context.Context) error { + if err := rc.stopHostEnvironment(ctx); err != nil { + return err + } + if rc.JobContainer == nil { + return nil + } + return rc.JobContainer.Remove()(ctx) } - rc.cleanUpJobContainer = rc.JobContainer.Remove() for k, v := range rc.JobContainer.GetRunnerContext(ctx) { if v, ok := v.(string); ok { rc.Env[fmt.Sprintf("RUNNER_%s", strings.ToUpper(k))] = v @@ -391,15 +404,44 @@ func (rc *RunContext) startHostEnvironment() common.Executor { } } -func (rc *RunContext) getNetworkName(_ context.Context) (networkName string, createAndDeleteNetwork bool) { - // specify the network to which the container will connect when `docker create` stage. (like execute command line: docker create --network ) - networkName = string(rc.Config.ContainerNetworkMode) - if networkName == "" { - // if networkName is empty string, will create a new network for the containers. - // and it will be removed after at last. - networkName, createAndDeleteNetwork = rc.networkName() +func (rc *RunContext) ensureRandomName(ctx context.Context) { + if rc.randomName == "" { + logger := common.Logger(ctx) + if rc.Parent != nil { + // composite actions inherit their run context from the parent job + rootRunContext := rc + for rootRunContext.Parent != nil { + rootRunContext = rootRunContext.Parent + } + rootRunContext.ensureRandomName(ctx) + rc.randomName = rootRunContext.randomName + logger.Debugf("RunContext inherited random name %s from its parent", rc.Name, rc.randomName) + } else { + rc.randomName = common.MustRandName(16) + logger.Debugf("RunContext %s is assigned random name %s", rc.Name, rc.randomName) + } + } +} + +func (rc *RunContext) getNetworkCreated(ctx context.Context) bool { + rc.ensureNetworkName(ctx) + return rc.networkCreated +} + +func (rc *RunContext) getNetworkName(ctx context.Context) string { + rc.ensureNetworkName(ctx) + return rc.networkName +} + +func (rc *RunContext) ensureNetworkName(ctx context.Context) { + if rc.networkName == "" { + rc.ensureRandomName(ctx) + rc.networkName = string(rc.Config.ContainerNetworkMode) + if len(rc.Run.Job().Services) > 0 || rc.networkName == "" { + rc.networkName = fmt.Sprintf("WORKFLOW-%s", rc.randomName) + rc.networkCreated = true + } } - return networkName, createAndDeleteNetwork } var sanitizeNetworkAliasRegex = regexp.MustCompile("[^a-z0-9-]") @@ -446,9 +488,8 @@ func (rc *RunContext) prepareJobContainer(ctx context.Context) error { envList = append(envList, fmt.Sprintf("%s=%s", "LANG", "C.UTF-8")) // Use same locale as GitHub Actions ext := container.LinuxContainerEnvironmentExtensions{} - binds, mounts := rc.GetBindsAndMounts() + binds, mounts, validVolumes := rc.GetBindsAndMounts(ctx) - networkName, createAndDeleteNetwork := rc.getNetworkName(ctx) // add service containers for serviceID, spec := range rc.Run.Job().Services { // interpolate env @@ -501,8 +542,7 @@ func (rc *RunContext) prepareJobContainer(ctx context.Context) error { Privileged: rc.Config.Privileged, UsernsMode: rc.Config.UsernsMode, Platform: rc.Config.ContainerArchitecture, - AutoRemove: rc.Config.AutoRemove, - NetworkMode: networkName, + NetworkMode: rc.getNetworkName(ctx), NetworkAliases: []string{sanitizeNetworkAlias(ctx, serviceID)}, ExposedPorts: exposedPorts, PortBindings: portBindings, @@ -525,8 +565,7 @@ func (rc *RunContext) prepareJobContainer(ctx context.Context) error { if rc.JobContainer != nil { return rc.JobContainer.Remove().IfNot(reuseJobContainer). - Then(container.NewDockerVolumeRemoveExecutor(rc.jobContainerName(), false)).IfNot(reuseJobContainer). - Then(container.NewDockerVolumeRemoveExecutor(rc.jobContainerName()+"-env", false)).IfNot(reuseJobContainer). + Then(container.NewDockerVolumesRemoveExecutor(rc.getInternalVolumeNames(ctx))).IfNot(reuseJobContainer). Then(func(ctx context.Context) error { if len(rc.ServiceContainers) > 0 { logger.Infof("Cleaning up services for job %s", rc.JobName) @@ -534,13 +573,9 @@ func (rc *RunContext) prepareJobContainer(ctx context.Context) error { logger.Errorf("Error while cleaning services: %v", err) } } - if createAndDeleteNetwork { - // clean network if it has been created by act - // if using service containers - // it means that the network to which containers are connecting is created by `act_runner`, - // so, we should remove the network at last. - logger.Infof("Cleaning up network for job %s, and network name is: %s", rc.JobName, networkName) - if err := container.NewDockerNetworkRemoveExecutor(networkName)(ctx); err != nil { + if rc.getNetworkCreated(ctx) { + logger.Infof("Cleaning up network for job %s, and network name is: %s", rc.JobName, rc.getNetworkName(ctx)) + if err := container.NewDockerNetworkRemoveExecutor(rc.getNetworkName(ctx))(ctx); err != nil { logger.Errorf("Error while cleaning network: %v", err) } } @@ -561,7 +596,7 @@ func (rc *RunContext) prepareJobContainer(ctx context.Context) error { Env: envList, ToolCache: rc.getToolCache(ctx), Mounts: mounts, - NetworkMode: networkName, + NetworkMode: rc.getNetworkName(ctx), NetworkAliases: []string{sanitizeNetworkAlias(ctx, rc.Name)}, Binds: binds, Stdout: logWriter, @@ -569,8 +604,7 @@ func (rc *RunContext) prepareJobContainer(ctx context.Context) error { Privileged: rc.Config.Privileged, UsernsMode: rc.Config.UsernsMode, Platform: rc.Config.ContainerArchitecture, - AutoRemove: rc.Config.AutoRemove, - ValidVolumes: rc.Config.ValidVolumes, + ValidVolumes: validVolumes, JobOptions: rc.options(ctx), ConfigOptions: rc.Config.ContainerOptions, @@ -587,7 +621,6 @@ func (rc *RunContext) startJobContainer() common.Executor { if err := rc.prepareJobContainer(ctx); err != nil { return err } - networkName, _ := rc.getNetworkName(ctx) networkConfig := network.CreateOptions{ Driver: "bridge", Scope: "local", @@ -597,8 +630,8 @@ func (rc *RunContext) startJobContainer() common.Executor { rc.pullServicesImages(rc.Config.ForcePull), rc.JobContainer.Pull(rc.Config.ForcePull), rc.stopJobContainer(), - container.NewDockerNetworkCreateExecutor(networkName, &networkConfig).IfBool(!rc.IsHostEnv(ctx) && rc.Config.ContainerNetworkMode == ""), // if the value of `ContainerNetworkMode` is empty string, then will create a new network for containers. - rc.startServiceContainers(networkName), + container.NewDockerNetworkCreateExecutor(rc.getNetworkName(ctx), &networkConfig).IfBool(!rc.IsHostEnv(ctx) && rc.Config.ContainerNetworkMode == ""), // if the value of `ContainerNetworkMode` is empty string, then will create a new network for containers. + rc.startServiceContainers(rc.getNetworkName(ctx)), rc.JobContainer.Create(rc.Config.ContainerCapAdd, rc.Config.ContainerCapDrop), rc.JobContainer.Start(false), rc.JobContainer.Copy(rc.JobContainer.GetActPath()+"/", &container.FileEntry{ @@ -610,6 +643,7 @@ func (rc *RunContext) startJobContainer() common.Executor { Mode: 0o666, Body: "", }), + rc.waitForServiceContainers(), )(ctx) } } @@ -621,14 +655,9 @@ func (rc *RunContext) sh(ctx context.Context, script string) (stdout, stderr str herr := &bytes.Buffer{} env := map[string]string{} - for k, v := range rc.Env { - env[k] = v - } + maps.Copy(env, rc.Env) - base, err := common.RandName(8) - if err != nil { - return "", "", err - } + base := common.MustRandName(8) name := base + ".sh" oldStdout, oldStderr := rc.JobContainer.ReplaceLogWriter(hout, herr) err = rc.JobContainer.Copy(rc.JobContainer.GetActPath(), &container.FileEntry{ @@ -744,6 +773,35 @@ func (rc *RunContext) startServiceContainers(_ string) common.Executor { } } +func waitForServiceContainer(ctx context.Context, c container.ExecutionsEnvironment) error { + for { + wait, err := c.IsHealthy(ctx) + if err != nil { + return err + } + if wait == time.Duration(0) { + return nil + } + select { + case <-ctx.Done(): + return nil + case <-time.After(wait): + } + } +} + +func (rc *RunContext) waitForServiceContainers() common.Executor { + return func(ctx context.Context) error { + execs := []common.Executor{} + for _, c := range rc.ServiceContainers { + execs = append(execs, func(ctx context.Context) error { + return waitForServiceContainer(ctx, c) + }) + } + return common.NewParallelExecutor(len(execs), execs...)(ctx) + } +} + func (rc *RunContext) stopServiceContainers() common.Executor { return func(ctx context.Context) error { execs := []common.Executor{} @@ -843,9 +901,6 @@ func (rc *RunContext) IsHostEnv(ctx context.Context) bool { func (rc *RunContext) stopContainer() common.Executor { return func(ctx context.Context) error { - if rc.IsLXCHostEnv(ctx) { - return rc.stopHostEnvironment(ctx) - } return rc.stopJobContainer()(ctx) } } @@ -853,16 +908,13 @@ func (rc *RunContext) stopContainer() common.Executor { func (rc *RunContext) closeContainer() common.Executor { return func(ctx context.Context) error { if rc.JobContainer != nil { - if rc.IsLXCHostEnv(ctx) { - return rc.stopHostEnvironment(ctx) - } return rc.JobContainer.Close()(ctx) } return nil } } -func (rc *RunContext) matrix() map[string]interface{} { +func (rc *RunContext) matrix() map[string]any { return rc.Matrix } @@ -896,7 +948,10 @@ func (rc *RunContext) Executor() (common.Executor, error) { return err } if res { - return executor(ctx) + timeoutctx, cancelTimeOut := evaluateTimeout(ctx, "job", rc.ExprEval, rc.Run.Job().TimeoutMinutes) + defer cancelTimeOut() + + return executor(timeoutctx) } return nil }, nil @@ -946,12 +1001,17 @@ func (rc *RunContext) runsOnPlatformNames(ctx context.Context) []string { return []string{} } - if err := rc.ExprEval.EvaluateYamlNode(ctx, &job.RawRunsOn); err != nil { + // Copy rawRunsOn from the job. `EvaluateYamlNode` later will mutate the yaml node in-place applying expression + // evaluation to it from the RunContext -- but the job object is shared in matrix executions between multiple + // running matrix jobs and `rc.EvalExpr` is specific to one matrix job. By copying the object we avoid mutating the + // shared field as it is accessed by multiple goroutines. + rawRunsOn := job.RawRunsOn + if err := rc.ExprEval.EvaluateYamlNode(ctx, &rawRunsOn); err != nil { common.Logger(ctx).Errorf("Error while evaluating runs-on: %v", err) return []string{} } - return job.RunsOn() + return model.FlattenRunsOnNode(rawRunsOn) } func (rc *RunContext) platformImage(ctx context.Context) string { @@ -975,11 +1035,11 @@ func (rc *RunContext) options(ctx context.Context) string { func (rc *RunContext) isEnabled(ctx context.Context) (bool, error) { job := rc.Run.Job() l := common.Logger(ctx) - runJob, runJobErr := EvalBool(ctx, rc.ExprEval, job.If.Value, exprparser.DefaultStatusCheckSuccess) + runJob, runJobErr := EvalBool(ctx, rc.ExprEval, job.IfClause(), exprparser.DefaultStatusCheckSuccess) jobType, jobTypeErr := job.Type() if runJobErr != nil { - return false, fmt.Errorf(" \u274C Error in if-expression: \"if: %s\" (%s)", job.If.Value, runJobErr) + return false, fmt.Errorf(" \u274C Error in if-expression: \"if: %s\" (%s)", job.IfClause(), runJobErr) } if jobType == model.JobTypeInvalid { @@ -988,7 +1048,7 @@ func (rc *RunContext) isEnabled(ctx context.Context) (bool, error) { if !runJob { rc.result("skipped") - l.WithField("jobResult", "skipped").Infof("Skipping job '%s' due to '%s'", job.Name, job.If.Value) + l.WithField("jobResult", "skipped").Infof("Skipping job '%s' due to '%s'", job.Name, job.IfClause()) return false, nil } @@ -1006,12 +1066,10 @@ func (rc *RunContext) isEnabled(ctx context.Context) (bool, error) { return true, nil } -func mergeMaps(maps ...map[string]string) map[string]string { +func mergeMaps(args ...map[string]string) map[string]string { rtnMap := make(map[string]string) - for _, m := range maps { - for k, v := range m { - rtnMap[k] = v - } + for _, m := range args { + maps.Copy(rtnMap, m) } return rtnMap } @@ -1076,8 +1134,9 @@ func (rc *RunContext) getStepsContext() map[string]*model.StepResult { func (rc *RunContext) getGithubContext(ctx context.Context) *model.GithubContext { logger := common.Logger(ctx) ghc := &model.GithubContext{ - Event: make(map[string]interface{}), + Event: make(map[string]any), Workflow: rc.Run.Workflow.Name, + RunAttempt: rc.Config.Env["GITHUB_RUN_ATTEMPT"], RunID: rc.Config.Env["GITHUB_RUN_ID"], RunNumber: rc.Config.Env["GITHUB_RUN_NUMBER"], Actor: rc.Config.Actor, @@ -1106,6 +1165,10 @@ func (rc *RunContext) getGithubContext(ctx context.Context) *model.GithubContext ghc.Workspace = rc.JobContainer.ToContainerPath(rc.Config.Workdir) } + if ghc.RunAttempt == "" { + ghc.RunAttempt = "1" + } + if ghc.RunID == "" { ghc.RunID = "1" } @@ -1147,7 +1210,7 @@ func (rc *RunContext) getGithubContext(ctx context.Context) *model.GithubContext ghc.RetentionDays = preset.RetentionDays instance := rc.Config.GitHubInstance - if !strings.HasPrefix(instance, "http://") && + if instance != "" && !strings.HasPrefix(instance, "http://") && !strings.HasPrefix(instance, "https://") { instance = "https://" + instance } @@ -1190,7 +1253,7 @@ func (rc *RunContext) getGithubContext(ctx context.Context) *model.GithubContext { // Adapt to Gitea instance := rc.Config.GitHubInstance - if !strings.HasPrefix(instance, "http://") && + if instance != "" && !strings.HasPrefix(instance, "http://") && !strings.HasPrefix(instance, "https://") { instance = "https://" + instance } @@ -1239,7 +1302,7 @@ func isLocalCheckout(ghc *model.GithubContext, step *model.Step) bool { return true } -func nestedMapLookup(m map[string]interface{}, ks ...string) (rval interface{}) { +func nestedMapLookup(m map[string]any, ks ...string) (rval any) { var ok bool if len(ks) == 0 { // degenerate input @@ -1249,7 +1312,7 @@ func nestedMapLookup(m map[string]interface{}, ks ...string) (rval interface{}) return nil } else if len(ks) == 1 { // we've reached the final key return rval - } else if m, ok = rval.(map[string]interface{}); !ok { + } else if m, ok = rval.(map[string]any); !ok { return nil } // 1+ more keys @@ -1264,6 +1327,7 @@ func (rc *RunContext) withGithubEnv(ctx context.Context, github *model.GithubCon } env["CI"] = "true" set("WORKFLOW", github.Workflow) + set("RUN_ATTEMPT", github.RunAttempt) set("RUN_ID", github.RunID) set("RUN_NUMBER", github.RunNumber) set("ACTION", github.Action) @@ -1291,16 +1355,6 @@ func (rc *RunContext) withGithubEnv(ctx context.Context, github *model.GithubCon set("SERVER_URL", github.ServerURL) set("API_URL", github.APIURL) - { // Adapt to Forgejo - instance := rc.Config.GitHubInstance - if !strings.HasPrefix(instance, "http://") && - !strings.HasPrefix(instance, "https://") { - instance = "https://" + instance - } - set("SERVER_URL", instance) - set("API_URL", instance+"/api/v1") - } - if rc.Config.ArtifactServerPath != "" { setActionRuntimeVars(rc, env) } @@ -1392,12 +1446,10 @@ func (rc *RunContext) handleServiceCredentials(ctx context.Context, creds map[st // GetServiceBindsAndMounts returns the binds and mounts for the service container, resolving paths as appopriate func (rc *RunContext) GetServiceBindsAndMounts(svcVolumes []string) ([]string, map[string]string) { - if rc.Config.ContainerDaemonSocket == "" { - rc.Config.ContainerDaemonSocket = "/var/run/docker.sock" - } + containerDaemonSocket := rc.Config.GetContainerDaemonSocket() binds := []string{} - if rc.Config.ContainerDaemonSocket != "-" { - daemonPath := getDockerDaemonSocketMountPath(rc.Config.ContainerDaemonSocket) + if containerDaemonSocket != "-" { + daemonPath := getDockerDaemonSocketMountPath(containerDaemonSocket) binds = append(binds, fmt.Sprintf("%s:%s", daemonPath, "/var/run/docker.sock")) } diff --git a/act/runner/run_context_test.go b/act/runner/run_context_test.go index 24caa4ad..4e11ba62 100644 --- a/act/runner/run_context_test.go +++ b/act/runner/run_context_test.go @@ -3,28 +3,30 @@ package runner import ( "cmp" "context" + "errors" "fmt" - "os" "runtime" "slices" "strings" "testing" + "time" - "code.forgejo.org/forgejo/runner/v9/act/container" - "code.forgejo.org/forgejo/runner/v9/act/exprparser" - "code.forgejo.org/forgejo/runner/v9/act/model" - "code.forgejo.org/forgejo/runner/v9/testutils" + "code.forgejo.org/forgejo/runner/v11/act/container" + "code.forgejo.org/forgejo/runner/v11/act/exprparser" + "code.forgejo.org/forgejo/runner/v11/act/model" + "code.forgejo.org/forgejo/runner/v11/testutils" "github.com/docker/go-connections/nat" log "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" - yaml "gopkg.in/yaml.v3" + yaml "go.yaml.in/yaml/v3" ) func TestRunContext_EvalBool(t *testing.T) { var yml yaml.Node - err := yml.Encode(map[string][]interface{}{ + err := yml.Encode(map[string][]any{ "os": {"Linux", "Windows"}, "foo": {"bar", "baz"}, }) @@ -52,7 +54,7 @@ func TestRunContext_EvalBool(t *testing.T) { }, }, }, - Matrix: map[string]interface{}{ + Matrix: map[string]any{ "os": "Linux", "foo": "bar", }, @@ -66,7 +68,7 @@ func TestRunContext_EvalBool(t *testing.T) { }, }, } - rc.ExprEval = rc.NewExpressionEvaluator(context.Background()) + rc.ExprEval = rc.NewExpressionEvaluator(t.Context()) tables := []struct { in string @@ -157,10 +159,9 @@ func TestRunContext_EvalBool(t *testing.T) { } for _, table := range tables { - table := table t.Run(table.in, func(t *testing.T) { assertObject := assert.New(t) - b, err := EvalBool(context.Background(), rc.ExprEval, table.in, exprparser.DefaultStatusCheckSuccess) + b, err := EvalBool(t.Context(), rc.ExprEval, table.in, exprparser.DefaultStatusCheckSuccess) if table.wantErr { assertObject.Error(err) } @@ -200,11 +201,7 @@ func TestRunContext_GetBindsAndMounts(t *testing.T) { isWindows := runtime.GOOS == "windows" for _, testcase := range tests { - // pin for scopelint - testcase := testcase for _, bindWorkDir := range []bool{true, false} { - // pin for scopelint - bindWorkDir := bindWorkDir testBindSuffix := "" if bindWorkDir { testBindSuffix = "Bind" @@ -216,7 +213,7 @@ func TestRunContext_GetBindsAndMounts(t *testing.T) { config := testcase.rc.Config config.Workdir = testcase.name config.BindWorkdir = bindWorkDir - gotbind, gotmount := rctemplate.GetBindsAndMounts() + gotbind, gotmount, _ := rctemplate.GetBindsAndMounts(t.Context()) // Name binds/mounts are either/or if config.BindWorkdir { @@ -226,7 +223,7 @@ func TestRunContext_GetBindsAndMounts(t *testing.T) { } assert.Contains(t, gotbind, fullBind) } else { - mountkey := testcase.rc.jobContainerName() + mountkey := testcase.rc.getInternalVolumeWorkdir(t.Context()) assert.EqualValues(t, testcase.wantmount, gotmount[mountkey]) } }) @@ -268,7 +265,7 @@ func TestRunContext_GetBindsAndMounts(t *testing.T) { rc.Run.JobID = "job1" rc.Run.Workflow.Jobs = map[string]*model.Job{"job1": job} - gotbind, gotmount := rc.GetBindsAndMounts() + gotbind, gotmount, _ := rc.GetBindsAndMounts(t.Context()) if len(testcase.wantbind) > 0 { assert.Contains(t, gotbind, testcase.wantbind) @@ -283,61 +280,37 @@ func TestRunContext_GetBindsAndMounts(t *testing.T) { }) } -func TestRunContext_GetGitHubContext(t *testing.T) { - log.SetLevel(log.DebugLevel) - - cwd, err := os.Getwd() - assert.Nil(t, err) - - rc := &RunContext{ - Config: &Config{ - EventName: "push", - Workdir: cwd, - }, - Run: &model.Run{ - Workflow: &model.Workflow{ - Name: "GitHubContextTest", - }, - }, - Name: "GitHubContextTest", - CurrentStep: "step", - Matrix: map[string]interface{}{}, - Env: map[string]string{}, - ExtraPath: []string{}, - StepResults: map[string]*model.StepResult{}, - OutputMappings: map[MappableOutput]MappableOutput{}, +func TestRunContext_GetGithubContextURL(t *testing.T) { + table := []struct { + instance string + serverURL string + APIURL string + }{ + {instance: "", serverURL: "", APIURL: "/api/v1"}, + {instance: "example.com", serverURL: "https://example.com", APIURL: "https://example.com/api/v1"}, + {instance: "http://example.com", serverURL: "http://example.com", APIURL: "http://example.com/api/v1"}, + {instance: "https://example.com", serverURL: "https://example.com", APIURL: "https://example.com/api/v1"}, } - rc.Run.JobID = "job1" + for _, data := range table { + t.Run(data.instance, func(t *testing.T) { + rc := &RunContext{ + EventJSON: "{}", + Config: &Config{ + GitHubInstance: data.instance, + }, + Run: &model.Run{ + Workflow: &model.Workflow{ + Name: "GitHubContextTest", + }, + }, + } - ghc := rc.getGithubContext(context.Background()) + ghc := rc.getGithubContext(t.Context()) - log.Debugf("%v", ghc) - - actor := "nektos/act" - if a := os.Getenv("ACT_ACTOR"); a != "" { - actor = a + assert.Equal(t, data.serverURL, ghc.ServerURL) + assert.Equal(t, data.APIURL, ghc.APIURL) + }) } - - repo := "forgejo/runner" - if r := os.Getenv("ACT_REPOSITORY"); r != "" { - repo = r - } - - owner := "code.forgejo.org" - if o := os.Getenv("ACT_OWNER"); o != "" { - owner = o - } - - assert.Equal(t, ghc.RunID, "1") - assert.Equal(t, ghc.RunNumber, "1") - assert.Equal(t, ghc.RetentionDays, "0") - assert.Equal(t, ghc.Actor, actor) - assert.True(t, strings.HasSuffix(ghc.Repository, repo)) - assert.Equal(t, ghc.RepositoryOwner, owner) - assert.Equal(t, ghc.RunnerPerflog, "/dev/null") - assert.Equal(t, ghc.Token, rc.Config.Secrets["GITHUB_TOKEN"]) - assert.Equal(t, ghc.Token, rc.Config.Secrets["FORGEJO_TOKEN"]) - assert.Equal(t, ghc.Job, "job1") } func TestRunContext_GetGithubContextRef(t *testing.T) { @@ -360,7 +333,6 @@ func TestRunContext_GetGithubContextRef(t *testing.T) { } for _, data := range table { - data := data t.Run(data.event, func(t *testing.T) { rc := &RunContext{ EventJSON: data.json, @@ -375,7 +347,7 @@ func TestRunContext_GetGithubContextRef(t *testing.T) { }, } - ghc := rc.getGithubContext(context.Background()) + ghc := rc.getGithubContext(t.Context()) assert.Equal(t, data.ref, ghc.Ref) }) @@ -420,44 +392,44 @@ func TestRunContext_RunsOnPlatformNames(t *testing.T) { rc := createIfTestRunContext(map[string]*model.Job{ "job1": createJob(t, `runs-on: ubuntu-latest`, ""), }) - assertObject.Equal([]string{"ubuntu-latest"}, rc.runsOnPlatformNames(context.Background())) + assertObject.Equal([]string{"ubuntu-latest"}, rc.runsOnPlatformNames(t.Context())) rc = createIfTestRunContext(map[string]*model.Job{ "job1": createJob(t, `runs-on: ${{ 'ubuntu-latest' }}`, ""), }) - assertObject.Equal([]string{"ubuntu-latest"}, rc.runsOnPlatformNames(context.Background())) + assertObject.Equal([]string{"ubuntu-latest"}, rc.runsOnPlatformNames(t.Context())) rc = createIfTestRunContext(map[string]*model.Job{ "job1": createJob(t, `runs-on: [self-hosted, my-runner]`, ""), }) - assertObject.Equal([]string{"self-hosted", "my-runner"}, rc.runsOnPlatformNames(context.Background())) + assertObject.Equal([]string{"self-hosted", "my-runner"}, rc.runsOnPlatformNames(t.Context())) rc = createIfTestRunContext(map[string]*model.Job{ "job1": createJob(t, `runs-on: [self-hosted, "${{ 'my-runner' }}"]`, ""), }) - assertObject.Equal([]string{"self-hosted", "my-runner"}, rc.runsOnPlatformNames(context.Background())) + assertObject.Equal([]string{"self-hosted", "my-runner"}, rc.runsOnPlatformNames(t.Context())) rc = createIfTestRunContext(map[string]*model.Job{ "job1": createJob(t, `runs-on: ${{ fromJSON('["ubuntu-latest"]') }}`, ""), }) - assertObject.Equal([]string{"ubuntu-latest"}, rc.runsOnPlatformNames(context.Background())) + assertObject.Equal([]string{"ubuntu-latest"}, rc.runsOnPlatformNames(t.Context())) // test missing / invalid runs-on rc = createIfTestRunContext(map[string]*model.Job{ "job1": createJob(t, `name: something`, ""), }) - assertObject.Equal([]string{}, rc.runsOnPlatformNames(context.Background())) + assertObject.Equal([]string{}, rc.runsOnPlatformNames(t.Context())) rc = createIfTestRunContext(map[string]*model.Job{ "job1": createJob(t, `runs-on: mapping: value`, ""), }) - assertObject.Equal([]string{}, rc.runsOnPlatformNames(context.Background())) + assertObject.Equal([]string{}, rc.runsOnPlatformNames(t.Context())) rc = createIfTestRunContext(map[string]*model.Job{ "job1": createJob(t, `runs-on: ${{ invalid expression }}`, ""), }) - assertObject.Equal([]string{}, rc.runsOnPlatformNames(context.Background())) + assertObject.Equal([]string{}, rc.runsOnPlatformNames(t.Context())) } func TestRunContext_IsEnabled(t *testing.T) { @@ -469,7 +441,7 @@ func TestRunContext_IsEnabled(t *testing.T) { "job1": createJob(t, `runs-on: ubuntu-latest if: success()`, ""), }) - assertObject.True(rc.isEnabled(context.Background())) + assertObject.True(rc.isEnabled(t.Context())) rc = createIfTestRunContext(map[string]*model.Job{ "job1": createJob(t, `runs-on: ubuntu-latest`, "failure"), @@ -478,7 +450,7 @@ needs: [job1] if: success()`, ""), }) rc.Run.JobID = "job2" - assertObject.False(rc.isEnabled(context.Background())) + assertObject.False(rc.isEnabled(t.Context())) rc = createIfTestRunContext(map[string]*model.Job{ "job1": createJob(t, `runs-on: ubuntu-latest`, "success"), @@ -487,7 +459,7 @@ needs: [job1] if: success()`, ""), }) rc.Run.JobID = "job2" - assertObject.True(rc.isEnabled(context.Background())) + assertObject.True(rc.isEnabled(t.Context())) rc = createIfTestRunContext(map[string]*model.Job{ "job1": createJob(t, `runs-on: ubuntu-latest`, "failure"), @@ -495,14 +467,14 @@ if: success()`, ""), if: success()`, ""), }) rc.Run.JobID = "job2" - assertObject.True(rc.isEnabled(context.Background())) + assertObject.True(rc.isEnabled(t.Context())) // failure() rc = createIfTestRunContext(map[string]*model.Job{ "job1": createJob(t, `runs-on: ubuntu-latest if: failure()`, ""), }) - assertObject.False(rc.isEnabled(context.Background())) + assertObject.False(rc.isEnabled(t.Context())) rc = createIfTestRunContext(map[string]*model.Job{ "job1": createJob(t, `runs-on: ubuntu-latest`, "failure"), @@ -511,7 +483,7 @@ needs: [job1] if: failure()`, ""), }) rc.Run.JobID = "job2" - assertObject.True(rc.isEnabled(context.Background())) + assertObject.True(rc.isEnabled(t.Context())) rc = createIfTestRunContext(map[string]*model.Job{ "job1": createJob(t, `runs-on: ubuntu-latest`, "success"), @@ -520,7 +492,7 @@ needs: [job1] if: failure()`, ""), }) rc.Run.JobID = "job2" - assertObject.False(rc.isEnabled(context.Background())) + assertObject.False(rc.isEnabled(t.Context())) rc = createIfTestRunContext(map[string]*model.Job{ "job1": createJob(t, `runs-on: ubuntu-latest`, "failure"), @@ -528,14 +500,14 @@ if: failure()`, ""), if: failure()`, ""), }) rc.Run.JobID = "job2" - assertObject.False(rc.isEnabled(context.Background())) + assertObject.False(rc.isEnabled(t.Context())) // always() rc = createIfTestRunContext(map[string]*model.Job{ "job1": createJob(t, `runs-on: ubuntu-latest if: always()`, ""), }) - assertObject.True(rc.isEnabled(context.Background())) + assertObject.True(rc.isEnabled(t.Context())) rc = createIfTestRunContext(map[string]*model.Job{ "job1": createJob(t, `runs-on: ubuntu-latest`, "failure"), @@ -544,7 +516,7 @@ needs: [job1] if: always()`, ""), }) rc.Run.JobID = "job2" - assertObject.True(rc.isEnabled(context.Background())) + assertObject.True(rc.isEnabled(t.Context())) rc = createIfTestRunContext(map[string]*model.Job{ "job1": createJob(t, `runs-on: ubuntu-latest`, "success"), @@ -553,7 +525,7 @@ needs: [job1] if: always()`, ""), }) rc.Run.JobID = "job2" - assertObject.True(rc.isEnabled(context.Background())) + assertObject.True(rc.isEnabled(t.Context())) rc = createIfTestRunContext(map[string]*model.Job{ "job1": createJob(t, `runs-on: ubuntu-latest`, "success"), @@ -561,18 +533,18 @@ if: always()`, ""), if: always()`, ""), }) rc.Run.JobID = "job2" - assertObject.True(rc.isEnabled(context.Background())) + assertObject.True(rc.isEnabled(t.Context())) rc = createIfTestRunContext(map[string]*model.Job{ "job1": createJob(t, `uses: ./.github/workflows/reusable.yml`, ""), }) - assertObject.True(rc.isEnabled(context.Background())) + assertObject.True(rc.isEnabled(t.Context())) rc = createIfTestRunContext(map[string]*model.Job{ "job1": createJob(t, `uses: ./.github/workflows/reusable.yml if: false`, ""), }) - assertObject.False(rc.isEnabled(context.Background())) + assertObject.False(rc.isEnabled(t.Context())) } func TestRunContext_GetEnv(t *testing.T) { @@ -647,12 +619,94 @@ func TestRunContext_CreateSimpleContainerName(t *testing.T) { } } +func TestRunContext_ensureNetworkName(t *testing.T) { + t.Run("CreateNetworkForServices", func(t *testing.T) { + yaml := ` +on: + push: + +jobs: + job: + runs-on: docker + container: + image: some:image + services: + service1: + image: service1:image + steps: + - run: echo ok +` + workflow, err := model.ReadWorkflow(strings.NewReader(yaml), true) + require.NoError(t, err) + + rc := &RunContext{ + Config: &Config{ + ContainerNetworkMode: "host", + }, + Run: &model.Run{ + JobID: "job", + Workflow: workflow, + }, + } + + rc.ensureNetworkName(t.Context()) + assert.True(t, rc.getNetworkCreated(t.Context())) + assert.True(t, strings.HasPrefix(rc.getNetworkName(t.Context()), "WORKFLOW-"), rc.getNetworkName(t.Context())) + }) + + yaml := ` +on: + push: + +jobs: + job: + runs-on: docker + container: + image: some:image + steps: + - run: echo ok +` + workflow, err := model.ReadWorkflow(strings.NewReader(yaml), true) + require.NoError(t, err) + + run := &model.Run{ + JobID: "job", + Workflow: workflow, + } + + t.Run("CreateNetworkIfEmptyNetworkMode", func(t *testing.T) { + rc := &RunContext{ + Config: &Config{ + ContainerNetworkMode: "", + }, + Run: run, + } + + rc.ensureNetworkName(t.Context()) + assert.True(t, rc.getNetworkCreated(t.Context())) + assert.True(t, strings.HasPrefix(rc.getNetworkName(t.Context()), "WORKFLOW-"), rc.getNetworkName(t.Context())) + }) + + t.Run("FixedNetworkIfSetByNetworkMode", func(t *testing.T) { + rc := &RunContext{ + Config: &Config{ + ContainerNetworkMode: "host", + }, + Run: run, + } + + rc.ensureNetworkName(t.Context()) + assert.False(t, rc.getNetworkCreated(t.Context())) + assert.Equal(t, "host", rc.getNetworkName(t.Context())) + }) +} + func TestRunContext_SanitizeNetworkAlias(t *testing.T) { same := "same" - assert.Equal(t, same, sanitizeNetworkAlias(context.Background(), same)) + assert.Equal(t, same, sanitizeNetworkAlias(t.Context(), same)) original := "or.igin'A-L" sanitized := "or_igin_a-l" - assert.Equal(t, sanitized, sanitizeNetworkAlias(context.Background(), original)) + assert.Equal(t, sanitized, sanitizeNetworkAlias(t.Context(), original)) } func TestRunContext_PrepareJobContainer(t *testing.T) { @@ -709,21 +763,16 @@ jobs: }, inputs: []container.NewContainerInput{ { - Name: "WORKFLOW-e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855_JOB", - Image: "some:image", - Username: "containerusername", - Password: "containerpassword", - Entrypoint: []string{"tail", "-f", "/dev/null"}, - Cmd: nil, - WorkingDir: "/my/workdir", - Env: []string{}, - ToolCache: "/opt/hostedtoolcache", - Binds: []string{"/var/run/docker.sock:/var/run/docker.sock"}, - Mounts: map[string]string{ - "WORKFLOW-e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855_JOB": "/my/workdir", - "WORKFLOW-e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855_JOB-env": "/var/run/act", - }, - NetworkMode: "WORKFLOW-e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855_JOB-job-network", + Name: "WORKFLOW-8a5edab282632443219e051e4ade2d1d5bbc671c781051bf1437897cbdfea0f1_JOB", + Image: "some:image", + Username: "containerusername", + Password: "containerpassword", + Entrypoint: []string{"tail", "-f", "/dev/null"}, + Cmd: nil, + WorkingDir: "/my/workdir", + Env: []string{}, + ToolCache: "/opt/hostedtoolcache", + Binds: []string{"/var/run/docker.sock:/var/run/docker.sock"}, Privileged: false, UsernsMode: "", Platform: "", @@ -732,15 +781,9 @@ jobs: PortBindings: nil, ConfigOptions: "", JobOptions: "", - AutoRemove: false, - ValidVolumes: []string{ - "WORKFLOW-e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855_JOB", - "WORKFLOW-e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855_JOB-env", - "/var/run/docker.sock", - }, }, { - Name: "WORKFLOW-e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca49599-fe7f4c0058dbd2161ebe4aafa71cd83bd96ee19d3ca8043d5e4bc477a664a80c", + Name: "WORKFLOW-8a5edab282632443219e051e4ade2d1d5bbc671c781051bf143789-d083efaebdcab24d231fa091b85dbb8768b47136582b340132c197d9cb5e7430", Image: "service1:image", Username: "service1username", Password: "service1password", @@ -750,8 +793,6 @@ jobs: Env: []string{}, ToolCache: "/opt/hostedtoolcache", Binds: []string{"/var/run/docker.sock:/var/run/docker.sock"}, - Mounts: map[string]string{}, - NetworkMode: "WORKFLOW-e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855_JOB-job-network", Privileged: false, UsernsMode: "", Platform: "", @@ -760,15 +801,9 @@ jobs: PortBindings: nat.PortMap{}, ConfigOptions: "", JobOptions: "", - AutoRemove: false, - ValidVolumes: []string{ - "WORKFLOW-e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855_JOB", - "WORKFLOW-e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855_JOB-env", - "/var/run/docker.sock", - }, }, { - Name: "WORKFLOW-e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca49599-c233cf913e1d0c90cc1404ee09917e625f9cb82156ca3d7cb10b729d563728ea", + Name: "WORKFLOW-8a5edab282632443219e051e4ade2d1d5bbc671c781051bf143789-4c44cd5731ec445ebe33780a3e39ed5e20e80f8a697a6e34c7acdd3675e631a8", Image: "service2:image", Username: "service2username", Password: "service2password", @@ -778,8 +813,6 @@ jobs: Env: []string{}, ToolCache: "/opt/hostedtoolcache", Binds: []string{"/var/run/docker.sock:/var/run/docker.sock"}, - Mounts: map[string]string{}, - NetworkMode: "WORKFLOW-e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855_JOB-job-network", Privileged: false, UsernsMode: "", Platform: "", @@ -788,12 +821,6 @@ jobs: PortBindings: nat.PortMap{}, ConfigOptions: "", JobOptions: "", - AutoRemove: false, - ValidVolumes: []string{ - "WORKFLOW-e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855_JOB", - "WORKFLOW-e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855_JOB-env", - "/var/run/docker.sock", - }, }, }, }, @@ -812,15 +839,88 @@ jobs: return newContainer(input) })() - ctx := context.Background() + ctx := t.Context() rc := testCase.step.getRunContext() rc.ExprEval = rc.NewExpressionEvaluator(ctx) require.NoError(t, rc.prepareJobContainer(ctx)) slices.SortFunc(containerInputs, func(a, b container.NewContainerInput) int { return cmp.Compare(a.Username, b.Username) }) + jobContainerInput := containerInputs[0] + require.Equal(t, "containerusername", jobContainerInput.Username) + require.NotEmpty(t, jobContainerInput.NetworkMode) + for source := range jobContainerInput.Mounts { + assert.Contains(t, jobContainerInput.ValidVolumes, source) + } for i := 0; i < len(containerInputs); i++ { + + assert.Equal(t, jobContainerInput.NetworkMode, containerInputs[i].NetworkMode, containerInputs[i].Username) + containerInputs[i].NetworkMode = "" + + if strings.HasPrefix(containerInputs[i].Username, "service") { + assert.Empty(t, containerInputs[i].Mounts) + assert.Empty(t, containerInputs[i].ValidVolumes) + } + containerInputs[i].Mounts = nil + containerInputs[i].ValidVolumes = nil + assert.EqualValues(t, testCase.inputs[i], containerInputs[i], containerInputs[i].Username) } }) } } + +type waitForServiceContainerMock struct { + mock.Mock + container.Container + container.LinuxContainerEnvironmentExtensions +} + +func (o *waitForServiceContainerMock) IsHealthy(ctx context.Context) (time.Duration, error) { + args := o.Called(ctx) + return args.Get(0).(time.Duration), args.Error(1) +} + +func Test_waitForServiceContainer(t *testing.T) { + t.Run("Wait", func(t *testing.T) { + m := &waitForServiceContainerMock{} + ctx := t.Context() + mock.InOrder( + m.On("IsHealthy", ctx).Return(1*time.Millisecond, nil).Once(), + m.On("IsHealthy", ctx).Return(time.Duration(0), nil).Once(), + ) + require.NoError(t, waitForServiceContainer(ctx, m)) + m.AssertExpectations(t) + }) + + t.Run("Cancel", func(t *testing.T) { + m := &waitForServiceContainerMock{} + ctx, cancel := context.WithCancel(t.Context()) + cancel() + m.On("IsHealthy", ctx).Return(1*time.Millisecond, nil).Once() + require.NoError(t, waitForServiceContainer(ctx, m)) + m.AssertExpectations(t) + }) + + t.Run("Error", func(t *testing.T) { + m := &waitForServiceContainerMock{} + ctx := t.Context() + m.On("IsHealthy", ctx).Return(time.Duration(0), errors.New("ERROR")) + require.ErrorContains(t, waitForServiceContainer(ctx, m), "ERROR") + m.AssertExpectations(t) + }) +} + +func TestRunContext_ensureRandomName(t *testing.T) { + parent := &RunContext{ + Name: "parentname", + } + rc := &RunContext{ + Name: "runname", + Parent: parent, + } + + parent.ensureRandomName(t.Context()) + assert.NotEmpty(t, parent.randomName) + rc.ensureRandomName(t.Context()) + assert.Equal(t, parent.randomName, rc.randomName) +} diff --git a/act/runner/runner.go b/act/runner/runner.go index 19aea849..8e8fba3a 100644 --- a/act/runner/runner.go +++ b/act/runner/runner.go @@ -11,8 +11,8 @@ import ( docker_container "github.com/docker/docker/api/types/container" log "github.com/sirupsen/logrus" - "code.forgejo.org/forgejo/runner/v9/act/common" - "code.forgejo.org/forgejo/runner/v9/act/model" + "code.forgejo.org/forgejo/runner/v11/act/common" + "code.forgejo.org/forgejo/runner/v11/act/model" ) // Runner provides capabilities to run GitHub actions @@ -52,7 +52,6 @@ type Config struct { GitHubInstance string // GitHub instance to use, default "github.com" ContainerCapAdd []string // list of kernel capabilities to add to the containers ContainerCapDrop []string // list of kernel capabilities to remove from the containers - AutoRemove bool // controls if the container is automatically removed upon workflow completion ArtifactServerPath string // the path where the artifact server stores uploads ArtifactServerAddr string // the address the artifact server binds to ArtifactServerPort string // the port the artifact server binds to @@ -86,6 +85,13 @@ func (c Config) GetToken() string { return token } +func (c *Config) GetContainerDaemonSocket() string { + if c.ContainerDaemonSocket == "" { + return "/var/run/docker.sock" + } + return c.ContainerDaemonSocket +} + type caller struct { runContext *RunContext } @@ -148,7 +154,7 @@ func (runner *runnerImpl) NewPlanExecutor(plan *model.Plan) common.Executor { log.Debugf("Job.RawNeeds: %v", job.RawNeeds) log.Debugf("Job.RawRunsOn: %v", job.RawRunsOn) log.Debugf("Job.Env: %v", job.Env) - log.Debugf("Job.If: %v", job.If) + log.Debugf("Job.If: %v", job.IfClause()) for step := range job.Steps { if nil != job.Steps[step] { log.Debugf("Job.Steps: %v", job.Steps[step].String()) @@ -179,7 +185,7 @@ func (runner *runnerImpl) NewPlanExecutor(plan *model.Plan) common.Executor { } } - var matrixes []map[string]interface{} + var matrixes []map[string]any if m, err := job.GetMatrixes(); err != nil { log.Errorf("Error while get job's matrix: %v", err) } else { @@ -199,7 +205,6 @@ func (runner *runnerImpl) NewPlanExecutor(plan *model.Plan) common.Executor { } for i, matrix := range matrixes { - matrix := matrix rc := runner.newRunContext(ctx, run, matrix) rc.JobName = rc.Name if len(matrixes) > 1 { @@ -220,10 +225,7 @@ func (runner *runnerImpl) NewPlanExecutor(plan *model.Plan) common.Executor { } pipeline = append(pipeline, common.NewParallelExecutor(maxParallel, stageExecutor...)) } - ncpu := runtime.NumCPU() - if 1 > ncpu { - ncpu = 1 - } + ncpu := max(1, runtime.NumCPU()) log.Debugf("Detected CPUs: %d", ncpu) return common.NewParallelExecutor(ncpu, pipeline...)(ctx) }) @@ -245,8 +247,8 @@ func handleFailure(plan *model.Plan) common.Executor { } } -func selectMatrixes(originalMatrixes []map[string]interface{}, targetMatrixValues map[string]map[string]bool) []map[string]interface{} { - matrixes := make([]map[string]interface{}, 0) +func selectMatrixes(originalMatrixes []map[string]any, targetMatrixValues map[string]map[string]bool) []map[string]any { + matrixes := make([]map[string]any, 0) for _, original := range originalMatrixes { flag := true for key, val := range original { @@ -264,7 +266,7 @@ func selectMatrixes(originalMatrixes []map[string]interface{}, targetMatrixValue return matrixes } -func (runner *runnerImpl) newRunContext(ctx context.Context, run *model.Run, matrix map[string]interface{}) *RunContext { +func (runner *runnerImpl) newRunContext(ctx context.Context, run *model.Run, matrix map[string]any) *RunContext { rc := &RunContext{ Config: runner.config, Run: run, diff --git a/act/runner/runner_test.go b/act/runner/runner_test.go index 0a11a5ea..f8b3918c 100644 --- a/act/runner/runner_test.go +++ b/act/runner/runner_test.go @@ -9,16 +9,17 @@ import ( "path" "path/filepath" "strings" + "sync" "testing" "github.com/joho/godotenv" log "github.com/sirupsen/logrus" assert "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "gopkg.in/yaml.v3" + "go.yaml.in/yaml/v3" - "code.forgejo.org/forgejo/runner/v9/act/common" - "code.forgejo.org/forgejo/runner/v9/act/model" + "code.forgejo.org/forgejo/runner/v11/act/common" + "code.forgejo.org/forgejo/runner/v11/act/model" ) var ( @@ -191,7 +192,6 @@ func (j *TestJobFileInfo) runTest(ctx context.Context, t *testing.T, cfg *Config Matrix: cfg.Matrix, JobLoggerLevel: cfg.JobLoggerLevel, ActionCache: cfg.ActionCache, - AutoRemove: true, } runner, err := New(runnerConfig) @@ -229,7 +229,7 @@ func TestRunner_RunEvent(t *testing.T) { t.Skip("skipping integration test") } - ctx := context.Background() + ctx := t.Context() tables := []TestJobFileInfo{ // Shells @@ -245,20 +245,25 @@ func TestRunner_RunEvent(t *testing.T) { {workdir, "local-action-fails-schema-validation", "push", "Job 'test' failed", platforms, secrets}, {workdir, "local-action-docker-url", "push", "", platforms, secrets}, {workdir, "local-action-dockerfile", "push", "", platforms, secrets}, + {workdir + "/local-action-dockerfile-tag/example1", "local-action-dockerfile-example1", "push", "", platforms, secrets}, + {workdir + "/local-action-dockerfile-tag/example2", "local-action-dockerfile-example2", "push", "", platforms, secrets}, {workdir, "local-action-via-composite-dockerfile", "push", "", platforms, secrets}, {workdir, "local-action-js", "push", "", platforms, secrets}, // Uses {workdir, "uses-composite", "push", "", platforms, secrets}, {workdir, "uses-composite-with-error", "push", "Job 'failing-composite-action' failed", platforms, secrets}, + {workdir, "uses-composite-check-for-input-collision", "push", "", platforms, secrets}, + {workdir, "uses-composite-check-for-input-shadowing", "push", "", platforms, secrets}, {workdir, "uses-nested-composite", "push", "", platforms, secrets}, + {workdir, "uses-composite-check-for-input-in-if-uses", "push", "", platforms, secrets}, // {workdir, "remote-action-composite-js-pre-with-defaults", "push", "", platforms, secrets}, {workdir, "remote-action-composite-action-ref", "push", "", platforms, secrets}, - // reusable workflow not fully implemented yet - // {workdir, "uses-workflow", "push", "", platforms, map[string]string{"secret": "keep_it_private"}}, - // {workdir, "uses-workflow", "pull_request", "", platforms, map[string]string{"secret": "keep_it_private"}}, + {workdir, "uses-workflow", "push", "", platforms, map[string]string{"secret": "keep_it_private"}}, + {workdir, "uses-workflow", "pull_request", "", platforms, map[string]string{"secret": "keep_it_private"}}, {workdir, "uses-docker-url", "push", "", platforms, secrets}, {workdir, "act-composite-env-test", "push", "", platforms, secrets}, + {workdir, "uses-workflow-env-input", "push", "", platforms, secrets}, // Eval {workdir, "evalmatrix", "push", "", platforms, secrets}, @@ -268,6 +273,8 @@ func TestRunner_RunEvent(t *testing.T) { {workdir, "evalmatrix-merge-array", "push", "", platforms, secrets}, {workdir, "basic", "push", "", platforms, secrets}, + {workdir, "timeout-minutes-stop", "push", "Job 'check' failed", platforms, secrets}, + {workdir, "timeout-minutes-job", "push", "context deadline exceeded", platforms, secrets}, {workdir, "fail", "push", "Job 'build' failed", platforms, secrets}, {workdir, "runs-on", "push", "", platforms, secrets}, {workdir, "checkout", "push", "", platforms, secrets}, @@ -281,6 +288,7 @@ func TestRunner_RunEvent(t *testing.T) { {workdir, "matrix", "push", "", platforms, secrets}, {workdir, "matrix-include-exclude", "push", "", platforms, secrets}, {workdir, "matrix-exitcode", "push", "Job 'test' failed", platforms, secrets}, + {workdir, "matrix-shell", "push", "", platforms, secrets}, {workdir, "commands", "push", "", platforms, secrets}, {workdir, "workdir", "push", "", platforms, secrets}, {workdir, "defaults-run", "push", "", platforms, secrets}, @@ -310,6 +318,7 @@ func TestRunner_RunEvent(t *testing.T) { {workdir, "workflow_dispatch_no_inputs_mapping", "workflow_dispatch", "", platforms, secrets}, {workdir, "workflow_dispatch-scalar", "workflow_dispatch", "", platforms, secrets}, {workdir, "workflow_dispatch-scalar-composite-action", "workflow_dispatch", "", platforms, secrets}, + {workdir, "uses-workflow-defaults", "workflow_dispatch", "", platforms, secrets}, {workdir, "job-needs-context-contains-result", "push", "", platforms, secrets}, {"../model/testdata", "strategy", "push", "", platforms, secrets}, // TODO: move all testdata into pkg so we can validate it with planner and runner {workdir, "path-handling", "push", "", platforms, secrets}, @@ -317,11 +326,14 @@ func TestRunner_RunEvent(t *testing.T) { {workdir, "set-env-step-env-override", "push", "", platforms, secrets}, {workdir, "set-env-new-env-file-per-step", "push", "", platforms, secrets}, {workdir, "no-panic-on-invalid-composite-action", "push", "missing steps in composite action", platforms, secrets}, + {workdir, "stepsummary", "push", "", platforms, secrets}, {workdir, "tool-cache", "push", "", platforms, secrets}, // services {workdir, "services", "push", "", platforms, secrets}, {workdir, "services-with-container", "push", "", platforms, secrets}, + {workdir, "mysql-service-container-with-health-check", "push", "", platforms, secrets}, + {workdir, "mysql-service-container-premature-terminate", "push", "service [maindb]", platforms, secrets}, } for _, table := range tables { @@ -362,7 +374,7 @@ func TestRunner_DryrunEvent(t *testing.T) { t.Skip("skipping integration test") } - ctx := common.WithDryrun(context.Background(), true) + ctx := common.WithDryrun(t.Context(), true) tables := []TestJobFileInfo{ // Shells @@ -391,7 +403,7 @@ func TestRunner_DockerActionForcePullForceRebuild(t *testing.T) { t.Skip("skipping integration test") } - ctx := context.Background() + ctx := t.Context() config := &Config{ ForcePull: true, @@ -424,7 +436,7 @@ func TestRunner_RunDifferentArchitecture(t *testing.T) { platforms: platforms, } - tjfi.runTest(context.Background(), t, &Config{ContainerArchitecture: "linux/arm64"}) + tjfi.runTest(t.Context(), t, &Config{ContainerArchitecture: "linux/arm64"}) } type runSkippedHook struct { @@ -459,7 +471,7 @@ func TestRunner_RunSkipped(t *testing.T) { } h := &runSkippedHook{resultKey: what + "Result"} - ctx := common.WithLoggerHook(context.Background(), h) + ctx := common.WithLoggerHook(t.Context(), h) jobLoggerLevel := log.InfoLevel tjfi.runTest(ctx, t, &Config{ContainerArchitecture: "linux/arm64", JobLoggerLevel: &jobLoggerLevel}) @@ -470,7 +482,24 @@ func TestRunner_RunSkipped(t *testing.T) { } type maskJobLoggerFactory struct { - Output bytes.Buffer + Output syncBuffer +} + +type syncBuffer struct { + buffer bytes.Buffer + mutex sync.Mutex +} + +func (sb *syncBuffer) Write(p []byte) (n int, err error) { + sb.mutex.Lock() + defer sb.mutex.Unlock() + return sb.buffer.Write(p) +} + +func (sb *syncBuffer) String() string { + sb.mutex.Lock() + defer sb.mutex.Unlock() + return sb.buffer.String() } func (f *maskJobLoggerFactory) WithJobLogger() *log.Logger { @@ -504,7 +533,7 @@ func TestRunner_MaskValues(t *testing.T) { } logger := &maskJobLoggerFactory{} - tjfi.runTest(WithJobLoggerFactory(common.WithLogger(context.Background(), logger.WithJobLogger()), logger), t, &Config{}) + tjfi.runTest(WithJobLoggerFactory(common.WithLogger(t.Context(), logger.WithJobLogger()), logger), t, &Config{}) output := logger.Output.String() assertNoSecret(output, "secret value") @@ -530,7 +559,7 @@ func TestRunner_RunEventSecrets(t *testing.T) { secrets, _ := godotenv.Read(filepath.Join(workdir, workflowPath, ".secrets")) assert.NoError(t, err, "Failed to read .secrets") - tjfi.runTest(context.Background(), t, &Config{Secrets: secrets, Env: env}) + tjfi.runTest(t.Context(), t, &Config{Secrets: secrets, Env: env}) } func TestRunner_RunWithService(t *testing.T) { @@ -539,7 +568,7 @@ func TestRunner_RunWithService(t *testing.T) { } log.SetLevel(log.DebugLevel) - ctx := context.Background() + ctx := t.Context() platforms := map[string]string{ "ubuntu-latest": "code.forgejo.org/oci/node:22", @@ -588,7 +617,7 @@ func TestRunner_RunActionInputs(t *testing.T) { "SOME_INPUT": "input", } - tjfi.runTest(context.Background(), t, &Config{Inputs: inputs}) + tjfi.runTest(t.Context(), t, &Config{Inputs: inputs}) } func TestRunner_RunEventPullRequest(t *testing.T) { @@ -606,7 +635,7 @@ func TestRunner_RunEventPullRequest(t *testing.T) { platforms: platforms, } - tjfi.runTest(context.Background(), t, &Config{EventPath: filepath.Join(workdir, workflowPath, "event.json")}) + tjfi.runTest(t.Context(), t, &Config{EventPath: filepath.Join(workdir, workflowPath, "event.json")}) } func TestRunner_RunMatrixWithUserDefinedInclusions(t *testing.T) { @@ -633,5 +662,33 @@ func TestRunner_RunMatrixWithUserDefinedInclusions(t *testing.T) { }, } - tjfi.runTest(context.Background(), t, &Config{Matrix: matrix}) + tjfi.runTest(t.Context(), t, &Config{Matrix: matrix}) +} + +// Regression test against `runs-on` in a matrix run, which references the matrix values, being corrupted by multiple +// concurrent goroutines. +func TestRunner_RunsOnMatrix(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test") + } + + log.SetLevel(log.DebugLevel) + + tjfi := TestJobFileInfo{ + workdir: workdir, + workflowPath: "matrix-runs-on", + eventName: "push", + errorMessage: "", + platforms: platforms, + } + + logger := &maskJobLoggerFactory{} + tjfi.runTest(WithJobLoggerFactory(common.WithLogger(t.Context(), logger.WithJobLogger()), logger), t, &Config{}) + output := logger.Output.String() + + // job 1 should succeed because `ubuntu-latest` is a valid runs-on target... + assert.Contains(t, output, "msg=\"🏁 Job succeeded\" dryrun=false job=test/matrix-runs-on-1", "expected job 1 to succeed, but did not find success message") + + // job 2 should be skipped because `ubuntu-20.04` is not a valid runs-on target in `platforms`. + assert.Contains(t, output, "msg=\"🚧 Skipping unsupported platform -- Try running with `-P ubuntu-20.04=...`\" dryrun=false job=test/matrix-runs-on-2", "expected job 2 to be skipped, but it was not") } diff --git a/act/runner/step.go b/act/runner/step.go index 72102e8c..c93291c4 100644 --- a/act/runner/step.go +++ b/act/runner/step.go @@ -1,17 +1,23 @@ package runner import ( + "archive/tar" "context" + "errors" "fmt" + "io" + "maps" "path" "strconv" "strings" "time" - "code.forgejo.org/forgejo/runner/v9/act/common" - "code.forgejo.org/forgejo/runner/v9/act/container" - "code.forgejo.org/forgejo/runner/v9/act/exprparser" - "code.forgejo.org/forgejo/runner/v9/act/model" + "code.forgejo.org/forgejo/runner/v11/act/common" + "code.forgejo.org/forgejo/runner/v11/act/container" + "code.forgejo.org/forgejo/runner/v11/act/exprparser" + "code.forgejo.org/forgejo/runner/v11/act/model" + + "github.com/sirupsen/logrus" ) type step interface { @@ -49,6 +55,32 @@ func (s stepStage) String() string { return "Unknown" } +func processRunnerSummaryCommand(ctx context.Context, fileName string, rc *RunContext) error { + if common.Dryrun(ctx) { + return nil + } + pathTar, err := rc.JobContainer.GetContainerArchive(ctx, path.Join(rc.JobContainer.GetActPath(), fileName)) + if err != nil { + return err + } + defer pathTar.Close() + + reader := tar.NewReader(pathTar) + _, err = reader.Next() + if err != nil && err != io.EOF { + return err + } + summary, err := io.ReadAll(reader) + if err != nil { + return err + } + if len(summary) == 0 { + return nil + } + common.Logger(ctx).WithFields(logrus.Fields{"command": "summary", "content": string(summary)}).Infof(" \U00002699 Summary - %s", string(summary)) + return nil +} + func processRunnerEnvFileCommand(ctx context.Context, fileName string, rc *RunContext, setter func(context.Context, map[string]string, string)) error { env := map[string]string{} err := rc.JobContainer.UpdateFromEnv(path.Join(rc.JobContainer.GetActPath(), fileName), &env)(ctx) @@ -145,7 +177,7 @@ func runStepExecutor(step step, stage stepStage, executor common.Executor) commo Mode: 0o666, })(ctx) - timeoutctx, cancelTimeOut := evaluateStepTimeout(ctx, rc.ExprEval, stepModel) + timeoutctx, cancelTimeOut := evaluateTimeout(ctx, "step", rc.ExprEval, stepModel.TimeoutMinutes) defer cancelTimeOut() err = executor(timeoutctx) @@ -171,36 +203,25 @@ func runStepExecutor(step step, stage stepStage, executor common.Executor) commo logger.WithField("stepResult", stepResult.Outcome).Errorf(" \u274C Failure - %s %s", stage, stepString) } // Process Runner File Commands - orgerr := err - err = processRunnerEnvFileCommand(ctx, envFileCommand, rc, rc.setEnv) - if err != nil { - return err - } - err = processRunnerEnvFileCommand(ctx, stateFileCommand, rc, rc.saveState) - if err != nil { - return err - } - err = processRunnerEnvFileCommand(ctx, outputFileCommand, rc, rc.setOutput) - if err != nil { - return err - } - err = rc.UpdateExtraPath(ctx, path.Join(actPath, pathFileCommand)) - if err != nil { - return err - } - if orgerr != nil { - return orgerr - } - return err + ferrors := []error{err} + ferrors = append(ferrors, processRunnerEnvFileCommand(ctx, envFileCommand, rc, rc.setEnv)) + ferrors = append(ferrors, processRunnerEnvFileCommand(ctx, stateFileCommand, rc, rc.saveState)) + ferrors = append(ferrors, processRunnerEnvFileCommand(ctx, outputFileCommand, rc, rc.setOutput)) + ferrors = append(ferrors, processRunnerSummaryCommand(ctx, summaryFileCommand, rc)) + ferrors = append(ferrors, rc.UpdateExtraPath(ctx, path.Join(actPath, pathFileCommand))) + return errors.Join(ferrors...) } } -func evaluateStepTimeout(ctx context.Context, exprEval ExpressionEvaluator, stepModel *model.Step) (context.Context, context.CancelFunc) { - timeout := exprEval.Interpolate(ctx, stepModel.TimeoutMinutes) +func evaluateTimeout(ctx context.Context, contextType string, exprEval ExpressionEvaluator, timeoutMinutes string) (context.Context, context.CancelFunc) { + timeout := exprEval.Interpolate(ctx, timeoutMinutes) if timeout != "" { - if timeOutMinutes, err := strconv.ParseInt(timeout, 10, 64); err == nil { + timeOutMinutes, err := strconv.ParseInt(timeout, 10, 64) + if err == nil { + common.Logger(ctx).Debugf("the %s will stop in timeout-minutes %s", contextType, timeout) return context.WithTimeout(ctx, time.Duration(timeOutMinutes)*time.Minute) } + common.Logger(ctx).Errorf("timeout-minutes %s cannot be parsed and will be ignored: %w", timeout, err) } return ctx, func() {} } @@ -245,6 +266,16 @@ func mergeEnv(ctx context.Context, step step) { } rc.withGithubEnv(ctx, step.getGithubContext(ctx), *env) + + if step.getStepModel().Uses != "" { + // prevent uses action input pollution of unset parameters, skip this for run steps + // due to design flaw + for key := range *env { + if strings.HasPrefix(key, "INPUT_") { + delete(*env, key) + } + } + } } func isStepEnabled(ctx context.Context, expr string, step step, stage stepStage) (bool, error) { @@ -257,7 +288,7 @@ func isStepEnabled(ctx context.Context, expr string, step step, stage stepStage) defaultStatusCheck = exprparser.DefaultStatusCheckSuccess } - runStep, err := EvalBool(ctx, rc.NewStepExpressionEvaluator(ctx, step), expr, defaultStatusCheck) + runStep, err := EvalBool(ctx, rc.NewStepExpressionEvaluatorExt(ctx, step, stage == stepStageMain), expr, defaultStatusCheck) if err != nil { return false, fmt.Errorf(" \u274C Error in if-expression: \"if: %s\" (%s)", expr, err) } @@ -289,11 +320,9 @@ func mergeIntoMap(step step, target *map[string]string, maps ...map[string]strin } } -func mergeIntoMapCaseSensitive(target map[string]string, maps ...map[string]string) { - for _, m := range maps { - for k, v := range m { - target[k] = v - } +func mergeIntoMapCaseSensitive(target map[string]string, args ...map[string]string) { + for _, m := range args { + maps.Copy(target, m) } } diff --git a/act/runner/step_action_local.go b/act/runner/step_action_local.go index 79f1a542..ef44e1c2 100644 --- a/act/runner/step_action_local.go +++ b/act/runner/step_action_local.go @@ -11,8 +11,8 @@ import ( "path" "path/filepath" - "code.forgejo.org/forgejo/runner/v9/act/common" - "code.forgejo.org/forgejo/runner/v9/act/model" + "code.forgejo.org/forgejo/runner/v11/act/common" + "code.forgejo.org/forgejo/runner/v11/act/model" ) type stepActionLocal struct { @@ -46,7 +46,7 @@ func (sal *stepActionLocal) main() common.Executor { _, cpath := getContainerActionPaths(sal.Step, path.Join(actionDir, ""), sal.RunContext) return func(filename string) (io.Reader, io.Closer, error) { spath := path.Join(cpath, filename) - for i := 0; i < maxSymlinkDepth; i++ { + for range maxSymlinkDepth { tars, err := sal.RunContext.JobContainer.GetContainerArchive(ctx, spath) if errors.Is(err, fs.ErrNotExist) { return nil, nil, err diff --git a/act/runner/step_action_local_test.go b/act/runner/step_action_local_test.go index c988d3b1..6286305a 100644 --- a/act/runner/step_action_local_test.go +++ b/act/runner/step_action_local_test.go @@ -8,11 +8,11 @@ import ( "strings" "testing" - "code.forgejo.org/forgejo/runner/v9/act/common" - "code.forgejo.org/forgejo/runner/v9/act/model" + "code.forgejo.org/forgejo/runner/v11/act/common" + "code.forgejo.org/forgejo/runner/v11/act/model" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" - "gopkg.in/yaml.v3" + "go.yaml.in/yaml/v3" ) type stepActionLocalMocks struct { @@ -30,7 +30,7 @@ func (salm *stepActionLocalMocks) readAction(_ context.Context, step *model.Step } func TestStepActionLocalTest(t *testing.T) { - ctx := context.Background() + ctx := t.Context() cm := &containerMock{} salm := &stepActionLocalMocks{} @@ -85,6 +85,7 @@ func TestStepActionLocalTest(t *testing.T) { return nil }) + cm.On("GetContainerArchive", ctx, "/var/run/act/workflow/SUMMARY.md").Return(io.NopCloser(&bytes.Buffer{}), nil) cm.On("GetContainerArchive", ctx, "/var/run/act/workflow/pathcmd.txt").Return(io.NopCloser(&bytes.Buffer{}), nil) salm.On("runAction", sal, filepath.Clean("/tmp/path/to/action"), (*remoteAction)(nil)).Return(func(ctx context.Context) error { @@ -230,7 +231,7 @@ func TestStepActionLocalPost(t *testing.T) { for _, tt := range table { t.Run(tt.name, func(t *testing.T) { - ctx := context.Background() + ctx := t.Context() cm := &containerMock{} @@ -257,7 +258,7 @@ func TestStepActionLocalPost(t *testing.T) { sal.RunContext.ExprEval = sal.RunContext.NewExpressionEvaluator(ctx) if tt.mocks.exec { - suffixMatcher := func(suffix string) interface{} { + suffixMatcher := func(suffix string) any { return mock.MatchedBy(func(array []string) bool { return strings.HasSuffix(array[1], suffix) }) @@ -280,6 +281,7 @@ func TestStepActionLocalPost(t *testing.T) { return nil }) + cm.On("GetContainerArchive", ctx, "/var/run/act/workflow/SUMMARY.md").Return(io.NopCloser(&bytes.Buffer{}), nil) cm.On("GetContainerArchive", ctx, "/var/run/act/workflow/pathcmd.txt").Return(io.NopCloser(&bytes.Buffer{}), nil) } diff --git a/act/runner/step_action_remote.go b/act/runner/step_action_remote.go index fdbbf794..9051c812 100644 --- a/act/runner/step_action_remote.go +++ b/act/runner/step_action_remote.go @@ -14,9 +14,9 @@ import ( gogit "github.com/go-git/go-git/v5" - "code.forgejo.org/forgejo/runner/v9/act/common" - "code.forgejo.org/forgejo/runner/v9/act/common/git" - "code.forgejo.org/forgejo/runner/v9/act/model" + "code.forgejo.org/forgejo/runner/v11/act/common" + "code.forgejo.org/forgejo/runner/v11/act/common/git" + "code.forgejo.org/forgejo/runner/v11/act/model" ) type stepActionRemote struct { @@ -79,7 +79,7 @@ func (sar *stepActionRemote) prepareActionExecutor() common.Executor { remoteReader := func(ctx context.Context) actionYamlReader { return func(filename string) (io.Reader, io.Closer, error) { spath := path.Join(sar.remoteAction.Path, filename) - for i := 0; i < maxSymlinkDepth; i++ { + for range maxSymlinkDepth { tars, err := cache.GetTarArchive(ctx, sar.cacheDir, sar.resolvedSha, spath) if err != nil { return nil, nil, os.ErrNotExist @@ -305,8 +305,8 @@ func (ra *remoteAction) IsCheckout() bool { func newRemoteAction(action string) *remoteAction { // support http(s)://host/owner/repo@v3 for _, schema := range []string{"https://", "http://"} { - if strings.HasPrefix(action, schema) { - splits := strings.SplitN(strings.TrimPrefix(action, schema), "/", 2) + if after, ok := strings.CutPrefix(action, schema); ok { + splits := strings.SplitN(after, "/", 2) if len(splits) != 2 { return nil } diff --git a/act/runner/step_action_remote_test.go b/act/runner/step_action_remote_test.go index 864a580e..fb302da5 100644 --- a/act/runner/step_action_remote_test.go +++ b/act/runner/step_action_remote_test.go @@ -9,11 +9,11 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" - "gopkg.in/yaml.v3" + "go.yaml.in/yaml/v3" - "code.forgejo.org/forgejo/runner/v9/act/common" - "code.forgejo.org/forgejo/runner/v9/act/common/git" - "code.forgejo.org/forgejo/runner/v9/act/model" + "code.forgejo.org/forgejo/runner/v11/act/common" + "code.forgejo.org/forgejo/runner/v11/act/common/git" + "code.forgejo.org/forgejo/runner/v11/act/model" ) type stepActionRemoteMocks struct { @@ -118,7 +118,7 @@ func TestStepActionRemoteOK(t *testing.T) { for _, tt := range table { t.Run(tt.name, func(t *testing.T) { - ctx := context.Background() + ctx := t.Context() cm := &containerMock{} sarm := &stepActionRemoteMocks{} @@ -180,6 +180,7 @@ func TestStepActionRemoteOK(t *testing.T) { return nil }) + cm.On("GetContainerArchive", ctx, "/var/run/act/workflow/SUMMARY.md").Return(io.NopCloser(&bytes.Buffer{}), nil) cm.On("GetContainerArchive", ctx, "/var/run/act/workflow/pathcmd.txt").Return(io.NopCloser(&bytes.Buffer{}), nil) } @@ -188,9 +189,9 @@ func TestStepActionRemoteOK(t *testing.T) { err = sar.main()(ctx) } - assert.Equal(t, tt.runError, err) + assert.ErrorIs(t, err, tt.runError) assert.Equal(t, tt.mocks.cloned, clonedAction) - assert.Equal(t, tt.result, sar.RunContext.StepResults["step"]) + assert.Equal(t, sar.RunContext.StepResults["step"], tt.result) sarm.AssertExpectations(t) cm.AssertExpectations(t) @@ -213,7 +214,7 @@ func TestStepActionRemotePre(t *testing.T) { for _, tt := range table { t.Run(tt.name, func(t *testing.T) { - ctx := context.Background() + ctx := t.Context() clonedAction := false sarm := &stepActionRemoteMocks{} @@ -274,7 +275,7 @@ func TestStepActionRemotePreThroughAction(t *testing.T) { for _, tt := range table { t.Run(tt.name, func(t *testing.T) { - ctx := context.Background() + ctx := t.Context() clonedAction := false sarm := &stepActionRemoteMocks{} @@ -462,7 +463,7 @@ func TestStepActionRemotePost(t *testing.T) { for _, tt := range table { t.Run(tt.name, func(t *testing.T) { - ctx := context.Background() + ctx := t.Context() cm := &containerMock{} @@ -508,6 +509,7 @@ func TestStepActionRemotePost(t *testing.T) { return nil }) + cm.On("GetContainerArchive", ctx, "/var/run/act/workflow/SUMMARY.md").Return(io.NopCloser(&bytes.Buffer{}), nil) cm.On("GetContainerArchive", ctx, "/var/run/act/workflow/pathcmd.txt").Return(io.NopCloser(&bytes.Buffer{}), nil) } diff --git a/act/runner/step_docker.go b/act/runner/step_docker.go index 5ecddd21..6f652674 100644 --- a/act/runner/step_docker.go +++ b/act/runner/step_docker.go @@ -5,9 +5,9 @@ import ( "fmt" "strings" - "code.forgejo.org/forgejo/runner/v9/act/common" - "code.forgejo.org/forgejo/runner/v9/act/container" - "code.forgejo.org/forgejo/runner/v9/act/model" + "code.forgejo.org/forgejo/runner/v11/act/common" + "code.forgejo.org/forgejo/runner/v11/act/container" + "code.forgejo.org/forgejo/runner/v11/act/model" "github.com/kballard/go-shellquote" ) @@ -110,7 +110,7 @@ func (sd *stepDocker) newStepContainer(ctx context.Context, image string, cmd, e envList = append(envList, fmt.Sprintf("%s=%s", "RUNNER_ARCH", container.RunnerArch(ctx))) envList = append(envList, fmt.Sprintf("%s=%s", "RUNNER_TEMP", "/tmp")) - binds, mounts := rc.GetBindsAndMounts() + binds, mounts, validVolumes := rc.GetBindsAndMounts(ctx) stepContainer := ContainerNewContainer(&container.NewContainerInput{ Cmd: cmd, Entrypoint: entrypoint, @@ -129,8 +129,7 @@ func (sd *stepDocker) newStepContainer(ctx context.Context, image string, cmd, e Privileged: rc.Config.Privileged, UsernsMode: rc.Config.UsernsMode, Platform: rc.Config.ContainerArchitecture, - AutoRemove: rc.Config.AutoRemove, - ValidVolumes: rc.Config.ValidVolumes, + ValidVolumes: validVolumes, }) return stepContainer } diff --git a/act/runner/step_docker_test.go b/act/runner/step_docker_test.go index 9608978e..1ba08858 100644 --- a/act/runner/step_docker_test.go +++ b/act/runner/step_docker_test.go @@ -6,8 +6,8 @@ import ( "io" "testing" - "code.forgejo.org/forgejo/runner/v9/act/container" - "code.forgejo.org/forgejo/runner/v9/act/model" + "code.forgejo.org/forgejo/runner/v11/act/container" + "code.forgejo.org/forgejo/runner/v11/act/model" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" ) @@ -27,7 +27,7 @@ func TestStepDockerMain(t *testing.T) { ContainerNewContainer = origContainerNewContainer })() - ctx := context.Background() + ctx := t.Context() sd := &stepDocker{ RunContext: &RunContext{ @@ -93,6 +93,7 @@ func TestStepDockerMain(t *testing.T) { return nil }) + cm.On("GetContainerArchive", ctx, "/var/run/act/workflow/SUMMARY.md").Return(io.NopCloser(&bytes.Buffer{}), nil) cm.On("GetContainerArchive", ctx, "/var/run/act/workflow/pathcmd.txt").Return(io.NopCloser(&bytes.Buffer{}), nil) err := sd.main()(ctx) @@ -104,7 +105,7 @@ func TestStepDockerMain(t *testing.T) { } func TestStepDockerPrePost(t *testing.T) { - ctx := context.Background() + ctx := t.Context() sd := &stepDocker{} err := sd.pre()(ctx) diff --git a/act/runner/step_factory.go b/act/runner/step_factory.go index 6435d37e..e791c99b 100644 --- a/act/runner/step_factory.go +++ b/act/runner/step_factory.go @@ -3,7 +3,7 @@ package runner import ( "fmt" - "code.forgejo.org/forgejo/runner/v9/act/model" + "code.forgejo.org/forgejo/runner/v11/act/model" ) type stepFactory interface { diff --git a/act/runner/step_factory_test.go b/act/runner/step_factory_test.go index af8cf968..3a434bf9 100644 --- a/act/runner/step_factory_test.go +++ b/act/runner/step_factory_test.go @@ -3,7 +3,7 @@ package runner import ( "testing" - "code.forgejo.org/forgejo/runner/v9/act/model" + "code.forgejo.org/forgejo/runner/v11/act/model" "github.com/stretchr/testify/assert" ) diff --git a/act/runner/step_run.go b/act/runner/step_run.go index 3c84b554..524314ca 100644 --- a/act/runner/step_run.go +++ b/act/runner/step_run.go @@ -3,15 +3,16 @@ package runner import ( "context" "fmt" + "maps" "runtime" "strings" "github.com/kballard/go-shellquote" - "code.forgejo.org/forgejo/runner/v9/act/common" - "code.forgejo.org/forgejo/runner/v9/act/container" - "code.forgejo.org/forgejo/runner/v9/act/lookpath" - "code.forgejo.org/forgejo/runner/v9/act/model" + "code.forgejo.org/forgejo/runner/v11/act/common" + "code.forgejo.org/forgejo/runner/v11/act/container" + "code.forgejo.org/forgejo/runner/v11/act/lookpath" + "code.forgejo.org/forgejo/runner/v11/act/model" ) type stepRun struct { @@ -93,6 +94,28 @@ func getScriptName(rc *RunContext, step *model.Step) string { return fmt.Sprintf("workflow/%s", scriptName) } +func shellCommand(shell string) string { + // Reference: https://github.com/actions/runner/blob/8109c962f09d9acc473d92c595ff43afceddb347/src/Runner.Worker/Handlers/ScriptHandlerHelpers.cs#L9-L17 + switch shell { + case "", "bash": + return "bash --noprofile --norc -e -o pipefail {0}" + case "pwsh": + return "pwsh -command . '{0}'" + case "python": + return "python {0}" + case "sh": + return "sh -e {0}" + case "cmd": + return "cmd /D /E:ON /V:OFF /S /C \"CALL \"{0}\"\"" + case "powershell": + return "powershell -command . '{0}'" + case "node": + return "node {0}" + default: + return shell + } +} + // TODO: Currently we just ignore top level keys, BUT we should return proper error on them // BUTx2 I leave this for when we rewrite act to use actionlint for workflow validation // so we return proper errors before any execution or spawning containers @@ -100,22 +123,21 @@ func getScriptName(rc *RunContext, step *model.Step) string { // OCI runtime exec failed: exec failed: container_linux.go:380: starting container process caused: exec: "${{": executable file not found in $PATH: unknown func (sr *stepRun) setupShellCommand(ctx context.Context) (name, script string, err error) { logger := common.Logger(ctx) - sr.setupShell(ctx) + shell := sr.interpretShell(ctx) sr.setupWorkingDirectory(ctx) step := sr.Step script = sr.RunContext.NewStepExpressionEvaluator(ctx, sr).Interpolate(ctx, step.Run) - scCmd := step.ShellCommand() - + shellCommand := shellCommand(shell) name = getScriptName(sr.RunContext, step) // Reference: https://github.com/actions/runner/blob/8109c962f09d9acc473d92c595ff43afceddb347/src/Runner.Worker/Handlers/ScriptHandlerHelpers.cs#L47-L64 // Reference: https://github.com/actions/runner/blob/8109c962f09d9acc473d92c595ff43afceddb347/src/Runner.Worker/Handlers/ScriptHandlerHelpers.cs#L19-L27 runPrepend := "" runAppend := "" - switch step.Shell { + switch shell { case "bash", "sh": name += ".sh" case "pwsh", "powershell": @@ -141,7 +163,7 @@ func (sr *stepRun) setupShellCommand(ctx context.Context) (name, script string, rc := sr.getRunContext() scriptPath := fmt.Sprintf("%s/%s", rc.JobContainer.GetActPath(), name) - sr.cmdline = strings.Replace(scCmd, `{0}`, scriptPath, 1) + sr.cmdline = strings.Replace(shellCommand, `{0}`, scriptPath, 1) sr.cmd, err = shellquote.Split(sr.cmdline) return name, script, err @@ -163,36 +185,34 @@ func (l *localEnv) Getenv(name string) string { return l.env[name] } -func (sr *stepRun) setupShell(ctx context.Context) { +func (sr *stepRun) interpretShell(ctx context.Context) string { rc := sr.RunContext - step := sr.Step + shell := sr.Step.RawShell - if step.Shell == "" { - step.Shell = rc.Run.Job().Defaults.Run.Shell + if shell == "" { + shell = rc.Run.Job().Defaults.Run.Shell } - step.Shell = rc.NewExpressionEvaluator(ctx).Interpolate(ctx, step.Shell) + shell = rc.NewExpressionEvaluator(ctx).Interpolate(ctx, shell) - if step.Shell == "" { - step.Shell = rc.Run.Workflow.Defaults.Run.Shell + if shell == "" { + shell = rc.Run.Workflow.Defaults.Run.Shell } - if step.Shell == "" { + if shell == "" { if _, ok := rc.JobContainer.(*container.HostEnvironment); ok { shellWithFallback := []string{"bash", "sh"} // Don't use bash on windows by default, if not using a docker container if runtime.GOOS == "windows" { shellWithFallback = []string{"pwsh", "powershell"} } - step.Shell = shellWithFallback[0] + shell = shellWithFallback[0] lenv := &localEnv{env: map[string]string{}} - for k, v := range sr.env { - lenv.env[k] = v - } + maps.Copy(lenv.env, sr.env) sr.getRunContext().ApplyExtraPath(ctx, &lenv.env) _, err := lookpath.LookPath2(shellWithFallback[0], lenv) if err != nil { - step.Shell = shellWithFallback[1] + shell = shellWithFallback[1] } } else { shellFallback := ` @@ -205,11 +225,13 @@ fi stdout, _, err := rc.sh(ctx, shellFallback) if err != nil { common.Logger(ctx).Error("fail to run %q: %v", shellFallback, err) - return + } else { + shell = stdout } - step.Shell = stdout } } + + return shell } func (sr *stepRun) setupWorkingDirectory(ctx context.Context) { diff --git a/act/runner/step_run_test.go b/act/runner/step_run_test.go index 611b1274..3246c433 100644 --- a/act/runner/step_run_test.go +++ b/act/runner/step_run_test.go @@ -9,8 +9,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" - "code.forgejo.org/forgejo/runner/v9/act/container" - "code.forgejo.org/forgejo/runner/v9/act/model" + "code.forgejo.org/forgejo/runner/v11/act/container" + "code.forgejo.org/forgejo/runner/v11/act/model" ) func TestStepRun(t *testing.T) { @@ -72,8 +72,9 @@ func TestStepRun(t *testing.T) { return nil }) - ctx := context.Background() + ctx := t.Context() + cm.On("GetContainerArchive", ctx, "/var/run/act/workflow/SUMMARY.md").Return(io.NopCloser(&bytes.Buffer{}), nil) cm.On("GetContainerArchive", ctx, "/var/run/act/workflow/pathcmd.txt").Return(io.NopCloser(&bytes.Buffer{}), nil) err := sr.main()(ctx) @@ -83,7 +84,7 @@ func TestStepRun(t *testing.T) { } func TestStepRunPrePost(t *testing.T) { - ctx := context.Background() + ctx := t.Context() sr := &stepRun{} err := sr.pre()(ctx) @@ -92,3 +93,22 @@ func TestStepRunPrePost(t *testing.T) { err = sr.post()(ctx) assert.Nil(t, err) } + +func TestStepShellCommand(t *testing.T) { + tests := []struct { + shell string + want string + }{ + {"pwsh -v '. {0}'", "pwsh -v '. {0}'"}, + {"pwsh", "pwsh -command . '{0}'"}, + {"powershell", "powershell -command . '{0}'"}, + {"node", "node {0}"}, + {"python", "python {0}"}, + } + for _, tt := range tests { + t.Run(tt.shell, func(t *testing.T) { + got := shellCommand(tt.shell) + assert.Equal(t, got, tt.want) + }) + } +} diff --git a/act/runner/step_test.go b/act/runner/step_test.go index b7bb31dd..779e9966 100644 --- a/act/runner/step_test.go +++ b/act/runner/step_test.go @@ -4,12 +4,12 @@ import ( "context" "testing" - "code.forgejo.org/forgejo/runner/v9/act/common" - "code.forgejo.org/forgejo/runner/v9/act/model" + "code.forgejo.org/forgejo/runner/v11/act/common" + "code.forgejo.org/forgejo/runner/v11/act/model" log "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" - yaml "gopkg.in/yaml.v3" + yaml "go.yaml.in/yaml/v3" ) func TestStep_MergeIntoMap(t *testing.T) { @@ -146,51 +146,51 @@ func TestStep_IsStepEnabled(t *testing.T) { // success() step := createTestStep(t, "if: success()") - assertObject.True(isStepEnabled(context.Background(), step.getIfExpression(context.Background(), stepStageMain), step, stepStageMain)) + assertObject.True(isStepEnabled(t.Context(), step.getIfExpression(t.Context(), stepStageMain), step, stepStageMain)) step = createTestStep(t, "if: success()") step.getRunContext().StepResults["a"] = &model.StepResult{ Conclusion: model.StepStatusSuccess, } - assertObject.True(isStepEnabled(context.Background(), step.getStepModel().If.Value, step, stepStageMain)) + assertObject.True(isStepEnabled(t.Context(), step.getStepModel().If.Value, step, stepStageMain)) step = createTestStep(t, "if: success()") step.getRunContext().StepResults["a"] = &model.StepResult{ Conclusion: model.StepStatusFailure, } - assertObject.False(isStepEnabled(context.Background(), step.getStepModel().If.Value, step, stepStageMain)) + assertObject.False(isStepEnabled(t.Context(), step.getStepModel().If.Value, step, stepStageMain)) // failure() step = createTestStep(t, "if: failure()") - assertObject.False(isStepEnabled(context.Background(), step.getStepModel().If.Value, step, stepStageMain)) + assertObject.False(isStepEnabled(t.Context(), step.getStepModel().If.Value, step, stepStageMain)) step = createTestStep(t, "if: failure()") step.getRunContext().StepResults["a"] = &model.StepResult{ Conclusion: model.StepStatusSuccess, } - assertObject.False(isStepEnabled(context.Background(), step.getStepModel().If.Value, step, stepStageMain)) + assertObject.False(isStepEnabled(t.Context(), step.getStepModel().If.Value, step, stepStageMain)) step = createTestStep(t, "if: failure()") step.getRunContext().StepResults["a"] = &model.StepResult{ Conclusion: model.StepStatusFailure, } - assertObject.True(isStepEnabled(context.Background(), step.getStepModel().If.Value, step, stepStageMain)) + assertObject.True(isStepEnabled(t.Context(), step.getStepModel().If.Value, step, stepStageMain)) // always() step = createTestStep(t, "if: always()") - assertObject.True(isStepEnabled(context.Background(), step.getStepModel().If.Value, step, stepStageMain)) + assertObject.True(isStepEnabled(t.Context(), step.getStepModel().If.Value, step, stepStageMain)) step = createTestStep(t, "if: always()") step.getRunContext().StepResults["a"] = &model.StepResult{ Conclusion: model.StepStatusSuccess, } - assertObject.True(isStepEnabled(context.Background(), step.getStepModel().If.Value, step, stepStageMain)) + assertObject.True(isStepEnabled(t.Context(), step.getStepModel().If.Value, step, stepStageMain)) step = createTestStep(t, "if: always()") step.getRunContext().StepResults["a"] = &model.StepResult{ Conclusion: model.StepStatusFailure, } - assertObject.True(isStepEnabled(context.Background(), step.getStepModel().If.Value, step, stepStageMain)) + assertObject.True(isStepEnabled(t.Context(), step.getStepModel().If.Value, step, stepStageMain)) } func TestStep_IsContinueOnError(t *testing.T) { @@ -228,37 +228,37 @@ func TestStep_IsContinueOnError(t *testing.T) { // absent step := createTestStep(t, "name: test") - continueOnError, err := isContinueOnError(context.Background(), step.getStepModel().RawContinueOnError, step, stepStageMain) + continueOnError, err := isContinueOnError(t.Context(), step.getStepModel().RawContinueOnError, step, stepStageMain) assertObject.False(continueOnError) assertObject.Nil(err) // explicit true step = createTestStep(t, "continue-on-error: true") - continueOnError, err = isContinueOnError(context.Background(), step.getStepModel().RawContinueOnError, step, stepStageMain) + continueOnError, err = isContinueOnError(t.Context(), step.getStepModel().RawContinueOnError, step, stepStageMain) assertObject.True(continueOnError) assertObject.Nil(err) // explicit false step = createTestStep(t, "continue-on-error: false") - continueOnError, err = isContinueOnError(context.Background(), step.getStepModel().RawContinueOnError, step, stepStageMain) + continueOnError, err = isContinueOnError(t.Context(), step.getStepModel().RawContinueOnError, step, stepStageMain) assertObject.False(continueOnError) assertObject.Nil(err) // expression true step = createTestStep(t, "continue-on-error: ${{ 'test' == 'test' }}") - continueOnError, err = isContinueOnError(context.Background(), step.getStepModel().RawContinueOnError, step, stepStageMain) + continueOnError, err = isContinueOnError(t.Context(), step.getStepModel().RawContinueOnError, step, stepStageMain) assertObject.True(continueOnError) assertObject.Nil(err) // expression false step = createTestStep(t, "continue-on-error: ${{ 'test' != 'test' }}") - continueOnError, err = isContinueOnError(context.Background(), step.getStepModel().RawContinueOnError, step, stepStageMain) + continueOnError, err = isContinueOnError(t.Context(), step.getStepModel().RawContinueOnError, step, stepStageMain) assertObject.False(continueOnError) assertObject.Nil(err) // expression parse error step = createTestStep(t, "continue-on-error: ${{ 'test' != test }}") - continueOnError, err = isContinueOnError(context.Background(), step.getStepModel().RawContinueOnError, step, stepStageMain) + continueOnError, err = isContinueOnError(t.Context(), step.getStepModel().RawContinueOnError, step, stepStageMain) assertObject.False(continueOnError) assertObject.NotNil(err) } diff --git a/act/runner/testdata/.github/workflows/local-reusable-and-dispatch.yml b/act/runner/testdata/.github/workflows/local-reusable-and-dispatch.yml new file mode 100644 index 00000000..b43f092b --- /dev/null +++ b/act/runner/testdata/.github/workflows/local-reusable-and-dispatch.yml @@ -0,0 +1,30 @@ +name: reuse + +on: + workflow_dispatch: + inputs: + my-val: + type: string + required: true + default: "default_value_reuse_workflow_dispatch_call" + dispatch-val: + type: string + default: "I am a dispatch var for checking if I am being used in workflow_call" + + workflow_call: + inputs: + my-val: + type: string + required: true + default: "default_value_reuse_workflow_call" + +jobs: + reusable_workflow_job: + runs-on: ubuntu-latest + steps: + - uses: https://data.forgejo.org/actions/checkout@v4 + - name: Run a one-line script + run: echo "✅ 🚀 ✅ hello this is from workflow reuse. Value - " ${{ inputs.my-val }} ${{ github.event_name }} ${{ inputs.dispatch-val }} + - name: Assert + run: | + exit ${{ ( inputs.my-val == 'default_value_reuse_workflow_call' || inputs.my-val == 'passed value from main' ) && !inputs.dispatch-val && '0' || '1' }} diff --git a/act/runner/testdata/.github/workflows/local-reusable-env-input.yml b/act/runner/testdata/.github/workflows/local-reusable-env-input.yml new file mode 100644 index 00000000..eb1d9092 --- /dev/null +++ b/act/runner/testdata/.github/workflows/local-reusable-env-input.yml @@ -0,0 +1,21 @@ +name: "use-inputs-impl" + +on: + workflow_call: + inputs: + greet_target: + type: string + required: false + default: "Some Default Value" + +jobs: + works: + runs-on: ubuntu-latest + env: + MY_INPUT_TEST: ${{ inputs.greet_target }} + INPUT_TEST: ${{ inputs.greet_target }} + INPUT_GREET_TARGET: ${{ inputs.greet_target }} + steps: + - run: '[ "$MY_INPUT_TEST" = "Mona the Octocat" ] || exit 1' + - run: '[ "$INPUT_TEST" = "Mona the Octocat" ] || exit 1' + - run: '[ "$INPUT_GREET_TARGET" = "Mona the Octocat" ] || exit 1' diff --git a/act/runner/testdata/.github/workflows/local-reusable-workflow.yml b/act/runner/testdata/.github/workflows/local-reusable-workflow.yml index d32dc5b8..2befe5f3 100644 --- a/act/runner/testdata/.github/workflows/local-reusable-workflow.yml +++ b/act/runner/testdata/.github/workflows/local-reusable-workflow.yml @@ -45,23 +45,23 @@ jobs: - name: test required bool run: | - echo inputs.bool_required=${{ inputs.bool_required }} - [[ "${{ inputs.bool_required }}" = "true" ]] || exit 1 + echo inputs.bool_required=${{ tojson(inputs.bool_required) }} + [[ "${{ tojson(inputs.bool_required) }}" = "true" ]] || exit 1 - name: test optional bool run: | - echo inputs.bool_optional=${{ inputs.bool_optional }} - [[ "${{ inputs.bool_optional }}" = "true" ]] || exit 1 + echo inputs.bool_optional=${{ tojson(inputs.bool_optional) }} + [[ "${{ tojson(inputs.bool_optional) }}" = "true" ]] || exit 1 - name: test required number run: | - echo inputs.number_required=${{ inputs.number_required }} - [[ "${{ inputs.number_required == 1 }}" = "true" ]] || exit 1 + echo inputs.number_required=${{ tojson(inputs.number_required) }} + [[ "${{ tojson(inputs.number_required) == '1' }}" = "true" ]] || exit 1 - name: test optional number run: | - echo inputs.number_optional=${{ inputs.number_optional }} - [[ "${{ inputs.number_optional == 1 }}" = "true" ]] || exit 1 + echo inputs.number_optional=${{ tojson(inputs.number_optional) }} + [[ "${{ tojson(inputs.number_optional) == '1' }}" = "true" ]] || exit 1 - name: test secret run: | diff --git a/act/runner/testdata/actions-environment-and-context-tests/docker/Dockerfile b/act/runner/testdata/actions-environment-and-context-tests/docker/Dockerfile index bd8fcb22..f50a50da 100644 --- a/act/runner/testdata/actions-environment-and-context-tests/docker/Dockerfile +++ b/act/runner/testdata/actions-environment-and-context-tests/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3 +FROM code.forgejo.org/oci/alpine:latest COPY entrypoint.sh /entrypoint.sh diff --git a/act/runner/testdata/actions/node24/action.yml b/act/runner/testdata/actions/node24/action.yml new file mode 100644 index 00000000..0f83974a --- /dev/null +++ b/act/runner/testdata/actions/node24/action.yml @@ -0,0 +1,5 @@ +name: 'using node24 action' +description: 'An action that has using:node24' +runs: + using: 'node24' + main: 'index.js' diff --git a/act/runner/testdata/actions/node24/index.js b/act/runner/testdata/actions/node24/index.js new file mode 100644 index 00000000..679a6f70 --- /dev/null +++ b/act/runner/testdata/actions/node24/index.js @@ -0,0 +1,3 @@ +// Note that the runner does not actually invoke different versions of node depending on the `using` tag, so this output +// isn't checked/asserted to ensure that the right version is in-use. It might as well be an empty script. +console.log('Current node.js version:', process.versions.node); diff --git a/act/runner/testdata/basic/push.yml b/act/runner/testdata/basic/push.yml index 8cb1836f..8b40283e 100644 --- a/act/runner/testdata/basic/push.yml +++ b/act/runner/testdata/basic/push.yml @@ -18,7 +18,8 @@ jobs: - run: ls - run: echo 'hello world' - run: echo ${GITHUB_SHA} >> $(dirname "${GITHUB_WORKSPACE}")/sha.txt - - run: cat $(dirname "${GITHUB_WORKSPACE}")/sha.txt | grep ${GITHUB_SHA} + - timeout-minutes: 30 + run: cat $(dirname "${GITHUB_WORKSPACE}")/sha.txt | grep ${GITHUB_SHA} build: runs-on: ubuntu-latest needs: [check] diff --git a/act/runner/testdata/issue-597/spelling.yaml b/act/runner/testdata/issue-597/spelling.yaml index 469f1500..120e2c7a 100644 --- a/act/runner/testdata/issue-597/spelling.yaml +++ b/act/runner/testdata/issue-597/spelling.yaml @@ -15,7 +15,7 @@ jobs: fetch-depth: 5 - name: My first true step if: ${{endsWith('Hello world', 'ld')}} - uses: https://github.com/actions/hello-world-javascript-action@main + uses: https://github.com/actions/hello-world-javascript-action@v1 with: who-to-greet: "Renst the Octocat" - name: My second false step diff --git a/act/runner/testdata/issue-598/spelling.yml b/act/runner/testdata/issue-598/spelling.yml index 46dcc77f..ef741b22 100644 --- a/act/runner/testdata/issue-598/spelling.yml +++ b/act/runner/testdata/issue-598/spelling.yml @@ -1,31 +1,30 @@ name: issue-598 on: push - + jobs: my_first_job: - + runs-on: ubuntu-latest steps: - name: My first false step if: "endsWith('Hello world', 'o1')" - uses: https://github.com/actions/hello-world-javascript-action@main + uses: https://github.com/actions/hello-world-javascript-action@v1 with: who-to-greet: 'Mona the Octocat' - name: My first true step if: "!endsWith('Hello world', 'od')" - uses: https://github.com/actions/hello-world-javascript-action@main + uses: https://github.com/actions/hello-world-javascript-action@v1 with: who-to-greet: "Renst the Octocat" - name: My second false step if: "endsWith('Hello world', 'o2')" - uses: https://github.com/actions/hello-world-javascript-action@main + uses: https://github.com/actions/hello-world-javascript-action@v1 with: who-to-greet: 'Act the Octocat' - name: My third false step if: "endsWith('Hello world', 'o2')" - uses: https://github.com/actions/hello-world-javascript-action@main + uses: https://github.com/actions/hello-world-javascript-action@v1 with: who-to-greet: 'Git the Octocat' - - \ No newline at end of file + diff --git a/act/runner/testdata/local-action-dockerfile-tag/README.txt b/act/runner/testdata/local-action-dockerfile-tag/README.txt new file mode 100644 index 00000000..aba97f3b --- /dev/null +++ b/act/runner/testdata/local-action-dockerfile-tag/README.txt @@ -0,0 +1,4 @@ +example1 and example2 eacho use a local actions that have the same +path (actions/docker-local) but do not behave the same. This verifies +that the locally built images have different names and do not collide +despite both being called with `uses: ./actions/docker-local. diff --git a/act/runner/testdata/local-action-dockerfile-tag/example1/actions/docker-local/Dockerfile b/act/runner/testdata/local-action-dockerfile-tag/example1/actions/docker-local/Dockerfile new file mode 100644 index 00000000..b8054873 --- /dev/null +++ b/act/runner/testdata/local-action-dockerfile-tag/example1/actions/docker-local/Dockerfile @@ -0,0 +1,8 @@ +# Container image that runs your code +FROM code.forgejo.org/oci/alpine:latest + +# Copies your code file from your action repository to the filesystem path `/` of the container +COPY entrypoint.sh /entrypoint.sh + +# Code file to execute when the docker container starts up (`entrypoint.sh`) +ENTRYPOINT ["/entrypoint.sh"] diff --git a/act/runner/testdata/local-action-dockerfile-tag/example1/actions/docker-local/action.yml b/act/runner/testdata/local-action-dockerfile-tag/example1/actions/docker-local/action.yml new file mode 100644 index 00000000..a6b97460 --- /dev/null +++ b/act/runner/testdata/local-action-dockerfile-tag/example1/actions/docker-local/action.yml @@ -0,0 +1,12 @@ +inputs: + who-to-greet: +outputs: + whoami: + description: 'The time we greeted you' +runs: + using: 'docker' + image: 'Dockerfile' + env: + WHOAMI: ${{ inputs.who-to-greet }} + args: + - ${{ inputs.who-to-greet }} diff --git a/act/runner/testdata/local-action-dockerfile-tag/example1/actions/docker-local/entrypoint.sh b/act/runner/testdata/local-action-dockerfile-tag/example1/actions/docker-local/entrypoint.sh new file mode 100755 index 00000000..7047838a --- /dev/null +++ b/act/runner/testdata/local-action-dockerfile-tag/example1/actions/docker-local/entrypoint.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +echo ::set-output "name=whoami::example1 $WHOAMI" diff --git a/act/runner/testdata/local-action-dockerfile-tag/example1/local-action-dockerfile-example1/push.yml b/act/runner/testdata/local-action-dockerfile-tag/example1/local-action-dockerfile-example1/push.yml new file mode 100644 index 00000000..fd24e76e --- /dev/null +++ b/act/runner/testdata/local-action-dockerfile-tag/example1/local-action-dockerfile-example1/push.yml @@ -0,0 +1,11 @@ +on: push +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: ./actions/docker-local + id: dockerlocal + with: + who-to-greet: 'SOMEONE' + - run: '[[ "${{ steps.dockerlocal.outputs.whoami }}" == "example1 SOMEONE" ]]' diff --git a/act/runner/testdata/local-action-dockerfile-tag/example2/actions/docker-local/Dockerfile b/act/runner/testdata/local-action-dockerfile-tag/example2/actions/docker-local/Dockerfile new file mode 100644 index 00000000..b8054873 --- /dev/null +++ b/act/runner/testdata/local-action-dockerfile-tag/example2/actions/docker-local/Dockerfile @@ -0,0 +1,8 @@ +# Container image that runs your code +FROM code.forgejo.org/oci/alpine:latest + +# Copies your code file from your action repository to the filesystem path `/` of the container +COPY entrypoint.sh /entrypoint.sh + +# Code file to execute when the docker container starts up (`entrypoint.sh`) +ENTRYPOINT ["/entrypoint.sh"] diff --git a/act/runner/testdata/local-action-dockerfile-tag/example2/actions/docker-local/action.yml b/act/runner/testdata/local-action-dockerfile-tag/example2/actions/docker-local/action.yml new file mode 100644 index 00000000..a6b97460 --- /dev/null +++ b/act/runner/testdata/local-action-dockerfile-tag/example2/actions/docker-local/action.yml @@ -0,0 +1,12 @@ +inputs: + who-to-greet: +outputs: + whoami: + description: 'The time we greeted you' +runs: + using: 'docker' + image: 'Dockerfile' + env: + WHOAMI: ${{ inputs.who-to-greet }} + args: + - ${{ inputs.who-to-greet }} diff --git a/act/runner/testdata/local-action-dockerfile-tag/example2/actions/docker-local/entrypoint.sh b/act/runner/testdata/local-action-dockerfile-tag/example2/actions/docker-local/entrypoint.sh new file mode 100755 index 00000000..c6750948 --- /dev/null +++ b/act/runner/testdata/local-action-dockerfile-tag/example2/actions/docker-local/entrypoint.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +echo ::set-output "name=whoami::example2 $WHOAMI" diff --git a/act/runner/testdata/local-action-dockerfile-tag/example2/local-action-dockerfile-example2/push.yml b/act/runner/testdata/local-action-dockerfile-tag/example2/local-action-dockerfile-example2/push.yml new file mode 100644 index 00000000..b6f39f47 --- /dev/null +++ b/act/runner/testdata/local-action-dockerfile-tag/example2/local-action-dockerfile-example2/push.yml @@ -0,0 +1,11 @@ +on: push +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: ./actions/docker-local + id: dockerlocal + with: + who-to-greet: 'SOMEONE' + - run: '[[ "${{ steps.dockerlocal.outputs.whoami }}" == "example2 SOMEONE" ]]' diff --git a/act/runner/testdata/local-action-js/push.yml b/act/runner/testdata/local-action-js/push.yml index f97cf2ec..70638568 100644 --- a/act/runner/testdata/local-action-js/push.yml +++ b/act/runner/testdata/local-action-js/push.yml @@ -10,3 +10,4 @@ jobs: - uses: ./actions/node20 with: who-to-greet: 'Mona the Octocat' + - uses: ./actions/node24 diff --git a/act/runner/testdata/matrix-runs-on/push.yml b/act/runner/testdata/matrix-runs-on/push.yml new file mode 100644 index 00000000..8863b17f --- /dev/null +++ b/act/runner/testdata/matrix-runs-on/push.yml @@ -0,0 +1,14 @@ +name: test + +on: push + +jobs: + matrix-runs-on: + strategy: + matrix: + os: [ubuntu-latest, ubuntu-20.04] + runs-on: ${{ matrix.os }} + steps: + - run: | + echo TESTOUTPUT: matrix.os == ${{ matrix.os }} + echo TESTOUTPUT: runs-on == ${{ jobs.matrix-runs-on.runs-on }} diff --git a/act/runner/testdata/matrix-shell/push.yml b/act/runner/testdata/matrix-shell/push.yml new file mode 100644 index 00000000..d7e1e5ef --- /dev/null +++ b/act/runner/testdata/matrix-shell/push.yml @@ -0,0 +1,32 @@ +name: test + +on: push + +jobs: + matrix-shell: + runs-on: ubuntu-latest + strategy: + matrix: + shell-to-use: ["bash", "cat {0}", "sh"] + steps: + - shell: ${{ matrix.shell-to-use }} + run: | + expected=${{ matrix.shell-to-use }} + if [ "$expected" = "bash" ]; then + if [ -z "$BASH_VERSION" ]; then + echo "Expected to execute bash shell, but BASH_VERSION is unset" + exit 1 + else + echo "Successfully running in bash ($BASH_VERSION)" + fi + elif [ "$expected" = "sh" ]; then + if [ -n "$BASH_VERSION" ]; then + echo "Expected to execute in sh shell, but BASH_VERSION is set ($BASH_VERSION)" + exit 1 + else + echo "Probably running in sh" + fi + else + echo "Not sure what's happening; expected shell is $expected ?!" + exit 1 + fi diff --git a/act/runner/testdata/mysql-service-container-premature-terminate/push.yml b/act/runner/testdata/mysql-service-container-premature-terminate/push.yml new file mode 100644 index 00000000..768e3e3e --- /dev/null +++ b/act/runner/testdata/mysql-service-container-premature-terminate/push.yml @@ -0,0 +1,21 @@ +name: service-container +on: push +jobs: + service-container-test: + runs-on: ubuntu-latest + container: code.forgejo.org/oci/mysql:8.4 + services: + maindb: + image: code.forgejo.org/oci/mysql:8.4 + # This container should immediately exit due to missing env variable for password config. ... [ERROR] + # [Entrypoint]: Database is uninitialized and password option is not specified You need to specify one of the + # following as an environment variable: + # - MYSQL_ROOT_PASSWORD + # - MYSQL_ALLOW_EMPTY_PASSWORD + # - MYSQL_RANDOM_ROOT_PASSWORD + # + # This container should retain the same health check config as the mysql-service-container-with-health-check + # case. + options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3 + steps: + - run: exit 100 # should never be hit since service will never be healthy diff --git a/act/runner/testdata/mysql-service-container-with-health-check/push.yml b/act/runner/testdata/mysql-service-container-with-health-check/push.yml new file mode 100644 index 00000000..efb8b2e5 --- /dev/null +++ b/act/runner/testdata/mysql-service-container-with-health-check/push.yml @@ -0,0 +1,17 @@ +name: service-container +on: push +jobs: + service-container-test: + runs-on: ubuntu-latest + container: code.forgejo.org/oci/mysql:8.4 + services: + maindb: + image: code.forgejo.org/oci/mysql:8.4 + env: + MYSQL_DATABASE: dbname + MYSQL_USER: dbuser + MYSQL_PASSWORD: dbpass + MYSQL_RANDOM_ROOT_PASSWORD: yes + options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3 + steps: + - run: mysql -u dbuser -D dbname -pdbpass -h maindb -e "create table T(id INT NOT NULL AUTO_INCREMENT, val VARCHAR(255), PRIMARY KEY (id))" diff --git a/act/runner/testdata/parallel/push.yml b/act/runner/testdata/parallel/push.yml index 1af9165e..9eb420dc 100644 --- a/act/runner/testdata/parallel/push.yml +++ b/act/runner/testdata/parallel/push.yml @@ -9,7 +9,7 @@ jobs: - uses: ./actions/action1 with: args: echo 'build' - - uses: https://github.com/actions/hello-world-javascript-action@master + - uses: https://github.com/actions/hello-world-javascript-action@v1 with: who-to-greet: 'Mona the Octocat' test1: diff --git a/act/runner/testdata/services/push.yaml b/act/runner/testdata/services/push.yaml index 03ac0855..3884a43f 100644 --- a/act/runner/testdata/services/push.yaml +++ b/act/runner/testdata/services/push.yaml @@ -6,7 +6,7 @@ jobs: runs-on: ubuntu-latest services: postgres: - image: code.forgejo.org/oci/bitnami/postgresql:16 + image: code.forgejo.org/oci/postgres:16 env: POSTGRES_USER: runner POSTGRES_PASSWORD: mysecretdbpass @@ -15,7 +15,7 @@ jobs: --health-cmd pg_isready --health-interval 10s --health-timeout 5s - --health-retries 5 + --health-retries 20 ports: - 5432:5432 steps: diff --git a/act/runner/testdata/stepsummary/push.yml b/act/runner/testdata/stepsummary/push.yml new file mode 100644 index 00000000..cb6c8e75 --- /dev/null +++ b/act/runner/testdata/stepsummary/push.yml @@ -0,0 +1,27 @@ +name: Step Summary Example + +on: [push] + +jobs: + create_summary: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: https://data.forgejo.org/actions/checkout@v4 + + # GITHUB_STEP_SUMMARY test + - name: Create Step Summary + uses: https://github.com/actions/github-script@v7 + with: + script: | + const summary = ` + ## Workflow Summary + - **Repository**: ${context.repo.owner}/${context.repo.repo} + - **Branch**: ${context.ref} + - **Commit SHA**: ${context.sha} + - **Event**: ${context.eventName} + `; + console.log('Summary:', summary); + await core.summary.addRaw(summary); + await core.summary.write(); + github-token: none diff --git a/act/runner/testdata/timeout-minutes-job/push.yml b/act/runner/testdata/timeout-minutes-job/push.yml new file mode 100644 index 00000000..be5f6c2d --- /dev/null +++ b/act/runner/testdata/timeout-minutes-job/push.yml @@ -0,0 +1,12 @@ +name: timeout-minutes +on: push + +env: + TIMEOUT_MINUTES: 0 + +jobs: + check: + runs-on: ubuntu-latest + timeout-minutes: ${{ env.TIMEOUT_MINUTES }} + steps: + - run: sleep 10 diff --git a/act/runner/testdata/timeout-minutes-step/push.yml b/act/runner/testdata/timeout-minutes-step/push.yml new file mode 100644 index 00000000..a79bb63a --- /dev/null +++ b/act/runner/testdata/timeout-minutes-step/push.yml @@ -0,0 +1,12 @@ +name: timeout-minutes +on: push + +env: + TIMEOUT_MINUTES: 0 + +jobs: + check: + runs-on: ubuntu-latest + steps: + - timeout-minutes: ${{ env.TIMEOUT_MINUTES }} + run: sleep 10 diff --git a/act/runner/testdata/uses-composite-check-for-input-collision/action-with-pre-and-post/action.yml b/act/runner/testdata/uses-composite-check-for-input-collision/action-with-pre-and-post/action.yml new file mode 100644 index 00000000..1e9a8122 --- /dev/null +++ b/act/runner/testdata/uses-composite-check-for-input-collision/action-with-pre-and-post/action.yml @@ -0,0 +1,16 @@ +name: "Action with pre and post" +description: "Action with pre and post" + +inputs: + step: + description: "step" + required: true + cache: + required: false + default: false + +runs: + using: "node16" + pre: pre.js + main: main.js + post: post.js diff --git a/act/runner/testdata/uses-composite-check-for-input-collision/action-with-pre-and-post/main.js b/act/runner/testdata/uses-composite-check-for-input-collision/action-with-pre-and-post/main.js new file mode 100644 index 00000000..5a58515a --- /dev/null +++ b/act/runner/testdata/uses-composite-check-for-input-collision/action-with-pre-and-post/main.js @@ -0,0 +1,14 @@ +const { appendFileSync } = require('fs'); +const step = process.env['INPUT_STEP']; +appendFileSync(process.env['GITHUB_ENV'], `TEST=${step}`, { encoding:'utf-8' }) + +var cache = process.env['INPUT_CACHE'] +try { + var cache = JSON.parse(cache) +} catch { + +} +if(typeof cache !== 'boolean') { + console.log("Input Polluted boolean true/false expected, got " + cache) + process.exit(1); +} \ No newline at end of file diff --git a/act/runner/testdata/uses-composite-check-for-input-collision/action-with-pre-and-post/post.js b/act/runner/testdata/uses-composite-check-for-input-collision/action-with-pre-and-post/post.js new file mode 100644 index 00000000..2f06cfe8 --- /dev/null +++ b/act/runner/testdata/uses-composite-check-for-input-collision/action-with-pre-and-post/post.js @@ -0,0 +1,14 @@ +const { appendFileSync } = require('fs'); +const step = process.env['INPUT_STEP']; +appendFileSync(process.env['GITHUB_ENV'], `TEST=${step}-post`, { encoding:'utf-8' }) + +var cache = process.env['INPUT_CACHE'] +try { + var cache = JSON.parse(cache) +} catch { + +} +if(typeof cache !== 'boolean') { + console.log("Input Polluted boolean true/false expected, got " + cache) + process.exit(1); +} \ No newline at end of file diff --git a/act/runner/testdata/uses-composite-check-for-input-collision/action-with-pre-and-post/pre.js b/act/runner/testdata/uses-composite-check-for-input-collision/action-with-pre-and-post/pre.js new file mode 100644 index 00000000..5d545140 --- /dev/null +++ b/act/runner/testdata/uses-composite-check-for-input-collision/action-with-pre-and-post/pre.js @@ -0,0 +1,12 @@ +console.log('pre'); + +var cache = process.env['INPUT_CACHE'] +try { + var cache = JSON.parse(cache) +} catch { + +} +if(typeof cache !== 'boolean') { + console.log("Input Polluted boolean true/false expected, got " + cache) + process.exit(1); +} \ No newline at end of file diff --git a/act/runner/testdata/uses-composite-check-for-input-collision/composite_action/action.yml b/act/runner/testdata/uses-composite-check-for-input-collision/composite_action/action.yml new file mode 100644 index 00000000..d9683b77 --- /dev/null +++ b/act/runner/testdata/uses-composite-check-for-input-collision/composite_action/action.yml @@ -0,0 +1,16 @@ +name: "Test Composite Action" +description: "Test action uses composite" + +inputs: + cache: + default: none + +runs: + using: "composite" + steps: + - uses: ./uses-composite-check-for-input-collision/action-with-pre-and-post + with: + step: step1 + - uses: ./uses-composite-check-for-input-collision/action-with-pre-and-post + with: + step: step2 diff --git a/act/runner/testdata/uses-composite-check-for-input-collision/push.yml b/act/runner/testdata/uses-composite-check-for-input-collision/push.yml new file mode 100644 index 00000000..059d8215 --- /dev/null +++ b/act/runner/testdata/uses-composite-check-for-input-collision/push.yml @@ -0,0 +1,10 @@ +name: uses-composite-with-pre-and-post-steps +on: push + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: https://data.forgejo.org/actions/checkout@v4 + - run: echo -n "STEP_OUTPUT_TEST=empty" >> $GITHUB_ENV + - uses: ./uses-composite-check-for-input-collision/composite_action diff --git a/act/runner/testdata/uses-composite-check-for-input-in-if-uses/composite_action/action.yml b/act/runner/testdata/uses-composite-check-for-input-in-if-uses/composite_action/action.yml new file mode 100644 index 00000000..b743bf85 --- /dev/null +++ b/act/runner/testdata/uses-composite-check-for-input-in-if-uses/composite_action/action.yml @@ -0,0 +1,47 @@ +name: "Test Composite Action" +description: "Test action uses composite" + +inputs: + b: + default: true + b2: {} + +runs: + using: "composite" + steps: + - uses: https://github.com/actions/github-script@v7 + if: inputs.b == 'true' + with: + script: | + console.log(${{ tojson(inputs) }}) + if( ${{ tojson(inputs.b) }} != 'true' ) { + process.exit(-1); + } + github-token: noop + - uses: https://github.com/actions/github-script@v7 + if: inputs.b != 'true' + with: + script: | + console.log(${{ tojson(inputs) }}) + if( ${{ tojson(inputs.b) }} == 'true' ) { + process.exit(-1); + } + github-token: noop + - uses: https://github.com/actions/github-script@v7 + if: inputs.b2 == 'false' + with: + script: | + console.log(${{ tojson(inputs) }}) + if( ${{ tojson(inputs.b2) }} != 'false' ) { + process.exit(-1); + } + github-token: noop + - uses: https://github.com/actions/github-script@v7 + if: inputs.b2 != 'false' + with: + script: | + console.log(${{ tojson(inputs) }}) + if( ${{ tojson(inputs.b2) }} == 'false' ) { + process.exit(-1); + } + github-token: noop diff --git a/act/runner/testdata/uses-composite-check-for-input-in-if-uses/push.yml b/act/runner/testdata/uses-composite-check-for-input-in-if-uses/push.yml new file mode 100644 index 00000000..068ba833 --- /dev/null +++ b/act/runner/testdata/uses-composite-check-for-input-in-if-uses/push.yml @@ -0,0 +1,24 @@ +name: uses-composite-check-for-input-in-if-uses +on: push + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: https://data.forgejo.org/actions/checkout@v4 + - uses: ./uses-composite-check-for-input-in-if-uses/composite_action + with: + b: true + b2: true + - uses: ./uses-composite-check-for-input-in-if-uses/composite_action + with: + b: false + b2: false + - uses: ./uses-composite-check-for-input-in-if-uses/composite_action + with: + b: true + b2: false + - uses: ./uses-composite-check-for-input-in-if-uses/composite_action + with: + b: false + b2: true diff --git a/act/runner/testdata/uses-composite-check-for-input-shadowing/action-with-pre-and-post/action.yml b/act/runner/testdata/uses-composite-check-for-input-shadowing/action-with-pre-and-post/action.yml new file mode 100644 index 00000000..1e9a8122 --- /dev/null +++ b/act/runner/testdata/uses-composite-check-for-input-shadowing/action-with-pre-and-post/action.yml @@ -0,0 +1,16 @@ +name: "Action with pre and post" +description: "Action with pre and post" + +inputs: + step: + description: "step" + required: true + cache: + required: false + default: false + +runs: + using: "node16" + pre: pre.js + main: main.js + post: post.js diff --git a/act/runner/testdata/uses-composite-check-for-input-shadowing/action-with-pre-and-post/main.js b/act/runner/testdata/uses-composite-check-for-input-shadowing/action-with-pre-and-post/main.js new file mode 100644 index 00000000..5a58515a --- /dev/null +++ b/act/runner/testdata/uses-composite-check-for-input-shadowing/action-with-pre-and-post/main.js @@ -0,0 +1,14 @@ +const { appendFileSync } = require('fs'); +const step = process.env['INPUT_STEP']; +appendFileSync(process.env['GITHUB_ENV'], `TEST=${step}`, { encoding:'utf-8' }) + +var cache = process.env['INPUT_CACHE'] +try { + var cache = JSON.parse(cache) +} catch { + +} +if(typeof cache !== 'boolean') { + console.log("Input Polluted boolean true/false expected, got " + cache) + process.exit(1); +} \ No newline at end of file diff --git a/act/runner/testdata/uses-composite-check-for-input-shadowing/action-with-pre-and-post/post.js b/act/runner/testdata/uses-composite-check-for-input-shadowing/action-with-pre-and-post/post.js new file mode 100644 index 00000000..2f06cfe8 --- /dev/null +++ b/act/runner/testdata/uses-composite-check-for-input-shadowing/action-with-pre-and-post/post.js @@ -0,0 +1,14 @@ +const { appendFileSync } = require('fs'); +const step = process.env['INPUT_STEP']; +appendFileSync(process.env['GITHUB_ENV'], `TEST=${step}-post`, { encoding:'utf-8' }) + +var cache = process.env['INPUT_CACHE'] +try { + var cache = JSON.parse(cache) +} catch { + +} +if(typeof cache !== 'boolean') { + console.log("Input Polluted boolean true/false expected, got " + cache) + process.exit(1); +} \ No newline at end of file diff --git a/act/runner/testdata/uses-composite-check-for-input-shadowing/action-with-pre-and-post/pre.js b/act/runner/testdata/uses-composite-check-for-input-shadowing/action-with-pre-and-post/pre.js new file mode 100644 index 00000000..5d545140 --- /dev/null +++ b/act/runner/testdata/uses-composite-check-for-input-shadowing/action-with-pre-and-post/pre.js @@ -0,0 +1,12 @@ +console.log('pre'); + +var cache = process.env['INPUT_CACHE'] +try { + var cache = JSON.parse(cache) +} catch { + +} +if(typeof cache !== 'boolean') { + console.log("Input Polluted boolean true/false expected, got " + cache) + process.exit(1); +} \ No newline at end of file diff --git a/act/runner/testdata/uses-composite-check-for-input-shadowing/composite_action/action.yml b/act/runner/testdata/uses-composite-check-for-input-shadowing/composite_action/action.yml new file mode 100644 index 00000000..2cb5b4e9 --- /dev/null +++ b/act/runner/testdata/uses-composite-check-for-input-shadowing/composite_action/action.yml @@ -0,0 +1,18 @@ +name: "Test Composite Action" +description: "Test action uses composite" + +inputs: + cache: + default: true + +runs: + using: "composite" + steps: + - uses: ./uses-composite-check-for-input-shadowing/action-with-pre-and-post + with: + step: step1 + cache: ${{ inputs.cache || 'none' }} + - uses: ./uses-composite-check-for-input-shadowing/action-with-pre-and-post + with: + step: step2 + cache: ${{ inputs.cache || 'none' }} diff --git a/act/runner/testdata/uses-composite-check-for-input-shadowing/push.yml b/act/runner/testdata/uses-composite-check-for-input-shadowing/push.yml new file mode 100644 index 00000000..0a11a995 --- /dev/null +++ b/act/runner/testdata/uses-composite-check-for-input-shadowing/push.yml @@ -0,0 +1,12 @@ +name: uses-composite-with-pre-and-post-steps +on: push + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: https://data.forgejo.org/actions/checkout@v4 + - run: echo -n "STEP_OUTPUT_TEST=empty" >> $GITHUB_ENV + - uses: ./uses-composite-check-for-input-shadowing/composite_action + # with: + # cache: other diff --git a/act/runner/testdata/uses-workflow-defaults/workflow_dispatch.yml b/act/runner/testdata/uses-workflow-defaults/workflow_dispatch.yml new file mode 100644 index 00000000..2ebe0f75 --- /dev/null +++ b/act/runner/testdata/uses-workflow-defaults/workflow_dispatch.yml @@ -0,0 +1,21 @@ +name: CI + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: https://data.forgejo.org/actions/checkout@v4 + - name: Run a one-line script + run: echo "✅ 🚀 ✅ hello this is from workflow main" ${{ github.event_name }} + call-reuse-w-val: + uses: ./testdata/.github/workflows/local-reusable-and-dispatch.yml + with: + my-val: "passed value from main" diff --git a/act/runner/testdata/uses-workflow-env-input/push.yml b/act/runner/testdata/uses-workflow-env-input/push.yml new file mode 100644 index 00000000..d702c4bb --- /dev/null +++ b/act/runner/testdata/uses-workflow-env-input/push.yml @@ -0,0 +1,8 @@ +name: local-action-env-input +on: push +jobs: + test: + runs-on: docker + uses: ./testdata/.github/workflows/local-reusable-env-input.yml + with: + greet_target: 'Mona the Octocat' diff --git a/act/runner/testdata/uses-workflow/local-workflow.yml b/act/runner/testdata/uses-workflow/local-workflow.yml index 2e9a08d7..d571a9fb 100644 --- a/act/runner/testdata/uses-workflow/local-workflow.yml +++ b/act/runner/testdata/uses-workflow/local-workflow.yml @@ -3,7 +3,7 @@ on: pull_request jobs: reusable-workflow: - uses: ./.github/workflows/local-reusable-workflow.yml + uses: ./testdata/.github/workflows/local-reusable-workflow.yml with: string_required: string bool_required: ${{ true }} @@ -12,7 +12,7 @@ jobs: secret: keep_it_private reusable-workflow-with-inherited-secrets: - uses: ./.github/workflows/local-reusable-workflow.yml + uses: ./testdata/.github/workflows/local-reusable-workflow.yml with: string_required: string bool_required: ${{ true }} @@ -20,10 +20,10 @@ jobs: secrets: inherit reusable-workflow-with-on-string-notation: - uses: ./.github/workflows/local-reusable-workflow-no-inputs-string.yml + uses: ./testdata/.github/workflows/local-reusable-workflow-no-inputs-string.yml reusable-workflow-with-on-array-notation: - uses: ./.github/workflows/local-reusable-workflow-no-inputs-array.yml + uses: ./testdata/.github/workflows/local-reusable-workflow-no-inputs-array.yml output-test: runs-on: ubuntu-latest diff --git a/act/schema/action_schema.json b/act/schema/action_schema.json index c0f7b69a..f5252c9c 100644 --- a/act/schema/action_schema.json +++ b/act/schema/action_schema.json @@ -157,6 +157,7 @@ "output-value": { "context": [ "forge", + "forgejo", "github", "strategy", "matrix", @@ -171,7 +172,9 @@ "input-default-context": { "context": [ "forge", + "forgejo", "github", + "inputs", "env", "strategy", "matrix", @@ -190,6 +193,7 @@ "string-steps-context": { "context": [ "forge", + "forgejo", "github", "inputs", "strategy", @@ -206,6 +210,7 @@ "boolean-steps-context": { "context": [ "forge", + "forgejo", "github", "inputs", "strategy", @@ -222,6 +227,7 @@ "step-env": { "context": [ "forge", + "forgejo", "github", "inputs", "strategy", @@ -241,6 +247,7 @@ "step-if": { "context": [ "forge", + "forgejo", "github", "inputs", "strategy", @@ -261,6 +268,7 @@ "step-with": { "context": [ "forge", + "forgejo", "github", "inputs", "strategy", diff --git a/act/schema/schema.go b/act/schema/schema.go index d452f012..e1d41e96 100644 --- a/act/schema/schema.go +++ b/act/schema/schema.go @@ -7,11 +7,12 @@ import ( "fmt" "math" "regexp" + "slices" "strconv" "strings" "github.com/rhysd/actionlint" - "gopkg.in/yaml.v3" + "go.yaml.in/yaml/v3" ) //go:embed workflow_schema.json @@ -236,6 +237,9 @@ func (s *Node) UnmarshalYAML(node *yaml.Node) error { if node != nil && node.Kind == yaml.DocumentNode { return s.UnmarshalYAML(node.Content[0]) } + if node != nil && node.Kind == yaml.AliasNode { + return s.UnmarshalYAML(node.Alias) + } def := s.Schema.GetDefinition(s.Definition) if s.Context == nil { s.Context = def.Context @@ -270,10 +274,8 @@ func (s *Node) UnmarshalYAML(node *yaml.Node) error { return node.Decode(&b) } else if def.AllowedValues != nil { s := node.Value - for _, v := range *def.AllowedValues { - if s == v { - return nil - } + if slices.Contains(*def.AllowedValues, s) { + return nil } return fmt.Errorf("%sExpected one of %s got %s", formatLocation(node), strings.Join(*def.AllowedValues, ","), s) } else if def.Null != nil { @@ -361,6 +363,15 @@ func (s *Node) checkMapping(node *yaml.Node, def Definition) error { if node.Kind != yaml.MappingNode { return fmt.Errorf("%sExpected a mapping got %v", formatLocation(node), getStringKind(node.Kind)) } + // merges cannot be conveniently validated and are skipped + // https://yaml.org/type/merge.html + for i, n := range node.Content { + if i%2 == 0 { + if n.Kind == yaml.ScalarNode && n.Value == "<<" && (n.Tag == "" || n.ShortTag() == "!!merge") { + return nil + } + } + } insertDirective := regexp.MustCompile(`\${{\s*insert\s*}}`) var allErrors error for i, k := range node.Content { diff --git a/act/schema/schema_test.go b/act/schema/schema_test.go index d7bc383f..beb1c185 100644 --- a/act/schema/schema_test.go +++ b/act/schema/schema_test.go @@ -1,11 +1,12 @@ package schema import ( + "fmt" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "gopkg.in/yaml.v3" + "go.yaml.in/yaml/v3" ) func TestAdditionalFunctions(t *testing.T) { @@ -29,6 +30,140 @@ jobs: assert.NoError(t, err) } +func TestContextsInWorkflowMatrix(t *testing.T) { + t.Run("KnownContexts", func(t *testing.T) { + // Parse raw YAML snippet. + var node yaml.Node + err := yaml.Unmarshal([]byte(` +on: push + +jobs: + job: + uses: ./.forgejo/workflow/test.yaml + strategy: + matrix: + input1: + - ${{ forge.KEY }} + - ${{ forgejo.KEY }} + - ${{ github.KEY }} + - ${{ inputs.KEY }} + - ${{ vars.KEY }} + - ${{ needs.KEY }} + include: + - forge: ${{ forge.KEY }} + - forgejo: ${{ forgejo.KEY }} + - github: ${{ github.KEY }} + - inputs: ${{ inputs.KEY }} + - vars: ${{ vars.KEY }} + - needs: ${{ needs.KEY }} + exclude: + - forge: ${{ forge.KEY }} + - forgejo: ${{ forgejo.KEY }} + - github: ${{ github.KEY }} + - inputs: ${{ inputs.KEY }} + - vars: ${{ vars.KEY }} + - needs: ${{ needs.KEY }} +`), &node) + if !assert.NoError(t, err) { + return + } + + // Parse YAML node as a validated workflow. + err = (&Node{ + Definition: "workflow-root", + Schema: GetWorkflowSchema(), + }).UnmarshalYAML(&node) + assert.NoError(t, err) + }) + + t.Run("UnknownContext", func(t *testing.T) { + for _, property := range []string{"include", "exclude", "input1"} { + t.Run(property, func(t *testing.T) { + for _, context := range []string{"secrets", "job", "steps", "runner", "matrix", "strategy"} { + t.Run(context, func(t *testing.T) { + var node yaml.Node + err := yaml.Unmarshal([]byte(fmt.Sprintf(` +on: push + +jobs: + job: + uses: ./.forgejo/workflow/test.yaml + strategy: + matrix: + %[1]s: + - input1: ${{ %[2]s.KEY }} +`, property, context)), &node) + if !assert.NoError(t, err) { + return + } + err = (&Node{ + Definition: "workflow-root", + Schema: GetWorkflowSchema(), + }).UnmarshalYAML(&node) + assert.ErrorContains(t, err, "Unknown Variable Access "+context) + }) + } + }) + } + }) +} + +func TestReusableWorkflow(t *testing.T) { + t.Run("KnownContexts", func(t *testing.T) { + var node yaml.Node + err := yaml.Unmarshal([]byte(` +on: push + +jobs: + job: + uses: ./.forgejo/workflow/test.yaml + with: + input1: | + ${{ forge.KEY }} + ${{ github.KEY }} + ${{ inputs.KEY }} + ${{ vars.KEY }} + ${{ env.KEY }} + ${{ needs.KEY }} + ${{ strategy.KEY }} + ${{ matrix.KEY }} +`), &node) + if !assert.NoError(t, err) { + return + } + err = (&Node{ + Definition: "workflow-root", + Schema: GetWorkflowSchema(), + }).UnmarshalYAML(&node) + assert.NoError(t, err) + }) + + t.Run("UnknownContext", func(t *testing.T) { + for _, context := range []string{"secrets", "job", "steps", "runner"} { + t.Run(context, func(t *testing.T) { + var node yaml.Node + err := yaml.Unmarshal([]byte(fmt.Sprintf(` +on: push + +jobs: + job: + uses: ./.forgejo/workflow/test.yaml + with: + input1: ${{ %[1]s.KEY }} +`, context)), &node) + if !assert.NoError(t, err) { + return + } + err = (&Node{ + Definition: "workflow-root", + Schema: GetWorkflowSchema(), + }).UnmarshalYAML(&node) + assert.ErrorContains(t, err, "Unknown Variable Access "+context) + }) + } + }) +} + func TestAdditionalFunctionsFailure(t *testing.T) { var node yaml.Node err := yaml.Unmarshal([]byte(` @@ -107,7 +242,7 @@ jobs: name: Build Silo Frontend DEV runs-on: ubuntu-latest container: - image: code.forgejo.org/oci/node:22-bookworm + image: code.forgejo.org/oci/${{ env.IMAGE }} uses: ./.forgejo/workflows/${{ vars.PATHNAME }} with: STAGE: dev @@ -140,7 +275,7 @@ inputs: default: '${{ env.GITHUB_SERVER_URL }}' repo: description: 'repo description' - default: '${{ github.repository }} ${{ vars.VARIABLE }}' + default: '${{ github.repository }} ${{ vars.VARIABLE }} ${{ inputs.VARIABLE }}' runs: using: "composite" steps: @@ -186,3 +321,39 @@ runs: }) } } + +// https://yaml.org/spec/1.2.1/#id2785586 +// An anchor is denoted by the “&” indicator. It marks a node for future reference. +// https://yaml.org/type/merge.html +// Specify one or more mappings to be merged with the current one. +func TestSchema_AnchorAndReference(t *testing.T) { + var node yaml.Node + err := yaml.Unmarshal([]byte(` +on: [push] +jobs: + test1: + runs-on: docker + steps: + - &step + run: echo All good! + - *step + test2: + runs-on: docker + steps: + - << : *step + name: other name + test3: + runs-on: docker + steps: + - !!merge << : *step + name: other name +`), &node) + if !assert.NoError(t, err) { + return + } + err = (&Node{ + Definition: "workflow-root", + Schema: GetWorkflowSchema(), + }).UnmarshalYAML(&node) + assert.NoError(t, err) +} diff --git a/act/schema/workflow_schema.json b/act/schema/workflow_schema.json index 5ec66e17..99633cac 100644 --- a/act/schema/workflow_schema.json +++ b/act/schema/workflow_schema.json @@ -52,7 +52,7 @@ "boolean": {} }, "run-name": { - "context": ["forge", "github", "inputs", "vars"], + "context": ["forge", "forgejo", "github", "inputs", "vars"], "string": {}, "description": "The name for workflow runs generated from the workflow. GitHub displays the workflow run name in the list of workflow runs on your repository's 'Actions' tab.\n\n[Documentation](https://docs.github.com/actions/using-workflows/workflow-syntax-for-github-actions#run-name)" }, @@ -1139,7 +1139,7 @@ }, "workflow-call-input-default": { "description": "If a `default` parameter is not set, the default value of the input is `false` for boolean, `0` for a number, and `\"\"` for a string.", - "context": ["forge", "github", "inputs", "vars"], + "context": ["forge", "forgejo", "github", "inputs", "vars"], "one-of": ["string", "boolean", "number"] }, "workflow-call-secrets": { @@ -1201,7 +1201,7 @@ }, "workflow-output-context": { "description": "The value to assign to the output parameter.", - "context": ["forge", "github", "inputs", "vars", "jobs"], + "context": ["forge", "forgejo", "github", "inputs", "vars", "jobs"], "string": {} }, "workflow-dispatch-string": { @@ -1402,7 +1402,7 @@ }, "workflow-env": { "description": "A map of environment variables that are available to the steps of all jobs in the workflow. You can also set environment variables that are only available to the steps of a single job or to a single step.\n\n[Documentation](https://docs.github.com/actions/using-workflows/workflow-syntax-for-github-actions#env)", - "context": ["forge", "github", "inputs", "vars", "secrets", "env"], + "context": ["forge", "forgejo", "github", "inputs", "vars", "secrets", "env"], "mapping": { "loose-key-type": "non-empty-string", "loose-value-type": "string" @@ -1517,6 +1517,7 @@ "description": "You can use the `if` conditional to prevent a job from running unless a condition is met. You can use any supported context and expression to create a conditional.", "context": [ "forge", + "forgejo", "github", "inputs", "vars", @@ -1534,6 +1535,7 @@ "job-if-result": { "context": [ "forge", + "forgejo", "github", "inputs", "vars", @@ -1548,7 +1550,7 @@ }, "strategy": { "description": "Use `strategy` to use a matrix strategy for your jobs. A matrix strategy lets you use variables in a single job definition to automatically create multiple job runs that are based on the combinations of the variables. ", - "context": ["forge", "github", "inputs", "vars", "needs"], + "context": ["forge", "forgejo", "github", "inputs", "vars", "needs"], "mapping": { "properties": { "fail-fast": { @@ -1566,6 +1568,7 @@ "fail-fast": { "context": [ "forge", + "forgejo", "github", "inputs", "vars", @@ -1584,6 +1587,7 @@ "max-parallel": { "context": [ "forge", + "forgejo", "github", "inputs", "vars", @@ -1630,7 +1634,7 @@ "runs-on": { "description": "Use `runs-on` to define the type of machine to run the job on.\n* The destination machine can be either a GitHub-hosted runner, larger runner, or a self-hosted runner.\n* You can target runners based on the labels assigned to them, or their group membership, or a combination of these.\n* You can provide `runs-on` as a single string or as an array of strings.\n* If you specify an array of strings, your workflow will execute on any runner that matches all of the specified `runs-on` values.\n* If you would like to run your workflow on multiple machines, use `jobs..strategy`.", "required": true, - "context": ["forge", "github", "inputs", "vars", "needs", "strategy", "matrix"], + "context": ["forge", "forgejo", "github", "inputs", "vars", "needs", "strategy", "matrix"], "one-of": [ "non-empty-string", "sequence-of-non-empty-string", @@ -1656,6 +1660,7 @@ "description": "A map of variables that are available to all steps in the job.", "context": [ "forge", + "forgejo", "github", "inputs", "vars", @@ -1671,12 +1676,12 @@ }, "workflow-concurrency": { "description": "Concurrency ensures that only a single job or workflow using the same concurrency group will run at a time. A concurrency group can be any string or expression.\n\nYou can also specify `concurrency` at the job level.\n\n[Documentation](https://docs.github.com/actions/using-workflows/workflow-syntax-for-github-actions#concurrency)", - "context": ["forge", "github", "inputs", "vars"], + "context": ["forge", "forgejo", "github", "inputs", "vars"], "one-of": ["string", "concurrency-mapping"] }, "job-concurrency": { "description": "Concurrency ensures that only a single job using the same concurrency group will run at a time. A concurrency group can be any string or expression. The expression can use any context except for the `secrets` context.\n\nYou can also specify `concurrency` at the workflow level.", - "context": ["forge", "github", "inputs", "vars", "needs", "strategy", "matrix"], + "context": ["forge", "forgejo", "github", "inputs", "vars", "needs", "strategy", "matrix"], "one-of": ["non-empty-string", "concurrency-mapping"] }, "concurrency-mapping": { @@ -1689,7 +1694,7 @@ "description": "When a concurrent job or workflow is queued, if another job or workflow using the same concurrency group in the repository is in progress, the queued job or workflow will be `pending`. Any previously pending job or workflow in the concurrency group will be canceled. To also cancel any currently running job or workflow in the same concurrency group, specify `cancel-in-progress: true`." }, "cancel-in-progress": { - "type": "boolean", + "type": "non-empty-string", "description": "To cancel any currently running job or workflow in the same concurrency group, specify cancel-in-progress: true." } } @@ -1697,7 +1702,7 @@ }, "job-environment": { "description": "The environment that the job references. All environment protection rules must pass before a job referencing the environment is sent to a runner.", - "context": ["forge", "github", "inputs", "vars", "needs", "strategy", "matrix"], + "context": ["forge", "forgejo", "github", "inputs", "vars", "needs", "strategy", "matrix"], "one-of": ["string", "job-environment-mapping"] }, "job-environment-mapping": { @@ -1716,7 +1721,7 @@ }, "job-environment-name": { "description": "The name of the environment used by the job.", - "context": ["forge", "github", "inputs", "vars", "needs", "strategy", "matrix"], + "context": ["forge", "forgejo", "github", "inputs", "vars", "needs", "strategy", "matrix"], "string": {} }, "job-defaults": { @@ -1730,6 +1735,7 @@ "job-defaults-run": { "context": [ "forge", + "forgejo", "github", "inputs", "vars", @@ -1800,6 +1806,7 @@ "step-uses": { "context": [ "forge", + "forgejo", "github", "inputs", "vars", @@ -1821,6 +1828,7 @@ "job-uses": { "context": [ "forge", + "forgejo", "github", "inputs", "vars", @@ -1841,6 +1849,7 @@ "step-continue-on-error": { "context": [ "forge", + "forgejo", "github", "inputs", "vars", @@ -1866,6 +1875,7 @@ "step-if": { "context": [ "forge", + "forgejo", "github", "inputs", "vars", @@ -1891,6 +1901,7 @@ "step-if-result": { "context": [ "forge", + "forgejo", "github", "inputs", "vars", @@ -1913,6 +1924,7 @@ "description": "Sets variables for steps to use in the runner environment. You can also set variables for the entire workflow or a job.", "context": [ "forge", + "forgejo", "github", "inputs", "vars", @@ -1934,6 +1946,7 @@ "step-name": { "context": [ "forge", + "forgejo", "github", "inputs", "vars", @@ -1953,6 +1966,7 @@ "step-timeout-minutes": { "context": [ "forge", + "forgejo", "github", "inputs", "vars", @@ -1973,6 +1987,7 @@ "description": "A map of the input parameters defined by the action. Each input parameter is a key/value pair. Input parameters are set as variables. When you specify an input in a workflow file or use a default input value, GitHub creates a variable for the input with the name `INPUT_`. The variable created converts input names to uppercase letters and replaces spaces with `_`.", "context": [ "forge", + "forgejo", "github", "inputs", "vars", @@ -1993,7 +2008,7 @@ }, "container": { "description": "A container to run any steps in a job that don't already specify a container. If you have steps that use both script and container actions, the container actions will run as sibling containers on the same network with the same volume mounts.\n\nIf you do not set a container, all steps will run directly on the host specified by runs-on unless a step refers to an action configured to run in a container.", - "context": ["forge", "github", "inputs", "vars", "needs", "strategy", "matrix"], + "context": ["forge", "forgejo", "github", "inputs", "vars", "needs", "strategy", "matrix", "env"], "one-of": ["string", "container-mapping"] }, "container-mapping": { @@ -2026,19 +2041,19 @@ }, "services": { "description": "Additional containers to host services for a job in a workflow. These are useful for creating databases or cache services like redis. The runner on the virtual machine will automatically create a network and manage the life cycle of the service containers. When you use a service container for a job or your step uses container actions, you don't need to set port information to access the service. Docker automatically exposes all ports between containers on the same network. When both the job and the action run in a container, you can directly reference the container by its hostname. The hostname is automatically mapped to the service name. When a step does not use a container action, you must access the service using localhost and bind the ports.", - "context": ["forge", "github", "inputs", "vars", "needs", "strategy", "matrix"], + "context": ["forge", "forgejo", "github", "inputs", "vars", "needs", "strategy", "matrix"], "mapping": { "loose-key-type": "non-empty-string", "loose-value-type": "services-container" } }, "services-container": { - "context": ["forge", "github", "inputs", "vars", "needs", "strategy", "matrix"], + "context": ["forge", "forgejo", "github", "inputs", "vars", "needs", "strategy", "matrix"], "one-of": ["non-empty-string", "container-mapping"] }, "container-registry-credentials": { "description": "If the image's container registry requires authentication to pull the image, you can use `jobs..container.credentials` to set a map of the username and password. The credentials are the same values that you would provide to the `docker login` command.", - "context": ["forge", "github", "inputs", "vars", "secrets", "env"], + "context": ["forge", "forgejo", "github", "inputs", "vars", "secrets", "env"], "mapping": { "properties": { "username": "non-empty-string", @@ -2064,24 +2079,25 @@ } }, "boolean-needs-context": { - "context": ["forge", "github", "inputs", "vars", "needs"], + "context": ["forge", "forgejo", "github", "inputs", "vars", "needs"], "boolean": {} }, "number-needs-context": { - "context": ["forge", "github", "inputs", "vars", "needs"], + "context": ["forge", "forgejo", "github", "inputs", "vars", "needs"], "number": {} }, "string-needs-context": { - "context": ["forge", "github", "inputs", "vars", "needs"], + "context": ["forge", "forgejo", "github", "inputs", "vars", "needs"], "string": {} }, "scalar-needs-context": { - "context": ["forge", "github", "inputs", "vars", "needs", "strategy", "matrix"], + "context": ["forge", "forgejo", "github", "inputs", "vars", "env", "needs", "strategy", "matrix"], "one-of": ["string", "boolean", "number"] }, "scalar-needs-context-with-secrets": { "context": [ "forge", + "forgejo", "github", "inputs", "vars", @@ -2093,24 +2109,25 @@ "one-of": ["string", "boolean", "number"] }, "boolean-strategy-context": { - "context": ["forge", "github", "inputs", "vars", "needs", "strategy", "matrix"], + "context": ["forge", "forgejo", "github", "inputs", "vars", "needs", "strategy", "matrix"], "boolean": {} }, "number-strategy-context": { - "context": ["forge", "github", "inputs", "vars", "needs", "strategy", "matrix"], + "context": ["forge", "forgejo", "github", "inputs", "vars", "needs", "strategy", "matrix"], "number": {} }, "string-strategy-context": { - "context": ["forge", "github", "inputs", "vars", "needs", "strategy", "matrix"], + "context": ["forge", "forgejo", "github", "inputs", "vars", "needs", "strategy", "matrix"], "string": {} }, "job-timeout-minutes": { - "context": ["forge", "github", "inputs", "vars", "needs", "strategy", "matrix"], + "context": ["forge", "forgejo", "github", "inputs", "vars", "needs", "strategy", "matrix"], "one-of": ["number", "string"] }, "boolean-steps-context": { "context": [ "forge", + "forgejo", "github", "inputs", "vars", @@ -2129,6 +2146,7 @@ "number-steps-context": { "context": [ "forge", + "forgejo", "github", "inputs", "vars", @@ -2147,6 +2165,7 @@ "string-runner-context": { "context": [ "forge", + "forgejo", "github", "inputs", "vars", @@ -2164,6 +2183,7 @@ "string-runner-context-no-secrets": { "context": [ "forge", + "forgejo", "github", "inputs", "vars", @@ -2180,6 +2200,7 @@ "string-steps-context": { "context": [ "forge", + "forgejo", "github", "inputs", "vars", diff --git a/act/workflowpattern/trace_writer.go b/act/workflowpattern/trace_writer.go index d5d990f6..675aa187 100644 --- a/act/workflowpattern/trace_writer.go +++ b/act/workflowpattern/trace_writer.go @@ -3,16 +3,16 @@ package workflowpattern import "fmt" type TraceWriter interface { - Info(string, ...interface{}) + Info(string, ...any) } type EmptyTraceWriter struct{} -func (*EmptyTraceWriter) Info(string, ...interface{}) { +func (*EmptyTraceWriter) Info(string, ...any) { } type StdOutTraceWriter struct{} -func (*StdOutTraceWriter) Info(format string, args ...interface{}) { +func (*StdOutTraceWriter) Info(format string, args ...any) { fmt.Printf(format+"\n", args...) } diff --git a/examples/docker-compose/compose-forgejo-and-runner.yml b/examples/docker-compose/compose-forgejo-and-runner.yml index ac51a52b..201650ae 100644 --- a/examples/docker-compose/compose-forgejo-and-runner.yml +++ b/examples/docker-compose/compose-forgejo-and-runner.yml @@ -35,7 +35,7 @@ services: bash -c ' /usr/bin/s6-svscan /etc/s6 & sleep 10 ; - su -c "forgejo forgejo-cli actions register --secret {SHARED_SECRET}" git ; + su -c "forgejo forgejo-cli actions register --keep-labels --secret {SHARED_SECRET}" git ; su -c "forgejo admin user create --admin --username root --password {ROOT_PASSWORD} --email root@example.com" git ; sleep infinity ' @@ -51,7 +51,7 @@ services: - 8080:3000 runner-register: - image: code.forgejo.org/forgejo/runner:7.0.0 + image: code.forgejo.org/forgejo/runner:11.1.2 links: - docker-in-docker - forgejo @@ -77,7 +77,7 @@ services: ' runner-daemon: - image: code.forgejo.org/forgejo/runner:7.0.0 + image: code.forgejo.org/forgejo/runner:11.1.2 links: - docker-in-docker - forgejo diff --git a/examples/lxc-systemd/README.md b/examples/lxc-systemd/README.md index f2b3baf1..5449c6cc 100644 --- a/examples/lxc-systemd/README.md +++ b/examples/lxc-systemd/README.md @@ -5,7 +5,7 @@ forgejo-runner-service.sh installs a [Forgejo runner](https://forgejo.org/docs/n - Install: `sudo wget -O /usr/local/bin/forgejo-runner-service.sh https://code.forgejo.org/forgejo/runner/raw/branch/main/examples/lxc-systemd/forgejo-runner-service.sh && sudo chmod +x /usr/local/bin/forgejo-runner-service.sh` - Obtain a runner registration token ($TOKEN) - Choose a serial number that is not already in use in `/etc/forgejo-runner` -- Create a runner `INPUTS_SERIAL=30 INPUTS_TOKEN=$TOKEN INPUTS_FORGEJO=https://code.forgejo.org forgejo-runner-service.sh` +- Create a runner `export INPUTS_SERIAL=30 ; INPUTS_TOKEN=$TOKEN INPUTS_FORGEJO=https://code.forgejo.org forgejo-runner-service.sh` - Start `systemctl enable --now forgejo-runner@$INPUTS_SERIAL` - Monitor with: - `systemctl status forgejo-runner@$INPUTS_SERIAL` diff --git a/examples/lxc-systemd/forgejo-runner-service.sh b/examples/lxc-systemd/forgejo-runner-service.sh index 24a5d755..370d5fff 100755 --- a/examples/lxc-systemd/forgejo-runner-service.sh +++ b/examples/lxc-systemd/forgejo-runner-service.sh @@ -20,14 +20,14 @@ trap "rm -fr $TMPDIR" EXIT : ${INPUTS_TOKEN:=} : ${INPUTS_FORGEJO:=https://code.forgejo.org} : ${INPUTS_LIFETIME:=7d} -DEFAULT_LXC_HELPERS_VERSION=1.0.3 # renovate: datasource=forgejo-tags depName=forgejo/lxc-helpers +DEFAULT_LXC_HELPERS_VERSION=1.1.3 # renovate: datasource=forgejo-tags depName=forgejo/lxc-helpers : ${INPUTS_LXC_HELPERS_VERSION:=$DEFAULT_LXC_HELPERS_VERSION} -DEFAULT_RUNNER_VERSION=9.0.2 # renovate: datasource=forgejo-releases depName=forgejo/runner +DEFAULT_RUNNER_VERSION=11.1.2 # renovate: datasource=forgejo-releases depName=forgejo/runner : ${INPUTS_RUNNER_VERSION:=$DEFAULT_RUNNER_VERSION} : ${KILL_AFTER:=21600} # 6h == 21600 NODEJS_VERSION=20 -DEBIAN_RELEASE=bookworm +DEBIAN_RELEASE=trixie YQ_VERSION=v4.45.1 SELF=${BASH_SOURCE[0]} SELF_FILENAME=$(basename "$SELF") diff --git a/go.mod b/go.mod index 32b0fef1..009ac5f9 100644 --- a/go.mod +++ b/go.mod @@ -1,25 +1,26 @@ -module code.forgejo.org/forgejo/runner/v9 +module code.forgejo.org/forgejo/runner/v11 -go 1.23.0 +go 1.24.0 -toolchain go1.23.11 +toolchain go1.24.9 require ( - code.forgejo.org/forgejo/actions-proto v0.5.1 - connectrpc.com/connect v1.18.1 + code.forgejo.org/forgejo/actions-proto v0.5.3 + connectrpc.com/connect v1.19.1 + dario.cat/mergo v1.0.2 github.com/Masterminds/semver v1.5.0 - github.com/avast/retry-go/v4 v4.6.1 + github.com/avast/retry-go/v4 v4.7.0 github.com/containerd/errdefs v1.0.0 github.com/creack/pty v1.1.24 github.com/distribution/reference v0.6.0 - github.com/docker/cli v28.3.3+incompatible - github.com/docker/docker v28.3.3+incompatible - github.com/docker/go-connections v0.5.0 + github.com/docker/cli v28.5.1+incompatible + github.com/docker/docker v28.5.1+incompatible + github.com/docker/go-connections v0.6.0 github.com/go-git/go-billy/v5 v5.6.2 - github.com/go-git/go-git/v5 v5.16.2 + github.com/go-git/go-git/v5 v5.16.3 github.com/gobwas/glob v0.2.3 + github.com/google/go-cmp v0.7.0 github.com/google/uuid v1.6.0 - github.com/imdario/mergo v0.3.16 github.com/joho/godotenv v1.5.1 github.com/julienschmidt/httprouter v1.3.0 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 @@ -28,27 +29,25 @@ require ( github.com/moby/patternmatcher v0.6.0 github.com/opencontainers/image-spec v1.1.1 github.com/opencontainers/selinux v1.12.0 - github.com/pkg/errors v0.9.1 - github.com/rhysd/actionlint v1.7.7 + github.com/rhysd/actionlint v1.7.8 github.com/sirupsen/logrus v1.9.3 - github.com/spf13/cobra v1.9.1 - github.com/spf13/pflag v1.0.7 - github.com/stretchr/testify v1.10.0 + github.com/spf13/cobra v1.10.1 + github.com/spf13/pflag v1.0.10 + github.com/stretchr/testify v1.11.1 github.com/timshannon/bolthold v0.0.0-20240314194003-30aac6950928 - go.etcd.io/bbolt v1.4.2 - golang.org/x/term v0.33.0 - golang.org/x/time v0.12.0 - google.golang.org/protobuf v1.36.6 - gopkg.in/yaml.v3 v3.0.1 + go.etcd.io/bbolt v1.4.3 + go.yaml.in/yaml/v3 v3.0.4 + golang.org/x/term v0.36.0 + golang.org/x/time v0.14.0 + google.golang.org/protobuf v1.36.10 gotest.tools/v3 v3.5.2 ) require ( - dario.cat/mergo v1.0.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/ProtonMail/go-crypto v1.1.6 // indirect - github.com/bmatcuk/doublestar/v4 v4.8.0 // indirect + github.com/bmatcuk/doublestar/v4 v4.9.1 // indirect github.com/cloudflare/circl v1.6.1 // indirect github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/containerd/log v0.1.0 // indirect @@ -62,17 +61,15 @@ require ( github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-viper/mapstructure/v2 v2.3.0 // indirect - github.com/gogo/protobuf v1.3.2 // indirect + github.com/go-viper/mapstructure/v2 v2.4.0 // indirect github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect - github.com/google/go-cmp v0.7.0 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/klauspost/compress v1.18.0 // indirect github.com/mattn/go-colorable v0.1.14 // indirect - github.com/mattn/go-runewidth v0.0.16 // indirect + github.com/mattn/go-runewidth v0.0.17 // indirect github.com/mattn/go-shellwords v1.0.12 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/sys/atomicwriter v0.1.0 // indirect @@ -83,6 +80,7 @@ require ( github.com/morikuni/aec v1.0.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/pjbgf/sha1cd v0.3.2 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/robfig/cron/v3 v3.0.1 // indirect @@ -100,10 +98,12 @@ require ( go.opentelemetry.io/otel/metric v1.36.0 // indirect go.opentelemetry.io/otel/sdk v1.21.0 // indirect go.opentelemetry.io/otel/trace v1.36.0 // indirect + go.yaml.in/yaml/v4 v4.0.0-rc.2 // indirect golang.org/x/crypto v0.37.0 // indirect golang.org/x/net v0.39.0 // indirect - golang.org/x/sync v0.12.0 // indirect - golang.org/x/sys v0.34.0 // indirect + golang.org/x/sync v0.17.0 // indirect + golang.org/x/sys v0.37.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20231120223509-83a465c0220f // indirect gopkg.in/warnings.v0 v0.1.2 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 50aa8317..76b55827 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,9 @@ -code.forgejo.org/forgejo/actions-proto v0.5.1 h1:GCJHR/Y/Apk7Yl7CH9qOsKrdf/k0tRVFeVhz1EIZvb4= -code.forgejo.org/forgejo/actions-proto v0.5.1/go.mod h1:nu8N1HQLsu3c4T/PpYWbqwNBxsZnEOVxqV0mQWtIQvE= -connectrpc.com/connect v1.18.1 h1:PAg7CjSAGvscaf6YZKUefjoih5Z/qYkyaTrBW8xvYPw= -connectrpc.com/connect v1.18.1/go.mod h1:0292hj1rnx8oFrStN7cB4jjVBeqs+Yx5yDIC2prWDO8= -dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= -dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +code.forgejo.org/forgejo/actions-proto v0.5.3 h1:dDProRNB4CDvEl9gfo8jkiVfGdiW7fXAt5TM9Irka28= +code.forgejo.org/forgejo/actions-proto v0.5.3/go.mod h1:33iTdur/jVa/wAQP+BuciRTK9WZcVaxy0BNEnSWWFDM= +connectrpc.com/connect v1.19.1 h1:R5M57z05+90EfEvCY1b7hBxDVOUl45PrtXtAV2fOC14= +connectrpc.com/connect v1.19.1/go.mod h1:tN20fjdGlewnSFeZxLKb0xwIZ6ozc3OQs2hTXy4du9w= +dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= +dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk= github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= @@ -19,10 +19,10 @@ github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFI github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= -github.com/avast/retry-go/v4 v4.6.1 h1:VkOLRubHdisGrHnTu89g08aQEWEgRU7LVEop3GbIcMk= -github.com/avast/retry-go/v4 v4.6.1/go.mod h1:V6oF8njAwxJ5gRo1Q7Cxab24xs5NCWZBeaHHBklR8mA= -github.com/bmatcuk/doublestar/v4 v4.8.0 h1:DSXtrypQddoug1459viM9X9D3dp1Z7993fw36I2kNcQ= -github.com/bmatcuk/doublestar/v4 v4.8.0/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= +github.com/avast/retry-go/v4 v4.7.0 h1:yjDs35SlGvKwRNSykujfjdMxMhMQQM0TnIjJaHB+Zio= +github.com/avast/retry-go/v4 v4.7.0/go.mod h1:ZMPDa3sY2bKgpLtap9JRUgk2yTAba7cgiFhqxY2Sg6Q= +github.com/bmatcuk/doublestar/v4 v4.9.1 h1:X8jg9rRZmJd4yRy7ZeNDRnM+T3ZfHv15JiBJ/avrEXE= +github.com/bmatcuk/doublestar/v4 v4.9.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cloudflare/circl v1.6.1 h1:zqIqSPIndyBh1bjLVVDHMPpVKqp8Su/V+6MeDzzQBQ0= @@ -43,14 +43,14 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/docker/cli v28.3.3+incompatible h1:fp9ZHAr1WWPGdIWBM1b3zLtgCF+83gRdVMTJsUeiyAo= -github.com/docker/cli v28.3.3+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= -github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjYkaKubwuBdxEI= -github.com/docker/docker v28.3.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/cli v28.5.1+incompatible h1:ESutzBALAD6qyCLqbQSEf1a/U8Ybms5agw59yGVc+yY= +github.com/docker/cli v28.5.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/docker v28.5.1+incompatible h1:Bm8DchhSD2J6PsFzxC35TZo4TLGR2PdW/E69rU45NhM= +github.com/docker/docker v28.5.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker-credential-helpers v0.8.0 h1:YQFtbBQb4VrpoPxhFuzEBPQ9E16qz5SpHLS+uswaCp8= github.com/docker/docker-credential-helpers v0.8.0/go.mod h1:UGFXcuoQ5TxPiB54nHOZ32AWRqQdECoh/Mg0AlEYb40= -github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= -github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= +github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94= +github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o= @@ -69,19 +69,17 @@ github.com/go-git/go-billy/v5 v5.6.2 h1:6Q86EsPXMa7c3YZ3aLAQsMA0VlWmy43r6FHqa/UN github.com/go-git/go-billy/v5 v5.6.2/go.mod h1:rcFC2rAsp/erv7CMz9GczHcuD0D32fWzH+MJAU+jaUU= github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= -github.com/go-git/go-git/v5 v5.16.2 h1:fT6ZIOjE5iEnkzKyxTHK1W4HGAsPhqEqiSAssSO77hM= -github.com/go-git/go-git/v5 v5.16.2/go.mod h1:4Ge4alE/5gPs30F2H1esi2gPd69R0C39lolkucHBOp8= +github.com/go-git/go-git/v5 v5.16.3 h1:Z8BtvxZ09bYm/yYNgPKCzgWtaRqDTgIKRgIRHBfU6Z8= +github.com/go-git/go-git/v5 v5.16.3/go.mod h1:4Ge4alE/5gPs30F2H1esi2gPd69R0C39lolkucHBOp8= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-viper/mapstructure/v2 v2.3.0 h1:27XbWsHIqhbdR5TIC911OfYvgSaW93HM+dX7970Q7jk= -github.com/go-viper/mapstructure/v2 v2.3.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= +github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ= github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= @@ -92,8 +90,6 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= -github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= -github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= @@ -106,8 +102,6 @@ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNU github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -121,8 +115,8 @@ github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHP github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= -github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-runewidth v0.0.17 h1:78v8ZlW0bP43XfmAfPsdXcoNCelfMHsDmd/pkENfrjQ= +github.com/mattn/go-runewidth v0.0.17/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk= github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= @@ -157,8 +151,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rhysd/actionlint v1.7.7 h1:0KgkoNTrYY7vmOCs9BW2AHxLvvpoY9nEUzgBHiPUr0k= -github.com/rhysd/actionlint v1.7.7/go.mod h1:AE6I6vJEkNaIfWqC2GNE5spIJNhxf8NCtLEKU4NnUXg= +github.com/rhysd/actionlint v1.7.8 h1:3d+N9ourgAxVYG4z2IFxFIk/YiT6V+VnKASfXGwT60E= +github.com/rhysd/actionlint v1.7.8/go.mod h1:3kiS6egcbXG+vQsJIhFxTz+UKaF1JprsE0SKrpCZKvU= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= @@ -174,11 +168,11 @@ github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/skeema/knownhosts v1.3.1 h1:X2osQ+RAjK76shCbvhHHHVl3ZlgDm8apHEHFqRjnBY8= github.com/skeema/knownhosts v1.3.1/go.mod h1:r7KTdC8l4uxWRyK2TpQZ/1o5HaSzh06ePQNxPwTcfiY= -github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= -github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= -github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/pflag v1.0.7 h1:vN6T9TfwStFPFM5XzjsvmzZkLuaLX+HS+0SeFLRgU6M= -github.com/spf13/pflag v1.0.7/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s= +github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -191,8 +185,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/timshannon/bolthold v0.0.0-20240314194003-30aac6950928 h1:zjNCuOOhh1TKRU0Ru3PPPJt80z7eReswCao91gBLk00= github.com/timshannon/bolthold v0.0.0-20240314194003-30aac6950928/go.mod h1:PCFYfAEfKT+Nd6zWvUpsXduMR1bXFLf0uGSlEF05MCI= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= @@ -204,11 +198,9 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHo github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.8/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= -go.etcd.io/bbolt v1.4.2 h1:IrUHp260R8c+zYx/Tm8QZr04CX+qWS5PGfPdevhdm1I= -go.etcd.io/bbolt v1.4.2/go.mod h1:Is8rSHO/b4f3XigBC0lL0+4FwAQv3HXEEIgFMuKHceM= +go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo= +go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E= go.etcd.io/gofail v0.1.0/go.mod h1:VZBCXYGZhHAinaBiiqYvuDynvahNsAyLFwB3kEHKz1M= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= @@ -228,32 +220,21 @@ go.opentelemetry.io/otel/trace v1.36.0 h1:ahxWNuqZjpdiFAyrIoQ4GIiAIhxAunQR6MUoKr go.opentelemetry.io/otel/trace v1.36.0/go.mod h1:gQ+OnDZzrybY4k4seLzPAWNwVBBVlF2szhehOBB/tGA= go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +go.yaml.in/yaml/v4 v4.0.0-rc.2 h1:/FrI8D64VSr4HtGIlUtlFMGsm7H7pWTbj6vOLVZcA6s= +go.yaml.in/yaml/v4 v4.0.0-rc.2/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY= golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= -golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= +golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -263,26 +244,17 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= -golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= +golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.33.0 h1:NuFncQrRcaRvVmgRkvM3j/F00gWIAlcmlB8ACEKmGIg= -golang.org/x/term v0.33.0/go.mod h1:s18+ql9tYWp1IfpV9DmCtQDDSRBUjKaw9M1eAv5UeF0= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/term v0.36.0 h1:zMPR+aF8gfksFprF/Nc/rd1wRS1EI6nDBGyWAvDzx2Q= +golang.org/x/term v0.36.0/go.mod h1:Qu394IJq6V6dCBRgwqshf3mPF85AqzYEzofzRdZkWss= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= -golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= -golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= +golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= +golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17 h1:wpZ8pe2x1Q3f2KyT5f8oP/fa9rHAKgFPr/HZdNuS+PQ= google.golang.org/genproto/googleapis/api v0.0.0-20231120223509-83a465c0220f h1:2yNACc1O40tTnrsbk9Cv6oxiW8pxI/pXj0wRtdlYmgY= google.golang.org/genproto/googleapis/api v0.0.0-20231120223509-83a465c0220f/go.mod h1:Uy9bTZJqmfrw2rIBxgGLnamc78euZULUBrLZ9XTITKI= @@ -290,8 +262,8 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1: google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= google.golang.org/grpc v1.67.0 h1:IdH9y6PF5MPSdAntIcpjQ+tXO41pcQsfZV2RxtQgVcw= google.golang.org/grpc v1.67.0/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= -google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= -google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= +google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= diff --git a/internal/app/cmd/cache-server.go b/internal/app/cmd/cache-server.go index ff233f19..37bee855 100644 --- a/internal/app/cmd/cache-server.go +++ b/internal/app/cmd/cache-server.go @@ -8,10 +8,11 @@ import ( "fmt" "os" "os/signal" + "syscall" - "code.forgejo.org/forgejo/runner/v9/internal/pkg/config" + "code.forgejo.org/forgejo/runner/v11/internal/pkg/config" - "code.forgejo.org/forgejo/runner/v9/act/artifactcache" + "code.forgejo.org/forgejo/runner/v11/act/artifactcache" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" ) @@ -73,7 +74,7 @@ func runCacheServer(ctx context.Context, configFile *string, cacheArgs *cacheSer log.Infof("cache server is listening on %v", cacheHandler.ExternalURL()) c := make(chan os.Signal, 1) - signal.Notify(c, os.Interrupt) + signal.Notify(c, syscall.SIGINT, syscall.SIGTERM) <-c return nil diff --git a/internal/app/cmd/cmd.go b/internal/app/cmd/cmd.go index c3a98a88..aada3bbe 100644 --- a/internal/app/cmd/cmd.go +++ b/internal/app/cmd/cmd.go @@ -10,8 +10,8 @@ import ( "github.com/spf13/cobra" - "code.forgejo.org/forgejo/runner/v9/internal/pkg/config" - "code.forgejo.org/forgejo/runner/v9/internal/pkg/ver" + "code.forgejo.org/forgejo/runner/v11/internal/pkg/config" + "code.forgejo.org/forgejo/runner/v11/internal/pkg/ver" ) func Execute(ctx context.Context) { @@ -45,7 +45,7 @@ func Execute(ctx context.Context) { Use: "daemon", Short: "Run as a runner daemon", Args: cobra.MaximumNArgs(1), - RunE: runDaemon(ctx, &configFile), + RunE: getRunDaemonCommandProcessor(ctx, &configFile), } rootCmd.AddCommand(daemonCmd) diff --git a/internal/app/cmd/create-runner-file.go b/internal/app/cmd/create-runner-file.go index 0eeb713b..2a811b0a 100644 --- a/internal/app/cmd/create-runner-file.go +++ b/internal/app/cmd/create-runner-file.go @@ -14,10 +14,10 @@ import ( log "github.com/sirupsen/logrus" "github.com/spf13/cobra" - "code.forgejo.org/forgejo/runner/v9/internal/app/run" - "code.forgejo.org/forgejo/runner/v9/internal/pkg/client" - "code.forgejo.org/forgejo/runner/v9/internal/pkg/config" - "code.forgejo.org/forgejo/runner/v9/internal/pkg/ver" + "code.forgejo.org/forgejo/runner/v11/internal/app/run" + "code.forgejo.org/forgejo/runner/v11/internal/pkg/client" + "code.forgejo.org/forgejo/runner/v11/internal/pkg/config" + "code.forgejo.org/forgejo/runner/v11/internal/pkg/ver" ) type createRunnerFileArgs struct { diff --git a/internal/app/cmd/create-runner-file_test.go b/internal/app/cmd/create-runner-file_test.go index d12cc325..546a5d87 100644 --- a/internal/app/cmd/create-runner-file_test.go +++ b/internal/app/cmd/create-runner-file_test.go @@ -8,13 +8,13 @@ import ( "testing" runnerv1 "code.forgejo.org/forgejo/actions-proto/runner/v1" - "code.forgejo.org/forgejo/runner/v9/internal/pkg/client" - "code.forgejo.org/forgejo/runner/v9/internal/pkg/config" - "code.forgejo.org/forgejo/runner/v9/internal/pkg/ver" + "code.forgejo.org/forgejo/runner/v11/internal/pkg/client" + "code.forgejo.org/forgejo/runner/v11/internal/pkg/config" + "code.forgejo.org/forgejo/runner/v11/internal/pkg/ver" "connectrpc.com/connect" "github.com/stretchr/testify/assert" - "gopkg.in/yaml.v3" + "go.yaml.in/yaml/v3" ) func Test_createRunnerFileCmd(t *testing.T) { @@ -37,12 +37,18 @@ func Test_uuidFromSecret(t *testing.T) { assert.EqualValues(t, uuid, "41414141-4141-4141-4141-414141414141") } -func Test_ping(t *testing.T) { - cfg := &config.Config{} +func getForgejoFromEnv(t *testing.T) string { + t.Helper() address := os.Getenv("FORGEJO_URL") if address == "" { - address = "https://code.forgejo.org" + t.Skip("skipping because FORGEJO_URL is not set") } + return address +} + +func Test_ping(t *testing.T) { + cfg := &config.Config{} + address := getForgejoFromEnv(t) reg := &config.Registration{ Address: address, UUID: "create-runner-file_test.go", @@ -51,6 +57,8 @@ func Test_ping(t *testing.T) { } func Test_runCreateRunnerFile(t *testing.T) { + instance := getForgejoFromEnv(t) + // // Set the .runner file to be in a temporary directory // @@ -63,10 +71,6 @@ func Test_runCreateRunnerFile(t *testing.T) { assert.NoError(t, err) assert.NoError(t, os.WriteFile(configFile, yamlData, 0o666)) - instance, has := os.LookupEnv("FORGEJO_URL") - if !has { - instance = "https://code.forgejo.org" - } secret, has := os.LookupEnv("FORGEJO_RUNNER_SECRET") assert.True(t, has) name := "testrunner" diff --git a/internal/app/cmd/daemon.go b/internal/app/cmd/daemon.go index 0a9815ed..4edf2dbe 100644 --- a/internal/app/cmd/daemon.go +++ b/internal/app/cmd/daemon.go @@ -18,122 +18,85 @@ import ( log "github.com/sirupsen/logrus" "github.com/spf13/cobra" - "code.forgejo.org/forgejo/runner/v9/internal/app/poll" - "code.forgejo.org/forgejo/runner/v9/internal/app/run" - "code.forgejo.org/forgejo/runner/v9/internal/pkg/client" - "code.forgejo.org/forgejo/runner/v9/internal/pkg/config" - "code.forgejo.org/forgejo/runner/v9/internal/pkg/envcheck" - "code.forgejo.org/forgejo/runner/v9/internal/pkg/labels" - "code.forgejo.org/forgejo/runner/v9/internal/pkg/ver" + "code.forgejo.org/forgejo/runner/v11/internal/app/poll" + "code.forgejo.org/forgejo/runner/v11/internal/app/run" + "code.forgejo.org/forgejo/runner/v11/internal/pkg/client" + "code.forgejo.org/forgejo/runner/v11/internal/pkg/common" + "code.forgejo.org/forgejo/runner/v11/internal/pkg/config" + "code.forgejo.org/forgejo/runner/v11/internal/pkg/envcheck" + "code.forgejo.org/forgejo/runner/v11/internal/pkg/labels" + "code.forgejo.org/forgejo/runner/v11/internal/pkg/ver" ) -func runDaemon(ctx context.Context, configFile *string) func(cmd *cobra.Command, args []string) error { +func getRunDaemonCommandProcessor(signalContext context.Context, configFile *string) func(cmd *cobra.Command, args []string) error { return func(cmd *cobra.Command, args []string) error { - cfg, err := config.LoadDefault(*configFile) - if err != nil { - return fmt.Errorf("invalid configuration: %w", err) - } - - initLogging(cfg) - log.Infoln("Starting runner daemon") - - reg, err := config.LoadRegistration(cfg.Runner.File) - if os.IsNotExist(err) { - log.Error("registration file not found, please register the runner first") - return err - } else if err != nil { - return fmt.Errorf("failed to load registration file: %w", err) - } - - cfg.Tune(reg.Address) - - lbls := reg.Labels - if len(cfg.Runner.Labels) > 0 { - lbls = cfg.Runner.Labels - } - - ls := labels.Labels{} - for _, l := range lbls { - label, err := labels.Parse(l) - if err != nil { - log.WithError(err).Warnf("ignored invalid label %q", l) - continue - } - ls = append(ls, label) - } - if len(ls) == 0 { - log.Warn("no labels configured, runner may not be able to pick up jobs") - } - - if ls.RequireDocker() { - dockerSocketPath, err := getDockerSocketPath(cfg.Container.DockerHost) - if err != nil { - return err - } - if err := envcheck.CheckIfDockerRunning(ctx, dockerSocketPath); err != nil { - return err - } - os.Setenv("DOCKER_HOST", dockerSocketPath) - if cfg.Container.DockerHost == "automount" { - cfg.Container.DockerHost = dockerSocketPath - } - // check the scheme, if the scheme is not npipe or unix - // set cfg.Container.DockerHost to "-" because it can't be mounted to the job container - if protoIndex := strings.Index(cfg.Container.DockerHost, "://"); protoIndex != -1 { - scheme := cfg.Container.DockerHost[:protoIndex] - if !strings.EqualFold(scheme, "npipe") && !strings.EqualFold(scheme, "unix") { - cfg.Container.DockerHost = "-" - } - } - } - - cli := client.New( - reg.Address, - cfg.Runner.Insecure, - reg.UUID, - reg.Token, - ver.Version(), - ) - - runner := run.NewRunner(cfg, reg, cli) - // declare the labels of the runner before fetching tasks - resp, err := runner.Declare(ctx, ls.Names()) - if err != nil && connect.CodeOf(err) == connect.CodeUnimplemented { - log.Warn("Because the Forgejo instance is an old version, skipping declaring the labels and version.") - } else if err != nil { - log.WithError(err).Error("fail to invoke Declare") - return err - } else { - log.Infof("runner: %s, with version: %s, with labels: %v, declared successfully", - resp.Msg.GetRunner().GetName(), resp.Msg.GetRunner().GetVersion(), resp.Msg.GetRunner().GetLabels()) - // if declared successfully, override the labels in the.runner file with valid labels in the config file (if specified) - runner.Update(ctx, ls) - reg.Labels = ls.ToStrings() - if err := config.SaveRegistration(cfg.Runner.File, reg); err != nil { - return fmt.Errorf("failed to save runner config: %w", err) - } - } - - poller := poll.New(cfg, cli, runner) - - go poller.Poll() - - <-ctx.Done() - log.Infof("runner: %s shutdown initiated, waiting [runner].shutdown_timeout=%s for running jobs to complete before shutting down", resp.Msg.GetRunner().GetName(), cfg.Runner.ShutdownTimeout) - - ctx, cancel := context.WithTimeout(context.Background(), cfg.Runner.ShutdownTimeout) - defer cancel() - - err = poller.Shutdown(ctx) - if err != nil { - log.Warnf("runner: %s cancelled in progress jobs during shutdown", resp.Msg.GetRunner().GetName()) - } - return nil + return runDaemon(signalContext, configFile) } } +func runDaemon(signalContext context.Context, configFile *string) error { + // signalContext will be 'done' when we receive a graceful shutdown signal; daemonContext is not a derived context + // because we want it to 'outlive' the signalContext in order to perform graceful cleanup. + daemonContext, cancel := context.WithCancel(context.Background()) + defer cancel() + + ctx := common.WithDaemonContext(daemonContext, daemonContext) + + cfg, err := initializeConfig(configFile) + if err != nil { + return err + } + + initLogging(cfg) + log.Infoln("Starting runner daemon") + + reg, err := loadRegistration(cfg) + if err != nil { + return err + } + + cfg.Tune(reg.Address) + ls := extractLabels(cfg, reg) + + err = configCheck(ctx, cfg, ls) + if err != nil { + return err + } + + cli := createClient(cfg, reg) + + runner, runnerName, err := createRunner(ctx, cfg, reg, cli, ls) + if err != nil { + return err + } + + poller := createPoller(ctx, cfg, cli, runner) + + go poller.Poll() + + <-signalContext.Done() + log.Infof("runner: %s shutdown initiated, waiting [runner].shutdown_timeout=%s for running jobs to complete before shutting down", runnerName, cfg.Runner.ShutdownTimeout) + + shutdownCtx, cancel := context.WithTimeout(daemonContext, cfg.Runner.ShutdownTimeout) + defer cancel() + + err = poller.Shutdown(shutdownCtx) + if err != nil { + log.Warnf("runner: %s cancelled in progress jobs during shutdown", runnerName) + } + return nil +} + +var initializeConfig = func(configFile *string) (*config.Config, error) { + cfg, err := config.LoadDefault(*configFile) + if err != nil { + return nil, fmt.Errorf("invalid configuration: %w", err) + } + return cfg, nil +} + // initLogging setup the global logrus logger. -func initLogging(cfg *config.Config) { +var initLogging = func(cfg *config.Config) { isTerm := isatty.IsTerminal(os.Stdout.Fd()) format := &log.TextFormatter{ DisableColors: !isTerm, @@ -170,6 +133,99 @@ func initLogging(cfg *config.Config) { } } +var loadRegistration = func(cfg *config.Config) (*config.Registration, error) { + reg, err := config.LoadRegistration(cfg.Runner.File) + if os.IsNotExist(err) { + log.Error("registration file not found, please register the runner first") + return nil, err + } else if err != nil { + return nil, fmt.Errorf("failed to load registration file: %w", err) + } + return reg, nil +} + +var extractLabels = func(cfg *config.Config, reg *config.Registration) labels.Labels { + lbls := reg.Labels + if len(cfg.Runner.Labels) > 0 { + lbls = cfg.Runner.Labels + } + + ls := labels.Labels{} + for _, l := range lbls { + label, err := labels.Parse(l) + if err != nil { + log.WithError(err).Warnf("ignored invalid label %q", l) + continue + } + ls = append(ls, label) + } + if len(ls) == 0 { + log.Warn("no labels configured, runner may not be able to pick up jobs") + } + return ls +} + +var configCheck = func(ctx context.Context, cfg *config.Config, ls labels.Labels) error { + if ls.RequireDocker() { + dockerSocketPath, err := getDockerSocketPath(cfg.Container.DockerHost) + if err != nil { + return err + } + if err := envcheck.CheckIfDockerRunning(ctx, dockerSocketPath); err != nil { + return err + } + os.Setenv("DOCKER_HOST", dockerSocketPath) + if cfg.Container.DockerHost == "automount" { + cfg.Container.DockerHost = dockerSocketPath + } + // check the scheme, if the scheme is not npipe or unix + // set cfg.Container.DockerHost to "-" because it can't be mounted to the job container + if protoIndex := strings.Index(cfg.Container.DockerHost, "://"); protoIndex != -1 { + scheme := cfg.Container.DockerHost[:protoIndex] + if !strings.EqualFold(scheme, "npipe") && !strings.EqualFold(scheme, "unix") { + cfg.Container.DockerHost = "-" + } + } + } + return nil +} + +var createClient = func(cfg *config.Config, reg *config.Registration) client.Client { + return client.New( + reg.Address, + cfg.Runner.Insecure, + reg.UUID, + reg.Token, + ver.Version(), + ) +} + +var createRunner = func(ctx context.Context, cfg *config.Config, reg *config.Registration, cli client.Client, ls labels.Labels) (run.RunnerInterface, string, error) { + runner := run.NewRunner(cfg, reg, cli) + // declare the labels of the runner before fetching tasks + resp, err := runner.Declare(ctx, ls.Names()) + if err != nil && connect.CodeOf(err) == connect.CodeUnimplemented { + log.Warn("Because the Forgejo instance is an old version, skipping declaring the labels and version.") + return runner, "runner", nil + } else if err != nil { + log.WithError(err).Error("fail to invoke Declare") + return nil, "", err + } + + log.Infof("runner: %s, with version: %s, with labels: %v, declared successfully", + resp.Msg.GetRunner().GetName(), resp.Msg.GetRunner().GetVersion(), resp.Msg.GetRunner().GetLabels()) + // if declared successfully, override the labels in the.runner file with valid labels in the config file (if specified) + runner.Update(ctx, ls) + reg.Labels = ls.ToStrings() + if err := config.SaveRegistration(cfg.Runner.File, reg); err != nil { + return nil, "", fmt.Errorf("failed to save runner config: %w", err) + } + return runner, resp.Msg.GetRunner().GetName(), nil +} + +// func(ctx context.Context, cfg *config.Config, cli client.Client, runner run.RunnerInterface) poll.Poller +var createPoller = poll.New + var commonSocketPaths = []string{ "/var/run/docker.sock", "/run/podman/podman.sock", diff --git a/internal/app/cmd/daemon_test.go b/internal/app/cmd/daemon_test.go new file mode 100644 index 00000000..16ce8e50 --- /dev/null +++ b/internal/app/cmd/daemon_test.go @@ -0,0 +1,117 @@ +// Copyright 2025 The Forgejo Authors +// SPDX-License-Identifier: MIT + +package cmd + +import ( + "context" + "testing" + "time" + + "code.forgejo.org/forgejo/runner/v11/internal/app/poll" + mock_poller "code.forgejo.org/forgejo/runner/v11/internal/app/poll/mocks" + "code.forgejo.org/forgejo/runner/v11/internal/app/run" + mock_runner "code.forgejo.org/forgejo/runner/v11/internal/app/run/mocks" + "code.forgejo.org/forgejo/runner/v11/internal/pkg/client" + mock_client "code.forgejo.org/forgejo/runner/v11/internal/pkg/client/mocks" + "code.forgejo.org/forgejo/runner/v11/internal/pkg/config" + "code.forgejo.org/forgejo/runner/v11/internal/pkg/labels" + "code.forgejo.org/forgejo/runner/v11/testutils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +func TestRunDaemonGracefulShutdown(t *testing.T) { + // Key assertions for graceful shutdown test: + // + // - ctx passed to createRunner, createPoller, and Shutdown must outlive signalContext passed to runDaemon, allowing + // the poller to operate without errors after termination signal is received: #1 + // + // - When shutting down, the order of operations should be: close signalContext, which causes Shutdown mock to be + // invoked, and Shutdown mock causes the Poll method to be stopped: #2 + + mockClient := mock_client.NewClient(t) + mockRunner := mock_runner.NewRunnerInterface(t) + mockPoller := mock_poller.NewPoller(t) + + defer testutils.MockVariable(&initializeConfig, func(configFile *string) (*config.Config, error) { + return &config.Config{ + Runner: config.Runner{ + // Default ShutdownTimeout of 0s won't work for the graceful shutdown test. + ShutdownTimeout: 30 * time.Second, + }, + }, nil + })() + defer testutils.MockVariable(&initLogging, func(cfg *config.Config) {})() + defer testutils.MockVariable(&loadRegistration, func(cfg *config.Config) (*config.Registration, error) { + return &config.Registration{}, nil + })() + defer testutils.MockVariable(&extractLabels, func(cfg *config.Config, reg *config.Registration) labels.Labels { + return labels.Labels{} + })() + defer testutils.MockVariable(&configCheck, func(ctx context.Context, cfg *config.Config, ls labels.Labels) error { + return nil + })() + defer testutils.MockVariable(&createClient, func(cfg *config.Config, reg *config.Registration) client.Client { + return mockClient + })() + var runnerContext context.Context + defer testutils.MockVariable(&createRunner, func(ctx context.Context, cfg *config.Config, reg *config.Registration, cli client.Client, ls labels.Labels) (run.RunnerInterface, string, error) { + runnerContext = ctx + return mockRunner, "runner", nil + })() + var pollerContext context.Context + defer testutils.MockVariable(&createPoller, func(ctx context.Context, cfg *config.Config, cli client.Client, runner run.RunnerInterface) poll.Poller { + pollerContext = ctx + return mockPoller + })() + + pollBegunChannel := make(chan interface{}) + shutdownChannel := make(chan interface{}) + mockPoller.On("Poll").Run(func(args mock.Arguments) { + close(pollBegunChannel) + // Simulate running the poll by waiting and doing nothing until shutdownChannel says Shutdown was invoked + require.NotNil(t, pollerContext) + select { + case <-pollerContext.Done(): + assert.Fail(t, "pollerContext was closed before shutdownChannel") // #1 + return + case <-shutdownChannel: + return + } + }) + mockPoller.On("Shutdown", mock.Anything).Run(func(args mock.Arguments) { + shutdownContext := args.Get(0).(context.Context) + select { + case <-shutdownContext.Done(): + assert.Fail(t, "shutdownContext was closed, but was expected to be open") // #1 + return + case <-runnerContext.Done(): + assert.Fail(t, "runnerContext was closed, but was expected to be open") // #1 + return + case <-time.After(time.Microsecond): + close(shutdownChannel) + return + } + }).Return(nil) + + // When runDaemon is begun, it will run "forever" until the passed-in context is done. So, let's start that in a goroutine... + mockSignalContext, cancelSignal := context.WithCancel(t.Context()) + runDaemonComplete := make(chan interface{}) + go func() { + configFile := "config.yaml" + err := runDaemon(mockSignalContext, &configFile) + close(runDaemonComplete) + require.NoError(t, err) + }() + + // Wait until runDaemon reaches poller.Poll(), where we expect graceful shutdown to trigger + <-pollBegunChannel + + // Now we'll signal to the daemon to begin graceful shutdown; this begins the events described in #2 + cancelSignal() + + // Wait for the daemon goroutine to stop + <-runDaemonComplete +} diff --git a/internal/app/cmd/exec.go b/internal/app/cmd/exec.go index 8c197711..f88fe327 100644 --- a/internal/app/cmd/exec.go +++ b/internal/app/cmd/exec.go @@ -7,16 +7,17 @@ package cmd import ( "context" "fmt" + "maps" "os" "path/filepath" "strconv" "strings" "time" - "code.forgejo.org/forgejo/runner/v9/act/artifactcache" - "code.forgejo.org/forgejo/runner/v9/act/common" - "code.forgejo.org/forgejo/runner/v9/act/model" - "code.forgejo.org/forgejo/runner/v9/act/runner" + "code.forgejo.org/forgejo/runner/v11/act/artifactcache" + "code.forgejo.org/forgejo/runner/v11/act/common" + "code.forgejo.org/forgejo/runner/v11/act/model" + "code.forgejo.org/forgejo/runner/v11/act/runner" "github.com/docker/docker/api/types/container" "github.com/joho/godotenv" log "github.com/sirupsen/logrus" @@ -53,7 +54,7 @@ type executeArgs struct { debug bool dryrun bool image string - cacheHandler *artifactcache.Handler + cacheHandler artifactcache.Handler network string enableIPv6 bool githubInstance string @@ -101,9 +102,7 @@ func readEnvs(path string, envs map[string]string) bool { if err != nil { log.Fatalf("Error loading from %s: %v", path, err) } - for k, v := range env { - envs[k] = v - } + maps.Copy(envs, env) return true } return false @@ -372,6 +371,10 @@ func runExec(ctx context.Context, execArgs *executeArgs) func(cmd *cobra.Command log.Infof("cache handler listens on: %v", handler.ExternalURL()) execArgs.cacheHandler = handler + if execArgs.containerDaemonSocket != "/var/run/docker.sock" { + log.Warnf("--container-daemon-socket %s: please use the DOCKER_HOST environment variable as documented at https://forgejo.org/docs/next/admin/actions/runner-installation/#setting-up-the-container-environment instead. See https://code.forgejo.org/forgejo/runner/issues/577 for more information.", execArgs.containerDaemonSocket) + } + // run the plan config := &runner.Config{ Workdir: execArgs.Workdir(), @@ -394,7 +397,6 @@ func runExec(ctx context.Context, execArgs *executeArgs) func(cmd *cobra.Command ContainerCapAdd: execArgs.containerCapAdd, ContainerCapDrop: execArgs.containerCapDrop, ContainerOptions: execArgs.containerOptions, - AutoRemove: true, NoSkipCheckout: execArgs.noSkipCheckout, // PresetGitHubContext: preset, // EventJSON: string(eventJSON), @@ -464,7 +466,7 @@ func loadExecCmd(ctx context.Context) *cobra.Command { execCmd.Flags().BoolVar(&execArg.privileged, "privileged", false, "use privileged mode") execCmd.Flags().StringVar(&execArg.usernsMode, "userns", "", "user namespace to use") execCmd.PersistentFlags().StringVarP(&execArg.containerArchitecture, "container-architecture", "", "", "Architecture which should be used to run containers, e.g.: linux/amd64. If not specified, will use host default architecture. Requires Docker server API Version 1.41+. Ignored on earlier Docker server platforms.") - execCmd.PersistentFlags().StringVarP(&execArg.containerDaemonSocket, "container-daemon-socket", "", "/var/run/docker.sock", "Path to Docker daemon socket which will be mounted to containers") + execCmd.PersistentFlags().StringVarP(&execArg.containerDaemonSocket, "container-daemon-socket", "", "/var/run/docker.sock", "Please use the DOCKER_HOST environment variable as documented at https://forgejo.org/docs/next/admin/actions/runner-installation/#setting-up-the-container-environment instead.") execCmd.Flags().BoolVar(&execArg.useGitIgnore, "use-gitignore", true, "Controls whether paths specified in .gitignore should be copied into container") execCmd.Flags().StringArrayVarP(&execArg.containerCapAdd, "container-cap-add", "", []string{}, "kernel capabilities to add to the workflow containers (e.g. --container-cap-add SYS_PTRACE)") execCmd.Flags().StringArrayVarP(&execArg.containerCapDrop, "container-cap-drop", "", []string{}, "kernel capabilities to remove from the workflow containers (e.g. --container-cap-drop SYS_PTRACE)") diff --git a/internal/app/cmd/job.go b/internal/app/cmd/job.go index 3be1f635..d2e2c103 100644 --- a/internal/app/cmd/job.go +++ b/internal/app/cmd/job.go @@ -13,13 +13,13 @@ import ( log "github.com/sirupsen/logrus" "github.com/spf13/cobra" - "code.forgejo.org/forgejo/runner/v9/internal/app/job" - "code.forgejo.org/forgejo/runner/v9/internal/app/run" - "code.forgejo.org/forgejo/runner/v9/internal/pkg/client" - "code.forgejo.org/forgejo/runner/v9/internal/pkg/config" - "code.forgejo.org/forgejo/runner/v9/internal/pkg/envcheck" - "code.forgejo.org/forgejo/runner/v9/internal/pkg/labels" - "code.forgejo.org/forgejo/runner/v9/internal/pkg/ver" + "code.forgejo.org/forgejo/runner/v11/internal/app/job" + "code.forgejo.org/forgejo/runner/v11/internal/app/run" + "code.forgejo.org/forgejo/runner/v11/internal/pkg/client" + "code.forgejo.org/forgejo/runner/v11/internal/pkg/config" + "code.forgejo.org/forgejo/runner/v11/internal/pkg/envcheck" + "code.forgejo.org/forgejo/runner/v11/internal/pkg/labels" + "code.forgejo.org/forgejo/runner/v11/internal/pkg/ver" ) func runJob(ctx context.Context, configFile *string) func(cmd *cobra.Command, args []string) error { diff --git a/internal/app/cmd/main_test.go b/internal/app/cmd/main_test.go index 588490e8..c83b122c 100644 --- a/internal/app/cmd/main_test.go +++ b/internal/app/cmd/main_test.go @@ -10,7 +10,7 @@ import ( "os" "testing" - "code.forgejo.org/forgejo/runner/v9/testutils" + "code.forgejo.org/forgejo/runner/v11/testutils" "github.com/spf13/cobra" "github.com/stretchr/testify/require" diff --git a/internal/app/cmd/register.go b/internal/app/cmd/register.go index dd369381..03831822 100644 --- a/internal/app/cmd/register.go +++ b/internal/app/cmd/register.go @@ -20,10 +20,10 @@ import ( log "github.com/sirupsen/logrus" "github.com/spf13/cobra" - "code.forgejo.org/forgejo/runner/v9/internal/pkg/client" - "code.forgejo.org/forgejo/runner/v9/internal/pkg/config" - "code.forgejo.org/forgejo/runner/v9/internal/pkg/labels" - "code.forgejo.org/forgejo/runner/v9/internal/pkg/ver" + "code.forgejo.org/forgejo/runner/v11/internal/pkg/client" + "code.forgejo.org/forgejo/runner/v11/internal/pkg/config" + "code.forgejo.org/forgejo/runner/v11/internal/pkg/labels" + "code.forgejo.org/forgejo/runner/v11/internal/pkg/ver" ) // runRegister registers a runner to the server @@ -172,7 +172,7 @@ func (r *registerInputs) assignToNext(stage registerStage, value string, cfg *co case StageInputLabels: r.Labels = defaultLabels if value != "" { - r.Labels = strings.Split(value, ",") + r.Labels = commaSplit(value) } if validateLabels(r.Labels) != nil { @@ -260,7 +260,7 @@ func registerNoInteractive(ctx context.Context, configFile string, regArgs *regi regArgs.Labels = strings.TrimSpace(regArgs.Labels) // command line flag. if regArgs.Labels != "" { - inputs.Labels = strings.Split(regArgs.Labels, ",") + inputs.Labels = commaSplit(regArgs.Labels) } // specify labels in config file. if len(cfg.Runner.Labels) > 0 { @@ -276,7 +276,7 @@ func registerNoInteractive(ctx context.Context, configFile string, regArgs *regi } if err := inputs.validate(); err != nil { log.WithError(err).Errorf("Invalid input, please re-run act command.") - return nil + return err } if err := doRegister(ctx, cfg, inputs); err != nil { return fmt.Errorf("Failed to register runner: %w", err) @@ -353,3 +353,24 @@ func doRegister(ctx context.Context, cfg *config.Config, inputs *registerInputs) } return nil } + +// Splits a string by commas, but trims whitespace around the commas. Used for label parsing to avoid a rogue whitespace +// like "docker:docker://node:22-bookworm, self-hosted:host, lxc:lxc://debian:bookworm" becoming labeled with +// " self-hosted" and " lxc". +func commaSplit(s string) []string { + if s == "" { + return make([]string, 0) + } + + parts := strings.Split(s, ",") + result := make([]string, 0, len(parts)) + + for _, part := range parts { + trimmed := strings.TrimSpace(part) + if trimmed != "" { + result = append(result, trimmed) + } + } + + return result +} diff --git a/internal/app/cmd/register_test.go b/internal/app/cmd/register_test.go new file mode 100644 index 00000000..1283db97 --- /dev/null +++ b/internal/app/cmd/register_test.go @@ -0,0 +1,42 @@ +// Copyright 2025 The Forgejo Authors +// SPDX-License-Identifier: MIT + +package cmd + +import ( + "slices" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCommaSplit(t *testing.T) { + tests := []struct { + input string + expected []string + }{ + {"", []string{}}, + {"abc", []string{"abc"}}, + {"abc,def", []string{"abc", "def"}}, + {"abc, def", []string{"abc", "def"}}, + {" abc , def ", []string{"abc", "def"}}, + {"abc, ,def", []string{"abc", "def"}}, + {" , , ", []string{}}, + } + + for _, test := range tests { + result := commaSplit(test.input) + if !slices.Equal(result, test.expected) { + t.Errorf("commaSplit(%q) = %v, want %v", test.input, result, test.expected) + } + } +} + +func TestRegisterNonInteractiveReturnsLabelValidationError(t *testing.T) { + err := registerNoInteractive(t.Context(), "", ®isterArgs{ + Labels: "label:invalid", + Token: "token", + InstanceAddr: "http://localhost:3000", + }) + assert.Error(t, err, "unsupported schema: invalid") +} diff --git a/internal/app/cmd/testdata/validate/bad-directory/.forgejo/workflows/workflow1.yml b/internal/app/cmd/testdata/validate/bad-directory/.forgejo/workflows/workflow1.yml new file mode 100644 index 00000000..802c1a24 --- /dev/null +++ b/internal/app/cmd/testdata/validate/bad-directory/.forgejo/workflows/workflow1.yml @@ -0,0 +1,6 @@ +on: [push] +jobs: + test: + ruins-on: docker + steps: + - run: echo All good! diff --git a/internal/app/cmd/testdata/validate/bad-directory/action.yml b/internal/app/cmd/testdata/validate/bad-directory/action.yml new file mode 100644 index 00000000..6f70f275 --- /dev/null +++ b/internal/app/cmd/testdata/validate/bad-directory/action.yml @@ -0,0 +1,67 @@ +name: 'Forgejo release download and upload' +author: 'Forgejo authors' +description: | + Upload or download the assets of a release to a Forgejo instance. +inputs: + badinput: scalarinsteadofmap + url: + description: 'URL of the Forgejo instance' + default: '${{ env.FORGEJO_SERVER_URL }}' + repo: + description: 'owner/project relative to the URL' + default: '${{ forge.repository }}' + tag: + description: 'Tag of the release' + default: '${{ forge.ref_name }}' + title: + description: 'Title of the release (defaults to tag)' + sha: + description: 'SHA of the release' + default: '${{ forge.sha }}' + token: + description: 'Forgejo application token' + default: '${{ forge.token }}' + release-dir: + description: 'Directory in whichs release assets are uploaded or downloaded' + required: true + release-notes: + description: 'Release notes' + direction: + description: 'Can either be `download` or `upload`' + required: true + gpg-private-key: + description: 'GPG Private Key to sign the release artifacts' + gpg-passphrase: + description: 'Passphrase of the GPG Private Key' + download-retry: + description: 'Number of times to retry if the release is not ready (default 1)' + download-latest: + description: 'Download the latest release' + default: false + verbose: + description: 'Increase the verbosity level' + default: false + override: + description: 'Override an existing release by the same `{tag}`' + default: false + prerelease: + description: 'Mark Release as Pre-Release' + default: false + release-notes-assistant: + description: 'Generate release notes with Release Notes Assistant' + default: false + hide-archive-link: + description: 'Hide the archive links' + default: false + +runs: + using: "composite" + steps: + - if: ${{ inputs.release-notes-assistant }} + uses: https://data.forgejo.org/actions/cache@v4 + with: + key: rna-${{ inputs.repo }} + path: ${{ forge.action_path }}/rna + + - run: echo "${{ forge.action_path }}" >> $FORGEJO_PATH + shell: bash diff --git a/internal/app/cmd/testdata/validate/good-directory/.forgejo/workflows/action.yml b/internal/app/cmd/testdata/validate/good-directory/.forgejo/workflows/action.yml new file mode 100644 index 00000000..efdc13fc --- /dev/null +++ b/internal/app/cmd/testdata/validate/good-directory/.forgejo/workflows/action.yml @@ -0,0 +1,6 @@ +on: [push] +jobs: + test: + runs-on: docker + steps: + - run: echo All good! diff --git a/internal/app/cmd/testdata/validate/good-directory/.forgejo/workflows/workflow1.yml b/internal/app/cmd/testdata/validate/good-directory/.forgejo/workflows/workflow1.yml new file mode 100644 index 00000000..efdc13fc --- /dev/null +++ b/internal/app/cmd/testdata/validate/good-directory/.forgejo/workflows/workflow1.yml @@ -0,0 +1,6 @@ +on: [push] +jobs: + test: + runs-on: docker + steps: + - run: echo All good! diff --git a/internal/app/cmd/testdata/validate/good-directory/.forgejo/workflows/workflow2.yaml b/internal/app/cmd/testdata/validate/good-directory/.forgejo/workflows/workflow2.yaml new file mode 100644 index 00000000..efdc13fc --- /dev/null +++ b/internal/app/cmd/testdata/validate/good-directory/.forgejo/workflows/workflow2.yaml @@ -0,0 +1,6 @@ +on: [push] +jobs: + test: + runs-on: docker + steps: + - run: echo All good! diff --git a/internal/app/cmd/testdata/validate/good-directory/.gitea/workflows/bad.yml b/internal/app/cmd/testdata/validate/good-directory/.gitea/workflows/bad.yml new file mode 100644 index 00000000..802c1a24 --- /dev/null +++ b/internal/app/cmd/testdata/validate/good-directory/.gitea/workflows/bad.yml @@ -0,0 +1,6 @@ +on: [push] +jobs: + test: + ruins-on: docker + steps: + - run: echo All good! diff --git a/internal/app/cmd/testdata/validate/good-directory/.github/workflows/bad.yml b/internal/app/cmd/testdata/validate/good-directory/.github/workflows/bad.yml new file mode 100644 index 00000000..802c1a24 --- /dev/null +++ b/internal/app/cmd/testdata/validate/good-directory/.github/workflows/bad.yml @@ -0,0 +1,6 @@ +on: [push] +jobs: + test: + ruins-on: docker + steps: + - run: echo All good! diff --git a/internal/app/cmd/testdata/validate/good-directory/action.yml b/internal/app/cmd/testdata/validate/good-directory/action.yml new file mode 100644 index 00000000..cb66d230 --- /dev/null +++ b/internal/app/cmd/testdata/validate/good-directory/action.yml @@ -0,0 +1,67 @@ +# SPDX-License-Identifier: MIT +name: 'Forgejo release download and upload' +author: 'Forgejo authors' +description: | + Upload or download the assets of a release to a Forgejo instance. +inputs: + url: + description: 'URL of the Forgejo instance' + default: '${{ env.FORGEJO_SERVER_URL }}' + repo: + description: 'owner/project relative to the URL' + default: '${{ forge.repository }}' + tag: + description: 'Tag of the release' + default: '${{ forge.ref_name }}' + title: + description: 'Title of the release (defaults to tag)' + sha: + description: 'SHA of the release' + default: '${{ forge.sha }}' + token: + description: 'Forgejo application token' + default: '${{ forge.token }}' + release-dir: + description: 'Directory in whichs release assets are uploaded or downloaded' + required: true + release-notes: + description: 'Release notes' + direction: + description: 'Can either be `download` or `upload`' + required: true + gpg-private-key: + description: 'GPG Private Key to sign the release artifacts' + gpg-passphrase: + description: 'Passphrase of the GPG Private Key' + download-retry: + description: 'Number of times to retry if the release is not ready (default 1)' + download-latest: + description: 'Download the latest release' + default: false + verbose: + description: 'Increase the verbosity level' + default: false + override: + description: 'Override an existing release by the same `{tag}`' + default: false + prerelease: + description: 'Mark Release as Pre-Release' + default: false + release-notes-assistant: + description: 'Generate release notes with Release Notes Assistant' + default: false + hide-archive-link: + description: 'Hide the archive links' + default: false + +runs: + using: "composite" + steps: + - if: ${{ inputs.release-notes-assistant }} + uses: https://data.forgejo.org/actions/cache@v4 + with: + key: rna-${{ inputs.repo }} + path: ${{ forge.action_path }}/rna + + - run: echo "${{ forge.action_path }}" >> $FORGEJO_PATH + shell: bash diff --git a/internal/app/cmd/testdata/validate/good-directory/subaction/action.yaml b/internal/app/cmd/testdata/validate/good-directory/subaction/action.yaml new file mode 100644 index 00000000..cb66d230 --- /dev/null +++ b/internal/app/cmd/testdata/validate/good-directory/subaction/action.yaml @@ -0,0 +1,67 @@ +# SPDX-License-Identifier: MIT +name: 'Forgejo release download and upload' +author: 'Forgejo authors' +description: | + Upload or download the assets of a release to a Forgejo instance. +inputs: + url: + description: 'URL of the Forgejo instance' + default: '${{ env.FORGEJO_SERVER_URL }}' + repo: + description: 'owner/project relative to the URL' + default: '${{ forge.repository }}' + tag: + description: 'Tag of the release' + default: '${{ forge.ref_name }}' + title: + description: 'Title of the release (defaults to tag)' + sha: + description: 'SHA of the release' + default: '${{ forge.sha }}' + token: + description: 'Forgejo application token' + default: '${{ forge.token }}' + release-dir: + description: 'Directory in whichs release assets are uploaded or downloaded' + required: true + release-notes: + description: 'Release notes' + direction: + description: 'Can either be `download` or `upload`' + required: true + gpg-private-key: + description: 'GPG Private Key to sign the release artifacts' + gpg-passphrase: + description: 'Passphrase of the GPG Private Key' + download-retry: + description: 'Number of times to retry if the release is not ready (default 1)' + download-latest: + description: 'Download the latest release' + default: false + verbose: + description: 'Increase the verbosity level' + default: false + override: + description: 'Override an existing release by the same `{tag}`' + default: false + prerelease: + description: 'Mark Release as Pre-Release' + default: false + release-notes-assistant: + description: 'Generate release notes with Release Notes Assistant' + default: false + hide-archive-link: + description: 'Hide the archive links' + default: false + +runs: + using: "composite" + steps: + - if: ${{ inputs.release-notes-assistant }} + uses: https://data.forgejo.org/actions/cache@v4 + with: + key: rna-${{ inputs.repo }} + path: ${{ forge.action_path }}/rna + + - run: echo "${{ forge.action_path }}" >> $FORGEJO_PATH + shell: bash diff --git a/internal/app/cmd/testdata/validate/make-repositories.sh b/internal/app/cmd/testdata/validate/make-repositories.sh index 6f3918ee..0047cb4c 100755 --- a/internal/app/cmd/testdata/validate/make-repositories.sh +++ b/internal/app/cmd/testdata/validate/make-repositories.sh @@ -35,6 +35,10 @@ git clone --bare $tmpdir/good good-repository rm -fr good-repository/hooks touch good-repository/refs/placeholder +rm -fr good-directory +git clone $tmpdir/good good-directory +rm -fr good-directory/.git + # bad mkdir $tmpdir/bad @@ -54,3 +58,7 @@ rm -fr bad-repository git clone --bare $tmpdir/bad bad-repository rm -fr bad-repository/hooks touch bad-repository/refs/placeholder + +rm -fr bad-directory +git clone $tmpdir/bad bad-directory +rm -fr bad-directory/.git diff --git a/internal/app/cmd/validate.go b/internal/app/cmd/validate.go index 3285ff6f..564922cc 100644 --- a/internal/app/cmd/validate.go +++ b/internal/app/cmd/validate.go @@ -13,8 +13,8 @@ import ( "path/filepath" "strings" - "code.forgejo.org/forgejo/runner/v9/act/model" - "code.forgejo.org/forgejo/runner/v9/testutils" + "code.forgejo.org/forgejo/runner/v11/act/model" + "code.forgejo.org/forgejo/runner/v11/testutils" "github.com/spf13/cobra" ) @@ -23,6 +23,7 @@ type validateArgs struct { path string repository string clonedir string + directory string workflow bool action bool } @@ -49,12 +50,13 @@ func validate(dir, path string, isWorkflow, isAction bool) error { kind = "action" } if err != nil { - fmt.Printf("%s %s schema validation failed:\n%s\n", shortPath, kind, err.Error()) + err = fmt.Errorf("%s %s schema validation failed:\n%s", shortPath, kind, err.Error()) + fmt.Printf("%s\n", err.Error()) } else { fmt.Printf("%s %s schema validation OK\n", shortPath, kind) } - return nil + return err } func validatePath(validateArgs *validateArgs) error { @@ -64,8 +66,17 @@ func validatePath(validateArgs *validateArgs) error { return validate("", validateArgs.path, validateArgs.workflow, validateArgs.action) } -func validateHasYamlSuffix(s, suffix string) bool { - return strings.HasSuffix(s, suffix+".yml") || strings.HasSuffix(s, suffix+".yaml") +func validatePathMatch(existing, search string) bool { + if !validateHasYamlSuffix(existing) { + return false + } + existing = strings.TrimSuffix(existing, ".yml") + existing = strings.TrimSuffix(existing, ".yaml") + return existing == search || strings.HasSuffix(existing, "/"+search) +} + +func validateHasYamlSuffix(s string) bool { + return strings.HasSuffix(s, ".yml") || strings.HasSuffix(s, ".yaml") } func validateRepository(validateArgs *validateArgs) error { @@ -105,15 +116,17 @@ func validateRepository(validateArgs *validateArgs) error { } } + var validationErrors error + if err := filepath.Walk(clonedir, func(path string, fi fs.FileInfo, err error) error { - if validateHasYamlSuffix(path, "/.forgejo/workflows/action") { + if validatePathMatch(path, ".forgejo/workflows/action") { return nil } isWorkflow := false isAction := true - if validateHasYamlSuffix(path, "/action") { + if validatePathMatch(path, "action") { if err := validate(clonedir, path, isWorkflow, isAction); err != nil { - return err + validationErrors = errors.Join(validationErrors, err) } } return nil @@ -131,9 +144,9 @@ func validateRepository(validateArgs *validateArgs) error { if err := filepath.Walk(workflowdir, func(path string, fi fs.FileInfo, err error) error { isWorkflow := true isAction := false - if validateHasYamlSuffix(path, "") { + if validateHasYamlSuffix(path) { if err := validate(clonedir, path, isWorkflow, isAction); err != nil { - return err + validationErrors = errors.Join(validationErrors, err) } } return nil @@ -142,11 +155,19 @@ func validateRepository(validateArgs *validateArgs) error { } } - return nil + return validationErrors +} + +func processDirectory(validateArgs *validateArgs) { + if len(validateArgs.directory) > 0 { + validateArgs.repository = validateArgs.directory + validateArgs.clonedir = validateArgs.directory + } } func runValidate(_ context.Context, validateArgs *validateArgs) func(cmd *cobra.Command, args []string) error { return func(cmd *cobra.Command, args []string) error { + processDirectory(validateArgs) if len(validateArgs.path) > 0 { return validatePath(validateArgs) } else if len(validateArgs.repository) > 0 { @@ -168,9 +189,15 @@ Validate workflows or actions with a schema verifying they are conformant. The --path argument is a filename that will be validated as a workflow (if the --workflow flag is set) or as an action (if the --action flag is set). -The --repository argument is a URL to a Git repository. It will be -cloned (in the --clonedir directory or a temporary location removed -when the validation completes). The following files will be validated: +The --repository argument is a URL to a Git repository that contains +workflows or actions. It will be cloned (in the --clonedir directory +or a temporary location removed when the validation completes). + +The --directory argument is the path a repository to be explored for +files to validate. + +The following files will be validated when exploring the clone of a repository +(--repository) or a directory (--directory): - All .forgejo/workflows/*.{yml,yaml} files as workflows - All **/action.{yml,yaml} files as actions @@ -185,9 +212,11 @@ when the validation completes). The following files will be validated: validateCmd.Flags().StringVar(&validateArgs.clonedir, "clonedir", "", "directory in which the repository will be cloned") validateCmd.Flags().StringVar(&validateArgs.repository, "repository", "", "URL to a repository to validate") + validateCmd.Flags().StringVar(&validateArgs.directory, "directory", "", "directory to a repository to validate") validateCmd.Flags().StringVar(&validateArgs.path, "path", "", "path to the file") - validateCmd.MarkFlagsOneRequired("repository", "path") - validateCmd.MarkFlagsMutuallyExclusive("repository", "path") + validateCmd.MarkFlagsOneRequired("repository", "path", "directory") + validateCmd.MarkFlagsMutuallyExclusive("repository", "path", "directory") + validateCmd.MarkFlagsMutuallyExclusive("directory", "clonedir") return validateCmd } diff --git a/internal/app/cmd/validate_test.go b/internal/app/cmd/validate_test.go index 7442d1d2..9b132d9f 100644 --- a/internal/app/cmd/validate_test.go +++ b/internal/app/cmd/validate_test.go @@ -10,6 +10,15 @@ import ( "github.com/stretchr/testify/assert" ) +func Test_validatePathMatch(t *testing.T) { + assert.False(t, validatePathMatch("nosuffix", "nosuffix")) + assert.True(t, validatePathMatch("something.yml", "something")) + assert.True(t, validatePathMatch("something.yaml", "something")) + assert.False(t, validatePathMatch("entire_something.yaml", "something")) + assert.True(t, validatePathMatch("nested/in/directory/something.yaml", "something")) + assert.False(t, validatePathMatch("nested/in/directory/entire_something.yaml", "something")) +} + func Test_validateCmd(t *testing.T) { ctx := context.Background() for _, testCase := range []struct { @@ -27,19 +36,30 @@ func Test_validateCmd(t *testing.T) { message: "one of --workflow or --action must be set", }, { - name: "MutuallyExclusive", + name: "MutuallyExclusiveActionWorkflow", args: []string{"--action", "--workflow", "--path", "/tmp"}, message: "[action workflow] were all set", }, + { + name: "MutuallyExclusiveRepositoryDirectory", + args: []string{"--repository", "example.com", "--directory", "."}, + message: "[directory repository] were all set", + }, + { + name: "MutuallyExclusiveClonedirDirectory", + args: []string{"--clonedir", ".", "--directory", "."}, + message: "[clonedir directory] were all set", + }, { name: "PathActionOK", args: []string{"--action", "--path", "testdata/validate/good-action.yml"}, stdOut: "schema validation OK", }, { - name: "PathActionNOK", - args: []string{"--action", "--path", "testdata/validate/bad-action.yml"}, - stdOut: "Expected a mapping got scalar", + name: "PathActionNOK", + args: []string{"--action", "--path", "testdata/validate/bad-action.yml"}, + stdOut: "Expected a mapping got scalar", + message: "testdata/validate/bad-action.yml action schema validation failed", }, { name: "PathWorkflowOK", @@ -47,9 +67,27 @@ func Test_validateCmd(t *testing.T) { stdOut: "schema validation OK", }, { - name: "PathWorkflowNOK", - args: []string{"--workflow", "--path", "testdata/validate/bad-workflow.yml"}, - stdOut: "Unknown Property ruins-on", + name: "PathWorkflowNOK", + args: []string{"--workflow", "--path", "testdata/validate/bad-workflow.yml"}, + stdOut: "Unknown Property ruins-on", + message: "testdata/validate/bad-workflow.yml workflow schema validation failed", + }, + { + name: "DirectoryOK", + args: []string{"--directory", "testdata/validate/good-directory"}, + stdOut: "action.yml action schema validation OK\nsubaction/action.yaml action schema validation OK\n.forgejo/workflows/action.yml workflow schema validation OK\n.forgejo/workflows/workflow1.yml workflow schema validation OK\n.forgejo/workflows/workflow2.yaml workflow schema validation OK", + }, + { + name: "DirectoryActionNOK", + args: []string{"--directory", "testdata/validate/bad-directory"}, + stdOut: "action.yml action schema validation failed", + message: "action.yml action schema validation failed", + }, + { + name: "DirectoryWorkflowNOK", + args: []string{"--directory", "testdata/validate/bad-directory"}, + stdOut: ".forgejo/workflows/workflow1.yml workflow schema validation failed", + message: ".forgejo/workflows/workflow1.yml workflow schema validation failed", }, { name: "RepositoryOK", @@ -57,14 +95,16 @@ func Test_validateCmd(t *testing.T) { stdOut: "action.yml action schema validation OK\nsubaction/action.yaml action schema validation OK\n.forgejo/workflows/action.yml workflow schema validation OK\n.forgejo/workflows/workflow1.yml workflow schema validation OK\n.forgejo/workflows/workflow2.yaml workflow schema validation OK", }, { - name: "RepositoryActionNOK", - args: []string{"--repository", "testdata/validate/bad-repository"}, - stdOut: "action.yml action schema validation failed", + name: "RepositoryActionNOK", + args: []string{"--repository", "testdata/validate/bad-repository"}, + stdOut: "action.yml action schema validation failed", + message: "action.yml action schema validation failed", }, { - name: "RepositoryWorkflowNOK", - args: []string{"--repository", "testdata/validate/bad-repository"}, - stdOut: ".forgejo/workflows/workflow1.yml workflow schema validation failed", + name: "RepositoryWorkflowNOK", + args: []string{"--repository", "testdata/validate/bad-repository"}, + stdOut: ".forgejo/workflows/workflow1.yml workflow schema validation failed", + message: ".forgejo/workflows/workflow1.yml workflow schema validation failed", }, } { t.Run(testCase.name, func(t *testing.T) { diff --git a/internal/app/job/job.go b/internal/app/job/job.go index 2dd06852..654bc154 100644 --- a/internal/app/job/job.go +++ b/internal/app/job/job.go @@ -13,9 +13,9 @@ import ( log "github.com/sirupsen/logrus" runnerv1 "code.forgejo.org/forgejo/actions-proto/runner/v1" - "code.forgejo.org/forgejo/runner/v9/internal/app/run" - "code.forgejo.org/forgejo/runner/v9/internal/pkg/client" - "code.forgejo.org/forgejo/runner/v9/internal/pkg/config" + "code.forgejo.org/forgejo/runner/v11/internal/app/run" + "code.forgejo.org/forgejo/runner/v11/internal/pkg/client" + "code.forgejo.org/forgejo/runner/v11/internal/pkg/config" ) type Job struct { @@ -36,11 +36,20 @@ func NewJob(cfg *config.Config, client client.Client, runner run.RunnerInterface } func (j *Job) Run(ctx context.Context) error { - task, ok := j.fetchTask(ctx) - if !ok { - return fmt.Errorf("could not fetch task") + log.Info("Polling for a job...") + for { + task, ok := j.fetchTask(ctx) + if ok { + return j.runTaskWithRecover(ctx, task) + } + // No task available, continue polling + select { + case <-ctx.Done(): + return ctx.Err() + default: + // Continue to next iteration + } } - return j.runTaskWithRecover(ctx, task) } func (j *Job) runTaskWithRecover(ctx context.Context, task *runnerv1.Task) error { diff --git a/internal/app/job/job_test.go b/internal/app/job/job_test.go index 9799425b..225d744c 100644 --- a/internal/app/job/job_test.go +++ b/internal/app/job/job_test.go @@ -11,7 +11,7 @@ import ( "code.forgejo.org/forgejo/actions-proto/ping/v1/pingv1connect" runnerv1 "code.forgejo.org/forgejo/actions-proto/runner/v1" "code.forgejo.org/forgejo/actions-proto/runner/v1/runnerv1connect" - "code.forgejo.org/forgejo/runner/v9/internal/pkg/config" + "code.forgejo.org/forgejo/runner/v11/internal/pkg/config" log "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" diff --git a/internal/app/poll/mocks/Poller.go b/internal/app/poll/mocks/Poller.go new file mode 100644 index 00000000..9d2b1e4d --- /dev/null +++ b/internal/app/poll/mocks/Poller.go @@ -0,0 +1,52 @@ +// Code generated by mockery v2.53.5. DO NOT EDIT. + +package mocks + +import ( + context "context" + + mock "github.com/stretchr/testify/mock" +) + +// Poller is an autogenerated mock type for the Poller type +type Poller struct { + mock.Mock +} + +// Poll provides a mock function with no fields +func (_m *Poller) Poll() { + _m.Called() +} + +// Shutdown provides a mock function with given fields: ctx +func (_m *Poller) Shutdown(ctx context.Context) error { + ret := _m.Called(ctx) + + if len(ret) == 0 { + panic("no return value specified for Shutdown") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context) error); ok { + r0 = rf(ctx) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// NewPoller creates a new instance of Poller. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewPoller(t interface { + mock.TestingT + Cleanup(func()) +}, +) *Poller { + mock := &Poller{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/internal/app/poll/poller.go b/internal/app/poll/poller.go index 4ebe4405..c880bc67 100644 --- a/internal/app/poll/poller.go +++ b/internal/app/poll/poller.go @@ -15,13 +15,14 @@ import ( log "github.com/sirupsen/logrus" "golang.org/x/time/rate" - "code.forgejo.org/forgejo/runner/v9/internal/app/run" - "code.forgejo.org/forgejo/runner/v9/internal/pkg/client" - "code.forgejo.org/forgejo/runner/v9/internal/pkg/config" + "code.forgejo.org/forgejo/runner/v11/internal/app/run" + "code.forgejo.org/forgejo/runner/v11/internal/pkg/client" + "code.forgejo.org/forgejo/runner/v11/internal/pkg/config" ) const PollerID = "PollerID" +//go:generate mockery --name Poller type Poller interface { Poll() Shutdown(ctx context.Context) error @@ -42,14 +43,14 @@ type poller struct { done chan any } -func New(cfg *config.Config, client client.Client, runner run.RunnerInterface) Poller { - return (&poller{}).init(cfg, client, runner) +func New(ctx context.Context, cfg *config.Config, client client.Client, runner run.RunnerInterface) Poller { + return (&poller{}).init(ctx, cfg, client, runner) } -func (p *poller) init(cfg *config.Config, client client.Client, runner run.RunnerInterface) Poller { - pollingCtx, shutdownPolling := context.WithCancel(context.Background()) +func (p *poller) init(ctx context.Context, cfg *config.Config, client client.Client, runner run.RunnerInterface) Poller { + pollingCtx, shutdownPolling := context.WithCancel(ctx) - jobsCtx, shutdownJobs := context.WithCancel(context.Background()) + jobsCtx, shutdownJobs := context.WithCancel(ctx) done := make(chan any) @@ -89,10 +90,10 @@ func (p *poller) Shutdown(ctx context.Context) error { return nil case <-ctx.Done(): - log.Trace("forcing the jobs to shutdown") + log.Info("forcing the jobs to shutdown") p.shutdownJobs() <-p.done - log.Trace("all jobs have been shutdown") + log.Info("all jobs have been shutdown") return ctx.Err() } } diff --git a/internal/app/poll/poller_test.go b/internal/app/poll/poller_test.go index 0c5febfe..e3b23eb3 100644 --- a/internal/app/poll/poller_test.go +++ b/internal/app/poll/poller_test.go @@ -14,7 +14,7 @@ import ( "code.forgejo.org/forgejo/actions-proto/ping/v1/pingv1connect" runnerv1 "code.forgejo.org/forgejo/actions-proto/runner/v1" "code.forgejo.org/forgejo/actions-proto/runner/v1/runnerv1connect" - "code.forgejo.org/forgejo/runner/v9/internal/pkg/config" + "code.forgejo.org/forgejo/runner/v11/internal/pkg/config" log "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" @@ -118,7 +118,7 @@ func setTrace(t *testing.T) { } func TestPoller_New(t *testing.T) { - p := New(&config.Config{}, &mockClient{}, &mockRunner{}) + p := New(t.Context(), &config.Config{}, &mockClient{}, &mockRunner{}) assert.NotNil(t, p) } @@ -172,6 +172,7 @@ func TestPoller_Runner(t *testing.T) { } p := &mockPoller{} p.init( + t.Context(), &config.Config{ Runner: configRunner, }, @@ -239,6 +240,7 @@ func TestPoller_Fetch(t *testing.T) { } p := &mockPoller{} p.init( + t.Context(), &config.Config{ Runner: configRunner, }, diff --git a/internal/app/run/mocks/RunnerInterface.go b/internal/app/run/mocks/RunnerInterface.go new file mode 100644 index 00000000..16dbfecb --- /dev/null +++ b/internal/app/run/mocks/RunnerInterface.go @@ -0,0 +1,49 @@ +// Code generated by mockery v2.53.5. DO NOT EDIT. + +package mocks + +import ( + context "context" + + mock "github.com/stretchr/testify/mock" + + runnerv1 "code.forgejo.org/forgejo/actions-proto/runner/v1" +) + +// RunnerInterface is an autogenerated mock type for the RunnerInterface type +type RunnerInterface struct { + mock.Mock +} + +// Run provides a mock function with given fields: ctx, task +func (_m *RunnerInterface) Run(ctx context.Context, task *runnerv1.Task) error { + ret := _m.Called(ctx, task) + + if len(ret) == 0 { + panic("no return value specified for Run") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, *runnerv1.Task) error); ok { + r0 = rf(ctx, task) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// NewRunnerInterface creates a new instance of RunnerInterface. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewRunnerInterface(t interface { + mock.TestingT + Cleanup(func()) +}, +) *RunnerInterface { + mock := &RunnerInterface{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/internal/app/run/runner.go b/internal/app/run/runner.go index e8d91822..c709b462 100644 --- a/internal/app/run/runner.go +++ b/internal/app/run/runner.go @@ -9,6 +9,7 @@ import ( "encoding/hex" "encoding/json" "fmt" + "maps" "path/filepath" "strconv" "strings" @@ -16,20 +17,20 @@ import ( "time" runnerv1 "code.forgejo.org/forgejo/actions-proto/runner/v1" - "code.forgejo.org/forgejo/runner/v9/act/artifactcache" - "code.forgejo.org/forgejo/runner/v9/act/cacheproxy" - "code.forgejo.org/forgejo/runner/v9/act/common" - "code.forgejo.org/forgejo/runner/v9/act/model" - "code.forgejo.org/forgejo/runner/v9/act/runner" + "code.forgejo.org/forgejo/runner/v11/act/artifactcache" + "code.forgejo.org/forgejo/runner/v11/act/cacheproxy" + "code.forgejo.org/forgejo/runner/v11/act/common" + "code.forgejo.org/forgejo/runner/v11/act/model" + "code.forgejo.org/forgejo/runner/v11/act/runner" "connectrpc.com/connect" "github.com/docker/docker/api/types/container" log "github.com/sirupsen/logrus" - "code.forgejo.org/forgejo/runner/v9/internal/pkg/client" - "code.forgejo.org/forgejo/runner/v9/internal/pkg/config" - "code.forgejo.org/forgejo/runner/v9/internal/pkg/labels" - "code.forgejo.org/forgejo/runner/v9/internal/pkg/report" - "code.forgejo.org/forgejo/runner/v9/internal/pkg/ver" + "code.forgejo.org/forgejo/runner/v11/internal/pkg/client" + "code.forgejo.org/forgejo/runner/v11/internal/pkg/config" + "code.forgejo.org/forgejo/runner/v11/internal/pkg/labels" + "code.forgejo.org/forgejo/runner/v11/internal/pkg/report" + "code.forgejo.org/forgejo/runner/v11/internal/pkg/ver" ) // Runner runs the pipeline. @@ -47,6 +48,7 @@ type Runner struct { runningTasks sync.Map } +//go:generate mockery --name RunnerInterface type RunnerInterface interface { Run(ctx context.Context, task *runnerv1.Task) error } @@ -66,9 +68,7 @@ func NewRunner(cfg *config.Config, reg *config.Registration, cli client.Client) cfg.Runner.Envs["GITHUB_SERVER_URL"] = reg.Address envs := make(map[string]string, len(cfg.Runner.Envs)) - for k, v := range cfg.Runner.Envs { - envs[k] = v - } + maps.Copy(envs, cfg.Runner.Envs) var cacheProxy *cacheproxy.Handler if cfg.Cache.Enabled == nil || *cfg.Cache.Enabled { @@ -117,13 +117,13 @@ func setupCache(cfg *config.Config, envs map[string]string) *cacheproxy.Handler cacheServer, err := artifactcache.StartHandler( cfg.Cache.Dir, - cfg.Cache.Host, + "", // automatically detect cfg.Cache.Port, cacheSecret, log.StandardLogger().WithField("module", "cache_request"), ) if err != nil { - log.Error("Could not start the cache server, cache will be disabled") + log.Errorf("Could not start the cache server, cache will be disabled: %v", err) return nil } @@ -144,16 +144,14 @@ func setupCache(cfg *config.Config, envs map[string]string) *cacheproxy.Handler cacheURL, cfg.Cache.Host, cfg.Cache.ProxyPort, + cfg.Cache.ActionsCacheURLOverride, cacheSecret, log.StandardLogger().WithField("module", "cache_proxy"), ) if err != nil { log.Errorf("cannot init cache proxy, cache will be disabled: %v", err) - } - - envs["ACTIONS_CACHE_URL"] = cacheProxy.ExternalURL() - if cfg.Cache.ActionsCacheURLOverride != "" { - envs["ACTIONS_CACHE_URL"] = cfg.Cache.ActionsCacheURLOverride + } else { + envs["ACTIONS_CACHE_URL"] = cacheProxy.ExternalURL() } return cacheProxy @@ -171,11 +169,7 @@ func (r *Runner) Run(ctx context.Context, task *runnerv1.Task) error { reporter := report.NewReporter(ctx, cancel, r.client, task, r.cfg.Runner.ReportInterval) var runErr error defer func() { - lastWords := "" - if runErr != nil { - lastWords = runErr.Error() - } - _ = reporter.Close(lastWords) + _ = reporter.Close(runErr) }() reporter.RunDaemon() runErr = r.run(ctx, task, reporter) @@ -193,12 +187,50 @@ func explainFailedGenerateWorkflow(task *runnerv1.Task, log func(message string, log("%5d: %s", n+1, line) } log("Errors were found and although they tend to be cryptic the line number they refer to gives a hint as to where the problem might be.") - for _, line := range strings.Split(err.Error(), "\n") { + for line := range strings.SplitSeq(err.Error(), "\n") { log("%s", line) } return fmt.Errorf("the workflow file is not usable") } +func getWriteIsolationKey(ctx context.Context, eventName, ref string, event map[string]any) (string, error) { + if eventName == "pull_request" { + // The "closed" action of a pull request event runs in the context of the base repository + // and was merged by a user with write access to the base repository. It is authorized to + // write the repository cache. + if event["action"] == "closed" { + pullRequest, ok := event["pull_request"].(map[string]any) + if !ok { + return "", fmt.Errorf("getWriteIsolationKey: event.pull_request is not a map[string]any but %T", event["pull_request"]) + } + merged, ok := pullRequest["merged"].(bool) + if !ok { + return "", fmt.Errorf("getWriteIsolationKey: event.pull_request.merged is not a bool but %T", pullRequest["merged"]) + } + if merged { + return "", nil + } + // a pull request that is closed but not merged falls thru and is expected to obey the same + // constraints as an opened pull request, it may be closed by a user with no write permissions to the + // base repository + } + // When performing an action on an event from an opened PR, provide a "write isolation key" to the cache. The generated + // ACTIONS_CACHE_URL will be able to read the cache, and write to a cache, but its writes will be isolated to + // future runs of the PR's workflows and won't be shared with other pull requests or actions. This is a security + // measure to prevent a malicious pull request from poisoning the cache with secret-stealing code which would + // later be executed on another action. + // Ensure that `ref` has the expected format so that we don't end up with a useless write isolation key + if !strings.HasPrefix(ref, "refs/pull/") { + return "", fmt.Errorf("getWriteIsolationKey: expected ref to be refs/pull/..., but was %q", ref) + } + return ref, nil + } + + // Other events do not allow the trigger user to modify the content of the repository and + // are allowed to write the cache without an isolation key + return "", nil +} + func (r *Runner) run(ctx context.Context, task *runnerv1.Task, reporter *report.Reporter) (err error) { defer func() { if r := recover(); r != nil { @@ -232,15 +264,18 @@ func (r *Runner) run(ctx context.Context, task *runnerv1.Task, reporter *report. defaultActionURL, r.client.Address()) + eventName := taskContext["event_name"].GetStringValue() + ref := taskContext["ref"].GetStringValue() + event := taskContext["event"].GetStructValue().AsMap() preset := &model.GithubContext{ - Event: taskContext["event"].GetStructValue().AsMap(), + Event: event, RunID: taskContext["run_id"].GetStringValue(), RunNumber: taskContext["run_number"].GetStringValue(), Actor: taskContext["actor"].GetStringValue(), Repository: taskContext["repository"].GetStringValue(), - EventName: taskContext["event_name"].GetStringValue(), + EventName: eventName, Sha: taskContext["sha"].GetStringValue(), - Ref: taskContext["ref"].GetStringValue(), + Ref: ref, RefName: taskContext["ref_name"].GetStringValue(), RefType: taskContext["ref_type"].GetStringValue(), HeadRef: taskContext["head_ref"].GetStringValue(), @@ -259,9 +294,7 @@ func (r *Runner) run(ctx context.Context, task *runnerv1.Task, reporter *report. // Clone the runner default envs into a local envs map runEnvs := make(map[string]string) - for id, v := range r.envs { - runEnvs[id] = v - } + maps.Copy(runEnvs, r.envs) runtimeToken := client.BackwardCompatibleContext(task, "runtime_token") if runtimeToken == "" { @@ -272,8 +305,13 @@ func (r *Runner) run(ctx context.Context, task *runnerv1.Task, reporter *report. // Register the run with the cacheproxy and modify the CACHE_URL if r.cacheProxy != nil { + writeIsolationKey, err := getWriteIsolationKey(ctx, eventName, ref, event) + if err != nil { + return err + } + timestamp := strconv.FormatInt(time.Now().Unix(), 10) - cacheRunData := r.cacheProxy.CreateRunData(preset.Repository, preset.RunID, timestamp) + cacheRunData := r.cacheProxy.CreateRunData(preset.Repository, preset.RunID, timestamp, writeIsolationKey) cacheRunID, err := r.cacheProxy.AddRun(cacheRunData) if err == nil { defer func() { _ = r.cacheProxy.RemoveRun(cacheRunID) }() @@ -314,7 +352,6 @@ func (r *Runner) run(ctx context.Context, task *runnerv1.Task, reporter *report. Env: runEnvs, Secrets: task.Secrets, GitHubInstance: strings.TrimSuffix(r.client.Address(), "/"), - AutoRemove: true, NoSkipCheckout: true, PresetGitHubContext: preset, EventJSON: string(eventJSON), @@ -358,7 +395,6 @@ func (r *Runner) run(ctx context.Context, task *runnerv1.Task, reporter *report. } execErr := executor(ctx) - _ = reporter.SetOutputs(job.Outputs) return execErr } diff --git a/internal/app/run/runner_test.go b/internal/app/run/runner_test.go index a9a8445d..c46fefb4 100644 --- a/internal/app/run/runner_test.go +++ b/internal/app/run/runner_test.go @@ -4,14 +4,29 @@ import ( "context" "errors" "fmt" + "net" + "os" "testing" + "time" + pingv1 "code.forgejo.org/forgejo/actions-proto/ping/v1" runnerv1 "code.forgejo.org/forgejo/actions-proto/runner/v1" - "code.forgejo.org/forgejo/runner/v9/internal/pkg/labels" + "code.forgejo.org/forgejo/runner/v11/internal/pkg/config" + "code.forgejo.org/forgejo/runner/v11/internal/pkg/labels" + "code.forgejo.org/forgejo/runner/v11/internal/pkg/report" + "connectrpc.com/connect" + log "github.com/sirupsen/logrus" + "google.golang.org/protobuf/types/known/structpb" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" ) +func init() { + log.SetLevel(log.TraceLevel) +} + func TestExplainFailedGenerateWorkflow(t *testing.T) { logged := "" log := func(message string, args ...any) { @@ -53,3 +68,690 @@ func TestLabelUpdate(t *testing.T) { assert.Contains(t, runner.labels, initialLabel) assert.Contains(t, runner.labels, newLabel) } + +type forgejoClientMock struct { + mock.Mock + sent string +} + +func (m *forgejoClientMock) Address() string { + args := m.Called() + return args.String(0) +} + +func (m *forgejoClientMock) Insecure() bool { + args := m.Called() + return args.Bool(0) +} + +func (m *forgejoClientMock) Ping(ctx context.Context, request *connect.Request[pingv1.PingRequest]) (*connect.Response[pingv1.PingResponse], error) { + args := m.Called(ctx, request) + if args.Get(0) == nil { + return nil, args.Error(1) + } + return args.Get(0).(*connect.Response[pingv1.PingResponse]), args.Error(1) +} + +func (m *forgejoClientMock) Register(ctx context.Context, request *connect.Request[runnerv1.RegisterRequest]) (*connect.Response[runnerv1.RegisterResponse], error) { + args := m.Called(ctx, request) + if args.Get(0) == nil { + return nil, args.Error(1) + } + return args.Get(0).(*connect.Response[runnerv1.RegisterResponse]), args.Error(1) +} + +func (m *forgejoClientMock) Declare(ctx context.Context, request *connect.Request[runnerv1.DeclareRequest]) (*connect.Response[runnerv1.DeclareResponse], error) { + args := m.Called(ctx, request) + if args.Get(0) == nil { + return nil, args.Error(1) + } + return args.Get(0).(*connect.Response[runnerv1.DeclareResponse]), args.Error(1) +} + +func (m *forgejoClientMock) FetchTask(ctx context.Context, request *connect.Request[runnerv1.FetchTaskRequest]) (*connect.Response[runnerv1.FetchTaskResponse], error) { + args := m.Called(ctx, request) + if args.Get(0) == nil { + return nil, args.Error(1) + } + return args.Get(0).(*connect.Response[runnerv1.FetchTaskResponse]), args.Error(1) +} + +func (m *forgejoClientMock) UpdateTask(ctx context.Context, request *connect.Request[runnerv1.UpdateTaskRequest]) (*connect.Response[runnerv1.UpdateTaskResponse], error) { + args := m.Called(ctx, request) + if args.Get(0) == nil { + return nil, args.Error(1) + } + return args.Get(0).(*connect.Response[runnerv1.UpdateTaskResponse]), args.Error(1) +} + +func rowsToString(rows []*runnerv1.LogRow) string { + s := "" + for _, row := range rows { + s += row.Content + "\n" + } + return s +} + +func (m *forgejoClientMock) UpdateLog(ctx context.Context, request *connect.Request[runnerv1.UpdateLogRequest]) (*connect.Response[runnerv1.UpdateLogResponse], error) { + // Enable for log output from runs if needed. + // for _, row := range request.Msg.Rows { + // println(fmt.Sprintf("UpdateLog: %q", row.Content)) + // } + m.sent += rowsToString(request.Msg.Rows) + args := m.Called(ctx, request) + mockRetval := args.Get(0) + mockError := args.Error(1) + if mockRetval != nil { + return mockRetval.(*connect.Response[runnerv1.UpdateLogResponse]), mockError + } else if mockError != nil { + return nil, mockError + } + // Unless overridden by mock, default to returning indication that logs were received successfully + return connect.NewResponse(&runnerv1.UpdateLogResponse{ + AckIndex: request.Msg.Index + int64(len(request.Msg.Rows)), + }), nil +} + +func TestRunner_getWriteIsolationKey(t *testing.T) { + t.Run("push", func(t *testing.T) { + key, err := getWriteIsolationKey(t.Context(), "push", "whatever", nil) + require.NoError(t, err) + assert.Empty(t, key) + }) + + t.Run("pull_request synchronized key is ref", func(t *testing.T) { + expectedKey := "refs/pull/1/head" + actualKey, err := getWriteIsolationKey(t.Context(), "pull_request", expectedKey, map[string]any{ + "action": "synchronized", + }) + require.NoError(t, err) + assert.Equal(t, expectedKey, actualKey) + }) + + t.Run("pull_request synchronized ref is invalid", func(t *testing.T) { + invalidKey := "refs/is/invalid" + key, err := getWriteIsolationKey(t.Context(), "pull_request", invalidKey, map[string]any{ + "action": "synchronized", + }) + require.Empty(t, key) + assert.ErrorContains(t, err, invalidKey) + }) + + t.Run("pull_request closed and not merged key is ref", func(t *testing.T) { + expectedKey := "refs/pull/1/head" + actualKey, err := getWriteIsolationKey(t.Context(), "pull_request", expectedKey, map[string]any{ + "action": "closed", + "pull_request": map[string]any{ + "merged": false, + }, + }) + require.NoError(t, err) + assert.Equal(t, expectedKey, actualKey) + }) + + t.Run("pull_request closed and merged key is empty", func(t *testing.T) { + key, err := getWriteIsolationKey(t.Context(), "pull_request", "whatever", map[string]any{ + "action": "closed", + "pull_request": map[string]any{ + "merged": true, + }, + }) + require.NoError(t, err) + assert.Empty(t, key) + }) + + t.Run("pull_request missing event.pull_request", func(t *testing.T) { + key, err := getWriteIsolationKey(t.Context(), "pull_request", "whatever", map[string]any{ + "action": "closed", + }) + require.Empty(t, key) + assert.ErrorContains(t, err, "event.pull_request is not a map") + }) + + t.Run("pull_request missing event.pull_request.merge", func(t *testing.T) { + key, err := getWriteIsolationKey(t.Context(), "pull_request", "whatever", map[string]any{ + "action": "closed", + "pull_request": map[string]any{}, + }) + require.Empty(t, key) + assert.ErrorContains(t, err, "event.pull_request.merged is not a bool") + }) + + t.Run("pull_request with event.pull_request.merge of an unexpected type", func(t *testing.T) { + key, err := getWriteIsolationKey(t.Context(), "pull_request", "whatever", map[string]any{ + "action": "closed", + "pull_request": map[string]any{ + "merged": "string instead of bool", + }, + }) + require.Empty(t, key) + assert.ErrorContains(t, err, "not a bool but string") + }) +} + +func TestRunnerCacheConfiguration(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test") + } + + forgejoClient := &forgejoClientMock{} + + forgejoClient.On("Address").Return("https://127.0.0.1:8080") // not expected to be used in this test + forgejoClient.On("UpdateLog", mock.Anything, mock.Anything).Return(nil, nil) + forgejoClient.On("UpdateTask", mock.Anything, mock.Anything). + Return(connect.NewResponse(&runnerv1.UpdateTaskResponse{}), nil) + + runner := NewRunner( + &config.Config{ + Cache: config.Cache{ + // Note that this test requires that containers on the local dev environment can access localhost to + // reach the cache proxy, and that the cache proxy can access localhost to reach the cache, both of + // which are embedded servers that the Runner will start. If any specific firewall config is needed + // it's easier to do that with statically configured ports, so... + Port: 40713, + ProxyPort: 40714, + Dir: t.TempDir(), + }, + Host: config.Host{ + WorkdirParent: t.TempDir(), + }, + }, + &config.Registration{ + Labels: []string{"ubuntu-latest:docker://code.forgejo.org/oci/node:20-bookworm"}, + }, + forgejoClient) + require.NotNil(t, runner) + + // Must set up cache for our test + require.NotNil(t, runner.cacheProxy) + + runWorkflow := func(ctx context.Context, cancel context.CancelFunc, yamlContent, eventName, ref, description string) { + task := &runnerv1.Task{ + WorkflowPayload: []byte(yamlContent), + Context: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "token": structpb.NewStringValue("some token here"), + "forgejo_default_actions_url": structpb.NewStringValue("https://data.forgejo.org"), + "repository": structpb.NewStringValue("runner"), + "event_name": structpb.NewStringValue(eventName), + "ref": structpb.NewStringValue(ref), + }, + }, + } + + reporter := report.NewReporter(ctx, cancel, forgejoClient, task, time.Second) + err := runner.run(ctx, task, reporter) + reporter.Close(nil) + require.NoError(t, err, description) + } + + t.Run("Cache accessible", func(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + // Step 1: Populate shared cache with push workflow + populateYaml := ` +name: Cache Testing Action +on: + push: +jobs: + job-cache-check-1: + runs-on: ubuntu-latest + steps: + - uses: actions/cache@v4 + with: + path: cache_path_1 + key: cache-key-1 + - run: | + mkdir -p cache_path_1 + echo "Hello from push workflow!" > cache_path_1/cache_content_1 +` + runWorkflow(ctx, cancel, populateYaml, "push", "refs/heads/main", "step 1: push cache populate expected to succeed") + + // Step 2: Validate that cache is accessible; mostly a sanity check that the test environment and mock context + // provides everything needed for the cache setup. + checkYaml := ` +name: Cache Testing Action +on: + push: +jobs: + job-cache-check-2: + runs-on: ubuntu-latest + steps: + - uses: actions/cache@v4 + with: + path: cache_path_1 + key: cache-key-1 + - run: | + [[ -f cache_path_1/cache_content_1 ]] && echo "Step 2: cache file found." || echo "Step 2: cache file missing!" + [[ -f cache_path_1/cache_content_1 ]] || exit 1 +` + runWorkflow(ctx, cancel, checkYaml, "push", "refs/heads/main", "step 2: push cache check expected to succeed") + }) + + t.Run("PR cache pollution prevented", func(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + // Step 1: Populate shared cache with push workflow + populateYaml := ` +name: Cache Testing Action +on: + push: +jobs: + job-cache-check-1: + runs-on: ubuntu-latest + steps: + - uses: actions/cache@v4 + with: + path: cache_path_1 + key: cache-key-1 + - run: | + mkdir -p cache_path_1 + echo "Hello from push workflow!" > cache_path_1/cache_content_1 +` + runWorkflow(ctx, cancel, populateYaml, "push", "refs/heads/main", "step 1: push cache populate expected to succeed") + + // Step 2: Check if pull_request can read push cache, should be available as it's a trusted cache. + checkPRYaml := ` +name: Cache Testing Action +on: + pull_request: +jobs: + job-cache-check-2: + runs-on: ubuntu-latest + steps: + - uses: actions/cache@v4 + with: + path: cache_path_1 + key: cache-key-1 + - run: | + [[ -f cache_path_1/cache_content_1 ]] && echo "Step 2: cache file found." || echo "Step 2: cache file missing!" + [[ -f cache_path_1/cache_content_1 ]] || exit 1 +` + runWorkflow(ctx, cancel, checkPRYaml, "pull_request", "refs/pull/1234/head", "step 2: PR should read push cache") + + // Step 3: Pull request writes to cache; here we need to use a new cache key because we'll get a cache-hit like we + // did in step #2 if we keep the same key, and then the cache contents won't be updated. + populatePRYaml := ` +name: Cache Testing Action +on: + pull_request: +jobs: + job-cache-check-3: + runs-on: ubuntu-latest + steps: + - uses: actions/cache@v4 + with: + path: cache_path_1 + key: cache-key-2 + - run: | + mkdir -p cache_path_1 + echo "Hello from PR workflow!" > cache_path_1/cache_content_2 +` + runWorkflow(ctx, cancel, populatePRYaml, "pull_request", "refs/pull/1234/head", "step 3: PR cache populate expected to succeed") + + // Step 4: Check if pull_request can read its own cache written by step #3. + checkPRKey2Yaml := ` +name: Cache Testing Action +on: + pull_request: +jobs: + job-cache-check-4: + runs-on: ubuntu-latest + steps: + - uses: actions/cache@v4 + with: + path: cache_path_1 + key: cache-key-2 + - run: | + [[ -f cache_path_1/cache_content_2 ]] && echo "Step 4 cache file found." || echo "Step 4 cache file missing!" + [[ -f cache_path_1/cache_content_2 ]] || exit 1 +` + runWorkflow(ctx, cancel, checkPRKey2Yaml, "pull_request", "refs/pull/1234/head", "step 4: PR should read its own cache") + + // Step 5: Check that the push workflow cannot access the isolated cache that was written by the pull_request in + // step #3, ensuring that it's not possible to pollute the cache by predicting cache keys. + checkKey2Yaml := ` +name: Cache Testing Action +on: + push: +jobs: + job-cache-check-6: + runs-on: ubuntu-latest + steps: + - uses: actions/cache@v4 + with: + path: cache_path_1 + key: cache-key-2 + - run: | + [[ -f cache_path_1/cache_content_2 ]] && echo "Step 5 cache file found, oh no!" || echo "Step 5: cache file missing as expected." + [[ -f cache_path_1/cache_content_2 ]] && exit 1 || exit 0 +` + runWorkflow(ctx, cancel, checkKey2Yaml, "push", "refs/heads/main", "step 5: push cache should not be polluted by PR") + }) +} + +func TestRunnerCacheStartupFailure(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test") + } + + testCases := []struct { + desc string + listen string + }{ + { + desc: "disable cache server", + listen: "127.0.0.1:40715", + }, + { + desc: "disable cache proxy server", + listen: "127.0.0.1:40716", + }, + } + + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + forgejoClient := &forgejoClientMock{} + + forgejoClient.On("Address").Return("https://127.0.0.1:8080") // not expected to be used in this test + forgejoClient.On("UpdateLog", mock.Anything, mock.Anything).Return(nil, nil) + forgejoClient.On("UpdateTask", mock.Anything, mock.Anything). + Return(connect.NewResponse(&runnerv1.UpdateTaskResponse{}), nil) + + // We'll be listening on some network port in this test that will conflict with the cache configuration... + l, err := net.Listen("tcp4", tc.listen) + require.NoError(t, err) + defer l.Close() + + runner := NewRunner( + &config.Config{ + Cache: config.Cache{ + Port: 40715, + ProxyPort: 40716, + Dir: t.TempDir(), + }, + Host: config.Host{ + WorkdirParent: t.TempDir(), + }, + }, + &config.Registration{ + Labels: []string{"ubuntu-latest:docker://code.forgejo.org/oci/node:20-bookworm"}, + }, + forgejoClient) + require.NotNil(t, runner) + + // Ensure that cacheProxy failed to start + assert.Nil(t, runner.cacheProxy) + + runWorkflow := func(ctx context.Context, cancel context.CancelFunc, yamlContent string) { + task := &runnerv1.Task{ + WorkflowPayload: []byte(yamlContent), + Context: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "token": structpb.NewStringValue("some token here"), + "forgejo_default_actions_url": structpb.NewStringValue("https://data.forgejo.org"), + "repository": structpb.NewStringValue("runner"), + "event_name": structpb.NewStringValue("push"), + "ref": structpb.NewStringValue("refs/heads/main"), + }, + }, + } + + reporter := report.NewReporter(ctx, cancel, forgejoClient, task, time.Second) + err := runner.run(ctx, task, reporter) + reporter.Close(nil) + require.NoError(t, err) + } + + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + checkCacheYaml := ` +name: Verify No ACTIONS_CACHE_URL +on: + push: +jobs: + job-cache-check-1: + runs-on: ubuntu-latest + steps: + - run: echo $ACTIONS_CACHE_URL + - run: '[[ "$ACTIONS_CACHE_URL" = "" ]] || exit 1' +` + runWorkflow(ctx, cancel, checkCacheYaml) + }) + } +} + +func TestRunnerLXC(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test") + } + + forgejoClient := &forgejoClientMock{} + + forgejoClient.On("Address").Return("https://127.0.0.1:8080") // not expected to be used in this test + forgejoClient.On("UpdateLog", mock.Anything, mock.Anything).Return(nil, nil) + forgejoClient.On("UpdateTask", mock.Anything, mock.Anything). + Return(connect.NewResponse(&runnerv1.UpdateTaskResponse{}), nil) + + workdirParent := t.TempDir() + runner := NewRunner( + &config.Config{ + Log: config.Log{ + JobLevel: "trace", + }, + Host: config.Host{ + WorkdirParent: workdirParent, + }, + }, + &config.Registration{ + Labels: []string{"lxc:lxc://debian:bookworm"}, + }, + forgejoClient) + require.NotNil(t, runner) + + runWorkflow := func(ctx context.Context, cancel context.CancelFunc, yamlContent, eventName, ref, description string) { + task := &runnerv1.Task{ + WorkflowPayload: []byte(yamlContent), + Context: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "token": structpb.NewStringValue("some token here"), + "forgejo_default_actions_url": structpb.NewStringValue("https://data.forgejo.org"), + "repository": structpb.NewStringValue("runner"), + "event_name": structpb.NewStringValue(eventName), + "ref": structpb.NewStringValue(ref), + }, + }, + } + + reporter := report.NewReporter(ctx, cancel, forgejoClient, task, time.Second) + err := runner.run(ctx, task, reporter) + reporter.Close(nil) + require.NoError(t, err, description) + // verify there are no leftovers + assertDirectoryEmpty := func(t *testing.T, dir string) { + f, err := os.Open(dir) + require.NoError(t, err) + defer f.Close() + + names, err := f.Readdirnames(-1) + require.NoError(t, err) + assert.Empty(t, names) + } + assertDirectoryEmpty(t, workdirParent) + } + + t.Run("OK", func(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + workflow := ` +on: + push: +jobs: + job: + runs-on: lxc + steps: + - run: mkdir -p some/directory/owned/by/root +` + runWorkflow(ctx, cancel, workflow, "push", "refs/heads/main", "OK") + }) +} + +func TestRunnerResources(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test") + } + + forgejoClient := &forgejoClientMock{} + + forgejoClient.On("Address").Return("https://127.0.0.1:8080") // not expected to be used in this test + forgejoClient.On("UpdateLog", mock.Anything, mock.Anything).Return(nil, nil) + forgejoClient.On("UpdateTask", mock.Anything, mock.Anything). + Return(connect.NewResponse(&runnerv1.UpdateTaskResponse{}), nil) + + workdirParent := t.TempDir() + + runWorkflow := func(ctx context.Context, cancel context.CancelFunc, yamlContent, options, errorMessage, logMessage string) { + task := &runnerv1.Task{ + WorkflowPayload: []byte(yamlContent), + Context: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "token": structpb.NewStringValue("some token here"), + "forgejo_default_actions_url": structpb.NewStringValue("https://data.forgejo.org"), + "repository": structpb.NewStringValue("runner"), + "event_name": structpb.NewStringValue("push"), + "ref": structpb.NewStringValue("refs/heads/main"), + }, + }, + } + + runner := NewRunner( + &config.Config{ + Log: config.Log{ + JobLevel: "trace", + }, + Host: config.Host{ + WorkdirParent: workdirParent, + }, + Container: config.Container{ + Options: options, + }, + }, + &config.Registration{ + Labels: []string{"docker:docker://code.forgejo.org/oci/node:20-bookworm"}, + }, + forgejoClient) + require.NotNil(t, runner) + + reporter := report.NewReporter(ctx, cancel, forgejoClient, task, time.Second) + err := runner.run(ctx, task, reporter) + reporter.Close(nil) + if len(errorMessage) > 0 { + require.Error(t, err) + assert.ErrorContains(t, err, errorMessage) + } else { + require.NoError(t, err) + } + if len(logMessage) > 0 { + assert.Contains(t, forgejoClient.sent, logMessage) + } + } + + t.Run("config.yaml --memory set and enforced", func(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + workflow := ` +on: + push: +jobs: + job: + runs-on: docker + steps: + - run: | + # more than 300MB + perl -e '$a = "a" x (300 * 1024 * 1024)' +` + runWorkflow(ctx, cancel, workflow, "--memory 200M", "Job 'job' failed", "Killed") + }) + + t.Run("config.yaml --memory set and within limits", func(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + workflow := ` +on: + push: +jobs: + job: + runs-on: docker + steps: + - run: echo OK +` + runWorkflow(ctx, cancel, workflow, "--memory 200M", "", "") + }) + + t.Run("config.yaml --memory set and container fails to increase it", func(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + workflow := ` +on: + push: +jobs: + job: + runs-on: docker + container: + image: code.forgejo.org/oci/node:20-bookworm + options: --memory 4G + steps: + - run: | + # more than 300MB + perl -e '$a = "a" x (300 * 1024 * 1024)' +` + runWorkflow(ctx, cancel, workflow, "--memory 200M", "option found in the workflow cannot be greater than", "") + }) + + t.Run("container --memory set and enforced", func(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + workflow := ` +on: + push: +jobs: + job: + runs-on: docker + container: + image: code.forgejo.org/oci/node:20-bookworm + options: --memory 200M + steps: + - run: | + # more than 300MB + perl -e '$a = "a" x (300 * 1024 * 1024)' +` + runWorkflow(ctx, cancel, workflow, "", "Job 'job' failed", "Killed") + }) + + t.Run("container --memory set and within limits", func(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + workflow := ` +on: + push: +jobs: + job: + runs-on: docker + container: + image: code.forgejo.org/oci/node:20-bookworm + options: --memory 200M + steps: + - run: echo OK +` + runWorkflow(ctx, cancel, workflow, "", "", "") + }) +} diff --git a/internal/app/run/workflow.go b/internal/app/run/workflow.go index 601f36d1..155eddb4 100644 --- a/internal/app/run/workflow.go +++ b/internal/app/run/workflow.go @@ -10,8 +10,8 @@ import ( "strings" runnerv1 "code.forgejo.org/forgejo/actions-proto/runner/v1" - "code.forgejo.org/forgejo/runner/v9/act/model" - "gopkg.in/yaml.v3" + "code.forgejo.org/forgejo/runner/v11/act/model" + "go.yaml.in/yaml/v3" ) func generateWorkflow(task *runnerv1.Task) (*model.Workflow, string, error) { diff --git a/internal/app/run/workflow_test.go b/internal/app/run/workflow_test.go index f0ac579c..c98d85f6 100644 --- a/internal/app/run/workflow_test.go +++ b/internal/app/run/workflow_test.go @@ -7,7 +7,7 @@ import ( "testing" runnerv1 "code.forgejo.org/forgejo/actions-proto/runner/v1" - "code.forgejo.org/forgejo/runner/v9/act/model" + "code.forgejo.org/forgejo/runner/v11/act/model" "github.com/stretchr/testify/require" "gotest.tools/v3/assert" ) diff --git a/internal/pkg/client/mocks/Client.go b/internal/pkg/client/mocks/Client.go index f2ea5fe2..3b3ab6cb 100644 --- a/internal/pkg/client/mocks/Client.go +++ b/internal/pkg/client/mocks/Client.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.53.4. DO NOT EDIT. +// Code generated by mockery v2.53.5. DO NOT EDIT. package mocks diff --git a/internal/pkg/common/daemon_context.go b/internal/pkg/common/daemon_context.go new file mode 100644 index 00000000..8cb2f70a --- /dev/null +++ b/internal/pkg/common/daemon_context.go @@ -0,0 +1,26 @@ +// Copyright 2025 The Forgejo Authors +// SPDX-License-Identifier: MIT + +package common + +import ( + "context" +) + +type daemonContextKey string + +const daemonContextKeyVal = daemonContextKey("daemon") + +func DaemonContext(ctx context.Context) context.Context { + val := ctx.Value(daemonContextKeyVal) + if val != nil { + if daemon, ok := val.(context.Context); ok { + return daemon + } + } + return context.Background() +} + +func WithDaemonContext(ctx, daemon context.Context) context.Context { + return context.WithValue(ctx, daemonContextKeyVal, daemon) +} diff --git a/internal/pkg/config/config.example.yaml b/internal/pkg/config/config.example.yaml index a3131b0b..5addac41 100644 --- a/internal/pkg/config/config.example.yaml +++ b/internal/pkg/config/config.example.yaml @@ -3,10 +3,19 @@ # You don't have to copy this file to your instance, # just run `forgejo-runner generate-config > config.yaml` to generate a config file. +# +# The value of level or job_level can be trace, debug, info, warn, error or fatal +# log: - # The level of logging, can be trace, debug, info, warn, error, fatal + # + # What is displayed in the output of the runner process but not sent + # to the Forgejo instance. + # level: info - # The level of logging for jobs, can be trace, debug, info, earn, error, fatal + # + # What is sent to the Forgejo instance and therefore + # visible in the web UI for a given job. + # job_level: info runner: @@ -101,25 +110,20 @@ cache: # external_server: "" # - ####################################################################### - # - # Common to the internal and external cache server - # - ####################################################################### - # # The shared cache secret used to secure the communications between # the cache proxy and the cache server. # # If empty, it will be generated to a new secret automatically when # the server starts and it will stay the same until it restarts. # - # Every time the secret is modified, all cache entries that were - # created with it are invalidated. In order to ensure that the cache - # content is reused when the runner restarts, this secret must be - # set, for instance with the output of openssl rand -hex 40. - # secret: "" # + ####################################################################### + # + # Common to the internal and external cache server + # + ####################################################################### + # # The IP or hostname (195.84.20.30 or example.com) to use when constructing # ACTIONS_CACHE_URL which is the URL of the cache proxy. # @@ -129,7 +133,7 @@ cache: # different network than the Forgejo runner (for instance when the # docker server used to create containers is not running on the same # host as the Forgejo runner), it may be impossible to figure that - # out automatically. In that case you can specifify which IP or + # out automatically. In that case you can specify which IP or # hostname to use to reach the internal cache server created by the # Forgejo runner. # @@ -140,10 +144,10 @@ cache: # proxy_port: 0 # - # Overrides the ACTIONS_CACHE_URL passed to workflow - # containers. This should only be used if the runner host is not - # reachable from the workflow containers, and requires further - # setup. + # Overrides the ACTIONS_CACHE_URL variable passed to workflow + # containers. The URL should generally not end with "/". This should only + # be used if the runner host is not reachable from the workflow containers, + # and requires further setup. # actions_cache_url_override: "" @@ -172,10 +176,12 @@ container: # valid_volumes: # - '**' valid_volumes: [] - # overrides the docker client host with the specified one. - # If "-" or "", an available docker host will automatically be found. + # Overrides the docker host set by the DOCKER_HOST environment variable, and mounts on the job container. + # If "-" or "", no docker host will be mounted in the job container # If "automount", an available docker host will automatically be found and mounted in the job container (e.g. /var/run/docker.sock). - # Otherwise the specified docker host will be used and an error will be returned if it doesn't work. + # If it's a url, the specified docker host will be mounted in the job container + # Example urls: unix:///run/docker.socket or ssh://user@host + # The specified socket is mounted within the job container at /var/run/docker.sock docker_host: "-" # Pull docker image(s) even if already present force_pull: false diff --git a/internal/pkg/config/config.go b/internal/pkg/config/config.go index 170db177..3390e91c 100644 --- a/internal/pkg/config/config.go +++ b/internal/pkg/config/config.go @@ -5,13 +5,14 @@ package config import ( "fmt" + "maps" "os" "path/filepath" "time" "github.com/joho/godotenv" log "github.com/sirupsen/logrus" - "gopkg.in/yaml.v3" + "go.yaml.in/yaml/v3" ) // Log represents the configuration for logging. @@ -109,9 +110,7 @@ func LoadDefault(file string) (*Config, error) { if cfg.Runner.Envs == nil { cfg.Runner.Envs = map[string]string{} } - for k, v := range envs { - cfg.Runner.Envs[k] = v - } + maps.Copy(cfg.Runner.Envs, envs) } } diff --git a/internal/pkg/config/registration.go b/internal/pkg/config/registration.go index be66b4fb..14d592bd 100644 --- a/internal/pkg/config/registration.go +++ b/internal/pkg/config/registration.go @@ -1,3 +1,4 @@ +// Copyright 2025 The Forgejo Authors. All rights reserved. // Copyright 2023 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT @@ -6,6 +7,8 @@ package config import ( "encoding/json" "os" + + "github.com/google/go-cmp/cmp" ) const registrationWarning = "This file is automatically generated by act-runner. Do not edit it manually unless you know what you are doing. Removing this file will cause act runner to re-register as a new runner." @@ -34,12 +37,28 @@ func LoadRegistration(file string) (*Registration, error) { return nil, err } - reg.Warning = "" - return ®, nil } +func isEqualRegistration(file string, reg *Registration) (bool, error) { + existing, err := LoadRegistration(file) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + return cmp.Equal(*reg, *existing), nil +} + func SaveRegistration(file string, reg *Registration) error { + equal, err := isEqualRegistration(file, reg) + if err != nil { + return err + } + if equal { + return nil + } f, err := os.Create(file) if err != nil { return err diff --git a/internal/pkg/config/registration_test.go b/internal/pkg/config/registration_test.go new file mode 100644 index 00000000..fe89410b --- /dev/null +++ b/internal/pkg/config/registration_test.go @@ -0,0 +1,63 @@ +// Copyright 2025 The Forgejo Authors. All rights reserved. +// SPDX-License-Identifier: GPL-3.0-or-later + +package config + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestConfig_Registration(t *testing.T) { + reg := Registration{ + Warning: registrationWarning, + ID: 1234, + UUID: "UUID", + Name: "NAME", + Token: "TOKEN", + Address: "ADDRESS", + Labels: []string{"LABEL1", "LABEL2"}, + } + + file := filepath.Join(t.TempDir(), ".runner") + + // when the file does not exist, it is never equal + equal, err := isEqualRegistration(file, ®) + require.NoError(t, err) + assert.False(t, equal) + + require.NoError(t, SaveRegistration(file, ®)) + + regReloaded, err := LoadRegistration(file) + require.NoError(t, err) + assert.Equal(t, reg, *regReloaded) + + equal, err = isEqualRegistration(file, ®) + require.NoError(t, err) + assert.True(t, equal) + + // if the registration is not modified, it is not saved + time.Sleep(2 * time.Second) // file system precision on modification time is one second + before, err := os.Stat(file) + require.NoError(t, err) + require.NoError(t, SaveRegistration(file, ®)) + after, err := os.Stat(file) + require.NoError(t, err) + assert.Equal(t, before.ModTime(), after.ModTime()) + + reg.Labels = []string{"LABEL3"} + equal, err = isEqualRegistration(file, ®) + require.NoError(t, err) + assert.False(t, equal) + + // if the registration is modified, it is saved + require.NoError(t, SaveRegistration(file, ®)) + after, err = os.Stat(file) + require.NoError(t, err) + assert.NotEqual(t, before.ModTime(), after.ModTime()) +} diff --git a/internal/pkg/labels/labels.go b/internal/pkg/labels/labels.go index 7befaf42..e256964a 100644 --- a/internal/pkg/labels/labels.go +++ b/internal/pkg/labels/labels.go @@ -31,6 +31,10 @@ func Parse(str string) (*Label, error) { Schema: "docker", } + if strings.TrimSpace(label.Name) != label.Name { + return nil, fmt.Errorf("invalid label %q: starting or ending with a space is invalid", label.Name) + } + if len(splits) >= 2 { label.Schema = splits[1] if label.Schema != SchemeHost && label.Schema != SchemeDocker && label.Schema != SchemeLXC { diff --git a/internal/pkg/labels/labels_test.go b/internal/pkg/labels/labels_test.go index db2f0adf..bf798fa2 100644 --- a/internal/pkg/labels/labels_test.go +++ b/internal/pkg/labels/labels_test.go @@ -63,7 +63,6 @@ func TestParse(t *testing.T) { }, wantErr: false, }, - { args: "label1:host", want: &Label{ @@ -78,12 +77,21 @@ func TestParse(t *testing.T) { want: nil, wantErr: true, }, - { args: "label1:invalidscheme", want: nil, wantErr: true, }, + { + args: " label1:lxc://debian:buster", + want: nil, + wantErr: true, + }, + { + args: "label1 :lxc://debian:buster", + want: nil, + wantErr: true, + }, } for _, tt := range tests { t.Run(tt.args, func(t *testing.T) { diff --git a/internal/pkg/report/mask.go b/internal/pkg/report/mask.go index 286a3fc5..3dbb6f42 100644 --- a/internal/pkg/report/mask.go +++ b/internal/pkg/report/mask.go @@ -36,13 +36,16 @@ func (o *masker) add(secret string) { slices.SortFunc(o.multiLines, func(a, b []string) int { return cmp.Compare(len(b), len(a)) }) - } else { - o.lines = append(o.lines, lines[0]) - // make sure the longest secret are replaced first - slices.SortFunc(o.lines, func(a, b string) int { - return cmp.Compare(len(b), len(a)) - }) + // a multiline secret transformed into a single line by replacing + // newlines with \ followed by n must also be redacted + o.lines = append(o.lines, strings.Join(lines, "\\n")) } + + o.lines = append(o.lines, secret) + // make sure the longest secret are replaced first + slices.SortFunc(o.lines, func(a, b string) int { + return cmp.Compare(len(b), len(a)) + }) } func (o *masker) getReplacer() *strings.Replacer { diff --git a/internal/pkg/report/mask_test.go b/internal/pkg/report/mask_test.go index f6f332e4..b785c4dc 100644 --- a/internal/pkg/report/mask_test.go +++ b/internal/pkg/report/mask_test.go @@ -7,6 +7,8 @@ import ( "fmt" "testing" + runnerv1 "code.forgejo.org/forgejo/actions-proto/runner/v1" + "github.com/stretchr/testify/assert" ) @@ -41,6 +43,19 @@ SIX` out: "line before\n***\n***\n***\nline after\n", needMore: false, }, + { + // + // a multiline secret where newlines are represented + // as \ followed by n is masked + // + name: "MultilineTransformedIsMasked", + secrets: []string{ + multiLineOne, + }, + in: fmt.Sprintf("line before\n%[1]s\\nTWO\\nTHREE\nline after", lineOne), + out: "line before\n***\nline after\n", + needMore: false, + }, { // // in a multiline secret \r\n is equivalent to \n and does @@ -254,4 +269,17 @@ SIX` assert.Equal(t, testCase.out, rowsToString(rows)) }) } + + t.Run("MultilineSecretInSingleRow", func(t *testing.T) { + secret := "ABC\nDEF\nGHI" + m := newMasker() + m.add(secret) + rows := []*runnerv1.LogRow{ + {Content: fmt.Sprintf("BEFORE%sAFTER", secret)}, + } + noMore := false + needMore := m.replace(rows, noMore) + assert.False(t, needMore) + assert.Equal(t, "BEFORE***AFTER\n", rowsToString(rows)) + }) } diff --git a/internal/pkg/report/reporter.go b/internal/pkg/report/reporter.go index 6d08269c..804744cc 100644 --- a/internal/pkg/report/reporter.go +++ b/internal/pkg/report/reporter.go @@ -13,13 +13,15 @@ import ( "time" runnerv1 "code.forgejo.org/forgejo/actions-proto/runner/v1" + "code.forgejo.org/forgejo/runner/v11/act/runner" "connectrpc.com/connect" retry "github.com/avast/retry-go/v4" log "github.com/sirupsen/logrus" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/timestamppb" - "code.forgejo.org/forgejo/runner/v9/internal/pkg/client" + "code.forgejo.org/forgejo/runner/v11/internal/pkg/client" + "code.forgejo.org/forgejo/runner/v11/internal/pkg/common" ) var ( @@ -46,6 +48,7 @@ type Reporter struct { debugOutputEnabled bool stopCommandEndToken string + issuedLocalCancel bool } func NewReporter(ctx context.Context, cancel context.CancelFunc, c client.Client, task *runnerv1.Task, reportInterval time.Duration) *Reporter { @@ -61,7 +64,10 @@ func NewReporter(ctx context.Context, cancel context.CancelFunc, c client.Client } rv := &Reporter{ - ctx: ctx, + // ctx & cancel are related: the daemon context allows the reporter + // to continue to operate even after the context is canceled, to + // conclude the converation + ctx: common.DaemonContext(ctx), cancel: cancel, client: c, masker: masker, @@ -81,7 +87,7 @@ func NewReporter(ctx context.Context, cancel context.CancelFunc, c client.Client func (r *Reporter) ResetSteps(l int) { r.stateMu.Lock() defer r.stateMu.Unlock() - for i := 0; i < l; i++ { + for i := range l { r.state.Steps = append(r.state.Steps, &runnerv1.StepState{ Id: int64(i), }) @@ -126,6 +132,13 @@ func (r *Reporter) Fire(entry *log.Entry) error { } } } + if r.state.Result == runnerv1.Result_RESULT_SUCCESS { + if v, ok := entry.Data["jobOutputs"]; ok { + _ = r.setOutputs(v.(map[string]string)) + } else { + log.Panicf("received log entry with successful jobResult, but without jobOutputs -- outputs will be corrupted for this job") + } + } } if !r.duringSteps() { r.logRows = appendIfNotNil(r.logRows, r.parseLogRow(entry)) @@ -162,7 +175,7 @@ func (r *Reporter) Fire(entry *log.Entry) error { } else if !r.duringSteps() { r.logRows = appendIfNotNil(r.logRows, r.parseLogRow(entry)) } - if v, ok := entry.Data["stepResult"]; ok { + if v := runner.GetOuterStepResult(entry); v != nil { if stepResult, ok := r.parseResult(v); ok { if step.LogLength == 0 { step.LogIndex = int64(r.logOffset + len(r.logRows)) @@ -180,23 +193,31 @@ func (r *Reporter) RunDaemon() { return } if r.ctx.Err() != nil { + // This shouldn't happen because DaemonContext is used for `r.ctx` which should outlive any running job. + log.Warnf("Terminating RunDaemon on an active job due to error: %v", r.ctx.Err()) return } - _ = r.ReportLog(false) - _ = r.ReportState() + err := r.ReportLog(false) + if err != nil { + log.Warnf("ReportLog error: %v", err) + } + err = r.ReportState() + if err != nil { + log.Warnf("ReportState error: %v", err) + } time.AfterFunc(r.reportInterval, r.RunDaemon) } -func (r *Reporter) Logf(format string, a ...interface{}) { +func (r *Reporter) Logf(format string, a ...any) { r.stateMu.Lock() defer r.stateMu.Unlock() r.logf(format, a...) } -func (r *Reporter) logf(format string, a ...interface{}) { +func (r *Reporter) logf(format string, a ...any) { if !r.duringSteps() { r.logRows = append(r.logRows, &runnerv1.LogRow{ Time: timestamppb.Now(), @@ -205,12 +226,22 @@ func (r *Reporter) logf(format string, a ...interface{}) { } } -func (r *Reporter) SetOutputs(outputs map[string]string) error { - r.stateMu.Lock() - defer r.stateMu.Unlock() +func (r *Reporter) cloneOutputs() map[string]string { + outputs := make(map[string]string) + r.outputs.Range(func(k, v any) bool { + if val, ok := v.(string); ok { + outputs[k.(string)] = val + } + return true + }) + return outputs +} +// Errors from setOutputs are logged into the reporter automatically; the `errors` return value is only used for unit +// tests. +func (r *Reporter) setOutputs(outputs map[string]string) error { var errs []error - recordError := func(format string, a ...interface{}) { + recordError := func(format string, a ...any) { r.logf(format, a...) errs = append(errs, fmt.Errorf(format, a...)) } @@ -232,31 +263,47 @@ func (r *Reporter) SetOutputs(outputs map[string]string) error { return errors.Join(errs...) } -func (r *Reporter) Close(lastWords string) error { +const ( + closeTimeoutMessage = "The runner cancelled the job because it exceeds the maximum run time" + closeCancelledMessage = "Cancelled" +) + +func (r *Reporter) Close(runErr error) error { r.closed = true - r.stateMu.Lock() - if r.state.Result == runnerv1.Result_RESULT_UNSPECIFIED { - if lastWords == "" { - lastWords = "Early termination" - } + setStepCancel := func() { for _, v := range r.state.Steps { if v.Result == runnerv1.Result_RESULT_UNSPECIFIED { v.Result = runnerv1.Result_RESULT_CANCELLED } } + } + + r.stateMu.Lock() + var lastWords string + if errors.Is(runErr, context.DeadlineExceeded) { + lastWords = closeTimeoutMessage + r.state.Result = runnerv1.Result_RESULT_CANCELLED + setStepCancel() + } else if r.state.Result == runnerv1.Result_RESULT_UNSPECIFIED { + if runErr == nil { + lastWords = closeCancelledMessage + } else { + lastWords = runErr.Error() + } r.state.Result = runnerv1.Result_RESULT_FAILURE - r.logRows = append(r.logRows, &runnerv1.LogRow{ - Time: timestamppb.Now(), - Content: lastWords, - }) - r.state.StoppedAt = timestamppb.Now() - } else if lastWords != "" { + setStepCancel() + } else if runErr != nil { + lastWords = runErr.Error() + } + + if lastWords != "" { r.logRows = append(r.logRows, &runnerv1.LogRow{ Time: timestamppb.Now(), Content: lastWords, }) } + r.state.StoppedAt = timestamppb.Now() r.stateMu.Unlock() return retry.Do(func() error { @@ -338,16 +385,9 @@ func (r *Reporter) ReportState() error { r.stateMu.RLock() state := proto.Clone(r.state).(*runnerv1.TaskState) + outputs := r.cloneOutputs() r.stateMu.RUnlock() - outputs := make(map[string]string) - r.outputs.Range(func(k, v interface{}) bool { - if val, ok := v.(string); ok { - outputs[k.(string)] = val - } - return true - }) - resp, err := r.client.UpdateTask(r.ctx, connect.NewRequest(&runnerv1.UpdateTaskRequest{ State: state, Outputs: outputs, @@ -360,12 +400,22 @@ func (r *Reporter) ReportState() error { r.outputs.Store(k, struct{}{}) } - if resp.Msg.GetState().GetResult() == runnerv1.Result_RESULT_CANCELLED { + localResultState := state.GetResult() + remoteResultState := resp.Msg.GetState().GetResult() + switch remoteResultState { + case runnerv1.Result_RESULT_CANCELLED, runnerv1.Result_RESULT_FAILURE: + // issuedLocalCancel is just used to deduplicate this log message if our local state doesn't catch up with our + // remote state as quickly as the report-interval, which would cause this message to repeat in the logs. + if !r.issuedLocalCancel && remoteResultState != localResultState { + log.Infof("UpdateTask returned task result %v for a task that was in local state %v - beginning local task termination", + remoteResultState, localResultState) + r.issuedLocalCancel = true + } r.cancel() } var noSent []string - r.outputs.Range(func(k, v interface{}) bool { + r.outputs.Range(func(k, v any) bool { if _, ok := v.(string); ok { noSent = append(noSent, k.(string)) } @@ -396,7 +446,7 @@ var stringToResult = map[string]runnerv1.Result{ "cancelled": runnerv1.Result_RESULT_CANCELLED, } -func (r *Reporter) parseResult(result interface{}) (runnerv1.Result, bool) { +func (r *Reporter) parseResult(result any) (runnerv1.Result, bool) { str := "" if v, ok := result.(string); ok { // for jobResult str = v diff --git a/internal/pkg/report/reporter_test.go b/internal/pkg/report/reporter_test.go index 2839c847..f41282cb 100644 --- a/internal/pkg/report/reporter_test.go +++ b/internal/pkg/report/reporter_test.go @@ -20,8 +20,9 @@ import ( "github.com/stretchr/testify/require" "google.golang.org/protobuf/types/known/structpb" - "code.forgejo.org/forgejo/runner/v9/internal/pkg/client/mocks" - "code.forgejo.org/forgejo/runner/v9/testutils" + "code.forgejo.org/forgejo/runner/v11/internal/pkg/client/mocks" + "code.forgejo.org/forgejo/runner/v11/internal/pkg/common" + "code.forgejo.org/forgejo/runner/v11/testutils" ) func rowsToString(rows []*runnerv1.LogRow) string { @@ -49,13 +50,13 @@ func mockReporter(t *testing.T) (*Reporter, *mocks.Client, func()) { client := mocks.NewClient(t) ctx, cancel := context.WithCancel(context.Background()) - taskCtx, err := structpb.NewStruct(map[string]interface{}{}) + taskCtx, err := structpb.NewStruct(map[string]any{}) require.NoError(t, err) - reporter := NewReporter(ctx, cancel, client, &runnerv1.Task{ + reporter := NewReporter(common.WithDaemonContext(ctx, t.Context()), cancel, client, &runnerv1.Task{ Context: taskCtx, }, time.Second) close := func() { - assert.NoError(t, reporter.Close("")) + assert.NoError(t, reporter.Close(nil)) } return reporter, client, close } @@ -64,7 +65,7 @@ func TestReporterSetOutputs(t *testing.T) { assertEqual := func(t *testing.T, expected map[string]string, actual *sync.Map) { t.Helper() actualMap := map[string]string{} - actual.Range(func(k, v interface{}) bool { + actual.Range(func(k, v any) bool { val, ok := v.(string) require.True(t, ok) actualMap[k.(string)] = val @@ -77,7 +78,7 @@ func TestReporterSetOutputs(t *testing.T) { reporter, _, _ := mockReporter(t) expected := map[string]string{"a": "b", "c": "d"} - assert.NoError(t, reporter.SetOutputs(expected)) + assert.NoError(t, reporter.setOutputs(expected)) assertEqual(t, expected, &reporter.outputs) }) @@ -92,7 +93,7 @@ func TestReporterSetOutputs(t *testing.T) { "c": "ABCDEFG", // value too big "d": "e", } - err := reporter.SetOutputs(in) + err := reporter.setOutputs(in) assert.ErrorContains(t, err, "ignore output because the length of the value for \"c\" is 7 (the maximum is 5)") assert.ErrorContains(t, err, "ignore output because the key is longer than 5: \"0123456\"") expected := map[string]string{"d": "e"} @@ -103,11 +104,11 @@ func TestReporterSetOutputs(t *testing.T) { reporter, _, _ := mockReporter(t) first := map[string]string{"a": "b", "c": "d"} - assert.NoError(t, reporter.SetOutputs(first)) + assert.NoError(t, reporter.setOutputs(first)) assertEqual(t, first, &reporter.outputs) second := map[string]string{"c": "d", "e": "f"} - assert.ErrorContains(t, reporter.SetOutputs(second), "ignore output because a value already exists for the key \"c\"") + assert.ErrorContains(t, reporter.setOutputs(second), "ignore output because a value already exists for the key \"c\"") expected := map[string]string{"a": "b", "c": "d", "e": "f"} assertEqual(t, expected, &reporter.outputs) @@ -268,7 +269,7 @@ func TestReporter_Fire(t *testing.T) { return connect_go.NewResponse(&runnerv1.UpdateTaskResponse{}), nil }) - dataStep0 := map[string]interface{}{ + dataStep0 := map[string]any{ "stage": "Main", "stepNumber": 0, "raw_output": true, @@ -283,6 +284,143 @@ func TestReporter_Fire(t *testing.T) { assert.Equal(t, int64(3), reporter.state.Steps[0].LogLength) }) + + t.Run("jobResult jobOutputs extracted from log entry", func(t *testing.T) { + reporter, _, _ := mockReporter(t) + + dataStep0 := map[string]any{ + "stage": "Post", + "stepNumber": 0, + "raw_output": true, + "jobResult": "success", + "jobOutputs": map[string]string{"key1": "value1"}, + } + assert.NoError(t, reporter.Fire(&log.Entry{Message: "success!", Data: dataStep0})) + + assert.EqualValues(t, runnerv1.Result_RESULT_SUCCESS, reporter.state.Result) + value, _ := reporter.outputs.Load("key1") + assert.EqualValues(t, "value1", value) + }) + + t.Run("jobResult jobOutputs is absent if not success", func(t *testing.T) { + reporter, _, _ := mockReporter(t) + + dataStep0 := map[string]any{ + "stage": "Post", + "stepNumber": 0, + "raw_output": true, + "jobResult": "skipped", + } + assert.NoError(t, reporter.Fire(&log.Entry{Message: "skipped!", Data: dataStep0})) + + assert.EqualValues(t, runnerv1.Result_RESULT_SKIPPED, reporter.state.Result) + }) +} + +func TestReporterReportState(t *testing.T) { + for _, testCase := range []struct { + name string + fixture func(t *testing.T, reporter *Reporter, client *mocks.Client) + assert func(t *testing.T, reporter *Reporter, ctx context.Context, err error) + }{ + { + name: "PartialOutputs", + fixture: func(t *testing.T, reporter *Reporter, client *mocks.Client) { + t.Helper() + outputKey1 := "KEY1" + outputValue1 := "VALUE1" + outputKey2 := "KEY2" + outputValue2 := "VALUE2" + reporter.setOutputs(map[string]string{ + outputKey1: outputValue1, + outputKey2: outputValue2, + }) + + client.On("UpdateTask", mock.Anything, mock.Anything).Return(func(_ context.Context, req *connect_go.Request[runnerv1.UpdateTaskRequest]) (*connect_go.Response[runnerv1.UpdateTaskResponse], error) { + t.Logf("Received UpdateTask: %s", req.Msg.String()) + return connect_go.NewResponse(&runnerv1.UpdateTaskResponse{ + SentOutputs: []string{outputKey1}, + }), nil + }) + }, + assert: func(t *testing.T, reporter *Reporter, ctx context.Context, err error) { + t.Helper() + require.ErrorContains(t, err, "not all logs are submitted 1 remain") + outputs := reporter.cloneOutputs() + assert.Equal(t, map[string]string{ + "KEY2": "VALUE2", + }, outputs) + assert.NoError(t, ctx.Err()) + }, + }, + { + name: "AllDone", + fixture: func(t *testing.T, reporter *Reporter, client *mocks.Client) { + t.Helper() + client.On("UpdateTask", mock.Anything, mock.Anything).Return(func(_ context.Context, req *connect_go.Request[runnerv1.UpdateTaskRequest]) (*connect_go.Response[runnerv1.UpdateTaskResponse], error) { + t.Logf("Received UpdateTask: %s", req.Msg.String()) + return connect_go.NewResponse(&runnerv1.UpdateTaskResponse{}), nil + }) + }, + assert: func(t *testing.T, reporter *Reporter, ctx context.Context, err error) { + t.Helper() + require.NoError(t, err) + assert.NoError(t, ctx.Err()) + }, + }, + { + name: "Canceled", + fixture: func(t *testing.T, reporter *Reporter, client *mocks.Client) { + t.Helper() + client.On("UpdateTask", mock.Anything, mock.Anything).Return(func(_ context.Context, req *connect_go.Request[runnerv1.UpdateTaskRequest]) (*connect_go.Response[runnerv1.UpdateTaskResponse], error) { + t.Logf("Received UpdateTask: %s", req.Msg.String()) + return connect_go.NewResponse(&runnerv1.UpdateTaskResponse{ + State: &runnerv1.TaskState{ + Result: runnerv1.Result_RESULT_CANCELLED, + }, + }), nil + }) + }, + assert: func(t *testing.T, reporter *Reporter, ctx context.Context, err error) { + t.Helper() + require.NoError(t, err) + assert.ErrorIs(t, ctx.Err(), context.Canceled) + }, + }, + { + name: "Failed", + fixture: func(t *testing.T, reporter *Reporter, client *mocks.Client) { + t.Helper() + client.On("UpdateTask", mock.Anything, mock.Anything).Return(func(_ context.Context, req *connect_go.Request[runnerv1.UpdateTaskRequest]) (*connect_go.Response[runnerv1.UpdateTaskResponse], error) { + t.Logf("Received UpdateTask: %s", req.Msg.String()) + return connect_go.NewResponse(&runnerv1.UpdateTaskResponse{ + State: &runnerv1.TaskState{ + Result: runnerv1.Result_RESULT_FAILURE, + }, + }), nil + }) + }, + assert: func(t *testing.T, reporter *Reporter, ctx context.Context, err error) { + t.Helper() + require.NoError(t, err) + assert.ErrorIs(t, ctx.Err(), context.Canceled) + }, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + client := mocks.NewClient(t) + ctx, cancel := context.WithCancel(context.Background()) + taskCtx, err := structpb.NewStruct(map[string]any{}) + require.NoError(t, err) + reporter := NewReporter(common.WithDaemonContext(ctx, t.Context()), cancel, client, &runnerv1.Task{ + Context: taskCtx, + }, time.Second) + + testCase.fixture(t, reporter, client) + err = reporter.ReportState() + testCase.assert(t, reporter, ctx, err) + }) + } } func TestReporterReportLogLost(t *testing.T) { @@ -407,3 +545,98 @@ func TestReporterReportLog(t *testing.T) { }) } } + +func TestReporterClose(t *testing.T) { + mockReporterCloser := func(t *testing.T, message *string, result *runnerv1.Result) *Reporter { + t.Helper() + reporter, client, _ := mockReporter(t) + if message != nil { + client.On("UpdateLog", mock.Anything, mock.Anything).Return(func(_ context.Context, req *connect_go.Request[runnerv1.UpdateLogRequest]) (*connect_go.Response[runnerv1.UpdateLogResponse], error) { + t.Logf("UpdateLogRequest: %s", req.Msg.String()) + assert.Equal(t, (*message)+"\n", rowsToString(req.Msg.Rows)) + resp := &runnerv1.UpdateLogResponse{ + AckIndex: req.Msg.Index + 1, + } + t.Logf("UpdateLogResponse: %s", resp.String()) + return connect_go.NewResponse(resp), nil + }) + } + + if result != nil { + client.On("UpdateTask", mock.Anything, mock.Anything).Return(func(_ context.Context, req *connect_go.Request[runnerv1.UpdateTaskRequest]) (*connect_go.Response[runnerv1.UpdateTaskResponse], error) { + t.Logf("Received UpdateTask: %s", req.Msg.String()) + assert.Equal(t, result.String(), req.Msg.State.Result.String()) + return connect_go.NewResponse(&runnerv1.UpdateTaskResponse{}), nil + }) + } + return reporter + } + + for _, testCase := range []struct { + name string + err error + expectedMessage string + result runnerv1.Result + expectedStateResult runnerv1.Result + expectedStepStateResult runnerv1.Result + }{ + { + name: "ResultSuccessAndNilErrorIsResultSuccess", + err: nil, + expectedMessage: "", + result: runnerv1.Result_RESULT_SUCCESS, + expectedStateResult: runnerv1.Result_RESULT_SUCCESS, + }, + { + name: "ResultUnspecifiedAndErrorIsResultFailure", + err: errors.New("ERROR_MESSAGE"), + expectedMessage: "ERROR_MESSAGE", + result: runnerv1.Result_RESULT_UNSPECIFIED, + expectedStateResult: runnerv1.Result_RESULT_FAILURE, + expectedStepStateResult: runnerv1.Result_RESULT_CANCELLED, + }, + { + name: "ResultUnspecifiedAndNilErrorIsResultFailure", + err: nil, + expectedMessage: closeCancelledMessage, + result: runnerv1.Result_RESULT_UNSPECIFIED, + expectedStateResult: runnerv1.Result_RESULT_FAILURE, + expectedStepStateResult: runnerv1.Result_RESULT_CANCELLED, + }, + { + name: "ResultSkippedAndErrorIsResultSkipped", + err: errors.New("ERROR_MESSAGE"), + expectedMessage: "ERROR_MESSAGE", + result: runnerv1.Result_RESULT_SKIPPED, + expectedStateResult: runnerv1.Result_RESULT_SKIPPED, + }, + { + name: "Timeout", + err: context.DeadlineExceeded, + expectedMessage: closeTimeoutMessage, + result: runnerv1.Result_RESULT_UNSPECIFIED, + expectedStateResult: runnerv1.Result_RESULT_CANCELLED, + expectedStepStateResult: runnerv1.Result_RESULT_CANCELLED, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + var message *string + if testCase.expectedMessage != "" { + message = &testCase.expectedMessage + } + reporter := mockReporterCloser(t, message, &testCase.expectedStateResult) + + // cancel() verifies Close can operate after the context is cancelled + // because it uses the daemon context instead + reporter.cancel() + reporter.state.Result = testCase.result + reporter.state.Steps = []*runnerv1.StepState{ + { + Result: runnerv1.Result_RESULT_UNSPECIFIED, + }, + } + require.NoError(t, reporter.Close(testCase.err)) + assert.Equal(t, testCase.expectedStepStateResult, reporter.state.Steps[0].Result) + }) + } +} diff --git a/internal/pkg/ver/version.go b/internal/pkg/ver/version.go index 9efc8739..7be5cca1 100644 --- a/internal/pkg/ver/version.go +++ b/internal/pkg/ver/version.go @@ -3,7 +3,7 @@ package ver -// go build -ldflags "-X code.forgejo.org/forgejo/runner/v9/internal/pkg/ver.version=1.2.3" +// go build -ldflags "-X code.forgejo.org/forgejo/runner/v11/internal/pkg/ver.version=1.2.3" var version = "dev" func Version() string { diff --git a/main.go b/main.go index 498bd129..1356572f 100644 --- a/main.go +++ b/main.go @@ -8,7 +8,7 @@ import ( "os/signal" "syscall" - "code.forgejo.org/forgejo/runner/v9/internal/app/cmd" + "code.forgejo.org/forgejo/runner/v11/internal/app/cmd" ) func main() { diff --git a/renovate.json b/renovate.json index e654a18d..b421427f 100644 --- a/renovate.json +++ b/renovate.json @@ -4,10 +4,22 @@ "prConcurrentLimit": 1, "packageRules": [ { - "description": "Separate minor and patch for some packages", + "description": "separate minor and patch for some packages", "matchDepNames": ["github.com/rhysd/actionlint"], "separateMinorPatch": true + }, + { + "description": "separate multiple major and minor for forgejo", + "matchDepNames": ["code.forgejo.org/forgejo/forgejo"], + "separateMultipleMajor": true, + "separateMultipleMinor": true + }, + { + "description": "group runner updates", + "matchDepNames": ["code.forgejo.org/forgejo/runner", "forgejo/runner"], + "groupName": "forgejo-runner" } ], + "labels": ["Kind/DependencyUpdate", "run-end-to-end-tests"], "ignorePaths": ["**/testdata/**", "**/node_modules/**"] }