diff --git a/.github/actions/ci-go-prep/action.yml b/.github/actions/ci-go-prep/action.yml new file mode 100644 index 0000000..0744083 --- /dev/null +++ b/.github/actions/ci-go-prep/action.yml @@ -0,0 +1,16 @@ +name: CI Go prep +description: Prepare Go directories and restore the Go module cache. +runs: + using: composite + steps: + - name: Prepare Go directories + shell: bash + run: | + set -euo pipefail + # Keep writable Go directories outside the runner template's built-in GOPATH. + mkdir -p "${GOCACHE}" "${GOMODCACHE}" "${GOPATH}/bin" + + - name: Restore Go module cache from S3 + continue-on-error: true + shell: bash + run: bash "${GITHUB_WORKSPACE}/.github/scripts/ci/restore-go-cache.sh" diff --git a/.github/scripts/ci/restore-go-cache.sh b/.github/scripts/ci/restore-go-cache.sh new file mode 100755 index 0000000..cefc33f --- /dev/null +++ b/.github/scripts/ci/restore-go-cache.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +set -euo pipefail + +source "${GITHUB_WORKSPACE}/.github/scripts/ci/s3-cache-common.sh" + +mkdir -p "${GOMODCACHE}/cache/download" "${GOPATH}/bin" "${GOCACHE}" "${GITHUB_WORKSPACE}/.cache" +rm -rf "${GITHUB_WORKSPACE}/.cache/go/build-cache" + +if ! s3_cache_enabled; then + echo "S3 Go module cache restore skipped: missing RUNNER_S3_AK/RUNNER_S3_SK or rclone" + exit 0 +fi + +if [[ -z "${CI_GO_MOD_HASH:-}" ]]; then + echo "S3 Go module cache restore skipped: CI_GO_MOD_HASH is empty" + exit 0 +fi + +cache_key="${RUNNER_S3_PREFIX}/go-download/hash/${RUNNER_OS:-Linux}/${CI_GO_MOD_HASH}.tar.zst" +archive_path="$(mktemp)" +trap 'rm -f "${archive_path}"' EXIT + +if s3_cache_get "${cache_key}" "${archive_path}"; then + zstd -dc "${archive_path}" | tar -C "${GITHUB_WORKSPACE}/.cache" -xf - + echo "Restored S3 Go module download cache: ${cache_key}" +else + echo "S3 Go module download cache not found: ${cache_key}" +fi diff --git a/.github/scripts/ci/s3-cache-common.sh b/.github/scripts/ci/s3-cache-common.sh new file mode 100755 index 0000000..e112d69 --- /dev/null +++ b/.github/scripts/ci/s3-cache-common.sh @@ -0,0 +1,106 @@ +#!/usr/bin/env bash +set -euo pipefail + +: "${RUNNER_S3_BUCKET:=las-github-runner-dal}" +: "${RUNNER_S3_ENDPOINT:=s3.us-north-1.qiniucs.com}" +: "${RUNNER_S3_REGION:=us-north-1}" +: "${RUNNER_S3_PREFIX:=ci-cache/${GITHUB_REPOSITORY:?GITHUB_REPOSITORY is required}}" + +raw_job_name="${GITHUB_JOB:-unknown-job}" +RUNNER_S3_JOB_SCOPE="$(printf '%s' "${raw_job_name}" | sed 's/[^[:alnum:]._-]/-/g')" +export RUNNER_S3_JOB_SCOPE + +RUNNER_S3_RCLONE_CONFIG="" + +s3_cache_enabled() { + [[ -n "${RUNNER_S3_AK:-}" && -n "${RUNNER_S3_SK:-}" ]] && command -v rclone >/dev/null 2>&1 +} + +s3_cache_configure_rclone() { + if [[ -n "${RUNNER_S3_RCLONE_CONFIG}" ]]; then + return 0 + fi + + local endpoint="${RUNNER_S3_ENDPOINT}" + + if [[ ! "${endpoint}" =~ ^https?:// ]]; then + endpoint="https://${endpoint}" + fi + + RUNNER_S3_RCLONE_CONFIG="$(mktemp)" + chmod 600 "${RUNNER_S3_RCLONE_CONFIG}" + cat > "${RUNNER_S3_RCLONE_CONFIG}" < "${hash_path}" + + if s3_cache_get "${object_key}.sha256" "${remote_hash_path}"; then + remote_hash="$(tr -d '[:space:]' < "${remote_hash_path}")" + if [[ "${remote_hash}" == "${local_hash}" ]]; then + echo "S3 cache unchanged: ${object_key}" + rm -rf "${tmp_dir}" + return 0 + fi + echo "S3 cache changed: ${object_key} remote_sha256=${remote_hash} local_sha256=${local_hash}" + else + echo "No existing S3 cache hash for ${object_key}; uploading" + fi + + s3_cache_put "${object_key}" "${archive_path}" + s3_cache_put "${object_key}.sha256" "${hash_path}" + echo "Uploaded S3 cache: ${object_key} sha256=${local_hash}" + rm -rf "${tmp_dir}" +} + +s3_cache_upload_if_missing() { + local object_key="$1" + local archive_path="$2" + + echo "Uploading S3 cache if missing: ${object_key}" + s3_cache_rclone copyto --ignore-existing "${archive_path}" "$(s3_cache_remote_path "${object_key}")" +} diff --git a/.github/scripts/ci/save-go-cache.sh b/.github/scripts/ci/save-go-cache.sh new file mode 100755 index 0000000..2bc09ae --- /dev/null +++ b/.github/scripts/ci/save-go-cache.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +set -euo pipefail + +source "${GITHUB_WORKSPACE}/.github/scripts/ci/s3-cache-common.sh" + +if ! s3_cache_enabled; then + echo "S3 Go module cache save skipped: missing RUNNER_S3_AK/RUNNER_S3_SK or rclone" + exit 0 +fi + +if [[ -z "${CI_GO_MOD_HASH:-}" ]]; then + echo "S3 Go module cache save skipped: CI_GO_MOD_HASH is empty" + exit 0 +fi + +download_cache="${GOMODCACHE}/cache/download" +if [[ ! -d "${download_cache}" ]]; then + echo "S3 Go module download cache save skipped: ${download_cache} does not exist" + exit 0 +fi + +rm -rf "${GITHUB_WORKSPACE}/.cache/go/build-cache" +find "${download_cache}" -type f -name '*.lock' -delete + +tmp_root="$(mktemp -d)" +tmp_tar="${tmp_root}/go-cache.tar" +tmp_archive="${tmp_tar}.zst" +trap 'rm -rf "${tmp_root}"' EXIT + +mkdir -p "${GITHUB_WORKSPACE}/.cache/go/pkg/mod/cache/download" +tar \ + --sort=name \ + --mtime='UTC 1970-01-01' \ + --owner=0 \ + --group=0 \ + --numeric-owner \ + -C "${GITHUB_WORKSPACE}/.cache" \ + -cf "${tmp_tar}" \ + go/pkg/mod/cache/download +zstd -T0 -3 -q "${tmp_tar}" -o "${tmp_archive}" + +cache_key="${RUNNER_S3_PREFIX}/go-download/hash/${RUNNER_OS:-Linux}/${CI_GO_MOD_HASH}.tar.zst" +s3_cache_upload_if_missing "${cache_key}" "${tmp_archive}" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dcc47ce..68588cf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,6 +1,7 @@ name: CI on: + pull_request: push: branches: - main @@ -12,26 +13,85 @@ concurrency: group: ci-${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +env: + # Keep writable Go directories outside the runner template's built-in GOPATH. + GOPATH: ${{ github.workspace }}/.cache/go + GOMODCACHE: ${{ github.workspace }}/.cache/go/pkg/mod + GOCACHE: /tmp/go-build-cache + GOPRIVATE: github.com/qbox/* + + # Shared Qiniu S3 cache configuration. Credentials come from repository secrets. + RUNNER_S3_BUCKET: las-github-runner-dal + RUNNER_S3_ENDPOINT: s3.us-north-1.qiniucs.com + RUNNER_S3_REGION: us-north-1 + RUNNER_S3_AK: ${{ secrets.RUNNER_S3_AK }} + RUNNER_S3_SK: ${{ secrets.RUNNER_S3_SK }} + jobs: + prepare: + if: github.repository == '1024XEngineer/xinfra' + name: Prepare CI context + runs-on: github-runner-ubuntu-24-04 + outputs: + ci_go_mod_hash: ${{ steps.context.outputs.ci_go_mod_hash }} + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Calculate Go module cache key + id: context + shell: bash + run: | + set -euo pipefail + ci_go_mod_hash="$(sha256sum server/go.mod server/go.sum | sha256sum | awk '{print $1}')" + echo "ci_go_mod_hash=${ci_go_mod_hash}" >> "${GITHUB_OUTPUT}" + echo "CI_GO_MOD_HASH=${ci_go_mod_hash}" + + go-mod-cache: + if: github.repository == '1024XEngineer/xinfra' + name: Prepare Go module cache + needs: prepare + runs-on: github-runner-ubuntu-24-04 + env: + CI_GO_MOD_HASH: ${{ needs.prepare.outputs.ci_go_mod_hash }} + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Prepare Go environment + uses: ./.github/actions/ci-go-prep + + - name: Download Go modules + working-directory: server + shell: bash + run: | + set -euo pipefail + go mod download + + - name: Save Go module cache to S3 + if: success() + continue-on-error: true + shell: bash + run: bash "${GITHUB_WORKSPACE}/.github/scripts/ci/save-go-cache.sh" + backend-unit: + if: github.repository == '1024XEngineer/xinfra' name: Backend unit tests - runs-on: ubuntu-latest + needs: + - prepare + - go-mod-cache + runs-on: github-runner-ubuntu-24-04 + env: + CI_GO_MOD_HASH: ${{ needs.prepare.outputs.ci_go_mod_hash }} defaults: run: working-directory: server steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version-file: server/go.mod - cache: true - cache-dependency-path: server/go.sum - - - name: Download Go modules - run: go mod download + - name: Prepare Go environment + uses: ./.github/actions/ci-go-prep - name: Go vet run: go vet ./... @@ -48,21 +108,15 @@ jobs: if-no-files-found: ignore frontend-validation: + if: github.repository == '1024XEngineer/xinfra' name: Frontend typecheck and build - runs-on: ubuntu-latest + runs-on: github-runner-ubuntu-24-04 defaults: run: working-directory: frontend steps: - name: Checkout - uses: actions/checkout@v4 - - - name: Set up Node.js - uses: actions/setup-node@v4 - with: - node-version: 22 - cache: npm - cache-dependency-path: frontend/package-lock.json + uses: actions/checkout@v6 - name: Install dependencies run: npm ci @@ -71,11 +125,16 @@ jobs: run: npm run build integration: + if: github.repository == '1024XEngineer/xinfra' name: Backend integration smoke test - runs-on: ubuntu-latest needs: + - prepare + - go-mod-cache - backend-unit - frontend-validation + runs-on: github-runner-ubuntu-24-04 + env: + CI_GO_MOD_HASH: ${{ needs.prepare.outputs.ci_go_mod_hash }} services: mysql: image: mysql:8.0.46 @@ -94,18 +153,10 @@ jobs: --health-retries=12 steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version-file: server/go.mod - cache: true - cache-dependency-path: server/go.sum - - - name: Download Go modules - working-directory: server - run: go mod download + - name: Prepare Go environment + uses: ./.github/actions/ci-go-prep - name: Run integration smoke test working-directory: server