diff --git a/.github/actions/ci-go-prep/action.yml b/.github/actions/ci-go-prep/action.yml new file mode 100644 index 0000000..0744083 --- /dev/null +++ b/.github/actions/ci-go-prep/action.yml @@ -0,0 +1,16 @@ +name: CI Go prep +description: Prepare Go directories and restore the Go module cache. +runs: + using: composite + steps: + - name: Prepare Go directories + shell: bash + run: | + set -euo pipefail + # Keep writable Go directories outside the runner template's built-in GOPATH. + mkdir -p "${GOCACHE}" "${GOMODCACHE}" "${GOPATH}/bin" + + - name: Restore Go module cache from S3 + continue-on-error: true + shell: bash + run: bash "${GITHUB_WORKSPACE}/.github/scripts/ci/restore-go-cache.sh" diff --git a/.github/scripts/ci/restore-go-cache.sh b/.github/scripts/ci/restore-go-cache.sh new file mode 100755 index 0000000..cefc33f --- /dev/null +++ b/.github/scripts/ci/restore-go-cache.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +set -euo pipefail + +source "${GITHUB_WORKSPACE}/.github/scripts/ci/s3-cache-common.sh" + +mkdir -p "${GOMODCACHE}/cache/download" "${GOPATH}/bin" "${GOCACHE}" "${GITHUB_WORKSPACE}/.cache" +rm -rf "${GITHUB_WORKSPACE}/.cache/go/build-cache" + +if ! s3_cache_enabled; then + echo "S3 Go module cache restore skipped: missing RUNNER_S3_AK/RUNNER_S3_SK or rclone" + exit 0 +fi + +if [[ -z "${CI_GO_MOD_HASH:-}" ]]; then + echo "S3 Go module cache restore skipped: CI_GO_MOD_HASH is empty" + exit 0 +fi + +cache_key="${RUNNER_S3_PREFIX}/go-download/hash/${RUNNER_OS:-Linux}/${CI_GO_MOD_HASH}.tar.zst" +archive_path="$(mktemp)" +trap 'rm -f "${archive_path}"' EXIT + +if s3_cache_get "${cache_key}" "${archive_path}"; then + zstd -dc "${archive_path}" | tar -C "${GITHUB_WORKSPACE}/.cache" -xf - + echo "Restored S3 Go module download cache: ${cache_key}" +else + echo "S3 Go module download cache not found: ${cache_key}" +fi diff --git a/.github/scripts/ci/s3-cache-common.sh b/.github/scripts/ci/s3-cache-common.sh new file mode 100755 index 0000000..e112d69 --- /dev/null +++ b/.github/scripts/ci/s3-cache-common.sh @@ -0,0 +1,106 @@ +#!/usr/bin/env bash +set -euo pipefail + +: "${RUNNER_S3_BUCKET:=las-github-runner-dal}" +: "${RUNNER_S3_ENDPOINT:=s3.us-north-1.qiniucs.com}" +: "${RUNNER_S3_REGION:=us-north-1}" +: "${RUNNER_S3_PREFIX:=ci-cache/${GITHUB_REPOSITORY:?GITHUB_REPOSITORY is required}}" + +raw_job_name="${GITHUB_JOB:-unknown-job}" +RUNNER_S3_JOB_SCOPE="$(printf '%s' "${raw_job_name}" | sed 's/[^[:alnum:]._-]/-/g')" +export RUNNER_S3_JOB_SCOPE + +RUNNER_S3_RCLONE_CONFIG="" + +s3_cache_enabled() { + [[ -n "${RUNNER_S3_AK:-}" && -n "${RUNNER_S3_SK:-}" ]] && command -v rclone >/dev/null 2>&1 +} + +s3_cache_configure_rclone() { + if [[ -n "${RUNNER_S3_RCLONE_CONFIG}" ]]; then + return 0 + fi + + local endpoint="${RUNNER_S3_ENDPOINT}" + + if [[ ! "${endpoint}" =~ ^https?:// ]]; then + endpoint="https://${endpoint}" + fi + + RUNNER_S3_RCLONE_CONFIG="$(mktemp)" + chmod 600 "${RUNNER_S3_RCLONE_CONFIG}" + cat > "${RUNNER_S3_RCLONE_CONFIG}" < "${hash_path}" + + if s3_cache_get "${object_key}.sha256" "${remote_hash_path}"; then + remote_hash="$(tr -d '[:space:]' < "${remote_hash_path}")" + if [[ "${remote_hash}" == "${local_hash}" ]]; then + echo "S3 cache unchanged: ${object_key}" + rm -rf "${tmp_dir}" + return 0 + fi + echo "S3 cache changed: ${object_key} remote_sha256=${remote_hash} local_sha256=${local_hash}" + else + echo "No existing S3 cache hash for ${object_key}; uploading" + fi + + s3_cache_put "${object_key}" "${archive_path}" + s3_cache_put "${object_key}.sha256" "${hash_path}" + echo "Uploaded S3 cache: ${object_key} sha256=${local_hash}" + rm -rf "${tmp_dir}" +} + +s3_cache_upload_if_missing() { + local object_key="$1" + local archive_path="$2" + + echo "Uploading S3 cache if missing: ${object_key}" + s3_cache_rclone copyto --ignore-existing "${archive_path}" "$(s3_cache_remote_path "${object_key}")" +} diff --git a/.github/scripts/ci/save-go-cache.sh b/.github/scripts/ci/save-go-cache.sh new file mode 100755 index 0000000..2bc09ae --- /dev/null +++ b/.github/scripts/ci/save-go-cache.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +set -euo pipefail + +source "${GITHUB_WORKSPACE}/.github/scripts/ci/s3-cache-common.sh" + +if ! s3_cache_enabled; then + echo "S3 Go module cache save skipped: missing RUNNER_S3_AK/RUNNER_S3_SK or rclone" + exit 0 +fi + +if [[ -z "${CI_GO_MOD_HASH:-}" ]]; then + echo "S3 Go module cache save skipped: CI_GO_MOD_HASH is empty" + exit 0 +fi + +download_cache="${GOMODCACHE}/cache/download" +if [[ ! -d "${download_cache}" ]]; then + echo "S3 Go module download cache save skipped: ${download_cache} does not exist" + exit 0 +fi + +rm -rf "${GITHUB_WORKSPACE}/.cache/go/build-cache" +find "${download_cache}" -type f -name '*.lock' -delete + +tmp_root="$(mktemp -d)" +tmp_tar="${tmp_root}/go-cache.tar" +tmp_archive="${tmp_tar}.zst" +trap 'rm -rf "${tmp_root}"' EXIT + +mkdir -p "${GITHUB_WORKSPACE}/.cache/go/pkg/mod/cache/download" +tar \ + --sort=name \ + --mtime='UTC 1970-01-01' \ + --owner=0 \ + --group=0 \ + --numeric-owner \ + -C "${GITHUB_WORKSPACE}/.cache" \ + -cf "${tmp_tar}" \ + go/pkg/mod/cache/download +zstd -T0 -3 -q "${tmp_tar}" -o "${tmp_archive}" + +cache_key="${RUNNER_S3_PREFIX}/go-download/hash/${RUNNER_OS:-Linux}/${CI_GO_MOD_HASH}.tar.zst" +s3_cache_upload_if_missing "${cache_key}" "${tmp_archive}" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b061bd4..11a0387 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,6 +1,7 @@ name: CI on: + pull_request: push: branches: - main @@ -12,27 +13,87 @@ concurrency: group: ci-${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +env: + # Keep writable Go directories outside the runner template's built-in GOPATH. + GOPATH: ${{ github.workspace }}/.cache/go + GOMODCACHE: ${{ github.workspace }}/.cache/go/pkg/mod + GOCACHE: /tmp/go-build-cache + GOPRIVATE: github.com/qbox/* + + # Shared Qiniu S3 cache configuration. Credentials come from repository secrets. + RUNNER_S3_BUCKET: las-github-runner-dal + RUNNER_S3_ENDPOINT: s3.us-north-1.qiniucs.com + RUNNER_S3_REGION: us-north-1 + RUNNER_S3_AK: ${{ secrets.RUNNER_S3_AK }} + RUNNER_S3_SK: ${{ secrets.RUNNER_S3_SK }} + jobs: + prepare: + if: github.repository == '1024XEngineer/xinfra' + name: Prepare CI context + runs-on: github-runner-ubuntu-24-04 + outputs: + ci_go_mod_hash: ${{ steps.context.outputs.ci_go_mod_hash }} + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Calculate Go module cache key + id: context + shell: bash + run: | + set -euo pipefail + ci_go_mod_hash="$(sha256sum server/go.mod server/go.sum | sha256sum | awk '{print $1}')" + echo "ci_go_mod_hash=${ci_go_mod_hash}" >> "${GITHUB_OUTPUT}" + echo "CI_GO_MOD_HASH=${ci_go_mod_hash}" + + go-mod-cache: + if: github.repository == '1024XEngineer/xinfra' + name: Prepare Go module cache + needs: prepare + runs-on: github-runner-ubuntu-24-04 + env: + CI_GO_MOD_HASH: ${{ needs.prepare.outputs.ci_go_mod_hash }} + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Prepare Go environment + uses: ./.github/actions/ci-go-prep + + - name: Download and verify Go modules + working-directory: server + shell: bash + run: | + set -euo pipefail + go mod download + go mod tidy + git diff --exit-code -- go.mod go.sum + + - name: Save Go module cache to S3 + if: success() + continue-on-error: true + shell: bash + run: bash "${GITHUB_WORKSPACE}/.github/scripts/ci/save-go-cache.sh" + backend-unit: if: github.repository == '1024XEngineer/xinfra' name: Backend unit tests - runs-on: ubuntu-latest + needs: + - prepare + - go-mod-cache + runs-on: github-runner-ubuntu-24-04 + env: + CI_GO_MOD_HASH: ${{ needs.prepare.outputs.ci_go_mod_hash }} defaults: run: working-directory: server steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version-file: server/go.mod - cache: true - cache-dependency-path: server/go.sum - - - name: Download Go modules - run: go mod download + - name: Prepare Go environment + uses: ./.github/actions/ci-go-prep - name: Go vet run: go vet ./... @@ -51,20 +112,13 @@ jobs: frontend-validation: if: github.repository == '1024XEngineer/xinfra' name: Frontend typecheck and build - runs-on: ubuntu-latest + runs-on: github-runner-ubuntu-24-04 defaults: run: working-directory: frontend steps: - name: Checkout - uses: actions/checkout@v4 - - - name: Set up Node.js - uses: actions/setup-node@v4 - with: - node-version: 22 - cache: npm - cache-dependency-path: frontend/package-lock.json + uses: actions/checkout@v6 - name: Install dependencies run: npm ci @@ -75,10 +129,14 @@ jobs: integration: if: github.repository == '1024XEngineer/xinfra' name: Backend integration smoke test - runs-on: ubuntu-latest needs: + - prepare + - go-mod-cache - backend-unit - frontend-validation + runs-on: github-runner-ubuntu-24-04 + env: + CI_GO_MOD_HASH: ${{ needs.prepare.outputs.ci_go_mod_hash }} services: mysql: image: mysql:8.0.46 @@ -97,18 +155,10 @@ jobs: --health-retries=12 steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version-file: server/go.mod - cache: true - cache-dependency-path: server/go.sum - - - name: Download Go modules - working-directory: server - run: go mod download + - name: Prepare Go environment + uses: ./.github/actions/ci-go-prep - name: Run integration smoke test working-directory: server diff --git a/server/go.mod b/server/go.mod index 49286f2..7a9c1d3 100644 --- a/server/go.mod +++ b/server/go.mod @@ -20,7 +20,6 @@ require ( github.com/bytedance/sonic v1.14.0 // indirect github.com/bytedance/sonic/loader v0.3.0 // indirect github.com/cloudwego/base64x v0.1.6 // indirect - github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d // indirect github.com/gabriel-vasile/mimetype v1.4.8 // indirect github.com/gin-contrib/sse v1.1.0 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect @@ -46,11 +45,8 @@ require ( github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/quic-go/qpack v0.5.1 // indirect github.com/quic-go/quic-go v0.54.0 // indirect - github.com/russross/blackfriday/v2 v2.0.1 // indirect - github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.3.0 // indirect - github.com/urfave/cli/v2 v2.3.0 // indirect go.uber.org/mock v0.5.0 // indirect golang.org/x/arch v0.20.0 // indirect golang.org/x/crypto v0.54.0 // indirect @@ -62,5 +58,4 @@ require ( golang.org/x/tools v0.47.0 // indirect google.golang.org/protobuf v1.36.9 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - sigs.k8s.io/yaml v1.3.0 // indirect ) diff --git a/server/go.sum b/server/go.sum index 48fc442..4bd0797 100644 --- a/server/go.sum +++ b/server/go.sum @@ -1,6 +1,5 @@ filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= @@ -13,8 +12,6 @@ github.com/bytedance/sonic/loader v0.3.0 h1:dskwH8edlzNMctoruo8FPTJDF3vLtDT0sXZw github.com/bytedance/sonic/loader v0.3.0/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= -github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= -github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -94,10 +91,6 @@ github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg= github.com/quic-go/quic-go v0.54.0 h1:6s1YB9QotYI6Ospeiguknbp2Znb/jZYjZLRXn9kMQBg= github.com/quic-go/quic-go v0.54.0/go.mod h1:e68ZEaCdyviluZmy44P6Iey98v/Wfz6HCjQEm+l8zTY= -github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -118,8 +111,6 @@ github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v1.3.0 h1:Qd2W2sQawAfG8XSvzwhBeoGq71zXOC/Q1E9y/wUcsUA= github.com/ugorji/go/codec v1.3.0/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4= -github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M= -github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU= go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM= @@ -176,7 +167,6 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= @@ -187,5 +177,3 @@ gorm.io/driver/mysql v1.6.0 h1:eNbLmNTpPpTOVZi8MMxCi2aaIm0ZpInbORNXDwyLGvg= gorm.io/driver/mysql v1.6.0/go.mod h1:D/oCC2GWK3M/dqoLxnOlaNKmXz8WNTfcS9y5ovaSqKo= gorm.io/gorm v1.30.1 h1:lSHg33jJTBxs2mgJRfRZeLDG+WZaHYCk3Wtfl6Ngzo4= gorm.io/gorm v1.30.1/go.mod h1:8Z33v652h4//uMA76KjeDH8mJXPm1QNCYrMeatR0DOE= -sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= -sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=