Merge pull request #131 from Gmaker689/ci/github-actions
Ci/GitHub actions
This commit is contained in:
GitHub
@@ -0,0 +1,16 @@
|
||||
name: CI Go prep
|
||||
description: Prepare Go directories and restore the Go module cache.
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Prepare Go directories
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# Keep writable Go directories outside the runner template's built-in GOPATH.
|
||||
mkdir -p "${GOCACHE}" "${GOMODCACHE}" "${GOPATH}/bin"
|
||||
|
||||
- name: Restore Go module cache from S3
|
||||
continue-on-error: true
|
||||
shell: bash
|
||||
run: bash "${GITHUB_WORKSPACE}/.github/scripts/ci/restore-go-cache.sh"
|
||||
Executable
+28
@@ -0,0 +1,28 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
source "${GITHUB_WORKSPACE}/.github/scripts/ci/s3-cache-common.sh"
|
||||
|
||||
mkdir -p "${GOMODCACHE}/cache/download" "${GOPATH}/bin" "${GOCACHE}" "${GITHUB_WORKSPACE}/.cache"
|
||||
rm -rf "${GITHUB_WORKSPACE}/.cache/go/build-cache"
|
||||
|
||||
if ! s3_cache_enabled; then
|
||||
echo "S3 Go module cache restore skipped: missing RUNNER_S3_AK/RUNNER_S3_SK or rclone"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ -z "${CI_GO_MOD_HASH:-}" ]]; then
|
||||
echo "S3 Go module cache restore skipped: CI_GO_MOD_HASH is empty"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
cache_key="${RUNNER_S3_PREFIX}/go-download/hash/${RUNNER_OS:-Linux}/${CI_GO_MOD_HASH}.tar.zst"
|
||||
archive_path="$(mktemp)"
|
||||
trap 'rm -f "${archive_path}"' EXIT
|
||||
|
||||
if s3_cache_get "${cache_key}" "${archive_path}"; then
|
||||
zstd -dc "${archive_path}" | tar -C "${GITHUB_WORKSPACE}/.cache" -xf -
|
||||
echo "Restored S3 Go module download cache: ${cache_key}"
|
||||
else
|
||||
echo "S3 Go module download cache not found: ${cache_key}"
|
||||
fi
|
||||
Executable
+106
@@ -0,0 +1,106 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
: "${RUNNER_S3_BUCKET:=las-github-runner-dal}"
|
||||
: "${RUNNER_S3_ENDPOINT:=s3.us-north-1.qiniucs.com}"
|
||||
: "${RUNNER_S3_REGION:=us-north-1}"
|
||||
: "${RUNNER_S3_PREFIX:=ci-cache/${GITHUB_REPOSITORY:?GITHUB_REPOSITORY is required}}"
|
||||
|
||||
raw_job_name="${GITHUB_JOB:-unknown-job}"
|
||||
RUNNER_S3_JOB_SCOPE="$(printf '%s' "${raw_job_name}" | sed 's/[^[:alnum:]._-]/-/g')"
|
||||
export RUNNER_S3_JOB_SCOPE
|
||||
|
||||
RUNNER_S3_RCLONE_CONFIG=""
|
||||
|
||||
s3_cache_enabled() {
|
||||
[[ -n "${RUNNER_S3_AK:-}" && -n "${RUNNER_S3_SK:-}" ]] && command -v rclone >/dev/null 2>&1
|
||||
}
|
||||
|
||||
s3_cache_configure_rclone() {
|
||||
if [[ -n "${RUNNER_S3_RCLONE_CONFIG}" ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
local endpoint="${RUNNER_S3_ENDPOINT}"
|
||||
|
||||
if [[ ! "${endpoint}" =~ ^https?:// ]]; then
|
||||
endpoint="https://${endpoint}"
|
||||
fi
|
||||
|
||||
RUNNER_S3_RCLONE_CONFIG="$(mktemp)"
|
||||
chmod 600 "${RUNNER_S3_RCLONE_CONFIG}"
|
||||
cat > "${RUNNER_S3_RCLONE_CONFIG}" <<CONFIG
|
||||
[cis3]
|
||||
type = s3
|
||||
provider = Qiniu
|
||||
access_key_id = ${RUNNER_S3_AK}
|
||||
secret_access_key = ${RUNNER_S3_SK}
|
||||
region = ${RUNNER_S3_REGION}
|
||||
endpoint = ${endpoint}
|
||||
force_path_style = false
|
||||
no_check_bucket = true
|
||||
CONFIG
|
||||
export RUNNER_S3_RCLONE_CONFIG
|
||||
}
|
||||
|
||||
s3_cache_remote_path() {
|
||||
printf 'cis3:%s/%s' "${RUNNER_S3_BUCKET}" "${1#/}"
|
||||
}
|
||||
|
||||
s3_cache_rclone() {
|
||||
s3_cache_configure_rclone
|
||||
rclone --config "${RUNNER_S3_RCLONE_CONFIG}" --stats=0 --retries "${RUNNER_S3_RCLONE_RETRIES:-4}" "$@"
|
||||
}
|
||||
|
||||
s3_cache_get() {
|
||||
local output_path="$2"
|
||||
|
||||
rm -f "${output_path}"
|
||||
s3_cache_rclone copyto "$(s3_cache_remote_path "$1")" "${output_path}"
|
||||
[[ -s "${output_path}" ]]
|
||||
}
|
||||
|
||||
s3_cache_put() {
|
||||
s3_cache_rclone copyto "$2" "$(s3_cache_remote_path "$1")"
|
||||
}
|
||||
|
||||
s3_cache_upload_if_changed() {
|
||||
local object_key="$1"
|
||||
local archive_path="$2"
|
||||
local tmp_dir
|
||||
local hash_path
|
||||
local remote_hash_path
|
||||
local local_hash
|
||||
local remote_hash
|
||||
|
||||
tmp_dir="$(mktemp -d)"
|
||||
hash_path="${tmp_dir}/local.sha256"
|
||||
remote_hash_path="${tmp_dir}/remote.sha256"
|
||||
local_hash="$(sha256sum "${archive_path}" | awk '{print $1}')"
|
||||
printf '%s\n' "${local_hash}" > "${hash_path}"
|
||||
|
||||
if s3_cache_get "${object_key}.sha256" "${remote_hash_path}"; then
|
||||
remote_hash="$(tr -d '[:space:]' < "${remote_hash_path}")"
|
||||
if [[ "${remote_hash}" == "${local_hash}" ]]; then
|
||||
echo "S3 cache unchanged: ${object_key}"
|
||||
rm -rf "${tmp_dir}"
|
||||
return 0
|
||||
fi
|
||||
echo "S3 cache changed: ${object_key} remote_sha256=${remote_hash} local_sha256=${local_hash}"
|
||||
else
|
||||
echo "No existing S3 cache hash for ${object_key}; uploading"
|
||||
fi
|
||||
|
||||
s3_cache_put "${object_key}" "${archive_path}"
|
||||
s3_cache_put "${object_key}.sha256" "${hash_path}"
|
||||
echo "Uploaded S3 cache: ${object_key} sha256=${local_hash}"
|
||||
rm -rf "${tmp_dir}"
|
||||
}
|
||||
|
||||
s3_cache_upload_if_missing() {
|
||||
local object_key="$1"
|
||||
local archive_path="$2"
|
||||
|
||||
echo "Uploading S3 cache if missing: ${object_key}"
|
||||
s3_cache_rclone copyto --ignore-existing "${archive_path}" "$(s3_cache_remote_path "${object_key}")"
|
||||
}
|
||||
Executable
+43
@@ -0,0 +1,43 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
source "${GITHUB_WORKSPACE}/.github/scripts/ci/s3-cache-common.sh"
|
||||
|
||||
if ! s3_cache_enabled; then
|
||||
echo "S3 Go module cache save skipped: missing RUNNER_S3_AK/RUNNER_S3_SK or rclone"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ -z "${CI_GO_MOD_HASH:-}" ]]; then
|
||||
echo "S3 Go module cache save skipped: CI_GO_MOD_HASH is empty"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
download_cache="${GOMODCACHE}/cache/download"
|
||||
if [[ ! -d "${download_cache}" ]]; then
|
||||
echo "S3 Go module download cache save skipped: ${download_cache} does not exist"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
rm -rf "${GITHUB_WORKSPACE}/.cache/go/build-cache"
|
||||
find "${download_cache}" -type f -name '*.lock' -delete
|
||||
|
||||
tmp_root="$(mktemp -d)"
|
||||
tmp_tar="${tmp_root}/go-cache.tar"
|
||||
tmp_archive="${tmp_tar}.zst"
|
||||
trap 'rm -rf "${tmp_root}"' EXIT
|
||||
|
||||
mkdir -p "${GITHUB_WORKSPACE}/.cache/go/pkg/mod/cache/download"
|
||||
tar \
|
||||
--sort=name \
|
||||
--mtime='UTC 1970-01-01' \
|
||||
--owner=0 \
|
||||
--group=0 \
|
||||
--numeric-owner \
|
||||
-C "${GITHUB_WORKSPACE}/.cache" \
|
||||
-cf "${tmp_tar}" \
|
||||
go/pkg/mod/cache/download
|
||||
zstd -T0 -3 -q "${tmp_tar}" -o "${tmp_archive}"
|
||||
|
||||
cache_key="${RUNNER_S3_PREFIX}/go-download/hash/${RUNNER_OS:-Linux}/${CI_GO_MOD_HASH}.tar.zst"
|
||||
s3_cache_upload_if_missing "${cache_key}" "${tmp_archive}"
|
||||
+83
-32
@@ -1,6 +1,7 @@
|
||||
name: CI
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
@@ -12,26 +13,85 @@ concurrency:
|
||||
group: ci-${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
# Keep writable Go directories outside the runner template's built-in GOPATH.
|
||||
GOPATH: ${{ github.workspace }}/.cache/go
|
||||
GOMODCACHE: ${{ github.workspace }}/.cache/go/pkg/mod
|
||||
GOCACHE: /tmp/go-build-cache
|
||||
GOPRIVATE: github.com/qbox/*
|
||||
|
||||
# Shared Qiniu S3 cache configuration. Credentials come from repository secrets.
|
||||
RUNNER_S3_BUCKET: las-github-runner-dal
|
||||
RUNNER_S3_ENDPOINT: s3.us-north-1.qiniucs.com
|
||||
RUNNER_S3_REGION: us-north-1
|
||||
RUNNER_S3_AK: ${{ secrets.RUNNER_S3_AK }}
|
||||
RUNNER_S3_SK: ${{ secrets.RUNNER_S3_SK }}
|
||||
|
||||
jobs:
|
||||
prepare:
|
||||
if: github.repository == '1024XEngineer/xinfra'
|
||||
name: Prepare CI context
|
||||
runs-on: github-runner-ubuntu-24-04
|
||||
outputs:
|
||||
ci_go_mod_hash: ${{ steps.context.outputs.ci_go_mod_hash }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Calculate Go module cache key
|
||||
id: context
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
ci_go_mod_hash="$(sha256sum server/go.mod server/go.sum | sha256sum | awk '{print $1}')"
|
||||
echo "ci_go_mod_hash=${ci_go_mod_hash}" >> "${GITHUB_OUTPUT}"
|
||||
echo "CI_GO_MOD_HASH=${ci_go_mod_hash}"
|
||||
|
||||
go-mod-cache:
|
||||
if: github.repository == '1024XEngineer/xinfra'
|
||||
name: Prepare Go module cache
|
||||
needs: prepare
|
||||
runs-on: github-runner-ubuntu-24-04
|
||||
env:
|
||||
CI_GO_MOD_HASH: ${{ needs.prepare.outputs.ci_go_mod_hash }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Prepare Go environment
|
||||
uses: ./.github/actions/ci-go-prep
|
||||
|
||||
- name: Download Go modules
|
||||
working-directory: server
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
go mod download
|
||||
|
||||
- name: Save Go module cache to S3
|
||||
if: success()
|
||||
continue-on-error: true
|
||||
shell: bash
|
||||
run: bash "${GITHUB_WORKSPACE}/.github/scripts/ci/save-go-cache.sh"
|
||||
|
||||
backend-unit:
|
||||
if: github.repository == '1024XEngineer/xinfra'
|
||||
name: Backend unit tests
|
||||
runs-on: ubuntu-latest
|
||||
needs:
|
||||
- prepare
|
||||
- go-mod-cache
|
||||
runs-on: github-runner-ubuntu-24-04
|
||||
env:
|
||||
CI_GO_MOD_HASH: ${{ needs.prepare.outputs.ci_go_mod_hash }}
|
||||
defaults:
|
||||
run:
|
||||
working-directory: server
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: server/go.mod
|
||||
cache: true
|
||||
cache-dependency-path: server/go.sum
|
||||
|
||||
- name: Download Go modules
|
||||
run: go mod download
|
||||
- name: Prepare Go environment
|
||||
uses: ./.github/actions/ci-go-prep
|
||||
|
||||
- name: Go vet
|
||||
run: go vet ./...
|
||||
@@ -48,21 +108,15 @@ jobs:
|
||||
if-no-files-found: ignore
|
||||
|
||||
frontend-validation:
|
||||
if: github.repository == '1024XEngineer/xinfra'
|
||||
name: Frontend typecheck and build
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: github-runner-ubuntu-24-04
|
||||
defaults:
|
||||
run:
|
||||
working-directory: frontend
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 22
|
||||
cache: npm
|
||||
cache-dependency-path: frontend/package-lock.json
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
@@ -71,11 +125,16 @@ jobs:
|
||||
run: npm run build
|
||||
|
||||
integration:
|
||||
if: github.repository == '1024XEngineer/xinfra'
|
||||
name: Backend integration smoke test
|
||||
runs-on: ubuntu-latest
|
||||
needs:
|
||||
- prepare
|
||||
- go-mod-cache
|
||||
- backend-unit
|
||||
- frontend-validation
|
||||
runs-on: github-runner-ubuntu-24-04
|
||||
env:
|
||||
CI_GO_MOD_HASH: ${{ needs.prepare.outputs.ci_go_mod_hash }}
|
||||
services:
|
||||
mysql:
|
||||
image: mysql:8.0.46
|
||||
@@ -94,18 +153,10 @@ jobs:
|
||||
--health-retries=12
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: server/go.mod
|
||||
cache: true
|
||||
cache-dependency-path: server/go.sum
|
||||
|
||||
- name: Download Go modules
|
||||
working-directory: server
|
||||
run: go mod download
|
||||
- name: Prepare Go environment
|
||||
uses: ./.github/actions/ci-go-prep
|
||||
|
||||
- name: Run integration smoke test
|
||||
working-directory: server
|
||||
|
||||
Reference in New Issue
Block a user