ci: migrate to Qiniu sandbox runner
This commit is contained in:
Executable
+28
@@ -0,0 +1,28 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
source "${GITHUB_WORKSPACE}/.github/scripts/ci/s3-cache-common.sh"
|
||||
|
||||
mkdir -p "${GOMODCACHE}/cache/download" "${GOPATH}/bin" "${GOCACHE}" "${GITHUB_WORKSPACE}/.cache"
|
||||
rm -rf "${GITHUB_WORKSPACE}/.cache/go/build-cache"
|
||||
|
||||
if ! s3_cache_enabled; then
|
||||
echo "S3 Go module cache restore skipped: missing RUNNER_S3_AK/RUNNER_S3_SK or rclone"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ -z "${CI_GO_MOD_HASH:-}" ]]; then
|
||||
echo "S3 Go module cache restore skipped: CI_GO_MOD_HASH is empty"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
cache_key="${RUNNER_S3_PREFIX}/go-download/hash/${RUNNER_OS:-Linux}/${CI_GO_MOD_HASH}.tar.zst"
|
||||
archive_path="$(mktemp)"
|
||||
trap 'rm -f "${archive_path}"' EXIT
|
||||
|
||||
if s3_cache_get "${cache_key}" "${archive_path}"; then
|
||||
zstd -dc "${archive_path}" | tar -C "${GITHUB_WORKSPACE}/.cache" -xf -
|
||||
echo "Restored S3 Go module download cache: ${cache_key}"
|
||||
else
|
||||
echo "S3 Go module download cache not found: ${cache_key}"
|
||||
fi
|
||||
Executable
+106
@@ -0,0 +1,106 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
: "${RUNNER_S3_BUCKET:=las-github-runner-dal}"
|
||||
: "${RUNNER_S3_ENDPOINT:=s3.us-north-1.qiniucs.com}"
|
||||
: "${RUNNER_S3_REGION:=us-north-1}"
|
||||
: "${RUNNER_S3_PREFIX:=ci-cache/${GITHUB_REPOSITORY:?GITHUB_REPOSITORY is required}}"
|
||||
|
||||
raw_job_name="${GITHUB_JOB:-unknown-job}"
|
||||
RUNNER_S3_JOB_SCOPE="$(printf '%s' "${raw_job_name}" | sed 's/[^[:alnum:]._-]/-/g')"
|
||||
export RUNNER_S3_JOB_SCOPE
|
||||
|
||||
RUNNER_S3_RCLONE_CONFIG=""
|
||||
|
||||
s3_cache_enabled() {
|
||||
[[ -n "${RUNNER_S3_AK:-}" && -n "${RUNNER_S3_SK:-}" ]] && command -v rclone >/dev/null 2>&1
|
||||
}
|
||||
|
||||
s3_cache_configure_rclone() {
|
||||
if [[ -n "${RUNNER_S3_RCLONE_CONFIG}" ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
local endpoint="${RUNNER_S3_ENDPOINT}"
|
||||
|
||||
if [[ ! "${endpoint}" =~ ^https?:// ]]; then
|
||||
endpoint="https://${endpoint}"
|
||||
fi
|
||||
|
||||
RUNNER_S3_RCLONE_CONFIG="$(mktemp)"
|
||||
chmod 600 "${RUNNER_S3_RCLONE_CONFIG}"
|
||||
cat > "${RUNNER_S3_RCLONE_CONFIG}" <<CONFIG
|
||||
[cis3]
|
||||
type = s3
|
||||
provider = Qiniu
|
||||
access_key_id = ${RUNNER_S3_AK}
|
||||
secret_access_key = ${RUNNER_S3_SK}
|
||||
region = ${RUNNER_S3_REGION}
|
||||
endpoint = ${endpoint}
|
||||
force_path_style = false
|
||||
no_check_bucket = true
|
||||
CONFIG
|
||||
export RUNNER_S3_RCLONE_CONFIG
|
||||
}
|
||||
|
||||
s3_cache_remote_path() {
|
||||
printf 'cis3:%s/%s' "${RUNNER_S3_BUCKET}" "${1#/}"
|
||||
}
|
||||
|
||||
s3_cache_rclone() {
|
||||
s3_cache_configure_rclone
|
||||
rclone --config "${RUNNER_S3_RCLONE_CONFIG}" --stats=0 --retries "${RUNNER_S3_RCLONE_RETRIES:-4}" "$@"
|
||||
}
|
||||
|
||||
s3_cache_get() {
|
||||
local output_path="$2"
|
||||
|
||||
rm -f "${output_path}"
|
||||
s3_cache_rclone copyto "$(s3_cache_remote_path "$1")" "${output_path}"
|
||||
[[ -s "${output_path}" ]]
|
||||
}
|
||||
|
||||
s3_cache_put() {
|
||||
s3_cache_rclone copyto "$2" "$(s3_cache_remote_path "$1")"
|
||||
}
|
||||
|
||||
s3_cache_upload_if_changed() {
|
||||
local object_key="$1"
|
||||
local archive_path="$2"
|
||||
local tmp_dir
|
||||
local hash_path
|
||||
local remote_hash_path
|
||||
local local_hash
|
||||
local remote_hash
|
||||
|
||||
tmp_dir="$(mktemp -d)"
|
||||
hash_path="${tmp_dir}/local.sha256"
|
||||
remote_hash_path="${tmp_dir}/remote.sha256"
|
||||
local_hash="$(sha256sum "${archive_path}" | awk '{print $1}')"
|
||||
printf '%s\n' "${local_hash}" > "${hash_path}"
|
||||
|
||||
if s3_cache_get "${object_key}.sha256" "${remote_hash_path}"; then
|
||||
remote_hash="$(tr -d '[:space:]' < "${remote_hash_path}")"
|
||||
if [[ "${remote_hash}" == "${local_hash}" ]]; then
|
||||
echo "S3 cache unchanged: ${object_key}"
|
||||
rm -rf "${tmp_dir}"
|
||||
return 0
|
||||
fi
|
||||
echo "S3 cache changed: ${object_key} remote_sha256=${remote_hash} local_sha256=${local_hash}"
|
||||
else
|
||||
echo "No existing S3 cache hash for ${object_key}; uploading"
|
||||
fi
|
||||
|
||||
s3_cache_put "${object_key}" "${archive_path}"
|
||||
s3_cache_put "${object_key}.sha256" "${hash_path}"
|
||||
echo "Uploaded S3 cache: ${object_key} sha256=${local_hash}"
|
||||
rm -rf "${tmp_dir}"
|
||||
}
|
||||
|
||||
s3_cache_upload_if_missing() {
|
||||
local object_key="$1"
|
||||
local archive_path="$2"
|
||||
|
||||
echo "Uploading S3 cache if missing: ${object_key}"
|
||||
s3_cache_rclone copyto --ignore-existing "${archive_path}" "$(s3_cache_remote_path "${object_key}")"
|
||||
}
|
||||
Executable
+43
@@ -0,0 +1,43 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
source "${GITHUB_WORKSPACE}/.github/scripts/ci/s3-cache-common.sh"
|
||||
|
||||
if ! s3_cache_enabled; then
|
||||
echo "S3 Go module cache save skipped: missing RUNNER_S3_AK/RUNNER_S3_SK or rclone"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ -z "${CI_GO_MOD_HASH:-}" ]]; then
|
||||
echo "S3 Go module cache save skipped: CI_GO_MOD_HASH is empty"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
download_cache="${GOMODCACHE}/cache/download"
|
||||
if [[ ! -d "${download_cache}" ]]; then
|
||||
echo "S3 Go module download cache save skipped: ${download_cache} does not exist"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
rm -rf "${GITHUB_WORKSPACE}/.cache/go/build-cache"
|
||||
find "${download_cache}" -type f -name '*.lock' -delete
|
||||
|
||||
tmp_root="$(mktemp -d)"
|
||||
tmp_tar="${tmp_root}/go-cache.tar"
|
||||
tmp_archive="${tmp_tar}.zst"
|
||||
trap 'rm -rf "${tmp_root}"' EXIT
|
||||
|
||||
mkdir -p "${GITHUB_WORKSPACE}/.cache/go/pkg/mod/cache/download"
|
||||
tar \
|
||||
--sort=name \
|
||||
--mtime='UTC 1970-01-01' \
|
||||
--owner=0 \
|
||||
--group=0 \
|
||||
--numeric-owner \
|
||||
-C "${GITHUB_WORKSPACE}/.cache" \
|
||||
-cf "${tmp_tar}" \
|
||||
go/pkg/mod/cache/download
|
||||
zstd -T0 -3 -q "${tmp_tar}" -o "${tmp_archive}"
|
||||
|
||||
cache_key="${RUNNER_S3_PREFIX}/go-download/hash/${RUNNER_OS:-Linux}/${CI_GO_MOD_HASH}.tar.zst"
|
||||
s3_cache_upload_if_missing "${cache_key}" "${tmp_archive}"
|
||||
Reference in New Issue
Block a user