107 lines
2.9 KiB
Bash
Executable File
107 lines
2.9 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
: "${RUNNER_S3_BUCKET:=las-github-runner-dal}"
|
|
: "${RUNNER_S3_ENDPOINT:=s3.us-north-1.qiniucs.com}"
|
|
: "${RUNNER_S3_REGION:=us-north-1}"
|
|
: "${RUNNER_S3_PREFIX:=ci-cache/${GITHUB_REPOSITORY:?GITHUB_REPOSITORY is required}}"
|
|
|
|
raw_job_name="${GITHUB_JOB:-unknown-job}"
|
|
RUNNER_S3_JOB_SCOPE="$(printf '%s' "${raw_job_name}" | sed 's/[^[:alnum:]._-]/-/g')"
|
|
export RUNNER_S3_JOB_SCOPE
|
|
|
|
RUNNER_S3_RCLONE_CONFIG=""
|
|
|
|
s3_cache_enabled() {
|
|
[[ -n "${RUNNER_S3_AK:-}" && -n "${RUNNER_S3_SK:-}" ]] && command -v rclone >/dev/null 2>&1
|
|
}
|
|
|
|
s3_cache_configure_rclone() {
|
|
if [[ -n "${RUNNER_S3_RCLONE_CONFIG}" ]]; then
|
|
return 0
|
|
fi
|
|
|
|
local endpoint="${RUNNER_S3_ENDPOINT}"
|
|
|
|
if [[ ! "${endpoint}" =~ ^https?:// ]]; then
|
|
endpoint="https://${endpoint}"
|
|
fi
|
|
|
|
RUNNER_S3_RCLONE_CONFIG="$(mktemp)"
|
|
chmod 600 "${RUNNER_S3_RCLONE_CONFIG}"
|
|
cat > "${RUNNER_S3_RCLONE_CONFIG}" <<CONFIG
|
|
[cis3]
|
|
type = s3
|
|
provider = Qiniu
|
|
access_key_id = ${RUNNER_S3_AK}
|
|
secret_access_key = ${RUNNER_S3_SK}
|
|
region = ${RUNNER_S3_REGION}
|
|
endpoint = ${endpoint}
|
|
force_path_style = false
|
|
no_check_bucket = true
|
|
CONFIG
|
|
export RUNNER_S3_RCLONE_CONFIG
|
|
}
|
|
|
|
s3_cache_remote_path() {
|
|
printf 'cis3:%s/%s' "${RUNNER_S3_BUCKET}" "${1#/}"
|
|
}
|
|
|
|
s3_cache_rclone() {
|
|
s3_cache_configure_rclone
|
|
rclone --config "${RUNNER_S3_RCLONE_CONFIG}" --stats=0 --retries "${RUNNER_S3_RCLONE_RETRIES:-4}" "$@"
|
|
}
|
|
|
|
s3_cache_get() {
|
|
local output_path="$2"
|
|
|
|
rm -f "${output_path}"
|
|
s3_cache_rclone copyto "$(s3_cache_remote_path "$1")" "${output_path}"
|
|
[[ -s "${output_path}" ]]
|
|
}
|
|
|
|
s3_cache_put() {
|
|
s3_cache_rclone copyto "$2" "$(s3_cache_remote_path "$1")"
|
|
}
|
|
|
|
s3_cache_upload_if_changed() {
|
|
local object_key="$1"
|
|
local archive_path="$2"
|
|
local tmp_dir
|
|
local hash_path
|
|
local remote_hash_path
|
|
local local_hash
|
|
local remote_hash
|
|
|
|
tmp_dir="$(mktemp -d)"
|
|
hash_path="${tmp_dir}/local.sha256"
|
|
remote_hash_path="${tmp_dir}/remote.sha256"
|
|
local_hash="$(sha256sum "${archive_path}" | awk '{print $1}')"
|
|
printf '%s\n' "${local_hash}" > "${hash_path}"
|
|
|
|
if s3_cache_get "${object_key}.sha256" "${remote_hash_path}"; then
|
|
remote_hash="$(tr -d '[:space:]' < "${remote_hash_path}")"
|
|
if [[ "${remote_hash}" == "${local_hash}" ]]; then
|
|
echo "S3 cache unchanged: ${object_key}"
|
|
rm -rf "${tmp_dir}"
|
|
return 0
|
|
fi
|
|
echo "S3 cache changed: ${object_key} remote_sha256=${remote_hash} local_sha256=${local_hash}"
|
|
else
|
|
echo "No existing S3 cache hash for ${object_key}; uploading"
|
|
fi
|
|
|
|
s3_cache_put "${object_key}" "${archive_path}"
|
|
s3_cache_put "${object_key}.sha256" "${hash_path}"
|
|
echo "Uploaded S3 cache: ${object_key} sha256=${local_hash}"
|
|
rm -rf "${tmp_dir}"
|
|
}
|
|
|
|
s3_cache_upload_if_missing() {
|
|
local object_key="$1"
|
|
local archive_path="$2"
|
|
|
|
echo "Uploading S3 cache if missing: ${object_key}"
|
|
s3_cache_rclone copyto --ignore-existing "${archive_path}" "$(s3_cache_remote_path "${object_key}")"
|
|
}
|