feat(delivery): integrate PostgreSQL delivery

This commit is contained in:
l x
2026-07-31 10:30:03 +08:00
parent 048ac6507e
commit 66fde73792
25 changed files with 2110 additions and 139 deletions
+29
View File
@@ -0,0 +1,29 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
REPO_ROOT=$(cd -- "${SCRIPT_DIR}/.." && pwd)
CONFIG_FILE=${1:-"${REPO_ROOT}/config/local/server-postgresql.env"}
[[ -f ${CONFIG_FILE} ]] || {
printf 'error: configuration file not found: %s\n' "${CONFIG_FILE}" >&2
exit 1
}
set -a
# shellcheck disable=SC1090
source "${CONFIG_FILE}"
set +a
if [[ -z ${AWX_TOKEN:-} && -z ${AWX_PASSWORD:-} ]]; then
AWX_K8S_NAMESPACE=${AWX_K8S_NAMESPACE:-awx}
AWX_K8S_ADMIN_PASSWORD_SECRET=${AWX_K8S_ADMIN_PASSWORD_SECRET:-awx-demo-admin-password}
AWX_PASSWORD=$(kubectl get secret \
-n "${AWX_K8S_NAMESPACE}" "${AWX_K8S_ADMIN_PASSWORD_SECRET}" \
-o jsonpath='{.data.password}' | base64 --decode)
export AWX_PASSWORD
fi
cd "${REPO_ROOT}/server"
exec go run ./cmd/server
+343
View File
@@ -0,0 +1,343 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
REPO_ROOT=$(cd -- "${SCRIPT_DIR}/.." && pwd)
CONFIG_FILE=${1:-"${REPO_ROOT}/config/local/postgresql-awx.env"}
die() {
printf 'error: %s\n' "$*" >&2
exit 1
}
require_command() {
command -v "$1" >/dev/null 2>&1 || die "required command not found: $1"
}
require_value() {
local name=$1
[[ -n ${!name:-} ]] || die "${name} must be set in ${CONFIG_FILE}"
}
[[ -f ${CONFIG_FILE} ]] || die "configuration file not found: ${CONFIG_FILE}"
set -a
# shellcheck disable=SC1090
source "${CONFIG_FILE}"
set +a
for command_name in curl jq kubectl base64 openssl; do
require_command "${command_name}"
done
for variable_name in \
AWX_BASE_URL AWX_ORGANIZATION AWX_PROJECT_NAME AWX_PROJECT_SCM_URL \
AWX_PROJECT_SOURCE_PATH AWX_PROJECT_SOURCE_DIR AWX_INVENTORY_NAME \
AWX_HOST_NAME AWX_HOST_ADDRESS \
AWX_MACHINE_CREDENTIAL_NAME AWX_SSH_PRIVATE_KEY_FILE \
AWX_POSTGRES_CREDENTIAL_TYPE_NAME AWX_POSTGRES_CREDENTIAL_NAME \
AWX_JOB_TEMPLATE_NAME AWX_PLAYBOOK; do
require_value "${variable_name}"
done
AWX_BASE_URL=${AWX_BASE_URL%/}
AWX_USERNAME=${AWX_USERNAME:-admin}
AWX_K8S_NAMESPACE=${AWX_K8S_NAMESPACE:-awx}
AWX_K8S_ADMIN_PASSWORD_SECRET=${AWX_K8S_ADMIN_PASSWORD_SECRET:-awx-demo-admin-password}
AWX_K8S_TASK_APP=${AWX_K8S_TASK_APP:-awx-demo-task}
AWX_K8S_TASK_CONTAINER=${AWX_K8S_TASK_CONTAINER:-${AWX_K8S_TASK_APP}}
AWX_HOST_SSH_PORT=${AWX_HOST_SSH_PORT:-22}
AWX_HOST_SSH_USER=${AWX_HOST_SSH_USER:-root}
AWX_HOST_ANSIBLE_ADDRESS=${AWX_HOST_ANSIBLE_ADDRESS:-${AWX_HOST_ADDRESS}}
XINFRA_POSTGRES_REPLICATION_USER=${XINFRA_POSTGRES_REPLICATION_USER:-xinfra_replication}
[[ -r ${AWX_SSH_PRIVATE_KEY_FILE} ]] || die "SSH private key is not readable: ${AWX_SSH_PRIVATE_KEY_FILE}"
case ${AWX_PROJECT_SOURCE_PATH} in
_[A-Za-z0-9_-]*) ;;
*) die "AWX_PROJECT_SOURCE_PATH must be a safe name beginning with '_'" ;;
esac
PROJECT_SOURCE=$(cd -- "${REPO_ROOT}/${AWX_PROJECT_SOURCE_DIR}" && pwd)
case ${PROJECT_SOURCE}/ in
"${REPO_ROOT}/"*) ;;
*) die "AWX_PROJECT_SOURCE_DIR must resolve inside the repository" ;;
esac
[[ -f ${PROJECT_SOURCE}/${AWX_PLAYBOOK} ]] || die "playbook not found: ${PROJECT_SOURCE}/${AWX_PLAYBOOK}"
if [[ -z ${AWX_TOKEN:-} && -z ${AWX_PASSWORD:-} ]]; then
AWX_PASSWORD=$(kubectl get secret \
-n "${AWX_K8S_NAMESPACE}" "${AWX_K8S_ADMIN_PASSWORD_SECRET}" \
-o jsonpath='{.data.password}' | base64 --decode)
fi
[[ -n ${AWX_TOKEN:-} || -n ${AWX_PASSWORD:-} ]] || die "AWX_TOKEN or AWX_PASSWORD is required"
AWX_AUTH_ARGS=()
if [[ -n ${AWX_TOKEN:-} ]]; then
AWX_AUTH_ARGS=(-H "Authorization: Bearer ${AWX_TOKEN}")
else
AWX_AUTH_ARGS=(-u "${AWX_USERNAME}:${AWX_PASSWORD}")
fi
awx_request() {
local method=$1
local path=$2
local payload=${3:-}
local args=(-sS --fail-with-body -X "${method}" -H 'Accept: application/json')
args+=("${AWX_AUTH_ARGS[@]}")
if [[ -n ${payload} ]]; then
args+=(-H 'Content-Type: application/json' --data "${payload}")
fi
curl "${args[@]}" "${AWX_BASE_URL}${path}"
}
find_named_id() {
local endpoint=$1
local name=$2
awx_request GET "${endpoint}?page_size=200" | jq -er --arg name "${name}" \
'.results[] | select(.name == $name) | .id' | head -n 1
}
find_inventory_host_id() {
local inventory_id=$1
awx_request GET "/api/v2/inventories/${inventory_id}/hosts/?page_size=200" | \
jq -er --arg name "${AWX_HOST_NAME}" '.results[] | select(.name == $name) | .id' | head -n 1
}
upsert_named_object() {
local endpoint=$1
local name=$2
local payload=$3
local id
id=$(find_named_id "${endpoint}" "${name}" 2>/dev/null || true)
if [[ -n ${id} ]]; then
awx_request PATCH "${endpoint}${id}/" "${payload}" | jq -er '.id'
else
awx_request POST "${endpoint}" "${payload}" | jq -er '.id'
fi
}
printf 'Checking AWX API at %s...\n' "${AWX_BASE_URL}"
awx_request GET '/api/v2/ping/' >/dev/null
if [[ -z ${AWX_K8S_TASK_POD:-} ]]; then
AWX_K8S_TASK_POD=$(kubectl get pods -n "${AWX_K8S_NAMESPACE}" \
-l "app.kubernetes.io/name=${AWX_K8S_TASK_APP}" \
-o jsonpath='{.items[0].metadata.name}')
fi
[[ -n ${AWX_K8S_TASK_POD} ]] || die "could not find the AWX task pod"
PROJECT_SOURCE_REMOTE_PATH=/var/lib/awx/projects/${AWX_PROJECT_SOURCE_PATH}
printf 'Refreshing local SCM source in %s/%s:%s...\n' \
"${AWX_K8S_NAMESPACE}" "${AWX_K8S_TASK_POD}" "${PROJECT_SOURCE_REMOTE_PATH}"
kubectl exec -n "${AWX_K8S_NAMESPACE}" "${AWX_K8S_TASK_POD}" \
-c "${AWX_K8S_TASK_CONTAINER}" -- \
sh -c 'mkdir -p "$1"; find "$1" -mindepth 1 -maxdepth 1 -exec rm -rf -- {} +' sh "${PROJECT_SOURCE_REMOTE_PATH}"
kubectl cp "${PROJECT_SOURCE}/." \
"${AWX_K8S_NAMESPACE}/${AWX_K8S_TASK_POD}:${PROJECT_SOURCE_REMOTE_PATH}" \
-c "${AWX_K8S_TASK_CONTAINER}"
kubectl exec -n "${AWX_K8S_NAMESPACE}" "${AWX_K8S_TASK_POD}" \
-c "${AWX_K8S_TASK_CONTAINER}" -- sh -c \
'cd "$1" && git init -q && git config user.email xinfra-local@localhost && git config user.name xinfra-local && git add -A && if ! git diff --cached --quiet; then git commit -qm local; fi' \
sh "${PROJECT_SOURCE_REMOTE_PATH}"
organization_id=$(find_named_id '/api/v2/organizations/' "${AWX_ORGANIZATION}" 2>/dev/null || true)
[[ -n ${organization_id} ]] || die "AWX organization not found: ${AWX_ORGANIZATION}"
project_payload=$(jq -nc \
--arg name "${AWX_PROJECT_NAME}" \
--arg description "${AWX_PROJECT_DESCRIPTION:-}" \
--arg scm_url "${AWX_PROJECT_SCM_URL}" \
--argjson organization "${organization_id}" \
'{name:$name, description:$description, organization:$organization, scm_type:"git", scm_url:$scm_url, scm_update_on_launch:false, scm_clean:false, scm_delete_on_update:false}')
project_id=$(upsert_named_object '/api/v2/projects/' "${AWX_PROJECT_NAME}" "${project_payload}")
project_local_path=$(awx_request GET "/api/v2/projects/${project_id}/" | jq -er '.local_path')
case ${project_local_path} in
_[A-Za-z0-9_-]*) ;;
*) die "AWX returned an unsafe project local_path: ${project_local_path}" ;;
esac
PROJECT_REMOTE_PATH=/var/lib/awx/projects/${project_local_path}
project_status=$(awx_request GET "/api/v2/projects/${project_id}/" | jq -r '.status')
if [[ ${project_status} == pending || ${project_status} == running ]]; then
for _ in $(seq 1 60); do
sleep 2
project_status=$(awx_request GET "/api/v2/projects/${project_id}/" | jq -r '.status')
[[ ${project_status} == pending || ${project_status} == running ]] || break
done
fi
if [[ ${project_status} != successful ]]; then
printf 'Refreshing the AWX-managed project directory...\n'
kubectl exec -n "${AWX_K8S_NAMESPACE}" "${AWX_K8S_TASK_POD}" \
-c "${AWX_K8S_TASK_CONTAINER}" -- \
sh -c 'mkdir -p "$1"; find "$1" -mindepth 1 -maxdepth 1 -exec rm -rf -- {} +' sh "${PROJECT_REMOTE_PATH}"
fi
project_update_id=$(awx_request POST "/api/v2/projects/${project_id}/update/" '{}' | jq -er '.id')
for _ in $(seq 1 60); do
project_status=$(awx_request GET "/api/v2/project_updates/${project_update_id}/" | jq -r '.status')
[[ ${project_status} == pending || ${project_status} == waiting || ${project_status} == running ]] || break
sleep 2
done
[[ ${project_status} == successful ]] || die "AWX project update ${project_update_id} finished with status ${project_status}"
if ! awx_request GET "/api/v2/projects/${project_id}/playbooks/" | \
jq -e --arg playbook "${AWX_PLAYBOOK}" 'index($playbook) != null' >/dev/null; then
die "AWX project ${project_id} does not expose playbook ${AWX_PLAYBOOK}"
fi
inventory_payload=$(jq -nc \
--arg name "${AWX_INVENTORY_NAME}" \
--arg description "${AWX_INVENTORY_DESCRIPTION:-}" \
--argjson organization "${organization_id}" \
'{name:$name, description:$description, organization:$organization, kind:""}')
inventory_id=$(upsert_named_object '/api/v2/inventories/' "${AWX_INVENTORY_NAME}" "${inventory_payload}")
host_variables=$(jq -nc \
--arg ansible_host "${AWX_HOST_ANSIBLE_ADDRESS}" \
--arg xinfra_public_address "${AWX_HOST_ADDRESS}" \
--arg ansible_user "${AWX_HOST_SSH_USER}" \
--argjson ansible_port "${AWX_HOST_SSH_PORT}" \
'{ansible_host:$ansible_host, xinfra_public_address:$xinfra_public_address, ansible_user:$ansible_user, ansible_port:$ansible_port, ansible_python_interpreter:"/usr/bin/python3"}')
host_payload=$(jq -nc \
--arg name "${AWX_HOST_NAME}" \
--arg variables "${host_variables}" \
--argjson inventory "${inventory_id}" \
'{name:$name, inventory:$inventory, enabled:true, variables:$variables}')
host_id=$(find_inventory_host_id "${inventory_id}" 2>/dev/null || true)
if [[ -n ${host_id} ]]; then
host_id=$(awx_request PATCH "/api/v2/hosts/${host_id}/" "${host_payload}" | jq -er '.id')
else
host_id=$(awx_request POST '/api/v2/hosts/' "${host_payload}" | jq -er '.id')
fi
machine_type_id=$(awx_request GET '/api/v2/credential_types/?page_size=200' | \
jq -er '.results[] | select(.kind == "ssh") | .id' | head -n 1)
ssh_key_data=$(<"${AWX_SSH_PRIVATE_KEY_FILE}")
machine_credential_payload=$(jq -nc \
--arg name "${AWX_MACHINE_CREDENTIAL_NAME}" \
--arg description "SSH access for the XINFRA PostgreSQL host pool" \
--arg username "${AWX_HOST_SSH_USER}" \
--arg ssh_key_data "${ssh_key_data}" \
--argjson organization "${organization_id}" \
--argjson credential_type "${machine_type_id}" \
'{name:$name, description:$description, organization:$organization, credential_type:$credential_type, inputs:{username:$username, ssh_key_data:$ssh_key_data}}')
machine_credential_id=$(upsert_named_object '/api/v2/credentials/' "${AWX_MACHINE_CREDENTIAL_NAME}" "${machine_credential_payload}")
postgres_credential_type_payload=$(jq -nc \
--arg name "${AWX_POSTGRES_CREDENTIAL_TYPE_NAME}" \
'{
name:$name,
description:"Injects PostgreSQL delivery secrets as execution environment variables",
kind:"cloud",
inputs:{
fields:[
{id:"admin_password", label:"PostgreSQL administrator password", type:"string", secret:true},
{id:"replication_user", label:"PostgreSQL replication user", type:"string", default:"xinfra_replication"},
{id:"replication_password", label:"PostgreSQL replication password", type:"string", secret:true}
],
required:["admin_password", "replication_password"]
},
injectors:{env:{
XINFRA_POSTGRES_ADMIN_PASSWORD:"{{ admin_password }}",
XINFRA_POSTGRES_REPLICATION_USER:"{{ replication_user }}",
XINFRA_POSTGRES_REPLICATION_PASSWORD:"{{ replication_password }}"
}}
}')
postgres_credential_type_id=$(upsert_named_object '/api/v2/credential_types/' \
"${AWX_POSTGRES_CREDENTIAL_TYPE_NAME}" "${postgres_credential_type_payload}")
postgres_credential_id=$(find_named_id '/api/v2/credentials/' "${AWX_POSTGRES_CREDENTIAL_NAME}" 2>/dev/null || true)
if [[ -z ${postgres_credential_id} ]]; then
admin_password=${XINFRA_POSTGRES_ADMIN_PASSWORD:-$(openssl rand -base64 24 | tr -d '\n')}
replication_password=${XINFRA_POSTGRES_REPLICATION_PASSWORD:-$(openssl rand -base64 24 | tr -d '\n')}
postgres_credential_payload=$(jq -nc \
--arg name "${AWX_POSTGRES_CREDENTIAL_NAME}" \
--arg description "Runtime secrets for XINFRA PostgreSQL delivery" \
--arg admin_password "${admin_password}" \
--arg replication_user "${XINFRA_POSTGRES_REPLICATION_USER}" \
--arg replication_password "${replication_password}" \
--argjson organization "${organization_id}" \
--argjson credential_type "${postgres_credential_type_id}" \
'{name:$name, description:$description, organization:$organization, credential_type:$credential_type, inputs:{admin_password:$admin_password, replication_user:$replication_user, replication_password:$replication_password}}')
postgres_credential_id=$(awx_request POST '/api/v2/credentials/' "${postgres_credential_payload}" | jq -er '.id')
elif [[ -n ${XINFRA_POSTGRES_ADMIN_PASSWORD:-} || -n ${XINFRA_POSTGRES_REPLICATION_PASSWORD:-} ]]; then
[[ -n ${XINFRA_POSTGRES_ADMIN_PASSWORD:-} && -n ${XINFRA_POSTGRES_REPLICATION_PASSWORD:-} ]] || \
die "set both PostgreSQL passwords together when rotating an existing credential"
postgres_credential_payload=$(jq -nc \
--arg admin_password "${XINFRA_POSTGRES_ADMIN_PASSWORD}" \
--arg replication_user "${XINFRA_POSTGRES_REPLICATION_USER}" \
--arg replication_password "${XINFRA_POSTGRES_REPLICATION_PASSWORD}" \
'{inputs:{admin_password:$admin_password, replication_user:$replication_user, replication_password:$replication_password}}')
postgres_credential_id=$(awx_request PATCH "/api/v2/credentials/${postgres_credential_id}/" \
"${postgres_credential_payload}" | jq -er '.id')
fi
prevent_fallback=false
instance_group_id=
if [[ -n ${AWX_INSTANCE_GROUP:-} ]]; then
instance_group_id=$(find_named_id '/api/v2/instance_groups/' "${AWX_INSTANCE_GROUP}" 2>/dev/null || true)
[[ -n ${instance_group_id} ]] || die "AWX instance group not found: ${AWX_INSTANCE_GROUP}"
prevent_fallback=true
fi
job_template_payload=$(jq -nc \
--arg name "${AWX_JOB_TEMPLATE_NAME}" \
--arg description "${AWX_JOB_TEMPLATE_DESCRIPTION:-PostgreSQL delivery template}" \
--arg playbook "${AWX_PLAYBOOK}" \
--argjson organization "${organization_id}" \
--argjson inventory "${inventory_id}" \
--argjson project "${project_id}" \
--argjson prevent_fallback "${prevent_fallback}" \
'{
name:$name,
description:$description,
organization:$organization,
inventory:$inventory,
project:$project,
playbook:$playbook,
job_type:"run",
ask_inventory_on_launch:true,
ask_variables_on_launch:true,
ask_limit_on_launch:true,
allow_simultaneous:true,
prevent_instance_group_fallback:$prevent_fallback
}')
job_template_id=$(upsert_named_object '/api/v2/job_templates/' \
"${AWX_JOB_TEMPLATE_NAME}" "${job_template_payload}")
while IFS= read -r existing_group_id; do
[[ -z ${existing_group_id} || ${existing_group_id} == "${instance_group_id}" ]] && continue
awx_request POST "/api/v2/job_templates/${job_template_id}/instance_groups/" \
"$(jq -nc --argjson id "${existing_group_id}" '{id:$id,disassociate:true}')" >/dev/null
done < <(awx_request GET "/api/v2/job_templates/${job_template_id}/instance_groups/" | jq -r '.results[].id')
if ! awx_request GET "/api/v2/job_templates/${job_template_id}/credentials/" | \
jq -e --argjson id "${machine_credential_id}" '.results | any(.id == $id)' >/dev/null; then
awx_request POST "/api/v2/job_templates/${job_template_id}/credentials/" \
"$(jq -nc --argjson id "${machine_credential_id}" '{id:$id}')" >/dev/null
fi
if ! awx_request GET "/api/v2/job_templates/${job_template_id}/credentials/" | \
jq -e --argjson id "${postgres_credential_id}" '.results | any(.id == $id)' >/dev/null; then
awx_request POST "/api/v2/job_templates/${job_template_id}/credentials/" \
"$(jq -nc --argjson id "${postgres_credential_id}" '{id:$id}')" >/dev/null
fi
if [[ -n ${instance_group_id} ]]; then
if ! awx_request GET "/api/v2/job_templates/${job_template_id}/instance_groups/" | \
jq -e --argjson id "${instance_group_id}" '.results | any(.id == $id)' >/dev/null; then
awx_request POST "/api/v2/job_templates/${job_template_id}/instance_groups/" \
"$(jq -nc --argjson id "${instance_group_id}" '{id:$id}')" >/dev/null
fi
fi
printf '\nPostgreSQL AWX configuration is ready:\n'
printf ' Project: %s (id=%s)\n' "${AWX_PROJECT_NAME}" "${project_id}"
printf ' Inventory: %s (id=%s)\n' "${AWX_INVENTORY_NAME}" "${inventory_id}"
printf ' Host: %s -> %s (id=%s)\n' "${AWX_HOST_NAME}" "${AWX_HOST_ADDRESS}" "${host_id}"
printf ' SSH credential: %s (id=%s)\n' "${AWX_MACHINE_CREDENTIAL_NAME}" "${machine_credential_id}"
printf ' PG credential: %s (id=%s)\n' "${AWX_POSTGRES_CREDENTIAL_NAME}" "${postgres_credential_id}"
printf ' Job Template: %s (id=%s)\n' "${AWX_JOB_TEMPLATE_NAME}" "${job_template_id}"
if [[ -n ${instance_group_id} ]]; then
printf ' Instance Group: %s (id=%s)\n' "${AWX_INSTANCE_GROUP}" "${instance_group_id}"
fi