xinfra/scripts/setup-postgresql-awx.sh

#!/usr/bin/env bash

set -euo pipefail

SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
REPO_ROOT=$(cd -- "${SCRIPT_DIR}/.." && pwd)
CONFIG_FILE=${1:-"${REPO_ROOT}/config/local/postgresql-awx.env"}

die() {
  printf 'error: %s\n' "$*" >&2
  exit 1
}

require_command() {
  command -v "$1" >/dev/null 2>&1 || die "required command not found: $1"
}

require_value() {
  local name=$1
  [[ -n ${!name:-} ]] || die "${name} must be set in ${CONFIG_FILE}"
}

[[ -f ${CONFIG_FILE} ]] || die "configuration file not found: ${CONFIG_FILE}"

set -a
# shellcheck disable=SC1090
source "${CONFIG_FILE}"
set +a

for command_name in curl jq kubectl base64 openssl; do
  require_command "${command_name}"
done

for variable_name in \
  AWX_BASE_URL AWX_ORGANIZATION AWX_PROJECT_NAME AWX_PROJECT_SCM_URL \
  AWX_PROJECT_SOURCE_PATH AWX_PROJECT_SOURCE_DIR AWX_INVENTORY_NAME \
  AWX_HOST_NAME AWX_HOST_ADDRESS \
  AWX_MACHINE_CREDENTIAL_NAME AWX_SSH_PRIVATE_KEY_FILE \
  AWX_POSTGRES_CREDENTIAL_TYPE_NAME AWX_POSTGRES_CREDENTIAL_NAME \
  AWX_JOB_TEMPLATE_NAME AWX_PLAYBOOK \
  AWX_ROLLBACK_JOB_TEMPLATE_NAME AWX_ROLLBACK_PLAYBOOK; do
  require_value "${variable_name}"
done

AWX_BASE_URL=${AWX_BASE_URL%/}
AWX_USERNAME=${AWX_USERNAME:-admin}
AWX_K8S_NAMESPACE=${AWX_K8S_NAMESPACE:-awx}
AWX_K8S_ADMIN_PASSWORD_SECRET=${AWX_K8S_ADMIN_PASSWORD_SECRET:-awx-demo-admin-password}
AWX_K8S_TASK_APP=${AWX_K8S_TASK_APP:-awx-demo-task}
AWX_K8S_TASK_CONTAINER=${AWX_K8S_TASK_CONTAINER:-${AWX_K8S_TASK_APP}}
AWX_HOST_SSH_PORT=${AWX_HOST_SSH_PORT:-22}
AWX_HOST_SSH_USER=${AWX_HOST_SSH_USER:-root}
AWX_HOST_ANSIBLE_ADDRESS=${AWX_HOST_ANSIBLE_ADDRESS:-${AWX_HOST_ADDRESS}}
XINFRA_POSTGRES_REPLICATION_USER=${XINFRA_POSTGRES_REPLICATION_USER:-xinfra_replication}

[[ -r ${AWX_SSH_PRIVATE_KEY_FILE} ]] || die "SSH private key is not readable: ${AWX_SSH_PRIVATE_KEY_FILE}"
case ${AWX_PROJECT_SOURCE_PATH} in
  _[A-Za-z0-9_-]*) ;;
  *) die "AWX_PROJECT_SOURCE_PATH must be a safe name beginning with '_'" ;;
esac

PROJECT_SOURCE=$(cd -- "${REPO_ROOT}/${AWX_PROJECT_SOURCE_DIR}" && pwd)
case ${PROJECT_SOURCE}/ in
  "${REPO_ROOT}/"*) ;;
  *) die "AWX_PROJECT_SOURCE_DIR must resolve inside the repository" ;;
esac
[[ -f ${PROJECT_SOURCE}/${AWX_PLAYBOOK} ]] || die "playbook not found: ${PROJECT_SOURCE}/${AWX_PLAYBOOK}"
[[ -f ${PROJECT_SOURCE}/${AWX_ROLLBACK_PLAYBOOK} ]] || die "rollback playbook not found: ${PROJECT_SOURCE}/${AWX_ROLLBACK_PLAYBOOK}"

if [[ -z ${AWX_TOKEN:-} && -z ${AWX_PASSWORD:-} ]]; then
  AWX_PASSWORD=$(kubectl get secret \
    -n "${AWX_K8S_NAMESPACE}" "${AWX_K8S_ADMIN_PASSWORD_SECRET}" \
    -o jsonpath='{.data.password}' | base64 --decode)
fi
[[ -n ${AWX_TOKEN:-} || -n ${AWX_PASSWORD:-} ]] || die "AWX_TOKEN or AWX_PASSWORD is required"

AWX_AUTH_ARGS=()
if [[ -n ${AWX_TOKEN:-} ]]; then
  AWX_AUTH_ARGS=(-H "Authorization: Bearer ${AWX_TOKEN}")
else
  AWX_AUTH_ARGS=(-u "${AWX_USERNAME}:${AWX_PASSWORD}")
fi

awx_request() {
  local method=$1
  local path=$2
  local payload=${3:-}
  local args=(-sS --fail-with-body -X "${method}" -H 'Accept: application/json')
  args+=("${AWX_AUTH_ARGS[@]}")
  if [[ -n ${payload} ]]; then
    args+=(-H 'Content-Type: application/json' --data "${payload}")
  fi
  curl "${args[@]}" "${AWX_BASE_URL}${path}"
}

find_named_id() {
  local endpoint=$1
  local name=$2
  awx_request GET "${endpoint}?page_size=200" | jq -er --arg name "${name}" \
    '.results[] | select(.name == $name) | .id' | head -n 1
}

find_inventory_host_id() {
  local inventory_id=$1
  awx_request GET "/api/v2/inventories/${inventory_id}/hosts/?page_size=200" | \
    jq -er --arg name "${AWX_HOST_NAME}" '.results[] | select(.name == $name) | .id' | head -n 1
}

upsert_named_object() {
  local endpoint=$1
  local name=$2
  local payload=$3
  local id
  id=$(find_named_id "${endpoint}" "${name}" 2>/dev/null || true)
  if [[ -n ${id} ]]; then
    awx_request PATCH "${endpoint}${id}/" "${payload}" | jq -er '.id'
  else
    awx_request POST "${endpoint}" "${payload}" | jq -er '.id'
  fi
}

printf 'Checking AWX API at %s...\n' "${AWX_BASE_URL}"
awx_request GET '/api/v2/ping/' >/dev/null

if [[ -z ${AWX_K8S_TASK_POD:-} ]]; then
  AWX_K8S_TASK_POD=$(kubectl get pods -n "${AWX_K8S_NAMESPACE}" \
    -l "app.kubernetes.io/name=${AWX_K8S_TASK_APP}" \
    -o jsonpath='{.items[0].metadata.name}')
fi
[[ -n ${AWX_K8S_TASK_POD} ]] || die "could not find the AWX task pod"

PROJECT_SOURCE_REMOTE_PATH=/var/lib/awx/projects/${AWX_PROJECT_SOURCE_PATH}
printf 'Refreshing local SCM source in %s/%s:%s...\n' \
  "${AWX_K8S_NAMESPACE}" "${AWX_K8S_TASK_POD}" "${PROJECT_SOURCE_REMOTE_PATH}"
kubectl exec -n "${AWX_K8S_NAMESPACE}" "${AWX_K8S_TASK_POD}" \
  -c "${AWX_K8S_TASK_CONTAINER}" -- \
  sh -c 'mkdir -p "$1"; find "$1" -mindepth 1 -maxdepth 1 -exec rm -rf -- {} +' sh "${PROJECT_SOURCE_REMOTE_PATH}"
kubectl cp "${PROJECT_SOURCE}/." \
  "${AWX_K8S_NAMESPACE}/${AWX_K8S_TASK_POD}:${PROJECT_SOURCE_REMOTE_PATH}" \
  -c "${AWX_K8S_TASK_CONTAINER}"
kubectl exec -n "${AWX_K8S_NAMESPACE}" "${AWX_K8S_TASK_POD}" \
  -c "${AWX_K8S_TASK_CONTAINER}" -- sh -c \
  'cd "$1" && git init -q && git config user.email xinfra-local@localhost && git config user.name xinfra-local && git add -A && if ! git diff --cached --quiet; then git commit -qm local; fi' \
  sh "${PROJECT_SOURCE_REMOTE_PATH}"

organization_id=$(find_named_id '/api/v2/organizations/' "${AWX_ORGANIZATION}" 2>/dev/null || true)
[[ -n ${organization_id} ]] || die "AWX organization not found: ${AWX_ORGANIZATION}"

project_payload=$(jq -nc \
  --arg name "${AWX_PROJECT_NAME}" \
  --arg description "${AWX_PROJECT_DESCRIPTION:-}" \
  --arg scm_url "${AWX_PROJECT_SCM_URL}" \
  --argjson organization "${organization_id}" \
  '{name:$name, description:$description, organization:$organization, scm_type:"git", scm_url:$scm_url, scm_update_on_launch:false, scm_clean:false, scm_delete_on_update:false}')
project_id=$(upsert_named_object '/api/v2/projects/' "${AWX_PROJECT_NAME}" "${project_payload}")

project_local_path=$(awx_request GET "/api/v2/projects/${project_id}/" | jq -er '.local_path')
case ${project_local_path} in
  _[A-Za-z0-9_-]*) ;;
  *) die "AWX returned an unsafe project local_path: ${project_local_path}" ;;
esac

PROJECT_REMOTE_PATH=/var/lib/awx/projects/${project_local_path}
project_status=$(awx_request GET "/api/v2/projects/${project_id}/" | jq -r '.status')
if [[ ${project_status} == pending || ${project_status} == running ]]; then
  for _ in $(seq 1 60); do
    sleep 2
    project_status=$(awx_request GET "/api/v2/projects/${project_id}/" | jq -r '.status')
    [[ ${project_status} == pending || ${project_status} == running ]] || break
  done
fi
if [[ ${project_status} != successful ]]; then
  printf 'Refreshing the AWX-managed project directory...\n'
  kubectl exec -n "${AWX_K8S_NAMESPACE}" "${AWX_K8S_TASK_POD}" \
    -c "${AWX_K8S_TASK_CONTAINER}" -- \
    sh -c 'mkdir -p "$1"; find "$1" -mindepth 1 -maxdepth 1 -exec rm -rf -- {} +' sh "${PROJECT_REMOTE_PATH}"
fi

project_update_id=$(awx_request POST "/api/v2/projects/${project_id}/update/" '{}' | jq -er '.id')
for _ in $(seq 1 60); do
  project_status=$(awx_request GET "/api/v2/project_updates/${project_update_id}/" | jq -r '.status')
  [[ ${project_status} == pending || ${project_status} == waiting || ${project_status} == running ]] || break
  sleep 2
done
[[ ${project_status} == successful ]] || die "AWX project update ${project_update_id} finished with status ${project_status}"

if ! awx_request GET "/api/v2/projects/${project_id}/playbooks/" | \
  jq -e --arg playbook "${AWX_PLAYBOOK}" 'index($playbook) != null' >/dev/null; then
  die "AWX project ${project_id} does not expose playbook ${AWX_PLAYBOOK}"
fi
if ! awx_request GET "/api/v2/projects/${project_id}/playbooks/" | \
  jq -e --arg playbook "${AWX_ROLLBACK_PLAYBOOK}" 'index($playbook) != null' >/dev/null; then
  die "AWX project ${project_id} does not expose rollback playbook ${AWX_ROLLBACK_PLAYBOOK}"
fi

inventory_payload=$(jq -nc \
  --arg name "${AWX_INVENTORY_NAME}" \
  --arg description "${AWX_INVENTORY_DESCRIPTION:-}" \
  --argjson organization "${organization_id}" \
  '{name:$name, description:$description, organization:$organization, kind:""}')
inventory_id=$(upsert_named_object '/api/v2/inventories/' "${AWX_INVENTORY_NAME}" "${inventory_payload}")

host_variables=$(jq -nc \
  --arg ansible_host "${AWX_HOST_ANSIBLE_ADDRESS}" \
  --arg xinfra_public_address "${AWX_HOST_ADDRESS}" \
  --arg ansible_user "${AWX_HOST_SSH_USER}" \
  --argjson ansible_port "${AWX_HOST_SSH_PORT}" \
  '{ansible_host:$ansible_host, xinfra_public_address:$xinfra_public_address, ansible_user:$ansible_user, ansible_port:$ansible_port, ansible_python_interpreter:"/usr/bin/python3"}')
host_payload=$(jq -nc \
  --arg name "${AWX_HOST_NAME}" \
  --arg variables "${host_variables}" \
  --argjson inventory "${inventory_id}" \
  '{name:$name, inventory:$inventory, enabled:true, variables:$variables}')
host_id=$(find_inventory_host_id "${inventory_id}" 2>/dev/null || true)
if [[ -n ${host_id} ]]; then
  host_id=$(awx_request PATCH "/api/v2/hosts/${host_id}/" "${host_payload}" | jq -er '.id')
else
  host_id=$(awx_request POST '/api/v2/hosts/' "${host_payload}" | jq -er '.id')
fi

machine_type_id=$(awx_request GET '/api/v2/credential_types/?page_size=200' | \
  jq -er '.results[] | select(.kind == "ssh") | .id' | head -n 1)
ssh_key_data=$(<"${AWX_SSH_PRIVATE_KEY_FILE}")
machine_credential_payload=$(jq -nc \
  --arg name "${AWX_MACHINE_CREDENTIAL_NAME}" \
  --arg description "SSH access for the XINFRA PostgreSQL host pool" \
  --arg username "${AWX_HOST_SSH_USER}" \
  --arg ssh_key_data "${ssh_key_data}" \
  --argjson organization "${organization_id}" \
  --argjson credential_type "${machine_type_id}" \
  '{name:$name, description:$description, organization:$organization, credential_type:$credential_type, inputs:{username:$username, ssh_key_data:$ssh_key_data}}')
machine_credential_id=$(upsert_named_object '/api/v2/credentials/' "${AWX_MACHINE_CREDENTIAL_NAME}" "${machine_credential_payload}")

postgres_credential_type_payload=$(jq -nc \
  --arg name "${AWX_POSTGRES_CREDENTIAL_TYPE_NAME}" \
  '{
    name:$name,
    description:"Injects PostgreSQL delivery secrets as execution environment variables",
    kind:"cloud",
    inputs:{
      fields:[
        {id:"admin_password", label:"PostgreSQL administrator password", type:"string", secret:true},
        {id:"replication_user", label:"PostgreSQL replication user", type:"string", default:"xinfra_replication"},
        {id:"replication_password", label:"PostgreSQL replication password", type:"string", secret:true}
      ],
      required:["admin_password", "replication_password"]
    },
    injectors:{env:{
      XINFRA_POSTGRES_ADMIN_PASSWORD:"{{ admin_password }}",
      XINFRA_POSTGRES_REPLICATION_USER:"{{ replication_user }}",
      XINFRA_POSTGRES_REPLICATION_PASSWORD:"{{ replication_password }}"
    }}
  }')
postgres_credential_type_id=$(upsert_named_object '/api/v2/credential_types/' \
  "${AWX_POSTGRES_CREDENTIAL_TYPE_NAME}" "${postgres_credential_type_payload}")

postgres_credential_id=$(find_named_id '/api/v2/credentials/' "${AWX_POSTGRES_CREDENTIAL_NAME}" 2>/dev/null || true)
if [[ -z ${postgres_credential_id} ]]; then
  admin_password=${XINFRA_POSTGRES_ADMIN_PASSWORD:-$(openssl rand -base64 24 | tr -d '\n')}
  replication_password=${XINFRA_POSTGRES_REPLICATION_PASSWORD:-$(openssl rand -base64 24 | tr -d '\n')}
  postgres_credential_payload=$(jq -nc \
    --arg name "${AWX_POSTGRES_CREDENTIAL_NAME}" \
    --arg description "Runtime secrets for XINFRA PostgreSQL delivery" \
    --arg admin_password "${admin_password}" \
    --arg replication_user "${XINFRA_POSTGRES_REPLICATION_USER}" \
    --arg replication_password "${replication_password}" \
    --argjson organization "${organization_id}" \
    --argjson credential_type "${postgres_credential_type_id}" \
    '{name:$name, description:$description, organization:$organization, credential_type:$credential_type, inputs:{admin_password:$admin_password, replication_user:$replication_user, replication_password:$replication_password}}')
  postgres_credential_id=$(awx_request POST '/api/v2/credentials/' "${postgres_credential_payload}" | jq -er '.id')
elif [[ -n ${XINFRA_POSTGRES_ADMIN_PASSWORD:-} || -n ${XINFRA_POSTGRES_REPLICATION_PASSWORD:-} ]]; then
  [[ -n ${XINFRA_POSTGRES_ADMIN_PASSWORD:-} && -n ${XINFRA_POSTGRES_REPLICATION_PASSWORD:-} ]] || \
    die "set both PostgreSQL passwords together when rotating an existing credential"
  postgres_credential_payload=$(jq -nc \
    --arg admin_password "${XINFRA_POSTGRES_ADMIN_PASSWORD}" \
    --arg replication_user "${XINFRA_POSTGRES_REPLICATION_USER}" \
    --arg replication_password "${XINFRA_POSTGRES_REPLICATION_PASSWORD}" \
    '{inputs:{admin_password:$admin_password, replication_user:$replication_user, replication_password:$replication_password}}')
  postgres_credential_id=$(awx_request PATCH "/api/v2/credentials/${postgres_credential_id}/" \
    "${postgres_credential_payload}" | jq -er '.id')
fi

prevent_fallback=false
instance_group_id=
if [[ -n ${AWX_INSTANCE_GROUP:-} ]]; then
  instance_group_id=$(find_named_id '/api/v2/instance_groups/' "${AWX_INSTANCE_GROUP}" 2>/dev/null || true)
  [[ -n ${instance_group_id} ]] || die "AWX instance group not found: ${AWX_INSTANCE_GROUP}"
  prevent_fallback=true
fi

job_template_payload=$(jq -nc \
  --arg name "${AWX_JOB_TEMPLATE_NAME}" \
  --arg description "${AWX_JOB_TEMPLATE_DESCRIPTION:-PostgreSQL delivery template}" \
  --arg playbook "${AWX_PLAYBOOK}" \
  --argjson organization "${organization_id}" \
  --argjson inventory "${inventory_id}" \
  --argjson project "${project_id}" \
  --argjson prevent_fallback "${prevent_fallback}" \
  '{
    name:$name,
    description:$description,
    organization:$organization,
    inventory:$inventory,
    project:$project,
    playbook:$playbook,
    job_type:"run",
    ask_inventory_on_launch:true,
    ask_variables_on_launch:true,
    ask_limit_on_launch:true,
    allow_simultaneous:true,
    prevent_instance_group_fallback:$prevent_fallback
  }')
job_template_id=$(upsert_named_object '/api/v2/job_templates/' \
  "${AWX_JOB_TEMPLATE_NAME}" "${job_template_payload}")

rollback_template_payload=$(jq -nc \
  --arg name "${AWX_ROLLBACK_JOB_TEMPLATE_NAME}" \
  --arg description "${AWX_ROLLBACK_JOB_TEMPLATE_DESCRIPTION:-PostgreSQL rollback template}" \
  --arg playbook "${AWX_ROLLBACK_PLAYBOOK}" \
  --argjson organization "${organization_id}" \
  --argjson inventory "${inventory_id}" \
  --argjson project "${project_id}" \
  --argjson prevent_fallback "${prevent_fallback}" \
  '{
    name:$name,
    description:$description,
    organization:$organization,
    inventory:$inventory,
    project:$project,
    playbook:$playbook,
    job_type:"run",
    ask_inventory_on_launch:true,
    ask_variables_on_launch:true,
    ask_limit_on_launch:true,
    allow_simultaneous:true,
    prevent_instance_group_fallback:$prevent_fallback
  }')
rollback_template_id=$(upsert_named_object '/api/v2/job_templates/' \
  "${AWX_ROLLBACK_JOB_TEMPLATE_NAME}" "${rollback_template_payload}")

for configured_template_id in "${job_template_id}" "${rollback_template_id}"; do
  while IFS= read -r existing_group_id; do
    [[ -z ${existing_group_id} || ${existing_group_id} == "${instance_group_id}" ]] && continue
    awx_request POST "/api/v2/job_templates/${configured_template_id}/instance_groups/" \
      "$(jq -nc --argjson id "${existing_group_id}" '{id:$id,disassociate:true}')" >/dev/null
  done < <(awx_request GET "/api/v2/job_templates/${configured_template_id}/instance_groups/" | jq -r '.results[].id')

  if ! awx_request GET "/api/v2/job_templates/${configured_template_id}/credentials/" | \
    jq -e --argjson id "${machine_credential_id}" '.results | any(.id == $id)' >/dev/null; then
    awx_request POST "/api/v2/job_templates/${configured_template_id}/credentials/" \
      "$(jq -nc --argjson id "${machine_credential_id}" '{id:$id}')" >/dev/null
  fi
  if [[ -n ${instance_group_id} ]]; then
    if ! awx_request GET "/api/v2/job_templates/${configured_template_id}/instance_groups/" | \
      jq -e --argjson id "${instance_group_id}" '.results | any(.id == $id)' >/dev/null; then
      awx_request POST "/api/v2/job_templates/${configured_template_id}/instance_groups/" \
        "$(jq -nc --argjson id "${instance_group_id}" '{id:$id}')" >/dev/null
    fi
  fi
done

if ! awx_request GET "/api/v2/job_templates/${job_template_id}/credentials/" | \
  jq -e --argjson id "${postgres_credential_id}" '.results | any(.id == $id)' >/dev/null; then
  awx_request POST "/api/v2/job_templates/${job_template_id}/credentials/" \
    "$(jq -nc --argjson id "${postgres_credential_id}" '{id:$id}')" >/dev/null
fi

printf '\nPostgreSQL AWX configuration is ready:\n'
printf '  Project:        %s (id=%s)\n' "${AWX_PROJECT_NAME}" "${project_id}"
printf '  Inventory:      %s (id=%s)\n' "${AWX_INVENTORY_NAME}" "${inventory_id}"
printf '  Host:           %s -> %s (id=%s)\n' "${AWX_HOST_NAME}" "${AWX_HOST_ADDRESS}" "${host_id}"
printf '  SSH credential: %s (id=%s)\n' "${AWX_MACHINE_CREDENTIAL_NAME}" "${machine_credential_id}"
printf '  PG credential:  %s (id=%s)\n' "${AWX_POSTGRES_CREDENTIAL_NAME}" "${postgres_credential_id}"
printf '  Job Template:   %s (id=%s)\n' "${AWX_JOB_TEMPLATE_NAME}" "${job_template_id}"
printf '  Rollback:       %s (id=%s)\n' "${AWX_ROLLBACK_JOB_TEMPLATE_NAME}" "${rollback_template_id}"
printf '  Server env:     DELIVERY_POSTGRESQL_ROLLBACK_TEMPLATE_ID=%s\n' "${rollback_template_id}"
if [[ -n ${instance_group_id} ]]; then
  printf '  Instance Group: %s (id=%s)\n' "${AWX_INSTANCE_GROUP}" "${instance_group_id}"
fi