fix(delivery): add PostgreSQL rollback and CloudDM isolation

This commit is contained in:
l x
2026-07-31 11:16:19 +08:00
parent 66fde73792
commit 363ad5f9be
15 changed files with 872 additions and 224 deletions
+86
View File
@@ -0,0 +1,86 @@
---
- name: Roll back native PostgreSQL delivery instances
hosts: "{{ target_hosts }}"
become: true
gather_facts: false
any_errors_fatal: true
vars:
postgresql_instance_plan: "{{ postgresql_instances[inventory_hostname] | default({}) }}"
postgresql_instance_id: "{{ postgresql_instance_plan.instance_id | default('') }}"
postgresql_data_root: /data/postgresql
postgresql_instance_root: "{{ postgresql_data_root }}/{{ postgresql_instance_id }}"
postgresql_data_dir: "{{ postgresql_instance_plan.data_dir | default('') }}"
postgresql_config_dir: "{{ postgresql_instance_plan.config_dir | default('') }}"
postgresql_log_dir: "{{ postgresql_instance_plan.log_dir | default('') }}"
postgresql_systemd_unit: "{{ postgresql_instance_plan.systemd_unit | default('') }}"
postgresql_dropin_dir: "/etc/systemd/system/{{ postgresql_systemd_unit }}.d"
postgresql_run_dir: "/run/postgresql-xinfra/{{ postgresql_instance_id }}"
pre_tasks:
- name: Validate PostgreSQL rollback target
ansible.builtin.assert:
that:
- postgresql_instance_plan is mapping
- postgresql_instance_id is match('^[a-z0-9][a-z0-9-]{0,62}$')
- postgresql_instance_root == '/data/postgresql/' + postgresql_instance_id
- postgresql_data_dir == postgresql_instance_root + '/data'
- postgresql_config_dir == postgresql_instance_root + '/conf'
- postgresql_log_dir == postgresql_instance_root + '/log'
- postgresql_systemd_unit == 'postgresql-xinfra@' + postgresql_instance_id + '.service'
- postgresql_instance_root != '/data/postgresql'
- "'..' not in postgresql_instance_root"
fail_msg: "Rollback target is outside the native PostgreSQL delivery layout"
quiet: true
tasks:
- name: Stop the PostgreSQL delivery instance when present
ansible.builtin.systemd_service:
name: "{{ postgresql_systemd_unit }}"
state: stopped
enabled: false
failed_when: false
- name: Reset the PostgreSQL instance failure state
ansible.builtin.command:
argv:
- systemctl
- reset-failed
- "{{ postgresql_systemd_unit }}"
changed_when: false
failed_when: false
- name: Remove the PostgreSQL instance systemd drop-in
ansible.builtin.file:
path: "{{ postgresql_dropin_dir }}"
state: absent
- name: Remove the PostgreSQL instance data, configuration, and logs
ansible.builtin.file:
path: "{{ postgresql_instance_root }}"
state: absent
- name: Remove the PostgreSQL instance runtime directory
ansible.builtin.file:
path: "{{ postgresql_run_dir }}"
state: absent
- name: Reload systemd after removing the instance drop-in
ansible.builtin.systemd_service:
daemon_reload: true
- name: Confirm PostgreSQL instance artifacts are gone
ansible.builtin.stat:
path: "{{ item }}"
loop:
- "{{ postgresql_dropin_dir }}"
- "{{ postgresql_instance_root }}"
- "{{ postgresql_run_dir }}"
register: postgresql_rollback_artifacts
- name: Assert all PostgreSQL instance artifacts are gone
ansible.builtin.assert:
that:
- not item.stat.exists
fail_msg: "PostgreSQL rollback left an instance artifact on the target host"
quiet: true
loop: "{{ postgresql_rollback_artifacts.results }}"
+9
View File
@@ -17,6 +17,8 @@ The local object mapping is:
| Secret credential | `XINFRA PostgreSQL Runtime Secrets` | | Secret credential | `XINFRA PostgreSQL Runtime Secrets` |
| Job Template | `XINFRA PostgreSQL Delivery` | | Job Template | `XINFRA PostgreSQL Delivery` |
| Playbook | `postgresql-deploy.yml` | | Playbook | `postgresql-deploy.yml` |
| Rollback Template | `XINFRA PostgreSQL Rollback` |
| Rollback Playbook | `postgresql-rollback.yml` |
| Instance Group | `xinfra-e2e-execution` | | Instance Group | `xinfra-e2e-execution` |
Run the idempotent setup whenever the AWX task pod restarts or the local Run the idempotent setup whenever the AWX task pod restarts or the local
@@ -25,6 +27,13 @@ cannot use a Manual Project, so the Project is Git-backed with launch updates
disabled. The script refreshes a local Git source inside the AWX task pod and disabled. The script refreshes a local Git source inside the AWX task pod and
runs a project update so AWX rebuilds its playbook index. runs a project update so AWX rebuilds its playbook index.
The setup output includes `DELIVERY_POSTGRESQL_ROLLBACK_TEMPLATE_ID`; copy that
value into `config/local/server-postgresql.env` before starting the scheduler.
PostgreSQL CloudDM registration is optional and requires the dedicated URL,
API token, username, and password documented in the server env example. The
CloudDM password must match the PostgreSQL administrator password configured
in the AWX runtime credential.
```bash ```bash
./scripts/setup-postgresql-awx.sh ./scripts/setup-postgresql-awx.sh
``` ```
+3
View File
@@ -40,4 +40,7 @@ XINFRA_POSTGRES_REPLICATION_PASSWORD=
AWX_JOB_TEMPLATE_NAME="XINFRA PostgreSQL Delivery" AWX_JOB_TEMPLATE_NAME="XINFRA PostgreSQL Delivery"
AWX_JOB_TEMPLATE_DESCRIPTION="PostgreSQL host_pool delivery template for XINFRA" AWX_JOB_TEMPLATE_DESCRIPTION="PostgreSQL host_pool delivery template for XINFRA"
AWX_PLAYBOOK=postgresql-deploy.yml AWX_PLAYBOOK=postgresql-deploy.yml
AWX_ROLLBACK_JOB_TEMPLATE_NAME="XINFRA PostgreSQL Rollback"
AWX_ROLLBACK_JOB_TEMPLATE_DESCRIPTION="PostgreSQL compensating cleanup template for XINFRA"
AWX_ROLLBACK_PLAYBOOK=postgresql-rollback.yml
AWX_INSTANCE_GROUP=xinfra-e2e-execution AWX_INSTANCE_GROUP=xinfra-e2e-execution
@@ -14,3 +14,11 @@ DELIVERY_GLOBAL_LIMIT=2
DELIVERY_TARGET_LIMIT=2 DELIVERY_TARGET_LIMIT=2
DELIVERY_BUSINESS_LIMIT=1 DELIVERY_BUSINESS_LIMIT=1
DELIVERY_DATA_DISKS=/data DELIVERY_DATA_DISKS=/data
DELIVERY_POSTGRESQL_ROLLBACK_TEMPLATE_ID=0
# Optional PostgreSQL CloudDM registration. Keep the URL empty to disable it.
# The password must match the postgres administrator password delivered by AWX.
CLOUDDM_POSTGRESQL_REGISTER_URL=
CLOUDDM_API_TOKEN=
CLOUDDM_POSTGRESQL_USERNAME=postgres
CLOUDDM_POSTGRESQL_PASSWORD=
+3 -3
View File
@@ -732,7 +732,7 @@ const basicServices = ref<Service[]>([
charsets: ['UTF8'], charsets: ['UTF8'],
defaultPort: 0, defaultPort: 0,
defaultPaths: { install: '/usr/lib/postgresql', data: '/data/postgresql', log: '/data/postgresql' }, defaultPaths: { install: '/usr/lib/postgresql', data: '/data/postgresql', log: '/data/postgresql' },
registerTo: 'PostgreSQL 专用资源台账', registerTo: 'PostgreSQL 资源台账 / CloudDM(可选)',
configId: 'PG-NATIVE-V1', configId: 'PG-NATIVE-V1',
assetId: '', assetId: '',
healthText: 'PostgreSQL 就绪 · TCP 探测通过', healthText: 'PostgreSQL 就绪 · TCP 探测通过',
@@ -835,7 +835,7 @@ const rollbackReleasing = ref(false)
const cloudDMRetrying = ref(false) const cloudDMRetrying = ref(false)
const rollbackActionBusy = computed(() => rollbackRetrying.value || rollbackReleasing.value) const rollbackActionBusy = computed(() => rollbackRetrying.value || rollbackReleasing.value)
const canManageRollback = computed(() => authStore.isAdmin && lastDeliveryStatus.value === 'rollback_failed' && Boolean(deploymentId.value)) const canManageRollback = computed(() => authStore.isAdmin && lastDeliveryStatus.value === 'rollback_failed' && Boolean(deploymentId.value))
const canRetryCloudDM = computed(() => !isPostgreSQL.value && deliveryRegisterFailed.value && Boolean(deploymentId.value)) const canRetryCloudDM = computed(() => deliveryRegisterFailed.value && Boolean(deploymentId.value))
const deliveryForm = reactive({ const deliveryForm = reactive({
instanceName: generateInstanceName(activeServiceKey.value, currentName.value), instanceName: generateInstanceName(activeServiceKey.value, currentName.value),
@@ -1318,7 +1318,7 @@ function defaultSteps(): DeliveryStep[] {
['应用实例配置', 'postgresql.conf、pg_hba.conf、systemd'], ['应用实例配置', 'postgresql.conf、pg_hba.conf、systemd'],
['配置复制拓扑', '复制账号、slot、primary_conninfo'], ['配置复制拓扑', '复制账号、slot、primary_conninfo'],
['数据库健康检查', '端口、角色、版本和数据目录'], ['数据库健康检查', '端口、角色、版本和数据目录'],
['资源入账与交付归档', 'PostgreSQL 集群、实例与资源台账'], ['资源入账与交付归档', 'PostgreSQL 资源台账与可选 CloudDM 注册'],
].map(([name, desc]) => ({ name, desc, state: 'pending' as StepState })) ].map(([name, desc]) => ({ name, desc, state: 'pending' as StepState }))
} }
const isMysql = activeServiceKey.value === 'mysql' const isMysql = activeServiceKey.value === 'mysql'
+54 -18
View File
@@ -37,7 +37,8 @@ for variable_name in \
AWX_HOST_NAME AWX_HOST_ADDRESS \ AWX_HOST_NAME AWX_HOST_ADDRESS \
AWX_MACHINE_CREDENTIAL_NAME AWX_SSH_PRIVATE_KEY_FILE \ AWX_MACHINE_CREDENTIAL_NAME AWX_SSH_PRIVATE_KEY_FILE \
AWX_POSTGRES_CREDENTIAL_TYPE_NAME AWX_POSTGRES_CREDENTIAL_NAME \ AWX_POSTGRES_CREDENTIAL_TYPE_NAME AWX_POSTGRES_CREDENTIAL_NAME \
AWX_JOB_TEMPLATE_NAME AWX_PLAYBOOK; do AWX_JOB_TEMPLATE_NAME AWX_PLAYBOOK \
AWX_ROLLBACK_JOB_TEMPLATE_NAME AWX_ROLLBACK_PLAYBOOK; do
require_value "${variable_name}" require_value "${variable_name}"
done done
@@ -64,6 +65,7 @@ case ${PROJECT_SOURCE}/ in
*) die "AWX_PROJECT_SOURCE_DIR must resolve inside the repository" ;; *) die "AWX_PROJECT_SOURCE_DIR must resolve inside the repository" ;;
esac esac
[[ -f ${PROJECT_SOURCE}/${AWX_PLAYBOOK} ]] || die "playbook not found: ${PROJECT_SOURCE}/${AWX_PLAYBOOK}" [[ -f ${PROJECT_SOURCE}/${AWX_PLAYBOOK} ]] || die "playbook not found: ${PROJECT_SOURCE}/${AWX_PLAYBOOK}"
[[ -f ${PROJECT_SOURCE}/${AWX_ROLLBACK_PLAYBOOK} ]] || die "rollback playbook not found: ${PROJECT_SOURCE}/${AWX_ROLLBACK_PLAYBOOK}"
if [[ -z ${AWX_TOKEN:-} && -z ${AWX_PASSWORD:-} ]]; then if [[ -z ${AWX_TOKEN:-} && -z ${AWX_PASSWORD:-} ]]; then
AWX_PASSWORD=$(kubectl get secret \ AWX_PASSWORD=$(kubectl get secret \
@@ -186,6 +188,10 @@ if ! awx_request GET "/api/v2/projects/${project_id}/playbooks/" | \
jq -e --arg playbook "${AWX_PLAYBOOK}" 'index($playbook) != null' >/dev/null; then jq -e --arg playbook "${AWX_PLAYBOOK}" 'index($playbook) != null' >/dev/null; then
die "AWX project ${project_id} does not expose playbook ${AWX_PLAYBOOK}" die "AWX project ${project_id} does not expose playbook ${AWX_PLAYBOOK}"
fi fi
if ! awx_request GET "/api/v2/projects/${project_id}/playbooks/" | \
jq -e --arg playbook "${AWX_ROLLBACK_PLAYBOOK}" 'index($playbook) != null' >/dev/null; then
die "AWX project ${project_id} does not expose rollback playbook ${AWX_ROLLBACK_PLAYBOOK}"
fi
inventory_payload=$(jq -nc \ inventory_payload=$(jq -nc \
--arg name "${AWX_INVENTORY_NAME}" \ --arg name "${AWX_INVENTORY_NAME}" \
@@ -307,29 +313,57 @@ job_template_payload=$(jq -nc \
job_template_id=$(upsert_named_object '/api/v2/job_templates/' \ job_template_id=$(upsert_named_object '/api/v2/job_templates/' \
"${AWX_JOB_TEMPLATE_NAME}" "${job_template_payload}") "${AWX_JOB_TEMPLATE_NAME}" "${job_template_payload}")
while IFS= read -r existing_group_id; do rollback_template_payload=$(jq -nc \
[[ -z ${existing_group_id} || ${existing_group_id} == "${instance_group_id}" ]] && continue --arg name "${AWX_ROLLBACK_JOB_TEMPLATE_NAME}" \
awx_request POST "/api/v2/job_templates/${job_template_id}/instance_groups/" \ --arg description "${AWX_ROLLBACK_JOB_TEMPLATE_DESCRIPTION:-PostgreSQL rollback template}" \
"$(jq -nc --argjson id "${existing_group_id}" '{id:$id,disassociate:true}')" >/dev/null --arg playbook "${AWX_ROLLBACK_PLAYBOOK}" \
done < <(awx_request GET "/api/v2/job_templates/${job_template_id}/instance_groups/" | jq -r '.results[].id') --argjson organization "${organization_id}" \
--argjson inventory "${inventory_id}" \
--argjson project "${project_id}" \
--argjson prevent_fallback "${prevent_fallback}" \
'{
name:$name,
description:$description,
organization:$organization,
inventory:$inventory,
project:$project,
playbook:$playbook,
job_type:"run",
ask_inventory_on_launch:true,
ask_variables_on_launch:true,
ask_limit_on_launch:true,
allow_simultaneous:true,
prevent_instance_group_fallback:$prevent_fallback
}')
rollback_template_id=$(upsert_named_object '/api/v2/job_templates/' \
"${AWX_ROLLBACK_JOB_TEMPLATE_NAME}" "${rollback_template_payload}")
for configured_template_id in "${job_template_id}" "${rollback_template_id}"; do
while IFS= read -r existing_group_id; do
[[ -z ${existing_group_id} || ${existing_group_id} == "${instance_group_id}" ]] && continue
awx_request POST "/api/v2/job_templates/${configured_template_id}/instance_groups/" \
"$(jq -nc --argjson id "${existing_group_id}" '{id:$id,disassociate:true}')" >/dev/null
done < <(awx_request GET "/api/v2/job_templates/${configured_template_id}/instance_groups/" | jq -r '.results[].id')
if ! awx_request GET "/api/v2/job_templates/${configured_template_id}/credentials/" | \
jq -e --argjson id "${machine_credential_id}" '.results | any(.id == $id)' >/dev/null; then
awx_request POST "/api/v2/job_templates/${configured_template_id}/credentials/" \
"$(jq -nc --argjson id "${machine_credential_id}" '{id:$id}')" >/dev/null
fi
if [[ -n ${instance_group_id} ]]; then
if ! awx_request GET "/api/v2/job_templates/${configured_template_id}/instance_groups/" | \
jq -e --argjson id "${instance_group_id}" '.results | any(.id == $id)' >/dev/null; then
awx_request POST "/api/v2/job_templates/${configured_template_id}/instance_groups/" \
"$(jq -nc --argjson id "${instance_group_id}" '{id:$id}')" >/dev/null
fi
fi
done
if ! awx_request GET "/api/v2/job_templates/${job_template_id}/credentials/" | \
jq -e --argjson id "${machine_credential_id}" '.results | any(.id == $id)' >/dev/null; then
awx_request POST "/api/v2/job_templates/${job_template_id}/credentials/" \
"$(jq -nc --argjson id "${machine_credential_id}" '{id:$id}')" >/dev/null
fi
if ! awx_request GET "/api/v2/job_templates/${job_template_id}/credentials/" | \ if ! awx_request GET "/api/v2/job_templates/${job_template_id}/credentials/" | \
jq -e --argjson id "${postgres_credential_id}" '.results | any(.id == $id)' >/dev/null; then jq -e --argjson id "${postgres_credential_id}" '.results | any(.id == $id)' >/dev/null; then
awx_request POST "/api/v2/job_templates/${job_template_id}/credentials/" \ awx_request POST "/api/v2/job_templates/${job_template_id}/credentials/" \
"$(jq -nc --argjson id "${postgres_credential_id}" '{id:$id}')" >/dev/null "$(jq -nc --argjson id "${postgres_credential_id}" '{id:$id}')" >/dev/null
fi fi
if [[ -n ${instance_group_id} ]]; then
if ! awx_request GET "/api/v2/job_templates/${job_template_id}/instance_groups/" | \
jq -e --argjson id "${instance_group_id}" '.results | any(.id == $id)' >/dev/null; then
awx_request POST "/api/v2/job_templates/${job_template_id}/instance_groups/" \
"$(jq -nc --argjson id "${instance_group_id}" '{id:$id}')" >/dev/null
fi
fi
printf '\nPostgreSQL AWX configuration is ready:\n' printf '\nPostgreSQL AWX configuration is ready:\n'
printf ' Project: %s (id=%s)\n' "${AWX_PROJECT_NAME}" "${project_id}" printf ' Project: %s (id=%s)\n' "${AWX_PROJECT_NAME}" "${project_id}"
@@ -338,6 +372,8 @@ printf ' Host: %s -> %s (id=%s)\n' "${AWX_HOST_NAME}" "${AWX_HOST_ADD
printf ' SSH credential: %s (id=%s)\n' "${AWX_MACHINE_CREDENTIAL_NAME}" "${machine_credential_id}" printf ' SSH credential: %s (id=%s)\n' "${AWX_MACHINE_CREDENTIAL_NAME}" "${machine_credential_id}"
printf ' PG credential: %s (id=%s)\n' "${AWX_POSTGRES_CREDENTIAL_NAME}" "${postgres_credential_id}" printf ' PG credential: %s (id=%s)\n' "${AWX_POSTGRES_CREDENTIAL_NAME}" "${postgres_credential_id}"
printf ' Job Template: %s (id=%s)\n' "${AWX_JOB_TEMPLATE_NAME}" "${job_template_id}" printf ' Job Template: %s (id=%s)\n' "${AWX_JOB_TEMPLATE_NAME}" "${job_template_id}"
printf ' Rollback: %s (id=%s)\n' "${AWX_ROLLBACK_JOB_TEMPLATE_NAME}" "${rollback_template_id}"
printf ' Server env: DELIVERY_POSTGRESQL_ROLLBACK_TEMPLATE_ID=%s\n' "${rollback_template_id}"
if [[ -n ${instance_group_id} ]]; then if [[ -n ${instance_group_id} ]]; then
printf ' Instance Group: %s (id=%s)\n' "${AWX_INSTANCE_GROUP}" "${instance_group_id}" printf ' Instance Group: %s (id=%s)\n' "${AWX_INSTANCE_GROUP}" "${instance_group_id}"
fi fi
+9 -2
View File
@@ -68,16 +68,23 @@ DELIVERY_MYSQL_INSPECT_TEMPLATE_NAME=XINFRA MySQL Inspect
DELIVERY_MYSQL_INSPECT_TIMEOUT_SECONDS=90 DELIVERY_MYSQL_INSPECT_TIMEOUT_SECONDS=90
# AWX Job Template ID for ansible/mysql-rollback.yml; required for automatic cleanup # AWX Job Template ID for ansible/mysql-rollback.yml; required for automatic cleanup
DELIVERY_ROLLBACK_TEMPLATE_ID=0 DELIVERY_ROLLBACK_TEMPLATE_ID=0
# AWX Job Template ID for ansible/postgresql-rollback.yml
DELIVERY_POSTGRESQL_ROLLBACK_TEMPLATE_ID=0
DELIVERY_SERVICE_TOKEN= DELIVERY_SERVICE_TOKEN=
# PostgreSQL runtime secrets are injected by an AWX Credential and are never # PostgreSQL deployment secrets are injected by an AWX Credential and are never
# persisted in xinfra task payloads: # persisted in xinfra task payloads. CloudDM registration, when enabled, uses
# the separate CLOUDDM_POSTGRESQL_* credential below.
# XINFRA_POSTGRES_ADMIN_PASSWORD # XINFRA_POSTGRES_ADMIN_PASSWORD
# XINFRA_POSTGRES_REPLICATION_PASSWORD # XINFRA_POSTGRES_REPLICATION_PASSWORD
# XINFRA_POSTGRES_REPLICATION_USER (optional, defaults to xinfra_replication) # XINFRA_POSTGRES_REPLICATION_USER (optional, defaults to xinfra_replication)
CLOUDDM_REGISTER_URL= CLOUDDM_REGISTER_URL=
CLOUDDM_DELETE_URL= CLOUDDM_DELETE_URL=
CLOUDDM_API_TOKEN= CLOUDDM_API_TOKEN=
# PostgreSQL uses an explicit contract and separate endpoint/credential.
CLOUDDM_POSTGRESQL_REGISTER_URL=
CLOUDDM_POSTGRESQL_USERNAME=postgres
CLOUDDM_POSTGRESQL_PASSWORD=
JWT_SECRET=change-this-secret JWT_SECRET=change-this-secret
JWT_ISSUER=authserver JWT_ISSUER=authserver
+160 -152
View File
@@ -14,82 +14,86 @@ type OAuthClient struct {
} }
type Config struct { type Config struct {
AppEnv string AppEnv string
HTTPAddr string HTTPAddr string
PublicBaseURL string PublicBaseURL string
MySQLDSN string MySQLDSN string
AutoMigrate bool AutoMigrate bool
SSOEnabled bool SSOEnabled bool
JWTSecret string JWTSecret string
JWTIssuer string JWTIssuer string
JWTTTLMinutes int JWTTTLMinutes int
SAMLEntityID string SAMLEntityID string
SAMLACSURL string SAMLACSURL string
SAMLSPCert string SAMLSPCert string
SAMLSPKey string SAMLSPKey string
SAMLIDPMetaURL string SAMLIDPMetaURL string
SAMLLogoutURL string SAMLLogoutURL string
WayenLoginURL string WayenLoginURL string
WayenTargetURL string WayenTargetURL string
WayenUsernameKey string WayenUsernameKey string
WayenPasswordKey string WayenPasswordKey string
WayenLoginFormat string WayenLoginFormat string
WayenLoginValue string WayenLoginValue string
WayenOAuthRef string WayenOAuthRef string
WayenOAuthLoginURL string WayenOAuthLoginURL string
WayneAPIBaseURL string WayneAPIBaseURL string
WayneAdminUsername string WayneAdminUsername string
WayneAdminPassword string WayneAdminPassword string
WayneTokenTTLMinutes int WayneTokenTTLMinutes int
WayneInternalAPIBaseURL string WayneInternalAPIBaseURL string
WayneServiceName string WayneServiceName string
WayneServiceAPISecretKey string WayneServiceAPISecretKey string
OAuthClientID string OAuthClientID string
OAuthClientSecret string OAuthClientSecret string
OAuthRedirectURI string OAuthRedirectURI string
OAuthCodeTTLSeconds int OAuthCodeTTLSeconds int
OIDCIssuer string OIDCIssuer string
OIDCAuthorizeURL string OIDCAuthorizeURL string
OIDCTokenURL string OIDCTokenURL string
OIDCUserInfoURL string OIDCUserInfoURL string
OIDCJWKSURL string OIDCJWKSURL string
CloudDMClientID string CloudDMClientID string
CloudDMClientSecret string CloudDMClientSecret string
CloudDMRedirectURI string CloudDMRedirectURI string
CloudDMTargetURL string CloudDMTargetURL string
CloudDMPublicURL string CloudDMPublicURL string
CloudDMLoginURL string CloudDMLoginURL string
CloudDMAdminUsername string CloudDMAdminUsername string
CloudDMAdminPassword string CloudDMAdminPassword string
CloudDMRegisterURL string CloudDMRegisterURL string
CloudDMDeleteURL string CloudDMDeleteURL string
CloudDMAPIToken string CloudDMAPIToken string
AWXBaseURL string CloudDMPostgreSQLRegisterURL string
AWXToken string CloudDMPostgreSQLUsername string
AWXUsername string CloudDMPostgreSQLPassword string
AWXPassword string AWXBaseURL string
AWXWebhookToken string AWXToken string
AWXFactsTemplateID uint64 AWXUsername string
AWXFactsTimeoutSeconds int AWXPassword string
MySQLDeliveryTemplateName string AWXWebhookToken string
MySQLInspectTemplateName string AWXFactsTemplateID uint64
MySQLInspectTimeoutSeconds int AWXFactsTimeoutSeconds int
RollbackTemplateID uint64 MySQLDeliveryTemplateName string
DeliveryServiceToken string MySQLInspectTemplateName string
DeliverySchedulerEnabled bool MySQLInspectTimeoutSeconds int
DeliveryDispatchSeconds int RollbackTemplateID uint64
DeliveryPollSeconds int PostgreSQLRollbackTemplateID uint64
DeliveryCallbackBaseURL string DeliveryServiceToken string
DeliveryCredentialSecret string DeliverySchedulerEnabled bool
ReservationTTLMinutes int DeliveryDispatchSeconds int
DeliveryGlobalLimit int DeliveryPollSeconds int
DeliveryTargetLimit int DeliveryCallbackBaseURL string
DeliveryHostInstanceLimit int DeliveryCredentialSecret string
DeliveryBusinessLimit int ReservationTTLMinutes int
DeliveryDataDisks []string DeliveryGlobalLimit int
SINABaseURL string DeliveryTargetLimit int
SINAUsername string DeliveryHostInstanceLimit int
SINAPassword string DeliveryBusinessLimit int
DeliveryDataDisks []string
SINABaseURL string
SINAUsername string
SINAPassword string
} }
func Load() Config { func Load() Config {
@@ -102,82 +106,86 @@ func Load() Config {
oidcIssuer = strings.TrimRight(oidcIssuer, "/") oidcIssuer = strings.TrimRight(oidcIssuer, "/")
return Config{ return Config{
AppEnv: env("APP_ENV", "dev"), AppEnv: env("APP_ENV", "dev"),
HTTPAddr: httpAddr, HTTPAddr: httpAddr,
PublicBaseURL: publicBaseURL, PublicBaseURL: publicBaseURL,
MySQLDSN: env("MYSQL_DSN", "auth:auth@tcp(127.0.0.1:3306)/authserver?charset=utf8mb4&parseTime=True&loc=Local"), MySQLDSN: env("MYSQL_DSN", "auth:auth@tcp(127.0.0.1:3306)/authserver?charset=utf8mb4&parseTime=True&loc=Local"),
AutoMigrate: envBool("AUTO_MIGRATE", true), AutoMigrate: envBool("AUTO_MIGRATE", true),
SSOEnabled: envBool("SSO_ENABLED", true), SSOEnabled: envBool("SSO_ENABLED", true),
JWTSecret: env("JWT_SECRET", "change-this-secret"), JWTSecret: env("JWT_SECRET", "change-this-secret"),
JWTIssuer: env("JWT_ISSUER", "authserver"), JWTIssuer: env("JWT_ISSUER", "authserver"),
JWTTTLMinutes: envInt("JWT_TTL_MINUTES", 120), JWTTTLMinutes: envInt("JWT_TTL_MINUTES", 120),
SAMLEntityID: samlEntityID, SAMLEntityID: samlEntityID,
SAMLACSURL: samlACSURL, SAMLACSURL: samlACSURL,
SAMLSPCert: env("SAML_SP_CERT_FILE", "certs/sp.crt"), SAMLSPCert: env("SAML_SP_CERT_FILE", "certs/sp.crt"),
SAMLSPKey: env("SAML_SP_KEY_FILE", "certs/sp.key"), SAMLSPKey: env("SAML_SP_KEY_FILE", "certs/sp.key"),
SAMLIDPMetaURL: env("SAML_IDP_METADATA_URL", "http://sso-internal.dev.qiniu.io/saml2/meta"), SAMLIDPMetaURL: env("SAML_IDP_METADATA_URL", "http://sso-internal.dev.qiniu.io/saml2/meta"),
SAMLLogoutURL: trimURL(env("SAML_LOGOUT_URL", "")), SAMLLogoutURL: trimURL(env("SAML_LOGOUT_URL", "")),
WayenLoginURL: env("WAYEN_LOGIN_URL", ""), WayenLoginURL: env("WAYEN_LOGIN_URL", ""),
WayenTargetURL: env("WAYEN_TARGET_URL", ""), WayenTargetURL: env("WAYEN_TARGET_URL", ""),
WayenUsernameKey: env("WAYEN_USERNAME_KEY", "email"), WayenUsernameKey: env("WAYEN_USERNAME_KEY", "email"),
WayenPasswordKey: env("WAYEN_PASSWORD_KEY", "password"), WayenPasswordKey: env("WAYEN_PASSWORD_KEY", "password"),
WayenLoginFormat: env("WAYEN_LOGIN_FORMAT", "form"), WayenLoginFormat: env("WAYEN_LOGIN_FORMAT", "form"),
WayenLoginValue: env("WAYEN_LOGIN_VALUE", "email"), WayenLoginValue: env("WAYEN_LOGIN_VALUE", "email"),
WayenOAuthRef: env("WAYEN_OAUTH_REF", "/portal/namespace/1/app"), WayenOAuthRef: env("WAYEN_OAUTH_REF", "/portal/namespace/1/app"),
WayenOAuthLoginURL: trimURL(env("WAYEN_OAUTH_LOGIN_URL", "")), WayenOAuthLoginURL: trimURL(env("WAYEN_OAUTH_LOGIN_URL", "")),
WayneAPIBaseURL: trimURL(env("WAYNE_API_BASE_URL", env("WAYNE_INTERNAL_API_BASE_URL", ""))), WayneAPIBaseURL: trimURL(env("WAYNE_API_BASE_URL", env("WAYNE_INTERNAL_API_BASE_URL", ""))),
WayneAdminUsername: env("WAYNE_ADMIN_USERNAME", ""), WayneAdminUsername: env("WAYNE_ADMIN_USERNAME", ""),
WayneAdminPassword: env("WAYNE_ADMIN_PASSWORD", ""), WayneAdminPassword: env("WAYNE_ADMIN_PASSWORD", ""),
WayneTokenTTLMinutes: envInt("WAYNE_TOKEN_TTL_MINUTES", 1440), WayneTokenTTLMinutes: envInt("WAYNE_TOKEN_TTL_MINUTES", 1440),
WayneInternalAPIBaseURL: trimURL(env("WAYNE_INTERNAL_API_BASE_URL", "")), WayneInternalAPIBaseURL: trimURL(env("WAYNE_INTERNAL_API_BASE_URL", "")),
WayneServiceName: env("WAYNE_SERVICE_NAME", "xinfra"), WayneServiceName: env("WAYNE_SERVICE_NAME", "xinfra"),
WayneServiceAPISecretKey: env("WAYNE_SERVICE_API_SECRET_KEY", ""), WayneServiceAPISecretKey: env("WAYNE_SERVICE_API_SECRET_KEY", ""),
OAuthClientID: env("OAUTH_WAYNE_CLIENT_ID", "wayne"), OAuthClientID: env("OAUTH_WAYNE_CLIENT_ID", "wayne"),
OAuthClientSecret: env("OAUTH_WAYNE_CLIENT_SECRET", "wayne-secret"), OAuthClientSecret: env("OAUTH_WAYNE_CLIENT_SECRET", "wayne-secret"),
OAuthRedirectURI: env("OAUTH_WAYNE_REDIRECT_URI", ""), OAuthRedirectURI: env("OAUTH_WAYNE_REDIRECT_URI", ""),
OAuthCodeTTLSeconds: envInt("OAUTH_CODE_TTL_SECONDS", 120), OAuthCodeTTLSeconds: envInt("OAUTH_CODE_TTL_SECONDS", 120),
OIDCIssuer: oidcIssuer, OIDCIssuer: oidcIssuer,
OIDCAuthorizeURL: trimURL(env("OIDC_AUTHORIZATION_ENDPOINT", oidcIssuer+"/oauth/authorize")), OIDCAuthorizeURL: trimURL(env("OIDC_AUTHORIZATION_ENDPOINT", oidcIssuer+"/oauth/authorize")),
OIDCTokenURL: trimURL(env("OIDC_TOKEN_ENDPOINT", oidcIssuer+"/oauth/token")), OIDCTokenURL: trimURL(env("OIDC_TOKEN_ENDPOINT", oidcIssuer+"/oauth/token")),
OIDCUserInfoURL: trimURL(env("OIDC_USERINFO_ENDPOINT", oidcIssuer+"/oauth/userinfo")), OIDCUserInfoURL: trimURL(env("OIDC_USERINFO_ENDPOINT", oidcIssuer+"/oauth/userinfo")),
OIDCJWKSURL: trimURL(env("OIDC_JWKS_URI", oidcIssuer+"/oauth/jwks")), OIDCJWKSURL: trimURL(env("OIDC_JWKS_URI", oidcIssuer+"/oauth/jwks")),
CloudDMClientID: env("OIDC_CLOUDDM_CLIENT_ID", "clouddm"), CloudDMClientID: env("OIDC_CLOUDDM_CLIENT_ID", "clouddm"),
CloudDMClientSecret: env("OIDC_CLOUDDM_CLIENT_SECRET", ""), CloudDMClientSecret: env("OIDC_CLOUDDM_CLIENT_SECRET", ""),
CloudDMRedirectURI: env("OIDC_CLOUDDM_REDIRECT_URI", ""), CloudDMRedirectURI: env("OIDC_CLOUDDM_REDIRECT_URI", ""),
CloudDMTargetURL: env("CLOUDDM_TARGET_URL", ""), CloudDMTargetURL: env("CLOUDDM_TARGET_URL", ""),
CloudDMPublicURL: trimURL(env("CLOUDDM_PUBLIC_URL", "")), CloudDMPublicURL: trimURL(env("CLOUDDM_PUBLIC_URL", "")),
CloudDMLoginURL: trimURL(env("CLOUDDM_LOGIN_URL", "")), CloudDMLoginURL: trimURL(env("CLOUDDM_LOGIN_URL", "")),
CloudDMAdminUsername: env("CLOUDDM_ADMIN_USERNAME", ""), CloudDMAdminUsername: env("CLOUDDM_ADMIN_USERNAME", ""),
CloudDMAdminPassword: env("CLOUDDM_ADMIN_PASSWORD", ""), CloudDMAdminPassword: env("CLOUDDM_ADMIN_PASSWORD", ""),
CloudDMRegisterURL: trimURL(env("CLOUDDM_REGISTER_URL", "")), CloudDMRegisterURL: trimURL(env("CLOUDDM_REGISTER_URL", "")),
CloudDMDeleteURL: trimURL(env("CLOUDDM_DELETE_URL", "")), CloudDMDeleteURL: trimURL(env("CLOUDDM_DELETE_URL", "")),
CloudDMAPIToken: env("CLOUDDM_API_TOKEN", ""), CloudDMAPIToken: env("CLOUDDM_API_TOKEN", ""),
AWXBaseURL: trimURL(env("AWX_BASE_URL", "")), CloudDMPostgreSQLRegisterURL: trimURL(env("CLOUDDM_POSTGRESQL_REGISTER_URL", "")),
AWXToken: env("AWX_TOKEN", ""), CloudDMPostgreSQLUsername: env("CLOUDDM_POSTGRESQL_USERNAME", "postgres"),
AWXUsername: env("AWX_USERNAME", ""), CloudDMPostgreSQLPassword: env("CLOUDDM_POSTGRESQL_PASSWORD", ""),
AWXPassword: env("AWX_PASSWORD", ""), AWXBaseURL: trimURL(env("AWX_BASE_URL", "")),
AWXWebhookToken: env("AWX_WEBHOOK_TOKEN", ""), AWXToken: env("AWX_TOKEN", ""),
AWXFactsTemplateID: envUint64("AWX_FACTS_TEMPLATE_ID", 0), AWXUsername: env("AWX_USERNAME", ""),
AWXFactsTimeoutSeconds: envInt("AWX_FACTS_TIMEOUT_SECONDS", 45), AWXPassword: env("AWX_PASSWORD", ""),
MySQLDeliveryTemplateName: env("DELIVERY_MYSQL_TEMPLATE_NAME", "XINFRA MySQL Native Prototype"), AWXWebhookToken: env("AWX_WEBHOOK_TOKEN", ""),
MySQLInspectTemplateName: env("DELIVERY_MYSQL_INSPECT_TEMPLATE_NAME", "XINFRA MySQL Inspect"), AWXFactsTemplateID: envUint64("AWX_FACTS_TEMPLATE_ID", 0),
MySQLInspectTimeoutSeconds: envInt("DELIVERY_MYSQL_INSPECT_TIMEOUT_SECONDS", 90), AWXFactsTimeoutSeconds: envInt("AWX_FACTS_TIMEOUT_SECONDS", 45),
RollbackTemplateID: uint64(envInt("DELIVERY_ROLLBACK_TEMPLATE_ID", 0)), MySQLDeliveryTemplateName: env("DELIVERY_MYSQL_TEMPLATE_NAME", "XINFRA MySQL Native Prototype"),
DeliveryServiceToken: env("DELIVERY_SERVICE_TOKEN", ""), MySQLInspectTemplateName: env("DELIVERY_MYSQL_INSPECT_TEMPLATE_NAME", "XINFRA MySQL Inspect"),
DeliverySchedulerEnabled: envBool("DELIVERY_SCHEDULER_ENABLED", false), MySQLInspectTimeoutSeconds: envInt("DELIVERY_MYSQL_INSPECT_TIMEOUT_SECONDS", 90),
DeliveryDispatchSeconds: envInt("DELIVERY_DISPATCH_SECONDS", 5), RollbackTemplateID: uint64(envInt("DELIVERY_ROLLBACK_TEMPLATE_ID", 0)),
DeliveryPollSeconds: envInt("DELIVERY_POLL_SECONDS", 5), PostgreSQLRollbackTemplateID: uint64(envInt("DELIVERY_POSTGRESQL_ROLLBACK_TEMPLATE_ID", 0)),
DeliveryCallbackBaseURL: trimURL(env("DELIVERY_CALLBACK_BASE_URL", publicBaseURL)), DeliveryServiceToken: env("DELIVERY_SERVICE_TOKEN", ""),
DeliveryCredentialSecret: env("DELIVERY_CREDENTIAL_SECRET", env("JWT_SECRET", "change-this-secret")), DeliverySchedulerEnabled: envBool("DELIVERY_SCHEDULER_ENABLED", false),
ReservationTTLMinutes: envInt("DELIVERY_RESERVATION_TTL_MINUTES", 120), DeliveryDispatchSeconds: envInt("DELIVERY_DISPATCH_SECONDS", 5),
DeliveryGlobalLimit: envInt("DELIVERY_GLOBAL_LIMIT", 2), DeliveryPollSeconds: envInt("DELIVERY_POLL_SECONDS", 5),
DeliveryTargetLimit: envInt("DELIVERY_TARGET_LIMIT", 2), DeliveryCallbackBaseURL: trimURL(env("DELIVERY_CALLBACK_BASE_URL", publicBaseURL)),
DeliveryHostInstanceLimit: envInt("DELIVERY_HOST_INSTANCE_LIMIT", 4), DeliveryCredentialSecret: env("DELIVERY_CREDENTIAL_SECRET", env("JWT_SECRET", "change-this-secret")),
DeliveryBusinessLimit: envInt("DELIVERY_BUSINESS_LIMIT", 1), ReservationTTLMinutes: envInt("DELIVERY_RESERVATION_TTL_MINUTES", 120),
DeliveryDataDisks: splitCSV(env("DELIVERY_DATA_DISKS", "/data,/disk1,/mnt,/opt/mysql-delivery")), DeliveryGlobalLimit: envInt("DELIVERY_GLOBAL_LIMIT", 2),
SINABaseURL: trimURL(env("SINA_BASE_URL", "https://sinai.qiniu.io:443")), DeliveryTargetLimit: envInt("DELIVERY_TARGET_LIMIT", 2),
SINAUsername: env("SINA_USERNAME", ""), DeliveryHostInstanceLimit: envInt("DELIVERY_HOST_INSTANCE_LIMIT", 4),
SINAPassword: env("SINA_PASSWORD", ""), DeliveryBusinessLimit: envInt("DELIVERY_BUSINESS_LIMIT", 1),
DeliveryDataDisks: splitCSV(env("DELIVERY_DATA_DISKS", "/data,/disk1,/mnt,/opt/mysql-delivery")),
SINABaseURL: trimURL(env("SINA_BASE_URL", "https://sinai.qiniu.io:443")),
SINAUsername: env("SINA_USERNAME", ""),
SINAPassword: env("SINA_PASSWORD", ""),
} }
} }
+28 -7
View File
@@ -16,15 +16,16 @@ import (
) )
type DeliveryHandler struct { type DeliveryHandler struct {
service *service.DeliveryService service *service.DeliveryService
postgresqlService *service.PostgreSQLDeliveryService
} }
type PostgreSQLDeliveryHandler struct { type PostgreSQLDeliveryHandler struct {
service *service.PostgreSQLDeliveryService service *service.PostgreSQLDeliveryService
} }
func NewDeliveryHandler(s *service.DeliveryService) *DeliveryHandler { func NewDeliveryHandler(s *service.DeliveryService, postgresqlService *service.PostgreSQLDeliveryService) *DeliveryHandler {
return &DeliveryHandler{service: s} return &DeliveryHandler{service: s, postgresqlService: postgresqlService}
} }
func NewPostgreSQLDeliveryHandler(s *service.PostgreSQLDeliveryService) *PostgreSQLDeliveryHandler { func NewPostgreSQLDeliveryHandler(s *service.PostgreSQLDeliveryService) *PostgreSQLDeliveryHandler {
@@ -345,7 +346,14 @@ func (h *DeliveryHandler) RetryRollback(c *gin.Context) {
if !requirePlatformAdmin(c) { if !requirePlatformAdmin(c) {
return return
} }
if err := h.service.RetryRollback(c.Request.Context(), c.Param("id")); err != nil { taskID := c.Param("id")
task, _, err := h.service.GetTask(c.Request.Context(), taskID, 0, true)
if err == nil && task.ServiceType == "postgresql" && h.postgresqlService != nil {
err = h.postgresqlService.RetryRollback(c.Request.Context(), taskID)
} else if err == nil {
err = h.service.RetryRollback(c.Request.Context(), taskID)
}
if err != nil {
c.JSON(http.StatusConflict, gin.H{"error": err.Error()}) c.JSON(http.StatusConflict, gin.H{"error": err.Error()})
return return
} }
@@ -358,7 +366,14 @@ func (h *DeliveryHandler) AcknowledgeRollbackRelease(c *gin.Context) {
if !requirePlatformAdmin(c) { if !requirePlatformAdmin(c) {
return return
} }
if err := h.service.AcknowledgeRollbackRelease(c.Request.Context(), c.Param("id")); err != nil { taskID := c.Param("id")
task, _, err := h.service.GetTask(c.Request.Context(), taskID, 0, true)
if err == nil && task.ServiceType == "postgresql" && h.postgresqlService != nil {
err = h.postgresqlService.AcknowledgeRollbackRelease(c.Request.Context(), taskID)
} else if err == nil {
err = h.service.AcknowledgeRollbackRelease(c.Request.Context(), taskID)
}
if err != nil {
c.JSON(http.StatusConflict, gin.H{"error": err.Error()}) c.JSON(http.StatusConflict, gin.H{"error": err.Error()})
return return
} }
@@ -374,7 +389,8 @@ func (h *DeliveryHandler) RetryCloudDMRegistration(c *gin.Context) {
return return
} }
taskID := c.Param("id") taskID := c.Param("id")
if _, _, err := h.service.GetTask(c.Request.Context(), taskID, claims.UserID, claims.IsAdmin); err != nil { task, _, err := h.service.GetTask(c.Request.Context(), taskID, claims.UserID, claims.IsAdmin)
if err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) { if errors.Is(err, gorm.ErrRecordNotFound) {
c.JSON(http.StatusNotFound, gin.H{"error": "task not found"}) c.JSON(http.StatusNotFound, gin.H{"error": "task not found"})
return return
@@ -382,7 +398,12 @@ func (h *DeliveryHandler) RetryCloudDMRegistration(c *gin.Context) {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return return
} }
if err := h.service.RetryCloudDMRegistration(c.Request.Context(), taskID); err != nil { if task.ServiceType == "postgresql" && h.postgresqlService != nil {
err = h.postgresqlService.RetryCloudDMRegistration(c.Request.Context(), taskID)
} else {
err = h.service.RetryCloudDMRegistration(c.Request.Context(), taskID)
}
if err != nil {
c.JSON(http.StatusConflict, gin.H{"error": err.Error()}) c.JSON(http.StatusConflict, gin.H{"error": err.Error()})
return return
} }
+1
View File
@@ -167,6 +167,7 @@ type PostgreSQLInstance struct {
UpstreamInstanceID string `gorm:"size:63;not null;default:''" json:"upstream_instance_id"` UpstreamInstanceID string `gorm:"size:63;not null;default:''" json:"upstream_instance_id"`
ReplicationSlotName string `gorm:"size:63;not null;default:''" json:"replication_slot_name"` ReplicationSlotName string `gorm:"size:63;not null;default:''" json:"replication_slot_name"`
Status string `gorm:"size:32;not null;index" json:"status"` Status string `gorm:"size:32;not null;index" json:"status"`
Metadata string `gorm:"type:json" json:"metadata"`
BackupStatus string `gorm:"size:32;not null;default:'not_configured'" json:"backup_status"` BackupStatus string `gorm:"size:32;not null;default:'not_configured'" json:"backup_status"`
MonitoringStatus string `gorm:"size:32;not null;default:'not_configured'" json:"monitoring_status"` MonitoringStatus string `gorm:"size:32;not null;default:'not_configured'" json:"monitoring_status"`
CreatedAt time.Time `json:"created_at"` CreatedAt time.Time `json:"created_at"`
+1 -1
View File
@@ -90,7 +90,7 @@ func registerAuthServerRoutes(r *gin.Engine, deps Dependencies) {
clouddmHandler := handler.NewCloudDMHandler(deps.Config, auditService) clouddmHandler := handler.NewCloudDMHandler(deps.Config, auditService)
samlHandler := handler.NewSAMLHandler(deps.Config, authService) samlHandler := handler.NewSAMLHandler(deps.Config, authService)
oauthHandler := handler.NewOAuthHandler(deps.Config, deps.DB, auditService) oauthHandler := handler.NewOAuthHandler(deps.Config, deps.DB, auditService)
deliveryHandler := handler.NewDeliveryHandler(deliveryService) deliveryHandler := handler.NewDeliveryHandler(deliveryService, postgresqlDeliveryService)
deliveryCallbackHandler := handler.NewDeliveryCallbackHandler(deliveryService, deps.Config.AWXWebhookToken) deliveryCallbackHandler := handler.NewDeliveryCallbackHandler(deliveryService, deps.Config.AWXWebhookToken)
postgresqlDeliveryHandler := handler.NewPostgreSQLDeliveryHandler(postgresqlDeliveryService) postgresqlDeliveryHandler := handler.NewPostgreSQLDeliveryHandler(postgresqlDeliveryService)
containerServiceHandler := handler.NewContainerServiceHandler(deps.DB, wayneRoleBindingService) containerServiceHandler := handler.NewContainerServiceHandler(deps.DB, wayneRoleBindingService)
+7 -1
View File
@@ -405,6 +405,9 @@ func (s *DeliveryService) ListTargets(ctx context.Context, component string) ([]
if !strings.Contains(text, component) { if !strings.Contains(text, component) {
continue continue
} }
if component == postgresqlServiceType && strings.Contains(text, "rollback") {
continue
}
} }
target, err := s.awxDeliveryTarget(ctx, template) target, err := s.awxDeliveryTarget(ctx, template)
if err != nil { if err != nil {
@@ -2860,7 +2863,10 @@ func (s *DeliveryService) markRollbackFailed(ctx context.Context, taskID, messag
func (s *DeliveryService) PollRollbackOnce(ctx context.Context) error { func (s *DeliveryService) PollRollbackOnce(ctx context.Context) error {
var jobs []model.RollbackJob var jobs []model.RollbackJob
if err := s.db.WithContext(ctx).Where("status IN ?", []string{"launching", "running"}).Find(&jobs).Error; err != nil { if err := s.db.WithContext(ctx).
Joins("JOIN delivery_tasks ON delivery_tasks.id = rollback_jobs.task_id").
Where("rollback_jobs.status IN ? AND (delivery_tasks.service_type = '' OR delivery_tasks.service_type = ?)", []string{"launching", "running"}, "mysql").
Find(&jobs).Error; err != nil {
return err return err
} }
for _, rollback := range jobs { for _, rollback := range jobs {
+35 -40
View File
@@ -28,6 +28,7 @@ const (
var supportedPostgreSQLVersions = map[string]bool{"15": true, "16": true} var supportedPostgreSQLVersions = map[string]bool{"15": true, "16": true}
var supportedPostgreSQLTopologies = map[string]bool{"standalone": true, "primary_replica": true} var supportedPostgreSQLTopologies = map[string]bool{"standalone": true, "primary_replica": true}
var postgresqlNamePattern = regexp.MustCompile(`^[a-z0-9](?:[-a-z0-9]*[a-z0-9])?$`) var postgresqlNamePattern = regexp.MustCompile(`^[a-z0-9](?:[-a-z0-9]*[a-z0-9])?$`)
var errPostgreSQLTaskHandled = errors.New("postgresql task validation completed without dispatch")
type PostgreSQLDeliveryInput struct { type PostgreSQLDeliveryInput struct {
BusinessLineID uint64 `json:"business_line_id" binding:"required"` BusinessLineID uint64 `json:"business_line_id" binding:"required"`
@@ -230,7 +231,8 @@ func getPostgreSQLTarget(ctx context.Context, awx *AWXClient, templateID uint64)
if err != nil { if err != nil {
return DeliveryTarget{}, fmt.Errorf("deployment target is unavailable: %w", err) return DeliveryTarget{}, fmt.Errorf("deployment target is unavailable: %w", err)
} }
if !strings.Contains(strings.ToLower(template.Name+" "+template.Description), "postgresql") { templateText := strings.ToLower(template.Name + " " + template.Description)
if !strings.Contains(templateText, "postgresql") || strings.Contains(templateText, "rollback") {
return DeliveryTarget{}, fmt.Errorf("AWX job template %d is not a PostgreSQL target", templateID) return DeliveryTarget{}, fmt.Errorf("AWX job template %d is not a PostgreSQL target", templateID)
} }
hosts, err := awx.ListInventoryHosts(ctx, template.Inventory) hosts, err := awx.ListInventoryHosts(ctx, template.Inventory)
@@ -342,7 +344,12 @@ func (s *PostgreSQLDeliveryService) claimAndReserve(ctx context.Context) (*model
} }
port, err := allocateReachablePostgreSQLPort(ctx, host.IP, usedPorts, postgresqlPortInUse) port, err := allocateReachablePostgreSQLPort(ctx, host.IP, usedPorts, postgresqlPortInUse)
if err != nil { if err != nil {
_ = tx.Delete(&cluster).Error if cleanupErr := tx.Where("task_id = ?", task.ID).Delete(&model.PostgreSQLInstance{}).Error; cleanupErr != nil {
return cleanupErr
}
if cleanupErr := tx.Delete(&cluster).Error; cleanupErr != nil {
return cleanupErr
}
return s.common.transitionTx(tx, &task, model.TaskValidationFailed, err.Error(), err.Error()) return s.common.transitionTx(tx, &task, model.TaskValidationFailed, err.Error(), err.Error())
} }
role := "replica" role := "replica"
@@ -390,6 +397,9 @@ func (s *PostgreSQLDeliveryService) claimAndReserve(ctx context.Context) (*model
} }
return s.common.transitionTx(tx, &task, model.TaskDispatching, "PostgreSQL resources, ports and directories reserved", "") return s.common.transitionTx(tx, &task, model.TaskDispatching, "PostgreSQL resources, ports and directories reserved", "")
}) })
if err == nil && task.Status != model.TaskDispatching {
return nil, errPostgreSQLTaskHandled
}
return &task, err return &task, err
} }
@@ -463,13 +473,13 @@ func (s *PostgreSQLDeliveryService) CreateExecution(ctx context.Context, task *m
func (s *PostgreSQLDeliveryService) DispatchOnce(ctx context.Context) error { func (s *PostgreSQLDeliveryService) DispatchOnce(ctx context.Context) error {
task, err := s.claimAndReserve(ctx) task, err := s.claimAndReserve(ctx)
if err != nil { if err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) || strings.HasPrefix(err.Error(), "defer:") { if errors.Is(err, gorm.ErrRecordNotFound) || errors.Is(err, errPostgreSQLTaskHandled) || strings.HasPrefix(err.Error(), "defer:") {
return nil return nil
} }
return err return err
} }
if err := s.CreateExecution(ctx, task); err != nil { if err := s.CreateExecution(ctx, task); err != nil {
return s.fail(ctx, task.ID, model.TaskExecutionFailed, err.Error()) return s.beginRollback(ctx, task.ID, "PostgreSQL deployment could not be started: "+err.Error())
} }
return nil return nil
} }
@@ -490,13 +500,13 @@ func (s *PostgreSQLDeliveryService) PollOnce(ctx context.Context) error {
case "pending", "waiting", "running", "new": case "pending", "waiting", "running", "new":
continue continue
case "canceled": case "canceled":
_ = s.fail(ctx, execution.TaskID, model.TaskCanceled, "AWX job was canceled") _ = s.beginRollback(ctx, execution.TaskID, "PostgreSQL AWX job was canceled")
case "successful": case "successful":
if err := s.complete(ctx, execution.TaskID); err != nil { if err := s.complete(ctx, execution.TaskID); err != nil {
_ = s.fail(ctx, execution.TaskID, model.TaskValidationFailed, err.Error()) _ = s.beginRollback(ctx, execution.TaskID, err.Error())
} }
default: default:
_ = s.fail(ctx, execution.TaskID, model.TaskExecutionFailed, "AWX job finished with status "+job.Status) _ = s.beginRollback(ctx, execution.TaskID, "PostgreSQL AWX job finished with status "+job.Status)
} }
} }
return nil return nil
@@ -524,7 +534,7 @@ func (s *PostgreSQLDeliveryService) complete(ctx context.Context, taskID string)
} }
} }
now := time.Now() now := time.Now()
return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { if err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
if err := s.common.transitionTx(tx, &task, model.TaskRegistering, "PostgreSQL health checks passed", ""); err != nil { if err := s.common.transitionTx(tx, &task, model.TaskRegistering, "PostgreSQL health checks passed", ""); err != nil {
return err return err
} }
@@ -545,8 +555,22 @@ func (s *PostgreSQLDeliveryService) complete(ctx context.Context, taskID string)
if err := tx.Model(&model.ExecutionJob{}).Where("task_id = ?", task.ID).Updates(map[string]any{"status": "successful", "finished_at": now}).Error; err != nil { if err := tx.Model(&model.ExecutionJob{}).Where("task_id = ?", task.ID).Updates(map[string]any{"status": "successful", "finished_at": now}).Error; err != nil {
return err return err
} }
return s.common.transitionTx(tx, &task, model.TaskFinished, "PostgreSQL delivery completed and recorded in the PostgreSQL resource ledger", "") return nil
}) }); err != nil {
return err
}
if err := s.RegisterCloudDM(ctx, task.ID); err != nil {
message := fmt.Sprintf("PostgreSQL delivered; CloudDM registration failed: %v", err)
if transitionErr := s.common.transition(ctx, &task, model.TaskRegisterFailed, "PostgreSQL delivered; CloudDM registration failed and can be retried", message); transitionErr != nil {
return fmt.Errorf("%s; cannot record register_failed: %w", message, transitionErr)
}
return nil
}
message := "PostgreSQL delivery completed; CloudDM registration was skipped because no PostgreSQL endpoint is configured"
if strings.TrimSpace(s.cfg.CloudDMPostgreSQLRegisterURL) != "" {
message = "PostgreSQL delivery completed, registered and recorded in the resource ledger"
}
return s.common.transition(ctx, &task, model.TaskFinished, message, "")
} }
func postgresReady(ctx context.Context, address string) error { func postgresReady(ctx context.Context, address string) error {
@@ -558,36 +582,6 @@ func postgresReady(ctx context.Context, address string) error {
return conn.Close() return conn.Close()
} }
func (s *PostgreSQLDeliveryService) fail(ctx context.Context, taskID, status, message string) error {
return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
var task model.DeliveryTask
if err := tx.First(&task, "id = ? AND service_type = ?", taskID, postgresqlServiceType).Error; err != nil {
return err
}
if err := s.common.transitionTx(tx, &task, status, message, message); err != nil {
return err
}
reservationStatus := "released"
if status == model.TaskExecutionFailed || (status == model.TaskValidationFailed && strings.Contains(strings.ToLower(message), "health")) {
reservationStatus = "quarantined"
}
if err := tx.Model(&model.ResourceReservation{}).Where("task_id = ? AND status = ?", taskID, "reserved").Update("status", reservationStatus).Error; err != nil {
return err
}
instanceStatus := "failed"
if reservationStatus == "quarantined" {
instanceStatus = "quarantined"
}
if err := tx.Model(&model.PostgreSQLInstance{}).Where("task_id = ? AND status = ?", taskID, "provisioning").Update("status", instanceStatus).Error; err != nil {
return err
}
if err := tx.Model(&model.PostgreSQLCluster{}).Where("task_id = ? AND status = ?", taskID, "provisioning").Update("status", instanceStatus).Error; err != nil {
return err
}
return tx.Model(&model.ExecutionJob{}).Where("task_id = ? AND status IN ?", taskID, []string{"launching", "running"}).Updates(map[string]any{"status": "failed", "finished_at": time.Now()}).Error
})
}
func (s *PostgreSQLDeliveryService) Run(ctx context.Context) { func (s *PostgreSQLDeliveryService) Run(ctx context.Context) {
interval := time.Duration(s.cfg.DeliveryPollSeconds) * time.Second interval := time.Duration(s.cfg.DeliveryPollSeconds) * time.Second
if interval < time.Second { if interval < time.Second {
@@ -602,6 +596,7 @@ func (s *PostgreSQLDeliveryService) Run(ctx context.Context) {
case <-ticker.C: case <-ticker.C:
_ = s.DispatchOnce(ctx) _ = s.DispatchOnce(ctx)
_ = s.PollOnce(ctx) _ = s.PollOnce(ctx)
_ = s.PollRollbackOnce(ctx)
} }
} }
} }
@@ -2,7 +2,10 @@ package service
import ( import (
"context" "context"
"encoding/json"
"testing" "testing"
"github.com/1024XEngineer/xinfra/server/internal/model"
) )
func TestValidatePostgreSQLDeliveryInput(t *testing.T) { func TestValidatePostgreSQLDeliveryInput(t *testing.T) {
@@ -102,3 +105,42 @@ func TestSelectPostgreSQLHosts(t *testing.T) {
t.Fatal("host outside the pool was accepted") t.Fatal("host outside the pool was accepted")
} }
} }
func TestBuildPostgreSQLCloudDMRegisterRequest(t *testing.T) {
req := buildPostgreSQLCloudDMRegisterRequest(
model.PostgreSQLInstance{ID: 42, InstanceID: "orders-pg-primary", HostIP: "10.0.0.10", Port: 15432, VersionMajor: "16", Role: "primary"},
model.PostgreSQLCluster{ID: 7, Name: "orders-pg"},
"postgres",
"secret",
)
if req.SourceSystem != "xinfra" || req.ResourceType != "POSTGRESQL_INSTANCE" {
t.Fatalf("unexpected request metadata: %#v", req)
}
if req.ExternalResourceID != "postgresql-instance:42" {
t.Fatalf("unexpected externalResourceId: %q", req.ExternalResourceID)
}
if req.DataSource.DSType != "PostgreSQL" || req.DataSource.Host != "10.0.0.10:15432" || req.DataSource.UserName != "postgres" || req.DataSource.Password != "secret" {
t.Fatalf("unexpected PostgreSQL data source: %#v", req.DataSource)
}
if req.DataSource.PostgreSQLVersion != "16" || req.DataSource.DefaultSchema != "postgres" {
t.Fatalf("unexpected PostgreSQL version/schema: %#v", req.DataSource)
}
raw, err := json.Marshal(req)
if err != nil {
t.Fatalf("marshal PostgreSQL CloudDM request: %v", err)
}
var decoded map[string]any
if err := json.Unmarshal(raw, &decoded); err != nil {
t.Fatalf("decode PostgreSQL CloudDM request: %v", err)
}
dataSource := decoded["dataSource"].(map[string]any)
if dataSource["postgresql_version"] != "16" {
t.Fatalf("postgresql_version missing from contract: %#v", dataSource)
}
}
func TestPostgreSQLCloudDMExternalResourceID(t *testing.T) {
if got := postgresqlCloudDMExternalResourceID(99); got != "postgresql-instance:99" {
t.Fatalf("external resource id = %q", got)
}
}
@@ -0,0 +1,426 @@
package service
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net"
"net/http"
"strconv"
"strings"
"time"
"github.com/1024XEngineer/xinfra/server/internal/model"
"gorm.io/gorm"
)
type postgresqlCloudDMRegisterRequest struct {
SourceSystem string `json:"sourceSystem"`
ResourceType string `json:"resourceType"`
ExternalResourceID string `json:"externalResourceId"`
DataSource postgresqlCloudDMDataSource `json:"dataSource"`
}
type postgresqlCloudDMDataSource struct {
InstanceName string `json:"instanceName"`
InstanceDesc string `json:"instanceDesc"`
DSType string `json:"dsType"`
Host string `json:"host"`
ClusterID *uint64 `json:"cluster_id"`
PostgreSQLVersion string `json:"postgresql_version"`
SecurityType string `json:"securityType"`
UserName string `json:"userName"`
Password string `json:"password"`
DefaultSchema string `json:"defaultSchema"`
ClientTimeZone string `json:"clientTimeZone"`
ConnectTimeoutMs int `json:"connectTimeoutMs"`
SocketTimeoutSecs int `json:"socketTimeoutSeconds"`
ConnectionCharset string `json:"connectionCharset"`
}
func postgresqlCloudDMExternalResourceID(instanceID uint64) string {
return "postgresql-instance:" + strconv.FormatUint(instanceID, 10)
}
func buildPostgreSQLCloudDMRegisterRequest(instance model.PostgreSQLInstance, cluster model.PostgreSQLCluster, username, password string) postgresqlCloudDMRegisterRequest {
description := fmt.Sprintf("%s PostgreSQL %s", cluster.Name, instance.Role)
return postgresqlCloudDMRegisterRequest{
SourceSystem: "xinfra",
ResourceType: "POSTGRESQL_INSTANCE",
ExternalResourceID: postgresqlCloudDMExternalResourceID(instance.ID),
DataSource: postgresqlCloudDMDataSource{
InstanceName: instance.InstanceID,
InstanceDesc: description,
DSType: "PostgreSQL",
Host: net.JoinHostPort(instance.HostIP, strconv.Itoa(instance.Port)),
ClusterID: nil,
PostgreSQLVersion: instance.VersionMajor,
SecurityType: "USER_PASSWD",
UserName: username,
Password: password,
DefaultSchema: "postgres",
ClientTimeZone: "Asia/Shanghai",
ConnectTimeoutMs: 5000,
SocketTimeoutSecs: 10,
ConnectionCharset: "utf8",
},
}
}
func postgresqlCloudDMMetadata(existing map[string]any, instance model.PostgreSQLInstance, dataSourceID uint64) map[string]any {
meta := map[string]any{}
for key, value := range existing {
meta[key] = value
}
meta["clouddm"] = map[string]any{
"data_source_id": dataSourceID,
"external_resource_id": postgresqlCloudDMExternalResourceID(instance.ID),
"registered_at": time.Now().Format(time.RFC3339),
}
return meta
}
func (s *PostgreSQLDeliveryService) RegisterCloudDM(ctx context.Context, taskID string) error {
registerURL := strings.TrimSpace(s.cfg.CloudDMPostgreSQLRegisterURL)
if registerURL == "" {
return nil
}
if strings.TrimSpace(s.cfg.CloudDMAPIToken) == "" {
return fmt.Errorf("PostgreSQL CloudDM registration is enabled but CLOUDDM_API_TOKEN is not configured")
}
username := strings.TrimSpace(s.cfg.CloudDMPostgreSQLUsername)
password := s.cfg.CloudDMPostgreSQLPassword
if username == "" || password == "" {
return fmt.Errorf("PostgreSQL CloudDM registration requires CLOUDDM_POSTGRESQL_USERNAME and CLOUDDM_POSTGRESQL_PASSWORD")
}
var cluster model.PostgreSQLCluster
if err := s.db.WithContext(ctx).First(&cluster, "task_id = ?", taskID).Error; err != nil {
return err
}
var instances []model.PostgreSQLInstance
if err := s.db.WithContext(ctx).Where("task_id = ?", taskID).Order("id ASC").Find(&instances).Error; err != nil {
return err
}
if len(instances) == 0 {
return fmt.Errorf("PostgreSQL task has no instances to register")
}
client := &http.Client{Timeout: 30 * time.Second}
for _, instance := range instances {
body := buildPostgreSQLCloudDMRegisterRequest(instance, cluster, username, password)
raw, err := json.Marshal(body)
if err != nil {
return fmt.Errorf("encode PostgreSQL CloudDM registration request: %w", err)
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, registerURL, bytes.NewReader(raw))
if err != nil {
return err
}
req.Header.Set("Authorization", "Bearer "+strings.TrimSpace(s.cfg.CloudDMAPIToken))
req.Header.Set("Content-Type", "application/json")
req.Header.Set("X-Request-ID", randomUUID())
resp, err := client.Do(req)
if err != nil {
return err
}
respBody, readErr := io.ReadAll(resp.Body)
closeErr := resp.Body.Close()
if readErr != nil {
return readErr
}
if closeErr != nil {
return closeErr
}
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return fmt.Errorf("PostgreSQL CloudDM registration returned %s: %s", resp.Status, strings.TrimSpace(string(respBody)))
}
dataSourceID, ok := cloudDMDataSourceIDFromResponse(respBody)
if !ok {
return fmt.Errorf("PostgreSQL CloudDM registration response did not include dataSourceId: %s", strings.TrimSpace(string(respBody)))
}
meta := postgresqlCloudDMMetadata(metadataMap(instance.Metadata), instance, dataSourceID)
if err := s.db.WithContext(ctx).Model(&instance).Update("metadata", string(mustJSON(meta))).Error; err != nil {
return fmt.Errorf("save PostgreSQL CloudDM dataSourceId: %w", err)
}
}
return nil
}
func (s *PostgreSQLDeliveryService) RetryCloudDMRegistration(ctx context.Context, taskID string) error {
var task model.DeliveryTask
if err := s.db.WithContext(ctx).First(&task, "id = ? AND service_type = ?", taskID, postgresqlServiceType).Error; err != nil {
return err
}
if task.Status != model.TaskRegisterFailed {
return fmt.Errorf("task %s is in state %q and cannot retry CloudDM registration", taskID, task.Status)
}
if err := s.common.transition(ctx, &task, model.TaskRegistering, "PostgreSQL CloudDM registration retry started", ""); err != nil {
return err
}
if err := s.RegisterCloudDM(ctx, taskID); err != nil {
message := fmt.Sprintf("PostgreSQL CloudDM registration failed: %v", err)
if transitionErr := s.common.transition(ctx, &task, model.TaskRegisterFailed, "PostgreSQL CloudDM registration retry failed; instances remain active", message); transitionErr != nil {
return fmt.Errorf("%s; cannot restore register_failed: %w", message, transitionErr)
}
return err
}
return s.common.transition(ctx, &task, model.TaskFinished, "PostgreSQL CloudDM registration completed", "")
}
func (s *PostgreSQLDeliveryService) beginRollback(ctx context.Context, taskID, reason string) error {
var task model.DeliveryTask
if err := s.db.WithContext(ctx).First(&task, "id = ? AND service_type = ?", taskID, postgresqlServiceType).Error; err != nil {
return err
}
if rollbackProtectedStatus(task.Status) {
return nil
}
if err := s.common.holdReservationForRollback(ctx, task.ID); err != nil {
return s.markRollbackFailed(ctx, task.ID, "cannot hold PostgreSQL resources during rollback: "+err.Error())
}
if s.cfg.PostgreSQLRollbackTemplateID == 0 {
return s.markRollbackFailed(ctx, task.ID, "PostgreSQL automatic rollback is not configured; manual cleanup is required: "+reason)
}
if err := s.common.transition(ctx, &task, model.TaskRollbackPending, "PostgreSQL automatic rollback queued", reason); err != nil {
return err
}
if err := s.setRollbackResourceStatus(ctx, task.ID, "rollback_pending"); err != nil {
return s.markRollbackFailed(ctx, task.ID, "cannot mark PostgreSQL resources for rollback: "+err.Error())
}
return s.launchRollbackJob(ctx, &task, reason)
}
func (s *PostgreSQLDeliveryService) launchRollbackJob(ctx context.Context, task *model.DeliveryTask, reason string) error {
target, err := getPostgreSQLTarget(ctx, s.awx, task.TargetID)
if err != nil {
return s.markRollbackFailed(ctx, task.ID, "PostgreSQL rollback target is unavailable: "+err.Error())
}
var instances []model.PostgreSQLInstance
if err := s.db.WithContext(ctx).Where("task_id = ?", task.ID).Order("id ASC").Find(&instances).Error; err != nil {
return s.markRollbackFailed(ctx, task.ID, "cannot load PostgreSQL rollback plan: "+err.Error())
}
if len(instances) == 0 {
return s.markRollbackFailed(ctx, task.ID, "PostgreSQL rollback plan has no instances")
}
instanceVars := make(map[string]any, len(instances))
for _, instance := range instances {
instanceVars[instance.Hostname] = map[string]any{
"instance_id": instance.InstanceID,
"data_dir": instance.DataDir,
"config_dir": instance.ConfigDir,
"log_dir": instance.LogDir,
"systemd_unit": instance.SystemdUnit,
}
}
now := time.Now()
var rollback model.RollbackJob
if err := s.db.WithContext(ctx).Where("task_id = ?", task.ID).First(&rollback).Error; errors.Is(err, gorm.ErrRecordNotFound) {
rollback = model.RollbackJob{TaskID: task.ID}
} else if err != nil {
return s.markRollbackFailed(ctx, task.ID, "cannot load PostgreSQL rollback job: "+err.Error())
}
rollback.ExecutorJobID = "pending-postgresql-rollback-" + task.ID
rollback.Status = "launching"
rollback.Reason = reason
rollback.StartedAt = &now
rollback.FinishedAt = nil
if err := s.db.WithContext(ctx).Save(&rollback).Error; err != nil {
return s.markRollbackFailed(ctx, task.ID, "cannot persist PostgreSQL rollback job: "+err.Error())
}
job, err := s.awx.Launch(ctx, s.cfg.PostgreSQLRollbackTemplateID, AWXLaunchRequest{
InventoryID: target.AWXInventoryID,
Limit: task.TargetHost,
ExtraVars: map[string]any{
"target_hosts": task.TargetHost,
"postgresql_instances": instanceVars,
"task_id": task.ID,
"rollback": true,
},
})
if err != nil {
_ = s.db.WithContext(ctx).Model(&rollback).Updates(map[string]any{"status": "failed", "finished_at": time.Now()}).Error
return s.markRollbackFailed(ctx, task.ID, "cannot launch PostgreSQL rollback AWX job: "+err.Error())
}
if err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
var current model.DeliveryTask
if err := tx.First(&current, "id = ? AND service_type = ?", task.ID, postgresqlServiceType).Error; err != nil {
return err
}
if current.Status != model.TaskRollbackPending {
return fmt.Errorf("task %s is no longer pending rollback (state %q)", task.ID, current.Status)
}
if err := tx.Model(&rollback).Updates(map[string]any{"executor_job_id": fmt.Sprint(job.ID), "status": "running"}).Error; err != nil {
return err
}
if err := tx.Model(&model.PostgreSQLInstance{}).Where("task_id = ?", task.ID).Update("status", "rolling_back").Error; err != nil {
return err
}
if err := tx.Model(&model.PostgreSQLCluster{}).Where("task_id = ?", task.ID).Update("status", "rolling_back").Error; err != nil {
return err
}
return s.common.transitionTx(tx, &current, model.TaskRollingBack, "PostgreSQL rollback AWX job started", reason)
}); err != nil {
return s.markRollbackFailed(ctx, task.ID, "cannot record PostgreSQL rollback AWX job: "+err.Error())
}
return nil
}
func (s *PostgreSQLDeliveryService) RetryRollback(ctx context.Context, taskID string) error {
var task model.DeliveryTask
if err := s.db.WithContext(ctx).First(&task, "id = ? AND service_type = ?", taskID, postgresqlServiceType).Error; err != nil {
return err
}
if task.Status != model.TaskRollbackFailed {
return fmt.Errorf("task %s is in state %q and cannot retry rollback", taskID, task.Status)
}
if s.cfg.PostgreSQLRollbackTemplateID == 0 {
return fmt.Errorf("PostgreSQL automatic rollback is not configured")
}
reason := "manual PostgreSQL rollback retry requested"
if err := s.common.transition(ctx, &task, model.TaskRollbackPending, "manual PostgreSQL rollback retry queued", reason); err != nil {
return err
}
if err := s.common.holdReservationForRollback(ctx, task.ID); err != nil {
_ = s.markRollbackFailed(ctx, task.ID, err.Error())
return err
}
if err := s.setRollbackResourceStatus(ctx, task.ID, "rollback_pending"); err != nil {
return s.markRollbackFailed(ctx, task.ID, err.Error())
}
return s.launchRollbackJob(ctx, &task, reason)
}
func (s *PostgreSQLDeliveryService) AcknowledgeRollbackRelease(ctx context.Context, taskID string) error {
now := time.Now()
return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
var task model.DeliveryTask
if err := tx.First(&task, "id = ? AND service_type = ?", taskID, postgresqlServiceType).Error; err != nil {
return err
}
if task.Status != model.TaskRollbackFailed {
return fmt.Errorf("task %s is in state %q and cannot acknowledge rollback release", taskID, task.Status)
}
if err := tx.Model(&model.PostgreSQLInstance{}).Where("task_id = ?", taskID).Update("status", "rollback_acknowledged").Error; err != nil {
return err
}
if err := tx.Model(&model.PostgreSQLCluster{}).Where("task_id = ?", taskID).Update("status", "rollback_acknowledged").Error; err != nil {
return err
}
if err := tx.Model(&model.PostgreSQLResourceUsage{}).Where("task_id = ? AND status = ?", taskID, "active").Updates(map[string]any{"status": "released", "released_at": now}).Error; err != nil {
return err
}
if err := tx.Model(&model.ResourceReservation{}).Where("task_id = ? AND status = ?", taskID, "rollback").Update("status", "released").Error; err != nil {
return err
}
return s.common.transitionTx(tx, &task, model.TaskRollbackAck, "PostgreSQL rollback release acknowledged by platform admin", "")
})
}
func (s *PostgreSQLDeliveryService) setRollbackResourceStatus(ctx context.Context, taskID, status string) error {
return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
if err := tx.Model(&model.PostgreSQLInstance{}).Where("task_id = ?", taskID).Update("status", status).Error; err != nil {
return err
}
return tx.Model(&model.PostgreSQLCluster{}).Where("task_id = ?", taskID).Update("status", status).Error
})
}
func (s *PostgreSQLDeliveryService) markRollbackFailed(ctx context.Context, taskID, message string) error {
return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
var task model.DeliveryTask
if err := tx.First(&task, "id = ? AND service_type = ?", taskID, postgresqlServiceType).Error; err != nil {
return err
}
if task.Status == model.TaskRolledBack || task.Status == model.TaskFinished || task.Status == model.TaskRegisterFailed || task.Status == model.TaskRollbackAck {
return nil
}
if task.Status != model.TaskRollbackFailed {
if err := s.common.transitionTx(tx, &task, model.TaskRollbackFailed, "PostgreSQL automatic rollback failed; manual cleanup is required", message); err != nil {
return err
}
}
if err := tx.Model(&model.PostgreSQLInstance{}).Where("task_id = ?", taskID).Update("status", "rollback_failed").Error; err != nil {
return err
}
if err := tx.Model(&model.PostgreSQLCluster{}).Where("task_id = ?", taskID).Update("status", "rollback_failed").Error; err != nil {
return err
}
return tx.Model(&model.ExecutionJob{}).Where("task_id = ? AND status IN ?", taskID, []string{"launching", "running"}).Updates(map[string]any{"status": "failed", "finished_at": time.Now()}).Error
})
}
func (s *PostgreSQLDeliveryService) PollRollbackOnce(ctx context.Context) error {
var jobs []model.RollbackJob
if err := s.db.WithContext(ctx).
Joins("JOIN delivery_tasks ON delivery_tasks.id = rollback_jobs.task_id").
Where("rollback_jobs.status IN ? AND delivery_tasks.service_type = ?", []string{"launching", "running"}, postgresqlServiceType).
Find(&jobs).Error; err != nil {
return err
}
for _, rollback := range jobs {
if rollback.Status == "launching" {
if !rollbackLaunchExpired(rollback, time.Now()) {
continue
}
expired, err := s.common.expireRollbackLaunch(ctx, rollback.ID)
if err != nil {
return err
}
if expired {
_ = s.markRollbackFailed(ctx, rollback.TaskID, "PostgreSQL rollback launch timed out before its AWX job ID was recorded; manual retry is required")
}
continue
}
job, err := s.awx.GetJob(ctx, rollback.ExecutorJobID)
if err != nil {
continue
}
switch strings.ToLower(job.Status) {
case "pending", "waiting", "running", "new":
continue
case "successful":
_ = s.common.finishRollbackJob(ctx, &rollback, "successful")
if err := s.completeRollback(ctx, rollback.TaskID); err != nil {
_ = s.markRollbackFailed(ctx, rollback.TaskID, err.Error())
}
case "canceled":
_ = s.common.finishRollbackJob(ctx, &rollback, "canceled")
_ = s.markRollbackFailed(ctx, rollback.TaskID, "PostgreSQL rollback AWX job was canceled")
default:
_ = s.common.finishRollbackJob(ctx, &rollback, "failed")
_ = s.markRollbackFailed(ctx, rollback.TaskID, "PostgreSQL rollback AWX job finished with status "+job.Status)
}
}
return nil
}
func (s *PostgreSQLDeliveryService) completeRollback(ctx context.Context, taskID string) error {
now := time.Now()
return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
var task model.DeliveryTask
if err := tx.First(&task, "id = ? AND service_type = ?", taskID, postgresqlServiceType).Error; err != nil {
return err
}
if task.Status != model.TaskRollingBack {
return fmt.Errorf("task %s is in state %q, cannot complete PostgreSQL rollback", taskID, task.Status)
}
if err := tx.Model(&model.PostgreSQLInstance{}).Where("task_id = ?", taskID).Update("status", "rolled_back").Error; err != nil {
return err
}
if err := tx.Model(&model.PostgreSQLCluster{}).Where("task_id = ?", taskID).Update("status", "rolled_back").Error; err != nil {
return err
}
if err := tx.Model(&model.PostgreSQLResourceUsage{}).Where("task_id = ? AND status = ?", taskID, "active").Updates(map[string]any{"status": "released", "released_at": now}).Error; err != nil {
return err
}
if err := tx.Model(&model.ResourceReservation{}).Where("task_id = ? AND status IN ?", taskID, []string{"reserved", "consumed", "rollback"}).Update("status", "released").Error; err != nil {
return err
}
if err := tx.Model(&model.ExecutionJob{}).Where("task_id = ? AND status IN ?", taskID, []string{"launching", "running"}).Updates(map[string]any{"status": "failed", "finished_at": now}).Error; err != nil {
return err
}
return s.common.transitionTx(tx, &task, model.TaskRolledBack, "PostgreSQL instance rollback completed", "")
})
}