fix(delivery): add PostgreSQL rollback and CloudDM isolation

This commit is contained in:
l x
2026-07-31 11:16:19 +08:00
parent 66fde73792
commit 363ad5f9be
15 changed files with 872 additions and 224 deletions
+86
View File
@@ -0,0 +1,86 @@
---
- name: Roll back native PostgreSQL delivery instances
hosts: "{{ target_hosts }}"
become: true
gather_facts: false
any_errors_fatal: true
vars:
postgresql_instance_plan: "{{ postgresql_instances[inventory_hostname] | default({}) }}"
postgresql_instance_id: "{{ postgresql_instance_plan.instance_id | default('') }}"
postgresql_data_root: /data/postgresql
postgresql_instance_root: "{{ postgresql_data_root }}/{{ postgresql_instance_id }}"
postgresql_data_dir: "{{ postgresql_instance_plan.data_dir | default('') }}"
postgresql_config_dir: "{{ postgresql_instance_plan.config_dir | default('') }}"
postgresql_log_dir: "{{ postgresql_instance_plan.log_dir | default('') }}"
postgresql_systemd_unit: "{{ postgresql_instance_plan.systemd_unit | default('') }}"
postgresql_dropin_dir: "/etc/systemd/system/{{ postgresql_systemd_unit }}.d"
postgresql_run_dir: "/run/postgresql-xinfra/{{ postgresql_instance_id }}"
pre_tasks:
- name: Validate PostgreSQL rollback target
ansible.builtin.assert:
that:
- postgresql_instance_plan is mapping
- postgresql_instance_id is match('^[a-z0-9][a-z0-9-]{0,62}$')
- postgresql_instance_root == '/data/postgresql/' + postgresql_instance_id
- postgresql_data_dir == postgresql_instance_root + '/data'
- postgresql_config_dir == postgresql_instance_root + '/conf'
- postgresql_log_dir == postgresql_instance_root + '/log'
- postgresql_systemd_unit == 'postgresql-xinfra@' + postgresql_instance_id + '.service'
- postgresql_instance_root != '/data/postgresql'
- "'..' not in postgresql_instance_root"
fail_msg: "Rollback target is outside the native PostgreSQL delivery layout"
quiet: true
tasks:
- name: Stop the PostgreSQL delivery instance when present
ansible.builtin.systemd_service:
name: "{{ postgresql_systemd_unit }}"
state: stopped
enabled: false
failed_when: false
- name: Reset the PostgreSQL instance failure state
ansible.builtin.command:
argv:
- systemctl
- reset-failed
- "{{ postgresql_systemd_unit }}"
changed_when: false
failed_when: false
- name: Remove the PostgreSQL instance systemd drop-in
ansible.builtin.file:
path: "{{ postgresql_dropin_dir }}"
state: absent
- name: Remove the PostgreSQL instance data, configuration, and logs
ansible.builtin.file:
path: "{{ postgresql_instance_root }}"
state: absent
- name: Remove the PostgreSQL instance runtime directory
ansible.builtin.file:
path: "{{ postgresql_run_dir }}"
state: absent
- name: Reload systemd after removing the instance drop-in
ansible.builtin.systemd_service:
daemon_reload: true
- name: Confirm PostgreSQL instance artifacts are gone
ansible.builtin.stat:
path: "{{ item }}"
loop:
- "{{ postgresql_dropin_dir }}"
- "{{ postgresql_instance_root }}"
- "{{ postgresql_run_dir }}"
register: postgresql_rollback_artifacts
- name: Assert all PostgreSQL instance artifacts are gone
ansible.builtin.assert:
that:
- not item.stat.exists
fail_msg: "PostgreSQL rollback left an instance artifact on the target host"
quiet: true
loop: "{{ postgresql_rollback_artifacts.results }}"
+9
View File
@@ -17,6 +17,8 @@ The local object mapping is:
| Secret credential | `XINFRA PostgreSQL Runtime Secrets` |
| Job Template | `XINFRA PostgreSQL Delivery` |
| Playbook | `postgresql-deploy.yml` |
| Rollback Template | `XINFRA PostgreSQL Rollback` |
| Rollback Playbook | `postgresql-rollback.yml` |
| Instance Group | `xinfra-e2e-execution` |
Run the idempotent setup whenever the AWX task pod restarts or the local
@@ -25,6 +27,13 @@ cannot use a Manual Project, so the Project is Git-backed with launch updates
disabled. The script refreshes a local Git source inside the AWX task pod and
runs a project update so AWX rebuilds its playbook index.
The setup output includes `DELIVERY_POSTGRESQL_ROLLBACK_TEMPLATE_ID`; copy that
value into `config/local/server-postgresql.env` before starting the scheduler.
PostgreSQL CloudDM registration is optional and requires the dedicated URL,
API token, username, and password documented in the server env example. The
CloudDM password must match the PostgreSQL administrator password configured
in the AWX runtime credential.
```bash
./scripts/setup-postgresql-awx.sh
```
+3
View File
@@ -40,4 +40,7 @@ XINFRA_POSTGRES_REPLICATION_PASSWORD=
AWX_JOB_TEMPLATE_NAME="XINFRA PostgreSQL Delivery"
AWX_JOB_TEMPLATE_DESCRIPTION="PostgreSQL host_pool delivery template for XINFRA"
AWX_PLAYBOOK=postgresql-deploy.yml
AWX_ROLLBACK_JOB_TEMPLATE_NAME="XINFRA PostgreSQL Rollback"
AWX_ROLLBACK_JOB_TEMPLATE_DESCRIPTION="PostgreSQL compensating cleanup template for XINFRA"
AWX_ROLLBACK_PLAYBOOK=postgresql-rollback.yml
AWX_INSTANCE_GROUP=xinfra-e2e-execution
@@ -14,3 +14,11 @@ DELIVERY_GLOBAL_LIMIT=2
DELIVERY_TARGET_LIMIT=2
DELIVERY_BUSINESS_LIMIT=1
DELIVERY_DATA_DISKS=/data
DELIVERY_POSTGRESQL_ROLLBACK_TEMPLATE_ID=0
# Optional PostgreSQL CloudDM registration. Keep the URL empty to disable it.
# The password must match the postgres administrator password delivered by AWX.
CLOUDDM_POSTGRESQL_REGISTER_URL=
CLOUDDM_API_TOKEN=
CLOUDDM_POSTGRESQL_USERNAME=postgres
CLOUDDM_POSTGRESQL_PASSWORD=
+3 -3
View File
@@ -732,7 +732,7 @@ const basicServices = ref<Service[]>([
charsets: ['UTF8'],
defaultPort: 0,
defaultPaths: { install: '/usr/lib/postgresql', data: '/data/postgresql', log: '/data/postgresql' },
registerTo: 'PostgreSQL 专用资源台账',
registerTo: 'PostgreSQL 资源台账 / CloudDM(可选)',
configId: 'PG-NATIVE-V1',
assetId: '',
healthText: 'PostgreSQL 就绪 · TCP 探测通过',
@@ -835,7 +835,7 @@ const rollbackReleasing = ref(false)
const cloudDMRetrying = ref(false)
const rollbackActionBusy = computed(() => rollbackRetrying.value || rollbackReleasing.value)
const canManageRollback = computed(() => authStore.isAdmin && lastDeliveryStatus.value === 'rollback_failed' && Boolean(deploymentId.value))
const canRetryCloudDM = computed(() => !isPostgreSQL.value && deliveryRegisterFailed.value && Boolean(deploymentId.value))
const canRetryCloudDM = computed(() => deliveryRegisterFailed.value && Boolean(deploymentId.value))
const deliveryForm = reactive({
instanceName: generateInstanceName(activeServiceKey.value, currentName.value),
@@ -1318,7 +1318,7 @@ function defaultSteps(): DeliveryStep[] {
['应用实例配置', 'postgresql.conf、pg_hba.conf、systemd'],
['配置复制拓扑', '复制账号、slot、primary_conninfo'],
['数据库健康检查', '端口、角色、版本和数据目录'],
['资源入账与交付归档', 'PostgreSQL 集群、实例与资源台账'],
['资源入账与交付归档', 'PostgreSQL 资源台账与可选 CloudDM 注册'],
].map(([name, desc]) => ({ name, desc, state: 'pending' as StepState }))
}
const isMysql = activeServiceKey.value === 'mysql'
+52 -16
View File
@@ -37,7 +37,8 @@ for variable_name in \
AWX_HOST_NAME AWX_HOST_ADDRESS \
AWX_MACHINE_CREDENTIAL_NAME AWX_SSH_PRIVATE_KEY_FILE \
AWX_POSTGRES_CREDENTIAL_TYPE_NAME AWX_POSTGRES_CREDENTIAL_NAME \
AWX_JOB_TEMPLATE_NAME AWX_PLAYBOOK; do
AWX_JOB_TEMPLATE_NAME AWX_PLAYBOOK \
AWX_ROLLBACK_JOB_TEMPLATE_NAME AWX_ROLLBACK_PLAYBOOK; do
require_value "${variable_name}"
done
@@ -64,6 +65,7 @@ case ${PROJECT_SOURCE}/ in
*) die "AWX_PROJECT_SOURCE_DIR must resolve inside the repository" ;;
esac
[[ -f ${PROJECT_SOURCE}/${AWX_PLAYBOOK} ]] || die "playbook not found: ${PROJECT_SOURCE}/${AWX_PLAYBOOK}"
[[ -f ${PROJECT_SOURCE}/${AWX_ROLLBACK_PLAYBOOK} ]] || die "rollback playbook not found: ${PROJECT_SOURCE}/${AWX_ROLLBACK_PLAYBOOK}"
if [[ -z ${AWX_TOKEN:-} && -z ${AWX_PASSWORD:-} ]]; then
AWX_PASSWORD=$(kubectl get secret \
@@ -186,6 +188,10 @@ if ! awx_request GET "/api/v2/projects/${project_id}/playbooks/" | \
jq -e --arg playbook "${AWX_PLAYBOOK}" 'index($playbook) != null' >/dev/null; then
die "AWX project ${project_id} does not expose playbook ${AWX_PLAYBOOK}"
fi
if ! awx_request GET "/api/v2/projects/${project_id}/playbooks/" | \
jq -e --arg playbook "${AWX_ROLLBACK_PLAYBOOK}" 'index($playbook) != null' >/dev/null; then
die "AWX project ${project_id} does not expose rollback playbook ${AWX_ROLLBACK_PLAYBOOK}"
fi
inventory_payload=$(jq -nc \
--arg name "${AWX_INVENTORY_NAME}" \
@@ -307,29 +313,57 @@ job_template_payload=$(jq -nc \
job_template_id=$(upsert_named_object '/api/v2/job_templates/' \
"${AWX_JOB_TEMPLATE_NAME}" "${job_template_payload}")
while IFS= read -r existing_group_id; do
[[ -z ${existing_group_id} || ${existing_group_id} == "${instance_group_id}" ]] && continue
awx_request POST "/api/v2/job_templates/${job_template_id}/instance_groups/" \
"$(jq -nc --argjson id "${existing_group_id}" '{id:$id,disassociate:true}')" >/dev/null
done < <(awx_request GET "/api/v2/job_templates/${job_template_id}/instance_groups/" | jq -r '.results[].id')
rollback_template_payload=$(jq -nc \
--arg name "${AWX_ROLLBACK_JOB_TEMPLATE_NAME}" \
--arg description "${AWX_ROLLBACK_JOB_TEMPLATE_DESCRIPTION:-PostgreSQL rollback template}" \
--arg playbook "${AWX_ROLLBACK_PLAYBOOK}" \
--argjson organization "${organization_id}" \
--argjson inventory "${inventory_id}" \
--argjson project "${project_id}" \
--argjson prevent_fallback "${prevent_fallback}" \
'{
name:$name,
description:$description,
organization:$organization,
inventory:$inventory,
project:$project,
playbook:$playbook,
job_type:"run",
ask_inventory_on_launch:true,
ask_variables_on_launch:true,
ask_limit_on_launch:true,
allow_simultaneous:true,
prevent_instance_group_fallback:$prevent_fallback
}')
rollback_template_id=$(upsert_named_object '/api/v2/job_templates/' \
"${AWX_ROLLBACK_JOB_TEMPLATE_NAME}" "${rollback_template_payload}")
if ! awx_request GET "/api/v2/job_templates/${job_template_id}/credentials/" | \
for configured_template_id in "${job_template_id}" "${rollback_template_id}"; do
while IFS= read -r existing_group_id; do
[[ -z ${existing_group_id} || ${existing_group_id} == "${instance_group_id}" ]] && continue
awx_request POST "/api/v2/job_templates/${configured_template_id}/instance_groups/" \
"$(jq -nc --argjson id "${existing_group_id}" '{id:$id,disassociate:true}')" >/dev/null
done < <(awx_request GET "/api/v2/job_templates/${configured_template_id}/instance_groups/" | jq -r '.results[].id')
if ! awx_request GET "/api/v2/job_templates/${configured_template_id}/credentials/" | \
jq -e --argjson id "${machine_credential_id}" '.results | any(.id == $id)' >/dev/null; then
awx_request POST "/api/v2/job_templates/${job_template_id}/credentials/" \
awx_request POST "/api/v2/job_templates/${configured_template_id}/credentials/" \
"$(jq -nc --argjson id "${machine_credential_id}" '{id:$id}')" >/dev/null
fi
fi
if [[ -n ${instance_group_id} ]]; then
if ! awx_request GET "/api/v2/job_templates/${configured_template_id}/instance_groups/" | \
jq -e --argjson id "${instance_group_id}" '.results | any(.id == $id)' >/dev/null; then
awx_request POST "/api/v2/job_templates/${configured_template_id}/instance_groups/" \
"$(jq -nc --argjson id "${instance_group_id}" '{id:$id}')" >/dev/null
fi
fi
done
if ! awx_request GET "/api/v2/job_templates/${job_template_id}/credentials/" | \
jq -e --argjson id "${postgres_credential_id}" '.results | any(.id == $id)' >/dev/null; then
awx_request POST "/api/v2/job_templates/${job_template_id}/credentials/" \
"$(jq -nc --argjson id "${postgres_credential_id}" '{id:$id}')" >/dev/null
fi
if [[ -n ${instance_group_id} ]]; then
if ! awx_request GET "/api/v2/job_templates/${job_template_id}/instance_groups/" | \
jq -e --argjson id "${instance_group_id}" '.results | any(.id == $id)' >/dev/null; then
awx_request POST "/api/v2/job_templates/${job_template_id}/instance_groups/" \
"$(jq -nc --argjson id "${instance_group_id}" '{id:$id}')" >/dev/null
fi
fi
printf '\nPostgreSQL AWX configuration is ready:\n'
printf ' Project: %s (id=%s)\n' "${AWX_PROJECT_NAME}" "${project_id}"
@@ -338,6 +372,8 @@ printf ' Host: %s -> %s (id=%s)\n' "${AWX_HOST_NAME}" "${AWX_HOST_ADD
printf ' SSH credential: %s (id=%s)\n' "${AWX_MACHINE_CREDENTIAL_NAME}" "${machine_credential_id}"
printf ' PG credential: %s (id=%s)\n' "${AWX_POSTGRES_CREDENTIAL_NAME}" "${postgres_credential_id}"
printf ' Job Template: %s (id=%s)\n' "${AWX_JOB_TEMPLATE_NAME}" "${job_template_id}"
printf ' Rollback: %s (id=%s)\n' "${AWX_ROLLBACK_JOB_TEMPLATE_NAME}" "${rollback_template_id}"
printf ' Server env: DELIVERY_POSTGRESQL_ROLLBACK_TEMPLATE_ID=%s\n' "${rollback_template_id}"
if [[ -n ${instance_group_id} ]]; then
printf ' Instance Group: %s (id=%s)\n' "${AWX_INSTANCE_GROUP}" "${instance_group_id}"
fi
+9 -2
View File
@@ -68,16 +68,23 @@ DELIVERY_MYSQL_INSPECT_TEMPLATE_NAME=XINFRA MySQL Inspect
DELIVERY_MYSQL_INSPECT_TIMEOUT_SECONDS=90
# AWX Job Template ID for ansible/mysql-rollback.yml; required for automatic cleanup
DELIVERY_ROLLBACK_TEMPLATE_ID=0
# AWX Job Template ID for ansible/postgresql-rollback.yml
DELIVERY_POSTGRESQL_ROLLBACK_TEMPLATE_ID=0
DELIVERY_SERVICE_TOKEN=
# PostgreSQL runtime secrets are injected by an AWX Credential and are never
# persisted in xinfra task payloads:
# PostgreSQL deployment secrets are injected by an AWX Credential and are never
# persisted in xinfra task payloads. CloudDM registration, when enabled, uses
# the separate CLOUDDM_POSTGRESQL_* credential below.
# XINFRA_POSTGRES_ADMIN_PASSWORD
# XINFRA_POSTGRES_REPLICATION_PASSWORD
# XINFRA_POSTGRES_REPLICATION_USER (optional, defaults to xinfra_replication)
CLOUDDM_REGISTER_URL=
CLOUDDM_DELETE_URL=
CLOUDDM_API_TOKEN=
# PostgreSQL uses an explicit contract and separate endpoint/credential.
CLOUDDM_POSTGRESQL_REGISTER_URL=
CLOUDDM_POSTGRESQL_USERNAME=postgres
CLOUDDM_POSTGRESQL_PASSWORD=
JWT_SECRET=change-this-secret
JWT_ISSUER=authserver
+8
View File
@@ -64,6 +64,9 @@ type Config struct {
CloudDMRegisterURL string
CloudDMDeleteURL string
CloudDMAPIToken string
CloudDMPostgreSQLRegisterURL string
CloudDMPostgreSQLUsername string
CloudDMPostgreSQLPassword string
AWXBaseURL string
AWXToken string
AWXUsername string
@@ -75,6 +78,7 @@ type Config struct {
MySQLInspectTemplateName string
MySQLInspectTimeoutSeconds int
RollbackTemplateID uint64
PostgreSQLRollbackTemplateID uint64
DeliveryServiceToken string
DeliverySchedulerEnabled bool
DeliveryDispatchSeconds int
@@ -152,6 +156,9 @@ func Load() Config {
CloudDMRegisterURL: trimURL(env("CLOUDDM_REGISTER_URL", "")),
CloudDMDeleteURL: trimURL(env("CLOUDDM_DELETE_URL", "")),
CloudDMAPIToken: env("CLOUDDM_API_TOKEN", ""),
CloudDMPostgreSQLRegisterURL: trimURL(env("CLOUDDM_POSTGRESQL_REGISTER_URL", "")),
CloudDMPostgreSQLUsername: env("CLOUDDM_POSTGRESQL_USERNAME", "postgres"),
CloudDMPostgreSQLPassword: env("CLOUDDM_POSTGRESQL_PASSWORD", ""),
AWXBaseURL: trimURL(env("AWX_BASE_URL", "")),
AWXToken: env("AWX_TOKEN", ""),
AWXUsername: env("AWX_USERNAME", ""),
@@ -163,6 +170,7 @@ func Load() Config {
MySQLInspectTemplateName: env("DELIVERY_MYSQL_INSPECT_TEMPLATE_NAME", "XINFRA MySQL Inspect"),
MySQLInspectTimeoutSeconds: envInt("DELIVERY_MYSQL_INSPECT_TIMEOUT_SECONDS", 90),
RollbackTemplateID: uint64(envInt("DELIVERY_ROLLBACK_TEMPLATE_ID", 0)),
PostgreSQLRollbackTemplateID: uint64(envInt("DELIVERY_POSTGRESQL_ROLLBACK_TEMPLATE_ID", 0)),
DeliveryServiceToken: env("DELIVERY_SERVICE_TOKEN", ""),
DeliverySchedulerEnabled: envBool("DELIVERY_SCHEDULER_ENABLED", false),
DeliveryDispatchSeconds: envInt("DELIVERY_DISPATCH_SECONDS", 5),
+27 -6
View File
@@ -17,14 +17,15 @@ import (
type DeliveryHandler struct {
service *service.DeliveryService
postgresqlService *service.PostgreSQLDeliveryService
}
type PostgreSQLDeliveryHandler struct {
service *service.PostgreSQLDeliveryService
}
func NewDeliveryHandler(s *service.DeliveryService) *DeliveryHandler {
return &DeliveryHandler{service: s}
func NewDeliveryHandler(s *service.DeliveryService, postgresqlService *service.PostgreSQLDeliveryService) *DeliveryHandler {
return &DeliveryHandler{service: s, postgresqlService: postgresqlService}
}
func NewPostgreSQLDeliveryHandler(s *service.PostgreSQLDeliveryService) *PostgreSQLDeliveryHandler {
@@ -345,7 +346,14 @@ func (h *DeliveryHandler) RetryRollback(c *gin.Context) {
if !requirePlatformAdmin(c) {
return
}
if err := h.service.RetryRollback(c.Request.Context(), c.Param("id")); err != nil {
taskID := c.Param("id")
task, _, err := h.service.GetTask(c.Request.Context(), taskID, 0, true)
if err == nil && task.ServiceType == "postgresql" && h.postgresqlService != nil {
err = h.postgresqlService.RetryRollback(c.Request.Context(), taskID)
} else if err == nil {
err = h.service.RetryRollback(c.Request.Context(), taskID)
}
if err != nil {
c.JSON(http.StatusConflict, gin.H{"error": err.Error()})
return
}
@@ -358,7 +366,14 @@ func (h *DeliveryHandler) AcknowledgeRollbackRelease(c *gin.Context) {
if !requirePlatformAdmin(c) {
return
}
if err := h.service.AcknowledgeRollbackRelease(c.Request.Context(), c.Param("id")); err != nil {
taskID := c.Param("id")
task, _, err := h.service.GetTask(c.Request.Context(), taskID, 0, true)
if err == nil && task.ServiceType == "postgresql" && h.postgresqlService != nil {
err = h.postgresqlService.AcknowledgeRollbackRelease(c.Request.Context(), taskID)
} else if err == nil {
err = h.service.AcknowledgeRollbackRelease(c.Request.Context(), taskID)
}
if err != nil {
c.JSON(http.StatusConflict, gin.H{"error": err.Error()})
return
}
@@ -374,7 +389,8 @@ func (h *DeliveryHandler) RetryCloudDMRegistration(c *gin.Context) {
return
}
taskID := c.Param("id")
if _, _, err := h.service.GetTask(c.Request.Context(), taskID, claims.UserID, claims.IsAdmin); err != nil {
task, _, err := h.service.GetTask(c.Request.Context(), taskID, claims.UserID, claims.IsAdmin)
if err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
c.JSON(http.StatusNotFound, gin.H{"error": "task not found"})
return
@@ -382,7 +398,12 @@ func (h *DeliveryHandler) RetryCloudDMRegistration(c *gin.Context) {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
if err := h.service.RetryCloudDMRegistration(c.Request.Context(), taskID); err != nil {
if task.ServiceType == "postgresql" && h.postgresqlService != nil {
err = h.postgresqlService.RetryCloudDMRegistration(c.Request.Context(), taskID)
} else {
err = h.service.RetryCloudDMRegistration(c.Request.Context(), taskID)
}
if err != nil {
c.JSON(http.StatusConflict, gin.H{"error": err.Error()})
return
}
+1
View File
@@ -167,6 +167,7 @@ type PostgreSQLInstance struct {
UpstreamInstanceID string `gorm:"size:63;not null;default:''" json:"upstream_instance_id"`
ReplicationSlotName string `gorm:"size:63;not null;default:''" json:"replication_slot_name"`
Status string `gorm:"size:32;not null;index" json:"status"`
Metadata string `gorm:"type:json" json:"metadata"`
BackupStatus string `gorm:"size:32;not null;default:'not_configured'" json:"backup_status"`
MonitoringStatus string `gorm:"size:32;not null;default:'not_configured'" json:"monitoring_status"`
CreatedAt time.Time `json:"created_at"`
+1 -1
View File
@@ -90,7 +90,7 @@ func registerAuthServerRoutes(r *gin.Engine, deps Dependencies) {
clouddmHandler := handler.NewCloudDMHandler(deps.Config, auditService)
samlHandler := handler.NewSAMLHandler(deps.Config, authService)
oauthHandler := handler.NewOAuthHandler(deps.Config, deps.DB, auditService)
deliveryHandler := handler.NewDeliveryHandler(deliveryService)
deliveryHandler := handler.NewDeliveryHandler(deliveryService, postgresqlDeliveryService)
deliveryCallbackHandler := handler.NewDeliveryCallbackHandler(deliveryService, deps.Config.AWXWebhookToken)
postgresqlDeliveryHandler := handler.NewPostgreSQLDeliveryHandler(postgresqlDeliveryService)
containerServiceHandler := handler.NewContainerServiceHandler(deps.DB, wayneRoleBindingService)
+7 -1
View File
@@ -405,6 +405,9 @@ func (s *DeliveryService) ListTargets(ctx context.Context, component string) ([]
if !strings.Contains(text, component) {
continue
}
if component == postgresqlServiceType && strings.Contains(text, "rollback") {
continue
}
}
target, err := s.awxDeliveryTarget(ctx, template)
if err != nil {
@@ -2860,7 +2863,10 @@ func (s *DeliveryService) markRollbackFailed(ctx context.Context, taskID, messag
func (s *DeliveryService) PollRollbackOnce(ctx context.Context) error {
var jobs []model.RollbackJob
if err := s.db.WithContext(ctx).Where("status IN ?", []string{"launching", "running"}).Find(&jobs).Error; err != nil {
if err := s.db.WithContext(ctx).
Joins("JOIN delivery_tasks ON delivery_tasks.id = rollback_jobs.task_id").
Where("rollback_jobs.status IN ? AND (delivery_tasks.service_type = '' OR delivery_tasks.service_type = ?)", []string{"launching", "running"}, "mysql").
Find(&jobs).Error; err != nil {
return err
}
for _, rollback := range jobs {
+35 -40
View File
@@ -28,6 +28,7 @@ const (
var supportedPostgreSQLVersions = map[string]bool{"15": true, "16": true}
var supportedPostgreSQLTopologies = map[string]bool{"standalone": true, "primary_replica": true}
var postgresqlNamePattern = regexp.MustCompile(`^[a-z0-9](?:[-a-z0-9]*[a-z0-9])?$`)
var errPostgreSQLTaskHandled = errors.New("postgresql task validation completed without dispatch")
type PostgreSQLDeliveryInput struct {
BusinessLineID uint64 `json:"business_line_id" binding:"required"`
@@ -230,7 +231,8 @@ func getPostgreSQLTarget(ctx context.Context, awx *AWXClient, templateID uint64)
if err != nil {
return DeliveryTarget{}, fmt.Errorf("deployment target is unavailable: %w", err)
}
if !strings.Contains(strings.ToLower(template.Name+" "+template.Description), "postgresql") {
templateText := strings.ToLower(template.Name + " " + template.Description)
if !strings.Contains(templateText, "postgresql") || strings.Contains(templateText, "rollback") {
return DeliveryTarget{}, fmt.Errorf("AWX job template %d is not a PostgreSQL target", templateID)
}
hosts, err := awx.ListInventoryHosts(ctx, template.Inventory)
@@ -342,7 +344,12 @@ func (s *PostgreSQLDeliveryService) claimAndReserve(ctx context.Context) (*model
}
port, err := allocateReachablePostgreSQLPort(ctx, host.IP, usedPorts, postgresqlPortInUse)
if err != nil {
_ = tx.Delete(&cluster).Error
if cleanupErr := tx.Where("task_id = ?", task.ID).Delete(&model.PostgreSQLInstance{}).Error; cleanupErr != nil {
return cleanupErr
}
if cleanupErr := tx.Delete(&cluster).Error; cleanupErr != nil {
return cleanupErr
}
return s.common.transitionTx(tx, &task, model.TaskValidationFailed, err.Error(), err.Error())
}
role := "replica"
@@ -390,6 +397,9 @@ func (s *PostgreSQLDeliveryService) claimAndReserve(ctx context.Context) (*model
}
return s.common.transitionTx(tx, &task, model.TaskDispatching, "PostgreSQL resources, ports and directories reserved", "")
})
if err == nil && task.Status != model.TaskDispatching {
return nil, errPostgreSQLTaskHandled
}
return &task, err
}
@@ -463,13 +473,13 @@ func (s *PostgreSQLDeliveryService) CreateExecution(ctx context.Context, task *m
func (s *PostgreSQLDeliveryService) DispatchOnce(ctx context.Context) error {
task, err := s.claimAndReserve(ctx)
if err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) || strings.HasPrefix(err.Error(), "defer:") {
if errors.Is(err, gorm.ErrRecordNotFound) || errors.Is(err, errPostgreSQLTaskHandled) || strings.HasPrefix(err.Error(), "defer:") {
return nil
}
return err
}
if err := s.CreateExecution(ctx, task); err != nil {
return s.fail(ctx, task.ID, model.TaskExecutionFailed, err.Error())
return s.beginRollback(ctx, task.ID, "PostgreSQL deployment could not be started: "+err.Error())
}
return nil
}
@@ -490,13 +500,13 @@ func (s *PostgreSQLDeliveryService) PollOnce(ctx context.Context) error {
case "pending", "waiting", "running", "new":
continue
case "canceled":
_ = s.fail(ctx, execution.TaskID, model.TaskCanceled, "AWX job was canceled")
_ = s.beginRollback(ctx, execution.TaskID, "PostgreSQL AWX job was canceled")
case "successful":
if err := s.complete(ctx, execution.TaskID); err != nil {
_ = s.fail(ctx, execution.TaskID, model.TaskValidationFailed, err.Error())
_ = s.beginRollback(ctx, execution.TaskID, err.Error())
}
default:
_ = s.fail(ctx, execution.TaskID, model.TaskExecutionFailed, "AWX job finished with status "+job.Status)
_ = s.beginRollback(ctx, execution.TaskID, "PostgreSQL AWX job finished with status "+job.Status)
}
}
return nil
@@ -524,7 +534,7 @@ func (s *PostgreSQLDeliveryService) complete(ctx context.Context, taskID string)
}
}
now := time.Now()
return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
if err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
if err := s.common.transitionTx(tx, &task, model.TaskRegistering, "PostgreSQL health checks passed", ""); err != nil {
return err
}
@@ -545,8 +555,22 @@ func (s *PostgreSQLDeliveryService) complete(ctx context.Context, taskID string)
if err := tx.Model(&model.ExecutionJob{}).Where("task_id = ?", task.ID).Updates(map[string]any{"status": "successful", "finished_at": now}).Error; err != nil {
return err
}
return s.common.transitionTx(tx, &task, model.TaskFinished, "PostgreSQL delivery completed and recorded in the PostgreSQL resource ledger", "")
})
return nil
}); err != nil {
return err
}
if err := s.RegisterCloudDM(ctx, task.ID); err != nil {
message := fmt.Sprintf("PostgreSQL delivered; CloudDM registration failed: %v", err)
if transitionErr := s.common.transition(ctx, &task, model.TaskRegisterFailed, "PostgreSQL delivered; CloudDM registration failed and can be retried", message); transitionErr != nil {
return fmt.Errorf("%s; cannot record register_failed: %w", message, transitionErr)
}
return nil
}
message := "PostgreSQL delivery completed; CloudDM registration was skipped because no PostgreSQL endpoint is configured"
if strings.TrimSpace(s.cfg.CloudDMPostgreSQLRegisterURL) != "" {
message = "PostgreSQL delivery completed, registered and recorded in the resource ledger"
}
return s.common.transition(ctx, &task, model.TaskFinished, message, "")
}
func postgresReady(ctx context.Context, address string) error {
@@ -558,36 +582,6 @@ func postgresReady(ctx context.Context, address string) error {
return conn.Close()
}
func (s *PostgreSQLDeliveryService) fail(ctx context.Context, taskID, status, message string) error {
return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
var task model.DeliveryTask
if err := tx.First(&task, "id = ? AND service_type = ?", taskID, postgresqlServiceType).Error; err != nil {
return err
}
if err := s.common.transitionTx(tx, &task, status, message, message); err != nil {
return err
}
reservationStatus := "released"
if status == model.TaskExecutionFailed || (status == model.TaskValidationFailed && strings.Contains(strings.ToLower(message), "health")) {
reservationStatus = "quarantined"
}
if err := tx.Model(&model.ResourceReservation{}).Where("task_id = ? AND status = ?", taskID, "reserved").Update("status", reservationStatus).Error; err != nil {
return err
}
instanceStatus := "failed"
if reservationStatus == "quarantined" {
instanceStatus = "quarantined"
}
if err := tx.Model(&model.PostgreSQLInstance{}).Where("task_id = ? AND status = ?", taskID, "provisioning").Update("status", instanceStatus).Error; err != nil {
return err
}
if err := tx.Model(&model.PostgreSQLCluster{}).Where("task_id = ? AND status = ?", taskID, "provisioning").Update("status", instanceStatus).Error; err != nil {
return err
}
return tx.Model(&model.ExecutionJob{}).Where("task_id = ? AND status IN ?", taskID, []string{"launching", "running"}).Updates(map[string]any{"status": "failed", "finished_at": time.Now()}).Error
})
}
func (s *PostgreSQLDeliveryService) Run(ctx context.Context) {
interval := time.Duration(s.cfg.DeliveryPollSeconds) * time.Second
if interval < time.Second {
@@ -602,6 +596,7 @@ func (s *PostgreSQLDeliveryService) Run(ctx context.Context) {
case <-ticker.C:
_ = s.DispatchOnce(ctx)
_ = s.PollOnce(ctx)
_ = s.PollRollbackOnce(ctx)
}
}
}
@@ -2,7 +2,10 @@ package service
import (
"context"
"encoding/json"
"testing"
"github.com/1024XEngineer/xinfra/server/internal/model"
)
func TestValidatePostgreSQLDeliveryInput(t *testing.T) {
@@ -102,3 +105,42 @@ func TestSelectPostgreSQLHosts(t *testing.T) {
t.Fatal("host outside the pool was accepted")
}
}
func TestBuildPostgreSQLCloudDMRegisterRequest(t *testing.T) {
req := buildPostgreSQLCloudDMRegisterRequest(
model.PostgreSQLInstance{ID: 42, InstanceID: "orders-pg-primary", HostIP: "10.0.0.10", Port: 15432, VersionMajor: "16", Role: "primary"},
model.PostgreSQLCluster{ID: 7, Name: "orders-pg"},
"postgres",
"secret",
)
if req.SourceSystem != "xinfra" || req.ResourceType != "POSTGRESQL_INSTANCE" {
t.Fatalf("unexpected request metadata: %#v", req)
}
if req.ExternalResourceID != "postgresql-instance:42" {
t.Fatalf("unexpected externalResourceId: %q", req.ExternalResourceID)
}
if req.DataSource.DSType != "PostgreSQL" || req.DataSource.Host != "10.0.0.10:15432" || req.DataSource.UserName != "postgres" || req.DataSource.Password != "secret" {
t.Fatalf("unexpected PostgreSQL data source: %#v", req.DataSource)
}
if req.DataSource.PostgreSQLVersion != "16" || req.DataSource.DefaultSchema != "postgres" {
t.Fatalf("unexpected PostgreSQL version/schema: %#v", req.DataSource)
}
raw, err := json.Marshal(req)
if err != nil {
t.Fatalf("marshal PostgreSQL CloudDM request: %v", err)
}
var decoded map[string]any
if err := json.Unmarshal(raw, &decoded); err != nil {
t.Fatalf("decode PostgreSQL CloudDM request: %v", err)
}
dataSource := decoded["dataSource"].(map[string]any)
if dataSource["postgresql_version"] != "16" {
t.Fatalf("postgresql_version missing from contract: %#v", dataSource)
}
}
func TestPostgreSQLCloudDMExternalResourceID(t *testing.T) {
if got := postgresqlCloudDMExternalResourceID(99); got != "postgresql-instance:99" {
t.Fatalf("external resource id = %q", got)
}
}
@@ -0,0 +1,426 @@
package service
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net"
"net/http"
"strconv"
"strings"
"time"
"github.com/1024XEngineer/xinfra/server/internal/model"
"gorm.io/gorm"
)
type postgresqlCloudDMRegisterRequest struct {
SourceSystem string `json:"sourceSystem"`
ResourceType string `json:"resourceType"`
ExternalResourceID string `json:"externalResourceId"`
DataSource postgresqlCloudDMDataSource `json:"dataSource"`
}
type postgresqlCloudDMDataSource struct {
InstanceName string `json:"instanceName"`
InstanceDesc string `json:"instanceDesc"`
DSType string `json:"dsType"`
Host string `json:"host"`
ClusterID *uint64 `json:"cluster_id"`
PostgreSQLVersion string `json:"postgresql_version"`
SecurityType string `json:"securityType"`
UserName string `json:"userName"`
Password string `json:"password"`
DefaultSchema string `json:"defaultSchema"`
ClientTimeZone string `json:"clientTimeZone"`
ConnectTimeoutMs int `json:"connectTimeoutMs"`
SocketTimeoutSecs int `json:"socketTimeoutSeconds"`
ConnectionCharset string `json:"connectionCharset"`
}
func postgresqlCloudDMExternalResourceID(instanceID uint64) string {
return "postgresql-instance:" + strconv.FormatUint(instanceID, 10)
}
func buildPostgreSQLCloudDMRegisterRequest(instance model.PostgreSQLInstance, cluster model.PostgreSQLCluster, username, password string) postgresqlCloudDMRegisterRequest {
description := fmt.Sprintf("%s PostgreSQL %s", cluster.Name, instance.Role)
return postgresqlCloudDMRegisterRequest{
SourceSystem: "xinfra",
ResourceType: "POSTGRESQL_INSTANCE",
ExternalResourceID: postgresqlCloudDMExternalResourceID(instance.ID),
DataSource: postgresqlCloudDMDataSource{
InstanceName: instance.InstanceID,
InstanceDesc: description,
DSType: "PostgreSQL",
Host: net.JoinHostPort(instance.HostIP, strconv.Itoa(instance.Port)),
ClusterID: nil,
PostgreSQLVersion: instance.VersionMajor,
SecurityType: "USER_PASSWD",
UserName: username,
Password: password,
DefaultSchema: "postgres",
ClientTimeZone: "Asia/Shanghai",
ConnectTimeoutMs: 5000,
SocketTimeoutSecs: 10,
ConnectionCharset: "utf8",
},
}
}
func postgresqlCloudDMMetadata(existing map[string]any, instance model.PostgreSQLInstance, dataSourceID uint64) map[string]any {
meta := map[string]any{}
for key, value := range existing {
meta[key] = value
}
meta["clouddm"] = map[string]any{
"data_source_id": dataSourceID,
"external_resource_id": postgresqlCloudDMExternalResourceID(instance.ID),
"registered_at": time.Now().Format(time.RFC3339),
}
return meta
}
func (s *PostgreSQLDeliveryService) RegisterCloudDM(ctx context.Context, taskID string) error {
registerURL := strings.TrimSpace(s.cfg.CloudDMPostgreSQLRegisterURL)
if registerURL == "" {
return nil
}
if strings.TrimSpace(s.cfg.CloudDMAPIToken) == "" {
return fmt.Errorf("PostgreSQL CloudDM registration is enabled but CLOUDDM_API_TOKEN is not configured")
}
username := strings.TrimSpace(s.cfg.CloudDMPostgreSQLUsername)
password := s.cfg.CloudDMPostgreSQLPassword
if username == "" || password == "" {
return fmt.Errorf("PostgreSQL CloudDM registration requires CLOUDDM_POSTGRESQL_USERNAME and CLOUDDM_POSTGRESQL_PASSWORD")
}
var cluster model.PostgreSQLCluster
if err := s.db.WithContext(ctx).First(&cluster, "task_id = ?", taskID).Error; err != nil {
return err
}
var instances []model.PostgreSQLInstance
if err := s.db.WithContext(ctx).Where("task_id = ?", taskID).Order("id ASC").Find(&instances).Error; err != nil {
return err
}
if len(instances) == 0 {
return fmt.Errorf("PostgreSQL task has no instances to register")
}
client := &http.Client{Timeout: 30 * time.Second}
for _, instance := range instances {
body := buildPostgreSQLCloudDMRegisterRequest(instance, cluster, username, password)
raw, err := json.Marshal(body)
if err != nil {
return fmt.Errorf("encode PostgreSQL CloudDM registration request: %w", err)
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, registerURL, bytes.NewReader(raw))
if err != nil {
return err
}
req.Header.Set("Authorization", "Bearer "+strings.TrimSpace(s.cfg.CloudDMAPIToken))
req.Header.Set("Content-Type", "application/json")
req.Header.Set("X-Request-ID", randomUUID())
resp, err := client.Do(req)
if err != nil {
return err
}
respBody, readErr := io.ReadAll(resp.Body)
closeErr := resp.Body.Close()
if readErr != nil {
return readErr
}
if closeErr != nil {
return closeErr
}
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return fmt.Errorf("PostgreSQL CloudDM registration returned %s: %s", resp.Status, strings.TrimSpace(string(respBody)))
}
dataSourceID, ok := cloudDMDataSourceIDFromResponse(respBody)
if !ok {
return fmt.Errorf("PostgreSQL CloudDM registration response did not include dataSourceId: %s", strings.TrimSpace(string(respBody)))
}
meta := postgresqlCloudDMMetadata(metadataMap(instance.Metadata), instance, dataSourceID)
if err := s.db.WithContext(ctx).Model(&instance).Update("metadata", string(mustJSON(meta))).Error; err != nil {
return fmt.Errorf("save PostgreSQL CloudDM dataSourceId: %w", err)
}
}
return nil
}
func (s *PostgreSQLDeliveryService) RetryCloudDMRegistration(ctx context.Context, taskID string) error {
var task model.DeliveryTask
if err := s.db.WithContext(ctx).First(&task, "id = ? AND service_type = ?", taskID, postgresqlServiceType).Error; err != nil {
return err
}
if task.Status != model.TaskRegisterFailed {
return fmt.Errorf("task %s is in state %q and cannot retry CloudDM registration", taskID, task.Status)
}
if err := s.common.transition(ctx, &task, model.TaskRegistering, "PostgreSQL CloudDM registration retry started", ""); err != nil {
return err
}
if err := s.RegisterCloudDM(ctx, taskID); err != nil {
message := fmt.Sprintf("PostgreSQL CloudDM registration failed: %v", err)
if transitionErr := s.common.transition(ctx, &task, model.TaskRegisterFailed, "PostgreSQL CloudDM registration retry failed; instances remain active", message); transitionErr != nil {
return fmt.Errorf("%s; cannot restore register_failed: %w", message, transitionErr)
}
return err
}
return s.common.transition(ctx, &task, model.TaskFinished, "PostgreSQL CloudDM registration completed", "")
}
func (s *PostgreSQLDeliveryService) beginRollback(ctx context.Context, taskID, reason string) error {
var task model.DeliveryTask
if err := s.db.WithContext(ctx).First(&task, "id = ? AND service_type = ?", taskID, postgresqlServiceType).Error; err != nil {
return err
}
if rollbackProtectedStatus(task.Status) {
return nil
}
if err := s.common.holdReservationForRollback(ctx, task.ID); err != nil {
return s.markRollbackFailed(ctx, task.ID, "cannot hold PostgreSQL resources during rollback: "+err.Error())
}
if s.cfg.PostgreSQLRollbackTemplateID == 0 {
return s.markRollbackFailed(ctx, task.ID, "PostgreSQL automatic rollback is not configured; manual cleanup is required: "+reason)
}
if err := s.common.transition(ctx, &task, model.TaskRollbackPending, "PostgreSQL automatic rollback queued", reason); err != nil {
return err
}
if err := s.setRollbackResourceStatus(ctx, task.ID, "rollback_pending"); err != nil {
return s.markRollbackFailed(ctx, task.ID, "cannot mark PostgreSQL resources for rollback: "+err.Error())
}
return s.launchRollbackJob(ctx, &task, reason)
}
func (s *PostgreSQLDeliveryService) launchRollbackJob(ctx context.Context, task *model.DeliveryTask, reason string) error {
target, err := getPostgreSQLTarget(ctx, s.awx, task.TargetID)
if err != nil {
return s.markRollbackFailed(ctx, task.ID, "PostgreSQL rollback target is unavailable: "+err.Error())
}
var instances []model.PostgreSQLInstance
if err := s.db.WithContext(ctx).Where("task_id = ?", task.ID).Order("id ASC").Find(&instances).Error; err != nil {
return s.markRollbackFailed(ctx, task.ID, "cannot load PostgreSQL rollback plan: "+err.Error())
}
if len(instances) == 0 {
return s.markRollbackFailed(ctx, task.ID, "PostgreSQL rollback plan has no instances")
}
instanceVars := make(map[string]any, len(instances))
for _, instance := range instances {
instanceVars[instance.Hostname] = map[string]any{
"instance_id": instance.InstanceID,
"data_dir": instance.DataDir,
"config_dir": instance.ConfigDir,
"log_dir": instance.LogDir,
"systemd_unit": instance.SystemdUnit,
}
}
now := time.Now()
var rollback model.RollbackJob
if err := s.db.WithContext(ctx).Where("task_id = ?", task.ID).First(&rollback).Error; errors.Is(err, gorm.ErrRecordNotFound) {
rollback = model.RollbackJob{TaskID: task.ID}
} else if err != nil {
return s.markRollbackFailed(ctx, task.ID, "cannot load PostgreSQL rollback job: "+err.Error())
}
rollback.ExecutorJobID = "pending-postgresql-rollback-" + task.ID
rollback.Status = "launching"
rollback.Reason = reason
rollback.StartedAt = &now
rollback.FinishedAt = nil
if err := s.db.WithContext(ctx).Save(&rollback).Error; err != nil {
return s.markRollbackFailed(ctx, task.ID, "cannot persist PostgreSQL rollback job: "+err.Error())
}
job, err := s.awx.Launch(ctx, s.cfg.PostgreSQLRollbackTemplateID, AWXLaunchRequest{
InventoryID: target.AWXInventoryID,
Limit: task.TargetHost,
ExtraVars: map[string]any{
"target_hosts": task.TargetHost,
"postgresql_instances": instanceVars,
"task_id": task.ID,
"rollback": true,
},
})
if err != nil {
_ = s.db.WithContext(ctx).Model(&rollback).Updates(map[string]any{"status": "failed", "finished_at": time.Now()}).Error
return s.markRollbackFailed(ctx, task.ID, "cannot launch PostgreSQL rollback AWX job: "+err.Error())
}
if err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
var current model.DeliveryTask
if err := tx.First(&current, "id = ? AND service_type = ?", task.ID, postgresqlServiceType).Error; err != nil {
return err
}
if current.Status != model.TaskRollbackPending {
return fmt.Errorf("task %s is no longer pending rollback (state %q)", task.ID, current.Status)
}
if err := tx.Model(&rollback).Updates(map[string]any{"executor_job_id": fmt.Sprint(job.ID), "status": "running"}).Error; err != nil {
return err
}
if err := tx.Model(&model.PostgreSQLInstance{}).Where("task_id = ?", task.ID).Update("status", "rolling_back").Error; err != nil {
return err
}
if err := tx.Model(&model.PostgreSQLCluster{}).Where("task_id = ?", task.ID).Update("status", "rolling_back").Error; err != nil {
return err
}
return s.common.transitionTx(tx, &current, model.TaskRollingBack, "PostgreSQL rollback AWX job started", reason)
}); err != nil {
return s.markRollbackFailed(ctx, task.ID, "cannot record PostgreSQL rollback AWX job: "+err.Error())
}
return nil
}
func (s *PostgreSQLDeliveryService) RetryRollback(ctx context.Context, taskID string) error {
var task model.DeliveryTask
if err := s.db.WithContext(ctx).First(&task, "id = ? AND service_type = ?", taskID, postgresqlServiceType).Error; err != nil {
return err
}
if task.Status != model.TaskRollbackFailed {
return fmt.Errorf("task %s is in state %q and cannot retry rollback", taskID, task.Status)
}
if s.cfg.PostgreSQLRollbackTemplateID == 0 {
return fmt.Errorf("PostgreSQL automatic rollback is not configured")
}
reason := "manual PostgreSQL rollback retry requested"
if err := s.common.transition(ctx, &task, model.TaskRollbackPending, "manual PostgreSQL rollback retry queued", reason); err != nil {
return err
}
if err := s.common.holdReservationForRollback(ctx, task.ID); err != nil {
_ = s.markRollbackFailed(ctx, task.ID, err.Error())
return err
}
if err := s.setRollbackResourceStatus(ctx, task.ID, "rollback_pending"); err != nil {
return s.markRollbackFailed(ctx, task.ID, err.Error())
}
return s.launchRollbackJob(ctx, &task, reason)
}
func (s *PostgreSQLDeliveryService) AcknowledgeRollbackRelease(ctx context.Context, taskID string) error {
now := time.Now()
return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
var task model.DeliveryTask
if err := tx.First(&task, "id = ? AND service_type = ?", taskID, postgresqlServiceType).Error; err != nil {
return err
}
if task.Status != model.TaskRollbackFailed {
return fmt.Errorf("task %s is in state %q and cannot acknowledge rollback release", taskID, task.Status)
}
if err := tx.Model(&model.PostgreSQLInstance{}).Where("task_id = ?", taskID).Update("status", "rollback_acknowledged").Error; err != nil {
return err
}
if err := tx.Model(&model.PostgreSQLCluster{}).Where("task_id = ?", taskID).Update("status", "rollback_acknowledged").Error; err != nil {
return err
}
if err := tx.Model(&model.PostgreSQLResourceUsage{}).Where("task_id = ? AND status = ?", taskID, "active").Updates(map[string]any{"status": "released", "released_at": now}).Error; err != nil {
return err
}
if err := tx.Model(&model.ResourceReservation{}).Where("task_id = ? AND status = ?", taskID, "rollback").Update("status", "released").Error; err != nil {
return err
}
return s.common.transitionTx(tx, &task, model.TaskRollbackAck, "PostgreSQL rollback release acknowledged by platform admin", "")
})
}
func (s *PostgreSQLDeliveryService) setRollbackResourceStatus(ctx context.Context, taskID, status string) error {
return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
if err := tx.Model(&model.PostgreSQLInstance{}).Where("task_id = ?", taskID).Update("status", status).Error; err != nil {
return err
}
return tx.Model(&model.PostgreSQLCluster{}).Where("task_id = ?", taskID).Update("status", status).Error
})
}
func (s *PostgreSQLDeliveryService) markRollbackFailed(ctx context.Context, taskID, message string) error {
return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
var task model.DeliveryTask
if err := tx.First(&task, "id = ? AND service_type = ?", taskID, postgresqlServiceType).Error; err != nil {
return err
}
if task.Status == model.TaskRolledBack || task.Status == model.TaskFinished || task.Status == model.TaskRegisterFailed || task.Status == model.TaskRollbackAck {
return nil
}
if task.Status != model.TaskRollbackFailed {
if err := s.common.transitionTx(tx, &task, model.TaskRollbackFailed, "PostgreSQL automatic rollback failed; manual cleanup is required", message); err != nil {
return err
}
}
if err := tx.Model(&model.PostgreSQLInstance{}).Where("task_id = ?", taskID).Update("status", "rollback_failed").Error; err != nil {
return err
}
if err := tx.Model(&model.PostgreSQLCluster{}).Where("task_id = ?", taskID).Update("status", "rollback_failed").Error; err != nil {
return err
}
return tx.Model(&model.ExecutionJob{}).Where("task_id = ? AND status IN ?", taskID, []string{"launching", "running"}).Updates(map[string]any{"status": "failed", "finished_at": time.Now()}).Error
})
}
func (s *PostgreSQLDeliveryService) PollRollbackOnce(ctx context.Context) error {
var jobs []model.RollbackJob
if err := s.db.WithContext(ctx).
Joins("JOIN delivery_tasks ON delivery_tasks.id = rollback_jobs.task_id").
Where("rollback_jobs.status IN ? AND delivery_tasks.service_type = ?", []string{"launching", "running"}, postgresqlServiceType).
Find(&jobs).Error; err != nil {
return err
}
for _, rollback := range jobs {
if rollback.Status == "launching" {
if !rollbackLaunchExpired(rollback, time.Now()) {
continue
}
expired, err := s.common.expireRollbackLaunch(ctx, rollback.ID)
if err != nil {
return err
}
if expired {
_ = s.markRollbackFailed(ctx, rollback.TaskID, "PostgreSQL rollback launch timed out before its AWX job ID was recorded; manual retry is required")
}
continue
}
job, err := s.awx.GetJob(ctx, rollback.ExecutorJobID)
if err != nil {
continue
}
switch strings.ToLower(job.Status) {
case "pending", "waiting", "running", "new":
continue
case "successful":
_ = s.common.finishRollbackJob(ctx, &rollback, "successful")
if err := s.completeRollback(ctx, rollback.TaskID); err != nil {
_ = s.markRollbackFailed(ctx, rollback.TaskID, err.Error())
}
case "canceled":
_ = s.common.finishRollbackJob(ctx, &rollback, "canceled")
_ = s.markRollbackFailed(ctx, rollback.TaskID, "PostgreSQL rollback AWX job was canceled")
default:
_ = s.common.finishRollbackJob(ctx, &rollback, "failed")
_ = s.markRollbackFailed(ctx, rollback.TaskID, "PostgreSQL rollback AWX job finished with status "+job.Status)
}
}
return nil
}
func (s *PostgreSQLDeliveryService) completeRollback(ctx context.Context, taskID string) error {
now := time.Now()
return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
var task model.DeliveryTask
if err := tx.First(&task, "id = ? AND service_type = ?", taskID, postgresqlServiceType).Error; err != nil {
return err
}
if task.Status != model.TaskRollingBack {
return fmt.Errorf("task %s is in state %q, cannot complete PostgreSQL rollback", taskID, task.Status)
}
if err := tx.Model(&model.PostgreSQLInstance{}).Where("task_id = ?", taskID).Update("status", "rolled_back").Error; err != nil {
return err
}
if err := tx.Model(&model.PostgreSQLCluster{}).Where("task_id = ?", taskID).Update("status", "rolled_back").Error; err != nil {
return err
}
if err := tx.Model(&model.PostgreSQLResourceUsage{}).Where("task_id = ? AND status = ?", taskID, "active").Updates(map[string]any{"status": "released", "released_at": now}).Error; err != nil {
return err
}
if err := tx.Model(&model.ResourceReservation{}).Where("task_id = ? AND status IN ?", taskID, []string{"reserved", "consumed", "rollback"}).Update("status", "released").Error; err != nil {
return err
}
if err := tx.Model(&model.ExecutionJob{}).Where("task_id = ? AND status IN ?", taskID, []string{"launching", "running"}).Updates(map[string]any{"status": "failed", "finished_at": now}).Error; err != nil {
return err
}
return s.common.transitionTx(tx, &task, model.TaskRolledBack, "PostgreSQL instance rollback completed", "")
})
}