Merge remote-tracking branch 'upstream/main' into feat/base-service-delivery

# Conflicts:
#	ansible/mysql-deploy.yml
#	frontend/src/api/delivery.ts
#	frontend/src/views/service/Catalog.vue
#	server/.env.example
#	server/internal/config/config.go
#	server/internal/handler/task_log.go
#	server/internal/service/delivery.go

ansible/files/mysql-delivery@.service

@@ -1,20 +1,53 @@
+# =============================================================================
+# XINFRA MySQL 交付实例 Systemd 模板单元
+# =============================================================================
+# 这是 systemd 的模板单元（template unit），文件名中的 @ 是模板标识
+# 使用方式：systemctl start mysql-delivery@{实例名}.service
+# 例如：mysql-delivery@db01.service、mysql-delivery@db02.service
+#
+# %i 参数会被 systemd 自动替换为 @ 后面的实例名
+# 这样同一台主机可以运行多个 MySQL 实例，彼此隔离
+# =============================================================================
+
 [Unit]
+# 服务描述，%i 会被替换为实际实例名
 Description=XINFRA MySQL delivery instance %i
+# 依赖：等待网络在线后再启动（MySQL 需要绑定网络端口）
 After=network-online.target
 Wants=network-online.target
 
 [Service]
+# Type=notify：mysqld 启动完成后会通过 sd_notify 通知 systemd
+# 比 Type=simple 更可靠，systemd 能准确知道服务何时真正就绪
 Type=notify
+
+# 以 mysql 用户/组身份运行（最小权限原则）
 User=mysql
 Group=mysql
+
+# 创建运行时目录 /run/mysql-delivery-{实例名}/
+# 用于存放 socket 和 pid 文件，放在 tmpfs（内存文件系统）上，重启自动清理
 RuntimeDirectory=mysql-delivery-%i
 RuntimeDirectoryMode=0755
+
+# 启动命令：使用实例专属配置文件
+# %i 被替换为实例名，指向 /etc/mysql/mysql-delivery/{实例名}.cnf
 ExecStart=/usr/sbin/mysqld --defaults-file=/etc/mysql/mysql-delivery/%i.cnf
+
+# 故障自动重启：崩溃后 5 秒自动重启
 Restart=on-failure
 RestartSec=5s
+
+# 启动超时：120 秒内未完成启动则判定为失败
 TimeoutStartSec=120s
+
+# 最大打开文件数：MySQL 需要大量文件描述符（连接、临时表等）
 LimitNOFILE=65535
+
+# OOM 评分调整：-200 表示被 OOM Killer 杀掉的优先级较低
+# MySQL 是数据库服务，被误杀影响较大，尽量保护
 OOMScoreAdjust=-200
 
 [Install]
+# 开机自启：加入 multi-user.target（多用户模式，即正常启动完成后）
 WantedBy=multi-user.target

ansible/inventory.example.yml

@@ -1,11 +1,35 @@
+# =============================================================================
+# Ansible Inventory 示例文件
+# =============================================================================
+# Inventory 定义了要管理的主机列表和分组
+# 实际使用时，平台会动态生成 inventory 并传入变量
+#
+# 使用方式：
+#   ansible-playbook -i inventory.example.yml mysql-deploy.yml \
+#     -e "target_hosts=mysql_hosts" \
+#     -e "instance_name=db01" \
+#     -e "topology=mgr_3" \
+#     -e "mysql_version=8.0" \
+#     -e "memory_mb=8192" \
+#     -e "storage_gb=100"
+# =============================================================================
+
 all:
   children:
+    # --- MySQL 主机组 ---
+    # 按拓扑需求选择对应数量的主机：
+    #   standalone = 1 台
+    #   primary_replica = 2 台（一主一从）
+    #   mgr_3 = 3 台（三节点组复制）
     mysql_hosts:
       hosts:
+        # 格式：主机名: { ansible_host: IP地址 }
         k8s-server-01: { ansible_host: 192.168.1.4 }
         k8s-server-02: { ansible_host: 192.168.1.5 }
         k8s-server-03: { ansible_host: 192.168.1.2 }
         k8s-worker-01: { ansible_host: 192.168.1.3 }
+
+  # --- 全局变量 ---
   vars:
-    ansible_user: root
+    ansible_user: root                                              # SSH 登录用户（需要 root 权限执行系统级操作）
-    ansible_python_interpreter: /usr/bin/python3
+    ansible_python_interpreter: /usr/bin/python3                    # Python 解释器路径（Ubuntu 默认位置）

ansible/mysql-rollback.yml

@@ -0,0 +1,79 @@
+---
+- name: Roll back native MySQL delivery instance
+  hosts: "{{ target_hosts }}"
+  become: true
+  gather_facts: false
+  any_errors_fatal: true
+  vars:
+    mysql_instance: "{{ instance_name }}"
+    mysql_data_disk: "{{ data_disk }}"
+    mysql_base_dir: "{{ mysql_data_disk }}/mysql-delivery/{{ mysql_instance }}"
+    mysql_install_dir: "/opt/mysql-delivery/{{ mysql_instance }}"
+    mysql_config_file: "/etc/mysql/mysql-delivery/{{ mysql_instance }}.cnf"
+    mysql_run_dir: "/run/mysql-delivery-{{ mysql_instance }}"
+
+  pre_tasks:
+    - name: Validate rollback target
+      ansible.builtin.assert:
+        that:
+          - mysql_instance is match('^[a-z0-9][a-z0-9-]{0,62}$')
+          - mysql_data_disk is match('^/')
+          - mysql_data_disk != '/'
+          - "'..' not in mysql_data_disk"
+        fail_msg: "Rollback target is outside the native MySQL delivery layout"
+        quiet: true
+
+  tasks:
+    - name: Stop the delivery instance when present
+      ansible.builtin.systemd_service:
+        name: "mysql-delivery@{{ mysql_instance }}.service"
+        state: stopped
+        enabled: false
+      register: mysql_stop_result
+      failed_when: false
+
+    - name: Remove the instance systemd failure state
+      ansible.builtin.command:
+        argv:
+          - systemctl
+          - reset-failed
+          - "mysql-delivery@{{ mysql_instance }}.service"
+      changed_when: false
+      failed_when: false
+
+    - name: Remove the instance configuration
+      ansible.builtin.file:
+        path: "{{ mysql_config_file }}"
+        state: absent
+
+    - name: Remove instance-local binary links
+      ansible.builtin.file:
+        path: "{{ mysql_install_dir }}"
+        state: absent
+
+    - name: Remove the instance data and log tree
+      ansible.builtin.file:
+        path: "{{ mysql_base_dir }}"
+        state: absent
+
+    - name: Remove the instance runtime directory
+      ansible.builtin.file:
+        path: "{{ mysql_run_dir }}"
+        state: absent
+
+    - name: Confirm the instance artifacts are gone
+      ansible.builtin.stat:
+        path: "{{ item }}"
+      loop:
+        - "{{ mysql_config_file }}"
+        - "{{ mysql_install_dir }}"
+        - "{{ mysql_base_dir }}"
+        - "{{ mysql_run_dir }}"
+      register: mysql_rollback_artifacts
+
+    - name: Assert that all instance artifacts are gone
+      ansible.builtin.assert:
+        that:
+          - mysql_rollback_artifacts.results | selectattr('stat.exists') | list | length == 0
+        fail_msg: "One or more MySQL delivery artifacts remain after rollback"
+        quiet: true

ansible/templates/mysql-instance.cnf.j2

@@ -1,82 +1,134 @@
-# Managed by XINFRA MySQL delivery - generated, do not edit by hand
+# =============================================================================
-# instance={{ mysql_instance }} version={{ mysql_version_value }} topology={{ topology }}
+# XINFRA MySQL 实例配置文件（Jinja2 模板）
+# =============================================================================
+# 此文件由 Playbook 自动生成，禁止手动编辑
+# 修改配置请通过平台参数传递，重新部署即可
+# =============================================================================
+
+# 客户端连接配置（mysql 命令行工具使用）
 [client]
-socket={{ mysql_run_dir }}/mysql.sock
+socket={{ mysql_run_dir }}/mysql.sock     # Unix socket 连接路径（本地连接更快）
-port={{ mysql_port_value }}
+port={{ mysql_port_value }}                # TCP 端口（远程连接使用）
 
+# =============================================================================
+# MySQL 服务端核心配置
+# =============================================================================
 [mysqld]
-user=mysql
+# --- 基础运行参数 ---
-basedir=/usr
+user=mysql                                 # 运行用户（与 systemd 单元一致）
-datadir={{ mysql_data_dir }}
+basedir=/usr                               # MySQL 安装根目录（Ubuntu apt 安装位置）
-tmpdir={{ mysql_tmp_dir }}
+datadir={{ mysql_data_dir }}               # 数据目录（所有表、索引、数据文件）
-socket={{ mysql_run_dir }}/mysql.sock
+tmpdir={{ mysql_tmp_dir }}                 # 临时文件目录（排序、临时表等）
-pid-file={{ mysql_run_dir }}/mysql.pid
+socket={{ mysql_run_dir }}/mysql.sock      # Unix socket 路径（本地客户端连接）
-port={{ mysql_port_value }}
+pid-file={{ mysql_run_dir }}/mysql.pid     # 进程 PID 文件路径
-bind-address=0.0.0.0
+port={{ mysql_port_value }}                # 监听端口
-skip-name-resolve=ON
+bind-address=0.0.0.0                       # 监听所有网络接口（允许远程连接）
-server-id={{ mysql_server_id_value }}
+skip-name-resolve=ON                       # 跳过 DNS 解析（提升连接速度，授权时需用 IP）
-local-infile=OFF
+server-id={{ mysql_server_id_value }}      # 服务器唯一 ID（复制必须，全机唯一）
+local-infile=OFF                           # 禁用 LOAD DATA LOCAL（安全考虑，防止读取客户端文件）
 
-# --- charset / collation / timezone / identifier case ---
+# --- 字符集 / 排序规则 / 时区 / 标识符大小写 ---
-character-set-server={{ mysql_character_set }}
+character-set-server={{ mysql_character_set }}           # 服务端字符集，默认 utf8mb4
-collation-server={{ mysql_collation }}
+collation-server={{ mysql_collation }}                   # 排序规则，默认 utf8mb4_general_ci
-default-time-zone={{ mysql_timezone }}
+default-time-zone={{ mysql_timezone }}                   # 默认时区，东八区
-lower-case-table-names={{ mysql_lower_case_table_names }}
+lower-case-table-names={{ mysql_lower_case_table_names }} # 表名大小写敏感：1=不敏感（Linux 推荐）
 
-# --- error / slow log ---
+# --- 错误日志 / 慢查询日志 ---
-log-error={{ mysql_log_dir }}/error.log
+log-error={{ mysql_log_dir }}/error.log                  # 错误日志路径
-slow-query-log=ON
+slow-query-log=ON                                        # 启用慢查询日志
-slow-query-log-file={{ mysql_log_dir }}/slow.log
+slow-query-log-file={{ mysql_log_dir }}/slow.log         # 慢查询日志路径
-long-query-time={{ mysql_long_query_time }}
+long-query-time={{ mysql_long_query_time }}              # 慢查询阈值（秒），超过此时间记录
 
-# --- binlog (PITR + replication base) ---
+# =============================================================================
-log-bin={{ mysql_binlog_dir }}/binlog
+# Binlog 配置（用于数据恢复 PITR + 复制基础）
-binlog-format=ROW
+# =============================================================================
-sync-binlog={{ mysql_sync_binlog }}
+log-bin={{ mysql_binlog_dir }}/binlog                    # binlog 文件前缀（二进制日志）
-max-binlog-size={{ mysql_max_binlog_size }}
+binlog-format=ROW                                        # 格式：ROW（行变更），复制最安全
-binlog-expire-logs-seconds={{ mysql_binlog_expire_logs_seconds }}
+sync-binlog={{ mysql_sync_binlog }}                      # 每次提交同步 binlog 到磁盘，1=最安全
+max-binlog-size={{ mysql_max_binlog_size }}              # 单个 binlog 文件最大大小
+binlog-expire-logs-seconds={{ mysql_binlog_expire_logs_seconds }} # binlog 过期时间（秒），默认 7 天
+
+# --- 非 standalone 拓扑才启用的复制参数 ---
 {% if topology != 'standalone' %}
-gtid-mode=ON
+gtid-mode=ON                                             # 启用 GTID（全局事务标识符）
-enforce-gtid-consistency=ON
+enforce-gtid-consistency=ON                              # 强制 GTID 一致性（确保复制安全）
-relay-log={{ mysql_binlog_dir }}/relay-bin
+relay-log={{ mysql_binlog_dir }}/relay-bin               # 中继日志路径（从库重放 binlog 使用）
 {% endif %}
 
-# --- InnoDB core ---
+# =============================================================================
+# InnoDB 存储引擎核心配置
+# =============================================================================
+# 缓冲池大小：分配内存的 55%（MySQL 官方推荐 50%-75%）
 innodb-buffer-pool-size={{ ((mysql_memory_mb_value | int) * 55 / 100) | int }}M
+
+# 刷盘方式：O_DIRECT 绕过操作系统缓存，直接写磁盘
+# 避免双重缓存，适合有电池保护写缓存的 RAID 控制器
 innodb-flush-method=O_DIRECT
+
+# 事务提交时刷日志到磁盘：1=每次提交都刷（最安全，性能最低）
 innodb-flush-log-at-trx-commit={{ mysql_flush_log_at_trx_commit }}
+
+# InnoDB I/O 能力：告诉 InnoDB 磁盘的 IOPS 能力
+# 2000 适合 SSD，HDD 建议 200-400
 innodb-io-capacity={{ mysql_io_capacity }}
+
+# 最大连接数（按内存梯度计算，默认值见 Playbook）
 max-connections={{ mysql_max_connections }}
 
-# --- redo log (version-sensitive) ---
+# =============================================================================
-innodb-log-group-home-dir={{ mysql_redo_dir }}
+# Redo Log 配置（InnoDB 崩溃恢复必需）
+# =============================================================================
+innodb-log-group-home-dir={{ mysql_redo_dir }}           # redo log 目录
+
+# MySQL 8.0.30+ 使用 innodb-redo-log-capacity 控制 redo log 总大小
+# 旧版本使用 innodb-log-file-size 控制单个文件大小
 {% if mysql_version_value in ['8.0', '8.4'] %}
 innodb-redo-log-capacity={{ mysql_redo_capacity }}
 {% else %}
 innodb-log-file-size={{ mysql_redo_capacity }}
 {% endif %}
 
-# --- X Protocol disabled (version-sensitive) ---
+# =============================================================================
+# X Protocol 禁用（版本相关）
+# =============================================================================
+# X Protocol 是 MySQL 的 NoSQL 部分，我们不需要，禁用节省资源
 {% if mysql_version_value in ['8.0', '8.4'] %}
-mysqlx=0
+mysqlx=0                                                   # 8.0/8.4 直接禁用
 {% elif mysql_version_value == '5.7' %}
-loose-mysqlx=0
+loose-mysqlx=0                                             # 5.7 使用 loose 前缀（参数可能不存在）
 {% endif %}
 
+# =============================================================================
+# Group Replication 配置（仅 mgr_3 拓扑）
+# =============================================================================
+# 注意：这里只做配置就绪，实际启动组复制需要平台侧执行 START GROUP_REPLICATION
 {% if topology == 'mgr_3' %}
-# --- Group Replication (config-ready; runtime bootstrap handled out-of-band) ---
+# 加载 Group Replication 插件
 plugin-load-add=group_replication.so
+
+# MySQL 8.0.26 之前需要设置事务写集提取（8.4 已移除此参数）
 {% if mysql_version_value != '8.4' %}
-{# deprecated since 8.0.26 and removed in 8.3+; only inject for older series #}
+transaction-write-set-extraction=XXHASH64                 # 使用 XXHASH64 算法提取写集
-transaction-write-set-extraction=XXHASH64
 {% endif %}
+
+# GR 组 UUID（所有节点必须相同）
 loose-group-replication-group-name={{ mgr_group_name }}
+
+# 不在启动时自动加入组（由平台侧手动控制加入时机）
 loose-group-replication-start-on-boot=OFF
+
+# 本节点用于 GR 通信的地址和端口
 loose-group-replication-local-address={{ ansible_host | default(inventory_hostname) }}:{{ mysql_gr_port_value }}
+
+# 组内所有节点的种子列表（用于新节点发现和连接）
 loose-group-replication-group-seeds={% for host in ansible_play_hosts_all %}{{ hostvars[host].ansible_host | default(host) }}:{{ mysql_gr_port_value }}{% if not loop.last %},{% endif %}{% endfor %}
 
+# IP 白名单：只允许组内节点 IP 加入（安全限制）
 loose-group-replication-ip-allowlist={% for host in ansible_play_hosts_all %}{{ hostvars[host].ansible_host | default(host) }}{% if not loop.last %},{% endif %}{% endfor %}
 
+# 单主模式：只有 primary 节点可写，replica 只读（推荐生产使用）
 loose-group-replication-single-primary-mode=ON
+
+# 关闭强制更新检查：单主模式下不需要（多主模式才需要）
 loose-group-replication-enforce-update-everywhere-checks=OFF
 {% endif %}

frontend/src/api/delivery.ts

@@ -136,6 +136,24 @@ export const deliveryApi = {
     const query = search.toString()
     return new EventSource(`/auth/api/v1/delivery/tasks/${encodeURIComponent(taskId)}/stream${query ? `?${query}` : ''}`)
   },
+
+  async retryRollback(taskId: string): Promise<void> {
+    await authRequest(`/auth/api/v1/delivery/tasks/${encodeURIComponent(taskId)}/rollback/retry`, {
+      method: 'POST',
+    })
+  },
+
+  async releaseRollback(taskId: string): Promise<void> {
+    await authRequest(`/auth/api/v1/delivery/tasks/${encodeURIComponent(taskId)}/rollback/release`, {
+      method: 'POST',
+    })
+  },
+
+  async retryCloudDMRegistration(taskId: string): Promise<void> {
+    await authRequest(`/auth/api/v1/delivery/tasks/${encodeURIComponent(taskId)}/clouddm/retry`, {
+      method: 'POST',
+    })
+  },
 }
 
 function createIdempotencyKey(payload: CreateMySQLDeliveryPayload) {

frontend/src/views/service/Catalog.vue

@@ -279,7 +279,7 @@
         </section>
 
         <section v-else class="result-view">
-          <div class="result-head" :class="{ failed: deliveryFailed }">
+          <div class="result-head" :class="{ failed: deliveryFailed, warning: deliveryRegisterFailed }">
             <div class="result-icon">{{ deliveryFailed ? '!' : '✓' }}</div>
             <div>
               <h3>{{ resultTitle }}</h3>
@@ -287,6 +287,27 @@
             </div>
           </div>
 
+          <div v-if="canManageRollback" class="rollback-actions">
+            <div>
+              <strong>目标机清理未确认</strong>
+              <p>资源目前仍被保护性占用。可重试自动回退，或确认已完成目标机清理后释放。</p>
+            </div>
+            <div class="head-actions">
+              <el-button :loading="rollbackRetrying" :disabled="rollbackActionBusy" @click="retryRollback">重试回退</el-button>
+              <el-button type="warning" :loading="rollbackReleasing" :disabled="rollbackActionBusy" @click="releaseRollback">确认清理并释放资源</el-button>
+            </div>
+          </div>
+
+          <div v-if="canRetryCloudDM" class="rollback-actions registration-actions">
+            <div>
+              <strong>实例已交付，CloudDM 注册未完成</strong>
+              <p>数据库健康实例和资源账本已保留，仅需重试 CloudDM 注册。</p>
+            </div>
+            <div class="head-actions">
+              <el-button type="warning" :loading="cloudDMRetrying" @click="retryCloudDMRegistration">重试 CloudDM 注册</el-button>
+            </div>
+          </div>
+
           <div class="result-grid">
             <div class="result-item">
               <span>访问地址</span>
@@ -326,7 +347,7 @@
                   <td>{{ deliveryForm.version }}</td>
                   <td>{{ currentModeLabel }}</td>
                   <td><code>{{ resultAddress }}</code></td>
-                  <td><span class="tag" :class="deliveryFailed ? 'tag-red' : 'tag-green'">{{ deliveryFailed ? '已回退' : '正常' }}</span></td>
+                  <td><span class="tag" :class="deliveryFailed ? (deliveryRolledBack || deliveryAcknowledged || deliveryRegisterFailed ? 'tag-amber' : 'tag-red') : 'tag-green'">{{ deliveryRolledBack ? '已回退' : deliveryAcknowledged ? '已确认释放' : deliveryRegisterFailed ? '实例保留' : deliveryFailed ? '需处理' : '正常' }}</span></td>
                   <td><code>{{ activeService.registerTo }}</code></td>
                 </tr>
               </tbody>
@@ -348,10 +369,11 @@
 
 <script setup lang="ts">
 import { computed, onMounted, onUnmounted, reactive, ref, watch } from 'vue'
-import { ElMessage } from 'element-plus'
+import { ElMessage, ElMessageBox } from 'element-plus'
 import { Back, CircleCheck, Promotion, Refresh } from '@element-plus/icons-vue'
 import { useRoute, useRouter } from 'vue-router'
 import { deliveryApi, type DeliveryMountPath, type DeliveryTarget, type DeliveryTask, type TaskEvent } from '@/api/delivery'
+import { useAuthStore } from '@/stores/auth'
 import { useBusinessLineStore } from '@/stores/businessLine'
 import { useBusinessLineMockProfile } from '@/utils/businessLineMock'
 
@@ -404,6 +426,7 @@ interface DeliveryStep {
 const { currentName } = useBusinessLineMockProfile()
 const route = useRoute()
 const router = useRouter()
+const authStore = useAuthStore()
 const businessLineStore = useBusinessLineStore()
 
 const basicServices = ref<Service[]>([
@@ -573,6 +596,15 @@ const taskRestoring = ref(false)
 const seenEventIds = ref(new Set<number>())
 const deliveryLog = ref('[ready] 等待创建交付任务...')
 let taskEventSource: EventSource | undefined
+const deliveryRolledBack = computed(() => currentTaskStatus.value === 'rolled_back')
+const deliveryAcknowledged = computed(() => currentTaskStatus.value === 'rollback_acknowledged')
+const deliveryRegisterFailed = computed(() => currentTaskStatus.value === 'register_failed')
+const rollbackRetrying = ref(false)
+const rollbackReleasing = ref(false)
+const cloudDMRetrying = ref(false)
+const rollbackActionBusy = computed(() => rollbackRetrying.value || rollbackReleasing.value)
+const canManageRollback = computed(() => authStore.isAdmin && currentTaskStatus.value === 'rollback_failed' && Boolean(deploymentId.value))
+const canRetryCloudDM = computed(() => deliveryRegisterFailed.value && Boolean(deploymentId.value))
 
 const deliveryForm = reactive({
   instanceName: `mysql-${currentName.value}-billing-02`,
@@ -645,21 +677,31 @@ const runnerPreview = computed(() => {
 })
 const resultAddress = computed(() => `${deliveredHost.value || topologyNodes.value[0]?.ip || '10.24.18.21'}:${deliveredPort.value || deliveryForm.port || 'auto'}`)
 const resultTitle = computed(() => {
-  if (currentTaskStatus.value === 'rollback_failed') return '交付失败，回滚失败'
+  if (deliveryRolledBack.value) return '交付失败，已完成回退'
-  if (currentTaskStatus.value === 'rolled_back') return '交付失败，已回滚'
+  if (deliveryAcknowledged.value) return '交付失败，已确认清理'
+  if (deliveryRegisterFailed.value) return 'MySQL 已交付，CloudDM 注册失败'
+  if (currentTaskStatus.value === 'rollback_failed') return '交付失败，回退失败'
+  if (currentTaskStatus.value === 'canceled') return '交付任务已取消'
   if (deliveryFailed.value) return '交付失败'
   return activeServiceKey.value === 'mysql' ? 'MySQL 实例已交付' : 'OpenResty 集群已交付'
 })
 const resultSubtitle = computed(() => {
-  if (currentTaskStatus.value === 'rollback_failed') return deliveryError.value || '自动回滚失败 · 需要人工清理'
+  if (deliveryRolledBack.value) return deliveryError.value || '目标机实例文件、配置和资源记录已清理'
-  if (currentTaskStatus.value === 'rolled_back') return deliveryError.value || '资源已释放 · 变更未交付'
+  if (deliveryAcknowledged.value) return deliveryError.value || '管理员已确认目标机清理完成，资源预留已释放（不代表自动回退成功）'
-  if (deliveryFailed.value) return deliveryError.value || '交付失败 · 资源已释放 · 变更未交付'
+  if (deliveryRegisterFailed.value) return deliveryError.value || '实例已通过健康检查并完成资源入账，CloudDM 注册失败；实例保留，可重试注册'
+  if (currentTaskStatus.value === 'rollback_failed') return deliveryError.value || '自动回退失败 · 目标机可能存在残留 · 需要人工处理'
+  if (currentTaskStatus.value === 'canceled') return deliveryError.value || '任务在部署完成前被取消'
+  if (deliveryFailed.value) return deliveryError.value || '交付失败 · 未确认目标机已清理'
   return activeServiceKey.value === 'mysql' ? '全部步骤执行成功 · 用时 06:42' : '全部步骤执行成功 · 用时 02:18'
 })
 const deliveryStateText = computed(() => {
-  if (currentTaskStatus.value === 'rollback_failed') return '回滚失败'
+  if (currentTaskStatus.value === 'rollback_failed') return '回退失败'
-  if (isRollbackRunningStatus(currentTaskStatus.value)) return '回滚中'
+  if (isRollbackRunningStatus(currentTaskStatus.value)) return '回退中'
-  if (deliveryFailed.value) return isRolledBackStatus(currentTaskStatus.value) ? '已回滚' : '失败'
+  if (deliveryRolledBack.value) return '已回退'
+  if (deliveryAcknowledged.value) return '已确认释放'
+  if (deliveryRegisterFailed.value) return '注册失败（实例保留）'
+  if (currentTaskStatus.value === 'canceled') return '已取消'
+  if (deliveryFailed.value) return '交付失败'
   if (deliveryDone.value) return '已交付'
   if (running.value) return '交付中'
   if (precheckPassed.value) return '待执行'
@@ -668,6 +710,9 @@ const deliveryStateText = computed(() => {
 const canShowCancelDeployment = computed(() => Boolean(deploymentId.value) && isCancelableDeliveryStatus(currentTaskStatus.value))
 const canCancelDeployment = computed(() => Boolean(deploymentId.value) && isCancelableDeliveryStatus(currentTaskStatus.value))
 const deliveryStateClass = computed(() => {
+  if (currentTaskStatus.value === 'rollback_failed') return 'tag-red'
+  if (deliveryRolledBack.value || deliveryAcknowledged.value) return 'tag-amber'
+  if (deliveryRegisterFailed.value) return 'tag-amber'
   if (deliveryFailed.value) return 'tag-red'
   if (deliveryDone.value) return 'tag-green'
   if (precheckPassed.value || running.value) return 'tag-amber'
@@ -799,6 +844,7 @@ function defaultSteps(): DeliveryStep[] {
         ['应用标准参数与复制配置', '基础参数、GTID、主从关系'],
         ['数据库健康检查', '端口、读写、复制延迟、参数基线'],
         ['注册资产与交付归档', 'CloudDM、CMDB、Prometheus'],
+        ['失败回退与清理', '停止实例、清理目录、释放资源'],
       ]
     : [
         ['资源锁定与配置检查', '节点、端口、证书、Ingress 冲突'],
@@ -1153,6 +1199,13 @@ function applyDeliveryStatus(status: string, message: string) {
     steps.value = steps.value.map((step) => ({ ...step, state: 'done' }))
     return
   }
+  if (status === 'rolled_back') {
+    running.value = false
+    deliveryDone.value = false
+    deliveryFailed.value = true
+    steps.value = steps.value.map((step) => ({ ...step, state: 'done' }))
+    return
+  }
   if (isFailedDeliveryStatus(status)) {
     running.value = false
     deliveryDone.value = false
@@ -1307,7 +1360,7 @@ function isTerminalDeliveryStatus(status: string) {
 }
 
 const activeDeliveryStatuses = ['pending', 'validating', 'dispatching', 'running', 'registering', 'canceling', 'rollback_pending', 'rolling_back']
-const failedDeliveryStatuses = ['execution_failed', 'validation_failed', 'register_failed', 'canceled', 'rollback_failed', 'rolled_back']
+const failedDeliveryStatuses = ['execution_failed', 'validation_failed', 'register_failed', 'canceled', 'rollback_failed', 'rolled_back', 'rollback_acknowledged']
 const cancelableDeliveryStatuses = ['pending', 'dispatching', 'running']
 
 function isActiveDeliveryStatus(status: string) {
@@ -1326,8 +1379,63 @@ function isRollbackRunningStatus(status: string) {
   return ['rollback_pending', 'rolling_back'].includes(status)
 }
 
-function isRolledBackStatus(status: string) {
+async function retryRollback() {
-  return status === 'rolled_back'
+  if (!deploymentId.value) return
+  rollbackRetrying.value = true
+  try {
+    await deliveryApi.retryRollback(deploymentId.value)
+    appendLog('[rollback] manual retry requested by platform admin')
+    activeView.value = 'execution'
+    startTaskStream(deploymentId.value)
+    ElMessage.success('已重新发起回退')
+  } catch (error) {
+    ElMessage.error(error instanceof Error ? error.message : '重试回退失败')
+  } finally {
+    rollbackRetrying.value = false
+  }
+}
+
+async function retryCloudDMRegistration() {
+  if (!deploymentId.value) return
+  cloudDMRetrying.value = true
+  try {
+    await deliveryApi.retryCloudDMRegistration(deploymentId.value)
+    appendLog('[clouddm] registration retry requested')
+    activeView.value = 'execution'
+    startTaskStream(deploymentId.value)
+    ElMessage.success('已重新发起 CloudDM 注册')
+  } catch (error) {
+    ElMessage.error(error instanceof Error ? error.message : '重试 CloudDM 注册失败')
+  } finally {
+    cloudDMRetrying.value = false
+  }
+}
+
+async function releaseRollback() {
+  if (!deploymentId.value) return
+  try {
+    await ElMessageBox.confirm('请确认已在目标机清理实例服务、配置、数据和日志目录。该操作会释放资源记录，但不会声称自动回退成功。', '确认释放资源', {
+      type: 'warning',
+      confirmButtonText: '确认释放',
+      cancelButtonText: '取消',
+    })
+  } catch {
+    return
+  }
+  rollbackReleasing.value = true
+  try {
+    await deliveryApi.releaseRollback(deploymentId.value)
+    appendLog('[rollback] resource release acknowledged by platform admin')
+    deliveryError.value = ''
+    applyDeliveryStatus('rollback_acknowledged', '')
+    stopTaskStream()
+    activeView.value = 'result'
+    ElMessage.success('已确认清理，资源已释放')
+  } catch (error) {
+    ElMessage.error(error instanceof Error ? error.message : '释放资源失败')
+  } finally {
+    rollbackReleasing.value = false
+  }
 }
 
 function markStepRunning() {
@@ -1800,11 +1908,42 @@ h4 {
   color: #FFFFFF;
 }
 
+.result-head.warning .result-icon {
+  background: var(--tag-amber-bg);
+  color: var(--tag-amber-text);
+  border: 1px solid var(--tag-amber-border);
+}
+
 .result-head h3 {
   margin: 0 0 4px;
   font-size: 16px;
 }
 
+.rollback-actions {
+  display: flex;
+  align-items: center;
+  justify-content: space-between;
+  gap: 16px;
+  margin: 16px 0;
+  padding: 14px 16px;
+  border: 1px solid var(--tag-amber-border);
+  border-radius: 8px;
+  background: var(--tag-amber-bg);
+}
+
+.rollback-actions strong {
+  display: block;
+  margin-bottom: 4px;
+  color: var(--tag-amber-text);
+  font-size: 13px;
+}
+
+.rollback-actions p {
+  margin: 0;
+  color: var(--text-mid);
+  font-size: 12px;
+}
+
 .result-grid {
   display: grid;
   grid-template-columns: repeat(4, minmax(0, 1fr));
@@ -1946,6 +2085,11 @@ code {
     flex-direction: column;
   }
 
+  .rollback-actions {
+    align-items: stretch;
+    flex-direction: column;
+  }
+
   .head-actions {
     margin-left: 0;
   }

server/.env.example

@@ -33,6 +33,9 @@ AWX_PASSWORD=
 AWX_WEBHOOK_TOKEN=
 AWX_FACTS_TEMPLATE_ID=
 AWX_FACTS_TIMEOUT_SECONDS=45
+# AWX Job Template ID for ansible/mysql-rollback.yml; required for automatic cleanup
+DELIVERY_ROLLBACK_TEMPLATE_ID=0
+DELIVERY_SERVICE_TOKEN=
 CLOUDDM_REGISTER_URL=
 CLOUDDM_API_TOKEN=
 

server/internal/config/config.go

@@ -66,6 +66,8 @@ type Config struct {
 	AWXWebhookToken           string
 	AWXFactsTemplateID        uint64
 	AWXFactsTimeoutSeconds    int
+	RollbackTemplateID        uint64
+	DeliveryServiceToken      string
 	DeliverySchedulerEnabled  bool
 	DeliveryDispatchSeconds   int
 	DeliveryCallbackBaseURL   string
@@ -138,6 +140,8 @@ func Load() Config {
 		AWXWebhookToken:           env("AWX_WEBHOOK_TOKEN", ""),
 		AWXFactsTemplateID:        envUint64("AWX_FACTS_TEMPLATE_ID", 0),
 		AWXFactsTimeoutSeconds:    envInt("AWX_FACTS_TIMEOUT_SECONDS", 45),
+		RollbackTemplateID:        uint64(envInt("DELIVERY_ROLLBACK_TEMPLATE_ID", 0)),
+		DeliveryServiceToken:      env("DELIVERY_SERVICE_TOKEN", ""),
 		DeliverySchedulerEnabled:  envBool("DELIVERY_SCHEDULER_ENABLED", false),
 		DeliveryDispatchSeconds:   envInt("DELIVERY_DISPATCH_SECONDS", 5),
 		DeliveryCallbackBaseURL:   trimURL(env("DELIVERY_CALLBACK_BASE_URL", publicBaseURL)),

server/internal/database/database.go

@@ -27,6 +27,7 @@ func AutoMigrate(db *gorm.DB) error {
 		&model.MySQLInstance{},
 		&model.ResourceUsage{},
 		&model.ExecutionJob{},
+		&model.RollbackJob{},
 		&model.TaskEvent{},
 	)
 }

server/internal/handler/delivery.go

@@ -242,6 +242,55 @@ func (h *DeliveryHandler) Cancel(c *gin.Context) {
 	c.JSON(http.StatusOK, gin.H{"ok": true})
 }
 
+// RetryRollback retries the compensating AWX job after a previous cleanup failure.
+func (h *DeliveryHandler) RetryRollback(c *gin.Context) {
+	if !requirePlatformAdmin(c) {
+		return
+	}
+	if err := h.service.RetryRollback(c.Request.Context(), c.Param("id")); err != nil {
+		c.JSON(http.StatusConflict, gin.H{"error": err.Error()})
+		return
+	}
+	c.JSON(http.StatusAccepted, gin.H{"ok": true, "status": model.TaskRollbackPending})
+}
+
+// AcknowledgeRollbackRelease releases bookkeeping after an administrator has
+// independently verified that no instance artifacts remain on the target host.
+func (h *DeliveryHandler) AcknowledgeRollbackRelease(c *gin.Context) {
+	if !requirePlatformAdmin(c) {
+		return
+	}
+	if err := h.service.AcknowledgeRollbackRelease(c.Request.Context(), c.Param("id")); err != nil {
+		c.JSON(http.StatusConflict, gin.H{"error": err.Error()})
+		return
+	}
+	c.JSON(http.StatusOK, gin.H{"ok": true, "status": model.TaskRollbackAck})
+}
+
+// RetryCloudDMRegistration retries only the CloudDM registration step for an
+// already healthy and accounted-for MySQL instance.
+func (h *DeliveryHandler) RetryCloudDMRegistration(c *gin.Context) {
+	claims, ok := CurrentClaims(c)
+	if !ok {
+		c.JSON(http.StatusUnauthorized, gin.H{"error": "missing current user"})
+		return
+	}
+	taskID := c.Param("id")
+	if _, _, err := h.service.GetTask(c.Request.Context(), taskID, claims.UserID, claims.IsAdmin); err != nil {
+		if errors.Is(err, gorm.ErrRecordNotFound) {
+			c.JSON(http.StatusNotFound, gin.H{"error": "task not found"})
+			return
+		}
+		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+		return
+	}
+	if err := h.service.RetryCloudDMRegistration(c.Request.Context(), taskID); err != nil {
+		c.JSON(http.StatusConflict, gin.H{"error": err.Error()})
+		return
+	}
+	c.JSON(http.StatusAccepted, gin.H{"ok": true, "status": model.TaskFinished})
+}
+
 // Targets 获取可用部署目标
 // @Summary      获取可用部署目标
 // @Description  从 AWX 动态返回可用 Job Template 及其 Inventory hosts

server/internal/handler/task_log.go

@@ -138,7 +138,7 @@ func (h *TaskLogHandler) getAWXTask(c *gin.Context, taskID string, userID uint64
 		lines = append(lines, taskLogLine{Time: formatTaskLogTime(event.CreatedAt), Message: "[" + event.ToState + "] " + event.Message, Class: classForTaskStatus(event.ToState)})
 	}
 	var execution model.ExecutionJob
-	if err := h.db.WithContext(c.Request.Context()).Where("task_id = ?", task.ID).First(&execution).Error; err == nil && execution.ExecutorJobID != "" && execution.ExecutorJobID != "pending" && !strings.HasPrefix(execution.ExecutorJobID, "pending:") {
+	if err := h.db.WithContext(c.Request.Context()).Where("task_id = ?", task.ID).First(&execution).Error; err == nil && execution.ExecutorJobID != "" && execution.ExecutorJobID != "pending" && !strings.HasPrefix(execution.ExecutorJobID, "pending:") && !strings.HasPrefix(execution.ExecutorJobID, "pending-") {
 		stdout, stdoutErr := h.delivery.AWXJobStdout(c.Request.Context(), execution.ExecutorJobID)
 		if stdoutErr != nil {
 			lines = append(lines, taskLogLine{Time: formatTaskLogTime(time.Now()), Message: "[awx] stdout fetch failed: " + stdoutErr.Error(), Class: "err"})
@@ -146,6 +146,15 @@ func (h *TaskLogHandler) getAWXTask(c *gin.Context, taskID string, userID uint64
 			lines = append(lines, splitStdoutLines(stdout)...)
 		}
 	}
+	var rollback model.RollbackJob
+	if err := h.db.WithContext(c.Request.Context()).Where("task_id = ?", task.ID).First(&rollback).Error; err == nil && rollback.ExecutorJobID != "" && rollback.ExecutorJobID != "pending" && !strings.HasPrefix(rollback.ExecutorJobID, "pending-") {
+		stdout, stdoutErr := h.delivery.AWXJobStdout(c.Request.Context(), rollback.ExecutorJobID)
+		if stdoutErr != nil {
+			lines = append(lines, taskLogLine{Time: formatTaskLogTime(time.Now()), Message: "[rollback awx] stdout fetch failed: " + stdoutErr.Error(), Class: "err"})
+		} else {
+			lines = append(lines, splitStdoutLines(stdout)...)
+		}
+	}
 	c.JSON(http.StatusOK, gin.H{"task": awxTaskSummary(*task), "lines": lines})
 }
 
@@ -257,8 +266,18 @@ func textForTaskStatus(status string) string {
 		return "等待"
 	case model.TaskRunning, model.TaskRegistering, model.TaskCanceling:
 		return "执行中"
+	case model.TaskRollbackPending, model.TaskRollingBack:
+		return "回退中"
 	case model.TaskFinished:
 		return "成功"
+	case model.TaskRolledBack:
+		return "已回退"
+	case model.TaskRollbackFailed:
+		return "回退失败"
+	case model.TaskRollbackAck:
+		return "已确认释放"
+	case model.TaskRegisterFailed:
+		return "注册失败（实例保留）"
 	case model.TaskCanceled:
 		return "已取消"
 	default:
@@ -270,10 +289,14 @@ func classForTaskStatus(status string) string {
 	switch status {
 	case model.TaskFinished:
 		return "ok"
-	case model.TaskExecutionFailed, model.TaskValidationFailed, model.TaskRegisterFailed, model.TaskCanceled:
+	case model.TaskExecutionFailed, model.TaskValidationFailed, model.TaskCanceled, model.TaskRollbackFailed:
 		return "err"
-	case model.TaskRunning, model.TaskDispatching, model.TaskRegistering, model.TaskCanceling:
+	case model.TaskRollbackAck, model.TaskRegisterFailed:
 		return "warn"
+	case model.TaskRunning, model.TaskDispatching, model.TaskRegistering, model.TaskCanceling, model.TaskRollbackPending, model.TaskRollingBack:
+		return "warn"
+	case model.TaskRolledBack:
+		return "ok"
 	default:
 		return ""
 	}

server/internal/model/delivery.go

@@ -14,6 +14,11 @@ const (
 	TaskValidationFailed = "validation_failed"
 	TaskCanceling        = "canceling"
 	TaskCanceled         = "canceled"
+	TaskRollbackPending  = "rollback_pending"
+	TaskRollingBack      = "rolling_back"
+	TaskRolledBack       = "rolled_back"
+	TaskRollbackFailed   = "rollback_failed"
+	TaskRollbackAck      = "rollback_acknowledged"
 )
 
 type ResourceQuota struct {
@@ -111,6 +116,20 @@ type ExecutionJob struct {
 	UpdatedAt      time.Time  `json:"updated_at"`
 }
 
+// RollbackJob tracks the compensating AWX run independently from the deploy run.
+// Keeping a separate record preserves both job IDs for audit and retry tooling.
+type RollbackJob struct {
+	ID            uint64     `gorm:"primaryKey" json:"id"`
+	TaskID        string     `gorm:"size:36;not null;uniqueIndex" json:"task_id"`
+	ExecutorJobID string     `gorm:"size:128;not null" json:"executor_job_id"`
+	Status        string     `gorm:"size:32;not null;index" json:"status"`
+	Reason        string     `gorm:"type:text" json:"reason"`
+	StartedAt     *time.Time `json:"started_at,omitempty"`
+	FinishedAt    *time.Time `json:"finished_at,omitempty"`
+	CreatedAt     time.Time  `json:"created_at"`
+	UpdatedAt     time.Time  `json:"updated_at"`
+}
+
 type TaskEvent struct {
 	ID          uint64    `gorm:"primaryKey" json:"id"`
 	TaskID      string    `gorm:"size:36;not null;index" json:"task_id"`

server/internal/router/router.go

@@ -152,6 +152,9 @@ func registerAuthServerRoutes(r *gin.Engine, deps Dependencies) {
 		protected.GET("/delivery/tasks/:id", deliveryHandler.Get)
 		protected.GET("/delivery/tasks/:id/stream", deliveryHandler.Stream)
 		protected.POST("/delivery/tasks/:id/cancel", deliveryHandler.Cancel)
+		protected.POST("/delivery/tasks/:id/rollback/retry", deliveryHandler.RetryRollback)
+		protected.POST("/delivery/tasks/:id/rollback/release", deliveryHandler.AcknowledgeRollbackRelease)
+		protected.POST("/delivery/tasks/:id/clouddm/retry", deliveryHandler.RetryCloudDMRegistration)
 		protected.GET("/task-logs", taskLogHandler.List)
 		protected.GET("/task-logs/:id", taskLogHandler.Get)
 	}

server/internal/service/delivery_test.go

@@ -1,6 +1,11 @@
 package service
 
-import "testing"
+import (
+	"testing"
+	"time"
+
+	"github.com/1024XEngineer/xinfra/server/internal/model"
+)
 
 func TestValidateDeliveryInput(t *testing.T) {
 	dataDisks := []string{"/data", "/disk1"}
@@ -18,15 +23,18 @@ func TestValidateDeliveryInput(t *testing.T) {
 	full.CPUMilli = 2000
 	full.MemoryMi = 8192
 	full.StorageGi = 2000
-	full.TimeZone = "+08:00"
+	full.Timezone = "+08:00"
-	full.LowerCaseTableNames = 0
+	lowerCaseZero := 0
+	full.LowerCaseTableNames = &lowerCaseZero
 	full.CharacterSet = "utf8mb4"
 	full.Collation = "utf8mb4_general_ci"
 	full.MaxConnections = "auto"
-	full.InnoDBRedoLogCapacity = "256M"
+	full.InnodbRedoLogCapacity = "256M"
-	full.InnoDBFlushLogAtTrxCommit = 2
+	flushLog := 2
-	full.SyncBinlog = 0
+	full.InnodbFlushLogAtTrxCommit = &flushLog
-	full.InnoDBIOCapacity = 2000
+	syncBinlog := 0
+	full.SyncBinlog = &syncBinlog
+	full.InnodbIOCapacity = 2000
 	full.LongQueryTime = 0.5
 	full.BinlogExpireLogsSeconds = 604800
 	full.MaxBinlogSize = "512M"
@@ -34,7 +42,7 @@ func TestValidateDeliveryInput(t *testing.T) {
 		t.Fatalf("valid full input rejected: %v", err)
 	}
 	namedZone := valid
-	namedZone.TimeZone = "Asia/Shanghai"
+	namedZone.Timezone = "Asia/Shanghai"
 	if err := validateDeliveryInput(namedZone, dataDisks); err != nil {
 		t.Fatalf("named timezone rejected: %v", err)
 	}
@@ -58,15 +66,15 @@ func TestValidateDeliveryInput(t *testing.T) {
 		"port below pool":      func(in *MySQLDeliveryInput) { in.MySQLPort = 3307 },
 		"port above pool":      func(in *MySQLDeliveryInput) { in.MySQLPort = 14000 },
 		"bad target host":      func(in *MySQLDeliveryInput) { in.TargetHost = "-bad-host" },
-		"bad timezone":         func(in *MySQLDeliveryInput) { in.TimeZone = "UTC+8" },
+		"bad timezone":         func(in *MySQLDeliveryInput) { in.Timezone = "UTC+8" },
-		"bad lower case":       func(in *MySQLDeliveryInput) { in.LowerCaseTableNames = 2 },
+		"bad lower case":       func(in *MySQLDeliveryInput) { v := 2; in.LowerCaseTableNames = &v },
 		"bad charset":          func(in *MySQLDeliveryInput) { in.CharacterSet = "big5" },
 		"collation mismatch":   func(in *MySQLDeliveryInput) { in.CharacterSet = "gbk"; in.Collation = "utf8mb4_general_ci" },
 		"bad max connections":  func(in *MySQLDeliveryInput) { in.MaxConnections = "300" },
-		"bad redo capacity":    func(in *MySQLDeliveryInput) { in.InnoDBRedoLogCapacity = "2G" },
+		"bad redo capacity":    func(in *MySQLDeliveryInput) { in.InnodbRedoLogCapacity = "2G" },
-		"bad flush log":        func(in *MySQLDeliveryInput) { in.InnoDBFlushLogAtTrxCommit = 3 },
+		"bad flush log":        func(in *MySQLDeliveryInput) { v := 3; in.InnodbFlushLogAtTrxCommit = &v },
-		"bad sync binlog":      func(in *MySQLDeliveryInput) { in.SyncBinlog = 2 },
+		"bad sync binlog":      func(in *MySQLDeliveryInput) { v := 2; in.SyncBinlog = &v },
-		"bad io capacity":      func(in *MySQLDeliveryInput) { in.InnoDBIOCapacity = 500 },
+		"bad io capacity":      func(in *MySQLDeliveryInput) { in.InnodbIOCapacity = 500 },
 		"bad long query time":  func(in *MySQLDeliveryInput) { in.LongQueryTime = 3 },
 		"bad binlog expire":    func(in *MySQLDeliveryInput) { in.BinlogExpireLogsSeconds = 3600 },
 		"bad max binlog size":  func(in *MySQLDeliveryInput) { in.MaxBinlogSize = "64M" },
@@ -122,3 +130,40 @@ func TestAllocatePort(t *testing.T) {
 		t.Fatal("exhausted pool still allocated a port")
 	}
 }
+
+func TestRollbackExtraVarsTargetsOnlyTheAllocatedInstance(t *testing.T) {
+	task := &model.DeliveryTask{ID: "task-1", TargetHost: "db-01"}
+	payload := deliveryPayload{MySQLDeliveryInput: MySQLDeliveryInput{InstanceName: "mysql-a", DataDisk: "/disk1"}}
+	vars := rollbackExtraVars(task, payload)
+	if vars["target_hosts"] != "db-01" || vars["instance_name"] != "mysql-a" || vars["data_disk"] != "/disk1" {
+		t.Fatalf("rollback vars target the wrong instance: %#v", vars)
+	}
+	if vars["rollback"] != true {
+		t.Fatalf("rollback marker missing: %#v", vars)
+	}
+}
+
+func TestRegisterFailedIsProtectedFromRollback(t *testing.T) {
+	if !rollbackProtectedStatus(model.TaskRegisterFailed) {
+		t.Fatal("register_failed must preserve the healthy instance and resource usage")
+	}
+	if rollbackProtectedStatus(model.TaskValidationFailed) {
+		t.Fatal("validation_failed must still be eligible for cleanup rollback")
+	}
+}
+
+func TestRollbackLaunchExpired(t *testing.T) {
+	now := time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC)
+	started := now.Add(-rollbackLaunchTimeout - time.Second)
+	if !rollbackLaunchExpired(model.RollbackJob{Status: "launching", StartedAt: &started}, now) {
+		t.Fatal("stale launching rollback job must be recoverable")
+	}
+	if rollbackLaunchExpired(model.RollbackJob{Status: "launching", StartedAt: ptrTime(now.Add(-rollbackLaunchTimeout + time.Second))}, now) {
+		t.Fatal("recent launching rollback job must remain pending")
+	}
+	if rollbackLaunchExpired(model.RollbackJob{Status: "running", StartedAt: &started}, now) {
+		t.Fatal("running rollback job is not a launch timeout")
+	}
+}
+
+func ptrTime(v time.Time) *time.Time { return &v }