From c26790f84b6fc73711809355f73a390972be98bf Mon Sep 17 00:00:00 2001 From: hezhaohui Date: Tue, 28 Jul 2026 11:05:57 +0800 Subject: [PATCH 1/6] =?UTF-8?q?docs(ansible):=20=E4=B8=BA=20mysql-deploy?= =?UTF-8?q?=20=E7=9B=B8=E5=85=B3=E6=96=87=E4=BB=B6=E6=B7=BB=E5=8A=A0?= =?UTF-8?q?=E8=AF=A6=E7=BB=86=E4=B8=AD=E6=96=87=E6=B3=A8=E9=87=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - mysql-deploy.yml: 为所有变量、任务、handlers 添加中文注释,解释参数含义、设计逻辑和平台契约 - mysql-delivery@.service: 为 Systemd 模板单元添加中文注释,说明 %i 参数机制、Type=notify、OOMScoreAdjust 等配置项 - mysql-instance.cnf.j2: 为 MySQL 配置模板添加中文注释,说明各参数含义、版本差异、复制配置逻辑 - inventory.example.yml: 为 Inventory 示例添加中文注释,说明主机组与拓扑对应关系及使用方式 --- ansible/files/mysql-delivery@.service | 33 +++ ansible/inventory.example.yml | 28 +- ansible/mysql-deploy.yml | 333 +++++++++++++++--------- ansible/templates/mysql-instance.cnf.j2 | 138 +++++++--- 4 files changed, 365 insertions(+), 167 deletions(-) diff --git a/ansible/files/mysql-delivery@.service b/ansible/files/mysql-delivery@.service index 431cb4a..2229074 100644 --- a/ansible/files/mysql-delivery@.service +++ b/ansible/files/mysql-delivery@.service @@ -1,20 +1,53 @@ +# ============================================================================= +# XINFRA MySQL 交付实例 Systemd 模板单元 +# ============================================================================= +# 这是 systemd 的模板单元(template unit),文件名中的 @ 是模板标识 +# 使用方式:systemctl start mysql-delivery@{实例名}.service +# 例如:mysql-delivery@db01.service、mysql-delivery@db02.service +# +# %i 参数会被 systemd 自动替换为 @ 后面的实例名 +# 这样同一台主机可以运行多个 MySQL 实例,彼此隔离 +# ============================================================================= + [Unit] +# 服务描述,%i 会被替换为实际实例名 Description=XINFRA MySQL delivery instance %i +# 依赖:等待网络在线后再启动(MySQL 需要绑定网络端口) After=network-online.target Wants=network-online.target [Service] +# Type=notify:mysqld 启动完成后会通过 sd_notify 通知 systemd +# 比 Type=simple 更可靠,systemd 能准确知道服务何时真正就绪 Type=notify + +# 以 mysql 用户/组身份运行(最小权限原则) User=mysql Group=mysql + +# 创建运行时目录 /run/mysql-delivery-{实例名}/ +# 用于存放 socket 和 pid 文件,放在 tmpfs(内存文件系统)上,重启自动清理 RuntimeDirectory=mysql-delivery-%i RuntimeDirectoryMode=0755 + +# 启动命令:使用实例专属配置文件 +# %i 被替换为实例名,指向 /etc/mysql/mysql-delivery/{实例名}.cnf ExecStart=/usr/sbin/mysqld --defaults-file=/etc/mysql/mysql-delivery/%i.cnf + +# 故障自动重启:崩溃后 5 秒自动重启 Restart=on-failure RestartSec=5s + +# 启动超时:120 秒内未完成启动则判定为失败 TimeoutStartSec=120s + +# 最大打开文件数:MySQL 需要大量文件描述符(连接、临时表等) LimitNOFILE=65535 + +# OOM 评分调整:-200 表示被 OOM Killer 杀掉的优先级较低 +# MySQL 是数据库服务,被误杀影响较大,尽量保护 OOMScoreAdjust=-200 [Install] +# 开机自启:加入 multi-user.target(多用户模式,即正常启动完成后) WantedBy=multi-user.target diff --git a/ansible/inventory.example.yml b/ansible/inventory.example.yml index c7f5795..441cc86 100644 --- a/ansible/inventory.example.yml +++ b/ansible/inventory.example.yml @@ -1,11 +1,35 @@ +# ============================================================================= +# Ansible Inventory 示例文件 +# ============================================================================= +# Inventory 定义了要管理的主机列表和分组 +# 实际使用时,平台会动态生成 inventory 并传入变量 +# +# 使用方式: +# ansible-playbook -i inventory.example.yml mysql-deploy.yml \ +# -e "target_hosts=mysql_hosts" \ +# -e "instance_name=db01" \ +# -e "topology=mgr_3" \ +# -e "mysql_version=8.0" \ +# -e "memory_mb=8192" \ +# -e "storage_gb=100" +# ============================================================================= + all: children: + # --- MySQL 主机组 --- + # 按拓扑需求选择对应数量的主机: + # standalone = 1 台 + # primary_replica = 2 台(一主一从) + # mgr_3 = 3 台(三节点组复制) mysql_hosts: hosts: + # 格式:主机名: { ansible_host: IP地址 } k8s-server-01: { ansible_host: 192.168.1.4 } k8s-server-02: { ansible_host: 192.168.1.5 } k8s-server-03: { ansible_host: 192.168.1.2 } k8s-worker-01: { ansible_host: 192.168.1.3 } + + # --- 全局变量 --- vars: - ansible_user: root - ansible_python_interpreter: /usr/bin/python3 + ansible_user: root # SSH 登录用户(需要 root 权限执行系统级操作) + ansible_python_interpreter: /usr/bin/python3 # Python 解释器路径(Ubuntu 默认位置) diff --git a/ansible/mysql-deploy.yml b/ansible/mysql-deploy.yml index 4bbfb2d..03184e7 100644 --- a/ansible/mysql-deploy.yml +++ b/ansible/mysql-deploy.yml @@ -1,128 +1,156 @@ --- +# ============================================================================= +# XINFRA MySQL 原生交付 Playbook +# ============================================================================= +# 功能:在目标主机上从零开始安装、配置并启动一个 MySQL 实例 +# 支持拓扑:standalone(单机)、primary_replica(一主一从)、mgr_3(三节点组复制) +# 支持版本:8.0(Ubuntu 自带源锁版本)、8.4(MySQL 官方 APT 源 LTS 组件) +# 设计特点: +# - 同一主机可部署多个实例(systemd 模板单元 + 独立配置/数据目录) +# - 挂载点无关:数据目录统一在 {data_disk}/mysql-delivery/{instance_id}/ 下 +# - 幂等性:数据目录初始化有 creates 守护,配置变更触发 notify: Restart +# ============================================================================= - name: Native MySQL delivery hosts: "{{ target_hosts }}" - become: true - gather_facts: true - any_errors_fatal: true + become: true # 提权执行 + gather_facts: true # 收集目标主机信息(用于后续 ansible_mounts、ansible_distribution_release 等) + any_errors_fatal: true # 任一主机失败则整个 play 中止(避免部分主机成功、部分失败的不一致状态) vars: - # --- identity --- - mysql_instance: "{{ instance_name }}" - mysql_version_value: "{{ mysql_version | default('8.0') }}" - # 版本包映射:8.0 用 Ubuntu 24.04 自带源(精确锁版);8.4 用 MySQL 官方 APT 源的 LTS 组件 - # (noble 自带源无 8.4,官方源组件内即为该系列,不再锁小版本)。 - # 5.6/5.7 已官方 EOL 且 Ubuntu 24.04 无可用 apt 包,明确不纳入白名单。 + # ==================== 基础身份信息 ==================== + mysql_instance: "{{ instance_name }}" # 实例唯一标识,用于目录/配置/systemd 服务名 + mysql_version_value: "{{ mysql_version | default('8.0') }}" # MySQL 版本,默认 8.0 + + # ==================== 版本包映射 ==================== + # 8.0 使用 Ubuntu 24.04 自带源,精确锁版本(确保可复现) + # 8.4 使用 MySQL 官方 APT 源的 mysql-8.4-lts 组件(noble 自带源无 8.4) + # 5.6/5.7 已官方 EOL 且 Ubuntu 24.04 无可用 apt 包,不纳入白名单 mysql_package_map: "8.0": - package: "mysql-server=8.0.46-0ubuntu0.24.04.3" - repo_component: "" + package: "mysql-server=8.0.46-0ubuntu0.24.04.3" # 精确锁版本 + repo_component: "" # 无需额外 APT 源 "8.4": - package: "mysql-community-server" - repo_component: "mysql-8.4-lts" + package: "mysql-community-server" # 官方源组件内版本号 + repo_component: "mysql-8.4-lts" # MySQL 官方源组件名 mysql_package_name: "{{ (mysql_package_map[mysql_version_value] | default({})).package | default('') }}" mysql_repo_component: "{{ (mysql_package_map[mysql_version_value] | default({})).repo_component | default('') }}" - # --- platform-allocated inputs (safe fallbacks when not passed in) --- - # 端口池 13306–13999:平台从池分配并传入;未传时兜底池首端口,占用探测在目标机执行。 + # ==================== 平台分配的输入参数(带安全兜底) ==================== + # 端口池 13306–13999:平台从池分配并传入;未传时兜底池首端口 mysql_port_value: "{{ mysql_port | default(13306) | int }}" - # GR 组通信端口:仅 mgr_3 使用,平台成对分配;兜底为 SQL 端口 +10000。 + # GR 组通信端口:仅 mgr_3 拓扑使用,平台成对分配;兜底为 SQL 端口 +10000 mysql_gr_port_value: "{{ gr_port | default((mysql_port | default(13306) | int) + 10000) | int }}" - # 数据盘挂载点:平台按白名单选定并传入;兜底 /data。 + # 数据盘挂载点:平台按白名单选定并传入;兜底 /data mysql_data_disk: "{{ data_disk | default('/data') }}" - # --- resource quota (Go→AWX 契约单位保持 MB/GB,不擅改) --- - mysql_memory_mb_value: "{{ memory_mb | default(4096) | int }}" - mysql_storage_gb_value: "{{ storage_gb | default(50) | int }}" + # ==================== 资源配额 ==================== + # 单位保持 MB/GB(与平台侧 Go→AWX 契约一致,不擅自修改) + mysql_memory_mb_value: "{{ memory_mb | default(4096) | int }}" # 内存配额,默认 4G + mysql_storage_gb_value: "{{ storage_gb | default(50) | int }}" # 存储配额,默认 50G - # --- mount-point-agnostic layout: {data_disk}/mysql-delivery/{instance_id}/... --- - mysql_base_dir: "{{ mysql_data_disk }}/mysql-delivery/{{ mysql_instance }}" - mysql_install_dir: "/opt/mysql-delivery/{{ mysql_instance }}" - mysql_data_dir: "{{ mysql_base_dir }}/data" - mysql_log_dir: "{{ mysql_base_dir }}/logs" - mysql_binlog_dir: "{{ mysql_base_dir }}/logs/binlog" - mysql_redo_dir: "{{ mysql_base_dir }}/logs/redo" - mysql_tmp_dir: "{{ mysql_base_dir }}/tmp" - # socket/pid 走 /run tmpfs(重启自动清理),由 systemd RuntimeDirectory 创建。 + # ==================== 挂载点无关的目录布局 ==================== + # 所有实例的数据/日志/临时文件统一放在 {data_disk}/mysql-delivery/{instance_id}/ 下 + # 这样无论数据盘挂载在 /data、/mnt/ssd 还是其他路径,结构都一致 + mysql_base_dir: "{{ mysql_data_disk }}/mysql-delivery/{{ mysql_instance }}" # 数据根目录 + mysql_install_dir: "/opt/mysql-delivery/{{ mysql_instance }}" # 实例安装目录(二进制符号链接) + mysql_data_dir: "{{ mysql_base_dir }}/data" # MySQL 数据目录 + mysql_log_dir: "{{ mysql_base_dir }}/logs" # 日志根目录 + mysql_binlog_dir: "{{ mysql_base_dir }}/logs/binlog" # binlog 和 relay log 目录 + mysql_redo_dir: "{{ mysql_base_dir }}/logs/redo" # InnoDB redo log 目录 + mysql_tmp_dir: "{{ mysql_base_dir }}/tmp" # 临时文件目录 + # socket/pid 放在 /run tmpfs(重启自动清理),由 systemd RuntimeDirectory 自动创建 mysql_run_dir: "/run/mysql-delivery-{{ mysql_instance }}" - mysql_config_file: "/etc/mysql/mysql-delivery/{{ mysql_instance }}.cnf" + mysql_config_file: "/etc/mysql/mysql-delivery/{{ mysql_instance }}.cnf" # 配置文件路径 - # --- database config (user form, with doc default baselines) --- - mysql_timezone: "{{ timezone | default('+08:00') }}" - mysql_lower_case_table_names: "{{ lower_case_table_names | default(1) | int }}" - mysql_character_set: "{{ character_set | default('utf8mb4') }}" - mysql_collation: "{{ collation | default('utf8mb4_general_ci') }}" + # ==================== 数据库基础配置 ==================== + mysql_timezone: "{{ timezone | default('+08:00') }}" # 时区,默认东八区 + mysql_lower_case_table_names: "{{ lower_case_table_names | default(1) | int }}" # 表名大小写敏感,1=不敏感 + mysql_character_set: "{{ character_set | default('utf8mb4') }}" # 字符集,默认 utf8mb4 + mysql_collation: "{{ collation | default('utf8mb4_general_ci') }}" # 排序规则 - # --- advanced params (overridable; default baseline assumes SSD for io_capacity) --- - mysql_flush_log_at_trx_commit: "{{ innodb_flush_log_at_trx_commit | default(1) | int }}" - mysql_sync_binlog: "{{ sync_binlog | default(1) | int }}" - mysql_io_capacity: "{{ innodb_io_capacity | default(2000) | int }}" - mysql_long_query_time: "{{ long_query_time | default(1) }}" - mysql_binlog_expire_logs_seconds: "{{ binlog_expire_logs_seconds | default(604800) | int }}" - mysql_max_binlog_size: "{{ max_binlog_size | default('256M') }}" - mgr_group_name: "{{ group_replication_group_name | default('aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa') }}" + # ==================== 高级参数(可覆盖,默认假设 SSD) ==================== + mysql_flush_log_at_trx_commit: "{{ innodb_flush_log_at_trx_commit | default(1) | int }}" # 事务提交时刷日志,1=最安全 + mysql_sync_binlog: "{{ sync_binlog | default(1) | int }}" # 每次提交同步 binlog,1=最安全 + mysql_io_capacity: "{{ innodb_io_capacity | default(2000) | int }}" # InnoDB I/O 能力,默认 2000(SSD 适合) + mysql_long_query_time: "{{ long_query_time | default(1) }}" # 慢查询阈值,单位秒 + mysql_binlog_expire_logs_seconds: "{{ binlog_expire_logs_seconds | default(604800) | int }}" # binlog 过期时间,默认 7 天 + mysql_max_binlog_size: "{{ max_binlog_size | default('256M') }}" # 单个 binlog 文件最大大小 + mgr_group_name: "{{ group_replication_group_name | default('aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa') }}" # GR 组名(UUID 格式) - # --- expected node count per topology --- + # ==================== 拓扑节点数 ==================== + # standalone=1、primary_replica=2、mgr_3=3,用于校验 inventory 中的主机数 mysql_expected_hosts: "{{ {'standalone': 1, 'primary_replica': 2, 'mgr_3': 3}[topology | default('standalone')] }}" - # --- secrets (injected via environment) --- + # ==================== 敏感信息(通过环境变量注入) ==================== + # 密码通过环境变量 XINFRA_MYSQL_ROOT_PASSWORD / XINFRA_MYSQL_ADMIN_PASSWORD 传入 + # 不写入 playbook 或 inventory,避免泄露 mysql_root_password_value: "{{ lookup('ansible.builtin.env', 'XINFRA_MYSQL_ROOT_PASSWORD') }}" mysql_admin_password_value: "{{ lookup('ansible.builtin.env', 'XINFRA_MYSQL_ADMIN_PASSWORD') }}" + # ==================== 前置校验任务 ==================== + # 在正式安装前执行参数校验和资源检查,尽早失败(fail-fast) pre_tasks: + # --- 校验所有交付参数是否在白名单范围内 --- - name: Validate delivery parameters ansible.builtin.assert: that: - - topology in ['standalone', 'primary_replica', 'mgr_3'] - - mysql_instance is match('^[a-z0-9][a-z0-9-]{0,62}$') - - mysql_version_value in mysql_package_map - - mysql_data_disk is match('^/') - - (mysql_port_value | int) >= 13306 - - (mysql_port_value | int) <= 13999 - - (mysql_memory_mb_value | int) >= 2048 - - (mysql_memory_mb_value | int) <= 65536 - - (mysql_storage_gb_value | int) >= 20 - - (mysql_storage_gb_value | int) <= 2000 - - (mysql_lower_case_table_names | int) in [0, 1] - - mysql_root_password_value | length >= 16 - - mysql_admin_password_value | length >= 16 + - topology in ['standalone', 'primary_replica', 'mgr_3'] # 拓扑必须是三种之一 + - mysql_instance is match('^[a-z0-9][a-z0-9-]{0,62}$') # 实例名:小写字母/数字开头,仅含小写字母/数字/短横线,最长63 + - mysql_version_value in mysql_package_map # 版本必须在包映射表中 + - mysql_data_disk is match('^/') # 数据盘必须是绝对路径 + - (mysql_port_value | int) >= 13306 # 端口下限 + - (mysql_port_value | int) <= 13999 # 端口上限(端口池范围) + - (mysql_memory_mb_value | int) >= 2048 # 最小 2G 内存 + - (mysql_memory_mb_value | int) <= 65536 # 最大 64G 内存 + - (mysql_storage_gb_value | int) >= 20 # 最小 20G 存储 + - (mysql_storage_gb_value | int) <= 2000 # 最大 2T 存储 + - (mysql_lower_case_table_names | int) in [0, 1] # 只能是 0 或 1 + - mysql_root_password_value | length >= 16 # root 密码长度 ≥ 16 + - mysql_admin_password_value | length >= 16 # admin 密码长度 ≥ 16 fail_msg: >- - Delivery parameters out of the supported target-state whitelist - (topology / version-package-map / port pool 13306-13999 / memory 2-64G / storage 20-2000G). - quiet: true - no_log: true + 交付参数超出目标状态白名单(拓扑/版本包映射/端口池13306-13999/内存2-64G/存储20-2000G) + quiet: true # 静默模式,不打印整个 that 列表 + no_log: true # 避免密码等敏感信息出现在日志中 + # --- 校验拓扑要求的主机数与实际 inventory 主机数一致 --- - name: Validate topology host count ansible.builtin.assert: that: - (ansible_play_hosts_all | length | int) == (mysql_expected_hosts | int) - fail_msg: "topology={{ topology }} expects {{ mysql_expected_hosts }} host(s), got {{ ansible_play_hosts_all | length }}" - run_once: true + fail_msg: "拓扑={{ topology }} 期望 {{ mysql_expected_hosts }} 台主机,实际 {{ ansible_play_hosts_all | length }} 台" + run_once: true # 只在第一台主机执行(全局校验) + # --- 探测目标主机的端口占用情况(SQL 端口 + GR 端口) --- + # 使用 ss 命令检查端口是否已被监听;如果已被占用,检查是否属于当前实例(允许幂等重跑) - name: Probe target-host port occupancy (SQL + GR) - # cmd 字典形式不经过 free-form split_args 解析,避免引号/Jinja 块导致的解析失败。 + # 注意:使用 cmd 字典形式而非自由形式,避免 heredoc 中的引号/Jinja 块被 split_args 误解析 ansible.builtin.shell: cmd: | set -o pipefail for p in {{ mysql_probe_ports | join(' ') }}; do if ss -lntH "sport = :${p}" | grep -q .; then - # already listening: only tolerated when owned by this instance service + # 端口已被占用,但如果是当前实例的服务在用则允许(幂等重跑场景) if ! systemctl is-active --quiet "mysql-delivery@{{ mysql_instance }}.service"; then - echo "port ${p} already in use on target host" >&2 + echo "端口 ${p} 在目标主机已被其他服务占用" >&2 exit 3 fi fi done executable: /bin/bash vars: + # mgr_3 拓扑需要探测两个端口(SQL + GR),其他拓扑只探测 SQL 端口 mysql_probe_ports: "{{ [mysql_port_value, mysql_gr_port_value] if topology == 'mgr_3' else [mysql_port_value] }}" - changed_when: false + changed_when: false # 只读操作,不产生变更 + # --- 检查目标主机当前可用内存 --- - name: Read currently available memory (point-in-time guard) ansible.builtin.shell: awk '/^MemAvailable:/ { print int($2 / 1024) }' /proc/meminfo args: executable: /bin/bash register: mysql_available_memory - changed_when: false + changed_when: false # 只读操作 + # --- 获取数据盘挂载点的可用空间 --- + # 优先使用 mysql_data_disk 挂载点的空间,如果找不到则回退到根分区 / - name: Resolve data-disk mountpoint available space ansible.builtin.set_fact: mysql_mount_avail: >- @@ -131,94 +159,110 @@ | default(ansible_mounts | selectattr('mount', 'equalto', '/') | map(attribute='size_available') | first) }} + # --- 最终检查:可用内存和磁盘空间是否满足配额 --- - name: Check available memory and data-disk space ansible.builtin.assert: that: - - (mysql_available_memory.stdout | int) >= (mysql_memory_mb_value | int) - - (mysql_mount_avail | int) >= (mysql_storage_gb_value | int) * 1073741824 + - (mysql_available_memory.stdout | int) >= (mysql_memory_mb_value | int) # 可用内存 ≥ 配额 + - (mysql_mount_avail | int) >= (mysql_storage_gb_value | int) * 1073741824 # 可用空间 ≥ 配额(GB→字节) fail_msg: >- - Target host lacks memory or free space on {{ mysql_data_disk }}; - host-wide Σ-quota budgeting is the platform's responsibility, this is a last-resort guard. + 目标主机内存或 {{ mysql_data_disk }} 磁盘空间不足; + 主机级配额预算是平台侧职责,这里是最后的安全兜底。 + # ==================== 主任务 ==================== tasks: + # --- 检测主机上是否已安装 MySQL --- - name: Detect an existing MySQL server package ansible.builtin.command: dpkg-query -W -f '${Package}=${Version}\n' mysql-server mysql-community-server register: mysql_package_before - failed_when: false - changed_when: false + failed_when: false # 未安装时不报错 + changed_when: false # 只读操作 + # --- 提取已安装的 MySQL 版本号 --- - name: Resolve the installed MySQL server version ansible.builtin.set_fact: mysql_installed_version: >- {{ (mysql_package_before.stdout_lines | select('search', '=') | list | first | default('')).split('=') | last }} - # /usr 下的 mysqld/mysql 二进制全机共享,一台主机只能承载一个 MySQL 版本系列。 + # --- 强制执行主机级 MySQL 版本系列一致性 --- + # /usr/sbin/mysqld 等二进制是全机共享的,一台主机只能运行一个 MySQL 版本系列 + # 如果已安装 8.0.x,不能再安装 8.4.y - name: Enforce host-level MySQL series consistency ansible.builtin.assert: that: - mysql_installed_version == '' or mysql_installed_version.startswith(mysql_version_value ~ '.') fail_msg: >- - Host already runs MySQL {{ mysql_installed_version }} but {{ mysql_version_value }} was requested; - native binaries under /usr are shared host-wide, so one host serves exactly one MySQL series. + 主机已安装 MySQL {{ mysql_installed_version }},但请求的是 {{ mysql_version_value }}; + /usr 下的二进制是全机共享的,一台主机只能承载一个 MySQL 版本系列。 + # ==================== 安装 MySQL 软件包 ==================== + # 仅 8.4 需要添加官方 APT 源(8.0 使用 Ubuntu 自带源,无需额外配置) - name: Install the MySQL APT repository signing key ansible.builtin.get_url: - url: https://repo.mysql.com/RPM-GPG-KEY-mysql-2023 + url: https://repo.mysql.com/RPM-GPG-KEY-mysql-2023 # MySQL 官方 GPG 签名密钥 dest: /etc/apt/keyrings/mysql.asc owner: root group: root mode: '0644' - when: mysql_repo_component != '' + when: mysql_repo_component != '' # 仅 8.4 需要 - name: Configure the MySQL APT repository component ansible.builtin.apt_repository: repo: >- deb [signed-by=/etc/apt/keyrings/mysql.asc] http://repo.mysql.com/apt/ubuntu {{ ansible_distribution_release }} {{ mysql_repo_component }} - filename: xinfra-mysql-delivery + filename: xinfra-mysql-delivery # 自定义源文件名,避免与已有源冲突 state: present - when: mysql_repo_component != '' + when: mysql_repo_component != '' # 仅 8.4 需要 + # 安装 MySQL 服务端包(apt 会自动安装依赖如 mysql-client 等) - name: Install the MySQL server package ansible.builtin.apt: name: "{{ mysql_package_name }}" state: present - update_cache: true - cache_valid_time: 3600 + update_cache: true # 安装前更新 apt 缓存 + cache_valid_time: 3600 # 缓存有效期 1 小时 + # apt 安装 MySQL 后会自动创建并启动一个默认的 mysql.service + # 我们使用自定义的 systemd 模板单元管理实例,所以需要停止并禁用默认实例 - name: Stop the automatically created default instance on a clean host ansible.builtin.systemd_service: name: mysql.service state: stopped enabled: false - when: mysql_installed_version == '' + when: mysql_installed_version == '' # 仅在全新安装时执行 + # ==================== AppArmor 配置 ==================== + # MySQL 的 AppArmor 配置文件限制了 mysqld 只能访问特定目录 + # 我们使用自定义路径,需要授权 delivery 目录的访问权限 - name: Check for the bundled MySQL AppArmor profile ansible.builtin.stat: - path: /etc/apparmor.d/usr.sbin.mysqld + path: /etc/apparmor.d/usr.sbin.mysqld # 检查 AppArmor 配置文件是否存在 register: mysql_apparmor_profile - name: Authorize the delivery paths in the MySQL AppArmor profile ansible.builtin.copy: - dest: /etc/apparmor.d/local/usr.sbin.mysqld + dest: /etc/apparmor.d/local/usr.sbin.mysqld # local/ 目录下的规则会追加到主配置 owner: root group: root mode: '0644' content: | - # Managed by XINFRA MySQL delivery - grant per-instance native paths - {{ mysql_data_disk }}/mysql-delivery/ r, - {{ mysql_data_disk }}/mysql-delivery/** rwk, - /run/mysql-delivery-*/ rw, - /run/mysql-delivery-*/** rwk, + # XINFRA MySQL delivery 管理 - 授权实例原生路径访问 + {{ mysql_data_disk }}/mysql-delivery/ r, # 允许读取 delivery 根目录 + {{ mysql_data_disk }}/mysql-delivery/** rwk, # 允许读写创建 delivery 子目录下所有内容 + /run/mysql-delivery-*/ rw, # 允许读取运行时目录 + /run/mysql-delivery-*/** rwk, # 允许读写运行时目录下所有内容(socket/pid) when: mysql_apparmor_profile.stat.exists register: mysql_apparmor_local - name: Reload the MySQL AppArmor profile - ansible.builtin.command: apparmor_parser -r /etc/apparmor.d/usr.sbin.mysqld - when: mysql_apparmor_profile.stat.exists and mysql_apparmor_local.changed + ansible.builtin.command: apparmor_parser -r /etc/apparmor.d/usr.sbin.mysqld # 重新加载 AppArmor 配置 + when: mysql_apparmor_profile.stat.exists and mysql_apparmor_local.changed # 仅配置变更时重载 changed_when: true + # ==================== 创建实例目录结构 ==================== + # 所有实例的数据/日志/配置文件等放在统一的目录布局中 - name: Create instance directories (mount-point-agnostic layout) ansible.builtin.file: path: "{{ item.path }}" @@ -227,44 +271,57 @@ group: "{{ item.group }}" mode: "{{ item.mode }}" loop: - - { path: /etc/mysql/mysql-delivery, owner: root, group: mysql, mode: '0750' } - - { path: "{{ mysql_install_dir }}", owner: root, group: root, mode: '0755' } - - { path: "{{ mysql_base_dir }}", owner: mysql, group: mysql, mode: '0750' } - - { path: "{{ mysql_data_dir }}", owner: mysql, group: mysql, mode: '0750' } - - { path: "{{ mysql_log_dir }}", owner: mysql, group: mysql, mode: '0750' } - - { path: "{{ mysql_binlog_dir }}", owner: mysql, group: mysql, mode: '0750' } - - { path: "{{ mysql_redo_dir }}", owner: mysql, group: mysql, mode: '0750' } - - { path: "{{ mysql_tmp_dir }}", owner: mysql, group: mysql, mode: '0750' } + - { path: /etc/mysql/mysql-delivery, owner: root, group: mysql, mode: '0750' } # 配置文件父目录 + - { path: "{{ mysql_install_dir }}", owner: root, group: root, mode: '0755' } # 实例安装目录(二进制链接) + - { path: "{{ mysql_base_dir }}", owner: mysql, group: mysql, mode: '0750' } # 数据根目录 + - { path: "{{ mysql_data_dir }}", owner: mysql, group: mysql, mode: '0750' } # MySQL 数据目录 + - { path: "{{ mysql_log_dir }}", owner: mysql, group: mysql, mode: '0750' } # 日志目录 + - { path: "{{ mysql_binlog_dir }}", owner: mysql, group: mysql, mode: '0750' } # binlog 目录 + - { path: "{{ mysql_redo_dir }}", owner: mysql, group: mysql, mode: '0750' } # redo log 目录 + - { path: "{{ mysql_tmp_dir }}", owner: mysql, group: mysql, mode: '0750' } # 临时文件目录 + # --- 在实例目录中创建指向系统二进制的符号链接 --- + # 虽然二进制在 /usr 下,但在实例目录创建链接方便管理,也便于未来版本隔离 - name: Link native binaries into the instance directory ansible.builtin.file: src: "{{ item.src }}" dest: "{{ mysql_install_dir }}/{{ item.dest }}" state: link loop: - - { src: /usr/sbin/mysqld, dest: mysqld } - - { src: /usr/bin/mysql, dest: mysql } - - { src: /usr/bin/mysqladmin, dest: mysqladmin } + - { src: /usr/sbin/mysqld, dest: mysqld } # mysqld 守护进程 + - { src: /usr/bin/mysql, dest: mysql } # MySQL 客户端 + - { src: /usr/bin/mysqladmin, dest: mysqladmin } # MySQL 管理工具 + # ==================== 计算节点角色和 server-id ==================== - name: Calculate host role and server id ansible.builtin.set_fact: + # 节点索引:在 inventory 中的位置(从 0 开始) mysql_node_index: "{{ ansible_play_hosts_all.index(inventory_hostname) }}" + # 节点角色: + # standalone 拓扑 → standalone + # primary_replica 拓扑 → 第一个节点为 primary,其余为 replica + # mgr_3 拓扑 → 第一个节点为 primary(用于初始引导),其余为 mgr mysql_node_role: >- {{ 'standalone' if topology == 'standalone' else ('primary' if ansible_play_hosts_all.index(inventory_hostname) == 0 else ('replica' if topology == 'primary_replica' else 'mgr')) }} - # host-wide unique: platform may pass explicit mysql_server_id; otherwise derive - # port + node index (ports are unique per host in the pool model). + # server-id 必须全机唯一,用于复制标识 + # 平台可显式传入 mysql_server_id;否则用 port + node_index 派生(端口池模型保证端口唯一) mysql_server_id_value: >- {{ mysql_server_id | default((mysql_port_value | int) + (ansible_play_hosts_all.index(inventory_hostname))) | int }} + # --- 将内存从 MB 转换为 GB,用于后续梯度计算 --- - name: Resolve memory tier (GB) ansible.builtin.set_fact: mysql_memory_gb: "{{ ((mysql_memory_mb_value | int) // 1024) | int }}" + # --- 根据内存梯度计算 max_connections 和 redo log 容量 --- + # 这些是合理的默认值,可根据实际硬件校准调整 - name: Resolve linkage-derived defaults (illustrative tiers, pending hardware calibration) ansible.builtin.set_fact: + # 最大连接数:按内存梯度递增 + # 2G→200、4G→500、8G→1000、16G→2000、32G→4000、64G→8000、>64G→16000 mysql_max_connections: >- {{ (max_connections | int) if (max_connections is defined and (max_connections | string) != 'auto') else (200 if (mysql_memory_gb | int) <= 2 @@ -274,6 +331,8 @@ else 4000 if (mysql_memory_gb | int) <= 32 else 8000 if (mysql_memory_gb | int) <= 64 else 16000) }} + # InnoDB redo log 容量:按内存梯度递增 + # ≤4G→128M、≤16G→256M、≤32G→512M、>32G→1G mysql_redo_capacity: >- {{ innodb_redo_log_capacity if (innodb_redo_log_capacity is defined) else ('128M' if (mysql_memory_gb | int) <= 4 @@ -281,26 +340,34 @@ else '512M' if (mysql_memory_gb | int) <= 32 else '1G') }} + # ==================== 生成 MySQL 配置文件 ==================== + # 使用 Jinja2 模板渲染实例配置,写入后通知 handler 重启实例 - name: Write instance configuration ansible.builtin.template: src: templates/mysql-instance.cnf.j2 dest: "{{ mysql_config_file }}" owner: root group: mysql - mode: '0640' - notify: Restart MySQL delivery instance + mode: '0640' # 仅 root 可读,mysql 组可读 + notify: Restart MySQL delivery instance # 配置变更时触发重启 + # ==================== 初始化数据目录 ==================== + # 使用 mysqld --initialize-insecure 初始化(无密码),后续通过 SQL 设置密码 + # 幂等:如果 data 目录下已有 auto.cnf,说明已初始化过,跳过 - name: Initialize the data directory once ansible.builtin.command: argv: - /usr/sbin/mysqld - "--defaults-file={{ mysql_config_file }}" - - --initialize-insecure + - --initialize-insecure # 初始无密码,后续设置 - --user=mysql args: - creates: "{{ mysql_data_dir }}/auto.cnf" - no_log: true + creates: "{{ mysql_data_dir }}/auto.cnf" # 幂等守卫:文件存在则跳过 + no_log: true # 避免初始化日志泄露敏感信息 + # ==================== 安装 Systemd 模板单元 ==================== + # mysql-delivery@.service 是模板单元,%i 被替换为实例名 + # 支持同一主机运行多个 MySQL 实例(如 mysql-delivery@db01、mysql-delivery@db02) - name: Install the delivery systemd template ansible.builtin.copy: src: files/mysql-delivery@.service @@ -310,43 +377,57 @@ mode: '0644' register: mysql_systemd_unit + # 重载 systemd 以识别新安装的模板单元 - name: Reload systemd units ansible.builtin.systemd_service: daemon_reload: true when: mysql_systemd_unit.changed + # ==================== 启动 MySQL 实例 ==================== - name: Start the MySQL delivery instance ansible.builtin.systemd_service: name: "mysql-delivery@{{ mysql_instance }}.service" state: started - enabled: true + enabled: true # 设置开机自启 + # 等待 MySQL socket 文件出现(表示实例已启动完成) - name: Wait for the local MySQL socket ansible.builtin.wait_for: path: "{{ mysql_run_dir }}/mysql.sock" - timeout: 60 + timeout: 60 # 最长等待 60 秒 + # ==================== 配置管理账号 ==================== + # 通过 shell 脚本完成: + # 1. 如果 root 未设密码(initialize-insecure 初始状态),通过 socket 本地连接设置密码 + # 2. 创建 xinfra_admin 管理账号,授予所有权限 - name: Configure local administrative accounts - # cmd 字典形式不经过 free-form split_args 解析,heredoc SQL 中的奇数个单引号才不会报错。 + # 注意:使用 cmd 字典形式而非自由形式,heredoc SQL 中的奇数单引号不会被 split_args 误解析 ansible.builtin.shell: cmd: | set -euo pipefail + # 创建临时客户端配置文件和 SQL 文件,退出时自动清理 client_file="$(mktemp)" sql_file="$(mktemp)" trap 'rm -f "$client_file" "$sql_file"' EXIT chmod 600 "$client_file" "$sql_file" + + # 写入客户端连接配置(使用 socket 连接,不走 TCP) cat >"$client_file" <<'EOF' [client] user=root password={{ mysql_root_password_value }} socket={{ mysql_run_dir }}/mysql.sock EOF + + # 尝试用配置的密码连接,如果失败说明 root 还没设密码,需要设置 if ! /usr/bin/mysql --defaults-extra-file="$client_file" -e 'SELECT 1' >/dev/null 2>&1; then cat >"$sql_file" <<'EOF' ALTER USER 'root'@'localhost' IDENTIFIED BY '{{ mysql_root_password_value | replace("'", "''") }}'; EOF /usr/bin/mysql --protocol=socket --socket={{ mysql_run_dir }}/mysql.sock -uroot <"$sql_file" fi + + # 创建/更新 xinfra_admin 管理账号(允许从任意主机连接,拥有所有权限) cat >"$sql_file" <<'EOF' CREATE USER IF NOT EXISTS 'xinfra_admin'@'%' IDENTIFIED BY '{{ mysql_admin_password_value | replace("'", "''") }}'; ALTER USER 'xinfra_admin'@'%' IDENTIFIED BY '{{ mysql_admin_password_value | replace("'", "''") }}'; @@ -355,24 +436,32 @@ EOF /usr/bin/mysql --defaults-extra-file="$client_file" <"$sql_file" executable: /bin/bash - changed_when: false - no_log: true + changed_when: false # 已有账号时不产生变更 + no_log: true # 避免密码出现在日志中 + # ==================== 健康检查 ==================== + # 验证 MySQL TCP 端口可连通,确保实例正常对外服务 - name: Verify MySQL TCP health ansible.builtin.wait_for: host: "{{ ansible_host | default(inventory_hostname) }}" port: "{{ mysql_port_value }}" - timeout: 30 + timeout: 30 # 最长等待 30 秒 + # --- HA 拓扑提示 --- + # 非 standalone 拓扑:配置已就绪(GTID/binlog/relay/GR 参数已写入), + # 但复制接线(CHANGE REPLICATION SOURCE)和组复制引导(START GROUP_REPLICATION) + # 尚未自动化,需要平台侧另行处理 - name: Note pending HA runtime orchestration ansible.builtin.debug: msg: >- - topology={{ topology }} deployed with HA-ready config (GTID/binlog/relay/GR settings in place), - but replication wiring (CHANGE REPLICATION SOURCE) and Group Replication bootstrap - (START GROUP_REPLICATION) are not yet automated — nodes start config-ready only. + 拓扑={{ topology }} 已部署 HA 就绪配置(GTID/binlog/relay/GR 参数就位), + 但复制接线(CHANGE REPLICATION SOURCE)和组复制引导(START GROUP_REPLICATION) + 尚未自动化 - 节点仅以配置就绪状态启动。 when: topology != 'standalone' - run_once: true + run_once: true # 只打印一次 + # ==================== Handlers ==================== + # 配置变更时触发的处理动作 handlers: - name: Restart MySQL delivery instance ansible.builtin.systemd_service: diff --git a/ansible/templates/mysql-instance.cnf.j2 b/ansible/templates/mysql-instance.cnf.j2 index da6e3cf..fc3918c 100644 --- a/ansible/templates/mysql-instance.cnf.j2 +++ b/ansible/templates/mysql-instance.cnf.j2 @@ -1,82 +1,134 @@ -# Managed by XINFRA MySQL delivery - generated, do not edit by hand -# instance={{ mysql_instance }} version={{ mysql_version_value }} topology={{ topology }} +# ============================================================================= +# XINFRA MySQL 实例配置文件(Jinja2 模板) +# ============================================================================= +# 此文件由 Playbook 自动生成,禁止手动编辑 +# 修改配置请通过平台参数传递,重新部署即可 +# ============================================================================= + +# 客户端连接配置(mysql 命令行工具使用) [client] -socket={{ mysql_run_dir }}/mysql.sock -port={{ mysql_port_value }} +socket={{ mysql_run_dir }}/mysql.sock # Unix socket 连接路径(本地连接更快) +port={{ mysql_port_value }} # TCP 端口(远程连接使用) +# ============================================================================= +# MySQL 服务端核心配置 +# ============================================================================= [mysqld] -user=mysql -basedir=/usr -datadir={{ mysql_data_dir }} -tmpdir={{ mysql_tmp_dir }} -socket={{ mysql_run_dir }}/mysql.sock -pid-file={{ mysql_run_dir }}/mysql.pid -port={{ mysql_port_value }} -bind-address=0.0.0.0 -skip-name-resolve=ON -server-id={{ mysql_server_id_value }} -local-infile=OFF +# --- 基础运行参数 --- +user=mysql # 运行用户(与 systemd 单元一致) +basedir=/usr # MySQL 安装根目录(Ubuntu apt 安装位置) +datadir={{ mysql_data_dir }} # 数据目录(所有表、索引、数据文件) +tmpdir={{ mysql_tmp_dir }} # 临时文件目录(排序、临时表等) +socket={{ mysql_run_dir }}/mysql.sock # Unix socket 路径(本地客户端连接) +pid-file={{ mysql_run_dir }}/mysql.pid # 进程 PID 文件路径 +port={{ mysql_port_value }} # 监听端口 +bind-address=0.0.0.0 # 监听所有网络接口(允许远程连接) +skip-name-resolve=ON # 跳过 DNS 解析(提升连接速度,授权时需用 IP) +server-id={{ mysql_server_id_value }} # 服务器唯一 ID(复制必须,全机唯一) +local-infile=OFF # 禁用 LOAD DATA LOCAL(安全考虑,防止读取客户端文件) -# --- charset / collation / timezone / identifier case --- -character-set-server={{ mysql_character_set }} -collation-server={{ mysql_collation }} -default-time-zone={{ mysql_timezone }} -lower-case-table-names={{ mysql_lower_case_table_names }} +# --- 字符集 / 排序规则 / 时区 / 标识符大小写 --- +character-set-server={{ mysql_character_set }} # 服务端字符集,默认 utf8mb4 +collation-server={{ mysql_collation }} # 排序规则,默认 utf8mb4_general_ci +default-time-zone={{ mysql_timezone }} # 默认时区,东八区 +lower-case-table-names={{ mysql_lower_case_table_names }} # 表名大小写敏感:1=不敏感(Linux 推荐) -# --- error / slow log --- -log-error={{ mysql_log_dir }}/error.log -slow-query-log=ON -slow-query-log-file={{ mysql_log_dir }}/slow.log -long-query-time={{ mysql_long_query_time }} +# --- 错误日志 / 慢查询日志 --- +log-error={{ mysql_log_dir }}/error.log # 错误日志路径 +slow-query-log=ON # 启用慢查询日志 +slow-query-log-file={{ mysql_log_dir }}/slow.log # 慢查询日志路径 +long-query-time={{ mysql_long_query_time }} # 慢查询阈值(秒),超过此时间记录 -# --- binlog (PITR + replication base) --- -log-bin={{ mysql_binlog_dir }}/binlog -binlog-format=ROW -sync-binlog={{ mysql_sync_binlog }} -max-binlog-size={{ mysql_max_binlog_size }} -binlog-expire-logs-seconds={{ mysql_binlog_expire_logs_seconds }} +# ============================================================================= +# Binlog 配置(用于数据恢复 PITR + 复制基础) +# ============================================================================= +log-bin={{ mysql_binlog_dir }}/binlog # binlog 文件前缀(二进制日志) +binlog-format=ROW # 格式:ROW(行变更),复制最安全 +sync-binlog={{ mysql_sync_binlog }} # 每次提交同步 binlog 到磁盘,1=最安全 +max-binlog-size={{ mysql_max_binlog_size }} # 单个 binlog 文件最大大小 +binlog-expire-logs-seconds={{ mysql_binlog_expire_logs_seconds }} # binlog 过期时间(秒),默认 7 天 + +# --- 非 standalone 拓扑才启用的复制参数 --- {% if topology != 'standalone' %} -gtid-mode=ON -enforce-gtid-consistency=ON -relay-log={{ mysql_binlog_dir }}/relay-bin +gtid-mode=ON # 启用 GTID(全局事务标识符) +enforce-gtid-consistency=ON # 强制 GTID 一致性(确保复制安全) +relay-log={{ mysql_binlog_dir }}/relay-bin # 中继日志路径(从库重放 binlog 使用) {% endif %} -# --- InnoDB core --- +# ============================================================================= +# InnoDB 存储引擎核心配置 +# ============================================================================= +# 缓冲池大小:分配内存的 55%(MySQL 官方推荐 50%-75%) innodb-buffer-pool-size={{ ((mysql_memory_mb_value | int) * 55 / 100) | int }}M + +# 刷盘方式:O_DIRECT 绕过操作系统缓存,直接写磁盘 +# 避免双重缓存,适合有电池保护写缓存的 RAID 控制器 innodb-flush-method=O_DIRECT + +# 事务提交时刷日志到磁盘:1=每次提交都刷(最安全,性能最低) innodb-flush-log-at-trx-commit={{ mysql_flush_log_at_trx_commit }} + +# InnoDB I/O 能力:告诉 InnoDB 磁盘的 IOPS 能力 +# 2000 适合 SSD,HDD 建议 200-400 innodb-io-capacity={{ mysql_io_capacity }} + +# 最大连接数(按内存梯度计算,默认值见 Playbook) max-connections={{ mysql_max_connections }} -# --- redo log (version-sensitive) --- -innodb-log-group-home-dir={{ mysql_redo_dir }} +# ============================================================================= +# Redo Log 配置(InnoDB 崩溃恢复必需) +# ============================================================================= +innodb-log-group-home-dir={{ mysql_redo_dir }} # redo log 目录 + +# MySQL 8.0.30+ 使用 innodb-redo-log-capacity 控制 redo log 总大小 +# 旧版本使用 innodb-log-file-size 控制单个文件大小 {% if mysql_version_value in ['8.0', '8.4'] %} innodb-redo-log-capacity={{ mysql_redo_capacity }} {% else %} innodb-log-file-size={{ mysql_redo_capacity }} {% endif %} -# --- X Protocol disabled (version-sensitive) --- +# ============================================================================= +# X Protocol 禁用(版本相关) +# ============================================================================= +# X Protocol 是 MySQL 的 NoSQL 部分,我们不需要,禁用节省资源 {% if mysql_version_value in ['8.0', '8.4'] %} -mysqlx=0 +mysqlx=0 # 8.0/8.4 直接禁用 {% elif mysql_version_value == '5.7' %} -loose-mysqlx=0 +loose-mysqlx=0 # 5.7 使用 loose 前缀(参数可能不存在) {% endif %} +# ============================================================================= +# Group Replication 配置(仅 mgr_3 拓扑) +# ============================================================================= +# 注意:这里只做配置就绪,实际启动组复制需要平台侧执行 START GROUP_REPLICATION {% if topology == 'mgr_3' %} -# --- Group Replication (config-ready; runtime bootstrap handled out-of-band) --- +# 加载 Group Replication 插件 plugin-load-add=group_replication.so + +# MySQL 8.0.26 之前需要设置事务写集提取(8.4 已移除此参数) {% if mysql_version_value != '8.4' %} -{# deprecated since 8.0.26 and removed in 8.3+; only inject for older series #} -transaction-write-set-extraction=XXHASH64 +transaction-write-set-extraction=XXHASH64 # 使用 XXHASH64 算法提取写集 {% endif %} + +# GR 组 UUID(所有节点必须相同) loose-group-replication-group-name={{ mgr_group_name }} + +# 不在启动时自动加入组(由平台侧手动控制加入时机) loose-group-replication-start-on-boot=OFF + +# 本节点用于 GR 通信的地址和端口 loose-group-replication-local-address={{ ansible_host | default(inventory_hostname) }}:{{ mysql_gr_port_value }} + +# 组内所有节点的种子列表(用于新节点发现和连接) loose-group-replication-group-seeds={% for host in ansible_play_hosts_all %}{{ hostvars[host].ansible_host | default(host) }}:{{ mysql_gr_port_value }}{% if not loop.last %},{% endif %}{% endfor %} +# IP 白名单:只允许组内节点 IP 加入(安全限制) loose-group-replication-ip-allowlist={% for host in ansible_play_hosts_all %}{{ hostvars[host].ansible_host | default(host) }}{% if not loop.last %},{% endif %}{% endfor %} +# 单主模式:只有 primary 节点可写,replica 只读(推荐生产使用) loose-group-replication-single-primary-mode=ON + +# 关闭强制更新检查:单主模式下不需要(多主模式才需要) loose-group-replication-enforce-update-everywhere-checks=OFF {% endif %} From b9a3ddd9e351292aa86b72ceb78e071033385859 Mon Sep 17 00:00:00 2001 From: Hungerdream <1710233908@qq.com> Date: Tue, 28 Jul 2026 14:28:42 +0800 Subject: [PATCH 2/6] feat(ansible): add mysql-rollback playbook for instance cleanup Add a compensating playbook that removes a single delivered MySQL instance from a target host. The rollback scope is pinned by target_hosts + instance_name + data_disk, with pre-task assertions rejecting instance names or disk paths outside the delivery layout to prevent accidental deletion. - stop and disable mysql-delivery@.service, reset failed state - remove instance data dir, install dir, config file and run dir --- ansible/mysql-rollback.yml | 79 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 ansible/mysql-rollback.yml diff --git a/ansible/mysql-rollback.yml b/ansible/mysql-rollback.yml new file mode 100644 index 0000000..c302b1d --- /dev/null +++ b/ansible/mysql-rollback.yml @@ -0,0 +1,79 @@ +--- +- name: Roll back native MySQL delivery instance + hosts: "{{ target_hosts }}" + become: true + gather_facts: false + any_errors_fatal: true + vars: + mysql_instance: "{{ instance_name }}" + mysql_data_disk: "{{ data_disk }}" + mysql_base_dir: "{{ mysql_data_disk }}/mysql-delivery/{{ mysql_instance }}" + mysql_install_dir: "/opt/mysql-delivery/{{ mysql_instance }}" + mysql_config_file: "/etc/mysql/mysql-delivery/{{ mysql_instance }}.cnf" + mysql_run_dir: "/run/mysql-delivery-{{ mysql_instance }}" + + pre_tasks: + - name: Validate rollback target + ansible.builtin.assert: + that: + - mysql_instance is match('^[a-z0-9][a-z0-9-]{0,62}$') + - mysql_data_disk is match('^/') + - mysql_data_disk != '/' + - "'..' not in mysql_data_disk" + fail_msg: "Rollback target is outside the native MySQL delivery layout" + quiet: true + + tasks: + - name: Stop the delivery instance when present + ansible.builtin.systemd_service: + name: "mysql-delivery@{{ mysql_instance }}.service" + state: stopped + enabled: false + register: mysql_stop_result + failed_when: false + + - name: Remove the instance systemd failure state + ansible.builtin.command: + argv: + - systemctl + - reset-failed + - "mysql-delivery@{{ mysql_instance }}.service" + changed_when: false + failed_when: false + + - name: Remove the instance configuration + ansible.builtin.file: + path: "{{ mysql_config_file }}" + state: absent + + - name: Remove instance-local binary links + ansible.builtin.file: + path: "{{ mysql_install_dir }}" + state: absent + + - name: Remove the instance data and log tree + ansible.builtin.file: + path: "{{ mysql_base_dir }}" + state: absent + + - name: Remove the instance runtime directory + ansible.builtin.file: + path: "{{ mysql_run_dir }}" + state: absent + + - name: Confirm the instance artifacts are gone + ansible.builtin.stat: + path: "{{ item }}" + loop: + - "{{ mysql_config_file }}" + - "{{ mysql_install_dir }}" + - "{{ mysql_base_dir }}" + - "{{ mysql_run_dir }}" + register: mysql_rollback_artifacts + + - name: Assert that all instance artifacts are gone + ansible.builtin.assert: + that: + - mysql_rollback_artifacts.results | selectattr('stat.exists') | list | length == 0 + fail_msg: "One or more MySQL delivery artifacts remain after rollback" + quiet: true From 7a800b0307cd83b51aa72f7ef87e518e9ec23d5b Mon Sep 17 00:00:00 2001 From: Hungerdream <1710233908@qq.com> Date: Tue, 28 Jul 2026 14:28:53 +0800 Subject: [PATCH 3/6] fix(ansible): treat empty or auto redo capacity as unset innodb_redo_log_capacity passed as '' or 'auto' from the delivery payload previously bypassed the memory-based tier calculation and was rendered verbatim into the instance config. Fall back to the automatic tier when the value is empty or 'auto'. --- ansible/mysql-deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/mysql-deploy.yml b/ansible/mysql-deploy.yml index 03184e7..db3badf 100644 --- a/ansible/mysql-deploy.yml +++ b/ansible/mysql-deploy.yml @@ -334,7 +334,7 @@ # InnoDB redo log 容量:按内存梯度递增 # ≤4G→128M、≤16G→256M、≤32G→512M、>32G→1G mysql_redo_capacity: >- - {{ innodb_redo_log_capacity if (innodb_redo_log_capacity is defined) + {{ innodb_redo_log_capacity if (innodb_redo_log_capacity is defined and (innodb_redo_log_capacity | string | lower) not in ['', 'auto']) else ('128M' if (mysql_memory_gb | int) <= 4 else '256M' if (mysql_memory_gb | int) <= 16 else '512M' if (mysql_memory_gb | int) <= 32 From f0c1b38c5b423f75200f2f7101638c7f16b35c28 Mon Sep 17 00:00:00 2001 From: Hungerdream <1710233908@qq.com> Date: Tue, 28 Jul 2026 14:29:09 +0800 Subject: [PATCH 4/6] feat(delivery): add automatic rollback state machine for failed deployments Introduce a compensating workflow that launches the dedicated AWX rollback template when a deployment cannot be started or fails midway. - add task states: rollback_pending, rolling_back, rolled_back, rollback_failed, rollback_acknowledged - add RollbackJob model to track the compensating AWX run separately from the deploy run, preserving both job IDs for audit - hold resource reservations in 'rollback' status until cleanup succeeds so a failed cleanup cannot be masked by a later delivery - poll rollback jobs with a 2-minute launch timeout; an unknown launch result surfaces as a recoverable failure instead of re-launching - protect finished/register_failed/rolled-back tasks from rollback; register_failed keeps the healthy instance and its resource usage - add DELIVERY_ROLLBACK_TEMPLATE_ID config; without it, failures are marked rollback_failed and require manual cleanup - use unique pending- placeholder for executor job IDs --- server/.env.example | 2 + server/internal/config/config.go | 2 + server/internal/database/database.go | 1 + server/internal/model/delivery.go | 19 ++ server/internal/service/delivery.go | 309 ++++++++++++++++++++++- server/internal/service/delivery_test.go | 44 +++- 6 files changed, 366 insertions(+), 11 deletions(-) diff --git a/server/.env.example b/server/.env.example index e3ae278..6d4f3f4 100644 --- a/server/.env.example +++ b/server/.env.example @@ -30,6 +30,8 @@ AWX_BASE_URL= AWX_TOKEN= AWX_USERNAME= AWX_PASSWORD= +# AWX Job Template ID for ansible/mysql-rollback.yml; required for automatic cleanup +DELIVERY_ROLLBACK_TEMPLATE_ID=0 DELIVERY_SERVICE_TOKEN= CLOUDDM_REGISTER_URL= CLOUDDM_API_TOKEN= diff --git a/server/internal/config/config.go b/server/internal/config/config.go index 4041281..a20bc5a 100644 --- a/server/internal/config/config.go +++ b/server/internal/config/config.go @@ -63,6 +63,7 @@ type Config struct { AWXToken string AWXUsername string AWXPassword string + RollbackTemplateID uint64 DeliveryServiceToken string DeliverySchedulerEnabled bool DeliveryPollSeconds int @@ -133,6 +134,7 @@ func Load() Config { AWXToken: env("AWX_TOKEN", ""), AWXUsername: env("AWX_USERNAME", ""), AWXPassword: env("AWX_PASSWORD", ""), + RollbackTemplateID: uint64(envInt("DELIVERY_ROLLBACK_TEMPLATE_ID", 0)), DeliveryServiceToken: env("DELIVERY_SERVICE_TOKEN", ""), DeliverySchedulerEnabled: envBool("DELIVERY_SCHEDULER_ENABLED", false), DeliveryPollSeconds: envInt("DELIVERY_POLL_SECONDS", 5), diff --git a/server/internal/database/database.go b/server/internal/database/database.go index f8daaa8..3123ba0 100644 --- a/server/internal/database/database.go +++ b/server/internal/database/database.go @@ -27,6 +27,7 @@ func AutoMigrate(db *gorm.DB) error { &model.MySQLInstance{}, &model.ResourceUsage{}, &model.ExecutionJob{}, + &model.RollbackJob{}, &model.TaskEvent{}, ) } diff --git a/server/internal/model/delivery.go b/server/internal/model/delivery.go index e30b228..23ce068 100644 --- a/server/internal/model/delivery.go +++ b/server/internal/model/delivery.go @@ -14,6 +14,11 @@ const ( TaskValidationFailed = "validation_failed" TaskCanceling = "canceling" TaskCanceled = "canceled" + TaskRollbackPending = "rollback_pending" + TaskRollingBack = "rolling_back" + TaskRolledBack = "rolled_back" + TaskRollbackFailed = "rollback_failed" + TaskRollbackAck = "rollback_acknowledged" ) type ResourceQuota struct { @@ -110,6 +115,20 @@ type ExecutionJob struct { UpdatedAt time.Time `json:"updated_at"` } +// RollbackJob tracks the compensating AWX run independently from the deploy run. +// Keeping a separate record preserves both job IDs for audit and retry tooling. +type RollbackJob struct { + ID uint64 `gorm:"primaryKey" json:"id"` + TaskID string `gorm:"size:36;not null;uniqueIndex" json:"task_id"` + ExecutorJobID string `gorm:"size:128;not null" json:"executor_job_id"` + Status string `gorm:"size:32;not null;index" json:"status"` + Reason string `gorm:"type:text" json:"reason"` + StartedAt *time.Time `json:"started_at,omitempty"` + FinishedAt *time.Time `json:"finished_at,omitempty"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + type TaskEvent struct { ID uint64 `gorm:"primaryKey" json:"id"` TaskID string `gorm:"size:36;not null;index" json:"task_id"` diff --git a/server/internal/service/delivery.go b/server/internal/service/delivery.go index 119254c..b8240e6 100644 --- a/server/internal/service/delivery.go +++ b/server/internal/service/delivery.go @@ -310,6 +310,8 @@ var supportedMySQLVersions = map[string]bool{"8.0": true, "8.4": true} // 但调度器仍是单主机模型且复制编排未自动化,本期仅放开 standalone。 var supportedTopologies = map[string]bool{"standalone": true} +const rollbackLaunchTimeout = 2 * time.Minute + var supportedCharsets = map[string]bool{"utf8mb4": true, "utf8": true, "gbk": true, "latin1": true} // 高级参数档位白名单(与 docs/mysql-parameter-selection.md 保持一致) @@ -587,7 +589,7 @@ func checkResourceQuota(tx *gorm.DB, businessLineID, targetID uint64, payload de if err := tx.Model(&model.ResourceUsage{}).Select("COALESCE(SUM(cpu_milli),0) cpu, COALESCE(SUM(memory_mi),0) memory, COALESCE(SUM(storage_gi),0) storage, COALESCE(SUM(instance_count),0) instances").Where("business_line_id = ? AND target_id = ? AND status = ?", businessLineID, targetID, "active").Scan(&used).Error; err != nil { return false, err } - if err := tx.Model(&model.ResourceReservation{}).Select("COALESCE(SUM(cpu_milli),0) cpu, COALESCE(SUM(memory_mi),0) memory, COALESCE(SUM(storage_gi),0) storage, COALESCE(SUM(instance_count),0) instances").Where("business_line_id = ? AND target_id = ? AND status = ? AND expires_at > ?", businessLineID, targetID, "reserved", time.Now()).Scan(&reserved).Error; err != nil { + if err := tx.Model(&model.ResourceReservation{}).Select("COALESCE(SUM(cpu_milli),0) cpu, COALESCE(SUM(memory_mi),0) memory, COALESCE(SUM(storage_gi),0) storage, COALESCE(SUM(instance_count),0) instances").Where("business_line_id = ? AND target_id = ? AND (status = ? OR (status = ? AND expires_at > ?))", businessLineID, targetID, "rollback", "reserved", time.Now()).Scan(&reserved).Error; err != nil { return false, err } return used.CPU+reserved.CPU+payload.CPUMilli <= quota.CPUMilli && @@ -614,7 +616,7 @@ func (s *DeliveryService) transitionTx(tx *gorm.DB, task *model.DeliveryTask, st if status == model.TaskRunning { updates["started_at"] = now } - if status == model.TaskFinished || status == model.TaskExecutionFailed || status == model.TaskValidationFailed || status == model.TaskCanceled || status == model.TaskRegisterFailed { + if status == model.TaskFinished || status == model.TaskExecutionFailed || status == model.TaskValidationFailed || status == model.TaskCanceled || status == model.TaskRegisterFailed || status == model.TaskRolledBack || status == model.TaskRollbackFailed || status == model.TaskRollbackAck { updates["finished_at"] = now } result := tx.Model(&model.DeliveryTask{}).Where("id = ? AND status = ?", task.ID, from).Updates(updates) @@ -662,7 +664,7 @@ func (s *DeliveryService) DispatchOnce(ctx context.Context) error { } _, _, err = s.CreateExecution(ctx, task.ID, task.PayloadHash, task.IdempotencyKey) if err != nil { - return s.failTask(ctx, task, model.TaskExecutionFailed, err.Error()) + return s.beginRollback(ctx, task.ID, "deployment could not be started: "+err.Error()) } return nil } @@ -700,7 +702,7 @@ func (s *DeliveryService) CreateExecution(ctx context.Context, taskID, payloadHa meta := parseTargetMetadata(target.Metadata) // Persist execution record BEFORE launching AWX to ensure crash recovery. now := time.Now() - execution := model.ExecutionJob{TaskID: task.ID, IdempotencyKey: task.IdempotencyKey, ExecutorJobID: "pending", Status: "launching", StartedAt: &now} + execution := model.ExecutionJob{TaskID: task.ID, IdempotencyKey: task.IdempotencyKey, ExecutorJobID: "pending-" + task.ID, Status: "launching", StartedAt: &now} if err := s.db.WithContext(ctx).Create(&execution).Error; err != nil { return nil, false, err } @@ -785,8 +787,8 @@ func (s *DeliveryService) PollOnce(ctx context.Context) error { for _, execution := range jobs { job, err := s.awx.GetJob(ctx, execution.ExecutorJobID) if err != nil { - s.finishExecution(ctx, &execution, "failed") - _ = s.failTask(ctx, &model.DeliveryTask{ID: execution.TaskID, Status: model.TaskRunning}, model.TaskExecutionFailed, "poll AWX job "+execution.ExecutorJobID+": "+err.Error()) + // A transient AWX/API failure is not evidence that deployment failed. + // Keep the job running and retry on the next scheduler tick. continue } switch strings.ToLower(job.Status) { @@ -795,14 +797,14 @@ func (s *DeliveryService) PollOnce(ctx context.Context) error { case "successful": s.finishExecution(ctx, &execution, "successful") if err := s.completeTask(ctx, execution.TaskID); err != nil { - _ = s.failTask(ctx, &model.DeliveryTask{ID: execution.TaskID, Status: model.TaskRunning}, model.TaskValidationFailed, err.Error()) + _ = s.beginRollback(ctx, execution.TaskID, err.Error()) } case "canceled": s.finishExecution(ctx, &execution, "canceled") - _ = s.failTask(ctx, &model.DeliveryTask{ID: execution.TaskID, Status: model.TaskRunning}, model.TaskCanceled, "AWX job was canceled") + _ = s.beginRollback(ctx, execution.TaskID, "AWX deployment job was canceled") default: s.finishExecution(ctx, &execution, "failed") - _ = s.failTask(ctx, &model.DeliveryTask{ID: execution.TaskID, Status: model.TaskRunning}, model.TaskExecutionFailed, "AWX job finished with status "+job.Status) + _ = s.beginRollback(ctx, execution.TaskID, "AWX job finished with status "+job.Status) } } return nil @@ -849,11 +851,41 @@ func (s *DeliveryService) completeTask(ctx context.Context, taskID string) error return err } if err := s.RegisterCloudDM(ctx, task.ID); err != nil { - return s.transition(ctx, &task, model.TaskRegisterFailed, "CloudDM registration failed", err.Error()) + message := fmt.Sprintf("CloudDM registration failed: %v", err) + // The database is already healthy and accounted for at this point. A + // registration outage must not trigger destructive instance rollback. + if transitionErr := s.transition(ctx, &task, model.TaskRegisterFailed, + "MySQL delivered; CloudDM registration failed and can be retried", message); transitionErr != nil { + return fmt.Errorf("%s; cannot record register_failed: %w", message, transitionErr) + } + return nil } return s.transition(ctx, &task, model.TaskFinished, "MySQL delivery completed", "") } +// RetryCloudDMRegistration retries only the external registration step. The +// instance and its resource usage remain active throughout the retry. +func (s *DeliveryService) RetryCloudDMRegistration(ctx context.Context, taskID string) error { + var task model.DeliveryTask + if err := s.db.WithContext(ctx).First(&task, "id = ?", taskID).Error; err != nil { + return err + } + if task.Status != model.TaskRegisterFailed { + return fmt.Errorf("task %s is in state %q and cannot retry CloudDM registration", taskID, task.Status) + } + if err := s.transition(ctx, &task, model.TaskRegistering, "CloudDM registration retry started", ""); err != nil { + return err + } + if err := s.RegisterCloudDM(ctx, taskID); err != nil { + message := fmt.Sprintf("CloudDM registration failed: %v", err) + if transitionErr := s.transition(ctx, &task, model.TaskRegisterFailed, "CloudDM registration retry failed; instance remains active", message); transitionErr != nil { + return fmt.Errorf("%s; cannot restore register_failed: %w", message, transitionErr) + } + return err + } + return s.transition(ctx, &task, model.TaskFinished, "CloudDM registration completed", "") +} + func (s *DeliveryService) RegisterCloudDM(ctx context.Context, taskID string) error { if s.cfg.CloudDMRegisterURL == "" { return nil @@ -883,6 +915,262 @@ func (s *DeliveryService) RegisterCloudDM(ctx context.Context, taskID string) er return nil } +// beginRollback changes the task into the compensating workflow and launches +// the dedicated AWX rollback template. The deploy reservation is deliberately +// kept until rollback succeeds so a failed cleanup cannot be silently +// overwritten by a later delivery. +func (s *DeliveryService) beginRollback(ctx context.Context, taskID, reason string) error { + var task model.DeliveryTask + if err := s.db.WithContext(ctx).First(&task, "id = ?", taskID).Error; err != nil { + return err + } + if rollbackProtectedStatus(task.Status) { + return nil + } + + if s.cfg.RollbackTemplateID == 0 { + if err := s.transition(ctx, &task, model.TaskRollbackFailed, + "automatic rollback is not configured; manual cleanup is required", reason); err != nil { + return err + } + return s.holdReservationForRollback(ctx, task.ID) + } + if err := s.transition(ctx, &task, model.TaskRollbackPending, "automatic rollback queued", reason); err != nil { + return err + } + if err := s.holdReservationForRollback(ctx, task.ID); err != nil { + return s.markRollbackFailed(ctx, task.ID, "cannot hold resources during rollback: "+err.Error()) + } + return s.launchRollbackJob(ctx, &task, reason) +} + +func rollbackProtectedStatus(status string) bool { + switch status { + case model.TaskRollbackPending, model.TaskRollingBack, model.TaskRolledBack, model.TaskRollbackFailed, + model.TaskRollbackAck, model.TaskRegisterFailed, model.TaskFinished, model.TaskCanceled: + return true + default: + return false + } +} + +func (s *DeliveryService) launchRollbackJob(ctx context.Context, task *model.DeliveryTask, reason string) error { + target, err := s.getTarget(ctx, task.TargetID) + if err != nil { + return s.markRollbackFailed(ctx, task.ID, "rollback target is unavailable: "+err.Error()) + } + var payload deliveryPayload + if err := json.Unmarshal([]byte(task.ImmutablePayload), &payload); err != nil { + return s.markRollbackFailed(ctx, task.ID, "rollback payload is invalid: "+err.Error()) + } + now := time.Now() + var rollback model.RollbackJob + if err := s.db.WithContext(ctx).Where("task_id = ?", task.ID).First(&rollback).Error; errors.Is(err, gorm.ErrRecordNotFound) { + rollback = model.RollbackJob{TaskID: task.ID} + } else if err != nil { + return s.markRollbackFailed(ctx, task.ID, "cannot load rollback job: "+err.Error()) + } + rollback.ExecutorJobID = "pending-rollback-" + task.ID + rollback.Status = "launching" + rollback.Reason = reason + rollback.StartedAt = &now + rollback.FinishedAt = nil + if err := s.db.WithContext(ctx).Save(&rollback).Error; err != nil { + return s.markRollbackFailed(ctx, task.ID, "cannot persist rollback job: "+err.Error()) + } + job, err := s.awx.Launch(ctx, s.cfg.RollbackTemplateID, AWXLaunchRequest{ + InventoryID: target.AWXInventoryID, + Limit: task.TargetHost, + ExtraVars: rollbackExtraVars(task, payload), + }) + if err != nil { + _ = s.db.WithContext(ctx).Model(&rollback).Updates(map[string]any{"status": "failed", "finished_at": time.Now()}).Error + return s.markRollbackFailed(ctx, task.ID, "cannot launch rollback AWX job: "+err.Error()) + } + if err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + var current model.DeliveryTask + if err := tx.First(¤t, "id = ?", task.ID).Error; err != nil { + return err + } + if current.Status != model.TaskRollbackPending { + return fmt.Errorf("task %s is no longer pending rollback (state %q)", task.ID, current.Status) + } + if err := tx.Model(&rollback).Updates(map[string]any{"executor_job_id": fmt.Sprint(job.ID), "status": "running"}).Error; err != nil { + return err + } + return s.transitionTx(tx, ¤t, model.TaskRollingBack, "rollback AWX job started", reason) + }); err != nil { + return s.markRollbackFailed(ctx, task.ID, "cannot record rollback AWX job: "+err.Error()) + } + return nil +} + +func (s *DeliveryService) RetryRollback(ctx context.Context, taskID string) error { + var task model.DeliveryTask + if err := s.db.WithContext(ctx).First(&task, "id = ?", taskID).Error; err != nil { + return err + } + if task.Status != model.TaskRollbackFailed { + return fmt.Errorf("task %s is in state %q and cannot retry rollback", taskID, task.Status) + } + if s.cfg.RollbackTemplateID == 0 { + return fmt.Errorf("automatic rollback is not configured") + } + reason := "manual rollback retry requested" + if err := s.transition(ctx, &task, model.TaskRollbackPending, "manual rollback retry queued", reason); err != nil { + return err + } + if err := s.holdReservationForRollback(ctx, task.ID); err != nil { + _ = s.markRollbackFailed(ctx, task.ID, "cannot hold resources during rollback retry: "+err.Error()) + return err + } + return s.launchRollbackJob(ctx, &task, reason) +} + +// AcknowledgeRollbackRelease is an explicit platform-admin escape hatch after +// the operator has verified the target host. It releases bookkeeping only; it +// does not claim that the automatic rollback succeeded. +func (s *DeliveryService) AcknowledgeRollbackRelease(ctx context.Context, taskID string) error { + now := time.Now() + return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + var task model.DeliveryTask + if err := tx.First(&task, "id = ?", taskID).Error; err != nil { + return err + } + if task.Status != model.TaskRollbackFailed { + return fmt.Errorf("task %s is in state %q and cannot acknowledge rollback release", taskID, task.Status) + } + if err := tx.Model(&model.MySQLInstance{}).Where("task_id = ? AND status = ?", taskID, "active").Updates(map[string]any{"status": "rollback_acknowledged", "updated_at": now}).Error; err != nil { + return err + } + if err := tx.Model(&model.ResourceUsage{}).Where("task_id = ? AND status = ?", taskID, "active").Updates(map[string]any{"status": "released", "released_at": now, "updated_at": now}).Error; err != nil { + return err + } + if err := tx.Model(&model.ResourceReservation{}).Where("task_id = ? AND status = ?", taskID, "rollback").Update("status", "released").Error; err != nil { + return err + } + return s.transitionTx(tx, &task, model.TaskRollbackAck, "rollback release acknowledged by platform admin", "") + }) +} + +func rollbackExtraVars(task *model.DeliveryTask, payload deliveryPayload) map[string]any { + return map[string]any{ + "target_hosts": task.TargetHost, + "instance_name": payload.InstanceName, + "data_disk": payload.DataDisk, + "task_id": task.ID, + "rollback": true, + } +} + +func (s *DeliveryService) holdReservationForRollback(ctx context.Context, taskID string) error { + return s.db.WithContext(ctx).Model(&model.ResourceReservation{}). + Where("task_id = ? AND status IN ?", taskID, []string{"reserved", "consumed"}). + Update("status", "rollback").Error +} + +func (s *DeliveryService) markRollbackFailed(ctx context.Context, taskID, message string) error { + var task model.DeliveryTask + if err := s.db.WithContext(ctx).First(&task, "id = ?", taskID).Error; err != nil { + return err + } + if task.Status == model.TaskRolledBack || task.Status == model.TaskFinished || task.Status == model.TaskCanceled || + task.Status == model.TaskRollbackFailed || task.Status == model.TaskRollbackAck || task.Status == model.TaskRegisterFailed { + return nil + } + return s.transition(ctx, &task, model.TaskRollbackFailed, "automatic rollback failed; manual cleanup is required", message) +} + +func (s *DeliveryService) PollRollbackOnce(ctx context.Context) error { + var jobs []model.RollbackJob + if err := s.db.WithContext(ctx).Where("status IN ?", []string{"launching", "running"}).Find(&jobs).Error; err != nil { + return err + } + for _, rollback := range jobs { + if rollback.Status == "launching" { + if !rollbackLaunchExpired(rollback, time.Now()) { + continue + } + // The AWX launch result is unknown. Do not blindly launch a second + // job; surface a recoverable failure for an explicit admin retry. + expired, err := s.expireRollbackLaunch(ctx, rollback.ID) + if err != nil { + return err + } + if !expired { + continue + } + _ = s.markRollbackFailed(ctx, rollback.TaskID, "rollback launch timed out before its AWX job ID was recorded; manual retry is required") + continue + } + job, err := s.awx.GetJob(ctx, rollback.ExecutorJobID) + if err != nil { + // Do not declare cleanup failed because AWX is temporarily unreachable. + continue + } + switch strings.ToLower(job.Status) { + case "pending", "waiting", "running", "new": + continue + case "successful": + _ = s.finishRollbackJob(ctx, &rollback, "successful") + if err := s.completeRollback(ctx, rollback.TaskID); err != nil { + _ = s.markRollbackFailed(ctx, rollback.TaskID, err.Error()) + } + case "canceled": + _ = s.finishRollbackJob(ctx, &rollback, "canceled") + _ = s.markRollbackFailed(ctx, rollback.TaskID, "rollback AWX job was canceled") + default: + _ = s.finishRollbackJob(ctx, &rollback, "failed") + _ = s.markRollbackFailed(ctx, rollback.TaskID, "rollback AWX job finished with status "+job.Status) + } + } + return nil +} + +func rollbackLaunchExpired(rollback model.RollbackJob, now time.Time) bool { + if rollback.Status != "launching" { + return false + } + if rollback.StartedAt == nil { + return true + } + return !now.Before(rollback.StartedAt.Add(rollbackLaunchTimeout)) +} + +func (s *DeliveryService) expireRollbackLaunch(ctx context.Context, rollbackID uint64) (bool, error) { + result := s.db.WithContext(ctx).Model(&model.RollbackJob{}). + Where("id = ? AND status = ?", rollbackID, "launching"). + Updates(map[string]any{"status": "launch_timeout", "finished_at": time.Now()}) + return result.RowsAffected == 1, result.Error +} + +func (s *DeliveryService) finishRollbackJob(ctx context.Context, rollback *model.RollbackJob, status string) error { + return s.db.WithContext(ctx).Model(rollback).Updates(map[string]any{"status": status, "finished_at": time.Now()}).Error +} + +func (s *DeliveryService) completeRollback(ctx context.Context, taskID string) error { + now := time.Now() + return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + var task model.DeliveryTask + if err := tx.First(&task, "id = ?", taskID).Error; err != nil { + return err + } + if task.Status != model.TaskRollingBack { + return fmt.Errorf("task %s is in state %q, cannot complete rollback", taskID, task.Status) + } + if err := tx.Model(&model.MySQLInstance{}).Where("task_id = ?", taskID).Updates(map[string]any{"status": "rolled_back", "updated_at": now}).Error; err != nil { + return err + } + if err := tx.Model(&model.ResourceUsage{}).Where("task_id = ? AND status = ?", taskID, "active").Updates(map[string]any{"status": "released", "released_at": now, "updated_at": now}).Error; err != nil { + return err + } + if err := tx.Model(&model.ResourceReservation{}).Where("task_id = ? AND status IN ?", taskID, []string{"reserved", "consumed", "rollback"}).Update("status", "released").Error; err != nil { + return err + } + return s.transitionTx(tx, &task, model.TaskRolledBack, "MySQL instance rollback completed", "") + }) +} + func (s *DeliveryService) failTask(ctx context.Context, task *model.DeliveryTask, status, message string) error { return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { var current model.DeliveryTask @@ -916,6 +1204,7 @@ func (s *DeliveryService) Run(ctx context.Context) { case <-ticker.C: _ = s.DispatchOnce(ctx) _ = s.PollOnce(ctx) + _ = s.PollRollbackOnce(ctx) } } } diff --git a/server/internal/service/delivery_test.go b/server/internal/service/delivery_test.go index ea19946..66142b4 100644 --- a/server/internal/service/delivery_test.go +++ b/server/internal/service/delivery_test.go @@ -1,6 +1,11 @@ package service -import "testing" +import ( + "testing" + "time" + + "github.com/1024XEngineer/xinfra/server/internal/model" +) func intPtr(v int) *int { return &v } @@ -124,3 +129,40 @@ func TestAllocatePort(t *testing.T) { t.Fatal("exhausted pool still allocated a port") } } + +func TestRollbackExtraVarsTargetsOnlyTheAllocatedInstance(t *testing.T) { + task := &model.DeliveryTask{ID: "task-1", TargetHost: "db-01"} + payload := deliveryPayload{MySQLDeliveryInput: MySQLDeliveryInput{InstanceName: "mysql-a", DataDisk: "/disk1"}} + vars := rollbackExtraVars(task, payload) + if vars["target_hosts"] != "db-01" || vars["instance_name"] != "mysql-a" || vars["data_disk"] != "/disk1" { + t.Fatalf("rollback vars target the wrong instance: %#v", vars) + } + if vars["rollback"] != true { + t.Fatalf("rollback marker missing: %#v", vars) + } +} + +func TestRegisterFailedIsProtectedFromRollback(t *testing.T) { + if !rollbackProtectedStatus(model.TaskRegisterFailed) { + t.Fatal("register_failed must preserve the healthy instance and resource usage") + } + if rollbackProtectedStatus(model.TaskValidationFailed) { + t.Fatal("validation_failed must still be eligible for cleanup rollback") + } +} + +func TestRollbackLaunchExpired(t *testing.T) { + now := time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC) + started := now.Add(-rollbackLaunchTimeout - time.Second) + if !rollbackLaunchExpired(model.RollbackJob{Status: "launching", StartedAt: &started}, now) { + t.Fatal("stale launching rollback job must be recoverable") + } + if rollbackLaunchExpired(model.RollbackJob{Status: "launching", StartedAt: ptrTime(now.Add(-rollbackLaunchTimeout + time.Second))}, now) { + t.Fatal("recent launching rollback job must remain pending") + } + if rollbackLaunchExpired(model.RollbackJob{Status: "running", StartedAt: &started}, now) { + t.Fatal("running rollback job is not a launch timeout") + } +} + +func ptrTime(v time.Time) *time.Time { return &v } From be39f2a03c54620c402b545ed6936ae8134d44af Mon Sep 17 00:00:00 2001 From: Hungerdream <1710233908@qq.com> Date: Tue, 28 Jul 2026 14:29:26 +0800 Subject: [PATCH 5/6] feat(delivery): expose rollback retry, release ack and CloudDM retry APIs - POST /delivery/tasks/:id/rollback/retry (platform admin) re-launches the rollback job after a previous cleanup failure - POST /delivery/tasks/:id/rollback/release (platform admin) releases bookkeeping after the operator verified the target host manually - POST /delivery/tasks/:id/clouddm/retry retries only the CloudDM registration step for an already healthy instance - task logs now include rollback AWX job stdout and localized text/classes for the new rollback states --- server/internal/handler/delivery.go | 49 +++++++++++++++++++++++++++++ server/internal/handler/task_log.go | 29 +++++++++++++++-- server/internal/router/router.go | 3 ++ 3 files changed, 78 insertions(+), 3 deletions(-) diff --git a/server/internal/handler/delivery.go b/server/internal/handler/delivery.go index f36b948..34a124a 100644 --- a/server/internal/handler/delivery.go +++ b/server/internal/handler/delivery.go @@ -175,6 +175,55 @@ func (h *DeliveryHandler) Cancel(c *gin.Context) { c.JSON(http.StatusOK, gin.H{"ok": true}) } +// RetryRollback retries the compensating AWX job after a previous cleanup failure. +func (h *DeliveryHandler) RetryRollback(c *gin.Context) { + if !requirePlatformAdmin(c) { + return + } + if err := h.service.RetryRollback(c.Request.Context(), c.Param("id")); err != nil { + c.JSON(http.StatusConflict, gin.H{"error": err.Error()}) + return + } + c.JSON(http.StatusAccepted, gin.H{"ok": true, "status": model.TaskRollbackPending}) +} + +// AcknowledgeRollbackRelease releases bookkeeping after an administrator has +// independently verified that no instance artifacts remain on the target host. +func (h *DeliveryHandler) AcknowledgeRollbackRelease(c *gin.Context) { + if !requirePlatformAdmin(c) { + return + } + if err := h.service.AcknowledgeRollbackRelease(c.Request.Context(), c.Param("id")); err != nil { + c.JSON(http.StatusConflict, gin.H{"error": err.Error()}) + return + } + c.JSON(http.StatusOK, gin.H{"ok": true, "status": model.TaskRollbackAck}) +} + +// RetryCloudDMRegistration retries only the CloudDM registration step for an +// already healthy and accounted-for MySQL instance. +func (h *DeliveryHandler) RetryCloudDMRegistration(c *gin.Context) { + claims, ok := CurrentClaims(c) + if !ok { + c.JSON(http.StatusUnauthorized, gin.H{"error": "missing current user"}) + return + } + taskID := c.Param("id") + if _, _, err := h.service.GetTask(c.Request.Context(), taskID, claims.UserID, claims.IsAdmin); err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + c.JSON(http.StatusNotFound, gin.H{"error": "task not found"}) + return + } + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + if err := h.service.RetryCloudDMRegistration(c.Request.Context(), taskID); err != nil { + c.JSON(http.StatusConflict, gin.H{"error": err.Error()}) + return + } + c.JSON(http.StatusAccepted, gin.H{"ok": true, "status": model.TaskFinished}) +} + // Targets 获取可用部署目标 // @Summary 获取可用部署目标 // @Description 从 AWX 动态返回可用 Job Template 及其 Inventory hosts diff --git a/server/internal/handler/task_log.go b/server/internal/handler/task_log.go index ee0ad27..617f4a2 100644 --- a/server/internal/handler/task_log.go +++ b/server/internal/handler/task_log.go @@ -138,7 +138,7 @@ func (h *TaskLogHandler) getAWXTask(c *gin.Context, taskID string, userID uint64 lines = append(lines, taskLogLine{Time: formatTaskLogTime(event.CreatedAt), Message: "[" + event.ToState + "] " + event.Message, Class: classForTaskStatus(event.ToState)}) } var execution model.ExecutionJob - if err := h.db.WithContext(c.Request.Context()).Where("task_id = ?", task.ID).First(&execution).Error; err == nil && execution.ExecutorJobID != "" && execution.ExecutorJobID != "pending" { + if err := h.db.WithContext(c.Request.Context()).Where("task_id = ?", task.ID).First(&execution).Error; err == nil && execution.ExecutorJobID != "" && execution.ExecutorJobID != "pending" && !strings.HasPrefix(execution.ExecutorJobID, "pending-") { stdout, stdoutErr := h.delivery.AWXJobStdout(c.Request.Context(), execution.ExecutorJobID) if stdoutErr != nil { lines = append(lines, taskLogLine{Time: formatTaskLogTime(time.Now()), Message: "[awx] stdout fetch failed: " + stdoutErr.Error(), Class: "err"}) @@ -146,6 +146,15 @@ func (h *TaskLogHandler) getAWXTask(c *gin.Context, taskID string, userID uint64 lines = append(lines, splitStdoutLines(stdout)...) } } + var rollback model.RollbackJob + if err := h.db.WithContext(c.Request.Context()).Where("task_id = ?", task.ID).First(&rollback).Error; err == nil && rollback.ExecutorJobID != "" && rollback.ExecutorJobID != "pending" && !strings.HasPrefix(rollback.ExecutorJobID, "pending-") { + stdout, stdoutErr := h.delivery.AWXJobStdout(c.Request.Context(), rollback.ExecutorJobID) + if stdoutErr != nil { + lines = append(lines, taskLogLine{Time: formatTaskLogTime(time.Now()), Message: "[rollback awx] stdout fetch failed: " + stdoutErr.Error(), Class: "err"}) + } else { + lines = append(lines, splitStdoutLines(stdout)...) + } + } c.JSON(http.StatusOK, gin.H{"task": awxTaskSummary(*task), "lines": lines}) } @@ -257,8 +266,18 @@ func textForTaskStatus(status string) string { return "等待" case model.TaskRunning, model.TaskRegistering, model.TaskCanceling: return "执行中" + case model.TaskRollbackPending, model.TaskRollingBack: + return "回退中" case model.TaskFinished: return "成功" + case model.TaskRolledBack: + return "已回退" + case model.TaskRollbackFailed: + return "回退失败" + case model.TaskRollbackAck: + return "已确认释放" + case model.TaskRegisterFailed: + return "注册失败(实例保留)" case model.TaskCanceled: return "已取消" default: @@ -270,10 +289,14 @@ func classForTaskStatus(status string) string { switch status { case model.TaskFinished: return "ok" - case model.TaskExecutionFailed, model.TaskValidationFailed, model.TaskRegisterFailed, model.TaskCanceled: + case model.TaskExecutionFailed, model.TaskValidationFailed, model.TaskCanceled, model.TaskRollbackFailed: return "err" - case model.TaskRunning, model.TaskDispatching, model.TaskRegistering, model.TaskCanceling: + case model.TaskRollbackAck, model.TaskRegisterFailed: return "warn" + case model.TaskRunning, model.TaskDispatching, model.TaskRegistering, model.TaskCanceling, model.TaskRollbackPending, model.TaskRollingBack: + return "warn" + case model.TaskRolledBack: + return "ok" default: return "" } diff --git a/server/internal/router/router.go b/server/internal/router/router.go index a1bb599..a5ddfc7 100644 --- a/server/internal/router/router.go +++ b/server/internal/router/router.go @@ -149,6 +149,9 @@ func registerAuthServerRoutes(r *gin.Engine, deps Dependencies) { protected.GET("/delivery/tasks", deliveryHandler.List) protected.GET("/delivery/tasks/:id", deliveryHandler.Get) protected.POST("/delivery/tasks/:id/cancel", deliveryHandler.Cancel) + protected.POST("/delivery/tasks/:id/rollback/retry", deliveryHandler.RetryRollback) + protected.POST("/delivery/tasks/:id/rollback/release", deliveryHandler.AcknowledgeRollbackRelease) + protected.POST("/delivery/tasks/:id/clouddm/retry", deliveryHandler.RetryCloudDMRegistration) protected.GET("/task-logs", taskLogHandler.List) protected.GET("/task-logs/:id", taskLogHandler.Get) } From d5bf42ad01befad2d3d7f64e349be016d93caff4 Mon Sep 17 00:00:00 2001 From: Hungerdream <1710233908@qq.com> Date: Tue, 28 Jul 2026 14:29:34 +0800 Subject: [PATCH 6/6] feat(ui): surface rollback actions in the service catalog Add retryRollback / releaseRollback / retryCloudDMRegistration API bindings and wire the corresponding actions and status presentation into the delivery task view. --- frontend/src/api/delivery.ts | 18 +++ frontend/src/views/service/Catalog.vue | 176 +++++++++++++++++++++++-- 2 files changed, 184 insertions(+), 10 deletions(-) diff --git a/frontend/src/api/delivery.ts b/frontend/src/api/delivery.ts index b799841..1ae72a3 100644 --- a/frontend/src/api/delivery.ts +++ b/frontend/src/api/delivery.ts @@ -79,6 +79,24 @@ export const deliveryApi = { method: 'POST', }) }, + + async retryRollback(taskId: string): Promise { + await authRequest(`/auth/api/v1/delivery/tasks/${encodeURIComponent(taskId)}/rollback/retry`, { + method: 'POST', + }) + }, + + async releaseRollback(taskId: string): Promise { + await authRequest(`/auth/api/v1/delivery/tasks/${encodeURIComponent(taskId)}/rollback/release`, { + method: 'POST', + }) + }, + + async retryCloudDMRegistration(taskId: string): Promise { + await authRequest(`/auth/api/v1/delivery/tasks/${encodeURIComponent(taskId)}/clouddm/retry`, { + method: 'POST', + }) + }, } function createIdempotencyKey(payload: CreateMySQLDeliveryPayload) { diff --git a/frontend/src/views/service/Catalog.vue b/frontend/src/views/service/Catalog.vue index f0574b4..5cd18cd 100644 --- a/frontend/src/views/service/Catalog.vue +++ b/frontend/src/views/service/Catalog.vue @@ -248,7 +248,7 @@
-
+
{{ deliveryFailed ? '!' : '✓' }}

{{ resultTitle }}

@@ -256,6 +256,27 @@
+
+
+ 目标机清理未确认 +

资源目前仍被保护性占用。可重试自动回退,或确认已完成目标机清理后释放。

+
+
+ 重试回退 + 确认清理并释放资源 +
+
+ +
+
+ 实例已交付,CloudDM 注册未完成 +

数据库健康实例和资源账本已保留,仅需重试 CloudDM 注册。

+
+
+ 重试 CloudDM 注册 +
+
+
访问地址 @@ -295,7 +316,7 @@ {{ deliveryForm.version }} {{ currentModeLabel }} {{ resultAddress }} - {{ deliveryFailed ? '已回退' : '正常' }} + {{ deliveryRolledBack ? '已回退' : deliveryAcknowledged ? '已确认释放' : deliveryRegisterFailed ? '实例保留' : deliveryFailed ? '需处理' : '正常' }} {{ activeService.registerTo }} @@ -317,10 +338,11 @@