xinfra/server/internal/service/delivery.go

package service

import (
	"bytes"
	"context"
	"crypto/aes"
	"crypto/cipher"
	"crypto/rand"
	"crypto/sha256"
	"encoding/base64"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"net"
	"net/http"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/1024XEngineer/xinfra/server/internal/config"
	"github.com/1024XEngineer/xinfra/server/internal/model"
	"gorm.io/gorm"
	"gorm.io/gorm/clause"
)

var dnsLabelPattern = regexp.MustCompile(`^[a-z0-9](?:[-a-z0-9]*[a-z0-9])?$`)

// AWX inventory 主机名：允许字母数字、中划线与点（FQDN）。
var hostNamePattern = regexp.MustCompile(`^[a-zA-Z0-9](?:[-a-zA-Z0-9.]*[a-zA-Z0-9])?$`)

type MySQLDeliveryInput struct {
	BusinessLineID uint64 `json:"business_line_id" binding:"required"`
	TargetID       uint64 `json:"target_id" binding:"required"`
	Namespace      string `json:"namespace" binding:"required"`
	InstanceName   string `json:"instance_name" binding:"required"`
	MySQLVersion   string `json:"mysql_version"`
	Topology       string `json:"topology"`
	MySQLPort      int    `json:"mysql_port"`
	DataDisk       string `json:"data_disk"`
	// 调度控制（选填）：点名候选池内主机跳过自动选机，端口/配额/实机守卫照常执行
	TargetHost string `json:"target_host"`
	CPUMilli   int64  `json:"cpu_milli" binding:"required"`
	MemoryMi   int64  `json:"memory_mi" binding:"required"`
	StorageGi  int64  `json:"storage_gi" binding:"required"`
	CPUCores   int64  `json:"cpu_cores,omitempty"`
	MemoryGB   int64  `json:"memory_gb,omitempty"`
	StorageGB  int64  `json:"storage_gb,omitempty"`
	// 数据库配置（选填，缺省由 playbook 基线兜底）
	Timezone            string `json:"timezone"`
	LowerCaseTableNames *int   `json:"lower_case_table_names"`
	CharacterSet        string `json:"character_set"`
	Collation           string `json:"collation"`
	// 高级参数（选填，零值视为未设置）
	MaxConnections            string  `json:"max_connections"`
	InnodbRedoLogCapacity     string  `json:"innodb_redo_log_capacity"`
	InnodbFlushLogAtTrxCommit *int    `json:"innodb_flush_log_at_trx_commit"`
	SyncBinlog                *int    `json:"sync_binlog"`
	InnodbIOCapacity          int     `json:"innodb_io_capacity"`
	LongQueryTime             float64 `json:"long_query_time"`
	BinlogExpireLogsSeconds   int64   `json:"binlog_expire_logs_seconds"`
	MaxBinlogSize             string  `json:"max_binlog_size"`
	MySQLRootPassword         string  `json:"mysql_root_password"`
	MySQLAdminPassword        string  `json:"mysql_admin_password"`
}

type deliveryPayload struct {
	MySQLDeliveryInput
	TargetType string `json:"target_type"`
}

type DeliveryTarget struct {
	ID             uint64 `json:"id"`
	Name           string `json:"name"`
	TargetType     string `json:"target_type"`
	AWXInventoryID uint64 `json:"awx_inventory_id"`
	AWXTemplateID  uint64 `json:"awx_template_id"`
	Enabled        bool   `json:"enabled"`
	Metadata       string `json:"metadata"`
}

type DeliveryMountPath struct {
	Path        string `json:"path"`
	AvailableGi int64  `json:"available_gi"`
	FSType      string `json:"fstype,omitempty"`
}

type DeliveryStageEventInput struct {
	Stage    string `json:"stage" binding:"required"`
	Status   string `json:"status" binding:"required"`
	Message  string `json:"message"`
	AWXJobID string `json:"awx_job_id"`
}

type AWXJobNotificationInput struct {
	ID        uint64 `json:"id"`
	Status    string `json:"status"`
	Name      string `json:"name"`
	URL       string `json:"url"`
	Traceback string `json:"traceback"`
	ExtraVars any    `json:"extra_vars"`
}

type DeliveryTaskListFilter struct {
	BusinessLineID uint64
	Component      string
	ActiveOnly     bool
}

type DeliveryTaskSnapshot struct {
	Task   *model.DeliveryTask `json:"task"`
	Events []model.TaskEvent   `json:"events"`
}

type MySQLServiceLedgerItem struct {
	Name        string `json:"name"`
	Datacenter  string `json:"datacenter"`
	BusinessTag string `json:"business_tag"`
	Instances   int    `json:"instances"`
	Healthy     int    `json:"healthy"`
	Address     string `json:"address"`
	Status      string `json:"status"`
	StatusClass string `json:"status_class"`
	Version     string `json:"version"`
	Namespace   string `json:"namespace"`
}

type DeploymentCredentialView struct {
	Username string `json:"username"`
	Host     string `json:"host"`
	Password string `json:"password"`
}

// targetMetadata describes the native VM候选节点池以及部署形态，由 AWX inventory hosts 动态组装。
type targetMetadata struct {
	Topology  string       `json:"topology"`
	MySQLPort int          `json:"mysql_port"`
	Hosts     []targetHost `json:"hosts"`
}

type targetHost struct {
	Name string `json:"name"`
	IP   string `json:"ip"`
}

func parseTargetMetadata(raw string) targetMetadata {
	meta := targetMetadata{}
	if strings.TrimSpace(raw) != "" {
		_ = json.Unmarshal([]byte(raw), &meta)
	}
	if meta.Topology == "" {
		meta.Topology = "standalone"
	}
	if meta.MySQLPort == 0 {
		meta.MySQLPort = 3307
	}
	return meta
}

func mountPathsFromFacts(facts map[string]any) []DeliveryMountPath {
	rawMounts, ok := facts["ansible_mounts"].([]any)
	if !ok {
		return nil
	}
	items := make([]DeliveryMountPath, 0, len(rawMounts))
	seen := map[string]struct{}{}
	for _, raw := range rawMounts {
		mount, ok := raw.(map[string]any)
		if !ok {
			continue
		}
		path := strings.TrimSpace(stringValue(mount["mount"]))
		if path == "" || !strings.HasPrefix(path, "/") {
			continue
		}
		if _, exists := seen[path]; exists {
			continue
		}
		seen[path] = struct{}{}
		items = append(items, DeliveryMountPath{
			Path:        path,
			AvailableGi: bytesToGi(int64Value(mount["size_available"])),
			FSType:      strings.TrimSpace(stringValue(mount["fstype"])),
		})
	}
	sort.Slice(items, func(i, j int) bool {
		if items[i].Path == "/" {
			return false
		}
		if items[j].Path == "/" {
			return true
		}
		return items[i].Path < items[j].Path
	})
	return items
}

func stringValue(value any) string {
	if value == nil {
		return ""
	}
	switch v := value.(type) {
	case string:
		return v
	case fmt.Stringer:
		return v.String()
	default:
		return fmt.Sprintf("%v", value)
	}
}

func int64Value(value any) int64 {
	switch v := value.(type) {
	case int:
		return int64(v)
	case int64:
		return v
	case float64:
		return int64(v)
	case json.Number:
		n, _ := v.Int64()
		return n
	default:
		return 0
	}
}

func bytesToGi(bytes int64) int64 {
	if bytes <= 0 {
		return 0
	}
	return bytes / 1073741824
}

func credentialKey(secret string) []byte {
	sum := sha256.Sum256([]byte(strings.TrimSpace(secret)))
	return sum[:]
}

func encryptCredential(secret, plain string) (string, string, error) {
	block, err := aes.NewCipher(credentialKey(secret))
	if err != nil {
		return "", "", err
	}
	gcm, err := cipher.NewGCM(block)
	if err != nil {
		return "", "", err
	}
	nonce := make([]byte, gcm.NonceSize())
	if _, err := rand.Read(nonce); err != nil {
		return "", "", err
	}
	ciphertext := gcm.Seal(nil, nonce, []byte(plain), nil)
	return base64.StdEncoding.EncodeToString(ciphertext), base64.StdEncoding.EncodeToString(nonce), nil
}

func decryptCredential(secret, ciphertextValue, nonceValue string) (string, error) {
	ciphertext, err := base64.StdEncoding.DecodeString(ciphertextValue)
	if err != nil {
		return "", err
	}
	nonce, err := base64.StdEncoding.DecodeString(nonceValue)
	if err != nil {
		return "", err
	}
	block, err := aes.NewCipher(credentialKey(secret))
	if err != nil {
		return "", err
	}
	gcm, err := cipher.NewGCM(block)
	if err != nil {
		return "", err
	}
	plain, err := gcm.Open(nil, nonce, ciphertext, nil)
	if err != nil {
		return "", err
	}
	return string(plain), nil
}

// firstFreeHost 返回候选池中非失败任务数未达单机实例上限的第一个节点；
// limit < 1 时按 1 兜底（退化为旧的一机一实例语义）。顺序遍历天然形成"先摊开、摊满一轮再叠加"。
func firstFreeHost(hosts []targetHost, occupied []string, limit int) *targetHost {
	if limit < 1 {
		limit = 1
	}
	for i := range hosts {
		if hostTaskCount(occupied, hosts[i].Name) < limit {
			return &hosts[i]
		}
	}
	return nil
}

// hostTaskCount 统计某主机在占用清单（非失败任务的 target_host 列表，含重复）中的出现次数。
func hostTaskCount(occupied []string, name string) int {
	count := 0
	for _, h := range occupied {
		if h == name {
			count++
		}
	}
	return count
}

// 端口池 13306-13999：混合分配模型，用户留空时自动分配，可覆盖为池内指定端口。
const (
	mysqlPortPoolStart = 13306
	mysqlPortPoolEnd   = 13999
)

// allocatePort 在目标主机已占用端口集上做混合分配：指定端口验冲突，未指定则取池内首个空闲端口。
func allocatePort(requested int, used []int) (int, error) {
	taken := make(map[int]bool, len(used))
	for _, p := range used {
		taken[p] = true
	}
	if requested != 0 {
		if taken[requested] {
			return 0, fmt.Errorf("mysql_port %d is already allocated on the target host", requested)
		}
		return requested, nil
	}
	for p := mysqlPortPoolStart; p <= mysqlPortPoolEnd; p++ {
		if !taken[p] {
			return p, nil
		}
	}
	return 0, fmt.Errorf("mysql port pool %d-%d is exhausted on the target host", mysqlPortPoolStart, mysqlPortPoolEnd)
}

type DeliveryService struct {
	db          *gorm.DB
	cfg         config.Config
	awx         *AWXClient
	audit       *AuditService
	executionMu sync.Mutex
	streamMu    sync.Mutex
	streams     map[string]map[chan DeliveryTaskSnapshot]struct{}
}

func (s *DeliveryService) DB() *gorm.DB { return s.db }

func NewDeliveryService(cfg config.Config, db *gorm.DB, audit *AuditService) *DeliveryService {
	return &DeliveryService{db: db, cfg: cfg, awx: NewAWXClient(cfg.AWXBaseURL, cfg.AWXToken, cfg.AWXUsername, cfg.AWXPassword), audit: audit, streams: make(map[string]map[chan DeliveryTaskSnapshot]struct{})}
}

func (s *DeliveryService) ListTargets(ctx context.Context, component string) ([]DeliveryTarget, error) {
	templates, err := s.awx.ListJobTemplates(ctx)
	if err != nil {
		return nil, err
	}
	component = strings.ToLower(strings.TrimSpace(component))
	var targets []DeliveryTarget
	for _, template := range templates {
		if template.Inventory == 0 {
			continue
		}
		if component != "" && component != "all" {
			text := strings.ToLower(template.Name + " " + template.Description)
			if !strings.Contains(text, component) {
				continue
			}
		}
		target, err := s.awxDeliveryTarget(ctx, template)
		if err != nil {
			continue
		}
		targets = append(targets, target)
	}
	return targets, nil
}

func (s *DeliveryService) getTarget(ctx context.Context, templateID uint64) (DeliveryTarget, error) {
	template, err := s.awx.GetJobTemplate(ctx, templateID)
	if err != nil {
		return DeliveryTarget{}, fmt.Errorf("deployment target is unavailable: %w", err)
	}
	return s.awxDeliveryTarget(ctx, *template)
}

func (s *DeliveryService) ListHostMountPaths(ctx context.Context, targetID uint64, hostName string) ([]DeliveryMountPath, error) {
	if targetID == 0 {
		return nil, fmt.Errorf("target_id is required")
	}
	if hostName == "" || len(hostName) > 253 || !hostNamePattern.MatchString(hostName) {
		return nil, fmt.Errorf("host must be a valid inventory host name")
	}
	template, err := s.awx.GetJobTemplate(ctx, targetID)
	if err != nil {
		return nil, fmt.Errorf("deployment target is unavailable: %w", err)
	}
	hosts, err := s.awx.ListInventoryHosts(ctx, template.Inventory)
	if err != nil {
		return nil, err
	}
	var matched *AWXInventoryHost
	for i := range hosts {
		if hosts[i].Enabled && hosts[i].Name == hostName {
			matched = &hosts[i]
			break
		}
	}
	if matched == nil {
		return nil, fmt.Errorf("host %q is not in the deployment target inventory", hostName)
	}
	if s.cfg.AWXFactsTemplateID != 0 {
		if err := s.refreshHostFacts(ctx, hostName); err != nil {
			return nil, err
		}
	}
	facts, err := s.awx.GetHostFacts(ctx, matched.ID)
	if err != nil {
		return nil, err
	}
	items := mountPathsFromFacts(facts)
	if items == nil {
		items = []DeliveryMountPath{}
	}
	return items, nil
}

func (s *DeliveryService) refreshHostFacts(ctx context.Context, hostName string) error {
	job, err := s.awx.Launch(ctx, s.cfg.AWXFactsTemplateID, AWXLaunchRequest{
		Limit: hostName,
	})
	if err != nil {
		return fmt.Errorf("launch AWX facts job: %w", err)
	}
	done, err := s.awx.WaitJob(ctx, strconv.FormatUint(job.ID, 10), time.Duration(s.cfg.AWXFactsTimeoutSeconds)*time.Second)
	if err != nil {
		return err
	}
	if done.Status != "successful" || done.Failed {
		return fmt.Errorf("AWX facts job %d finished with status %s", done.ID, done.Status)
	}
	return nil
}

func (s *DeliveryService) awxDeliveryTarget(ctx context.Context, template AWXJobTemplate) (DeliveryTarget, error) {
	hosts, err := s.awx.ListInventoryHosts(ctx, template.Inventory)
	if err != nil {
		return DeliveryTarget{}, err
	}
	meta := targetMetadata{Topology: "standalone", MySQLPort: 3307}
	for _, host := range hosts {
		if !host.Enabled {
			continue
		}
		meta.Hosts = append(meta.Hosts, targetHost{Name: host.Name, IP: AWXHostIP(host)})
	}
	raw, err := json.Marshal(meta)
	if err != nil {
		return DeliveryTarget{}, err
	}
	return DeliveryTarget{
		ID:             template.ID,
		Name:           template.Name,
		TargetType:     "k8s",
		AWXInventoryID: template.Inventory,
		AWXTemplateID:  template.ID,
		Enabled:        true,
		Metadata:       string(raw),
	}, nil
}

func (s *DeliveryService) CreateTask(ctx context.Context, userID uint64, isAdmin bool, idempotencyKey string, input MySQLDeliveryInput) (*model.DeliveryTask, bool, error) {
	idempotencyKey = strings.TrimSpace(idempotencyKey)
	if idempotencyKey == "" || len(idempotencyKey) > 128 {
		return nil, false, fmt.Errorf("Idempotency-Key header is required and must not exceed 128 characters")
	}
	normalizeMySQLDeliveryInput(&input)
	if err := validateDeliveryInput(input, s.cfg.DeliveryDataDisks); err != nil {
		return nil, false, err
	}
	credentialInput := map[string]string{
		"root@localhost": strings.TrimSpace(input.MySQLRootPassword),
		"xinfra_admin@%": strings.TrimSpace(input.MySQLAdminPassword),
	}
	hasCredentialInput := credentialInput["root@localhost"] != "" || credentialInput["xinfra_admin@%"] != ""
	if hasCredentialInput {
		if credentialInput["root@localhost"] == "" || credentialInput["xinfra_admin@%"] == "" {
			return nil, false, fmt.Errorf("mysql_root_password and mysql_admin_password must be provided together")
		}
		if len(credentialInput["root@localhost"]) < 16 || len(credentialInput["xinfra_admin@%"]) < 16 {
			return nil, false, fmt.Errorf("mysql passwords must be at least 16 characters")
		}
	}
	input.MySQLRootPassword = ""
	input.MySQLAdminPassword = ""

	var existing model.DeliveryTask
	if err := s.db.WithContext(ctx).Where("idempotency_key = ?", idempotencyKey).First(&existing).Error; err == nil {
		if existing.RequestedBy != userID {
			return nil, false, fmt.Errorf("idempotency key is already in use by another user")
		}
		return &existing, true, nil
	} else if !errors.Is(err, gorm.ErrRecordNotFound) {
		return nil, false, err
	}

	target, err := s.getTarget(ctx, input.TargetID)
	if err != nil {
		return nil, false, err
	}
	if target.TargetType != "k8s" {
		return nil, false, fmt.Errorf("target type %q is not supported in the MVP", target.TargetType)
	}
	if !isAdmin {
		var count int64
		if err := s.db.WithContext(ctx).Model(&model.BusinessLineUser{}).
			Where("business_line_id = ? AND user_id = ?", input.BusinessLineID, userID).Count(&count).Error; err != nil {
			return nil, false, err
		}
		if count == 0 {
			return nil, false, fmt.Errorf("user is not authorized for this business line")
		}
	}
	if input.MySQLVersion == "" {
		input.MySQLVersion = "8.0"
	}
	if input.Topology == "" {
		input.Topology = "standalone"
	}
	if input.DataDisk == "" {
		if len(s.cfg.DeliveryDataDisks) > 0 {
			input.DataDisk = s.cfg.DeliveryDataDisks[0]
		} else {
			input.DataDisk = "/data"
		}
	}
	payload := deliveryPayload{MySQLDeliveryInput: input, TargetType: target.TargetType}
	raw, err := json.Marshal(payload)
	if err != nil {
		return nil, false, err
	}
	digest := sha256.Sum256(raw)
	task := model.DeliveryTask{
		ID:               randomUUID(),
		BusinessLineID:   input.BusinessLineID,
		RequestedBy:      userID,
		Component:        "mysql",
		TargetType:       target.TargetType,
		TargetID:         target.ID,
		Namespace:        input.Namespace,
		InstanceName:     input.InstanceName,
		Status:           model.TaskPending,
		ImmutablePayload: string(raw),
		PayloadHash:      hex.EncodeToString(digest[:]),
		IdempotencyKey:   idempotencyKey,
	}
	if err := s.db.WithContext(ctx).Create(&task).Error; err != nil {
		if lookupErr := s.db.WithContext(ctx).Where("idempotency_key = ?", idempotencyKey).First(&existing).Error; lookupErr == nil {
			return &existing, true, nil
		}
		return nil, false, err
	}
	if hasCredentialInput {
		if err := s.storeDeploymentCredentials(ctx, &task, credentialInput, "pending"); err != nil {
			return nil, false, err
		}
	}
	_ = s.db.WithContext(ctx).Create(&model.TaskEvent{TaskID: task.ID, ToState: model.TaskPending, Message: "delivery task created"}).Error
	return &task, false, nil
}

// 版本白名单与 playbook 的 mysql_package_map 保持同步：
// 8.0 走 Ubuntu 自带源，8.4 走 MySQL 官方 APT 源；5.6/5.7 已 EOL 且无 noble 包，不支持。
var supportedMySQLVersions = map[string]bool{"8.0": true, "8.4": true}

// 拓扑白名单：playbook 已支持 primary_replica/mgr_3 的配置渲染，
// 但调度器仍是单主机模型且复制编排未自动化，本期仅放开 standalone。
var supportedTopologies = map[string]bool{"standalone": true}

const rollbackLaunchTimeout = 2 * time.Minute

var supportedCharsets = map[string]bool{"utf8mb4": true, "utf8": true, "gbk": true, "latin1": true}

// 高级参数档位白名单（与 docs/mysql-parameter-selection.md 保持一致）
var (
	supportedMaxConnections   = map[string]bool{"auto": true, "200": true, "500": true, "1000": true, "2000": true, "4000": true, "8000": true, "16000": true}
	supportedLogSizes         = map[string]bool{"128M": true, "256M": true, "512M": true, "1G": true}
	supportedRedoLogSizes     = map[string]bool{"auto": true, "128M": true, "256M": true, "512M": true, "1G": true}
	supportedIOCapacities     = map[int]bool{200: true, 2000: true, 5000: true}
	supportedLongQueryTimes   = map[float64]bool{0.5: true, 1: true, 2: true, 5: true, 10: true}
	supportedBinlogExpireSecs = map[int64]bool{86400: true, 259200: true, 604800: true, 1209600: true}
)

// timezone 仅接受偏移量（±HH:MM）、SYSTEM 或命名时区（如 Asia/Shanghai）。
var timezonePattern = regexp.MustCompile(`^([+-](0\d|1[0-4]):[0-5]\d|SYSTEM|[A-Za-z]+(?:/[A-Za-z0-9_+-]+)+)$`)

func validateDeliveryInput(input MySQLDeliveryInput, _ []string) error {
	if len(input.Namespace) > 63 || !dnsLabelPattern.MatchString(input.Namespace) {
		return fmt.Errorf("namespace must be a valid Kubernetes DNS label")
	}
	if len(input.InstanceName) > 63 || !dnsLabelPattern.MatchString(input.InstanceName) {
		return fmt.Errorf("instance_name must be a valid Kubernetes DNS label")
	}
	// 与文档目标态一致（memory 2048-65536 MiB / storage 20-2000 GiB），playbook assert 同步。
	if input.CPUMilli < 100 || input.CPUMilli > 64000 || input.MemoryMi < 2048 || input.MemoryMi > 65536 || input.StorageGi < 20 || input.StorageGi > 2000 {
		return fmt.Errorf("requested resources are outside the supported range (memory: 2048-65536 MiB, storage: 20-2000 GiB)")
	}
	if input.MySQLVersion != "" && !supportedMySQLVersions[input.MySQLVersion] {
		return fmt.Errorf("unsupported mysql_version %q, supported: 8.0, 8.4 (5.6/5.7 are EOL and have no Ubuntu 24.04 packages)", input.MySQLVersion)
	}
	if input.Topology != "" && !supportedTopologies[input.Topology] {
		return fmt.Errorf("unsupported topology %q, supported: standalone (primary_replica/mgr_3 pending scheduler support)", input.Topology)
	}
	if input.MySQLPort != 0 && (input.MySQLPort < mysqlPortPoolStart || input.MySQLPort > mysqlPortPoolEnd) {
		return fmt.Errorf("mysql_port must be left empty for auto allocation or within the pool %d-%d", mysqlPortPoolStart, mysqlPortPoolEnd)
	}
	if input.TargetHost != "" && (len(input.TargetHost) > 253 || !hostNamePattern.MatchString(input.TargetHost)) {
		return fmt.Errorf("target_host must be a valid inventory host name")
	}
	if input.Timezone != "" && !timezonePattern.MatchString(input.Timezone) {
		return fmt.Errorf("timezone must be an offset like +08:00, SYSTEM, or a named zone like Asia/Shanghai")
	}
	if input.LowerCaseTableNames != nil && *input.LowerCaseTableNames != 0 && *input.LowerCaseTableNames != 1 {
		return fmt.Errorf("lower_case_table_names must be 0 or 1")
	}
	if input.CharacterSet != "" && !supportedCharsets[input.CharacterSet] {
		return fmt.Errorf("unsupported character_set %q, supported: utf8mb4, utf8, gbk, latin1", input.CharacterSet)
	}
	if input.Collation != "" {
		charset := input.CharacterSet
		if charset == "" {
			charset = "utf8mb4"
		}
		if !strings.HasPrefix(input.Collation, charset+"_") {
			return fmt.Errorf("collation %q does not match character_set %q", input.Collation, charset)
		}
	}
	if input.MaxConnections != "" && !supportedMaxConnections[input.MaxConnections] {
		return fmt.Errorf("max_connections must be one of auto, 200, 500, 1000, 2000, 4000, 8000, 16000")
	}
	if input.InnodbRedoLogCapacity != "" && !supportedRedoLogSizes[input.InnodbRedoLogCapacity] {
		return fmt.Errorf("innodb_redo_log_capacity must be one of auto, 128M, 256M, 512M, 1G")
	}
	if input.InnodbFlushLogAtTrxCommit != nil && (*input.InnodbFlushLogAtTrxCommit < 0 || *input.InnodbFlushLogAtTrxCommit > 2) {
		return fmt.Errorf("innodb_flush_log_at_trx_commit must be 0, 1 or 2")
	}
	if input.SyncBinlog != nil && *input.SyncBinlog != 0 && *input.SyncBinlog != 1 {
		return fmt.Errorf("sync_binlog must be 0 or 1")
	}
	if input.InnodbIOCapacity != 0 && !supportedIOCapacities[input.InnodbIOCapacity] {
		return fmt.Errorf("innodb_io_capacity must be one of 200, 2000, 5000")
	}
	if input.LongQueryTime != 0 && !supportedLongQueryTimes[input.LongQueryTime] {
		return fmt.Errorf("long_query_time must be one of 0.5, 1, 2, 5, 10")
	}
	if input.BinlogExpireLogsSeconds != 0 && !supportedBinlogExpireSecs[input.BinlogExpireLogsSeconds] {
		return fmt.Errorf("binlog_expire_logs_seconds must be one of 86400, 259200, 604800, 1209600")
	}
	if input.MaxBinlogSize != "" && !supportedLogSizes[input.MaxBinlogSize] {
		return fmt.Errorf("max_binlog_size must be one of 128M, 256M, 512M, 1G")
	}
	return nil
}

func normalizeMySQLDeliveryInput(input *MySQLDeliveryInput) {
	input.Namespace = normalizeDNSLabel(input.Namespace)
	input.InstanceName = normalizeDNSLabel(input.InstanceName)
	input.TargetHost = strings.TrimSpace(input.TargetHost)
	input.DataDisk = strings.TrimSpace(input.DataDisk)
	if input.MySQLVersion == "" {
		input.MySQLVersion = "8.0"
	}
	if input.Topology == "" {
		input.Topology = "standalone"
	}
	if input.CPUMilli == 0 && input.CPUCores != 0 {
		input.CPUMilli = input.CPUCores * 1000
	}
	if input.MemoryMi == 0 && input.MemoryGB != 0 {
		input.MemoryMi = input.MemoryGB * 1024
	}
	if input.StorageGi == 0 && input.StorageGB != 0 {
		input.StorageGi = input.StorageGB
	}
}

func normalizeDNSLabel(value string) string {
	value = strings.ToLower(strings.TrimSpace(value))
	value = regexp.MustCompile(`[^a-z0-9-]+`).ReplaceAllString(value, "-")
	value = regexp.MustCompile(`-+`).ReplaceAllString(value, "-")
	value = strings.Trim(value, "-")
	if len(value) > 63 {
		value = strings.Trim(value[:63], "-")
	}
	return value
}

func (s *DeliveryService) ListTasks(ctx context.Context, userID uint64, isAdmin bool, filter DeliveryTaskListFilter) ([]model.DeliveryTask, error) {
	query := s.db.WithContext(ctx).Order("created_at DESC")
	if filter.BusinessLineID != 0 {
		query = query.Where("business_line_id = ?", filter.BusinessLineID)
	}
	if filter.Component != "" {
		query = query.Where("component = ?", filter.Component)
	}
	if filter.ActiveOnly {
		query = query.Where("status NOT IN ?", terminalTaskStatuses())
	}
	if !isAdmin {
		query = query.Where("business_line_id IN (?)", s.db.Model(&model.BusinessLineUser{}).Select("business_line_id").Where("user_id = ?", userID))
	}
	var tasks []model.DeliveryTask
	return tasks, query.Find(&tasks).Error
}

func (s *DeliveryService) ListMySQLServiceLedger(ctx context.Context, userID uint64, isAdmin bool, businessLineID uint64) ([]MySQLServiceLedgerItem, error) {
	if businessLineID == 0 {
		return nil, fmt.Errorf("business_line_id is required")
	}
	if !isAdmin {
		var count int64
		if err := s.db.WithContext(ctx).Model(&model.BusinessLineUser{}).
			Where("business_line_id = ? AND user_id = ?", businessLineID, userID).
			Count(&count).Error; err != nil {
			return nil, err
		}
		if count == 0 {
			return nil, fmt.Errorf("user is not authorized for this business line")
		}
	}
	var businessLine model.BusinessLine
	if err := s.db.WithContext(ctx).First(&businessLine, "id = ?", businessLineID).Error; err != nil {
		return nil, err
	}
	var instances []model.DeploymentResult
	if err := s.db.WithContext(ctx).
		Where("business_line_id = ? AND component = ? AND service_type = ? AND status = ?", businessLineID, "mysql", "database", "active").
		Order("created_at DESC").
		Find(&instances).Error; err != nil {
		return nil, err
	}
	taskIDs := make([]string, 0, len(instances))
	for _, instance := range instances {
		taskIDs = append(taskIDs, instance.TaskID)
	}
	taskStatuses := map[string]string{}
	if len(taskIDs) > 0 {
		var tasks []model.DeliveryTask
		if err := s.db.WithContext(ctx).Select("id", "status").Where("id IN ?", taskIDs).Find(&tasks).Error; err != nil {
			return nil, err
		}
		for _, task := range tasks {
			taskStatuses[task.ID] = task.Status
		}
	}
	items := make([]MySQLServiceLedgerItem, 0, len(instances))
	for _, instance := range instances {
		status := "健康"
		statusClass := "ok"
		healthy := 1
		if taskStatuses[instance.TaskID] == model.TaskRegisterFailed {
			status = "注册异常"
			statusClass = "warn"
		}
		if instance.Host == "" || instance.Port == 0 {
			status = "部分异常"
			statusClass = "warn"
			healthy = 0
		}
		items = append(items, MySQLServiceLedgerItem{
			Name:        instance.InstanceName,
			Datacenter:  instance.NodeName,
			BusinessTag: businessLine.Name,
			Instances:   1,
			Healthy:     healthy,
			Address:     fmt.Sprintf("%s:%d", instance.Host, instance.Port),
			Status:      status,
			StatusClass: statusClass,
			Version:     instance.Version,
			Namespace:   instance.Namespace,
		})
	}
	return items, nil
}

func (s *DeliveryService) storeDeploymentCredentials(ctx context.Context, task *model.DeliveryTask, credentials map[string]string, status string) error {
	for key, password := range credentials {
		username, host, ok := strings.Cut(key, "@")
		if !ok || strings.TrimSpace(username) == "" {
			return fmt.Errorf("invalid credential account %q", key)
		}
		ciphertext, nonce, err := encryptCredential(s.cfg.DeliveryCredentialSecret, password)
		if err != nil {
			return err
		}
		item := model.DeploymentCredential{
			TaskID:         task.ID,
			BusinessLineID: task.BusinessLineID,
			Component:      task.Component,
			InstanceName:   task.InstanceName,
			Username:       strings.TrimSpace(username),
			AccountHost:    strings.TrimSpace(host),
			Ciphertext:     ciphertext,
			Nonce:          nonce,
			Status:         status,
			Source:         "user_input",
		}
		if item.Component == "" {
			item.Component = "mysql"
		}
		if err := s.db.WithContext(ctx).Create(&item).Error; err != nil {
			return err
		}
	}
	return nil
}

func (s *DeliveryService) deploymentCredentialVars(ctx context.Context, taskID string) (map[string]string, error) {
	var items []model.DeploymentCredential
	if err := s.db.WithContext(ctx).
		Where("task_id = ? AND status IN ?", taskID, []string{"pending", "available"}).
		Find(&items).Error; err != nil {
		return nil, err
	}
	if len(items) == 0 {
		return map[string]string{}, nil
	}
	values := map[string]string{}
	for _, item := range items {
		password, err := decryptCredential(s.cfg.DeliveryCredentialSecret, item.Ciphertext, item.Nonce)
		if err != nil {
			return nil, err
		}
		switch item.Username + "@" + item.AccountHost {
		case "root@localhost":
			values["mysql_root_password"] = password
		case "xinfra_admin@%":
			values["mysql_admin_password"] = password
		}
	}
	if values["mysql_root_password"] == "" || values["mysql_admin_password"] == "" {
		return nil, fmt.Errorf("deployment credentials are missing for task %s", taskID)
	}
	return values, nil
}

func (s *DeliveryService) GetTask(ctx context.Context, taskID string, userID uint64, isAdmin bool) (*model.DeliveryTask, []model.TaskEvent, error) {
	query := s.db.WithContext(ctx).Where("id = ?", taskID)
	if !isAdmin {
		query = query.Where("business_line_id IN (?)", s.db.Model(&model.BusinessLineUser{}).Select("business_line_id").Where("user_id = ?", userID))
	}
	var task model.DeliveryTask
	if err := query.First(&task).Error; err != nil {
		return nil, nil, err
	}
	var events []model.TaskEvent
	if err := s.db.WithContext(ctx).Where("task_id = ?", taskID).Order("id ASC").Find(&events).Error; err != nil {
		return nil, nil, err
	}
	return &task, events, nil
}

func (s *DeliveryService) RevealDeploymentCredentials(ctx context.Context, taskID string, userID uint64, isAdmin bool) ([]DeploymentCredentialView, error) {
	task, _, err := s.GetTask(ctx, taskID, userID, isAdmin)
	if err != nil {
		return nil, err
	}
	if task.Status != model.TaskFinished && task.Status != model.TaskRegisterFailed {
		return nil, fmt.Errorf("task credentials are available only after a successful deployment")
	}
	var out []DeploymentCredentialView
	err = s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
		var credentials []model.DeploymentCredential
		if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).
			Where("task_id = ? AND status = ?", taskID, "available").
			Order("id ASC").
			Find(&credentials).Error; err != nil {
			return err
		}
		if len(credentials) == 0 {
			return gorm.ErrRecordNotFound
		}
		for _, credential := range credentials {
			password, err := decryptCredential(s.cfg.DeliveryCredentialSecret, credential.Ciphertext, credential.Nonce)
			if err != nil {
				return err
			}
			out = append(out, DeploymentCredentialView{
				Username: credential.Username,
				Host:     credential.AccountHost,
				Password: password,
			})
		}
		ids := make([]uint64, 0, len(credentials))
		for _, credential := range credentials {
			ids = append(ids, credential.ID)
		}
		now := time.Now()
		return tx.Model(&model.DeploymentCredential{}).Where("id IN ?", ids).Updates(map[string]any{
			"status":     "viewed",
			"viewed_by":  userID,
			"viewed_at":  now,
			"updated_at": now,
		}).Error
	})
	if err != nil {
		return nil, err
	}
	return out, nil
}

func (s *DeliveryService) SubscribeTask(taskID string) (<-chan DeliveryTaskSnapshot, func()) {
	ch := make(chan DeliveryTaskSnapshot, 8)
	s.streamMu.Lock()
	if s.streams[taskID] == nil {
		s.streams[taskID] = make(map[chan DeliveryTaskSnapshot]struct{})
	}
	s.streams[taskID][ch] = struct{}{}
	s.streamMu.Unlock()
	cancel := func() {
		s.streamMu.Lock()
		if subscribers := s.streams[taskID]; subscribers != nil {
			delete(subscribers, ch)
			if len(subscribers) == 0 {
				delete(s.streams, taskID)
			}
		}
		s.streamMu.Unlock()
		close(ch)
	}
	return ch, cancel
}

func (s *DeliveryService) taskSnapshot(ctx context.Context, taskID string) (DeliveryTaskSnapshot, error) {
	task, events, err := s.GetTask(ctx, taskID, 0, true)
	if err != nil {
		return DeliveryTaskSnapshot{}, err
	}
	return DeliveryTaskSnapshot{Task: task, Events: events}, nil
}

func (s *DeliveryService) broadcastTask(ctx context.Context, taskID string) {
	snapshot, err := s.taskSnapshot(ctx, taskID)
	if err != nil {
		return
	}
	s.streamMu.Lock()
	defer s.streamMu.Unlock()
	for ch := range s.streams[taskID] {
		select {
		case ch <- snapshot:
		default:
		}
	}
}

func (s *DeliveryService) AWXJobStdout(ctx context.Context, jobID string) (string, error) {
	return s.awx.JobStdout(ctx, jobID)
}

func (s *DeliveryService) Cancel(ctx context.Context, taskID string, userID uint64, isAdmin bool) error {
	task, _, err := s.GetTask(ctx, taskID, userID, isAdmin)
	if err != nil {
		return err
	}
	if task.Status == model.TaskPending {
		return s.transition(ctx, task, model.TaskCanceled, "canceled before dispatch", "")
	}
	if task.Status != model.TaskRunning && task.Status != model.TaskDispatching {
		return fmt.Errorf("task in state %q cannot be canceled", task.Status)
	}
	var job model.ExecutionJob
	if err := s.db.WithContext(ctx).Where("task_id = ?", task.ID).First(&job).Error; err != nil {
		return err
	}
	if err := s.awx.Cancel(ctx, job.ExecutorJobID); err != nil {
		return err
	}
	return s.transition(ctx, task, model.TaskCanceling, "cancel requested in AWX", "")
}

func (s *DeliveryService) claimAndReserve(ctx context.Context) (*model.DeliveryTask, error) {
	var task model.DeliveryTask
	var payload deliveryPayload
	var target DeliveryTarget
	dispatchable := false
	err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
		if err := tx.Clauses(clause.Locking{Strength: "UPDATE", Options: "SKIP LOCKED"}).Where("status = ?", model.TaskPending).Order("created_at ASC").First(&task).Error; err != nil {
			return err
		}
		var targetErr error
		target, targetErr = s.getTarget(ctx, task.TargetID)
		if targetErr != nil {
			return s.failInTransaction(tx, &task, model.TaskValidationFailed, targetErr.Error())
		}
		if err := json.Unmarshal([]byte(task.ImmutablePayload), &payload); err != nil {
			return s.failInTransaction(tx, &task, model.TaskValidationFailed, "stored deployment payload is invalid")
		}
		activeStates := []string{model.TaskValidating, model.TaskDispatching, model.TaskRunning, model.TaskCanceling}
		checks := []struct {
			query   string
			args    []any
			limit   int
			message string
		}{
			{"status IN ?", []any{activeStates}, s.cfg.DeliveryGlobalLimit, "global concurrency limit reached"},
			{"status IN ? AND target_id = ?", []any{activeStates, task.TargetID}, s.cfg.DeliveryTargetLimit, "target concurrency limit reached"},
			{"status IN ? AND target_id = ? AND namespace = ?", []any{activeStates, task.TargetID, task.Namespace}, 1, "namespace already has an active MySQL delivery"},
		}
		for _, check := range checks {
			var count int64
			if check.limit > 0 {
				if err := tx.Model(&model.DeliveryTask{}).Where(check.query, check.args...).Count(&count).Error; err != nil {
					return err
				}
				if count >= int64(check.limit) {
					return fmt.Errorf("defer: %s", check.message)
				}
			}
		}
		quotaOK, err := checkResourceQuota(tx, task.BusinessLineID, task.TargetID, payload)
		if err != nil {
			return err
		}
		if !quotaOK {
			return s.failInTransaction(tx, &task, model.TaskValidationFailed, "resource quota is insufficient")
		}
		meta := parseTargetMetadata(target.Metadata)
		if len(meta.Hosts) == 0 {
			return s.failInTransaction(tx, &task, model.TaskValidationFailed, "deployment target has no candidate hosts")
		}
		var occupied []string
		occupiedExclude := []string{model.TaskExecutionFailed, model.TaskValidationFailed, model.TaskCanceled}
		if err := tx.Model(&model.DeliveryTask{}).Where("target_id = ? AND target_host <> ? AND status NOT IN ?", task.TargetID, "", occupiedExclude).Pluck("target_host", &occupied).Error; err != nil {
			return err
		}
		limit := s.cfg.DeliveryHostInstanceLimit
		var host *targetHost
		if payload.TargetHost != "" {
			// 点名主机：仅校验池内存在性与单机实例上限，端口/配额/实机守卫照常执行。
			for i := range meta.Hosts {
				if meta.Hosts[i].Name == payload.TargetHost {
					host = &meta.Hosts[i]
					break
				}
			}
			if host == nil {
				return s.failInTransaction(tx, &task, model.TaskValidationFailed, fmt.Sprintf("target_host %q is not in the candidate host pool", payload.TargetHost))
			}
			if effective := max(limit, 1); hostTaskCount(occupied, host.Name) >= effective {
				return fmt.Errorf("defer: pinned host %s reached the per-host instance limit %d", host.Name, effective)
			}
		} else {
			host = firstFreeHost(meta.Hosts, occupied, limit)
		}
		if host == nil {
			return fmt.Errorf("defer: no free host available on target")
		}
		// 端口池混合分配：同主机已占端口 = 非终态任务分配端口 ∪ 存量 active 实例端口。
		var usedPorts []int
		if err := tx.Model(&model.DeliveryTask{}).Where("target_host = ? AND status NOT IN ?", host.Name, occupiedExclude).Pluck("mysql_port", &usedPorts).Error; err != nil {
			return err
		}
		var instancePorts []int
		if err := tx.Model(&model.DeploymentResult{}).
			Where("component = ? AND service_type = ? AND node_name = ? AND status = ?", "mysql", "database", host.Name, "active").
			Pluck("port", &instancePorts).Error; err != nil {
			return err
		}
		port, portErr := allocatePort(payload.MySQLPort, append(usedPorts, instancePorts...))
		if portErr != nil {
			return s.failInTransaction(tx, &task, model.TaskValidationFailed, portErr.Error())
		}
		reservation := model.ResourceReservation{TaskID: task.ID, BusinessLineID: task.BusinessLineID, TargetID: task.TargetID, CPUMilli: payload.CPUMilli, MemoryMi: payload.MemoryMi, StorageGi: payload.StorageGi, InstanceCount: 1, Status: "reserved", ExpiresAt: time.Now().Add(time.Duration(s.cfg.ReservationTTLMinutes) * time.Minute)}
		if err := tx.Create(&reservation).Error; err != nil {
			return err
		}
		if err := tx.Model(&model.DeliveryTask{}).Where("id = ?", task.ID).Updates(map[string]any{"target_host": host.Name, "target_host_ip": host.IP, "mysql_port": port}).Error; err != nil {
			return err
		}
		task.TargetHost = host.Name
		task.TargetHostIP = host.IP
		task.MySQLPort = port
		if err := s.transitionTx(tx, &task, model.TaskDispatching, "resources reserved", ""); err != nil {
			return err
		}
		dispatchable = true
		return nil
	})
	if err == nil && !dispatchable {
		err = gorm.ErrRecordNotFound
	}
	return &task, err
}

func checkResourceQuota(tx *gorm.DB, businessLineID, targetID uint64, payload deliveryPayload) (bool, error) {
	var quota model.ResourceQuota
	if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Where("business_line_id = ? AND target_id = ?", businessLineID, targetID).First(&quota).Error; err != nil {
		if errors.Is(err, gorm.ErrRecordNotFound) {
			return true, nil
		}
		return false, err
	}
	type totals struct{ CPU, Memory, Storage, Instances int64 }
	var used, reserved totals
	if err := tx.Model(&model.ResourceUsage{}).Select("COALESCE(SUM(cpu_milli),0) cpu, COALESCE(SUM(memory_mi),0) memory, COALESCE(SUM(storage_gi),0) storage, COALESCE(SUM(instance_count),0) instances").Where("business_line_id = ? AND target_id = ? AND status = ?", businessLineID, targetID, "active").Scan(&used).Error; err != nil {
		return false, err
	}
	if err := tx.Model(&model.ResourceReservation{}).Select("COALESCE(SUM(cpu_milli),0) cpu, COALESCE(SUM(memory_mi),0) memory, COALESCE(SUM(storage_gi),0) storage, COALESCE(SUM(instance_count),0) instances").Where("business_line_id = ? AND target_id = ? AND (status = ? OR (status = ? AND expires_at > ?))", businessLineID, targetID, "rollback", "reserved", time.Now()).Scan(&reserved).Error; err != nil {
		return false, err
	}
	return used.CPU+reserved.CPU+payload.CPUMilli <= quota.CPUMilli &&
		used.Memory+reserved.Memory+payload.MemoryMi <= quota.MemoryMi &&
		used.Storage+reserved.Storage+payload.StorageGi <= quota.StorageGi &&
		used.Instances+reserved.Instances+1 <= quota.InstanceLimit, nil
}

func (s *DeliveryService) failInTransaction(tx *gorm.DB, task *model.DeliveryTask, status, message string) error {
	if err := s.transitionTx(tx, task, status, message, message); err != nil {
		return err
	}
	return nil
}

func (s *DeliveryService) transition(ctx context.Context, task *model.DeliveryTask, status, message, errorMessage string) error {
	return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { return s.transitionTx(tx, task, status, message, errorMessage) })
}

func (s *DeliveryService) transitionTx(tx *gorm.DB, task *model.DeliveryTask, status, message, errorMessage string) error {
	from := task.Status
	updates := map[string]any{"status": status, "error_message": errorMessage}
	now := time.Now()
	if status == model.TaskRunning {
		updates["started_at"] = now
	}
	if status == model.TaskFinished || status == model.TaskExecutionFailed || status == model.TaskValidationFailed || status == model.TaskCanceled || status == model.TaskRegisterFailed || status == model.TaskRolledBack || status == model.TaskRollbackFailed || status == model.TaskRollbackAck {
		updates["finished_at"] = now
	}
	result := tx.Model(&model.DeliveryTask{}).Where("id = ? AND status = ?", task.ID, from).Updates(updates)
	if result.Error != nil {
		return result.Error
	}
	if result.RowsAffected != 1 {
		return fmt.Errorf("task %s changed concurrently", task.ID)
	}
	task.Status = status
	task.ErrorMessage = errorMessage
	return tx.Create(&model.TaskEvent{TaskID: task.ID, FromState: from, ToState: status, Message: message}).Error
}

func (s *DeliveryService) releaseReservation(tx *gorm.DB, taskID string) error {
	return tx.Model(&model.ResourceReservation{}).Where("task_id = ? AND status = ?", taskID, "reserved").Update("status", "released").Error
}

func randomUUID() string {
	b := make([]byte, 16)
	if _, err := rand.Read(b); err != nil {
		panic(err)
	}
	b[6] = (b[6] & 0x0f) | 0x40
	b[8] = (b[8] & 0x3f) | 0x80
	return fmt.Sprintf("%08x-%04x-%04x-%04x-%012x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:16])
}

func mustJSON(value any) []byte {
	raw, err := json.Marshal(value)
	if err != nil {
		return []byte(`{}`)
	}
	return raw
}

func mysqlReady(ctx context.Context, address string) error {
	dialer := net.Dialer{Timeout: 5 * time.Second}
	conn, err := dialer.DialContext(ctx, "tcp", address)
	if err != nil {
		return err
	}
	return conn.Close()
}

func (s *DeliveryService) DispatchOnce(ctx context.Context) error {
	task, err := s.claimAndReserve(ctx)
	if err != nil {
		if errors.Is(err, gorm.ErrRecordNotFound) || strings.HasPrefix(err.Error(), "defer:") {
			return nil
		}
		return err
	}
	_, _, err = s.CreateExecution(ctx, task.ID, task.PayloadHash, task.IdempotencyKey)
	if err != nil {
		failErr := s.beginRollback(ctx, task.ID, "deployment could not be started: "+err.Error())
		s.broadcastTask(ctx, task.ID)
		return failErr
	}
	s.broadcastTask(ctx, task.ID)
	return nil
}

func (s *DeliveryService) CreateExecution(ctx context.Context, taskID, payloadHash, idempotencyKey string) (*model.ExecutionJob, bool, error) {
	s.executionMu.Lock()
	defer s.executionMu.Unlock()
	var existing model.ExecutionJob
	if err := s.db.WithContext(ctx).Where("task_id = ? OR idempotency_key = ?", taskID, idempotencyKey).First(&existing).Error; err == nil {
		if existing.TaskID != taskID || existing.IdempotencyKey != idempotencyKey {
			return nil, false, fmt.Errorf("idempotency key is already bound to another task")
		}
		return &existing, true, nil
	} else if !errors.Is(err, gorm.ErrRecordNotFound) {
		return nil, false, err
	}
	var task model.DeliveryTask
	if err := s.db.WithContext(ctx).First(&task, "id = ?", taskID).Error; err != nil {
		return nil, false, err
	}
	if task.PayloadHash != payloadHash || task.IdempotencyKey != idempotencyKey {
		return nil, false, fmt.Errorf("execution request does not match the immutable task payload")
	}
	if task.Status != model.TaskDispatching {
		return nil, false, fmt.Errorf("task in state %q is not ready for execution", task.Status)
	}
	target, err := s.getTarget(ctx, task.TargetID)
	if err != nil {
		return nil, false, err
	}
	var payload deliveryPayload
	if err := json.Unmarshal([]byte(task.ImmutablePayload), &payload); err != nil {
		return nil, false, err
	}
	meta := parseTargetMetadata(target.Metadata)
	// Persist execution record BEFORE launching AWX to ensure crash recovery.
	now := time.Now()
	execution := model.ExecutionJob{TaskID: task.ID, IdempotencyKey: task.IdempotencyKey, ExecutorJobID: "pending-" + task.ID, Status: "launching", StartedAt: &now}
	if err := s.db.WithContext(ctx).Create(&execution).Error; err != nil {
		return nil, false, err
	}
	extraVars := deliveryExtraVars(&task, payload, meta)
	credentialVars, err := s.deploymentCredentialVars(ctx, task.ID)
	if err != nil {
		_ = s.db.WithContext(ctx).Model(&execution).Updates(map[string]any{"status": "failed", "finished_at": time.Now()})
		return nil, false, err
	}
	for key, value := range credentialVars {
		extraVars[key] = value
	}
	extraVars["delivery_callback_url"] = s.deliveryCallbackURL(task.ID)
	extraVars["delivery_callback_token"] = s.cfg.AWXWebhookToken
	job, err := s.awx.Launch(ctx, target.AWXTemplateID, AWXLaunchRequest{InventoryID: target.AWXInventoryID, Limit: task.TargetHost, ExtraVars: extraVars})
	if err != nil {
		_ = s.db.WithContext(ctx).Model(&execution).Updates(map[string]any{"status": "failed", "finished_at": time.Now()})
		return nil, false, err
	}
	if err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
		if err := tx.Model(&execution).Updates(map[string]any{"executor_job_id": fmt.Sprint(job.ID), "status": "running"}).Error; err != nil {
			return err
		}
		return s.transitionTx(tx, &task, model.TaskRunning, "AWX job started", "")
	}); err != nil {
		return nil, false, err
	}
	return &execution, false, nil
}

func (s *DeliveryService) deliveryCallbackURL(taskID string) string {
	base := strings.TrimRight(strings.TrimSpace(s.cfg.DeliveryCallbackBaseURL), "/")
	if base == "" {
		return ""
	}
	return base + "/auth/internal/delivery/tasks/" + taskID + "/events"
}

// deliveryExtraVars 组装传给 playbook 的变量：必传项固定注入，
// 选填项仅在用户显式设置时下发，未设置时由 playbook 默认基线兜底。
func deliveryExtraVars(task *model.DeliveryTask, payload deliveryPayload, meta targetMetadata) map[string]any {
	topology := payload.Topology
	if topology == "" {
		// 存量任务的 ImmutablePayload 无 topology 字段，回退到 target metadata。
		topology = meta.Topology
	}
	vars := map[string]any{
		"task_id": task.ID, "payload_hash": task.PayloadHash,
		"target_hosts": task.TargetHost, "topology": topology,
		"instance_name": payload.InstanceName, "mysql_port": task.MySQLPort,
		"memory_mb": payload.MemoryMi, "storage_gb": payload.StorageGi,
		"mysql_version": payload.MySQLVersion,
	}
	if payload.DataDisk != "" {
		vars["data_disk"] = payload.DataDisk
	}
	if payload.Timezone != "" {
		vars["timezone"] = payload.Timezone
	}
	if payload.LowerCaseTableNames != nil {
		vars["lower_case_table_names"] = *payload.LowerCaseTableNames
	}
	if payload.CharacterSet != "" {
		vars["character_set"] = payload.CharacterSet
	}
	if payload.Collation != "" {
		vars["collation"] = payload.Collation
	}
	if payload.MaxConnections != "" {
		vars["max_connections"] = payload.MaxConnections
	}
	if payload.InnodbRedoLogCapacity != "" {
		vars["innodb_redo_log_capacity"] = payload.InnodbRedoLogCapacity
	}
	if payload.InnodbFlushLogAtTrxCommit != nil {
		vars["innodb_flush_log_at_trx_commit"] = *payload.InnodbFlushLogAtTrxCommit
	}
	if payload.SyncBinlog != nil {
		vars["sync_binlog"] = *payload.SyncBinlog
	}
	if payload.InnodbIOCapacity != 0 {
		vars["innodb_io_capacity"] = payload.InnodbIOCapacity
	}
	if payload.LongQueryTime != 0 {
		vars["long_query_time"] = payload.LongQueryTime
	}
	if payload.BinlogExpireLogsSeconds != 0 {
		vars["binlog_expire_logs_seconds"] = payload.BinlogExpireLogsSeconds
	}
	if payload.MaxBinlogSize != "" {
		vars["max_binlog_size"] = payload.MaxBinlogSize
	}
	return vars
}

func (s *DeliveryService) HandleStageEvent(ctx context.Context, taskID string, input DeliveryStageEventInput) error {
	stage := strings.ToLower(strings.TrimSpace(input.Stage))
	status := strings.ToLower(strings.TrimSpace(input.Status))
	if !validDeliveryStage(stage) {
		return fmt.Errorf("invalid delivery stage %q", input.Stage)
	}
	if !validDeliveryStageStatus(status) {
		return fmt.Errorf("invalid delivery stage status %q", input.Status)
	}
	message := strings.TrimSpace(input.Message)
	if message == "" {
		message = stage + " " + status
	}
	eventState := "stage_" + stage + "_" + status
	var failedTaskID string
	err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
		var task model.DeliveryTask
		if err := tx.First(&task, "id = ?", taskID).Error; err != nil {
			return err
		}
		if input.AWXJobID != "" {
			var execution model.ExecutionJob
			if err := tx.Where("task_id = ?", task.ID).First(&execution).Error; err != nil {
				return err
			}
			if execution.ExecutorJobID != strings.TrimSpace(input.AWXJobID) {
				return fmt.Errorf("AWX job %s does not match task %s", input.AWXJobID, task.ID)
			}
		}
		if isTerminalTaskStatus(task.Status) {
			return nil
		}
		if err := tx.Create(&model.TaskEvent{
			TaskID:      task.ID,
			FromState:   task.Status,
			ToState:     eventState,
			Stage:       stage,
			EventStatus: status,
			Message:     message,
		}).Error; err != nil {
			return err
		}
		if status == "failed" {
			now := time.Now()
			if err := tx.Model(&model.ExecutionJob{}).Where("task_id = ?", task.ID).Updates(map[string]any{"status": "failed", "finished_at": now}).Error; err != nil {
				return err
			}
			failedTaskID = task.ID
		}
		return nil
	})
	if err != nil {
		return err
	}
	if failedTaskID != "" {
		if rollbackErr := s.beginRollback(ctx, failedTaskID, message); rollbackErr != nil {
			return rollbackErr
		}
	}
	s.broadcastTask(ctx, taskID)
	return nil
}

func (s *DeliveryService) HandleAWXJobNotification(ctx context.Context, input AWXJobNotificationInput) error {
	status := strings.ToLower(strings.TrimSpace(input.Status))
	if status == "" && input.ID == 0 && awxNotificationTaskID(input.ExtraVars) == "" {
		return nil
	}
	if status == "" {
		return fmt.Errorf("missing AWX job status")
	}
	execution, err := s.findExecutionForAWXNotification(ctx, input)
	if err != nil {
		return err
	}
	message := awxNotificationMessage(input)
	defer s.broadcastTask(ctx, execution.TaskID)
	switch status {
	case "pending", "waiting", "running", "new":
		return s.recordAWXEvent(ctx, execution.TaskID, status, message)
	case "successful":
		s.finishExecution(ctx, execution, "successful")
		if err := s.completeTask(ctx, execution.TaskID); err != nil {
			return s.beginRollback(ctx, execution.TaskID, err.Error())
		}
		return nil
	case "canceled", "cancelled":
		s.finishExecution(ctx, execution, "canceled")
		return s.failTask(ctx, &model.DeliveryTask{ID: execution.TaskID, Status: model.TaskRunning}, model.TaskCanceled, message)
	case "failed", "error":
		s.finishExecution(ctx, execution, "failed")
		return s.beginRollback(ctx, execution.TaskID, message)
	default:
		s.finishExecution(ctx, execution, "failed")
		return s.beginRollback(ctx, execution.TaskID, "AWX job finished with status "+status)
	}
}

func (s *DeliveryService) findExecutionForAWXNotification(ctx context.Context, input AWXJobNotificationInput) (*model.ExecutionJob, error) {
	var execution model.ExecutionJob
	if input.ID != 0 {
		if err := s.db.WithContext(ctx).Where("executor_job_id = ?", fmt.Sprint(input.ID)).First(&execution).Error; err == nil {
			return &execution, nil
		} else if !errors.Is(err, gorm.ErrRecordNotFound) {
			return nil, err
		}
	}
	taskID := awxNotificationTaskID(input.ExtraVars)
	if taskID == "" {
		return nil, fmt.Errorf("AWX notification did not include a known job id or task_id")
	}
	if err := s.db.WithContext(ctx).Where("task_id = ?", taskID).First(&execution).Error; err != nil {
		return nil, err
	}
	return &execution, nil
}

func (s *DeliveryService) recordAWXEvent(ctx context.Context, taskID, status, message string) error {
	var task model.DeliveryTask
	if err := s.db.WithContext(ctx).First(&task, "id = ?", taskID).Error; err != nil {
		return err
	}
	if isTerminalTaskStatus(task.Status) {
		return nil
	}
	return s.db.WithContext(ctx).Create(&model.TaskEvent{
		TaskID:      task.ID,
		FromState:   task.Status,
		ToState:     "awx_" + status,
		Stage:       "awx",
		EventStatus: status,
		Message:     message,
	}).Error
}

func validDeliveryStage(stage string) bool {
	switch stage {
	case "precheck", "install", "configure", "healthcheck", "register":
		return true
	default:
		return false
	}
}

func validDeliveryStageStatus(status string) bool {
	switch status {
	case "running", "success", "failed":
		return true
	default:
		return false
	}
}

func isTerminalTaskStatus(status string) bool {
	for _, item := range terminalTaskStatuses() {
		if status == item {
			return true
		}
	}
	return false
}

func terminalTaskStatuses() []string {
	return []string{
		model.TaskFinished,
		model.TaskExecutionFailed,
		model.TaskValidationFailed,
		model.TaskRegisterFailed,
		model.TaskCanceled,
		model.TaskRolledBack,
		model.TaskRollbackFailed,
		model.TaskRollbackAck,
	}
}

func awxNotificationMessage(input AWXJobNotificationInput) string {
	status := strings.TrimSpace(input.Status)
	name := strings.TrimSpace(input.Name)
	if strings.TrimSpace(input.Traceback) != "" {
		return strings.TrimSpace(input.Traceback)
	}
	if name == "" {
		return "AWX job " + status
	}
	return "AWX job " + name + " " + status
}

func awxNotificationTaskID(extraVars any) string {
	var values map[string]any
	switch v := extraVars.(type) {
	case map[string]any:
		values = v
	case string:
		if strings.TrimSpace(v) == "" {
			return ""
		}
		_ = json.Unmarshal([]byte(v), &values)
	default:
		raw, err := json.Marshal(v)
		if err != nil {
			return ""
		}
		_ = json.Unmarshal(raw, &values)
	}
	if values == nil {
		return ""
	}
	if taskID, ok := values["task_id"].(string); ok {
		return strings.TrimSpace(taskID)
	}
	return ""
}

func (s *DeliveryService) finishExecution(ctx context.Context, execution *model.ExecutionJob, status string) {
	now := time.Now()
	_ = s.db.WithContext(ctx).Model(execution).Updates(map[string]any{"status": status, "finished_at": now})
}

func (s *DeliveryService) PollOnce(ctx context.Context) error {
	var jobs []model.ExecutionJob
	if err := s.db.WithContext(ctx).Where("status = ?", "running").Find(&jobs).Error; err != nil {
		return err
	}
	for _, execution := range jobs {
		job, err := s.awx.GetJob(ctx, execution.ExecutorJobID)
		if err != nil {
			// A transient AWX/API failure is not evidence that deployment failed.
			// Keep the job running and retry on the next scheduler tick.
			continue
		}
		switch strings.ToLower(job.Status) {
		case "pending", "waiting", "running", "new":
			continue
		case "successful":
			s.finishExecution(ctx, &execution, "successful")
			if err := s.completeTask(ctx, execution.TaskID); err != nil {
				_ = s.beginRollback(ctx, execution.TaskID, err.Error())
			}
		case "canceled":
			s.finishExecution(ctx, &execution, "canceled")
			_ = s.beginRollback(ctx, execution.TaskID, "AWX deployment job was canceled")
		default:
			s.finishExecution(ctx, &execution, "failed")
			_ = s.beginRollback(ctx, execution.TaskID, "AWX job finished with status "+job.Status)
		}
	}
	return nil
}

func (s *DeliveryService) completeTask(ctx context.Context, taskID string) error {
	var task model.DeliveryTask
	if err := s.db.WithContext(ctx).First(&task, "id = ?", taskID).Error; err != nil {
		return err
	}
	var payload deliveryPayload
	if err := json.Unmarshal([]byte(task.ImmutablePayload), &payload); err != nil {
		return err
	}
	addr := fmt.Sprintf("%s:%d", task.TargetHostIP, task.MySQLPort)
	if err := mysqlReady(ctx, addr); err != nil {
		return fmt.Errorf("MySQL health check failed: %w", err)
	}
	now := time.Now()
	if err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
		if err := s.transitionTx(tx, &task, model.TaskRegistering, "AWX succeeded and MySQL health check passed", ""); err != nil {
			return err
		}
		result := model.DeploymentResult{
			TaskID:         task.ID,
			BusinessLineID: task.BusinessLineID,
			Component:      task.Component,
			ServiceType:    "database",
			InstanceName:   payload.InstanceName,
			Namespace:      payload.Namespace,
			TargetID:       task.TargetID,
			NodeName:       task.TargetHost,
			Host:           task.TargetHostIP,
			Port:           task.MySQLPort,
			Version:        payload.MySQLVersion,
			Status:         "active",
			Metadata:       string(mustJSON(map[string]any{"data_disk": payload.DataDisk})),
		}
		if result.Component == "" {
			result.Component = "mysql"
		}
		if err := tx.Create(&result).Error; err != nil {
			return err
		}
		if err := tx.Model(&model.DeploymentCredential{}).Where("task_id = ? AND status = ?", task.ID, "pending").Updates(map[string]any{"deployment_result_id": result.ID, "status": "available"}).Error; err != nil {
			return err
		}
		if err := tx.Create(&model.ResourceUsage{TaskID: task.ID, InstanceID: result.ID, BusinessLineID: task.BusinessLineID, TargetID: task.TargetID, CPUMilli: payload.CPUMilli, MemoryMi: payload.MemoryMi, StorageGi: payload.StorageGi, InstanceCount: 1, Status: "active"}).Error; err != nil {
			return err
		}
		if err := tx.Model(&model.ResourceReservation{}).Where("task_id = ? AND status = ?", task.ID, "reserved").Update("status", "consumed").Error; err != nil {
			return err
		}
		if err := tx.Model(&model.ExecutionJob{}).Where("task_id = ?", task.ID).Updates(map[string]any{"status": "successful", "finished_at": now}).Error; err != nil {
			return err
		}
		return nil
	}); err != nil {
		return err
	}
	if err := s.RegisterCloudDM(ctx, task.ID); err != nil {
		message := fmt.Sprintf("CloudDM registration failed: %v", err)
		// The database is already healthy and accounted for at this point. A
		// registration outage must not trigger destructive instance rollback.
		if transitionErr := s.transition(ctx, &task, model.TaskRegisterFailed,
			"MySQL delivered; CloudDM registration failed and can be retried", message); transitionErr != nil {
			return fmt.Errorf("%s; cannot record register_failed: %w", message, transitionErr)
		}
		return nil
	}
	return s.transition(ctx, &task, model.TaskFinished, "MySQL delivery completed", "")
}

// RetryCloudDMRegistration retries only the external registration step. The
// instance and its resource usage remain active throughout the retry.
func (s *DeliveryService) RetryCloudDMRegistration(ctx context.Context, taskID string) error {
	var task model.DeliveryTask
	if err := s.db.WithContext(ctx).First(&task, "id = ?", taskID).Error; err != nil {
		return err
	}
	if task.Status != model.TaskRegisterFailed {
		return fmt.Errorf("task %s is in state %q and cannot retry CloudDM registration", taskID, task.Status)
	}
	if err := s.transition(ctx, &task, model.TaskRegistering, "CloudDM registration retry started", ""); err != nil {
		return err
	}
	if err := s.RegisterCloudDM(ctx, taskID); err != nil {
		message := fmt.Sprintf("CloudDM registration failed: %v", err)
		if transitionErr := s.transition(ctx, &task, model.TaskRegisterFailed, "CloudDM registration retry failed; instance remains active", message); transitionErr != nil {
			return fmt.Errorf("%s; cannot restore register_failed: %w", message, transitionErr)
		}
		return err
	}
	return s.transition(ctx, &task, model.TaskFinished, "CloudDM registration completed", "")
}

func (s *DeliveryService) RegisterCloudDM(ctx context.Context, taskID string) error {
	if s.cfg.CloudDMRegisterURL == "" {
		return nil
	}
	var instance model.DeploymentResult
	if err := s.db.WithContext(ctx).Where("task_id = ? AND component = ? AND service_type = ?", taskID, "mysql", "database").First(&instance).Error; err != nil {
		return err
	}
	body := map[string]any{"name": instance.InstanceName, "host": instance.Host, "port": instance.Port, "username": "root", "database_type": "mysql"}
	raw, _ := json.Marshal(body)
	req, err := http.NewRequestWithContext(ctx, http.MethodPost, s.cfg.CloudDMRegisterURL, bytes.NewReader(raw))
	if err != nil {
		return err
	}
	req.Header.Set("Content-Type", "application/json")
	if s.cfg.CloudDMAPIToken != "" {
		req.Header.Set("Authorization", "Bearer "+s.cfg.CloudDMAPIToken)
	}
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
		return fmt.Errorf("CloudDM returned %s", resp.Status)
	}
	return nil
}

// beginRollback changes the task into the compensating workflow and launches
// the dedicated AWX rollback template. The deploy reservation is deliberately
// kept until rollback succeeds so a failed cleanup cannot be silently
// overwritten by a later delivery.
func (s *DeliveryService) beginRollback(ctx context.Context, taskID, reason string) error {
	var task model.DeliveryTask
	if err := s.db.WithContext(ctx).First(&task, "id = ?", taskID).Error; err != nil {
		return err
	}
	if rollbackProtectedStatus(task.Status) {
		return nil
	}

	if s.cfg.RollbackTemplateID == 0 {
		if err := s.transition(ctx, &task, model.TaskRollbackFailed,
			"automatic rollback is not configured; manual cleanup is required", reason); err != nil {
			return err
		}
		return s.holdReservationForRollback(ctx, task.ID)
	}
	if err := s.transition(ctx, &task, model.TaskRollbackPending, "automatic rollback queued", reason); err != nil {
		return err
	}
	if err := s.holdReservationForRollback(ctx, task.ID); err != nil {
		return s.markRollbackFailed(ctx, task.ID, "cannot hold resources during rollback: "+err.Error())
	}
	return s.launchRollbackJob(ctx, &task, reason)
}

func rollbackProtectedStatus(status string) bool {
	switch status {
	case model.TaskRollbackPending, model.TaskRollingBack, model.TaskRolledBack, model.TaskRollbackFailed,
		model.TaskRollbackAck, model.TaskRegisterFailed, model.TaskFinished, model.TaskCanceled:
		return true
	default:
		return false
	}
}

func (s *DeliveryService) launchRollbackJob(ctx context.Context, task *model.DeliveryTask, reason string) error {
	target, err := s.getTarget(ctx, task.TargetID)
	if err != nil {
		return s.markRollbackFailed(ctx, task.ID, "rollback target is unavailable: "+err.Error())
	}
	var payload deliveryPayload
	if err := json.Unmarshal([]byte(task.ImmutablePayload), &payload); err != nil {
		return s.markRollbackFailed(ctx, task.ID, "rollback payload is invalid: "+err.Error())
	}
	now := time.Now()
	var rollback model.RollbackJob
	if err := s.db.WithContext(ctx).Where("task_id = ?", task.ID).First(&rollback).Error; errors.Is(err, gorm.ErrRecordNotFound) {
		rollback = model.RollbackJob{TaskID: task.ID}
	} else if err != nil {
		return s.markRollbackFailed(ctx, task.ID, "cannot load rollback job: "+err.Error())
	}
	rollback.ExecutorJobID = "pending-rollback-" + task.ID
	rollback.Status = "launching"
	rollback.Reason = reason
	rollback.StartedAt = &now
	rollback.FinishedAt = nil
	if err := s.db.WithContext(ctx).Save(&rollback).Error; err != nil {
		return s.markRollbackFailed(ctx, task.ID, "cannot persist rollback job: "+err.Error())
	}
	job, err := s.awx.Launch(ctx, s.cfg.RollbackTemplateID, AWXLaunchRequest{
		InventoryID: target.AWXInventoryID,
		Limit:       task.TargetHost,
		ExtraVars:   rollbackExtraVars(task, payload),
	})
	if err != nil {
		_ = s.db.WithContext(ctx).Model(&rollback).Updates(map[string]any{"status": "failed", "finished_at": time.Now()}).Error
		return s.markRollbackFailed(ctx, task.ID, "cannot launch rollback AWX job: "+err.Error())
	}
	if err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
		var current model.DeliveryTask
		if err := tx.First(&current, "id = ?", task.ID).Error; err != nil {
			return err
		}
		if current.Status != model.TaskRollbackPending {
			return fmt.Errorf("task %s is no longer pending rollback (state %q)", task.ID, current.Status)
		}
		if err := tx.Model(&rollback).Updates(map[string]any{"executor_job_id": fmt.Sprint(job.ID), "status": "running"}).Error; err != nil {
			return err
		}
		return s.transitionTx(tx, &current, model.TaskRollingBack, "rollback AWX job started", reason)
	}); err != nil {
		return s.markRollbackFailed(ctx, task.ID, "cannot record rollback AWX job: "+err.Error())
	}
	return nil
}

func (s *DeliveryService) RetryRollback(ctx context.Context, taskID string) error {
	var task model.DeliveryTask
	if err := s.db.WithContext(ctx).First(&task, "id = ?", taskID).Error; err != nil {
		return err
	}
	if task.Status != model.TaskRollbackFailed {
		return fmt.Errorf("task %s is in state %q and cannot retry rollback", taskID, task.Status)
	}
	if s.cfg.RollbackTemplateID == 0 {
		return fmt.Errorf("automatic rollback is not configured")
	}
	reason := "manual rollback retry requested"
	if err := s.transition(ctx, &task, model.TaskRollbackPending, "manual rollback retry queued", reason); err != nil {
		return err
	}
	if err := s.holdReservationForRollback(ctx, task.ID); err != nil {
		_ = s.markRollbackFailed(ctx, task.ID, "cannot hold resources during rollback retry: "+err.Error())
		return err
	}
	return s.launchRollbackJob(ctx, &task, reason)
}

// AcknowledgeRollbackRelease is an explicit platform-admin escape hatch after
// the operator has verified the target host. It releases bookkeeping only; it
// does not claim that the automatic rollback succeeded.
func (s *DeliveryService) AcknowledgeRollbackRelease(ctx context.Context, taskID string) error {
	now := time.Now()
	return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
		var task model.DeliveryTask
		if err := tx.First(&task, "id = ?", taskID).Error; err != nil {
			return err
		}
		if task.Status != model.TaskRollbackFailed {
			return fmt.Errorf("task %s is in state %q and cannot acknowledge rollback release", taskID, task.Status)
		}
		if err := tx.Model(&model.DeploymentResult{}).Where("task_id = ? AND component = ? AND service_type = ? AND status = ?", taskID, "mysql", "database", "active").Updates(map[string]any{"status": "rollback_acknowledged", "updated_at": now}).Error; err != nil {
			return err
		}
		if err := tx.Model(&model.ResourceUsage{}).Where("task_id = ? AND status = ?", taskID, "active").Updates(map[string]any{"status": "released", "released_at": now, "updated_at": now}).Error; err != nil {
			return err
		}
		if err := tx.Model(&model.ResourceReservation{}).Where("task_id = ? AND status = ?", taskID, "rollback").Update("status", "released").Error; err != nil {
			return err
		}
		return s.transitionTx(tx, &task, model.TaskRollbackAck, "rollback release acknowledged by platform admin", "")
	})
}

func rollbackExtraVars(task *model.DeliveryTask, payload deliveryPayload) map[string]any {
	return map[string]any{
		"target_hosts":  task.TargetHost,
		"instance_name": payload.InstanceName,
		"data_disk":     payload.DataDisk,
		"task_id":       task.ID,
		"rollback":      true,
	}
}

func (s *DeliveryService) holdReservationForRollback(ctx context.Context, taskID string) error {
	return s.db.WithContext(ctx).Model(&model.ResourceReservation{}).
		Where("task_id = ? AND status IN ?", taskID, []string{"reserved", "consumed"}).
		Update("status", "rollback").Error
}

func (s *DeliveryService) markRollbackFailed(ctx context.Context, taskID, message string) error {
	var task model.DeliveryTask
	if err := s.db.WithContext(ctx).First(&task, "id = ?", taskID).Error; err != nil {
		return err
	}
	if task.Status == model.TaskRolledBack || task.Status == model.TaskFinished || task.Status == model.TaskCanceled ||
		task.Status == model.TaskRollbackFailed || task.Status == model.TaskRollbackAck || task.Status == model.TaskRegisterFailed {
		return nil
	}
	return s.transition(ctx, &task, model.TaskRollbackFailed, "automatic rollback failed; manual cleanup is required", message)
}

func (s *DeliveryService) PollRollbackOnce(ctx context.Context) error {
	var jobs []model.RollbackJob
	if err := s.db.WithContext(ctx).Where("status IN ?", []string{"launching", "running"}).Find(&jobs).Error; err != nil {
		return err
	}
	for _, rollback := range jobs {
		if rollback.Status == "launching" {
			if !rollbackLaunchExpired(rollback, time.Now()) {
				continue
			}
			// The AWX launch result is unknown. Do not blindly launch a second
			// job; surface a recoverable failure for an explicit admin retry.
			expired, err := s.expireRollbackLaunch(ctx, rollback.ID)
			if err != nil {
				return err
			}
			if !expired {
				continue
			}
			_ = s.markRollbackFailed(ctx, rollback.TaskID, "rollback launch timed out before its AWX job ID was recorded; manual retry is required")
			continue
		}
		job, err := s.awx.GetJob(ctx, rollback.ExecutorJobID)
		if err != nil {
			// Do not declare cleanup failed because AWX is temporarily unreachable.
			continue
		}
		switch strings.ToLower(job.Status) {
		case "pending", "waiting", "running", "new":
			continue
		case "successful":
			_ = s.finishRollbackJob(ctx, &rollback, "successful")
			if err := s.completeRollback(ctx, rollback.TaskID); err != nil {
				_ = s.markRollbackFailed(ctx, rollback.TaskID, err.Error())
			}
		case "canceled":
			_ = s.finishRollbackJob(ctx, &rollback, "canceled")
			_ = s.markRollbackFailed(ctx, rollback.TaskID, "rollback AWX job was canceled")
		default:
			_ = s.finishRollbackJob(ctx, &rollback, "failed")
			_ = s.markRollbackFailed(ctx, rollback.TaskID, "rollback AWX job finished with status "+job.Status)
		}
	}
	return nil
}

func rollbackLaunchExpired(rollback model.RollbackJob, now time.Time) bool {
	if rollback.Status != "launching" {
		return false
	}
	if rollback.StartedAt == nil {
		return true
	}
	return !now.Before(rollback.StartedAt.Add(rollbackLaunchTimeout))
}

func (s *DeliveryService) expireRollbackLaunch(ctx context.Context, rollbackID uint64) (bool, error) {
	result := s.db.WithContext(ctx).Model(&model.RollbackJob{}).
		Where("id = ? AND status = ?", rollbackID, "launching").
		Updates(map[string]any{"status": "launch_timeout", "finished_at": time.Now()})
	return result.RowsAffected == 1, result.Error
}

func (s *DeliveryService) finishRollbackJob(ctx context.Context, rollback *model.RollbackJob, status string) error {
	return s.db.WithContext(ctx).Model(rollback).Updates(map[string]any{"status": status, "finished_at": time.Now()}).Error
}

func (s *DeliveryService) completeRollback(ctx context.Context, taskID string) error {
	now := time.Now()
	return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
		var task model.DeliveryTask
		if err := tx.First(&task, "id = ?", taskID).Error; err != nil {
			return err
		}
		if task.Status != model.TaskRollingBack {
			return fmt.Errorf("task %s is in state %q, cannot complete rollback", taskID, task.Status)
		}
		if err := tx.Model(&model.DeploymentResult{}).Where("task_id = ?", taskID).Updates(map[string]any{"status": "rolled_back", "updated_at": now}).Error; err != nil {
			return err
		}
		if err := tx.Unscoped().Where("task_id = ?", taskID).Delete(&model.DeploymentCredential{}).Error; err != nil {
			return err
		}
		if err := tx.Model(&model.ResourceUsage{}).Where("task_id = ? AND status = ?", taskID, "active").Updates(map[string]any{"status": "released", "released_at": now, "updated_at": now}).Error; err != nil {
			return err
		}
		if err := tx.Model(&model.ResourceReservation{}).Where("task_id = ? AND status IN ?", taskID, []string{"reserved", "consumed", "rollback"}).Update("status", "released").Error; err != nil {
			return err
		}
		return s.transitionTx(tx, &task, model.TaskRolledBack, "MySQL instance rollback completed", "")
	})
}

func (s *DeliveryService) failTask(ctx context.Context, task *model.DeliveryTask, status, message string) error {
	return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
		var current model.DeliveryTask
		if err := tx.First(&current, "id = ?", task.ID).Error; err != nil {
			return err
		}
		if current.Status != model.TaskPending && current.Status != model.TaskDispatching && current.Status != model.TaskRunning && current.Status != model.TaskRegistering && current.Status != model.TaskCanceling {
			return nil
		}
		if err := s.transitionTx(tx, &current, status, message, message); err != nil {
			return err
		}
		if status == model.TaskExecutionFailed || status == model.TaskValidationFailed || status == model.TaskCanceled {
			return s.releaseReservation(tx, current.ID)
		}
		return nil
	})
}

func (s *DeliveryService) Run(ctx context.Context) {
	interval := time.Duration(s.cfg.DeliveryDispatchSeconds) * time.Second
	if interval < time.Second {
		interval = time.Second
	}
	ticker := time.NewTicker(interval)
	defer ticker.Stop()
	for {
		select {
		case <-ctx.Done():
			return
		case <-ticker.C:
			_ = s.DispatchOnce(ctx)
			_ = s.PollOnce(ctx)
			_ = s.PollRollbackOnce(ctx)
		}
	}
}