xinfra/server/internal/service/delivery.go

package service

import (
	"bytes"
	"context"
	"crypto/rand"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"net"
	"net/http"
	"regexp"
	"strings"
	"sync"
	"time"

	"github.com/1024XEngineer/xinfra/server/internal/config"
	"github.com/1024XEngineer/xinfra/server/internal/model"
	"gorm.io/gorm"
	"gorm.io/gorm/clause"
)

var dnsLabelPattern = regexp.MustCompile(`^[a-z0-9](?:[-a-z0-9]*[a-z0-9])?$`)

// AWX inventory 主机名：允许字母数字、中划线与点（FQDN）。
var hostNamePattern = regexp.MustCompile(`^[a-zA-Z0-9](?:[-a-zA-Z0-9.]*[a-zA-Z0-9])?$`)

type MySQLDeliveryInput struct {
	BusinessLineID uint64 `json:"business_line_id" binding:"required"`
	TargetID       uint64 `json:"target_id" binding:"required"`
	Namespace      string `json:"namespace" binding:"required"`
	InstanceName   string `json:"instance_name" binding:"required"`
	MySQLVersion   string `json:"mysql_version"`
	Topology       string `json:"topology"`
	MySQLPort      int    `json:"mysql_port"`
	DataDisk       string `json:"data_disk"`
	// 调度控制（选填）：点名候选池内主机跳过自动选机，端口/配额/实机守卫照常执行
	TargetHost string `json:"target_host"`
	CPUMilli   int64  `json:"cpu_milli" binding:"required"`
	MemoryMi   int64  `json:"memory_mi" binding:"required"`
	StorageGi  int64  `json:"storage_gi" binding:"required"`
	// 数据库配置（选填，缺省由 playbook 基线兜底）
	Timezone            string `json:"timezone"`
	LowerCaseTableNames *int   `json:"lower_case_table_names"`
	CharacterSet        string `json:"character_set"`
	Collation           string `json:"collation"`
	// 高级参数（选填，零值视为未设置）
	MaxConnections            string  `json:"max_connections"`
	InnodbRedoLogCapacity     string  `json:"innodb_redo_log_capacity"`
	InnodbFlushLogAtTrxCommit *int    `json:"innodb_flush_log_at_trx_commit"`
	SyncBinlog                *int    `json:"sync_binlog"`
	InnodbIOCapacity          int     `json:"innodb_io_capacity"`
	LongQueryTime             float64 `json:"long_query_time"`
	BinlogExpireLogsSeconds   int64   `json:"binlog_expire_logs_seconds"`
	MaxBinlogSize             string  `json:"max_binlog_size"`
}

type deliveryPayload struct {
	MySQLDeliveryInput
	TargetType string `json:"target_type"`
}

type DeliveryTarget struct {
	ID             uint64 `json:"id"`
	Name           string `json:"name"`
	TargetType     string `json:"target_type"`
	AWXInventoryID uint64 `json:"awx_inventory_id"`
	AWXTemplateID  uint64 `json:"awx_template_id"`
	Enabled        bool   `json:"enabled"`
	Metadata       string `json:"metadata"`
}

// targetMetadata describes the native VM候选节点池以及部署形态，由 AWX inventory hosts 动态组装。
type targetMetadata struct {
	Topology  string       `json:"topology"`
	MySQLPort int          `json:"mysql_port"`
	Hosts     []targetHost `json:"hosts"`
}

type targetHost struct {
	Name string `json:"name"`
	IP   string `json:"ip"`
}

func parseTargetMetadata(raw string) targetMetadata {
	meta := targetMetadata{}
	if strings.TrimSpace(raw) != "" {
		_ = json.Unmarshal([]byte(raw), &meta)
	}
	if meta.Topology == "" {
		meta.Topology = "standalone"
	}
	if meta.MySQLPort == 0 {
		meta.MySQLPort = 3307
	}
	return meta
}

// firstFreeHost 返回候选池中非失败任务数未达单机实例上限的第一个节点；
// limit < 1 时按 1 兜底（退化为旧的一机一实例语义）。顺序遍历天然形成"先摊开、摊满一轮再叠加"。
func firstFreeHost(hosts []targetHost, occupied []string, limit int) *targetHost {
	if limit < 1 {
		limit = 1
	}
	for i := range hosts {
		if hostTaskCount(occupied, hosts[i].Name) < limit {
			return &hosts[i]
		}
	}
	return nil
}

// hostTaskCount 统计某主机在占用清单（非失败任务的 target_host 列表，含重复）中的出现次数。
func hostTaskCount(occupied []string, name string) int {
	count := 0
	for _, h := range occupied {
		if h == name {
			count++
		}
	}
	return count
}

// 端口池 13306-13999：混合分配模型，用户留空时自动分配，可覆盖为池内指定端口。
const (
	mysqlPortPoolStart = 13306
	mysqlPortPoolEnd   = 13999
)

// allocatePort 在目标主机已占用端口集上做混合分配：指定端口验冲突，未指定则取池内首个空闲端口。
func allocatePort(requested int, used []int) (int, error) {
	taken := make(map[int]bool, len(used))
	for _, p := range used {
		taken[p] = true
	}
	if requested != 0 {
		if taken[requested] {
			return 0, fmt.Errorf("mysql_port %d is already allocated on the target host", requested)
		}
		return requested, nil
	}
	for p := mysqlPortPoolStart; p <= mysqlPortPoolEnd; p++ {
		if !taken[p] {
			return p, nil
		}
	}
	return 0, fmt.Errorf("mysql port pool %d-%d is exhausted on the target host", mysqlPortPoolStart, mysqlPortPoolEnd)
}

type DeliveryService struct {
	db          *gorm.DB
	cfg         config.Config
	awx         *AWXClient
	audit       *AuditService
	executionMu sync.Mutex
}

func (s *DeliveryService) DB() *gorm.DB { return s.db }

func NewDeliveryService(cfg config.Config, db *gorm.DB, audit *AuditService) *DeliveryService {
	return &DeliveryService{db: db, cfg: cfg, awx: NewAWXClient(cfg.AWXBaseURL, cfg.AWXToken, cfg.AWXUsername, cfg.AWXPassword), audit: audit}
}

func (s *DeliveryService) ListTargets(ctx context.Context, component string) ([]DeliveryTarget, error) {
	templates, err := s.awx.ListJobTemplates(ctx)
	if err != nil {
		return nil, err
	}
	component = strings.ToLower(strings.TrimSpace(component))
	var targets []DeliveryTarget
	for _, template := range templates {
		if template.Inventory == 0 {
			continue
		}
		if component != "" && component != "all" {
			text := strings.ToLower(template.Name + " " + template.Description)
			if !strings.Contains(text, component) {
				continue
			}
		}
		target, err := s.awxDeliveryTarget(ctx, template)
		if err != nil {
			continue
		}
		targets = append(targets, target)
	}
	return targets, nil
}

func (s *DeliveryService) getTarget(ctx context.Context, templateID uint64) (DeliveryTarget, error) {
	template, err := s.awx.GetJobTemplate(ctx, templateID)
	if err != nil {
		return DeliveryTarget{}, fmt.Errorf("deployment target is unavailable: %w", err)
	}
	return s.awxDeliveryTarget(ctx, *template)
}

func (s *DeliveryService) awxDeliveryTarget(ctx context.Context, template AWXJobTemplate) (DeliveryTarget, error) {
	hosts, err := s.awx.ListInventoryHosts(ctx, template.Inventory)
	if err != nil {
		return DeliveryTarget{}, err
	}
	meta := targetMetadata{Topology: "standalone", MySQLPort: 3307}
	for _, host := range hosts {
		if !host.Enabled {
			continue
		}
		meta.Hosts = append(meta.Hosts, targetHost{Name: host.Name, IP: AWXHostIP(host)})
	}
	raw, err := json.Marshal(meta)
	if err != nil {
		return DeliveryTarget{}, err
	}
	return DeliveryTarget{
		ID:             template.ID,
		Name:           template.Name,
		TargetType:     "k8s",
		AWXInventoryID: template.Inventory,
		AWXTemplateID:  template.ID,
		Enabled:        true,
		Metadata:       string(raw),
	}, nil
}

func (s *DeliveryService) CreateTask(ctx context.Context, userID uint64, isAdmin bool, idempotencyKey string, input MySQLDeliveryInput) (*model.DeliveryTask, bool, error) {
	idempotencyKey = strings.TrimSpace(idempotencyKey)
	if idempotencyKey == "" || len(idempotencyKey) > 128 {
		return nil, false, fmt.Errorf("Idempotency-Key header is required and must not exceed 128 characters")
	}
	if err := validateDeliveryInput(input, s.cfg.DeliveryDataDisks); err != nil {
		return nil, false, err
	}

	var existing model.DeliveryTask
	if err := s.db.WithContext(ctx).Where("idempotency_key = ?", idempotencyKey).First(&existing).Error; err == nil {
		if existing.RequestedBy != userID {
			return nil, false, fmt.Errorf("idempotency key is already in use by another user")
		}
		return &existing, true, nil
	} else if !errors.Is(err, gorm.ErrRecordNotFound) {
		return nil, false, err
	}

	target, err := s.getTarget(ctx, input.TargetID)
	if err != nil {
		return nil, false, err
	}
	if target.TargetType != "k8s" {
		return nil, false, fmt.Errorf("target type %q is not supported in the MVP", target.TargetType)
	}
	if !isAdmin {
		var count int64
		if err := s.db.WithContext(ctx).Model(&model.BusinessLineUser{}).
			Where("business_line_id = ? AND user_id = ?", input.BusinessLineID, userID).Count(&count).Error; err != nil {
			return nil, false, err
		}
		if count == 0 {
			return nil, false, fmt.Errorf("user is not authorized for this business line")
		}
	}
	if input.MySQLVersion == "" {
		input.MySQLVersion = "8.0"
	}
	if input.Topology == "" {
		input.Topology = "standalone"
	}
	if input.DataDisk == "" {
		if len(s.cfg.DeliveryDataDisks) > 0 {
			input.DataDisk = s.cfg.DeliveryDataDisks[0]
		} else {
			input.DataDisk = "/data"
		}
	}
	payload := deliveryPayload{MySQLDeliveryInput: input, TargetType: target.TargetType}
	raw, err := json.Marshal(payload)
	if err != nil {
		return nil, false, err
	}
	digest := sha256.Sum256(raw)
	task := model.DeliveryTask{
		ID:               randomUUID(),
		BusinessLineID:   input.BusinessLineID,
		RequestedBy:      userID,
		TargetType:       target.TargetType,
		TargetID:         target.ID,
		Namespace:        input.Namespace,
		InstanceName:     input.InstanceName,
		Status:           model.TaskPending,
		ImmutablePayload: string(raw),
		PayloadHash:      hex.EncodeToString(digest[:]),
		IdempotencyKey:   idempotencyKey,
	}
	if err := s.db.WithContext(ctx).Create(&task).Error; err != nil {
		if lookupErr := s.db.WithContext(ctx).Where("idempotency_key = ?", idempotencyKey).First(&existing).Error; lookupErr == nil {
			return &existing, true, nil
		}
		return nil, false, err
	}
	_ = s.db.WithContext(ctx).Create(&model.TaskEvent{TaskID: task.ID, ToState: model.TaskPending, Message: "delivery task created"}).Error
	return &task, false, nil
}

// 版本白名单与 playbook 的 mysql_package_map 保持同步：
// 8.0 走 Ubuntu 自带源，8.4 走 MySQL 官方 APT 源；5.6/5.7 已 EOL 且无 noble 包，不支持。
var supportedMySQLVersions = map[string]bool{"8.0": true, "8.4": true}

// 拓扑白名单：playbook 已支持 primary_replica/mgr_3 的配置渲染，
// 但调度器仍是单主机模型且复制编排未自动化，本期仅放开 standalone。
var supportedTopologies = map[string]bool{"standalone": true}

const rollbackLaunchTimeout = 2 * time.Minute

var supportedCharsets = map[string]bool{"utf8mb4": true, "utf8": true, "gbk": true, "latin1": true}

// 高级参数档位白名单（与 docs/mysql-parameter-selection.md 保持一致）
var (
	supportedMaxConnections   = map[string]bool{"auto": true, "200": true, "500": true, "1000": true, "2000": true, "4000": true, "8000": true, "16000": true}
	supportedLogSizes         = map[string]bool{"128M": true, "256M": true, "512M": true, "1G": true}
	supportedIOCapacities     = map[int]bool{200: true, 2000: true, 5000: true}
	supportedLongQueryTimes   = map[float64]bool{0.5: true, 1: true, 2: true, 5: true, 10: true}
	supportedBinlogExpireSecs = map[int64]bool{86400: true, 259200: true, 604800: true, 1209600: true}
)

// timezone 仅接受偏移量（±HH:MM）、SYSTEM 或命名时区（如 Asia/Shanghai）。
var timezonePattern = regexp.MustCompile(`^([+-](0\d|1[0-4]):[0-5]\d|SYSTEM|[A-Za-z]+(?:/[A-Za-z0-9_+-]+)+)$`)

func validateDeliveryInput(input MySQLDeliveryInput, dataDisks []string) error {
	if len(input.Namespace) > 63 || !dnsLabelPattern.MatchString(input.Namespace) {
		return fmt.Errorf("namespace must be a valid Kubernetes DNS label")
	}
	if len(input.InstanceName) > 63 || !dnsLabelPattern.MatchString(input.InstanceName) {
		return fmt.Errorf("instance_name must be a valid Kubernetes DNS label")
	}
	// 与文档目标态一致（memory 2048-65536 MiB / storage 20-2000 GiB），playbook assert 同步。
	if input.CPUMilli < 100 || input.CPUMilli > 64000 || input.MemoryMi < 2048 || input.MemoryMi > 65536 || input.StorageGi < 20 || input.StorageGi > 2000 {
		return fmt.Errorf("requested resources are outside the supported range (memory: 2048-65536 MiB, storage: 20-2000 GiB)")
	}
	if input.MySQLVersion != "" && !supportedMySQLVersions[input.MySQLVersion] {
		return fmt.Errorf("unsupported mysql_version %q, supported: 8.0, 8.4 (5.6/5.7 are EOL and have no Ubuntu 24.04 packages)", input.MySQLVersion)
	}
	if input.Topology != "" && !supportedTopologies[input.Topology] {
		return fmt.Errorf("unsupported topology %q, supported: standalone (primary_replica/mgr_3 pending scheduler support)", input.Topology)
	}
	if input.MySQLPort != 0 && (input.MySQLPort < mysqlPortPoolStart || input.MySQLPort > mysqlPortPoolEnd) {
		return fmt.Errorf("mysql_port must be left empty for auto allocation or within the pool %d-%d", mysqlPortPoolStart, mysqlPortPoolEnd)
	}
	if input.DataDisk != "" {
		allowed := dataDisks
		if len(allowed) == 0 {
			allowed = []string{"/data"}
		}
		found := false
		for _, disk := range allowed {
			if input.DataDisk == disk {
				found = true
				break
			}
		}
		if !found {
			return fmt.Errorf("data_disk %q is not in the allowed mount point list %v", input.DataDisk, allowed)
		}
	}
	if input.TargetHost != "" && (len(input.TargetHost) > 253 || !hostNamePattern.MatchString(input.TargetHost)) {
		return fmt.Errorf("target_host must be a valid inventory host name")
	}
	if input.Timezone != "" && !timezonePattern.MatchString(input.Timezone) {
		return fmt.Errorf("timezone must be an offset like +08:00, SYSTEM, or a named zone like Asia/Shanghai")
	}
	if input.LowerCaseTableNames != nil && *input.LowerCaseTableNames != 0 && *input.LowerCaseTableNames != 1 {
		return fmt.Errorf("lower_case_table_names must be 0 or 1")
	}
	if input.CharacterSet != "" && !supportedCharsets[input.CharacterSet] {
		return fmt.Errorf("unsupported character_set %q, supported: utf8mb4, utf8, gbk, latin1", input.CharacterSet)
	}
	if input.Collation != "" {
		charset := input.CharacterSet
		if charset == "" {
			charset = "utf8mb4"
		}
		if !strings.HasPrefix(input.Collation, charset+"_") {
			return fmt.Errorf("collation %q does not match character_set %q", input.Collation, charset)
		}
	}
	if input.MaxConnections != "" && !supportedMaxConnections[input.MaxConnections] {
		return fmt.Errorf("max_connections must be one of auto, 200, 500, 1000, 2000, 4000, 8000, 16000")
	}
	if input.InnodbRedoLogCapacity != "" && !supportedLogSizes[input.InnodbRedoLogCapacity] {
		return fmt.Errorf("innodb_redo_log_capacity must be one of 128M, 256M, 512M, 1G")
	}
	if input.InnodbFlushLogAtTrxCommit != nil && (*input.InnodbFlushLogAtTrxCommit < 0 || *input.InnodbFlushLogAtTrxCommit > 2) {
		return fmt.Errorf("innodb_flush_log_at_trx_commit must be 0, 1 or 2")
	}
	if input.SyncBinlog != nil && *input.SyncBinlog != 0 && *input.SyncBinlog != 1 {
		return fmt.Errorf("sync_binlog must be 0 or 1")
	}
	if input.InnodbIOCapacity != 0 && !supportedIOCapacities[input.InnodbIOCapacity] {
		return fmt.Errorf("innodb_io_capacity must be one of 200, 2000, 5000")
	}
	if input.LongQueryTime != 0 && !supportedLongQueryTimes[input.LongQueryTime] {
		return fmt.Errorf("long_query_time must be one of 0.5, 1, 2, 5, 10")
	}
	if input.BinlogExpireLogsSeconds != 0 && !supportedBinlogExpireSecs[input.BinlogExpireLogsSeconds] {
		return fmt.Errorf("binlog_expire_logs_seconds must be one of 86400, 259200, 604800, 1209600")
	}
	if input.MaxBinlogSize != "" && !supportedLogSizes[input.MaxBinlogSize] {
		return fmt.Errorf("max_binlog_size must be one of 128M, 256M, 512M, 1G")
	}
	return nil
}

func (s *DeliveryService) ListTasks(ctx context.Context, userID uint64, isAdmin bool, businessLineID uint64) ([]model.DeliveryTask, error) {
	query := s.db.WithContext(ctx).Order("created_at DESC")
	if businessLineID != 0 {
		query = query.Where("business_line_id = ?", businessLineID)
	}
	if !isAdmin {
		query = query.Where("business_line_id IN (?)", s.db.Model(&model.BusinessLineUser{}).Select("business_line_id").Where("user_id = ?", userID))
	}
	var tasks []model.DeliveryTask
	return tasks, query.Find(&tasks).Error
}

func (s *DeliveryService) GetTask(ctx context.Context, taskID string, userID uint64, isAdmin bool) (*model.DeliveryTask, []model.TaskEvent, error) {
	query := s.db.WithContext(ctx).Where("id = ?", taskID)
	if !isAdmin {
		query = query.Where("business_line_id IN (?)", s.db.Model(&model.BusinessLineUser{}).Select("business_line_id").Where("user_id = ?", userID))
	}
	var task model.DeliveryTask
	if err := query.First(&task).Error; err != nil {
		return nil, nil, err
	}
	var events []model.TaskEvent
	if err := s.db.WithContext(ctx).Where("task_id = ?", taskID).Order("id ASC").Find(&events).Error; err != nil {
		return nil, nil, err
	}
	return &task, events, nil
}

func (s *DeliveryService) AWXJobStdout(ctx context.Context, jobID string) (string, error) {
	return s.awx.JobStdout(ctx, jobID)
}

func (s *DeliveryService) Cancel(ctx context.Context, taskID string, userID uint64, isAdmin bool) error {
	task, _, err := s.GetTask(ctx, taskID, userID, isAdmin)
	if err != nil {
		return err
	}
	if task.Status == model.TaskPending {
		return s.transition(ctx, task, model.TaskCanceled, "canceled before dispatch", "")
	}
	if task.Status != model.TaskRunning && task.Status != model.TaskDispatching {
		return fmt.Errorf("task in state %q cannot be canceled", task.Status)
	}
	var job model.ExecutionJob
	if err := s.db.WithContext(ctx).Where("task_id = ?", task.ID).First(&job).Error; err != nil {
		return err
	}
	if err := s.awx.Cancel(ctx, job.ExecutorJobID); err != nil {
		return err
	}
	return s.transition(ctx, task, model.TaskCanceling, "cancel requested in AWX", "")
}

func (s *DeliveryService) claimAndReserve(ctx context.Context) (*model.DeliveryTask, error) {
	var task model.DeliveryTask
	var payload deliveryPayload
	var target DeliveryTarget
	dispatchable := false
	err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
		if err := tx.Clauses(clause.Locking{Strength: "UPDATE", Options: "SKIP LOCKED"}).Where("status = ?", model.TaskPending).Order("created_at ASC").First(&task).Error; err != nil {
			return err
		}
		var targetErr error
		target, targetErr = s.getTarget(ctx, task.TargetID)
		if targetErr != nil {
			return s.failInTransaction(tx, &task, model.TaskValidationFailed, targetErr.Error())
		}
		if err := json.Unmarshal([]byte(task.ImmutablePayload), &payload); err != nil {
			return s.failInTransaction(tx, &task, model.TaskValidationFailed, "stored deployment payload is invalid")
		}
		activeStates := []string{model.TaskValidating, model.TaskDispatching, model.TaskRunning, model.TaskCanceling}
		checks := []struct {
			query   string
			args    []any
			limit   int
			message string
		}{
			{"status IN ?", []any{activeStates}, s.cfg.DeliveryGlobalLimit, "global concurrency limit reached"},
			{"status IN ? AND target_id = ?", []any{activeStates, task.TargetID}, s.cfg.DeliveryTargetLimit, "target concurrency limit reached"},
			{"status IN ? AND business_line_id = ?", []any{activeStates, task.BusinessLineID}, s.cfg.DeliveryBusinessLimit, "business line concurrency limit reached"},
			{"status IN ? AND target_id = ? AND namespace = ?", []any{activeStates, task.TargetID, task.Namespace}, 1, "namespace already has an active MySQL delivery"},
		}
		for _, check := range checks {
			var count int64
			if check.limit > 0 {
				if err := tx.Model(&model.DeliveryTask{}).Where(check.query, check.args...).Count(&count).Error; err != nil {
					return err
				}
				if count >= int64(check.limit) {
					return fmt.Errorf("defer: %s", check.message)
				}
			}
		}
		quotaOK, err := checkResourceQuota(tx, task.BusinessLineID, task.TargetID, payload)
		if err != nil {
			return err
		}
		if !quotaOK {
			return s.failInTransaction(tx, &task, model.TaskValidationFailed, "resource quota is insufficient")
		}
		meta := parseTargetMetadata(target.Metadata)
		if len(meta.Hosts) == 0 {
			return s.failInTransaction(tx, &task, model.TaskValidationFailed, "deployment target has no candidate hosts")
		}
		var occupied []string
		occupiedExclude := []string{model.TaskExecutionFailed, model.TaskValidationFailed, model.TaskCanceled}
		if err := tx.Model(&model.DeliveryTask{}).Where("target_id = ? AND target_host <> ? AND status NOT IN ?", task.TargetID, "", occupiedExclude).Pluck("target_host", &occupied).Error; err != nil {
			return err
		}
		limit := s.cfg.DeliveryHostInstanceLimit
		var host *targetHost
		if payload.TargetHost != "" {
			// 点名主机：仅校验池内存在性与单机实例上限，端口/配额/实机守卫照常执行。
			for i := range meta.Hosts {
				if meta.Hosts[i].Name == payload.TargetHost {
					host = &meta.Hosts[i]
					break
				}
			}
			if host == nil {
				return s.failInTransaction(tx, &task, model.TaskValidationFailed, fmt.Sprintf("target_host %q is not in the candidate host pool", payload.TargetHost))
			}
			if effective := max(limit, 1); hostTaskCount(occupied, host.Name) >= effective {
				return fmt.Errorf("defer: pinned host %s reached the per-host instance limit %d", host.Name, effective)
			}
		} else {
			host = firstFreeHost(meta.Hosts, occupied, limit)
		}
		if host == nil {
			return fmt.Errorf("defer: no free host available on target")
		}
		// 端口池混合分配：同主机已占端口 = 非终态任务分配端口 ∪ 存量 active 实例端口。
		var usedPorts []int
		if err := tx.Model(&model.DeliveryTask{}).Where("target_host = ? AND status NOT IN ?", host.Name, occupiedExclude).Pluck("mysql_port", &usedPorts).Error; err != nil {
			return err
		}
		var instancePorts []int
		if err := tx.Model(&model.MySQLInstance{}).Where("node_name = ? AND status = ?", host.Name, "active").Pluck("port", &instancePorts).Error; err != nil {
			return err
		}
		port, portErr := allocatePort(payload.MySQLPort, append(usedPorts, instancePorts...))
		if portErr != nil {
			return s.failInTransaction(tx, &task, model.TaskValidationFailed, portErr.Error())
		}
		reservation := model.ResourceReservation{TaskID: task.ID, BusinessLineID: task.BusinessLineID, TargetID: task.TargetID, CPUMilli: payload.CPUMilli, MemoryMi: payload.MemoryMi, StorageGi: payload.StorageGi, InstanceCount: 1, Status: "reserved", ExpiresAt: time.Now().Add(time.Duration(s.cfg.ReservationTTLMinutes) * time.Minute)}
		if err := tx.Create(&reservation).Error; err != nil {
			return err
		}
		if err := tx.Model(&model.DeliveryTask{}).Where("id = ?", task.ID).Updates(map[string]any{"target_host": host.Name, "target_host_ip": host.IP, "mysql_port": port}).Error; err != nil {
			return err
		}
		task.TargetHost = host.Name
		task.TargetHostIP = host.IP
		task.MySQLPort = port
		if err := s.transitionTx(tx, &task, model.TaskDispatching, "resources reserved", ""); err != nil {
			return err
		}
		dispatchable = true
		return nil
	})
	if err == nil && !dispatchable {
		err = gorm.ErrRecordNotFound
	}
	return &task, err
}

func checkResourceQuota(tx *gorm.DB, businessLineID, targetID uint64, payload deliveryPayload) (bool, error) {
	var quota model.ResourceQuota
	if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Where("business_line_id = ? AND target_id = ?", businessLineID, targetID).First(&quota).Error; err != nil {
		if errors.Is(err, gorm.ErrRecordNotFound) {
			return true, nil
		}
		return false, err
	}
	type totals struct{ CPU, Memory, Storage, Instances int64 }
	var used, reserved totals
	if err := tx.Model(&model.ResourceUsage{}).Select("COALESCE(SUM(cpu_milli),0) cpu, COALESCE(SUM(memory_mi),0) memory, COALESCE(SUM(storage_gi),0) storage, COALESCE(SUM(instance_count),0) instances").Where("business_line_id = ? AND target_id = ? AND status = ?", businessLineID, targetID, "active").Scan(&used).Error; err != nil {
		return false, err
	}
	if err := tx.Model(&model.ResourceReservation{}).Select("COALESCE(SUM(cpu_milli),0) cpu, COALESCE(SUM(memory_mi),0) memory, COALESCE(SUM(storage_gi),0) storage, COALESCE(SUM(instance_count),0) instances").Where("business_line_id = ? AND target_id = ? AND (status = ? OR (status = ? AND expires_at > ?))", businessLineID, targetID, "rollback", "reserved", time.Now()).Scan(&reserved).Error; err != nil {
		return false, err
	}
	return used.CPU+reserved.CPU+payload.CPUMilli <= quota.CPUMilli &&
		used.Memory+reserved.Memory+payload.MemoryMi <= quota.MemoryMi &&
		used.Storage+reserved.Storage+payload.StorageGi <= quota.StorageGi &&
		used.Instances+reserved.Instances+1 <= quota.InstanceLimit, nil
}

func (s *DeliveryService) failInTransaction(tx *gorm.DB, task *model.DeliveryTask, status, message string) error {
	if err := s.transitionTx(tx, task, status, message, message); err != nil {
		return err
	}
	return nil
}

func (s *DeliveryService) transition(ctx context.Context, task *model.DeliveryTask, status, message, errorMessage string) error {
	return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { return s.transitionTx(tx, task, status, message, errorMessage) })
}

func (s *DeliveryService) transitionTx(tx *gorm.DB, task *model.DeliveryTask, status, message, errorMessage string) error {
	from := task.Status
	updates := map[string]any{"status": status, "error_message": errorMessage}
	now := time.Now()
	if status == model.TaskRunning {
		updates["started_at"] = now
	}
	if status == model.TaskFinished || status == model.TaskExecutionFailed || status == model.TaskValidationFailed || status == model.TaskCanceled || status == model.TaskRegisterFailed || status == model.TaskRolledBack || status == model.TaskRollbackFailed || status == model.TaskRollbackAck {
		updates["finished_at"] = now
	}
	result := tx.Model(&model.DeliveryTask{}).Where("id = ? AND status = ?", task.ID, from).Updates(updates)
	if result.Error != nil {
		return result.Error
	}
	if result.RowsAffected != 1 {
		return fmt.Errorf("task %s changed concurrently", task.ID)
	}
	task.Status = status
	task.ErrorMessage = errorMessage
	return tx.Create(&model.TaskEvent{TaskID: task.ID, FromState: from, ToState: status, Message: message}).Error
}

func (s *DeliveryService) releaseReservation(tx *gorm.DB, taskID string) error {
	return tx.Model(&model.ResourceReservation{}).Where("task_id = ? AND status = ?", taskID, "reserved").Update("status", "released").Error
}

func randomUUID() string {
	b := make([]byte, 16)
	if _, err := rand.Read(b); err != nil {
		panic(err)
	}
	b[6] = (b[6] & 0x0f) | 0x40
	b[8] = (b[8] & 0x3f) | 0x80
	return fmt.Sprintf("%08x-%04x-%04x-%04x-%012x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:16])
}

func mysqlReady(ctx context.Context, address string) error {
	dialer := net.Dialer{Timeout: 5 * time.Second}
	conn, err := dialer.DialContext(ctx, "tcp", address)
	if err != nil {
		return err
	}
	return conn.Close()
}

func (s *DeliveryService) DispatchOnce(ctx context.Context) error {
	task, err := s.claimAndReserve(ctx)
	if err != nil {
		if errors.Is(err, gorm.ErrRecordNotFound) || strings.HasPrefix(err.Error(), "defer:") {
			return nil
		}
		return err
	}
	_, _, err = s.CreateExecution(ctx, task.ID, task.PayloadHash, task.IdempotencyKey)
	if err != nil {
		return s.beginRollback(ctx, task.ID, "deployment could not be started: "+err.Error())
	}
	return nil
}

func (s *DeliveryService) CreateExecution(ctx context.Context, taskID, payloadHash, idempotencyKey string) (*model.ExecutionJob, bool, error) {
	s.executionMu.Lock()
	defer s.executionMu.Unlock()
	var existing model.ExecutionJob
	if err := s.db.WithContext(ctx).Where("task_id = ? OR idempotency_key = ?", taskID, idempotencyKey).First(&existing).Error; err == nil {
		if existing.TaskID != taskID || existing.IdempotencyKey != idempotencyKey {
			return nil, false, fmt.Errorf("idempotency key is already bound to another task")
		}
		return &existing, true, nil
	} else if !errors.Is(err, gorm.ErrRecordNotFound) {
		return nil, false, err
	}
	var task model.DeliveryTask
	if err := s.db.WithContext(ctx).First(&task, "id = ?", taskID).Error; err != nil {
		return nil, false, err
	}
	if task.PayloadHash != payloadHash || task.IdempotencyKey != idempotencyKey {
		return nil, false, fmt.Errorf("execution request does not match the immutable task payload")
	}
	if task.Status != model.TaskDispatching {
		return nil, false, fmt.Errorf("task in state %q is not ready for execution", task.Status)
	}
	target, err := s.getTarget(ctx, task.TargetID)
	if err != nil {
		return nil, false, err
	}
	var payload deliveryPayload
	if err := json.Unmarshal([]byte(task.ImmutablePayload), &payload); err != nil {
		return nil, false, err
	}
	meta := parseTargetMetadata(target.Metadata)
	// Persist execution record BEFORE launching AWX to ensure crash recovery.
	now := time.Now()
	execution := model.ExecutionJob{TaskID: task.ID, IdempotencyKey: task.IdempotencyKey, ExecutorJobID: "pending-" + task.ID, Status: "launching", StartedAt: &now}
	if err := s.db.WithContext(ctx).Create(&execution).Error; err != nil {
		return nil, false, err
	}
	job, err := s.awx.Launch(ctx, target.AWXTemplateID, AWXLaunchRequest{InventoryID: target.AWXInventoryID, Limit: task.TargetHost, ExtraVars: deliveryExtraVars(&task, payload, meta)})
	if err != nil {
		_ = s.db.WithContext(ctx).Model(&execution).Updates(map[string]any{"status": "failed", "finished_at": time.Now()})
		return nil, false, err
	}
	if err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
		if err := tx.Model(&execution).Updates(map[string]any{"executor_job_id": fmt.Sprint(job.ID), "status": "running"}).Error; err != nil {
			return err
		}
		return s.transitionTx(tx, &task, model.TaskRunning, "AWX job started", "")
	}); err != nil {
		return nil, false, err
	}
	return &execution, false, nil
}

// deliveryExtraVars 组装传给 playbook 的变量：必传项固定注入，
// 选填项仅在用户显式设置时下发，未设置时由 playbook 默认基线兜底。
func deliveryExtraVars(task *model.DeliveryTask, payload deliveryPayload, meta targetMetadata) map[string]any {
	topology := payload.Topology
	if topology == "" {
		// 存量任务的 ImmutablePayload 无 topology 字段，回退到 target metadata。
		topology = meta.Topology
	}
	vars := map[string]any{
		"task_id": task.ID, "payload_hash": task.PayloadHash,
		"target_hosts": task.TargetHost, "topology": topology,
		"instance_name": payload.InstanceName, "mysql_port": task.MySQLPort,
		"memory_mb": payload.MemoryMi, "storage_gb": payload.StorageGi,
		"mysql_version": payload.MySQLVersion,
	}
	if payload.DataDisk != "" {
		vars["data_disk"] = payload.DataDisk
	}
	if payload.Timezone != "" {
		vars["timezone"] = payload.Timezone
	}
	if payload.LowerCaseTableNames != nil {
		vars["lower_case_table_names"] = *payload.LowerCaseTableNames
	}
	if payload.CharacterSet != "" {
		vars["character_set"] = payload.CharacterSet
	}
	if payload.Collation != "" {
		vars["collation"] = payload.Collation
	}
	if payload.MaxConnections != "" {
		vars["max_connections"] = payload.MaxConnections
	}
	if payload.InnodbRedoLogCapacity != "" {
		vars["innodb_redo_log_capacity"] = payload.InnodbRedoLogCapacity
	}
	if payload.InnodbFlushLogAtTrxCommit != nil {
		vars["innodb_flush_log_at_trx_commit"] = *payload.InnodbFlushLogAtTrxCommit
	}
	if payload.SyncBinlog != nil {
		vars["sync_binlog"] = *payload.SyncBinlog
	}
	if payload.InnodbIOCapacity != 0 {
		vars["innodb_io_capacity"] = payload.InnodbIOCapacity
	}
	if payload.LongQueryTime != 0 {
		vars["long_query_time"] = payload.LongQueryTime
	}
	if payload.BinlogExpireLogsSeconds != 0 {
		vars["binlog_expire_logs_seconds"] = payload.BinlogExpireLogsSeconds
	}
	if payload.MaxBinlogSize != "" {
		vars["max_binlog_size"] = payload.MaxBinlogSize
	}
	return vars
}

func (s *DeliveryService) PollOnce(ctx context.Context) error {
	var jobs []model.ExecutionJob
	if err := s.db.WithContext(ctx).Where("status = ?", "running").Find(&jobs).Error; err != nil {
		return err
	}
	for _, execution := range jobs {
		job, err := s.awx.GetJob(ctx, execution.ExecutorJobID)
		if err != nil {
			// A transient AWX/API failure is not evidence that deployment failed.
			// Keep the job running and retry on the next scheduler tick.
			continue
		}
		switch strings.ToLower(job.Status) {
		case "pending", "waiting", "running", "new":
			continue
		case "successful":
			s.finishExecution(ctx, &execution, "successful")
			if err := s.completeTask(ctx, execution.TaskID); err != nil {
				_ = s.beginRollback(ctx, execution.TaskID, err.Error())
			}
		case "canceled":
			s.finishExecution(ctx, &execution, "canceled")
			_ = s.beginRollback(ctx, execution.TaskID, "AWX deployment job was canceled")
		default:
			s.finishExecution(ctx, &execution, "failed")
			_ = s.beginRollback(ctx, execution.TaskID, "AWX job finished with status "+job.Status)
		}
	}
	return nil
}

func (s *DeliveryService) finishExecution(ctx context.Context, execution *model.ExecutionJob, status string) {
	now := time.Now()
	_ = s.db.WithContext(ctx).Model(execution).Updates(map[string]any{"status": status, "finished_at": now})
}

func (s *DeliveryService) completeTask(ctx context.Context, taskID string) error {
	var task model.DeliveryTask
	if err := s.db.WithContext(ctx).First(&task, "id = ?", taskID).Error; err != nil {
		return err
	}
	var payload deliveryPayload
	if err := json.Unmarshal([]byte(task.ImmutablePayload), &payload); err != nil {
		return err
	}
	addr := fmt.Sprintf("%s:%d", task.TargetHostIP, task.MySQLPort)
	if err := mysqlReady(ctx, addr); err != nil {
		return fmt.Errorf("MySQL health check failed: %w", err)
	}
	now := time.Now()
	if err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
		if err := s.transitionTx(tx, &task, model.TaskRegistering, "AWX succeeded and MySQL health check passed", ""); err != nil {
			return err
		}
		instance := model.MySQLInstance{TaskID: task.ID, BusinessLineID: task.BusinessLineID, TargetID: task.TargetID, Namespace: payload.Namespace, Name: payload.InstanceName, NodeName: task.TargetHost, Host: task.TargetHostIP, Port: task.MySQLPort, Version: payload.MySQLVersion, Status: "active"}
		if err := tx.Create(&instance).Error; err != nil {
			return err
		}
		if err := tx.Create(&model.ResourceUsage{TaskID: task.ID, InstanceID: instance.ID, BusinessLineID: task.BusinessLineID, TargetID: task.TargetID, CPUMilli: payload.CPUMilli, MemoryMi: payload.MemoryMi, StorageGi: payload.StorageGi, InstanceCount: 1, Status: "active"}).Error; err != nil {
			return err
		}
		if err := tx.Model(&model.ResourceReservation{}).Where("task_id = ? AND status = ?", task.ID, "reserved").Update("status", "consumed").Error; err != nil {
			return err
		}
		if err := tx.Model(&model.ExecutionJob{}).Where("task_id = ?", task.ID).Updates(map[string]any{"status": "successful", "finished_at": now}).Error; err != nil {
			return err
		}
		return nil
	}); err != nil {
		return err
	}
	if err := s.RegisterCloudDM(ctx, task.ID); err != nil {
		message := fmt.Sprintf("CloudDM registration failed: %v", err)
		// The database is already healthy and accounted for at this point. A
		// registration outage must not trigger destructive instance rollback.
		if transitionErr := s.transition(ctx, &task, model.TaskRegisterFailed,
			"MySQL delivered; CloudDM registration failed and can be retried", message); transitionErr != nil {
			return fmt.Errorf("%s; cannot record register_failed: %w", message, transitionErr)
		}
		return nil
	}
	return s.transition(ctx, &task, model.TaskFinished, "MySQL delivery completed", "")
}

// RetryCloudDMRegistration retries only the external registration step. The
// instance and its resource usage remain active throughout the retry.
func (s *DeliveryService) RetryCloudDMRegistration(ctx context.Context, taskID string) error {
	var task model.DeliveryTask
	if err := s.db.WithContext(ctx).First(&task, "id = ?", taskID).Error; err != nil {
		return err
	}
	if task.Status != model.TaskRegisterFailed {
		return fmt.Errorf("task %s is in state %q and cannot retry CloudDM registration", taskID, task.Status)
	}
	if err := s.transition(ctx, &task, model.TaskRegistering, "CloudDM registration retry started", ""); err != nil {
		return err
	}
	if err := s.RegisterCloudDM(ctx, taskID); err != nil {
		message := fmt.Sprintf("CloudDM registration failed: %v", err)
		if transitionErr := s.transition(ctx, &task, model.TaskRegisterFailed, "CloudDM registration retry failed; instance remains active", message); transitionErr != nil {
			return fmt.Errorf("%s; cannot restore register_failed: %w", message, transitionErr)
		}
		return err
	}
	return s.transition(ctx, &task, model.TaskFinished, "CloudDM registration completed", "")
}

func (s *DeliveryService) RegisterCloudDM(ctx context.Context, taskID string) error {
	if s.cfg.CloudDMRegisterURL == "" {
		return nil
	}
	var instance model.MySQLInstance
	if err := s.db.WithContext(ctx).Where("task_id = ?", taskID).First(&instance).Error; err != nil {
		return err
	}
	body := map[string]any{"name": instance.Name, "host": instance.Host, "port": instance.Port, "username": "root", "database_type": "mysql"}
	raw, _ := json.Marshal(body)
	req, err := http.NewRequestWithContext(ctx, http.MethodPost, s.cfg.CloudDMRegisterURL, bytes.NewReader(raw))
	if err != nil {
		return err
	}
	req.Header.Set("Content-Type", "application/json")
	if s.cfg.CloudDMAPIToken != "" {
		req.Header.Set("Authorization", "Bearer "+s.cfg.CloudDMAPIToken)
	}
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
		return fmt.Errorf("CloudDM returned %s", resp.Status)
	}
	return nil
}

// beginRollback changes the task into the compensating workflow and launches
// the dedicated AWX rollback template. The deploy reservation is deliberately
// kept until rollback succeeds so a failed cleanup cannot be silently
// overwritten by a later delivery.
func (s *DeliveryService) beginRollback(ctx context.Context, taskID, reason string) error {
	var task model.DeliveryTask
	if err := s.db.WithContext(ctx).First(&task, "id = ?", taskID).Error; err != nil {
		return err
	}
	if rollbackProtectedStatus(task.Status) {
		return nil
	}

	if s.cfg.RollbackTemplateID == 0 {
		if err := s.transition(ctx, &task, model.TaskRollbackFailed,
			"automatic rollback is not configured; manual cleanup is required", reason); err != nil {
			return err
		}
		return s.holdReservationForRollback(ctx, task.ID)
	}
	if err := s.transition(ctx, &task, model.TaskRollbackPending, "automatic rollback queued", reason); err != nil {
		return err
	}
	if err := s.holdReservationForRollback(ctx, task.ID); err != nil {
		return s.markRollbackFailed(ctx, task.ID, "cannot hold resources during rollback: "+err.Error())
	}
	return s.launchRollbackJob(ctx, &task, reason)
}

func rollbackProtectedStatus(status string) bool {
	switch status {
	case model.TaskRollbackPending, model.TaskRollingBack, model.TaskRolledBack, model.TaskRollbackFailed,
		model.TaskRollbackAck, model.TaskRegisterFailed, model.TaskFinished, model.TaskCanceled:
		return true
	default:
		return false
	}
}

func (s *DeliveryService) launchRollbackJob(ctx context.Context, task *model.DeliveryTask, reason string) error {
	target, err := s.getTarget(ctx, task.TargetID)
	if err != nil {
		return s.markRollbackFailed(ctx, task.ID, "rollback target is unavailable: "+err.Error())
	}
	var payload deliveryPayload
	if err := json.Unmarshal([]byte(task.ImmutablePayload), &payload); err != nil {
		return s.markRollbackFailed(ctx, task.ID, "rollback payload is invalid: "+err.Error())
	}
	now := time.Now()
	var rollback model.RollbackJob
	if err := s.db.WithContext(ctx).Where("task_id = ?", task.ID).First(&rollback).Error; errors.Is(err, gorm.ErrRecordNotFound) {
		rollback = model.RollbackJob{TaskID: task.ID}
	} else if err != nil {
		return s.markRollbackFailed(ctx, task.ID, "cannot load rollback job: "+err.Error())
	}
	rollback.ExecutorJobID = "pending-rollback-" + task.ID
	rollback.Status = "launching"
	rollback.Reason = reason
	rollback.StartedAt = &now
	rollback.FinishedAt = nil
	if err := s.db.WithContext(ctx).Save(&rollback).Error; err != nil {
		return s.markRollbackFailed(ctx, task.ID, "cannot persist rollback job: "+err.Error())
	}
	job, err := s.awx.Launch(ctx, s.cfg.RollbackTemplateID, AWXLaunchRequest{
		InventoryID: target.AWXInventoryID,
		Limit:       task.TargetHost,
		ExtraVars:   rollbackExtraVars(task, payload),
	})
	if err != nil {
		_ = s.db.WithContext(ctx).Model(&rollback).Updates(map[string]any{"status": "failed", "finished_at": time.Now()}).Error
		return s.markRollbackFailed(ctx, task.ID, "cannot launch rollback AWX job: "+err.Error())
	}
	if err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
		var current model.DeliveryTask
		if err := tx.First(&current, "id = ?", task.ID).Error; err != nil {
			return err
		}
		if current.Status != model.TaskRollbackPending {
			return fmt.Errorf("task %s is no longer pending rollback (state %q)", task.ID, current.Status)
		}
		if err := tx.Model(&rollback).Updates(map[string]any{"executor_job_id": fmt.Sprint(job.ID), "status": "running"}).Error; err != nil {
			return err
		}
		return s.transitionTx(tx, &current, model.TaskRollingBack, "rollback AWX job started", reason)
	}); err != nil {
		return s.markRollbackFailed(ctx, task.ID, "cannot record rollback AWX job: "+err.Error())
	}
	return nil
}

func (s *DeliveryService) RetryRollback(ctx context.Context, taskID string) error {
	var task model.DeliveryTask
	if err := s.db.WithContext(ctx).First(&task, "id = ?", taskID).Error; err != nil {
		return err
	}
	if task.Status != model.TaskRollbackFailed {
		return fmt.Errorf("task %s is in state %q and cannot retry rollback", taskID, task.Status)
	}
	if s.cfg.RollbackTemplateID == 0 {
		return fmt.Errorf("automatic rollback is not configured")
	}
	reason := "manual rollback retry requested"
	if err := s.transition(ctx, &task, model.TaskRollbackPending, "manual rollback retry queued", reason); err != nil {
		return err
	}
	if err := s.holdReservationForRollback(ctx, task.ID); err != nil {
		_ = s.markRollbackFailed(ctx, task.ID, "cannot hold resources during rollback retry: "+err.Error())
		return err
	}
	return s.launchRollbackJob(ctx, &task, reason)
}

// AcknowledgeRollbackRelease is an explicit platform-admin escape hatch after
// the operator has verified the target host. It releases bookkeeping only; it
// does not claim that the automatic rollback succeeded.
func (s *DeliveryService) AcknowledgeRollbackRelease(ctx context.Context, taskID string) error {
	now := time.Now()
	return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
		var task model.DeliveryTask
		if err := tx.First(&task, "id = ?", taskID).Error; err != nil {
			return err
		}
		if task.Status != model.TaskRollbackFailed {
			return fmt.Errorf("task %s is in state %q and cannot acknowledge rollback release", taskID, task.Status)
		}
		if err := tx.Model(&model.MySQLInstance{}).Where("task_id = ? AND status = ?", taskID, "active").Updates(map[string]any{"status": "rollback_acknowledged", "updated_at": now}).Error; err != nil {
			return err
		}
		if err := tx.Model(&model.ResourceUsage{}).Where("task_id = ? AND status = ?", taskID, "active").Updates(map[string]any{"status": "released", "released_at": now, "updated_at": now}).Error; err != nil {
			return err
		}
		if err := tx.Model(&model.ResourceReservation{}).Where("task_id = ? AND status = ?", taskID, "rollback").Update("status", "released").Error; err != nil {
			return err
		}
		return s.transitionTx(tx, &task, model.TaskRollbackAck, "rollback release acknowledged by platform admin", "")
	})
}

func rollbackExtraVars(task *model.DeliveryTask, payload deliveryPayload) map[string]any {
	return map[string]any{
		"target_hosts":  task.TargetHost,
		"instance_name": payload.InstanceName,
		"data_disk":     payload.DataDisk,
		"task_id":       task.ID,
		"rollback":      true,
	}
}

func (s *DeliveryService) holdReservationForRollback(ctx context.Context, taskID string) error {
	return s.db.WithContext(ctx).Model(&model.ResourceReservation{}).
		Where("task_id = ? AND status IN ?", taskID, []string{"reserved", "consumed"}).
		Update("status", "rollback").Error
}

func (s *DeliveryService) markRollbackFailed(ctx context.Context, taskID, message string) error {
	var task model.DeliveryTask
	if err := s.db.WithContext(ctx).First(&task, "id = ?", taskID).Error; err != nil {
		return err
	}
	if task.Status == model.TaskRolledBack || task.Status == model.TaskFinished || task.Status == model.TaskCanceled ||
		task.Status == model.TaskRollbackFailed || task.Status == model.TaskRollbackAck || task.Status == model.TaskRegisterFailed {
		return nil
	}
	return s.transition(ctx, &task, model.TaskRollbackFailed, "automatic rollback failed; manual cleanup is required", message)
}

func (s *DeliveryService) PollRollbackOnce(ctx context.Context) error {
	var jobs []model.RollbackJob
	if err := s.db.WithContext(ctx).Where("status IN ?", []string{"launching", "running"}).Find(&jobs).Error; err != nil {
		return err
	}
	for _, rollback := range jobs {
		if rollback.Status == "launching" {
			if !rollbackLaunchExpired(rollback, time.Now()) {
				continue
			}
			// The AWX launch result is unknown. Do not blindly launch a second
			// job; surface a recoverable failure for an explicit admin retry.
			expired, err := s.expireRollbackLaunch(ctx, rollback.ID)
			if err != nil {
				return err
			}
			if !expired {
				continue
			}
			_ = s.markRollbackFailed(ctx, rollback.TaskID, "rollback launch timed out before its AWX job ID was recorded; manual retry is required")
			continue
		}
		job, err := s.awx.GetJob(ctx, rollback.ExecutorJobID)
		if err != nil {
			// Do not declare cleanup failed because AWX is temporarily unreachable.
			continue
		}
		switch strings.ToLower(job.Status) {
		case "pending", "waiting", "running", "new":
			continue
		case "successful":
			_ = s.finishRollbackJob(ctx, &rollback, "successful")
			if err := s.completeRollback(ctx, rollback.TaskID); err != nil {
				_ = s.markRollbackFailed(ctx, rollback.TaskID, err.Error())
			}
		case "canceled":
			_ = s.finishRollbackJob(ctx, &rollback, "canceled")
			_ = s.markRollbackFailed(ctx, rollback.TaskID, "rollback AWX job was canceled")
		default:
			_ = s.finishRollbackJob(ctx, &rollback, "failed")
			_ = s.markRollbackFailed(ctx, rollback.TaskID, "rollback AWX job finished with status "+job.Status)
		}
	}
	return nil
}

func rollbackLaunchExpired(rollback model.RollbackJob, now time.Time) bool {
	if rollback.Status != "launching" {
		return false
	}
	if rollback.StartedAt == nil {
		return true
	}
	return !now.Before(rollback.StartedAt.Add(rollbackLaunchTimeout))
}

func (s *DeliveryService) expireRollbackLaunch(ctx context.Context, rollbackID uint64) (bool, error) {
	result := s.db.WithContext(ctx).Model(&model.RollbackJob{}).
		Where("id = ? AND status = ?", rollbackID, "launching").
		Updates(map[string]any{"status": "launch_timeout", "finished_at": time.Now()})
	return result.RowsAffected == 1, result.Error
}

func (s *DeliveryService) finishRollbackJob(ctx context.Context, rollback *model.RollbackJob, status string) error {
	return s.db.WithContext(ctx).Model(rollback).Updates(map[string]any{"status": status, "finished_at": time.Now()}).Error
}

func (s *DeliveryService) completeRollback(ctx context.Context, taskID string) error {
	now := time.Now()
	return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
		var task model.DeliveryTask
		if err := tx.First(&task, "id = ?", taskID).Error; err != nil {
			return err
		}
		if task.Status != model.TaskRollingBack {
			return fmt.Errorf("task %s is in state %q, cannot complete rollback", taskID, task.Status)
		}
		if err := tx.Model(&model.MySQLInstance{}).Where("task_id = ?", taskID).Updates(map[string]any{"status": "rolled_back", "updated_at": now}).Error; err != nil {
			return err
		}
		if err := tx.Model(&model.ResourceUsage{}).Where("task_id = ? AND status = ?", taskID, "active").Updates(map[string]any{"status": "released", "released_at": now, "updated_at": now}).Error; err != nil {
			return err
		}
		if err := tx.Model(&model.ResourceReservation{}).Where("task_id = ? AND status IN ?", taskID, []string{"reserved", "consumed", "rollback"}).Update("status", "released").Error; err != nil {
			return err
		}
		return s.transitionTx(tx, &task, model.TaskRolledBack, "MySQL instance rollback completed", "")
	})
}

func (s *DeliveryService) failTask(ctx context.Context, task *model.DeliveryTask, status, message string) error {
	return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
		var current model.DeliveryTask
		if err := tx.First(&current, "id = ?", task.ID).Error; err != nil {
			return err
		}
		if current.Status != model.TaskPending && current.Status != model.TaskDispatching && current.Status != model.TaskRunning && current.Status != model.TaskRegistering && current.Status != model.TaskCanceling {
			return nil
		}
		if err := s.transitionTx(tx, &current, status, message, message); err != nil {
			return err
		}
		if status == model.TaskExecutionFailed || status == model.TaskValidationFailed || status == model.TaskCanceled {
			return s.releaseReservation(tx, current.ID)
		}
		return nil
	})
}

func (s *DeliveryService) Run(ctx context.Context) {
	interval := time.Duration(s.cfg.DeliveryPollSeconds) * time.Second
	if interval < time.Second {
		interval = time.Second
	}
	ticker := time.NewTicker(interval)
	defer ticker.Stop()
	for {
		select {
		case <-ctx.Done():
			return
		case <-ticker.C:
			_ = s.DispatchOnce(ctx)
			_ = s.PollOnce(ctx)
			_ = s.PollRollbackOnce(ctx)
		}
	}
}