package service import ( "bytes" "context" "crypto/rand" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "fmt" "net" "net/http" "regexp" "strings" "sync" "time" "github.com/1024XEngineer/xinfra/server/internal/config" "github.com/1024XEngineer/xinfra/server/internal/model" "gorm.io/gorm" "gorm.io/gorm/clause" ) var dnsLabelPattern = regexp.MustCompile(`^[a-z0-9](?:[-a-z0-9]*[a-z0-9])?$`) type MySQLDeliveryInput struct { BusinessLineID uint64 `json:"business_line_id" binding:"required"` TargetID uint64 `json:"target_id" binding:"required"` Namespace string `json:"namespace" binding:"required"` InstanceName string `json:"instance_name" binding:"required"` MySQLVersion string `json:"mysql_version"` Topology string `json:"topology"` MySQLPort int `json:"mysql_port"` DataDisk string `json:"data_disk"` CPUMilli int64 `json:"cpu_milli" binding:"required"` MemoryMi int64 `json:"memory_mi" binding:"required"` StorageGi int64 `json:"storage_gi" binding:"required"` // 数据库配置(选填,缺省由 playbook 基线兜底) Timezone string `json:"timezone"` LowerCaseTableNames *int `json:"lower_case_table_names"` CharacterSet string `json:"character_set"` Collation string `json:"collation"` // 高级参数(选填,零值视为未设置) MaxConnections string `json:"max_connections"` InnodbRedoLogCapacity string `json:"innodb_redo_log_capacity"` InnodbFlushLogAtTrxCommit *int `json:"innodb_flush_log_at_trx_commit"` SyncBinlog *int `json:"sync_binlog"` InnodbIOCapacity int `json:"innodb_io_capacity"` LongQueryTime float64 `json:"long_query_time"` BinlogExpireLogsSeconds int64 `json:"binlog_expire_logs_seconds"` MaxBinlogSize string `json:"max_binlog_size"` } type deliveryPayload struct { MySQLDeliveryInput TargetType string `json:"target_type"` } type DeliveryTarget struct { ID uint64 `json:"id"` Name string `json:"name"` TargetType string `json:"target_type"` AWXInventoryID uint64 `json:"awx_inventory_id"` AWXTemplateID uint64 `json:"awx_template_id"` Enabled bool `json:"enabled"` Metadata string `json:"metadata"` } // targetMetadata describes the native VM候选节点池以及部署形态,由 AWX inventory hosts 动态组装。 type targetMetadata struct { Topology string `json:"topology"` MySQLPort int `json:"mysql_port"` Hosts []targetHost `json:"hosts"` } type targetHost struct { Name string `json:"name"` IP string `json:"ip"` } func parseTargetMetadata(raw string) targetMetadata { meta := targetMetadata{} if strings.TrimSpace(raw) != "" { _ = json.Unmarshal([]byte(raw), &meta) } if meta.Topology == "" { meta.Topology = "standalone" } if meta.MySQLPort == 0 { meta.MySQLPort = 3307 } return meta } // firstFreeHost 返回候选池中第一个未被占用的节点。 func firstFreeHost(hosts []targetHost, occupied []string) *targetHost { taken := make(map[string]bool, len(occupied)) for _, h := range occupied { taken[h] = true } for i := range hosts { if !taken[hosts[i].Name] { return &hosts[i] } } return nil } // 端口池 13306-13999:混合分配模型,用户留空时自动分配,可覆盖为池内指定端口。 const ( mysqlPortPoolStart = 13306 mysqlPortPoolEnd = 13999 ) // allocatePort 在目标主机已占用端口集上做混合分配:指定端口验冲突,未指定则取池内首个空闲端口。 func allocatePort(requested int, used []int) (int, error) { taken := make(map[int]bool, len(used)) for _, p := range used { taken[p] = true } if requested != 0 { if taken[requested] { return 0, fmt.Errorf("mysql_port %d is already allocated on the target host", requested) } return requested, nil } for p := mysqlPortPoolStart; p <= mysqlPortPoolEnd; p++ { if !taken[p] { return p, nil } } return 0, fmt.Errorf("mysql port pool %d-%d is exhausted on the target host", mysqlPortPoolStart, mysqlPortPoolEnd) } type DeliveryService struct { db *gorm.DB cfg config.Config awx *AWXClient audit *AuditService executionMu sync.Mutex } func (s *DeliveryService) DB() *gorm.DB { return s.db } func NewDeliveryService(cfg config.Config, db *gorm.DB, audit *AuditService) *DeliveryService { return &DeliveryService{db: db, cfg: cfg, awx: NewAWXClient(cfg.AWXBaseURL, cfg.AWXToken, cfg.AWXUsername, cfg.AWXPassword), audit: audit} } func (s *DeliveryService) ListTargets(ctx context.Context, component string) ([]DeliveryTarget, error) { templates, err := s.awx.ListJobTemplates(ctx) if err != nil { return nil, err } component = strings.ToLower(strings.TrimSpace(component)) var targets []DeliveryTarget for _, template := range templates { if template.Inventory == 0 { continue } if component != "" && component != "all" { text := strings.ToLower(template.Name + " " + template.Description) if !strings.Contains(text, component) { continue } } target, err := s.awxDeliveryTarget(ctx, template) if err != nil { continue } targets = append(targets, target) } return targets, nil } func (s *DeliveryService) getTarget(ctx context.Context, templateID uint64) (DeliveryTarget, error) { template, err := s.awx.GetJobTemplate(ctx, templateID) if err != nil { return DeliveryTarget{}, fmt.Errorf("deployment target is unavailable: %w", err) } return s.awxDeliveryTarget(ctx, *template) } func (s *DeliveryService) awxDeliveryTarget(ctx context.Context, template AWXJobTemplate) (DeliveryTarget, error) { hosts, err := s.awx.ListInventoryHosts(ctx, template.Inventory) if err != nil { return DeliveryTarget{}, err } meta := targetMetadata{Topology: "standalone", MySQLPort: 3307} for _, host := range hosts { if !host.Enabled { continue } meta.Hosts = append(meta.Hosts, targetHost{Name: host.Name, IP: AWXHostIP(host)}) } raw, err := json.Marshal(meta) if err != nil { return DeliveryTarget{}, err } return DeliveryTarget{ ID: template.ID, Name: template.Name, TargetType: "k8s", AWXInventoryID: template.Inventory, AWXTemplateID: template.ID, Enabled: true, Metadata: string(raw), }, nil } func (s *DeliveryService) CreateTask(ctx context.Context, userID uint64, isAdmin bool, idempotencyKey string, input MySQLDeliveryInput) (*model.DeliveryTask, bool, error) { idempotencyKey = strings.TrimSpace(idempotencyKey) if idempotencyKey == "" || len(idempotencyKey) > 128 { return nil, false, fmt.Errorf("Idempotency-Key header is required and must not exceed 128 characters") } if err := validateDeliveryInput(input, s.cfg.DeliveryDataDisks); err != nil { return nil, false, err } var existing model.DeliveryTask if err := s.db.WithContext(ctx).Where("idempotency_key = ?", idempotencyKey).First(&existing).Error; err == nil { if existing.RequestedBy != userID { return nil, false, fmt.Errorf("idempotency key is already in use by another user") } return &existing, true, nil } else if !errors.Is(err, gorm.ErrRecordNotFound) { return nil, false, err } target, err := s.getTarget(ctx, input.TargetID) if err != nil { return nil, false, err } if target.TargetType != "k8s" { return nil, false, fmt.Errorf("target type %q is not supported in the MVP", target.TargetType) } if !isAdmin { var count int64 if err := s.db.WithContext(ctx).Model(&model.BusinessLineUser{}). Where("business_line_id = ? AND user_id = ?", input.BusinessLineID, userID).Count(&count).Error; err != nil { return nil, false, err } if count == 0 { return nil, false, fmt.Errorf("user is not authorized for this business line") } } if input.MySQLVersion == "" { input.MySQLVersion = "8.0" } if input.Topology == "" { input.Topology = "standalone" } if input.DataDisk == "" { if len(s.cfg.DeliveryDataDisks) > 0 { input.DataDisk = s.cfg.DeliveryDataDisks[0] } else { input.DataDisk = "/data" } } payload := deliveryPayload{MySQLDeliveryInput: input, TargetType: target.TargetType} raw, err := json.Marshal(payload) if err != nil { return nil, false, err } digest := sha256.Sum256(raw) task := model.DeliveryTask{ ID: randomUUID(), BusinessLineID: input.BusinessLineID, RequestedBy: userID, TargetType: target.TargetType, TargetID: target.ID, Namespace: input.Namespace, InstanceName: input.InstanceName, Status: model.TaskPending, ImmutablePayload: string(raw), PayloadHash: hex.EncodeToString(digest[:]), IdempotencyKey: idempotencyKey, } if err := s.db.WithContext(ctx).Create(&task).Error; err != nil { if lookupErr := s.db.WithContext(ctx).Where("idempotency_key = ?", idempotencyKey).First(&existing).Error; lookupErr == nil { return &existing, true, nil } return nil, false, err } _ = s.db.WithContext(ctx).Create(&model.TaskEvent{TaskID: task.ID, ToState: model.TaskPending, Message: "delivery task created"}).Error return &task, false, nil } // 版本白名单与 playbook 的 mysql_package_map 保持同步。 var supportedMySQLVersions = map[string]bool{"8.0": true} // 拓扑白名单:playbook 已支持 primary_replica/mgr_3 的配置渲染, // 但调度器仍是单主机模型且复制编排未自动化,本期仅放开 standalone。 var supportedTopologies = map[string]bool{"standalone": true} var supportedCharsets = map[string]bool{"utf8mb4": true, "utf8": true, "gbk": true, "latin1": true} // 高级参数档位白名单(与 docs/mysql-parameter-selection.md 保持一致) var ( supportedMaxConnections = map[string]bool{"auto": true, "200": true, "500": true, "1000": true, "2000": true, "4000": true, "8000": true, "16000": true} supportedLogSizes = map[string]bool{"128M": true, "256M": true, "512M": true, "1G": true} supportedIOCapacities = map[int]bool{200: true, 2000: true, 5000: true} supportedLongQueryTimes = map[float64]bool{0.5: true, 1: true, 2: true, 5: true, 10: true} supportedBinlogExpireSecs = map[int64]bool{86400: true, 259200: true, 604800: true, 1209600: true} ) // timezone 仅接受偏移量(±HH:MM)、SYSTEM 或命名时区(如 Asia/Shanghai)。 var timezonePattern = regexp.MustCompile(`^([+-](0\d|1[0-4]):[0-5]\d|SYSTEM|[A-Za-z]+(?:/[A-Za-z0-9_+-]+)+)$`) func validateDeliveryInput(input MySQLDeliveryInput, dataDisks []string) error { if len(input.Namespace) > 63 || !dnsLabelPattern.MatchString(input.Namespace) { return fmt.Errorf("namespace must be a valid Kubernetes DNS label") } if len(input.InstanceName) > 63 || !dnsLabelPattern.MatchString(input.InstanceName) { return fmt.Errorf("instance_name must be a valid Kubernetes DNS label") } // 与文档目标态一致(memory 2048-65536 MiB / storage 20-2000 GiB),playbook assert 同步。 if input.CPUMilli < 100 || input.CPUMilli > 64000 || input.MemoryMi < 2048 || input.MemoryMi > 65536 || input.StorageGi < 20 || input.StorageGi > 2000 { return fmt.Errorf("requested resources are outside the supported range (memory: 2048-65536 MiB, storage: 20-2000 GiB)") } if input.MySQLVersion != "" && !supportedMySQLVersions[input.MySQLVersion] { return fmt.Errorf("unsupported mysql_version %q, supported: 8.0", input.MySQLVersion) } if input.Topology != "" && !supportedTopologies[input.Topology] { return fmt.Errorf("unsupported topology %q, supported: standalone (primary_replica/mgr_3 pending scheduler support)", input.Topology) } if input.MySQLPort != 0 && (input.MySQLPort < mysqlPortPoolStart || input.MySQLPort > mysqlPortPoolEnd) { return fmt.Errorf("mysql_port must be left empty for auto allocation or within the pool %d-%d", mysqlPortPoolStart, mysqlPortPoolEnd) } if input.DataDisk != "" { allowed := dataDisks if len(allowed) == 0 { allowed = []string{"/data"} } found := false for _, disk := range allowed { if input.DataDisk == disk { found = true break } } if !found { return fmt.Errorf("data_disk %q is not in the allowed mount point list %v", input.DataDisk, allowed) } } if input.Timezone != "" && !timezonePattern.MatchString(input.Timezone) { return fmt.Errorf("timezone must be an offset like +08:00, SYSTEM, or a named zone like Asia/Shanghai") } if input.LowerCaseTableNames != nil && *input.LowerCaseTableNames != 0 && *input.LowerCaseTableNames != 1 { return fmt.Errorf("lower_case_table_names must be 0 or 1") } if input.CharacterSet != "" && !supportedCharsets[input.CharacterSet] { return fmt.Errorf("unsupported character_set %q, supported: utf8mb4, utf8, gbk, latin1", input.CharacterSet) } if input.Collation != "" { charset := input.CharacterSet if charset == "" { charset = "utf8mb4" } if !strings.HasPrefix(input.Collation, charset+"_") { return fmt.Errorf("collation %q does not match character_set %q", input.Collation, charset) } } if input.MaxConnections != "" && !supportedMaxConnections[input.MaxConnections] { return fmt.Errorf("max_connections must be one of auto, 200, 500, 1000, 2000, 4000, 8000, 16000") } if input.InnodbRedoLogCapacity != "" && !supportedLogSizes[input.InnodbRedoLogCapacity] { return fmt.Errorf("innodb_redo_log_capacity must be one of 128M, 256M, 512M, 1G") } if input.InnodbFlushLogAtTrxCommit != nil && (*input.InnodbFlushLogAtTrxCommit < 0 || *input.InnodbFlushLogAtTrxCommit > 2) { return fmt.Errorf("innodb_flush_log_at_trx_commit must be 0, 1 or 2") } if input.SyncBinlog != nil && *input.SyncBinlog != 0 && *input.SyncBinlog != 1 { return fmt.Errorf("sync_binlog must be 0 or 1") } if input.InnodbIOCapacity != 0 && !supportedIOCapacities[input.InnodbIOCapacity] { return fmt.Errorf("innodb_io_capacity must be one of 200, 2000, 5000") } if input.LongQueryTime != 0 && !supportedLongQueryTimes[input.LongQueryTime] { return fmt.Errorf("long_query_time must be one of 0.5, 1, 2, 5, 10") } if input.BinlogExpireLogsSeconds != 0 && !supportedBinlogExpireSecs[input.BinlogExpireLogsSeconds] { return fmt.Errorf("binlog_expire_logs_seconds must be one of 86400, 259200, 604800, 1209600") } if input.MaxBinlogSize != "" && !supportedLogSizes[input.MaxBinlogSize] { return fmt.Errorf("max_binlog_size must be one of 128M, 256M, 512M, 1G") } return nil } func (s *DeliveryService) ListTasks(ctx context.Context, userID uint64, isAdmin bool, businessLineID uint64) ([]model.DeliveryTask, error) { query := s.db.WithContext(ctx).Order("created_at DESC") if businessLineID != 0 { query = query.Where("business_line_id = ?", businessLineID) } if !isAdmin { query = query.Where("business_line_id IN (?)", s.db.Model(&model.BusinessLineUser{}).Select("business_line_id").Where("user_id = ?", userID)) } var tasks []model.DeliveryTask return tasks, query.Find(&tasks).Error } func (s *DeliveryService) GetTask(ctx context.Context, taskID string, userID uint64, isAdmin bool) (*model.DeliveryTask, []model.TaskEvent, error) { query := s.db.WithContext(ctx).Where("id = ?", taskID) if !isAdmin { query = query.Where("business_line_id IN (?)", s.db.Model(&model.BusinessLineUser{}).Select("business_line_id").Where("user_id = ?", userID)) } var task model.DeliveryTask if err := query.First(&task).Error; err != nil { return nil, nil, err } var events []model.TaskEvent if err := s.db.WithContext(ctx).Where("task_id = ?", taskID).Order("id ASC").Find(&events).Error; err != nil { return nil, nil, err } return &task, events, nil } func (s *DeliveryService) AWXJobStdout(ctx context.Context, jobID string) (string, error) { return s.awx.JobStdout(ctx, jobID) } func (s *DeliveryService) Cancel(ctx context.Context, taskID string, userID uint64, isAdmin bool) error { task, _, err := s.GetTask(ctx, taskID, userID, isAdmin) if err != nil { return err } if task.Status == model.TaskPending { return s.transition(ctx, task, model.TaskCanceled, "canceled before dispatch", "") } if task.Status != model.TaskRunning && task.Status != model.TaskDispatching { return fmt.Errorf("task in state %q cannot be canceled", task.Status) } var job model.ExecutionJob if err := s.db.WithContext(ctx).Where("task_id = ?", task.ID).First(&job).Error; err != nil { return err } if err := s.awx.Cancel(ctx, job.ExecutorJobID); err != nil { return err } return s.transition(ctx, task, model.TaskCanceling, "cancel requested in AWX", "") } func (s *DeliveryService) claimAndReserve(ctx context.Context) (*model.DeliveryTask, error) { var task model.DeliveryTask var payload deliveryPayload var target DeliveryTarget dispatchable := false err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { if err := tx.Clauses(clause.Locking{Strength: "UPDATE", Options: "SKIP LOCKED"}).Where("status = ?", model.TaskPending).Order("created_at ASC").First(&task).Error; err != nil { return err } var targetErr error target, targetErr = s.getTarget(ctx, task.TargetID) if targetErr != nil { return s.failInTransaction(tx, &task, model.TaskValidationFailed, targetErr.Error()) } if err := json.Unmarshal([]byte(task.ImmutablePayload), &payload); err != nil { return s.failInTransaction(tx, &task, model.TaskValidationFailed, "stored deployment payload is invalid") } activeStates := []string{model.TaskValidating, model.TaskDispatching, model.TaskRunning, model.TaskCanceling} checks := []struct { query string args []any limit int message string }{ {"status IN ?", []any{activeStates}, s.cfg.DeliveryGlobalLimit, "global concurrency limit reached"}, {"status IN ? AND target_id = ?", []any{activeStates, task.TargetID}, s.cfg.DeliveryTargetLimit, "target concurrency limit reached"}, {"status IN ? AND business_line_id = ?", []any{activeStates, task.BusinessLineID}, s.cfg.DeliveryBusinessLimit, "business line concurrency limit reached"}, {"status IN ? AND target_id = ? AND namespace = ?", []any{activeStates, task.TargetID, task.Namespace}, 1, "namespace already has an active MySQL delivery"}, } for _, check := range checks { var count int64 if check.limit > 0 { if err := tx.Model(&model.DeliveryTask{}).Where(check.query, check.args...).Count(&count).Error; err != nil { return err } if count >= int64(check.limit) { return fmt.Errorf("defer: %s", check.message) } } } quotaOK, err := checkResourceQuota(tx, task.BusinessLineID, task.TargetID, payload) if err != nil { return err } if !quotaOK { return s.failInTransaction(tx, &task, model.TaskValidationFailed, "resource quota is insufficient") } meta := parseTargetMetadata(target.Metadata) if len(meta.Hosts) == 0 { return s.failInTransaction(tx, &task, model.TaskValidationFailed, "deployment target has no candidate hosts") } var occupied []string occupiedExclude := []string{model.TaskExecutionFailed, model.TaskValidationFailed, model.TaskCanceled} if err := tx.Model(&model.DeliveryTask{}).Where("target_id = ? AND target_host <> ? AND status NOT IN ?", task.TargetID, "", occupiedExclude).Pluck("target_host", &occupied).Error; err != nil { return err } host := firstFreeHost(meta.Hosts, occupied) if host == nil { return fmt.Errorf("defer: no free host available on target") } // 端口池混合分配:同主机已占端口 = 非终态任务分配端口 ∪ 存量 active 实例端口。 var usedPorts []int if err := tx.Model(&model.DeliveryTask{}).Where("target_host = ? AND status NOT IN ?", host.Name, occupiedExclude).Pluck("mysql_port", &usedPorts).Error; err != nil { return err } var instancePorts []int if err := tx.Model(&model.MySQLInstance{}).Where("node_name = ? AND status = ?", host.Name, "active").Pluck("port", &instancePorts).Error; err != nil { return err } port, portErr := allocatePort(payload.MySQLPort, append(usedPorts, instancePorts...)) if portErr != nil { return s.failInTransaction(tx, &task, model.TaskValidationFailed, portErr.Error()) } reservation := model.ResourceReservation{TaskID: task.ID, BusinessLineID: task.BusinessLineID, TargetID: task.TargetID, CPUMilli: payload.CPUMilli, MemoryMi: payload.MemoryMi, StorageGi: payload.StorageGi, InstanceCount: 1, Status: "reserved", ExpiresAt: time.Now().Add(time.Duration(s.cfg.ReservationTTLMinutes) * time.Minute)} if err := tx.Create(&reservation).Error; err != nil { return err } if err := tx.Model(&model.DeliveryTask{}).Where("id = ?", task.ID).Updates(map[string]any{"target_host": host.Name, "target_host_ip": host.IP, "mysql_port": port}).Error; err != nil { return err } task.TargetHost = host.Name task.TargetHostIP = host.IP task.MySQLPort = port if err := s.transitionTx(tx, &task, model.TaskDispatching, "resources reserved", ""); err != nil { return err } dispatchable = true return nil }) if err == nil && !dispatchable { err = gorm.ErrRecordNotFound } return &task, err } func checkResourceQuota(tx *gorm.DB, businessLineID, targetID uint64, payload deliveryPayload) (bool, error) { var quota model.ResourceQuota if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Where("business_line_id = ? AND target_id = ?", businessLineID, targetID).First("a).Error; err != nil { if errors.Is(err, gorm.ErrRecordNotFound) { return true, nil } return false, err } type totals struct{ CPU, Memory, Storage, Instances int64 } var used, reserved totals if err := tx.Model(&model.ResourceUsage{}).Select("COALESCE(SUM(cpu_milli),0) cpu, COALESCE(SUM(memory_mi),0) memory, COALESCE(SUM(storage_gi),0) storage, COALESCE(SUM(instance_count),0) instances").Where("business_line_id = ? AND target_id = ? AND status = ?", businessLineID, targetID, "active").Scan(&used).Error; err != nil { return false, err } if err := tx.Model(&model.ResourceReservation{}).Select("COALESCE(SUM(cpu_milli),0) cpu, COALESCE(SUM(memory_mi),0) memory, COALESCE(SUM(storage_gi),0) storage, COALESCE(SUM(instance_count),0) instances").Where("business_line_id = ? AND target_id = ? AND status = ? AND expires_at > ?", businessLineID, targetID, "reserved", time.Now()).Scan(&reserved).Error; err != nil { return false, err } return used.CPU+reserved.CPU+payload.CPUMilli <= quota.CPUMilli && used.Memory+reserved.Memory+payload.MemoryMi <= quota.MemoryMi && used.Storage+reserved.Storage+payload.StorageGi <= quota.StorageGi && used.Instances+reserved.Instances+1 <= quota.InstanceLimit, nil } func (s *DeliveryService) failInTransaction(tx *gorm.DB, task *model.DeliveryTask, status, message string) error { if err := s.transitionTx(tx, task, status, message, message); err != nil { return err } return nil } func (s *DeliveryService) transition(ctx context.Context, task *model.DeliveryTask, status, message, errorMessage string) error { return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { return s.transitionTx(tx, task, status, message, errorMessage) }) } func (s *DeliveryService) transitionTx(tx *gorm.DB, task *model.DeliveryTask, status, message, errorMessage string) error { from := task.Status updates := map[string]any{"status": status, "error_message": errorMessage} now := time.Now() if status == model.TaskRunning { updates["started_at"] = now } if status == model.TaskFinished || status == model.TaskExecutionFailed || status == model.TaskValidationFailed || status == model.TaskCanceled || status == model.TaskRegisterFailed { updates["finished_at"] = now } result := tx.Model(&model.DeliveryTask{}).Where("id = ? AND status = ?", task.ID, from).Updates(updates) if result.Error != nil { return result.Error } if result.RowsAffected != 1 { return fmt.Errorf("task %s changed concurrently", task.ID) } task.Status = status task.ErrorMessage = errorMessage return tx.Create(&model.TaskEvent{TaskID: task.ID, FromState: from, ToState: status, Message: message}).Error } func (s *DeliveryService) releaseReservation(tx *gorm.DB, taskID string) error { return tx.Model(&model.ResourceReservation{}).Where("task_id = ? AND status = ?", taskID, "reserved").Update("status", "released").Error } func randomUUID() string { b := make([]byte, 16) if _, err := rand.Read(b); err != nil { panic(err) } b[6] = (b[6] & 0x0f) | 0x40 b[8] = (b[8] & 0x3f) | 0x80 return fmt.Sprintf("%08x-%04x-%04x-%04x-%012x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:16]) } func mysqlReady(ctx context.Context, address string) error { dialer := net.Dialer{Timeout: 5 * time.Second} conn, err := dialer.DialContext(ctx, "tcp", address) if err != nil { return err } return conn.Close() } func (s *DeliveryService) DispatchOnce(ctx context.Context) error { task, err := s.claimAndReserve(ctx) if err != nil { if errors.Is(err, gorm.ErrRecordNotFound) || strings.HasPrefix(err.Error(), "defer:") { return nil } return err } _, _, err = s.CreateExecution(ctx, task.ID, task.PayloadHash, task.IdempotencyKey) if err != nil { return s.failTask(ctx, task, model.TaskExecutionFailed, err.Error()) } return nil } func (s *DeliveryService) CreateExecution(ctx context.Context, taskID, payloadHash, idempotencyKey string) (*model.ExecutionJob, bool, error) { s.executionMu.Lock() defer s.executionMu.Unlock() var existing model.ExecutionJob if err := s.db.WithContext(ctx).Where("task_id = ? OR idempotency_key = ?", taskID, idempotencyKey).First(&existing).Error; err == nil { if existing.TaskID != taskID || existing.IdempotencyKey != idempotencyKey { return nil, false, fmt.Errorf("idempotency key is already bound to another task") } return &existing, true, nil } else if !errors.Is(err, gorm.ErrRecordNotFound) { return nil, false, err } var task model.DeliveryTask if err := s.db.WithContext(ctx).First(&task, "id = ?", taskID).Error; err != nil { return nil, false, err } if task.PayloadHash != payloadHash || task.IdempotencyKey != idempotencyKey { return nil, false, fmt.Errorf("execution request does not match the immutable task payload") } if task.Status != model.TaskDispatching { return nil, false, fmt.Errorf("task in state %q is not ready for execution", task.Status) } target, err := s.getTarget(ctx, task.TargetID) if err != nil { return nil, false, err } var payload deliveryPayload if err := json.Unmarshal([]byte(task.ImmutablePayload), &payload); err != nil { return nil, false, err } meta := parseTargetMetadata(target.Metadata) // Persist execution record BEFORE launching AWX to ensure crash recovery. now := time.Now() execution := model.ExecutionJob{TaskID: task.ID, IdempotencyKey: task.IdempotencyKey, ExecutorJobID: "pending", Status: "launching", StartedAt: &now} if err := s.db.WithContext(ctx).Create(&execution).Error; err != nil { return nil, false, err } job, err := s.awx.Launch(ctx, target.AWXTemplateID, AWXLaunchRequest{InventoryID: target.AWXInventoryID, Limit: task.TargetHost, ExtraVars: deliveryExtraVars(&task, payload, meta)}) if err != nil { _ = s.db.WithContext(ctx).Model(&execution).Updates(map[string]any{"status": "failed", "finished_at": time.Now()}) return nil, false, err } if err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { if err := tx.Model(&execution).Updates(map[string]any{"executor_job_id": fmt.Sprint(job.ID), "status": "running"}).Error; err != nil { return err } return s.transitionTx(tx, &task, model.TaskRunning, "AWX job started", "") }); err != nil { return nil, false, err } return &execution, false, nil } // deliveryExtraVars 组装传给 playbook 的变量:必传项固定注入, // 选填项仅在用户显式设置时下发,未设置时由 playbook 默认基线兜底。 func deliveryExtraVars(task *model.DeliveryTask, payload deliveryPayload, meta targetMetadata) map[string]any { topology := payload.Topology if topology == "" { // 存量任务的 ImmutablePayload 无 topology 字段,回退到 target metadata。 topology = meta.Topology } vars := map[string]any{ "task_id": task.ID, "payload_hash": task.PayloadHash, "target_hosts": task.TargetHost, "topology": topology, "instance_name": payload.InstanceName, "mysql_port": task.MySQLPort, "memory_mb": payload.MemoryMi, "storage_gb": payload.StorageGi, "mysql_version": payload.MySQLVersion, } if payload.DataDisk != "" { vars["data_disk"] = payload.DataDisk } if payload.Timezone != "" { vars["timezone"] = payload.Timezone } if payload.LowerCaseTableNames != nil { vars["lower_case_table_names"] = *payload.LowerCaseTableNames } if payload.CharacterSet != "" { vars["character_set"] = payload.CharacterSet } if payload.Collation != "" { vars["collation"] = payload.Collation } if payload.MaxConnections != "" { vars["max_connections"] = payload.MaxConnections } if payload.InnodbRedoLogCapacity != "" { vars["innodb_redo_log_capacity"] = payload.InnodbRedoLogCapacity } if payload.InnodbFlushLogAtTrxCommit != nil { vars["innodb_flush_log_at_trx_commit"] = *payload.InnodbFlushLogAtTrxCommit } if payload.SyncBinlog != nil { vars["sync_binlog"] = *payload.SyncBinlog } if payload.InnodbIOCapacity != 0 { vars["innodb_io_capacity"] = payload.InnodbIOCapacity } if payload.LongQueryTime != 0 { vars["long_query_time"] = payload.LongQueryTime } if payload.BinlogExpireLogsSeconds != 0 { vars["binlog_expire_logs_seconds"] = payload.BinlogExpireLogsSeconds } if payload.MaxBinlogSize != "" { vars["max_binlog_size"] = payload.MaxBinlogSize } return vars } func (s *DeliveryService) PollOnce(ctx context.Context) error { var jobs []model.ExecutionJob if err := s.db.WithContext(ctx).Where("status = ?", "running").Find(&jobs).Error; err != nil { return err } for _, execution := range jobs { job, err := s.awx.GetJob(ctx, execution.ExecutorJobID) if err != nil { s.finishExecution(ctx, &execution, "failed") _ = s.failTask(ctx, &model.DeliveryTask{ID: execution.TaskID, Status: model.TaskRunning}, model.TaskExecutionFailed, "poll AWX job "+execution.ExecutorJobID+": "+err.Error()) continue } switch strings.ToLower(job.Status) { case "pending", "waiting", "running", "new": continue case "successful": s.finishExecution(ctx, &execution, "successful") if err := s.completeTask(ctx, execution.TaskID); err != nil { _ = s.failTask(ctx, &model.DeliveryTask{ID: execution.TaskID, Status: model.TaskRunning}, model.TaskValidationFailed, err.Error()) } case "canceled": s.finishExecution(ctx, &execution, "canceled") _ = s.failTask(ctx, &model.DeliveryTask{ID: execution.TaskID, Status: model.TaskRunning}, model.TaskCanceled, "AWX job was canceled") default: s.finishExecution(ctx, &execution, "failed") _ = s.failTask(ctx, &model.DeliveryTask{ID: execution.TaskID, Status: model.TaskRunning}, model.TaskExecutionFailed, "AWX job finished with status "+job.Status) } } return nil } func (s *DeliveryService) finishExecution(ctx context.Context, execution *model.ExecutionJob, status string) { now := time.Now() _ = s.db.WithContext(ctx).Model(execution).Updates(map[string]any{"status": status, "finished_at": now}) } func (s *DeliveryService) completeTask(ctx context.Context, taskID string) error { var task model.DeliveryTask if err := s.db.WithContext(ctx).First(&task, "id = ?", taskID).Error; err != nil { return err } var payload deliveryPayload if err := json.Unmarshal([]byte(task.ImmutablePayload), &payload); err != nil { return err } addr := fmt.Sprintf("%s:%d", task.TargetHostIP, task.MySQLPort) if err := mysqlReady(ctx, addr); err != nil { return fmt.Errorf("MySQL health check failed: %w", err) } now := time.Now() if err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { if err := s.transitionTx(tx, &task, model.TaskRegistering, "AWX succeeded and MySQL health check passed", ""); err != nil { return err } instance := model.MySQLInstance{TaskID: task.ID, BusinessLineID: task.BusinessLineID, TargetID: task.TargetID, Namespace: payload.Namespace, Name: payload.InstanceName, NodeName: task.TargetHost, Host: task.TargetHostIP, Port: task.MySQLPort, Version: payload.MySQLVersion, Status: "active"} if err := tx.Create(&instance).Error; err != nil { return err } if err := tx.Create(&model.ResourceUsage{TaskID: task.ID, InstanceID: instance.ID, BusinessLineID: task.BusinessLineID, TargetID: task.TargetID, CPUMilli: payload.CPUMilli, MemoryMi: payload.MemoryMi, StorageGi: payload.StorageGi, InstanceCount: 1, Status: "active"}).Error; err != nil { return err } if err := tx.Model(&model.ResourceReservation{}).Where("task_id = ? AND status = ?", task.ID, "reserved").Update("status", "consumed").Error; err != nil { return err } if err := tx.Model(&model.ExecutionJob{}).Where("task_id = ?", task.ID).Updates(map[string]any{"status": "successful", "finished_at": now}).Error; err != nil { return err } return nil }); err != nil { return err } if err := s.RegisterCloudDM(ctx, task.ID); err != nil { return s.transition(ctx, &task, model.TaskRegisterFailed, "CloudDM registration failed", err.Error()) } return s.transition(ctx, &task, model.TaskFinished, "MySQL delivery completed", "") } func (s *DeliveryService) RegisterCloudDM(ctx context.Context, taskID string) error { if s.cfg.CloudDMRegisterURL == "" { return nil } var instance model.MySQLInstance if err := s.db.WithContext(ctx).Where("task_id = ?", taskID).First(&instance).Error; err != nil { return err } body := map[string]any{"name": instance.Name, "host": instance.Host, "port": instance.Port, "username": "root", "database_type": "mysql"} raw, _ := json.Marshal(body) req, err := http.NewRequestWithContext(ctx, http.MethodPost, s.cfg.CloudDMRegisterURL, bytes.NewReader(raw)) if err != nil { return err } req.Header.Set("Content-Type", "application/json") if s.cfg.CloudDMAPIToken != "" { req.Header.Set("Authorization", "Bearer "+s.cfg.CloudDMAPIToken) } resp, err := http.DefaultClient.Do(req) if err != nil { return err } defer resp.Body.Close() if resp.StatusCode < 200 || resp.StatusCode >= 300 { return fmt.Errorf("CloudDM returned %s", resp.Status) } return nil } func (s *DeliveryService) failTask(ctx context.Context, task *model.DeliveryTask, status, message string) error { return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { var current model.DeliveryTask if err := tx.First(¤t, "id = ?", task.ID).Error; err != nil { return err } if current.Status != model.TaskPending && current.Status != model.TaskDispatching && current.Status != model.TaskRunning && current.Status != model.TaskRegistering && current.Status != model.TaskCanceling { return nil } if err := s.transitionTx(tx, ¤t, status, message, message); err != nil { return err } if status == model.TaskExecutionFailed || status == model.TaskValidationFailed || status == model.TaskCanceled { return s.releaseReservation(tx, current.ID) } return nil }) } func (s *DeliveryService) Run(ctx context.Context) { interval := time.Duration(s.cfg.DeliveryPollSeconds) * time.Second if interval < time.Second { interval = time.Second } ticker := time.NewTicker(interval) defer ticker.Stop() for { select { case <-ctx.Done(): return case <-ticker.C: _ = s.DispatchOnce(ctx) _ = s.PollOnce(ctx) } } }