f0c1b38c5b
Introduce a compensating workflow that launches the dedicated AWX rollback template when a deployment cannot be started or fails midway. - add task states: rollback_pending, rolling_back, rolled_back, rollback_failed, rollback_acknowledged - add RollbackJob model to track the compensating AWX run separately from the deploy run, preserving both job IDs for audit - hold resource reservations in 'rollback' status until cleanup succeeds so a failed cleanup cannot be masked by a later delivery - poll rollback jobs with a 2-minute launch timeout; an unknown launch result surfaces as a recoverable failure instead of re-launching - protect finished/register_failed/rolled-back tasks from rollback; register_failed keeps the healthy instance and its resource usage - add DELIVERY_ROLLBACK_TEMPLATE_ID config; without it, failures are marked rollback_failed and require manual cleanup - use unique pending-<task_id> placeholder for executor job IDs
169 lines
7.5 KiB
Go
169 lines
7.5 KiB
Go
package service
|
|
|
|
import (
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/1024XEngineer/xinfra/server/internal/model"
|
|
)
|
|
|
|
func intPtr(v int) *int { return &v }
|
|
|
|
func TestValidateDeliveryInput(t *testing.T) {
|
|
dataDisks := []string{"/data", "/disk1"}
|
|
valid := MySQLDeliveryInput{BusinessLineID: 1, TargetID: 1, Namespace: "team-a", InstanceName: "mysql-01", CPUMilli: 500, MemoryMi: 2048, StorageGi: 20}
|
|
if err := validateDeliveryInput(valid, dataDisks); err != nil {
|
|
t.Fatalf("valid input rejected: %v", err)
|
|
}
|
|
full := valid
|
|
full.MySQLVersion = "8.0"
|
|
full.Topology = "standalone"
|
|
full.MySQLPort = 13306
|
|
full.DataDisk = "/disk1"
|
|
full.TargetHost = "k8s-server-03"
|
|
full.MemoryMi = 8192
|
|
full.StorageGi = 2000
|
|
full.Timezone = "+08:00"
|
|
full.LowerCaseTableNames = intPtr(0)
|
|
full.CharacterSet = "utf8mb4"
|
|
full.Collation = "utf8mb4_general_ci"
|
|
full.MaxConnections = "auto"
|
|
full.InnodbRedoLogCapacity = "256M"
|
|
full.InnodbFlushLogAtTrxCommit = intPtr(2)
|
|
full.SyncBinlog = intPtr(0)
|
|
full.InnodbIOCapacity = 2000
|
|
full.LongQueryTime = 0.5
|
|
full.BinlogExpireLogsSeconds = 604800
|
|
full.MaxBinlogSize = "512M"
|
|
if err := validateDeliveryInput(full, dataDisks); err != nil {
|
|
t.Fatalf("valid full input rejected: %v", err)
|
|
}
|
|
namedZone := valid
|
|
namedZone.Timezone = "Asia/Shanghai"
|
|
if err := validateDeliveryInput(namedZone, dataDisks); err != nil {
|
|
t.Fatalf("named timezone rejected: %v", err)
|
|
}
|
|
lts := valid
|
|
lts.MySQLVersion = "8.4"
|
|
if err := validateDeliveryInput(lts, dataDisks); err != nil {
|
|
t.Fatalf("8.4 LTS rejected: %v", err)
|
|
}
|
|
for name, mutate := range map[string]func(*MySQLDeliveryInput){
|
|
"uppercase namespace": func(in *MySQLDeliveryInput) { in.Namespace = "Team-A" },
|
|
"bad instance": func(in *MySQLDeliveryInput) { in.InstanceName = "mysql_01" },
|
|
"too little memory": func(in *MySQLDeliveryInput) { in.MemoryMi = 1024 },
|
|
"too much memory": func(in *MySQLDeliveryInput) { in.MemoryMi = 131072 },
|
|
"too little storage": func(in *MySQLDeliveryInput) { in.StorageGi = 10 },
|
|
"too much storage": func(in *MySQLDeliveryInput) { in.StorageGi = 4000 },
|
|
"unsupported version": func(in *MySQLDeliveryInput) { in.MySQLVersion = "5.7" },
|
|
"eol version": func(in *MySQLDeliveryInput) { in.MySQLVersion = "5.6" },
|
|
"unsupported topology": func(in *MySQLDeliveryInput) { in.Topology = "mgr_3" },
|
|
"port below pool": func(in *MySQLDeliveryInput) { in.MySQLPort = 3307 },
|
|
"port above pool": func(in *MySQLDeliveryInput) { in.MySQLPort = 14000 },
|
|
"data disk not in list": func(in *MySQLDeliveryInput) { in.DataDisk = "/mnt/other" },
|
|
"bad target host": func(in *MySQLDeliveryInput) { in.TargetHost = "-bad-host" },
|
|
"bad timezone": func(in *MySQLDeliveryInput) { in.Timezone = "UTC+8" },
|
|
"bad lower case": func(in *MySQLDeliveryInput) { in.LowerCaseTableNames = intPtr(2) },
|
|
"bad charset": func(in *MySQLDeliveryInput) { in.CharacterSet = "big5" },
|
|
"collation mismatch": func(in *MySQLDeliveryInput) { in.CharacterSet = "gbk"; in.Collation = "utf8mb4_general_ci" },
|
|
"collation vs default": func(in *MySQLDeliveryInput) { in.Collation = "gbk_chinese_ci" },
|
|
"bad max connections": func(in *MySQLDeliveryInput) { in.MaxConnections = "300" },
|
|
"bad redo capacity": func(in *MySQLDeliveryInput) { in.InnodbRedoLogCapacity = "2G" },
|
|
"bad flush log": func(in *MySQLDeliveryInput) { in.InnodbFlushLogAtTrxCommit = intPtr(3) },
|
|
"bad sync binlog": func(in *MySQLDeliveryInput) { in.SyncBinlog = intPtr(2) },
|
|
"bad io capacity": func(in *MySQLDeliveryInput) { in.InnodbIOCapacity = 500 },
|
|
"bad long query time": func(in *MySQLDeliveryInput) { in.LongQueryTime = 3 },
|
|
"bad binlog expire": func(in *MySQLDeliveryInput) { in.BinlogExpireLogsSeconds = 3600 },
|
|
"bad max binlog size": func(in *MySQLDeliveryInput) { in.MaxBinlogSize = "64M" },
|
|
} {
|
|
input := valid
|
|
mutate(&input)
|
|
if err := validateDeliveryInput(input, dataDisks); err == nil {
|
|
t.Errorf("%s was accepted", name)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestFirstFreeHost(t *testing.T) {
|
|
hosts := []targetHost{{Name: "node-a"}, {Name: "node-b"}}
|
|
if h := firstFreeHost(hosts, nil, 1); h == nil || h.Name != "node-a" {
|
|
t.Fatalf("expected node-a on empty occupancy, got %+v", h)
|
|
}
|
|
if h := firstFreeHost(hosts, []string{"node-a"}, 1); h == nil || h.Name != "node-b" {
|
|
t.Fatalf("expected node-b when node-a is full at limit 1, got %+v", h)
|
|
}
|
|
if h := firstFreeHost(hosts, []string{"node-a", "node-b"}, 1); h != nil {
|
|
t.Fatalf("limit 1 with all hosts taken should return nil, got %+v", h)
|
|
}
|
|
// 摊满一轮后回到首台叠加第二个实例
|
|
if h := firstFreeHost(hosts, []string{"node-a", "node-b"}, 2); h == nil || h.Name != "node-a" {
|
|
t.Fatalf("expected node-a for second round at limit 2, got %+v", h)
|
|
}
|
|
if h := firstFreeHost(hosts, []string{"node-a", "node-a", "node-b", "node-b"}, 2); h != nil {
|
|
t.Fatalf("limit 2 with all hosts saturated should return nil, got %+v", h)
|
|
}
|
|
// limit < 1 退化为一机一实例
|
|
if h := firstFreeHost(hosts, []string{"node-a"}, 0); h == nil || h.Name != "node-b" {
|
|
t.Fatalf("limit 0 should degrade to 1, got %+v", h)
|
|
}
|
|
}
|
|
|
|
func TestAllocatePort(t *testing.T) {
|
|
if port, err := allocatePort(0, nil); err != nil || port != mysqlPortPoolStart {
|
|
t.Fatalf("expected first pool port %d, got %d err=%v", mysqlPortPoolStart, port, err)
|
|
}
|
|
if port, err := allocatePort(0, []int{13306, 13307}); err != nil || port != 13308 {
|
|
t.Fatalf("expected 13308 skipping occupied, got %d err=%v", port, err)
|
|
}
|
|
if port, err := allocatePort(13400, []int{13306}); err != nil || port != 13400 {
|
|
t.Fatalf("expected requested port 13400, got %d err=%v", port, err)
|
|
}
|
|
if _, err := allocatePort(13306, []int{13306}); err == nil {
|
|
t.Fatal("requested occupied port was accepted")
|
|
}
|
|
used := make([]int, 0, mysqlPortPoolEnd-mysqlPortPoolStart+1)
|
|
for p := mysqlPortPoolStart; p <= mysqlPortPoolEnd; p++ {
|
|
used = append(used, p)
|
|
}
|
|
if _, err := allocatePort(0, used); err == nil {
|
|
t.Fatal("exhausted pool still allocated a port")
|
|
}
|
|
}
|
|
|
|
func TestRollbackExtraVarsTargetsOnlyTheAllocatedInstance(t *testing.T) {
|
|
task := &model.DeliveryTask{ID: "task-1", TargetHost: "db-01"}
|
|
payload := deliveryPayload{MySQLDeliveryInput: MySQLDeliveryInput{InstanceName: "mysql-a", DataDisk: "/disk1"}}
|
|
vars := rollbackExtraVars(task, payload)
|
|
if vars["target_hosts"] != "db-01" || vars["instance_name"] != "mysql-a" || vars["data_disk"] != "/disk1" {
|
|
t.Fatalf("rollback vars target the wrong instance: %#v", vars)
|
|
}
|
|
if vars["rollback"] != true {
|
|
t.Fatalf("rollback marker missing: %#v", vars)
|
|
}
|
|
}
|
|
|
|
func TestRegisterFailedIsProtectedFromRollback(t *testing.T) {
|
|
if !rollbackProtectedStatus(model.TaskRegisterFailed) {
|
|
t.Fatal("register_failed must preserve the healthy instance and resource usage")
|
|
}
|
|
if rollbackProtectedStatus(model.TaskValidationFailed) {
|
|
t.Fatal("validation_failed must still be eligible for cleanup rollback")
|
|
}
|
|
}
|
|
|
|
func TestRollbackLaunchExpired(t *testing.T) {
|
|
now := time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC)
|
|
started := now.Add(-rollbackLaunchTimeout - time.Second)
|
|
if !rollbackLaunchExpired(model.RollbackJob{Status: "launching", StartedAt: &started}, now) {
|
|
t.Fatal("stale launching rollback job must be recoverable")
|
|
}
|
|
if rollbackLaunchExpired(model.RollbackJob{Status: "launching", StartedAt: ptrTime(now.Add(-rollbackLaunchTimeout + time.Second))}, now) {
|
|
t.Fatal("recent launching rollback job must remain pending")
|
|
}
|
|
if rollbackLaunchExpired(model.RollbackJob{Status: "running", StartedAt: &started}, now) {
|
|
t.Fatal("running rollback job is not a launch timeout")
|
|
}
|
|
}
|
|
|
|
func ptrTime(v time.Time) *time.Time { return &v }
|