feat(delivery): add automatic rollback state machine for failed deployments
Introduce a compensating workflow that launches the dedicated AWX rollback template when a deployment cannot be started or fails midway. - add task states: rollback_pending, rolling_back, rolled_back, rollback_failed, rollback_acknowledged - add RollbackJob model to track the compensating AWX run separately from the deploy run, preserving both job IDs for audit - hold resource reservations in 'rollback' status until cleanup succeeds so a failed cleanup cannot be masked by a later delivery - poll rollback jobs with a 2-minute launch timeout; an unknown launch result surfaces as a recoverable failure instead of re-launching - protect finished/register_failed/rolled-back tasks from rollback; register_failed keeps the healthy instance and its resource usage - add DELIVERY_ROLLBACK_TEMPLATE_ID config; without it, failures are marked rollback_failed and require manual cleanup - use unique pending-<task_id> placeholder for executor job IDs
This commit is contained in:
@@ -1,6 +1,11 @@
|
||||
package service
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/1024XEngineer/xinfra/server/internal/model"
|
||||
)
|
||||
|
||||
func intPtr(v int) *int { return &v }
|
||||
|
||||
@@ -124,3 +129,40 @@ func TestAllocatePort(t *testing.T) {
|
||||
t.Fatal("exhausted pool still allocated a port")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRollbackExtraVarsTargetsOnlyTheAllocatedInstance(t *testing.T) {
|
||||
task := &model.DeliveryTask{ID: "task-1", TargetHost: "db-01"}
|
||||
payload := deliveryPayload{MySQLDeliveryInput: MySQLDeliveryInput{InstanceName: "mysql-a", DataDisk: "/disk1"}}
|
||||
vars := rollbackExtraVars(task, payload)
|
||||
if vars["target_hosts"] != "db-01" || vars["instance_name"] != "mysql-a" || vars["data_disk"] != "/disk1" {
|
||||
t.Fatalf("rollback vars target the wrong instance: %#v", vars)
|
||||
}
|
||||
if vars["rollback"] != true {
|
||||
t.Fatalf("rollback marker missing: %#v", vars)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegisterFailedIsProtectedFromRollback(t *testing.T) {
|
||||
if !rollbackProtectedStatus(model.TaskRegisterFailed) {
|
||||
t.Fatal("register_failed must preserve the healthy instance and resource usage")
|
||||
}
|
||||
if rollbackProtectedStatus(model.TaskValidationFailed) {
|
||||
t.Fatal("validation_failed must still be eligible for cleanup rollback")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRollbackLaunchExpired(t *testing.T) {
|
||||
now := time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC)
|
||||
started := now.Add(-rollbackLaunchTimeout - time.Second)
|
||||
if !rollbackLaunchExpired(model.RollbackJob{Status: "launching", StartedAt: &started}, now) {
|
||||
t.Fatal("stale launching rollback job must be recoverable")
|
||||
}
|
||||
if rollbackLaunchExpired(model.RollbackJob{Status: "launching", StartedAt: ptrTime(now.Add(-rollbackLaunchTimeout + time.Second))}, now) {
|
||||
t.Fatal("recent launching rollback job must remain pending")
|
||||
}
|
||||
if rollbackLaunchExpired(model.RollbackJob{Status: "running", StartedAt: &started}, now) {
|
||||
t.Fatal("running rollback job is not a launch timeout")
|
||||
}
|
||||
}
|
||||
|
||||
func ptrTime(v time.Time) *time.Time { return &v }
|
||||
|
||||
Reference in New Issue
Block a user