feat(delivery): add automatic rollback state machine for failed deployments

Introduce a compensating workflow that launches the dedicated AWX
rollback template when a deployment cannot be started or fails midway.

- add task states: rollback_pending, rolling_back, rolled_back,
  rollback_failed, rollback_acknowledged
- add RollbackJob model to track the compensating AWX run separately
  from the deploy run, preserving both job IDs for audit
- hold resource reservations in 'rollback' status until cleanup
  succeeds so a failed cleanup cannot be masked by a later delivery
- poll rollback jobs with a 2-minute launch timeout; an unknown launch
  result surfaces as a recoverable failure instead of re-launching
- protect finished/register_failed/rolled-back tasks from rollback;
  register_failed keeps the healthy instance and its resource usage
- add DELIVERY_ROLLBACK_TEMPLATE_ID config; without it, failures are
  marked rollback_failed and require manual cleanup
- use unique pending-<task_id> placeholder for executor job IDs
This commit is contained in:
Hungerdream
2026-07-28 14:29:09 +08:00
parent 7a800b0307
commit f0c1b38c5b
6 changed files with 366 additions and 11 deletions
+43 -1
View File
@@ -1,6 +1,11 @@
package service
import "testing"
import (
"testing"
"time"
"github.com/1024XEngineer/xinfra/server/internal/model"
)
func intPtr(v int) *int { return &v }
@@ -124,3 +129,40 @@ func TestAllocatePort(t *testing.T) {
t.Fatal("exhausted pool still allocated a port")
}
}
func TestRollbackExtraVarsTargetsOnlyTheAllocatedInstance(t *testing.T) {
task := &model.DeliveryTask{ID: "task-1", TargetHost: "db-01"}
payload := deliveryPayload{MySQLDeliveryInput: MySQLDeliveryInput{InstanceName: "mysql-a", DataDisk: "/disk1"}}
vars := rollbackExtraVars(task, payload)
if vars["target_hosts"] != "db-01" || vars["instance_name"] != "mysql-a" || vars["data_disk"] != "/disk1" {
t.Fatalf("rollback vars target the wrong instance: %#v", vars)
}
if vars["rollback"] != true {
t.Fatalf("rollback marker missing: %#v", vars)
}
}
func TestRegisterFailedIsProtectedFromRollback(t *testing.T) {
if !rollbackProtectedStatus(model.TaskRegisterFailed) {
t.Fatal("register_failed must preserve the healthy instance and resource usage")
}
if rollbackProtectedStatus(model.TaskValidationFailed) {
t.Fatal("validation_failed must still be eligible for cleanup rollback")
}
}
func TestRollbackLaunchExpired(t *testing.T) {
now := time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC)
started := now.Add(-rollbackLaunchTimeout - time.Second)
if !rollbackLaunchExpired(model.RollbackJob{Status: "launching", StartedAt: &started}, now) {
t.Fatal("stale launching rollback job must be recoverable")
}
if rollbackLaunchExpired(model.RollbackJob{Status: "launching", StartedAt: ptrTime(now.Add(-rollbackLaunchTimeout + time.Second))}, now) {
t.Fatal("recent launching rollback job must remain pending")
}
if rollbackLaunchExpired(model.RollbackJob{Status: "running", StartedAt: &started}, now) {
t.Fatal("running rollback job is not a launch timeout")
}
}
func ptrTime(v time.Time) *time.Time { return &v }