feat(delivery): add automatic rollback state machine for failed deployments

Introduce a compensating workflow that launches the dedicated AWX
rollback template when a deployment cannot be started or fails midway.

- add task states: rollback_pending, rolling_back, rolled_back,
  rollback_failed, rollback_acknowledged
- add RollbackJob model to track the compensating AWX run separately
  from the deploy run, preserving both job IDs for audit
- hold resource reservations in 'rollback' status until cleanup
  succeeds so a failed cleanup cannot be masked by a later delivery
- poll rollback jobs with a 2-minute launch timeout; an unknown launch
  result surfaces as a recoverable failure instead of re-launching
- protect finished/register_failed/rolled-back tasks from rollback;
  register_failed keeps the healthy instance and its resource usage
- add DELIVERY_ROLLBACK_TEMPLATE_ID config; without it, failures are
  marked rollback_failed and require manual cleanup
- use unique pending-<task_id> placeholder for executor job IDs
This commit is contained in:
Hungerdream
2026-07-28 14:29:09 +08:00
parent 7a800b0307
commit f0c1b38c5b
6 changed files with 366 additions and 11 deletions
+19
View File
@@ -14,6 +14,11 @@ const (
TaskValidationFailed = "validation_failed"
TaskCanceling = "canceling"
TaskCanceled = "canceled"
TaskRollbackPending = "rollback_pending"
TaskRollingBack = "rolling_back"
TaskRolledBack = "rolled_back"
TaskRollbackFailed = "rollback_failed"
TaskRollbackAck = "rollback_acknowledged"
)
type ResourceQuota struct {
@@ -110,6 +115,20 @@ type ExecutionJob struct {
UpdatedAt time.Time `json:"updated_at"`
}
// RollbackJob tracks the compensating AWX run independently from the deploy run.
// Keeping a separate record preserves both job IDs for audit and retry tooling.
type RollbackJob struct {
ID uint64 `gorm:"primaryKey" json:"id"`
TaskID string `gorm:"size:36;not null;uniqueIndex" json:"task_id"`
ExecutorJobID string `gorm:"size:128;not null" json:"executor_job_id"`
Status string `gorm:"size:32;not null;index" json:"status"`
Reason string `gorm:"type:text" json:"reason"`
StartedAt *time.Time `json:"started_at,omitempty"`
FinishedAt *time.Time `json:"finished_at,omitempty"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
}
type TaskEvent struct {
ID uint64 `gorm:"primaryKey" json:"id"`
TaskID string `gorm:"size:36;not null;index" json:"task_id"`