feat(delivery): add automatic rollback state machine for failed deployments
Introduce a compensating workflow that launches the dedicated AWX rollback template when a deployment cannot be started or fails midway. - add task states: rollback_pending, rolling_back, rolled_back, rollback_failed, rollback_acknowledged - add RollbackJob model to track the compensating AWX run separately from the deploy run, preserving both job IDs for audit - hold resource reservations in 'rollback' status until cleanup succeeds so a failed cleanup cannot be masked by a later delivery - poll rollback jobs with a 2-minute launch timeout; an unknown launch result surfaces as a recoverable failure instead of re-launching - protect finished/register_failed/rolled-back tasks from rollback; register_failed keeps the healthy instance and its resource usage - add DELIVERY_ROLLBACK_TEMPLATE_ID config; without it, failures are marked rollback_failed and require manual cleanup - use unique pending-<task_id> placeholder for executor job IDs
This commit is contained in:
@@ -14,6 +14,11 @@ const (
|
||||
TaskValidationFailed = "validation_failed"
|
||||
TaskCanceling = "canceling"
|
||||
TaskCanceled = "canceled"
|
||||
TaskRollbackPending = "rollback_pending"
|
||||
TaskRollingBack = "rolling_back"
|
||||
TaskRolledBack = "rolled_back"
|
||||
TaskRollbackFailed = "rollback_failed"
|
||||
TaskRollbackAck = "rollback_acknowledged"
|
||||
)
|
||||
|
||||
type ResourceQuota struct {
|
||||
@@ -110,6 +115,20 @@ type ExecutionJob struct {
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
// RollbackJob tracks the compensating AWX run independently from the deploy run.
|
||||
// Keeping a separate record preserves both job IDs for audit and retry tooling.
|
||||
type RollbackJob struct {
|
||||
ID uint64 `gorm:"primaryKey" json:"id"`
|
||||
TaskID string `gorm:"size:36;not null;uniqueIndex" json:"task_id"`
|
||||
ExecutorJobID string `gorm:"size:128;not null" json:"executor_job_id"`
|
||||
Status string `gorm:"size:32;not null;index" json:"status"`
|
||||
Reason string `gorm:"type:text" json:"reason"`
|
||||
StartedAt *time.Time `json:"started_at,omitempty"`
|
||||
FinishedAt *time.Time `json:"finished_at,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
type TaskEvent struct {
|
||||
ID uint64 `gorm:"primaryKey" json:"id"`
|
||||
TaskID string `gorm:"size:36;not null;index" json:"task_id"`
|
||||
|
||||
Reference in New Issue
Block a user