wonder/xinfra
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(delivery): add PostgreSQL rollback and CloudDM isolation

Browse Source
This commit is contained in:
l x
2026-07-31 11:16:19 +08:00
parent 66fde73792
commit 363ad5f9be
15 changed files with 872 additions and 224 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/internal/service/postgresql_delivery.go
+35 -40
View File
@@ -28,6 +28,7 @@ const (
var supportedPostgreSQLVersions = map[string]bool{"15": true, "16": true}
var supportedPostgreSQLTopologies = map[string]bool{"standalone": true, "primary_replica": true}
var postgresqlNamePattern = regexp.MustCompile(`^[a-z0-9](?:[-a-z0-9]*[a-z0-9])?$`)
var errPostgreSQLTaskHandled = errors.New("postgresql task validation completed without dispatch")
type PostgreSQLDeliveryInput struct {
BusinessLineID uint64 `json:"business_line_id" binding:"required"`
@@ -230,7 +231,8 @@ func getPostgreSQLTarget(ctx context.Context, awx *AWXClient, templateID uint64)
if err != nil {
return DeliveryTarget{}, fmt.Errorf("deployment target is unavailable: %w", err)
}
if !strings.Contains(strings.ToLower(template.Name+" "+template.Description), "postgresql") {
templateText := strings.ToLower(template.Name + " " + template.Description)
if !strings.Contains(templateText, "postgresql") || strings.Contains(templateText, "rollback") {
return DeliveryTarget{}, fmt.Errorf("AWX job template %d is not a PostgreSQL target", templateID)
}
hosts, err := awx.ListInventoryHosts(ctx, template.Inventory)
@@ -342,7 +344,12 @@ func (s *PostgreSQLDeliveryService) claimAndReserve(ctx context.Context) (*model
}
port, err := allocateReachablePostgreSQLPort(ctx, host.IP, usedPorts, postgresqlPortInUse)
if err != nil {
_ = tx.Delete(&cluster).Error
if cleanupErr := tx.Where("task_id = ?", task.ID).Delete(&model.PostgreSQLInstance{}).Error; cleanupErr != nil {
return cleanupErr
}
if cleanupErr := tx.Delete(&cluster).Error; cleanupErr != nil {
return cleanupErr
}
return s.common.transitionTx(tx, &task, model.TaskValidationFailed, err.Error(), err.Error())
}
role := "replica"
@@ -390,6 +397,9 @@ func (s *PostgreSQLDeliveryService) claimAndReserve(ctx context.Context) (*model
}
return s.common.transitionTx(tx, &task, model.TaskDispatching, "PostgreSQL resources, ports and directories reserved", "")
})
if err == nil && task.Status != model.TaskDispatching {
return nil, errPostgreSQLTaskHandled
}
return &task, err
}
@@ -463,13 +473,13 @@ func (s *PostgreSQLDeliveryService) CreateExecution(ctx context.Context, task *m
func (s *PostgreSQLDeliveryService) DispatchOnce(ctx context.Context) error {
task, err := s.claimAndReserve(ctx)
if err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) || strings.HasPrefix(err.Error(), "defer:") {
if errors.Is(err, gorm.ErrRecordNotFound) || errors.Is(err, errPostgreSQLTaskHandled) || strings.HasPrefix(err.Error(), "defer:") {
return nil
}
return err
}
if err := s.CreateExecution(ctx, task); err != nil {
return s.fail(ctx, task.ID, model.TaskExecutionFailed, err.Error())
return s.beginRollback(ctx, task.ID, "PostgreSQL deployment could not be started: "+err.Error())
}
return nil
}
@@ -490,13 +500,13 @@ func (s *PostgreSQLDeliveryService) PollOnce(ctx context.Context) error {
case "pending", "waiting", "running", "new":
continue
case "canceled":
_ = s.fail(ctx, execution.TaskID, model.TaskCanceled, "AWX job was canceled")
_ = s.beginRollback(ctx, execution.TaskID, "PostgreSQL AWX job was canceled")
case "successful":
if err := s.complete(ctx, execution.TaskID); err != nil {
_ = s.fail(ctx, execution.TaskID, model.TaskValidationFailed, err.Error())
_ = s.beginRollback(ctx, execution.TaskID, err.Error())
}
default:
_ = s.fail(ctx, execution.TaskID, model.TaskExecutionFailed, "AWX job finished with status "+job.Status)
_ = s.beginRollback(ctx, execution.TaskID, "PostgreSQL AWX job finished with status "+job.Status)
}
}
return nil
@@ -524,7 +534,7 @@ func (s *PostgreSQLDeliveryService) complete(ctx context.Context, taskID string)
}
}
now := time.Now()
return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
if err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
if err := s.common.transitionTx(tx, &task, model.TaskRegistering, "PostgreSQL health checks passed", ""); err != nil {
return err
}
@@ -545,8 +555,22 @@ func (s *PostgreSQLDeliveryService) complete(ctx context.Context, taskID string)
if err := tx.Model(&model.ExecutionJob{}).Where("task_id = ?", task.ID).Updates(map[string]any{"status": "successful", "finished_at": now}).Error; err != nil {
return err
}
return s.common.transitionTx(tx, &task, model.TaskFinished, "PostgreSQL delivery completed and recorded in the PostgreSQL resource ledger", "")
})
return nil
}); err != nil {
return err
}
if err := s.RegisterCloudDM(ctx, task.ID); err != nil {
message := fmt.Sprintf("PostgreSQL delivered; CloudDM registration failed: %v", err)
if transitionErr := s.common.transition(ctx, &task, model.TaskRegisterFailed, "PostgreSQL delivered; CloudDM registration failed and can be retried", message); transitionErr != nil {
return fmt.Errorf("%s; cannot record register_failed: %w", message, transitionErr)
}
return nil
}
message := "PostgreSQL delivery completed; CloudDM registration was skipped because no PostgreSQL endpoint is configured"
if strings.TrimSpace(s.cfg.CloudDMPostgreSQLRegisterURL) != "" {
message = "PostgreSQL delivery completed, registered and recorded in the resource ledger"
}
return s.common.transition(ctx, &task, model.TaskFinished, message, "")
}
func postgresReady(ctx context.Context, address string) error {
@@ -558,36 +582,6 @@ func postgresReady(ctx context.Context, address string) error {
return conn.Close()
}
func (s *PostgreSQLDeliveryService) fail(ctx context.Context, taskID, status, message string) error {
return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
var task model.DeliveryTask
if err := tx.First(&task, "id = ? AND service_type = ?", taskID, postgresqlServiceType).Error; err != nil {
return err
}
if err := s.common.transitionTx(tx, &task, status, message, message); err != nil {
return err
}
reservationStatus := "released"
if status == model.TaskExecutionFailed || (status == model.TaskValidationFailed && strings.Contains(strings.ToLower(message), "health")) {
reservationStatus = "quarantined"
}
if err := tx.Model(&model.ResourceReservation{}).Where("task_id = ? AND status = ?", taskID, "reserved").Update("status", reservationStatus).Error; err != nil {
return err
}
instanceStatus := "failed"
if reservationStatus == "quarantined" {
instanceStatus = "quarantined"
}
if err := tx.Model(&model.PostgreSQLInstance{}).Where("task_id = ? AND status = ?", taskID, "provisioning").Update("status", instanceStatus).Error; err != nil {
return err
}
if err := tx.Model(&model.PostgreSQLCluster{}).Where("task_id = ? AND status = ?", taskID, "provisioning").Update("status", instanceStatus).Error; err != nil {
return err
}
return tx.Model(&model.ExecutionJob{}).Where("task_id = ? AND status IN ?", taskID, []string{"launching", "running"}).Updates(map[string]any{"status": "failed", "finished_at": time.Now()}).Error
})
}
func (s *PostgreSQLDeliveryService) Run(ctx context.Context) {
interval := time.Duration(s.cfg.DeliveryPollSeconds) * time.Second
if interval < time.Second {
@@ -602,6 +596,7 @@ func (s *PostgreSQLDeliveryService) Run(ctx context.Context) {
case <-ticker.C:
_ = s.DispatchOnce(ctx)
_ = s.PollOnce(ctx)
_ = s.PollRollbackOnce(ctx)
}
}
}