feat(delivery): integrate PostgreSQL delivery
This commit is contained in:
+10
-1
@@ -41,9 +41,11 @@ SSO_ENABLED=true
|
||||
BOOTSTRAP_ADMIN_USERNAME=admin
|
||||
BOOTSTRAP_ADMIN_PASSWORD=change-this-bootstrap-admin-password
|
||||
|
||||
# MySQL service delivery (AWX is required when the scheduler is enabled)
|
||||
# Database delivery (MySQL/PostgreSQL; AWX is required when the scheduler is enabled)
|
||||
DELIVERY_SCHEDULER_ENABLED=false
|
||||
DELIVERY_DISPATCH_SECONDS=5
|
||||
DELIVERY_POLL_SECONDS=5
|
||||
DELIVERY_BUSINESS_LIMIT=1
|
||||
DELIVERY_CALLBACK_BASE_URL=http://authserver-backend.authserver.svc.cluster.local:8083
|
||||
DELIVERY_CREDENTIAL_SECRET=
|
||||
DELIVERY_RESERVATION_TTL_MINUTES=120
|
||||
@@ -54,6 +56,7 @@ DELIVERY_HOST_INSTANCE_LIMIT=4
|
||||
# 数据盘挂载点白名单(逗号分隔,第一项为默认值)
|
||||
DELIVERY_DATA_DISKS=/data,/disk1,/mnt,/opt/mysql-delivery
|
||||
AWX_BASE_URL=
|
||||
# AWX API must point at the HTTP API endpoint, not a Minikube SSH port.
|
||||
AWX_TOKEN=
|
||||
AWX_USERNAME=
|
||||
AWX_PASSWORD=
|
||||
@@ -66,6 +69,12 @@ DELIVERY_MYSQL_INSPECT_TIMEOUT_SECONDS=90
|
||||
# AWX Job Template ID for ansible/mysql-rollback.yml; required for automatic cleanup
|
||||
DELIVERY_ROLLBACK_TEMPLATE_ID=0
|
||||
DELIVERY_SERVICE_TOKEN=
|
||||
|
||||
# PostgreSQL runtime secrets are injected by an AWX Credential and are never
|
||||
# persisted in xinfra task payloads:
|
||||
# XINFRA_POSTGRES_ADMIN_PASSWORD
|
||||
# XINFRA_POSTGRES_REPLICATION_PASSWORD
|
||||
# XINFRA_POSTGRES_REPLICATION_USER (optional, defaults to xinfra_replication)
|
||||
CLOUDDM_REGISTER_URL=
|
||||
CLOUDDM_DELETE_URL=
|
||||
CLOUDDM_API_TOKEN=
|
||||
|
||||
@@ -78,12 +78,14 @@ type Config struct {
|
||||
DeliveryServiceToken string
|
||||
DeliverySchedulerEnabled bool
|
||||
DeliveryDispatchSeconds int
|
||||
DeliveryPollSeconds int
|
||||
DeliveryCallbackBaseURL string
|
||||
DeliveryCredentialSecret string
|
||||
ReservationTTLMinutes int
|
||||
DeliveryGlobalLimit int
|
||||
DeliveryTargetLimit int
|
||||
DeliveryHostInstanceLimit int
|
||||
DeliveryBusinessLimit int
|
||||
DeliveryDataDisks []string
|
||||
SINABaseURL string
|
||||
SINAUsername string
|
||||
@@ -164,12 +166,14 @@ func Load() Config {
|
||||
DeliveryServiceToken: env("DELIVERY_SERVICE_TOKEN", ""),
|
||||
DeliverySchedulerEnabled: envBool("DELIVERY_SCHEDULER_ENABLED", false),
|
||||
DeliveryDispatchSeconds: envInt("DELIVERY_DISPATCH_SECONDS", 5),
|
||||
DeliveryPollSeconds: envInt("DELIVERY_POLL_SECONDS", 5),
|
||||
DeliveryCallbackBaseURL: trimURL(env("DELIVERY_CALLBACK_BASE_URL", publicBaseURL)),
|
||||
DeliveryCredentialSecret: env("DELIVERY_CREDENTIAL_SECRET", env("JWT_SECRET", "change-this-secret")),
|
||||
ReservationTTLMinutes: envInt("DELIVERY_RESERVATION_TTL_MINUTES", 120),
|
||||
DeliveryGlobalLimit: envInt("DELIVERY_GLOBAL_LIMIT", 2),
|
||||
DeliveryTargetLimit: envInt("DELIVERY_TARGET_LIMIT", 2),
|
||||
DeliveryHostInstanceLimit: envInt("DELIVERY_HOST_INSTANCE_LIMIT", 4),
|
||||
DeliveryBusinessLimit: envInt("DELIVERY_BUSINESS_LIMIT", 1),
|
||||
DeliveryDataDisks: splitCSV(env("DELIVERY_DATA_DISKS", "/data,/disk1,/mnt,/opt/mysql-delivery")),
|
||||
SINABaseURL: trimURL(env("SINA_BASE_URL", "https://sinai.qiniu.io:443")),
|
||||
SINAUsername: env("SINA_USERNAME", ""),
|
||||
|
||||
@@ -27,6 +27,9 @@ func AutoMigrate(db *gorm.DB) error {
|
||||
&model.ResourceReservation{},
|
||||
&model.DeploymentResult{},
|
||||
&model.DeploymentCredential{},
|
||||
&model.PostgreSQLCluster{},
|
||||
&model.PostgreSQLInstance{},
|
||||
&model.PostgreSQLResourceUsage{},
|
||||
&model.ResourceUsage{},
|
||||
&model.ExecutionJob{},
|
||||
&model.RollbackJob{},
|
||||
|
||||
@@ -15,12 +15,22 @@ import (
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
type DeliveryHandler struct{ service *service.DeliveryService }
|
||||
type DeliveryHandler struct {
|
||||
service *service.DeliveryService
|
||||
}
|
||||
|
||||
type PostgreSQLDeliveryHandler struct {
|
||||
service *service.PostgreSQLDeliveryService
|
||||
}
|
||||
|
||||
func NewDeliveryHandler(s *service.DeliveryService) *DeliveryHandler {
|
||||
return &DeliveryHandler{service: s}
|
||||
}
|
||||
|
||||
func NewPostgreSQLDeliveryHandler(s *service.PostgreSQLDeliveryService) *PostgreSQLDeliveryHandler {
|
||||
return &PostgreSQLDeliveryHandler{service: s}
|
||||
}
|
||||
|
||||
type DeliveryCallbackHandler struct {
|
||||
service *service.DeliveryService
|
||||
token string
|
||||
@@ -108,6 +118,30 @@ func (h *DeliveryHandler) CreateMySQL(c *gin.Context) {
|
||||
c.JSON(status, gin.H{"task": task, "idempotent_replay": existing})
|
||||
}
|
||||
|
||||
// CreatePostgreSQL submits a standalone or primary/replica PostgreSQL delivery.
|
||||
func (h *PostgreSQLDeliveryHandler) CreatePostgreSQL(c *gin.Context) {
|
||||
claims, ok := CurrentClaims(c)
|
||||
if !ok {
|
||||
c.JSON(http.StatusUnauthorized, gin.H{"error": "missing current user"})
|
||||
return
|
||||
}
|
||||
var req service.PostgreSQLDeliveryInput
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
task, existing, err := h.service.CreateTask(c.Request.Context(), claims.UserID, claims.IsAdmin, c.GetHeader("Idempotency-Key"), req)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
status := http.StatusAccepted
|
||||
if existing {
|
||||
status = http.StatusOK
|
||||
}
|
||||
c.JSON(status, gin.H{"task": task, "idempotent_replay": existing})
|
||||
}
|
||||
|
||||
// List 获取交付任务列表
|
||||
// @Summary 获取交付任务列表
|
||||
// @Description 返回当前用户可见的交付任务列表(管理员可见全部)
|
||||
|
||||
@@ -39,12 +39,14 @@ type DeliveryTask struct {
|
||||
RequestedBy uint64 `gorm:"not null;index" json:"requested_by"`
|
||||
Component string `gorm:"size:32;not null;default:mysql;index" json:"component"`
|
||||
TargetType string `gorm:"size:32;not null" json:"target_type"`
|
||||
ServiceType string `gorm:"size:32;not null;default:'mysql';index" json:"service_type"`
|
||||
TargetID uint64 `gorm:"not null;index" json:"target_id"`
|
||||
Namespace string `gorm:"size:63;not null;index" json:"namespace"`
|
||||
InstanceName string `gorm:"size:63;not null" json:"instance_name"`
|
||||
TargetHost string `gorm:"size:128" json:"target_host,omitempty"`
|
||||
TargetHost string `gorm:"size:1024" json:"target_host,omitempty"`
|
||||
TargetHostIP string `gorm:"size:64" json:"target_host_ip,omitempty"`
|
||||
MySQLPort int `gorm:"column:mysql_port;not null;default:3307" json:"mysql_port"`
|
||||
PostgreSQLPort int `gorm:"column:postgresql_port;not null;default:15432" json:"postgresql_port,omitempty"`
|
||||
Status string `gorm:"size:32;not null;index" json:"status"`
|
||||
ImmutablePayload string `gorm:"type:json;not null" json:"immutable_payload"`
|
||||
PayloadHash string `gorm:"size:64;not null" json:"payload_hash"`
|
||||
@@ -126,6 +128,69 @@ type DeploymentCredential struct {
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
type PostgreSQLCluster struct {
|
||||
ID uint64 `gorm:"primaryKey" json:"id"`
|
||||
TaskID string `gorm:"size:36;not null;uniqueIndex" json:"task_id"`
|
||||
BusinessLineID uint64 `gorm:"not null;index" json:"business_line_id"`
|
||||
TargetID uint64 `gorm:"not null;index" json:"target_id"`
|
||||
Name string `gorm:"size:63;not null;uniqueIndex" json:"name"`
|
||||
VersionMajor string `gorm:"size:8;not null" json:"version_major"`
|
||||
Topology string `gorm:"size:32;not null" json:"topology"`
|
||||
ReplicationMode string `gorm:"size:32;not null;default:'async'" json:"replication_mode"`
|
||||
FailoverMode string `gorm:"size:32;not null;default:'manual'" json:"failover_mode"`
|
||||
PrimaryInstanceID uint64 `gorm:"not null;default:0" json:"primary_instance_id"`
|
||||
Status string `gorm:"size:32;not null;index" json:"status"`
|
||||
BackupStatus string `gorm:"size:32;not null;default:'not_configured'" json:"backup_status"`
|
||||
MonitoringStatus string `gorm:"size:32;not null;default:'not_configured'" json:"monitoring_status"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
type PostgreSQLInstance struct {
|
||||
ID uint64 `gorm:"primaryKey" json:"id"`
|
||||
TaskID string `gorm:"size:36;not null;index" json:"task_id"`
|
||||
ClusterID uint64 `gorm:"not null;index" json:"cluster_id"`
|
||||
BusinessLineID uint64 `gorm:"not null;index" json:"business_line_id"`
|
||||
TargetID uint64 `gorm:"not null;index" json:"target_id"`
|
||||
InstanceID string `gorm:"size:63;not null;uniqueIndex" json:"instance_id"`
|
||||
Hostname string `gorm:"size:255;not null;uniqueIndex:idx_postgresql_host_port,priority:1" json:"hostname"`
|
||||
HostIP string `gorm:"size:64;not null" json:"host_ip"`
|
||||
Port int `gorm:"not null;index:idx_postgresql_host_port,priority:2" json:"port"`
|
||||
DataDir string `gorm:"size:512;not null;uniqueIndex" json:"data_dir"`
|
||||
ConfigDir string `gorm:"size:512;not null" json:"config_dir"`
|
||||
LogDir string `gorm:"size:512;not null" json:"log_dir"`
|
||||
SystemdUnit string `gorm:"size:128;not null" json:"systemd_unit"`
|
||||
VersionMajor string `gorm:"size:8;not null" json:"version_major"`
|
||||
VersionFull string `gorm:"size:32;not null;default:''" json:"version_full"`
|
||||
Role string `gorm:"size:16;not null" json:"role"`
|
||||
HAComponentRole string `gorm:"size:16;not null;default:'database'" json:"ha_component_role"`
|
||||
UpstreamInstanceID string `gorm:"size:63;not null;default:''" json:"upstream_instance_id"`
|
||||
ReplicationSlotName string `gorm:"size:63;not null;default:''" json:"replication_slot_name"`
|
||||
Status string `gorm:"size:32;not null;index" json:"status"`
|
||||
BackupStatus string `gorm:"size:32;not null;default:'not_configured'" json:"backup_status"`
|
||||
MonitoringStatus string `gorm:"size:32;not null;default:'not_configured'" json:"monitoring_status"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
type PostgreSQLResourceUsage struct {
|
||||
ID uint64 `gorm:"primaryKey" json:"id"`
|
||||
TaskID string `gorm:"size:36;not null;index" json:"task_id"`
|
||||
InstanceID uint64 `gorm:"not null;uniqueIndex" json:"instance_id"`
|
||||
ClusterID uint64 `gorm:"not null;index" json:"cluster_id"`
|
||||
BusinessLineID uint64 `gorm:"not null;index" json:"business_line_id"`
|
||||
TargetID uint64 `gorm:"not null;index" json:"target_id"`
|
||||
CPUMilli int64 `gorm:"not null" json:"cpu_milli"`
|
||||
MemoryMi int64 `gorm:"not null" json:"memory_mi"`
|
||||
StorageGi int64 `gorm:"not null" json:"storage_gi"`
|
||||
Port int `gorm:"not null" json:"port"`
|
||||
DataDir string `gorm:"size:512;not null" json:"data_dir"`
|
||||
Status string `gorm:"size:32;not null;index" json:"status"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
ReleasedAt *time.Time `json:"released_at,omitempty"`
|
||||
}
|
||||
|
||||
type ExecutionJob struct {
|
||||
ID uint64 `gorm:"primaryKey" json:"id"`
|
||||
TaskID string `gorm:"size:36;not null;uniqueIndex" json:"task_id"`
|
||||
|
||||
@@ -73,8 +73,10 @@ func registerAuthServerRoutes(r *gin.Engine, deps Dependencies) {
|
||||
wayneRoleBindingService := service.NewWayneRoleBindingService(deps.Config, deps.DB)
|
||||
deliveryService := service.NewDeliveryService(deps.Config, deps.DB, auditService)
|
||||
machineService := service.NewMachineService(deps.Config, deps.DB)
|
||||
postgresqlDeliveryService := service.NewPostgreSQLDeliveryService(deps.Config, deps.DB, deliveryService)
|
||||
if deps.Config.DeliverySchedulerEnabled {
|
||||
go deliveryService.Run(context.Background())
|
||||
go postgresqlDeliveryService.Run(context.Background())
|
||||
}
|
||||
machineService.Run(context.Background())
|
||||
|
||||
@@ -90,6 +92,7 @@ func registerAuthServerRoutes(r *gin.Engine, deps Dependencies) {
|
||||
oauthHandler := handler.NewOAuthHandler(deps.Config, deps.DB, auditService)
|
||||
deliveryHandler := handler.NewDeliveryHandler(deliveryService)
|
||||
deliveryCallbackHandler := handler.NewDeliveryCallbackHandler(deliveryService, deps.Config.AWXWebhookToken)
|
||||
postgresqlDeliveryHandler := handler.NewPostgreSQLDeliveryHandler(postgresqlDeliveryService)
|
||||
containerServiceHandler := handler.NewContainerServiceHandler(deps.DB, wayneRoleBindingService)
|
||||
taskLogHandler := handler.NewTaskLogHandler(deps.DB, deliveryService, wayneRoleBindingService)
|
||||
machineHandler := handler.NewMachineHandler(machineService)
|
||||
@@ -155,6 +158,7 @@ func registerAuthServerRoutes(r *gin.Engine, deps Dependencies) {
|
||||
protected.GET("/delivery/targets/:target_id/hosts/:host/mount-paths", deliveryHandler.TargetHostMountPaths)
|
||||
protected.PUT("/delivery/quotas", deliveryHandler.UpsertQuota)
|
||||
protected.POST("/delivery/mysql", deliveryHandler.CreateMySQL)
|
||||
protected.POST("/delivery/postgresql", postgresqlDeliveryHandler.CreatePostgreSQL)
|
||||
protected.GET("/delivery/tasks", deliveryHandler.List)
|
||||
protected.GET("/delivery/business-lines/:id/mysql-services", deliveryHandler.MySQLServiceLedger)
|
||||
protected.POST("/delivery/business-lines/:id/mysql-services/sync", deliveryHandler.SyncMySQLServiceLedger)
|
||||
|
||||
@@ -363,6 +363,11 @@ var ansibleHostPattern = regexp.MustCompile(`(?m)^\s*ansible_host\s*:\s*"?([^"\s
|
||||
func AWXHostIP(host AWXInventoryHost) string {
|
||||
var parsed map[string]any
|
||||
if err := json.Unmarshal([]byte(host.Variables), &parsed); err == nil {
|
||||
for _, key := range []string{"xinfra_public_address", "xinfra_host_address"} {
|
||||
if value, ok := parsed[key].(string); ok && strings.TrimSpace(value) != "" {
|
||||
return strings.TrimSpace(value)
|
||||
}
|
||||
}
|
||||
if value, ok := parsed["ansible_host"].(string); ok && strings.TrimSpace(value) != "" {
|
||||
return strings.TrimSpace(value)
|
||||
}
|
||||
|
||||
@@ -73,3 +73,13 @@ func TestAWXClientCreatesTokenFromCredentials(t *testing.T) {
|
||||
t.Fatalf("token requests = %d, want 1", tokenRequests)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAWXHostIPPrefersXInfraPublicAddress(t *testing.T) {
|
||||
host := AWXInventoryHost{
|
||||
Name: "postgresql-218-11-5-224",
|
||||
Variables: `{"ansible_host":"127.0.0.1","xinfra_public_address":"218.11.5.224"}`,
|
||||
}
|
||||
if got := AWXHostIP(host); got != "218.11.5.224" {
|
||||
t.Fatalf("AWXHostIP = %q, want public address", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -160,7 +160,7 @@ type DeploymentCredentialView struct {
|
||||
// targetMetadata describes the native VM候选节点池以及部署形态,由 AWX inventory hosts 动态组装。
|
||||
type targetMetadata struct {
|
||||
Topology string `json:"topology"`
|
||||
MySQLPort int `json:"mysql_port"`
|
||||
MySQLPort int `json:"mysql_port,omitempty"`
|
||||
Hosts []targetHost `json:"hosts"`
|
||||
}
|
||||
|
||||
@@ -410,6 +410,14 @@ func (s *DeliveryService) ListTargets(ctx context.Context, component string) ([]
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if component == postgresqlServiceType {
|
||||
target.TargetType = "host_pool"
|
||||
meta := parseTargetMetadata(target.Metadata)
|
||||
meta.MySQLPort = 0
|
||||
if raw, marshalErr := json.Marshal(meta); marshalErr == nil {
|
||||
target.Metadata = string(raw)
|
||||
}
|
||||
}
|
||||
targets = append(targets, target)
|
||||
}
|
||||
return targets, nil
|
||||
@@ -692,6 +700,7 @@ func (s *DeliveryService) CreateTask(ctx context.Context, userID uint64, isAdmin
|
||||
RequestedBy: userID,
|
||||
Component: "mysql",
|
||||
TargetType: target.TargetType,
|
||||
ServiceType: "mysql",
|
||||
TargetID: target.ID,
|
||||
Namespace: input.Namespace,
|
||||
InstanceName: input.InstanceName,
|
||||
@@ -1376,6 +1385,9 @@ func (s *DeliveryService) RevealDeploymentCredentials(ctx context.Context, taskI
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !isMySQLServiceType(task.ServiceType) {
|
||||
return nil, fmt.Errorf("task service type %q does not provide MySQL credentials", task.ServiceType)
|
||||
}
|
||||
if task.Status != model.TaskFinished && task.Status != model.TaskRegisterFailed {
|
||||
return nil, fmt.Errorf("task credentials are available only after a successful deployment")
|
||||
}
|
||||
@@ -1593,7 +1605,7 @@ func (s *DeliveryService) claimAndReserve(ctx context.Context) (*model.DeliveryT
|
||||
var target DeliveryTarget
|
||||
dispatchable := false
|
||||
err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
if err := tx.Clauses(clause.Locking{Strength: "UPDATE", Options: "SKIP LOCKED"}).Where("status = ?", model.TaskPending).Order("created_at ASC").First(&task).Error; err != nil {
|
||||
if err := tx.Clauses(clause.Locking{Strength: "UPDATE", Options: "SKIP LOCKED"}).Where("status = ? AND (service_type = '' OR service_type = 'mysql')", model.TaskPending).Order("created_at ASC").First(&task).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
var targetErr error
|
||||
@@ -1930,6 +1942,9 @@ func (s *DeliveryService) CreateExecution(ctx context.Context, taskID, payloadHa
|
||||
if err := s.db.WithContext(ctx).First(&task, "id = ?", taskID).Error; err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if task.ServiceType != "" && task.ServiceType != "mysql" {
|
||||
return nil, false, fmt.Errorf("task service type %q is not handled by the MySQL executor", task.ServiceType)
|
||||
}
|
||||
if task.PayloadHash != payloadHash || task.IdempotencyKey != idempotencyKey {
|
||||
return nil, false, fmt.Errorf("execution request does not match the immutable task payload")
|
||||
}
|
||||
@@ -2063,6 +2078,9 @@ func (s *DeliveryService) HandleStageEvent(ctx context.Context, taskID string, i
|
||||
if err := tx.First(&task, "id = ?", taskID).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if !isMySQLServiceType(task.ServiceType) {
|
||||
return fmt.Errorf("task service type %q is not handled by MySQL stage callbacks", task.ServiceType)
|
||||
}
|
||||
if input.AWXJobID != "" {
|
||||
var execution model.ExecutionJob
|
||||
if err := tx.Where("task_id = ?", task.ID).First(&execution).Error; err != nil {
|
||||
@@ -2118,6 +2136,14 @@ func (s *DeliveryService) HandleAWXJobNotification(ctx context.Context, input AW
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var task model.DeliveryTask
|
||||
if err := s.db.WithContext(ctx).Select("id", "service_type").First(&task, "id = ?", execution.TaskID).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if !isMySQLServiceType(task.ServiceType) {
|
||||
// PostgreSQL jobs are finalized by PostgreSQLDeliveryService.PollOnce.
|
||||
return nil
|
||||
}
|
||||
message := awxNotificationMessage(input)
|
||||
defer s.broadcastTask(ctx, execution.TaskID)
|
||||
switch status {
|
||||
@@ -2275,7 +2301,10 @@ func (s *DeliveryService) finishExecution(ctx context.Context, execution *model.
|
||||
|
||||
func (s *DeliveryService) PollOnce(ctx context.Context) error {
|
||||
var jobs []model.ExecutionJob
|
||||
if err := s.db.WithContext(ctx).Where("status = ?", "running").Find(&jobs).Error; err != nil {
|
||||
if err := s.db.WithContext(ctx).
|
||||
Joins("JOIN delivery_tasks ON delivery_tasks.id = execution_jobs.task_id").
|
||||
Where("execution_jobs.status = ? AND (delivery_tasks.service_type = '' OR delivery_tasks.service_type = ?)", "running", "mysql").
|
||||
Find(&jobs).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
for _, execution := range jobs {
|
||||
@@ -2309,6 +2338,9 @@ func (s *DeliveryService) completeTask(ctx context.Context, taskID string) error
|
||||
if err := s.db.WithContext(ctx).First(&task, "id = ?", taskID).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if !isMySQLServiceType(task.ServiceType) {
|
||||
return fmt.Errorf("task service type %q cannot be completed by the MySQL delivery service", task.ServiceType)
|
||||
}
|
||||
var payload deliveryPayload
|
||||
if err := json.Unmarshal([]byte(task.ImmutablePayload), &payload); err != nil {
|
||||
return err
|
||||
@@ -2380,6 +2412,9 @@ func (s *DeliveryService) RetryCloudDMRegistration(ctx context.Context, taskID s
|
||||
if err := s.db.WithContext(ctx).First(&task, "id = ?", taskID).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if !isMySQLServiceType(task.ServiceType) {
|
||||
return fmt.Errorf("task service type %q cannot use the MySQL CloudDM registration flow", task.ServiceType)
|
||||
}
|
||||
if task.Status != model.TaskRegisterFailed {
|
||||
return fmt.Errorf("task %s is in state %q and cannot retry CloudDM registration", taskID, task.Status)
|
||||
}
|
||||
@@ -2657,6 +2692,9 @@ func (s *DeliveryService) beginRollback(ctx context.Context, taskID, reason stri
|
||||
if err := s.db.WithContext(ctx).First(&task, "id = ?", taskID).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if !isMySQLServiceType(task.ServiceType) {
|
||||
return fmt.Errorf("task service type %q cannot use the MySQL rollback flow", task.ServiceType)
|
||||
}
|
||||
if rollbackProtectedStatus(task.Status) {
|
||||
return nil
|
||||
}
|
||||
@@ -2743,6 +2781,9 @@ func (s *DeliveryService) RetryRollback(ctx context.Context, taskID string) erro
|
||||
if err := s.db.WithContext(ctx).First(&task, "id = ?", taskID).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if !isMySQLServiceType(task.ServiceType) {
|
||||
return fmt.Errorf("task service type %q cannot use the MySQL rollback flow", task.ServiceType)
|
||||
}
|
||||
if task.Status != model.TaskRollbackFailed {
|
||||
return fmt.Errorf("task %s is in state %q and cannot retry rollback", taskID, task.Status)
|
||||
}
|
||||
@@ -2770,6 +2811,9 @@ func (s *DeliveryService) AcknowledgeRollbackRelease(ctx context.Context, taskID
|
||||
if err := tx.First(&task, "id = ?", taskID).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if !isMySQLServiceType(task.ServiceType) {
|
||||
return fmt.Errorf("task service type %q cannot use the MySQL rollback flow", task.ServiceType)
|
||||
}
|
||||
if task.Status != model.TaskRollbackFailed {
|
||||
return fmt.Errorf("task %s is in state %q and cannot acknowledge rollback release", taskID, task.Status)
|
||||
}
|
||||
@@ -2888,6 +2932,9 @@ func (s *DeliveryService) completeRollback(ctx context.Context, taskID string) e
|
||||
if err := tx.First(&task, "id = ?", taskID).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if !isMySQLServiceType(task.ServiceType) {
|
||||
return fmt.Errorf("task service type %q cannot use the MySQL rollback flow", task.ServiceType)
|
||||
}
|
||||
if task.Status != model.TaskRollingBack {
|
||||
return fmt.Errorf("task %s is in state %q, cannot complete rollback", taskID, task.Status)
|
||||
}
|
||||
@@ -2907,6 +2954,10 @@ func (s *DeliveryService) completeRollback(ctx context.Context, taskID string) e
|
||||
})
|
||||
}
|
||||
|
||||
func isMySQLServiceType(serviceType string) bool {
|
||||
return serviceType == "" || serviceType == "mysql"
|
||||
}
|
||||
|
||||
func (s *DeliveryService) failTask(ctx context.Context, task *model.DeliveryTask, status, message string) error {
|
||||
return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
var current model.DeliveryTask
|
||||
|
||||
@@ -293,6 +293,15 @@ func TestRegisterFailedIsProtectedFromRollback(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMySQLServiceTypeCompatibility(t *testing.T) {
|
||||
if !isMySQLServiceType("") || !isMySQLServiceType("mysql") {
|
||||
t.Fatal("legacy and explicit MySQL tasks must remain supported")
|
||||
}
|
||||
if isMySQLServiceType(postgresqlServiceType) {
|
||||
t.Fatal("PostgreSQL tasks must not enter MySQL execution paths")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRollbackLaunchExpired(t *testing.T) {
|
||||
now := time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC)
|
||||
started := now.Add(-rollbackLaunchTimeout - time.Second)
|
||||
|
||||
@@ -0,0 +1,607 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/1024XEngineer/xinfra/server/internal/config"
|
||||
"github.com/1024XEngineer/xinfra/server/internal/model"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
)
|
||||
|
||||
const (
|
||||
postgresqlPortPoolStart = 15432
|
||||
postgresqlPortPoolEnd = 15999
|
||||
postgresqlServiceType = "postgresql"
|
||||
)
|
||||
|
||||
var supportedPostgreSQLVersions = map[string]bool{"15": true, "16": true}
|
||||
var supportedPostgreSQLTopologies = map[string]bool{"standalone": true, "primary_replica": true}
|
||||
var postgresqlNamePattern = regexp.MustCompile(`^[a-z0-9](?:[-a-z0-9]*[a-z0-9])?$`)
|
||||
|
||||
type PostgreSQLDeliveryInput struct {
|
||||
BusinessLineID uint64 `json:"business_line_id" binding:"required"`
|
||||
TargetID uint64 `json:"target_id" binding:"required"`
|
||||
Namespace string `json:"namespace" binding:"required"`
|
||||
ClusterName string `json:"cluster_name" binding:"required"`
|
||||
VersionMajor string `json:"version_major"`
|
||||
PostgreSQLVersion string `json:"postgresql_version,omitempty"`
|
||||
Topology string `json:"topology"`
|
||||
ReplicaCount int `json:"replica_count"`
|
||||
TargetHosts []string `json:"target_hosts,omitempty"`
|
||||
CPUMilli int64 `json:"cpu_milli" binding:"required"`
|
||||
MemoryMi int64 `json:"memory_mi" binding:"required"`
|
||||
StorageGi int64 `json:"storage_gi" binding:"required"`
|
||||
DataRoot string `json:"data_root"`
|
||||
MaxConnections int `json:"max_connections"`
|
||||
}
|
||||
|
||||
type postgresqlDeliveryPayload struct {
|
||||
PostgreSQLDeliveryInput
|
||||
TargetType string `json:"target_type"`
|
||||
}
|
||||
|
||||
func validatePostgreSQLDeliveryInput(input PostgreSQLDeliveryInput, dataDisks []string) error {
|
||||
_ = dataDisks
|
||||
if input.VersionMajor == "" {
|
||||
input.VersionMajor = input.PostgreSQLVersion
|
||||
}
|
||||
if input.PostgreSQLVersion != "" && input.VersionMajor != "" && input.PostgreSQLVersion != input.VersionMajor {
|
||||
return fmt.Errorf("version_major and postgresql_version must match")
|
||||
}
|
||||
if len(input.Namespace) > 63 || !dnsLabelPattern.MatchString(input.Namespace) {
|
||||
return fmt.Errorf("namespace must be a valid Kubernetes DNS label")
|
||||
}
|
||||
if len(input.ClusterName) > 63 || !postgresqlNamePattern.MatchString(input.ClusterName) {
|
||||
return fmt.Errorf("cluster_name must be a valid DNS label")
|
||||
}
|
||||
if !supportedPostgreSQLVersions[input.VersionMajor] {
|
||||
return fmt.Errorf("unsupported version_major %q, supported: 15, 16", input.VersionMajor)
|
||||
}
|
||||
topology := input.Topology
|
||||
if topology == "" {
|
||||
topology = "standalone"
|
||||
}
|
||||
if !supportedPostgreSQLTopologies[topology] {
|
||||
return fmt.Errorf("unsupported topology %q, supported: standalone, primary_replica", topology)
|
||||
}
|
||||
if topology == "standalone" && input.ReplicaCount != 0 {
|
||||
return fmt.Errorf("standalone topology cannot have replicas")
|
||||
}
|
||||
if topology == "primary_replica" && (input.ReplicaCount < 1 || input.ReplicaCount > 7) {
|
||||
return fmt.Errorf("replica_count must be between 1 and 7 for primary_replica")
|
||||
}
|
||||
if input.CPUMilli < 100 || input.CPUMilli > 64000 || input.MemoryMi < 2048 || input.MemoryMi > 65536 || input.StorageGi < 20 || input.StorageGi > 2000 {
|
||||
return fmt.Errorf("requested resources are outside the supported range (memory: 2048-65536 MiB, storage: 20-2000 GiB)")
|
||||
}
|
||||
nodes := 1
|
||||
if topology == "primary_replica" {
|
||||
nodes += input.ReplicaCount
|
||||
}
|
||||
if len(input.TargetHosts) > 0 && len(input.TargetHosts) != nodes {
|
||||
return fmt.Errorf("target_hosts must contain exactly %d distinct hosts", nodes)
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, host := range input.TargetHosts {
|
||||
if len(host) > 253 || !hostNamePattern.MatchString(host) || seen[host] {
|
||||
return fmt.Errorf("target_hosts must contain unique valid inventory host names")
|
||||
}
|
||||
seen[host] = true
|
||||
}
|
||||
if input.DataRoot != "" {
|
||||
if input.DataRoot != "/data/postgresql" {
|
||||
return fmt.Errorf("data_root is fixed to /data/postgresql in the first release")
|
||||
}
|
||||
}
|
||||
if input.MaxConnections < 0 || input.MaxConnections > 10000 {
|
||||
return fmt.Errorf("max_connections must be between 0 and 10000")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func allocatePostgreSQLPort(requested int, used []int) (int, error) {
|
||||
taken := make(map[int]bool, len(used))
|
||||
for _, port := range used {
|
||||
taken[port] = true
|
||||
}
|
||||
if requested != 0 {
|
||||
if requested < postgresqlPortPoolStart || requested > postgresqlPortPoolEnd {
|
||||
return 0, fmt.Errorf("postgresql_port must be within %d-%d", postgresqlPortPoolStart, postgresqlPortPoolEnd)
|
||||
}
|
||||
if taken[requested] {
|
||||
return 0, fmt.Errorf("postgresql port %d is already allocated on the target host", requested)
|
||||
}
|
||||
return requested, nil
|
||||
}
|
||||
for port := postgresqlPortPoolStart; port <= postgresqlPortPoolEnd; port++ {
|
||||
if !taken[port] {
|
||||
return port, nil
|
||||
}
|
||||
}
|
||||
return 0, fmt.Errorf("postgresql port pool %d-%d is exhausted on the target host", postgresqlPortPoolStart, postgresqlPortPoolEnd)
|
||||
}
|
||||
|
||||
type postgresqlPortProbe func(context.Context, string, int) bool
|
||||
|
||||
func allocateReachablePostgreSQLPort(ctx context.Context, host string, used []int, inUse postgresqlPortProbe) (int, error) {
|
||||
occupied := append([]int(nil), used...)
|
||||
for {
|
||||
port, err := allocatePostgreSQLPort(0, occupied)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if !inUse(ctx, host, port) {
|
||||
return port, nil
|
||||
}
|
||||
occupied = append(occupied, port)
|
||||
}
|
||||
}
|
||||
|
||||
func postgresqlPortInUse(ctx context.Context, host string, port int) bool {
|
||||
probeCtx, cancel := context.WithTimeout(ctx, 500*time.Millisecond)
|
||||
defer cancel()
|
||||
conn, err := (&net.Dialer{}).DialContext(probeCtx, "tcp", net.JoinHostPort(host, strconv.Itoa(port)))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
_ = conn.Close()
|
||||
return true
|
||||
}
|
||||
|
||||
type PostgreSQLDeliveryService struct {
|
||||
db *gorm.DB
|
||||
cfg config.Config
|
||||
awx *AWXClient
|
||||
common *DeliveryService
|
||||
}
|
||||
|
||||
func NewPostgreSQLDeliveryService(cfg config.Config, db *gorm.DB, common *DeliveryService) *PostgreSQLDeliveryService {
|
||||
return &PostgreSQLDeliveryService{db: db, cfg: cfg, awx: NewAWXClient(cfg.AWXBaseURL, cfg.AWXToken, cfg.AWXUsername, cfg.AWXPassword), common: common}
|
||||
}
|
||||
|
||||
func (s *PostgreSQLDeliveryService) CreateTask(ctx context.Context, userID uint64, isAdmin bool, idempotencyKey string, input PostgreSQLDeliveryInput) (*model.DeliveryTask, bool, error) {
|
||||
idempotencyKey = strings.TrimSpace(idempotencyKey)
|
||||
if idempotencyKey == "" || len(idempotencyKey) > 128 {
|
||||
return nil, false, fmt.Errorf("Idempotency-Key header is required and must not exceed 128 characters")
|
||||
}
|
||||
if err := validatePostgreSQLDeliveryInput(input, s.cfg.DeliveryDataDisks); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if input.VersionMajor == "" {
|
||||
input.VersionMajor = input.PostgreSQLVersion
|
||||
}
|
||||
var existing model.DeliveryTask
|
||||
if err := s.db.WithContext(ctx).Where("idempotency_key = ?", idempotencyKey).First(&existing).Error; err == nil {
|
||||
if existing.RequestedBy != userID || existing.ServiceType != postgresqlServiceType {
|
||||
return nil, false, fmt.Errorf("idempotency key is already in use")
|
||||
}
|
||||
return &existing, true, nil
|
||||
} else if !errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil, false, err
|
||||
}
|
||||
if !isAdmin {
|
||||
var count int64
|
||||
if err := s.db.WithContext(ctx).Model(&model.BusinessLineUser{}).Where("business_line_id = ? AND user_id = ?", input.BusinessLineID, userID).Count(&count).Error; err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if count == 0 {
|
||||
return nil, false, fmt.Errorf("user is not authorized for this business line")
|
||||
}
|
||||
}
|
||||
if input.Topology == "" {
|
||||
input.Topology = "standalone"
|
||||
}
|
||||
if input.DataRoot == "" {
|
||||
input.DataRoot = "/data/postgresql"
|
||||
}
|
||||
target, err := getPostgreSQLTarget(ctx, s.awx, input.TargetID)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
payload := postgresqlDeliveryPayload{PostgreSQLDeliveryInput: input, TargetType: target.TargetType}
|
||||
raw, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
digest := sha256.Sum256(raw)
|
||||
task := model.DeliveryTask{ID: randomUUID(), BusinessLineID: input.BusinessLineID, RequestedBy: userID, Component: postgresqlServiceType, TargetType: target.TargetType, ServiceType: postgresqlServiceType, TargetID: input.TargetID, Namespace: input.Namespace, InstanceName: input.ClusterName, Status: model.TaskPending, ImmutablePayload: string(raw), PayloadHash: hex.EncodeToString(digest[:]), IdempotencyKey: idempotencyKey}
|
||||
if err := s.db.WithContext(ctx).Create(&task).Error; err != nil {
|
||||
if lookupErr := s.db.WithContext(ctx).Where("idempotency_key = ?", idempotencyKey).First(&existing).Error; lookupErr == nil {
|
||||
return &existing, true, nil
|
||||
}
|
||||
return nil, false, err
|
||||
}
|
||||
_ = s.db.WithContext(ctx).Create(&model.TaskEvent{TaskID: task.ID, ToState: model.TaskPending, Message: "postgresql delivery task created"}).Error
|
||||
return &task, false, nil
|
||||
}
|
||||
|
||||
func getPostgreSQLTarget(ctx context.Context, awx *AWXClient, templateID uint64) (DeliveryTarget, error) {
|
||||
template, err := awx.GetJobTemplate(ctx, templateID)
|
||||
if err != nil {
|
||||
return DeliveryTarget{}, fmt.Errorf("deployment target is unavailable: %w", err)
|
||||
}
|
||||
if !strings.Contains(strings.ToLower(template.Name+" "+template.Description), "postgresql") {
|
||||
return DeliveryTarget{}, fmt.Errorf("AWX job template %d is not a PostgreSQL target", templateID)
|
||||
}
|
||||
hosts, err := awx.ListInventoryHosts(ctx, template.Inventory)
|
||||
if err != nil {
|
||||
return DeliveryTarget{}, err
|
||||
}
|
||||
meta := targetMetadata{Topology: "standalone"}
|
||||
for _, host := range hosts {
|
||||
if host.Enabled {
|
||||
meta.Hosts = append(meta.Hosts, targetHost{Name: host.Name, IP: AWXHostIP(host)})
|
||||
}
|
||||
}
|
||||
raw, err := json.Marshal(meta)
|
||||
if err != nil {
|
||||
return DeliveryTarget{}, err
|
||||
}
|
||||
return DeliveryTarget{ID: template.ID, Name: template.Name, TargetType: "host_pool", AWXInventoryID: template.Inventory, AWXTemplateID: template.ID, Enabled: true, Metadata: string(raw)}, nil
|
||||
}
|
||||
|
||||
func selectPostgreSQLHosts(hosts []targetHost, requested []string, count int) ([]targetHost, error) {
|
||||
if len(requested) == 0 {
|
||||
if len(hosts) < count {
|
||||
return nil, fmt.Errorf("deployment target has %d hosts, but %d PostgreSQL instances are required", len(hosts), count)
|
||||
}
|
||||
return append([]targetHost(nil), hosts[:count]...), nil
|
||||
}
|
||||
byName := map[string]targetHost{}
|
||||
for _, host := range hosts {
|
||||
byName[host.Name] = host
|
||||
}
|
||||
selected := make([]targetHost, 0, len(requested))
|
||||
for _, name := range requested {
|
||||
host, ok := byName[name]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("target host %q is not in the candidate host pool", name)
|
||||
}
|
||||
selected = append(selected, host)
|
||||
}
|
||||
return selected, nil
|
||||
}
|
||||
|
||||
func (s *PostgreSQLDeliveryService) claimAndReserve(ctx context.Context) (*model.DeliveryTask, error) {
|
||||
var task model.DeliveryTask
|
||||
var payload postgresqlDeliveryPayload
|
||||
err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
if err := tx.Clauses(clause.Locking{Strength: "UPDATE", Options: "SKIP LOCKED"}).Where("status = ? AND service_type = ?", model.TaskPending, postgresqlServiceType).Order("created_at ASC").First(&task).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := json.Unmarshal([]byte(task.ImmutablePayload), &payload); err != nil {
|
||||
return s.common.transitionTx(tx, &task, model.TaskValidationFailed, "stored PostgreSQL deployment payload is invalid", "stored PostgreSQL deployment payload is invalid")
|
||||
}
|
||||
activeStates := []string{model.TaskValidating, model.TaskDispatching, model.TaskRunning, model.TaskCanceling}
|
||||
checks := []struct {
|
||||
query string
|
||||
args []any
|
||||
limit int
|
||||
}{
|
||||
{"status IN ?", []any{activeStates}, s.cfg.DeliveryGlobalLimit},
|
||||
{"status IN ? AND target_id = ?", []any{activeStates, task.TargetID}, s.cfg.DeliveryTargetLimit},
|
||||
{"status IN ? AND business_line_id = ?", []any{activeStates, task.BusinessLineID}, s.cfg.DeliveryBusinessLimit},
|
||||
}
|
||||
for _, check := range checks {
|
||||
if check.limit <= 0 {
|
||||
continue
|
||||
}
|
||||
var count int64
|
||||
if err := tx.Model(&model.DeliveryTask{}).Where(check.query, check.args...).Count(&count).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if count >= int64(check.limit) {
|
||||
return fmt.Errorf("defer: delivery concurrency limit reached")
|
||||
}
|
||||
}
|
||||
target, err := getPostgreSQLTarget(ctx, s.awx, task.TargetID)
|
||||
if err != nil {
|
||||
return s.common.transitionTx(tx, &task, model.TaskValidationFailed, err.Error(), err.Error())
|
||||
}
|
||||
meta := parseTargetMetadata(target.Metadata)
|
||||
nodes := 1
|
||||
if payload.Topology == "primary_replica" {
|
||||
nodes += payload.ReplicaCount
|
||||
}
|
||||
selected, err := selectPostgreSQLHosts(meta.Hosts, payload.TargetHosts, nodes)
|
||||
if err != nil {
|
||||
return s.common.transitionTx(tx, &task, model.TaskValidationFailed, err.Error(), err.Error())
|
||||
}
|
||||
quotaOK, err := checkPostgreSQLResourceQuota(tx, task.BusinessLineID, task.TargetID, payload, int64(nodes))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !quotaOK {
|
||||
return s.common.transitionTx(tx, &task, model.TaskValidationFailed, "resource quota is insufficient", "resource quota is insufficient")
|
||||
}
|
||||
var existingCluster model.PostgreSQLCluster
|
||||
if err := tx.Where("name = ?", payload.ClusterName).First(&existingCluster).Error; err == nil {
|
||||
return s.common.transitionTx(tx, &task, model.TaskValidationFailed, "PostgreSQL cluster name is already in use", "PostgreSQL cluster name is already in use")
|
||||
} else if !errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return err
|
||||
}
|
||||
cluster := model.PostgreSQLCluster{TaskID: task.ID, BusinessLineID: task.BusinessLineID, TargetID: task.TargetID, Name: payload.ClusterName, VersionMajor: payload.VersionMajor, Topology: payload.Topology, ReplicationMode: "async", FailoverMode: "manual", Status: "provisioning", BackupStatus: "not_configured", MonitoringStatus: "not_configured"}
|
||||
if err := tx.Create(&cluster).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
primaryPort := 0
|
||||
for index, host := range selected {
|
||||
var usedPorts []int
|
||||
if err := tx.Model(&model.PostgreSQLInstance{}).Where("hostname = ? AND status IN ?", host.Name, []string{"provisioning", "active", "quarantined"}).Pluck("port", &usedPorts).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
port, err := allocateReachablePostgreSQLPort(ctx, host.IP, usedPorts, postgresqlPortInUse)
|
||||
if err != nil {
|
||||
_ = tx.Delete(&cluster).Error
|
||||
return s.common.transitionTx(tx, &task, model.TaskValidationFailed, err.Error(), err.Error())
|
||||
}
|
||||
role := "replica"
|
||||
instanceID := fmt.Sprintf("%s-replica-%d", payload.ClusterName, index)
|
||||
upstream := payload.ClusterName + "-primary"
|
||||
slot := fmt.Sprintf("xinfra_%s_replica_%d", strings.ReplaceAll(payload.ClusterName, "-", "_"), index)
|
||||
if index == 0 {
|
||||
role = "primary"
|
||||
instanceID = payload.ClusterName + "-primary"
|
||||
upstream = ""
|
||||
slot = ""
|
||||
primaryPort = port
|
||||
}
|
||||
if payload.Topology == "standalone" {
|
||||
role = "standalone"
|
||||
instanceID = payload.ClusterName
|
||||
upstream = ""
|
||||
slot = ""
|
||||
}
|
||||
root := strings.TrimRight(payload.DataRoot, "/") + "/" + instanceID
|
||||
instance := model.PostgreSQLInstance{TaskID: task.ID, ClusterID: cluster.ID, BusinessLineID: task.BusinessLineID, TargetID: task.TargetID, InstanceID: instanceID, Hostname: host.Name, HostIP: host.IP, Port: port, DataDir: root + "/data", ConfigDir: root + "/conf", LogDir: root + "/log", SystemdUnit: "postgresql-xinfra@" + instanceID + ".service", VersionMajor: payload.VersionMajor, Role: role, HAComponentRole: "database", UpstreamInstanceID: upstream, ReplicationSlotName: slot, Status: "provisioning", BackupStatus: "not_configured", MonitoringStatus: "not_configured"}
|
||||
if err := tx.Create(&instance).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if index == 0 {
|
||||
cluster.PrimaryInstanceID = instance.ID
|
||||
}
|
||||
}
|
||||
if err := tx.Save(&cluster).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
reservation := model.ResourceReservation{TaskID: task.ID, BusinessLineID: task.BusinessLineID, TargetID: task.TargetID, CPUMilli: payload.CPUMilli * int64(nodes), MemoryMi: payload.MemoryMi * int64(nodes), StorageGi: payload.StorageGi * int64(nodes), InstanceCount: int64(nodes), Status: "reserved", ExpiresAt: time.Now().Add(time.Duration(s.cfg.ReservationTTLMinutes) * time.Minute)}
|
||||
if err := tx.Create(&reservation).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
hostNames := make([]string, 0, len(selected))
|
||||
for _, host := range selected {
|
||||
hostNames = append(hostNames, host.Name)
|
||||
}
|
||||
task.TargetHost = strings.Join(hostNames, ",")
|
||||
task.TargetHostIP = selected[0].IP
|
||||
task.PostgreSQLPort = primaryPort
|
||||
if err := tx.Model(&model.DeliveryTask{}).Where("id = ?", task.ID).Updates(map[string]any{"target_host": task.TargetHost, "target_host_ip": task.TargetHostIP, "postgresql_port": primaryPort}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
return s.common.transitionTx(tx, &task, model.TaskDispatching, "PostgreSQL resources, ports and directories reserved", "")
|
||||
})
|
||||
return &task, err
|
||||
}
|
||||
|
||||
func checkPostgreSQLResourceQuota(tx *gorm.DB, businessLineID, targetID uint64, payload postgresqlDeliveryPayload, nodes int64) (bool, error) {
|
||||
var quota model.ResourceQuota
|
||||
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Where("business_line_id = ? AND target_id = ?", businessLineID, targetID).First("a).Error; err != nil {
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return true, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
type totals struct{ CPU, Memory, Storage, Instances int64 }
|
||||
var mysqlUsed, postgresUsed, reserved totals
|
||||
if err := tx.Model(&model.ResourceUsage{}).Select("COALESCE(SUM(cpu_milli),0) cpu, COALESCE(SUM(memory_mi),0) memory, COALESCE(SUM(storage_gi),0) storage, COALESCE(SUM(instance_count),0) instances").Where("business_line_id = ? AND target_id = ? AND status = ?", businessLineID, targetID, "active").Scan(&mysqlUsed).Error; err != nil {
|
||||
return false, err
|
||||
}
|
||||
if err := tx.Model(&model.PostgreSQLResourceUsage{}).Select("COALESCE(SUM(cpu_milli),0) cpu, COALESCE(SUM(memory_mi),0) memory, COALESCE(SUM(storage_gi),0) storage, COUNT(*) instances").Where("business_line_id = ? AND target_id = ? AND status = ?", businessLineID, targetID, "active").Scan(&postgresUsed).Error; err != nil {
|
||||
return false, err
|
||||
}
|
||||
if err := tx.Model(&model.ResourceReservation{}).Select("COALESCE(SUM(cpu_milli),0) cpu, COALESCE(SUM(memory_mi),0) memory, COALESCE(SUM(storage_gi),0) storage, COALESCE(SUM(instance_count),0) instances").Where("business_line_id = ? AND target_id = ? AND status = ? AND expires_at > ?", businessLineID, targetID, "reserved", time.Now()).Scan(&reserved).Error; err != nil {
|
||||
return false, err
|
||||
}
|
||||
requestedCPU := payload.CPUMilli * nodes
|
||||
requestedMemory := payload.MemoryMi * nodes
|
||||
requestedStorage := payload.StorageGi * nodes
|
||||
return mysqlUsed.CPU+postgresUsed.CPU+reserved.CPU+requestedCPU <= quota.CPUMilli &&
|
||||
mysqlUsed.Memory+postgresUsed.Memory+reserved.Memory+requestedMemory <= quota.MemoryMi &&
|
||||
mysqlUsed.Storage+postgresUsed.Storage+reserved.Storage+requestedStorage <= quota.StorageGi &&
|
||||
mysqlUsed.Instances+postgresUsed.Instances+reserved.Instances+nodes <= quota.InstanceLimit, nil
|
||||
}
|
||||
|
||||
func (s *PostgreSQLDeliveryService) CreateExecution(ctx context.Context, task *model.DeliveryTask) error {
|
||||
var payload postgresqlDeliveryPayload
|
||||
if err := json.Unmarshal([]byte(task.ImmutablePayload), &payload); err != nil {
|
||||
return err
|
||||
}
|
||||
var instances []model.PostgreSQLInstance
|
||||
if err := s.db.WithContext(ctx).Where("task_id = ?", task.ID).Order("id ASC").Find(&instances).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
instanceVars := make(map[string]any, len(instances))
|
||||
for _, instance := range instances {
|
||||
instanceVars[instance.Hostname] = map[string]any{"instance_id": instance.InstanceID, "port": instance.Port, "data_dir": instance.DataDir, "config_dir": instance.ConfigDir, "log_dir": instance.LogDir, "role": instance.Role, "replication_slot": instance.ReplicationSlotName}
|
||||
}
|
||||
now := time.Now()
|
||||
execution := model.ExecutionJob{TaskID: task.ID, IdempotencyKey: task.IdempotencyKey, ExecutorJobID: "pending", Status: "launching", StartedAt: &now}
|
||||
if err := s.db.WithContext(ctx).Create(&execution).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
target, err := getPostgreSQLTarget(ctx, s.awx, task.TargetID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
extraVars := map[string]any{"task_id": task.ID, "payload_hash": task.PayloadHash, "target_hosts": task.TargetHost, "cluster_name": payload.ClusterName, "postgresql_version": payload.VersionMajor, "postgresql_port": task.PostgreSQLPort, "topology": payload.Topology, "replica_count": payload.ReplicaCount, "postgresql_instances": instanceVars, "memory_mb": payload.MemoryMi, "storage_gb": payload.StorageGi}
|
||||
if payload.MaxConnections > 0 {
|
||||
extraVars["max_connections"] = payload.MaxConnections
|
||||
}
|
||||
job, err := s.awx.Launch(ctx, target.AWXTemplateID, AWXLaunchRequest{InventoryID: target.AWXInventoryID, Limit: task.TargetHost, ExtraVars: extraVars})
|
||||
if err != nil {
|
||||
_ = s.db.WithContext(ctx).Model(&execution).Updates(map[string]any{"status": "failed", "finished_at": time.Now()})
|
||||
return err
|
||||
}
|
||||
return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
if err := tx.Model(&execution).Updates(map[string]any{"executor_job_id": fmt.Sprint(job.ID), "status": "running"}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
return s.common.transitionTx(tx, task, model.TaskRunning, "PostgreSQL AWX job started", "")
|
||||
})
|
||||
}
|
||||
|
||||
func (s *PostgreSQLDeliveryService) DispatchOnce(ctx context.Context) error {
|
||||
task, err := s.claimAndReserve(ctx)
|
||||
if err != nil {
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) || strings.HasPrefix(err.Error(), "defer:") {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if err := s.CreateExecution(ctx, task); err != nil {
|
||||
return s.fail(ctx, task.ID, model.TaskExecutionFailed, err.Error())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *PostgreSQLDeliveryService) PollOnce(ctx context.Context) error {
|
||||
var jobs []model.ExecutionJob
|
||||
if err := s.db.WithContext(ctx).Joins("JOIN delivery_tasks ON delivery_tasks.id = execution_jobs.task_id").Where("execution_jobs.status = ? AND delivery_tasks.service_type = ?", "running", postgresqlServiceType).Find(&jobs).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
for _, execution := range jobs {
|
||||
job, err := s.awx.GetJob(ctx, execution.ExecutorJobID)
|
||||
if err != nil {
|
||||
// A temporary AWX failure does not mean the deployment failed. Retry on
|
||||
// the next scheduler tick while the execution remains running.
|
||||
continue
|
||||
}
|
||||
switch strings.ToLower(job.Status) {
|
||||
case "pending", "waiting", "running", "new":
|
||||
continue
|
||||
case "canceled":
|
||||
_ = s.fail(ctx, execution.TaskID, model.TaskCanceled, "AWX job was canceled")
|
||||
case "successful":
|
||||
if err := s.complete(ctx, execution.TaskID); err != nil {
|
||||
_ = s.fail(ctx, execution.TaskID, model.TaskValidationFailed, err.Error())
|
||||
}
|
||||
default:
|
||||
_ = s.fail(ctx, execution.TaskID, model.TaskExecutionFailed, "AWX job finished with status "+job.Status)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *PostgreSQLDeliveryService) complete(ctx context.Context, taskID string) error {
|
||||
var task model.DeliveryTask
|
||||
if err := s.db.WithContext(ctx).First(&task, "id = ? AND service_type = ?", taskID, postgresqlServiceType).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
var payload postgresqlDeliveryPayload
|
||||
if err := json.Unmarshal([]byte(task.ImmutablePayload), &payload); err != nil {
|
||||
return err
|
||||
}
|
||||
var instances []model.PostgreSQLInstance
|
||||
if err := s.db.WithContext(ctx).Where("task_id = ?", taskID).Order("id ASC").Find(&instances).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if len(instances) == 0 {
|
||||
return fmt.Errorf("PostgreSQL task has no planned instances")
|
||||
}
|
||||
for _, instance := range instances {
|
||||
if err := postgresReady(ctx, net.JoinHostPort(instance.HostIP, fmt.Sprint(instance.Port))); err != nil {
|
||||
return fmt.Errorf("postgresql health check failed for %s: %w", instance.InstanceID, err)
|
||||
}
|
||||
}
|
||||
now := time.Now()
|
||||
return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
if err := s.common.transitionTx(tx, &task, model.TaskRegistering, "PostgreSQL health checks passed", ""); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, instance := range instances {
|
||||
if err := tx.Model(&instance).Updates(map[string]any{"status": "active", "version_full": instance.VersionMajor}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Create(&model.PostgreSQLResourceUsage{TaskID: task.ID, InstanceID: instance.ID, ClusterID: instance.ClusterID, BusinessLineID: instance.BusinessLineID, TargetID: instance.TargetID, CPUMilli: payload.CPUMilli, MemoryMi: payload.MemoryMi, StorageGi: payload.StorageGi, Port: instance.Port, DataDir: instance.DataDir, Status: "active"}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := tx.Model(&model.ResourceReservation{}).Where("task_id = ? AND status = ?", task.ID, "reserved").Update("status", "consumed").Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Model(&model.PostgreSQLCluster{}).Where("task_id = ?", task.ID).Update("status", "active").Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Model(&model.ExecutionJob{}).Where("task_id = ?", task.ID).Updates(map[string]any{"status": "successful", "finished_at": now}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
return s.common.transitionTx(tx, &task, model.TaskFinished, "PostgreSQL delivery completed and recorded in the PostgreSQL resource ledger", "")
|
||||
})
|
||||
}
|
||||
|
||||
func postgresReady(ctx context.Context, address string) error {
|
||||
dialer := net.Dialer{Timeout: 5 * time.Second}
|
||||
conn, err := dialer.DialContext(ctx, "tcp", address)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return conn.Close()
|
||||
}
|
||||
|
||||
func (s *PostgreSQLDeliveryService) fail(ctx context.Context, taskID, status, message string) error {
|
||||
return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
var task model.DeliveryTask
|
||||
if err := tx.First(&task, "id = ? AND service_type = ?", taskID, postgresqlServiceType).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.common.transitionTx(tx, &task, status, message, message); err != nil {
|
||||
return err
|
||||
}
|
||||
reservationStatus := "released"
|
||||
if status == model.TaskExecutionFailed || (status == model.TaskValidationFailed && strings.Contains(strings.ToLower(message), "health")) {
|
||||
reservationStatus = "quarantined"
|
||||
}
|
||||
if err := tx.Model(&model.ResourceReservation{}).Where("task_id = ? AND status = ?", taskID, "reserved").Update("status", reservationStatus).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
instanceStatus := "failed"
|
||||
if reservationStatus == "quarantined" {
|
||||
instanceStatus = "quarantined"
|
||||
}
|
||||
if err := tx.Model(&model.PostgreSQLInstance{}).Where("task_id = ? AND status = ?", taskID, "provisioning").Update("status", instanceStatus).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Model(&model.PostgreSQLCluster{}).Where("task_id = ? AND status = ?", taskID, "provisioning").Update("status", instanceStatus).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Model(&model.ExecutionJob{}).Where("task_id = ? AND status IN ?", taskID, []string{"launching", "running"}).Updates(map[string]any{"status": "failed", "finished_at": time.Now()}).Error
|
||||
})
|
||||
}
|
||||
|
||||
func (s *PostgreSQLDeliveryService) Run(ctx context.Context) {
|
||||
interval := time.Duration(s.cfg.DeliveryPollSeconds) * time.Second
|
||||
if interval < time.Second {
|
||||
interval = time.Second
|
||||
}
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
_ = s.DispatchOnce(ctx)
|
||||
_ = s.PollOnce(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestValidatePostgreSQLDeliveryInput(t *testing.T) {
|
||||
valid := PostgreSQLDeliveryInput{BusinessLineID: 1, TargetID: 2, Namespace: "team-a", ClusterName: "orders-pg", VersionMajor: "16", Topology: "standalone", CPUMilli: 500, MemoryMi: 2048, StorageGi: 20}
|
||||
if err := validatePostgreSQLDeliveryInput(valid, []string{"/data"}); err != nil {
|
||||
t.Fatalf("valid standalone input rejected: %v", err)
|
||||
}
|
||||
aliasVersion := valid
|
||||
aliasVersion.VersionMajor = ""
|
||||
aliasVersion.PostgreSQLVersion = "15"
|
||||
if err := validatePostgreSQLDeliveryInput(aliasVersion, []string{"/data"}); err != nil {
|
||||
t.Fatalf("postgresql_version alias rejected: %v", err)
|
||||
}
|
||||
replicated := valid
|
||||
replicated.VersionMajor = "15"
|
||||
replicated.Topology = "primary_replica"
|
||||
replicated.ReplicaCount = 2
|
||||
replicated.TargetHosts = []string{"pg-a", "pg-b", "pg-c"}
|
||||
replicated.DataRoot = "/data/postgresql"
|
||||
replicated.MaxConnections = 500
|
||||
if err := validatePostgreSQLDeliveryInput(replicated, []string{"/data"}); err != nil {
|
||||
t.Fatalf("valid primary_replica input rejected: %v", err)
|
||||
}
|
||||
for name, mutate := range map[string]func(*PostgreSQLDeliveryInput){
|
||||
"unsupported version": func(in *PostgreSQLDeliveryInput) { in.VersionMajor = "14" },
|
||||
"bad topology": func(in *PostgreSQLDeliveryInput) { in.Topology = "patroni" },
|
||||
"standalone replicas": func(in *PostgreSQLDeliveryInput) { in.ReplicaCount = 1 },
|
||||
"missing replicas": func(in *PostgreSQLDeliveryInput) { in.Topology = "primary_replica" },
|
||||
"too many replicas": func(in *PostgreSQLDeliveryInput) { in.Topology = "primary_replica"; in.ReplicaCount = 8 },
|
||||
"duplicate hosts": func(in *PostgreSQLDeliveryInput) {
|
||||
in.Topology = "primary_replica"
|
||||
in.ReplicaCount = 1
|
||||
in.TargetHosts = []string{"pg-a", "pg-a"}
|
||||
},
|
||||
"wrong host count": func(in *PostgreSQLDeliveryInput) {
|
||||
in.Topology = "primary_replica"
|
||||
in.ReplicaCount = 2
|
||||
in.TargetHosts = []string{"pg-a", "pg-b"}
|
||||
},
|
||||
"bad root": func(in *PostgreSQLDeliveryInput) { in.DataRoot = "/tmp/postgresql" },
|
||||
"unsupported root disk": func(in *PostgreSQLDeliveryInput) { in.DataRoot = "/disk1/postgresql" },
|
||||
} {
|
||||
input := valid
|
||||
mutate(&input)
|
||||
if err := validatePostgreSQLDeliveryInput(input, []string{"/data"}); err == nil {
|
||||
t.Errorf("%s was accepted", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAllocatePostgreSQLPort(t *testing.T) {
|
||||
if port, err := allocatePostgreSQLPort(0, nil); err != nil || port != postgresqlPortPoolStart {
|
||||
t.Fatalf("expected pool start %d, got %d err=%v", postgresqlPortPoolStart, port, err)
|
||||
}
|
||||
if port, err := allocatePostgreSQLPort(0, []int{15432, 15433}); err != nil || port != 15434 {
|
||||
t.Fatalf("expected 15434, got %d err=%v", port, err)
|
||||
}
|
||||
if _, err := allocatePostgreSQLPort(15432, []int{15432}); err == nil {
|
||||
t.Fatal("occupied requested port was accepted")
|
||||
}
|
||||
if _, err := allocatePostgreSQLPort(5432, nil); err == nil {
|
||||
t.Fatal("port outside the PostgreSQL pool was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAllocateReachablePostgreSQLPortSkipsListeningPorts(t *testing.T) {
|
||||
probed := []int{}
|
||||
port, err := allocateReachablePostgreSQLPort(context.Background(), "pg.example", nil, func(_ context.Context, host string, port int) bool {
|
||||
if host != "pg.example" {
|
||||
t.Fatalf("unexpected probe host %q", host)
|
||||
}
|
||||
probed = append(probed, port)
|
||||
return port == postgresqlPortPoolStart
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("allocate reachable port: %v", err)
|
||||
}
|
||||
if port != postgresqlPortPoolStart+1 {
|
||||
t.Fatalf("port = %d, want %d", port, postgresqlPortPoolStart+1)
|
||||
}
|
||||
if len(probed) != 2 || probed[0] != postgresqlPortPoolStart || probed[1] != postgresqlPortPoolStart+1 {
|
||||
t.Fatalf("unexpected probes: %v", probed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectPostgreSQLHosts(t *testing.T) {
|
||||
hosts := []targetHost{{Name: "pg-a"}, {Name: "pg-b"}, {Name: "pg-c"}}
|
||||
selected, err := selectPostgreSQLHosts(hosts, nil, 2)
|
||||
if err != nil || len(selected) != 2 || selected[0].Name != "pg-a" || selected[1].Name != "pg-b" {
|
||||
t.Fatalf("unexpected automatic selection: %+v err=%v", selected, err)
|
||||
}
|
||||
selected, err = selectPostgreSQLHosts(hosts, []string{"pg-c", "pg-a"}, 2)
|
||||
if err != nil || selected[0].Name != "pg-c" || selected[1].Name != "pg-a" {
|
||||
t.Fatalf("unexpected pinned selection: %+v err=%v", selected, err)
|
||||
}
|
||||
if _, err := selectPostgreSQLHosts(hosts, []string{"missing"}, 1); err == nil {
|
||||
t.Fatal("host outside the pool was accepted")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user