feat: wire delivery result and infra data backends

This commit is contained in:
mac
2026-07-28 17:56:00 +08:00
parent 897670b721
commit f7be87c7d9
14 changed files with 1468 additions and 34 deletions
+321 -9
View File
@@ -3,8 +3,11 @@ package service
import (
"bytes"
"context"
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
@@ -60,6 +63,8 @@ type MySQLDeliveryInput struct {
LongQueryTime float64 `json:"long_query_time"`
BinlogExpireLogsSeconds int64 `json:"binlog_expire_logs_seconds"`
MaxBinlogSize string `json:"max_binlog_size"`
MySQLRootPassword string `json:"mysql_root_password"`
MySQLAdminPassword string `json:"mysql_admin_password"`
}
type deliveryPayload struct {
@@ -110,6 +115,25 @@ type DeliveryTaskSnapshot struct {
Events []model.TaskEvent `json:"events"`
}
type MySQLServiceLedgerItem struct {
Name string `json:"name"`
Datacenter string `json:"datacenter"`
BusinessTag string `json:"business_tag"`
Instances int `json:"instances"`
Healthy int `json:"healthy"`
Address string `json:"address"`
Status string `json:"status"`
StatusClass string `json:"status_class"`
Version string `json:"version"`
Namespace string `json:"namespace"`
}
type DeploymentCredentialView struct {
Username string `json:"username"`
Host string `json:"host"`
Password string `json:"password"`
}
// targetMetadata describes the native VM候选节点池以及部署形态,由 AWX inventory hosts 动态组装。
type targetMetadata struct {
Topology string `json:"topology"`
@@ -211,6 +235,52 @@ func bytesToGi(bytes int64) int64 {
return bytes / 1073741824
}
func credentialKey(secret string) []byte {
sum := sha256.Sum256([]byte(strings.TrimSpace(secret)))
return sum[:]
}
func encryptCredential(secret, plain string) (string, string, error) {
block, err := aes.NewCipher(credentialKey(secret))
if err != nil {
return "", "", err
}
gcm, err := cipher.NewGCM(block)
if err != nil {
return "", "", err
}
nonce := make([]byte, gcm.NonceSize())
if _, err := rand.Read(nonce); err != nil {
return "", "", err
}
ciphertext := gcm.Seal(nil, nonce, []byte(plain), nil)
return base64.StdEncoding.EncodeToString(ciphertext), base64.StdEncoding.EncodeToString(nonce), nil
}
func decryptCredential(secret, ciphertextValue, nonceValue string) (string, error) {
ciphertext, err := base64.StdEncoding.DecodeString(ciphertextValue)
if err != nil {
return "", err
}
nonce, err := base64.StdEncoding.DecodeString(nonceValue)
if err != nil {
return "", err
}
block, err := aes.NewCipher(credentialKey(secret))
if err != nil {
return "", err
}
gcm, err := cipher.NewGCM(block)
if err != nil {
return "", err
}
plain, err := gcm.Open(nil, nonce, ciphertext, nil)
if err != nil {
return "", err
}
return string(plain), nil
}
// firstFreeHost 返回候选池中非失败任务数未达单机实例上限的第一个节点;
// limit < 1 时按 1 兜底(退化为旧的一机一实例语义)。顺序遍历天然形成"先摊开、摊满一轮再叠加"。
func firstFreeHost(hosts []targetHost, occupied []string, limit int) *targetHost {
@@ -406,6 +476,21 @@ func (s *DeliveryService) CreateTask(ctx context.Context, userID uint64, isAdmin
if err := validateDeliveryInput(input, s.cfg.DeliveryDataDisks); err != nil {
return nil, false, err
}
credentialInput := map[string]string{
"root@localhost": strings.TrimSpace(input.MySQLRootPassword),
"xinfra_admin@%": strings.TrimSpace(input.MySQLAdminPassword),
}
hasCredentialInput := credentialInput["root@localhost"] != "" || credentialInput["xinfra_admin@%"] != ""
if hasCredentialInput {
if credentialInput["root@localhost"] == "" || credentialInput["xinfra_admin@%"] == "" {
return nil, false, fmt.Errorf("mysql_root_password and mysql_admin_password must be provided together")
}
if len(credentialInput["root@localhost"]) < 16 || len(credentialInput["xinfra_admin@%"]) < 16 {
return nil, false, fmt.Errorf("mysql passwords must be at least 16 characters")
}
}
input.MySQLRootPassword = ""
input.MySQLAdminPassword = ""
var existing model.DeliveryTask
if err := s.db.WithContext(ctx).Where("idempotency_key = ?", idempotencyKey).First(&existing).Error; err == nil {
@@ -457,6 +542,7 @@ func (s *DeliveryService) CreateTask(ctx context.Context, userID uint64, isAdmin
ID: randomUUID(),
BusinessLineID: input.BusinessLineID,
RequestedBy: userID,
Component: "mysql",
TargetType: target.TargetType,
TargetID: target.ID,
Namespace: input.Namespace,
@@ -472,6 +558,11 @@ func (s *DeliveryService) CreateTask(ctx context.Context, userID uint64, isAdmin
}
return nil, false, err
}
if hasCredentialInput {
if err := s.storeDeploymentCredentials(ctx, &task, credentialInput, "pending"); err != nil {
return nil, false, err
}
}
_ = s.db.WithContext(ctx).Create(&model.TaskEvent{TaskID: task.ID, ToState: model.TaskPending, Message: "delivery task created"}).Error
return &task, false, nil
}
@@ -620,6 +711,137 @@ func (s *DeliveryService) ListTasks(ctx context.Context, userID uint64, isAdmin
return tasks, query.Find(&tasks).Error
}
func (s *DeliveryService) ListMySQLServiceLedger(ctx context.Context, userID uint64, isAdmin bool, businessLineID uint64) ([]MySQLServiceLedgerItem, error) {
if businessLineID == 0 {
return nil, fmt.Errorf("business_line_id is required")
}
if !isAdmin {
var count int64
if err := s.db.WithContext(ctx).Model(&model.BusinessLineUser{}).
Where("business_line_id = ? AND user_id = ?", businessLineID, userID).
Count(&count).Error; err != nil {
return nil, err
}
if count == 0 {
return nil, fmt.Errorf("user is not authorized for this business line")
}
}
var businessLine model.BusinessLine
if err := s.db.WithContext(ctx).First(&businessLine, "id = ?", businessLineID).Error; err != nil {
return nil, err
}
var instances []model.DeploymentResult
if err := s.db.WithContext(ctx).
Where("business_line_id = ? AND component = ? AND service_type = ? AND status = ?", businessLineID, "mysql", "database", "active").
Order("created_at DESC").
Find(&instances).Error; err != nil {
return nil, err
}
taskIDs := make([]string, 0, len(instances))
for _, instance := range instances {
taskIDs = append(taskIDs, instance.TaskID)
}
taskStatuses := map[string]string{}
if len(taskIDs) > 0 {
var tasks []model.DeliveryTask
if err := s.db.WithContext(ctx).Select("id", "status").Where("id IN ?", taskIDs).Find(&tasks).Error; err != nil {
return nil, err
}
for _, task := range tasks {
taskStatuses[task.ID] = task.Status
}
}
items := make([]MySQLServiceLedgerItem, 0, len(instances))
for _, instance := range instances {
status := "健康"
statusClass := "ok"
healthy := 1
if taskStatuses[instance.TaskID] == model.TaskRegisterFailed {
status = "注册异常"
statusClass = "warn"
}
if instance.Host == "" || instance.Port == 0 {
status = "部分异常"
statusClass = "warn"
healthy = 0
}
items = append(items, MySQLServiceLedgerItem{
Name: instance.InstanceName,
Datacenter: instance.NodeName,
BusinessTag: businessLine.Name,
Instances: 1,
Healthy: healthy,
Address: fmt.Sprintf("%s:%d", instance.Host, instance.Port),
Status: status,
StatusClass: statusClass,
Version: instance.Version,
Namespace: instance.Namespace,
})
}
return items, nil
}
func (s *DeliveryService) storeDeploymentCredentials(ctx context.Context, task *model.DeliveryTask, credentials map[string]string, status string) error {
for key, password := range credentials {
username, host, ok := strings.Cut(key, "@")
if !ok || strings.TrimSpace(username) == "" {
return fmt.Errorf("invalid credential account %q", key)
}
ciphertext, nonce, err := encryptCredential(s.cfg.DeliveryCredentialSecret, password)
if err != nil {
return err
}
item := model.DeploymentCredential{
TaskID: task.ID,
BusinessLineID: task.BusinessLineID,
Component: task.Component,
InstanceName: task.InstanceName,
Username: strings.TrimSpace(username),
AccountHost: strings.TrimSpace(host),
Ciphertext: ciphertext,
Nonce: nonce,
Status: status,
Source: "user_input",
}
if item.Component == "" {
item.Component = "mysql"
}
if err := s.db.WithContext(ctx).Create(&item).Error; err != nil {
return err
}
}
return nil
}
func (s *DeliveryService) deploymentCredentialVars(ctx context.Context, taskID string) (map[string]string, error) {
var items []model.DeploymentCredential
if err := s.db.WithContext(ctx).
Where("task_id = ? AND status IN ?", taskID, []string{"pending", "available"}).
Find(&items).Error; err != nil {
return nil, err
}
if len(items) == 0 {
return map[string]string{}, nil
}
values := map[string]string{}
for _, item := range items {
password, err := decryptCredential(s.cfg.DeliveryCredentialSecret, item.Ciphertext, item.Nonce)
if err != nil {
return nil, err
}
switch item.Username + "@" + item.AccountHost {
case "root@localhost":
values["mysql_root_password"] = password
case "xinfra_admin@%":
values["mysql_admin_password"] = password
}
}
if values["mysql_root_password"] == "" || values["mysql_admin_password"] == "" {
return nil, fmt.Errorf("deployment credentials are missing for task %s", taskID)
}
return values, nil
}
func (s *DeliveryService) GetTask(ctx context.Context, taskID string, userID uint64, isAdmin bool) (*model.DeliveryTask, []model.TaskEvent, error) {
query := s.db.WithContext(ctx).Where("id = ?", taskID)
if !isAdmin {
@@ -636,6 +858,55 @@ func (s *DeliveryService) GetTask(ctx context.Context, taskID string, userID uin
return &task, events, nil
}
func (s *DeliveryService) RevealDeploymentCredentials(ctx context.Context, taskID string, userID uint64, isAdmin bool) ([]DeploymentCredentialView, error) {
task, _, err := s.GetTask(ctx, taskID, userID, isAdmin)
if err != nil {
return nil, err
}
if task.Status != model.TaskFinished && task.Status != model.TaskRegisterFailed {
return nil, fmt.Errorf("task credentials are available only after a successful deployment")
}
var out []DeploymentCredentialView
err = s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
var credentials []model.DeploymentCredential
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).
Where("task_id = ? AND status = ?", taskID, "available").
Order("id ASC").
Find(&credentials).Error; err != nil {
return err
}
if len(credentials) == 0 {
return gorm.ErrRecordNotFound
}
for _, credential := range credentials {
password, err := decryptCredential(s.cfg.DeliveryCredentialSecret, credential.Ciphertext, credential.Nonce)
if err != nil {
return err
}
out = append(out, DeploymentCredentialView{
Username: credential.Username,
Host: credential.AccountHost,
Password: password,
})
}
ids := make([]uint64, 0, len(credentials))
for _, credential := range credentials {
ids = append(ids, credential.ID)
}
now := time.Now()
return tx.Model(&model.DeploymentCredential{}).Where("id IN ?", ids).Updates(map[string]any{
"status": "viewed",
"viewed_by": userID,
"viewed_at": now,
"updated_at": now,
}).Error
})
if err != nil {
return nil, err
}
return out, nil
}
func (s *DeliveryService) SubscribeTask(taskID string) (<-chan DeliveryTaskSnapshot, func()) {
ch := make(chan DeliveryTaskSnapshot, 8)
s.streamMu.Lock()
@@ -789,7 +1060,9 @@ func (s *DeliveryService) claimAndReserve(ctx context.Context) (*model.DeliveryT
return err
}
var instancePorts []int
if err := tx.Model(&model.MySQLInstance{}).Where("node_name = ? AND status = ?", host.Name, "active").Pluck("port", &instancePorts).Error; err != nil {
if err := tx.Model(&model.DeploymentResult{}).
Where("component = ? AND service_type = ? AND node_name = ? AND status = ?", "mysql", "database", host.Name, "active").
Pluck("port", &instancePorts).Error; err != nil {
return err
}
port, portErr := allocatePort(payload.MySQLPort, append(usedPorts, instancePorts...))
@@ -887,6 +1160,14 @@ func randomUUID() string {
return fmt.Sprintf("%08x-%04x-%04x-%04x-%012x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:16])
}
func mustJSON(value any) []byte {
raw, err := json.Marshal(value)
if err != nil {
return []byte(`{}`)
}
return raw
}
func mysqlReady(ctx context.Context, address string) error {
dialer := net.Dialer{Timeout: 5 * time.Second}
conn, err := dialer.DialContext(ctx, "tcp", address)
@@ -952,6 +1233,14 @@ func (s *DeliveryService) CreateExecution(ctx context.Context, taskID, payloadHa
return nil, false, err
}
extraVars := deliveryExtraVars(&task, payload, meta)
credentialVars, err := s.deploymentCredentialVars(ctx, task.ID)
if err != nil {
_ = s.db.WithContext(ctx).Model(&execution).Updates(map[string]any{"status": "failed", "finished_at": time.Now()})
return nil, false, err
}
for key, value := range credentialVars {
extraVars[key] = value
}
extraVars["delivery_callback_url"] = s.deliveryCallbackURL(task.ID)
extraVars["delivery_callback_token"] = s.cfg.AWXWebhookToken
job, err := s.awx.Launch(ctx, target.AWXTemplateID, AWXLaunchRequest{InventoryID: target.AWXInventoryID, Limit: task.TargetHost, ExtraVars: extraVars})
@@ -1302,11 +1591,31 @@ func (s *DeliveryService) completeTask(ctx context.Context, taskID string) error
if err := s.transitionTx(tx, &task, model.TaskRegistering, "AWX succeeded and MySQL health check passed", ""); err != nil {
return err
}
instance := model.MySQLInstance{TaskID: task.ID, BusinessLineID: task.BusinessLineID, TargetID: task.TargetID, Namespace: payload.Namespace, Name: payload.InstanceName, NodeName: task.TargetHost, Host: task.TargetHostIP, Port: task.MySQLPort, Version: payload.MySQLVersion, Status: "active"}
if err := tx.Create(&instance).Error; err != nil {
result := model.DeploymentResult{
TaskID: task.ID,
BusinessLineID: task.BusinessLineID,
Component: task.Component,
ServiceType: "database",
InstanceName: payload.InstanceName,
Namespace: payload.Namespace,
TargetID: task.TargetID,
NodeName: task.TargetHost,
Host: task.TargetHostIP,
Port: task.MySQLPort,
Version: payload.MySQLVersion,
Status: "active",
Metadata: string(mustJSON(map[string]any{"data_disk": payload.DataDisk})),
}
if result.Component == "" {
result.Component = "mysql"
}
if err := tx.Create(&result).Error; err != nil {
return err
}
if err := tx.Create(&model.ResourceUsage{TaskID: task.ID, InstanceID: instance.ID, BusinessLineID: task.BusinessLineID, TargetID: task.TargetID, CPUMilli: payload.CPUMilli, MemoryMi: payload.MemoryMi, StorageGi: payload.StorageGi, InstanceCount: 1, Status: "active"}).Error; err != nil {
if err := tx.Model(&model.DeploymentCredential{}).Where("task_id = ? AND status = ?", task.ID, "pending").Updates(map[string]any{"deployment_result_id": result.ID, "status": "available"}).Error; err != nil {
return err
}
if err := tx.Create(&model.ResourceUsage{TaskID: task.ID, InstanceID: result.ID, BusinessLineID: task.BusinessLineID, TargetID: task.TargetID, CPUMilli: payload.CPUMilli, MemoryMi: payload.MemoryMi, StorageGi: payload.StorageGi, InstanceCount: 1, Status: "active"}).Error; err != nil {
return err
}
if err := tx.Model(&model.ResourceReservation{}).Where("task_id = ? AND status = ?", task.ID, "reserved").Update("status", "consumed").Error; err != nil {
@@ -1359,11 +1668,11 @@ func (s *DeliveryService) RegisterCloudDM(ctx context.Context, taskID string) er
if s.cfg.CloudDMRegisterURL == "" {
return nil
}
var instance model.MySQLInstance
if err := s.db.WithContext(ctx).Where("task_id = ?", taskID).First(&instance).Error; err != nil {
var instance model.DeploymentResult
if err := s.db.WithContext(ctx).Where("task_id = ? AND component = ? AND service_type = ?", taskID, "mysql", "database").First(&instance).Error; err != nil {
return err
}
body := map[string]any{"name": instance.Name, "host": instance.Host, "port": instance.Port, "username": "root", "database_type": "mysql"}
body := map[string]any{"name": instance.InstanceName, "host": instance.Host, "port": instance.Port, "username": "root", "database_type": "mysql"}
raw, _ := json.Marshal(body)
req, err := http.NewRequestWithContext(ctx, http.MethodPost, s.cfg.CloudDMRegisterURL, bytes.NewReader(raw))
if err != nil {
@@ -1509,7 +1818,7 @@ func (s *DeliveryService) AcknowledgeRollbackRelease(ctx context.Context, taskID
if task.Status != model.TaskRollbackFailed {
return fmt.Errorf("task %s is in state %q and cannot acknowledge rollback release", taskID, task.Status)
}
if err := tx.Model(&model.MySQLInstance{}).Where("task_id = ? AND status = ?", taskID, "active").Updates(map[string]any{"status": "rollback_acknowledged", "updated_at": now}).Error; err != nil {
if err := tx.Model(&model.DeploymentResult{}).Where("task_id = ? AND component = ? AND service_type = ? AND status = ?", taskID, "mysql", "database", "active").Updates(map[string]any{"status": "rollback_acknowledged", "updated_at": now}).Error; err != nil {
return err
}
if err := tx.Model(&model.ResourceUsage{}).Where("task_id = ? AND status = ?", taskID, "active").Updates(map[string]any{"status": "released", "released_at": now, "updated_at": now}).Error; err != nil {
@@ -1627,7 +1936,10 @@ func (s *DeliveryService) completeRollback(ctx context.Context, taskID string) e
if task.Status != model.TaskRollingBack {
return fmt.Errorf("task %s is in state %q, cannot complete rollback", taskID, task.Status)
}
if err := tx.Model(&model.MySQLInstance{}).Where("task_id = ?", taskID).Updates(map[string]any{"status": "rolled_back", "updated_at": now}).Error; err != nil {
if err := tx.Model(&model.DeploymentResult{}).Where("task_id = ?", taskID).Updates(map[string]any{"status": "rolled_back", "updated_at": now}).Error; err != nil {
return err
}
if err := tx.Unscoped().Where("task_id = ?", taskID).Delete(&model.DeploymentCredential{}).Error; err != nil {
return err
}
if err := tx.Model(&model.ResourceUsage{}).Where("task_id = ? AND status = ?", taskID, "active").Updates(map[string]any{"status": "released", "released_at": now, "updated_at": now}).Error; err != nil {