feat(delivery): support multi-instance scheduling on a single host

- Per-host instance limit (DELIVERY_HOST_INSTANCE_LIMIT, default 4)
- Optional target_host to pin a host from the candidate pool
- Regenerate swagger docs
This commit is contained in:
Hungerdream
2026-07-27 15:19:44 +08:00
parent bfb4f0df4d
commit 41b2f45420
7 changed files with 355 additions and 439 deletions
+2
View File
@@ -22,6 +22,8 @@ DELIVERY_RESERVATION_TTL_MINUTES=120
DELIVERY_GLOBAL_LIMIT=2 DELIVERY_GLOBAL_LIMIT=2
DELIVERY_TARGET_LIMIT=2 DELIVERY_TARGET_LIMIT=2
DELIVERY_BUSINESS_LIMIT=1 DELIVERY_BUSINESS_LIMIT=1
# 单机 MySQL 实例数上限(同机多实例,容量由配额 + playbook 实机守卫兜底)
DELIVERY_HOST_INSTANCE_LIMIT=4
# 数据盘挂载点白名单(逗号分隔,第一项为默认值) # 数据盘挂载点白名单(逗号分隔,第一项为默认值)
DELIVERY_DATA_DISKS=/data DELIVERY_DATA_DISKS=/data
AWX_BASE_URL= AWX_BASE_URL=
+61 -118
View File
@@ -152,7 +152,7 @@ const docTemplate = `{
"BearerAuth": [] "BearerAuth": []
} }
], ],
"description": "返回所有已启用的部署目标(如 k8s 集群、主机池)", "description": "从 AWX 动态返回可用 Job Template 及其 Inventory hosts",
"produces": [ "produces": [
"application/json" "application/json"
], ],
@@ -160,6 +160,14 @@ const docTemplate = `{
"delivery" "delivery"
], ],
"summary": "获取可用部署目标", "summary": "获取可用部署目标",
"parameters": [
{
"type": "string",
"description": "组件过滤,例如 mysql",
"name": "component",
"in": "query"
}
],
"responses": { "responses": {
"200": { "200": {
"description": "items: 部署目标数组", "description": "items: 部署目标数组",
@@ -176,64 +184,6 @@ const docTemplate = `{
} }
} }
} }
},
"post": {
"security": [
{
"BearerAuth": []
}
],
"description": "管理员创建新的部署目标(目前仅支持 k8s 类型)",
"consumes": [
"application/json"
],
"produces": [
"application/json"
],
"tags": [
"delivery"
],
"summary": "创建部署目标",
"parameters": [
{
"description": "目标配置",
"name": "body",
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/handler.targetPayload"
}
}
],
"responses": {
"201": {
"description": "目标已创建",
"schema": {
"$ref": "#/definitions/model.DeploymentTarget"
}
},
"400": {
"description": "参数错误",
"schema": {
"type": "object",
"additionalProperties": true
}
},
"401": {
"description": "未授权",
"schema": {
"type": "object",
"additionalProperties": true
}
},
"409": {
"description": "名称冲突",
"schema": {
"type": "object",
"additionalProperties": true
}
}
}
} }
}, },
"/auth/api/v1/delivery/tasks": { "/auth/api/v1/delivery/tasks": {
@@ -254,6 +204,7 @@ const docTemplate = `{
"parameters": [ "parameters": [
{ {
"type": "integer", "type": "integer",
"format": "int64",
"description": "业务线 ID 过滤", "description": "业务线 ID 过滤",
"name": "business_line_id", "name": "business_line_id",
"in": "query" "in": "query"
@@ -689,65 +640,6 @@ const docTemplate = `{
} }
} }
}, },
"handler.targetPayload": {
"type": "object",
"required": [
"awx_inventory_id",
"awx_template_id",
"name",
"target_type"
],
"properties": {
"awx_inventory_id": {
"type": "integer"
},
"awx_template_id": {
"type": "integer"
},
"metadata": {
"type": "object",
"additionalProperties": {}
},
"name": {
"type": "string"
},
"target_type": {
"type": "string"
}
}
},
"model.DeploymentTarget": {
"type": "object",
"properties": {
"awx_inventory_id": {
"type": "integer"
},
"awx_template_id": {
"type": "integer"
},
"created_at": {
"type": "string"
},
"enabled": {
"type": "boolean"
},
"id": {
"type": "integer"
},
"metadata": {
"type": "string"
},
"name": {
"type": "string"
},
"target_type": {
"type": "string"
},
"updated_at": {
"type": "string"
}
}
},
"model.ResourceQuota": { "model.ResourceQuota": {
"type": "object", "type": "object",
"properties": { "properties": {
@@ -816,18 +708,55 @@ const docTemplate = `{
"target_id" "target_id"
], ],
"properties": { "properties": {
"binlog_expire_logs_seconds": {
"type": "integer"
},
"business_line_id": { "business_line_id": {
"type": "integer" "type": "integer"
}, },
"character_set": {
"type": "string"
},
"collation": {
"type": "string"
},
"cpu_milli": { "cpu_milli": {
"type": "integer" "type": "integer"
}, },
"data_disk": {
"type": "string"
},
"innodb_flush_log_at_trx_commit": {
"type": "integer"
},
"innodb_io_capacity": {
"type": "integer"
},
"innodb_redo_log_capacity": {
"type": "string"
},
"instance_name": { "instance_name": {
"type": "string" "type": "string"
}, },
"long_query_time": {
"type": "number"
},
"lower_case_table_names": {
"type": "integer"
},
"max_binlog_size": {
"type": "string"
},
"max_connections": {
"description": "高级参数(选填,零值视为未设置)",
"type": "string"
},
"memory_mi": { "memory_mi": {
"type": "integer" "type": "integer"
}, },
"mysql_port": {
"type": "integer"
},
"mysql_version": { "mysql_version": {
"type": "string" "type": "string"
}, },
@@ -837,8 +766,22 @@ const docTemplate = `{
"storage_gi": { "storage_gi": {
"type": "integer" "type": "integer"
}, },
"sync_binlog": {
"type": "integer"
},
"target_host": {
"description": "调度控制(选填):点名候选池内主机跳过自动选机,端口/配额/实机守卫照常执行",
"type": "string"
},
"target_id": { "target_id": {
"type": "integer" "type": "integer"
},
"timezone": {
"description": "数据库配置(选填,缺省由 playbook 基线兜底)",
"type": "string"
},
"topology": {
"type": "string"
} }
} }
} }
+61 -118
View File
@@ -145,7 +145,7 @@
"BearerAuth": [] "BearerAuth": []
} }
], ],
"description": "返回所有已启用的部署目标(如 k8s 集群、主机池)", "description": "从 AWX 动态返回可用 Job Template 及其 Inventory hosts",
"produces": [ "produces": [
"application/json" "application/json"
], ],
@@ -153,6 +153,14 @@
"delivery" "delivery"
], ],
"summary": "获取可用部署目标", "summary": "获取可用部署目标",
"parameters": [
{
"type": "string",
"description": "组件过滤,例如 mysql",
"name": "component",
"in": "query"
}
],
"responses": { "responses": {
"200": { "200": {
"description": "items: 部署目标数组", "description": "items: 部署目标数组",
@@ -169,64 +177,6 @@
} }
} }
} }
},
"post": {
"security": [
{
"BearerAuth": []
}
],
"description": "管理员创建新的部署目标(目前仅支持 k8s 类型)",
"consumes": [
"application/json"
],
"produces": [
"application/json"
],
"tags": [
"delivery"
],
"summary": "创建部署目标",
"parameters": [
{
"description": "目标配置",
"name": "body",
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/handler.targetPayload"
}
}
],
"responses": {
"201": {
"description": "目标已创建",
"schema": {
"$ref": "#/definitions/model.DeploymentTarget"
}
},
"400": {
"description": "参数错误",
"schema": {
"type": "object",
"additionalProperties": true
}
},
"401": {
"description": "未授权",
"schema": {
"type": "object",
"additionalProperties": true
}
},
"409": {
"description": "名称冲突",
"schema": {
"type": "object",
"additionalProperties": true
}
}
}
} }
}, },
"/auth/api/v1/delivery/tasks": { "/auth/api/v1/delivery/tasks": {
@@ -247,6 +197,7 @@
"parameters": [ "parameters": [
{ {
"type": "integer", "type": "integer",
"format": "int64",
"description": "业务线 ID 过滤", "description": "业务线 ID 过滤",
"name": "business_line_id", "name": "business_line_id",
"in": "query" "in": "query"
@@ -682,65 +633,6 @@
} }
} }
}, },
"handler.targetPayload": {
"type": "object",
"required": [
"awx_inventory_id",
"awx_template_id",
"name",
"target_type"
],
"properties": {
"awx_inventory_id": {
"type": "integer"
},
"awx_template_id": {
"type": "integer"
},
"metadata": {
"type": "object",
"additionalProperties": {}
},
"name": {
"type": "string"
},
"target_type": {
"type": "string"
}
}
},
"model.DeploymentTarget": {
"type": "object",
"properties": {
"awx_inventory_id": {
"type": "integer"
},
"awx_template_id": {
"type": "integer"
},
"created_at": {
"type": "string"
},
"enabled": {
"type": "boolean"
},
"id": {
"type": "integer"
},
"metadata": {
"type": "string"
},
"name": {
"type": "string"
},
"target_type": {
"type": "string"
},
"updated_at": {
"type": "string"
}
}
},
"model.ResourceQuota": { "model.ResourceQuota": {
"type": "object", "type": "object",
"properties": { "properties": {
@@ -809,18 +701,55 @@
"target_id" "target_id"
], ],
"properties": { "properties": {
"binlog_expire_logs_seconds": {
"type": "integer"
},
"business_line_id": { "business_line_id": {
"type": "integer" "type": "integer"
}, },
"character_set": {
"type": "string"
},
"collation": {
"type": "string"
},
"cpu_milli": { "cpu_milli": {
"type": "integer" "type": "integer"
}, },
"data_disk": {
"type": "string"
},
"innodb_flush_log_at_trx_commit": {
"type": "integer"
},
"innodb_io_capacity": {
"type": "integer"
},
"innodb_redo_log_capacity": {
"type": "string"
},
"instance_name": { "instance_name": {
"type": "string" "type": "string"
}, },
"long_query_time": {
"type": "number"
},
"lower_case_table_names": {
"type": "integer"
},
"max_binlog_size": {
"type": "string"
},
"max_connections": {
"description": "高级参数(选填,零值视为未设置)",
"type": "string"
},
"memory_mi": { "memory_mi": {
"type": "integer" "type": "integer"
}, },
"mysql_port": {
"type": "integer"
},
"mysql_version": { "mysql_version": {
"type": "string" "type": "string"
}, },
@@ -830,8 +759,22 @@
"storage_gi": { "storage_gi": {
"type": "integer" "type": "integer"
}, },
"sync_binlog": {
"type": "integer"
},
"target_host": {
"description": "调度控制(选填):点名候选池内主机跳过自动选机,端口/配额/实机守卫照常执行",
"type": "string"
},
"target_id": { "target_id": {
"type": "integer" "type": "integer"
},
"timezone": {
"description": "数据库配置(选填,缺省由 playbook 基线兜底)",
"type": "string"
},
"topology": {
"type": "string"
} }
} }
} }
+42 -79
View File
@@ -22,46 +22,6 @@ definitions:
- storage_gi - storage_gi
- target_id - target_id
type: object type: object
handler.targetPayload:
properties:
awx_inventory_id:
type: integer
awx_template_id:
type: integer
metadata:
additionalProperties: {}
type: object
name:
type: string
target_type:
type: string
required:
- awx_inventory_id
- awx_template_id
- name
- target_type
type: object
model.DeploymentTarget:
properties:
awx_inventory_id:
type: integer
awx_template_id:
type: integer
created_at:
type: string
enabled:
type: boolean
id:
type: integer
metadata:
type: string
name:
type: string
target_type:
type: string
updated_at:
type: string
type: object
model.ResourceQuota: model.ResourceQuota:
properties: properties:
business_line_id: business_line_id:
@@ -101,22 +61,57 @@ definitions:
type: object type: object
service.MySQLDeliveryInput: service.MySQLDeliveryInput:
properties: properties:
binlog_expire_logs_seconds:
type: integer
business_line_id: business_line_id:
type: integer type: integer
character_set:
type: string
collation:
type: string
cpu_milli: cpu_milli:
type: integer type: integer
data_disk:
type: string
innodb_flush_log_at_trx_commit:
type: integer
innodb_io_capacity:
type: integer
innodb_redo_log_capacity:
type: string
instance_name: instance_name:
type: string type: string
long_query_time:
type: number
lower_case_table_names:
type: integer
max_binlog_size:
type: string
max_connections:
description: 高级参数(选填,零值视为未设置)
type: string
memory_mi: memory_mi:
type: integer type: integer
mysql_port:
type: integer
mysql_version: mysql_version:
type: string type: string
namespace: namespace:
type: string type: string
storage_gi: storage_gi:
type: integer type: integer
sync_binlog:
type: integer
target_host:
description: 调度控制(选填):点名候选池内主机跳过自动选机,端口/配额/实机守卫照常执行
type: string
target_id: target_id:
type: integer type: integer
timezone:
description: 数据库配置(选填,缺省由 playbook 基线兜底)
type: string
topology:
type: string
required: required:
- business_line_id - business_line_id
- cpu_milli - cpu_milli
@@ -220,7 +215,12 @@ paths:
- delivery - delivery
/auth/api/v1/delivery/targets: /auth/api/v1/delivery/targets:
get: get:
description: 返回所有已启用的部署目标(如 k8s 集群、主机池) description: 从 AWX 动态返回可用 Job Template 及其 Inventory hosts
parameters:
- description: 组件过滤,例如 mysql
in: query
name: component
type: string
produces: produces:
- application/json - application/json
responses: responses:
@@ -239,49 +239,12 @@ paths:
summary: 获取可用部署目标 summary: 获取可用部署目标
tags: tags:
- delivery - delivery
post:
consumes:
- application/json
description: 管理员创建新的部署目标(目前仅支持 k8s 类型)
parameters:
- description: 目标配置
in: body
name: body
required: true
schema:
$ref: '#/definitions/handler.targetPayload'
produces:
- application/json
responses:
"201":
description: 目标已创建
schema:
$ref: '#/definitions/model.DeploymentTarget'
"400":
description: 参数错误
schema:
additionalProperties: true
type: object
"401":
description: 未授权
schema:
additionalProperties: true
type: object
"409":
description: 名称冲突
schema:
additionalProperties: true
type: object
security:
- BearerAuth: []
summary: 创建部署目标
tags:
- delivery
/auth/api/v1/delivery/tasks: /auth/api/v1/delivery/tasks:
get: get:
description: 返回当前用户可见的交付任务列表(管理员可见全部) description: 返回当前用户可见的交付任务列表(管理员可见全部)
parameters: parameters:
- description: 业务线 ID 过滤 - description: 业务线 ID 过滤
format: int64
in: query in: query
name: business_line_id name: business_line_id
type: integer type: integer
+116 -114
View File
@@ -14,63 +14,64 @@ type OAuthClient struct {
} }
type Config struct { type Config struct {
AppEnv string AppEnv string
HTTPAddr string HTTPAddr string
PublicBaseURL string PublicBaseURL string
MySQLDSN string MySQLDSN string
AutoMigrate bool AutoMigrate bool
SSOEnabled bool SSOEnabled bool
JWTSecret string JWTSecret string
JWTIssuer string JWTIssuer string
JWTTTLMinutes int JWTTTLMinutes int
SAMLEntityID string SAMLEntityID string
SAMLACSURL string SAMLACSURL string
SAMLSPCert string SAMLSPCert string
SAMLSPKey string SAMLSPKey string
SAMLIDPMetaURL string SAMLIDPMetaURL string
SAMLLogoutURL string SAMLLogoutURL string
WayenLoginURL string WayenLoginURL string
WayenTargetURL string WayenTargetURL string
WayenUsernameKey string WayenUsernameKey string
WayenPasswordKey string WayenPasswordKey string
WayenLoginFormat string WayenLoginFormat string
WayenLoginValue string WayenLoginValue string
WayenOAuthRef string WayenOAuthRef string
WayenOAuthLoginURL string WayenOAuthLoginURL string
WayneAPIBaseURL string WayneAPIBaseURL string
WayneAdminUsername string WayneAdminUsername string
WayneAdminPassword string WayneAdminPassword string
WayneTokenTTLMinutes int WayneTokenTTLMinutes int
WayneInternalAPIBaseURL string WayneInternalAPIBaseURL string
WayneServiceName string WayneServiceName string
WayneServiceAPISecretKey string WayneServiceAPISecretKey string
OAuthClientID string OAuthClientID string
OAuthClientSecret string OAuthClientSecret string
OAuthRedirectURI string OAuthRedirectURI string
OAuthCodeTTLSeconds int OAuthCodeTTLSeconds int
OIDCIssuer string OIDCIssuer string
OIDCAuthorizeURL string OIDCAuthorizeURL string
OIDCTokenURL string OIDCTokenURL string
OIDCUserInfoURL string OIDCUserInfoURL string
OIDCJWKSURL string OIDCJWKSURL string
CloudDMClientID string CloudDMClientID string
CloudDMClientSecret string CloudDMClientSecret string
CloudDMRedirectURI string CloudDMRedirectURI string
CloudDMTargetURL string CloudDMTargetURL string
CloudDMRegisterURL string CloudDMRegisterURL string
CloudDMAPIToken string CloudDMAPIToken string
AWXBaseURL string AWXBaseURL string
AWXToken string AWXToken string
AWXUsername string AWXUsername string
AWXPassword string AWXPassword string
DeliveryServiceToken string DeliveryServiceToken string
DeliverySchedulerEnabled bool DeliverySchedulerEnabled bool
DeliveryPollSeconds int DeliveryPollSeconds int
ReservationTTLMinutes int ReservationTTLMinutes int
DeliveryGlobalLimit int DeliveryGlobalLimit int
DeliveryTargetLimit int DeliveryTargetLimit int
DeliveryBusinessLimit int DeliveryBusinessLimit int
DeliveryDataDisks []string DeliveryHostInstanceLimit int
DeliveryDataDisks []string
} }
func Load() Config { func Load() Config {
@@ -83,63 +84,64 @@ func Load() Config {
oidcIssuer = strings.TrimRight(oidcIssuer, "/") oidcIssuer = strings.TrimRight(oidcIssuer, "/")
return Config{ return Config{
AppEnv: env("APP_ENV", "dev"), AppEnv: env("APP_ENV", "dev"),
HTTPAddr: httpAddr, HTTPAddr: httpAddr,
PublicBaseURL: publicBaseURL, PublicBaseURL: publicBaseURL,
MySQLDSN: env("MYSQL_DSN", "auth:auth@tcp(127.0.0.1:3306)/authserver?charset=utf8mb4&parseTime=True&loc=Local"), MySQLDSN: env("MYSQL_DSN", "auth:auth@tcp(127.0.0.1:3306)/authserver?charset=utf8mb4&parseTime=True&loc=Local"),
AutoMigrate: envBool("AUTO_MIGRATE", true), AutoMigrate: envBool("AUTO_MIGRATE", true),
SSOEnabled: envBool("SSO_ENABLED", true), SSOEnabled: envBool("SSO_ENABLED", true),
JWTSecret: env("JWT_SECRET", "change-this-secret"), JWTSecret: env("JWT_SECRET", "change-this-secret"),
JWTIssuer: env("JWT_ISSUER", "authserver"), JWTIssuer: env("JWT_ISSUER", "authserver"),
JWTTTLMinutes: envInt("JWT_TTL_MINUTES", 120), JWTTTLMinutes: envInt("JWT_TTL_MINUTES", 120),
SAMLEntityID: samlEntityID, SAMLEntityID: samlEntityID,
SAMLACSURL: samlACSURL, SAMLACSURL: samlACSURL,
SAMLSPCert: env("SAML_SP_CERT_FILE", "certs/sp.crt"), SAMLSPCert: env("SAML_SP_CERT_FILE", "certs/sp.crt"),
SAMLSPKey: env("SAML_SP_KEY_FILE", "certs/sp.key"), SAMLSPKey: env("SAML_SP_KEY_FILE", "certs/sp.key"),
SAMLIDPMetaURL: env("SAML_IDP_METADATA_URL", "http://sso-internal.dev.qiniu.io/saml2/meta"), SAMLIDPMetaURL: env("SAML_IDP_METADATA_URL", "http://sso-internal.dev.qiniu.io/saml2/meta"),
SAMLLogoutURL: trimURL(env("SAML_LOGOUT_URL", "")), SAMLLogoutURL: trimURL(env("SAML_LOGOUT_URL", "")),
WayenLoginURL: env("WAYEN_LOGIN_URL", ""), WayenLoginURL: env("WAYEN_LOGIN_URL", ""),
WayenTargetURL: env("WAYEN_TARGET_URL", ""), WayenTargetURL: env("WAYEN_TARGET_URL", ""),
WayenUsernameKey: env("WAYEN_USERNAME_KEY", "email"), WayenUsernameKey: env("WAYEN_USERNAME_KEY", "email"),
WayenPasswordKey: env("WAYEN_PASSWORD_KEY", "password"), WayenPasswordKey: env("WAYEN_PASSWORD_KEY", "password"),
WayenLoginFormat: env("WAYEN_LOGIN_FORMAT", "form"), WayenLoginFormat: env("WAYEN_LOGIN_FORMAT", "form"),
WayenLoginValue: env("WAYEN_LOGIN_VALUE", "email"), WayenLoginValue: env("WAYEN_LOGIN_VALUE", "email"),
WayenOAuthRef: env("WAYEN_OAUTH_REF", "/portal/namespace/1/app"), WayenOAuthRef: env("WAYEN_OAUTH_REF", "/portal/namespace/1/app"),
WayenOAuthLoginURL: trimURL(env("WAYEN_OAUTH_LOGIN_URL", "")), WayenOAuthLoginURL: trimURL(env("WAYEN_OAUTH_LOGIN_URL", "")),
WayneAPIBaseURL: trimURL(env("WAYNE_API_BASE_URL", env("WAYNE_INTERNAL_API_BASE_URL", ""))), WayneAPIBaseURL: trimURL(env("WAYNE_API_BASE_URL", env("WAYNE_INTERNAL_API_BASE_URL", ""))),
WayneAdminUsername: env("WAYNE_ADMIN_USERNAME", ""), WayneAdminUsername: env("WAYNE_ADMIN_USERNAME", ""),
WayneAdminPassword: env("WAYNE_ADMIN_PASSWORD", ""), WayneAdminPassword: env("WAYNE_ADMIN_PASSWORD", ""),
WayneTokenTTLMinutes: envInt("WAYNE_TOKEN_TTL_MINUTES", 1440), WayneTokenTTLMinutes: envInt("WAYNE_TOKEN_TTL_MINUTES", 1440),
WayneInternalAPIBaseURL: trimURL(env("WAYNE_INTERNAL_API_BASE_URL", "")), WayneInternalAPIBaseURL: trimURL(env("WAYNE_INTERNAL_API_BASE_URL", "")),
WayneServiceName: env("WAYNE_SERVICE_NAME", "xinfra"), WayneServiceName: env("WAYNE_SERVICE_NAME", "xinfra"),
WayneServiceAPISecretKey: env("WAYNE_SERVICE_API_SECRET_KEY", ""), WayneServiceAPISecretKey: env("WAYNE_SERVICE_API_SECRET_KEY", ""),
OAuthClientID: env("OAUTH_WAYNE_CLIENT_ID", "wayne"), OAuthClientID: env("OAUTH_WAYNE_CLIENT_ID", "wayne"),
OAuthClientSecret: env("OAUTH_WAYNE_CLIENT_SECRET", "wayne-secret"), OAuthClientSecret: env("OAUTH_WAYNE_CLIENT_SECRET", "wayne-secret"),
OAuthRedirectURI: env("OAUTH_WAYNE_REDIRECT_URI", ""), OAuthRedirectURI: env("OAUTH_WAYNE_REDIRECT_URI", ""),
OAuthCodeTTLSeconds: envInt("OAUTH_CODE_TTL_SECONDS", 120), OAuthCodeTTLSeconds: envInt("OAUTH_CODE_TTL_SECONDS", 120),
OIDCIssuer: oidcIssuer, OIDCIssuer: oidcIssuer,
OIDCAuthorizeURL: trimURL(env("OIDC_AUTHORIZATION_ENDPOINT", oidcIssuer+"/oauth/authorize")), OIDCAuthorizeURL: trimURL(env("OIDC_AUTHORIZATION_ENDPOINT", oidcIssuer+"/oauth/authorize")),
OIDCTokenURL: trimURL(env("OIDC_TOKEN_ENDPOINT", oidcIssuer+"/oauth/token")), OIDCTokenURL: trimURL(env("OIDC_TOKEN_ENDPOINT", oidcIssuer+"/oauth/token")),
OIDCUserInfoURL: trimURL(env("OIDC_USERINFO_ENDPOINT", oidcIssuer+"/oauth/userinfo")), OIDCUserInfoURL: trimURL(env("OIDC_USERINFO_ENDPOINT", oidcIssuer+"/oauth/userinfo")),
OIDCJWKSURL: trimURL(env("OIDC_JWKS_URI", oidcIssuer+"/oauth/jwks")), OIDCJWKSURL: trimURL(env("OIDC_JWKS_URI", oidcIssuer+"/oauth/jwks")),
CloudDMClientID: env("OIDC_CLOUDDM_CLIENT_ID", "clouddm"), CloudDMClientID: env("OIDC_CLOUDDM_CLIENT_ID", "clouddm"),
CloudDMClientSecret: env("OIDC_CLOUDDM_CLIENT_SECRET", ""), CloudDMClientSecret: env("OIDC_CLOUDDM_CLIENT_SECRET", ""),
CloudDMRedirectURI: env("OIDC_CLOUDDM_REDIRECT_URI", ""), CloudDMRedirectURI: env("OIDC_CLOUDDM_REDIRECT_URI", ""),
CloudDMTargetURL: env("CLOUDDM_TARGET_URL", ""), CloudDMTargetURL: env("CLOUDDM_TARGET_URL", ""),
CloudDMRegisterURL: trimURL(env("CLOUDDM_REGISTER_URL", "")), CloudDMRegisterURL: trimURL(env("CLOUDDM_REGISTER_URL", "")),
CloudDMAPIToken: env("CLOUDDM_API_TOKEN", ""), CloudDMAPIToken: env("CLOUDDM_API_TOKEN", ""),
AWXBaseURL: trimURL(env("AWX_BASE_URL", "")), AWXBaseURL: trimURL(env("AWX_BASE_URL", "")),
AWXToken: env("AWX_TOKEN", ""), AWXToken: env("AWX_TOKEN", ""),
AWXUsername: env("AWX_USERNAME", ""), AWXUsername: env("AWX_USERNAME", ""),
AWXPassword: env("AWX_PASSWORD", ""), AWXPassword: env("AWX_PASSWORD", ""),
DeliveryServiceToken: env("DELIVERY_SERVICE_TOKEN", ""), DeliveryServiceToken: env("DELIVERY_SERVICE_TOKEN", ""),
DeliverySchedulerEnabled: envBool("DELIVERY_SCHEDULER_ENABLED", false), DeliverySchedulerEnabled: envBool("DELIVERY_SCHEDULER_ENABLED", false),
DeliveryPollSeconds: envInt("DELIVERY_POLL_SECONDS", 5), DeliveryPollSeconds: envInt("DELIVERY_POLL_SECONDS", 5),
ReservationTTLMinutes: envInt("DELIVERY_RESERVATION_TTL_MINUTES", 120), ReservationTTLMinutes: envInt("DELIVERY_RESERVATION_TTL_MINUTES", 120),
DeliveryGlobalLimit: envInt("DELIVERY_GLOBAL_LIMIT", 2), DeliveryGlobalLimit: envInt("DELIVERY_GLOBAL_LIMIT", 2),
DeliveryTargetLimit: envInt("DELIVERY_TARGET_LIMIT", 2), DeliveryTargetLimit: envInt("DELIVERY_TARGET_LIMIT", 2),
DeliveryBusinessLimit: envInt("DELIVERY_BUSINESS_LIMIT", 1), DeliveryBusinessLimit: envInt("DELIVERY_BUSINESS_LIMIT", 1),
DeliveryDataDisks: envCSV("DELIVERY_DATA_DISKS", "/data"), DeliveryHostInstanceLimit: envInt("DELIVERY_HOST_INSTANCE_LIMIT", 4),
DeliveryDataDisks: envCSV("DELIVERY_DATA_DISKS", "/data"),
} }
} }
+47 -10
View File
@@ -24,6 +24,9 @@ import (
var dnsLabelPattern = regexp.MustCompile(`^[a-z0-9](?:[-a-z0-9]*[a-z0-9])?$`) var dnsLabelPattern = regexp.MustCompile(`^[a-z0-9](?:[-a-z0-9]*[a-z0-9])?$`)
// AWX inventory 主机名:允许字母数字、中划线与点(FQDN)。
var hostNamePattern = regexp.MustCompile(`^[a-zA-Z0-9](?:[-a-zA-Z0-9.]*[a-zA-Z0-9])?$`)
type MySQLDeliveryInput struct { type MySQLDeliveryInput struct {
BusinessLineID uint64 `json:"business_line_id" binding:"required"` BusinessLineID uint64 `json:"business_line_id" binding:"required"`
TargetID uint64 `json:"target_id" binding:"required"` TargetID uint64 `json:"target_id" binding:"required"`
@@ -33,9 +36,11 @@ type MySQLDeliveryInput struct {
Topology string `json:"topology"` Topology string `json:"topology"`
MySQLPort int `json:"mysql_port"` MySQLPort int `json:"mysql_port"`
DataDisk string `json:"data_disk"` DataDisk string `json:"data_disk"`
CPUMilli int64 `json:"cpu_milli" binding:"required"` // 调度控制(选填):点名候选池内主机跳过自动选机,端口/配额/实机守卫照常执行
MemoryMi int64 `json:"memory_mi" binding:"required"` TargetHost string `json:"target_host"`
StorageGi int64 `json:"storage_gi" binding:"required"` CPUMilli int64 `json:"cpu_milli" binding:"required"`
MemoryMi int64 `json:"memory_mi" binding:"required"`
StorageGi int64 `json:"storage_gi" binding:"required"`
// 数据库配置(选填,缺省由 playbook 基线兜底) // 数据库配置(选填,缺省由 playbook 基线兜底)
Timezone string `json:"timezone"` Timezone string `json:"timezone"`
LowerCaseTableNames *int `json:"lower_case_table_names"` LowerCaseTableNames *int `json:"lower_case_table_names"`
@@ -93,20 +98,31 @@ func parseTargetMetadata(raw string) targetMetadata {
return meta return meta
} }
// firstFreeHost 返回候选池中第一个未被占用的节点。 // firstFreeHost 返回候选池中非失败任务数未达单机实例上限的第一个节点;
func firstFreeHost(hosts []targetHost, occupied []string) *targetHost { // limit < 1 时按 1 兜底(退化为旧的一机一实例语义)。顺序遍历天然形成"先摊开、摊满一轮再叠加"。
taken := make(map[string]bool, len(occupied)) func firstFreeHost(hosts []targetHost, occupied []string, limit int) *targetHost {
for _, h := range occupied { if limit < 1 {
taken[h] = true limit = 1
} }
for i := range hosts { for i := range hosts {
if !taken[hosts[i].Name] { if hostTaskCount(occupied, hosts[i].Name) < limit {
return &hosts[i] return &hosts[i]
} }
} }
return nil return nil
} }
// hostTaskCount 统计某主机在占用清单(非失败任务的 target_host 列表,含重复)中的出现次数。
func hostTaskCount(occupied []string, name string) int {
count := 0
for _, h := range occupied {
if h == name {
count++
}
}
return count
}
// 端口池 13306-13999:混合分配模型,用户留空时自动分配,可覆盖为池内指定端口。 // 端口池 13306-13999:混合分配模型,用户留空时自动分配,可覆盖为池内指定端口。
const ( const (
mysqlPortPoolStart = 13306 mysqlPortPoolStart = 13306
@@ -344,6 +360,9 @@ func validateDeliveryInput(input MySQLDeliveryInput, dataDisks []string) error {
return fmt.Errorf("data_disk %q is not in the allowed mount point list %v", input.DataDisk, allowed) return fmt.Errorf("data_disk %q is not in the allowed mount point list %v", input.DataDisk, allowed)
} }
} }
if input.TargetHost != "" && (len(input.TargetHost) > 253 || !hostNamePattern.MatchString(input.TargetHost)) {
return fmt.Errorf("target_host must be a valid inventory host name")
}
if input.Timezone != "" && !timezonePattern.MatchString(input.Timezone) { if input.Timezone != "" && !timezonePattern.MatchString(input.Timezone) {
return fmt.Errorf("timezone must be an offset like +08:00, SYSTEM, or a named zone like Asia/Shanghai") return fmt.Errorf("timezone must be an offset like +08:00, SYSTEM, or a named zone like Asia/Shanghai")
} }
@@ -498,7 +517,25 @@ func (s *DeliveryService) claimAndReserve(ctx context.Context) (*model.DeliveryT
if err := tx.Model(&model.DeliveryTask{}).Where("target_id = ? AND target_host <> ? AND status NOT IN ?", task.TargetID, "", occupiedExclude).Pluck("target_host", &occupied).Error; err != nil { if err := tx.Model(&model.DeliveryTask{}).Where("target_id = ? AND target_host <> ? AND status NOT IN ?", task.TargetID, "", occupiedExclude).Pluck("target_host", &occupied).Error; err != nil {
return err return err
} }
host := firstFreeHost(meta.Hosts, occupied) limit := s.cfg.DeliveryHostInstanceLimit
var host *targetHost
if payload.TargetHost != "" {
// 点名主机:仅校验池内存在性与单机实例上限,端口/配额/实机守卫照常执行。
for i := range meta.Hosts {
if meta.Hosts[i].Name == payload.TargetHost {
host = &meta.Hosts[i]
break
}
}
if host == nil {
return s.failInTransaction(tx, &task, model.TaskValidationFailed, fmt.Sprintf("target_host %q is not in the candidate host pool", payload.TargetHost))
}
if effective := max(limit, 1); hostTaskCount(occupied, host.Name) >= effective {
return fmt.Errorf("defer: pinned host %s reached the per-host instance limit %d", host.Name, effective)
}
} else {
host = firstFreeHost(meta.Hosts, occupied, limit)
}
if host == nil { if host == nil {
return fmt.Errorf("defer: no free host available on target") return fmt.Errorf("defer: no free host available on target")
} }
+26
View File
@@ -15,6 +15,7 @@ func TestValidateDeliveryInput(t *testing.T) {
full.Topology = "standalone" full.Topology = "standalone"
full.MySQLPort = 13306 full.MySQLPort = 13306
full.DataDisk = "/disk1" full.DataDisk = "/disk1"
full.TargetHost = "k8s-server-03"
full.MemoryMi = 8192 full.MemoryMi = 8192
full.StorageGi = 2000 full.StorageGi = 2000
full.Timezone = "+08:00" full.Timezone = "+08:00"
@@ -55,6 +56,7 @@ func TestValidateDeliveryInput(t *testing.T) {
"port below pool": func(in *MySQLDeliveryInput) { in.MySQLPort = 3307 }, "port below pool": func(in *MySQLDeliveryInput) { in.MySQLPort = 3307 },
"port above pool": func(in *MySQLDeliveryInput) { in.MySQLPort = 14000 }, "port above pool": func(in *MySQLDeliveryInput) { in.MySQLPort = 14000 },
"data disk not in list": func(in *MySQLDeliveryInput) { in.DataDisk = "/mnt/other" }, "data disk not in list": func(in *MySQLDeliveryInput) { in.DataDisk = "/mnt/other" },
"bad target host": func(in *MySQLDeliveryInput) { in.TargetHost = "-bad-host" },
"bad timezone": func(in *MySQLDeliveryInput) { in.Timezone = "UTC+8" }, "bad timezone": func(in *MySQLDeliveryInput) { in.Timezone = "UTC+8" },
"bad lower case": func(in *MySQLDeliveryInput) { in.LowerCaseTableNames = intPtr(2) }, "bad lower case": func(in *MySQLDeliveryInput) { in.LowerCaseTableNames = intPtr(2) },
"bad charset": func(in *MySQLDeliveryInput) { in.CharacterSet = "big5" }, "bad charset": func(in *MySQLDeliveryInput) { in.CharacterSet = "big5" },
@@ -77,6 +79,30 @@ func TestValidateDeliveryInput(t *testing.T) {
} }
} }
func TestFirstFreeHost(t *testing.T) {
hosts := []targetHost{{Name: "node-a"}, {Name: "node-b"}}
if h := firstFreeHost(hosts, nil, 1); h == nil || h.Name != "node-a" {
t.Fatalf("expected node-a on empty occupancy, got %+v", h)
}
if h := firstFreeHost(hosts, []string{"node-a"}, 1); h == nil || h.Name != "node-b" {
t.Fatalf("expected node-b when node-a is full at limit 1, got %+v", h)
}
if h := firstFreeHost(hosts, []string{"node-a", "node-b"}, 1); h != nil {
t.Fatalf("limit 1 with all hosts taken should return nil, got %+v", h)
}
// 摊满一轮后回到首台叠加第二个实例
if h := firstFreeHost(hosts, []string{"node-a", "node-b"}, 2); h == nil || h.Name != "node-a" {
t.Fatalf("expected node-a for second round at limit 2, got %+v", h)
}
if h := firstFreeHost(hosts, []string{"node-a", "node-a", "node-b", "node-b"}, 2); h != nil {
t.Fatalf("limit 2 with all hosts saturated should return nil, got %+v", h)
}
// limit < 1 退化为一机一实例
if h := firstFreeHost(hosts, []string{"node-a"}, 0); h == nil || h.Name != "node-b" {
t.Fatalf("limit 0 should degrade to 1, got %+v", h)
}
}
func TestAllocatePort(t *testing.T) { func TestAllocatePort(t *testing.T) {
if port, err := allocatePort(0, nil); err != nil || port != mysqlPortPoolStart { if port, err := allocatePort(0, nil); err != nil || port != mysqlPortPoolStart {
t.Fatalf("expected first pool port %d, got %d err=%v", mysqlPortPoolStart, port, err) t.Fatalf("expected first pool port %d, got %d err=%v", mysqlPortPoolStart, port, err)