From 41b2f4542003b39033d9090e6e731a0ca540e22e Mon Sep 17 00:00:00 2001 From: Hungerdream <1710233908@qq.com> Date: Mon, 27 Jul 2026 15:19:44 +0800 Subject: [PATCH] feat(delivery): support multi-instance scheduling on a single host - Per-host instance limit (DELIVERY_HOST_INSTANCE_LIMIT, default 4) - Optional target_host to pin a host from the candidate pool - Regenerate swagger docs --- server/.env.example | 2 + server/docs/docs.go | 179 ++++++------------ server/docs/swagger.json | 179 ++++++------------ server/docs/swagger.yaml | 121 +++++------- server/internal/config/config.go | 230 ++++++++++++----------- server/internal/service/delivery.go | 57 +++++- server/internal/service/delivery_test.go | 26 +++ 7 files changed, 355 insertions(+), 439 deletions(-) diff --git a/server/.env.example b/server/.env.example index 5e6d1a5..e3ae278 100644 --- a/server/.env.example +++ b/server/.env.example @@ -22,6 +22,8 @@ DELIVERY_RESERVATION_TTL_MINUTES=120 DELIVERY_GLOBAL_LIMIT=2 DELIVERY_TARGET_LIMIT=2 DELIVERY_BUSINESS_LIMIT=1 +# 单机 MySQL 实例数上限（同机多实例，容量由配额 + playbook 实机守卫兜底） +DELIVERY_HOST_INSTANCE_LIMIT=4 # 数据盘挂载点白名单（逗号分隔，第一项为默认值） DELIVERY_DATA_DISKS=/data AWX_BASE_URL= diff --git a/server/docs/docs.go b/server/docs/docs.go index 763ede7..9444b0a 100644 --- a/server/docs/docs.go +++ b/server/docs/docs.go @@ -152,7 +152,7 @@ const docTemplate = `{ "BearerAuth": [] } ], - "description": "返回所有已启用的部署目标（如 k8s 集群、主机池）", + "description": "从 AWX 动态返回可用 Job Template 及其 Inventory hosts", "produces": [ "application/json" ], @@ -160,6 +160,14 @@ const docTemplate = `{ "delivery" ], "summary": "获取可用部署目标", + "parameters": [ + { + "type": "string", + "description": "组件过滤，例如 mysql", + "name": "component", + "in": "query" + } + ], "responses": { "200": { "description": "items: 部署目标数组", @@ -176,64 +184,6 @@ const docTemplate = `{ } } } - }, - "post": { - "security": [ - { - "BearerAuth": [] - } - ], - "description": "管理员创建新的部署目标（目前仅支持 k8s 类型）", - "consumes": [ - "application/json" - ], - "produces": [ - "application/json" - ], - "tags": [ - "delivery" - ], - "summary": "创建部署目标", - "parameters": [ - { - "description": "目标配置", - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/handler.targetPayload" - } - } - ], - "responses": { - "201": { - "description": "目标已创建", - "schema": { - "$ref": "#/definitions/model.DeploymentTarget" - } - }, - "400": { - "description": "参数错误", - "schema": { - "type": "object", - "additionalProperties": true - } - }, - "401": { - "description": "未授权", - "schema": { - "type": "object", - "additionalProperties": true - } - }, - "409": { - "description": "名称冲突", - "schema": { - "type": "object", - "additionalProperties": true - } - } - } } }, "/auth/api/v1/delivery/tasks": { @@ -254,6 +204,7 @@ const docTemplate = `{ "parameters": [ { "type": "integer", + "format": "int64", "description": "业务线 ID 过滤", "name": "business_line_id", "in": "query" @@ -689,65 +640,6 @@ const docTemplate = `{ } } }, - "handler.targetPayload": { - "type": "object", - "required": [ - "awx_inventory_id", - "awx_template_id", - "name", - "target_type" - ], - "properties": { - "awx_inventory_id": { - "type": "integer" - }, - "awx_template_id": { - "type": "integer" - }, - "metadata": { - "type": "object", - "additionalProperties": {} - }, - "name": { - "type": "string" - }, - "target_type": { - "type": "string" - } - } - }, - "model.DeploymentTarget": { - "type": "object", - "properties": { - "awx_inventory_id": { - "type": "integer" - }, - "awx_template_id": { - "type": "integer" - }, - "created_at": { - "type": "string" - }, - "enabled": { - "type": "boolean" - }, - "id": { - "type": "integer" - }, - "metadata": { - "type": "string" - }, - "name": { - "type": "string" - }, - "target_type": { - "type": "string" - }, - "updated_at": { - "type": "string" - } - } - }, "model.ResourceQuota": { "type": "object", "properties": { @@ -816,18 +708,55 @@ const docTemplate = `{ "target_id" ], "properties": { + "binlog_expire_logs_seconds": { + "type": "integer" + }, "business_line_id": { "type": "integer" }, + "character_set": { + "type": "string" + }, + "collation": { + "type": "string" + }, "cpu_milli": { "type": "integer" }, + "data_disk": { + "type": "string" + }, + "innodb_flush_log_at_trx_commit": { + "type": "integer" + }, + "innodb_io_capacity": { + "type": "integer" + }, + "innodb_redo_log_capacity": { + "type": "string" + }, "instance_name": { "type": "string" }, + "long_query_time": { + "type": "number" + }, + "lower_case_table_names": { + "type": "integer" + }, + "max_binlog_size": { + "type": "string" + }, + "max_connections": { + "description": "高级参数（选填，零值视为未设置）", + "type": "string" + }, "memory_mi": { "type": "integer" }, + "mysql_port": { + "type": "integer" + }, "mysql_version": { "type": "string" }, @@ -837,8 +766,22 @@ const docTemplate = `{ "storage_gi": { "type": "integer" }, + "sync_binlog": { + "type": "integer" + }, + "target_host": { + "description": "调度控制（选填）：点名候选池内主机跳过自动选机，端口/配额/实机守卫照常执行", + "type": "string" + }, "target_id": { "type": "integer" + }, + "timezone": { + "description": "数据库配置（选填，缺省由 playbook 基线兜底）", + "type": "string" + }, + "topology": { + "type": "string" } } } diff --git a/server/docs/swagger.json b/server/docs/swagger.json index a794ca0..663bdd5 100644 --- a/server/docs/swagger.json +++ b/server/docs/swagger.json @@ -145,7 +145,7 @@ "BearerAuth": [] } ], - "description": "返回所有已启用的部署目标（如 k8s 集群、主机池）", + "description": "从 AWX 动态返回可用 Job Template 及其 Inventory hosts", "produces": [ "application/json" ], @@ -153,6 +153,14 @@ "delivery" ], "summary": "获取可用部署目标", + "parameters": [ + { + "type": "string", + "description": "组件过滤，例如 mysql", + "name": "component", + "in": "query" + } + ], "responses": { "200": { "description": "items: 部署目标数组", @@ -169,64 +177,6 @@ } } } - }, - "post": { - "security": [ - { - "BearerAuth": [] - } - ], - "description": "管理员创建新的部署目标（目前仅支持 k8s 类型）", - "consumes": [ - "application/json" - ], - "produces": [ - "application/json" - ], - "tags": [ - "delivery" - ], - "summary": "创建部署目标", - "parameters": [ - { - "description": "目标配置", - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/handler.targetPayload" - } - } - ], - "responses": { - "201": { - "description": "目标已创建", - "schema": { - "$ref": "#/definitions/model.DeploymentTarget" - } - }, - "400": { - "description": "参数错误", - "schema": { - "type": "object", - "additionalProperties": true - } - }, - "401": { - "description": "未授权", - "schema": { - "type": "object", - "additionalProperties": true - } - }, - "409": { - "description": "名称冲突", - "schema": { - "type": "object", - "additionalProperties": true - } - } - } } }, "/auth/api/v1/delivery/tasks": { @@ -247,6 +197,7 @@ "parameters": [ { "type": "integer", + "format": "int64", "description": "业务线 ID 过滤", "name": "business_line_id", "in": "query" @@ -682,65 +633,6 @@ } } }, - "handler.targetPayload": { - "type": "object", - "required": [ - "awx_inventory_id", - "awx_template_id", - "name", - "target_type" - ], - "properties": { - "awx_inventory_id": { - "type": "integer" - }, - "awx_template_id": { - "type": "integer" - }, - "metadata": { - "type": "object", - "additionalProperties": {} - }, - "name": { - "type": "string" - }, - "target_type": { - "type": "string" - } - } - }, - "model.DeploymentTarget": { - "type": "object", - "properties": { - "awx_inventory_id": { - "type": "integer" - }, - "awx_template_id": { - "type": "integer" - }, - "created_at": { - "type": "string" - }, - "enabled": { - "type": "boolean" - }, - "id": { - "type": "integer" - }, - "metadata": { - "type": "string" - }, - "name": { - "type": "string" - }, - "target_type": { - "type": "string" - }, - "updated_at": { - "type": "string" - } - } - }, "model.ResourceQuota": { "type": "object", "properties": { @@ -809,18 +701,55 @@ "target_id" ], "properties": { + "binlog_expire_logs_seconds": { + "type": "integer" + }, "business_line_id": { "type": "integer" }, + "character_set": { + "type": "string" + }, + "collation": { + "type": "string" + }, "cpu_milli": { "type": "integer" }, + "data_disk": { + "type": "string" + }, + "innodb_flush_log_at_trx_commit": { + "type": "integer" + }, + "innodb_io_capacity": { + "type": "integer" + }, + "innodb_redo_log_capacity": { + "type": "string" + }, "instance_name": { "type": "string" }, + "long_query_time": { + "type": "number" + }, + "lower_case_table_names": { + "type": "integer" + }, + "max_binlog_size": { + "type": "string" + }, + "max_connections": { + "description": "高级参数（选填，零值视为未设置）", + "type": "string" + }, "memory_mi": { "type": "integer" }, + "mysql_port": { + "type": "integer" + }, "mysql_version": { "type": "string" }, @@ -830,8 +759,22 @@ "storage_gi": { "type": "integer" }, + "sync_binlog": { + "type": "integer" + }, + "target_host": { + "description": "调度控制（选填）：点名候选池内主机跳过自动选机，端口/配额/实机守卫照常执行", + "type": "string" + }, "target_id": { "type": "integer" + }, + "timezone": { + "description": "数据库配置（选填，缺省由 playbook 基线兜底）", + "type": "string" + }, + "topology": { + "type": "string" } } } diff --git a/server/docs/swagger.yaml b/server/docs/swagger.yaml index 97d66ff..55f406c 100644 --- a/server/docs/swagger.yaml +++ b/server/docs/swagger.yaml @@ -22,46 +22,6 @@ definitions: - storage_gi - target_id type: object - handler.targetPayload: - properties: - awx_inventory_id: - type: integer - awx_template_id: - type: integer - metadata: - additionalProperties: {} - type: object - name: - type: string - target_type: - type: string - required: - - awx_inventory_id - - awx_template_id - - name - - target_type - type: object - model.DeploymentTarget: - properties: - awx_inventory_id: - type: integer - awx_template_id: - type: integer - created_at: - type: string - enabled: - type: boolean - id: - type: integer - metadata: - type: string - name: - type: string - target_type: - type: string - updated_at: - type: string - type: object model.ResourceQuota: properties: business_line_id: @@ -101,22 +61,57 @@ definitions: type: object service.MySQLDeliveryInput: properties: + binlog_expire_logs_seconds: + type: integer business_line_id: type: integer + character_set: + type: string + collation: + type: string cpu_milli: type: integer + data_disk: + type: string + innodb_flush_log_at_trx_commit: + type: integer + innodb_io_capacity: + type: integer + innodb_redo_log_capacity: + type: string instance_name: type: string + long_query_time: + type: number + lower_case_table_names: + type: integer + max_binlog_size: + type: string + max_connections: + description: 高级参数（选填，零值视为未设置） + type: string memory_mi: type: integer + mysql_port: + type: integer mysql_version: type: string namespace: type: string storage_gi: type: integer + sync_binlog: + type: integer + target_host: + description: 调度控制（选填）：点名候选池内主机跳过自动选机，端口/配额/实机守卫照常执行 + type: string target_id: type: integer + timezone: + description: 数据库配置（选填，缺省由 playbook 基线兜底） + type: string + topology: + type: string required: - business_line_id - cpu_milli @@ -220,7 +215,12 @@ paths: - delivery /auth/api/v1/delivery/targets: get: - description: 返回所有已启用的部署目标（如 k8s 集群、主机池） + description: 从 AWX 动态返回可用 Job Template 及其 Inventory hosts + parameters: + - description: 组件过滤，例如 mysql + in: query + name: component + type: string produces: - application/json responses: @@ -239,49 +239,12 @@ paths: summary: 获取可用部署目标 tags: - delivery - post: - consumes: - - application/json - description: 管理员创建新的部署目标（目前仅支持 k8s 类型） - parameters: - - description: 目标配置 - in: body - name: body - required: true - schema: - $ref: '#/definitions/handler.targetPayload' - produces: - - application/json - responses: - "201": - description: 目标已创建 - schema: - $ref: '#/definitions/model.DeploymentTarget' - "400": - description: 参数错误 - schema: - additionalProperties: true - type: object - "401": - description: 未授权 - schema: - additionalProperties: true - type: object - "409": - description: 名称冲突 - schema: - additionalProperties: true - type: object - security: - - BearerAuth: [] - summary: 创建部署目标 - tags: - - delivery /auth/api/v1/delivery/tasks: get: description: 返回当前用户可见的交付任务列表（管理员可见全部） parameters: - description: 业务线 ID 过滤 + format: int64 in: query name: business_line_id type: integer diff --git a/server/internal/config/config.go b/server/internal/config/config.go index 30f46d6..4041281 100644 --- a/server/internal/config/config.go +++ b/server/internal/config/config.go @@ -14,63 +14,64 @@ type OAuthClient struct { } type Config struct { - AppEnv string - HTTPAddr string - PublicBaseURL string - MySQLDSN string - AutoMigrate bool - SSOEnabled bool - JWTSecret string - JWTIssuer string - JWTTTLMinutes int - SAMLEntityID string - SAMLACSURL string - SAMLSPCert string - SAMLSPKey string - SAMLIDPMetaURL string - SAMLLogoutURL string - WayenLoginURL string - WayenTargetURL string - WayenUsernameKey string - WayenPasswordKey string - WayenLoginFormat string - WayenLoginValue string - WayenOAuthRef string - WayenOAuthLoginURL string - WayneAPIBaseURL string - WayneAdminUsername string - WayneAdminPassword string - WayneTokenTTLMinutes int - WayneInternalAPIBaseURL string - WayneServiceName string - WayneServiceAPISecretKey string - OAuthClientID string - OAuthClientSecret string - OAuthRedirectURI string - OAuthCodeTTLSeconds int - OIDCIssuer string - OIDCAuthorizeURL string - OIDCTokenURL string - OIDCUserInfoURL string - OIDCJWKSURL string - CloudDMClientID string - CloudDMClientSecret string - CloudDMRedirectURI string - CloudDMTargetURL string - CloudDMRegisterURL string - CloudDMAPIToken string - AWXBaseURL string - AWXToken string - AWXUsername string - AWXPassword string - DeliveryServiceToken string - DeliverySchedulerEnabled bool - DeliveryPollSeconds int - ReservationTTLMinutes int - DeliveryGlobalLimit int - DeliveryTargetLimit int - DeliveryBusinessLimit int - DeliveryDataDisks []string + AppEnv string + HTTPAddr string + PublicBaseURL string + MySQLDSN string + AutoMigrate bool + SSOEnabled bool + JWTSecret string + JWTIssuer string + JWTTTLMinutes int + SAMLEntityID string + SAMLACSURL string + SAMLSPCert string + SAMLSPKey string + SAMLIDPMetaURL string + SAMLLogoutURL string + WayenLoginURL string + WayenTargetURL string + WayenUsernameKey string + WayenPasswordKey string + WayenLoginFormat string + WayenLoginValue string + WayenOAuthRef string + WayenOAuthLoginURL string + WayneAPIBaseURL string + WayneAdminUsername string + WayneAdminPassword string + WayneTokenTTLMinutes int + WayneInternalAPIBaseURL string + WayneServiceName string + WayneServiceAPISecretKey string + OAuthClientID string + OAuthClientSecret string + OAuthRedirectURI string + OAuthCodeTTLSeconds int + OIDCIssuer string + OIDCAuthorizeURL string + OIDCTokenURL string + OIDCUserInfoURL string + OIDCJWKSURL string + CloudDMClientID string + CloudDMClientSecret string + CloudDMRedirectURI string + CloudDMTargetURL string + CloudDMRegisterURL string + CloudDMAPIToken string + AWXBaseURL string + AWXToken string + AWXUsername string + AWXPassword string + DeliveryServiceToken string + DeliverySchedulerEnabled bool + DeliveryPollSeconds int + ReservationTTLMinutes int + DeliveryGlobalLimit int + DeliveryTargetLimit int + DeliveryBusinessLimit int + DeliveryHostInstanceLimit int + DeliveryDataDisks []string } func Load() Config { @@ -83,63 +84,64 @@ func Load() Config { oidcIssuer = strings.TrimRight(oidcIssuer, "/") return Config{ - AppEnv: env("APP_ENV", "dev"), - HTTPAddr: httpAddr, - PublicBaseURL: publicBaseURL, - MySQLDSN: env("MYSQL_DSN", "auth:auth@tcp(127.0.0.1:3306)/authserver?charset=utf8mb4&parseTime=True&loc=Local"), - AutoMigrate: envBool("AUTO_MIGRATE", true), - SSOEnabled: envBool("SSO_ENABLED", true), - JWTSecret: env("JWT_SECRET", "change-this-secret"), - JWTIssuer: env("JWT_ISSUER", "authserver"), - JWTTTLMinutes: envInt("JWT_TTL_MINUTES", 120), - SAMLEntityID: samlEntityID, - SAMLACSURL: samlACSURL, - SAMLSPCert: env("SAML_SP_CERT_FILE", "certs/sp.crt"), - SAMLSPKey: env("SAML_SP_KEY_FILE", "certs/sp.key"), - SAMLIDPMetaURL: env("SAML_IDP_METADATA_URL", "http://sso-internal.dev.qiniu.io/saml2/meta"), - SAMLLogoutURL: trimURL(env("SAML_LOGOUT_URL", "")), - WayenLoginURL: env("WAYEN_LOGIN_URL", ""), - WayenTargetURL: env("WAYEN_TARGET_URL", ""), - WayenUsernameKey: env("WAYEN_USERNAME_KEY", "email"), - WayenPasswordKey: env("WAYEN_PASSWORD_KEY", "password"), - WayenLoginFormat: env("WAYEN_LOGIN_FORMAT", "form"), - WayenLoginValue: env("WAYEN_LOGIN_VALUE", "email"), - WayenOAuthRef: env("WAYEN_OAUTH_REF", "/portal/namespace/1/app"), - WayenOAuthLoginURL: trimURL(env("WAYEN_OAUTH_LOGIN_URL", "")), - WayneAPIBaseURL: trimURL(env("WAYNE_API_BASE_URL", env("WAYNE_INTERNAL_API_BASE_URL", ""))), - WayneAdminUsername: env("WAYNE_ADMIN_USERNAME", ""), - WayneAdminPassword: env("WAYNE_ADMIN_PASSWORD", ""), - WayneTokenTTLMinutes: envInt("WAYNE_TOKEN_TTL_MINUTES", 1440), - WayneInternalAPIBaseURL: trimURL(env("WAYNE_INTERNAL_API_BASE_URL", "")), - WayneServiceName: env("WAYNE_SERVICE_NAME", "xinfra"), - WayneServiceAPISecretKey: env("WAYNE_SERVICE_API_SECRET_KEY", ""), - OAuthClientID: env("OAUTH_WAYNE_CLIENT_ID", "wayne"), - OAuthClientSecret: env("OAUTH_WAYNE_CLIENT_SECRET", "wayne-secret"), - OAuthRedirectURI: env("OAUTH_WAYNE_REDIRECT_URI", ""), - OAuthCodeTTLSeconds: envInt("OAUTH_CODE_TTL_SECONDS", 120), - OIDCIssuer: oidcIssuer, - OIDCAuthorizeURL: trimURL(env("OIDC_AUTHORIZATION_ENDPOINT", oidcIssuer+"/oauth/authorize")), - OIDCTokenURL: trimURL(env("OIDC_TOKEN_ENDPOINT", oidcIssuer+"/oauth/token")), - OIDCUserInfoURL: trimURL(env("OIDC_USERINFO_ENDPOINT", oidcIssuer+"/oauth/userinfo")), - OIDCJWKSURL: trimURL(env("OIDC_JWKS_URI", oidcIssuer+"/oauth/jwks")), - CloudDMClientID: env("OIDC_CLOUDDM_CLIENT_ID", "clouddm"), - CloudDMClientSecret: env("OIDC_CLOUDDM_CLIENT_SECRET", ""), - CloudDMRedirectURI: env("OIDC_CLOUDDM_REDIRECT_URI", ""), - CloudDMTargetURL: env("CLOUDDM_TARGET_URL", ""), - CloudDMRegisterURL: trimURL(env("CLOUDDM_REGISTER_URL", "")), - CloudDMAPIToken: env("CLOUDDM_API_TOKEN", ""), - AWXBaseURL: trimURL(env("AWX_BASE_URL", "")), - AWXToken: env("AWX_TOKEN", ""), - AWXUsername: env("AWX_USERNAME", ""), - AWXPassword: env("AWX_PASSWORD", ""), - DeliveryServiceToken: env("DELIVERY_SERVICE_TOKEN", ""), - DeliverySchedulerEnabled: envBool("DELIVERY_SCHEDULER_ENABLED", false), - DeliveryPollSeconds: envInt("DELIVERY_POLL_SECONDS", 5), - ReservationTTLMinutes: envInt("DELIVERY_RESERVATION_TTL_MINUTES", 120), - DeliveryGlobalLimit: envInt("DELIVERY_GLOBAL_LIMIT", 2), - DeliveryTargetLimit: envInt("DELIVERY_TARGET_LIMIT", 2), - DeliveryBusinessLimit: envInt("DELIVERY_BUSINESS_LIMIT", 1), - DeliveryDataDisks: envCSV("DELIVERY_DATA_DISKS", "/data"), + AppEnv: env("APP_ENV", "dev"), + HTTPAddr: httpAddr, + PublicBaseURL: publicBaseURL, + MySQLDSN: env("MYSQL_DSN", "auth:auth@tcp(127.0.0.1:3306)/authserver?charset=utf8mb4&parseTime=True&loc=Local"), + AutoMigrate: envBool("AUTO_MIGRATE", true), + SSOEnabled: envBool("SSO_ENABLED", true), + JWTSecret: env("JWT_SECRET", "change-this-secret"), + JWTIssuer: env("JWT_ISSUER", "authserver"), + JWTTTLMinutes: envInt("JWT_TTL_MINUTES", 120), + SAMLEntityID: samlEntityID, + SAMLACSURL: samlACSURL, + SAMLSPCert: env("SAML_SP_CERT_FILE", "certs/sp.crt"), + SAMLSPKey: env("SAML_SP_KEY_FILE", "certs/sp.key"), + SAMLIDPMetaURL: env("SAML_IDP_METADATA_URL", "http://sso-internal.dev.qiniu.io/saml2/meta"), + SAMLLogoutURL: trimURL(env("SAML_LOGOUT_URL", "")), + WayenLoginURL: env("WAYEN_LOGIN_URL", ""), + WayenTargetURL: env("WAYEN_TARGET_URL", ""), + WayenUsernameKey: env("WAYEN_USERNAME_KEY", "email"), + WayenPasswordKey: env("WAYEN_PASSWORD_KEY", "password"), + WayenLoginFormat: env("WAYEN_LOGIN_FORMAT", "form"), + WayenLoginValue: env("WAYEN_LOGIN_VALUE", "email"), + WayenOAuthRef: env("WAYEN_OAUTH_REF", "/portal/namespace/1/app"), + WayenOAuthLoginURL: trimURL(env("WAYEN_OAUTH_LOGIN_URL", "")), + WayneAPIBaseURL: trimURL(env("WAYNE_API_BASE_URL", env("WAYNE_INTERNAL_API_BASE_URL", ""))), + WayneAdminUsername: env("WAYNE_ADMIN_USERNAME", ""), + WayneAdminPassword: env("WAYNE_ADMIN_PASSWORD", ""), + WayneTokenTTLMinutes: envInt("WAYNE_TOKEN_TTL_MINUTES", 1440), + WayneInternalAPIBaseURL: trimURL(env("WAYNE_INTERNAL_API_BASE_URL", "")), + WayneServiceName: env("WAYNE_SERVICE_NAME", "xinfra"), + WayneServiceAPISecretKey: env("WAYNE_SERVICE_API_SECRET_KEY", ""), + OAuthClientID: env("OAUTH_WAYNE_CLIENT_ID", "wayne"), + OAuthClientSecret: env("OAUTH_WAYNE_CLIENT_SECRET", "wayne-secret"), + OAuthRedirectURI: env("OAUTH_WAYNE_REDIRECT_URI", ""), + OAuthCodeTTLSeconds: envInt("OAUTH_CODE_TTL_SECONDS", 120), + OIDCIssuer: oidcIssuer, + OIDCAuthorizeURL: trimURL(env("OIDC_AUTHORIZATION_ENDPOINT", oidcIssuer+"/oauth/authorize")), + OIDCTokenURL: trimURL(env("OIDC_TOKEN_ENDPOINT", oidcIssuer+"/oauth/token")), + OIDCUserInfoURL: trimURL(env("OIDC_USERINFO_ENDPOINT", oidcIssuer+"/oauth/userinfo")), + OIDCJWKSURL: trimURL(env("OIDC_JWKS_URI", oidcIssuer+"/oauth/jwks")), + CloudDMClientID: env("OIDC_CLOUDDM_CLIENT_ID", "clouddm"), + CloudDMClientSecret: env("OIDC_CLOUDDM_CLIENT_SECRET", ""), + CloudDMRedirectURI: env("OIDC_CLOUDDM_REDIRECT_URI", ""), + CloudDMTargetURL: env("CLOUDDM_TARGET_URL", ""), + CloudDMRegisterURL: trimURL(env("CLOUDDM_REGISTER_URL", "")), + CloudDMAPIToken: env("CLOUDDM_API_TOKEN", ""), + AWXBaseURL: trimURL(env("AWX_BASE_URL", "")), + AWXToken: env("AWX_TOKEN", ""), + AWXUsername: env("AWX_USERNAME", ""), + AWXPassword: env("AWX_PASSWORD", ""), + DeliveryServiceToken: env("DELIVERY_SERVICE_TOKEN", ""), + DeliverySchedulerEnabled: envBool("DELIVERY_SCHEDULER_ENABLED", false), + DeliveryPollSeconds: envInt("DELIVERY_POLL_SECONDS", 5), + ReservationTTLMinutes: envInt("DELIVERY_RESERVATION_TTL_MINUTES", 120), + DeliveryGlobalLimit: envInt("DELIVERY_GLOBAL_LIMIT", 2), + DeliveryTargetLimit: envInt("DELIVERY_TARGET_LIMIT", 2), + DeliveryBusinessLimit: envInt("DELIVERY_BUSINESS_LIMIT", 1), + DeliveryHostInstanceLimit: envInt("DELIVERY_HOST_INSTANCE_LIMIT", 4), + DeliveryDataDisks: envCSV("DELIVERY_DATA_DISKS", "/data"), } } diff --git a/server/internal/service/delivery.go b/server/internal/service/delivery.go index f40a8dd..119254c 100644 --- a/server/internal/service/delivery.go +++ b/server/internal/service/delivery.go @@ -24,6 +24,9 @@ import ( var dnsLabelPattern = regexp.MustCompile(`^[a-z0-9](?:[-a-z0-9]*[a-z0-9])?$`) +// AWX inventory 主机名：允许字母数字、中划线与点（FQDN）。 +var hostNamePattern = regexp.MustCompile(`^[a-zA-Z0-9](?:[-a-zA-Z0-9.]*[a-zA-Z0-9])?$`) + type MySQLDeliveryInput struct { BusinessLineID uint64 `json:"business_line_id" binding:"required"` TargetID uint64 `json:"target_id" binding:"required"` @@ -33,9 +36,11 @@ type MySQLDeliveryInput struct { Topology string `json:"topology"` MySQLPort int `json:"mysql_port"` DataDisk string `json:"data_disk"` - CPUMilli int64 `json:"cpu_milli" binding:"required"` - MemoryMi int64 `json:"memory_mi" binding:"required"` - StorageGi int64 `json:"storage_gi" binding:"required"` + // 调度控制（选填）：点名候选池内主机跳过自动选机，端口/配额/实机守卫照常执行 + TargetHost string `json:"target_host"` + CPUMilli int64 `json:"cpu_milli" binding:"required"` + MemoryMi int64 `json:"memory_mi" binding:"required"` + StorageGi int64 `json:"storage_gi" binding:"required"` // 数据库配置（选填，缺省由 playbook 基线兜底） Timezone string `json:"timezone"` LowerCaseTableNames *int `json:"lower_case_table_names"` @@ -93,20 +98,31 @@ func parseTargetMetadata(raw string) targetMetadata { return meta } -// firstFreeHost 返回候选池中第一个未被占用的节点。 -func firstFreeHost(hosts []targetHost, occupied []string) *targetHost { - taken := make(map[string]bool, len(occupied)) - for _, h := range occupied { - taken[h] = true +// firstFreeHost 返回候选池中非失败任务数未达单机实例上限的第一个节点； +// limit < 1 时按 1 兜底（退化为旧的一机一实例语义）。顺序遍历天然形成"先摊开、摊满一轮再叠加"。 +func firstFreeHost(hosts []targetHost, occupied []string, limit int) *targetHost { + if limit < 1 { + limit = 1 } for i := range hosts { - if !taken[hosts[i].Name] { + if hostTaskCount(occupied, hosts[i].Name) < limit { return &hosts[i] } } return nil } +// hostTaskCount 统计某主机在占用清单（非失败任务的 target_host 列表，含重复）中的出现次数。 +func hostTaskCount(occupied []string, name string) int { + count := 0 + for _, h := range occupied { + if h == name { + count++ + } + } + return count +} + // 端口池 13306-13999：混合分配模型，用户留空时自动分配，可覆盖为池内指定端口。 const ( mysqlPortPoolStart = 13306 @@ -344,6 +360,9 @@ func validateDeliveryInput(input MySQLDeliveryInput, dataDisks []string) error { return fmt.Errorf("data_disk %q is not in the allowed mount point list %v", input.DataDisk, allowed) } } + if input.TargetHost != "" && (len(input.TargetHost) > 253 || !hostNamePattern.MatchString(input.TargetHost)) { + return fmt.Errorf("target_host must be a valid inventory host name") + } if input.Timezone != "" && !timezonePattern.MatchString(input.Timezone) { return fmt.Errorf("timezone must be an offset like +08:00, SYSTEM, or a named zone like Asia/Shanghai") } @@ -498,7 +517,25 @@ func (s *DeliveryService) claimAndReserve(ctx context.Context) (*model.DeliveryT if err := tx.Model(&model.DeliveryTask{}).Where("target_id = ? AND target_host <> ? AND status NOT IN ?", task.TargetID, "", occupiedExclude).Pluck("target_host", &occupied).Error; err != nil { return err } - host := firstFreeHost(meta.Hosts, occupied) + limit := s.cfg.DeliveryHostInstanceLimit + var host *targetHost + if payload.TargetHost != "" { + // 点名主机：仅校验池内存在性与单机实例上限，端口/配额/实机守卫照常执行。 + for i := range meta.Hosts { + if meta.Hosts[i].Name == payload.TargetHost { + host = &meta.Hosts[i] + break + } + } + if host == nil { + return s.failInTransaction(tx, &task, model.TaskValidationFailed, fmt.Sprintf("target_host %q is not in the candidate host pool", payload.TargetHost)) + } + if effective := max(limit, 1); hostTaskCount(occupied, host.Name) >= effective { + return fmt.Errorf("defer: pinned host %s reached the per-host instance limit %d", host.Name, effective) + } + } else { + host = firstFreeHost(meta.Hosts, occupied, limit) + } if host == nil { return fmt.Errorf("defer: no free host available on target") } diff --git a/server/internal/service/delivery_test.go b/server/internal/service/delivery_test.go index f929cba..ea19946 100644 --- a/server/internal/service/delivery_test.go +++ b/server/internal/service/delivery_test.go @@ -15,6 +15,7 @@ func TestValidateDeliveryInput(t *testing.T) { full.Topology = "standalone" full.MySQLPort = 13306 full.DataDisk = "/disk1" + full.TargetHost = "k8s-server-03" full.MemoryMi = 8192 full.StorageGi = 2000 full.Timezone = "+08:00" @@ -55,6 +56,7 @@ func TestValidateDeliveryInput(t *testing.T) { "port below pool": func(in *MySQLDeliveryInput) { in.MySQLPort = 3307 }, "port above pool": func(in *MySQLDeliveryInput) { in.MySQLPort = 14000 }, "data disk not in list": func(in *MySQLDeliveryInput) { in.DataDisk = "/mnt/other" }, + "bad target host": func(in *MySQLDeliveryInput) { in.TargetHost = "-bad-host" }, "bad timezone": func(in *MySQLDeliveryInput) { in.Timezone = "UTC+8" }, "bad lower case": func(in *MySQLDeliveryInput) { in.LowerCaseTableNames = intPtr(2) }, "bad charset": func(in *MySQLDeliveryInput) { in.CharacterSet = "big5" }, @@ -77,6 +79,30 @@ func TestValidateDeliveryInput(t *testing.T) { } } +func TestFirstFreeHost(t *testing.T) { + hosts := []targetHost{{Name: "node-a"}, {Name: "node-b"}} + if h := firstFreeHost(hosts, nil, 1); h == nil || h.Name != "node-a" { + t.Fatalf("expected node-a on empty occupancy, got %+v", h) + } + if h := firstFreeHost(hosts, []string{"node-a"}, 1); h == nil || h.Name != "node-b" { + t.Fatalf("expected node-b when node-a is full at limit 1, got %+v", h) + } + if h := firstFreeHost(hosts, []string{"node-a", "node-b"}, 1); h != nil { + t.Fatalf("limit 1 with all hosts taken should return nil, got %+v", h) + } + // 摊满一轮后回到首台叠加第二个实例 + if h := firstFreeHost(hosts, []string{"node-a", "node-b"}, 2); h == nil || h.Name != "node-a" { + t.Fatalf("expected node-a for second round at limit 2, got %+v", h) + } + if h := firstFreeHost(hosts, []string{"node-a", "node-a", "node-b", "node-b"}, 2); h != nil { + t.Fatalf("limit 2 with all hosts saturated should return nil, got %+v", h) + } + // limit < 1 退化为一机一实例 + if h := firstFreeHost(hosts, []string{"node-a"}, 0); h == nil || h.Name != "node-b" { + t.Fatalf("limit 0 should degrade to 1, got %+v", h) + } +} + func TestAllocatePort(t *testing.T) { if port, err := allocatePort(0, nil); err != nil || port != mysqlPortPoolStart { t.Fatalf("expected first pool port %d, got %d err=%v", mysqlPortPoolStart, port, err)