Merge remote-tracking branch 'upstream/main' into feat/base-service-delivery

# Conflicts:
#	server/.env.example
#	server/internal/config/config.go
#	server/internal/service/delivery.go
#	server/internal/service/delivery_test.go
This commit is contained in:
mac
2026-07-28 09:34:37 +08:00
11 changed files with 786 additions and 602 deletions
+208 -64
View File
@@ -1,82 +1,186 @@
---
- name: Preflight native MySQL delivery
- name: Native MySQL delivery
hosts: "{{ target_hosts }}"
become: true
gather_facts: true
any_errors_fatal: true
vars:
# --- identity ---
mysql_instance: "{{ instance_name }}"
mysql_version_value: "{{ mysql_version | default('8.0') }}"
# 版本包映射:8.0 用 Ubuntu 24.04 自带源(精确锁版);8.4 用 MySQL 官方 APT 源的 LTS 组件
# (noble 自带源无 8.4,官方源组件内即为该系列,不再锁小版本)。
# 5.6/5.7 已官方 EOL 且 Ubuntu 24.04 无可用 apt 包,明确不纳入白名单。
mysql_package_map:
"8.0": "mysql-server=8.0.46-0ubuntu0.24.04.3"
mysql_package_name: "{{ mysql_package_map[mysql_version_value] }}"
mysql_port_value: "{{ mysql_port | default(3307) | int }}"
mysql_memory_mb_value: "{{ memory_mb | default(2048) | int }}"
mysql_storage_gb_value: "{{ storage_gb | default(20) | int }}"
"8.0":
package: "mysql-server=8.0.46-0ubuntu0.24.04.3"
repo_component: ""
"8.4":
package: "mysql-community-server"
repo_component: "mysql-8.4-lts"
mysql_package_name: "{{ (mysql_package_map[mysql_version_value] | default({})).package | default('') }}"
mysql_repo_component: "{{ (mysql_package_map[mysql_version_value] | default({})).repo_component | default('') }}"
# --- platform-allocated inputs (safe fallbacks when not passed in) ---
# 端口池 13306–13999:平台从池分配并传入;未传时兜底池首端口,占用探测在目标机执行。
mysql_port_value: "{{ mysql_port | default(13306) | int }}"
# GR 组通信端口:仅 mgr_3 使用,平台成对分配;兜底为 SQL 端口 +10000。
mysql_gr_port_value: "{{ gr_port | default((mysql_port | default(13306) | int) + 10000) | int }}"
# 数据盘挂载点:平台按白名单选定并传入;兜底 /data。
mysql_data_disk: "{{ data_disk | default('/data') }}"
# --- resource quota (Go→AWX 契约单位保持 MB/GB,不擅改) ---
mysql_memory_mb_value: "{{ memory_mb | default(4096) | int }}"
mysql_storage_gb_value: "{{ storage_gb | default(50) | int }}"
# --- mount-point-agnostic layout: {data_disk}/mysql-delivery/{instance_id}/... ---
mysql_base_dir: "{{ mysql_data_disk }}/mysql-delivery/{{ mysql_instance }}"
mysql_install_dir: "/opt/mysql-delivery/{{ mysql_instance }}"
mysql_data_dir: "/data/mysql-delivery/{{ mysql_instance }}/data"
mysql_log_dir: "/data/mysql-delivery/{{ mysql_instance }}/log"
mysql_data_dir: "{{ mysql_base_dir }}/data"
mysql_log_dir: "{{ mysql_base_dir }}/logs"
mysql_binlog_dir: "{{ mysql_base_dir }}/logs/binlog"
mysql_redo_dir: "{{ mysql_base_dir }}/logs/redo"
mysql_tmp_dir: "{{ mysql_base_dir }}/tmp"
# socket/pid 走 /run tmpfs(重启自动清理),由 systemd RuntimeDirectory 创建。
mysql_run_dir: "/run/mysql-delivery-{{ mysql_instance }}"
mysql_config_file: "/etc/mysql/mysql-delivery/{{ mysql_instance }}.cnf"
# --- database config (user form, with doc default baselines) ---
mysql_timezone: "{{ timezone | default('+08:00') }}"
mysql_lower_case_table_names: "{{ lower_case_table_names | default(1) | int }}"
mysql_character_set: "{{ character_set | default('utf8mb4') }}"
mysql_collation: "{{ collation | default('utf8mb4_general_ci') }}"
# --- advanced params (overridable; default baseline assumes SSD for io_capacity) ---
mysql_flush_log_at_trx_commit: "{{ innodb_flush_log_at_trx_commit | default(1) | int }}"
mysql_sync_binlog: "{{ sync_binlog | default(1) | int }}"
mysql_io_capacity: "{{ innodb_io_capacity | default(2000) | int }}"
mysql_long_query_time: "{{ long_query_time | default(1) }}"
mysql_binlog_expire_logs_seconds: "{{ binlog_expire_logs_seconds | default(604800) | int }}"
mysql_max_binlog_size: "{{ max_binlog_size | default('256M') }}"
mgr_group_name: "{{ group_replication_group_name | default('aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa') }}"
# --- expected node count per topology ---
mysql_expected_hosts: "{{ {'standalone': 1, 'primary_replica': 2, 'mgr_3': 3}[topology | default('standalone')] }}"
# --- secrets (injected via environment) ---
mysql_root_password_value: "{{ lookup('ansible.builtin.env', 'XINFRA_MYSQL_ROOT_PASSWORD') }}"
mysql_admin_password_value: "{{ lookup('ansible.builtin.env', 'XINFRA_MYSQL_ADMIN_PASSWORD') }}"
pre_tasks:
- name: Validate prototype parameters
- name: Validate delivery parameters
ansible.builtin.assert:
that:
- topology == 'standalone'
- topology in ['standalone', 'primary_replica', 'mgr_3']
- mysql_instance is match('^[a-z0-9][a-z0-9-]{0,62}$')
- mysql_version_value in mysql_package_map
- (mysql_port_value | int) >= 1024
- (mysql_port_value | int) <= 65535
- (mysql_memory_mb_value | int) >= 1024
- (mysql_memory_mb_value | int) <= 4096
- (mysql_storage_gb_value | int) >= 10
- (mysql_storage_gb_value | int) <= 100
- mysql_data_disk is match('^/')
- (mysql_port_value | int) >= 13306
- (mysql_port_value | int) <= 13999
- (mysql_memory_mb_value | int) >= 2048
- (mysql_memory_mb_value | int) <= 65536
- (mysql_storage_gb_value | int) >= 20
- (mysql_storage_gb_value | int) <= 2000
- (mysql_lower_case_table_names | int) in [0, 1]
- mysql_root_password_value | length >= 16
- mysql_admin_password_value | length >= 16
fail_msg: The first machine prototype only supports safe standalone parameters
fail_msg: >-
Delivery parameters out of the supported target-state whitelist
(topology / version-package-map / port pool 13306-13999 / memory 2-64G / storage 20-2000G).
quiet: true
no_log: true
- name: Validate topology host count
ansible.builtin.assert:
that:
- ansible_play_hosts_all | length == 1
fail_msg: The selected topology does not match the target host count
- (ansible_play_hosts_all | length | int) == (mysql_expected_hosts | int)
fail_msg: "topology={{ topology }} expects {{ mysql_expected_hosts }} host(s), got {{ ansible_play_hosts_all | length }}"
run_once: true
- name: Check port ownership
ansible.builtin.shell: |
set -o pipefail
if ss -lntH "sport = :{{ mysql_port_value }}" | grep -q .; then
systemctl is-active --quiet "mysql-delivery@{{ mysql_instance }}.service"
fi
args:
- name: Probe target-host port occupancy (SQL + GR)
# cmd 字典形式不经过 free-form split_args 解析,避免引号/Jinja 块导致的解析失败。
ansible.builtin.shell:
cmd: |
set -o pipefail
for p in {{ mysql_probe_ports | join(' ') }}; do
if ss -lntH "sport = :${p}" | grep -q .; then
# already listening: only tolerated when owned by this instance service
if ! systemctl is-active --quiet "mysql-delivery@{{ mysql_instance }}.service"; then
echo "port ${p} already in use on target host" >&2
exit 3
fi
fi
done
executable: /bin/bash
vars:
mysql_probe_ports: "{{ [mysql_port_value, mysql_gr_port_value] if topology == 'mgr_3' else [mysql_port_value] }}"
changed_when: false
- name: Read currently available memory
- name: Read currently available memory (point-in-time guard)
ansible.builtin.shell: awk '/^MemAvailable:/ { print int($2 / 1024) }' /proc/meminfo
args:
executable: /bin/bash
register: mysql_available_memory
changed_when: false
- name: Check available memory and disk
- name: Resolve data-disk mountpoint available space
ansible.builtin.set_fact:
mysql_mount_avail: >-
{{ (ansible_mounts | selectattr('mount', 'equalto', mysql_data_disk)
| map(attribute='size_available') | first)
| default(ansible_mounts | selectattr('mount', 'equalto', '/')
| map(attribute='size_available') | first) }}
- name: Check available memory and data-disk space
ansible.builtin.assert:
that:
- (mysql_available_memory.stdout | int) >= (mysql_memory_mb_value | int)
- (ansible_mounts | selectattr('mount', 'equalto', '/') | map(attribute='size_available') | first | int) >= (mysql_storage_gb_value | int) * 1073741824
fail_msg: Target host does not have enough available memory or disk
- (mysql_mount_avail | int) >= (mysql_storage_gb_value | int) * 1073741824
fail_msg: >-
Target host lacks memory or free space on {{ mysql_data_disk }};
host-wide Σ-quota budgeting is the platform's responsibility, this is a last-resort guard.
tasks:
- name: Detect an existing Ubuntu MySQL package
ansible.builtin.command: dpkg-query -W mysql-server
- name: Detect an existing MySQL server package
ansible.builtin.command: dpkg-query -W -f '${Package}=${Version}\n' mysql-server mysql-community-server
register: mysql_package_before
failed_when: false
changed_when: false
- name: Install Ubuntu MySQL package
- name: Resolve the installed MySQL server version
ansible.builtin.set_fact:
mysql_installed_version: >-
{{ (mysql_package_before.stdout_lines | select('search', '=') | list
| first | default('')).split('=') | last }}
# /usr 下的 mysqld/mysql 二进制全机共享,一台主机只能承载一个 MySQL 版本系列。
- name: Enforce host-level MySQL series consistency
ansible.builtin.assert:
that:
- mysql_installed_version == '' or mysql_installed_version.startswith(mysql_version_value ~ '.')
fail_msg: >-
Host already runs MySQL {{ mysql_installed_version }} but {{ mysql_version_value }} was requested;
native binaries under /usr are shared host-wide, so one host serves exactly one MySQL series.
- name: Install the MySQL APT repository signing key
ansible.builtin.get_url:
url: https://repo.mysql.com/RPM-GPG-KEY-mysql-2023
dest: /etc/apt/keyrings/mysql.asc
owner: root
group: root
mode: '0644'
when: mysql_repo_component != ''
- name: Configure the MySQL APT repository component
ansible.builtin.apt_repository:
repo: >-
deb [signed-by=/etc/apt/keyrings/mysql.asc]
http://repo.mysql.com/apt/ubuntu {{ ansible_distribution_release }} {{ mysql_repo_component }}
filename: xinfra-mysql-delivery
state: present
when: mysql_repo_component != ''
- name: Install the MySQL server package
ansible.builtin.apt:
name: "{{ mysql_package_name }}"
state: present
@@ -88,14 +192,14 @@
name: mysql.service
state: stopped
enabled: false
when: mysql_package_before.rc != 0
when: mysql_installed_version == ''
- name: Check for the bundled MySQL AppArmor profile
ansible.builtin.stat:
path: /etc/apparmor.d/usr.sbin.mysqld
register: mysql_apparmor_profile
- name: Allow the delivery data and run paths in the MySQL AppArmor profile
- name: Authorize the delivery paths in the MySQL AppArmor profile
ansible.builtin.copy:
dest: /etc/apparmor.d/local/usr.sbin.mysqld
owner: root
@@ -103,8 +207,8 @@
mode: '0644'
content: |
# Managed by XINFRA MySQL delivery - grant per-instance native paths
/data/mysql-delivery/ r,
/data/mysql-delivery/** rwk,
{{ mysql_data_disk }}/mysql-delivery/ r,
{{ mysql_data_disk }}/mysql-delivery/** rwk,
/run/mysql-delivery-*/ rw,
/run/mysql-delivery-*/** rwk,
when: mysql_apparmor_profile.stat.exists
@@ -115,7 +219,7 @@
when: mysql_apparmor_profile.stat.exists and mysql_apparmor_local.changed
changed_when: true
- name: Create instance directories
- name: Create instance directories (mount-point-agnostic layout)
ansible.builtin.file:
path: "{{ item.path }}"
state: directory
@@ -125,9 +229,12 @@
loop:
- { path: /etc/mysql/mysql-delivery, owner: root, group: mysql, mode: '0750' }
- { path: "{{ mysql_install_dir }}", owner: root, group: root, mode: '0755' }
- { path: "{{ mysql_base_dir }}", owner: mysql, group: mysql, mode: '0750' }
- { path: "{{ mysql_data_dir }}", owner: mysql, group: mysql, mode: '0750' }
- { path: "{{ mysql_log_dir }}", owner: mysql, group: mysql, mode: '0750' }
- { path: "{{ mysql_run_dir }}", owner: mysql, group: mysql, mode: '0755' }
- { path: "{{ mysql_binlog_dir }}", owner: mysql, group: mysql, mode: '0750' }
- { path: "{{ mysql_redo_dir }}", owner: mysql, group: mysql, mode: '0750' }
- { path: "{{ mysql_tmp_dir }}", owner: mysql, group: mysql, mode: '0750' }
- name: Link native binaries into the instance directory
ansible.builtin.file:
@@ -146,6 +253,33 @@
{{ 'standalone' if topology == 'standalone' else
('primary' if ansible_play_hosts_all.index(inventory_hostname) == 0 else
('replica' if topology == 'primary_replica' else 'mgr')) }}
# host-wide unique: platform may pass explicit mysql_server_id; otherwise derive
# port + node index (ports are unique per host in the pool model).
mysql_server_id_value: >-
{{ mysql_server_id | default((mysql_port_value | int)
+ (ansible_play_hosts_all.index(inventory_hostname))) | int }}
- name: Resolve memory tier (GB)
ansible.builtin.set_fact:
mysql_memory_gb: "{{ ((mysql_memory_mb_value | int) // 1024) | int }}"
- name: Resolve linkage-derived defaults (illustrative tiers, pending hardware calibration)
ansible.builtin.set_fact:
mysql_max_connections: >-
{{ (max_connections | int) if (max_connections is defined and (max_connections | string) != 'auto')
else (200 if (mysql_memory_gb | int) <= 2
else 500 if (mysql_memory_gb | int) <= 4
else 1000 if (mysql_memory_gb | int) <= 8
else 2000 if (mysql_memory_gb | int) <= 16
else 4000 if (mysql_memory_gb | int) <= 32
else 8000 if (mysql_memory_gb | int) <= 64
else 16000) }}
mysql_redo_capacity: >-
{{ innodb_redo_log_capacity if (innodb_redo_log_capacity is defined)
else ('128M' if (mysql_memory_gb | int) <= 4
else '256M' if (mysql_memory_gb | int) <= 16
else '512M' if (mysql_memory_gb | int) <= 32
else '1G') }}
- name: Write instance configuration
ansible.builtin.template:
@@ -193,32 +327,33 @@
timeout: 60
- name: Configure local administrative accounts
ansible.builtin.shell: |
set -euo pipefail
client_file="$(mktemp)"
sql_file="$(mktemp)"
trap 'rm -f "$client_file" "$sql_file"' EXIT
chmod 600 "$client_file" "$sql_file"
cat >"$client_file" <<'EOF'
[client]
user=root
password={{ mysql_root_password_value }}
socket={{ mysql_run_dir }}/mysql.sock
EOF
if ! /usr/bin/mysql --defaults-extra-file="$client_file" -e 'SELECT 1' >/dev/null 2>&1; then
# cmd 字典形式不经过 free-form split_args 解析,heredoc SQL 中的奇数个单引号才不会报错。
ansible.builtin.shell:
cmd: |
set -euo pipefail
client_file="$(mktemp)"
sql_file="$(mktemp)"
trap 'rm -f "$client_file" "$sql_file"' EXIT
chmod 600 "$client_file" "$sql_file"
cat >"$client_file" <<'EOF'
[client]
user=root
password={{ mysql_root_password_value }}
socket={{ mysql_run_dir }}/mysql.sock
EOF
if ! /usr/bin/mysql --defaults-extra-file="$client_file" -e 'SELECT 1' >/dev/null 2>&1; then
cat >"$sql_file" <<'EOF'
ALTER USER 'root'@'localhost' IDENTIFIED BY '{{ mysql_root_password_value | replace("'", "''") }}';
EOF
/usr/bin/mysql --protocol=socket --socket={{ mysql_run_dir }}/mysql.sock -uroot <"$sql_file"
fi
cat >"$sql_file" <<'EOF'
ALTER USER 'root'@'localhost' IDENTIFIED BY '{{ mysql_root_password_value | replace("'", "''") }}';
EOF
/usr/bin/mysql --protocol=socket --socket={{ mysql_run_dir }}/mysql.sock -uroot <"$sql_file"
fi
cat >"$sql_file" <<'EOF'
CREATE USER IF NOT EXISTS 'xinfra_admin'@'%' IDENTIFIED BY '{{ mysql_admin_password_value | replace("'", "''") }}';
ALTER USER 'xinfra_admin'@'%' IDENTIFIED BY '{{ mysql_admin_password_value | replace("'", "''") }}';
GRANT ALL PRIVILEGES ON *.* TO 'xinfra_admin'@'%' WITH GRANT OPTION;
FLUSH PRIVILEGES;
EOF
/usr/bin/mysql --defaults-extra-file="$client_file" <"$sql_file"
args:
CREATE USER IF NOT EXISTS 'xinfra_admin'@'%' IDENTIFIED BY '{{ mysql_admin_password_value | replace("'", "''") }}';
ALTER USER 'xinfra_admin'@'%' IDENTIFIED BY '{{ mysql_admin_password_value | replace("'", "''") }}';
GRANT ALL PRIVILEGES ON *.* TO 'xinfra_admin'@'%' WITH GRANT OPTION;
FLUSH PRIVILEGES;
EOF
/usr/bin/mysql --defaults-extra-file="$client_file" <"$sql_file"
executable: /bin/bash
changed_when: false
no_log: true
@@ -229,6 +364,15 @@
port: "{{ mysql_port_value }}"
timeout: 30
- name: Note pending HA runtime orchestration
ansible.builtin.debug:
msg: >-
topology={{ topology }} deployed with HA-ready config (GTID/binlog/relay/GR settings in place),
but replication wiring (CHANGE REPLICATION SOURCE) and Group Replication bootstrap
(START GROUP_REPLICATION) are not yet automated — nodes start config-ready only.
when: topology != 'standalone'
run_once: true
handlers:
- name: Restart MySQL delivery instance
ansible.builtin.systemd_service:
+58 -11
View File
@@ -1,35 +1,82 @@
# Managed by XINFRA MySQL delivery - generated, do not edit by hand
# instance={{ mysql_instance }} version={{ mysql_version_value }} topology={{ topology }}
[client]
socket={{ mysql_run_dir }}/mysql.sock
port={{ mysql_port_value }}
[mysqld]
user=mysql
basedir=/usr
datadir={{ mysql_data_dir }}
tmpdir={{ mysql_tmp_dir }}
socket={{ mysql_run_dir }}/mysql.sock
pid-file={{ mysql_run_dir }}/mysql.pid
port={{ mysql_port_value }}
bind-address=0.0.0.0
mysqlx=0
server-id={{ 101 + mysql_node_index | int }}
skip-name-resolve=ON
server-id={{ mysql_server_id_value }}
local-infile=OFF
# --- charset / collation / timezone / identifier case ---
character-set-server={{ mysql_character_set }}
collation-server={{ mysql_collation }}
default-time-zone={{ mysql_timezone }}
lower-case-table-names={{ mysql_lower_case_table_names }}
# --- error / slow log ---
log-error={{ mysql_log_dir }}/error.log
slow-query-log=ON
slow-query-log-file={{ mysql_log_dir }}/slow.log
skip-name-resolve=ON
max-connections=100
innodb-buffer-pool-size={{ ((mysql_memory_mb_value | int) * 55 / 100) | int }}M
innodb-log-file-size=128M
log-bin={{ mysql_log_dir }}/mysql-bin
long-query-time={{ mysql_long_query_time }}
# --- binlog (PITR + replication base) ---
log-bin={{ mysql_binlog_dir }}/binlog
binlog-format=ROW
sync-binlog={{ mysql_sync_binlog }}
max-binlog-size={{ mysql_max_binlog_size }}
binlog-expire-logs-seconds={{ mysql_binlog_expire_logs_seconds }}
{% if topology != 'standalone' %}
gtid-mode=ON
enforce-gtid-consistency=ON
relay-log={{ mysql_log_dir }}/relay-bin
relay-log={{ mysql_binlog_dir }}/relay-bin
{% endif %}
# --- InnoDB core ---
innodb-buffer-pool-size={{ ((mysql_memory_mb_value | int) * 55 / 100) | int }}M
innodb-flush-method=O_DIRECT
innodb-flush-log-at-trx-commit={{ mysql_flush_log_at_trx_commit }}
innodb-io-capacity={{ mysql_io_capacity }}
max-connections={{ mysql_max_connections }}
# --- redo log (version-sensitive) ---
innodb-log-group-home-dir={{ mysql_redo_dir }}
{% if mysql_version_value in ['8.0', '8.4'] %}
innodb-redo-log-capacity={{ mysql_redo_capacity }}
{% else %}
innodb-log-file-size={{ mysql_redo_capacity }}
{% endif %}
# --- X Protocol disabled (version-sensitive) ---
{% if mysql_version_value in ['8.0', '8.4'] %}
mysqlx=0
{% elif mysql_version_value == '5.7' %}
loose-mysqlx=0
{% endif %}
{% if topology == 'mgr_3' %}
# --- Group Replication (config-ready; runtime bootstrap handled out-of-band) ---
plugin-load-add=group_replication.so
{% if mysql_version_value != '8.4' %}
{# deprecated since 8.0.26 and removed in 8.3+; only inject for older series #}
transaction-write-set-extraction=XXHASH64
{% endif %}
loose-group-replication-group-name={{ mgr_group_name }}
loose-group-replication-start-on-boot=OFF
loose-group-replication-local-address={{ ansible_host | default(inventory_hostname) }}:{{ mysql_port_value | int + 10000 }}
loose-group-replication-group-seeds={% for host in ansible_play_hosts_all %}{{ hostvars[host].ansible_host | default(host) }}:{{ mysql_port_value | int + 10000 }}{% if not loop.last %},{% endif %}{% endfor %}
loose-group-replication-local-address={{ ansible_host | default(inventory_hostname) }}:{{ mysql_gr_port_value }}
loose-group-replication-group-seeds={% for host in ansible_play_hosts_all %}{{ hostvars[host].ansible_host | default(host) }}:{{ mysql_gr_port_value }}{% if not loop.last %},{% endif %}{% endfor %}
loose-group-replication-ip-allowlist={% for host in ansible_play_hosts_all %}{{ hostvars[host].ansible_host | default(host) }}{% if not loop.last %},{% endif %}{% endfor %}
loose-group-replication-single-primary-mode=ON
loose-group-replication-enforce-update-everywhere-checks=OFF
{% endif %}
+4
View File
@@ -19,9 +19,13 @@ export interface CreateMySQLDeliveryPayload {
topology: string
mysql_port?: number
data_disk: string
target_host?: string
cpu_cores: number
memory_gb: number
storage_gb: number
cpu_milli: number
memory_mi: number
storage_gi: number
param_template: string
timezone: string
lower_case_table_names: number
+3
View File
@@ -959,6 +959,9 @@ function mysqlDeliveryPayload(businessLineId: number) {
cpu_cores: resources.cpuCores,
memory_gb: resources.memoryGb,
storage_gb: parseStorageGb(deliveryForm.disk),
cpu_milli: resources.cpuCores * 1000,
memory_mi: resources.memoryGb * 1024,
storage_gi: parseStorageGb(deliveryForm.disk),
param_template: deliveryForm.paramTemplate,
timezone: deliveryForm.timezone,
lower_case_table_names: deliveryForm.lowerCaseTableNames,
+4
View File
@@ -22,6 +22,10 @@ DELIVERY_CALLBACK_BASE_URL=http://authserver-backend.authserver.svc.cluster.loca
DELIVERY_RESERVATION_TTL_MINUTES=120
DELIVERY_GLOBAL_LIMIT=2
DELIVERY_TARGET_LIMIT=2
# 单机 MySQL 实例数上限(同机多实例,容量由配额 + playbook 实机守卫兜底)
DELIVERY_HOST_INSTANCE_LIMIT=4
# 数据盘挂载点白名单(逗号分隔,第一项为默认值)
DELIVERY_DATA_DISKS=/data
AWX_BASE_URL=
AWX_TOKEN=
AWX_USERNAME=
+61 -118
View File
@@ -152,7 +152,7 @@ const docTemplate = `{
"BearerAuth": []
}
],
"description": "返回所有已启用的部署目标(如 k8s 集群、主机池)",
"description": "从 AWX 动态返回可用 Job Template 及其 Inventory hosts",
"produces": [
"application/json"
],
@@ -160,6 +160,14 @@ const docTemplate = `{
"delivery"
],
"summary": "获取可用部署目标",
"parameters": [
{
"type": "string",
"description": "组件过滤,例如 mysql",
"name": "component",
"in": "query"
}
],
"responses": {
"200": {
"description": "items: 部署目标数组",
@@ -176,64 +184,6 @@ const docTemplate = `{
}
}
}
},
"post": {
"security": [
{
"BearerAuth": []
}
],
"description": "管理员创建新的部署目标(目前仅支持 k8s 类型)",
"consumes": [
"application/json"
],
"produces": [
"application/json"
],
"tags": [
"delivery"
],
"summary": "创建部署目标",
"parameters": [
{
"description": "目标配置",
"name": "body",
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/handler.targetPayload"
}
}
],
"responses": {
"201": {
"description": "目标已创建",
"schema": {
"$ref": "#/definitions/model.DeploymentTarget"
}
},
"400": {
"description": "参数错误",
"schema": {
"type": "object",
"additionalProperties": true
}
},
"401": {
"description": "未授权",
"schema": {
"type": "object",
"additionalProperties": true
}
},
"409": {
"description": "名称冲突",
"schema": {
"type": "object",
"additionalProperties": true
}
}
}
}
},
"/auth/api/v1/delivery/tasks": {
@@ -254,6 +204,7 @@ const docTemplate = `{
"parameters": [
{
"type": "integer",
"format": "int64",
"description": "业务线 ID 过滤",
"name": "business_line_id",
"in": "query"
@@ -689,65 +640,6 @@ const docTemplate = `{
}
}
},
"handler.targetPayload": {
"type": "object",
"required": [
"awx_inventory_id",
"awx_template_id",
"name",
"target_type"
],
"properties": {
"awx_inventory_id": {
"type": "integer"
},
"awx_template_id": {
"type": "integer"
},
"metadata": {
"type": "object",
"additionalProperties": {}
},
"name": {
"type": "string"
},
"target_type": {
"type": "string"
}
}
},
"model.DeploymentTarget": {
"type": "object",
"properties": {
"awx_inventory_id": {
"type": "integer"
},
"awx_template_id": {
"type": "integer"
},
"created_at": {
"type": "string"
},
"enabled": {
"type": "boolean"
},
"id": {
"type": "integer"
},
"metadata": {
"type": "string"
},
"name": {
"type": "string"
},
"target_type": {
"type": "string"
},
"updated_at": {
"type": "string"
}
}
},
"model.ResourceQuota": {
"type": "object",
"properties": {
@@ -816,18 +708,55 @@ const docTemplate = `{
"target_id"
],
"properties": {
"binlog_expire_logs_seconds": {
"type": "integer"
},
"business_line_id": {
"type": "integer"
},
"character_set": {
"type": "string"
},
"collation": {
"type": "string"
},
"cpu_milli": {
"type": "integer"
},
"data_disk": {
"type": "string"
},
"innodb_flush_log_at_trx_commit": {
"type": "integer"
},
"innodb_io_capacity": {
"type": "integer"
},
"innodb_redo_log_capacity": {
"type": "string"
},
"instance_name": {
"type": "string"
},
"long_query_time": {
"type": "number"
},
"lower_case_table_names": {
"type": "integer"
},
"max_binlog_size": {
"type": "string"
},
"max_connections": {
"description": "高级参数(选填,零值视为未设置)",
"type": "string"
},
"memory_mi": {
"type": "integer"
},
"mysql_port": {
"type": "integer"
},
"mysql_version": {
"type": "string"
},
@@ -837,8 +766,22 @@ const docTemplate = `{
"storage_gi": {
"type": "integer"
},
"sync_binlog": {
"type": "integer"
},
"target_host": {
"description": "调度控制(选填):点名候选池内主机跳过自动选机,端口/配额/实机守卫照常执行",
"type": "string"
},
"target_id": {
"type": "integer"
},
"timezone": {
"description": "数据库配置(选填,缺省由 playbook 基线兜底)",
"type": "string"
},
"topology": {
"type": "string"
}
}
}
+61 -118
View File
@@ -145,7 +145,7 @@
"BearerAuth": []
}
],
"description": "返回所有已启用的部署目标(如 k8s 集群、主机池)",
"description": "从 AWX 动态返回可用 Job Template 及其 Inventory hosts",
"produces": [
"application/json"
],
@@ -153,6 +153,14 @@
"delivery"
],
"summary": "获取可用部署目标",
"parameters": [
{
"type": "string",
"description": "组件过滤,例如 mysql",
"name": "component",
"in": "query"
}
],
"responses": {
"200": {
"description": "items: 部署目标数组",
@@ -169,64 +177,6 @@
}
}
}
},
"post": {
"security": [
{
"BearerAuth": []
}
],
"description": "管理员创建新的部署目标(目前仅支持 k8s 类型)",
"consumes": [
"application/json"
],
"produces": [
"application/json"
],
"tags": [
"delivery"
],
"summary": "创建部署目标",
"parameters": [
{
"description": "目标配置",
"name": "body",
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/handler.targetPayload"
}
}
],
"responses": {
"201": {
"description": "目标已创建",
"schema": {
"$ref": "#/definitions/model.DeploymentTarget"
}
},
"400": {
"description": "参数错误",
"schema": {
"type": "object",
"additionalProperties": true
}
},
"401": {
"description": "未授权",
"schema": {
"type": "object",
"additionalProperties": true
}
},
"409": {
"description": "名称冲突",
"schema": {
"type": "object",
"additionalProperties": true
}
}
}
}
},
"/auth/api/v1/delivery/tasks": {
@@ -247,6 +197,7 @@
"parameters": [
{
"type": "integer",
"format": "int64",
"description": "业务线 ID 过滤",
"name": "business_line_id",
"in": "query"
@@ -682,65 +633,6 @@
}
}
},
"handler.targetPayload": {
"type": "object",
"required": [
"awx_inventory_id",
"awx_template_id",
"name",
"target_type"
],
"properties": {
"awx_inventory_id": {
"type": "integer"
},
"awx_template_id": {
"type": "integer"
},
"metadata": {
"type": "object",
"additionalProperties": {}
},
"name": {
"type": "string"
},
"target_type": {
"type": "string"
}
}
},
"model.DeploymentTarget": {
"type": "object",
"properties": {
"awx_inventory_id": {
"type": "integer"
},
"awx_template_id": {
"type": "integer"
},
"created_at": {
"type": "string"
},
"enabled": {
"type": "boolean"
},
"id": {
"type": "integer"
},
"metadata": {
"type": "string"
},
"name": {
"type": "string"
},
"target_type": {
"type": "string"
},
"updated_at": {
"type": "string"
}
}
},
"model.ResourceQuota": {
"type": "object",
"properties": {
@@ -809,18 +701,55 @@
"target_id"
],
"properties": {
"binlog_expire_logs_seconds": {
"type": "integer"
},
"business_line_id": {
"type": "integer"
},
"character_set": {
"type": "string"
},
"collation": {
"type": "string"
},
"cpu_milli": {
"type": "integer"
},
"data_disk": {
"type": "string"
},
"innodb_flush_log_at_trx_commit": {
"type": "integer"
},
"innodb_io_capacity": {
"type": "integer"
},
"innodb_redo_log_capacity": {
"type": "string"
},
"instance_name": {
"type": "string"
},
"long_query_time": {
"type": "number"
},
"lower_case_table_names": {
"type": "integer"
},
"max_binlog_size": {
"type": "string"
},
"max_connections": {
"description": "高级参数(选填,零值视为未设置)",
"type": "string"
},
"memory_mi": {
"type": "integer"
},
"mysql_port": {
"type": "integer"
},
"mysql_version": {
"type": "string"
},
@@ -830,8 +759,22 @@
"storage_gi": {
"type": "integer"
},
"sync_binlog": {
"type": "integer"
},
"target_host": {
"description": "调度控制(选填):点名候选池内主机跳过自动选机,端口/配额/实机守卫照常执行",
"type": "string"
},
"target_id": {
"type": "integer"
},
"timezone": {
"description": "数据库配置(选填,缺省由 playbook 基线兜底)",
"type": "string"
},
"topology": {
"type": "string"
}
}
}
+42 -79
View File
@@ -22,46 +22,6 @@ definitions:
- storage_gi
- target_id
type: object
handler.targetPayload:
properties:
awx_inventory_id:
type: integer
awx_template_id:
type: integer
metadata:
additionalProperties: {}
type: object
name:
type: string
target_type:
type: string
required:
- awx_inventory_id
- awx_template_id
- name
- target_type
type: object
model.DeploymentTarget:
properties:
awx_inventory_id:
type: integer
awx_template_id:
type: integer
created_at:
type: string
enabled:
type: boolean
id:
type: integer
metadata:
type: string
name:
type: string
target_type:
type: string
updated_at:
type: string
type: object
model.ResourceQuota:
properties:
business_line_id:
@@ -101,22 +61,57 @@ definitions:
type: object
service.MySQLDeliveryInput:
properties:
binlog_expire_logs_seconds:
type: integer
business_line_id:
type: integer
character_set:
type: string
collation:
type: string
cpu_milli:
type: integer
data_disk:
type: string
innodb_flush_log_at_trx_commit:
type: integer
innodb_io_capacity:
type: integer
innodb_redo_log_capacity:
type: string
instance_name:
type: string
long_query_time:
type: number
lower_case_table_names:
type: integer
max_binlog_size:
type: string
max_connections:
description: 高级参数(选填,零值视为未设置)
type: string
memory_mi:
type: integer
mysql_port:
type: integer
mysql_version:
type: string
namespace:
type: string
storage_gi:
type: integer
sync_binlog:
type: integer
target_host:
description: 调度控制(选填):点名候选池内主机跳过自动选机,端口/配额/实机守卫照常执行
type: string
target_id:
type: integer
timezone:
description: 数据库配置(选填,缺省由 playbook 基线兜底)
type: string
topology:
type: string
required:
- business_line_id
- cpu_milli
@@ -220,7 +215,12 @@ paths:
- delivery
/auth/api/v1/delivery/targets:
get:
description: 返回所有已启用的部署目标(如 k8s 集群、主机池)
description: 从 AWX 动态返回可用 Job Template 及其 Inventory hosts
parameters:
- description: 组件过滤,例如 mysql
in: query
name: component
type: string
produces:
- application/json
responses:
@@ -239,49 +239,12 @@ paths:
summary: 获取可用部署目标
tags:
- delivery
post:
consumes:
- application/json
description: 管理员创建新的部署目标(目前仅支持 k8s 类型)
parameters:
- description: 目标配置
in: body
name: body
required: true
schema:
$ref: '#/definitions/handler.targetPayload'
produces:
- application/json
responses:
"201":
description: 目标已创建
schema:
$ref: '#/definitions/model.DeploymentTarget'
"400":
description: 参数错误
schema:
additionalProperties: true
type: object
"401":
description: 未授权
schema:
additionalProperties: true
type: object
"409":
description: 名称冲突
schema:
additionalProperties: true
type: object
security:
- BearerAuth: []
summary: 创建部署目标
tags:
- delivery
/auth/api/v1/delivery/tasks:
get:
description: 返回当前用户可见的交付任务列表(管理员可见全部)
parameters:
- description: 业务线 ID 过滤
format: int64
in: query
name: business_line_id
type: integer
+116 -112
View File
@@ -14,62 +14,64 @@ type OAuthClient struct {
}
type Config struct {
AppEnv string
HTTPAddr string
PublicBaseURL string
MySQLDSN string
AutoMigrate bool
SSOEnabled bool
JWTSecret string
JWTIssuer string
JWTTTLMinutes int
SAMLEntityID string
SAMLACSURL string
SAMLSPCert string
SAMLSPKey string
SAMLIDPMetaURL string
SAMLLogoutURL string
WayenLoginURL string
WayenTargetURL string
WayenUsernameKey string
WayenPasswordKey string
WayenLoginFormat string
WayenLoginValue string
WayenOAuthRef string
WayenOAuthLoginURL string
WayneAPIBaseURL string
WayneAdminUsername string
WayneAdminPassword string
WayneTokenTTLMinutes int
WayneInternalAPIBaseURL string
WayneServiceName string
WayneServiceAPISecretKey string
OAuthClientID string
OAuthClientSecret string
OAuthRedirectURI string
OAuthCodeTTLSeconds int
OIDCIssuer string
OIDCAuthorizeURL string
OIDCTokenURL string
OIDCUserInfoURL string
OIDCJWKSURL string
CloudDMClientID string
CloudDMClientSecret string
CloudDMRedirectURI string
CloudDMTargetURL string
CloudDMRegisterURL string
CloudDMAPIToken string
AWXBaseURL string
AWXToken string
AWXUsername string
AWXPassword string
AWXWebhookToken string
DeliverySchedulerEnabled bool
DeliveryDispatchSeconds int
DeliveryCallbackBaseURL string
ReservationTTLMinutes int
DeliveryGlobalLimit int
DeliveryTargetLimit int
AppEnv string
HTTPAddr string
PublicBaseURL string
MySQLDSN string
AutoMigrate bool
SSOEnabled bool
JWTSecret string
JWTIssuer string
JWTTTLMinutes int
SAMLEntityID string
SAMLACSURL string
SAMLSPCert string
SAMLSPKey string
SAMLIDPMetaURL string
SAMLLogoutURL string
WayenLoginURL string
WayenTargetURL string
WayenUsernameKey string
WayenPasswordKey string
WayenLoginFormat string
WayenLoginValue string
WayenOAuthRef string
WayenOAuthLoginURL string
WayneAPIBaseURL string
WayneAdminUsername string
WayneAdminPassword string
WayneTokenTTLMinutes int
WayneInternalAPIBaseURL string
WayneServiceName string
WayneServiceAPISecretKey string
OAuthClientID string
OAuthClientSecret string
OAuthRedirectURI string
OAuthCodeTTLSeconds int
OIDCIssuer string
OIDCAuthorizeURL string
OIDCTokenURL string
OIDCUserInfoURL string
OIDCJWKSURL string
CloudDMClientID string
CloudDMClientSecret string
CloudDMRedirectURI string
CloudDMTargetURL string
CloudDMRegisterURL string
CloudDMAPIToken string
AWXBaseURL string
AWXToken string
AWXUsername string
AWXPassword string
AWXWebhookToken string
DeliverySchedulerEnabled bool
DeliveryDispatchSeconds int
DeliveryCallbackBaseURL string
ReservationTTLMinutes int
DeliveryGlobalLimit int
DeliveryTargetLimit int
DeliveryHostInstanceLimit int
DeliveryDataDisks []string
}
func Load() Config {
@@ -82,62 +84,64 @@ func Load() Config {
oidcIssuer = strings.TrimRight(oidcIssuer, "/")
return Config{
AppEnv: env("APP_ENV", "dev"),
HTTPAddr: httpAddr,
PublicBaseURL: publicBaseURL,
MySQLDSN: env("MYSQL_DSN", "auth:auth@tcp(127.0.0.1:3306)/authserver?charset=utf8mb4&parseTime=True&loc=Local"),
AutoMigrate: envBool("AUTO_MIGRATE", true),
SSOEnabled: envBool("SSO_ENABLED", true),
JWTSecret: env("JWT_SECRET", "change-this-secret"),
JWTIssuer: env("JWT_ISSUER", "authserver"),
JWTTTLMinutes: envInt("JWT_TTL_MINUTES", 120),
SAMLEntityID: samlEntityID,
SAMLACSURL: samlACSURL,
SAMLSPCert: env("SAML_SP_CERT_FILE", "certs/sp.crt"),
SAMLSPKey: env("SAML_SP_KEY_FILE", "certs/sp.key"),
SAMLIDPMetaURL: env("SAML_IDP_METADATA_URL", "http://sso-internal.dev.qiniu.io/saml2/meta"),
SAMLLogoutURL: trimURL(env("SAML_LOGOUT_URL", "")),
WayenLoginURL: env("WAYEN_LOGIN_URL", ""),
WayenTargetURL: env("WAYEN_TARGET_URL", ""),
WayenUsernameKey: env("WAYEN_USERNAME_KEY", "email"),
WayenPasswordKey: env("WAYEN_PASSWORD_KEY", "password"),
WayenLoginFormat: env("WAYEN_LOGIN_FORMAT", "form"),
WayenLoginValue: env("WAYEN_LOGIN_VALUE", "email"),
WayenOAuthRef: env("WAYEN_OAUTH_REF", "/portal/namespace/1/app"),
WayenOAuthLoginURL: trimURL(env("WAYEN_OAUTH_LOGIN_URL", "")),
WayneAPIBaseURL: trimURL(env("WAYNE_API_BASE_URL", env("WAYNE_INTERNAL_API_BASE_URL", ""))),
WayneAdminUsername: env("WAYNE_ADMIN_USERNAME", ""),
WayneAdminPassword: env("WAYNE_ADMIN_PASSWORD", ""),
WayneTokenTTLMinutes: envInt("WAYNE_TOKEN_TTL_MINUTES", 1440),
WayneInternalAPIBaseURL: trimURL(env("WAYNE_INTERNAL_API_BASE_URL", "")),
WayneServiceName: env("WAYNE_SERVICE_NAME", "xinfra"),
WayneServiceAPISecretKey: env("WAYNE_SERVICE_API_SECRET_KEY", ""),
OAuthClientID: env("OAUTH_WAYNE_CLIENT_ID", "wayne"),
OAuthClientSecret: env("OAUTH_WAYNE_CLIENT_SECRET", "wayne-secret"),
OAuthRedirectURI: env("OAUTH_WAYNE_REDIRECT_URI", ""),
OAuthCodeTTLSeconds: envInt("OAUTH_CODE_TTL_SECONDS", 120),
OIDCIssuer: oidcIssuer,
OIDCAuthorizeURL: trimURL(env("OIDC_AUTHORIZATION_ENDPOINT", oidcIssuer+"/oauth/authorize")),
OIDCTokenURL: trimURL(env("OIDC_TOKEN_ENDPOINT", oidcIssuer+"/oauth/token")),
OIDCUserInfoURL: trimURL(env("OIDC_USERINFO_ENDPOINT", oidcIssuer+"/oauth/userinfo")),
OIDCJWKSURL: trimURL(env("OIDC_JWKS_URI", oidcIssuer+"/oauth/jwks")),
CloudDMClientID: env("OIDC_CLOUDDM_CLIENT_ID", "clouddm"),
CloudDMClientSecret: env("OIDC_CLOUDDM_CLIENT_SECRET", ""),
CloudDMRedirectURI: env("OIDC_CLOUDDM_REDIRECT_URI", ""),
CloudDMTargetURL: env("CLOUDDM_TARGET_URL", ""),
CloudDMRegisterURL: trimURL(env("CLOUDDM_REGISTER_URL", "")),
CloudDMAPIToken: env("CLOUDDM_API_TOKEN", ""),
AWXBaseURL: trimURL(env("AWX_BASE_URL", "")),
AWXToken: env("AWX_TOKEN", ""),
AWXUsername: env("AWX_USERNAME", ""),
AWXPassword: env("AWX_PASSWORD", ""),
AWXWebhookToken: env("AWX_WEBHOOK_TOKEN", ""),
DeliverySchedulerEnabled: envBool("DELIVERY_SCHEDULER_ENABLED", false),
DeliveryDispatchSeconds: envInt("DELIVERY_DISPATCH_SECONDS", 5),
DeliveryCallbackBaseURL: trimURL(env("DELIVERY_CALLBACK_BASE_URL", publicBaseURL)),
ReservationTTLMinutes: envInt("DELIVERY_RESERVATION_TTL_MINUTES", 120),
DeliveryGlobalLimit: envInt("DELIVERY_GLOBAL_LIMIT", 2),
DeliveryTargetLimit: envInt("DELIVERY_TARGET_LIMIT", 2),
AppEnv: env("APP_ENV", "dev"),
HTTPAddr: httpAddr,
PublicBaseURL: publicBaseURL,
MySQLDSN: env("MYSQL_DSN", "auth:auth@tcp(127.0.0.1:3306)/authserver?charset=utf8mb4&parseTime=True&loc=Local"),
AutoMigrate: envBool("AUTO_MIGRATE", true),
SSOEnabled: envBool("SSO_ENABLED", true),
JWTSecret: env("JWT_SECRET", "change-this-secret"),
JWTIssuer: env("JWT_ISSUER", "authserver"),
JWTTTLMinutes: envInt("JWT_TTL_MINUTES", 120),
SAMLEntityID: samlEntityID,
SAMLACSURL: samlACSURL,
SAMLSPCert: env("SAML_SP_CERT_FILE", "certs/sp.crt"),
SAMLSPKey: env("SAML_SP_KEY_FILE", "certs/sp.key"),
SAMLIDPMetaURL: env("SAML_IDP_METADATA_URL", "http://sso-internal.dev.qiniu.io/saml2/meta"),
SAMLLogoutURL: trimURL(env("SAML_LOGOUT_URL", "")),
WayenLoginURL: env("WAYEN_LOGIN_URL", ""),
WayenTargetURL: env("WAYEN_TARGET_URL", ""),
WayenUsernameKey: env("WAYEN_USERNAME_KEY", "email"),
WayenPasswordKey: env("WAYEN_PASSWORD_KEY", "password"),
WayenLoginFormat: env("WAYEN_LOGIN_FORMAT", "form"),
WayenLoginValue: env("WAYEN_LOGIN_VALUE", "email"),
WayenOAuthRef: env("WAYEN_OAUTH_REF", "/portal/namespace/1/app"),
WayenOAuthLoginURL: trimURL(env("WAYEN_OAUTH_LOGIN_URL", "")),
WayneAPIBaseURL: trimURL(env("WAYNE_API_BASE_URL", env("WAYNE_INTERNAL_API_BASE_URL", ""))),
WayneAdminUsername: env("WAYNE_ADMIN_USERNAME", ""),
WayneAdminPassword: env("WAYNE_ADMIN_PASSWORD", ""),
WayneTokenTTLMinutes: envInt("WAYNE_TOKEN_TTL_MINUTES", 1440),
WayneInternalAPIBaseURL: trimURL(env("WAYNE_INTERNAL_API_BASE_URL", "")),
WayneServiceName: env("WAYNE_SERVICE_NAME", "xinfra"),
WayneServiceAPISecretKey: env("WAYNE_SERVICE_API_SECRET_KEY", ""),
OAuthClientID: env("OAUTH_WAYNE_CLIENT_ID", "wayne"),
OAuthClientSecret: env("OAUTH_WAYNE_CLIENT_SECRET", "wayne-secret"),
OAuthRedirectURI: env("OAUTH_WAYNE_REDIRECT_URI", ""),
OAuthCodeTTLSeconds: envInt("OAUTH_CODE_TTL_SECONDS", 120),
OIDCIssuer: oidcIssuer,
OIDCAuthorizeURL: trimURL(env("OIDC_AUTHORIZATION_ENDPOINT", oidcIssuer+"/oauth/authorize")),
OIDCTokenURL: trimURL(env("OIDC_TOKEN_ENDPOINT", oidcIssuer+"/oauth/token")),
OIDCUserInfoURL: trimURL(env("OIDC_USERINFO_ENDPOINT", oidcIssuer+"/oauth/userinfo")),
OIDCJWKSURL: trimURL(env("OIDC_JWKS_URI", oidcIssuer+"/oauth/jwks")),
CloudDMClientID: env("OIDC_CLOUDDM_CLIENT_ID", "clouddm"),
CloudDMClientSecret: env("OIDC_CLOUDDM_CLIENT_SECRET", ""),
CloudDMRedirectURI: env("OIDC_CLOUDDM_REDIRECT_URI", ""),
CloudDMTargetURL: env("CLOUDDM_TARGET_URL", ""),
CloudDMRegisterURL: trimURL(env("CLOUDDM_REGISTER_URL", "")),
CloudDMAPIToken: env("CLOUDDM_API_TOKEN", ""),
AWXBaseURL: trimURL(env("AWX_BASE_URL", "")),
AWXToken: env("AWX_TOKEN", ""),
AWXUsername: env("AWX_USERNAME", ""),
AWXPassword: env("AWX_PASSWORD", ""),
AWXWebhookToken: env("AWX_WEBHOOK_TOKEN", ""),
DeliverySchedulerEnabled: envBool("DELIVERY_SCHEDULER_ENABLED", false),
DeliveryDispatchSeconds: envInt("DELIVERY_DISPATCH_SECONDS", 5),
DeliveryCallbackBaseURL: trimURL(env("DELIVERY_CALLBACK_BASE_URL", publicBaseURL)),
ReservationTTLMinutes: envInt("DELIVERY_RESERVATION_TTL_MINUTES", 120),
DeliveryGlobalLimit: envInt("DELIVERY_GLOBAL_LIMIT", 2),
DeliveryTargetLimit: envInt("DELIVERY_TARGET_LIMIT", 2),
DeliveryHostInstanceLimit: envInt("DELIVERY_HOST_INSTANCE_LIMIT", 4),
DeliveryDataDisks: splitCSV(env("DELIVERY_DATA_DISKS", "/data")),
}
}
+119 -54
View File
@@ -24,6 +24,7 @@ import (
)
var dnsLabelPattern = regexp.MustCompile(`^[a-z0-9](?:[-a-z0-9]*[a-z0-9])?$`)
var hostNamePattern = regexp.MustCompile(`^[a-zA-Z0-9](?:[-a-zA-Z0-9.]*[a-zA-Z0-9])?$`)
type MySQLDeliveryInput struct {
BusinessLineID uint64 `json:"business_line_id" binding:"required"`
@@ -34,9 +35,10 @@ type MySQLDeliveryInput struct {
Topology string `json:"topology"`
MySQLPort int `json:"mysql_port"`
DataDisk string `json:"data_disk"`
CPUCores int64 `json:"cpu_cores" binding:"required"`
MemoryGB int64 `json:"memory_gb" binding:"required"`
StorageGB int64 `json:"storage_gb" binding:"required"`
TargetHost string `json:"target_host"`
CPUCores int64 `json:"cpu_cores"`
MemoryGB int64 `json:"memory_gb"`
StorageGB int64 `json:"storage_gb"`
ParamTemplate string `json:"param_template"`
TimeZone string `json:"timezone"`
LowerCaseTableNames int `json:"lower_case_table_names"`
@@ -50,9 +52,9 @@ type MySQLDeliveryInput struct {
LongQueryTime float64 `json:"long_query_time"`
BinlogExpireLogsSeconds int64 `json:"binlog_expire_logs_seconds"`
MaxBinlogSize string `json:"max_binlog_size"`
CPUMilli int64 `json:"-"`
MemoryMi int64 `json:"-"`
StorageGi int64 `json:"-"`
CPUMilli int64 `json:"cpu_milli"`
MemoryMi int64 `json:"memory_mi"`
StorageGi int64 `json:"storage_gi"`
}
type deliveryPayload struct {
@@ -118,31 +120,51 @@ func parseTargetMetadata(raw string) targetMetadata {
return meta
}
// firstFreeHost 返回候选池中第一个未被占用的节点。
func firstFreeHost(hosts []targetHost, occupied []string) *targetHost {
taken := make(map[string]bool, len(occupied))
for _, h := range occupied {
taken[h] = true
// firstFreeHost 返回候选池中非失败任务数未达单机实例上限的第一个节点。
func firstFreeHost(hosts []targetHost, occupied []string, limit int) *targetHost {
if limit < 1 {
limit = 1
}
for i := range hosts {
if !taken[hosts[i].Name] {
if hostTaskCount(occupied, hosts[i].Name) < limit {
return &hosts[i]
}
}
return nil
}
func firstFreePort(occupied []int) int {
taken := make(map[int]bool, len(occupied))
for _, port := range occupied {
taken[port] = true
}
for port := 13306; port <= 13999; port++ {
if !taken[port] {
return port
func hostTaskCount(occupied []string, name string) int {
count := 0
for _, h := range occupied {
if h == name {
count++
}
}
return 0
return count
}
const (
mysqlPortPoolStart = 13306
mysqlPortPoolEnd = 13999
)
func allocatePort(requested int, used []int) (int, error) {
taken := make(map[int]bool, len(used))
for _, p := range used {
taken[p] = true
}
if requested != 0 {
if taken[requested] {
return 0, fmt.Errorf("mysql_port %d is already allocated on the target host", requested)
}
return requested, nil
}
for p := mysqlPortPoolStart; p <= mysqlPortPoolEnd; p++ {
if !taken[p] {
return p, nil
}
}
return 0, fmt.Errorf("mysql port pool %d-%d is exhausted on the target host", mysqlPortPoolStart, mysqlPortPoolEnd)
}
type DeliveryService struct {
@@ -228,8 +250,11 @@ func (s *DeliveryService) CreateTask(ctx context.Context, userID uint64, isAdmin
if idempotencyKey == "" || len(idempotencyKey) > 128 {
return nil, false, fmt.Errorf("Idempotency-Key header is required and must not exceed 128 characters")
}
if input.DataDisk == "" && len(s.cfg.DeliveryDataDisks) > 0 {
input.DataDisk = s.cfg.DeliveryDataDisks[0]
}
normalizeMySQLDeliveryInput(&input)
if err := validateDeliveryInput(input); err != nil {
if err := validateDeliveryInput(input, s.cfg.DeliveryDataDisks); err != nil {
return nil, false, err
}
@@ -290,9 +315,8 @@ func (s *DeliveryService) CreateTask(ctx context.Context, userID uint64, isAdmin
return &task, false, nil
}
var supportedMySQLVersions = map[string]bool{"8.0": true}
var supportedMySQLVersions = map[string]bool{"8.0": true, "8.4": true}
var supportedMySQLTopologies = map[string]bool{"standalone": true}
var supportedDataDisks = map[string]bool{"/data": true, "/disk1": true, "/mnt/vol-1": true}
var supportedParamTemplates = map[string]bool{"default": true, "high_performance": true, "high_safety": true}
var supportedCharacterSets = map[string]bool{"utf8mb4": true, "utf8": true, "gbk": true, "latin1": true}
var supportedCollations = map[string]bool{
@@ -301,6 +325,7 @@ var supportedCollations = map[string]bool{
}
var supportedRedoLogCapacity = map[string]bool{"auto": true, "128M": true, "256M": true, "512M": true, "1G": true}
var supportedMaxBinlogSize = map[string]bool{"128M": true, "256M": true, "512M": true, "1G": true}
var timezonePattern = regexp.MustCompile(`^([+-](0\d|1[0-4]):[0-5]\d|SYSTEM|[A-Za-z]+(?:/[A-Za-z0-9_+-]+)+)$`)
func normalizeMySQLDeliveryInput(input *MySQLDeliveryInput) {
if input.MySQLVersion == "" {
@@ -348,9 +373,15 @@ func normalizeMySQLDeliveryInput(input *MySQLDeliveryInput) {
if input.MaxBinlogSize == "" {
input.MaxBinlogSize = "256M"
}
input.CPUMilli = input.CPUCores * 1000
input.MemoryMi = input.MemoryGB * 1024
input.StorageGi = input.StorageGB
if input.CPUMilli == 0 && input.CPUCores != 0 {
input.CPUMilli = input.CPUCores * 1000
}
if input.MemoryMi == 0 && input.MemoryGB != 0 {
input.MemoryMi = input.MemoryGB * 1024
}
if input.StorageGi == 0 && input.StorageGB != 0 {
input.StorageGi = input.StorageGB
}
}
func defaultCollation(characterSet string) string {
@@ -366,38 +397,48 @@ func defaultCollation(characterSet string) string {
}
}
func validateDeliveryInput(input MySQLDeliveryInput) error {
func validateDeliveryInput(input MySQLDeliveryInput, dataDisks []string) error {
if len(input.Namespace) > 63 || !dnsLabelPattern.MatchString(input.Namespace) {
return fmt.Errorf("namespace must be a valid Kubernetes DNS label")
}
if len(input.InstanceName) > 63 || !dnsLabelPattern.MatchString(input.InstanceName) {
return fmt.Errorf("instance_name must be a valid Kubernetes DNS label")
}
if !oneOfInt64(input.CPUCores, []int64{1, 2, 4, 8, 16}) {
return fmt.Errorf("cpu_cores must be one of 1, 2, 4, 8, 16")
}
if !oneOfInt64(input.MemoryGB, []int64{2, 4, 8, 16, 32, 64}) {
return fmt.Errorf("memory_gb must be one of 2, 4, 8, 16, 32, 64")
}
if input.StorageGB < 20 || input.StorageGB > 2000 {
return fmt.Errorf("storage_gb must be between 20 and 2000")
if input.CPUMilli < 100 || input.CPUMilli > 64000 || input.MemoryMi < 2048 || input.MemoryMi > 65536 || input.StorageGi < 20 || input.StorageGi > 2000 {
return fmt.Errorf("requested resources are outside the supported range (cpu: 100-64000m, memory: 2048-65536 MiB, storage: 20-2000 GiB)")
}
if input.MySQLVersion != "" && !supportedMySQLVersions[input.MySQLVersion] {
return fmt.Errorf("unsupported mysql_version %q, supported: 8.0", input.MySQLVersion)
return fmt.Errorf("unsupported mysql_version %q, supported: 8.0, 8.4 (5.6/5.7 are EOL and have no Ubuntu 24.04 packages)", input.MySQLVersion)
}
if input.Topology != "" && !supportedMySQLTopologies[input.Topology] {
return fmt.Errorf("unsupported topology %q, supported: standalone", input.Topology)
}
if input.MySQLPort != 0 && (input.MySQLPort < 13306 || input.MySQLPort > 13999) {
return fmt.Errorf("mysql_port must be empty for auto assignment or between 13306 and 13999")
if input.MySQLPort != 0 && (input.MySQLPort < mysqlPortPoolStart || input.MySQLPort > mysqlPortPoolEnd) {
return fmt.Errorf("mysql_port must be empty for auto assignment or between %d and %d", mysqlPortPoolStart, mysqlPortPoolEnd)
}
if !supportedDataDisks[input.DataDisk] {
return fmt.Errorf("unsupported data_disk %q", input.DataDisk)
if input.DataDisk != "" {
allowed := dataDisks
if len(allowed) == 0 {
allowed = []string{"/data"}
}
found := false
for _, disk := range allowed {
if input.DataDisk == disk {
found = true
break
}
}
if !found {
return fmt.Errorf("data_disk %q is not in the allowed mount point list %v", input.DataDisk, allowed)
}
}
if input.TargetHost != "" && (len(input.TargetHost) > 253 || !hostNamePattern.MatchString(input.TargetHost)) {
return fmt.Errorf("target_host must be a valid inventory host name")
}
if !supportedParamTemplates[input.ParamTemplate] {
return fmt.Errorf("unsupported param_template %q", input.ParamTemplate)
}
if !validTimeZone(input.TimeZone) {
if !timezonePattern.MatchString(input.TimeZone) {
return fmt.Errorf("unsupported timezone %q", input.TimeZone)
}
if input.LowerCaseTableNames != 0 && input.LowerCaseTableNames != 1 {
@@ -593,20 +634,42 @@ func (s *DeliveryService) claimAndReserve(ctx context.Context) (*model.DeliveryT
if err := tx.Model(&model.DeliveryTask{}).Where("target_id = ? AND target_host <> ? AND status NOT IN ?", task.TargetID, "", occupiedExclude).Pluck("target_host", &occupied).Error; err != nil {
return err
}
host := firstFreeHost(meta.Hosts, occupied)
limit := s.cfg.DeliveryHostInstanceLimit
var host *targetHost
if payload.TargetHost != "" {
for i := range meta.Hosts {
if meta.Hosts[i].Name == payload.TargetHost {
host = &meta.Hosts[i]
break
}
}
if host == nil {
return s.failInTransaction(tx, &task, model.TaskValidationFailed, fmt.Sprintf("target_host %q is not in the candidate host pool", payload.TargetHost))
}
effectiveLimit := limit
if effectiveLimit < 1 {
effectiveLimit = 1
}
if hostTaskCount(occupied, host.Name) >= effectiveLimit {
return fmt.Errorf("defer: pinned host %s reached the per-host instance limit %d", host.Name, effectiveLimit)
}
} else {
host = firstFreeHost(meta.Hosts, occupied, limit)
}
if host == nil {
return fmt.Errorf("defer: no free host available on target")
}
mysqlPort := payload.MySQLPort
if mysqlPort == 0 {
var occupiedPorts []int
if err := tx.Model(&model.DeliveryTask{}).Where("target_id = ? AND target_host = ? AND mysql_port <> ? AND status NOT IN ?", task.TargetID, host.Name, 0, occupiedExclude).Pluck("mysql_port", &occupiedPorts).Error; err != nil {
return err
}
mysqlPort = firstFreePort(occupiedPorts)
if mysqlPort == 0 {
return fmt.Errorf("defer: no free MySQL port available on target host")
}
var usedPorts []int
if err := tx.Model(&model.DeliveryTask{}).Where("target_id = ? AND target_host = ? AND mysql_port <> ? AND status NOT IN ?", task.TargetID, host.Name, 0, occupiedExclude).Pluck("mysql_port", &usedPorts).Error; err != nil {
return err
}
var instancePorts []int
if err := tx.Model(&model.MySQLInstance{}).Where("target_id = ? AND node_name = ? AND status = ?", task.TargetID, host.Name, "active").Pluck("port", &instancePorts).Error; err != nil {
return err
}
mysqlPort, portErr := allocatePort(payload.MySQLPort, append(usedPorts, instancePorts...))
if portErr != nil {
return s.failInTransaction(tx, &task, model.TaskValidationFailed, portErr.Error())
}
reservation := model.ResourceReservation{TaskID: task.ID, BusinessLineID: task.BusinessLineID, TargetID: task.TargetID, CPUMilli: payload.CPUMilli, MemoryMi: payload.MemoryMi, StorageGi: payload.StorageGi, InstanceCount: 1, Status: "reserved", ExpiresAt: time.Now().Add(time.Duration(s.cfg.ReservationTTLMinutes) * time.Minute)}
if err := tx.Create(&reservation).Error; err != nil {
@@ -771,8 +834,10 @@ func (s *DeliveryService) CreateExecution(ctx context.Context, taskID, payloadHa
"instance_name": payload.InstanceName, "mysql_port": task.MySQLPort,
"data_disk": payload.DataDisk, "cpu_cores": payload.CPUCores,
"memory_gb": payload.MemoryGB, "storage_gb": payload.StorageGB,
"cpu_milli": payload.CPUMilli, "memory_mi": payload.MemoryMi, "storage_gi": payload.StorageGi,
"mysql_version": payload.MySQLVersion, "param_template": payload.ParamTemplate,
"timezone": payload.TimeZone, "lower_case_table_names": payload.LowerCaseTableNames,
"target_host": payload.TargetHost,
"timezone": payload.TimeZone, "lower_case_table_names": payload.LowerCaseTableNames,
"character_set": payload.CharacterSet, "collation": payload.Collation,
"max_connections": payload.MaxConnections, "innodb_redo_log_capacity": payload.InnoDBRedoLogCapacity,
"innodb_flush_log_at_trx_commit": payload.InnoDBFlushLogAtTrxCommit, "sync_binlog": payload.SyncBinlog,
+110 -46
View File
@@ -3,59 +3,123 @@ package service
import "testing"
func TestValidateDeliveryInput(t *testing.T) {
dataDisks := []string{"/data", "/disk1"}
valid := MySQLDeliveryInput{BusinessLineID: 1, TargetID: 1, Namespace: "team-a", InstanceName: "mysql-01", CPUCores: 2, MemoryGB: 4, StorageGB: 50}
normalizeMySQLDeliveryInput(&valid)
if err := validateDeliveryInput(valid); err != nil {
if err := validateDeliveryInput(valid, dataDisks); err != nil {
t.Fatalf("valid input rejected: %v", err)
}
withVersion := valid
withVersion.MySQLVersion = "8.0"
if err := validateDeliveryInput(withVersion); err != nil {
t.Fatalf("valid input with version 8.0 rejected: %v", err)
full := valid
full.MySQLVersion = "8.0"
full.Topology = "standalone"
full.MySQLPort = 13306
full.DataDisk = "/disk1"
full.TargetHost = "k8s-server-03"
full.CPUMilli = 2000
full.MemoryMi = 8192
full.StorageGi = 2000
full.TimeZone = "+08:00"
full.LowerCaseTableNames = 0
full.CharacterSet = "utf8mb4"
full.Collation = "utf8mb4_general_ci"
full.MaxConnections = "auto"
full.InnoDBRedoLogCapacity = "256M"
full.InnoDBFlushLogAtTrxCommit = 2
full.SyncBinlog = 0
full.InnoDBIOCapacity = 2000
full.LongQueryTime = 0.5
full.BinlogExpireLogsSeconds = 604800
full.MaxBinlogSize = "512M"
if err := validateDeliveryInput(full, dataDisks); err != nil {
t.Fatalf("valid full input rejected: %v", err)
}
for name, input := range map[string]MySQLDeliveryInput{
"uppercase namespace": valid,
"bad instance": valid,
"bad cpu cores": valid,
"too much memory": valid,
"too little storage": valid,
"too much storage": valid,
"unsupported version": valid,
"unsupported topology": valid,
"bad mysql port": valid,
"bad data disk": valid,
"unsupported charset": valid,
"bad collation": valid,
namedZone := valid
namedZone.TimeZone = "Asia/Shanghai"
if err := validateDeliveryInput(namedZone, dataDisks); err != nil {
t.Fatalf("named timezone rejected: %v", err)
}
lts := valid
lts.MySQLVersion = "8.4"
if err := validateDeliveryInput(lts, dataDisks); err != nil {
t.Fatalf("8.4 LTS rejected: %v", err)
}
for name, mutate := range map[string]func(*MySQLDeliveryInput){
"uppercase namespace": func(in *MySQLDeliveryInput) { in.Namespace = "Team-A" },
"bad instance": func(in *MySQLDeliveryInput) { in.InstanceName = "mysql_01" },
"too little cpu": func(in *MySQLDeliveryInput) { in.CPUMilli = 50 },
"too much cpu": func(in *MySQLDeliveryInput) { in.CPUMilli = 65000 },
"too little memory": func(in *MySQLDeliveryInput) { in.MemoryMi = 1024 },
"too much memory": func(in *MySQLDeliveryInput) { in.MemoryMi = 131072 },
"too little storage": func(in *MySQLDeliveryInput) { in.StorageGi = 10 },
"too much storage": func(in *MySQLDeliveryInput) { in.StorageGi = 4000 },
"unsupported version": func(in *MySQLDeliveryInput) { in.MySQLVersion = "5.7" },
"eol version": func(in *MySQLDeliveryInput) { in.MySQLVersion = "5.6" },
"unsupported topology": func(in *MySQLDeliveryInput) { in.Topology = "mgr_3" },
"port below pool": func(in *MySQLDeliveryInput) { in.MySQLPort = 3307 },
"port above pool": func(in *MySQLDeliveryInput) { in.MySQLPort = 14000 },
"data disk not in list": func(in *MySQLDeliveryInput) { in.DataDisk = "/mnt/other" },
"bad target host": func(in *MySQLDeliveryInput) { in.TargetHost = "-bad-host" },
"bad timezone": func(in *MySQLDeliveryInput) { in.TimeZone = "UTC+8" },
"bad lower case": func(in *MySQLDeliveryInput) { in.LowerCaseTableNames = 2 },
"bad charset": func(in *MySQLDeliveryInput) { in.CharacterSet = "big5" },
"collation mismatch": func(in *MySQLDeliveryInput) { in.CharacterSet = "gbk"; in.Collation = "utf8mb4_general_ci" },
"bad max connections": func(in *MySQLDeliveryInput) { in.MaxConnections = "300" },
"bad redo capacity": func(in *MySQLDeliveryInput) { in.InnoDBRedoLogCapacity = "2G" },
"bad flush log": func(in *MySQLDeliveryInput) { in.InnoDBFlushLogAtTrxCommit = 3 },
"bad sync binlog": func(in *MySQLDeliveryInput) { in.SyncBinlog = 2 },
"bad io capacity": func(in *MySQLDeliveryInput) { in.InnoDBIOCapacity = 500 },
"bad long query time": func(in *MySQLDeliveryInput) { in.LongQueryTime = 3 },
"bad binlog expire": func(in *MySQLDeliveryInput) { in.BinlogExpireLogsSeconds = 3600 },
"bad max binlog size": func(in *MySQLDeliveryInput) { in.MaxBinlogSize = "64M" },
} {
switch name {
case "uppercase namespace":
input.Namespace = "Team-A"
case "bad instance":
input.InstanceName = "mysql_01"
case "bad cpu cores":
input.CPUCores = 3
case "too much memory":
input.MemoryGB = 128
case "too little storage":
input.StorageGB = 10
case "too much storage":
input.StorageGB = 3000
case "unsupported version":
input.MySQLVersion = "5.7"
case "unsupported topology":
input.Topology = "mgr"
case "bad mysql port":
input.MySQLPort = 3306
case "bad data disk":
input.DataDisk = "/"
case "unsupported charset":
input.CharacterSet = "sjis"
case "bad collation":
input.CharacterSet = "utf8"
input.Collation = "utf8mb4_general_ci"
}
if err := validateDeliveryInput(input); err == nil {
input := valid
mutate(&input)
if err := validateDeliveryInput(input, dataDisks); err == nil {
t.Errorf("%s was accepted", name)
}
}
}
func TestFirstFreeHost(t *testing.T) {
hosts := []targetHost{{Name: "node-a"}, {Name: "node-b"}}
if h := firstFreeHost(hosts, nil, 1); h == nil || h.Name != "node-a" {
t.Fatalf("expected node-a on empty occupancy, got %+v", h)
}
if h := firstFreeHost(hosts, []string{"node-a"}, 1); h == nil || h.Name != "node-b" {
t.Fatalf("expected node-b when node-a is full at limit 1, got %+v", h)
}
if h := firstFreeHost(hosts, []string{"node-a", "node-b"}, 1); h != nil {
t.Fatalf("limit 1 with all hosts taken should return nil, got %+v", h)
}
if h := firstFreeHost(hosts, []string{"node-a", "node-b"}, 2); h == nil || h.Name != "node-a" {
t.Fatalf("expected node-a for second round at limit 2, got %+v", h)
}
if h := firstFreeHost(hosts, []string{"node-a", "node-a", "node-b", "node-b"}, 2); h != nil {
t.Fatalf("limit 2 with all hosts saturated should return nil, got %+v", h)
}
if h := firstFreeHost(hosts, []string{"node-a"}, 0); h == nil || h.Name != "node-b" {
t.Fatalf("limit 0 should degrade to 1, got %+v", h)
}
}
func TestAllocatePort(t *testing.T) {
if port, err := allocatePort(0, nil); err != nil || port != mysqlPortPoolStart {
t.Fatalf("expected first pool port %d, got %d err=%v", mysqlPortPoolStart, port, err)
}
if port, err := allocatePort(0, []int{13306, 13307}); err != nil || port != 13308 {
t.Fatalf("expected 13308 skipping occupied, got %d err=%v", port, err)
}
if port, err := allocatePort(13400, []int{13306}); err != nil || port != 13400 {
t.Fatalf("expected requested port 13400, got %d err=%v", port, err)
}
if _, err := allocatePort(13306, []int{13306}); err == nil {
t.Fatal("requested occupied port was accepted")
}
used := make([]int, 0, mysqlPortPoolEnd-mysqlPortPoolStart+1)
for p := mysqlPortPoolStart; p <= mysqlPortPoolEnd; p++ {
used = append(used, p)
}
if _, err := allocatePort(0, used); err == nil {
t.Fatal("exhausted pool still allocated a port")
}
}