Merge remote-tracking branch 'upstream/main' into feat/base-service-delivery

# Conflicts:
#	server/.env.example
#	server/internal/config/config.go
#	server/internal/service/delivery.go
#	server/internal/service/delivery_test.go
This commit is contained in:
mac
2026-07-28 09:34:37 +08:00
11 changed files with 786 additions and 602 deletions
+208 -64
View File
@@ -1,82 +1,186 @@
--- ---
- name: Preflight native MySQL delivery - name: Native MySQL delivery
hosts: "{{ target_hosts }}" hosts: "{{ target_hosts }}"
become: true become: true
gather_facts: true gather_facts: true
any_errors_fatal: true any_errors_fatal: true
vars: vars:
# --- identity ---
mysql_instance: "{{ instance_name }}" mysql_instance: "{{ instance_name }}"
mysql_version_value: "{{ mysql_version | default('8.0') }}" mysql_version_value: "{{ mysql_version | default('8.0') }}"
# 版本包映射:8.0 用 Ubuntu 24.04 自带源(精确锁版);8.4 用 MySQL 官方 APT 源的 LTS 组件
# (noble 自带源无 8.4,官方源组件内即为该系列,不再锁小版本)。
# 5.6/5.7 已官方 EOL 且 Ubuntu 24.04 无可用 apt 包,明确不纳入白名单。
mysql_package_map: mysql_package_map:
"8.0": "mysql-server=8.0.46-0ubuntu0.24.04.3" "8.0":
mysql_package_name: "{{ mysql_package_map[mysql_version_value] }}" package: "mysql-server=8.0.46-0ubuntu0.24.04.3"
mysql_port_value: "{{ mysql_port | default(3307) | int }}" repo_component: ""
mysql_memory_mb_value: "{{ memory_mb | default(2048) | int }}" "8.4":
mysql_storage_gb_value: "{{ storage_gb | default(20) | int }}" package: "mysql-community-server"
repo_component: "mysql-8.4-lts"
mysql_package_name: "{{ (mysql_package_map[mysql_version_value] | default({})).package | default('') }}"
mysql_repo_component: "{{ (mysql_package_map[mysql_version_value] | default({})).repo_component | default('') }}"
# --- platform-allocated inputs (safe fallbacks when not passed in) ---
# 端口池 13306–13999:平台从池分配并传入;未传时兜底池首端口,占用探测在目标机执行。
mysql_port_value: "{{ mysql_port | default(13306) | int }}"
# GR 组通信端口:仅 mgr_3 使用,平台成对分配;兜底为 SQL 端口 +10000。
mysql_gr_port_value: "{{ gr_port | default((mysql_port | default(13306) | int) + 10000) | int }}"
# 数据盘挂载点:平台按白名单选定并传入;兜底 /data。
mysql_data_disk: "{{ data_disk | default('/data') }}"
# --- resource quota (Go→AWX 契约单位保持 MB/GB,不擅改) ---
mysql_memory_mb_value: "{{ memory_mb | default(4096) | int }}"
mysql_storage_gb_value: "{{ storage_gb | default(50) | int }}"
# --- mount-point-agnostic layout: {data_disk}/mysql-delivery/{instance_id}/... ---
mysql_base_dir: "{{ mysql_data_disk }}/mysql-delivery/{{ mysql_instance }}"
mysql_install_dir: "/opt/mysql-delivery/{{ mysql_instance }}" mysql_install_dir: "/opt/mysql-delivery/{{ mysql_instance }}"
mysql_data_dir: "/data/mysql-delivery/{{ mysql_instance }}/data" mysql_data_dir: "{{ mysql_base_dir }}/data"
mysql_log_dir: "/data/mysql-delivery/{{ mysql_instance }}/log" mysql_log_dir: "{{ mysql_base_dir }}/logs"
mysql_binlog_dir: "{{ mysql_base_dir }}/logs/binlog"
mysql_redo_dir: "{{ mysql_base_dir }}/logs/redo"
mysql_tmp_dir: "{{ mysql_base_dir }}/tmp"
# socket/pid 走 /run tmpfs(重启自动清理),由 systemd RuntimeDirectory 创建。
mysql_run_dir: "/run/mysql-delivery-{{ mysql_instance }}" mysql_run_dir: "/run/mysql-delivery-{{ mysql_instance }}"
mysql_config_file: "/etc/mysql/mysql-delivery/{{ mysql_instance }}.cnf" mysql_config_file: "/etc/mysql/mysql-delivery/{{ mysql_instance }}.cnf"
# --- database config (user form, with doc default baselines) ---
mysql_timezone: "{{ timezone | default('+08:00') }}"
mysql_lower_case_table_names: "{{ lower_case_table_names | default(1) | int }}"
mysql_character_set: "{{ character_set | default('utf8mb4') }}"
mysql_collation: "{{ collation | default('utf8mb4_general_ci') }}"
# --- advanced params (overridable; default baseline assumes SSD for io_capacity) ---
mysql_flush_log_at_trx_commit: "{{ innodb_flush_log_at_trx_commit | default(1) | int }}"
mysql_sync_binlog: "{{ sync_binlog | default(1) | int }}"
mysql_io_capacity: "{{ innodb_io_capacity | default(2000) | int }}"
mysql_long_query_time: "{{ long_query_time | default(1) }}"
mysql_binlog_expire_logs_seconds: "{{ binlog_expire_logs_seconds | default(604800) | int }}"
mysql_max_binlog_size: "{{ max_binlog_size | default('256M') }}"
mgr_group_name: "{{ group_replication_group_name | default('aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa') }}"
# --- expected node count per topology ---
mysql_expected_hosts: "{{ {'standalone': 1, 'primary_replica': 2, 'mgr_3': 3}[topology | default('standalone')] }}"
# --- secrets (injected via environment) ---
mysql_root_password_value: "{{ lookup('ansible.builtin.env', 'XINFRA_MYSQL_ROOT_PASSWORD') }}" mysql_root_password_value: "{{ lookup('ansible.builtin.env', 'XINFRA_MYSQL_ROOT_PASSWORD') }}"
mysql_admin_password_value: "{{ lookup('ansible.builtin.env', 'XINFRA_MYSQL_ADMIN_PASSWORD') }}" mysql_admin_password_value: "{{ lookup('ansible.builtin.env', 'XINFRA_MYSQL_ADMIN_PASSWORD') }}"
pre_tasks: pre_tasks:
- name: Validate prototype parameters - name: Validate delivery parameters
ansible.builtin.assert: ansible.builtin.assert:
that: that:
- topology == 'standalone' - topology in ['standalone', 'primary_replica', 'mgr_3']
- mysql_instance is match('^[a-z0-9][a-z0-9-]{0,62}$') - mysql_instance is match('^[a-z0-9][a-z0-9-]{0,62}$')
- mysql_version_value in mysql_package_map - mysql_version_value in mysql_package_map
- (mysql_port_value | int) >= 1024 - mysql_data_disk is match('^/')
- (mysql_port_value | int) <= 65535 - (mysql_port_value | int) >= 13306
- (mysql_memory_mb_value | int) >= 1024 - (mysql_port_value | int) <= 13999
- (mysql_memory_mb_value | int) <= 4096 - (mysql_memory_mb_value | int) >= 2048
- (mysql_storage_gb_value | int) >= 10 - (mysql_memory_mb_value | int) <= 65536
- (mysql_storage_gb_value | int) <= 100 - (mysql_storage_gb_value | int) >= 20
- (mysql_storage_gb_value | int) <= 2000
- (mysql_lower_case_table_names | int) in [0, 1]
- mysql_root_password_value | length >= 16 - mysql_root_password_value | length >= 16
- mysql_admin_password_value | length >= 16 - mysql_admin_password_value | length >= 16
fail_msg: The first machine prototype only supports safe standalone parameters fail_msg: >-
Delivery parameters out of the supported target-state whitelist
(topology / version-package-map / port pool 13306-13999 / memory 2-64G / storage 20-2000G).
quiet: true
no_log: true no_log: true
- name: Validate topology host count - name: Validate topology host count
ansible.builtin.assert: ansible.builtin.assert:
that: that:
- ansible_play_hosts_all | length == 1 - (ansible_play_hosts_all | length | int) == (mysql_expected_hosts | int)
fail_msg: The selected topology does not match the target host count fail_msg: "topology={{ topology }} expects {{ mysql_expected_hosts }} host(s), got {{ ansible_play_hosts_all | length }}"
run_once: true run_once: true
- name: Check port ownership - name: Probe target-host port occupancy (SQL + GR)
ansible.builtin.shell: | # cmd 字典形式不经过 free-form split_args 解析,避免引号/Jinja 块导致的解析失败。
set -o pipefail ansible.builtin.shell:
if ss -lntH "sport = :{{ mysql_port_value }}" | grep -q .; then cmd: |
systemctl is-active --quiet "mysql-delivery@{{ mysql_instance }}.service" set -o pipefail
fi for p in {{ mysql_probe_ports | join(' ') }}; do
args: if ss -lntH "sport = :${p}" | grep -q .; then
# already listening: only tolerated when owned by this instance service
if ! systemctl is-active --quiet "mysql-delivery@{{ mysql_instance }}.service"; then
echo "port ${p} already in use on target host" >&2
exit 3
fi
fi
done
executable: /bin/bash executable: /bin/bash
vars:
mysql_probe_ports: "{{ [mysql_port_value, mysql_gr_port_value] if topology == 'mgr_3' else [mysql_port_value] }}"
changed_when: false changed_when: false
- name: Read currently available memory - name: Read currently available memory (point-in-time guard)
ansible.builtin.shell: awk '/^MemAvailable:/ { print int($2 / 1024) }' /proc/meminfo ansible.builtin.shell: awk '/^MemAvailable:/ { print int($2 / 1024) }' /proc/meminfo
args: args:
executable: /bin/bash executable: /bin/bash
register: mysql_available_memory register: mysql_available_memory
changed_when: false changed_when: false
- name: Check available memory and disk - name: Resolve data-disk mountpoint available space
ansible.builtin.set_fact:
mysql_mount_avail: >-
{{ (ansible_mounts | selectattr('mount', 'equalto', mysql_data_disk)
| map(attribute='size_available') | first)
| default(ansible_mounts | selectattr('mount', 'equalto', '/')
| map(attribute='size_available') | first) }}
- name: Check available memory and data-disk space
ansible.builtin.assert: ansible.builtin.assert:
that: that:
- (mysql_available_memory.stdout | int) >= (mysql_memory_mb_value | int) - (mysql_available_memory.stdout | int) >= (mysql_memory_mb_value | int)
- (ansible_mounts | selectattr('mount', 'equalto', '/') | map(attribute='size_available') | first | int) >= (mysql_storage_gb_value | int) * 1073741824 - (mysql_mount_avail | int) >= (mysql_storage_gb_value | int) * 1073741824
fail_msg: Target host does not have enough available memory or disk fail_msg: >-
Target host lacks memory or free space on {{ mysql_data_disk }};
host-wide Σ-quota budgeting is the platform's responsibility, this is a last-resort guard.
tasks: tasks:
- name: Detect an existing Ubuntu MySQL package - name: Detect an existing MySQL server package
ansible.builtin.command: dpkg-query -W mysql-server ansible.builtin.command: dpkg-query -W -f '${Package}=${Version}\n' mysql-server mysql-community-server
register: mysql_package_before register: mysql_package_before
failed_when: false failed_when: false
changed_when: false changed_when: false
- name: Install Ubuntu MySQL package - name: Resolve the installed MySQL server version
ansible.builtin.set_fact:
mysql_installed_version: >-
{{ (mysql_package_before.stdout_lines | select('search', '=') | list
| first | default('')).split('=') | last }}
# /usr 下的 mysqld/mysql 二进制全机共享,一台主机只能承载一个 MySQL 版本系列。
- name: Enforce host-level MySQL series consistency
ansible.builtin.assert:
that:
- mysql_installed_version == '' or mysql_installed_version.startswith(mysql_version_value ~ '.')
fail_msg: >-
Host already runs MySQL {{ mysql_installed_version }} but {{ mysql_version_value }} was requested;
native binaries under /usr are shared host-wide, so one host serves exactly one MySQL series.
- name: Install the MySQL APT repository signing key
ansible.builtin.get_url:
url: https://repo.mysql.com/RPM-GPG-KEY-mysql-2023
dest: /etc/apt/keyrings/mysql.asc
owner: root
group: root
mode: '0644'
when: mysql_repo_component != ''
- name: Configure the MySQL APT repository component
ansible.builtin.apt_repository:
repo: >-
deb [signed-by=/etc/apt/keyrings/mysql.asc]
http://repo.mysql.com/apt/ubuntu {{ ansible_distribution_release }} {{ mysql_repo_component }}
filename: xinfra-mysql-delivery
state: present
when: mysql_repo_component != ''
- name: Install the MySQL server package
ansible.builtin.apt: ansible.builtin.apt:
name: "{{ mysql_package_name }}" name: "{{ mysql_package_name }}"
state: present state: present
@@ -88,14 +192,14 @@
name: mysql.service name: mysql.service
state: stopped state: stopped
enabled: false enabled: false
when: mysql_package_before.rc != 0 when: mysql_installed_version == ''
- name: Check for the bundled MySQL AppArmor profile - name: Check for the bundled MySQL AppArmor profile
ansible.builtin.stat: ansible.builtin.stat:
path: /etc/apparmor.d/usr.sbin.mysqld path: /etc/apparmor.d/usr.sbin.mysqld
register: mysql_apparmor_profile register: mysql_apparmor_profile
- name: Allow the delivery data and run paths in the MySQL AppArmor profile - name: Authorize the delivery paths in the MySQL AppArmor profile
ansible.builtin.copy: ansible.builtin.copy:
dest: /etc/apparmor.d/local/usr.sbin.mysqld dest: /etc/apparmor.d/local/usr.sbin.mysqld
owner: root owner: root
@@ -103,8 +207,8 @@
mode: '0644' mode: '0644'
content: | content: |
# Managed by XINFRA MySQL delivery - grant per-instance native paths # Managed by XINFRA MySQL delivery - grant per-instance native paths
/data/mysql-delivery/ r, {{ mysql_data_disk }}/mysql-delivery/ r,
/data/mysql-delivery/** rwk, {{ mysql_data_disk }}/mysql-delivery/** rwk,
/run/mysql-delivery-*/ rw, /run/mysql-delivery-*/ rw,
/run/mysql-delivery-*/** rwk, /run/mysql-delivery-*/** rwk,
when: mysql_apparmor_profile.stat.exists when: mysql_apparmor_profile.stat.exists
@@ -115,7 +219,7 @@
when: mysql_apparmor_profile.stat.exists and mysql_apparmor_local.changed when: mysql_apparmor_profile.stat.exists and mysql_apparmor_local.changed
changed_when: true changed_when: true
- name: Create instance directories - name: Create instance directories (mount-point-agnostic layout)
ansible.builtin.file: ansible.builtin.file:
path: "{{ item.path }}" path: "{{ item.path }}"
state: directory state: directory
@@ -125,9 +229,12 @@
loop: loop:
- { path: /etc/mysql/mysql-delivery, owner: root, group: mysql, mode: '0750' } - { path: /etc/mysql/mysql-delivery, owner: root, group: mysql, mode: '0750' }
- { path: "{{ mysql_install_dir }}", owner: root, group: root, mode: '0755' } - { path: "{{ mysql_install_dir }}", owner: root, group: root, mode: '0755' }
- { path: "{{ mysql_base_dir }}", owner: mysql, group: mysql, mode: '0750' }
- { path: "{{ mysql_data_dir }}", owner: mysql, group: mysql, mode: '0750' } - { path: "{{ mysql_data_dir }}", owner: mysql, group: mysql, mode: '0750' }
- { path: "{{ mysql_log_dir }}", owner: mysql, group: mysql, mode: '0750' } - { path: "{{ mysql_log_dir }}", owner: mysql, group: mysql, mode: '0750' }
- { path: "{{ mysql_run_dir }}", owner: mysql, group: mysql, mode: '0755' } - { path: "{{ mysql_binlog_dir }}", owner: mysql, group: mysql, mode: '0750' }
- { path: "{{ mysql_redo_dir }}", owner: mysql, group: mysql, mode: '0750' }
- { path: "{{ mysql_tmp_dir }}", owner: mysql, group: mysql, mode: '0750' }
- name: Link native binaries into the instance directory - name: Link native binaries into the instance directory
ansible.builtin.file: ansible.builtin.file:
@@ -146,6 +253,33 @@
{{ 'standalone' if topology == 'standalone' else {{ 'standalone' if topology == 'standalone' else
('primary' if ansible_play_hosts_all.index(inventory_hostname) == 0 else ('primary' if ansible_play_hosts_all.index(inventory_hostname) == 0 else
('replica' if topology == 'primary_replica' else 'mgr')) }} ('replica' if topology == 'primary_replica' else 'mgr')) }}
# host-wide unique: platform may pass explicit mysql_server_id; otherwise derive
# port + node index (ports are unique per host in the pool model).
mysql_server_id_value: >-
{{ mysql_server_id | default((mysql_port_value | int)
+ (ansible_play_hosts_all.index(inventory_hostname))) | int }}
- name: Resolve memory tier (GB)
ansible.builtin.set_fact:
mysql_memory_gb: "{{ ((mysql_memory_mb_value | int) // 1024) | int }}"
- name: Resolve linkage-derived defaults (illustrative tiers, pending hardware calibration)
ansible.builtin.set_fact:
mysql_max_connections: >-
{{ (max_connections | int) if (max_connections is defined and (max_connections | string) != 'auto')
else (200 if (mysql_memory_gb | int) <= 2
else 500 if (mysql_memory_gb | int) <= 4
else 1000 if (mysql_memory_gb | int) <= 8
else 2000 if (mysql_memory_gb | int) <= 16
else 4000 if (mysql_memory_gb | int) <= 32
else 8000 if (mysql_memory_gb | int) <= 64
else 16000) }}
mysql_redo_capacity: >-
{{ innodb_redo_log_capacity if (innodb_redo_log_capacity is defined)
else ('128M' if (mysql_memory_gb | int) <= 4
else '256M' if (mysql_memory_gb | int) <= 16
else '512M' if (mysql_memory_gb | int) <= 32
else '1G') }}
- name: Write instance configuration - name: Write instance configuration
ansible.builtin.template: ansible.builtin.template:
@@ -193,32 +327,33 @@
timeout: 60 timeout: 60
- name: Configure local administrative accounts - name: Configure local administrative accounts
ansible.builtin.shell: | # cmd 字典形式不经过 free-form split_args 解析,heredoc SQL 中的奇数个单引号才不会报错。
set -euo pipefail ansible.builtin.shell:
client_file="$(mktemp)" cmd: |
sql_file="$(mktemp)" set -euo pipefail
trap 'rm -f "$client_file" "$sql_file"' EXIT client_file="$(mktemp)"
chmod 600 "$client_file" "$sql_file" sql_file="$(mktemp)"
cat >"$client_file" <<'EOF' trap 'rm -f "$client_file" "$sql_file"' EXIT
[client] chmod 600 "$client_file" "$sql_file"
user=root cat >"$client_file" <<'EOF'
password={{ mysql_root_password_value }} [client]
socket={{ mysql_run_dir }}/mysql.sock user=root
EOF password={{ mysql_root_password_value }}
if ! /usr/bin/mysql --defaults-extra-file="$client_file" -e 'SELECT 1' >/dev/null 2>&1; then socket={{ mysql_run_dir }}/mysql.sock
EOF
if ! /usr/bin/mysql --defaults-extra-file="$client_file" -e 'SELECT 1' >/dev/null 2>&1; then
cat >"$sql_file" <<'EOF'
ALTER USER 'root'@'localhost' IDENTIFIED BY '{{ mysql_root_password_value | replace("'", "''") }}';
EOF
/usr/bin/mysql --protocol=socket --socket={{ mysql_run_dir }}/mysql.sock -uroot <"$sql_file"
fi
cat >"$sql_file" <<'EOF' cat >"$sql_file" <<'EOF'
ALTER USER 'root'@'localhost' IDENTIFIED BY '{{ mysql_root_password_value | replace("'", "''") }}'; CREATE USER IF NOT EXISTS 'xinfra_admin'@'%' IDENTIFIED BY '{{ mysql_admin_password_value | replace("'", "''") }}';
EOF ALTER USER 'xinfra_admin'@'%' IDENTIFIED BY '{{ mysql_admin_password_value | replace("'", "''") }}';
/usr/bin/mysql --protocol=socket --socket={{ mysql_run_dir }}/mysql.sock -uroot <"$sql_file" GRANT ALL PRIVILEGES ON *.* TO 'xinfra_admin'@'%' WITH GRANT OPTION;
fi FLUSH PRIVILEGES;
cat >"$sql_file" <<'EOF' EOF
CREATE USER IF NOT EXISTS 'xinfra_admin'@'%' IDENTIFIED BY '{{ mysql_admin_password_value | replace("'", "''") }}'; /usr/bin/mysql --defaults-extra-file="$client_file" <"$sql_file"
ALTER USER 'xinfra_admin'@'%' IDENTIFIED BY '{{ mysql_admin_password_value | replace("'", "''") }}';
GRANT ALL PRIVILEGES ON *.* TO 'xinfra_admin'@'%' WITH GRANT OPTION;
FLUSH PRIVILEGES;
EOF
/usr/bin/mysql --defaults-extra-file="$client_file" <"$sql_file"
args:
executable: /bin/bash executable: /bin/bash
changed_when: false changed_when: false
no_log: true no_log: true
@@ -229,6 +364,15 @@
port: "{{ mysql_port_value }}" port: "{{ mysql_port_value }}"
timeout: 30 timeout: 30
- name: Note pending HA runtime orchestration
ansible.builtin.debug:
msg: >-
topology={{ topology }} deployed with HA-ready config (GTID/binlog/relay/GR settings in place),
but replication wiring (CHANGE REPLICATION SOURCE) and Group Replication bootstrap
(START GROUP_REPLICATION) are not yet automated — nodes start config-ready only.
when: topology != 'standalone'
run_once: true
handlers: handlers:
- name: Restart MySQL delivery instance - name: Restart MySQL delivery instance
ansible.builtin.systemd_service: ansible.builtin.systemd_service:
+58 -11
View File
@@ -1,35 +1,82 @@
# Managed by XINFRA MySQL delivery - generated, do not edit by hand
# instance={{ mysql_instance }} version={{ mysql_version_value }} topology={{ topology }}
[client]
socket={{ mysql_run_dir }}/mysql.sock
port={{ mysql_port_value }}
[mysqld] [mysqld]
user=mysql user=mysql
basedir=/usr basedir=/usr
datadir={{ mysql_data_dir }} datadir={{ mysql_data_dir }}
tmpdir={{ mysql_tmp_dir }}
socket={{ mysql_run_dir }}/mysql.sock socket={{ mysql_run_dir }}/mysql.sock
pid-file={{ mysql_run_dir }}/mysql.pid pid-file={{ mysql_run_dir }}/mysql.pid
port={{ mysql_port_value }} port={{ mysql_port_value }}
bind-address=0.0.0.0 bind-address=0.0.0.0
mysqlx=0 skip-name-resolve=ON
server-id={{ 101 + mysql_node_index | int }} server-id={{ mysql_server_id_value }}
local-infile=OFF
# --- charset / collation / timezone / identifier case ---
character-set-server={{ mysql_character_set }}
collation-server={{ mysql_collation }}
default-time-zone={{ mysql_timezone }}
lower-case-table-names={{ mysql_lower_case_table_names }}
# --- error / slow log ---
log-error={{ mysql_log_dir }}/error.log log-error={{ mysql_log_dir }}/error.log
slow-query-log=ON slow-query-log=ON
slow-query-log-file={{ mysql_log_dir }}/slow.log slow-query-log-file={{ mysql_log_dir }}/slow.log
skip-name-resolve=ON long-query-time={{ mysql_long_query_time }}
max-connections=100
innodb-buffer-pool-size={{ ((mysql_memory_mb_value | int) * 55 / 100) | int }}M # --- binlog (PITR + replication base) ---
innodb-log-file-size=128M log-bin={{ mysql_binlog_dir }}/binlog
log-bin={{ mysql_log_dir }}/mysql-bin
binlog-format=ROW binlog-format=ROW
sync-binlog={{ mysql_sync_binlog }}
max-binlog-size={{ mysql_max_binlog_size }}
binlog-expire-logs-seconds={{ mysql_binlog_expire_logs_seconds }}
{% if topology != 'standalone' %}
gtid-mode=ON gtid-mode=ON
enforce-gtid-consistency=ON enforce-gtid-consistency=ON
relay-log={{ mysql_log_dir }}/relay-bin relay-log={{ mysql_binlog_dir }}/relay-bin
{% endif %}
# --- InnoDB core ---
innodb-buffer-pool-size={{ ((mysql_memory_mb_value | int) * 55 / 100) | int }}M
innodb-flush-method=O_DIRECT
innodb-flush-log-at-trx-commit={{ mysql_flush_log_at_trx_commit }}
innodb-io-capacity={{ mysql_io_capacity }}
max-connections={{ mysql_max_connections }}
# --- redo log (version-sensitive) ---
innodb-log-group-home-dir={{ mysql_redo_dir }}
{% if mysql_version_value in ['8.0', '8.4'] %}
innodb-redo-log-capacity={{ mysql_redo_capacity }}
{% else %}
innodb-log-file-size={{ mysql_redo_capacity }}
{% endif %}
# --- X Protocol disabled (version-sensitive) ---
{% if mysql_version_value in ['8.0', '8.4'] %}
mysqlx=0
{% elif mysql_version_value == '5.7' %}
loose-mysqlx=0
{% endif %}
{% if topology == 'mgr_3' %} {% if topology == 'mgr_3' %}
# --- Group Replication (config-ready; runtime bootstrap handled out-of-band) ---
plugin-load-add=group_replication.so plugin-load-add=group_replication.so
{% if mysql_version_value != '8.4' %}
{# deprecated since 8.0.26 and removed in 8.3+; only inject for older series #}
transaction-write-set-extraction=XXHASH64 transaction-write-set-extraction=XXHASH64
{% endif %}
loose-group-replication-group-name={{ mgr_group_name }} loose-group-replication-group-name={{ mgr_group_name }}
loose-group-replication-start-on-boot=OFF loose-group-replication-start-on-boot=OFF
loose-group-replication-local-address={{ ansible_host | default(inventory_hostname) }}:{{ mysql_port_value | int + 10000 }} loose-group-replication-local-address={{ ansible_host | default(inventory_hostname) }}:{{ mysql_gr_port_value }}
loose-group-replication-group-seeds={% for host in ansible_play_hosts_all %}{{ hostvars[host].ansible_host | default(host) }}:{{ mysql_port_value | int + 10000 }}{% if not loop.last %},{% endif %}{% endfor %} loose-group-replication-group-seeds={% for host in ansible_play_hosts_all %}{{ hostvars[host].ansible_host | default(host) }}:{{ mysql_gr_port_value }}{% if not loop.last %},{% endif %}{% endfor %}
loose-group-replication-ip-allowlist={% for host in ansible_play_hosts_all %}{{ hostvars[host].ansible_host | default(host) }}{% if not loop.last %},{% endif %}{% endfor %} loose-group-replication-ip-allowlist={% for host in ansible_play_hosts_all %}{{ hostvars[host].ansible_host | default(host) }}{% if not loop.last %},{% endif %}{% endfor %}
loose-group-replication-single-primary-mode=ON loose-group-replication-single-primary-mode=ON
loose-group-replication-enforce-update-everywhere-checks=OFF loose-group-replication-enforce-update-everywhere-checks=OFF
{% endif %} {% endif %}
+4
View File
@@ -19,9 +19,13 @@ export interface CreateMySQLDeliveryPayload {
topology: string topology: string
mysql_port?: number mysql_port?: number
data_disk: string data_disk: string
target_host?: string
cpu_cores: number cpu_cores: number
memory_gb: number memory_gb: number
storage_gb: number storage_gb: number
cpu_milli: number
memory_mi: number
storage_gi: number
param_template: string param_template: string
timezone: string timezone: string
lower_case_table_names: number lower_case_table_names: number
+3
View File
@@ -959,6 +959,9 @@ function mysqlDeliveryPayload(businessLineId: number) {
cpu_cores: resources.cpuCores, cpu_cores: resources.cpuCores,
memory_gb: resources.memoryGb, memory_gb: resources.memoryGb,
storage_gb: parseStorageGb(deliveryForm.disk), storage_gb: parseStorageGb(deliveryForm.disk),
cpu_milli: resources.cpuCores * 1000,
memory_mi: resources.memoryGb * 1024,
storage_gi: parseStorageGb(deliveryForm.disk),
param_template: deliveryForm.paramTemplate, param_template: deliveryForm.paramTemplate,
timezone: deliveryForm.timezone, timezone: deliveryForm.timezone,
lower_case_table_names: deliveryForm.lowerCaseTableNames, lower_case_table_names: deliveryForm.lowerCaseTableNames,
+4
View File
@@ -22,6 +22,10 @@ DELIVERY_CALLBACK_BASE_URL=http://authserver-backend.authserver.svc.cluster.loca
DELIVERY_RESERVATION_TTL_MINUTES=120 DELIVERY_RESERVATION_TTL_MINUTES=120
DELIVERY_GLOBAL_LIMIT=2 DELIVERY_GLOBAL_LIMIT=2
DELIVERY_TARGET_LIMIT=2 DELIVERY_TARGET_LIMIT=2
# 单机 MySQL 实例数上限(同机多实例,容量由配额 + playbook 实机守卫兜底)
DELIVERY_HOST_INSTANCE_LIMIT=4
# 数据盘挂载点白名单(逗号分隔,第一项为默认值)
DELIVERY_DATA_DISKS=/data
AWX_BASE_URL= AWX_BASE_URL=
AWX_TOKEN= AWX_TOKEN=
AWX_USERNAME= AWX_USERNAME=
+61 -118
View File
@@ -152,7 +152,7 @@ const docTemplate = `{
"BearerAuth": [] "BearerAuth": []
} }
], ],
"description": "返回所有已启用的部署目标(如 k8s 集群、主机池)", "description": "从 AWX 动态返回可用 Job Template 及其 Inventory hosts",
"produces": [ "produces": [
"application/json" "application/json"
], ],
@@ -160,6 +160,14 @@ const docTemplate = `{
"delivery" "delivery"
], ],
"summary": "获取可用部署目标", "summary": "获取可用部署目标",
"parameters": [
{
"type": "string",
"description": "组件过滤,例如 mysql",
"name": "component",
"in": "query"
}
],
"responses": { "responses": {
"200": { "200": {
"description": "items: 部署目标数组", "description": "items: 部署目标数组",
@@ -176,64 +184,6 @@ const docTemplate = `{
} }
} }
} }
},
"post": {
"security": [
{
"BearerAuth": []
}
],
"description": "管理员创建新的部署目标(目前仅支持 k8s 类型)",
"consumes": [
"application/json"
],
"produces": [
"application/json"
],
"tags": [
"delivery"
],
"summary": "创建部署目标",
"parameters": [
{
"description": "目标配置",
"name": "body",
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/handler.targetPayload"
}
}
],
"responses": {
"201": {
"description": "目标已创建",
"schema": {
"$ref": "#/definitions/model.DeploymentTarget"
}
},
"400": {
"description": "参数错误",
"schema": {
"type": "object",
"additionalProperties": true
}
},
"401": {
"description": "未授权",
"schema": {
"type": "object",
"additionalProperties": true
}
},
"409": {
"description": "名称冲突",
"schema": {
"type": "object",
"additionalProperties": true
}
}
}
} }
}, },
"/auth/api/v1/delivery/tasks": { "/auth/api/v1/delivery/tasks": {
@@ -254,6 +204,7 @@ const docTemplate = `{
"parameters": [ "parameters": [
{ {
"type": "integer", "type": "integer",
"format": "int64",
"description": "业务线 ID 过滤", "description": "业务线 ID 过滤",
"name": "business_line_id", "name": "business_line_id",
"in": "query" "in": "query"
@@ -689,65 +640,6 @@ const docTemplate = `{
} }
} }
}, },
"handler.targetPayload": {
"type": "object",
"required": [
"awx_inventory_id",
"awx_template_id",
"name",
"target_type"
],
"properties": {
"awx_inventory_id": {
"type": "integer"
},
"awx_template_id": {
"type": "integer"
},
"metadata": {
"type": "object",
"additionalProperties": {}
},
"name": {
"type": "string"
},
"target_type": {
"type": "string"
}
}
},
"model.DeploymentTarget": {
"type": "object",
"properties": {
"awx_inventory_id": {
"type": "integer"
},
"awx_template_id": {
"type": "integer"
},
"created_at": {
"type": "string"
},
"enabled": {
"type": "boolean"
},
"id": {
"type": "integer"
},
"metadata": {
"type": "string"
},
"name": {
"type": "string"
},
"target_type": {
"type": "string"
},
"updated_at": {
"type": "string"
}
}
},
"model.ResourceQuota": { "model.ResourceQuota": {
"type": "object", "type": "object",
"properties": { "properties": {
@@ -816,18 +708,55 @@ const docTemplate = `{
"target_id" "target_id"
], ],
"properties": { "properties": {
"binlog_expire_logs_seconds": {
"type": "integer"
},
"business_line_id": { "business_line_id": {
"type": "integer" "type": "integer"
}, },
"character_set": {
"type": "string"
},
"collation": {
"type": "string"
},
"cpu_milli": { "cpu_milli": {
"type": "integer" "type": "integer"
}, },
"data_disk": {
"type": "string"
},
"innodb_flush_log_at_trx_commit": {
"type": "integer"
},
"innodb_io_capacity": {
"type": "integer"
},
"innodb_redo_log_capacity": {
"type": "string"
},
"instance_name": { "instance_name": {
"type": "string" "type": "string"
}, },
"long_query_time": {
"type": "number"
},
"lower_case_table_names": {
"type": "integer"
},
"max_binlog_size": {
"type": "string"
},
"max_connections": {
"description": "高级参数(选填,零值视为未设置)",
"type": "string"
},
"memory_mi": { "memory_mi": {
"type": "integer" "type": "integer"
}, },
"mysql_port": {
"type": "integer"
},
"mysql_version": { "mysql_version": {
"type": "string" "type": "string"
}, },
@@ -837,8 +766,22 @@ const docTemplate = `{
"storage_gi": { "storage_gi": {
"type": "integer" "type": "integer"
}, },
"sync_binlog": {
"type": "integer"
},
"target_host": {
"description": "调度控制(选填):点名候选池内主机跳过自动选机,端口/配额/实机守卫照常执行",
"type": "string"
},
"target_id": { "target_id": {
"type": "integer" "type": "integer"
},
"timezone": {
"description": "数据库配置(选填,缺省由 playbook 基线兜底)",
"type": "string"
},
"topology": {
"type": "string"
} }
} }
} }
+61 -118
View File
@@ -145,7 +145,7 @@
"BearerAuth": [] "BearerAuth": []
} }
], ],
"description": "返回所有已启用的部署目标(如 k8s 集群、主机池)", "description": "从 AWX 动态返回可用 Job Template 及其 Inventory hosts",
"produces": [ "produces": [
"application/json" "application/json"
], ],
@@ -153,6 +153,14 @@
"delivery" "delivery"
], ],
"summary": "获取可用部署目标", "summary": "获取可用部署目标",
"parameters": [
{
"type": "string",
"description": "组件过滤,例如 mysql",
"name": "component",
"in": "query"
}
],
"responses": { "responses": {
"200": { "200": {
"description": "items: 部署目标数组", "description": "items: 部署目标数组",
@@ -169,64 +177,6 @@
} }
} }
} }
},
"post": {
"security": [
{
"BearerAuth": []
}
],
"description": "管理员创建新的部署目标(目前仅支持 k8s 类型)",
"consumes": [
"application/json"
],
"produces": [
"application/json"
],
"tags": [
"delivery"
],
"summary": "创建部署目标",
"parameters": [
{
"description": "目标配置",
"name": "body",
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/handler.targetPayload"
}
}
],
"responses": {
"201": {
"description": "目标已创建",
"schema": {
"$ref": "#/definitions/model.DeploymentTarget"
}
},
"400": {
"description": "参数错误",
"schema": {
"type": "object",
"additionalProperties": true
}
},
"401": {
"description": "未授权",
"schema": {
"type": "object",
"additionalProperties": true
}
},
"409": {
"description": "名称冲突",
"schema": {
"type": "object",
"additionalProperties": true
}
}
}
} }
}, },
"/auth/api/v1/delivery/tasks": { "/auth/api/v1/delivery/tasks": {
@@ -247,6 +197,7 @@
"parameters": [ "parameters": [
{ {
"type": "integer", "type": "integer",
"format": "int64",
"description": "业务线 ID 过滤", "description": "业务线 ID 过滤",
"name": "business_line_id", "name": "business_line_id",
"in": "query" "in": "query"
@@ -682,65 +633,6 @@
} }
} }
}, },
"handler.targetPayload": {
"type": "object",
"required": [
"awx_inventory_id",
"awx_template_id",
"name",
"target_type"
],
"properties": {
"awx_inventory_id": {
"type": "integer"
},
"awx_template_id": {
"type": "integer"
},
"metadata": {
"type": "object",
"additionalProperties": {}
},
"name": {
"type": "string"
},
"target_type": {
"type": "string"
}
}
},
"model.DeploymentTarget": {
"type": "object",
"properties": {
"awx_inventory_id": {
"type": "integer"
},
"awx_template_id": {
"type": "integer"
},
"created_at": {
"type": "string"
},
"enabled": {
"type": "boolean"
},
"id": {
"type": "integer"
},
"metadata": {
"type": "string"
},
"name": {
"type": "string"
},
"target_type": {
"type": "string"
},
"updated_at": {
"type": "string"
}
}
},
"model.ResourceQuota": { "model.ResourceQuota": {
"type": "object", "type": "object",
"properties": { "properties": {
@@ -809,18 +701,55 @@
"target_id" "target_id"
], ],
"properties": { "properties": {
"binlog_expire_logs_seconds": {
"type": "integer"
},
"business_line_id": { "business_line_id": {
"type": "integer" "type": "integer"
}, },
"character_set": {
"type": "string"
},
"collation": {
"type": "string"
},
"cpu_milli": { "cpu_milli": {
"type": "integer" "type": "integer"
}, },
"data_disk": {
"type": "string"
},
"innodb_flush_log_at_trx_commit": {
"type": "integer"
},
"innodb_io_capacity": {
"type": "integer"
},
"innodb_redo_log_capacity": {
"type": "string"
},
"instance_name": { "instance_name": {
"type": "string" "type": "string"
}, },
"long_query_time": {
"type": "number"
},
"lower_case_table_names": {
"type": "integer"
},
"max_binlog_size": {
"type": "string"
},
"max_connections": {
"description": "高级参数(选填,零值视为未设置)",
"type": "string"
},
"memory_mi": { "memory_mi": {
"type": "integer" "type": "integer"
}, },
"mysql_port": {
"type": "integer"
},
"mysql_version": { "mysql_version": {
"type": "string" "type": "string"
}, },
@@ -830,8 +759,22 @@
"storage_gi": { "storage_gi": {
"type": "integer" "type": "integer"
}, },
"sync_binlog": {
"type": "integer"
},
"target_host": {
"description": "调度控制(选填):点名候选池内主机跳过自动选机,端口/配额/实机守卫照常执行",
"type": "string"
},
"target_id": { "target_id": {
"type": "integer" "type": "integer"
},
"timezone": {
"description": "数据库配置(选填,缺省由 playbook 基线兜底)",
"type": "string"
},
"topology": {
"type": "string"
} }
} }
} }
+42 -79
View File
@@ -22,46 +22,6 @@ definitions:
- storage_gi - storage_gi
- target_id - target_id
type: object type: object
handler.targetPayload:
properties:
awx_inventory_id:
type: integer
awx_template_id:
type: integer
metadata:
additionalProperties: {}
type: object
name:
type: string
target_type:
type: string
required:
- awx_inventory_id
- awx_template_id
- name
- target_type
type: object
model.DeploymentTarget:
properties:
awx_inventory_id:
type: integer
awx_template_id:
type: integer
created_at:
type: string
enabled:
type: boolean
id:
type: integer
metadata:
type: string
name:
type: string
target_type:
type: string
updated_at:
type: string
type: object
model.ResourceQuota: model.ResourceQuota:
properties: properties:
business_line_id: business_line_id:
@@ -101,22 +61,57 @@ definitions:
type: object type: object
service.MySQLDeliveryInput: service.MySQLDeliveryInput:
properties: properties:
binlog_expire_logs_seconds:
type: integer
business_line_id: business_line_id:
type: integer type: integer
character_set:
type: string
collation:
type: string
cpu_milli: cpu_milli:
type: integer type: integer
data_disk:
type: string
innodb_flush_log_at_trx_commit:
type: integer
innodb_io_capacity:
type: integer
innodb_redo_log_capacity:
type: string
instance_name: instance_name:
type: string type: string
long_query_time:
type: number
lower_case_table_names:
type: integer
max_binlog_size:
type: string
max_connections:
description: 高级参数(选填,零值视为未设置)
type: string
memory_mi: memory_mi:
type: integer type: integer
mysql_port:
type: integer
mysql_version: mysql_version:
type: string type: string
namespace: namespace:
type: string type: string
storage_gi: storage_gi:
type: integer type: integer
sync_binlog:
type: integer
target_host:
description: 调度控制(选填):点名候选池内主机跳过自动选机,端口/配额/实机守卫照常执行
type: string
target_id: target_id:
type: integer type: integer
timezone:
description: 数据库配置(选填,缺省由 playbook 基线兜底)
type: string
topology:
type: string
required: required:
- business_line_id - business_line_id
- cpu_milli - cpu_milli
@@ -220,7 +215,12 @@ paths:
- delivery - delivery
/auth/api/v1/delivery/targets: /auth/api/v1/delivery/targets:
get: get:
description: 返回所有已启用的部署目标(如 k8s 集群、主机池) description: 从 AWX 动态返回可用 Job Template 及其 Inventory hosts
parameters:
- description: 组件过滤,例如 mysql
in: query
name: component
type: string
produces: produces:
- application/json - application/json
responses: responses:
@@ -239,49 +239,12 @@ paths:
summary: 获取可用部署目标 summary: 获取可用部署目标
tags: tags:
- delivery - delivery
post:
consumes:
- application/json
description: 管理员创建新的部署目标(目前仅支持 k8s 类型)
parameters:
- description: 目标配置
in: body
name: body
required: true
schema:
$ref: '#/definitions/handler.targetPayload'
produces:
- application/json
responses:
"201":
description: 目标已创建
schema:
$ref: '#/definitions/model.DeploymentTarget'
"400":
description: 参数错误
schema:
additionalProperties: true
type: object
"401":
description: 未授权
schema:
additionalProperties: true
type: object
"409":
description: 名称冲突
schema:
additionalProperties: true
type: object
security:
- BearerAuth: []
summary: 创建部署目标
tags:
- delivery
/auth/api/v1/delivery/tasks: /auth/api/v1/delivery/tasks:
get: get:
description: 返回当前用户可见的交付任务列表(管理员可见全部) description: 返回当前用户可见的交付任务列表(管理员可见全部)
parameters: parameters:
- description: 业务线 ID 过滤 - description: 业务线 ID 过滤
format: int64
in: query in: query
name: business_line_id name: business_line_id
type: integer type: integer
+116 -112
View File
@@ -14,62 +14,64 @@ type OAuthClient struct {
} }
type Config struct { type Config struct {
AppEnv string AppEnv string
HTTPAddr string HTTPAddr string
PublicBaseURL string PublicBaseURL string
MySQLDSN string MySQLDSN string
AutoMigrate bool AutoMigrate bool
SSOEnabled bool SSOEnabled bool
JWTSecret string JWTSecret string
JWTIssuer string JWTIssuer string
JWTTTLMinutes int JWTTTLMinutes int
SAMLEntityID string SAMLEntityID string
SAMLACSURL string SAMLACSURL string
SAMLSPCert string SAMLSPCert string
SAMLSPKey string SAMLSPKey string
SAMLIDPMetaURL string SAMLIDPMetaURL string
SAMLLogoutURL string SAMLLogoutURL string
WayenLoginURL string WayenLoginURL string
WayenTargetURL string WayenTargetURL string
WayenUsernameKey string WayenUsernameKey string
WayenPasswordKey string WayenPasswordKey string
WayenLoginFormat string WayenLoginFormat string
WayenLoginValue string WayenLoginValue string
WayenOAuthRef string WayenOAuthRef string
WayenOAuthLoginURL string WayenOAuthLoginURL string
WayneAPIBaseURL string WayneAPIBaseURL string
WayneAdminUsername string WayneAdminUsername string
WayneAdminPassword string WayneAdminPassword string
WayneTokenTTLMinutes int WayneTokenTTLMinutes int
WayneInternalAPIBaseURL string WayneInternalAPIBaseURL string
WayneServiceName string WayneServiceName string
WayneServiceAPISecretKey string WayneServiceAPISecretKey string
OAuthClientID string OAuthClientID string
OAuthClientSecret string OAuthClientSecret string
OAuthRedirectURI string OAuthRedirectURI string
OAuthCodeTTLSeconds int OAuthCodeTTLSeconds int
OIDCIssuer string OIDCIssuer string
OIDCAuthorizeURL string OIDCAuthorizeURL string
OIDCTokenURL string OIDCTokenURL string
OIDCUserInfoURL string OIDCUserInfoURL string
OIDCJWKSURL string OIDCJWKSURL string
CloudDMClientID string CloudDMClientID string
CloudDMClientSecret string CloudDMClientSecret string
CloudDMRedirectURI string CloudDMRedirectURI string
CloudDMTargetURL string CloudDMTargetURL string
CloudDMRegisterURL string CloudDMRegisterURL string
CloudDMAPIToken string CloudDMAPIToken string
AWXBaseURL string AWXBaseURL string
AWXToken string AWXToken string
AWXUsername string AWXUsername string
AWXPassword string AWXPassword string
AWXWebhookToken string AWXWebhookToken string
DeliverySchedulerEnabled bool DeliverySchedulerEnabled bool
DeliveryDispatchSeconds int DeliveryDispatchSeconds int
DeliveryCallbackBaseURL string DeliveryCallbackBaseURL string
ReservationTTLMinutes int ReservationTTLMinutes int
DeliveryGlobalLimit int DeliveryGlobalLimit int
DeliveryTargetLimit int DeliveryTargetLimit int
DeliveryHostInstanceLimit int
DeliveryDataDisks []string
} }
func Load() Config { func Load() Config {
@@ -82,62 +84,64 @@ func Load() Config {
oidcIssuer = strings.TrimRight(oidcIssuer, "/") oidcIssuer = strings.TrimRight(oidcIssuer, "/")
return Config{ return Config{
AppEnv: env("APP_ENV", "dev"), AppEnv: env("APP_ENV", "dev"),
HTTPAddr: httpAddr, HTTPAddr: httpAddr,
PublicBaseURL: publicBaseURL, PublicBaseURL: publicBaseURL,
MySQLDSN: env("MYSQL_DSN", "auth:auth@tcp(127.0.0.1:3306)/authserver?charset=utf8mb4&parseTime=True&loc=Local"), MySQLDSN: env("MYSQL_DSN", "auth:auth@tcp(127.0.0.1:3306)/authserver?charset=utf8mb4&parseTime=True&loc=Local"),
AutoMigrate: envBool("AUTO_MIGRATE", true), AutoMigrate: envBool("AUTO_MIGRATE", true),
SSOEnabled: envBool("SSO_ENABLED", true), SSOEnabled: envBool("SSO_ENABLED", true),
JWTSecret: env("JWT_SECRET", "change-this-secret"), JWTSecret: env("JWT_SECRET", "change-this-secret"),
JWTIssuer: env("JWT_ISSUER", "authserver"), JWTIssuer: env("JWT_ISSUER", "authserver"),
JWTTTLMinutes: envInt("JWT_TTL_MINUTES", 120), JWTTTLMinutes: envInt("JWT_TTL_MINUTES", 120),
SAMLEntityID: samlEntityID, SAMLEntityID: samlEntityID,
SAMLACSURL: samlACSURL, SAMLACSURL: samlACSURL,
SAMLSPCert: env("SAML_SP_CERT_FILE", "certs/sp.crt"), SAMLSPCert: env("SAML_SP_CERT_FILE", "certs/sp.crt"),
SAMLSPKey: env("SAML_SP_KEY_FILE", "certs/sp.key"), SAMLSPKey: env("SAML_SP_KEY_FILE", "certs/sp.key"),
SAMLIDPMetaURL: env("SAML_IDP_METADATA_URL", "http://sso-internal.dev.qiniu.io/saml2/meta"), SAMLIDPMetaURL: env("SAML_IDP_METADATA_URL", "http://sso-internal.dev.qiniu.io/saml2/meta"),
SAMLLogoutURL: trimURL(env("SAML_LOGOUT_URL", "")), SAMLLogoutURL: trimURL(env("SAML_LOGOUT_URL", "")),
WayenLoginURL: env("WAYEN_LOGIN_URL", ""), WayenLoginURL: env("WAYEN_LOGIN_URL", ""),
WayenTargetURL: env("WAYEN_TARGET_URL", ""), WayenTargetURL: env("WAYEN_TARGET_URL", ""),
WayenUsernameKey: env("WAYEN_USERNAME_KEY", "email"), WayenUsernameKey: env("WAYEN_USERNAME_KEY", "email"),
WayenPasswordKey: env("WAYEN_PASSWORD_KEY", "password"), WayenPasswordKey: env("WAYEN_PASSWORD_KEY", "password"),
WayenLoginFormat: env("WAYEN_LOGIN_FORMAT", "form"), WayenLoginFormat: env("WAYEN_LOGIN_FORMAT", "form"),
WayenLoginValue: env("WAYEN_LOGIN_VALUE", "email"), WayenLoginValue: env("WAYEN_LOGIN_VALUE", "email"),
WayenOAuthRef: env("WAYEN_OAUTH_REF", "/portal/namespace/1/app"), WayenOAuthRef: env("WAYEN_OAUTH_REF", "/portal/namespace/1/app"),
WayenOAuthLoginURL: trimURL(env("WAYEN_OAUTH_LOGIN_URL", "")), WayenOAuthLoginURL: trimURL(env("WAYEN_OAUTH_LOGIN_URL", "")),
WayneAPIBaseURL: trimURL(env("WAYNE_API_BASE_URL", env("WAYNE_INTERNAL_API_BASE_URL", ""))), WayneAPIBaseURL: trimURL(env("WAYNE_API_BASE_URL", env("WAYNE_INTERNAL_API_BASE_URL", ""))),
WayneAdminUsername: env("WAYNE_ADMIN_USERNAME", ""), WayneAdminUsername: env("WAYNE_ADMIN_USERNAME", ""),
WayneAdminPassword: env("WAYNE_ADMIN_PASSWORD", ""), WayneAdminPassword: env("WAYNE_ADMIN_PASSWORD", ""),
WayneTokenTTLMinutes: envInt("WAYNE_TOKEN_TTL_MINUTES", 1440), WayneTokenTTLMinutes: envInt("WAYNE_TOKEN_TTL_MINUTES", 1440),
WayneInternalAPIBaseURL: trimURL(env("WAYNE_INTERNAL_API_BASE_URL", "")), WayneInternalAPIBaseURL: trimURL(env("WAYNE_INTERNAL_API_BASE_URL", "")),
WayneServiceName: env("WAYNE_SERVICE_NAME", "xinfra"), WayneServiceName: env("WAYNE_SERVICE_NAME", "xinfra"),
WayneServiceAPISecretKey: env("WAYNE_SERVICE_API_SECRET_KEY", ""), WayneServiceAPISecretKey: env("WAYNE_SERVICE_API_SECRET_KEY", ""),
OAuthClientID: env("OAUTH_WAYNE_CLIENT_ID", "wayne"), OAuthClientID: env("OAUTH_WAYNE_CLIENT_ID", "wayne"),
OAuthClientSecret: env("OAUTH_WAYNE_CLIENT_SECRET", "wayne-secret"), OAuthClientSecret: env("OAUTH_WAYNE_CLIENT_SECRET", "wayne-secret"),
OAuthRedirectURI: env("OAUTH_WAYNE_REDIRECT_URI", ""), OAuthRedirectURI: env("OAUTH_WAYNE_REDIRECT_URI", ""),
OAuthCodeTTLSeconds: envInt("OAUTH_CODE_TTL_SECONDS", 120), OAuthCodeTTLSeconds: envInt("OAUTH_CODE_TTL_SECONDS", 120),
OIDCIssuer: oidcIssuer, OIDCIssuer: oidcIssuer,
OIDCAuthorizeURL: trimURL(env("OIDC_AUTHORIZATION_ENDPOINT", oidcIssuer+"/oauth/authorize")), OIDCAuthorizeURL: trimURL(env("OIDC_AUTHORIZATION_ENDPOINT", oidcIssuer+"/oauth/authorize")),
OIDCTokenURL: trimURL(env("OIDC_TOKEN_ENDPOINT", oidcIssuer+"/oauth/token")), OIDCTokenURL: trimURL(env("OIDC_TOKEN_ENDPOINT", oidcIssuer+"/oauth/token")),
OIDCUserInfoURL: trimURL(env("OIDC_USERINFO_ENDPOINT", oidcIssuer+"/oauth/userinfo")), OIDCUserInfoURL: trimURL(env("OIDC_USERINFO_ENDPOINT", oidcIssuer+"/oauth/userinfo")),
OIDCJWKSURL: trimURL(env("OIDC_JWKS_URI", oidcIssuer+"/oauth/jwks")), OIDCJWKSURL: trimURL(env("OIDC_JWKS_URI", oidcIssuer+"/oauth/jwks")),
CloudDMClientID: env("OIDC_CLOUDDM_CLIENT_ID", "clouddm"), CloudDMClientID: env("OIDC_CLOUDDM_CLIENT_ID", "clouddm"),
CloudDMClientSecret: env("OIDC_CLOUDDM_CLIENT_SECRET", ""), CloudDMClientSecret: env("OIDC_CLOUDDM_CLIENT_SECRET", ""),
CloudDMRedirectURI: env("OIDC_CLOUDDM_REDIRECT_URI", ""), CloudDMRedirectURI: env("OIDC_CLOUDDM_REDIRECT_URI", ""),
CloudDMTargetURL: env("CLOUDDM_TARGET_URL", ""), CloudDMTargetURL: env("CLOUDDM_TARGET_URL", ""),
CloudDMRegisterURL: trimURL(env("CLOUDDM_REGISTER_URL", "")), CloudDMRegisterURL: trimURL(env("CLOUDDM_REGISTER_URL", "")),
CloudDMAPIToken: env("CLOUDDM_API_TOKEN", ""), CloudDMAPIToken: env("CLOUDDM_API_TOKEN", ""),
AWXBaseURL: trimURL(env("AWX_BASE_URL", "")), AWXBaseURL: trimURL(env("AWX_BASE_URL", "")),
AWXToken: env("AWX_TOKEN", ""), AWXToken: env("AWX_TOKEN", ""),
AWXUsername: env("AWX_USERNAME", ""), AWXUsername: env("AWX_USERNAME", ""),
AWXPassword: env("AWX_PASSWORD", ""), AWXPassword: env("AWX_PASSWORD", ""),
AWXWebhookToken: env("AWX_WEBHOOK_TOKEN", ""), AWXWebhookToken: env("AWX_WEBHOOK_TOKEN", ""),
DeliverySchedulerEnabled: envBool("DELIVERY_SCHEDULER_ENABLED", false), DeliverySchedulerEnabled: envBool("DELIVERY_SCHEDULER_ENABLED", false),
DeliveryDispatchSeconds: envInt("DELIVERY_DISPATCH_SECONDS", 5), DeliveryDispatchSeconds: envInt("DELIVERY_DISPATCH_SECONDS", 5),
DeliveryCallbackBaseURL: trimURL(env("DELIVERY_CALLBACK_BASE_URL", publicBaseURL)), DeliveryCallbackBaseURL: trimURL(env("DELIVERY_CALLBACK_BASE_URL", publicBaseURL)),
ReservationTTLMinutes: envInt("DELIVERY_RESERVATION_TTL_MINUTES", 120), ReservationTTLMinutes: envInt("DELIVERY_RESERVATION_TTL_MINUTES", 120),
DeliveryGlobalLimit: envInt("DELIVERY_GLOBAL_LIMIT", 2), DeliveryGlobalLimit: envInt("DELIVERY_GLOBAL_LIMIT", 2),
DeliveryTargetLimit: envInt("DELIVERY_TARGET_LIMIT", 2), DeliveryTargetLimit: envInt("DELIVERY_TARGET_LIMIT", 2),
DeliveryHostInstanceLimit: envInt("DELIVERY_HOST_INSTANCE_LIMIT", 4),
DeliveryDataDisks: splitCSV(env("DELIVERY_DATA_DISKS", "/data")),
} }
} }
+119 -54
View File
@@ -24,6 +24,7 @@ import (
) )
var dnsLabelPattern = regexp.MustCompile(`^[a-z0-9](?:[-a-z0-9]*[a-z0-9])?$`) var dnsLabelPattern = regexp.MustCompile(`^[a-z0-9](?:[-a-z0-9]*[a-z0-9])?$`)
var hostNamePattern = regexp.MustCompile(`^[a-zA-Z0-9](?:[-a-zA-Z0-9.]*[a-zA-Z0-9])?$`)
type MySQLDeliveryInput struct { type MySQLDeliveryInput struct {
BusinessLineID uint64 `json:"business_line_id" binding:"required"` BusinessLineID uint64 `json:"business_line_id" binding:"required"`
@@ -34,9 +35,10 @@ type MySQLDeliveryInput struct {
Topology string `json:"topology"` Topology string `json:"topology"`
MySQLPort int `json:"mysql_port"` MySQLPort int `json:"mysql_port"`
DataDisk string `json:"data_disk"` DataDisk string `json:"data_disk"`
CPUCores int64 `json:"cpu_cores" binding:"required"` TargetHost string `json:"target_host"`
MemoryGB int64 `json:"memory_gb" binding:"required"` CPUCores int64 `json:"cpu_cores"`
StorageGB int64 `json:"storage_gb" binding:"required"` MemoryGB int64 `json:"memory_gb"`
StorageGB int64 `json:"storage_gb"`
ParamTemplate string `json:"param_template"` ParamTemplate string `json:"param_template"`
TimeZone string `json:"timezone"` TimeZone string `json:"timezone"`
LowerCaseTableNames int `json:"lower_case_table_names"` LowerCaseTableNames int `json:"lower_case_table_names"`
@@ -50,9 +52,9 @@ type MySQLDeliveryInput struct {
LongQueryTime float64 `json:"long_query_time"` LongQueryTime float64 `json:"long_query_time"`
BinlogExpireLogsSeconds int64 `json:"binlog_expire_logs_seconds"` BinlogExpireLogsSeconds int64 `json:"binlog_expire_logs_seconds"`
MaxBinlogSize string `json:"max_binlog_size"` MaxBinlogSize string `json:"max_binlog_size"`
CPUMilli int64 `json:"-"` CPUMilli int64 `json:"cpu_milli"`
MemoryMi int64 `json:"-"` MemoryMi int64 `json:"memory_mi"`
StorageGi int64 `json:"-"` StorageGi int64 `json:"storage_gi"`
} }
type deliveryPayload struct { type deliveryPayload struct {
@@ -118,31 +120,51 @@ func parseTargetMetadata(raw string) targetMetadata {
return meta return meta
} }
// firstFreeHost 返回候选池中第一个未被占用的节点。 // firstFreeHost 返回候选池中非失败任务数未达单机实例上限的第一个节点。
func firstFreeHost(hosts []targetHost, occupied []string) *targetHost { func firstFreeHost(hosts []targetHost, occupied []string, limit int) *targetHost {
taken := make(map[string]bool, len(occupied)) if limit < 1 {
for _, h := range occupied { limit = 1
taken[h] = true
} }
for i := range hosts { for i := range hosts {
if !taken[hosts[i].Name] { if hostTaskCount(occupied, hosts[i].Name) < limit {
return &hosts[i] return &hosts[i]
} }
} }
return nil return nil
} }
func firstFreePort(occupied []int) int { func hostTaskCount(occupied []string, name string) int {
taken := make(map[int]bool, len(occupied)) count := 0
for _, port := range occupied { for _, h := range occupied {
taken[port] = true if h == name {
} count++
for port := 13306; port <= 13999; port++ {
if !taken[port] {
return port
} }
} }
return 0 return count
}
const (
mysqlPortPoolStart = 13306
mysqlPortPoolEnd = 13999
)
func allocatePort(requested int, used []int) (int, error) {
taken := make(map[int]bool, len(used))
for _, p := range used {
taken[p] = true
}
if requested != 0 {
if taken[requested] {
return 0, fmt.Errorf("mysql_port %d is already allocated on the target host", requested)
}
return requested, nil
}
for p := mysqlPortPoolStart; p <= mysqlPortPoolEnd; p++ {
if !taken[p] {
return p, nil
}
}
return 0, fmt.Errorf("mysql port pool %d-%d is exhausted on the target host", mysqlPortPoolStart, mysqlPortPoolEnd)
} }
type DeliveryService struct { type DeliveryService struct {
@@ -228,8 +250,11 @@ func (s *DeliveryService) CreateTask(ctx context.Context, userID uint64, isAdmin
if idempotencyKey == "" || len(idempotencyKey) > 128 { if idempotencyKey == "" || len(idempotencyKey) > 128 {
return nil, false, fmt.Errorf("Idempotency-Key header is required and must not exceed 128 characters") return nil, false, fmt.Errorf("Idempotency-Key header is required and must not exceed 128 characters")
} }
if input.DataDisk == "" && len(s.cfg.DeliveryDataDisks) > 0 {
input.DataDisk = s.cfg.DeliveryDataDisks[0]
}
normalizeMySQLDeliveryInput(&input) normalizeMySQLDeliveryInput(&input)
if err := validateDeliveryInput(input); err != nil { if err := validateDeliveryInput(input, s.cfg.DeliveryDataDisks); err != nil {
return nil, false, err return nil, false, err
} }
@@ -290,9 +315,8 @@ func (s *DeliveryService) CreateTask(ctx context.Context, userID uint64, isAdmin
return &task, false, nil return &task, false, nil
} }
var supportedMySQLVersions = map[string]bool{"8.0": true} var supportedMySQLVersions = map[string]bool{"8.0": true, "8.4": true}
var supportedMySQLTopologies = map[string]bool{"standalone": true} var supportedMySQLTopologies = map[string]bool{"standalone": true}
var supportedDataDisks = map[string]bool{"/data": true, "/disk1": true, "/mnt/vol-1": true}
var supportedParamTemplates = map[string]bool{"default": true, "high_performance": true, "high_safety": true} var supportedParamTemplates = map[string]bool{"default": true, "high_performance": true, "high_safety": true}
var supportedCharacterSets = map[string]bool{"utf8mb4": true, "utf8": true, "gbk": true, "latin1": true} var supportedCharacterSets = map[string]bool{"utf8mb4": true, "utf8": true, "gbk": true, "latin1": true}
var supportedCollations = map[string]bool{ var supportedCollations = map[string]bool{
@@ -301,6 +325,7 @@ var supportedCollations = map[string]bool{
} }
var supportedRedoLogCapacity = map[string]bool{"auto": true, "128M": true, "256M": true, "512M": true, "1G": true} var supportedRedoLogCapacity = map[string]bool{"auto": true, "128M": true, "256M": true, "512M": true, "1G": true}
var supportedMaxBinlogSize = map[string]bool{"128M": true, "256M": true, "512M": true, "1G": true} var supportedMaxBinlogSize = map[string]bool{"128M": true, "256M": true, "512M": true, "1G": true}
var timezonePattern = regexp.MustCompile(`^([+-](0\d|1[0-4]):[0-5]\d|SYSTEM|[A-Za-z]+(?:/[A-Za-z0-9_+-]+)+)$`)
func normalizeMySQLDeliveryInput(input *MySQLDeliveryInput) { func normalizeMySQLDeliveryInput(input *MySQLDeliveryInput) {
if input.MySQLVersion == "" { if input.MySQLVersion == "" {
@@ -348,9 +373,15 @@ func normalizeMySQLDeliveryInput(input *MySQLDeliveryInput) {
if input.MaxBinlogSize == "" { if input.MaxBinlogSize == "" {
input.MaxBinlogSize = "256M" input.MaxBinlogSize = "256M"
} }
input.CPUMilli = input.CPUCores * 1000 if input.CPUMilli == 0 && input.CPUCores != 0 {
input.MemoryMi = input.MemoryGB * 1024 input.CPUMilli = input.CPUCores * 1000
input.StorageGi = input.StorageGB }
if input.MemoryMi == 0 && input.MemoryGB != 0 {
input.MemoryMi = input.MemoryGB * 1024
}
if input.StorageGi == 0 && input.StorageGB != 0 {
input.StorageGi = input.StorageGB
}
} }
func defaultCollation(characterSet string) string { func defaultCollation(characterSet string) string {
@@ -366,38 +397,48 @@ func defaultCollation(characterSet string) string {
} }
} }
func validateDeliveryInput(input MySQLDeliveryInput) error { func validateDeliveryInput(input MySQLDeliveryInput, dataDisks []string) error {
if len(input.Namespace) > 63 || !dnsLabelPattern.MatchString(input.Namespace) { if len(input.Namespace) > 63 || !dnsLabelPattern.MatchString(input.Namespace) {
return fmt.Errorf("namespace must be a valid Kubernetes DNS label") return fmt.Errorf("namespace must be a valid Kubernetes DNS label")
} }
if len(input.InstanceName) > 63 || !dnsLabelPattern.MatchString(input.InstanceName) { if len(input.InstanceName) > 63 || !dnsLabelPattern.MatchString(input.InstanceName) {
return fmt.Errorf("instance_name must be a valid Kubernetes DNS label") return fmt.Errorf("instance_name must be a valid Kubernetes DNS label")
} }
if !oneOfInt64(input.CPUCores, []int64{1, 2, 4, 8, 16}) { if input.CPUMilli < 100 || input.CPUMilli > 64000 || input.MemoryMi < 2048 || input.MemoryMi > 65536 || input.StorageGi < 20 || input.StorageGi > 2000 {
return fmt.Errorf("cpu_cores must be one of 1, 2, 4, 8, 16") return fmt.Errorf("requested resources are outside the supported range (cpu: 100-64000m, memory: 2048-65536 MiB, storage: 20-2000 GiB)")
}
if !oneOfInt64(input.MemoryGB, []int64{2, 4, 8, 16, 32, 64}) {
return fmt.Errorf("memory_gb must be one of 2, 4, 8, 16, 32, 64")
}
if input.StorageGB < 20 || input.StorageGB > 2000 {
return fmt.Errorf("storage_gb must be between 20 and 2000")
} }
if input.MySQLVersion != "" && !supportedMySQLVersions[input.MySQLVersion] { if input.MySQLVersion != "" && !supportedMySQLVersions[input.MySQLVersion] {
return fmt.Errorf("unsupported mysql_version %q, supported: 8.0", input.MySQLVersion) return fmt.Errorf("unsupported mysql_version %q, supported: 8.0, 8.4 (5.6/5.7 are EOL and have no Ubuntu 24.04 packages)", input.MySQLVersion)
} }
if input.Topology != "" && !supportedMySQLTopologies[input.Topology] { if input.Topology != "" && !supportedMySQLTopologies[input.Topology] {
return fmt.Errorf("unsupported topology %q, supported: standalone", input.Topology) return fmt.Errorf("unsupported topology %q, supported: standalone", input.Topology)
} }
if input.MySQLPort != 0 && (input.MySQLPort < 13306 || input.MySQLPort > 13999) { if input.MySQLPort != 0 && (input.MySQLPort < mysqlPortPoolStart || input.MySQLPort > mysqlPortPoolEnd) {
return fmt.Errorf("mysql_port must be empty for auto assignment or between 13306 and 13999") return fmt.Errorf("mysql_port must be empty for auto assignment or between %d and %d", mysqlPortPoolStart, mysqlPortPoolEnd)
} }
if !supportedDataDisks[input.DataDisk] { if input.DataDisk != "" {
return fmt.Errorf("unsupported data_disk %q", input.DataDisk) allowed := dataDisks
if len(allowed) == 0 {
allowed = []string{"/data"}
}
found := false
for _, disk := range allowed {
if input.DataDisk == disk {
found = true
break
}
}
if !found {
return fmt.Errorf("data_disk %q is not in the allowed mount point list %v", input.DataDisk, allowed)
}
}
if input.TargetHost != "" && (len(input.TargetHost) > 253 || !hostNamePattern.MatchString(input.TargetHost)) {
return fmt.Errorf("target_host must be a valid inventory host name")
} }
if !supportedParamTemplates[input.ParamTemplate] { if !supportedParamTemplates[input.ParamTemplate] {
return fmt.Errorf("unsupported param_template %q", input.ParamTemplate) return fmt.Errorf("unsupported param_template %q", input.ParamTemplate)
} }
if !validTimeZone(input.TimeZone) { if !timezonePattern.MatchString(input.TimeZone) {
return fmt.Errorf("unsupported timezone %q", input.TimeZone) return fmt.Errorf("unsupported timezone %q", input.TimeZone)
} }
if input.LowerCaseTableNames != 0 && input.LowerCaseTableNames != 1 { if input.LowerCaseTableNames != 0 && input.LowerCaseTableNames != 1 {
@@ -593,20 +634,42 @@ func (s *DeliveryService) claimAndReserve(ctx context.Context) (*model.DeliveryT
if err := tx.Model(&model.DeliveryTask{}).Where("target_id = ? AND target_host <> ? AND status NOT IN ?", task.TargetID, "", occupiedExclude).Pluck("target_host", &occupied).Error; err != nil { if err := tx.Model(&model.DeliveryTask{}).Where("target_id = ? AND target_host <> ? AND status NOT IN ?", task.TargetID, "", occupiedExclude).Pluck("target_host", &occupied).Error; err != nil {
return err return err
} }
host := firstFreeHost(meta.Hosts, occupied) limit := s.cfg.DeliveryHostInstanceLimit
var host *targetHost
if payload.TargetHost != "" {
for i := range meta.Hosts {
if meta.Hosts[i].Name == payload.TargetHost {
host = &meta.Hosts[i]
break
}
}
if host == nil {
return s.failInTransaction(tx, &task, model.TaskValidationFailed, fmt.Sprintf("target_host %q is not in the candidate host pool", payload.TargetHost))
}
effectiveLimit := limit
if effectiveLimit < 1 {
effectiveLimit = 1
}
if hostTaskCount(occupied, host.Name) >= effectiveLimit {
return fmt.Errorf("defer: pinned host %s reached the per-host instance limit %d", host.Name, effectiveLimit)
}
} else {
host = firstFreeHost(meta.Hosts, occupied, limit)
}
if host == nil { if host == nil {
return fmt.Errorf("defer: no free host available on target") return fmt.Errorf("defer: no free host available on target")
} }
mysqlPort := payload.MySQLPort var usedPorts []int
if mysqlPort == 0 { if err := tx.Model(&model.DeliveryTask{}).Where("target_id = ? AND target_host = ? AND mysql_port <> ? AND status NOT IN ?", task.TargetID, host.Name, 0, occupiedExclude).Pluck("mysql_port", &usedPorts).Error; err != nil {
var occupiedPorts []int return err
if err := tx.Model(&model.DeliveryTask{}).Where("target_id = ? AND target_host = ? AND mysql_port <> ? AND status NOT IN ?", task.TargetID, host.Name, 0, occupiedExclude).Pluck("mysql_port", &occupiedPorts).Error; err != nil { }
return err var instancePorts []int
} if err := tx.Model(&model.MySQLInstance{}).Where("target_id = ? AND node_name = ? AND status = ?", task.TargetID, host.Name, "active").Pluck("port", &instancePorts).Error; err != nil {
mysqlPort = firstFreePort(occupiedPorts) return err
if mysqlPort == 0 { }
return fmt.Errorf("defer: no free MySQL port available on target host") mysqlPort, portErr := allocatePort(payload.MySQLPort, append(usedPorts, instancePorts...))
} if portErr != nil {
return s.failInTransaction(tx, &task, model.TaskValidationFailed, portErr.Error())
} }
reservation := model.ResourceReservation{TaskID: task.ID, BusinessLineID: task.BusinessLineID, TargetID: task.TargetID, CPUMilli: payload.CPUMilli, MemoryMi: payload.MemoryMi, StorageGi: payload.StorageGi, InstanceCount: 1, Status: "reserved", ExpiresAt: time.Now().Add(time.Duration(s.cfg.ReservationTTLMinutes) * time.Minute)} reservation := model.ResourceReservation{TaskID: task.ID, BusinessLineID: task.BusinessLineID, TargetID: task.TargetID, CPUMilli: payload.CPUMilli, MemoryMi: payload.MemoryMi, StorageGi: payload.StorageGi, InstanceCount: 1, Status: "reserved", ExpiresAt: time.Now().Add(time.Duration(s.cfg.ReservationTTLMinutes) * time.Minute)}
if err := tx.Create(&reservation).Error; err != nil { if err := tx.Create(&reservation).Error; err != nil {
@@ -771,8 +834,10 @@ func (s *DeliveryService) CreateExecution(ctx context.Context, taskID, payloadHa
"instance_name": payload.InstanceName, "mysql_port": task.MySQLPort, "instance_name": payload.InstanceName, "mysql_port": task.MySQLPort,
"data_disk": payload.DataDisk, "cpu_cores": payload.CPUCores, "data_disk": payload.DataDisk, "cpu_cores": payload.CPUCores,
"memory_gb": payload.MemoryGB, "storage_gb": payload.StorageGB, "memory_gb": payload.MemoryGB, "storage_gb": payload.StorageGB,
"cpu_milli": payload.CPUMilli, "memory_mi": payload.MemoryMi, "storage_gi": payload.StorageGi,
"mysql_version": payload.MySQLVersion, "param_template": payload.ParamTemplate, "mysql_version": payload.MySQLVersion, "param_template": payload.ParamTemplate,
"timezone": payload.TimeZone, "lower_case_table_names": payload.LowerCaseTableNames, "target_host": payload.TargetHost,
"timezone": payload.TimeZone, "lower_case_table_names": payload.LowerCaseTableNames,
"character_set": payload.CharacterSet, "collation": payload.Collation, "character_set": payload.CharacterSet, "collation": payload.Collation,
"max_connections": payload.MaxConnections, "innodb_redo_log_capacity": payload.InnoDBRedoLogCapacity, "max_connections": payload.MaxConnections, "innodb_redo_log_capacity": payload.InnoDBRedoLogCapacity,
"innodb_flush_log_at_trx_commit": payload.InnoDBFlushLogAtTrxCommit, "sync_binlog": payload.SyncBinlog, "innodb_flush_log_at_trx_commit": payload.InnoDBFlushLogAtTrxCommit, "sync_binlog": payload.SyncBinlog,
+110 -46
View File
@@ -3,59 +3,123 @@ package service
import "testing" import "testing"
func TestValidateDeliveryInput(t *testing.T) { func TestValidateDeliveryInput(t *testing.T) {
dataDisks := []string{"/data", "/disk1"}
valid := MySQLDeliveryInput{BusinessLineID: 1, TargetID: 1, Namespace: "team-a", InstanceName: "mysql-01", CPUCores: 2, MemoryGB: 4, StorageGB: 50} valid := MySQLDeliveryInput{BusinessLineID: 1, TargetID: 1, Namespace: "team-a", InstanceName: "mysql-01", CPUCores: 2, MemoryGB: 4, StorageGB: 50}
normalizeMySQLDeliveryInput(&valid) normalizeMySQLDeliveryInput(&valid)
if err := validateDeliveryInput(valid); err != nil { if err := validateDeliveryInput(valid, dataDisks); err != nil {
t.Fatalf("valid input rejected: %v", err) t.Fatalf("valid input rejected: %v", err)
} }
withVersion := valid full := valid
withVersion.MySQLVersion = "8.0" full.MySQLVersion = "8.0"
if err := validateDeliveryInput(withVersion); err != nil { full.Topology = "standalone"
t.Fatalf("valid input with version 8.0 rejected: %v", err) full.MySQLPort = 13306
full.DataDisk = "/disk1"
full.TargetHost = "k8s-server-03"
full.CPUMilli = 2000
full.MemoryMi = 8192
full.StorageGi = 2000
full.TimeZone = "+08:00"
full.LowerCaseTableNames = 0
full.CharacterSet = "utf8mb4"
full.Collation = "utf8mb4_general_ci"
full.MaxConnections = "auto"
full.InnoDBRedoLogCapacity = "256M"
full.InnoDBFlushLogAtTrxCommit = 2
full.SyncBinlog = 0
full.InnoDBIOCapacity = 2000
full.LongQueryTime = 0.5
full.BinlogExpireLogsSeconds = 604800
full.MaxBinlogSize = "512M"
if err := validateDeliveryInput(full, dataDisks); err != nil {
t.Fatalf("valid full input rejected: %v", err)
} }
for name, input := range map[string]MySQLDeliveryInput{ namedZone := valid
"uppercase namespace": valid, namedZone.TimeZone = "Asia/Shanghai"
"bad instance": valid, if err := validateDeliveryInput(namedZone, dataDisks); err != nil {
"bad cpu cores": valid, t.Fatalf("named timezone rejected: %v", err)
"too much memory": valid, }
"too little storage": valid, lts := valid
"too much storage": valid, lts.MySQLVersion = "8.4"
"unsupported version": valid, if err := validateDeliveryInput(lts, dataDisks); err != nil {
"unsupported topology": valid, t.Fatalf("8.4 LTS rejected: %v", err)
"bad mysql port": valid, }
"bad data disk": valid, for name, mutate := range map[string]func(*MySQLDeliveryInput){
"unsupported charset": valid, "uppercase namespace": func(in *MySQLDeliveryInput) { in.Namespace = "Team-A" },
"bad collation": valid, "bad instance": func(in *MySQLDeliveryInput) { in.InstanceName = "mysql_01" },
"too little cpu": func(in *MySQLDeliveryInput) { in.CPUMilli = 50 },
"too much cpu": func(in *MySQLDeliveryInput) { in.CPUMilli = 65000 },
"too little memory": func(in *MySQLDeliveryInput) { in.MemoryMi = 1024 },
"too much memory": func(in *MySQLDeliveryInput) { in.MemoryMi = 131072 },
"too little storage": func(in *MySQLDeliveryInput) { in.StorageGi = 10 },
"too much storage": func(in *MySQLDeliveryInput) { in.StorageGi = 4000 },
"unsupported version": func(in *MySQLDeliveryInput) { in.MySQLVersion = "5.7" },
"eol version": func(in *MySQLDeliveryInput) { in.MySQLVersion = "5.6" },
"unsupported topology": func(in *MySQLDeliveryInput) { in.Topology = "mgr_3" },
"port below pool": func(in *MySQLDeliveryInput) { in.MySQLPort = 3307 },
"port above pool": func(in *MySQLDeliveryInput) { in.MySQLPort = 14000 },
"data disk not in list": func(in *MySQLDeliveryInput) { in.DataDisk = "/mnt/other" },
"bad target host": func(in *MySQLDeliveryInput) { in.TargetHost = "-bad-host" },
"bad timezone": func(in *MySQLDeliveryInput) { in.TimeZone = "UTC+8" },
"bad lower case": func(in *MySQLDeliveryInput) { in.LowerCaseTableNames = 2 },
"bad charset": func(in *MySQLDeliveryInput) { in.CharacterSet = "big5" },
"collation mismatch": func(in *MySQLDeliveryInput) { in.CharacterSet = "gbk"; in.Collation = "utf8mb4_general_ci" },
"bad max connections": func(in *MySQLDeliveryInput) { in.MaxConnections = "300" },
"bad redo capacity": func(in *MySQLDeliveryInput) { in.InnoDBRedoLogCapacity = "2G" },
"bad flush log": func(in *MySQLDeliveryInput) { in.InnoDBFlushLogAtTrxCommit = 3 },
"bad sync binlog": func(in *MySQLDeliveryInput) { in.SyncBinlog = 2 },
"bad io capacity": func(in *MySQLDeliveryInput) { in.InnoDBIOCapacity = 500 },
"bad long query time": func(in *MySQLDeliveryInput) { in.LongQueryTime = 3 },
"bad binlog expire": func(in *MySQLDeliveryInput) { in.BinlogExpireLogsSeconds = 3600 },
"bad max binlog size": func(in *MySQLDeliveryInput) { in.MaxBinlogSize = "64M" },
} { } {
switch name { input := valid
case "uppercase namespace": mutate(&input)
input.Namespace = "Team-A" if err := validateDeliveryInput(input, dataDisks); err == nil {
case "bad instance":
input.InstanceName = "mysql_01"
case "bad cpu cores":
input.CPUCores = 3
case "too much memory":
input.MemoryGB = 128
case "too little storage":
input.StorageGB = 10
case "too much storage":
input.StorageGB = 3000
case "unsupported version":
input.MySQLVersion = "5.7"
case "unsupported topology":
input.Topology = "mgr"
case "bad mysql port":
input.MySQLPort = 3306
case "bad data disk":
input.DataDisk = "/"
case "unsupported charset":
input.CharacterSet = "sjis"
case "bad collation":
input.CharacterSet = "utf8"
input.Collation = "utf8mb4_general_ci"
}
if err := validateDeliveryInput(input); err == nil {
t.Errorf("%s was accepted", name) t.Errorf("%s was accepted", name)
} }
} }
} }
func TestFirstFreeHost(t *testing.T) {
hosts := []targetHost{{Name: "node-a"}, {Name: "node-b"}}
if h := firstFreeHost(hosts, nil, 1); h == nil || h.Name != "node-a" {
t.Fatalf("expected node-a on empty occupancy, got %+v", h)
}
if h := firstFreeHost(hosts, []string{"node-a"}, 1); h == nil || h.Name != "node-b" {
t.Fatalf("expected node-b when node-a is full at limit 1, got %+v", h)
}
if h := firstFreeHost(hosts, []string{"node-a", "node-b"}, 1); h != nil {
t.Fatalf("limit 1 with all hosts taken should return nil, got %+v", h)
}
if h := firstFreeHost(hosts, []string{"node-a", "node-b"}, 2); h == nil || h.Name != "node-a" {
t.Fatalf("expected node-a for second round at limit 2, got %+v", h)
}
if h := firstFreeHost(hosts, []string{"node-a", "node-a", "node-b", "node-b"}, 2); h != nil {
t.Fatalf("limit 2 with all hosts saturated should return nil, got %+v", h)
}
if h := firstFreeHost(hosts, []string{"node-a"}, 0); h == nil || h.Name != "node-b" {
t.Fatalf("limit 0 should degrade to 1, got %+v", h)
}
}
func TestAllocatePort(t *testing.T) {
if port, err := allocatePort(0, nil); err != nil || port != mysqlPortPoolStart {
t.Fatalf("expected first pool port %d, got %d err=%v", mysqlPortPoolStart, port, err)
}
if port, err := allocatePort(0, []int{13306, 13307}); err != nil || port != 13308 {
t.Fatalf("expected 13308 skipping occupied, got %d err=%v", port, err)
}
if port, err := allocatePort(13400, []int{13306}); err != nil || port != 13400 {
t.Fatalf("expected requested port 13400, got %d err=%v", port, err)
}
if _, err := allocatePort(13306, []int{13306}); err == nil {
t.Fatal("requested occupied port was accepted")
}
used := make([]int, 0, mysqlPortPoolEnd-mysqlPortPoolStart+1)
for p := mysqlPortPoolStart; p <= mysqlPortPoolEnd; p++ {
used = append(used, p)
}
if _, err := allocatePort(0, used); err == nil {
t.Fatal("exhausted pool still allocated a port")
}
}