Merge remote-tracking branch 'upstream/main' into feat/base-service-delivery
# Conflicts: # server/.env.example # server/internal/config/config.go # server/internal/service/delivery.go # server/internal/service/delivery_test.go
This commit is contained in:
+208
-64
@@ -1,82 +1,186 @@
|
||||
---
|
||||
- name: Preflight native MySQL delivery
|
||||
- name: Native MySQL delivery
|
||||
hosts: "{{ target_hosts }}"
|
||||
become: true
|
||||
gather_facts: true
|
||||
any_errors_fatal: true
|
||||
vars:
|
||||
# --- identity ---
|
||||
mysql_instance: "{{ instance_name }}"
|
||||
mysql_version_value: "{{ mysql_version | default('8.0') }}"
|
||||
# 版本包映射:8.0 用 Ubuntu 24.04 自带源(精确锁版);8.4 用 MySQL 官方 APT 源的 LTS 组件
|
||||
# (noble 自带源无 8.4,官方源组件内即为该系列,不再锁小版本)。
|
||||
# 5.6/5.7 已官方 EOL 且 Ubuntu 24.04 无可用 apt 包,明确不纳入白名单。
|
||||
mysql_package_map:
|
||||
"8.0": "mysql-server=8.0.46-0ubuntu0.24.04.3"
|
||||
mysql_package_name: "{{ mysql_package_map[mysql_version_value] }}"
|
||||
mysql_port_value: "{{ mysql_port | default(3307) | int }}"
|
||||
mysql_memory_mb_value: "{{ memory_mb | default(2048) | int }}"
|
||||
mysql_storage_gb_value: "{{ storage_gb | default(20) | int }}"
|
||||
"8.0":
|
||||
package: "mysql-server=8.0.46-0ubuntu0.24.04.3"
|
||||
repo_component: ""
|
||||
"8.4":
|
||||
package: "mysql-community-server"
|
||||
repo_component: "mysql-8.4-lts"
|
||||
mysql_package_name: "{{ (mysql_package_map[mysql_version_value] | default({})).package | default('') }}"
|
||||
mysql_repo_component: "{{ (mysql_package_map[mysql_version_value] | default({})).repo_component | default('') }}"
|
||||
|
||||
# --- platform-allocated inputs (safe fallbacks when not passed in) ---
|
||||
# 端口池 13306–13999:平台从池分配并传入;未传时兜底池首端口,占用探测在目标机执行。
|
||||
mysql_port_value: "{{ mysql_port | default(13306) | int }}"
|
||||
# GR 组通信端口:仅 mgr_3 使用,平台成对分配;兜底为 SQL 端口 +10000。
|
||||
mysql_gr_port_value: "{{ gr_port | default((mysql_port | default(13306) | int) + 10000) | int }}"
|
||||
# 数据盘挂载点:平台按白名单选定并传入;兜底 /data。
|
||||
mysql_data_disk: "{{ data_disk | default('/data') }}"
|
||||
|
||||
# --- resource quota (Go→AWX 契约单位保持 MB/GB,不擅改) ---
|
||||
mysql_memory_mb_value: "{{ memory_mb | default(4096) | int }}"
|
||||
mysql_storage_gb_value: "{{ storage_gb | default(50) | int }}"
|
||||
|
||||
# --- mount-point-agnostic layout: {data_disk}/mysql-delivery/{instance_id}/... ---
|
||||
mysql_base_dir: "{{ mysql_data_disk }}/mysql-delivery/{{ mysql_instance }}"
|
||||
mysql_install_dir: "/opt/mysql-delivery/{{ mysql_instance }}"
|
||||
mysql_data_dir: "/data/mysql-delivery/{{ mysql_instance }}/data"
|
||||
mysql_log_dir: "/data/mysql-delivery/{{ mysql_instance }}/log"
|
||||
mysql_data_dir: "{{ mysql_base_dir }}/data"
|
||||
mysql_log_dir: "{{ mysql_base_dir }}/logs"
|
||||
mysql_binlog_dir: "{{ mysql_base_dir }}/logs/binlog"
|
||||
mysql_redo_dir: "{{ mysql_base_dir }}/logs/redo"
|
||||
mysql_tmp_dir: "{{ mysql_base_dir }}/tmp"
|
||||
# socket/pid 走 /run tmpfs(重启自动清理),由 systemd RuntimeDirectory 创建。
|
||||
mysql_run_dir: "/run/mysql-delivery-{{ mysql_instance }}"
|
||||
mysql_config_file: "/etc/mysql/mysql-delivery/{{ mysql_instance }}.cnf"
|
||||
|
||||
# --- database config (user form, with doc default baselines) ---
|
||||
mysql_timezone: "{{ timezone | default('+08:00') }}"
|
||||
mysql_lower_case_table_names: "{{ lower_case_table_names | default(1) | int }}"
|
||||
mysql_character_set: "{{ character_set | default('utf8mb4') }}"
|
||||
mysql_collation: "{{ collation | default('utf8mb4_general_ci') }}"
|
||||
|
||||
# --- advanced params (overridable; default baseline assumes SSD for io_capacity) ---
|
||||
mysql_flush_log_at_trx_commit: "{{ innodb_flush_log_at_trx_commit | default(1) | int }}"
|
||||
mysql_sync_binlog: "{{ sync_binlog | default(1) | int }}"
|
||||
mysql_io_capacity: "{{ innodb_io_capacity | default(2000) | int }}"
|
||||
mysql_long_query_time: "{{ long_query_time | default(1) }}"
|
||||
mysql_binlog_expire_logs_seconds: "{{ binlog_expire_logs_seconds | default(604800) | int }}"
|
||||
mysql_max_binlog_size: "{{ max_binlog_size | default('256M') }}"
|
||||
mgr_group_name: "{{ group_replication_group_name | default('aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa') }}"
|
||||
|
||||
# --- expected node count per topology ---
|
||||
mysql_expected_hosts: "{{ {'standalone': 1, 'primary_replica': 2, 'mgr_3': 3}[topology | default('standalone')] }}"
|
||||
|
||||
# --- secrets (injected via environment) ---
|
||||
mysql_root_password_value: "{{ lookup('ansible.builtin.env', 'XINFRA_MYSQL_ROOT_PASSWORD') }}"
|
||||
mysql_admin_password_value: "{{ lookup('ansible.builtin.env', 'XINFRA_MYSQL_ADMIN_PASSWORD') }}"
|
||||
|
||||
pre_tasks:
|
||||
- name: Validate prototype parameters
|
||||
- name: Validate delivery parameters
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- topology == 'standalone'
|
||||
- topology in ['standalone', 'primary_replica', 'mgr_3']
|
||||
- mysql_instance is match('^[a-z0-9][a-z0-9-]{0,62}$')
|
||||
- mysql_version_value in mysql_package_map
|
||||
- (mysql_port_value | int) >= 1024
|
||||
- (mysql_port_value | int) <= 65535
|
||||
- (mysql_memory_mb_value | int) >= 1024
|
||||
- (mysql_memory_mb_value | int) <= 4096
|
||||
- (mysql_storage_gb_value | int) >= 10
|
||||
- (mysql_storage_gb_value | int) <= 100
|
||||
- mysql_data_disk is match('^/')
|
||||
- (mysql_port_value | int) >= 13306
|
||||
- (mysql_port_value | int) <= 13999
|
||||
- (mysql_memory_mb_value | int) >= 2048
|
||||
- (mysql_memory_mb_value | int) <= 65536
|
||||
- (mysql_storage_gb_value | int) >= 20
|
||||
- (mysql_storage_gb_value | int) <= 2000
|
||||
- (mysql_lower_case_table_names | int) in [0, 1]
|
||||
- mysql_root_password_value | length >= 16
|
||||
- mysql_admin_password_value | length >= 16
|
||||
fail_msg: The first machine prototype only supports safe standalone parameters
|
||||
fail_msg: >-
|
||||
Delivery parameters out of the supported target-state whitelist
|
||||
(topology / version-package-map / port pool 13306-13999 / memory 2-64G / storage 20-2000G).
|
||||
quiet: true
|
||||
no_log: true
|
||||
|
||||
- name: Validate topology host count
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- ansible_play_hosts_all | length == 1
|
||||
fail_msg: The selected topology does not match the target host count
|
||||
- (ansible_play_hosts_all | length | int) == (mysql_expected_hosts | int)
|
||||
fail_msg: "topology={{ topology }} expects {{ mysql_expected_hosts }} host(s), got {{ ansible_play_hosts_all | length }}"
|
||||
run_once: true
|
||||
|
||||
- name: Check port ownership
|
||||
ansible.builtin.shell: |
|
||||
set -o pipefail
|
||||
if ss -lntH "sport = :{{ mysql_port_value }}" | grep -q .; then
|
||||
systemctl is-active --quiet "mysql-delivery@{{ mysql_instance }}.service"
|
||||
fi
|
||||
args:
|
||||
- name: Probe target-host port occupancy (SQL + GR)
|
||||
# cmd 字典形式不经过 free-form split_args 解析,避免引号/Jinja 块导致的解析失败。
|
||||
ansible.builtin.shell:
|
||||
cmd: |
|
||||
set -o pipefail
|
||||
for p in {{ mysql_probe_ports | join(' ') }}; do
|
||||
if ss -lntH "sport = :${p}" | grep -q .; then
|
||||
# already listening: only tolerated when owned by this instance service
|
||||
if ! systemctl is-active --quiet "mysql-delivery@{{ mysql_instance }}.service"; then
|
||||
echo "port ${p} already in use on target host" >&2
|
||||
exit 3
|
||||
fi
|
||||
fi
|
||||
done
|
||||
executable: /bin/bash
|
||||
vars:
|
||||
mysql_probe_ports: "{{ [mysql_port_value, mysql_gr_port_value] if topology == 'mgr_3' else [mysql_port_value] }}"
|
||||
changed_when: false
|
||||
|
||||
- name: Read currently available memory
|
||||
- name: Read currently available memory (point-in-time guard)
|
||||
ansible.builtin.shell: awk '/^MemAvailable:/ { print int($2 / 1024) }' /proc/meminfo
|
||||
args:
|
||||
executable: /bin/bash
|
||||
register: mysql_available_memory
|
||||
changed_when: false
|
||||
|
||||
- name: Check available memory and disk
|
||||
- name: Resolve data-disk mountpoint available space
|
||||
ansible.builtin.set_fact:
|
||||
mysql_mount_avail: >-
|
||||
{{ (ansible_mounts | selectattr('mount', 'equalto', mysql_data_disk)
|
||||
| map(attribute='size_available') | first)
|
||||
| default(ansible_mounts | selectattr('mount', 'equalto', '/')
|
||||
| map(attribute='size_available') | first) }}
|
||||
|
||||
- name: Check available memory and data-disk space
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- (mysql_available_memory.stdout | int) >= (mysql_memory_mb_value | int)
|
||||
- (ansible_mounts | selectattr('mount', 'equalto', '/') | map(attribute='size_available') | first | int) >= (mysql_storage_gb_value | int) * 1073741824
|
||||
fail_msg: Target host does not have enough available memory or disk
|
||||
- (mysql_mount_avail | int) >= (mysql_storage_gb_value | int) * 1073741824
|
||||
fail_msg: >-
|
||||
Target host lacks memory or free space on {{ mysql_data_disk }};
|
||||
host-wide Σ-quota budgeting is the platform's responsibility, this is a last-resort guard.
|
||||
|
||||
tasks:
|
||||
- name: Detect an existing Ubuntu MySQL package
|
||||
ansible.builtin.command: dpkg-query -W mysql-server
|
||||
- name: Detect an existing MySQL server package
|
||||
ansible.builtin.command: dpkg-query -W -f '${Package}=${Version}\n' mysql-server mysql-community-server
|
||||
register: mysql_package_before
|
||||
failed_when: false
|
||||
changed_when: false
|
||||
|
||||
- name: Install Ubuntu MySQL package
|
||||
- name: Resolve the installed MySQL server version
|
||||
ansible.builtin.set_fact:
|
||||
mysql_installed_version: >-
|
||||
{{ (mysql_package_before.stdout_lines | select('search', '=') | list
|
||||
| first | default('')).split('=') | last }}
|
||||
|
||||
# /usr 下的 mysqld/mysql 二进制全机共享,一台主机只能承载一个 MySQL 版本系列。
|
||||
- name: Enforce host-level MySQL series consistency
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- mysql_installed_version == '' or mysql_installed_version.startswith(mysql_version_value ~ '.')
|
||||
fail_msg: >-
|
||||
Host already runs MySQL {{ mysql_installed_version }} but {{ mysql_version_value }} was requested;
|
||||
native binaries under /usr are shared host-wide, so one host serves exactly one MySQL series.
|
||||
|
||||
- name: Install the MySQL APT repository signing key
|
||||
ansible.builtin.get_url:
|
||||
url: https://repo.mysql.com/RPM-GPG-KEY-mysql-2023
|
||||
dest: /etc/apt/keyrings/mysql.asc
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0644'
|
||||
when: mysql_repo_component != ''
|
||||
|
||||
- name: Configure the MySQL APT repository component
|
||||
ansible.builtin.apt_repository:
|
||||
repo: >-
|
||||
deb [signed-by=/etc/apt/keyrings/mysql.asc]
|
||||
http://repo.mysql.com/apt/ubuntu {{ ansible_distribution_release }} {{ mysql_repo_component }}
|
||||
filename: xinfra-mysql-delivery
|
||||
state: present
|
||||
when: mysql_repo_component != ''
|
||||
|
||||
- name: Install the MySQL server package
|
||||
ansible.builtin.apt:
|
||||
name: "{{ mysql_package_name }}"
|
||||
state: present
|
||||
@@ -88,14 +192,14 @@
|
||||
name: mysql.service
|
||||
state: stopped
|
||||
enabled: false
|
||||
when: mysql_package_before.rc != 0
|
||||
when: mysql_installed_version == ''
|
||||
|
||||
- name: Check for the bundled MySQL AppArmor profile
|
||||
ansible.builtin.stat:
|
||||
path: /etc/apparmor.d/usr.sbin.mysqld
|
||||
register: mysql_apparmor_profile
|
||||
|
||||
- name: Allow the delivery data and run paths in the MySQL AppArmor profile
|
||||
- name: Authorize the delivery paths in the MySQL AppArmor profile
|
||||
ansible.builtin.copy:
|
||||
dest: /etc/apparmor.d/local/usr.sbin.mysqld
|
||||
owner: root
|
||||
@@ -103,8 +207,8 @@
|
||||
mode: '0644'
|
||||
content: |
|
||||
# Managed by XINFRA MySQL delivery - grant per-instance native paths
|
||||
/data/mysql-delivery/ r,
|
||||
/data/mysql-delivery/** rwk,
|
||||
{{ mysql_data_disk }}/mysql-delivery/ r,
|
||||
{{ mysql_data_disk }}/mysql-delivery/** rwk,
|
||||
/run/mysql-delivery-*/ rw,
|
||||
/run/mysql-delivery-*/** rwk,
|
||||
when: mysql_apparmor_profile.stat.exists
|
||||
@@ -115,7 +219,7 @@
|
||||
when: mysql_apparmor_profile.stat.exists and mysql_apparmor_local.changed
|
||||
changed_when: true
|
||||
|
||||
- name: Create instance directories
|
||||
- name: Create instance directories (mount-point-agnostic layout)
|
||||
ansible.builtin.file:
|
||||
path: "{{ item.path }}"
|
||||
state: directory
|
||||
@@ -125,9 +229,12 @@
|
||||
loop:
|
||||
- { path: /etc/mysql/mysql-delivery, owner: root, group: mysql, mode: '0750' }
|
||||
- { path: "{{ mysql_install_dir }}", owner: root, group: root, mode: '0755' }
|
||||
- { path: "{{ mysql_base_dir }}", owner: mysql, group: mysql, mode: '0750' }
|
||||
- { path: "{{ mysql_data_dir }}", owner: mysql, group: mysql, mode: '0750' }
|
||||
- { path: "{{ mysql_log_dir }}", owner: mysql, group: mysql, mode: '0750' }
|
||||
- { path: "{{ mysql_run_dir }}", owner: mysql, group: mysql, mode: '0755' }
|
||||
- { path: "{{ mysql_binlog_dir }}", owner: mysql, group: mysql, mode: '0750' }
|
||||
- { path: "{{ mysql_redo_dir }}", owner: mysql, group: mysql, mode: '0750' }
|
||||
- { path: "{{ mysql_tmp_dir }}", owner: mysql, group: mysql, mode: '0750' }
|
||||
|
||||
- name: Link native binaries into the instance directory
|
||||
ansible.builtin.file:
|
||||
@@ -146,6 +253,33 @@
|
||||
{{ 'standalone' if topology == 'standalone' else
|
||||
('primary' if ansible_play_hosts_all.index(inventory_hostname) == 0 else
|
||||
('replica' if topology == 'primary_replica' else 'mgr')) }}
|
||||
# host-wide unique: platform may pass explicit mysql_server_id; otherwise derive
|
||||
# port + node index (ports are unique per host in the pool model).
|
||||
mysql_server_id_value: >-
|
||||
{{ mysql_server_id | default((mysql_port_value | int)
|
||||
+ (ansible_play_hosts_all.index(inventory_hostname))) | int }}
|
||||
|
||||
- name: Resolve memory tier (GB)
|
||||
ansible.builtin.set_fact:
|
||||
mysql_memory_gb: "{{ ((mysql_memory_mb_value | int) // 1024) | int }}"
|
||||
|
||||
- name: Resolve linkage-derived defaults (illustrative tiers, pending hardware calibration)
|
||||
ansible.builtin.set_fact:
|
||||
mysql_max_connections: >-
|
||||
{{ (max_connections | int) if (max_connections is defined and (max_connections | string) != 'auto')
|
||||
else (200 if (mysql_memory_gb | int) <= 2
|
||||
else 500 if (mysql_memory_gb | int) <= 4
|
||||
else 1000 if (mysql_memory_gb | int) <= 8
|
||||
else 2000 if (mysql_memory_gb | int) <= 16
|
||||
else 4000 if (mysql_memory_gb | int) <= 32
|
||||
else 8000 if (mysql_memory_gb | int) <= 64
|
||||
else 16000) }}
|
||||
mysql_redo_capacity: >-
|
||||
{{ innodb_redo_log_capacity if (innodb_redo_log_capacity is defined)
|
||||
else ('128M' if (mysql_memory_gb | int) <= 4
|
||||
else '256M' if (mysql_memory_gb | int) <= 16
|
||||
else '512M' if (mysql_memory_gb | int) <= 32
|
||||
else '1G') }}
|
||||
|
||||
- name: Write instance configuration
|
||||
ansible.builtin.template:
|
||||
@@ -193,32 +327,33 @@
|
||||
timeout: 60
|
||||
|
||||
- name: Configure local administrative accounts
|
||||
ansible.builtin.shell: |
|
||||
set -euo pipefail
|
||||
client_file="$(mktemp)"
|
||||
sql_file="$(mktemp)"
|
||||
trap 'rm -f "$client_file" "$sql_file"' EXIT
|
||||
chmod 600 "$client_file" "$sql_file"
|
||||
cat >"$client_file" <<'EOF'
|
||||
[client]
|
||||
user=root
|
||||
password={{ mysql_root_password_value }}
|
||||
socket={{ mysql_run_dir }}/mysql.sock
|
||||
EOF
|
||||
if ! /usr/bin/mysql --defaults-extra-file="$client_file" -e 'SELECT 1' >/dev/null 2>&1; then
|
||||
# cmd 字典形式不经过 free-form split_args 解析,heredoc SQL 中的奇数个单引号才不会报错。
|
||||
ansible.builtin.shell:
|
||||
cmd: |
|
||||
set -euo pipefail
|
||||
client_file="$(mktemp)"
|
||||
sql_file="$(mktemp)"
|
||||
trap 'rm -f "$client_file" "$sql_file"' EXIT
|
||||
chmod 600 "$client_file" "$sql_file"
|
||||
cat >"$client_file" <<'EOF'
|
||||
[client]
|
||||
user=root
|
||||
password={{ mysql_root_password_value }}
|
||||
socket={{ mysql_run_dir }}/mysql.sock
|
||||
EOF
|
||||
if ! /usr/bin/mysql --defaults-extra-file="$client_file" -e 'SELECT 1' >/dev/null 2>&1; then
|
||||
cat >"$sql_file" <<'EOF'
|
||||
ALTER USER 'root'@'localhost' IDENTIFIED BY '{{ mysql_root_password_value | replace("'", "''") }}';
|
||||
EOF
|
||||
/usr/bin/mysql --protocol=socket --socket={{ mysql_run_dir }}/mysql.sock -uroot <"$sql_file"
|
||||
fi
|
||||
cat >"$sql_file" <<'EOF'
|
||||
ALTER USER 'root'@'localhost' IDENTIFIED BY '{{ mysql_root_password_value | replace("'", "''") }}';
|
||||
EOF
|
||||
/usr/bin/mysql --protocol=socket --socket={{ mysql_run_dir }}/mysql.sock -uroot <"$sql_file"
|
||||
fi
|
||||
cat >"$sql_file" <<'EOF'
|
||||
CREATE USER IF NOT EXISTS 'xinfra_admin'@'%' IDENTIFIED BY '{{ mysql_admin_password_value | replace("'", "''") }}';
|
||||
ALTER USER 'xinfra_admin'@'%' IDENTIFIED BY '{{ mysql_admin_password_value | replace("'", "''") }}';
|
||||
GRANT ALL PRIVILEGES ON *.* TO 'xinfra_admin'@'%' WITH GRANT OPTION;
|
||||
FLUSH PRIVILEGES;
|
||||
EOF
|
||||
/usr/bin/mysql --defaults-extra-file="$client_file" <"$sql_file"
|
||||
args:
|
||||
CREATE USER IF NOT EXISTS 'xinfra_admin'@'%' IDENTIFIED BY '{{ mysql_admin_password_value | replace("'", "''") }}';
|
||||
ALTER USER 'xinfra_admin'@'%' IDENTIFIED BY '{{ mysql_admin_password_value | replace("'", "''") }}';
|
||||
GRANT ALL PRIVILEGES ON *.* TO 'xinfra_admin'@'%' WITH GRANT OPTION;
|
||||
FLUSH PRIVILEGES;
|
||||
EOF
|
||||
/usr/bin/mysql --defaults-extra-file="$client_file" <"$sql_file"
|
||||
executable: /bin/bash
|
||||
changed_when: false
|
||||
no_log: true
|
||||
@@ -229,6 +364,15 @@
|
||||
port: "{{ mysql_port_value }}"
|
||||
timeout: 30
|
||||
|
||||
- name: Note pending HA runtime orchestration
|
||||
ansible.builtin.debug:
|
||||
msg: >-
|
||||
topology={{ topology }} deployed with HA-ready config (GTID/binlog/relay/GR settings in place),
|
||||
but replication wiring (CHANGE REPLICATION SOURCE) and Group Replication bootstrap
|
||||
(START GROUP_REPLICATION) are not yet automated — nodes start config-ready only.
|
||||
when: topology != 'standalone'
|
||||
run_once: true
|
||||
|
||||
handlers:
|
||||
- name: Restart MySQL delivery instance
|
||||
ansible.builtin.systemd_service:
|
||||
|
||||
@@ -1,35 +1,82 @@
|
||||
# Managed by XINFRA MySQL delivery - generated, do not edit by hand
|
||||
# instance={{ mysql_instance }} version={{ mysql_version_value }} topology={{ topology }}
|
||||
[client]
|
||||
socket={{ mysql_run_dir }}/mysql.sock
|
||||
port={{ mysql_port_value }}
|
||||
|
||||
[mysqld]
|
||||
user=mysql
|
||||
basedir=/usr
|
||||
datadir={{ mysql_data_dir }}
|
||||
tmpdir={{ mysql_tmp_dir }}
|
||||
socket={{ mysql_run_dir }}/mysql.sock
|
||||
pid-file={{ mysql_run_dir }}/mysql.pid
|
||||
port={{ mysql_port_value }}
|
||||
bind-address=0.0.0.0
|
||||
mysqlx=0
|
||||
server-id={{ 101 + mysql_node_index | int }}
|
||||
skip-name-resolve=ON
|
||||
server-id={{ mysql_server_id_value }}
|
||||
local-infile=OFF
|
||||
|
||||
# --- charset / collation / timezone / identifier case ---
|
||||
character-set-server={{ mysql_character_set }}
|
||||
collation-server={{ mysql_collation }}
|
||||
default-time-zone={{ mysql_timezone }}
|
||||
lower-case-table-names={{ mysql_lower_case_table_names }}
|
||||
|
||||
# --- error / slow log ---
|
||||
log-error={{ mysql_log_dir }}/error.log
|
||||
slow-query-log=ON
|
||||
slow-query-log-file={{ mysql_log_dir }}/slow.log
|
||||
skip-name-resolve=ON
|
||||
max-connections=100
|
||||
innodb-buffer-pool-size={{ ((mysql_memory_mb_value | int) * 55 / 100) | int }}M
|
||||
innodb-log-file-size=128M
|
||||
log-bin={{ mysql_log_dir }}/mysql-bin
|
||||
long-query-time={{ mysql_long_query_time }}
|
||||
|
||||
# --- binlog (PITR + replication base) ---
|
||||
log-bin={{ mysql_binlog_dir }}/binlog
|
||||
binlog-format=ROW
|
||||
sync-binlog={{ mysql_sync_binlog }}
|
||||
max-binlog-size={{ mysql_max_binlog_size }}
|
||||
binlog-expire-logs-seconds={{ mysql_binlog_expire_logs_seconds }}
|
||||
{% if topology != 'standalone' %}
|
||||
gtid-mode=ON
|
||||
enforce-gtid-consistency=ON
|
||||
relay-log={{ mysql_log_dir }}/relay-bin
|
||||
relay-log={{ mysql_binlog_dir }}/relay-bin
|
||||
{% endif %}
|
||||
|
||||
# --- InnoDB core ---
|
||||
innodb-buffer-pool-size={{ ((mysql_memory_mb_value | int) * 55 / 100) | int }}M
|
||||
innodb-flush-method=O_DIRECT
|
||||
innodb-flush-log-at-trx-commit={{ mysql_flush_log_at_trx_commit }}
|
||||
innodb-io-capacity={{ mysql_io_capacity }}
|
||||
max-connections={{ mysql_max_connections }}
|
||||
|
||||
# --- redo log (version-sensitive) ---
|
||||
innodb-log-group-home-dir={{ mysql_redo_dir }}
|
||||
{% if mysql_version_value in ['8.0', '8.4'] %}
|
||||
innodb-redo-log-capacity={{ mysql_redo_capacity }}
|
||||
{% else %}
|
||||
innodb-log-file-size={{ mysql_redo_capacity }}
|
||||
{% endif %}
|
||||
|
||||
# --- X Protocol disabled (version-sensitive) ---
|
||||
{% if mysql_version_value in ['8.0', '8.4'] %}
|
||||
mysqlx=0
|
||||
{% elif mysql_version_value == '5.7' %}
|
||||
loose-mysqlx=0
|
||||
{% endif %}
|
||||
|
||||
{% if topology == 'mgr_3' %}
|
||||
# --- Group Replication (config-ready; runtime bootstrap handled out-of-band) ---
|
||||
plugin-load-add=group_replication.so
|
||||
{% if mysql_version_value != '8.4' %}
|
||||
{# deprecated since 8.0.26 and removed in 8.3+; only inject for older series #}
|
||||
transaction-write-set-extraction=XXHASH64
|
||||
{% endif %}
|
||||
loose-group-replication-group-name={{ mgr_group_name }}
|
||||
loose-group-replication-start-on-boot=OFF
|
||||
loose-group-replication-local-address={{ ansible_host | default(inventory_hostname) }}:{{ mysql_port_value | int + 10000 }}
|
||||
loose-group-replication-group-seeds={% for host in ansible_play_hosts_all %}{{ hostvars[host].ansible_host | default(host) }}:{{ mysql_port_value | int + 10000 }}{% if not loop.last %},{% endif %}{% endfor %}
|
||||
loose-group-replication-local-address={{ ansible_host | default(inventory_hostname) }}:{{ mysql_gr_port_value }}
|
||||
loose-group-replication-group-seeds={% for host in ansible_play_hosts_all %}{{ hostvars[host].ansible_host | default(host) }}:{{ mysql_gr_port_value }}{% if not loop.last %},{% endif %}{% endfor %}
|
||||
|
||||
loose-group-replication-ip-allowlist={% for host in ansible_play_hosts_all %}{{ hostvars[host].ansible_host | default(host) }}{% if not loop.last %},{% endif %}{% endfor %}
|
||||
|
||||
loose-group-replication-single-primary-mode=ON
|
||||
loose-group-replication-enforce-update-everywhere-checks=OFF
|
||||
{% endif %}
|
||||
|
||||
|
||||
@@ -19,9 +19,13 @@ export interface CreateMySQLDeliveryPayload {
|
||||
topology: string
|
||||
mysql_port?: number
|
||||
data_disk: string
|
||||
target_host?: string
|
||||
cpu_cores: number
|
||||
memory_gb: number
|
||||
storage_gb: number
|
||||
cpu_milli: number
|
||||
memory_mi: number
|
||||
storage_gi: number
|
||||
param_template: string
|
||||
timezone: string
|
||||
lower_case_table_names: number
|
||||
|
||||
@@ -959,6 +959,9 @@ function mysqlDeliveryPayload(businessLineId: number) {
|
||||
cpu_cores: resources.cpuCores,
|
||||
memory_gb: resources.memoryGb,
|
||||
storage_gb: parseStorageGb(deliveryForm.disk),
|
||||
cpu_milli: resources.cpuCores * 1000,
|
||||
memory_mi: resources.memoryGb * 1024,
|
||||
storage_gi: parseStorageGb(deliveryForm.disk),
|
||||
param_template: deliveryForm.paramTemplate,
|
||||
timezone: deliveryForm.timezone,
|
||||
lower_case_table_names: deliveryForm.lowerCaseTableNames,
|
||||
|
||||
@@ -22,6 +22,10 @@ DELIVERY_CALLBACK_BASE_URL=http://authserver-backend.authserver.svc.cluster.loca
|
||||
DELIVERY_RESERVATION_TTL_MINUTES=120
|
||||
DELIVERY_GLOBAL_LIMIT=2
|
||||
DELIVERY_TARGET_LIMIT=2
|
||||
# 单机 MySQL 实例数上限(同机多实例,容量由配额 + playbook 实机守卫兜底)
|
||||
DELIVERY_HOST_INSTANCE_LIMIT=4
|
||||
# 数据盘挂载点白名单(逗号分隔,第一项为默认值)
|
||||
DELIVERY_DATA_DISKS=/data
|
||||
AWX_BASE_URL=
|
||||
AWX_TOKEN=
|
||||
AWX_USERNAME=
|
||||
|
||||
+61
-118
@@ -152,7 +152,7 @@ const docTemplate = `{
|
||||
"BearerAuth": []
|
||||
}
|
||||
],
|
||||
"description": "返回所有已启用的部署目标(如 k8s 集群、主机池)",
|
||||
"description": "从 AWX 动态返回可用 Job Template 及其 Inventory hosts",
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
@@ -160,6 +160,14 @@ const docTemplate = `{
|
||||
"delivery"
|
||||
],
|
||||
"summary": "获取可用部署目标",
|
||||
"parameters": [
|
||||
{
|
||||
"type": "string",
|
||||
"description": "组件过滤,例如 mysql",
|
||||
"name": "component",
|
||||
"in": "query"
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "items: 部署目标数组",
|
||||
@@ -176,64 +184,6 @@ const docTemplate = `{
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"post": {
|
||||
"security": [
|
||||
{
|
||||
"BearerAuth": []
|
||||
}
|
||||
],
|
||||
"description": "管理员创建新的部署目标(目前仅支持 k8s 类型)",
|
||||
"consumes": [
|
||||
"application/json"
|
||||
],
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"delivery"
|
||||
],
|
||||
"summary": "创建部署目标",
|
||||
"parameters": [
|
||||
{
|
||||
"description": "目标配置",
|
||||
"name": "body",
|
||||
"in": "body",
|
||||
"required": true,
|
||||
"schema": {
|
||||
"$ref": "#/definitions/handler.targetPayload"
|
||||
}
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"201": {
|
||||
"description": "目标已创建",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/model.DeploymentTarget"
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "参数错误",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": true
|
||||
}
|
||||
},
|
||||
"401": {
|
||||
"description": "未授权",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": true
|
||||
}
|
||||
},
|
||||
"409": {
|
||||
"description": "名称冲突",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/auth/api/v1/delivery/tasks": {
|
||||
@@ -254,6 +204,7 @@ const docTemplate = `{
|
||||
"parameters": [
|
||||
{
|
||||
"type": "integer",
|
||||
"format": "int64",
|
||||
"description": "业务线 ID 过滤",
|
||||
"name": "business_line_id",
|
||||
"in": "query"
|
||||
@@ -689,65 +640,6 @@ const docTemplate = `{
|
||||
}
|
||||
}
|
||||
},
|
||||
"handler.targetPayload": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"awx_inventory_id",
|
||||
"awx_template_id",
|
||||
"name",
|
||||
"target_type"
|
||||
],
|
||||
"properties": {
|
||||
"awx_inventory_id": {
|
||||
"type": "integer"
|
||||
},
|
||||
"awx_template_id": {
|
||||
"type": "integer"
|
||||
},
|
||||
"metadata": {
|
||||
"type": "object",
|
||||
"additionalProperties": {}
|
||||
},
|
||||
"name": {
|
||||
"type": "string"
|
||||
},
|
||||
"target_type": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"model.DeploymentTarget": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"awx_inventory_id": {
|
||||
"type": "integer"
|
||||
},
|
||||
"awx_template_id": {
|
||||
"type": "integer"
|
||||
},
|
||||
"created_at": {
|
||||
"type": "string"
|
||||
},
|
||||
"enabled": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"id": {
|
||||
"type": "integer"
|
||||
},
|
||||
"metadata": {
|
||||
"type": "string"
|
||||
},
|
||||
"name": {
|
||||
"type": "string"
|
||||
},
|
||||
"target_type": {
|
||||
"type": "string"
|
||||
},
|
||||
"updated_at": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"model.ResourceQuota": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@@ -816,18 +708,55 @@ const docTemplate = `{
|
||||
"target_id"
|
||||
],
|
||||
"properties": {
|
||||
"binlog_expire_logs_seconds": {
|
||||
"type": "integer"
|
||||
},
|
||||
"business_line_id": {
|
||||
"type": "integer"
|
||||
},
|
||||
"character_set": {
|
||||
"type": "string"
|
||||
},
|
||||
"collation": {
|
||||
"type": "string"
|
||||
},
|
||||
"cpu_milli": {
|
||||
"type": "integer"
|
||||
},
|
||||
"data_disk": {
|
||||
"type": "string"
|
||||
},
|
||||
"innodb_flush_log_at_trx_commit": {
|
||||
"type": "integer"
|
||||
},
|
||||
"innodb_io_capacity": {
|
||||
"type": "integer"
|
||||
},
|
||||
"innodb_redo_log_capacity": {
|
||||
"type": "string"
|
||||
},
|
||||
"instance_name": {
|
||||
"type": "string"
|
||||
},
|
||||
"long_query_time": {
|
||||
"type": "number"
|
||||
},
|
||||
"lower_case_table_names": {
|
||||
"type": "integer"
|
||||
},
|
||||
"max_binlog_size": {
|
||||
"type": "string"
|
||||
},
|
||||
"max_connections": {
|
||||
"description": "高级参数(选填,零值视为未设置)",
|
||||
"type": "string"
|
||||
},
|
||||
"memory_mi": {
|
||||
"type": "integer"
|
||||
},
|
||||
"mysql_port": {
|
||||
"type": "integer"
|
||||
},
|
||||
"mysql_version": {
|
||||
"type": "string"
|
||||
},
|
||||
@@ -837,8 +766,22 @@ const docTemplate = `{
|
||||
"storage_gi": {
|
||||
"type": "integer"
|
||||
},
|
||||
"sync_binlog": {
|
||||
"type": "integer"
|
||||
},
|
||||
"target_host": {
|
||||
"description": "调度控制(选填):点名候选池内主机跳过自动选机,端口/配额/实机守卫照常执行",
|
||||
"type": "string"
|
||||
},
|
||||
"target_id": {
|
||||
"type": "integer"
|
||||
},
|
||||
"timezone": {
|
||||
"description": "数据库配置(选填,缺省由 playbook 基线兜底)",
|
||||
"type": "string"
|
||||
},
|
||||
"topology": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+61
-118
@@ -145,7 +145,7 @@
|
||||
"BearerAuth": []
|
||||
}
|
||||
],
|
||||
"description": "返回所有已启用的部署目标(如 k8s 集群、主机池)",
|
||||
"description": "从 AWX 动态返回可用 Job Template 及其 Inventory hosts",
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
@@ -153,6 +153,14 @@
|
||||
"delivery"
|
||||
],
|
||||
"summary": "获取可用部署目标",
|
||||
"parameters": [
|
||||
{
|
||||
"type": "string",
|
||||
"description": "组件过滤,例如 mysql",
|
||||
"name": "component",
|
||||
"in": "query"
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "items: 部署目标数组",
|
||||
@@ -169,64 +177,6 @@
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"post": {
|
||||
"security": [
|
||||
{
|
||||
"BearerAuth": []
|
||||
}
|
||||
],
|
||||
"description": "管理员创建新的部署目标(目前仅支持 k8s 类型)",
|
||||
"consumes": [
|
||||
"application/json"
|
||||
],
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"delivery"
|
||||
],
|
||||
"summary": "创建部署目标",
|
||||
"parameters": [
|
||||
{
|
||||
"description": "目标配置",
|
||||
"name": "body",
|
||||
"in": "body",
|
||||
"required": true,
|
||||
"schema": {
|
||||
"$ref": "#/definitions/handler.targetPayload"
|
||||
}
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"201": {
|
||||
"description": "目标已创建",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/model.DeploymentTarget"
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "参数错误",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": true
|
||||
}
|
||||
},
|
||||
"401": {
|
||||
"description": "未授权",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": true
|
||||
}
|
||||
},
|
||||
"409": {
|
||||
"description": "名称冲突",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/auth/api/v1/delivery/tasks": {
|
||||
@@ -247,6 +197,7 @@
|
||||
"parameters": [
|
||||
{
|
||||
"type": "integer",
|
||||
"format": "int64",
|
||||
"description": "业务线 ID 过滤",
|
||||
"name": "business_line_id",
|
||||
"in": "query"
|
||||
@@ -682,65 +633,6 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"handler.targetPayload": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"awx_inventory_id",
|
||||
"awx_template_id",
|
||||
"name",
|
||||
"target_type"
|
||||
],
|
||||
"properties": {
|
||||
"awx_inventory_id": {
|
||||
"type": "integer"
|
||||
},
|
||||
"awx_template_id": {
|
||||
"type": "integer"
|
||||
},
|
||||
"metadata": {
|
||||
"type": "object",
|
||||
"additionalProperties": {}
|
||||
},
|
||||
"name": {
|
||||
"type": "string"
|
||||
},
|
||||
"target_type": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"model.DeploymentTarget": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"awx_inventory_id": {
|
||||
"type": "integer"
|
||||
},
|
||||
"awx_template_id": {
|
||||
"type": "integer"
|
||||
},
|
||||
"created_at": {
|
||||
"type": "string"
|
||||
},
|
||||
"enabled": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"id": {
|
||||
"type": "integer"
|
||||
},
|
||||
"metadata": {
|
||||
"type": "string"
|
||||
},
|
||||
"name": {
|
||||
"type": "string"
|
||||
},
|
||||
"target_type": {
|
||||
"type": "string"
|
||||
},
|
||||
"updated_at": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"model.ResourceQuota": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@@ -809,18 +701,55 @@
|
||||
"target_id"
|
||||
],
|
||||
"properties": {
|
||||
"binlog_expire_logs_seconds": {
|
||||
"type": "integer"
|
||||
},
|
||||
"business_line_id": {
|
||||
"type": "integer"
|
||||
},
|
||||
"character_set": {
|
||||
"type": "string"
|
||||
},
|
||||
"collation": {
|
||||
"type": "string"
|
||||
},
|
||||
"cpu_milli": {
|
||||
"type": "integer"
|
||||
},
|
||||
"data_disk": {
|
||||
"type": "string"
|
||||
},
|
||||
"innodb_flush_log_at_trx_commit": {
|
||||
"type": "integer"
|
||||
},
|
||||
"innodb_io_capacity": {
|
||||
"type": "integer"
|
||||
},
|
||||
"innodb_redo_log_capacity": {
|
||||
"type": "string"
|
||||
},
|
||||
"instance_name": {
|
||||
"type": "string"
|
||||
},
|
||||
"long_query_time": {
|
||||
"type": "number"
|
||||
},
|
||||
"lower_case_table_names": {
|
||||
"type": "integer"
|
||||
},
|
||||
"max_binlog_size": {
|
||||
"type": "string"
|
||||
},
|
||||
"max_connections": {
|
||||
"description": "高级参数(选填,零值视为未设置)",
|
||||
"type": "string"
|
||||
},
|
||||
"memory_mi": {
|
||||
"type": "integer"
|
||||
},
|
||||
"mysql_port": {
|
||||
"type": "integer"
|
||||
},
|
||||
"mysql_version": {
|
||||
"type": "string"
|
||||
},
|
||||
@@ -830,8 +759,22 @@
|
||||
"storage_gi": {
|
||||
"type": "integer"
|
||||
},
|
||||
"sync_binlog": {
|
||||
"type": "integer"
|
||||
},
|
||||
"target_host": {
|
||||
"description": "调度控制(选填):点名候选池内主机跳过自动选机,端口/配额/实机守卫照常执行",
|
||||
"type": "string"
|
||||
},
|
||||
"target_id": {
|
||||
"type": "integer"
|
||||
},
|
||||
"timezone": {
|
||||
"description": "数据库配置(选填,缺省由 playbook 基线兜底)",
|
||||
"type": "string"
|
||||
},
|
||||
"topology": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+42
-79
@@ -22,46 +22,6 @@ definitions:
|
||||
- storage_gi
|
||||
- target_id
|
||||
type: object
|
||||
handler.targetPayload:
|
||||
properties:
|
||||
awx_inventory_id:
|
||||
type: integer
|
||||
awx_template_id:
|
||||
type: integer
|
||||
metadata:
|
||||
additionalProperties: {}
|
||||
type: object
|
||||
name:
|
||||
type: string
|
||||
target_type:
|
||||
type: string
|
||||
required:
|
||||
- awx_inventory_id
|
||||
- awx_template_id
|
||||
- name
|
||||
- target_type
|
||||
type: object
|
||||
model.DeploymentTarget:
|
||||
properties:
|
||||
awx_inventory_id:
|
||||
type: integer
|
||||
awx_template_id:
|
||||
type: integer
|
||||
created_at:
|
||||
type: string
|
||||
enabled:
|
||||
type: boolean
|
||||
id:
|
||||
type: integer
|
||||
metadata:
|
||||
type: string
|
||||
name:
|
||||
type: string
|
||||
target_type:
|
||||
type: string
|
||||
updated_at:
|
||||
type: string
|
||||
type: object
|
||||
model.ResourceQuota:
|
||||
properties:
|
||||
business_line_id:
|
||||
@@ -101,22 +61,57 @@ definitions:
|
||||
type: object
|
||||
service.MySQLDeliveryInput:
|
||||
properties:
|
||||
binlog_expire_logs_seconds:
|
||||
type: integer
|
||||
business_line_id:
|
||||
type: integer
|
||||
character_set:
|
||||
type: string
|
||||
collation:
|
||||
type: string
|
||||
cpu_milli:
|
||||
type: integer
|
||||
data_disk:
|
||||
type: string
|
||||
innodb_flush_log_at_trx_commit:
|
||||
type: integer
|
||||
innodb_io_capacity:
|
||||
type: integer
|
||||
innodb_redo_log_capacity:
|
||||
type: string
|
||||
instance_name:
|
||||
type: string
|
||||
long_query_time:
|
||||
type: number
|
||||
lower_case_table_names:
|
||||
type: integer
|
||||
max_binlog_size:
|
||||
type: string
|
||||
max_connections:
|
||||
description: 高级参数(选填,零值视为未设置)
|
||||
type: string
|
||||
memory_mi:
|
||||
type: integer
|
||||
mysql_port:
|
||||
type: integer
|
||||
mysql_version:
|
||||
type: string
|
||||
namespace:
|
||||
type: string
|
||||
storage_gi:
|
||||
type: integer
|
||||
sync_binlog:
|
||||
type: integer
|
||||
target_host:
|
||||
description: 调度控制(选填):点名候选池内主机跳过自动选机,端口/配额/实机守卫照常执行
|
||||
type: string
|
||||
target_id:
|
||||
type: integer
|
||||
timezone:
|
||||
description: 数据库配置(选填,缺省由 playbook 基线兜底)
|
||||
type: string
|
||||
topology:
|
||||
type: string
|
||||
required:
|
||||
- business_line_id
|
||||
- cpu_milli
|
||||
@@ -220,7 +215,12 @@ paths:
|
||||
- delivery
|
||||
/auth/api/v1/delivery/targets:
|
||||
get:
|
||||
description: 返回所有已启用的部署目标(如 k8s 集群、主机池)
|
||||
description: 从 AWX 动态返回可用 Job Template 及其 Inventory hosts
|
||||
parameters:
|
||||
- description: 组件过滤,例如 mysql
|
||||
in: query
|
||||
name: component
|
||||
type: string
|
||||
produces:
|
||||
- application/json
|
||||
responses:
|
||||
@@ -239,49 +239,12 @@ paths:
|
||||
summary: 获取可用部署目标
|
||||
tags:
|
||||
- delivery
|
||||
post:
|
||||
consumes:
|
||||
- application/json
|
||||
description: 管理员创建新的部署目标(目前仅支持 k8s 类型)
|
||||
parameters:
|
||||
- description: 目标配置
|
||||
in: body
|
||||
name: body
|
||||
required: true
|
||||
schema:
|
||||
$ref: '#/definitions/handler.targetPayload'
|
||||
produces:
|
||||
- application/json
|
||||
responses:
|
||||
"201":
|
||||
description: 目标已创建
|
||||
schema:
|
||||
$ref: '#/definitions/model.DeploymentTarget'
|
||||
"400":
|
||||
description: 参数错误
|
||||
schema:
|
||||
additionalProperties: true
|
||||
type: object
|
||||
"401":
|
||||
description: 未授权
|
||||
schema:
|
||||
additionalProperties: true
|
||||
type: object
|
||||
"409":
|
||||
description: 名称冲突
|
||||
schema:
|
||||
additionalProperties: true
|
||||
type: object
|
||||
security:
|
||||
- BearerAuth: []
|
||||
summary: 创建部署目标
|
||||
tags:
|
||||
- delivery
|
||||
/auth/api/v1/delivery/tasks:
|
||||
get:
|
||||
description: 返回当前用户可见的交付任务列表(管理员可见全部)
|
||||
parameters:
|
||||
- description: 业务线 ID 过滤
|
||||
format: int64
|
||||
in: query
|
||||
name: business_line_id
|
||||
type: integer
|
||||
|
||||
+116
-112
@@ -14,62 +14,64 @@ type OAuthClient struct {
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
AppEnv string
|
||||
HTTPAddr string
|
||||
PublicBaseURL string
|
||||
MySQLDSN string
|
||||
AutoMigrate bool
|
||||
SSOEnabled bool
|
||||
JWTSecret string
|
||||
JWTIssuer string
|
||||
JWTTTLMinutes int
|
||||
SAMLEntityID string
|
||||
SAMLACSURL string
|
||||
SAMLSPCert string
|
||||
SAMLSPKey string
|
||||
SAMLIDPMetaURL string
|
||||
SAMLLogoutURL string
|
||||
WayenLoginURL string
|
||||
WayenTargetURL string
|
||||
WayenUsernameKey string
|
||||
WayenPasswordKey string
|
||||
WayenLoginFormat string
|
||||
WayenLoginValue string
|
||||
WayenOAuthRef string
|
||||
WayenOAuthLoginURL string
|
||||
WayneAPIBaseURL string
|
||||
WayneAdminUsername string
|
||||
WayneAdminPassword string
|
||||
WayneTokenTTLMinutes int
|
||||
WayneInternalAPIBaseURL string
|
||||
WayneServiceName string
|
||||
WayneServiceAPISecretKey string
|
||||
OAuthClientID string
|
||||
OAuthClientSecret string
|
||||
OAuthRedirectURI string
|
||||
OAuthCodeTTLSeconds int
|
||||
OIDCIssuer string
|
||||
OIDCAuthorizeURL string
|
||||
OIDCTokenURL string
|
||||
OIDCUserInfoURL string
|
||||
OIDCJWKSURL string
|
||||
CloudDMClientID string
|
||||
CloudDMClientSecret string
|
||||
CloudDMRedirectURI string
|
||||
CloudDMTargetURL string
|
||||
CloudDMRegisterURL string
|
||||
CloudDMAPIToken string
|
||||
AWXBaseURL string
|
||||
AWXToken string
|
||||
AWXUsername string
|
||||
AWXPassword string
|
||||
AWXWebhookToken string
|
||||
DeliverySchedulerEnabled bool
|
||||
DeliveryDispatchSeconds int
|
||||
DeliveryCallbackBaseURL string
|
||||
ReservationTTLMinutes int
|
||||
DeliveryGlobalLimit int
|
||||
DeliveryTargetLimit int
|
||||
AppEnv string
|
||||
HTTPAddr string
|
||||
PublicBaseURL string
|
||||
MySQLDSN string
|
||||
AutoMigrate bool
|
||||
SSOEnabled bool
|
||||
JWTSecret string
|
||||
JWTIssuer string
|
||||
JWTTTLMinutes int
|
||||
SAMLEntityID string
|
||||
SAMLACSURL string
|
||||
SAMLSPCert string
|
||||
SAMLSPKey string
|
||||
SAMLIDPMetaURL string
|
||||
SAMLLogoutURL string
|
||||
WayenLoginURL string
|
||||
WayenTargetURL string
|
||||
WayenUsernameKey string
|
||||
WayenPasswordKey string
|
||||
WayenLoginFormat string
|
||||
WayenLoginValue string
|
||||
WayenOAuthRef string
|
||||
WayenOAuthLoginURL string
|
||||
WayneAPIBaseURL string
|
||||
WayneAdminUsername string
|
||||
WayneAdminPassword string
|
||||
WayneTokenTTLMinutes int
|
||||
WayneInternalAPIBaseURL string
|
||||
WayneServiceName string
|
||||
WayneServiceAPISecretKey string
|
||||
OAuthClientID string
|
||||
OAuthClientSecret string
|
||||
OAuthRedirectURI string
|
||||
OAuthCodeTTLSeconds int
|
||||
OIDCIssuer string
|
||||
OIDCAuthorizeURL string
|
||||
OIDCTokenURL string
|
||||
OIDCUserInfoURL string
|
||||
OIDCJWKSURL string
|
||||
CloudDMClientID string
|
||||
CloudDMClientSecret string
|
||||
CloudDMRedirectURI string
|
||||
CloudDMTargetURL string
|
||||
CloudDMRegisterURL string
|
||||
CloudDMAPIToken string
|
||||
AWXBaseURL string
|
||||
AWXToken string
|
||||
AWXUsername string
|
||||
AWXPassword string
|
||||
AWXWebhookToken string
|
||||
DeliverySchedulerEnabled bool
|
||||
DeliveryDispatchSeconds int
|
||||
DeliveryCallbackBaseURL string
|
||||
ReservationTTLMinutes int
|
||||
DeliveryGlobalLimit int
|
||||
DeliveryTargetLimit int
|
||||
DeliveryHostInstanceLimit int
|
||||
DeliveryDataDisks []string
|
||||
}
|
||||
|
||||
func Load() Config {
|
||||
@@ -82,62 +84,64 @@ func Load() Config {
|
||||
oidcIssuer = strings.TrimRight(oidcIssuer, "/")
|
||||
|
||||
return Config{
|
||||
AppEnv: env("APP_ENV", "dev"),
|
||||
HTTPAddr: httpAddr,
|
||||
PublicBaseURL: publicBaseURL,
|
||||
MySQLDSN: env("MYSQL_DSN", "auth:auth@tcp(127.0.0.1:3306)/authserver?charset=utf8mb4&parseTime=True&loc=Local"),
|
||||
AutoMigrate: envBool("AUTO_MIGRATE", true),
|
||||
SSOEnabled: envBool("SSO_ENABLED", true),
|
||||
JWTSecret: env("JWT_SECRET", "change-this-secret"),
|
||||
JWTIssuer: env("JWT_ISSUER", "authserver"),
|
||||
JWTTTLMinutes: envInt("JWT_TTL_MINUTES", 120),
|
||||
SAMLEntityID: samlEntityID,
|
||||
SAMLACSURL: samlACSURL,
|
||||
SAMLSPCert: env("SAML_SP_CERT_FILE", "certs/sp.crt"),
|
||||
SAMLSPKey: env("SAML_SP_KEY_FILE", "certs/sp.key"),
|
||||
SAMLIDPMetaURL: env("SAML_IDP_METADATA_URL", "http://sso-internal.dev.qiniu.io/saml2/meta"),
|
||||
SAMLLogoutURL: trimURL(env("SAML_LOGOUT_URL", "")),
|
||||
WayenLoginURL: env("WAYEN_LOGIN_URL", ""),
|
||||
WayenTargetURL: env("WAYEN_TARGET_URL", ""),
|
||||
WayenUsernameKey: env("WAYEN_USERNAME_KEY", "email"),
|
||||
WayenPasswordKey: env("WAYEN_PASSWORD_KEY", "password"),
|
||||
WayenLoginFormat: env("WAYEN_LOGIN_FORMAT", "form"),
|
||||
WayenLoginValue: env("WAYEN_LOGIN_VALUE", "email"),
|
||||
WayenOAuthRef: env("WAYEN_OAUTH_REF", "/portal/namespace/1/app"),
|
||||
WayenOAuthLoginURL: trimURL(env("WAYEN_OAUTH_LOGIN_URL", "")),
|
||||
WayneAPIBaseURL: trimURL(env("WAYNE_API_BASE_URL", env("WAYNE_INTERNAL_API_BASE_URL", ""))),
|
||||
WayneAdminUsername: env("WAYNE_ADMIN_USERNAME", ""),
|
||||
WayneAdminPassword: env("WAYNE_ADMIN_PASSWORD", ""),
|
||||
WayneTokenTTLMinutes: envInt("WAYNE_TOKEN_TTL_MINUTES", 1440),
|
||||
WayneInternalAPIBaseURL: trimURL(env("WAYNE_INTERNAL_API_BASE_URL", "")),
|
||||
WayneServiceName: env("WAYNE_SERVICE_NAME", "xinfra"),
|
||||
WayneServiceAPISecretKey: env("WAYNE_SERVICE_API_SECRET_KEY", ""),
|
||||
OAuthClientID: env("OAUTH_WAYNE_CLIENT_ID", "wayne"),
|
||||
OAuthClientSecret: env("OAUTH_WAYNE_CLIENT_SECRET", "wayne-secret"),
|
||||
OAuthRedirectURI: env("OAUTH_WAYNE_REDIRECT_URI", ""),
|
||||
OAuthCodeTTLSeconds: envInt("OAUTH_CODE_TTL_SECONDS", 120),
|
||||
OIDCIssuer: oidcIssuer,
|
||||
OIDCAuthorizeURL: trimURL(env("OIDC_AUTHORIZATION_ENDPOINT", oidcIssuer+"/oauth/authorize")),
|
||||
OIDCTokenURL: trimURL(env("OIDC_TOKEN_ENDPOINT", oidcIssuer+"/oauth/token")),
|
||||
OIDCUserInfoURL: trimURL(env("OIDC_USERINFO_ENDPOINT", oidcIssuer+"/oauth/userinfo")),
|
||||
OIDCJWKSURL: trimURL(env("OIDC_JWKS_URI", oidcIssuer+"/oauth/jwks")),
|
||||
CloudDMClientID: env("OIDC_CLOUDDM_CLIENT_ID", "clouddm"),
|
||||
CloudDMClientSecret: env("OIDC_CLOUDDM_CLIENT_SECRET", ""),
|
||||
CloudDMRedirectURI: env("OIDC_CLOUDDM_REDIRECT_URI", ""),
|
||||
CloudDMTargetURL: env("CLOUDDM_TARGET_URL", ""),
|
||||
CloudDMRegisterURL: trimURL(env("CLOUDDM_REGISTER_URL", "")),
|
||||
CloudDMAPIToken: env("CLOUDDM_API_TOKEN", ""),
|
||||
AWXBaseURL: trimURL(env("AWX_BASE_URL", "")),
|
||||
AWXToken: env("AWX_TOKEN", ""),
|
||||
AWXUsername: env("AWX_USERNAME", ""),
|
||||
AWXPassword: env("AWX_PASSWORD", ""),
|
||||
AWXWebhookToken: env("AWX_WEBHOOK_TOKEN", ""),
|
||||
DeliverySchedulerEnabled: envBool("DELIVERY_SCHEDULER_ENABLED", false),
|
||||
DeliveryDispatchSeconds: envInt("DELIVERY_DISPATCH_SECONDS", 5),
|
||||
DeliveryCallbackBaseURL: trimURL(env("DELIVERY_CALLBACK_BASE_URL", publicBaseURL)),
|
||||
ReservationTTLMinutes: envInt("DELIVERY_RESERVATION_TTL_MINUTES", 120),
|
||||
DeliveryGlobalLimit: envInt("DELIVERY_GLOBAL_LIMIT", 2),
|
||||
DeliveryTargetLimit: envInt("DELIVERY_TARGET_LIMIT", 2),
|
||||
AppEnv: env("APP_ENV", "dev"),
|
||||
HTTPAddr: httpAddr,
|
||||
PublicBaseURL: publicBaseURL,
|
||||
MySQLDSN: env("MYSQL_DSN", "auth:auth@tcp(127.0.0.1:3306)/authserver?charset=utf8mb4&parseTime=True&loc=Local"),
|
||||
AutoMigrate: envBool("AUTO_MIGRATE", true),
|
||||
SSOEnabled: envBool("SSO_ENABLED", true),
|
||||
JWTSecret: env("JWT_SECRET", "change-this-secret"),
|
||||
JWTIssuer: env("JWT_ISSUER", "authserver"),
|
||||
JWTTTLMinutes: envInt("JWT_TTL_MINUTES", 120),
|
||||
SAMLEntityID: samlEntityID,
|
||||
SAMLACSURL: samlACSURL,
|
||||
SAMLSPCert: env("SAML_SP_CERT_FILE", "certs/sp.crt"),
|
||||
SAMLSPKey: env("SAML_SP_KEY_FILE", "certs/sp.key"),
|
||||
SAMLIDPMetaURL: env("SAML_IDP_METADATA_URL", "http://sso-internal.dev.qiniu.io/saml2/meta"),
|
||||
SAMLLogoutURL: trimURL(env("SAML_LOGOUT_URL", "")),
|
||||
WayenLoginURL: env("WAYEN_LOGIN_URL", ""),
|
||||
WayenTargetURL: env("WAYEN_TARGET_URL", ""),
|
||||
WayenUsernameKey: env("WAYEN_USERNAME_KEY", "email"),
|
||||
WayenPasswordKey: env("WAYEN_PASSWORD_KEY", "password"),
|
||||
WayenLoginFormat: env("WAYEN_LOGIN_FORMAT", "form"),
|
||||
WayenLoginValue: env("WAYEN_LOGIN_VALUE", "email"),
|
||||
WayenOAuthRef: env("WAYEN_OAUTH_REF", "/portal/namespace/1/app"),
|
||||
WayenOAuthLoginURL: trimURL(env("WAYEN_OAUTH_LOGIN_URL", "")),
|
||||
WayneAPIBaseURL: trimURL(env("WAYNE_API_BASE_URL", env("WAYNE_INTERNAL_API_BASE_URL", ""))),
|
||||
WayneAdminUsername: env("WAYNE_ADMIN_USERNAME", ""),
|
||||
WayneAdminPassword: env("WAYNE_ADMIN_PASSWORD", ""),
|
||||
WayneTokenTTLMinutes: envInt("WAYNE_TOKEN_TTL_MINUTES", 1440),
|
||||
WayneInternalAPIBaseURL: trimURL(env("WAYNE_INTERNAL_API_BASE_URL", "")),
|
||||
WayneServiceName: env("WAYNE_SERVICE_NAME", "xinfra"),
|
||||
WayneServiceAPISecretKey: env("WAYNE_SERVICE_API_SECRET_KEY", ""),
|
||||
OAuthClientID: env("OAUTH_WAYNE_CLIENT_ID", "wayne"),
|
||||
OAuthClientSecret: env("OAUTH_WAYNE_CLIENT_SECRET", "wayne-secret"),
|
||||
OAuthRedirectURI: env("OAUTH_WAYNE_REDIRECT_URI", ""),
|
||||
OAuthCodeTTLSeconds: envInt("OAUTH_CODE_TTL_SECONDS", 120),
|
||||
OIDCIssuer: oidcIssuer,
|
||||
OIDCAuthorizeURL: trimURL(env("OIDC_AUTHORIZATION_ENDPOINT", oidcIssuer+"/oauth/authorize")),
|
||||
OIDCTokenURL: trimURL(env("OIDC_TOKEN_ENDPOINT", oidcIssuer+"/oauth/token")),
|
||||
OIDCUserInfoURL: trimURL(env("OIDC_USERINFO_ENDPOINT", oidcIssuer+"/oauth/userinfo")),
|
||||
OIDCJWKSURL: trimURL(env("OIDC_JWKS_URI", oidcIssuer+"/oauth/jwks")),
|
||||
CloudDMClientID: env("OIDC_CLOUDDM_CLIENT_ID", "clouddm"),
|
||||
CloudDMClientSecret: env("OIDC_CLOUDDM_CLIENT_SECRET", ""),
|
||||
CloudDMRedirectURI: env("OIDC_CLOUDDM_REDIRECT_URI", ""),
|
||||
CloudDMTargetURL: env("CLOUDDM_TARGET_URL", ""),
|
||||
CloudDMRegisterURL: trimURL(env("CLOUDDM_REGISTER_URL", "")),
|
||||
CloudDMAPIToken: env("CLOUDDM_API_TOKEN", ""),
|
||||
AWXBaseURL: trimURL(env("AWX_BASE_URL", "")),
|
||||
AWXToken: env("AWX_TOKEN", ""),
|
||||
AWXUsername: env("AWX_USERNAME", ""),
|
||||
AWXPassword: env("AWX_PASSWORD", ""),
|
||||
AWXWebhookToken: env("AWX_WEBHOOK_TOKEN", ""),
|
||||
DeliverySchedulerEnabled: envBool("DELIVERY_SCHEDULER_ENABLED", false),
|
||||
DeliveryDispatchSeconds: envInt("DELIVERY_DISPATCH_SECONDS", 5),
|
||||
DeliveryCallbackBaseURL: trimURL(env("DELIVERY_CALLBACK_BASE_URL", publicBaseURL)),
|
||||
ReservationTTLMinutes: envInt("DELIVERY_RESERVATION_TTL_MINUTES", 120),
|
||||
DeliveryGlobalLimit: envInt("DELIVERY_GLOBAL_LIMIT", 2),
|
||||
DeliveryTargetLimit: envInt("DELIVERY_TARGET_LIMIT", 2),
|
||||
DeliveryHostInstanceLimit: envInt("DELIVERY_HOST_INSTANCE_LIMIT", 4),
|
||||
DeliveryDataDisks: splitCSV(env("DELIVERY_DATA_DISKS", "/data")),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -24,6 +24,7 @@ import (
|
||||
)
|
||||
|
||||
var dnsLabelPattern = regexp.MustCompile(`^[a-z0-9](?:[-a-z0-9]*[a-z0-9])?$`)
|
||||
var hostNamePattern = regexp.MustCompile(`^[a-zA-Z0-9](?:[-a-zA-Z0-9.]*[a-zA-Z0-9])?$`)
|
||||
|
||||
type MySQLDeliveryInput struct {
|
||||
BusinessLineID uint64 `json:"business_line_id" binding:"required"`
|
||||
@@ -34,9 +35,10 @@ type MySQLDeliveryInput struct {
|
||||
Topology string `json:"topology"`
|
||||
MySQLPort int `json:"mysql_port"`
|
||||
DataDisk string `json:"data_disk"`
|
||||
CPUCores int64 `json:"cpu_cores" binding:"required"`
|
||||
MemoryGB int64 `json:"memory_gb" binding:"required"`
|
||||
StorageGB int64 `json:"storage_gb" binding:"required"`
|
||||
TargetHost string `json:"target_host"`
|
||||
CPUCores int64 `json:"cpu_cores"`
|
||||
MemoryGB int64 `json:"memory_gb"`
|
||||
StorageGB int64 `json:"storage_gb"`
|
||||
ParamTemplate string `json:"param_template"`
|
||||
TimeZone string `json:"timezone"`
|
||||
LowerCaseTableNames int `json:"lower_case_table_names"`
|
||||
@@ -50,9 +52,9 @@ type MySQLDeliveryInput struct {
|
||||
LongQueryTime float64 `json:"long_query_time"`
|
||||
BinlogExpireLogsSeconds int64 `json:"binlog_expire_logs_seconds"`
|
||||
MaxBinlogSize string `json:"max_binlog_size"`
|
||||
CPUMilli int64 `json:"-"`
|
||||
MemoryMi int64 `json:"-"`
|
||||
StorageGi int64 `json:"-"`
|
||||
CPUMilli int64 `json:"cpu_milli"`
|
||||
MemoryMi int64 `json:"memory_mi"`
|
||||
StorageGi int64 `json:"storage_gi"`
|
||||
}
|
||||
|
||||
type deliveryPayload struct {
|
||||
@@ -118,31 +120,51 @@ func parseTargetMetadata(raw string) targetMetadata {
|
||||
return meta
|
||||
}
|
||||
|
||||
// firstFreeHost 返回候选池中第一个未被占用的节点。
|
||||
func firstFreeHost(hosts []targetHost, occupied []string) *targetHost {
|
||||
taken := make(map[string]bool, len(occupied))
|
||||
for _, h := range occupied {
|
||||
taken[h] = true
|
||||
// firstFreeHost 返回候选池中非失败任务数未达单机实例上限的第一个节点。
|
||||
func firstFreeHost(hosts []targetHost, occupied []string, limit int) *targetHost {
|
||||
if limit < 1 {
|
||||
limit = 1
|
||||
}
|
||||
for i := range hosts {
|
||||
if !taken[hosts[i].Name] {
|
||||
if hostTaskCount(occupied, hosts[i].Name) < limit {
|
||||
return &hosts[i]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func firstFreePort(occupied []int) int {
|
||||
taken := make(map[int]bool, len(occupied))
|
||||
for _, port := range occupied {
|
||||
taken[port] = true
|
||||
}
|
||||
for port := 13306; port <= 13999; port++ {
|
||||
if !taken[port] {
|
||||
return port
|
||||
func hostTaskCount(occupied []string, name string) int {
|
||||
count := 0
|
||||
for _, h := range occupied {
|
||||
if h == name {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return 0
|
||||
return count
|
||||
}
|
||||
|
||||
const (
|
||||
mysqlPortPoolStart = 13306
|
||||
mysqlPortPoolEnd = 13999
|
||||
)
|
||||
|
||||
func allocatePort(requested int, used []int) (int, error) {
|
||||
taken := make(map[int]bool, len(used))
|
||||
for _, p := range used {
|
||||
taken[p] = true
|
||||
}
|
||||
if requested != 0 {
|
||||
if taken[requested] {
|
||||
return 0, fmt.Errorf("mysql_port %d is already allocated on the target host", requested)
|
||||
}
|
||||
return requested, nil
|
||||
}
|
||||
for p := mysqlPortPoolStart; p <= mysqlPortPoolEnd; p++ {
|
||||
if !taken[p] {
|
||||
return p, nil
|
||||
}
|
||||
}
|
||||
return 0, fmt.Errorf("mysql port pool %d-%d is exhausted on the target host", mysqlPortPoolStart, mysqlPortPoolEnd)
|
||||
}
|
||||
|
||||
type DeliveryService struct {
|
||||
@@ -228,8 +250,11 @@ func (s *DeliveryService) CreateTask(ctx context.Context, userID uint64, isAdmin
|
||||
if idempotencyKey == "" || len(idempotencyKey) > 128 {
|
||||
return nil, false, fmt.Errorf("Idempotency-Key header is required and must not exceed 128 characters")
|
||||
}
|
||||
if input.DataDisk == "" && len(s.cfg.DeliveryDataDisks) > 0 {
|
||||
input.DataDisk = s.cfg.DeliveryDataDisks[0]
|
||||
}
|
||||
normalizeMySQLDeliveryInput(&input)
|
||||
if err := validateDeliveryInput(input); err != nil {
|
||||
if err := validateDeliveryInput(input, s.cfg.DeliveryDataDisks); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
|
||||
@@ -290,9 +315,8 @@ func (s *DeliveryService) CreateTask(ctx context.Context, userID uint64, isAdmin
|
||||
return &task, false, nil
|
||||
}
|
||||
|
||||
var supportedMySQLVersions = map[string]bool{"8.0": true}
|
||||
var supportedMySQLVersions = map[string]bool{"8.0": true, "8.4": true}
|
||||
var supportedMySQLTopologies = map[string]bool{"standalone": true}
|
||||
var supportedDataDisks = map[string]bool{"/data": true, "/disk1": true, "/mnt/vol-1": true}
|
||||
var supportedParamTemplates = map[string]bool{"default": true, "high_performance": true, "high_safety": true}
|
||||
var supportedCharacterSets = map[string]bool{"utf8mb4": true, "utf8": true, "gbk": true, "latin1": true}
|
||||
var supportedCollations = map[string]bool{
|
||||
@@ -301,6 +325,7 @@ var supportedCollations = map[string]bool{
|
||||
}
|
||||
var supportedRedoLogCapacity = map[string]bool{"auto": true, "128M": true, "256M": true, "512M": true, "1G": true}
|
||||
var supportedMaxBinlogSize = map[string]bool{"128M": true, "256M": true, "512M": true, "1G": true}
|
||||
var timezonePattern = regexp.MustCompile(`^([+-](0\d|1[0-4]):[0-5]\d|SYSTEM|[A-Za-z]+(?:/[A-Za-z0-9_+-]+)+)$`)
|
||||
|
||||
func normalizeMySQLDeliveryInput(input *MySQLDeliveryInput) {
|
||||
if input.MySQLVersion == "" {
|
||||
@@ -348,9 +373,15 @@ func normalizeMySQLDeliveryInput(input *MySQLDeliveryInput) {
|
||||
if input.MaxBinlogSize == "" {
|
||||
input.MaxBinlogSize = "256M"
|
||||
}
|
||||
input.CPUMilli = input.CPUCores * 1000
|
||||
input.MemoryMi = input.MemoryGB * 1024
|
||||
input.StorageGi = input.StorageGB
|
||||
if input.CPUMilli == 0 && input.CPUCores != 0 {
|
||||
input.CPUMilli = input.CPUCores * 1000
|
||||
}
|
||||
if input.MemoryMi == 0 && input.MemoryGB != 0 {
|
||||
input.MemoryMi = input.MemoryGB * 1024
|
||||
}
|
||||
if input.StorageGi == 0 && input.StorageGB != 0 {
|
||||
input.StorageGi = input.StorageGB
|
||||
}
|
||||
}
|
||||
|
||||
func defaultCollation(characterSet string) string {
|
||||
@@ -366,38 +397,48 @@ func defaultCollation(characterSet string) string {
|
||||
}
|
||||
}
|
||||
|
||||
func validateDeliveryInput(input MySQLDeliveryInput) error {
|
||||
func validateDeliveryInput(input MySQLDeliveryInput, dataDisks []string) error {
|
||||
if len(input.Namespace) > 63 || !dnsLabelPattern.MatchString(input.Namespace) {
|
||||
return fmt.Errorf("namespace must be a valid Kubernetes DNS label")
|
||||
}
|
||||
if len(input.InstanceName) > 63 || !dnsLabelPattern.MatchString(input.InstanceName) {
|
||||
return fmt.Errorf("instance_name must be a valid Kubernetes DNS label")
|
||||
}
|
||||
if !oneOfInt64(input.CPUCores, []int64{1, 2, 4, 8, 16}) {
|
||||
return fmt.Errorf("cpu_cores must be one of 1, 2, 4, 8, 16")
|
||||
}
|
||||
if !oneOfInt64(input.MemoryGB, []int64{2, 4, 8, 16, 32, 64}) {
|
||||
return fmt.Errorf("memory_gb must be one of 2, 4, 8, 16, 32, 64")
|
||||
}
|
||||
if input.StorageGB < 20 || input.StorageGB > 2000 {
|
||||
return fmt.Errorf("storage_gb must be between 20 and 2000")
|
||||
if input.CPUMilli < 100 || input.CPUMilli > 64000 || input.MemoryMi < 2048 || input.MemoryMi > 65536 || input.StorageGi < 20 || input.StorageGi > 2000 {
|
||||
return fmt.Errorf("requested resources are outside the supported range (cpu: 100-64000m, memory: 2048-65536 MiB, storage: 20-2000 GiB)")
|
||||
}
|
||||
if input.MySQLVersion != "" && !supportedMySQLVersions[input.MySQLVersion] {
|
||||
return fmt.Errorf("unsupported mysql_version %q, supported: 8.0", input.MySQLVersion)
|
||||
return fmt.Errorf("unsupported mysql_version %q, supported: 8.0, 8.4 (5.6/5.7 are EOL and have no Ubuntu 24.04 packages)", input.MySQLVersion)
|
||||
}
|
||||
if input.Topology != "" && !supportedMySQLTopologies[input.Topology] {
|
||||
return fmt.Errorf("unsupported topology %q, supported: standalone", input.Topology)
|
||||
}
|
||||
if input.MySQLPort != 0 && (input.MySQLPort < 13306 || input.MySQLPort > 13999) {
|
||||
return fmt.Errorf("mysql_port must be empty for auto assignment or between 13306 and 13999")
|
||||
if input.MySQLPort != 0 && (input.MySQLPort < mysqlPortPoolStart || input.MySQLPort > mysqlPortPoolEnd) {
|
||||
return fmt.Errorf("mysql_port must be empty for auto assignment or between %d and %d", mysqlPortPoolStart, mysqlPortPoolEnd)
|
||||
}
|
||||
if !supportedDataDisks[input.DataDisk] {
|
||||
return fmt.Errorf("unsupported data_disk %q", input.DataDisk)
|
||||
if input.DataDisk != "" {
|
||||
allowed := dataDisks
|
||||
if len(allowed) == 0 {
|
||||
allowed = []string{"/data"}
|
||||
}
|
||||
found := false
|
||||
for _, disk := range allowed {
|
||||
if input.DataDisk == disk {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
return fmt.Errorf("data_disk %q is not in the allowed mount point list %v", input.DataDisk, allowed)
|
||||
}
|
||||
}
|
||||
if input.TargetHost != "" && (len(input.TargetHost) > 253 || !hostNamePattern.MatchString(input.TargetHost)) {
|
||||
return fmt.Errorf("target_host must be a valid inventory host name")
|
||||
}
|
||||
if !supportedParamTemplates[input.ParamTemplate] {
|
||||
return fmt.Errorf("unsupported param_template %q", input.ParamTemplate)
|
||||
}
|
||||
if !validTimeZone(input.TimeZone) {
|
||||
if !timezonePattern.MatchString(input.TimeZone) {
|
||||
return fmt.Errorf("unsupported timezone %q", input.TimeZone)
|
||||
}
|
||||
if input.LowerCaseTableNames != 0 && input.LowerCaseTableNames != 1 {
|
||||
@@ -593,20 +634,42 @@ func (s *DeliveryService) claimAndReserve(ctx context.Context) (*model.DeliveryT
|
||||
if err := tx.Model(&model.DeliveryTask{}).Where("target_id = ? AND target_host <> ? AND status NOT IN ?", task.TargetID, "", occupiedExclude).Pluck("target_host", &occupied).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
host := firstFreeHost(meta.Hosts, occupied)
|
||||
limit := s.cfg.DeliveryHostInstanceLimit
|
||||
var host *targetHost
|
||||
if payload.TargetHost != "" {
|
||||
for i := range meta.Hosts {
|
||||
if meta.Hosts[i].Name == payload.TargetHost {
|
||||
host = &meta.Hosts[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if host == nil {
|
||||
return s.failInTransaction(tx, &task, model.TaskValidationFailed, fmt.Sprintf("target_host %q is not in the candidate host pool", payload.TargetHost))
|
||||
}
|
||||
effectiveLimit := limit
|
||||
if effectiveLimit < 1 {
|
||||
effectiveLimit = 1
|
||||
}
|
||||
if hostTaskCount(occupied, host.Name) >= effectiveLimit {
|
||||
return fmt.Errorf("defer: pinned host %s reached the per-host instance limit %d", host.Name, effectiveLimit)
|
||||
}
|
||||
} else {
|
||||
host = firstFreeHost(meta.Hosts, occupied, limit)
|
||||
}
|
||||
if host == nil {
|
||||
return fmt.Errorf("defer: no free host available on target")
|
||||
}
|
||||
mysqlPort := payload.MySQLPort
|
||||
if mysqlPort == 0 {
|
||||
var occupiedPorts []int
|
||||
if err := tx.Model(&model.DeliveryTask{}).Where("target_id = ? AND target_host = ? AND mysql_port <> ? AND status NOT IN ?", task.TargetID, host.Name, 0, occupiedExclude).Pluck("mysql_port", &occupiedPorts).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
mysqlPort = firstFreePort(occupiedPorts)
|
||||
if mysqlPort == 0 {
|
||||
return fmt.Errorf("defer: no free MySQL port available on target host")
|
||||
}
|
||||
var usedPorts []int
|
||||
if err := tx.Model(&model.DeliveryTask{}).Where("target_id = ? AND target_host = ? AND mysql_port <> ? AND status NOT IN ?", task.TargetID, host.Name, 0, occupiedExclude).Pluck("mysql_port", &usedPorts).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
var instancePorts []int
|
||||
if err := tx.Model(&model.MySQLInstance{}).Where("target_id = ? AND node_name = ? AND status = ?", task.TargetID, host.Name, "active").Pluck("port", &instancePorts).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
mysqlPort, portErr := allocatePort(payload.MySQLPort, append(usedPorts, instancePorts...))
|
||||
if portErr != nil {
|
||||
return s.failInTransaction(tx, &task, model.TaskValidationFailed, portErr.Error())
|
||||
}
|
||||
reservation := model.ResourceReservation{TaskID: task.ID, BusinessLineID: task.BusinessLineID, TargetID: task.TargetID, CPUMilli: payload.CPUMilli, MemoryMi: payload.MemoryMi, StorageGi: payload.StorageGi, InstanceCount: 1, Status: "reserved", ExpiresAt: time.Now().Add(time.Duration(s.cfg.ReservationTTLMinutes) * time.Minute)}
|
||||
if err := tx.Create(&reservation).Error; err != nil {
|
||||
@@ -771,8 +834,10 @@ func (s *DeliveryService) CreateExecution(ctx context.Context, taskID, payloadHa
|
||||
"instance_name": payload.InstanceName, "mysql_port": task.MySQLPort,
|
||||
"data_disk": payload.DataDisk, "cpu_cores": payload.CPUCores,
|
||||
"memory_gb": payload.MemoryGB, "storage_gb": payload.StorageGB,
|
||||
"cpu_milli": payload.CPUMilli, "memory_mi": payload.MemoryMi, "storage_gi": payload.StorageGi,
|
||||
"mysql_version": payload.MySQLVersion, "param_template": payload.ParamTemplate,
|
||||
"timezone": payload.TimeZone, "lower_case_table_names": payload.LowerCaseTableNames,
|
||||
"target_host": payload.TargetHost,
|
||||
"timezone": payload.TimeZone, "lower_case_table_names": payload.LowerCaseTableNames,
|
||||
"character_set": payload.CharacterSet, "collation": payload.Collation,
|
||||
"max_connections": payload.MaxConnections, "innodb_redo_log_capacity": payload.InnoDBRedoLogCapacity,
|
||||
"innodb_flush_log_at_trx_commit": payload.InnoDBFlushLogAtTrxCommit, "sync_binlog": payload.SyncBinlog,
|
||||
|
||||
@@ -3,59 +3,123 @@ package service
|
||||
import "testing"
|
||||
|
||||
func TestValidateDeliveryInput(t *testing.T) {
|
||||
dataDisks := []string{"/data", "/disk1"}
|
||||
valid := MySQLDeliveryInput{BusinessLineID: 1, TargetID: 1, Namespace: "team-a", InstanceName: "mysql-01", CPUCores: 2, MemoryGB: 4, StorageGB: 50}
|
||||
normalizeMySQLDeliveryInput(&valid)
|
||||
if err := validateDeliveryInput(valid); err != nil {
|
||||
if err := validateDeliveryInput(valid, dataDisks); err != nil {
|
||||
t.Fatalf("valid input rejected: %v", err)
|
||||
}
|
||||
withVersion := valid
|
||||
withVersion.MySQLVersion = "8.0"
|
||||
if err := validateDeliveryInput(withVersion); err != nil {
|
||||
t.Fatalf("valid input with version 8.0 rejected: %v", err)
|
||||
full := valid
|
||||
full.MySQLVersion = "8.0"
|
||||
full.Topology = "standalone"
|
||||
full.MySQLPort = 13306
|
||||
full.DataDisk = "/disk1"
|
||||
full.TargetHost = "k8s-server-03"
|
||||
full.CPUMilli = 2000
|
||||
full.MemoryMi = 8192
|
||||
full.StorageGi = 2000
|
||||
full.TimeZone = "+08:00"
|
||||
full.LowerCaseTableNames = 0
|
||||
full.CharacterSet = "utf8mb4"
|
||||
full.Collation = "utf8mb4_general_ci"
|
||||
full.MaxConnections = "auto"
|
||||
full.InnoDBRedoLogCapacity = "256M"
|
||||
full.InnoDBFlushLogAtTrxCommit = 2
|
||||
full.SyncBinlog = 0
|
||||
full.InnoDBIOCapacity = 2000
|
||||
full.LongQueryTime = 0.5
|
||||
full.BinlogExpireLogsSeconds = 604800
|
||||
full.MaxBinlogSize = "512M"
|
||||
if err := validateDeliveryInput(full, dataDisks); err != nil {
|
||||
t.Fatalf("valid full input rejected: %v", err)
|
||||
}
|
||||
for name, input := range map[string]MySQLDeliveryInput{
|
||||
"uppercase namespace": valid,
|
||||
"bad instance": valid,
|
||||
"bad cpu cores": valid,
|
||||
"too much memory": valid,
|
||||
"too little storage": valid,
|
||||
"too much storage": valid,
|
||||
"unsupported version": valid,
|
||||
"unsupported topology": valid,
|
||||
"bad mysql port": valid,
|
||||
"bad data disk": valid,
|
||||
"unsupported charset": valid,
|
||||
"bad collation": valid,
|
||||
namedZone := valid
|
||||
namedZone.TimeZone = "Asia/Shanghai"
|
||||
if err := validateDeliveryInput(namedZone, dataDisks); err != nil {
|
||||
t.Fatalf("named timezone rejected: %v", err)
|
||||
}
|
||||
lts := valid
|
||||
lts.MySQLVersion = "8.4"
|
||||
if err := validateDeliveryInput(lts, dataDisks); err != nil {
|
||||
t.Fatalf("8.4 LTS rejected: %v", err)
|
||||
}
|
||||
for name, mutate := range map[string]func(*MySQLDeliveryInput){
|
||||
"uppercase namespace": func(in *MySQLDeliveryInput) { in.Namespace = "Team-A" },
|
||||
"bad instance": func(in *MySQLDeliveryInput) { in.InstanceName = "mysql_01" },
|
||||
"too little cpu": func(in *MySQLDeliveryInput) { in.CPUMilli = 50 },
|
||||
"too much cpu": func(in *MySQLDeliveryInput) { in.CPUMilli = 65000 },
|
||||
"too little memory": func(in *MySQLDeliveryInput) { in.MemoryMi = 1024 },
|
||||
"too much memory": func(in *MySQLDeliveryInput) { in.MemoryMi = 131072 },
|
||||
"too little storage": func(in *MySQLDeliveryInput) { in.StorageGi = 10 },
|
||||
"too much storage": func(in *MySQLDeliveryInput) { in.StorageGi = 4000 },
|
||||
"unsupported version": func(in *MySQLDeliveryInput) { in.MySQLVersion = "5.7" },
|
||||
"eol version": func(in *MySQLDeliveryInput) { in.MySQLVersion = "5.6" },
|
||||
"unsupported topology": func(in *MySQLDeliveryInput) { in.Topology = "mgr_3" },
|
||||
"port below pool": func(in *MySQLDeliveryInput) { in.MySQLPort = 3307 },
|
||||
"port above pool": func(in *MySQLDeliveryInput) { in.MySQLPort = 14000 },
|
||||
"data disk not in list": func(in *MySQLDeliveryInput) { in.DataDisk = "/mnt/other" },
|
||||
"bad target host": func(in *MySQLDeliveryInput) { in.TargetHost = "-bad-host" },
|
||||
"bad timezone": func(in *MySQLDeliveryInput) { in.TimeZone = "UTC+8" },
|
||||
"bad lower case": func(in *MySQLDeliveryInput) { in.LowerCaseTableNames = 2 },
|
||||
"bad charset": func(in *MySQLDeliveryInput) { in.CharacterSet = "big5" },
|
||||
"collation mismatch": func(in *MySQLDeliveryInput) { in.CharacterSet = "gbk"; in.Collation = "utf8mb4_general_ci" },
|
||||
"bad max connections": func(in *MySQLDeliveryInput) { in.MaxConnections = "300" },
|
||||
"bad redo capacity": func(in *MySQLDeliveryInput) { in.InnoDBRedoLogCapacity = "2G" },
|
||||
"bad flush log": func(in *MySQLDeliveryInput) { in.InnoDBFlushLogAtTrxCommit = 3 },
|
||||
"bad sync binlog": func(in *MySQLDeliveryInput) { in.SyncBinlog = 2 },
|
||||
"bad io capacity": func(in *MySQLDeliveryInput) { in.InnoDBIOCapacity = 500 },
|
||||
"bad long query time": func(in *MySQLDeliveryInput) { in.LongQueryTime = 3 },
|
||||
"bad binlog expire": func(in *MySQLDeliveryInput) { in.BinlogExpireLogsSeconds = 3600 },
|
||||
"bad max binlog size": func(in *MySQLDeliveryInput) { in.MaxBinlogSize = "64M" },
|
||||
} {
|
||||
switch name {
|
||||
case "uppercase namespace":
|
||||
input.Namespace = "Team-A"
|
||||
case "bad instance":
|
||||
input.InstanceName = "mysql_01"
|
||||
case "bad cpu cores":
|
||||
input.CPUCores = 3
|
||||
case "too much memory":
|
||||
input.MemoryGB = 128
|
||||
case "too little storage":
|
||||
input.StorageGB = 10
|
||||
case "too much storage":
|
||||
input.StorageGB = 3000
|
||||
case "unsupported version":
|
||||
input.MySQLVersion = "5.7"
|
||||
case "unsupported topology":
|
||||
input.Topology = "mgr"
|
||||
case "bad mysql port":
|
||||
input.MySQLPort = 3306
|
||||
case "bad data disk":
|
||||
input.DataDisk = "/"
|
||||
case "unsupported charset":
|
||||
input.CharacterSet = "sjis"
|
||||
case "bad collation":
|
||||
input.CharacterSet = "utf8"
|
||||
input.Collation = "utf8mb4_general_ci"
|
||||
}
|
||||
if err := validateDeliveryInput(input); err == nil {
|
||||
input := valid
|
||||
mutate(&input)
|
||||
if err := validateDeliveryInput(input, dataDisks); err == nil {
|
||||
t.Errorf("%s was accepted", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFirstFreeHost(t *testing.T) {
|
||||
hosts := []targetHost{{Name: "node-a"}, {Name: "node-b"}}
|
||||
if h := firstFreeHost(hosts, nil, 1); h == nil || h.Name != "node-a" {
|
||||
t.Fatalf("expected node-a on empty occupancy, got %+v", h)
|
||||
}
|
||||
if h := firstFreeHost(hosts, []string{"node-a"}, 1); h == nil || h.Name != "node-b" {
|
||||
t.Fatalf("expected node-b when node-a is full at limit 1, got %+v", h)
|
||||
}
|
||||
if h := firstFreeHost(hosts, []string{"node-a", "node-b"}, 1); h != nil {
|
||||
t.Fatalf("limit 1 with all hosts taken should return nil, got %+v", h)
|
||||
}
|
||||
if h := firstFreeHost(hosts, []string{"node-a", "node-b"}, 2); h == nil || h.Name != "node-a" {
|
||||
t.Fatalf("expected node-a for second round at limit 2, got %+v", h)
|
||||
}
|
||||
if h := firstFreeHost(hosts, []string{"node-a", "node-a", "node-b", "node-b"}, 2); h != nil {
|
||||
t.Fatalf("limit 2 with all hosts saturated should return nil, got %+v", h)
|
||||
}
|
||||
if h := firstFreeHost(hosts, []string{"node-a"}, 0); h == nil || h.Name != "node-b" {
|
||||
t.Fatalf("limit 0 should degrade to 1, got %+v", h)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAllocatePort(t *testing.T) {
|
||||
if port, err := allocatePort(0, nil); err != nil || port != mysqlPortPoolStart {
|
||||
t.Fatalf("expected first pool port %d, got %d err=%v", mysqlPortPoolStart, port, err)
|
||||
}
|
||||
if port, err := allocatePort(0, []int{13306, 13307}); err != nil || port != 13308 {
|
||||
t.Fatalf("expected 13308 skipping occupied, got %d err=%v", port, err)
|
||||
}
|
||||
if port, err := allocatePort(13400, []int{13306}); err != nil || port != 13400 {
|
||||
t.Fatalf("expected requested port 13400, got %d err=%v", port, err)
|
||||
}
|
||||
if _, err := allocatePort(13306, []int{13306}); err == nil {
|
||||
t.Fatal("requested occupied port was accepted")
|
||||
}
|
||||
used := make([]int, 0, mysqlPortPoolEnd-mysqlPortPoolStart+1)
|
||||
for p := mysqlPortPoolStart; p <= mysqlPortPoolEnd; p++ {
|
||||
used = append(used, p)
|
||||
}
|
||||
if _, err := allocatePort(0, used); err == nil {
|
||||
t.Fatal("exhausted pool still allocated a port")
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user