Files
xinfra/server/internal/service/machine.go
T

711 lines
19 KiB
Go

package service
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"strings"
"sync"
"time"
"github.com/1024XEngineer/xinfra/server/internal/config"
"github.com/1024XEngineer/xinfra/server/internal/model"
"gorm.io/gorm"
)
const (
machineSyncSource = "sina"
machineClassName = "zion_server"
sinaLoginPath = "/sinai/v1/login"
sinaCILotPath = "/sinai/v1/3rd/ci/lot"
sinaSyncInterval = 30 * time.Minute
sinaRequestTimeout = 10 * time.Second
sinaPageSize = 500
)
type MachineService struct {
cfg config.Config
db *gorm.DB
httpClient *http.Client
mu sync.Mutex
running bool
}
type MachineListQuery struct {
BusinessLineID uint64
Page int
Size int
Hostname string
AssetNumber string
Type string
Location string
IP string
Spec string
Source string
Status string
}
type MachineResourceItem struct {
Hostname string `json:"hostname"`
AssetNumber string `json:"assetNumber"`
Type string `json:"type"`
Location string `json:"location"`
IP string `json:"ip"`
Spec string `json:"spec"`
BusinessLine string `json:"businessLine"`
Source string `json:"source"`
Status string `json:"status"`
}
type MachineResourceList struct {
Total int64 `json:"total"`
Items []MachineResourceItem `json:"items"`
}
type MachineOverview struct {
Total int64 `json:"total"`
Physical int64 `json:"physical"`
Virtual int64 `json:"virtual"`
CMDB MachineSourceStat `json:"cmdb"`
Aliyun MachineCloudStat `json:"aliyun"`
AWSQiniu MachineAWSQiniuStat `json:"awsQiniu"`
LastSync MachineLastSyncOverview `json:"lastSync"`
}
type MachineSourceStat struct {
Total int64 `json:"total"`
Physical int64 `json:"physical"`
Virtual int64 `json:"virtual"`
}
type MachineCloudStat struct {
Total int64 `json:"total"`
Label string `json:"label"`
}
type MachineAWSQiniuStat struct {
Total int64 `json:"total"`
AWS int64 `json:"aws"`
Qiniu int64 `json:"qiniu"`
}
type MachineLastSyncOverview struct {
Status string `json:"status"`
Time string `json:"time"`
Created int64 `json:"created"`
Error string `json:"error,omitempty"`
}
func NewMachineService(cfg config.Config, db *gorm.DB) *MachineService {
return &MachineService{
cfg: cfg,
db: db,
httpClient: &http.Client{
Timeout: sinaRequestTimeout,
},
}
}
func (s *MachineService) Run(ctx context.Context) {
go func() {
_, _ = s.SyncNow(ctx)
ticker := time.NewTicker(sinaSyncInterval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
_, _ = s.SyncNow(ctx)
}
}
}()
}
func (s *MachineService) SyncNow(ctx context.Context) (*model.MachineSyncState, error) {
if err := s.acquireSync(); err != nil {
state, _ := s.syncState(ctx)
return state, err
}
defer s.releaseSync()
started := time.Now()
if err := s.startSync(ctx, started); err != nil {
return nil, err
}
token, err := s.loginSina(ctx)
if err != nil {
_ = s.finishSync(ctx, 0, 0, 0, 0, err)
state, _ := s.syncState(ctx)
return state, err
}
page := 1
var total, fetched, created, updated int64
for {
resp, err := s.fetchSinaPage(ctx, token, page, sinaPageSize)
if err != nil {
_ = s.finishSync(ctx, total, fetched, created, updated, err)
state, _ := s.syncState(ctx)
return state, err
}
if page == 1 {
total = resp.Result.Count
}
if len(resp.Result.Items) == 0 {
break
}
for _, item := range resp.Result.Items {
isCreated, err := s.upsertMachine(ctx, item, started)
if err != nil {
_ = s.finishSync(ctx, total, fetched, created, updated, err)
state, _ := s.syncState(ctx)
return state, err
}
fetched++
if isCreated {
created++
} else {
updated++
}
}
if total > 0 && fetched >= total {
break
}
if len(resp.Result.Items) < sinaPageSize {
break
}
page++
}
if err := s.finishSync(ctx, total, fetched, created, updated, nil); err != nil {
return nil, err
}
return s.syncState(ctx)
}
func (s *MachineService) Overview(ctx context.Context, userID uint64, isAdmin bool, businessLineID uint64) (*MachineOverview, error) {
baseDB, err := s.scopedMachineDB(ctx, userID, isAdmin, businessLineID)
if err != nil {
return nil, err
}
var total, physical int64
if err := baseDB.Count(&total).Error; err != nil {
return nil, err
}
if err := baseDB.Session(&gorm.Session{}).Where("resource_type = ?", "physical").Count(&physical).Error; err != nil {
return nil, err
}
sourceCounts, err := s.sourceCounts(baseDB.Session(&gorm.Session{}))
if err != nil {
return nil, err
}
cloudSources := []string{"aliyun", "ali", "alicloud", "aws", "qiniu"}
cmdbTotal, err := s.countExcludingSources(baseDB.Session(&gorm.Session{}), cloudSources)
if err != nil {
return nil, err
}
cmdbPhysical, err := s.countExcludingSourcesAndType(baseDB.Session(&gorm.Session{}), cloudSources, "physical")
if err != nil {
return nil, err
}
state, err := s.syncState(ctx)
if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
return nil, err
}
return &MachineOverview{
Total: total,
Physical: physical,
Virtual: total - physical,
CMDB: MachineSourceStat{
Total: cmdbTotal,
Physical: cmdbPhysical,
Virtual: cmdbTotal - cmdbPhysical,
},
Aliyun: MachineCloudStat{
Total: sourceCounts["aliyun"],
Label: "ECS · 增量同步",
},
AWSQiniu: MachineAWSQiniuStat{
Total: sourceCounts["aws"] + sourceCounts["qiniu"],
AWS: sourceCounts["aws"],
Qiniu: sourceCounts["qiniu"],
},
LastSync: mapLastSync(state),
}, nil
}
func (s *MachineService) List(ctx context.Context, userID uint64, isAdmin bool, query MachineListQuery) (*MachineResourceList, error) {
if query.Page <= 0 {
query.Page = 1
}
if query.Size <= 0 {
query.Size = 20
}
if query.Size > 200 {
query.Size = 200
}
db, err := s.scopedMachineDB(ctx, userID, isAdmin, query.BusinessLineID)
if err != nil {
return nil, err
}
db = applyMachineFilters(db, query)
var total int64
if err := db.Count(&total).Error; err != nil {
return nil, err
}
var rows []model.MachineResource
if err := db.Order("updated_at DESC").Offset((query.Page - 1) * query.Size).Limit(query.Size).Find(&rows).Error; err != nil {
return nil, err
}
items := make([]MachineResourceItem, 0, len(rows))
for _, row := range rows {
items = append(items, mapMachineResource(row))
}
return &MachineResourceList{Total: total, Items: items}, nil
}
func (s *MachineService) scopedMachineDB(ctx context.Context, userID uint64, isAdmin bool, businessLineID uint64) (*gorm.DB, error) {
if businessLineID == 0 {
return nil, errors.New("business_line_id is required")
}
if !isAdmin {
var binding model.BusinessLineUser
err := s.db.WithContext(ctx).Where("business_line_id = ? AND user_id = ?", businessLineID, userID).First(&binding).Error
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil, errors.New("current user does not have business line permission")
}
if err != nil {
return nil, err
}
}
var mappings []model.BusinessLineSinaOrganization
if err := s.db.WithContext(ctx).Where("business_line_id = ?", businessLineID).Order("sina_organization_name ASC").Find(&mappings).Error; err != nil {
return nil, err
}
names := make([]string, 0, len(mappings))
for _, mapping := range mappings {
if name := strings.TrimSpace(mapping.SinaOrganizationName); name != "" {
names = append(names, name)
}
}
db := s.db.WithContext(ctx).Model(&model.MachineResource{})
if len(names) == 0 {
return db.Where("1 = 0"), nil
}
return db.Where("business_line IN ?", names), nil
}
func (s *MachineService) acquireSync() error {
s.mu.Lock()
defer s.mu.Unlock()
if s.running {
return errors.New("machine sync is already running")
}
s.running = true
return nil
}
func (s *MachineService) releaseSync() {
s.mu.Lock()
defer s.mu.Unlock()
s.running = false
}
func (s *MachineService) startSync(ctx context.Context, started time.Time) error {
state := model.MachineSyncState{
Source: machineSyncSource,
Status: "running",
LastStartedAt: &started,
LastError: "",
}
return s.db.WithContext(ctx).Where("source = ?", machineSyncSource).Assign(state).FirstOrCreate(&state).Error
}
func (s *MachineService) finishSync(ctx context.Context, total, fetched, created, updated int64, syncErr error) error {
now := time.Now()
status := "success"
lastErr := ""
if syncErr != nil {
status = "failed"
lastErr = syncErr.Error()
}
return s.db.WithContext(ctx).Model(&model.MachineSyncState{}).
Where("source = ?", machineSyncSource).
Updates(map[string]interface{}{
"status": status,
"last_finished_at": &now,
"last_error": lastErr,
"total": total,
"fetched": fetched,
"created_count": created,
"updated_count": updated,
}).Error
}
func (s *MachineService) syncState(ctx context.Context) (*model.MachineSyncState, error) {
var state model.MachineSyncState
err := s.db.WithContext(ctx).Where("source = ?", machineSyncSource).First(&state).Error
return &state, err
}
type sinaPageResp struct {
Success bool `json:"success"`
Result struct {
Items []map[string]interface{} `json:"items"`
Count int64 `json:"count"`
} `json:"result"`
Message string `json:"message"`
}
type sinaLoginResp struct {
Success bool `json:"success"`
Result map[string]interface{} `json:"result"`
Data map[string]interface{} `json:"data"`
Token string `json:"token"`
Message string `json:"message"`
}
func (s *MachineService) loginSina(ctx context.Context) (string, error) {
username := strings.TrimSpace(s.cfg.SINAUsername)
password := strings.TrimSpace(s.cfg.SINAPassword)
if username == "" || password == "" {
return "", errors.New("SINA_USERNAME or SINA_PASSWORD is not configured")
}
payload, err := json.Marshal(map[string]string{
"username": username,
"password": password,
})
if err != nil {
return "", err
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, s.cfg.SINABaseURL+sinaLoginPath, bytes.NewReader(payload))
if err != nil {
return "", err
}
req.Header.Set("content-type", "application/json")
resp, err := s.httpClient.Do(req)
if err != nil {
return "", err
}
defer resp.Body.Close()
respBody, err := io.ReadAll(resp.Body)
if err != nil {
return "", err
}
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return "", fmt.Errorf("sina login failed: status=%d body=%s", resp.StatusCode, string(respBody))
}
var parsed sinaLoginResp
if err := json.Unmarshal(respBody, &parsed); err != nil {
return "", err
}
if !parsed.Success {
return "", fmt.Errorf("sina login failed: %s", parsed.Message)
}
token := firstNonEmpty(
parsed.Token,
machineStringValue(parsed.Result["token"]),
machineStringValue(parsed.Result["access_token"]),
machineStringValue(parsed.Data["token"]),
machineStringValue(parsed.Data["access_token"]),
)
if token == "" {
return "", errors.New("sina login response missing token")
}
return token, nil
}
func (s *MachineService) fetchSinaPage(ctx context.Context, token string, page, size int) (*sinaPageResp, error) {
body := map[string]interface{}{
"className": machineClassName,
"page": page,
"size": size,
"field": map[string]interface{}{},
}
payload, err := json.Marshal(body)
if err != nil {
return nil, err
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, s.cfg.SINABaseURL+sinaCILotPath, bytes.NewReader(payload))
if err != nil {
return nil, err
}
req.Header.Set("content-type", "application/json")
req.Header.Set("Authorization", token)
resp, err := s.httpClient.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
respBody, err := io.ReadAll(resp.Body)
if err != nil {
return nil, err
}
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return nil, fmt.Errorf("sina request failed: status=%d body=%s", resp.StatusCode, string(respBody))
}
var parsed sinaPageResp
if err := json.Unmarshal(respBody, &parsed); err != nil {
return nil, err
}
if !parsed.Success {
return nil, fmt.Errorf("sina request failed: %s", parsed.Message)
}
return &parsed, nil
}
func (s *MachineService) upsertMachine(ctx context.Context, item map[string]interface{}, syncedAt time.Time) (bool, error) {
sinaID := machineInt64Value(item["id"])
if sinaID == 0 {
return false, nil
}
raw, _ := json.Marshal(item)
next := model.MachineResource{
SinaID: sinaID,
CIID: machineStringValue(item["ciId"]),
Hostname: machineStringValue(item["hostname"]),
Name: machineStringValue(item["name"]),
AssetNumber: machineStringValue(item["asset_number"]),
IDCNumber: machineStringValue(item["idc_number"]),
ResourceType: machineStringValue(item["type"]),
LocationName: machineStringValue(item["location_name"]),
LocationAlias: machineStringValue(item["location_alias"]),
IntranetIP: machineStringValue(item["intranet_ip"]),
Spec: machineStringValue(item["spec"]),
BusinessLine: machineStringValue(item["org_name"]),
Source: machineStringValue(item["source"]),
Status: machineStringValue(item["status"]),
Brand: machineStringValue(item["brand"]),
Model: machineStringValue(item["model"]),
SerialNumber: machineStringValue(item["serial_number"]),
PowerStatus: machineStringValue(item["power_status"]),
CPUCores: machineInt64Value(item["cpu_cores"]),
CPUName: machineStringValue(item["cpu_name"]),
Memory: machineStringValue(item["memory"]),
HardCapacity: machineStringValue(item["hard_capacity"]),
SSDCapacity: machineStringValue(item["ssd_capacity"]),
OOBIP: machineStringValue(item["oob_ip"]),
OSFamily: machineStringValue(item["os_family"]),
OSVersion: machineStringValue(item["os_version"]),
CollectTime: parseSinaTime(machineStringValue(item["collect_time"])),
SinaUpdatedAt: parseSinaTime(machineStringValue(item["updated"])),
RawJSON: string(raw),
LastSyncedAt: syncedAt,
}
var current model.MachineResource
err := s.db.WithContext(ctx).Where("sina_id = ?", sinaID).First(&current).Error
if errors.Is(err, gorm.ErrRecordNotFound) {
return true, s.db.WithContext(ctx).Create(&next).Error
}
if err != nil {
return false, err
}
next.ID = current.ID
next.CreatedAt = current.CreatedAt
return false, s.db.WithContext(ctx).Save(&next).Error
}
func applyMachineFilters(db *gorm.DB, query MachineListQuery) *gorm.DB {
switch query.Type {
case "physical":
db = db.Where("resource_type = ?", "physical")
case "vm", "virtual":
db = db.Where("resource_type <> ?", "physical")
}
switch query.Source {
case "cmdb":
db = db.Where("source NOT IN ?", []string{"aliyun", "ali", "alicloud", "aws", "qiniu"})
case "aliyun":
db = db.Where("source IN ?", []string{"aliyun", "ali", "alicloud"})
case "aws":
db = db.Where("source = ?", "aws")
case "qiniu":
db = db.Where("source = ?", "qiniu")
}
if query.Status != "" {
db = db.Where("status = ?", query.Status)
}
if query.Hostname != "" {
like := "%" + query.Hostname + "%"
db = db.Where("hostname LIKE ? OR name LIKE ?", like, like)
}
if query.AssetNumber != "" {
like := "%" + query.AssetNumber + "%"
db = db.Where("asset_number LIKE ? OR idc_number LIKE ?", like, like)
}
if query.Location != "" {
like := "%" + query.Location + "%"
db = db.Where("location_name LIKE ? OR location_alias LIKE ?", like, like)
}
if query.IP != "" {
db = db.Where("intranet_ip LIKE ?", "%"+query.IP+"%")
}
if query.Spec != "" {
db = db.Where("spec LIKE ?", "%"+query.Spec+"%")
}
return db
}
func (s *MachineService) sourceCounts(db *gorm.DB) (map[string]int64, error) {
var rows []struct {
Source string
Count int64
}
if err := db.Select("source, count(*) as count").Group("source").Scan(&rows).Error; err != nil {
return nil, err
}
out := map[string]int64{}
for _, row := range rows {
key := classifyMachineSource(row.Source)
out[key] += row.Count
}
return out, nil
}
func (s *MachineService) countExcludingSources(db *gorm.DB, sources []string) (int64, error) {
var count int64
err := db.Where("source NOT IN ?", sources).Count(&count).Error
return count, err
}
func (s *MachineService) countExcludingSourcesAndType(db *gorm.DB, sources []string, resourceType string) (int64, error) {
var count int64
err := db.Where("source NOT IN ? AND resource_type = ?", sources, resourceType).Count(&count).Error
return count, err
}
func mapMachineResource(row model.MachineResource) MachineResourceItem {
return MachineResourceItem{
Hostname: firstNonEmpty(row.Hostname, row.Name),
AssetNumber: firstNonEmpty(row.AssetNumber, row.IDCNumber, "—"),
Type: displayMachineType(row.ResourceType),
Location: firstNonEmpty(row.LocationName, row.LocationAlias),
IP: row.IntranetIP,
Spec: row.Spec,
BusinessLine: row.BusinessLine,
Source: displayMachineSource(row.Source),
Status: row.Status,
}
}
func mapLastSync(state *model.MachineSyncState) MachineLastSyncOverview {
if state == nil || state.ID == 0 || state.LastFinishedAt == nil {
return MachineLastSyncOverview{Status: "unknown", Time: "—"}
}
status := "normal"
if state.Status == "failed" {
status = "failed"
}
return MachineLastSyncOverview{
Status: status,
Time: state.LastFinishedAt.Format("15:04:05"),
Created: state.CreatedCount,
Error: state.LastError,
}
}
func classifyMachineSource(source string) string {
switch strings.ToLower(strings.TrimSpace(source)) {
case "aliyun", "ali", "alicloud":
return "aliyun"
case "aws":
return "aws"
case "qiniu":
return "qiniu"
default:
return "cmdb"
}
}
func displayMachineSource(source string) string {
switch classifyMachineSource(source) {
case "aliyun":
return "阿里云同步"
case "aws":
return "AWS 同步"
case "qiniu":
return "七牛同步"
default:
return "SINA CMDB"
}
}
func displayMachineType(value string) string {
if strings.EqualFold(value, "physical") {
return "物理机"
}
return "虚机"
}
func firstNonEmpty(values ...string) string {
for _, value := range values {
if strings.TrimSpace(value) != "" {
return value
}
}
return ""
}
func machineStringValue(value interface{}) string {
switch v := value.(type) {
case string:
return strings.TrimSpace(v)
case fmt.Stringer:
return strings.TrimSpace(v.String())
case nil:
return ""
default:
return strings.TrimSpace(fmt.Sprint(v))
}
}
func machineInt64Value(value interface{}) int64 {
switch v := value.(type) {
case int64:
return v
case int:
return int64(v)
case float64:
return int64(v)
case json.Number:
n, _ := v.Int64()
return n
default:
return 0
}
}
func parseSinaTime(value string) *time.Time {
value = strings.TrimSpace(value)
if value == "" || strings.HasPrefix(value, "0001-") {
return nil
}
t, err := time.ParseInLocation("2006-01-02 15:04:05", value, time.Local)
if err != nil {
return nil
}
return &t
}