平台系统后台管理模块

platform-admin/
├── go.mod
├── cmd/admin/main.go                  # 进程入口：装配 DB / 网关客户端 / gin 路由
├── deploy/
│   ├── admin-configmap.yaml           # 配置（网关地址 / DB / 留存天数）
│   ├── admin-rbac.yaml                # ServiceAccount + Role + RoleBinding
│   ├── admin-deployment.yaml          # 管理后台 Deployment + 探针
│   ├── db-backup-cronjob.yaml         # 元数据中心 DB 每日异地加密备份
│   └── admin-prometheus.yaml          # ServiceMonitor + 告警规则
└── internal/
    ├── model/
    │   ├── audit.go                    # GORM 审计表（90天+ 不可篡改）
    │   └── config.go                   # 全局配额 / 特性开关模型
    ├── store/
    │   └── postgres.go                 # GORM 连接与自动迁移
    ├── k8s/
    │   ├── client.go                   # client-go 经统一网关（GatewayRoundTripper）
    │   └── tenant.go                   # 租户 Namespace / ResourceQuota reconcile
    └── admin/
        ├── metrics.go                  # Prometheus 指标 + Metrics 中间件
        ├── middleware.go               # RBAC + 审计落库
        ├── handler.go                  # gin 管理 API（审计查询/配额管理）
        └── backup.go                   # 备份 Job 经网关触发

// file: go.mod
module github.com/example/paas/admin

go 1.22

require (
	github.com/gin-gonic/gin v1.10.0
	github.com/prometheus/client_golang v1.19.1
	gorm.io/driver/postgres v1.5.7
	gorm.io/gorm v1.25.10
	k8s.io/api v0.30.2
	k8s.io/apimachinery v0.30.2
	k8s.io/client-go v0.30.2
)

# file: deploy/admin-configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: platform-admin-config
  namespace: paas-system
  labels:
    app: platform-admin
    paas.io/managed: "true"
data:
  LOG_LEVEL: "info"
  GATEWAY_ADDR: "https://k8s-gateway.paas.internal"   # 统一 K8s API 网关地址
  DB_HOST: "pg-admin.paas-system.svc.cluster.local"
  DB_PORT: "5432"
  DB_NAME: "paas_admin"
  DB_SSLMODE: "require"
  AUDIT_RETENTION_DAYS: "90"                            # 审计留存 ≥90 天
  BACKUP_BUCKET: "s3://paas-backup-prod/metadata"       # 异地对象存储

# file: deploy/admin-rbac.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
  name: platform-admin
  namespace: paas-system
  labels:
    app: platform-admin
    paas.io/managed: "true"
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: platform-admin-role
  namespace: paas-system
rules:
  - apiGroups: [""]
    resources: ["namespaces", "resourcequotas", "configmaps", "secrets"]
    verbs: ["get", "list", "watch", "create", "update", "patch"]
  - apiGroups: ["batch"]
    resources: ["jobs", "cronjobs"]
    verbs: ["get", "list", "watch", "create", "delete"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: platform-admin-binding
  namespace: paas-system
subjects:
  - kind: ServiceAccount
    name: platform-admin
    namespace: paas-system
roleRef:
  kind: Role
  name: platform-admin-role
  apiGroup: rbac.authorization.k8s.io

// file: internal/model/audit.go
package model

import "time"

// AuditLog 全局审计视图，留存 90 天以上，不可篡改（数据库只读副本 + Thanos 备份）。
// 标签 tenant/app/env/cluster 与 Prometheus 联邦指标对齐。
type AuditLog struct {
	ID          uint64    `gorm:"primaryKey;autoIncrement" json:"id"`
	TraceID     string    `gorm:"size:64;index" json:"trace_id"`
	Actor       string    `gorm:"size:128;index" json:"actor"`     // 操作人
	Role        string    `gorm:"size:64" json:"role"`             // 平台管理员/审计员/审批人
	Tenant      string    `gorm:"size:128;index" json:"tenant"`    // 租户
	App         string    `gorm:"size:128;index" json:"app"`       // 应用
	Env         string    `gorm:"size:32;index" json:"env"`        // prod/staging/test/dev
	Cluster     string    `gorm:"size:128;index" json:"cluster"`   // 物理隔离集群标识
	Resource    string    `gorm:"size:256" json:"resource"`        // 资源类型
	Action      string    `gorm:"size:128" json:"action"`          // create/update/delete/quota
	BeforeValue string    `gorm:"type:text" json:"before_value"`   // 变更前值（敏感字段已加密）
	AfterValue  string    `gorm:"type:text" json:"after_value"`    // 变更后值
	ApprovalID  string    `gorm:"size:64;index" json:"approval_id"` // 关联审批单
	Result      string    `gorm:"size:16" json:"result"`           // success/fail
	ClientIP    string    `gorm:"size:64" json:"client_ip"`
	CreatedAt   time.Time `gorm:"index" json:"created_at"`
}

func (AuditLog) TableName() string { return "audit_logs" }

// file: internal/model/config.go
package model

import "time"

// GlobalQuota 全局配额（元数据中心逻辑管控 + 成本计量）。
// 真正生效的 ResourceQuota 由平台控制器经统一 K8s API 网关下发给租户 Namespace。
type GlobalQuota struct {
	ID               uint64    `gorm:"primaryKey" json:"id"`
	Tenant           string    `gorm:"size:128;uniqueIndex:uniq_tenant_env" json:"tenant"`
	Env              string    `gorm:"size:32;uniqueIndex:uniq_tenant_env" json:"env"`
	CPUQuota         int64     `json:"cpu_quota"`          // 单位：毫核
	MemQuota         int64     `json:"mem_quota"`          // 单位：Mi
	StorageQuota     int64     `json:"storage_quota"`      // 单位：Gi
	MiddlewareInstances int    `json:"middleware_instances"`
	LicenseEdition   string    `gorm:"size:32" json:"license_edition"` // standard/enterprise/finance
	CreatedAt        time.Time `json:"created_at"`
	UpdatedAt        time.Time `json:"updated_at"`
}

func (GlobalQuota) TableName() string { return "global_quotas" }

// FeatureFlag 平台级特性开关，支持按租户灰度。
type FeatureFlag struct {
	ID         uint64    `gorm:"primaryKey" json:"id"`
	Key        string    `gorm:"size:128;uniqueIndex" json:"key"`
	Enabled    bool      `json:"enabled"`
	GrayTenant string    `gorm:"size:128" json:"gray_tenant"` // 灰度租户，空表示全量
	CreatedAt  time.Time `json:"created_at"`
	UpdatedAt  time.Time `json:"updated_at"`
}

func (FeatureFlag) TableName() string { return "feature_flags" }

// file: internal/k8s/client.go
package k8s

import (
	"context"
	"net/http"

	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
)

// GatewayRoundTripper 在每次请求注入网关令牌与目标集群标签，
// 后台管理服务绝不直连 apiserver，所有请求经统一 K8s API 网关收敛。
type GatewayRoundTripper struct {
	GatewayToken string
	Cluster      string
	Env          string
	Next         http.RoundTripper
}

func (rt *GatewayRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
	req.Header.Set("Authorization", "Bearer "+rt.GatewayToken)
	req.Header.Set("X-PaaS-Cluster", rt.Cluster) // 网关按此标签路由到物理隔离集群
	req.Header.Set("X-PaaS-Env", rt.Env)
	return rt.Next.RoundTrip(req)
}

// Client 封装经网关访问某集群的 clientset。
type Client struct {
	Clientset *kubernetes.Clientset
	Cluster   string
	Env       string
}

// Dial 构造指向统一 K8s API 网关的 client-go 客户端。
func Dial(gatewayAddr, gatewayToken, cluster, env string) (*Client, error) {
	cfg := &rest.Config{
		Host: gatewayAddr,
		TLSClientConfig: rest.TLSClientConfig{
			Insecure: false, // 生产使用网关 CA，不跳过校验
		},
	}
	cfg.WrapTransport = func(rt http.RoundTripper) http.RoundTripper {
		return &GatewayRoundTripper{
			GatewayToken: gatewayToken,
			Cluster:      cluster,
			Env:          env,
			Next:         rt,
		}
	}
	cs, err := kubernetes.NewForConfig(cfg)
	if err != nil {
		return nil, err
	}
	return &Client{Clientset: cs, Cluster: cluster, Env: env}, nil
}

// Health 探测网关到目标集群的连接态。
func (c *Client) Health(ctx context.Context) error {
	_, err := c.Clientset.Discovery().ServerVersion()
	return err
}

// file: internal/k8s/tenant.go
package k8s

import (
	"context"
	"fmt"

	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/api/resource"

	"github.com/example/paas/admin/internal/model"
)

// ReconcileNamespace 确保租户 Namespace 存在且带 tenant/env 标签（资源强隔离）。
func (c *Client) ReconcileNamespace(ctx context.Context, tenant, env string) error {
	nsName := fmt.Sprintf("%s-%s", tenant, env)
	cs := c.Clientset
	ns, err := cs.CoreV1().Namespaces().Get(ctx, nsName, metav1.GetOptions{})
	if err != nil {
		ns = &corev1.Namespace{
			ObjectMeta: metav1.ObjectMeta{
				Name: nsName,
				Labels: map[string]string{
					"tenant":          tenant,
					"env":             env,
					"paas.io/managed": "true",
				},
			},
		}
		if _, err := cs.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}); err != nil {
			return fmt.Errorf("create namespace %s: %w", nsName, err)
		}
		return nil
	}
	if ns.Labels == nil {
		ns.Labels = map[string]string{}
	}
	ns.Labels["tenant"] = tenant
	ns.Labels["env"] = env
	_, err = cs.CoreV1().Namespaces().Update(ctx, ns, metav1.UpdateOptions{})
	return err
}

// ReconcileResourceQuota 将全局配额下发给租户 Namespace 的 K8s ResourceQuota，
// 避免"后台已满、集群仍可调度"的配额漂移。
func (c *Client) ReconcileResourceQuota(ctx context.Context, q model.GlobalQuota) error {
	nsName := fmt.Sprintf("%s-%s", q.Tenant, q.Env)
	rqName := fmt.Sprintf("quota-%s-%s", q.Tenant, q.Env)
	hard := corev1.ResourceList{
		corev1.ResourceCPU:     *resource.NewMilliQuantity(q.CPUQuota, resource.DecimalSI),
		corev1.ResourceMemory:  *resource.NewQuantity(q.MemQuota*1024*1024, resource.BinarySI),
		corev1.ResourceStorage: *resource.NewQuantity(q.StorageQuota*1024*1024*1024, resource.BinarySI),
	}
	rq := &corev1.ResourceQuota{
		ObjectMeta: metav1.ObjectMeta{
			Name:      rqName,
			Namespace: nsName,
			Labels: map[string]string{
				"tenant":          q.Tenant,
				"env":             q.Env,
				"paas.io/managed": "true",
			},
		},
		Spec: corev1.ResourceQuotaSpec{Hard: hard},
	}
	cs := c.Clientset
	if _, err := cs.CoreV1().ResourceQuotas(nsName).Get(ctx, rqName, metav1.GetOptions{}); err != nil {
		if _, err := cs.CoreV1().ResourceQuotas(nsName).Create(ctx, rq, metav1.CreateOptions{}); err != nil {
			return fmt.Errorf("create resourcequota %s: %w", rqName, err)
		}
		return nil
	}
	if _, err := cs.CoreV1().ResourceQuotas(nsName).Update(ctx, rq, metav1.UpdateOptions{}); err != nil {
		return fmt.Errorf("update resourcequota %s: %w", rqName, err)
	}
	return nil
}

// file: internal/admin/middleware.go
package admin

import (
	"context"
	"crypto/rand"
	"encoding/hex"
	"time"

	"github.com/gin-gonic/gin"
	"gorm.io/gorm"

	"github.com/example/paas/admin/internal/model"
)

// RequireRole 后台 RBAC：仅允许持对应角色的管理员执行敏感操作。
// 角色由接入层 SSO/OAuth2 注入到 X-Admin-Role 头，绝不下放集群凭据。
func RequireRole(roles ...string) gin.HandlerFunc {
	allowed := make(map[string]bool)
	for _, r := range roles {
		allowed[r] = true
	}
	return func(c *gin.Context) {
		role := c.GetHeader("X-Admin-Role")
		actor := c.GetHeader("X-Admin-User")
		if role == "" || !allowed[role] {
			c.AbortWithStatusJSON(403, gin.H{"error": "forbidden: insufficient platform role"})
			return
		}
		c.Set("actor", actor)
		c.Set("role", role)
		c.Next()
	}
}

// AuditContext 审计落库上下文。
type AuditContext struct {
	Ctx        context.Context
	Actor      string
	Role       string
	Tenant     string
	App        string
	Env        string
	Cluster    string
	Resource   string
	Action     string
	Before     string
	After      string
	ApprovalID string
	Result     string
	ClientIP   string
}

// writeAudit 写入一条审计记录（90天+ 不可篡改，配合 Thanos 只读层）。
func (s *Server) writeAudit(a AuditContext) {
	_ = s.db.WithContext(a.Ctx).Create(&model.AuditLog{
		TraceID:     newTraceID(),
		Actor:       a.Actor,
		Role:        a.Role,
		Tenant:      a.Tenant,
		App:         a.App,
		Env:         a.Env,
		Cluster:     a.Cluster,
		Resource:    a.Resource,
		Action:      a.Action,
		BeforeValue: a.Before,
		AfterValue:  a.After,
		ApprovalID:  a.ApprovalID,
		Result:      a.Result,
		ClientIP:    a.ClientIP,
		CreatedAt:   time.Now(),
	}).Error
}

func newTraceID() string {
	b := make([]byte, 16)
	_, _ = rand.Read(b)
	return hex.EncodeToString(b)
}

func strPtr(v interface{}) string {
	if v == nil {
		return ""
	}
	if s, ok := v.(string); ok {
		return s
	}
	return ""
}

// ensure gorm 被引用（Server.db 使用），避免未使用导入告警
var _ = gorm.ErrRecordNotFound

# 步骤 1：管理员登录后台，确认目标组件在测试集群状态正常
kubectl --context=paas-gateway get deploy platform-admin -n paas-system

# 步骤 2：在后台提交"配额调整 + 组件升级"草稿，标记 env=测试
curl -X PUT "https://admin.paas.internal/api/v1/quotas/acme/test" \
  -H "X-Admin-Role: platform-admin" -H "X-Admin-User: alice" \
  -H "X-Approval-Id: TEST-APR-0001" \
  -d '{"cluster":"test-cluster","cpu_quota":120000,"mem_quota":262144,"storage_quota":500,"middleware_instances":8}'

# 步骤 3：测试环境审批可自动通过（策略配置），网关下发变更
# 步骤 4：观察联邦 Grafana 后台大盘，确认组件版本与配额生效

# 步骤 1：管理员登录后台提交变更，env=生产，触发多级审批
curl -X PUT "https://admin.paas.internal/api/v1/quotas/acme/prod" \
  -H "X-Admin-Role: platform-admin" -H "X-Admin-User: alice" \
  -H "X-Approval-Id: PROD-APR-8842" \
  -d '{"cluster":"prod-cluster","cpu_quota":120000,"mem_quota":262144,"storage_quota":500,"middleware_instances":8}'

# 步骤 2：配额审批人 + 变更委员会两级审批，审批事件进审计
# 步骤 3：审批通过后网关下发；升级先对生产集群做 pre-flight 检查
# 步骤 4：滚动升级，分批灰度；每批经联邦健康检查，异常自动暂停
# 步骤 5：升级完成后审计员核验审计视图，确认 90 天可查

# file: deploy/admin-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: platform-admin
  namespace: paas-system
  labels:
    app: platform-admin
    tenant: platform
    env: prod
    cluster: prod-cluster
    paas.io/managed: "true"
spec:
  replicas: 2
  selector:
    matchLabels:
      app: platform-admin
  template:
    metadata:
      labels:
        app: platform-admin
        tenant: platform
        env: prod
        cluster: prod-cluster
    spec:
      serviceAccountName: platform-admin
      containers:
        - name: admin
          image: registry.paas.internal/platform-admin:v1.8.2
          ports:
            - containerPort: 8080
              name: http
            - containerPort: 9090
              name: metrics
          envFrom:
            - configMapRef:
                name: platform-admin-config
          env:
            - name: DB_PASSWORD
              valueFrom:
                secretKeyRef:
                  name: platform-admin-secret
                  key: db-password
            - name: GATEWAY_TOKEN
              valueFrom:
                secretKeyRef:
                  name: platform-admin-secret
                  key: gateway-token
          readinessProbe:
            httpGet:
              path: /healthz
              port: 8080
            initialDelaySeconds: 10
          livenessProbe:
            httpGet:
              path: /livez
              port: 8080
          resources:
            requests:
              cpu: 200m
              memory: 256Mi
            limits:
              cpu: 500m
              memory: 512Mi

# file: deploy/db-backup-cronjob.yaml
apiVersion: batch/v1
kind: CronJob
metadata:
  name: platform-admin-db-backup
  namespace: paas-system
  labels:
    app: platform-admin
    paas.io/managed: "true"
spec:
  schedule: "0 2 * * *"            # 每日 02:00 全量备份
  concurrencyPolicy: Forbid
  successfulJobsHistoryLimit: 7
  failedJobsHistoryLimit: 3
  jobTemplate:
    spec:
      backoffLimit: 2
      template:
        spec:
          serviceAccountName: platform-admin
          restartPolicy: Never
          containers:
            - name: backup
              image: registry.paas.internal/pg-backup:1.4.0
              env:
                - name: PGHOST
                  valueFrom:
                    configMapKeyRef:
                      name: platform-admin-config
                      key: DB_HOST
                - name: PGPASSWORD
                  valueFrom:
                    secretKeyRef:
                      name: platform-admin-secret
                      key: db-password
                - name: BACKUP_BUCKET
                  valueFrom:
                    configMapKeyRef:
                      name: platform-admin-config
                      key: BACKUP_BUCKET
              command:
                - /bin/sh
                - -c
                - |
                  ts=$(date +%Y%m%d-%H%M%S)
                  pg_dump -U paas_admin paas_admin | gzip > /tmp/meta-$ts.sql.gz
                  # 异地对象存储 + KMS 服务端加密上传
                  rclone copy /tmp/meta-$ts.sql.gz ${BACKUP_BUCKET}/ --s3-server-side-encryption aws:kms
                  echo "backup $ts done"                  

// file: internal/store/postgres.go
package store

import (
	"fmt"
	"time"

	"gorm.io/driver/postgres"
	"gorm.io/gorm"
	"gorm.io/gorm/logger"

	"github.com/example/paas/admin/internal/model"
)

// Config 数据库连接配置，从 ConfigMap / 环境变量注入。
type Config struct {
	Host     string
	Port     int
	User     string
	Password string
	DBName   string
	SSLMode  string
}

func NewPostgres(cfg Config) (*gorm.DB, error) {
	dsn := fmt.Sprintf(
		"host=%s port=%d user=%s password=%s dbname=%s sslmode=%s",
		cfg.Host, cfg.Port, cfg.User, cfg.Password, cfg.DBName, cfg.SSLMode,
	)
	db, err := gorm.Open(postgres.Open(dsn), &gorm.Config{
		Logger: logger.Default.LogMode(logger.Warn),
	})
	if err != nil {
		return nil, fmt.Errorf("open postgres: %w", err)
	}
	sqlDB, err := db.DB()
	if err != nil {
		return nil, err
	}
	sqlDB.SetMaxOpenConns(50)
	sqlDB.SetMaxIdleConns(10)
	sqlDB.SetConnMaxLifetime(30 * time.Minute)

	// 生产环境由独立迁移任务执行；此处兜底确保本地/测试可用
	if err := db.AutoMigrate(&model.AuditLog{}, &model.GlobalQuota{}, &model.FeatureFlag{}); err != nil {
		return nil, fmt.Errorf("migrate: %w", err)
	}
	return db, nil
}

// file: internal/admin/metrics.go
package admin

import (
	"github.com/gin-gonic/gin"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promauto"
)

var (
	apiRequests = promauto.NewCounterVec(prometheus.CounterOpts{
		Name: "paas_admin_api_requests_total",
		Help: "后台管理 API 调用量",
	}, []string{"tenant", "app", "env", "cluster", "action", "result"})

	apiLatency = promauto.NewHistogramVec(prometheus.HistogramOpts{
		Name:    "paas_admin_api_request_duration_seconds",
		Help:    "后台管理 API 耗时",
		Buckets: prometheus.DefBuckets,
	}, []string{"tenant", "app", "env", "cluster", "action"})

	backupRuns = promauto.NewCounterVec(prometheus.CounterOpts{
		Name: "paas_admin_backup_runs_total",
		Help: "DB 备份触发次数",
	}, []string{"tenant", "env", "cluster", "result"})
)

// MetricsMiddleware 记录后台管理 API 调用量（带 tenant/app/env/cluster 标签）。
func MetricsMiddleware() gin.HandlerFunc {
	return func(c *gin.Context) {
		c.Next()
		tenant := pick(c.Query("tenant"), c.Param("tenant"), "platform")
		env := pick(c.Query("env"), "prod")
		cluster := pick(c.Query("cluster"), "prod-cluster")
		result := "success"
		if c.Writer.Status() >= 400 {
			result = "fail"
		}
		apiRequests.WithLabelValues(tenant, "platform-admin", env, cluster, c.Request.Method, result).Inc()
	}
}

func pick(vals ...string) string {
	for _, v := range vals {
		if v != "" {
			return v
		}
	}
	return ""
}

// file: internal/admin/handler.go
package admin

import (
	"encoding/json"
	"net/http"

	"github.com/gin-gonic/gin"
	"gorm.io/gorm"

	"github.com/example/paas/admin/internal/k8s"
	"github.com/example/paas/admin/internal/model"
)

// Server 后台管理服务核心，持有 DB 与按集群缓存的网关客户端。
type Server struct {
	db           *gorm.DB
	gatewayAddr  string
	gatewayToken string
	clients      map[string]*k8s.Client
}

func NewServer(db *gorm.DB, gatewayAddr, gatewayToken string) *Server {
	return &Server{
		db:           db,
		gatewayAddr:  gatewayAddr,
		gatewayToken: gatewayToken,
		clients:      make(map[string]*k8s.Client),
	}
}

// k8sFor 按 cluster/env 返回经统一网关的客户端（带缓存，避免重复 Dial）。
func (s *Server) k8sFor(cluster, env string) (*k8s.Client, error) {
	key := cluster + "/" + env
	if c, ok := s.clients[key]; ok {
		return c, nil
	}
	c, err := k8s.Dial(s.gatewayAddr, s.gatewayToken, cluster, env)
	if err != nil {
		return nil, err
	}
	s.clients[key] = c
	return c, nil
}

// QueryAuditLogs GET /api/v1/audit/logs  全局审计视图检索（≥90天）。
func (s *Server) QueryAuditLogs(c *gin.Context) {
	var logs []model.AuditLog
	q := s.db.WithContext(c.Request.Context()).Order("created_at DESC")
	if v := c.Query("tenant"); v != "" {
		q = q.Where("tenant = ?", v)
	}
	if v := c.Query("env"); v != "" {
		q = q.Where("env = ?", v)
	}
	if v := c.Query("cluster"); v != "" {
		q = q.Where("cluster = ?", v)
	}
	if v := c.Query("actor"); v != "" {
		q = q.Where("actor = ?", v)
	}
	if v := c.Query("action"); v != "" {
		q = q.Where("action = ?", v)
	}
	if err := q.Limit(200).Find(&logs).Error; err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}
	c.JSON(http.StatusOK, gin.H{"total": len(logs), "items": logs})
}

// GetQuota GET /api/v1/quotas/:tenant/:env  查询元数据中心全局配额。
func (s *Server) GetQuota(c *gin.Context) {
	tenant := c.Param("tenant")
	env := c.Param("env")
	var q model.GlobalQuota
	if err := s.db.WithContext(c.Request.Context()).
		Where("tenant = ? AND env = ?", tenant, env).First(&q).Error; err != nil {
		c.JSON(http.StatusNotFound, gin.H{"error": "quota not found"})
		return
	}
	c.JSON(http.StatusOK, q)
}

// UpdateQuota PUT /api/v1/quotas/:tenant/:env
// 更新元数据中心全局配额，并经统一 K8s API 网关重下发 ResourceQuota 消除漂移。
func (s *Server) UpdateQuota(c *gin.Context) {
	tenant := c.Param("tenant")
	env := c.Param("env")
	var req struct {
		Cluster             string `json:"cluster"`
		CPUQuota            int64  `json:"cpu_quota"`
		MemQuota            int64  `json:"mem_quota"`
		StorageQuota        int64  `json:"storage_quota"`
		MiddlewareInstances int    `json:"middleware_instances"`
	}
	if err := c.ShouldBindJSON(&req); err != nil {
		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}
	ctx := c.Request.Context()
	var existing model.GlobalQuota
	if err := s.db.WithContext(ctx).Where("tenant = ? AND env = ?", tenant, env).First(&existing).Error; err != nil {
		c.JSON(http.StatusNotFound, gin.H{"error": "quota not found"})
		return
	}
	before, _ := json.Marshal(existing)

	existing.CPUQuota = req.CPUQuota
	existing.MemQuota = req.MemQuota
	existing.StorageQuota = req.StorageQuota
	existing.MiddlewareInstances = req.MiddlewareInstances
	if err := s.db.WithContext(ctx).Save(&existing).Error; err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}
	after, _ := json.Marshal(existing)

	// 经网关重下发 ResourceQuota 与 Namespace（消除"后台已满、集群可调度"漂移）
	client, err := s.k8sFor(req.Cluster, env)
	if err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": "dial gateway: " + err.Error()})
		return
	}
	if err := client.ReconcileNamespace(ctx, tenant, env); err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": "reconcile ns: " + err.Error()})
		return
	}
	if err := client.ReconcileResourceQuota(ctx, existing); err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": "reconcile rq: " + err.Error()})
		return
	}

	s.writeAudit(AuditContext{
		Ctx:        ctx,
		Actor:      strPtr(c.Get("actor")),
		Role:       strPtr(c.Get("role")),
		Tenant:     tenant,
		App:        "platform-admin",
		Env:        env,
		Cluster:    req.Cluster,
		Resource:   "global_quota",
		Action:     "update",
		Before:     string(before),
		After:      string(after),
		ApprovalID: c.GetHeader("X-Approval-Id"),
		Result:     "success",
		ClientIP:   c.ClientIP(),
	})
	c.JSON(http.StatusOK, existing)
}

// file: internal/admin/backup.go
package admin

import (
	"fmt"
	"time"

	batchv1 "k8s.io/api/batch/v1"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"github.com/gin-gonic/gin"
)

func int32Ptr(i int32) *int32 { return &i }

// TriggerBackup POST /api/v1/backup/trigger
// 经统一 K8s API 网关在目标集群 paas-system 创建一次性备份 Job。
func (s *Server) TriggerBackup(c *gin.Context) {
	var req struct {
		Tenant  string `json:"tenant"`
		Env     string `json:"env"`
		Cluster string `json:"cluster"`
	}
	if err := c.ShouldBindJSON(&req); err != nil {
		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}
	client, err := s.k8sFor(req.Cluster, req.Env)
	if err != nil {
		backupRuns.WithLabelValues(req.Tenant, req.Env, req.Cluster, "fail").Inc()
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}
	jobName := fmt.Sprintf("db-backup-%s-%s-%d", req.Tenant, req.Env, time.Now().Unix())
	job := &batchv1.Job{
		ObjectMeta: metav1.ObjectMeta{
			Name:      jobName,
			Namespace: "paas-system",
			Labels:    map[string]string{"tenant": req.Tenant, "env": req.Env, "paas.io/managed": "true"},
		},
		Spec: batchv1.JobSpec{
			BackoffLimit: int32Ptr(2),
			Template: corev1.PodTemplateSpec{
				Spec: corev1.PodSpec{
					RestartPolicy: corev1.RestartPolicyNever,
					Containers: []corev1.Container{{
						Name:  "backup",
						Image: "registry.paas.internal/pg-backup:1.4.0",
						Env: []corev1.EnvVar{
							{Name: "PGHOST", Value: "pg-admin.paas-system.svc.cluster.local"},
							{Name: "PGPASSWORD", ValueFrom: &corev1.EnvVarSource{
								SecretKeyRef: &corev1.SecretKeySelector{
									LocalObjectReference: corev1.LocalObjectReference{Name: "platform-admin-secret"},
									Key:                  "db-password",
								},
							}},
							{Name: "BACKUP_BUCKET", Value: "s3://paas-backup-prod/metadata"},
						},
						Command: []string{"/bin/sh", "-c",
							"ts=$(date +%Y%m%d-%H%M%S); pg_dump -U paas_admin paas_admin | gzip > /tmp/meta-$ts.sql.gz; " +
								"rclone copy /tmp/meta-$ts.sql.gz ${BACKUP_BUCKET}/ --s3-server-side-encryption aws:kms; echo backup $ts done"},
					}},
				},
			},
		},
	}
	if _, err := client.Clientset.BatchV1().Jobs("paas-system").Create(c.Request.Context(), job, metav1.CreateOptions{}); err != nil {
		backupRuns.WithLabelValues(req.Tenant, req.Env, req.Cluster, "fail").Inc()
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}
	backupRuns.WithLabelValues(req.Tenant, req.Env, req.Cluster, "success").Inc()
	c.JSON(http.StatusAccepted, gin.H{"job": jobName, "status": "triggered"})
}

// file: cmd/admin/main.go
package main

import (
	"fmt"
	"log"
	"net/http"
	"os"

	"github.com/gin-gonic/gin"
	"github.com/prometheus/client_golang/prometheus/promhttp"

	"github.com/example/paas/admin/internal/admin"
	"github.com/example/paas/admin/internal/store"
)

func atoiDefault(s string, def int) int {
	if s == "" {
		return def
	}
	var n int
	if _, err := fmt.Sscanf(s, "%d", &n); err != nil {
		return def
	}
	return n
}

func main() {
	dbCfg := store.Config{
		Host:     os.Getenv("DB_HOST"),
		Port:     atoiDefault(os.Getenv("DB_PORT"), 5432),
		User:     os.Getenv("DB_USER"),
		Password: os.Getenv("DB_PASSWORD"),
		DBName:   os.Getenv("DB_NAME"),
		SSLMode:  os.Getenv("DB_SSLMODE"),
	}
	db, err := store.NewPostgres(dbCfg)
	if err != nil {
		log.Fatalf("init postgres: %v", err)
	}

	srv := admin.NewServer(db, os.Getenv("GATEWAY_ADDR"), os.Getenv("GATEWAY_TOKEN"))

	r := gin.New()
	r.Use(gin.Recovery())
	r.Use(admin.MetricsMiddleware())

	r.GET("/healthz", func(c *gin.Context) { c.JSON(http.StatusOK, gin.H{"status": "ok"}) })
	r.GET("/livez", func(c *gin.Context) { c.JSON(http.StatusOK, gin.H{"status": "ok"}) })

	api := r.Group("/api/v1")
	api.Use(admin.RequireRole("platform-admin", "quota-approver", "audit-viewer"))
	{
		api.GET("/audit/logs", srv.QueryAuditLogs)
		api.GET("/quotas/:tenant/:env", srv.GetQuota)
		api.PUT("/quotas/:tenant/:env", admin.RequireRole("platform-admin", "quota-approver"), srv.UpdateQuota)
		api.POST("/backup/trigger", admin.RequireRole("platform-admin"), srv.TriggerBackup)
	}

	r.GET("/metrics", gin.WrapH(promhttp.Handler()))

	addr := os.Getenv("LISTEN_ADDR")
	if addr == "" {
		addr = ":8080"
	}
	log.Printf("platform-admin listening on %s", addr)
	if err := r.Run(addr); err != nil {
		log.Fatal(err)
	}
}

# ── 备份 / 恢复元数据中心 DB ──────────────────────────────
# 触发一次手动备份（经后台 API，走统一网关）
curl -X POST "https://admin.paas.internal/api/v1/backup/trigger" \
  -H "X-Admin-Role: platform-admin" -H "X-Admin-User: sre" \
  -d '{"tenant":"acme","env":"prod","cluster":"prod-cluster"}'

# 从异地对象存储恢复到新库（灾难演练）
kubectl --context=paas-gateway -n paas-system exec -it pg-admin-0 -- bash
# 容器内：rclone copy s3://paas-backup-prod/metadata/meta-20251015-020000.sql.gz /tmp/ && \
#        gunzip -c /tmp/meta-*.sql.gz | psql -U paas_admin paas_admin_restore

# ── 配额调整并核验审计 ────────────────────────────────────
curl -X PUT "https://admin.paas.internal/api/v1/quotas/acme/prod" \
  -H "X-Admin-Role: platform-admin" -H "X-Approval-Id: PROD-APR-8842" \
  -d '{"cluster":"prod-cluster","cpu_quota":120000,"mem_quota":262144,"storage_quota":500,"middleware_instances":8}'

# 在联邦 Grafana 后台大盘确认指标带 tenant/env/cluster 标签
# 检索审计是否可查（≥90天）
curl "https://admin.paas.internal/api/v1/audit/logs?tenant=acme&action=update&env=prod" \
  -H "X-Admin-Role: audit-viewer"

# ── 网关连接态巡检 ────────────────────────────────────────
kubectl --context=paas-gateway -n paas-system get deploy platform-admin -o wide
kubectl --context=paas-gateway -n paas-system logs deploy/platform-admin -l app=platform-admin --tail=100

# file: deploy/admin-prometheus.yaml
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: platform-admin
  namespace: paas-system
  labels:
    app: platform-admin
    release: kube-prometheus-stack
spec:
  selector:
    matchLabels:
      app: platform-admin
  endpoints:
    - port: metrics
      interval: 15s
      path: /metrics
---
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: platform-admin-rules
  namespace: paas-system
  labels:
    app: platform-admin
    release: kube-prometheus-stack
spec:
  groups:
    - name: platform-admin
      rules:
        - alert: AdminAPIHighErrorRate
          expr: |
            sum(rate(paas_admin_api_requests_total{result="fail"}[5m])) by (tenant,env,cluster)
            /
            sum(rate(paas_admin_api_requests_total[5m])) by (tenant,env,cluster) > 0.05            
          for: 10m
          labels:
            severity: warning
            env: "{{ $labels.env }}"
          annotations:
            summary: "后台API错误率超5% (tenant={{ $labels.tenant }} cluster={{ $labels.cluster }})"
        - alert: AdminBackupFailed
          expr: |
            increase(paas_admin_backup_runs_total{result="fail"}[1h]) > 0            
          for: 0m
          labels:
            severity: critical
          annotations:
            summary: "元数据中心备份失败，容灾 RPO 受影响"
        - alert: AdminGatewayUnreachable
          expr: |
            probe_success{job="platform-admin-gateway-health"} == 0            
          for: 5m
          labels:
            severity: critical
          annotations:
            summary: "统一K8s API网关连接失败，后台应进入只读降级"

# 重新下发 acme/prod 的 ResourceQuota（消除漂移）
curl -X PUT "https://admin.paas.internal/api/v1/quotas/acme/prod" \
  -H "X-Admin-Role: platform-admin" -H "X-Approval-Id: PROD-APR-8842" \
  -d '{"cluster":"prod-cluster","cpu_quota":120000,"mem_quota":262144,"storage_quota":500,"middleware_instances":8}'