中间件托管平台模块

paas-middleware/                           # 中间件托管平台源码（Go + gin + client-go）
├── go.mod
├── main.go                                # gin 入口 + /metrics（暴露 paas_middleware_* 指标）
├── pkg/
│   ├── gateway/
│   │   └── client.go                      # 统一 K8s API 网关多集群客户端
│   ├── middleware/
│   │   ├── instance.go                    # 经 dynamic client 创建 MiddlewareInstance（调 operator）
│   │   ├── backup.go                      # 触发备份 Job（PITR 基点）
│   │   └── status.go                      # 读取实例状态
│   ├── metrics/
│   │   └── metrics.go                     # prometheus 业务指标
│   └── handler/
│       └── middleware.go                  # gin 创建/备份/状态 Handler
├── manifests/                             # K8s IaC（kubectl apply 即用）
│   ├── mysql-instance.yaml               # 平台下发的中介件 CRD（MiddlewareInstance）
│   ├── mysql-statefulset.yaml            # MySQL StatefulSet + PVC + Service（group replication）
│   ├── redis-statefulset.yaml            # Redis Cluster StatefulSet + PVC + Service
│   ├── mysql-backup-cronjob.yaml         # 每日全量 + 增量备份 CronJob
│   ├── middleware-networkpolicy.yaml     # NetworkPolicy 严格隔离
│   └── rbac-prod.yaml                     # 生产实例 RBAC 约束
└── observability/                         # Prometheus / 联邦观测配置
    ├── mysql-servicemonitor.yaml         # MySQL/Redis ServiceMonitor
    ├── prometheus-mw-relabel.yaml        # 联邦抓取 + tenant/app/env/cluster 注入
    ├── mysql-recording-rules.yaml        # 慢查询/命中率/堆积 recording rules
    └── mysql-alert-rules.yaml            # 主库宕机/慢查询/热Key/堆积/备份失败告警

# file: manifests/mysql-instance.yaml
apiVersion: paas.example.com/v1
kind: MiddlewareInstance
metadata:
  name: mysql-order
  namespace: tenant-a-prod          # 租户生产 Namespace，强隔离
  labels:
    app: mysql-order
    tenant: tenant-a
    env: prod
    cluster: cluster-prod
spec:
  type: mysql
  version: "8.0"
  topology:
    mode: group-replication
    replicas: 3                      # 1主2从
  storage:
    class: ssd-encrypted             # ★ 加密存储类
    size: 200Gi
  backup:
    full: "0 2 * * *"                # 每日全量
    incremental: binlog              # 增量
    pitr: true                       # ★ 时间点恢复
    retention: 30d

// file: pkg/gateway/client.go
package gateway

import (
	"context"
	"fmt"

	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
)

// ClusterGateway 统一 K8s API 网关客户端管理器。
type ClusterGateway struct {
	clients     map[string]kubernetes.Interface
	gatewayREST *rest.Config
}

// NewClusterGateway 从网关 kubeconfig 初始化。
func NewClusterGateway(kubeconfigPath string) (*ClusterGateway, error) {
	cfg, err := clientcmd.BuildConfigFromFlags("", kubeconfigPath)
	if err != nil {
		return nil, fmt.Errorf("build gateway config: %w", err)
	}
	cfg.Impersonate = rest.ImpersonationConfig{
		UserName: "paas-platform",
		Groups:   []string{"paas:platform"},
	}
	clientset, err := kubernetes.NewForConfig(cfg)
	if err != nil {
		return nil, fmt.Errorf("new gateway clientset: %w", err)
	}
	return &ClusterGateway{
		clients:     map[string]kubernetes.Interface{"gateway": clientset},
		gatewayREST: cfg,
	}, nil
}

// RESTConfig 暴露底层 *rest.Config，供 dynamic client 复用。
func (g *ClusterGateway) RESTConfig() *rest.Config { return g.gatewayREST }

// ClientFor 返回指定集群（经网关纳管）的客户端。
func (g *ClusterGateway) ClientFor(ctx context.Context, clusterID string) (kubernetes.Interface, error) {
	if c, ok := g.clients[clusterID]; ok {
		return c, nil
	}
	cs, err := kubernetes.NewForConfig(g.gatewayREST)
	if err != nil {
		return nil, fmt.Errorf("client for cluster %s: %w", clusterID, err)
	}
	g.clients[clusterID] = cs
	return cs, nil
}

// file: pkg/middleware/instance.go
package middleware

import (
	"context"
	"fmt"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/client-go/dynamic"
)

var middlewareGVR = schema.GroupVersionResource{
	Group: "paas.example.com", Version: "v1", Resource: "middlewareinstances",
}

// MiddlewareClient 通过动态客户端操作中间件 CRD（由集群内 Operator 调谐）。
type MiddlewareClient struct {
	dyn dynamic.Interface
}

func NewMiddlewareClient(dyn dynamic.Interface) *MiddlewareClient {
	return &MiddlewareClient{dyn: dyn}
}

// CreateInstance 创建 MiddlewareInstance（client-go 调 operator 托管中间件的核心示例）。
func (m *MiddlewareClient) CreateInstance(ctx context.Context, ns string, inst *unstructured.Unstructured) error {
	_, err := m.dyn.Resource(middlewareGVR).Namespace(ns).Create(ctx, inst, metav1.CreateOptions{})
	if err != nil {
		return fmt.Errorf("create middlewareinstance: %w", err)
	}
	return nil
}

// BuildMySQLInstance 构造与 manifests/mysql-instance.yaml 完全一致的 MiddlewareInstance。
func BuildMySQLInstance(name, ns string, replicas int, size string) *unstructured.Unstructured {
	return &unstructured.Unstructured{Object: map[string]interface{}{
		"apiVersion": "paas.example.com/v1",
		"kind":       "MiddlewareInstance",
		"metadata": map[string]interface{}{
			"name":      name,
			"namespace": ns,
			"labels": map[string]interface{}{
				"app":     name,
				"tenant":  "tenant-a",
				"env":     "prod",
				"cluster": "cluster-prod",
			},
		},
		"spec": map[string]interface{}{
			"type":    "mysql",
			"version": "8.0",
			"topology": map[string]interface{}{
				"mode":     "group-replication",
				"replicas": int64(replicas),
			},
			"storage": map[string]interface{}{
				"class": "ssd-encrypted",
				"size":  size,
			},
			"backup": map[string]interface{}{
				"full":        "0 2 * * *",
				"incremental": "binlog",
				"pitr":        true,
				"retention":   "30d",
			},
		},
	}}
}

# file: observability/mysql-servicemonitor.yaml
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: mysql-order
  namespace: monitoring
  labels:
    tenant: tenant-a
    env: prod
    release: paas
spec:
  selector:
    matchLabels:
      app: mysql-order
  namespaceSelector:
    matchNames:
      - tenant-a-prod
  endpoints:
    - port: mysql
      path: /metrics
      interval: 15s
      metricRelabelings:
        - sourceLabels: [__name__]
          regex: "mysql_up|mysql_global_status_slow_queries|mysql_global_status_threads_connected|mysql_global_status_max_connections"
          action: keep

# file: observability/prometheus-mw-relabel.yaml
# 中心 Prometheus 抓取配置片段：把 Namespace/Pod 标签注入为 tenant/app/env/cluster 维度
scrape_configs:
  - job_name: paas-middleware
    kubernetes_sd_configs:
      - role: endpoints
        namespaces:
          names:
            - tenant-a-prod
            - tenant-a-test
    relabel_configs:
      - source_labels: [__meta_kubernetes_pod_label_tenant]
        target_label: tenant
      - source_labels: [__meta_kubernetes_pod_label_app]
        target_label: app
      - source_labels: [__meta_kubernetes_pod_label_env]
        target_label: env
      - source_labels: [__meta_kubernetes_pod_label_cluster]
        target_label: cluster
      - source_labels: [__meta_kubernetes_namespace]
        target_label: namespace
      - source_labels: [__address__]
        target_label: instance
  # 平台控制面暴露的 paas_middleware_* 业务指标（已自带 tenant/app/env/cluster 标签）
  - job_name: paas-middleware-platform
    static_configs:
      - targets: ["paas-middleware.platform.svc:8081"]
    relabel_configs:
      - source_labels: [__address__]
        target_label: instance

// file: pkg/middleware/backup.go
package middleware

import (
	"context"
	"fmt"

	batchv1 "k8s.io/api/batch/v1"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
)

// BackupClient 触发一次性备份 Job。
type BackupClient struct {
	cs kubernetes.Interface
}

func NewBackupClient(cs kubernetes.Interface) *BackupClient {
	return &BackupClient{cs: cs}
}

// TriggerBackup 立即触发一次备份 Job（等效 cron 之外的手动 PITR 基点）。
func (b *BackupClient) TriggerBackup(ctx context.Context, ns, instance, img string) (string, error) {
	jobName := fmt.Sprintf("%s-backup-%d", instance, metav1.Now().Unix())
	job := &batchv1.Job{
		ObjectMeta: metav1.ObjectMeta{
			Name:      jobName,
			Namespace: ns,
			Labels: map[string]string{
				"app": instance, "paas.example.com/type": "backup",
			},
		},
		Spec: batchv1.JobSpec{
			Template: corev1.PodTemplateSpec{
				ObjectMeta: metav1.ObjectMeta{
					Labels: map[string]string{
						"app": instance, "paas.example.com/type": "backup",
					},
				},
				Spec: corev1.PodSpec{
					RestartPolicy: corev1.RestartPolicyOnFailure,
					Containers: []corev1.Container{{
						Name:  "backup",
						Image: img,
						Args:  []string{"backup", "--instance", instance, "--pitr"},
					}},
				},
			},
		},
	}
	if _, err := b.cs.BatchV1().Jobs(ns).Create(ctx, job, metav1.CreateOptions{}); err != nil {
		return "", fmt.Errorf("trigger backup: %w", err)
	}
	return jobName, nil
}

// file: pkg/middleware/status.go
package middleware

import (
	"context"
	"fmt"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/client-go/dynamic"
)

// Status 读取 MiddlewareInstance 的 phase 与就绪副本数（Go 调用 K8s 读取状态）。
func (m *MiddlewareClient) Status(ctx context.Context, ns, name string) (phase string, replicas int64, err error) {
	obj, err := m.dyn.Resource(middlewareGVR).Namespace(ns).Get(ctx, name, metav1.GetOptions{})
	if err != nil {
		return "", 0, fmt.Errorf("get instance: %w", err)
	}
	status, found, _ := unstructured.NestedMap(obj.Object, "status")
	if !found {
		return "Unknown", 0, nil
	}
	phase, _, _ = unstructured.NestedString(status, "phase")
	replicas, _, _ = unstructured.NestedInt64(status, "readyReplicas")
	return phase, replicas, nil
}

// file: pkg/metrics/metrics.go
package metrics

import "github.com/prometheus/client_golang/prometheus"

var (
	InstanceProvisionTotal = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "paas_middleware_instance_total",
			Help: "Total middleware instances provisioned",
		},
		[]string{"tenant", "app", "env", "cluster", "type", "result"},
	)
	InstanceReady = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Name: "paas_middleware_instance_ready",
			Help: "1 if instance ready",
		},
		[]string{"tenant", "app", "env", "cluster", "type"},
	)
	BackupLastSuccess = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Name: "middleware_backup_last_success_timestamp",
			Help: "Timestamp of last successful backup",
		},
		[]string{"tenant", "app", "env", "cluster"},
	)
	BackupDuration = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Name:    "middleware_backup_duration_seconds",
			Help:    "Backup duration seconds",
			Buckets: prometheus.DefBuckets,
		},
		[]string{"tenant", "app", "env", "cluster"},
	)
)

func init() {
	prometheus.MustRegister(InstanceProvisionTotal, InstanceReady, BackupLastSuccess, BackupDuration)
}

// file: pkg/handler/middleware.go
package handler

import (
	"net/http"
	"time"

	"github.com/gin-gonic/gin"
	"k8s.io/client-go/dynamic"

	"github.com/example/paas-middleware/pkg/gateway"
	"github.com/example/paas-middleware/pkg/metrics"
	"github.com/example/paas-middleware/pkg/middleware"
)

// CreateInstanceRequest 创建实例请求体。
type CreateInstanceRequest struct {
	Tenant   string `json:"tenant" binding:"required"`
	App      string `json:"app" binding:"required"`
	Env      string `json:"env" binding:"required"`
	Cluster  string `json:"cluster" binding:"required"`
	Type     string `json:"type" binding:"required"` // mysql | redis | kafka
	Replicas int    `json:"replicas"`
	Size     string `json:"size"`
}

// CreateInstanceHandler 经 API 网关创建中间件实例（调 operator）。
func CreateInstanceHandler(gw *gateway.ClusterGateway, dyn dynamic.Interface) gin.HandlerFunc {
	return func(c *gin.Context) {
		var req CreateInstanceRequest
		if err := c.ShouldBindJSON(&req); err != nil {
			c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
			return
		}
		ctx := c.Request.Context()
		if _, err := gw.ClientFor(ctx, req.Cluster); err != nil {
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
			return
		}
		ns := req.Tenant + "-" + req.Env
		mw := middleware.NewMiddlewareClient(dyn)
		inst := middleware.BuildMySQLInstance(req.App, ns, req.Replicas, req.Size)
		if err := mw.CreateInstance(ctx, ns, inst); err != nil {
			metrics.InstanceProvisionTotal.WithLabelValues(req.Tenant, req.App, req.Env, req.Cluster, req.Type, "failed").Inc()
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
			return
		}
		metrics.InstanceProvisionTotal.WithLabelValues(req.Tenant, req.App, req.Env, req.Cluster, req.Type, "success").Inc()
		c.JSON(http.StatusOK, gin.H{"status": "provisioning", "namespace": ns})
	}
}

// BackupHandler 触发备份（Go 调用 K8s 创建 Job）。
func BackupHandler(gw *gateway.ClusterGateway) gin.HandlerFunc {
	return func(c *gin.Context) {
		var req struct {
			Tenant  string `json:"tenant" binding:"required"`
			App     string `json:"app" binding:"required"`
			Env     string `json:"env" binding:"required"`
			Cluster string `json:"cluster" binding:"required"`
			Image   string `json:"image"`
		}
		if err := c.ShouldBindJSON(&req); err != nil {
			c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
			return
		}
		ctx := c.Request.Context()
		cs, err := gw.ClientFor(ctx, req.Cluster)
		if err != nil {
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
			return
		}
		ns := req.Tenant + "-" + req.Env
		img := req.Image
		if img == "" {
			img = "harbor.example.com/paas-mysql-backup:latest"
		}
		bc := middleware.NewBackupClient(cs)
		job, err := bc.TriggerBackup(ctx, ns, req.App, img)
		if err != nil {
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
			return
		}
		metrics.BackupLastSuccess.WithLabelValues(req.Tenant, req.App, req.Env, req.Cluster).SetToCurrentTime()
		c.JSON(http.StatusOK, gin.H{"job": job, "status": "triggered"})
	}
}

// StatusHandler 读取实例状态（Go 调用 K8s 读取状态）。
func StatusHandler(gw *gateway.ClusterGateway, dyn dynamic.Interface) gin.HandlerFunc {
	return func(c *gin.Context) {
		tenant := c.Param("tenant")
		env := c.Param("env")
		app := c.Param("app")
		cluster := c.Query("cluster")
		if cluster == "" {
			cluster = "cluster-prod"
		}
		ctx := c.Request.Context()
		if _, err := gw.ClientFor(ctx, cluster); err != nil {
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
			return
		}
		mw := middleware.NewMiddlewareClient(dyn)
		phase, ready, err := mw.Status(ctx, tenant+"-"+env, app)
		if err != nil {
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
			return
		}
		readyVal := 0.0
		if phase == "Ready" {
			readyVal = 1
		}
		metrics.InstanceReady.WithLabelValues(tenant, app, env, cluster, "mysql").Set(readyVal)
		c.JSON(http.StatusOK, gin.H{"phase": phase, "readyReplicas": ready})
	}
}

var _ = time.Now

// file: go.mod
module github.com/example/paas-middleware

go 1.22

require (
	github.com/gin-gonic/gin v1.10.0
	github.com/prometheus/client_golang v1.19.0
	k8s.io/apimachinery v0.30.0
	k8s.io/client-go v0.30.0
)

// file: main.go
package main

import (
	"log"

	"github.com/gin-gonic/gin"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"k8s.io/client-go/dynamic"

	"github.com/example/paas-middleware/pkg/gateway"
	"github.com/example/paas-middleware/pkg/handler"
)

func main() {
	gw, err := gateway.NewClusterGateway("/etc/paas/gateway.kubeconfig")
	if err != nil {
		log.Fatalf("init gateway: %v", err)
	}
	dyn, err := dynamic.NewForConfig(gw.RESTConfig())
	if err != nil {
		log.Fatalf("init dynamic: %v", err)
	}

	r := gin.Default()
	r.POST("/api/v1/middleware", handler.CreateInstanceHandler(gw, dyn))
	r.POST("/api/v1/middleware/backup", handler.BackupHandler(gw))
	r.GET("/api/v1/middleware/:tenant/:env/:app", handler.StatusHandler(gw, dyn))
	r.GET("/metrics", gin.WrapH(promhttp.Handler()))
	if err := r.Run(":8081"); err != nil {
		log.Fatalf("server: %v", err)
	}
}

# file: manifests/mysql-statefulset.yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: mysql-order
  namespace: tenant-a-prod
  labels:
    app: mysql-order
    tenant: tenant-a
    env: prod
    cluster: cluster-prod
spec:
  serviceName: mysql-order-headless
  replicas: 3
  selector:
    matchLabels:
      app: mysql-order
  template:
    metadata:
      labels:
        app: mysql-order
        tenant: tenant-a
        env: prod
        cluster: cluster-prod
    spec:
      containers:
        - name: mysql
          image: harbor.example.com/mysql:8.0
          ports:
            - containerPort: 3306
              name: mysql
          env:
            - name: MYSQL_ROOT_PASSWORD
              valueFrom:
                secretKeyRef:
                  name: mysql-order-secret
                  key: ROOT_PASSWORD
          volumeMounts:
            - name: data
              mountPath: /var/lib/mysql
          resources:
            requests:
              cpu: "1"
              memory: 2Gi
            limits:
              cpu: "2"
              memory: 4Gi
  volumeClaimTemplates:
    - metadata:
        name: data
        labels:
          app: mysql-order
          tenant: tenant-a
          env: prod
      spec:
        accessModes: ["ReadWriteOnce"]
        storageClassName: ssd-encrypted   # ★ KMS 信封加密存储类
        resources:
          requests:
            storage: 200Gi
---
apiVersion: v1
kind: Service
metadata:
  name: mysql-order
  namespace: tenant-a-prod
  labels:
    app: mysql-order
    tenant: tenant-a
    env: prod
spec:
  selector:
    app: mysql-order
  ports:
    - name: mysql
      port: 3306
      targetPort: 3306
---
apiVersion: v1
kind: Service
metadata:
  name: mysql-order-headless
  namespace: tenant-a-prod
  labels:
    app: mysql-order
    tenant: tenant-a
    env: prod
spec:
  clusterIP: None
  selector:
    app: mysql-order
  ports:
    - name: mysql
      port: 3306
      targetPort: 3306

# file: manifests/redis-statefulset.yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: redis-cache
  namespace: tenant-a-prod
  labels:
    app: redis-cache
    tenant: tenant-a
    env: prod
    cluster: cluster-prod
spec:
  serviceName: redis-cache-headless
  replicas: 6
  selector:
    matchLabels:
      app: redis-cache
  template:
    metadata:
      labels:
        app: redis-cache
        tenant: tenant-a
        env: prod
        cluster: cluster-prod
    spec:
      containers:
        - name: redis
          image: harbor.example.com/redis:7.2
          ports:
            - containerPort: 6379
              name: redis
          volumeMounts:
            - name: data
              mountPath: /data
          resources:
            requests:
              cpu: "500m"
              memory: 1Gi
            limits:
              cpu: "1"
              memory: 2Gi
  volumeClaimTemplates:
    - metadata:
        name: data
        labels:
          app: redis-cache
          tenant: tenant-a
          env: prod
      spec:
        accessModes: ["ReadWriteOnce"]
        storageClassName: ssd-encrypted
        resources:
          requests:
            storage: 20Gi
---
apiVersion: v1
kind: Service
metadata:
  name: redis-cache
  namespace: tenant-a-prod
  labels:
    app: redis-cache
    tenant: tenant-a
    env: prod
spec:
  selector:
    app: redis-cache
  ports:
    - name: redis
      port: 6379
      targetPort: 6379
---
apiVersion: v1
kind: Service
metadata:
  name: redis-cache-headless
  namespace: tenant-a-prod
  labels:
    app: redis-cache
    tenant: tenant-a
    env: prod
spec:
  clusterIP: None
  selector:
    app: redis-cache
  ports:
    - name: redis
      port: 6379
      targetPort: 6379

# file: manifests/mysql-backup-cronjob.yaml
apiVersion: batch/v1
kind: CronJob
metadata:
  name: mysql-order-backup
  namespace: tenant-a-prod
  labels:
    app: mysql-order
    tenant: tenant-a
    env: prod
    cluster: cluster-prod
spec:
  schedule: "0 2 * * *"          # 每日全量
  concurrencyPolicy: Forbid
  jobTemplate:
    spec:
      template:
        metadata:
          labels:
            app: mysql-order
            paas.example.com/type: backup
        spec:
          restartPolicy: OnFailure
          containers:
            - name: backup
              image: harbor.example.com/paas-mysql-backup:latest
              args: ["backup", "--instance", "mysql-order", "--pitr", "--upload", "s3://paas-backup/tenant-a/"]
              env:
                - name: S3_ENDPOINT
                  value: "https://obs.example.com"
                - name: BACKUP_PASSWORD
                  valueFrom:
                    secretKeyRef:
                      name: mysql-order-secret
                      key: BACKUP_PASSWORD

# file: manifests/middleware-networkpolicy.yaml
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: mysql-order-netpol
  namespace: tenant-a-prod
  labels:
    app: mysql-order
    tenant: tenant-a
    env: prod
spec:
  podSelector:
    matchLabels:
      app: mysql-order
  policyTypes:
    - Ingress
    - Egress
  ingress:
    # 仅允许同租户 order-service 访问 3306
    - from:
        - podSelector:
            matchLabels:
              app: order-service
          namespaceSelector:
            matchLabels:
              tenant: tenant-a
      ports:
        - protocol: TCP
          port: 3306
  egress:
    # 允许访问备份对象存储与 DNS
    - to:
        - ipBlock:
            cidr: 10.20.0.0/16   # 备份存储网段
      ports:
        - protocol: TCP
          port: 443
    - to:
        - namespaceSelector: {}
      ports:
        - protocol: UDP
          port: 53

# 1) 经平台 API 申请生产 MySQL 主从实例（Go CreateInstanceHandler -> 下发 MiddlewareInstance CRD）
curl -XPOST http://paas-gateway/api/v1/middleware -H 'Content-Type: application/json' -d '{
  "tenant":"tenant-a","app":"mysql-order","env":"prod","cluster":"cluster-prod",
  "type":"mysql","replicas":3,"size":"200Gi"}'

# 2) 直接使用 IaC 落地（与平台下发等价）
kubectl apply -f manifests/mysql-instance.yaml
kubectl apply -f manifests/mysql-statefulset.yaml
kubectl apply -f manifests/mysql-backup-cronjob.yaml
kubectl apply -f manifests/middleware-networkpolicy.yaml

# 3) 查看实例状态（Pod/STS/PVC）
kubectl -n tenant-a-prod get middlewareinstance mysql-order
kubectl -n tenant-a-prod get sts,pods,pvc -l app=mysql-order

# 4) 手动触发一次 PITR 备份基点
kubectl -n tenant-a-prod create job mysql-order-backup-manual --from=cronjob/mysql-order-backup

# 5) 扩缩容（增加只读副本）
kubectl -n tenant-a-prod scale sts/mysql-order --replicas=5

# 6) 镜像安全（中间件基础镜像同样扫描）
trivy image --severity CRITICAL,HIGH --exit-code 1 harbor.example.com/mysql:8.0

# file: manifests/rbac-prod.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: tenant-a-prod-mw-ops
  namespace: tenant-a-prod
rules:
  - apiGroups: ["paas.example.com"]
    resources: ["middlewareinstances"]
    verbs: ["get", "list", "watch", "create", "update", "patch"]   # 仅运维可写
  - apiGroups: ["apps"]
    resources: ["statefulsets"]
    verbs: ["get", "list", "watch", "scale"]
  - apiGroups: ["batch"]
    resources: ["cronjobs", "jobs"]
    verbs: ["get", "list", "watch", "create"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: tenant-a-prod-mw-ops
  namespace: tenant-a-prod
subjects:
  - kind: User
    name: mw-ops@tenant-a
roleRef:
  kind: Role
  name: tenant-a-prod-mw-ops
  apiGroup: rbac.authorization.k8s.io

# file: observability/mysql-recording-rules.yaml
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: mysql-recording-rules
  namespace: monitoring
  labels:
    role: recording-rules
spec:
  groups:
    - name: middleware_slo
      interval: 30s
      rules:
        - record: redis_hit_rate
          expr: |
            sum(rate(redis_keyspace_hits_total[5m])) by (tenant,app,env,cluster)
            /
            (sum(rate(redis_keyspace_hits_total[5m])) by (tenant,app,env,cluster)
             + sum(rate(redis_keyspace_misses_total[5m])) by (tenant,app,env,cluster))            
        - record: mysql_slow_query_rate
          expr: rate(mysql_global_status_slow_queries[5m])
        - record: mysql_connection_ratio
          expr: mysql_global_status_threads_connected / mysql_global_status_max_connections
        - record: kafka_consumer_lag_sum
          expr: sum(kafka_consumer_group_lag) by (tenant,app,env,cluster)
        # 平台控制面指标（与 pkg/metrics/metrics.go 上报一致）
        - record: middleware_instance_available
          expr: paas_middleware_instance_ready

# file: observability/mysql-alert-rules.yaml
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: mysql-alert-rules
  namespace: monitoring
  labels:
    role: alert-rules
spec:
  groups:
    - name: middleware_alerts
      rules:
        - alert: MySQLDown
          expr: mysql_up == 0
          for: 2m
          labels:
            severity: critical
            tenant: "{{ $labels.tenant }}"
          annotations:
            summary: "MySQL 实例 {{ $labels.instance }} 不可达"
        - alert: MySQLSlowQuerySpike
          expr: mysql_slow_query_rate > 1
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "慢查询率升高"
        - alert: RedisHitRateLow
          expr: redis_hit_rate < 0.9
          for: 10m
          labels:
            severity: warning
          annotations:
            summary: "Redis 命中率低于 90%"
        - alert: KafkaConsumerLagHigh
          expr: kafka_consumer_lag_sum > 100000
          for: 10m
          labels:
            severity: warning
          annotations:
            summary: "Kafka 消费堆积超过 10 万"
        - alert: BackupFailed
          expr: (time() - middleware_backup_last_success_timestamp) > 86400
          for: 5m
          labels:
            severity: critical
          annotations:
            summary: "实例 {{ $labels.app }} 超过 24h 无成功备份"
        - alert: BackupJobFailed
          expr: kube_job_status_failed{job_name=~".*backup.*"} == 1
          for: 5m
          labels:
            severity: critical
          annotations:
            summary: "备份 Job 执行失败"

# 主库故障自动选主验证：删主库 Pod，观察副本顶替
kubectl -n tenant-a-prod delete pod mysql-order-0
kubectl -n tenant-a-prod get pods -l app=mysql-order -w

# 手动触发一次备份（PITR 基点）
kubectl -n tenant-a-prod create job mysql-order-backup-manual --from=cronjob/mysql-order-backup

# PITR 恢复到指定时间点（先恢复到临时实例校验，再切流）
kubectl -n tenant-a-prod run mysql-restore --image=harbor.example.com/paas-mysql-backup \
  --restart=Never -- backup --pitr --target-time="2025-10-15T02:00:00+08:00" \
  --src=s3://paas-backup/tenant-a/mysql-order/ --dst=/restore
kubectl -n tenant-a-prod exec mysql-restore -- mysql -h mysql-order < /restore/export.sql

# 确认备份成功时间戳指标（配合 BackupFailed 告警）
kubectl -n tenant-a-prod get pods -l paas.example.com/type=backup

# redis-benchmark 制造热 Key，验证联邦大盘热 Key 指标 + Alertmanager 告警
redis-benchmark -h redis-cache.tenant-a-prod.svc -t set -n 100000 -r 10