多K8s集群纳管模块

# file: deploy/crd-cluster.yaml
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
  name: clusters.paas.io
spec:
  group: paas.io
  names:
    kind: Cluster
    plural: clusters
    singular: cluster
  scope: Cluster
  versions:
    - name: v1
      served: true
      storage: true
      schema:
        openAPIV3Schema:
          type: object
          required: ["spec"]
          properties:
            spec:
              type: object
              required: ["env", "endpoint"]
              properties:
                env:
                  type: string
                  description: "production / staging / test / dev"
                endpoint:
                  type: string
                  description: "apiserver 地址，仅网关可达，不直接对外暴露"
                mode:
                  type: string
                  enum: ["reverse-tunnel", "center-proxy"]
                kubeconfigRef:
                  type: string
                  description: "KMS 信封加密后的 kubeconfig Secret 名"
            status:
              type: object
              properties:
                phase:
                  type: string
                  enum: ["Connected", "Degraded", "Evicted"]
      subresources:
        status: {}

// file: platform/k8s/registry.go
package k8s

import (
	"fmt"
	"sync"

	"k8s.io/client-go/dynamic"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
)

// ClusterEntry 网关侧保存的单集群连接（来自注册时 KMS 解密到内存的 kubeconfig）。
type ClusterEntry struct {
	ClusterID string
	Env       string
	Endpoint  string
	Config    *rest.Config
	Client    kubernetes.Interface
	Dyn       dynamic.Interface
	Phase     string // Connected / Degraded / Evicted
}

type ClusterRegistry struct {
	mu       sync.RWMutex
	clusters map[string]*ClusterEntry
}

func NewClusterRegistry() *ClusterRegistry {
	return &ClusterRegistry{clusters: make(map[string]*ClusterEntry)}
}

// LoadGatewayConfig 加载网关自身（中心控制面）rest.Config。
func LoadGatewayConfig(path string) (*rest.Config, error) {
	if path == "" {
		return rest.InClusterConfig()
	}
	return clientcmd.BuildConfigFromFlags("", path)
}

// RegisterCluster 用「KMS 信封解密后的 kubeconfig 字节」构建 rest.Config 并接入网关连接池。
// 明文 kubeconfig 不落盘：调用方从 Secret/内存解密后传入，函数返回后即被 GC。
func (r *ClusterRegistry) RegisterCluster(clusterID, env, endpoint string, kubeconfigBytes []byte) error {
	cfg, err := clientcmd.RESTConfigFromKubeConfig(kubeconfigBytes)
	if err != nil {
		return fmt.Errorf("build rest.Config from decrypted kubeconfig: %w", err)
	}
	cli, err := kubernetes.NewForConfig(cfg)
	if err != nil {
		return fmt.Errorf("new clientset: %w", err)
	}
	dyn, err := dynamic.NewForConfig(cfg)
	if err != nil {
		return fmt.Errorf("new dynamic client: %w", err)
	}
	r.mu.Lock()
	defer r.mu.Unlock()
	r.clusters[clusterID] = &ClusterEntry{
		ClusterID: clusterID, Env: env, Endpoint: endpoint,
		Config: cfg, Client: cli, Dyn: dyn, Phase: "Connected",
	}
	return nil
}

// ClientForCluster 网关按集群 ID 路由（反向隧道模式下实际转发到内存解密的 apiserver 连接）。
func (r *ClusterRegistry) ClientForCluster(id string) (*ClusterEntry, error) {
	r.mu.RLock()
	defer r.mu.RUnlock()
	c, ok := r.clusters[id]
	if !ok {
		return nil, fmt.Errorf("cluster %s not registered", id)
	}
	return c, nil
}

func (r *ClusterRegistry) SetPhase(id, phase string) {
	r.mu.Lock()
	defer r.mu.Unlock()
	if c, ok := r.clusters[id]; ok {
		c.Phase = phase
	}
}

// IDs 返回当前已纳管的所有集群 ID（健康巡检循环遍历用）。
func (r *ClusterRegistry) IDs() []string {
	r.mu.RLock()
	defer r.mu.RUnlock()
	ids := make([]string, 0, len(r.clusters))
	for id := range r.clusters {
		ids = append(ids, id)
	}
	return ids
}

// file: platform/k8s/kms.go
package k8s

import (
	"crypto/aes"
	"crypto/cipher"
	"crypto/rand"
	"encoding/base64"
	"errors"
	"io"
)

// MockKMS 模拟云 KMS：真实场景调用 AWS KMS / 腾讯云 KMS 的 Encrypt/Decrypt。
// 这里用固定主密钥演示「数据密钥 DEK 由 KMS 加密为密文，DEK 明文仅内存使用」。
type MockKMS struct {
	masterKey []byte
}

func NewMockKMS(masterKey string) *MockKMS {
	return &MockKMS{masterKey: []byte(masterKey)}
}

func splitDot(s string) []string {
	for i := 0; i < len(s); i++ {
		if s[i] == '.' {
			return []string{s[:i], s[i+1:]}
		}
	}
	return []string{s}
}

// EnvelopeEncrypt 信封加密：生成随机 DEK，用 DEK 加密明文，再用主密钥加密 DEK。
// 返回格式：base64(主IV+encDEK).base64(数据IV+数据密文)
func (k *MockKMS) EnvelopeEncrypt(plaintext []byte) (string, error) {
	dek := make([]byte, 32)
	if _, err := io.ReadFull(rand.Reader, dek); err != nil {
		return "", err
	}
	block, err := aes.NewCipher(dek)
	if err != nil {
		return "", err
	}
	gcm, err := cipher.NewGCM(block)
	if err != nil {
		return "", err
	}
	iv := make([]byte, gcm.NonceSize())
	if _, err := io.ReadFull(rand.Reader, iv); err != nil {
		return "", err
	}
	ct := gcm.Seal(nil, iv, plaintext, nil)

	mblock, err := aes.NewCipher(k.masterKey)
	if err != nil {
		return "", err
	}
	mgcm, err := cipher.NewGCM(mblock)
	if err != nil {
		return "", err
	}
	miv := make([]byte, mgcm.NonceSize())
	if _, err := io.ReadFull(rand.Reader, miv); err != nil {
		return "", err
	}
	encDEK := mgcm.Seal(nil, miv, dek, nil)

	outer := append(miv, encDEK...)
	inner := append(iv, ct...)
	return base64.StdEncoding.EncodeToString(outer) + "." + base64.StdEncoding.EncodeToString(inner), nil
}

// EnvelopeDecrypt 解密：主密钥解出 DEK，DEK 解出明文；DEK 明文仅函数内使用，用完即焚。
func (k *MockKMS) EnvelopeDecrypt(payload string) ([]byte, error) {
	parts := splitDot(payload)
	if len(parts) != 2 {
		return nil, errors.New("bad envelope payload")
	}
	outer, err := base64.StdEncoding.DecodeString(parts[0])
	if err != nil {
		return nil, err
	}
	inner, err := base64.StdEncoding.DecodeString(parts[1])
	if err != nil {
		return nil, err
	}
	mblock, err := aes.NewCipher(k.masterKey)
	if err != nil {
		return nil, err
	}
	mgcm, err := cipher.NewGCM(mblock)
	if err != nil {
		return nil, err
	}
	ns := mgcm.NonceSize()
	if len(outer) < ns {
		return nil, errors.New("short outer")
	}
	miv, encDEK := outer[:ns], outer[ns:]
	dek, err := mgcm.Open(nil, miv, encDEK, nil)
	if err != nil {
		return nil, err
	}
	block, err := aes.NewCipher(dek)
	if err != nil {
		return nil, err
	}
	gcm, err := cipher.NewGCM(block)
	if err != nil {
		return nil, err
	}
	if len(inner) < gcm.NonceSize() {
		return nil, errors.New("short inner")
	}
	iv, ct := inner[:gcm.NonceSize()], inner[gcm.NonceSize():]
	return gcm.Open(nil, iv, ct, nil)
}

// file: platform/k8s/cluster.go
package k8s

import (
	"context"
	"fmt"

	appsv1 "k8s.io/api/apps/v1"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/client-go/dynamic"
	"k8s.io/client-go/kubernetes"
)

var clusterGVR = schema.GroupVersionResource{Group: "paas.io", Version: "v1", Resource: "clusters"}

// RegisterCluster 完整纳管：加密托管 kubeconfig -> 建 Cluster CR -> 部署 Agent -> 接入连接池。
func RegisterCluster(ctx context.Context, reg *ClusterRegistry, kms *MockKMS,
	gwCli kubernetes.Interface, gwDyn dynamic.Interface,
	clusterID, env, endpoint, mode string, kubeconfigPlain []byte) error {

	// 1) KMS 信封加密 kubeconfig，仅密文落 Secret（明文不落盘）
	cipherText, err := kms.EnvelopeEncrypt(kubeconfigPlain)
	if err != nil {
		return fmt.Errorf("envelope encrypt kubeconfig: %w", err)
	}
	sec := &corev1.Secret{
		ObjectMeta: metav1.ObjectMeta{
			Name: "kubeconfig-" + clusterID, Namespace: "paas-system",
			Labels: map[string]string{"paas.io/cluster": clusterID},
		},
		StringData: map[string]string{"kubeconfig": cipherText},
	}
	if _, err := gwCli.CoreV1().Secrets("paas-system").Create(ctx, sec, metav1.CreateOptions{}); err != nil {
		return fmt.Errorf("store encrypted kubeconfig: %w", err)
	}

	// 2) 在网关侧建 Cluster CR（DynamicClient）
	c := &unstructured.Unstructured{Object: map[string]interface{}{
		"apiVersion": "paas.io/v1", "kind": "Cluster",
		"metadata": map[string]interface{}{"name": clusterID, "labels": map[string]interface{}{"env": env}},
		"spec": map[string]interface{}{
			"env": env, "endpoint": endpoint, "mode": mode,
			"kubeconfigRef": "kubeconfig-" + clusterID,
		},
		"status": map[string]interface{}{"phase": "Connected"},
	}}
	if _, err := gwDyn.Resource(clusterGVR).Create(ctx, c, metav1.CreateOptions{}); err != nil {
		return fmt.Errorf("create Cluster CR: %w", err)
	}

	// 3) 解密后的 kubeconfig 接入网关连接池（明文仅内存）
	if err := reg.RegisterCluster(clusterID, env, endpoint, kubeconfigPlain); err != nil {
		return err
	}

	// 4) 在该集群部署 Prometheus Agent（用目标集群 client）
	entry, err := reg.ClientForCluster(clusterID)
	if err != nil {
		return err
	}
	if err := DeployPrometheusAgent(ctx, entry.Client, clusterID, env); err != nil {
		return err
	}
	IncClusterRegister(env, mode)
	return nil
}

// DeployPrometheusAgent 在目标集群部署 Agent（Agent 模式只 remote_write，本地不长周期存）。
func DeployPrometheusAgent(ctx context.Context, cli kubernetes.Interface, clusterID, env string) error {
	cm := &corev1.ConfigMap{
		ObjectMeta: metav1.ObjectMeta{
			Name: "prom-agent-" + clusterID, Namespace: "paas-monitoring",
			Labels: map[string]string{"paas.io/cluster": clusterID},
		},
		Data: map[string]string{
			"prometheus.yml": fmt.Sprintf(`global:
  scrape_interval: 30s
  external_labels:
    cluster: %s
    env: %s
remote_write:
  - url: http://thanos-receiver.paas-monitoring.svc:19291/api/v1/receive
scrape_configs:
  - job_name: kube-pods
    kubernetes_sd_configs:
      - role: pod
    relabel_configs:
      - source_labels: [__meta_kubernetes_namespace]
        target_label: tenant
      - source_labels: [__meta_kubernetes_pod_label_app]
        target_label: app
`, clusterID, env),
		},
	}
	if _, err := cli.CoreV1().ConfigMaps("paas-monitoring").Create(ctx, cm, metav1.CreateOptions{}); err != nil {
		return err
	}
	replicas := int32(1)
	dep := &appsv1.Deployment{
		ObjectMeta: metav1.ObjectMeta{
			Name: "prom-agent-" + clusterID, Namespace: "paas-monitoring",
			Labels: map[string]string{"paas.io/cluster": clusterID},
		},
		Spec: appsv1.DeploymentSpec{
			Replicas: &replicas,
			Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "prom-agent", "paas.io/cluster": clusterID}},
			Template: corev1.PodTemplateSpec{
				ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"app": "prom-agent", "paas.io/cluster": clusterID}},
				Spec: corev1.PodSpec{
					Containers: []corev1.Container{{
						Name:  "agent",
						Image: "prom/prometheus:v2.51.0",
						Args:  []string{"--config.file=/etc/prometheus/prometheus.yml", "--enable-feature=agent"},
						VolumeMounts: []corev1.VolumeMount{
							{Name: "cfg", MountPath: "/etc/prometheus"},
						},
					}},
					Volumes: []corev1.Volume{{
						Name: "cfg",
						VolumeSource: corev1.VolumeSource{
							ConfigMap: &corev1.ConfigMapVolumeSource{
								LocalObjectReference: corev1.LocalObjectReference{Name: "prom-agent-" + clusterID},
							},
						},
					}},
				},
			},
		},
	}
	if _, err := cli.AppsV1().Deployments("paas-monitoring").Create(ctx, dep, metav1.CreateOptions{}); err != nil {
		return err
	}
	return nil
}

// file: platform/k8s/health.go
package k8s

import (
	"context"
	"fmt"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
)

// ProbeCluster 探活：apiserver 健康 + 节点 Ready 比例，返回 phase。
func ProbeCluster(ctx context.Context, cli kubernetes.Interface) (string, error) {
	nodes, err := cli.CoreV1().Nodes().List(ctx, metav1.ListOptions{Limit: 100})
	if err != nil {
		return "Degraded", fmt.Errorf("apiserver probe failed: %w", err)
	}
	ready := 0
	for _, n := range nodes.Items {
		for _, c := range n.Status.Conditions {
			if c.Type == "Ready" && c.Status == "True" {
				ready++
			}
		}
	}
	if ready == 0 && len(nodes.Items) > 0 {
		return "Degraded", nil
	}
	return "Connected", nil
}

// EvictIfUnhealthy 连续不可用达阈值则标记为已摘除并隔离流量；否则标记 Degraded。
func EvictIfUnhealthy(reg *ClusterRegistry, id string, failCount int) (evicted bool) {
	if failCount >= 3 {
		reg.SetPhase(id, "Evicted")
		return true
	}
	reg.SetPhase(id, "Degraded")
	return false
}

// file: platform/k8s/metrics.go
package k8s

import (
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promauto"
)

var (
	clusterHealthGauge = promauto.NewGaugeVec(
		prometheus.GaugeOpts{
			Name: "paas_cluster_health",
			Help: "集群健康状态：1=Connected 0=Degraded -1=Evicted",
		},
		[]string{"cluster", "env"},
	)
	agentUpGauge = promauto.NewGaugeVec(
		prometheus.GaugeOpts{
			Name: "paas_cluster_agent_up",
			Help: "Prometheus Agent 是否在线",
		},
		[]string{"cluster", "env"},
	)
	clusterRegisterTotal = promauto.NewCounterVec(
		prometheus.CounterOpts{
			Name: "paas_cluster_register_total",
			Help: "集群纳管总数",
		},
		[]string{"env", "mode"},
	)
)

func SetClusterHealth(cluster, env, phase string) {
	v := 1.0
	if phase == "Degraded" {
		v = 0
	} else if phase == "Evicted" {
		v = -1
	}
	clusterHealthGauge.WithLabelValues(cluster, env).Set(v)
}
func SetAgentUp(cluster, env string, up bool) {
	if up {
		agentUpGauge.WithLabelValues(cluster, env).Set(1)
	} else {
		agentUpGauge.WithLabelValues(cluster, env).Set(0)
	}
}
func IncClusterRegister(env, mode string) {
	clusterRegisterTotal.WithLabelValues(env, mode).Inc()
}

// file: platform/k8s/audit.go
package k8s

import (
	"context"
	"encoding/json"
	"os"
	"sync"
	"time"
)

// AuditRecord 审计记录（含操作人/对象/动作/集群/时间/来源IP/审批单号）。
type AuditRecord struct {
	Timestamp time.Time `json:"ts"`
	Operator  string    `json:"operator"`
	Action    string    `json:"action"`
	Cluster   string    `json:"cluster"`
	Target    string    `json:"target"`
	SrcIP     string    `json:"src_ip"`
	Ticket    string    `json:"ticket"`
}

type AuditSink struct {
	mu   sync.Mutex
	path string
}

func NewAuditSink(path string) *AuditSink { return &AuditSink{path: path} }

func (a *AuditSink) Write(ctx context.Context, rec AuditRecord) error {
	rec.Timestamp = time.Now()
	b, err := json.Marshal(rec)
	if err != nil {
		return err
	}
	a.mu.Lock()
	defer a.mu.Unlock()
	f, err := os.OpenFile(a.path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
	if err != nil {
		return err
	}
	defer f.Close()
	_, err = f.Write(append(b, '\n'))
	return err
}

// file: platform/go.mod
module paas.example.com

go 1.22

require (
	github.com/gin-gonic/gin v1.9.1
	github.com/prometheus/client_golang v1.18.0
	k8s.io/api v0.29.0
	k8s.io/apimachinery v0.29.0
	k8s.io/client-go v0.29.0
)

// file: platform/main.go
package main

import (
	"context"
	"log"
	"net/http"
	"time"

	"github.com/gin-gonic/gin"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"k8s.io/client-go/dynamic"
	"k8s.io/client-go/kubernetes"

	"paas.example.com/platform/k8s"
)

func main() {
	reg := k8s.NewClusterRegistry()
	kms := k8s.NewMockKMS("0123456789abcdef0123456789abcdef") // 真实为 KMS 主密钥，永不落盘

	// 网关自身连接（中心控制面：存 Cluster CR / 加密 kubeconfig Secret / 部署 Agent）
	gwCfg, err := k8s.LoadGatewayConfig("/etc/paas/gateway-kubeconfig")
	if err != nil {
		log.Fatalf("load gateway config: %v", err)
	}
	gwCli, err := kubernetes.NewForConfig(gwCfg)
	if err != nil {
		log.Fatalf("gateway clientset: %v", err)
	}
	gwDyn, err := dynamic.NewForConfig(gwCfg)
	if err != nil {
		log.Fatalf("gateway dynamic client: %v", err)
	}
	audit := k8s.NewAuditSink("/var/log/paas/cluster-audit.jsonl")

	r := gin.Default()
	r.GET("/metrics", gin.WrapH(promhttp.Handler()))

	// 集群注册（平台管理员 / 集群运维）
	r.POST("/api/v1/clusters", func(c *gin.Context) {
		var req struct {
			ID         string `json:"id"`
			Env        string `json:"env"`
			Endpoint   string `json:"endpoint"`
			Mode       string `json:"mode"`
			Kubeconfig string `json:"kubeconfig"` // 明文仅请求内存中，立即 KMS 加密
		}
		if err := c.ShouldBindJSON(&req); err != nil {
			c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
			return
		}
		// 生产环境注册强制二级审批：header 携带审批单号
		if req.Env == "production" && c.GetHeader("X-Approval-Ticket") == "" {
			c.JSON(http.StatusForbidden, gin.H{"error": "production register requires approval ticket"})
			return
		}
		if err := k8s.RegisterCluster(c.Request.Context(), reg, kms, gwCli, gwDyn,
			req.ID, req.Env, req.Endpoint, req.Mode, []byte(req.Kubeconfig)); err != nil {
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
			return
		}
		_ = audit.Write(c.Request.Context(), k8s.AuditRecord{
			Operator: c.GetHeader("X-User"), Action: "register-cluster",
			Cluster: req.ID, Target: "cluster/" + req.ID,
			SrcIP: c.ClientIP(), Ticket: c.GetHeader("X-Approval-Ticket"),
		})
		c.JSON(http.StatusOK, gin.H{"status": "registered"})
	})

	// 网关多集群代理：按集群 ID 路由到对应 apiserver（反向隧道/中心代理）
	r.Any("/proxy/:cluster/*path", func(c *gin.Context) {
		id := c.Param("cluster")
		entry, err := reg.ClientForCluster(id)
		if err != nil {
			c.JSON(http.StatusNotFound, gin.H{"error": err.Error()})
			return
		}
		// 真实实现：用 entry.Config 构造 transport 转发 K8s API；此处回显路由命中结果
		c.JSON(http.StatusOK, gin.H{
			"cluster": entry.ClusterID, "endpoint": entry.Endpoint, "phase": entry.Phase,
		})
	})

	// 健康检查与故障摘除循环（SRE）：每 30s 探活，连续失败达阈值则摘除
	probeFails := map[string]int{}
	go func() {
		ticker := time.NewTicker(30 * time.Second)
		defer ticker.Stop()
		for range ticker.C {
			for _, id := range reg.IDs() {
				entry, err := reg.ClientForCluster(id)
				if err != nil {
					continue
				}
				phase, perr := k8s.ProbeCluster(context.Background(), entry.Client)
				if perr != nil {
					probeFails[id]++
					if k8s.EvictIfUnhealthy(reg, id, probeFails[id]) {
						phase = "Evicted"
					} else {
						phase = "Degraded"
					}
				} else {
					probeFails[id] = 0
				}
				k8s.SetClusterHealth(id, entry.Env, phase)
				if phase == "Degraded" || phase == "Evicted" {
					_ = audit.Write(context.Background(), k8s.AuditRecord{
						Operator: "health-probe", Action: "cluster-probe",
						Cluster: id, Target: "cluster/" + id, SrcIP: "gateway", Ticket: "AUTO",
					})
				}
			}
		}
	}()

	log.Println("multi-cluster gateway listening on :8080")
	if err := r.Run(":8080"); err != nil {
		log.Fatal(err)
	}
}

# file: deploy/gateway-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: k8s-api-gateway
  namespace: paas-system
  labels:
    app: k8s-api-gateway
spec:
  replicas: 2
  selector:
    matchLabels:
      app: k8s-api-gateway
  template:
    metadata:
      labels:
        app: k8s-api-gateway
    spec:
      containers:
      - name: gateway
        image: paas/k8s-api-gateway:v1.0.0
        args: ["--kubeconfig=/etc/paas/gateway-kubeconfig", "--listen=:8080"]
        ports:
        - containerPort: 8080
        volumeMounts:
        - name: kc
          mountPath: /etc/paas
          readOnly: true
      volumes:
      - name: kc
        secret:
          secretName: gateway-kubeconfig   # 由 KMS 信封加密托管，挂载即内存解密

# 步骤 1：准备最小权限 kubeconfig（禁止 cluster-admin）
# 步骤 2：注册，平台自动 KMS 信封加密托管，并部署 Prometheus Agent
curl -X POST https://paas-gateway/api/v1/clusters \
  -H "Content-Type: application/json" \
  -d '{
    "id": "test-cluster-01",
    "env": "test",
    "endpoint": "https://10.10.10.10:6443",
    "mode": "reverse-tunnel",
    "kubeconfig": "<base64-or-plaintext-only-in-memory>"
  }'

# 步骤 3：注册成功后，平台在集群内生成租户 Namespace 与配额
kubectl --context test-cluster-01 get ns subcompany-a --show-labels
kubectl --context test-cluster-01 get resourcequota -n subcompany-a

# file: deploy/tenant-quota.yaml
apiVersion: v1
kind: ResourceQuota
metadata:
  name: tenant-subcompany-a
  namespace: subcompany-a
  labels:
    tenant: subcompany-a
spec:
  hard:
    requests.cpu: "50"
    requests.memory: 200Gi
    pods: "100"

# file: deploy/networkpolicy.yaml
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: default-deny-cross-tenant
  namespace: subcompany-a
spec:
  podSelector: {}
  policyTypes:
  - Ingress
  - Egress
  # 仅允许同租户 namespace 内通信，跨租户/跨环境默认拒绝
  ingress:
  - from:
    - namespaceSelector:
        matchLabels:
          tenant: subcompany-a
  egress:
  - to:
    - namespaceSelector:
        matchLabels:
          tenant: subcompany-a

# 步骤 1：给集群打 env 标签（物理隔离域）
paas-cli cluster label --name prod-cluster-01 --set env=production

# 步骤 2：创建节点池并绑定租户配额
paas-cli nodepool create --cluster prod-cluster-01 --name np-gold --zone az1
paas-cli tenant bind-nodepool --tenant subcompany-a --nodepool np-gold

# file: observability/prometheus-agent.yaml
# 集群内 Prometheus Agent：采集本集群指标，强制注入 tenant/app/env/cluster 标签
apiVersion: v1
kind: ConfigMap
metadata:
  name: prom-agent-prod-cluster-01
  namespace: paas-monitoring
data:
  prometheus.yml: |
    global:
      scrape_interval: 30s
      external_labels:
        cluster: prod-cluster-01
        env: production
    remote_write:
      - url: http://thanos-receiver.paas-monitoring.svc:19291/api/v1/receive
    scrape_configs:
      - job_name: kube-pods
        kubernetes_sd_configs:
          - role: pod
        relabel_configs:
          - source_labels: [__meta_kubernetes_namespace]
            target_label: tenant
          - source_labels: [__meta_kubernetes_pod_label_app]
            target_label: app
          - target_label: env
            replacement: production
          - target_label: cluster
            replacement: prod-cluster-01    
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: prom-agent-prod-cluster-01
  namespace: paas-monitoring
  labels:
    app: prom-agent
    paas.io/cluster: prod-cluster-01
spec:
  replicas: 1
  selector:
    matchLabels:
      app: prom-agent
      paas.io/cluster: prod-cluster-01
  template:
    metadata:
      labels:
        app: prom-agent
        paas.io/cluster: prod-cluster-01
    spec:
      containers:
      - name: agent
        image: prom/prometheus:v2.51.0
        args: ["--config.file=/etc/prometheus/prometheus.yml", "--enable-feature=agent"]
        volumeMounts:
        - name: cfg
          mountPath: /etc/prometheus
      volumes:
      - name: cfg
        configMap:
          name: prom-agent-prod-cluster-01

# file: observability/prometheus-federation.yaml
# 中心 Prometheus 联邦拉取各集群 Agent，honor_labels 保留 tenant/env/cluster 源标签
global:
  scrape_interval: 30s
scrape_configs:
  - job_name: federate-prod
    honor_labels: true
    metrics_path: /federate
    params:
      match[]:
        - '{__name__=~"paas_.*|kube_.*"}'
    static_configs:
      - targets: ['prom-agent-prod.paas-monitoring.svc:9090']
  - job_name: federate-test
    honor_labels: true
    metrics_path: /federate
    params:
      match[]:
        - '{__name__=~"paas_.*|kube_.*"}'
    static_configs:
      - targets: ['prom-agent-test.paas-monitoring.svc:9090']

# file: observability/recording-rules.yaml
groups:
  - name: paas_cluster_recording
    rules:
      - record: paas:cluster:health_ratio
        expr: sum(paas_cluster_health) / count(paas_cluster_health)
      - record: paas:cluster:agent_up_count
        expr: sum by (env) (paas_cluster_agent_up)

# file: observability/alert-rules.yaml
groups:
  - name: paas_cluster_alerts
    rules:
      - alert: ClusterDegraded
        expr: paas_cluster_health == 0
        for: 5m
        labels:
          severity: warning
          team: paas-sre
        annotations:
          summary: "集群 {{ $labels.cluster }} 降级，请检查反向隧道/防火墙"
      - alert: ClusterEvicted
        expr: paas_cluster_health == -1
        for: 1m
        labels:
          severity: critical
          team: paas-sre
        annotations:
          summary: "集群 {{ $labels.cluster }} 已摘除，触发流量隔离"
      - alert: PrometheusAgentDown
        expr: paas_cluster_agent_up == 0
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "集群 {{ $labels.cluster }} Prometheus Agent 离线，联邦监控缺口"

# file: observability/alertmanager.yaml
global:
  resolve_timeout: 5m
route:
  receiver: default
  group_by: ['cluster', 'env']
  routes:
    - matchers: ['env="production"']
      receiver: prod-oncall
      continue: true
    - matchers: ['severity="critical"']
      receiver: crit-page
receivers:
  - name: default
    webhook_configs:
      - url: http://paas-notify.svc/api/v1/alert
  - name: prod-oncall
    webhook_configs:
      - url: http://paas-notify.svc/api/v1/wechat
  - name: crit-page
    pagerduty_configs:
      - routing_key: ${PAGERDUTY_KEY}

#!/usr/bin/env bash
# file: runbook/cluster-ops.sh
set -euo pipefail

# ---- 章节四 SOP：注册测试集群 ----
curl -X POST https://paas-gateway/api/v1/clusters \
  -H "Content-Type: application/json" \
  -d '{"id":"test-cluster-01","env":"test","endpoint":"https://10.10.10.10:6443","mode":"reverse-tunnel","kubeconfig":"<in-memory-only>"}'
# 校验 Namespace 与配额
kubectl --context test-cluster-01 get ns subcompany-a --show-labels
kubectl --context test-cluster-01 get resourcequota -n subcompany-a

# ---- 环境划分与节点池 ----
paas-cli cluster label --name prod-cluster-01 --set env=production
paas-cli nodepool create --cluster prod-cluster-01 --name np-gold --zone az1

# ---- 跨集群查询（经网关，无需直连 apiserver）----
curl -s https://paas-gateway/proxy/prod-cluster-01/api/v1/namespaces

# ---- 故障1：查反向隧道/防火墙/agent ----
kubectl --context prod-cluster-01 -n paas-system get pods -l app=cluster-agent -o wide
curl -s 'http://prom-center:9090/api/v1/query?query=paas_cluster_health' | jq '.data.result[] | {cluster:.metric.cluster, health:.value[1]}'

# ---- 故障2：查 Agent 在线情况 ----
curl -s 'http://prom-center:9090/api/v1/query?query=paas_cluster_agent_up' | jq '.data.result[] | {cluster:.metric.cluster, up:.value[1]}'
kubectl --context prod-cluster-01 -n paas-monitoring get deploy -l app=prom-agent

# ---- 故障3：查租户配额与 Pod requests ----
kubectl --context prod-cluster-01 -n subcompany-a get resourcequota -o yaml
kubectl --context prod-cluster-01 -n subcompany-a get pods -o json \
  | jq '.items[] | select(.spec.containers[].resources.requests==null) | .metadata.name'

# ---- 查询审计（≥90 天）----
grep '"action":"register-cluster"' /var/log/paas/cluster-audit.jsonl | tail -5