统一观测平台（Prometheus 联邦对接）

paas-observability/
├── go.mod                                  # Go 依赖（gin / thanos / grpc）
├── cmd/
│   └── federation-proxy/
│       └── main.go                         # gin 服务入口：租户隔离查询 API
├── internal/
│   ├── auth/
│   │   └── tenant.go                       # gin 中间件：解析租户/管理员身份
│   ├── thanos/
│   │   ├── query.go                        # Thanos Query HTTP 客户端（/api/v1/query）
│   │   └── grpc/
│   │       └── grpc_query.go               # Thanos Query gRPC Store/Query 客户端
│   ├── grafana/
│   │   └── datasource.go                   # 按 tenant 动态创建 Grafana 数据源
│   └── alerting/
│       └── router.go                       # 告警分级路由 Go 实现（与 Alertmanager 一致）
└── deploy/                                 # K8s IaC（见第三、四章）
    ├── 00-namespace-rbac.yaml              # monitoring ns + Agent SA/ClusterRole
    ├── 01-prom-agent-configmap.yaml        # Agent scrape + relabel 注入标签
    ├── 02-prom-agent-statefulset.yaml      # Agent StatefulSet/Service/ServiceMonitor
    ├── 03-central-federation-configmap.yaml# 中心 Prometheus 联邦 scrape 配置
    ├── 04-thanos.yaml                      # Sidecar/Receiver/Query/Store 部署
    ├── 05-alertmanager.yaml                # Alertmanager 部署 + 分级路由/抑制
    ├── 06-grafana-provisioning.yaml        # Grafana 数据源 + 多租户文件夹
    └── 07-rules.yaml                       # recording rules + alert rules

# file: deploy/00-namespace-rbac.yaml
apiVersion: v1
kind: Namespace
metadata:
  name: monitoring
  labels:
    tenant: platform
    env: production
    cluster: prod-cluster-a
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: prom-agent
  namespace: monitoring
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: prom-agent
rules:
  - apiGroups: [""]
    resources: ["nodes", "nodes/metrics", "services", "endpoints", "pods"]
    verbs: ["get", "list", "watch"]
  - apiGroups: ["apps"]
    resources: ["deployments", "statefulsets", "daemonsets", "replicasets"]
    verbs: ["get", "list", "watch"]
  - nonResourceURLs: ["/metrics", "/federate"]
    verbs: ["get"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: prom-agent
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: prom-agent
subjects:
  - kind: ServiceAccount
    name: prom-agent
    namespace: monitoring

# file: deploy/01-prom-agent-configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: prom-agent-config
  namespace: monitoring
data:
  prometheus.yml: |
    global:
      scrape_interval: 15s
      evaluation_interval: 30s
      external_labels:
        cluster: prod-cluster-a
        env: production
    # Agent 模式：开 enable-feature=agent 时无本地存储/告警，仅采集+remote_write
    scrape_configs:
      - job_name: kube-pods
        kubernetes_sd_configs:
          - role: pod
        relabel_configs:
          # ① 从命名空间 tn-<租户> 提取 tenant 标签（★标签隔离核心）
          - source_labels: [__meta_kubernetes_namespace]
            regex: "tn-(.+)"
            target_label: tenant
            replacement: "$1"
          # ② 未遵循 tn- 前缀的命名空间，统一标记 tenant=unknown（供中心治理告警）
          #    regex: "" 仅匹配空值，即只有第一步没成功写入 tenant 时才落 unknown
          - source_labels: [tenant]
            regex: ""
            target_label: tenant
            replacement: unknown
          # ③ 从 Pod 标准 label 提取 app
          - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name]
            target_label: app
            regex: "(.+)"
            replacement: "$1"
          # ④ 注入静态环境/集群标签（与 global.external_labels 双保险）
          - target_label: env
            replacement: production
          - target_label: cluster
            replacement: prod-cluster-a
          # 只保留有 tenant 的真实业务 Pod，过滤系统组件噪声
          - source_labels: [tenant]
            regex: "unknown"
            action: drop
        metric_relabel_configs:
          # 仅保留关键指标，降低联邦拉取体积
          - source_labels: [__name__]
            regex: "up|kube_pod_.*|container_cpu_.*|container_memory_.*|http_requests_total"
            action: keep

    # 可选：remote_write 到中心 Thanos Receiver（联邦 pull 的补充通道）
    remote_write:
      - url: http://thanos-receiver.monitoring.svc:19291/api/v1/receive
        send_exemplars: true
        queue_config:
          max_samples_per_send: 2000
          capacity: 100000    

# file: deploy/02-prom-agent-statefulset.yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: prom-agent
  namespace: monitoring
  labels:
    app: prom-agent
spec:
  serviceName: prom-agent
  replicas: 1
  selector:
    matchLabels:
      app: prom-agent
  template:
    metadata:
      labels:
        app: prom-agent
    spec:
      serviceAccountName: prom-agent
      containers:
        - name: prometheus
          image: quay.io/prometheus/prometheus:v2.51.2
          args:
            - "--config.file=/etc/prometheus/prometheus.yml"
            - "--storage.tsdb.path=/prometheus"
            - "--storage.tsdb.retention.time=2h"   # Agent 本地仅短缓存
            - "--web.enable-lifecycle"
            - "--enable-feature=agent"             # Agent 模式：无本地告警/长期存储
          ports:
            - containerPort: 9090
              name: http
            - containerPort: 9091
              name: federate                      # /federate 端点由 http 端口暴露
          volumeMounts:
            - name: config
              mountPath: /etc/prometheus
            - name: data
              mountPath: /prometheus
      volumes:
        - name: config
          configMap:
            name: prom-agent-config
  volumeClaimTemplates:
    - metadata:
        name: data
      spec:
        accessModes: ["ReadWriteOnce"]
        resources:
          requests:
            storage: 5Gi
---
apiVersion: v1
kind: Service
metadata:
  name: prom-agent
  namespace: monitoring
  labels:
    app: prom-agent
spec:
  selector:
    app: prom-agent
  ports:
    - name: http
      port: 9090
      targetPort: 9090
    # 联邦拉取走 9090 /federate；网关只放行该端口
    - name: federate
      port: 9090
      targetPort: 9090
---
# ServiceMonitor 让中心以「服务发现」方式纳管本 Agent（可选，pull 模型下中心直接 scrape /federate）
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: prom-agent
  namespace: monitoring
  labels:
    tenant: platform
    cluster: prod-cluster-a
spec:
  selector:
    matchLabels:
      app: prom-agent
  endpoints:
    - port: http
      path: /federate
      interval: 30s
      params:
        match[]:
          - '{tenant!=""}'

# file: deploy/03-central-federation-configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: central-prometheus-config
  namespace: monitoring
data:
  prometheus.yml: |
    global:
      scrape_interval: 30s
      evaluation_interval: 30s
      external_labels:
        cluster: central
        env: production
    rule_files:
      - /etc/prometheus/rules/*.yml
    scrape_configs:
      # ===== 各物理隔离集群的联邦拉取任务 =====
      - job_name: federate-prod-cluster-a
        scrape_interval: 30s
        honor_labels: true        # ★ 保留 Agent 侧 tenant/app/env/cluster 标签
        metrics_path: /federate
        params:
          match[]:
            - '{__name__=~"up|kube_pod_.*|container_.*|http_requests_total"}'
            - '{tenant!=""}'      # 兜底过滤无标签噪声
        static_configs:
          - targets: ['prom-agent.monitoring.svc:9090']   # 实际经 API 网关隧道解析
        relabel_configs:
          - target_label: cluster
            replacement: prod-cluster-a

      - job_name: federate-prod-cluster-b
        scrape_interval: 30s
        honor_labels: true
        metrics_path: /federate
        params:
          match[]:
            - '{__name__=~"up|kube_pod_.*|container_.*|http_requests_total"}'
            - '{tenant!=""}'
        static_configs:
          - targets: ['prom-agent-prod-b.monitoring.svc:9090']
        relabel_configs:
          - target_label: cluster
            replacement: prod-cluster-b

      - job_name: federate-prestaging
        scrape_interval: 60s
        honor_labels: true
        metrics_path: /federate
        params:
          match[]:
            - '{tenant!=""}'
        static_configs:
          - targets: ['prom-agent-prestaging.monitoring.svc:9090']
        relabel_configs:
          - target_label: cluster
            replacement: prestaging-cluster

      - job_name: federate-testing
        scrape_interval: 60s
        honor_labels: true
        metrics_path: /federate
        params:
          match[]:
            - '{tenant!=""}'
        static_configs:
          - targets: ['prom-agent-testing.monitoring.svc:9090']
        relabel_configs:
          - target_label: cluster
            replacement: testing-cluster

      # 中心自身 Thanos Sidecar 暴露的本地指标（近期）
      - job_name: thanos-sidecar
        static_configs:
          - targets: ['thanos-sidecar.monitoring.svc:10902']    

# file: deploy/04-thanos.yaml
# ===== 对象存储凭证（KMS 信封加密后的 Secret，由网关解密下发）=====
apiVersion: v1
kind: Secret
metadata:
  name: thanos-objstore
  namespace: monitoring
type: Opaque
stringData:
  objstore.yaml: |
    type: s3
    config:
      bucket: paas-thanos-prod
      endpoint: oss.internal:443
      region: cn-north
      access_key: "{{KMS_DECIPHERED_AK}}"   # 占位，由网关注入真实值
      secret_key: "{{KMS_DECIPHERED_SK}}"    
---
# ===== Thanos Sidecar：随中心 Prometheus 上传 TSDB 块 =====
apiVersion: apps/v1
kind: Deployment
metadata:
  name: thanos-sidecar
  namespace: monitoring
  labels: {app: thanos-sidecar}
spec:
  replicas: 1
  selector:
    matchLabels: {app: thanos-sidecar}
  template:
    metadata:
      labels: {app: thanos-sidecar}
    spec:
      containers:
        - name: sidecar
          image: quay.io/thanos/thanos:v0.35.1
          args:
            - sidecar
            - --prometheus.url=http://central-prometheus.monitoring.svc:9090
            - --objstore.config-file=/etc/thanos/objstore.yaml
            - --tsdb.retention=2h
            - --grpc-address=0.0.0.0:10901
            - --http-address=0.0.0.0:10902
          ports:
            - {containerPort: 10901, name: grpc}
            - {containerPort: 10902, name: http}
          volumeMounts:
            - {name: objstore, mountPath: /etc/thanos}
      volumes:
        - name: objstore
          secret:
            secretName: thanos-objstore
---
# ===== Thanos Receiver：接收边缘 Agent remote_write =====
apiVersion: apps/v1
kind: Deployment
metadata:
  name: thanos-receiver
  namespace: monitoring
  labels: {app: thanos-receiver}
spec:
  replicas: 2
  selector:
    matchLabels: {app: thanos-receiver}
  template:
    metadata:
      labels: {app: thanos-receiver}
    spec:
      containers:
        - name: receiver
          image: quay.io/thanos/thanos:v0.35.1
          args:
            - receive
            - --receive.replication-factor=2
            - --receive.hashrings-file=/etc/thanos/hashrings.json
            - --objstore.config-file=/etc/thanos/objstore.yaml
            - --grpc-address=0.0.0.0:10901
            - --http-address=0.0.0.0:10902
            - --remote-write.address=0.0.0.0:19291
          ports:
            - {containerPort: 10901, name: grpc}
            - {containerPort: 10902, name: http}
            - {containerPort: 19291, name: remote-write}
          volumeMounts:
            - {name: objstore, mountPath: /etc/thanos}
      volumes:
        - name: objstore
          secret:
            secretName: thanos-objstore
---
# ===== Thanos Query：聚合 Sidecar + Receiver + Store =====
apiVersion: apps/v1
kind: Deployment
metadata:
  name: thanos-query
  namespace: monitoring
  labels: {app: thanos-query}
spec:
  replicas: 2
  selector:
    matchLabels: {app: thanos-query}
  template:
    metadata:
      labels: {app: thanos-query}
    spec:
      containers:
        - name: query
          image: quay.io/thanos/thanos:v0.35.1
          args:
            - query
            - --http-address=0.0.0.0:9090
            - --grpc-address=0.0.0.0:10901
            - --query.replica-label=replica
            - --store=thanos-sidecar.monitoring.svc:10901
            - --store=thanos-receiver.monitoring.svc:10901
            - --store=thanos-store.monitoring.svc:10901
          ports:
            - {containerPort: 9090, name: http}     # 暴露 Prometheus 兼容 HTTP API
            - {containerPort: 10901, name: grpc}    # 暴露 gRPC Store/Query
---
# ===== Thanos Store：提供历史块检索（对象存储）=====
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: thanos-store
  namespace: monitoring
  labels: {app: thanos-store}
spec:
  serviceName: thanos-store
  replicas: 2
  selector:
    matchLabels: {app: thanos-store}
  template:
    metadata:
      labels: {app: thanos-store}
    spec:
      containers:
        - name: store
          image: quay.io/thanos/thanos:v0.35.1
          args:
            - store
            - --objstore.config-file=/etc/thanos/objstore.yaml
            - --grpc-address=0.0.0.0:10901
            - --http-address=0.0.0.0:10902
            - --data-dir=/var/thanos/store
          ports:
            - {containerPort: 10901, name: grpc}
            - {containerPort: 10902, name: http}
          volumeMounts:
            - {name: objstore, mountPath: /etc/thanos}
            - {name: data, mountPath: /var/thanos/store}
      volumes:
        - name: objstore
          secret:
            secretName: thanos-objstore
  volumeClaimTemplates:
    - metadata: {name: data}
      spec:
        accessModes: ["ReadWriteOnce"]
        resources:
          requests: {storage: 50Gi}

sum by (tenant, cluster) (rate(http_requests_total{env="production"}[5m]))

# file: deploy/05-alertmanager.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: alertmanager-config
  namespace: monitoring
data:
  alertmanager.yml: |
    global:
      resolve_timeout: 5m
    route:
      receiver: 'default-webhook'
      group_by: ['tenant', 'env', 'cluster', 'alertname']
      group_wait: 30s
      group_interval: 5m
      repeat_interval: 4h
      routes:
        # 生产环境 P0/critical：电话 + 企业微信急件
        - matchers: ['severity="critical"', 'env="production"']
          receiver: 'prod-pager'
          repeat_interval: 30m
          continue: true
        # 预发布/测试 critical：仅急件不打电话
        - matchers: ['severity="critical"', 'env!="production"']
          receiver: 'nonprod-webhook'
          continue: true
        # 按租户二次分发到对应租户 IM 群
        - matchers: ['severity=~"warning|info"']
          receiver: 'ticket-system'
          group_wait: 2m
    receivers:
      - name: 'default-webhook'
        webhook_configs:
          - url: http://alert-hub.monitoring.svc:8080/v1/notify
      - name: 'prod-pager'
        webhook_configs:
          - url: http://alert-hub.monitoring.svc:8080/v1/pager
            send_resolved: true
      - name: 'nonprod-webhook'
        webhook_configs:
          - url: http://alert-hub.monitoring.svc:8080/v1/notify
      - name: 'ticket-system'
        webhook_configs:
          - url: http://alert-hub.monitoring.svc:8080/v1/ticket
    inhibit_rules:
      # ★ 节点不可用时，抑制该节点上的 Pod/工作负载副本告警
      - source_matchers: ['alertname="NodeDown"', 'severity="critical"']
        target_matchers: ['alertname=~"PodCrashLooping|KubePodNotReady"']
        equal: ['cluster', 'tenant', 'node']
      # 集群级采集中断时，抑制该集群全部业务告警（避免无数据误报）
      - source_matchers: ['alertname="FederationScrapeDown"']
        target_matchers: ['cluster=~".+"']
        equal: ['cluster']    
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: alertmanager
  namespace: monitoring
  labels: {app: alertmanager}
spec:
  replicas: 2
  selector:
    matchLabels: {app: alertmanager}
  template:
    metadata:
      labels: {app: alertmanager}
    spec:
      containers:
        - name: alertmanager
          image: quay.io/prometheus/alertmanager:v0.27.0
          args:
            - --config.file=/etc/alertmanager/alertmanager.yml
            - --storage.path=/alertmanager
          ports:
            - {containerPort: 9093, name: http}
          volumeMounts:
            - {name: config, mountPath: /etc/alertmanager}
      volumes:
        - name: config
          configMap:
            name: alertmanager-config

# file: deploy/06-grafana-provisioning.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: grafana-datasources
  namespace: monitoring
data:
  datasource-acme.yaml: |
    apiVersion: 1
    datasources:
      - name: thanos-acme
        type: prometheus
        # 走平台查询代理，由代理按 org(tenant) 注入标签过滤，实现数据级隔离
        url: http://federation-proxy.monitoring.svc:8080/api/v1
        access: proxy
        isDefault: true
        jsonData:
          httpHeaderName1: "X-Tenant"
          httpHeaderName2: "X-Env"
        secureJsonData:
          httpHeaderValue1: "acme"
          httpHeaderValue2: "production"    
  datasource-global.yaml: |
    apiVersion: 1
    datasources:
      - name: thanos-global
        type: prometheus
        url: http://thanos-query.monitoring.svc:9090
        access: proxy
        jsonData:
          httpHeaderName1: "X-Tenant"
        secureJsonData:
          httpHeaderValue1: "platform"   # 仅管理员可用全局源    
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: grafana-folders
  namespace: monitoring
data:
  folders.yaml: |
    apiVersion: 1
    folders:
      - title: "租户-acme"
        uid: tenant-acme
      - title: "租户-global"
        uid: tenant-global    

# file: deploy/07-rules.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: central-rules
  namespace: monitoring
data:
  recording.yml: |
    groups:
      - name: tenant_slo_recording
        interval: 30s
        rules:
          - record: job🆙sum_by_tenant
            expr: sum by (tenant, env, cluster) (up)
          - record: tenant:request_errors:rate5m
            expr: sum by (tenant) (rate(http_requests_total{code=~"5.."}[5m]))
          - record: tenant:request_qps:rate5m
            expr: sum by (tenant, cluster) (rate(http_requests_total[5m]))
          # 自动化任务成功率（供模块 06 复用，标签严格一致）
          - record: task:success_rate:rate7d
            expr: |
              sum by (tenant, env, cluster) (rate(task_executions_total{result="success"}[7d]))
              /
              sum by (tenant, env, cluster) (rate(task_executions_total[7d]))
      - name: node_recording
        interval: 30s
        rules:
          - record: instance:node_cpu_util:ratio
            expr: 1 - avg by (tenant, cluster, instance) (rate(node_cpu_seconds_total{mode="idle"}[5m]))    
  alerts.yml: |
    groups:
      - name: tenant_governance
        rules:
          # ★ 标签缺失治理：某租户带 unknown 标签的指标比例过高
          - alert: TenantLabelMissing
            expr: sum by (cluster) (up{tenant="unknown"}) / sum by (cluster) (up) > 0.05
            for: 10m
            labels:
              severity: warning
            annotations:
              summary: "集群 {{ $labels.cluster }} 存在大量 tenant=unknown 指标"
          # ★ 联邦采集中断
          - alert: FederationScrapeDown
            expr: up{job=~"federate-.*"} == 0
            for: 5m
            labels:
              severity: critical
            annotations:
              summary: "联邦任务 {{ $labels.job }} 拉取失败 (cluster={{ $labels.cluster }})"
          # 节点宕机（触发抑制源）
          - alert: NodeDown
            expr: up{job="kube-pods",tenant!="unknown"} == 0 and on(instance) kube_node_status_condition{condition="Ready",status="true"} == 0
            for: 3m
            labels:
              severity: critical
            annotations:
              summary: "节点 {{ $labels.instance }} 不可用"
          # 租户错误率超 SLO
          - alert: TenantErrorBudgetBurn
            expr: tenant:request_errors:rate5m / (tenant:request_qps:rate5m + 1) > 0.05
            for: 15m
            labels:
              severity: critical
            annotations:
              summary: "租户 {{ $labels.tenant }} 错误率超 5%"    

// file: cmd/federation-proxy/main.go
package main

import (
	"log"
	"net/http"
	"os"

	"paas-observability/internal/alerting"
	"paas-observability/internal/auth"
	"paas-observability/internal/grafana"
	"paas-observability/internal/thanos"
	"paas-observability/internal/thanos/grpc"

	"github.com/gin-gonic/gin"
)

// main 启动租户隔离的联邦查询代理。
// 所有查询都先经 auth.TenantMiddleware 解析租户身份，
// 再转发到 Thanos Query（HTTP / gRPC 两种后端）。
func main() {
	thanosHTTPAddr := getenv("THANOS_QUERY_HTTP_ADDR", "http://thanos-query.monitoring.svc:9090")
	thanosGRPCAddr := getenv("THANOS_QUERY_GRPC_ADDR", "thanos-query.monitoring.svc:10901")
	grafanaAddr := getenv("GRAFANA_ADDR", "http://grafana.monitoring.svc:3000")
	grafanaToken := os.Getenv("GRAFANA_TOKEN")

	httpClient := thanos.NewQueryClient(thanosHTTPAddr)
	grpcClient := grpcclient.NewGRPCClient(thanosGRPCAddr)
	grafanaClient := grafana.NewClient(grafanaAddr, grafanaToken)

	r := gin.Default()
	r.Use(auth.TenantMiddleware())

	v1 := r.Group("/api/v1")
	{
		// 即时查询：HTTP 后端
		v1.GET("/query", func(c *gin.Context) {
			tenant := c.GetString("tenant")
			admin := c.GetBool("admin")
			q := c.Query("query")
			if !admin {
				// ★ 非管理员强制按本租户过滤，物理隔离数据
				q = auth.InjectTenantLabel(q, tenant)
			}
			body, err := httpClient.Query(c.Request.Context(), q, c.Query("time"))
			if err != nil {
				c.JSON(http.StatusBadGateway, gin.H{"error": err.Error()})
				return
			}
			c.Data(http.StatusOK, "application/json", body)
		})

		// 区间查询：HTTP 后端
		v1.GET("/query_range", func(c *gin.Context) {
			tenant := c.GetString("tenant")
			q := c.Query("query")
			if !c.GetBool("admin") {
				q = auth.InjectTenantLabel(q, tenant)
			}
			body, err := httpClient.QueryRange(c.Request.Context(), q, c.Query("start"), c.Query("end"), c.Query("step"))
			if err != nil {
				c.JSON(http.StatusBadGateway, gin.H{"error": err.Error()})
				return
			}
			c.Data(http.StatusOK, "application/json", body)
		})

		// gRPC 后端即时查询（演示 Thanos Store/Query gRPC 调用）
		v1.GET("/query_grpc", func(c *gin.Context) {
			tenant := c.GetString("tenant")
			q := c.Query("query")
			if !c.GetBool("admin") {
				q = auth.InjectTenantLabel(q, tenant)
			}
			if err := grpcClient.Query(c.Request.Context(), q); err != nil {
				c.JSON(http.StatusBadGateway, gin.H{"error": err.Error()})
				return
			}
			c.JSON(http.StatusOK, gin.H{"grpc": "streamed", "tenant": tenant})
		})

		// 为某租户动态创建 Grafana 隔离数据源（★高级能力 ⑤）
		v1.POST("/tenants/:tenant/datasource", func(c *gin.Context) {
			if !c.GetBool("admin") {
				c.JSON(http.StatusForbidden, gin.H{"error": "only admin can provision datasources"})
				return
			}
			t := c.Param("tenant")
			if err := grafanaClient.CreateTenantDatasource(c.Request.Context(), t); err != nil {
				c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
				return
			}
			c.JSON(http.StatusOK, gin.H{"tenant": t, "status": "provisioned"})
		})
	}

	// 告警分级路由演示端点（与 Alertmanager 配置语义一致）
	v1.POST("/alert", func(c *gin.Context) {
		var a alerting.Alert
		if err := c.ShouldBindJSON(&a); err != nil {
			c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
			return
		}
		receivers, pager := alerting.Route(a)
		c.JSON(http.StatusOK, gin.H{"receivers": receivers, "pager": pager})
	})

	addr := getenv("LISTEN_ADDR", ":8080")
	if err := r.Run(addr); err != nil {
		log.Fatalf("federation-proxy exit: %v", err)
	}
}

func getenv(k, def string) string {
	if v := os.Getenv(k); v != "" {
		return v
	}
	return def
}

// file: internal/auth/tenant.go
package auth

import (
	"strings"

	"github.com/gin-gonic/gin"
)

// TenantMiddleware 从 X-Tenant 头（由 API 网关注入并经 JWT 校验）解析租户身份。
// admin=true 时放行全局查询；否则强制注入 tenant 过滤。
func TenantMiddleware() gin.HandlerFunc {
	return func(c *gin.Context) {
		tenant := c.GetHeader("X-Tenant")
		if tenant == "" {
			tenant = "unknown"
		}
		admin := c.GetHeader("X-Role") == "platform-admin"
		c.Set("tenant", tenant)
		c.Set("admin", admin)
		c.Next()
	}
}

// InjectTenantLabel 在 PromQL 中强制注入 {tenant="X"} 过滤，保证数据级隔离。
// 若查询已含 tenant 选择器则不再重复注入。
func InjectTenantLabel(query, tenant string) string {
	if tenant == "" || tenant == "unknown" {
		return query
	}
	if strings.Contains(query, "tenant=") {
		return query
	}
	// 在最外层 {} 选择器里追加 tenant 匹配；无选择器则整体包裹
	if strings.Contains(query, "{") {
		return strings.Replace(query, "{", "{tenant=\""+tenant+"\",", 1)
	}
	return query + "{tenant=\"" + tenant + "\"}"
}

// file: internal/thanos/query.go
package thanos

import (
	"context"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"time"
)

// QueryClient 封装 Thanos Query 的 Prometheus 兼容 HTTP API。
type QueryClient struct {
	BaseURL string
	HTTP    *http.Client
}

// NewQueryClient 构造 HTTP 查询客户端。
func NewQueryClient(baseURL string) *QueryClient {
	return &QueryClient{
		BaseURL: baseURL,
		HTTP:    &http.Client{Timeout: 30 * time.Second},
	}
}

// Query 调用 /api/v1/query 即时查询，返回原始 JSON（与 Prometheus 响应结构一致）。
func (c *QueryClient) Query(ctx context.Context, q, t string) ([]byte, error) {
	qry := url.Values{}
	qry.Set("query", q)
	if t != "" {
		qry.Set("time", t)
	}
	return c.get(ctx, "/api/v1/query?"+qry.Encode())
}

// QueryRange 调用 /api/v1/query_range 区间查询。
func (c *QueryClient) QueryRange(ctx context.Context, q, start, end, step string) ([]byte, error) {
	qry := url.Values{}
	qry.Set("query", q)
	qry.Set("start", start)
	qry.Set("end", end)
	if step == "" {
		step = "60"
	}
	qry.Set("step", step)
	return c.get(ctx, "/api/v1/query_range?"+qry.Encode())
}

func (c *QueryClient) get(ctx context.Context, path string) ([]byte, error) {
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.BaseURL+path, nil)
	if err != nil {
		return nil, err
	}
	resp, err := c.HTTP.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	body, err := io.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("thanos query http %d: %s", resp.StatusCode, string(body))
	}
	return body, nil
}

// file: internal/thanos/grpc/grpc_query.go
package grpcclient

import (
	"context"
	"io"
	"time"

	"github.com/thanos-io/thanos/pkg/store/storepb"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials/insecure"
)

// GRPCClient 封装 Thanos Query/Sidecar 的 gRPC Store API。
type GRPCClient struct {
	addr string
}

// NewGRPCClient 构造 gRPC 客户端（生产应启用 TLS+mTLS）。
func NewGRPCClient(addr string) *GRPCClient {
	return &GRPCClient{addr: addr}
}

// Query 通过 gRPC 流式查询，打印每个 Series 的租户标签，验证跨集群聚合。
func (g *GRPCClient) Query(ctx context.Context, query string) error {
	ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
	defer cancel()

	conn, err := grpc.DialContext(ctx, g.addr,
		grpc.WithTransportCredentials(insecure.NewCredentials()),
		grpc.WithBlock(),
	)
	if err != nil {
		return err
	}
	defer conn.Close()

	cli := storepb.NewStoreClient(conn)
	req := &storepb.QueryRequest{
		Query: query,
		// 聚合所有副本（与 --query.replica-label=replica 一致）
		ReplicaLabels: []string{"replica"},
	}

	stream, err := cli.Query(ctx, req)
	if err != nil {
		return err
	}
	for {
		resp, err := stream.Recv()
		if err == io.EOF {
			return nil
		}
		if err != nil {
			return err
		}
		// resp 含租户维度标签，可在平台侧做二次过滤/计费
		ts := resp.GetTimeseries()
		if ts == nil {
			continue
		}
		tenant := ""
		for _, l := range ts.GetLabels() {
			if l.GetName() == "tenant" {
				tenant = l.GetValue()
			}
		}
		_ = tenant
	}
}

// file: internal/grafana/datasource.go
package grafana

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"time"
)

// Client 封装 Grafana HTTP API，按租户创建隔离数据源。
type Client struct {
	BaseURL string
	Token   string
	HTTP    *http.Client
}

// NewClient 构造 Grafana 客户端。
func NewClient(baseURL, token string) *Client {
	return &Client{
		BaseURL: baseURL,
		Token:   token,
		HTTP:    &http.Client{Timeout: 20 * time.Second},
	}
}

// datasourcePayload 为 Grafana 数据源创建请求体。
// 关键：url 指向平台查询代理，并通过 X-Tenant 头让代理强制注入 tenant 过滤。
type datasourcePayload struct {
	Name      string            `json:"name"`
	Type      string            `json:"type"`
	URL       string            `json:"url"`
	Access    string            `json:"access"`
	IsDefault bool              `json:"isDefault"`
	JSONData  map[string]string `json:"jsonData"`
	Secure    map[string]string `json:"secureJsonData"`
}

// CreateTenantDatasource 为某租户创建隔离数据源（★数据级隔离）。
func (c *Client) CreateTenantDatasource(ctx context.Context, tenant string) error {
	payload := datasourcePayload{
		Name:      "thanos-" + tenant,
		Type:      "prometheus",
		URL:       "http://federation-proxy.monitoring.svc:8080/api/v1",
		Access:    "proxy",
		IsDefault: true,
		JSONData: map[string]string{
			"httpHeaderName1": "X-Tenant",
			"httpHeaderName2": "X-Env",
		},
		Secure: map[string]string{
			"httpHeaderValue1": tenant,
			"httpHeaderValue2": "production",
		},
	}
	body, err := json.Marshal(payload)
	if err != nil {
		return err
	}
	req, err := http.NewRequestWithContext(ctx, http.MethodPost,
		c.BaseURL+"/api/datasources", bytes.NewReader(body))
	if err != nil {
		return err
	}
	req.Header.Set("Authorization", "Bearer "+c.Token)
	req.Header.Set("Content-Type", "application/json")

	resp, err := c.HTTP.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode >= 300 {
		b, _ := io.ReadAll(resp.Body)
		return fmt.Errorf("grafana create datasource http %d: %s", resp.StatusCode, string(b))
	}
	return nil
}

// file: internal/alerting/router.go
package alerting

// Alert 与 Prometheus alert 标签对齐，字段名与 4.7 alert rules 一致。
type Alert struct {
	Tenant   string `json:"tenant"`
	Env      string `json:"env"`
	Cluster  string `json:"cluster"`
	Severity string `json:"severity"`
	Summary  string `json:"summary"`
}

// Route 根据 severity + env 决定接收方；返回接收方列表与是否触发电话(P0)。
// 逻辑与 deploy/05-alertmanager.yaml 的 route 配置一一对应。
func Route(a Alert) (receivers []string, pager bool) {
	switch {
	case a.Severity == "critical" && a.Env == "production":
		// 生产 P0：电话 + 企业微信急件
		return []string{"prod-pager", "alert-hub"}, true
	case a.Severity == "critical":
		// 非生产 critical：仅急件
		return []string{"nonprod-webhook"}, false
	case a.Severity == "warning" || a.Severity == "info":
		// 低级别：进工单
		return []string{"ticket-system"}, false
	default:
		return []string{"default-webhook"}, false
	}
}

// file: go.mod
module paas-observability

go 1.21

require (
	github.com/gin-gonic/gin v1.9.1
	github.com/thanos-io/thanos v0.35.1
	google.golang.org/grpc v1.60.1
)

# 构建并运行联邦查询代理（需先 go mod tidy 拉取依赖）
cd paas-observability
go mod tidy
go build ./...
go run ./cmd/federation-proxy

# 以租户 acme 身份查询（代理会强制注入 {tenant="acme"}）
curl -H "X-Tenant: acme" -H "X-Role: tenant" \
  'http://localhost:8080/api/v1/query?query=sum(rate(http_requests_total[5m]))'

# 以平台管理员查询全局
curl -H "X-Tenant: platform" -H "X-Role: platform-admin" \
  'http://localhost:8080/api/v1/query?query=sum by (tenant,cluster)(rate(http_requests_total[5m]))'

# 1. 在目标集群部署 Agent（见 deploy/00~02），确认 /federate 可达
kubectl -n monitoring get pods -l app=prom-agent
kubectl -n monitoring port-forward svc/prom-agent 9090:9090 &
curl 'http://localhost:9090/federate?match[]={tenant!=""}' | head

# 2. 在中心 Prometheus 增加 federate-<cluster> job（编辑 03-central-federation-configmap.yaml）
kubectl -n monitoring edit configmap central-prometheus-config
kubectl -n monitoring rollout restart deployment central-prometheus

# 3. 验证联邦 target 健康
kubectl -n monitoring exec deploy/central-prometheus -- \
  wget -qO- 'http://localhost:9090/api/v1/targets' | jq '.data.activeTargets[] | select(.scrapePool|test("federate"))'

# 检查某联邦任务 target 是否 up
curl -s 'http://thanos-query.monitoring.svc:9090/api/v1/query?query=up{job=~"federate-.*"}' | jq .

# 在 Agent 侧确认 tenant 标签是否已注入（relabel 是否生效）
kubectl -n monitoring exec deploy/prom-agent -- \
  wget -qO- 'http://localhost:9090/api/v1/label/tenant/values'

# relabel 丢失标签常见原因：命名空间未用 tn- 前缀
kubectl get ns | grep -v '^tn-'   # 找出未隔离的租户命名空间

# 查看 Query 已连接的 Store 节点
thanos query --help >/dev/null
curl -s 'http://thanos-query.monitoring.svc:9090/api/v1/stores' | jq '.data.stores[] | {name,lastSync,minTime,maxTime}'

# 查看 Sidecar 上传对象存储是否报错
kubectl -n monitoring logs deploy/thanos-sidecar | grep -i 'upload\|error'

# 对象存储凭证（KMS 解密后）是否就绪
kubectl -n monitoring get secret thanos-objstore -o jsonpath='{.data.objstore\.yaml}' | base64 -d

# 手动注入一条 critical+production 告警，验证走 prod-pager 通道
curl -XPOST http://federation-proxy.monitoring.svc:8080/api/v1/alert -H 'Content-Type: application/json' -d '{
  "tenant":"acme","env":"production","cluster":"prod-cluster-a",
  "severity":"critical","summary":"manual test"
}'
# 期望返回 receivers=["prod-pager","alert-hub"], pager=true

# 查看 Alertmanager 当前告警与抑制
kubectl -n monitoring port-forward svc/alertmanager 9093:9093 &
curl 'http://localhost:9093/api/v2/alerts' | jq '.[] | {labels, status}'

# 租户 acme 应只能查到 tenant="acme"
curl -s -H "X-Tenant: acme" -H "X-Role: tenant" \
  'http://federation-proxy.monitoring.svc:8080/api/v1/query?query=up' \
  | jq '.data.result[].metric.tenant' | sort -u
# 期望仅输出 "acme"

# 越权尝试（无 admin）查询他人应被注入过滤，返回空
curl -s -H "X-Tenant: acme" -H "X-Role: tenant" \
  'http://federation-proxy.monitoring.svc:8080/api/v1/query?query=up{tenant="other"}'