本文面向有一定 Kubernetes 基础的运维工程师，系统梳理容器编排在生产环境中的进阶实践，涵盖自动伸缩、资源管理、调度策略、RBAC、网络策略、存储进阶、Pod 调度、故障排查、集群运维与生产最佳实践。
一、HPA 自动伸缩
1.1 基于 CPU/内存的 HPA
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
name: web-app-hpa
spec:
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: web-app
minReplicas: 2
maxReplicas: 20
metrics:
- type: Resource
resource:
name: cpu
target:
type: Utilization
averageUtilization: 70
- type: Resource
resource:
name: memory
target:
type: Utilization
averageUtilization: 80
behavior:
scaleUp:
stabilizationWindowSeconds: 60
policies:
- type: Percent
value: 100
periodSeconds: 60
scaleDown:
stabilizationWindowSeconds: 300
policies:
- type: Percent
value: 10
periodSeconds: 60# 创建 HPA
kubectl apply -f hpa.yaml
# 查看 HPA 状态
kubectl get hpa web-app-hpa -o wide
# 手动触发扩容测试
kubectl run load-test --image=busybox -- /bin/sh -c "while true; do wget -q -O- http://web-app.default.svc.cluster.local; done"1.2 自定义指标 HPA
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
name: web-app-hpa-custom
spec:
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: web-app
minReplicas: 2
maxReplicas: 50
metrics:
- type: Pods
pods:
metric:
name: http_requests_per_second
target:
type: AverageValue
averageValue: "1000"
- type: Object
object:
metric:
name: requests_per_second
describedObject:
apiVersion: networking.k8s.io/v1
kind: Ingress
name: main-ingress
target:
type: Value
value: "10000"1.3 Prometheus Adapter 配置
# prometheus-adapter ConfigMap
apiVersion: v1
kind: ConfigMap
metadata:
name: prometheus-adapter-config
namespace: monitoring
data:
config.yaml: |
rules:
- seriesQuery: 'http_requests_total{namespace!="",pod!=""}'
resources:
overrides:
namespace: {resource: "namespace"}
pod: {resource: "pod"}
name:
matches: "^(.*)_total$"
as: "${1}_per_second"
metricsQuery: 'sum(rate(<<.Series>>{<<.LabelMatchers>>}[2m])) by (<<.GroupBy>>)'
- seriesQuery: 'nginx_connections_active{namespace!="",pod!=""}'
resources:
overrides:
namespace: {resource: "namespace"}
pod: {resource: "pod"}
name:
as: "nginx_active_connections"
metricsQuery: 'sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>)'# 安装 Prometheus Adapter
helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
helm install prometheus-adapter prometheus-community/prometheus-adapter \
--namespace monitoring \
--values prometheus-adapter-values.yaml
# 验证自定义指标是否可用
kubectl get --raw "/apis/custom.metrics.k8s.io/v1beta1" | jq .1.4 VPA（Vertical Pod Autoscaler）
apiVersion: autoscaling.k8s.io/v1
kind: VerticalPodAutoscaler
metadata:
name: web-app-vpa
spec:
targetRef:
apiVersion: apps/v1
kind: Deployment
name: web-app
updatePolicy:
updateMode: "Auto" # Off | Initial | Recreate | Auto
resourcePolicy:
containerPolicies:
- containerName: "*"
minAllowed:
cpu: 100m
memory: 128Mi
maxAllowed:
cpu: 4
memory: 8Gi
controlledResources: ["cpu", "memory"]# 安装 VPA
git clone https://github.com/kubernetes/autoscaler.git
cd autoscaler/vertical-pod-autoscaler
./hack/vpa-up.sh
# 查看 VPA 推荐
kubectl describe vpa web-app-vpa二、资源管理
2.1 Requests 与 Limits
apiVersion: apps/v1
kind: Deployment
metadata:
name: web-app
spec:
replicas: 3
selector:
matchLabels:
app: web-app
template:
metadata:
labels:
app: web-app
spec:
containers:
- name: web
image: nginx:1.25
resources:
requests:
cpu: 250m
memory: 256Mi
limits:
cpu: "1"
memory: 1Gi
ports:
- containerPort: 802.2 LimitRange
apiVersion: v1
kind: LimitRange
metadata:
name: default-limits
namespace: production
spec:
limits:
- type: Container
default:
cpu: 500m
memory: 512Mi
defaultRequest:
cpu: 100m
memory: 128Mi
max:
cpu: "4"
memory: 8Gi
min:
cpu: 50m
memory: 64Mi
- type: Pod
max:
cpu: "8"
memory: 16Gi
- type: PersistentVolumeClaim
max:
storage: 500Gi
min:
storage: 1Gi2.3 ResourceQuota
apiVersion: v1
kind: ResourceQuota
metadata:
name: team-quota
namespace: team-a
spec:
hard:
requests.cpu: "20"
requests.memory: 40Gi
limits.cpu: "40"
limits.memory: 80Gi
pods: "100"
services: "20"
persistentvolumeclaims: "30"
requests.storage: 2Ti
configmaps: "50"
secrets: "50"
scopes:
- NotTerminating# 查看配额使用情况
kubectl describe resourcequota team-quota -n team-a
# 查看所有命名空间的配额
kubectl get resourcequota --all-namespaces2.4 QoS 类别
# Guaranteed Pod
apiVersion: v1
kind: Pod
metadata:
name: guaranteed-pod
spec:
containers:
- name: app
image: nginx
resources:
requests:
cpu: "1"
memory: 1Gi
limits:
cpu: "1"
memory: 1Gi
---
# Burstable Pod
apiVersion: v1
kind: Pod
metadata:
name: burstable-pod
spec:
containers:
- name: app
image: nginx
resources:
requests:
cpu: 250m
memory: 256Mi
limits:
cpu: "2"
memory: 2Gi# 查看 Pod 的 QoS 类别
kubectl get pods -o custom-columns=\
NAME:.metadata.name,\
QOS:.status.qosClass,\
CPU_REQ:.spec.containers[*].resources.requests.cpu,\
CPU_LIM:.spec.containers[*].resources.limits.cpu三、调度策略
3.1 NodeSelector
apiVersion: v1
kind: Pod
metadata:
name: gpu-workload
spec:
nodeSelector:
gpu: "nvidia-a100"
disktype: ssd
containers:
- name: ml-training
image: tensorflow/tensorflow:latest-gpu
resources:
limits:
nvidia.com/gpu: 2# 给节点打标签
kubectl label nodes node-01 gpu=nvidia-a100 disktype=ssd
kubectl label nodes node-02 gpu=nvidia-v100 disktype=nvme3.2 NodeAffinity
apiVersion: v1
kind: Pod
metadata:
name: web-app
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/os
operator: In
values:
- linux
- key: node-role.kubernetes.io/worker
operator: Exists
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 80
preference:
matchExpressions:
- key: topology.kubernetes.io/zone
operator: In
values:
- cn-hangzhou-b
- cn-hangzhou-c
- weight: 20
preference:
matchExpressions:
- key: node.kubernetes.io/instance-type
operator: In
values:
- ecs.g7.2xlarge
- ecs.g7.4xlarge
containers:
- name: web
image: nginx:1.253.3 PodAffinity 与 PodAntiAffinity
apiVersion: apps/v1
kind: Deployment
metadata:
name: web-frontend
spec:
replicas: 3
selector:
matchLabels:
app: web-frontend
template:
metadata:
labels:
app: web-frontend
tier: frontend
spec:
affinity:
# Pod 亲和性：尽量与缓存服务在同一节点
podAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
labelSelector:
matchExpressions:
- key: app
operator: In
values:
- redis-cache
topologyKey: kubernetes.io/hostname
# Pod 反亲和性：前端副本分散到不同可用区
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchExpressions:
- key: app
operator: In
values:
- web-frontend
topologyKey: topology.kubernetes.io/zone
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
labelSelector:
matchExpressions:
- key: app
operator: In
values:
- web-frontend
topologyKey: kubernetes.io/hostname
containers:
- name: web
image: nginx:1.253.4 Taint 与 Toleration
# 给节点设置污点
kubectl taint nodes node-01 dedicated=gpu:NoSchedule
kubectl taint nodes node-02 dedicated=storage:NoSchedule
kubectl taint nodes node-03 special=true:NoExecute
# 查看节点污点
kubectl get nodes -o custom-columns=NAME:.metadata.name,TAINTS:.spec.taints
# 移除污点
kubectl taint nodes node-01 dedicated=gpu:NoSchedule-apiVersion: v1
kind: Pod
metadata:
name: gpu-task
spec:
tolerations:
- key: "dedicated"
operator: "Equal"
value: "gpu"
effect: "NoSchedule"
- key: "node.kubernetes.io/not-ready"
operator: "Exists"
effect: "NoExecute"
tolerationSeconds: 300
containers:
- name: ml-job
image: pytorch/pytorch:latest3.5 TopologySpreadConstraints
apiVersion: apps/v1
kind: Deployment
metadata:
name: web-app
spec:
replicas: 6
selector:
matchLabels:
app: web-app
template:
metadata:
labels:
app: web-app
spec:
topologySpreadConstraints:
- maxSkew: 1
topologyKey: topology.kubernetes.io/zone
whenUnsatisfiable: DoNotSchedule
labelSelector:
matchLabels:
app: web-app
- maxSkew: 1
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: ScheduleAnyway
labelSelector:
matchLabels:
app: web-app
containers:
- name: web
image: nginx:1.25
resources:
requests:
cpu: 250m
memory: 256Mi四、RBAC 权限管理
4.1 Role 与 RoleBinding
# Role：命名空间级别权限
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: pod-reader
namespace: production
rules:
- apiGroups: [""]
resources: ["pods", "pods/log"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["pods/exec"]
verbs: ["create"]
- apiGroups: [""]
resources: ["services", "endpoints"]
verbs: ["get", "list"]
---
# RoleBinding：绑定到用户和 ServiceAccount
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: read-pods
namespace: production
subjects:
- kind: User
name: developer@example.com
apiGroup: rbac.authorization.k8s.io
- kind: Group
name: dev-team
apiGroup: rbac.authorization.k8s.io
- kind: ServiceAccount
name: monitoring-sa
namespace: monitoring
roleRef:
kind: Role
name: pod-reader
apiGroup: rbac.authorization.k8s.io4.2 ClusterRole 与 ClusterRoleBinding
# ClusterRole：集群级别权限
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: cluster-reader
rules:
- apiGroups: [""]
resources: ["nodes", "namespaces", "persistentvolumes"]
verbs: ["get", "list", "watch"]
- apiGroups: ["metrics.k8s.io"]
resources: ["nodes", "pods"]
verbs: ["get", "list"]
- apiGroups: ["storage.k8s.io"]
resources: ["storageclasses"]
verbs: ["get", "list", "watch"]
---
# ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: cluster-reader-binding
subjects:
- kind: ServiceAccount
name: monitoring-sa
namespace: monitoring
roleRef:
kind: ClusterRole
name: cluster-reader
apiGroup: rbac.authorization.k8s.io4.3 ServiceAccount 实践
apiVersion: v1
kind: ServiceAccount
metadata:
name: app-sa
namespace: production
annotations:
eks.amazonaws.com/role-arn: arn:aws:iam::123456789012:role/app-role
automountServiceAccountToken: true
---
# 使用 ServiceAccount 的 Deployment
apiVersion: apps/v1
kind: Deployment
metadata:
name: web-app
namespace: production
spec:
replicas: 3
selector:
matchLabels:
app: web-app
template:
metadata:
labels:
app: web-app
spec:
serviceAccountName: app-sa
automountServiceAccountToken: true
containers:
- name: web
image: nginx:1.25# 查看 ServiceAccount 的权限
kubectl auth can-i --list --as=system:serviceaccount:production:app-sa
# 验证特定权限
kubectl auth can-i get pods --as=system:serviceaccount:production:app-sa -n production
kubectl auth can-i create deployments --as=system:serviceaccount:production:app-sa -n production
# 创建长期 Token（Kubernetes 1.24+）
kubectl create token app-sa -n production --duration=8760h五、网络策略
5.1 基本 NetworkPolicy
# 默认拒绝所有入站流量
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: default-deny-ingress
namespace: production
spec:
podSelector: {}
policyTypes:
- Ingress
---
# 默认拒绝所有出站流量
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: default-deny-egress
namespace: production
spec:
podSelector: {}
policyTypes:
- Egress
---
# 允许特定应用的入站和出站
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: web-app-policy
namespace: production
spec:
podSelector:
matchLabels:
app: web-app
policyTypes:
- Ingress
- Egress
ingress:
- from:
- namespaceSelector:
matchLabels:
name: ingress-nginx
- podSelector:
matchLabels:
app: api-gateway
ports:
- protocol: TCP
port: 8080
- from:
- podSelector:
matchLabels:
app: prometheus
ports:
- protocol: TCP
port: 9090
egress:
- to:
- podSelector:
matchLabels:
app: database
ports:
- protocol: TCP
port: 5432
- to: # 允许 DNS
- namespaceSelector: {}
podSelector:
matchLabels:
k8s-app: kube-dns
ports:
- protocol: UDP
port: 53
- protocol: TCP
port: 53
- to: # 允许访问外部 HTTPS
- ipBlock:
cidr: 0.0.0.0/0
except:
- 10.0.0.0/8
- 172.16.0.0/12
- 192.168.0.0/16
ports:
- protocol: TCP
port: 4435.2 Calico 网络策略
# Calico GlobalNetworkPolicy（集群级别）
apiVersion: projectcalico.org/v3
kind: GlobalNetworkPolicy
metadata:
name: deny-external-access
spec:
order: 100
selector: tenant == "internal"
types:
- Ingress
ingress:
- action: Deny
source:
nets:
- 0.0.0.0/0
notNets:
- 10.0.0.0/8
---
# Calico NetworkPolicy（支持更多功能）
apiVersion: projectcalico.org/v3
kind: NetworkPolicy
metadata:
name: advanced-web-policy
namespace: production
spec:
selector: app == "web-app"
types:
- Ingress
- Egress
ingress:
- action: Allow
protocol: TCP
source:
namespaceSelector: env == "staging"
serviceAccounts:
name: gateway-sa
destination:
ports:
- 8080
- 8443
egress:
- action: Allow
protocol: TCP
destination:
selector: app == "database"
ports:
- 5432
- action: Allow
protocol: UDP
destination:
ports:
- 535.3 Cilium 网络策略
# Cilium CiliumNetworkPolicy
apiVersion: cilium.io/v2
kind: CiliumNetworkPolicy
metadata:
name: l7-web-policy
namespace: production
spec:
endpointSelector:
matchLabels:
app: web-app
ingress:
- fromEndpoints:
- matchLabels:
app: api-gateway
toPorts:
- ports:
- port: "8080"
protocol: TCP
rules:
http:
- method: GET
path: "/api/v1/.*"
- method: POST
path: "/api/v1/submit"
headers:
- 'Content-Type: application/json'
egress:
- toEndpoints:
- matchLabels:
app: database
toPorts:
- ports:
- port: "5432"
protocol: TCP
- toFQDNs:
- matchName: "api.external.com"
- matchPattern: "*.s3.amazonaws.com"
toPorts:
- ports:
- port: "443"
protocol: TCP# 安装 Cilium
helm repo add cilium https://helm.cilium.io/
helm install cilium cilium/cilium --namespace kube-system \
--set hubble.enabled=true \
--set hubble.relay.enabled=true \
--set hubble.ui.enabled=true \
--set kubeProxyReplacement=strict
# 查看 Cilium 状态
cilium status
# 查看网络流量
cilium monitor --type drop六、存储进阶
6.1 CSI 驱动
# StorageClass（CSI）
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: fast-ssd
provisioner: disk.csi.aliyun.com
parameters:
type: cloud_essd
performanceLevel: PL1
fstype: ext4
encrypted: "true"
reclaimPolicy: Retain
allowVolumeExpansion: true
volumeBindingMode: WaitForFirstConsumer
mountOptions:
- noatime
- nodiratime
---
# PVC 使用 CSI StorageClass
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: app-data
namespace: production
spec:
accessModes:
- ReadWriteOnce
storageClassName: fast-ssd
resources:
requests:
storage: 100Gi6.2 VolumeSnapshot
# VolumeSnapshotClass
apiVersion: snapshot.storage.k8s.io/v1
kind: VolumeSnapshotClass
metadata:
name: csi-snapclass
driver: disk.csi.aliyun.com
deletionPolicy: Retain
parameters:
tags: "env=production,managedby=velero"
---
# 创建快照
apiVersion: snapshot.storage.k8s.io/v1
kind: VolumeSnapshot
metadata:
name: app-data-snapshot-20260610
namespace: production
spec:
volumeSnapshotClassName: csi-snapclass
source:
persistentVolumeClaimName: app-data# 查看快照
kubectl get volumesnapshot -n production
# 从快照恢复
cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: app-data-restored
namespace: production
spec:
accessModes:
- ReadWriteOnce
storageClassName: fast-ssd
resources:
requests:
storage: 100Gi
dataSource:
name: app-data-snapshot-20260610
kind: VolumeSnapshot
apiGroup: snapshot.storage.k8s.io
EOF6.3 Volume Clone
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: app-data-clone
namespace: production
spec:
accessModes:
- ReadWriteOnce
storageClassName: fast-ssd
resources:
requests:
storage: 100Gi
dataSource:
name: app-data
kind: PersistentVolumeClaim6.4 Velero 备份与恢复
# 安装 Velero
velero install \
--provider alibabacloud \
--plugins velero/velero-plugin-for-alibaba:v1.9 \
--bucket velero-backup \
--secret-file ./credentials-velero \
--backup-location-config region=cn-hangzhou \
--snapshot-location-config region=cn-hangzhou
# 创建备份
velero backup create production-backup \
--include-namespaces production \
--snapshot-volumes \
--ttl 720h
# 定时备份（每天凌晨 2 点）
velero schedule create production-daily \
--include-namespaces production \
--schedule="0 2 * * *" \
--ttl 720h
# 查看备份状态
velero backup describe production-backup
velero backup logs production-backup
# 恢复
velero restore create --from-backup production-backup# Velero Schedule CRD
apiVersion: velero.io/v1
kind: Schedule
metadata:
name: production-daily
namespace: velero
spec:
schedule: "0 2 * * *"
template:
includedNamespaces:
- production
snapshotVolumes: true
storageLocation: default
ttl: 720h
hooks:
resources:
- name: pre-backup-hook
includedNamespaces:
- production
labelSelector:
matchLabels:
backup-hook: "true"
pre:
- exec:
container: postgres
command:
- /bin/bash
- -c
- "PGPASSWORD=$POSTGRES_PASSWORD pg_dump -U postgres app_db > /backup/dump.sql"
onError: Fail
timeout: 300s七、Pod 调度进阶
7.1 DaemonSet
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: node-exporter
namespace: monitoring
spec:
selector:
matchLabels:
app: node-exporter
template:
metadata:
labels:
app: node-exporter
spec:
hostNetwork: true
hostPID: true
tolerations:
- operator: Exists
containers:
- name: node-exporter
image: prom/node-exporter:v1.7.0
ports:
- containerPort: 9100
hostPort: 9100
resources:
requests:
cpu: 50m
memory: 64Mi
limits:
cpu: 200m
memory: 256Mi
volumeMounts:
- name: proc
mountPath: /host/proc
readOnly: true
- name: sys
mountPath: /host/sys
readOnly: true
volumes:
- name: proc
hostPath:
path: /proc
- name: sys
hostPath:
path: /sys
updateStrategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 17.2 StatefulSet
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: postgres
namespace: production
spec:
serviceName: postgres
replicas: 3
selector:
matchLabels:
app: postgres
template:
metadata:
labels:
app: postgres
spec:
terminationGracePeriodSeconds: 30
containers:
- name: postgres
image: postgres:16
ports:
- containerPort: 5432
env:
- name: POSTGRES_PASSWORD
valueFrom:
secretKeyRef:
name: postgres-secret
key: password
- name: PGDATA
value: /var/lib/postgresql/data/pgdata
resources:
requests:
cpu: 500m
memory: 1Gi
limits:
cpu: "2"
memory: 4Gi
volumeMounts:
- name: data
mountPath: /var/lib/postgresql/data
livenessProbe:
exec:
command:
- pg_isready
- -U
- postgres
initialDelaySeconds: 30
periodSeconds: 10
readinessProbe:
exec:
command:
- pg_isready
- -U
- postgres
initialDelaySeconds: 5
periodSeconds: 5
volumeClaimTemplates:
- metadata:
name: data
spec:
accessModes:
- ReadWriteOnce
storageClassName: fast-ssd
resources:
requests:
storage: 100Gi
podManagementPolicy: Parallel
updateStrategy:
type: RollingUpdate7.3 Job 与 CronJob
# Job：一次性任务
apiVersion: batch/v1
kind: Job
metadata:
name: db-migration
namespace: production
spec:
backoffLimit: 3
activeDeadlineSeconds: 600
ttlSecondsAfterFinished: 3600
template:
spec:
restartPolicy: OnFailure
containers:
- name: migrate
image: myapp/migrations:v2.5.0
command: ["python", "manage.py", "migrate"]
env:
- name: DATABASE_URL
valueFrom:
secretKeyRef:
name: db-credentials
key: url
resources:
requests:
cpu: 250m
memory: 512Mi
---
# CronJob：定时任务
apiVersion: batch/v1
kind: CronJob
metadata:
name: nightly-report
namespace: production
spec:
schedule: "0 3 * * *"
concurrencyPolicy: Forbid
failedJobsHistoryLimit: 3
successfulJobsHistoryLimit: 5
startingDeadlineSeconds: 600
jobTemplate:
spec:
backoffLimit: 2
template:
spec:
restartPolicy: OnFailure
containers:
- name: report
image: myapp/report-generator:v1.0.0
command: ["python", "generate_report.py"]
resources:
requests:
cpu: 500m
memory: 1Gi
limits:
cpu: "2"
memory: 4Gi7.4 Init Container
apiVersion: v1
kind: Pod
metadata:
name: web-app
spec:
initContainers:
- name: wait-for-db
image: busybox:1.36
command:
- sh
- -c
- |
until nc -z postgres.production.svc.cluster.local 5432; do
echo "Waiting for database..."
sleep 2
done
- name: init-schema
image: myapp/migrations:v2.5.0
command: ["python", "manage.py", "init-schema"]
env:
- name: DATABASE_URL
valueFrom:
secretKeyRef:
name: db-credentials
key: url
- name: download-config
image: alpine:3.19
command:
- sh
- -c
- |
wget -O /config/app.conf https://config.internal.com/app.conf
wget -O /config/tls.crt https://vault.internal.com/certs/tls.crt
volumeMounts:
- name: config
mountPath: /config
containers:
- name: web
image: myapp/web:v2.5.0
volumeMounts:
- name: config
mountPath: /app/config
readOnly: true
volumes:
- name: config
emptyDir: {}7.5 Sidecar 模式
apiVersion: v1
kind: Pod
metadata:
name: app-with-sidecars
labels:
app: web-app
spec:
containers:
# 主容器
- name: web
image: nginx:1.25
ports:
- containerPort: 8080
volumeMounts:
- name: shared-logs
mountPath: /var/log/nginx
# Sidecar: 日志收集
- name: log-collector
image: fluent/fluent-bit:2.2
volumeMounts:
- name: shared-logs
mountPath: /var/log/nginx
readOnly: true
- name: fluent-config
mountPath: /fluent-bit/etc/
resources:
requests:
cpu: 50m
memory: 64Mi
# Sidecar: 代理
- name: envoy-proxy
image: envoyproxy/envoy:v1.28
ports:
- containerPort: 15001
name: envoy-port
volumeMounts:
- name: envoy-config
mountPath: /etc/envoy
resources:
requests:
cpu: 100m
memory: 128Mi
volumes:
- name: shared-logs
emptyDir: {}
- name: fluent-config
configMap:
name: fluent-bit-config
- name: envoy-config
configMap:
name: envoy-config八、故障排查
8.1 Pod 状态诊断
# 查看 Pod 详情（重点关注 Events 和 Conditions）
kubectl describe pod <pod-name> -n <namespace>
# 常见 Pod 状态及排查
# CrashLoopBackOff
kubectl logs <pod-name> -n <namespace> --previous
kubectl logs <pod-name> -n <namespace> -p --tail=100
# ImagePullBackOff
kubectl describe pod <pod-name> | grep -A5 "Events"
# 检查镜像名称、tag、仓库认证
kubectl get secret regcred -o jsonpath='{.data.\.dockerconfigjson}' | base64 -d
# Pending
kubectl describe pod <pod-name> | grep -A10 "Events"
# 常见原因：资源不足、节点选择器不匹配、PVC 未绑定
kubectl get nodes -o custom-columns=NAME:.metadata.name,CPU_ALLOC:.status.allocatable.cpu,MEM_ALLOC:.status.allocatable.memory
# OOMKilled
kubectl get pod <pod-name> -o jsonpath='{.status.containerStatuses[0].lastState.terminated.reason}'
kubectl top pod <pod-name> -n <namespace>8.2 Events 分析
# 按时间排序查看事件
kubectl get events -n <namespace> --sort-by='.lastTimestamp'
# 查看特定资源的事件
kubectl get events -n <namespace> --field-selector involvedObject.name=<pod-name>
# 集群级别警告事件
kubectl get events --all-namespaces --field-selector type=Warning --sort-by='.lastTimestamp' | tail -20
# 实时监控事件
kubectl get events -n <namespace> -w8.3 日志排查
# 实时日志
kubectl logs -f <pod-name> -n <namespace>
# 多容器 Pod 日志
kubectl logs <pod-name> -c <container-name> -n <namespace>
# 前一个容器实例的日志（崩溃后查看）
kubectl logs <pod-name> -p -n <namespace>
# 带时间范围的日志
kubectl logs <pod-name> -n <namespace> --since=1h
kubectl logs <pod-name> -n <namespace> --since-time=2026-06-10T06:00:00Z
# 按标签查询多个 Pod 日志
kubectl logs -l app=web-app -n <namespace> --all-containers=true --tail=50
# 日志导出
kubectl logs <pod-name> -n <namespace> > pod.log 2>&18.4 Exec 调试
# 进入容器
kubectl exec -it <pod-name> -n <namespace> -- /bin/sh
# 多容器 Pod
kubectl exec -it <pod-name> -c <container-name> -n <namespace> -- /bin/bash
# 网络诊断
kubectl exec -it <pod-name> -n <namespace> -- curl -v http://service-name:8080/health
kubectl exec -it <pod-name> -n <namespace> -- nslookup kubernetes.default
kubectl exec -it <pod-name> -n <namespace> -- nc -zv <service-name> <port>
# 文件系统检查
kubectl exec -it <pod-name> -n <namespace> -- df -h
kubectl exec -it <pod-name> -n <namespace> -- ls -la /app/config/8.5 网络诊断
# DNS 诊断
kubectl run dns-test --image=busybox:1.36 --rm -it --restart=Never -- nslookup kubernetes.default
kubectl run dns-test --image=busybox:1.36 --rm -it --restart=Never -- nslookup my-service.production.svc.cluster.local
# 连通性测试
kubectl run curl-test --image=curlimages/curl --rm -it --restart=Never -- \
curl -v http://web-app.production.svc.cluster.local:8080/health
# 使用临时调试容器（Kubernetes 1.25+）
kubectl debug -it <pod-name> -n <namespace> --image=nicolaka/netshoot --target=<container-name>
# 查看 Service 端点
kubectl get endpoints <service-name> -n <namespace>
# 查看 iptables 规则（在节点上执行）
iptables -t nat -L KUBE-SERVICES -n | grep <service-name>
# 查看 CoreDNS 日志
kubectl logs -l k8s-app=kube-dns -n kube-system --tail=100九、集群运维
9.1 节点维护
# 标记节点不可调度（新 Pod 不会调度到该节点）
kubectl cordon node-02
# 驱逐节点上的 Pod（自动调度到其他节点）
kubectl drain node-02 \
--ignore-daemonsets \
--delete-emptydir-data \
--grace-period=60 \
--timeout=300s
# 执行维护操作（升级内核、更换硬件等）
# ... 维护完成 ...
# 恢复节点调度
kubectl uncordon node-02
# 验证节点状态
kubectl get nodes
kubectl describe node node-02 | grep -A5 Conditions9.2 etcd 备份与恢复
# 备份 etcd
ETCDCTL_API=3 etcdctl snapshot save /backup/etcd-$(date +%Y%m%d-%H%M%S).db \
--endpoints=https://127.0.0.1:2379 \
--cacert=/etc/kubernetes/pki/etcd/ca.crt \
--cert=/etc/kubernetes/pki/etcd/server.crt \
--key=/etc/kubernetes/pki/etcd/server.key
# 验证备份
ETCDCTL_API=3 etcdctl snapshot status /backup/etcd-20260610-145800.db --write-table
# 自动备份 CronJob
cat <<EOF | kubectl apply -f -
apiVersion: batch/v1
kind: CronJob
metadata:
name: etcd-backup
namespace: kube-system
spec:
schedule: "0 */6 * * *"
jobTemplate:
spec:
template:
spec:
hostNetwork: true
nodeSelector:
node-role.kubernetes.io/control-plane: ""
tolerations:
- operator: Exists
containers:
- name: backup
image: registry.k8s.io/etcd:3.5.12
command:
- /bin/sh
- -c
- |
etcdctl snapshot save /backup/etcd-\$(date +%Y%m%d-%H%M%S).db
find /backup -name "etcd-*.db" -mtime +7 -delete
env:
- name: ETCDCTL_API
value: "3"
- name: ETCDCTL_ENDPOINTS
value: "https://127.0.0.1:2379"
- name: ETCDCTL_CACERT
value: "/etc/kubernetes/pki/etcd/ca.crt"
- name: ETCDCTL_CERT
value: "/etc/kubernetes/pki/etcd/server.crt"
- name: ETCDCTL_KEY
value: "/etc/kubernetes/pki/etcd/server.key"
volumeMounts:
- name: etcd-certs
mountPath: /etc/kubernetes/pki/etcd
readOnly: true
- name: backup
mountPath: /backup
volumes:
- name: etcd-certs
hostPath:
path: /etc/kubernetes/pki/etcd
- name: backup
hostPath:
path: /var/backups/etcd
restartPolicy: OnFailure
EOF
# 恢复 etcd（灾难恢复场景）
ETCDCTL_API=3 etcdctl snapshot restore /backup/etcd-20260610-145800.db \
--data-dir=/var/lib/etcd-restore \
--name=etcd-0 \
--initial-cluster=etcd-0=https://192.168.1.10:2380 \
--initial-advertise-peer-urls=https://192.168.1.10:23809.3 证书轮换
# 查看证书过期时间
kubeadm certs check-expiration
# 使用 kubeadm 轮换所有证书
kubeadm certs renew all
# 轮换特定证书
kubeadm certs renew apiserver
kubeadm certs renew apiserver-kubelet-client
kubeadm certs renew front-proxy-client
# 重启控制面组件
systemctl restart kubelet
# 手动轮换 kubeconfig
kubeadm kubeconfig user --client-name=admin --org=system:masters > /etc/kubernetes/admin.conf
cp /etc/kubernetes/admin.conf ~/.kube/config
# 验证新证书
openssl x509 -in /etc/kubernetes/pki/apiserver.crt -noout -dates9.4 集群升级
# 1. 升级 kubeadm
apt-get update && apt-get install -y kubeadm=1.30.0-*
# 2. 检查升级计划
kubeadm upgrade plan
# 3. 升级控制面
kubeadm upgrade apply v1.30.0
# 4. 升级 kubelet 和 kubectl
apt-get install -y kubelet=1.30.0-* kubectl=1.30.0-*
systemctl daemon-reload
systemctl restart kubelet
# 5. 逐个升级 Worker 节点
# 在 Master 上：
kubectl drain worker-01 --ignore-daemonsets --delete-emptydir-data
# 在 Worker 上：
apt-get update && apt-get install -y kubeadm=1.30.0-*
kubeadm upgrade node
apt-get install -y kubelet=1.30.0-* kubectl=1.30.0-*
systemctl daemon-reload
systemctl restart kubelet
# 在 Master 上：
kubectl uncordon worker-01
# 6. 验证升级
kubectl get nodes
kubectl version十、生产最佳实践
10.1 Namespace 规划
# 标准命名空间结构
apiVersion: v1
kind: Namespace
metadata:
name: production
labels:
env: production
managed-by: platform-team
---
apiVersion: v1
kind: Namespace
metadata:
name: staging
labels:
env: staging
managed-by: platform-team
---
apiVersion: v1
kind: Namespace
metadata:
name: monitoring
labels:
env: shared
managed-by: platform-team10.2 资源配额模板
apiVersion: v1
kind: ResourceQuota
metadata:
name: team-quota
namespace: ${TEAM_NS}
spec:
hard:
requests.cpu: "20"
requests.memory: 40Gi
limits.cpu: "40"
limits.memory: 80Gi
pods: "100"
services: "20"
persistentvolumeclaims: "30"
requests.storage: 2Ti
---
apiVersion: v1
kind: LimitRange
metadata:
name: default-limits
namespace: ${TEAM_NS}
spec:
limits:
- type: Container
default:
cpu: 500m
memory: 512Mi
defaultRequest:
cpu: 100m
memory: 128Mi10.3 镜像策略
# Kyverno 镜像策略
apiVersion: kyverno.io/v1
kind: ClusterPolicy
metadata:
name: image-policy
spec:
validationFailureAction: Enforce
background: true
rules:
- name: require-image-tag
match:
any:
- resources:
kinds:
- Pod
validate:
message: "镜像必须指定 tag，禁止使用 latest"
pattern:
spec:
containers:
- image: "!*:latest & *:*"
- name: verify-signature
match:
any:
- resources:
kinds:
- Pod
verifyImages:
- imageReferences:
- "registry.example.com/*"
attestors:
- entries:
- keys:
publicKeys: |-
-----BEGIN PUBLIC KEY-----
MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE...
-----END PUBLIC KEY-----
- name: restrict-registries
match:
any:
- resources:
kinds:
- Pod
validate:
message: "只允许从公司内部仓库拉取镜像"
pattern:
spec:
containers:
- image: "registry.example.com/*"10.4 安全加固
# Pod Security Standards
apiVersion: v1
kind: Namespace
metadata:
name: production
labels:
pod-security.kubernetes.io/enforce: restricted
pod-security.kubernetes.io/audit: restricted
pod-security.kubernetes.io/warn: restricted
---
# 安全上下文最佳实践
apiVersion: apps/v1
kind: Deployment
metadata:
name: secure-app
namespace: production
spec:
replicas: 3
selector:
matchLabels:
app: secure-app
template:
metadata:
labels:
app: secure-app
spec:
automountServiceAccountToken: false
securityContext:
runAsNonRoot: true
runAsUser: 1000
runAsGroup: 3000
fsGroup: 2000
seccompProfile:
type: RuntimeDefault
containers:
- name: app
image: registry.example.com/app:v2.5.0
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop:
- ALL
resources:
requests:
cpu: 250m
memory: 256Mi
limits:
cpu: "1"
memory: 1Gi
volumeMounts:
- name: tmp
mountPath: /tmp
- name: app-config
mountPath: /app/config
readOnly: true
livenessProbe:
httpGet:
path: /healthz
port: 8080
initialDelaySeconds: 15
periodSeconds: 10
readinessProbe:
httpGet:
path: /ready
port: 8080
initialDelaySeconds: 5
periodSeconds: 5
volumes:
- name: tmp
emptyDir:
medium: Memory
sizeLimit: 100Mi
- name: app-config
configMap:
name: app-config10.5 GitOps 工作流
# ArgoCD Application
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: web-app
namespace: argocd
spec:
project: production
source:
repoURL: https://git.internal.com/platform/k8s-manifests.git
targetRevision: main
path: apps/web-app/overlays/production
destination:
server: https://kubernetes.default.svc
namespace: production
syncPolicy:
automated:
prune: true
selfHeal: true
syncOptions:
- CreateNamespace=true
- PrunePropagationPolicy=foreground
- ServerSideApply=true
retry:
limit: 5
backoff:
duration: 5s
factor: 2
maxDuration: 3m
---
# ArgoCD AppProject
apiVersion: argoproj.io/v1alpha1
kind: AppProject
metadata:
name: production
namespace: argocd
spec:
description: "生产环境项目"
sourceRepos:
- "https://git.internal.com/platform/*"
destinations:
- namespace: "production"
server: "https://kubernetes.default.svc"
- namespace: "monitoring"
server: "https://kubernetes.default.svc"
clusterResourceWhitelist:
- group: "*"
kind: Namespace
namespaceResourceWhitelist:
- group: "*"
kind: "*"
roles:
- name: developer
description: "开发者角色"
policies:
- p, proj:production:developer, applications, get, production/*, allow
- p, proj:production:developer, applications, sync, production/*, allow# 安装 ArgoCD
kubectl create namespace argocd
kubectl apply -n argocd -f https://raw.githubusercontent.com/argoproj/argo-cd/stable/manifests/install.yaml
# 获取初始密码
argocd admin initial-password -n argocd
# 配置 GitOps 仓库
argocd repo add https://git.internal.com/platform/k8s-manifests.git \
--username deploy-token \
--password <token>
# 同步应用
argocd app sync web-app
argocd app get web-app附录：常用命令速查
参考文档