服务器运维笔记：Kubernetes 入门部署

┌─────────────────────────────────┐
│         Control Plane (Master)  │
│  ┌──────────┐  ┌─────────────┐  │
│  │ API Server│  │  Scheduler  │  │
│  └──────────┘  └─────────────┘  │
│  ┌──────────┐  ┌─────────────┐  │
│  │Controller │  │   etcd      │  │
│  │ Manager   │  │ (KV Store)  │  │
│  └──────────┘  └─────────────┘  │
└─────────────────────────────────┘
         ▲  API  ▲
         │       │
┌────────┴───────┴────────────────┐
│         Worker Nodes            │
│  ┌──────┐ ┌──────┐ ┌────────┐  │
│  │kubelet│ │kube- │ │Container│  │
│  │      │ │proxy │ │ Runtime │  │
│  └──────┘ └──────┘ └────────┘  │
│  ┌──────────────────────────┐   │
│  │   Pod  Pod  Pod  Pod     │   │
│  └──────────────────────────┘   │
└─────────────────────────────────┘

etcdctl --endpoints=https://127.0.0.1:2379 \
  --cacert=/etc/kubernetes/pki/etcd/ca.crt \
  --cert=/etc/kubernetes/pki/etcd/server.crt \
  --key=/etc/kubernetes/pki/etcd/server.key \
  endpoint health

# 关闭 swap（Kubernetes 要求）
sudo swapoff -a
sudo sed -i '/ swap / s/^/#/' /etc/fstab

# 关闭 SELinux（CentOS/RHEL）
# 仅在对应发行版 kubeadm 文档明确要求时评估 SELinux 模式；先保留现有策略
sudo sed -i 's/^SELINUX=enforcing/SELINUX=disabled/' /etc/selinux/config

# 加载内核模块
cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf
overlay
br_netfilter
EOF
sudo modprobe overlay
sudo modprobe br_netfilter

# 配置 sysctl
cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-iptables  = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward                 = 1
EOF
sudo sysctl --system

# 配置主机名和 hosts
sudo hostnamectl set-hostname k8s-master  # 各节点设置不同主机名
cat <<EOF | sudo tee -a /etc/hosts
192.168.1.10  k8s-master
192.168.1.11  k8s-node1
192.168.1.12  k8s-node2
EOF

# 安装 containerd
sudo apt-get update
sudo apt-get install -y containerd
sudo mkdir -p /etc/containerd
containerd config default | sudo tee /etc/containerd/config.toml
# 修改 SystemdCgroup = true
sudo sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml
sudo systemctl restart containerd
sudo systemctl enable containerd

# 添加 Kubernetes APT 源（以 v1.30 为例）
sudo apt-get install -y apt-transport-https ca-certificates curl gpg
curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.30/deb/Release.key | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.30/deb/ /' | sudo tee /etc/apt/sources.list.d/kubernetes.list

sudo apt-get update
sudo apt-get install -y kubelet kubeadm kubectl
sudo apt-mark hold kubelet kubeadm kubectl
sudo systemctl enable kubelet

# 添加 YUM 源
cat <<EOF | sudo tee /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://pkgs.k8s.io/core:/stable:/v1.30/rpm/
enabled=1
gpgcheck=1
gpgkey=https://pkgs.k8s.io/core:/stable:/v1.30/rpm/repodata/repomd.xml.key
EOF
sudo yum install -y kubelet kubeadm kubectl --disableexcludes=kubernetes
sudo systemctl enable kubelet

# 在 k8s-master 上执行
sudo kubeadm init \
  --apiserver-advertise-address=192.168.1.10 \
  --pod-network-cidr=10.244.0.0/16 \
  --service-cidr=10.96.0.0/12 \
  --image-repository registry.aliyuncs.com/google_containers

# 初始化完成后，配置 kubectl
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

kubeadm join 192.168.1.10:6443 \
  --token abcdef.0123456789abcdef \
  --discovery-token-ca-cert-hash sha256:xxxxxxxxxxxxxxxxxxxx

# 在每个 Worker 节点上执行
sudo kubeadm join 192.168.1.10:6443 \
  --token abcdef.0123456789abcdef \
  --discovery-token-ca-cert-hash sha256:xxxxxxxxxxxxxxxxxxxx

kubeadm token create --print-join-command

kubectl apply -f https://raw.githubusercontent.com/projectcalico/calico/v3.27.0/manifests/calico.yaml

kubectl apply -f https://github.com/flannel-io/flannel/releases/latest/download/kube-flannel.yml

kubectl get nodes
# NAME          STATUS   ROLES           AGE   VERSION
# k8s-master    Ready    control-plane   5m    v1.30.0
# k8s-node1     Ready    <none>          3m    v1.30.0
# k8s-node2     Ready    <none>          3m    v1.30.0

kubectl get pods -n kube-system

# 查看集群信息
kubectl cluster-info

# 查看所有节点
kubectl get nodes -o wide

# 查看所有命名空间的 Pod
kubectl get pods --all-namespaces

# 查看资源详情（以 Pod 为例）
kubectl describe pod <pod-name> -n <namespace>

# 查看资源的 YAML 定义
kubectl get deployment nginx -o yaml

# 以 JSON 格式输出
kubectl get svc -o json

# 查看所有资源类型
kubectl api-resources

# 从 YAML 文件创建资源
kubectl apply -f deployment.yaml

# 删除资源
kubectl delete -f deployment.yaml
kubectl delete pod <pod-name>

# 强制删除卡在 Terminating 的 Pod
kubectl delete pod <pod-name> --grace-period=0 --force

# 扩缩容
kubectl scale deployment nginx --replicas=5

# 滚动更新
kubectl set image deployment/nginx nginx=nginx:1.25

# 查看更新状态
kubectl rollout status deployment/nginx

# 回滚到上一版本
kubectl rollout undo deployment/nginx

# 查看修订历史
kubectl rollout history deployment/nginx

# 查看 Pod 日志
kubectl logs <pod-name>
kubectl logs <pod-name> -c <container-name>  # 多容器 Pod
kubectl logs -f <pod-name> --tail=100         # 实时尾部 100 行

# 进入容器
kubectl exec -it <pod-name> -- /bin/bash

# 查看资源使用（需要 metrics-server）
kubectl top pods
kubectl top nodes

# 快速创建临时 Pod 进行测试
kubectl run debug --rm -it --image=busybox -- /bin/sh

# 查看当前上下文
kubectl config current-context

# 切换上下文
kubectl config use-context <context-name>

# 设置默认命名空间
kubectl config set-context --current --namespace=production

# 创建命名空间
kubectl create namespace dev

# pod-demo.yaml
apiVersion: v1
kind: Pod
metadata:
  name: nginx-demo
  labels:
    app: nginx
    env: demo
spec:
  containers:
  - name: nginx
    image: nginx:1.25
    ports:
    - containerPort: 80

kubectl apply -f pod-demo.yaml
kubectl get pod nginx-demo -o wide

apiVersion: v1
kind: Pod
metadata:
  name: app-with-sidecar
spec:
  containers:
  # 主容器：Web 应用
  - name: app
    image: myapp:v1
    ports:
    - containerPort: 8080
    volumeMounts:
    - name: shared-logs
      mountPath: /var/log/app

  # Sidecar 容器：日志收集
  - name: log-collector
    image: fluentd:v1.16
    volumeMounts:
    - name: shared-logs
      mountPath: /var/log/app
      readOnly: true

  volumes:
  - name: shared-logs
    emptyDir: {}

apiVersion: v1
kind: Pod
metadata:
  name: resource-demo
spec:
  containers:
  - name: app
    image: nginx:1.25
    resources:
      requests:        # 调度依据：最低保障
        cpu: "250m"    # 0.25 核
        memory: "128Mi"
      limits:          # 硬上限：超出会被 OOM Kill
        cpu: "500m"
        memory: "256Mi"

apiVersion: v1
kind: Pod
metadata:
  name: healthcheck-demo
spec:
  containers:
  - name: app
    image: myapp:v1
    ports:
    - containerPort: 8080

    # 存活探针：失败则重启容器
    livenessProbe:
      httpGet:
        path: /healthz
        port: 8080
      initialDelaySeconds: 10
      periodSeconds: 10
      failureThreshold: 3

    # 就绪探针：失败则从 Service 端点移除
    readinessProbe:
      httpGet:
        path: /ready
        port: 8080
      initialDelaySeconds: 5
      periodSeconds: 5

    # 启动探针：保护慢启动应用
    startupProbe:
      httpGet:
        path: /healthz
        port: 8080
      failureThreshold: 30
      periodSeconds: 10

# deployment-nginx.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: nginx
  labels:
    app: nginx
spec:
  replicas: 3
  selector:
    matchLabels:
      app: nginx
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxSurge: 1          # 滚动更新时最多多出的 Pod 数
      maxUnavailable: 0     # 更新过程中不允许不可用
  template:
    metadata:
      labels:
        app: nginx
    spec:
      containers:
      - name: nginx
        image: nginx:1.25
        ports:
        - containerPort: 80
        resources:
          requests:
            cpu: "100m"
            memory: "64Mi"
          limits:
            cpu: "200m"
            memory: "128Mi"

kubectl apply -f deployment-nginx.yaml
kubectl get deployment nginx
kubectl get replicasets
kubectl get pods -l app=nginx

# 方式一：修改镜像
kubectl set image deployment/nginx nginx=nginx:1.26

# 方式二：编辑 YAML
kubectl edit deployment nginx

# 查看更新进度
kubectl rollout status deployment/nginx

# 暂停/恢复滚动更新（用于批量修改多个字段）
kubectl rollout pause deployment/nginx
kubectl set image deployment/nginx nginx=nginx:1.26
kubectl set resources deployment/nginx -c=nginx --limits=cpu=300m
kubectl rollout resume deployment/nginx

# 查看修订历史
kubectl rollout history deployment/nginx

# 查看特定版本的详情
kubectl rollout history deployment/nginx --revision=2

# 回滚到上一版本
kubectl rollout undo deployment/nginx

# 回滚到指定版本
kubectl rollout undo deployment/nginx --to-revision=1

# hpa-nginx.yaml
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: nginx-hpa
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: nginx
  minReplicas: 2
  maxReplicas: 10
  metrics:
  - type: Resource
    resource:
      name: cpu
      target:
        type: Utilization
        averageUtilization: 70
  - type: Resource
    resource:
      name: memory
      target:
        type: Utilization
        averageUtilization: 80
  behavior:
    scaleDown:
      stabilizationWindowSeconds: 300   # 缩容冷静期 5 分钟
      policies:
      - type: Percent
        value: 10
        periodSeconds: 60

# 前提：安装 metrics-server
kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml

kubectl apply -f hpa-nginx.yaml
kubectl get hpa
kubectl describe hpa nginx-hpa

apiVersion: v1
kind: Service
metadata:
  name: nginx-clusterip
spec:
  type: ClusterIP
  selector:
    app: nginx
  ports:
  - port: 80           # Service 端口
    targetPort: 80      # 容器端口
    protocol: TCP

kubectl get svc nginx-clusterip
# 内部访问：curl http://nginx-clusterip.default.svc.cluster.local

apiVersion: v1
kind: Service
metadata:
  name: nginx-nodeport
spec:
  type: NodePort
  selector:
    app: nginx
  ports:
  - port: 80
    targetPort: 80
    nodePort: 30080     # 可选：指定端口，不指定则自动分配

apiVersion: v1
kind: Service
metadata:
  name: nginx-lb
  annotations:
    service.beta.kubernetes.io/alibaba-cloud-loadbalancer-spec: slb.s1.small
spec:
  type: LoadBalancer
  selector:
    app: nginx
  ports:
  - port: 80
    targetPort: 80

kubectl get endpoints nginx-clusterip
# NAME               ENDPOINTS                                 AGE
# nginx-clusterip    10.244.1.5:80,10.244.2.8:80,10.244.3.3:80  5m

apiVersion: v1
kind: Endpoints
metadata:
  name: external-db
subsets:
- addresses:
  - ip: 10.0.0.100
  ports:
  - port: 5432
    protocol: TCP
---
apiVersion: v1
kind: Service
metadata:
  name: external-db
spec:
  ports:
  - port: 5432
    targetPort: 5432

# 使用 Helm 安装
helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx
helm repo update
helm install ingress-nginx ingress-nginx/ingress-nginx \
  --namespace ingress-nginx \
  --create-namespace \
  --set controller.service.type=NodePort \
  --set controller.service.nodePorts.http=30080 \
  --set controller.service.nodePorts.https=30443

kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.10.0/deploy/static/provider/cloud/deploy.yaml

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: app-ingress
  annotations:
    nginx.ingress.kubernetes.io/rewrite-target: /
spec:
  ingressClassName: nginx
  rules:
  - host: app.example.com
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: nginx-clusterip
            port:
              number: 80
      - path: /api
        pathType: Prefix
        backend:
          service:
            name: api-service
            port:
              number: 8080

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: tls-ingress
  annotations:
    nginx.ingress.kubernetes.io/ssl-redirect: "true"
spec:
  ingressClassName: nginx
  tls:
  - hosts:
    - app.example.com
    secretName: app-tls-secret    # 包含 tls.crt 和 tls.key 的 Secret
  rules:
  - host: app.example.com
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: nginx-clusterip
            port:
              number: 80

kubectl create secret tls app-tls-secret \
  --cert=tls.crt \
  --key=tls.key

metadata:
  annotations:
    # 限速
    nginx.ingress.kubernetes.io/limit-rps: "10"
    nginx.ingress.kubernetes.io/limit-connections: "5"

    # 请求体大小限制
    nginx.ingress.kubernetes.io/proxy-body-size: "50m"

    # 超时设置
    nginx.ingress.kubernetes.io/proxy-connect-timeout: "10"
    nginx.ingress.kubernetes.io/proxy-read-timeout: "60"
    nginx.ingress.kubernetes.io/proxy-send-timeout: "60"

    # CORS
    nginx.ingress.kubernetes.io/enable-cors: "true"
    nginx.ingress.kubernetes.io/cors-allow-origin: "https://frontend.example.com"

    # 自定义错误页面
    nginx.ingress.kubernetes.io/custom-http-errors: "404,503"

    # 跨域 WebSocket 支持
    nginx.ingress.kubernetes.io/proxy-http-version: "1.1"
    nginx.ingress.kubernetes.io/upstream-hash-by: "$remote_addr"

apiVersion: v1
kind: ConfigMap
metadata:
  name: app-config
data:
  APP_ENV: "production"
  APP_DEBUG: "false"
  database.properties: |
    db.host=mysql.default.svc.cluster.local
    db.port=3306
    db.name=myapp

kubectl create configmap nginx-conf --from-file=nginx.conf
kubectl create configmap app-config --from-env-file=config.env

apiVersion: v1
kind: Pod
metadata:
  name: config-demo
spec:
  containers:
  - name: app
    image: myapp:v1
    # 方式一：环境变量
    env:
    - name: APP_ENV
      valueFrom:
        configMapKeyRef:
          name: app-config
          key: APP_ENV
    # 方式二：整个 ConfigMap 映射为环境变量
    envFrom:
    - configMapRef:
        name: app-config
    # 方式三：挂载为文件
    volumeMounts:
    - name: config-volume
      mountPath: /etc/config
  volumes:
  - name: config-volume
    configMap:
      name: app-config

apiVersion: v1
kind: Secret
metadata:
  name: db-secret
type: Opaque
data:
  DB_USER: YWRtaW4=          # echo -n 'admin' | base64
  DB_PASS: cEBzc3cwcmQxMjM=  # echo -n 'p@ssw0rd123' | base64

# 使用 kubectl 更方便
kubectl create secret generic db-secret \
  --from-literal=DB_USER=admin \
  --from-literal=DB_PASS='p@ssw0rd123'

# 创建 TLS Secret
kubectl create secret tls my-tls --cert=cert.pem --key=key.pem

# 创建 Docker Registry Secret
kubectl create secret docker-registry regcred \
  --docker-server=registry.example.com \
  --docker-username=user \
  --docker-password=pass

spec:
  containers:
  - name: app
    image: myapp:v1
    env:
    - name: DB_USER
      valueFrom:
        secretKeyRef:
          name: db-secret
          key: DB_USER
    - name: DB_PASS
      valueFrom:
        secretKeyRef:
          name: db-secret
          key: DB_PASS
    volumeMounts:
    - name: secret-volume
      mountPath: /etc/secrets
      readOnly: true
  volumes:
  - name: secret-volume
    secret:
      secretName: db-secret

用户提交 PVC → Kubernetes 匹配合适的 PV → 绑定 → Pod 挂载使用

apiVersion: v1
kind: PersistentVolume
metadata:
  name: nfs-pv
spec:
  capacity:
    storage: 10Gi
  accessModes:
  - ReadWriteMany        # 多节点读写
  persistentVolumeReclaimPolicy: Retain   # 删除 PVC 后保留数据
  nfs:
    server: 192.168.1.100
    path: /data/k8s-volumes/pv1

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: nfs-pvc
spec:
  accessModes:
  - ReadWriteMany
  resources:
    requests:
      storage: 10Gi
  # storageClassName: ""   # 空字符串表示不使用动态供给

apiVersion: v1
kind: Pod
metadata:
  name: nfs-pod
spec:
  containers:
  - name: app
    image: nginx:1.25
    volumeMounts:
    - name: data
      mountPath: /usr/share/nginx/html
  volumes:
  - name: data
    persistentVolumeClaim:
      claimName: nfs-pvc

apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: nfs-client
provisioner: nfs-subdir-external-provisioner  # 需要部署 Provisioner
parameters:
  archiveOnDelete: "false"
  server: 192.168.1.100
  share: /data/k8s-volumes
reclaimPolicy: Delete
volumeBindingMode: Immediate

helm repo add nfs-subdir-external-provisioner https://kubernetes-sigs.github.io/nfs-subdir-external-provisioner
helm install nfs-provisioner nfs-subdir-external-provisioner/nfs-subdir-external-provisioner \
  --set nfs.server=192.168.1.100 \
  --set nfs.path=/data/k8s-volumes \
  --set storageClass.name=nfs-client

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: auto-pvc
spec:
  storageClassName: nfs-client
  accessModes:
  - ReadWriteMany
  resources:
    requests:
      storage: 5Gi

# 方式一：脚本安装
curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash

# 方式二：包管理器
# Ubuntu/Debian
sudo apt-get install helm
# CentOS/RHEL
sudo yum install helm

# 验证
helm version

mychart/
├── Chart.yaml          # Chart 元数据（名称、版本）
├── Chart.lock          # 依赖锁定
├── values.yaml         # 默认配置值
├── templates/          # 模板文件
│   ├── deployment.yaml
│   ├── service.yaml
│   ├── ingress.yaml
│   ├── _helpers.tpl    # 模板辅助函数
│   └── tests/
├── charts/             # 子 Chart 依赖
└── README.md

apiVersion: v2
name: myapp
description: A Helm chart for My Application
type: application
version: 1.0.0          # Chart 版本
appVersion: "2.1.0"     # 应用版本
dependencies:
- name: redis
  version: "18.x.x"
  repository: "https://charts.bitnami.com/bitnami"
  condition: redis.enabled

replicaCount: 3

image:
  repository: myapp
  tag: "2.1.0"
  pullPolicy: IfNotPresent

service:
  type: ClusterIP
  port: 80

ingress:
  enabled: true
  className: nginx
  hosts:
  - host: myapp.example.com
    paths:
    - path: /
      pathType: Prefix
  tls: []

resources:
  requests:
    cpu: 100m
    memory: 128Mi
  limits:
    cpu: 500m
    memory: 256Mi

# 添加仓库
helm repo add bitnami https://charts.bitnami.com/bitnami
helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx
helm repo update

# 搜索 Chart
helm search repo nginx
helm search hub wordpress

# 安装 Chart
helm install my-nginx bitnami/nginx -n web --create-namespace

# 自定义 values 安装
helm install my-app ./mychart -f custom-values.yaml
helm install my-app ./mychart --set replicaCount=5 --set image.tag=v3

# 查看已安装的 Release
helm list -A

# 查看 Release 状态
helm status my-nginx

# 查看当前 values
helm get values my-nginx

# 查看全部 values（包含默认值）
helm get values my-nginx --all

# 升级
helm upgrade my-nginx bitnami/nginx --set replicaCount=3

# 升级或安装（不存在则安装）
helm upgrade --install my-nginx bitnami/nginx -f values.yaml

# 回滚
helm rollback my-nginx 1          # 回滚到版本 1

# 查看修订历史
helm history my-nginx

# 卸载
helm uninstall my-nginx -n web

# 干跑（不实际执行）
helm install my-app ./mychart --dry-run --debug

# 模板渲染（调试模板）
helm template my-app ./mychart -f values.yaml

# 脚手架创建
helm create mychart

# 验证 Chart 语法
helm lint mychart

# 打包
helm package mychart

# 推送到 OCI Registry（Helm 3.8+）
helm push mychart-1.0.0.tgz oci://registry.example.com/charts

# 推送到 ChartMuseum
helm cm-push mychart-1.0.0.tgz my-repo