给 AI 工程师的 Kubernetes 速通：概念与最小可用方案

# 先确认当前操作的是哪个集群
kubectl config current-context

# 需要相应的集群级只读权限
kubectl get nodes -L kubernetes.io/arch,nvidia.com/gpu.product
kubectl get nodes -o custom-columns='NAME:.metadata.name,GPU:.status.allocatable.nvidia\.com/gpu'
kubectl get storageclass

apiVersion: v1
kind: Namespace
metadata:
  name: ai-demo

kubectl apply -f manifests/00-namespace.yaml

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: train-output
  namespace: ai-demo
spec:
  accessModes: [ReadWriteOnce]
  resources:
    requests:
      storage: 5Gi
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: train-code
  namespace: ai-demo
data:
  train.py: |
    import os
    from pathlib import Path
    import torch

    assert torch.cuda.is_available(), "CUDA is not available"
    torch.manual_seed(42)
    device = torch.device("cuda")
    print("GPU:", torch.cuda.get_device_name(0), flush=True)

    x = torch.randn(1024, 32, device=device)
    y = x.sum(dim=1, keepdim=True)
    model = torch.nn.Linear(32, 1).to(device)
    optimizer = torch.optim.Adam(model.parameters(), lr=0.03)
    for step in range(100):
        optimizer.zero_grad(set_to_none=True)
        loss = torch.nn.functional.mse_loss(model(x), y)
        loss.backward()
        optimizer.step()
        if step % 20 == 0:
            print(f"step={step} loss={loss.item():.6f}", flush=True)

    # 各 Pod 独立目录，避免重试或替代 Pod 覆盖同一路径
    output = Path("/outputs") / os.environ["POD_UID"]
    output.mkdir(parents=True, exist_ok=True)
    temporary = output / "checkpoint.tmp"
    torch.save(model.state_dict(), temporary)
    temporary.replace(output / "checkpoint.pt")
    print("saved:", output / "checkpoint.pt", flush=True)
---
apiVersion: batch/v1
kind: Job
metadata:
  name: gpu-train-demo
  namespace: ai-demo
spec:
  backoffLimit: 0
  activeDeadlineSeconds: 1800
  template:
    spec:
      restartPolicy: Never
      automountServiceAccountToken: false
      nodeSelector:
        kubernetes.io/arch: amd64
      containers:
        - name: trainer
          image: pytorch/pytorch:2.5.1-cuda12.4-cudnn9-runtime
          command: [python, -u, /app/train.py]
          env:
            - name: POD_UID
              valueFrom:
                fieldRef:
                  fieldPath: metadata.uid
          resources:
            requests:
              cpu: "2"
              memory: 4Gi
            limits:
              cpu: "4"
              memory: 8Gi
              nvidia.com/gpu: "1"
          volumeMounts:
            - name: code
              mountPath: /app
              readOnly: true
            - name: output
              mountPath: /outputs
      volumes:
        - name: code
          configMap:
            name: train-code
        - name: output
          persistentVolumeClaim:
            claimName: train-output

kubectl apply --dry-run=server -f manifests/01-train.yaml
kubectl apply -f manifests/01-train.yaml
kubectl -n ai-demo get pods -l job-name=gpu-train-demo -w
# 看完状态后按 Ctrl+C，继续读取日志
kubectl -n ai-demo logs -f job/gpu-train-demo -c trainer
kubectl -n ai-demo wait --for=condition=complete job/gpu-train-demo --timeout=30m

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: model-cache
  namespace: ai-demo
spec:
  accessModes: [ReadWriteOnce]
  resources:
    requests:
      storage: 10Gi
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: vllm-demo
  namespace: ai-demo
spec:
  replicas: 1
  strategy:
    type: Recreate
  progressDeadlineSeconds: 2400
  selector:
    matchLabels:
      app: vllm-demo
  template:
    metadata:
      labels:
        app: vllm-demo
    spec:
      automountServiceAccountToken: false
      terminationGracePeriodSeconds: 120
      nodeSelector:
        kubernetes.io/arch: amd64
      containers:
        - name: vllm
          image: vllm/vllm-openai:v0.11.0
          args:
            - --model=Qwen/Qwen2.5-0.5B-Instruct
            - --revision=7ae557604adf67be50417f59c2c2f167def9a775
            - --served-model-name=qwen-demo
            - --host=0.0.0.0
            - --port=8000
            - --tensor-parallel-size=1
            - --dtype=half
            - --max-model-len=2048
            - --gpu-memory-utilization=0.8
          resources:
            requests:
              cpu: "2"
              memory: 8Gi
            limits:
              cpu: "4"
              memory: 16Gi
              nvidia.com/gpu: "1"
          ports:
            - name: http
              containerPort: 8000
          startupProbe:
            httpGet:
              path: /health
              port: http
            periodSeconds: 10
            timeoutSeconds: 5
            failureThreshold: 180
          readinessProbe:
            httpGet:
              path: /health
              port: http
            periodSeconds: 5
            timeoutSeconds: 5
            failureThreshold: 3
          volumeMounts:
            - name: cache
              mountPath: /root/.cache/huggingface
            - name: shm
              mountPath: /dev/shm
      volumes:
        - name: cache
          persistentVolumeClaim:
            claimName: model-cache
        - name: shm
          emptyDir:
            medium: Memory
            sizeLimit: 2Gi
---
apiVersion: v1
kind: Service
metadata:
  name: vllm-demo
  namespace: ai-demo
spec:
  type: ClusterIP
  selector:
    app: vllm-demo
  ports:
    - name: http
      port: 80
      targetPort: http

kubectl apply --dry-run=server -f manifests/02-inference.yaml
kubectl apply -f manifests/02-inference.yaml
kubectl -n ai-demo rollout status deployment/vllm-demo --timeout=40m
kubectl -n ai-demo get pods,pvc,service
kubectl -n ai-demo get endpointslices -l kubernetes.io/service-name=vllm-demo

kubectl -n ai-demo port-forward service/vllm-demo 8000:80

curl -sS http://127.0.0.1:8000/v1/chat/completions \
  -H 'Content-Type: application/json' \
  -d '{"model":"qwen-demo","messages":[{"role":"user","content":"用一句话解释 Kubernetes Pod。"}],"max_tokens":64}'

kubectl -n ai-demo delete deployment vllm-demo
kubectl -n ai-demo delete service vllm-demo
kubectl -n ai-demo delete job gpu-train-demo