Kubernetes From Scratch - Complete Production Guide

Core Concepts

Every Kubernetes object fits a hierarchy: Pods run containers, Deployments manage Pods, Services expose them, Ingress routes external traffic. ConfigMaps hold config, Secrets hold credentials.

apiVersion: v1
kind: Pod
metadata:
  name: nginx-pod
  labels:
    app: nginx
spec:
  containers:
  - name: nginx
    image: nginx:1.27-alpine
    ports:
    - containerPort: 80
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: app-config
data:
  DATABASE_HOST: "postgres.default.svc.cluster.local"
  LOG_LEVEL: "info"
---
apiVersion: v1
kind: Secret
metadata:
  name: db-credentials
type: Opaque
data:
  username: YWRtaW4=
  password: cGFzc3dvcmQxMjM=

kubectl apply -f pod.yaml
kubectl get pods -o wide
kubectl describe pod nginx-pod
kubectl logs nginx-pod

Local Development

Use kind for a fast local cluster with a local registry and Skaffold for hot-reload.

# kind-config.yaml
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
nodes:
- role: control-plane
  extraPortMappings:
  - containerPort: 30000
    hostPort: 30000
- role: worker
- role: worker
containerdConfigPatches:
- |-
  [plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:5001"]
    endpoint = ["http://kind-registry:5001"]

kind create cluster --config kind-config.yaml --name dev
docker run -d --restart=always -p 5001:5000 --name kind-registry --network kind registry:2

# skaffold.yaml
apiVersion: skaffold/v4beta6
kind: Config
build:
  artifacts:
  - image: localhost:5001/myapp
    docker:
      dockerfile: Dockerfile
deploy:
  kubectl:
    manifests: ["k8s/*.yaml"]

skaffold dev --port-forward

Production Manifests

apiVersion: apps/v1
kind: Deployment
metadata:
  name: api-server
spec:
  replicas: 3
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxSurge: 1
      maxUnavailable: 0
  selector:
    matchLabels:
      app: api-server
  template:
    metadata:
      labels:
        app: api-server
    spec:
      containers:
      - name: api
        image: myregistry/api-server:1.4.2
        ports:
        - containerPort: 8080
        resources:
          requests: { cpu: 100m, memory: 128Mi }
          limits: { cpu: 500m, memory: 512Mi }
        livenessProbe:
          httpGet: { path: /healthz, port: 8080 }
          initialDelaySeconds: 10
          periodSeconds: 15
        readinessProbe:
          httpGet: { path: /ready, port: 8080 }
          initialDelaySeconds: 5
          periodSeconds: 5
        env:
        - name: DB_PASSWORD
          valueFrom:
            secretKeyRef: { name: db-credentials, key: password }
---
apiVersion: v1
kind: Service
metadata:
  name: api-server
spec:
  type: ClusterIP
  selector:
    app: api-server
  ports:
  - port: 80
    targetPort: 8080
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: api-ingress
  annotations:
    cert-manager.io/cluster-issuer: letsencrypt-prod
spec:
  ingressClassName: nginx
  tls:
  - hosts: [api.example.com]
    secretName: api-tls
  rules:
  - host: api.example.com
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service: { name: api-server, port: { number: 80 } }

kubectl apply -f deployment.yaml
kubectl rollout status deployment/api-server
kubectl rollout undo deployment/api-server

Stateful Workloads

apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: postgres
spec:
  serviceName: postgres
  replicas: 1
  selector:
    matchLabels:
      app: postgres
  template:
    metadata:
      labels:
        app: postgres
    spec:
      containers:
      - name: postgres
        image: postgres:16-alpine
        ports:
        - containerPort: 5432
        env:
        - name: POSTGRES_DB
          value: appdb
        - name: POSTGRES_PASSWORD
          valueFrom:
            secretKeyRef: { name: db-credentials, key: password }
        volumeMounts:
        - name: data
          mountPath: /var/lib/postgresql/data
        resources:
          requests: { cpu: 250m, memory: 512Mi }
          limits: { cpu: "1", memory: 1Gi }
  volumeClaimTemplates:
  - metadata:
      name: data
    spec:
      accessModes: ["ReadWriteOnce"]
      storageClassName: gp3
      resources:
        requests:
          storage: 20Gi

kubectl apply -f statefulset.yaml
kubectl get pvc
kubectl exec -it postgres-0 -- psql -U admin -d appdb

Autoscaling

apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: api-server-hpa
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: api-server
  minReplicas: 2
  maxReplicas: 20
  metrics:
  - type: Resource
    resource:
      name: cpu
      target: { type: Utilization, averageUtilization: 70 }
  - type: Resource
    resource:
      name: memory
      target: { type: Utilization, averageUtilization: 80 }
  - type: Pods
    pods:
      metric: { name: http_requests_per_second }
      target: { type: AverageValue, averageValue: "1000" }
  behavior:
    scaleDown:
      stabilizationWindowSeconds: 300

kubectl apply -f hpa.yaml
kubectl get hpa -w
kubectl top pods

Helm Charts

Helm packages manifests into reusable charts with templated values for multi-environment deployments.

# Chart structure
mychart/
  Chart.yaml
  values.yaml
  values-staging.yaml
  values-prod.yaml
  templates/
    deployment.yaml
    service.yaml
    _helpers.tpl

# values.yaml
replicaCount: 2
image:
  repository: myregistry/api-server
  tag: "1.4.2"
resources:
  requests: { cpu: 100m, memory: 128Mi }
  limits: { cpu: 500m, memory: 512Mi }
ingress:
  enabled: true
  host: api.example.com

# templates/deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: {{ include "mychart.fullname" . }}
spec:
  replicas: {{ .Values.replicaCount }}
  template:
    spec:
      containers:
      - name: {{ .Chart.Name }}
        image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
        resources: {{- toYaml .Values.resources | nindent 10 }}

helm install api ./mychart -f values-prod.yaml -n production
helm upgrade api ./mychart -f values-prod.yaml -n production
helm rollback api 1 -n production

Debugging

5-step troubleshooting: events, pod status, logs, exec, debug container.

# 1. Check events
kubectl get events --sort-by='.lastTimestamp'
# 2. Describe pod
kubectl describe pod api-server-7d4f8b6c9-x2k4m
# 3. Logs (current + previous crash)
kubectl logs api-server-7d4f8b6c9-x2k4m
kubectl logs api-server-7d4f8b6c9-x2k4m --previous
# 4. Exec into container
kubectl exec -it api-server-7d4f8b6c9-x2k4m -- /bin/sh
# 5. Ephemeral debug container
kubectl debug -it api-server-7d4f8b6c9-x2k4m --image=busybox --target=api
# Bonus: test service connectivity
kubectl run curl --rm -it --image=curlimages/curl -- curl http://api-server:80/healthz

Security

apiVersion: v1
kind: ServiceAccount
metadata:
  name: api-sa
  namespace: production
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: api-role
  namespace: production
rules:
- apiGroups: [""]
  resources: ["configmaps", "secrets"]
  verbs: ["get", "list"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: api-rolebinding
  namespace: production
subjects:
- kind: ServiceAccount
  name: api-sa
roleRef:
  kind: Role
  name: api-role
  apiGroup: rbac.authorization.k8s.io
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: api-netpol
  namespace: production
spec:
  podSelector:
    matchLabels: { app: api-server }
  policyTypes: [Ingress, Egress]
  ingress:
  - from:
    - podSelector:
        matchLabels: { app: ingress-nginx }
    ports: [{ port: 8080 }]
  egress:
  - to:
    - podSelector:
        matchLabels: { app: postgres }
    ports: [{ port: 5432 }]

kubectl label namespace production pod-security.kubernetes.io/enforce=restricted
kubectl auth can-i get secrets --as=system:serviceaccount:production:api-sa -n production

Observability

# Prometheus + Grafana
helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
helm install monitoring prometheus-community/kube-prometheus-stack \
  --namespace monitoring --create-namespace \
  --set grafana.adminPassword=changeme

# Loki for logs
helm repo add grafana https://grafana.github.io/helm-charts
helm install loki grafana/loki-stack --namespace monitoring --set promtail.enabled=true

# Useful PromQL
sum(rate(container_cpu_usage_seconds_total{namespace="production"}[5m])) by (pod)
container_memory_working_set_bytes / container_spec_memory_limit_bytes * 100
sum(rate(http_requests_total{status=~"5.."}[5m])) / sum(rate(http_requests_total[5m])) * 100

kubectl port-forward svc/monitoring-grafana 3000:80 -n monitoring

CI/CD and GitOps

# .github/workflows/deploy.yaml
name: Build and Deploy
on:
  push:
    branches: [main]
jobs:
  build:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v4
    - name: Build and push
      run: |
        docker build -t ${{ secrets.REGISTRY }}/api:${{ github.sha }} .
        docker push ${{ secrets.REGISTRY }}/api:${{ github.sha }}
    - name: Update manifests
      run: |
        cd k8s/overlays/production
        kustomize edit set image api=${{ secrets.REGISTRY }}/api:${{ github.sha }}
        git add . && git commit -m "deploy: ${{ github.sha }}" && git push

# ArgoCD Application
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
  name: api-server
  namespace: argocd
spec:
  project: default
  source:
    repoURL: https://github.com/myorg/k8s-manifests.git
    targetRevision: main
    path: k8s/overlays/production
  destination:
    server: https://kubernetes.default.svc
    namespace: production
  syncPolicy:
    automated: { prune: true, selfHeal: true }

kubectl apply -n argocd -f https://raw.githubusercontent.com/argoproj/argo-cd/stable/manifests/install.yaml
argocd app sync api-server

Production Hardening

apiVersion: v1
kind: ResourceQuota
metadata:
  name: production-quota
  namespace: production
spec:
  hard:
    requests.cpu: "10"
    requests.memory: 20Gi
    limits.cpu: "20"
    limits.memory: 40Gi
    pods: "50"
---
apiVersion: v1
kind: LimitRange
metadata:
  name: default-limits
  namespace: production
spec:
  limits:
  - default: { cpu: 500m, memory: 512Mi }
    defaultRequest: { cpu: 100m, memory: 128Mi }
    type: Container
---
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: api-pdb
spec:
  minAvailable: 2
  selector:
    matchLabels: { app: api-server }

# Node affinity + tolerations
spec:
  affinity:
    nodeAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
        nodeSelectorTerms:
        - matchExpressions:
          - key: node-type
            operator: In
            values: [compute-optimized]
  tolerations:
  - key: "dedicated"
    operator: "Equal"
    value: "api"
    effect: "NoSchedule"

kubectl apply -f quota.yaml
kubectl describe resourcequota production-quota -n production
kubectl get pdb -n production

Cost Optimization

Right-size workloads, use spot instances, and enable autoscaling to cut costs 40-70%.

# Karpenter NodePool for spot instances
apiVersion: karpenter.sh/v1beta1
kind: NodePool
metadata:
  name: spot-pool
spec:
  template:
    spec:
      requirements:
      - key: karpenter.sh/capacity-type
        operator: In
        values: ["spot"]
      - key: node.kubernetes.io/instance-type
        operator: In
        values: ["m5.large", "m5.xlarge", "m6i.large", "m6i.xlarge"]
  limits:
    cpu: "100"
    memory: 200Gi
  disruption:
    consolidationPolicy: WhenUnderutilized

# Cluster Autoscaler
helm install cluster-autoscaler autoscaler/cluster-autoscaler \
  --set autoDiscovery.clusterName=my-cluster \
  --set awsRegion=us-west-2 -n kube-system
# VPA for right-sizing recommendations
kubectl apply -f https://github.com/kubernetes/autoscaler/releases/latest/download/vpa.yaml
kubectl get vpa -n production -o yaml

EKS vs GKE vs AKS

# Quick cluster creation
eksctl create cluster --name prod --region us-west-2 --nodes 3 --node-type m5.large
gcloud container clusters create-auto prod --region us-central1
az aks create --resource-group myRG --name prod --node-count 3 --node-vm-size Standard_D4s_v3