kubectl
是 Kubernetes 管理员的日常工具,当调试或排查 pod 问题时, 一般情况使用 logs describe exec
子命令便可以找到问题原因,但当如果容器崩溃(CrashLoopBackoff) 或容器镜像不包含 shell 时,我们无法更进一步去排查或调试容器,所以这时我们需要一个工具来解决这个问题。在 Kubernetes 1.18 版本开始,kubectl 新增了一个可以创建用于调试的临时容器的 alpha 命令,即 debug , 交互式的临时容器对于我们排除故障很有用。
debug 默认是没有开启的,需要在 kube-apiserver kubelet 加入下列参数:
--feature-gates="EphemeralContainers=true"
复制
kubectl debug
内置的 kubectl debug
命令其实也比较很简单,通过 --target 参数指定容器,给正在运行中的 Pod 增加一个临时容器,共享进程命名空间,容器文件系统通过 /proc/$pid/root
链接对 pod 中的其他容器可见。
debug 源码:
https://github.com/kubernetes/kubectl/blob/master/pkg/cmd/debug/debug.go
这个就是 debug 主要源码,通过深复制 pod 信息,然后加入 debug 容器,最后进行 Patch
// debugByEphemeralContainer runs an EphemeralContainer in the target Pod for use as a debug container
func (o *DebugOptions) debugByEphemeralContainer(ctx context.Context, pod *corev1.Pod) (*corev1.Pod, string, error) {
klog.V(2).Infof("existing ephemeral containers: %v", pod.Spec.EphemeralContainers)
podJS, err := json.Marshal(pod)
if err != nil {
return nil, "", fmt.Errorf("error creating JSON for pod: %v", err)
}
debugContainer := o.generateDebugContainer(pod)
klog.V(2).Infof("new ephemeral container: %#v", debugContainer)
debugPod := pod.DeepCopy()
debugPod.Spec.EphemeralContainers = append(debugPod.Spec.EphemeralContainers, *debugContainer)
debugJS, err := json.Marshal(debugPod)
if err != nil {
return nil, "", fmt.Errorf("error creating JSON for debug container: %v", err)
}
patch, err := strategicpatch.CreateTwoWayMergePatch(podJS, debugJS, pod)
if err != nil {
return nil, "", fmt.Errorf("error creating patch to add debug container: %v", err)
}
klog.V(2).Infof("generated strategic merge patch for debug container: %s", patch)
pods := o.podClient.Pods(pod.Namespace)
result, err := pods.Patch(ctx, pod.Name, types.StrategicMergePatchType, patch, metav1.PatchOptions{}, "ephemeralcontainers")
if err != nil {
// The apiserver will return a 404 when the EphemeralContainers feature is disabled because the `/ephemeralcontainers` subresource
// is missing. Unlike the 404 returned by a missing pod, the status details will be empty.
if serr, ok := err.(*errors.StatusError); ok && serr.Status().Reason == metav1.StatusReasonNotFound && serr.ErrStatus.Details.Name == "" {
return nil, "", fmt.Errorf("ephemeral containers are disabled for this cluster (error from server: %q).", err)
}
// The Kind used for the /ephemeralcontainers subresource changed in 1.22. When presented with an unexpected
// Kind the api server will respond with a not-registered error. When this happens we can optimistically try
// using the old API.
if runtime.IsNotRegisteredError(err) {
klog.V(1).Infof("Falling back to legacy API because server returned error: %v", err)
return o.debugByEphemeralContainerLegacy(ctx, pod, debugContainer)
}
return nil, "", err
}
return result, debugContainer.Name, nil
}复制
运行一个没有 shell 的 pod
root@k8s-dev-master01:~# kubectl run ephemeral-demo --image=k8s.gcr.io/pause:3.5 --restart=Never
pod/ephemeral-demo created
root@k8s-dev-master01:~# kubectl get po
NAME READY STATUS RESTARTS AGE
ephemeral-demo 1/1 Running 0 4s
root@k8s-dev-master01:~# kubectl exec -it ephemeral-demo -- sh
ERRO[0000] exec failed: container_linux.go:380: starting container process caused: exec: "sh": executable file not found in $PATH
command terminated with exit code 1复制
运行 debug 命令, 把日志级别设置为 10 ,查看创建 debug 容器的过程:
GET 获取 pod -> PATCH 增加临时容器 -> GET 获取临时容器 -> POST 进入临时容器。
root@k8s-dev-master01:~# kubectl -v=10 debug -it ephemeral-demo --image=busybox:1.33.1 --target=ephemeral-demo
I0726 23:01:53.713938 1438765 loader.go:372] Config loaded from file: /root/.kube/config
Targeting container "ephemeral-demo". If you don't see processes from this container it may be because the container runtime doesn't support this feature.
I0726 23:01:53.715536 1438765 cached_discovery.go:114] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/servergroups.json
I0726 23:01:53.716698 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/autoscaling/v2beta1/serverresources.json
I0726 23:01:53.716698 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/authentication.k8s.io/v1/serverresources.json
I0726 23:01:53.716714 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/apiregistration.k8s.io/v1/serverresources.json
I0726 23:01:53.716884 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/authorization.k8s.io/v1/serverresources.json
I0726 23:01:53.716972 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/events.k8s.io/v1/serverresources.json
I0726 23:01:53.717006 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/autoscaling/v1/serverresources.json
I0726 23:01:53.717068 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/crd.projectcalico.org/v1/serverresources.json
I0726 23:01:53.717131 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/events.k8s.io/v1beta1/serverresources.json
I0726 23:01:53.717245 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/apps/v1/serverresources.json
I0726 23:01:53.717262 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/batch/v1/serverresources.json
I0726 23:01:53.717301 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/storage.k8s.io/v1alpha1/serverresources.json
I0726 23:01:53.717347 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/batch/v1beta1/serverresources.json
I0726 23:01:53.717431 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/certificates.k8s.io/v1/serverresources.json
I0726 23:01:53.717511 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/policy/v1beta1/serverresources.json
I0726 23:01:53.717533 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/networking.k8s.io/v1/serverresources.json
I0726 23:01:53.717747 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/v1/serverresources.json
I0726 23:01:53.717785 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/admissionregistration.k8s.io/v1/serverresources.json
I0726 23:01:53.717889 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/storage.k8s.io/v1/serverresources.json
I0726 23:01:53.717904 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/apiextensions.k8s.io/v1/serverresources.json
I0726 23:01:53.717973 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/rbac.authorization.k8s.io/v1/serverresources.json
I0726 23:01:53.717978 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/scheduling.k8s.io/v1/serverresources.json
I0726 23:01:53.717998 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/rbac.authorization.k8s.io/v1alpha1/serverresources.json
I0726 23:01:53.718037 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/scheduling.k8s.io/v1alpha1/serverresources.json
I0726 23:01:53.718077 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/storage.k8s.io/v1beta1/serverresources.json
I0726 23:01:53.718132 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/coordination.k8s.io/v1/serverresources.json
I0726 23:01:53.718138 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/discovery.k8s.io/v1/serverresources.json
I0726 23:01:53.718304 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/node.k8s.io/v1alpha1/serverresources.json
I0726 23:01:53.718194 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/node.k8s.io/v1/serverresources.json
I0726 23:01:53.718211 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/discovery.k8s.io/v1beta1/serverresources.json
I0726 23:01:53.718249 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/node.k8s.io/v1beta1/serverresources.json
I0726 23:01:53.718302 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/flowcontrol.apiserver.k8s.io/v1beta1/serverresources.json
I0726 23:01:53.718370 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/internal.apiserver.k8s.io/v1alpha1/serverresources.json
I0726 23:01:53.718425 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/autoscaling/v2beta2/serverresources.json
I0726 23:01:53.718506 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/policy/v1/serverresources.json
I0726 23:01:53.718998 1438765 cached_discovery.go:114] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/servergroups.json
I0726 23:01:53.719205 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/rbac.authorization.k8s.io/v1/serverresources.json
I0726 23:01:53.719245 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/crd.projectcalico.org/v1/serverresources.json
I0726 23:01:53.719286 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/scheduling.k8s.io/v1alpha1/serverresources.json
I0726 23:01:53.719305 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/node.k8s.io/v1alpha1/serverresources.json
I0726 23:01:53.719309 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/discovery.k8s.io/v1/serverresources.json
I0726 23:01:53.719354 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/discovery.k8s.io/v1beta1/serverresources.json
I0726 23:01:53.719456 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/node.k8s.io/v1beta1/serverresources.json
I0726 23:01:53.719360 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/coordination.k8s.io/v1/serverresources.json
I0726 23:01:53.719407 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/node.k8s.io/v1/serverresources.json
I0726 23:01:53.719420 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/flowcontrol.apiserver.k8s.io/v1beta1/serverresources.json
I0726 23:01:53.719471 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/internal.apiserver.k8s.io/v1alpha1/serverresources.json
I0726 23:01:53.719515 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/v1/serverresources.json
I0726 23:01:53.719524 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/storage.k8s.io/v1alpha1/serverresources.json
I0726 23:01:53.719575 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/storage.k8s.io/v1/serverresources.json
I0726 23:01:53.719583 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/apiregistration.k8s.io/v1/serverresources.json
I0726 23:01:53.719578 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/storage.k8s.io/v1beta1/serverresources.json
I0726 23:01:53.719645 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/apiextensions.k8s.io/v1/serverresources.json
I0726 23:01:53.719649 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/admissionregistration.k8s.io/v1/serverresources.json
I0726 23:01:53.719700 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/rbac.authorization.k8s.io/v1alpha1/serverresources.json
I0726 23:01:53.719704 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/autoscaling/v2beta2/serverresources.json
I0726 23:01:53.719704 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/authentication.k8s.io/v1/serverresources.json
I0726 23:01:53.719824 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/autoscaling/v1/serverresources.json
I0726 23:01:53.719831 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/events.k8s.io/v1beta1/serverresources.json
I0726 23:01:53.719715 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/apps/v1/serverresources.json
I0726 23:01:53.719750 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/events.k8s.io/v1/serverresources.json
I0726 23:01:53.719771 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/authorization.k8s.io/v1/serverresources.json
I0726 23:01:53.719819 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/batch/v1/serverresources.json
I0726 23:01:53.719870 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/batch/v1beta1/serverresources.json
I0726 23:01:53.719882 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/autoscaling/v2beta1/serverresources.json
I0726 23:01:53.719928 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/certificates.k8s.io/v1/serverresources.json
I0726 23:01:53.719934 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/policy/v1/serverresources.json
I0726 23:01:53.719954 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/policy/v1beta1/serverresources.json
I0726 23:01:53.719989 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/scheduling.k8s.io/v1/serverresources.json
I0726 23:01:53.720014 1438765 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/172.31.6.142_6443/networking.k8s.io/v1/serverresources.json
I0726 23:01:53.721027 1438765 round_trippers.go:435] curl -v -XGET -H "Accept: application/json, */*" -H "User-Agent: kubectl/v1.22.0 (linux/amd64) kubernetes/f27a086" 'https://172.31.6.142:6443/api/v1/namespaces/default/pods/ephemeral-demo'
I0726 23:01:53.731160 1438765 round_trippers.go:454] GET https://172.31.6.142:6443/api/v1/namespaces/default/pods/ephemeral-demo 200 OK in 10 milliseconds
I0726 23:01:53.731176 1438765 round_trippers.go:460] Response Headers:
I0726 23:01:53.731186 1438765 round_trippers.go:463] Audit-Id: 31a6d288-08e4-4b7f-8aca-27451d5d1e7e
I0726 23:01:53.731201 1438765 round_trippers.go:463] Cache-Control: no-cache, private
I0726 23:01:53.731213 1438765 round_trippers.go:463] Content-Type: application/json
I0726 23:01:53.731227 1438765 round_trippers.go:463] X-Kubernetes-Pf-Flowschema-Uid: e996f762-0052-428d-a993-54c03e671d1f
I0726 23:01:53.731239 1438765 round_trippers.go:463] X-Kubernetes-Pf-Prioritylevel-Uid: f0c48556-c734-443c-ac72-1f4b01d3d863
I0726 23:01:53.731251 1438765 round_trippers.go:463] Content-Length: 4040
I0726 23:01:53.731263 1438765 round_trippers.go:463] Date: Mon, 26 Jul 2021 15:01:53 GMT
I0726 23:01:53.731326 1438765 request.go:1181] Response Body: {"kind":"Pod","apiVersion":"v1","metadata":{"name":"ephemeral-demo","namespace":"default","uid":"4c7aac63-02ca-421e-9abd-665733b0de55","resourceVersion":"127631","creationTimestamp":"2021-07-26T14:58:10Z","labels":{"run":"ephemeral-demo"},"annotations":{"cni.projectcalico.org/podIP":"192.168.239.9/32","cni.projectcalico.org/podIPs":"192.168.239.9/32"},"managedFields":[{"manager":"calico","operation":"Update","apiVersion":"v1","time":"2021-07-26T14:58:10Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:cni.projectcalico.org/podIP":{},"f:cni.projectcalico.org/podIPs":{}}}},"subresource":"status"},{"manager":"kubectl-run","operation":"Update","apiVersion":"v1","time":"2021-07-26T14:58:10Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:labels":{".":{},"f:run":{}}},"f:spec":{"f:containers":{"k:{\"name\":\"ephemeral-demo\"}":{".":{},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:resources":{},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:terminationGracePeriodSeconds":{}}}},{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2021-07-26T14:58:10Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"192.168.239.9\"}":{".":{},"f:ip":{}}},"f:startTime":{}}},"subresource":"status"}]},"spec":{"volumes":[{"name":"kube-api-access-lx8x9","projected":{"sources":[{"serviceAccountToken":{"expirationSeconds":3607,"path":"token"}},{"configMap":{"name":"kube-root-ca.crt","items":[{"key":"ca.crt","path":"ca.crt"}]}},{"downwardAPI":{"items":[{"path":"namespace","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}]}}],"defaultMode":420}}],"containers":[{"name":"ephemeral-demo","image":"k8s.gcr.io/pause:3.5","resources":{},"volumeMounts":[{"name":"kube-api-access-lx8x9","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"IfNotPresent"}],"restartPolicy":"Never","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","serviceAccountName":"default","serviceAccount":"default","nodeName":"k8s-dev-master01","securityContext":{},"schedulerName":"default-scheduler","tolerations":[{"key":"node.kubernetes.io/not-ready","operator":"Exists","effect":"NoExecute","tolerationSeconds":360},{"key":"node.kubernetes.io/unreachable","operator":"Exists","effect":"NoExecute","tolerationSeconds":360}],"priority":0,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Running","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2021-07-26T14:58:10Z"},{"type":"Ready","status":"True","lastProbeTime":null,"lastTransitionTime":"2021-07-26T14:58:10Z"},{"type":"ContainersReady","status":"True","lastProbeTime":null,"lastTransitionTime":"2021-07-26T14:58:10Z"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2021-07-26T14:58:10Z"}],"hostIP":"172.31.6.142","podIP":"192.168.239.9","podIPs":[{"ip":"192.168.239.9"}],"startTime":"2021-07-26T14:58:10Z","containerStatuses":[{"name":"ephemeral-demo","state":{"running":{"startedAt":"2021-07-26T14:58:10Z"}},"lastState":{},"ready":true,"restartCount":0,"image":"k8s.gcr.io/pause:3.5","imageID":"k8s.gcr.io/pause@sha256:1ff6c18fbef2045af6b9c16bf034cc421a29027b800e4f9b68ae9b1cb3e9ae07","containerID":"cri-o://bb462da06947a7e8aed3245835e3e50a13f181915afa4443e022c6cac51ed69b","started":true}],"qosClass":"BestEffort"}}
I0726 23:01:53.736885 1438765 debug.go:396] existing ephemeral containers: []
Defaulting debug container name to debugger-c5dkn.
I0726 23:01:53.740053 1438765 debug.go:403] new ephemeral container: &v1.EphemeralContainer{EphemeralContainerCommon:v1.EphemeralContainerCommon{Name:"debugger-c5dkn", Image:"busybox:1.33.1", Command:[]string{}, Args:[]string(nil), WorkingDir:"", Ports:[]v1.ContainerPort(nil), EnvFrom:[]v1.EnvFromSource(nil), Env:[]v1.EnvVar(nil), Resources:v1.ResourceRequirements{Limits:v1.ResourceList(nil), Requests:v1.ResourceList(nil)}, VolumeMounts:[]v1.VolumeMount(nil), VolumeDevices:[]v1.VolumeDevice(nil), LivenessProbe:(*v1.Probe)(nil), ReadinessProbe:(*v1.Probe)(nil), StartupProbe:(*v1.Probe)(nil), Lifecycle:(*v1.Lifecycle)(nil), TerminationMessagePath:"", TerminationMessagePolicy:"File", ImagePullPolicy:"", SecurityContext:(*v1.SecurityContext)(nil), Stdin:true, StdinOnce:false, TTY:true}, TargetContainerName:"ephemeral-demo"}
I0726 23:01:53.740812 1438765 debug.go:415] generated strategic merge patch for debug container: {"spec":{"ephemeralContainers":[{"image":"busybox:1.33.1","name":"debugger-c5dkn","resources":{},"stdin":true,"targetContainerName":"ephemeral-demo","terminationMessagePolicy":"File","tty":true}]}}
I0726 23:01:53.740862 1438765 request.go:1181] Request Body: {"spec":{"ephemeralContainers":[{"image":"busybox:1.33.1","name":"debugger-c5dkn","resources":{},"stdin":true,"targetContainerName":"ephemeral-demo","terminationMessagePolicy":"File","tty":true}]}}
I0726 23:01:53.740932 1438765 round_trippers.go:435] curl -v -XPATCH -H "Accept: application/json, */*" -H "Content-Type: application/strategic-merge-patch+json" -H "User-Agent: kubectl/v1.22.0 (linux/amd64) kubernetes/f27a086" 'https://172.31.6.142:6443/api/v1/namespaces/default/pods/ephemeral-demo/ephemeralcontainers'
I0726 23:01:53.747285 1438765 round_trippers.go:454] PATCH https://172.31.6.142:6443/api/v1/namespaces/default/pods/ephemeral-demo/ephemeralcontainers 200 OK in 6 milliseconds
I0726 23:01:53.747306 1438765 round_trippers.go:460] Response Headers:
I0726 23:01:53.747317 1438765 round_trippers.go:463] Content-Length: 2838
I0726 23:01:53.747326 1438765 round_trippers.go:463] Date: Mon, 26 Jul 2021 15:01:53 GMT
I0726 23:01:53.747335 1438765 round_trippers.go:463] Audit-Id: bbe3c1cd-bb9b-40e0-b3b8-42eab1fe0a8e
I0726 23:01:53.747344 1438765 round_trippers.go:463] Cache-Control: no-cache, private
I0726 23:01:53.747353 1438765 round_trippers.go:463] Content-Type: application/json
I0726 23:01:53.747362 1438765 round_trippers.go:463] X-Kubernetes-Pf-Flowschema-Uid: e996f762-0052-428d-a993-54c03e671d1f
I0726 23:01:53.747378 1438765 round_trippers.go:463] X-Kubernetes-Pf-Prioritylevel-Uid: f0c48556-c734-443c-ac72-1f4b01d3d863
I0726 23:01:53.747423 1438765 request.go:1181] Response Body: {"kind":"Pod","apiVersion":"v1","metadata":{"name":"ephemeral-demo","namespace":"default","uid":"4c7aac63-02ca-421e-9abd-665733b0de55","resourceVersion":"127899","creationTimestamp":"2021-07-26T14:58:10Z","labels":{"run":"ephemeral-demo"},"annotations":{"cni.projectcalico.org/podIP":"192.168.239.9/32","cni.projectcalico.org/podIPs":"192.168.239.9/32"}},"spec":{"volumes":[{"name":"kube-api-access-lx8x9","projected":{"sources":[{"serviceAccountToken":{"expirationSeconds":3607,"path":"token"}},{"configMap":{"name":"kube-root-ca.crt","items":[{"key":"ca.crt","path":"ca.crt"}]}},{"downwardAPI":{"items":[{"path":"namespace","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}]}}],"defaultMode":420}}],"containers":[{"name":"ephemeral-demo","image":"k8s.gcr.io/pause:3.5","resources":{},"volumeMounts":[{"name":"kube-api-access-lx8x9","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"IfNotPresent"}],"ephemeralContainers":[{"name":"debugger-c5dkn","image":"busybox:1.33.1","resources":{},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"IfNotPresent","stdin":true,"tty":true,"targetContainerName":"ephemeral-demo"}],"restartPolicy":"Never","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","serviceAccountName":"default","serviceAccount":"default","nodeName":"k8s-dev-master01","securityContext":{},"schedulerName":"default-scheduler","tolerations":[{"key":"node.kubernetes.io/not-ready","operator":"Exists","effect":"NoExecute","tolerationSeconds":360},{"key":"node.kubernetes.io/unreachable","operator":"Exists","effect":"NoExecute","tolerationSeconds":360}],"priority":0,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Running","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2021-07-26T14:58:10Z"},{"type":"Ready","status":"True","lastProbeTime":null,"lastTransitionTime":"2021-07-26T14:58:10Z"},{"type":"ContainersReady","status":"True","lastProbeTime":null,"lastTransitionTime":"2021-07-26T14:58:10Z"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2021-07-26T14:58:10Z"}],"hostIP":"172.31.6.142","podIP":"192.168.239.9","podIPs":[{"ip":"192.168.239.9"}],"startTime":"2021-07-26T14:58:10Z","containerStatuses":[{"name":"ephemeral-demo","state":{"running":{"startedAt":"2021-07-26T14:58:10Z"}},"lastState":{},"ready":true,"restartCount":0,"image":"k8s.gcr.io/pause:3.5","imageID":"k8s.gcr.io/pause@sha256:1ff6c18fbef2045af6b9c16bf034cc421a29027b800e4f9b68ae9b1cb3e9ae07","containerID":"cri-o://bb462da06947a7e8aed3245835e3e50a13f181915afa4443e022c6cac51ed69b","started":true}],"qosClass":"BestEffort"}}
I0726 23:01:53.755960 1438765 reflector.go:219] Starting reflector *v1.Pod (0s) from k8s.io/client-go/tools/watch/informerwatcher.go:146
I0726 23:01:53.756037 1438765 reflector.go:255] Listing and watching *v1.Pod from k8s.io/client-go/tools/watch/informerwatcher.go:146
I0726 23:01:53.756274 1438765 round_trippers.go:435] curl -v -XGET -H "Accept: application/json, */*" -H "User-Agent: kubectl/v1.22.0 (linux/amd64) kubernetes/f27a086" 'https://172.31.6.142:6443/api/v1/namespaces/default/pods?fieldSelector=metadata.name%3Dephemeral-demo&limit=500&resourceVersion=0'
I0726 23:01:53.760585 1438765 round_trippers.go:454] GET https://172.31.6.142:6443/api/v1/namespaces/default/pods?fieldSelector=metadata.name%3Dephemeral-demo&limit=500&resourceVersion=0 200 OK in 4 milliseconds
I0726 23:01:53.760644 1438765 round_trippers.go:460] Response Headers:
I0726 23:01:53.760677 1438765 round_trippers.go:463] Cache-Control: no-cache, private
I0726 23:01:53.760707 1438765 round_trippers.go:463] Content-Type: application/json
I0726 23:01:53.760735 1438765 round_trippers.go:463] X-Kubernetes-Pf-Flowschema-Uid: e996f762-0052-428d-a993-54c03e671d1f
I0726 23:01:53.760764 1438765 round_trippers.go:463] X-Kubernetes-Pf-Prioritylevel-Uid: f0c48556-c734-443c-ac72-1f4b01d3d863
I0726 23:01:53.760793 1438765 round_trippers.go:463] Content-Length: 2894
I0726 23:01:53.760822 1438765 round_trippers.go:463] Date: Mon, 26 Jul 2021 15:01:53 GMT
I0726 23:01:53.760850 1438765 round_trippers.go:463] Audit-Id: 36c65278-7b17-44d1-9301-d0d9a882ef09
I0726 23:01:53.760929 1438765 request.go:1181] Response Body: {"kind":"PodList","apiVersion":"v1","metadata":{"resourceVersion":"127899"},"items":[{"metadata":{"name":"ephemeral-demo","namespace":"default","uid":"4c7aac63-02ca-421e-9abd-665733b0de55","resourceVersion":"127899","creationTimestamp":"2021-07-26T14:58:10Z","labels":{"run":"ephemeral-demo"},"annotations":{"cni.projectcalico.org/podIP":"192.168.239.9/32","cni.projectcalico.org/podIPs":"192.168.239.9/32"}},"spec":{"volumes":[{"name":"kube-api-access-lx8x9","projected":{"sources":[{"serviceAccountToken":{"expirationSeconds":3607,"path":"token"}},{"configMap":{"name":"kube-root-ca.crt","items":[{"key":"ca.crt","path":"ca.crt"}]}},{"downwardAPI":{"items":[{"path":"namespace","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}]}}],"defaultMode":420}}],"containers":[{"name":"ephemeral-demo","image":"k8s.gcr.io/pause:3.5","resources":{},"volumeMounts":[{"name":"kube-api-access-lx8x9","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"IfNotPresent"}],"ephemeralContainers":[{"name":"debugger-c5dkn","image":"busybox:1.33.1","resources":{},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"IfNotPresent","stdin":true,"tty":true,"targetContainerName":"ephemeral-demo"}],"restartPolicy":"Never","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","serviceAccountName":"default","serviceAccount":"default","nodeName":"k8s-dev-master01","securityContext":{},"schedulerName":"default-scheduler","tolerations":[{"key":"node.kubernetes.io/not-ready","operator":"Exists","effect":"NoExecute","tolerationSeconds":360},{"key":"node.kubernetes.io/unreachable","operator":"Exists","effect":"NoExecute","tolerationSeconds":360}],"priority":0,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Running","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2021-07-26T14:58:10Z"},{"type":"Ready","status":"True","lastProbeTime":null,"lastTransitionTime":"2021-07-26T14:58:10Z"},{"type":"ContainersReady","status":"True","lastProbeTime":null,"lastTransitionTime":"2021-07-26T14:58:10Z"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2021-07-26T14:58:10Z"}],"hostIP":"172.31.6.142","podIP":"192.168.239.9","podIPs":[{"ip":"192.168.239.9"}],"startTime":"2021-07-26T14:58:10Z","containerStatuses":[{"name":"ephemeral-demo","state":{"running":{"startedAt":"2021-07-26T14:58:10Z"}},"lastState":{},"ready":true,"restartCount":0,"image":"k8s.gcr.io/pause:3.5","imageID":"k8s.gcr.io/pause@sha256:1ff6c18fbef2045af6b9c16bf034cc421a29027b800e4f9b68ae9b1cb3e9ae07","containerID":"cri-o://bb462da06947a7e8aed3245835e3e50a13f181915afa4443e022c6cac51ed69b","started":true}],"qosClass":"BestEffort"}}]}
I0726 23:01:53.767672 1438765 round_trippers.go:435] curl -v -XGET -H "Accept: application/json, */*" -H "User-Agent: kubectl/v1.22.0 (linux/amd64) kubernetes/f27a086" 'https://172.31.6.142:6443/api/v1/namespaces/default/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dephemeral-demo&resourceVersion=127899&timeout=9m36s&timeoutSeconds=576&watch=true'
I0726 23:01:53.769655 1438765 round_trippers.go:454] GET https://172.31.6.142:6443/api/v1/namespaces/default/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dephemeral-demo&resourceVersion=127899&timeout=9m36s&timeoutSeconds=576&watch=true 200 OK in 1 milliseconds
I0726 23:01:53.769682 1438765 round_trippers.go:460] Response Headers:
I0726 23:01:53.769693 1438765 round_trippers.go:463] Audit-Id: a6c20ae9-d422-4b54-9e71-13260050f0cc
I0726 23:01:53.769699 1438765 round_trippers.go:463] Cache-Control: no-cache, private
I0726 23:01:53.769705 1438765 round_trippers.go:463] Content-Type: application/json
I0726 23:01:53.769712 1438765 round_trippers.go:463] X-Kubernetes-Pf-Flowschema-Uid: e996f762-0052-428d-a993-54c03e671d1f
I0726 23:01:53.769721 1438765 round_trippers.go:463] X-Kubernetes-Pf-Prioritylevel-Uid: f0c48556-c734-443c-ac72-1f4b01d3d863
I0726 23:01:53.769728 1438765 round_trippers.go:463] Date: Mon, 26 Jul 2021 15:01:53 GMT
I0726 23:01:53.770603 1438765 debug.go:739] debug container status is &ContainerStatus{Name:debugger-c5dkn,State:ContainerState{Waiting:&ContainerStateWaiting{Reason:ContainerCreating,Message:,},Running:nil,Terminated:nil,},LastTerminationState:ContainerState{Waiting:nil,Running:nil,Terminated:nil,},Ready:false,RestartCount:0,Image:busybox:1.33.1,ImageID:,ContainerID:,Started:nil,}
I0726 23:01:54.400822 1438765 debug.go:739] debug container status is &ContainerStatus{Name:debugger-c5dkn,State:ContainerState{Waiting:nil,Running:&ContainerStateRunning{StartedAt:2021-07-26 23:01:53 +0800 CST,},Terminated:nil,},LastTerminationState:ContainerState{Waiting:nil,Running:nil,Terminated:nil,},Ready:false,RestartCount:0,Image:docker.io/library/busybox:1.33.1,ImageID:docker.io/library/busybox@sha256:0f354ec1728d9ff32edcd7d1b8bbdfc798277ad36120dc3dc683be44524c8b60,ContainerID:cri-o://5f9c8d76734c3b81728ceffa8c1c1b96406ab873d11575eb89b3a9757dd346dc,Started:nil,}
I0726 23:01:54.400965 1438765 reflector.go:225] Stopping reflector *v1.Pod (0s) from k8s.io/client-go/tools/watch/informerwatcher.go:146
If you don't see a command prompt, try pressing enter.
I0726 23:01:54.401772 1438765 round_trippers.go:435] curl -v -XPOST -H "X-Stream-Protocol-Version: v4.channel.k8s.io" -H "X-Stream-Protocol-Version: v3.channel.k8s.io" -H "X-Stream-Protocol-Version: v2.channel.k8s.io" -H "X-Stream-Protocol-Version: channel.k8s.io" -H "User-Agent: kubectl/v1.22.0 (linux/amd64) kubernetes/f27a086" 'https://172.31.6.142:6443/api/v1/namespaces/default/pods/ephemeral-demo/attach?container=debugger-c5dkn&stdin=true&stdout=true&tty=true'
I0726 23:01:54.419552 1438765 round_trippers.go:454] POST https://172.31.6.142:6443/api/v1/namespaces/default/pods/ephemeral-demo/attach?container=debugger-c5dkn&stdin=true&stdout=true&tty=true 101 Switching Protocols in 17 milliseconds
I0726 23:01:54.419574 1438765 round_trippers.go:460] Response Headers:
I0726 23:01:54.419586 1438765 round_trippers.go:463] Upgrade: SPDY/3.1
I0726 23:01:54.419600 1438765 round_trippers.go:463] X-Stream-Protocol-Version: v4.channel.k8s.io
I0726 23:01:54.419612 1438765 round_trippers.go:463] Date: Mon, 26 Jul 2021 15:01:54 GMT
I0726 23:01:54.419616 1438765 round_trippers.go:463] Connection: Upgrade
/ #
/ # ls -lrt
total 36
drwxr-xr-x 2 root root 12288 Jun 7 17:34 bin
drwxrwxrwt 2 root root 4096 Jun 7 17:34 tmp
drwxr-xr-x 4 root root 4096 Jun 7 17:34 var
drwxr-xr-x 3 root root 4096 Jun 7 17:34 usr
drwxr-xr-x 2 nobody nobody 4096 Jun 7 17:34 home
dr-xr-xr-x 13 root root 0 Jul 26 14:58 sys
dr-xr-xr-x 209 root root 0 Jul 26 15:01 proc
drwxr-xr-x 1 root root 4096 Jul 26 15:01 etc
drwxr-xr-x 5 root root 380 Jul 26 15:01 dev
drwx------ 1 root root 4096 Jul 26 15:03 root
/ # ps
PID USER TIME COMMAND
1 root 0:00 sh
8 root 0:00 ps
/ # ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: tunl0@NONE: <NOARP> mtu 1480 qdisc noop qlen 1000
link/ipip 0.0.0.0 brd 0.0.0.0
4: eth0@if15: <BROADCAST,MULTICAST,UP,LOWER_UP,M-DOWN> mtu 1480 qdisc noqueue
link/ether 02:e3:83:67:97:7c brd ff:ff:ff:ff:ff:ff
inet 192.168.239.9/32 brd 192.168.239.9 scope global eth0
valid_lft forever preferred_lft forever
inet6 fe80::e3:83ff:fe67:977c/64 scope link
valid_lft forever preferred_lft forever
/ #复制
用 kubectl describe 查看新创建的临时容器的状态。
Ephemeral Containers:
debugger-c5dkn:
Container ID: cri-o://5f9c8d76734c3b81728ceffa8c1c1b96406ab873d11575eb89b3a9757dd346dc
Image: busybox:1.33.1
Image ID: docker.io/library/busybox@sha256:0f354ec1728d9ff32edcd7d1b8bbdfc798277ad36120dc3dc683be44524c8b60
Port: <none>
Host Port: <none>
State: Terminated
Reason: Error
Exit Code: 130
Started: Mon, 26 Jul 2021 23:01:53 +0800
Finished: Mon, 26 Jul 2021 23:04:34 +0800
Ready: False
Restart Count: 0
Environment: <none>
Mounts: <none>
Conditions:
Type Status
Initialized True
Ready True
ContainersReady True
PodScheduled True复制
值得注意的是,临时容器是一次性的,只对本次执行 debug 有效,退出后当前 debug 会话后便不能再进入了本次创建的临时容器了。再次执行 debug 时,会生成一个新的临时容器。
使用 client-go 实现 debug
参考 kubectl debug
的源码的步骤实现。
package main
import (
"context"
"encoding/json"
"flag"
"io"
"math/rand"
"os"
"time"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/strategicpatch"
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/tools/remotecommand"
"k8s.io/klog/v2"
"github.com/prodanlabs/client-go-debug/client"
)
func main() {
// init logs
klog.InitFlags(nil)
flag.Parse()
defer klog.Flush()
// 实例化 k8s 客户端
kubeConfig, err := client.InitKubeConfig(false)
if err != nil {
klog.Fatal("kubernetes Config failed to initialize.", err)
}
clientSet, err := client.NewClientSet(kubeConfig)
if err != nil {
klog.Fatal("kubernetes clientSet failed.", err)
}
id := generateID(5)
debugContainerName := "debugger-" + id
ec := corev1.EphemeralContainer{
EphemeralContainerCommon: corev1.EphemeralContainerCommon{
Name: debugContainerName,
Command: []string{"sh"},
Image: "busybox:1.33.1",
ImagePullPolicy: "IfNotPresent",
Stdin: true,
TTY: true,
TerminationMessagePolicy: corev1.TerminationMessageReadFile,
},
TargetContainerName: "ephemeral-demo",
}
klog.V(4).Infof("new ephemeral container : %s", ec)
pod, err := clientSet.CoreV1().Pods("default").Get(context.TODO(), "ephemeral-demo", metav1.GetOptions{})
if err != nil {
klog.Error(err)
}
podJS, err := json.Marshal(pod)
if err != nil {
klog.Errorf("error creating JSON for pod: %v", err)
}
debugPod := pod.DeepCopy()
debugPod.Spec.EphemeralContainers = append(debugPod.Spec.EphemeralContainers, ec)
debugJS, err := json.Marshal(debugPod)
if err != nil {
klog.Errorf("error creating JSON for debug container: %v", err)
}
patch, err := strategicpatch.CreateTwoWayMergePatch(podJS, debugJS, pod)
pods := clientSet.CoreV1().Pods("default")
result, err := pods.Patch(context.TODO(), "ephemeral-demo", types.StrategicMergePatchType, patch, metav1.PatchOptions{}, "ephemeralcontainers")
if err != nil {
klog.Error(err)
}
klog.Infof("debug container status is : %s", result.Spec.EphemeralContainers)
req := clientSet.CoreV1().RESTClient().Post().Namespace("default").
Resource("pods").Name("ephemeral-demo").SubResource("attach").
VersionedParams(&corev1.PodAttachOptions{
TypeMeta: metav1.TypeMeta{
Kind: "EphemeralContainers",
APIVersion: "v1",
},
Container: debugContainerName,
Stdin: true,
Stdout: true,
Stderr: true,
TTY: true,
}, scheme.ParameterCodec)
klog.Info(req.URL())
exec, err := remotecommand.NewSPDYExecutor(kubeConfig, "POST", req.URL())
if err != nil {
klog.Error(err)
}
screen := struct {
io.Reader
io.Writer
}{os.Stdin, os.Stdout}
// 建立链接之后从请求的sream中发送、读取数据
if err = exec.Stream(remotecommand.StreamOptions{
Stdin: screen,
Stdout: screen,
Stderr: screen,
Tty: true,
}); err != nil {
klog.Error(err)
}
}
func generateID(length int) string {
str := "qwertyuiopasdfghjklzxcvbnm1234567890"
bytes := []byte(str)
result := []byte{}
rand.Seed(time.Now().UnixNano() + int64(rand.Intn(100)))
for i := 0; i < length; i++ {
result = append(result, bytes[rand.Intn(len(bytes))])
}
return string(result)
}复制
代码运行结果: