这边已经将Prometheus安装到k8s中了
安装node_exporter，我这边是采用的DaemonSet来确保每个节点都有对应的pod
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: node-exporter
namespace: monitor
labels:
k8s-app: node-exporter
spec:
selector:
matchLabels:
k8s-app: node-exporter
template:
metadata:
labels:
k8s-app: node-exporter
spec:
hostPID: true #这几项是定义了该pod直接共享node的资源，这样也不需要用svc来暴露端口了
hostIPC: true
hostNetwork: true
containers:
- image: bitnami/node-exporter:latest
args:
- --web.listen-address=$(HOSTIP):9100
- --path.procfs=/host/proc
- --path.sysfs=/host/sys
- --path.rootfs=/host/root
- --collector.filesystem.ignored-mount-points=^/(dev|proc|sys|var/lib/docker/.+)($|/)
- --collector.filesystem.ignored-fs-types=^(autofs|binfmt_misc|cgroup|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|mqueue|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|sysfs|tracefs)$
name: node-exporter
env:
- name: HOSTIP
valueFrom:
fieldRef:
fieldPath: status.hostIP
resources:
requests:
cpu: 150m
memory: 180Mi
limits:
cpu: 150m
memory: 180Mi
securityContext:
runAsNonRoot: true
runAsUser: 65534
volumeMounts:
- name: proc # 我对应的卷都是用的hostPath，直接将宿主机卷挂给pod避免pod无法正常获取node信息
mountPath: /host/proc
- name: sys
mountPath: /host/sys
- name: root
mountPath: /host/root
mountPropagation: HostToContainer
readOnly: true
ports:
- containerPort: 9100
protocol: TCP
name: http
tolerations: #这里是为了让pod能在master上运行，加了容忍度
- key: node-role.kubernetes.io/control-plane
operator: Exists
effect: NoSchedule
volumes:
- name: proc
hostPath:
path: /proc
- name: dev
hostPath:
path: /dev
- name: sys
hostPath:
path: /sys
- name: root
hostPath:
path: /
root@master1:~/k8s-prometheus# kubectl apply -f node-export.yaml
root@master1:~/k8s-prometheus# kubectl get pod -n monitor
NAME READY STATUS RESTARTS AGE
grafana-core-5c68549dc7-t92fv 1/1 Running 0 2d17h
node-exporter-6fqvt 1/1 Running 0 2d19h
node-exporter-bgrjn 1/1 Running 0 2d19h
node-exporter-bk7m4 1/1 Running 0 2d19h
node-exporter-m7wgx 1/1 Running 0 2d19h
node-exporter-mbgtg 1/1 Running 0 2d19h
node-exporter-rdtcs 1/1 Running 0 2d19h
prometheus-7d659686d7-x62vt 1/1 Running 0 2d16h
root@master1:~/k8s-prometheus# ss -lntp |grep node_exporter
LISTEN 0 4096 10.10.21.170:9100 0.0.0.0:* users:(("node_exporter",pid=2597275,fd=3))
#可以看到它直接使用了宿主机的9100端口
coredns自身提供了/metrics接口，我们直接配置prometheus去它的9153拿数据即可
root@master1:~/k8s-prometheus# kubectl -n kube-system get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kube-dns ClusterIP 10.100.0.10 <none> 53/UDP,53/TCP,9153/TCP 22d
root@master1:~/k8s-prometheus# kubectl -n kube-system get cm
NAME DATA AGE
coredns 1 22d
extension-apiserver-authentication 6 22d
kube-proxy 2 22d
kube-root-ca.crt 1 22d
kubeadm-config 1 22d
kubelet-config 1 22d
root@master1:~/k8s-prometheus# kubectl -n kube-system get cm coredns -o yaml
apiVersion: v1
data:
Corefile: |
.:53 {
errors
health {
lameduck 5s
}
ready
kubernetes cluster.local in-addr.arpa ip6.arpa {
pods insecure
fallthrough in-addr.arpa ip6.arpa
ttl 30
}
prometheus :9153
forward . /etc/resolv.conf {
max_concurrent 1000
}
cache 30
loop
reload
loadbalance
}
kind: ConfigMap
metadata:
creationTimestamp: "2022-10-11T10:23:37Z"
name: coredns
namespace: kube-system
resourceVersion: "228"
uid: 632b41d8-6dda-40b7-81c1-b6cffefaf2e8
Ingress-nginx自身默认也提供了/metrics接口，但是需要我们先给他暴露10254端口，暴露之后我们直接配置prometheus去它的9153拿数据即可
暴露Ingress端口有多种方法，比如把对应deployment的网络模式改为hostNetwork，直接将node的10254给到Ingress
或者先在deployment中将端口加上，再去Ingress对应的svc里面将10254端口代理一下
这边我用的是第一种方式
root@master1:~# kubectl -n ingress-nginx edit deployments.apps ingress-nginx-controller
需要改的地方有两处
root@master1:~# kubectl -n ingress-nginx get deployments.apps ingress-nginx-controller -o yaml
apiVersion: apps/v1
kind: Deployment
metadata:
annotations:
deployment.kubernetes.io/revision: "8"
kubectl.kubernetes.io/last-applied-configuration: |
{"apiVersion":"apps/v1","kind":"Deployment","metadata":{"annotations":{},"labels":{"app.kubernetes.io/component":"controller","app.kubernetes.io/instance":"ingress-nginx","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/name":"ingress-nginx","app.kubernetes.io/version":"1.1.1","helm.sh/chart":"ingress-nginx-4.0.15"},"name":"ingress-nginx-controller","namespace":"ingress-nginx"},"spec":{"minReadySeconds":0,"revisionHistoryLimit":10,"selector":{"matchLabels":{"app.kubernetes.io/component":"controller","app.kubernetes.io/instance":"ingress-nginx","app.kubernetes.io/name":"ingress-nginx"}},"strategy":{"rollingUpdate":{"maxUnavailable":1},"type":"RollingUpdate"},"template":{"metadata":{"labels":{"app.kubernetes.io/component":"controller","app.kubernetes.io/instance":"ingress-nginx","app.kubernetes.io/name":"ingress-nginx"}},"spec":{"containers":[{"args":["/nginx-ingress-controller","--election-id=ingress-controller-leader","--controller-class=k8s.io/ingress-nginx","--configmap=$(POD_NAMESPACE)/ingress-nginx-controller","--validating-webhook=:8443","--validating-webhook-certificate=/usr/local/certificates/cert","--validating-webhook-key=/usr/local/certificates/key","--watch-ingress-without-class=true","--publish-status-address=localhost"],"env":[{"name":"POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"POD_NAMESPACE","valueFrom":{"fieldRef":{"fieldPath":"metadata.namespace"}}},{"name":"LD_PRELOAD","value":"/usr/local/lib/libmimalloc.so"}],"image":"registry.cn-qingdao.aliyuncs.com/kubernetes_xingej/nginx-ingress-controller:v1.1.1","imagePullPolicy":"IfNotPresent","lifecycle":{"preStop":{"exec":{"command":["/wait-shutdown"]}}},"livenessProbe":{"failureThreshold":5,"httpGet":{"path":"/healthz","port":10254,"scheme":"HTTP"},"initialDelaySeconds":10,"periodSeconds":10,"successThreshold":1,"timeoutSeconds":1},"name":"controller","ports":[{"containerPort":80,"hostPort":80,"name":"http","protocol":"TCP"},{"containerPort":443,"hostPort":443,"name":"https","protocol":"TCP"},{"containerPort":8443,"name":"webhook","protocol":"TCP"}],"readinessProbe":{"failureThreshold":3,"httpGet":{"path":"/healthz","port":10254,"scheme":"HTTP"},"initialDelaySeconds":10,"periodSeconds":10,"successThreshold":1,"timeoutSeconds":1},"resources":{"requests":{"cpu":"100m","memory":"90Mi"}},"securityContext":{"allowPrivilegeEscalation":true,"capabilities":{"add":["NET_BIND_SERVICE"],"drop":["ALL"]},"runAsUser":101},"volumeMounts":[{"mountPath":"/usr/local/certificates/","name":"webhook-cert","readOnly":true}]}],"dnsPolicy":"ClusterFirst","nodeSelector":{"ingress-ready":"true","kubernetes.io/os":"linux"},"serviceAccountName":"ingress-nginx","terminationGracePeriodSeconds":0,"tolerations":[{"effect":"NoSchedule","key":"node-role.kubernetes.io/master","operator":"Equal"}],"volumes":[{"name":"webhook-cert","secret":{"secretName":"ingress-nginx-admission"}}]}}}}
creationTimestamp: "2022-10-31T08:30:33Z"
generation: 8
labels:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/managed-by: Helm
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/version: 1.1.1
helm.sh/chart: ingress-nginx-4.0.15
name: ingress-nginx-controller
namespace: ingress-nginx
resourceVersion: "3780917"
uid: e4b28ba0-eb0b-45e3-a0b8-c03797c8f416
spec:
progressDeadlineSeconds: 600
replicas: 1
revisionHistoryLimit: 10
selector:
matchLabels:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
strategy:
rollingUpdate:
maxSurge: 25%
maxUnavailable: 1
type: RollingUpdate
template:
metadata:
creationTimestamp: null
labels:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
spec:
containers:
- args:
- /nginx-ingress-controller
- --election-id=ingress-controller-leader
- --controller-class=k8s.io/ingress-nginx
- --configmap=$(POD_NAMESPACE)/ingress-nginx-controller
- --validating-webhook=:8443
- --validating-webhook-certificate=/usr/local/certificates/cert
- --validating-webhook-key=/usr/local/certificates/key
- --watch-ingress-without-class=true
- --publish-status-address=localhost
env:
- name: POD_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.namespace
- name: LD_PRELOAD
value: /usr/local/lib/libmimalloc.so
image: bitnami/nginx-ingress-controller:1.1.1
imagePullPolicy: IfNotPresent
lifecycle:
preStop:
exec:
command:
- /wait-shutdown
livenessProbe:
failureThreshold: 5
httpGet:
path: /healthz
port: 10254
scheme: HTTP
initialDelaySeconds: 10
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
name: controller
ports:
- containerPort: 10254 #这两行是将10254暴露出来
hostPort: 10254
name: prometheus
protocol: TCP
- containerPort: 80
hostPort: 80
name: http
protocol: TCP
- containerPort: 443
hostPort: 443
name: https
protocol: TCP
- containerPort: 8443
hostPort: 8443
name: webhook
protocol: TCP
readinessProbe:
failureThreshold: 3
httpGet:
path: /healthz
port: 10254
scheme: HTTP
initialDelaySeconds: 10
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
resources:
requests:
cpu: 100m
memory: 90Mi
securityContext:
allowPrivilegeEscalation: true
capabilities:
add:
- NET_BIND_SERVICE
drop:
- ALL
runAsUser: 101
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /usr/local/certificates/
name: webhook-cert
readOnly: true
dnsPolicy: ClusterFirst
hostNetwork: true # 这里是声明直接将宿主机的端口给Ingress使用
nodeSelector:
ingress-ready: "true"
kubernetes.io/os: linux
restartPolicy: Always
schedulerName: default-scheduler
securityContext: {}
serviceAccount: ingress-nginx
serviceAccountName: ingress-nginx
terminationGracePeriodSeconds: 0
tolerations:
- effect: NoSchedule
key: node-role.kubernetes.io/master
operator: Equal
volumes:
- name: webhook-cert
secret:
defaultMode: 420
secretName: ingress-nginx-admission
status:
availableReplicas: 1
conditions:
- lastTransitionTime: "2022-10-31T08:30:33Z"
lastUpdateTime: "2022-10-31T08:30:33Z"
message: Deployment has minimum availability.
reason: MinimumReplicasAvailable
status: "True"
type: Available
- lastTransitionTime: "2022-10-31T08:46:21Z"
lastUpdateTime: "2022-11-03T10:25:05Z"
message: ReplicaSet "ingress-nginx-controller-87bc8b9c7" has successfully progressed.
reason: NewReplicaSetAvailable
status: "True"
type: Progressing
observedGeneration: 8
readyReplicas: 1
replicas: 1
updatedReplicas: 1
这时候可以检查一下宿主机的端口是否被ingress占用
root@node1:~# ss -lntp |grep nginx-ingress
LISTEN 0 4096 127.0.0.1:10245 0.0.0.0:* users:(("nginx-ingress-c",pid=1398881,fd=8))
root@master1:~# git clone https://github.com/kubernetes/kube-state-metrics.git
root@master1:~# cd kube-state-metrics/examples/standard/
root@master1:~/kube-state-metrics/examples/standard# ls
cluster-role-binding.yaml cluster-role.yaml deployment.yaml service-account.yaml service.yaml
# 克隆下来之后直接有几个yaml文件，稍微修改一下就可以直接用了
这边改一下service.yaml
root@master1:~/kube-state-metrics/examples/standard# cat service.yaml
apiVersion: v1
kind: Service
metadata:
labels:
app.kubernetes.io/component: exporter
app.kubernetes.io/name: kube-state-metrics
app.kubernetes.io/version: 2.6.0
name: kube-state-metrics
namespace: kube-system
spec:
type: ClusterIP # 刚clone下来是无头服务，我这边改为了ClusterIP
ports:
- name: http-metrics
port: 8080
targetPort: http-metrics
- name: telemetry
port: 8081
targetPort: telemetry
selector:
app.kubernetes.io/name: kube-state-metrics
然后apply一下并进行检查
root@master1:~/kube-state-metrics/examples/standard# kubectl apply -f . #.不要掉了，指的apply当前目录下的yaml
root@master1:~/kube-state-metrics/examples/standard# kubectl get svc,pod -n kube-system
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
service/kube-dns ClusterIP 10.100.0.10 <none> 53/UDP,53/TCP,9153/TCP 23d
service/kube-state-metrics ClusterIP 10.100.162.224 <none> 8080/TCP,8081/TCP 91m #这就是刚刚生成的svc
NAME READY STATUS RESTARTS AGE
pod/coredns-c676cc86f-7bd45 1/1 Running 5 (2d2h ago) 22d
pod/coredns-c676cc86f-7l2c8 1/1 Running 0 22d
pod/coredns-c676cc86f-gl6qc 1/1 Running 0 22d
pod/coredns-c676cc86f-rxcjn 1/1 Running 0 22d
pod/kube-apiserver-master1.org 1/1 Running 6 (2d2h ago) 17d
pod/kube-apiserver-master2.org 1/1 Running 0 17d
pod/kube-apiserver-master3.org 1/1 Running 0 17d
pod/kube-controller-manager-master1.org 1/1 Running 7 (2d2h ago) 23d
pod/kube-controller-manager-master2.org 1/1 Running 0 17d
pod/kube-controller-manager-master3.org 1/1 Running 0 17d
pod/kube-proxy-5pnh5 1/1 Running 0 21d
pod/kube-proxy-vws54 1/1 Running 0 21d
pod/kube-proxy-wlknz 1/1 Running 0 21d
pod/kube-proxy-wtnnf 1/1 Running 5 (2d2h ago) 21d
pod/kube-proxy-xkkhp 1/1 Running 0 21d
pod/kube-proxy-zf4vg 1/1 Running 0 21d
pod/kube-scheduler-master1.org 1/1 Running 7 (2d2h ago) 23d
pod/kube-scheduler-master2.org 1/1 Running 0 22d
pod/kube-scheduler-master3.org 1/1 Running 0 22d
pod/kube-state-metrics-757ff7b448-lwwlg 1/1 Running 0 90m #这个就是刚刚生成的pod
验证一下metrics是否有数据
root@node1:~# curl -I 10.100.162.224:8080/metrics
HTTP/1.1 200 OK
Content-Type: text/plain; version=0.0.4
Date: Thu, 03 Nov 2022 16:25:21 GMT
root@master1:~/k8s-prometheus# cat prometheus-configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: prometheus-config
namespace: monitor
data:
prometheus.yml: |
global:
scrape_interval: 15s
scrape_timeout: 15s
scrape_configs:
- job_name: 'prometheus'
static_configs:
- targets: ['localhost:9090']
- job_name: 'Linux Server'
static_configs:
- targets:
- 'xxx.xxx.xxx.xxx:9100'
- 'xxx.xxx.xxx.xxx:9100'
- 'xxx.xxx.xxx.xxx:9100'
- 'xxx.xxx.xxx.xxx:9100'
- 'xxx.xxx.xxx.xxx:9100'
- 'xxx.xxx.xxx.xxx:9100'
- job_name: 'coredns'
static_configs: #我这边基本上都是用IP写的target，实际上也可以用域名，默认的域名书写方式为 kube-dns.kube-system.svc.cluster.local
- targets: ['xxx.xxx.xxx.xxx:9153']
- job_name: 'ingress-nginx'
static_configs:
- targets: ['xxx.xxx.xxx.xxx:10254']
- job_name: 'k8s-state-metrics'
static_configs:
- targets:
- '10.100.162.224:8080'
- '10.100.162.224:8081'
root@master1:~/k8s-prometheus# kubectl apply -f prometheus-configmap.yaml
root@master1:~/k8s-prometheus# curl -XPOST 10.100.122.13:9090/-/reload
查看是否生效
基本上大部分需要import的都可以直接去Grafana官网搜索，搜索完成之后可以copy 对应id，也可以下载下来
下面演示导入ID的操作（如果是导入json的话，只需要复制json文件粘贴到对应的框里面即可）
这个dashboard导入的是9276
这里导入的是5926，效果如下
这里导入的是9614，效果如下
这个dashboard导入的是13105
