| 监控指标 | 具体实现 | 举例 |
|---|---|---|
| Pod性能 | cAdvisor | 容器CPU，内存利用率 |
| Node性能 | node-exporter | 节点CPU，内存利用率 |
| K8S资源对象 | kube-state-metrics | Pod/Deployment/Service |
参考链接：
Kubernetes RBAC 详解
k8s部署Prometheus
如何监控k8s apiserver
#创建一个monitor-sa的名称空间
kubectl create ns monitor-sa
#创建一个sa账号
kubectl create serviceaccount monitor -n monitor-sa
#把sa账号monitor通过clusterrolebing绑定到clusterrole上
kubectl create clusterrolebinding monitor-clusterrolebinding -n monitor-sa --clusterrole=cluster-admin --serviceaccount=monitor-sa:monitor
#在k8s集群的任何一个node节点操作，因为我的k8s集群只有一个node节点node1，所以我在node1上操作如下命令：
mkdir /data/prometheus
chmod 777 /data/prometheus
prometheus通过 sa,clusterrolebinding来解决token、证书挂载问题
sa等配置： prometheus yaml中需要配置对应的saserviceAccountName
kube-state-metrics github项目地址
.
├── cluster-role-binding.yaml
├── cluster-role.yaml
├── deployment.yaml
├── README.md
├── service-account.yaml
└── service.yaml
service-account.yaml文件
apiVersion: v1
kind: ServiceAccount
metadata:
labels:
app.kubernetes.io/name: kube-state-metrics
app.kubernetes.io/version: v1.8.0
name: kube-state-metrics
namespace: monitor-sa
cluster-role.yaml 文件
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
app.kubernetes.io/name: kube-state-metrics
app.kubernetes.io/version: v1.8.0
name: kube-state-metrics
rules:
- apiGroups:
- ""
resources:
- configmaps
- secrets
- nodes
- pods
- services
- resourcequotas
- replicationcontrollers
- limitranges
- persistentvolumeclaims
- persistentvolumes
- namespaces
- endpoints
verbs:
- list
- watch
- apiGroups:
- extensions
resources:
- daemonsets
- deployments
- replicasets
- ingresses
verbs:
- list
- watch
- apiGroups:
- apps
resources:
- statefulsets
- daemonsets
- deployments
- replicasets
verbs:
- list
- watch
- apiGroups:
- batch
resources:
- cronjobs
- jobs
verbs:
- list
- watch
- apiGroups:
- autoscaling
resources:
- horizontalpodautoscalers
verbs:
- list
- watch
- apiGroups:
- authentication.k8s.io
resources:
- tokenreviews
verbs:
- create
- apiGroups:
- authorization.k8s.io
resources:
- subjectaccessreviews
verbs:
- create
- apiGroups:
- policy
resources:
- poddisruptionbudgets
verbs:
- list
- watch
- apiGroups:
- certificates.k8s.io
resources:
- certificatesigningrequests
verbs:
- list
- watch
- apiGroups:
- storage.k8s.io
resources:
- storageclasses
- volumeattachments
verbs:
- list
- watch
- apiGroups:
- admissionregistration.k8s.io
resources:
- mutatingwebhookconfigurations
- validatingwebhookconfigurations
verbs:
- list
- watch
- apiGroups:
- networking.k8s.io
resources:
- networkpolicies
verbs:
- list
- watch
cluster-role-binding.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
labels:
app.kubernetes.io/name: kube-state-metrics
app.kubernetes.io/version: v1.8.0
name: kube-state-metrics
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: kube-state-metrics
subjects:
- kind: ServiceAccount
name: kube-state-metrics
namespace: monitor-sa
deployment.yaml 文件
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app.kubernetes.io/name: kube-state-metrics
app.kubernetes.io/version: v1.8.0
name: kube-state-metrics
namespace: monitor-sa
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: kube-state-metrics
template:
metadata:
labels:
app.kubernetes.io/name: kube-state-metrics
app.kubernetes.io/version: v1.8.0
spec:
containers:
- image: quay.io/coreos/kube-state-metrics:v1.8.0
livenessProbe:
httpGet:
path: /healthz
port: 8080
initialDelaySeconds: 5
timeoutSeconds: 5
name: kube-state-metrics
ports:
- containerPort: 8080
name: http-metrics
- containerPort: 8081
name: telemetry
readinessProbe:
httpGet:
path: /
port: 8081
initialDelaySeconds: 5
timeoutSeconds: 5
nodeSelector:
kubernetes.io/os: linux
serviceAccountName: kube-state-metrics
service.yaml文件
apiVersion: v1
kind: Service
metadata:
annotations:
prometheus.io/scrape: 'true'
name: kube-state-metrics
namespace: monitor-sa
labels:
app: kube-state-metrics
spec:
ports:
- name: kube-state-metrics
port: 8080
protocol: TCP
- name: telemetry
port: 8081
protocol: TCP
selector:
app.kubernetes.io/name: kube-state-metrics
1）创建一个configmap存储卷，用来存放prometheus配置信息
kubectl get sa monitor -n monitor-sa -o yaml
apiVersion: v1
kind: ServiceAccount
metadata:
creationTimestamp: "2021-05-23T14:18:14Z"
name: monitor
namespace: monitor-sa
resourceVersion: "18761312"
selfLink: /api/v1/namespaces/monitor-sa/serviceaccounts/monitor
uid: 12ed67ab-dae8-4704-87b8-5a073a7047d2
secrets:
- name: monitor-token-p6wgp
kubectl describe sa monitor -n monitor-sa
Name: monitor
Namespace: monitor-sa
Labels: <none>
Annotations: <none>
Image pull secrets: <none>
Mountable secrets: monitor-token-p6wgp
Tokens: monitor-token-p6wgp
Events: <none>
kubectl describe secrets monitor-token-p6wgp -n monitor-sa
Name: monitor-token-p6wgp
Namespace: monitor-sa
Labels: <none>
Annotations: kubernetes.io/service-account.name: monitor
kubernetes.io/service-account.uid: 12ed67ab-dae8-4704-87b8-5a073a7047d2
Type: kubernetes.io/service-account-token
Data
====
ca.crt: 1025 bytes
namespace: 10 bytes
token: eyJhbGciOiJSUzI1NiIsImtpZCI6IkY5eEJEUjZMRjNnejhxMVl6enJiYmVHX0RSOFYza1JfbVVpZmhCVXlucDQifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJtb25pdG9yLXNhIiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZWNyZXQubmFtZSI6Im1vbml0b3ItdG9rZW4tcDZ3Z3AiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC5uYW1lIjoibW9uaXRvciIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50LnVpZCI6IjEyZWQ2N2FiLWRhZTgtNDcwNC04N2I4LTVhMDczYTcwNDdkMiIsInN1YiI6InN5c3RlbTpzZXJ2aWNlYWNjb3VudDptb25pdG9yLXNhOm1vbml0b3IifQ.j2hWAze7aOZgVDg0j4NKOhoMUktu7XIJ56kU_RCRbt7RCaXYd_A4ijg7IJqVUHBitQKfx-_ZzNXcOqMZt5nCN5dtToKGWRK_Du0eqepKNcsfj9dzVvebaEbd-4t7LyhHvEdf5M1CviD0wnrw1O_9nXl1COpm9IojJB9I8tIzs9Y3fiMVd2oTUL3ctFKRSkwM4CTAEIm5SZN0QRgld7Ol8W7F-m8jjOh3c7MMm9FnnAn_NkQ57XSKJovMy_AdMA55gwZaufCYA225tubG9KS0eUyF70wgGvAKOMFn6yGpRZjHj26JcBDhoEZkwzFrBM4-blnGl9pMHXtPztAPlw-xQQ
cat >prometheus-cfg.yaml <<EOF
---
kind: ConfigMap
apiVersion: v1
metadata:
labels:
app: prometheus
name: prometheus-config
namespace: monitor-sa
data:
prometheus.yml: |
global:
scrape_interval: 15s
scrape_timeout: 10s
external_labels:
monitor: 'AIUI-ceshi-k8s'
evaluation_interval: 1m
scrape_configs:
- job_name: kubernetes-node
kubernetes_sd_configs:
- role: node
tls_config:
insecure_skip_verify: true
bearer_token_file: /opt/k8s/k8s.token
relabel_configs:
- source_labels: [__address__]
regex: '(.*):10250'
replacement: '${1}:9100'
target_label: __address__
action: replace
- action: labelmap
regex: __meta_kubernetes_node_label_(.+)
- source_labels: [instance]
regex: .*db002.*
action: drop
- job_name: 'kubernetes-apiservers'
kubernetes_sd_configs:
- role: endpoints
scheme: https
tls_config:
insecure_skip_verify: true
bearer_token_file: /opt/k8s/k8s.token
relabel_configs:
- source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
action: keep
regex: default;kubernetes;https
- job_name: 'kube-state-metrics'
static_configs:
- targets: ['kube-state-metrics:8080']
- job_name: 'kubernetes-node-cadvisor'
kubernetes_sd_configs:
- role: node
scheme: https
tls_config:
insecure_skip_verify: true
bearer_token_file: /opt/k8s/k8s.token
relabel_configs:
- action: labelmap
regex: __meta_kubernetes_node_label_(.+)
- target_label: __address__
replacement: 172.16.154.13:6443
- source_labels: [__meta_kubernetes_node_name]
regex: (.+)
target_label: __metrics_path__
replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
- job_name: 'kubernetes-service-endpoints'
scrape_timeout: 10s
kubernetes_sd_configs:
- role: endpoints
relabel_configs:
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
action: keep
regex: true
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
action: replace
target_label: __scheme__
regex: (https?)
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
action: replace
target_label: __metrics_path__
regex: (.+)
- source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
action: replace
target_label: __address__
regex: ([^:]+)(?::\d+)?;(\d+)
replacement: $1:$2
- action: labelmap
regex: __meta_kubernetes_service_label_(.+)
- source_labels: [__meta_kubernetes_namespace]
action: replace
target_label: kubernetes_namespace
- source_labels: [__meta_kubernetes_service_name]
action: replace
target_label: kubernetes_name
# - source_labels: [__meta_kubernetes_pod_container_port_number]
# action: replace
# target_label: container_port
EOF
cat >prometheus-deploy.yaml <<EOF
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: prometheus-server
namespace: monitor-sa
labels:
app: prometheus
spec:
replicas: 1
selector:
matchLabels:
app: prometheus
component: server
template:
metadata:
labels:
app: prometheus
component: server
annotations:
prometheus.io/scrape: 'false'
spec:
serviceAccountName: monitor
containers:
- name: prometheus
image: prom/prometheus:v2.2.1
imagePullPolicy: IfNotPresent
command:
- 'prometheus'
- '--config.file=/etc/prometheus/prometheus.yml'
- '--storage.tsdb.path=/prometheus'
- '--storage.tsdb.retention=720h'
- '--web.enable-lifecycle'
ports:
- containerPort: 9090
protocol: TCP
volumeMounts:
- mountPath: /etc/prometheus/prometheus.yml
name: prometheus-config
subPath: prometheus.yml
- mountPath: /prometheus/
name: prometheus-storage-volume
- mountPath: /opt/k8s/k8s.token
name: k8s-token
subPath: k8s.token
volumes:
- name: prometheus-config
configMap:
name: prometheus-config
items:
- key: prometheus.yml
path: prometheus.yml
mode: 0644
- name: prometheus-storage-volume
hostPath:
path: /data/prometheus
type: Directory
- name: k8s-token
hostPath:
path: /opt/k8s
type: Directory
cat > prometheus-svc.yaml <<EOF
---
apiVersion: v1
kind: Service
metadata:
annotations:
prometheus.io/scrape: 'true'
name: prometheus
namespace: monitor-sa
labels:
app: prometheus
spec:
type: NodePort
ports:
- port: 9090
targetPort: 9090
nodePort: 30090
protocol: TCP
selector:
app: prometheus
component: server
EOF
