prometheus手动添加k8s集群外的node-exporter监控

1、部署node-exporter
1）helm方式部署

root@iZj6c72dzbei17o2cuksmeZ:~# helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
"prometheus-community" has been added to your repositories
root@iZj6c72dzbei17o2cuksmeZ:~# helm repo update
Hang tight while we grab the latest from your chart repositories...
...Successfully got an update from the "prometheus-community" chart repository
Update Complete. ⎈Happy Helming!⎈
root@iZj6c72dzbei17o2cuksmeZ:~# helm install prometheus-node-exporter prometheus-community/prometheus-node-exporter
NAME: prometheus-node-exporter
LAST DEPLOYED: Sun Apr 27 10:56:21 2025
NAMESPACE: default
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
1. Get the application URL by running these commands:export POD_NAME=$(kubectl get pods --namespace default -l "app.kubernetes.io/name=prometheus-node-exporter,app.kubernetes.io/instance=prometheus-node-exporter" -o jsonpath="{.items[0].metadata.name}")echo "Visit http://127.0.0.1:9100 to use your application"kubectl port-forward --namespace default $POD_NAME 9100

2）docker-compose部署

root@iZj6c2vhsafoay7j7vyy89Z:~# cat /data/docker-compose/node-exporter/docker-compose.yaml 
version: '3.8'services:node-exporter:image: prom/node-exporter:latestcontainer_name: node-exporterrestart: unless-stoppedports:- "9100:9100"network_mode: "host"   # 使用宿主机网络，方便采集真实的主机数据pid: "host"            # 让容器可以访问宿主机的 /procvolumes:- /proc:/host/proc:ro- /sys:/host/sys:ro- /:/rootfs:rocommand:- '--path.procfs=/host/proc'- '--path.sysfs=/host/sys'- '--path.rootfs=/rootfs'- '--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|host|etc)($$|/)'

部署

root@iZj6c2vhsafoay7j7vyy89Z:/data/docker-compose/node-exporter# docker-compose up -d
Pulling node-exporter (prom/node-exporter:latest)...
latest: Pulling from prom/node-exporter
9fa9226be034: Pull complete
1617e25568b2: Pull complete
c6e37428e3b3: Pull complete
Digest: sha256:d00a542e409ee618a4edc67da14dd48c5da66726bbd5537ab2af9c1dfc442c8a
Status: Downloaded newer image for prom/node-exporter:latest
Creating node-exporter ... done

2、部署endpoints、service、ServiceMonitor
endpoints的yaml文件
root@iZj6c72dzbei17o2cuksmeZ:~/yaml# cat zjj-endpoints.yaml

apiVersion: v1
kind: Endpoints
metadata:annotations: {}        labels:app: zjjjobname: node-exportersource: externalname: zjj namespace: monitoring
subsets:
- addresses:- hostname: zjjip: 172.16.0.20ports:- name: metricsport: 9100 protocol: TCP

service的yaml文件
root@iZj6c72dzbei17o2cuksmeZ:~/yaml# cat zjj-nodeporter-service.yaml

---
apiVersion: v1
kind: Service
metadata:annotations: {}labels:app: zjj jobname: node-exportersource: externalname: zjjnamespace: monitoringresourceVersion: '552219945'
spec:internalTrafficPolicy: ClusteripFamilies:- IPv4ipFamilyPolicy: SingleStackports:- name: metricsport: 9100 protocol: TCPtargetPort: 9100sessionAffinity: Nonetype: ClusterIP

ServiceMonitor的yaml文件
root@iZj6c72dzbei17o2cuksmeZ:~/yaml# cat zjj-servicemonitor.yaml

---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:annotations: {}labels:app: zjj release: kube-prometheussource: externalname: zjjnamespace: monitoringresourceVersion: '294048548'
spec:endpoints:- port: metricsjobLabel: jobnamenamespaceSelector:matchNames:- monitoringselector:matchLabels:app: zjjsource: external

启动对应的服务

root@iZj6c72dzbei17o2cuksmeZ:~/yaml# kubectl apply -f zjj-endpoints.yaml 
endpoints/zjj created
root@iZj6c72dzbei17o2cuksmeZ:~/yaml# kubectl apply -f zjj-nodeporter-service.yaml 
service/zjj created
root@iZj6c72dzbei17o2cuksmeZ:~/yaml# kubectl apply -f zjj-servicemonitor.yaml 
servicemonitor.monitoring.coreos.com/zjj created

root@iZj6c72dzbei17o2cuksmeZ:~/yaml# kubectl get endpoints zjj -n monitoring
NAME   ENDPOINTS          AGE
zjj    172.16.0.20:9100   55m
root@iZj6c72dzbei17o2cuksmeZ:~/yaml# kubectl get service zjj -n monitoring
NAME   TYPE        CLUSTER-IP     EXTERNAL-IP   PORT(S)    AGE
zjj    ClusterIP   10.68.140.25   <none>        9100/TCP   54m
root@iZj6c72dzbei17o2cuksmeZ:~/yaml# kubectl get servicemonitor zjj -n monitoring
NAME   AGE
zjj    55m

3、创建alertmanager报警模块
root@iZj6c72dzbei17o2cuksmeZ:~/kube-prometheus-0.14.0# cat manifests/zjj-nodeExporter-prometheusRule.yaml

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:labels:app.kubernetes.io/component: nodeInstancesapp.kubernetes.io/name: NodeInstancesapp.kubernetes.io/part-of: kube-prometheusapp.kubernetes.io/version: 1.8.2prometheus: k8srole: alert-rulesname: node-instances-rulesnamespace: monitoring
spec:groups:- name: zjj-rulesrules:- alert: NodeInstancesDownannotations:description: Failed to scrape {{ $labels.job }} on {{ $labels.instance }} formore than 1 minutes. Node seems down.title: Node {{ $labels.instance }} is downexpr: up{job="node-exporter"} == 0for: 1mlabels:severity: critical

启动新的规则

root@iZj6c72dzbei17o2cuksmeZ:~/kube-prometheus-0.14.0# kubectl apply -f manifests/zjj-nodeExporter-prometheusRule.yaml