# Setup Prometheus + Grafana for monitoring multiple ec2 groups of instances #monitoring #prometheus #grafana ![[install prometheus+grafana.png]] ## Prometheus server - add this script to user data when creating EC2 instance or run it directly on the server - **always read scripts before running!** ```bash #!/bin/bash #-------------------------------------------------------------------- # Script to Install Prometheus Server on Linux Ubuntu # Tested on Ubuntu 22.04, 24.04 #-------------------------------------------------------------------- PROMETHEUS_VERSION="2.51.1" PROMETHEUS_FOLDER_CONFIG="/etc/prometheus" PROMETHEUS_FOLDER_TSDATA="/etc/prometheus/data" # Install Prometheus cd /tmp || exit wget https://github.com/prometheus/prometheus/releases/download/v$PROMETHEUS_VERSION/prometheus-$PROMETHEUS_VERSION.linux-amd64.tar.gz tar xvfz prometheus-$PROMETHEUS_VERSION.linux-amd64.tar.gz cd prometheus-$PROMETHEUS_VERSION.linux-amd64 || exit mv prometheus /usr/bin/ rm -rf /tmp/prometheus* # Create Prometheus Configuration Folder mkdir -p $PROMETHEUS_FOLDER_CONFIG # Create Prometheus Data Folder mkdir -p $PROMETHEUS_FOLDER_TSDATA # Create Prometheus Configuration File cat <<EOF> $PROMETHEUS_FOLDER_CONFIG/prometheus.yml global: scrape_interval: 15s scrape_configs: - job_name: "prometheus" static_configs: - targets: ["localhost:9090"] - job_name: "prod-servers" ec2_sd_configs: - port: 9100 filters: - name: tag:Environment values: ["prod"] - job_name: "dev-servers" ec2_sd_configs: - port: 9100 filters: - name: tag:Environment values: ["dev"] EOF useradd -rs /bin/false prometheus chown prometheus:prometheus /usr/bin/prometheus chown prometheus:prometheus $PROMETHEUS_FOLDER_CONFIG chown prometheus:prometheus $PROMETHEUS_FOLDER_CONFIG/prometheus.yml chown prometheus:prometheus $PROMETHEUS_FOLDER_TSDATA # Create Prometheus Service cat <<EOF> /etc/systemd/system/prometheus.service [Unit] Description=Prometheus Server After=network.target [Service] User=prometheus Group=prometheus Type=simple Restart=on-failure ExecStart=/usr/bin/prometheus \ --config.file ${PROMETHEUS_FOLDER_CONFIG}/prometheus.yml \ --storage.tsdb.path ${PROMETHEUS_FOLDER_TSDATA} [Install] WantedBy=multi-user.target EOF # Start Prometheus Service systemctl daemon-reload systemctl start prometheus systemctl enable prometheus systemctl status prometheus --no-pager prometheus --version # next steps # 1. make sure to open port 9090 on your security group # 2. make sure to add tags [dev] [prod] to your instances to be able to scrape them # 3. attach the iam role to prometheus server to be able to scrape the instances (EC2ReadOnlyAccess or custom policy with ec2:DescribeInstances permission) # 4. access the prometheus server on http://<prometheus-server-ip>:9090 # 5. open targets page to see the instances that are being scraped http://<prometheus-server-ip>:9090/targets # 6. for adding more targets, you can install node_exporter on the instances and add them to the prometheus.yml file (https://github.com/avramukk/lab/prometheus/install_prometheus_node_exporter.sh) ``` ## Grafana server - Set PROMETHEUS_URL if you want to add Prometheus as a default datasource ```bash #!/bin/bash #-------------------------------------------------------------------- # Script to Install Grafana Server on Linux Ubuntu (22.04, 24.04) # Include Prometheus DataSource Configuration (Optional) #-------------------------------------------------------------------- # https://grafana.com/grafana/download GRAFANA_VERSION="10.4.2" # Set PROMETHEUS_URL if you want to add Prometheus as a default datasource # PROMETHEUS_URL="http://your-prometheus-url:9090" apt-get install -y apt-transport-https software-properties-common wget mkdir -p /etc/apt/keyrings/ wget -q -O - https://apt.grafana.com/gpg.key | gpg --dearmor | sudo tee /etc/apt/keyrings/grafana.gpg > /dev/null echo "deb [signed-by=/etc/apt/keyrings/grafana.gpg] https://apt.grafana.com stable main" | sudo tee -a /etc/apt/sources.list.d/grafana.list apt-get update apt-get install -y adduser libfontconfig1 musl wget https://dl.grafana.com/oss/release/grafana_${GRAFANA_VERSION}_amd64.deb dpkg -i grafana_${GRAFANA_VERSION}_amd64.deb echo "export PATH=/usr/share/grafana/bin:$PATH" >> /etc/profile # Add Prometheus datasource only if PROMETHEUS_URL is set if [ ! -z "${PROMETHEUS_URL}" ]; then cat <<EOF> /etc/grafana/provisioning/datasources/prometheus.yaml apiVersion: 1 datasources: - name: Prometheus type: prometheus url: ${PROMETHEUS_URL} EOF fi systemctl daemon-reload systemctl enable grafana-server systemctl start grafana-server systemctl status grafana-server ``` ## Node exporter Node exporter is a Prometheus exporter for hardware and OS metrics exposed by *NIX kernels, written in Go with pluggable metric collectors. In simple words, it collects the metrics from the system and makes them available for Prometheus to scrape. ```bash #!/bin/bash #-------------------------------------------------------------------- # script to install prometheus node_exporter on linux # tested on ubuntu 22.04, 24.04, amazon linux 2023 #-------------------------------------------------------------------- # https://github.com/prometheus/node_exporter/releases node_exporter_version="1.7.0" cd /tmp || exit wget https://github.com/prometheus/node_exporter/releases/download/v$node_exporter_version/node_exporter-$node_exporter_version.linux-amd64.tar.gz tar xvfz node_exporter-$node_exporter_version.linux-amd64.tar.gz cd node_exporter-$node_exporter_version.linux-amd64 || exit mv node_exporter /usr/bin/ rm -rf /tmp/node_exporter* useradd -rs /bin/false node_exporter chown node_exporter:node_exporter /usr/bin/node_exporter cat <<EOF > /etc/systemd/system/node_exporter.service [Unit] Description=Prometheus Node Exporter After=network.target [Service] User=node_exporter Group=node_exporter Type=simple Restart=on-failure ExecStart=/usr/bin/node_exporter [Install] WantedBy=multi-user.target EOF systemctl daemon-reload systemctl start node_exporter systemctl enable node_exporter systemctl status node_exporter node_exporter --version ```