Grafana
Jump to navigation
Jump to search
Grafana Labs
FREE TIER !
Note, the free plan starts with a 14 trial Pro, but you can cancel the trial immediatly and get a complete but limited account for free
Grafana
Setup the grafana agent
Agent installation
- from https://<YOUR_HOST>.grafana.net/a/grafana-easystart-app/
Note: you can get YOUR_USER from grafana DataSources, and YOUR_PASS is an ApiKey you must create in Grafana Cloud Portal as MetricPublisher role.
# binary download
apt-get -y install unzip curl < /dev/null
apt-get -y clean < /dev/null
mkdir -m 0755 /srv/local/ /srv/local/grafana/ /srv/local/grafana/bin/
cd /srv/local/grafana/bin/
curl -O -L "https://github.com/grafana/agent/releases/download/v0.12.0/agent-linux-amd64.zip"
unzip "agent-linux-amd64.zip"
rm -f "agent-linux-amd64.zip"
chmod a+x "agent-linux-amd64"
# configuration
mkdir -m 0755 /srv/local/grafana/etc/
cd /srv/local/grafana/etc/
cat > agent-config.yaml << EOF
server:
http_listen_port: 12345
prometheus:
wal_directory: /tmp/grafana-agent-wal
global:
scrape_interval: 15s
configs: null
integrations:
node_exporter:
enabled: true
prometheus_remote_write:
- url: https://prometheus-us-central1.grafana.net/api/prom/push
basic_auth:
username: <YOUR_USER>
password: <YOUR_PASS>
EOF
chmod 644 /srv/local/grafana/etc/agent-config.yaml
# creation of specific user
useradd --no-create-home --home-dir /srv/local/grafana -g nogroup --shell /usr/sbin/nologin grafana-agent
Create a systemd service for the agent
# please note that such file must end with .service cat > /etc/systemd/system/grafana-agent.service << EOF [Unit] Description=Grafana Cloud Agent [Service] User=grafana-agent ExecStart=/srv/local/grafana/bin/agent-linux-amd64 --config.file=/srv/local/grafana/etc/agent-config.yaml Restart=always [Install] WantedBy=multi-user.target EOF
Enable and start the agent
# reload service files systemctl daemon-reload # start manually for this time systemctl start grafana-agent.service # check service status systemctl status grafana-agent.service # to check service logs journalctl -u grafana-agent.service # enable service at boot systemctl enable grafana-agent.service
Add Postgresql integration
# create a readonly postgres user
su - postgres
psql
CREATE USER grafanareader WITH PASSWORD '<YOUR_PASSWORD>';
\connect <YOUR_DATABASE>
GRANT USAGE ON SCHEMA <YOUR_SCHEMA> TO grafanareader;
GRANT SELECT ON ALL TABLES IN SCHEMA <YOUR_SCHEMA> TO grafanareader;
ALTER DEFAULT PRIVILEGES IN SCHEMA <YOUR_SCHEMA> GRANT SELECT ON TABLES TO grafanareader;
\q
exit
# improve config
sed -i 's|integrations:|integrations:\
postgres_exporter:\
enabled: true\
data_source_names:\
- "postgresql://grafanareader:<YOUR_PASSWORD>@localhost:5432/postgres?sslmode=disable"\
|g' /srv/local/grafana/etc/agent-config.yaml
Loki
Setup the fluent-bit agent
Fluent-bit agent installation
Build a minimal binary
apt-get install make cmake g++ flex bison apt-get clean mkdir /srv/tmp cd /srv/tmp curl -L -O https://fluentbit.io/releases/1.6/fluent-bit-1.6.10.tar.gz tar -xzf fluent-bit-1.6.10.tar.gz cd fluent-bit-1.6.10/build cmake .. -DCMAKE_INSTALL_PREFIX=/srv/local/fluent-bit -DFLB_ALL=No -DFLB_DEBUG=No -DFLB_TLS=Yes -DFLB_EXAMPLES=No -DFLB_SHARED_LIB=No -DFLB_INOTIFY=No -DFLB_SQLDB=No -DFLB_HTTP_SERVER=Yes -DFLB_BACKTRACE=No -DFLB_LUAJIT=No -DFLB_RECORD_ACCESSOR=Yes -DFLB_SIGNV4=No -DFLB_AWS=No -DFLB_STREAM_PROCESSOR=Yes -DFLB_AVRO_ENCODER=No -DFLB_PROXY_GO=No -DFLB_IN_DOCKER=No -DFLB_IN_DOCKER_EVENTS=No -DFLB_IN_SERIAL=No -DFLB_IN_MQTT=No -DFLB_IN_NETIF=No -DFLB_IN_WINLOG=No -DFLB_IN_COLLECTD=No -DFLB_IN_STATSD=No -DFLB_IN_STORAGE_BACKLOG=No -DFLB_IN_EMITTER=Yes -DFLB_OUT_AZURE=No -DFLB_OUT_AZURE_BLOB=No -DFLB_OUT_BIGQUERY=No -DFLB_OUT_COUNTER=No -DFLB_OUT_DATADOG=No -DFLB_OUT_ES=No -DFLB_OUT_EXIT=No -DFLB_OUT_FORWARD=No -DFLB_OUT_GELF=No -DFLB_OUT_INFLUXDB=No -DFLB_OUT_NATS=No -DFLB_OUT_NRLOGS=No -DFLB_OUT_PLOT=No -DFLB_OUT_TD=No -DFLB_OUT_RETRY=No -DFLB_OUT_PGSQL=No -DFLB_OUT_SLACK=No -DFLB_OUT_SPLUNK=No -DFLB_OUT_STACKDRIVER=No -DFLB_OUT_FLOWCOUNTER=No -DFLB_OUT_LOGDNA=No -DFLB_OUT_LOKI=Yes -DFLB_OUT_KAFKA=No -DFLB_OUT_KAFKA_REST=No -DFLB_OUT_CLOUDWATCH_LOGS=No -DFLB_OUT_KINESIS_FIREHOSE=No -DFLB_OUT_S3=No -DFLB_FILTER_AWS=No -DFLB_FILTER_KUBERNETES=No -DFLB_FILTER_LUA=No -DFLB_FILTER_TENSORFLOW=No make make install rm -rf /srv/tmp
Configuration
cd /srv/local/fluent-bit
mv etc etc_bkp
mkdir -p etc/fluent-bit/ log/
cat > etc/fluent-bit/fluent-bit.conf << 'EOF'
[SERVICE]
flush 5
daemon off
log_level info
http_server off
storage.metrics off
parsers_file parsers.conf
[INPUT]
name tail
path /srv/local/fluent-bit/log/springboot*.json
buffer_chunk_size 512k
buffer_max_size 8MB
mem_buf_limit 8MB
refresh_interval 10
parser springboot
[Output]
name loki
log_level info
match *
host logs-prod-us-central1.grafana.net
port 443
tls on
tls.verify off
http_user <YOUR_USERNAME>
http_passwd <YOUR_API_METRICPUBLISHER_KEY>
labels job=fluent-bit-loki
line_format json
EOF
cat > etc/fluent-bit/parsers.conf << 'EOF'
[PARSER]
name springboot
format json
time_key time
time_format %Y-%m-%dT%H:%M:%S.%L
time_keep on
EOF
# creation of specific user
useradd --no-create-home --home-dir /srv/local/fluent-bit -g nogroup --shell /usr/sbin/nologin fluent-bit-agent
Create a systemd service for the agent
# please note that such file must end with .service cat > /etc/systemd/system/fluent-bit-agent.service << EOF [Unit] Description=Fluent-bit Agent [Service] User=fluent-bit-agent ExecStart=/srv/local/fluent-bit/bin/fluent-bit -c /srv/local/fluent-bit/etc/fluent-bit/fluent-bit.conf Restart=always [Install] WantedBy=multi-user.target EOF
Enable and start the agent
# reload service files systemctl daemon-reload # start manually for this time systemctl start fluent-bit-agent.service # check service status systemctl status fluent-bit-agent.service # to check service logs journalctl -u fluent-bit-agent.service # enable service at boot systemctl enable fluent-bit-agent.service