telemetry: added tempo service

This commit is contained in:
Tigor Hutasuhut 2024-09-02 18:17:03 +07:00
parent a54dcef036
commit 4786f0562b
5 changed files with 129 additions and 10 deletions

View file

@ -73,6 +73,17 @@
"bind" "bind"
]; ];
}; };
"/nas/telemetry/tempo" = lib.mkIf config.profile.services.telemetry.tempo.enable {
device = "/var/lib/tempo";
fsType = "auto";
options = [
"defaults"
"nofail"
"nobootwait"
"bind"
];
};
}; };
swapDevices = [ ]; swapDevices = [ ];

View file

@ -9,6 +9,12 @@ loki:
username: ENC[AES256_GCM,data:MRwky3O8LGS/4w==,iv:CUHjGRNc8NU5FqhqvpqbATmVE3Kg9Z0jMBFlzsAwON0=,tag:uZlxw9skd0VNLfZTJ/6ZSQ==,type:str] username: ENC[AES256_GCM,data:MRwky3O8LGS/4w==,iv:CUHjGRNc8NU5FqhqvpqbATmVE3Kg9Z0jMBFlzsAwON0=,tag:uZlxw9skd0VNLfZTJ/6ZSQ==,type:str]
#ENC[AES256_GCM,data:t+u4g7nvpq2U27CHgmu1xi2Ppwv7cJf1s6Et,iv:LtwoPoxsQn1MujHRoD6SqDLm8uN4uBpuIVmn23DDgjI=,tag:PKOHF5vcQzHszpp1sfuU0Q==,type:comment] #ENC[AES256_GCM,data:t+u4g7nvpq2U27CHgmu1xi2Ppwv7cJf1s6Et,iv:LtwoPoxsQn1MujHRoD6SqDLm8uN4uBpuIVmn23DDgjI=,tag:PKOHF5vcQzHszpp1sfuU0Q==,type:comment]
password: ENC[AES256_GCM,data:GcQ6x8ewxInmAcQwhhwJgXMHxd/ygkscsp2vg7PILEeaOv1heBX0fTHb0sRyAfhLxwKDH84LtGb37656,iv:UEQ7dgqNEKisalpPXFffsVzn6kXDt9DmJP6ec3LOHRE=,tag:CyPPqG2Vf8eeSVzAASintQ==,type:str] password: ENC[AES256_GCM,data:GcQ6x8ewxInmAcQwhhwJgXMHxd/ygkscsp2vg7PILEeaOv1heBX0fTHb0sRyAfhLxwKDH84LtGb37656,iv:UEQ7dgqNEKisalpPXFffsVzn6kXDt9DmJP6ec3LOHRE=,tag:CyPPqG2Vf8eeSVzAASintQ==,type:str]
tempo:
caddy:
basic_auth:
username: ENC[AES256_GCM,data:B+8IkRh/MQ1f8g==,iv:0pmiHofPm+SvavQ1UsxOZcjdkWFuPpOs6cejTfkYH98=,tag:GiPPWqIKxhcxmnp709qyRg==,type:str]
#ENC[AES256_GCM,data:7/lgFtPiBjVhWba7cByCXNP2kmtdXTjqCEq6,iv:BAwTjTxXFy3vSxNaPRD9NsraMznrPFcjAQwtO0joOE8=,tag:7HqKeGXpGvT+Du3NoZskGQ==,type:comment]
password: ENC[AES256_GCM,data:tx/6SEqOxsStAKov6rZ83WJbO6CckvOw+tFkGSE0Co6ppSJB8CW4KiNGsUj9K6p+NjHalR1+48S32Bik,iv:MIJH6y2m3m+rks4+vwJAi6FrHqE8wv8aF30yvAHVTPM=,tag:uq/yUvoP/7KO0Z0jBEfYhw==,type:str]
sops: sops:
kms: [] kms: []
gcp_kms: [] gcp_kms: []
@ -24,8 +30,8 @@ sops:
bzYzSEQrRkN6WVVmN254a2NCcUxGVFkK4aHv8tiFiNEnd7I5LB0Jd/4upkEEEXis bzYzSEQrRkN6WVVmN254a2NCcUxGVFkK4aHv8tiFiNEnd7I5LB0Jd/4upkEEEXis
9A5hdTn20EqL62QuHeYRav1TRu42dp+R4iZAlVl9cRzThkzZKJdHlg== 9A5hdTn20EqL62QuHeYRav1TRu42dp+R4iZAlVl9cRzThkzZKJdHlg==
-----END AGE ENCRYPTED FILE----- -----END AGE ENCRYPTED FILE-----
lastmodified: "2024-08-29T05:57:46Z" lastmodified: "2024-09-02T09:51:05Z"
mac: ENC[AES256_GCM,data:JGrZUe8PLjlcHULvVa8Yi8ORAW5bMKOMxSGbJ2UFji9byRGu+JHaU7gdF45lrR5XKxJZLmZesWI9fRjsnlEd9WDTEroiwFo965mYFcdmbrJb37BIRO6Thy6C77GXMNcOVW2hBgcVDckNIbAk3qgvVG2QOZ5VKwxPLVQPsfWfCFs=,iv:Do05RY+cgahdv8/Nk6RIOxBA6x28GxyErrgSQRoIR80=,tag:HoqhdhZmcS3QxXGfZyxfFw==,type:str] mac: ENC[AES256_GCM,data:VsbdOVWBk49kiSS+3WXzJ209UENXkGqq4lRr0ETczp5PZAYbrUPzVlBrrqtvyxXMWZI+Q/J2vpoJmEQypyhwodtDp1NMbbiu5Nh0z5GQ1XVo6PBVUyIpCEaKkusWKzWmcWEP7HAvRzNdMEmlaRIaT07x0Ea8xLq79AeA0PZDT6A=,iv:JRIWfCZnO/9O458se670g5ei4xqXk9nEOTEV82bfGoc=,tag:LgX+9xgy5qPQ6EWhQCcFoA==,type:str]
pgp: [] pgp: []
unencrypted_suffix: _unencrypted unencrypted_suffix: _unencrypted
version: 3.8.1 version: 3.8.1

View file

@ -3,5 +3,6 @@
imports = [ imports = [
./grafana.nix ./grafana.nix
./loki.nix ./loki.nix
./tempo.nix
]; ];
} }

View file

@ -3,10 +3,10 @@ let
cfg = config.profile.services.telemetry.loki; cfg = config.profile.services.telemetry.loki;
inherit (lib) mkIf; inherit (lib) mkIf;
lokiDomain = "loki.tigor.web.id"; lokiDomain = "loki.tigor.web.id";
server = config.services.loki.configuration.server;
in in
{ {
config = mkIf cfg.enable { config = mkIf cfg.enable {
sops = sops =
let let
usernameKey = "loki/caddy/basic_auth/username"; usernameKey = "loki/caddy/basic_auth/username";
@ -36,7 +36,7 @@ in
basicauth { basicauth {
{$LOKI_USERNAME} {$LOKI_PASSWORD} {$LOKI_USERNAME} {$LOKI_PASSWORD}
} }
reverse_proxy ${config.services.loki.configuration.server.http_listen_address}:${toString config.services.loki.configuration.server.http_listen_port} reverse_proxy ${server.http_listen_address}:${toString server.http_listen_port}
''; '';
services.loki = services.loki =
@ -90,17 +90,14 @@ in
{ {
name = "Loki"; name = "Loki";
type = "loki"; type = "loki";
uid = "loki";
access = "proxy"; access = "proxy";
url = "http://${config.services.loki.configuration.server.http_listen_address}:${toString config.services.loki.configuration.server.http_listen_port}"; url = "http://${server.http_listen_address}:${toString server.http_listen_port}";
basicAuth = true; basicAuth = false;
basicAuthUser = "$__file{${config.sops.secrets."loki/caddy/basic_auth/username".path}}";
jsonData = { jsonData = {
timeout = 60; timeout = 60;
maxLines = 1000; maxLines = 1000;
}; };
secureJsonData = {
basicAuthPassword = "$__file{${config.sops.secrets."loki/caddy/basic_auth/password".path}}";
};
} }
]; ];
}; };

View file

@ -0,0 +1,104 @@
{ config, lib, ... }:
let
cfg = config.profile.services.telemetry.tempo;
inherit (lib) mkIf;
domain = "tempo.tigor.web.id";
basic_auth = {
username = "tempo/caddy/basic_auth/username";
password = "tempo/caddy/basic_auth/password";
template = "tempo/caddy/basic_auth";
};
server = config.services.tempo.settings.server;
in
{
config = mkIf cfg.enable {
sops = {
secrets =
let
opts = { sopsFile = ../../../secrets/telemetry.yaml; owner = "grafana"; };
in
{
${basic_auth.username} = opts;
${basic_auth.password} = opts;
};
templates = {
${basic_auth.template}.content = /*sh*/ ''
TEMPO_USERNAME=${config.sops.placeholder.${basic_auth.username}}
TEMPO_PASSWORD=${config.sops.placeholder.${basic_auth.password}}
'';
};
};
systemd.services."caddy".serviceConfig = {
EnvironmentFile = [ config.sops.templates.${basic_auth.template}.path ];
};
services.caddy.virtualHosts.${domain}.extraConfig = ''
basicauth {
{$TEMPO_USERNAME} {$TEMPO_PASSWORD}
}
reverse_proxy ${server.http_listen_address}:${toString server.http_listen_port}
'';
services.tempo = {
enable = true;
settings = {
server = {
http_listen_address = "0.0.0.0";
http_listen_port = 3200;
grpc_listen_port = 9096;
};
distributor = {
receivers = {
otlp = {
protocols = {
http = { };
};
};
};
};
storage.trace = {
backend = "local";
local.path = "/var/lib/tempo/traces";
wal.path = "/var/lib/tempo/wal";
};
ingester = {
lifecycler.ring.replication_factor = 1;
};
};
};
services.grafana.provision.datasources.settings.datasources = [
{
name = "Tempo";
type = "tempo";
access = "proxy";
url = "http://${server.http_listen_address}:${toString server.http_listen_port}";
basicAuth = false;
jsonData = {
nodeGraph.enabled = true;
search.hide = false;
traceQuery = {
timeShiftEnabled = true;
spanStartTimeShift = "1h";
spanEndTimeShift = "1h";
};
spanBar = {
type = "Tag";
tag = "http.path";
};
tracesToLogsV2 = mkIf config.profile.services.telemetry.loki.enable {
datasourceUid = "loki";
spanStartTimeShift = "-1h";
spanEndTimeShift = "1h";
tags = [ "job" "instance" "pod" "namespace" ];
filterByTraceID = false;
filterBySpanID = false;
customQuery = true;
query = ''method="$''${__span.tags.method}"'';
};
};
}
];
};
}