telemetry: added tempo service
This commit is contained in:
parent
a54dcef036
commit
4786f0562b
|
|
@ -73,6 +73,17 @@
|
|||
"bind"
|
||||
];
|
||||
};
|
||||
|
||||
"/nas/telemetry/tempo" = lib.mkIf config.profile.services.telemetry.tempo.enable {
|
||||
device = "/var/lib/tempo";
|
||||
fsType = "auto";
|
||||
options = [
|
||||
"defaults"
|
||||
"nofail"
|
||||
"nobootwait"
|
||||
"bind"
|
||||
];
|
||||
};
|
||||
};
|
||||
|
||||
swapDevices = [ ];
|
||||
|
|
|
|||
|
|
@ -9,6 +9,12 @@ loki:
|
|||
username: ENC[AES256_GCM,data:MRwky3O8LGS/4w==,iv:CUHjGRNc8NU5FqhqvpqbATmVE3Kg9Z0jMBFlzsAwON0=,tag:uZlxw9skd0VNLfZTJ/6ZSQ==,type:str]
|
||||
#ENC[AES256_GCM,data:t+u4g7nvpq2U27CHgmu1xi2Ppwv7cJf1s6Et,iv:LtwoPoxsQn1MujHRoD6SqDLm8uN4uBpuIVmn23DDgjI=,tag:PKOHF5vcQzHszpp1sfuU0Q==,type:comment]
|
||||
password: ENC[AES256_GCM,data:GcQ6x8ewxInmAcQwhhwJgXMHxd/ygkscsp2vg7PILEeaOv1heBX0fTHb0sRyAfhLxwKDH84LtGb37656,iv:UEQ7dgqNEKisalpPXFffsVzn6kXDt9DmJP6ec3LOHRE=,tag:CyPPqG2Vf8eeSVzAASintQ==,type:str]
|
||||
tempo:
|
||||
caddy:
|
||||
basic_auth:
|
||||
username: ENC[AES256_GCM,data:B+8IkRh/MQ1f8g==,iv:0pmiHofPm+SvavQ1UsxOZcjdkWFuPpOs6cejTfkYH98=,tag:GiPPWqIKxhcxmnp709qyRg==,type:str]
|
||||
#ENC[AES256_GCM,data:7/lgFtPiBjVhWba7cByCXNP2kmtdXTjqCEq6,iv:BAwTjTxXFy3vSxNaPRD9NsraMznrPFcjAQwtO0joOE8=,tag:7HqKeGXpGvT+Du3NoZskGQ==,type:comment]
|
||||
password: ENC[AES256_GCM,data:tx/6SEqOxsStAKov6rZ83WJbO6CckvOw+tFkGSE0Co6ppSJB8CW4KiNGsUj9K6p+NjHalR1+48S32Bik,iv:MIJH6y2m3m+rks4+vwJAi6FrHqE8wv8aF30yvAHVTPM=,tag:uq/yUvoP/7KO0Z0jBEfYhw==,type:str]
|
||||
sops:
|
||||
kms: []
|
||||
gcp_kms: []
|
||||
|
|
@ -24,8 +30,8 @@ sops:
|
|||
bzYzSEQrRkN6WVVmN254a2NCcUxGVFkK4aHv8tiFiNEnd7I5LB0Jd/4upkEEEXis
|
||||
9A5hdTn20EqL62QuHeYRav1TRu42dp+R4iZAlVl9cRzThkzZKJdHlg==
|
||||
-----END AGE ENCRYPTED FILE-----
|
||||
lastmodified: "2024-08-29T05:57:46Z"
|
||||
mac: ENC[AES256_GCM,data:JGrZUe8PLjlcHULvVa8Yi8ORAW5bMKOMxSGbJ2UFji9byRGu+JHaU7gdF45lrR5XKxJZLmZesWI9fRjsnlEd9WDTEroiwFo965mYFcdmbrJb37BIRO6Thy6C77GXMNcOVW2hBgcVDckNIbAk3qgvVG2QOZ5VKwxPLVQPsfWfCFs=,iv:Do05RY+cgahdv8/Nk6RIOxBA6x28GxyErrgSQRoIR80=,tag:HoqhdhZmcS3QxXGfZyxfFw==,type:str]
|
||||
lastmodified: "2024-09-02T09:51:05Z"
|
||||
mac: ENC[AES256_GCM,data:VsbdOVWBk49kiSS+3WXzJ209UENXkGqq4lRr0ETczp5PZAYbrUPzVlBrrqtvyxXMWZI+Q/J2vpoJmEQypyhwodtDp1NMbbiu5Nh0z5GQ1XVo6PBVUyIpCEaKkusWKzWmcWEP7HAvRzNdMEmlaRIaT07x0Ea8xLq79AeA0PZDT6A=,iv:JRIWfCZnO/9O458se670g5ei4xqXk9nEOTEV82bfGoc=,tag:LgX+9xgy5qPQ6EWhQCcFoA==,type:str]
|
||||
pgp: []
|
||||
unencrypted_suffix: _unencrypted
|
||||
version: 3.8.1
|
||||
|
|
|
|||
|
|
@ -3,5 +3,6 @@
|
|||
imports = [
|
||||
./grafana.nix
|
||||
./loki.nix
|
||||
./tempo.nix
|
||||
];
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,10 +3,10 @@ let
|
|||
cfg = config.profile.services.telemetry.loki;
|
||||
inherit (lib) mkIf;
|
||||
lokiDomain = "loki.tigor.web.id";
|
||||
server = config.services.loki.configuration.server;
|
||||
in
|
||||
{
|
||||
config = mkIf cfg.enable {
|
||||
|
||||
sops =
|
||||
let
|
||||
usernameKey = "loki/caddy/basic_auth/username";
|
||||
|
|
@ -36,7 +36,7 @@ in
|
|||
basicauth {
|
||||
{$LOKI_USERNAME} {$LOKI_PASSWORD}
|
||||
}
|
||||
reverse_proxy ${config.services.loki.configuration.server.http_listen_address}:${toString config.services.loki.configuration.server.http_listen_port}
|
||||
reverse_proxy ${server.http_listen_address}:${toString server.http_listen_port}
|
||||
'';
|
||||
|
||||
services.loki =
|
||||
|
|
@ -90,17 +90,14 @@ in
|
|||
{
|
||||
name = "Loki";
|
||||
type = "loki";
|
||||
uid = "loki";
|
||||
access = "proxy";
|
||||
url = "http://${config.services.loki.configuration.server.http_listen_address}:${toString config.services.loki.configuration.server.http_listen_port}";
|
||||
basicAuth = true;
|
||||
basicAuthUser = "$__file{${config.sops.secrets."loki/caddy/basic_auth/username".path}}";
|
||||
url = "http://${server.http_listen_address}:${toString server.http_listen_port}";
|
||||
basicAuth = false;
|
||||
jsonData = {
|
||||
timeout = 60;
|
||||
maxLines = 1000;
|
||||
};
|
||||
secureJsonData = {
|
||||
basicAuthPassword = "$__file{${config.sops.secrets."loki/caddy/basic_auth/password".path}}";
|
||||
};
|
||||
}
|
||||
];
|
||||
};
|
||||
|
|
|
|||
104
system/services/telemetry/tempo.nix
Normal file
104
system/services/telemetry/tempo.nix
Normal file
|
|
@ -0,0 +1,104 @@
|
|||
{ config, lib, ... }:
|
||||
let
|
||||
cfg = config.profile.services.telemetry.tempo;
|
||||
inherit (lib) mkIf;
|
||||
domain = "tempo.tigor.web.id";
|
||||
basic_auth = {
|
||||
username = "tempo/caddy/basic_auth/username";
|
||||
password = "tempo/caddy/basic_auth/password";
|
||||
template = "tempo/caddy/basic_auth";
|
||||
};
|
||||
server = config.services.tempo.settings.server;
|
||||
in
|
||||
{
|
||||
config = mkIf cfg.enable {
|
||||
sops = {
|
||||
secrets =
|
||||
let
|
||||
opts = { sopsFile = ../../../secrets/telemetry.yaml; owner = "grafana"; };
|
||||
in
|
||||
{
|
||||
${basic_auth.username} = opts;
|
||||
${basic_auth.password} = opts;
|
||||
};
|
||||
templates = {
|
||||
${basic_auth.template}.content = /*sh*/ ''
|
||||
TEMPO_USERNAME=${config.sops.placeholder.${basic_auth.username}}
|
||||
TEMPO_PASSWORD=${config.sops.placeholder.${basic_auth.password}}
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
systemd.services."caddy".serviceConfig = {
|
||||
EnvironmentFile = [ config.sops.templates.${basic_auth.template}.path ];
|
||||
};
|
||||
|
||||
services.caddy.virtualHosts.${domain}.extraConfig = ''
|
||||
basicauth {
|
||||
{$TEMPO_USERNAME} {$TEMPO_PASSWORD}
|
||||
}
|
||||
|
||||
reverse_proxy ${server.http_listen_address}:${toString server.http_listen_port}
|
||||
'';
|
||||
|
||||
services.tempo = {
|
||||
enable = true;
|
||||
settings = {
|
||||
server = {
|
||||
http_listen_address = "0.0.0.0";
|
||||
http_listen_port = 3200;
|
||||
grpc_listen_port = 9096;
|
||||
};
|
||||
distributor = {
|
||||
receivers = {
|
||||
otlp = {
|
||||
protocols = {
|
||||
http = { };
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
storage.trace = {
|
||||
backend = "local";
|
||||
local.path = "/var/lib/tempo/traces";
|
||||
wal.path = "/var/lib/tempo/wal";
|
||||
};
|
||||
ingester = {
|
||||
lifecycler.ring.replication_factor = 1;
|
||||
};
|
||||
};
|
||||
};
|
||||
services.grafana.provision.datasources.settings.datasources = [
|
||||
{
|
||||
name = "Tempo";
|
||||
type = "tempo";
|
||||
access = "proxy";
|
||||
url = "http://${server.http_listen_address}:${toString server.http_listen_port}";
|
||||
basicAuth = false;
|
||||
jsonData = {
|
||||
nodeGraph.enabled = true;
|
||||
search.hide = false;
|
||||
traceQuery = {
|
||||
timeShiftEnabled = true;
|
||||
spanStartTimeShift = "1h";
|
||||
spanEndTimeShift = "1h";
|
||||
};
|
||||
spanBar = {
|
||||
type = "Tag";
|
||||
tag = "http.path";
|
||||
};
|
||||
tracesToLogsV2 = mkIf config.profile.services.telemetry.loki.enable {
|
||||
datasourceUid = "loki";
|
||||
spanStartTimeShift = "-1h";
|
||||
spanEndTimeShift = "1h";
|
||||
tags = [ "job" "instance" "pod" "namespace" ];
|
||||
filterByTraceID = false;
|
||||
filterBySpanID = false;
|
||||
customQuery = true;
|
||||
query = ''method="$''${__span.tags.method}"'';
|
||||
};
|
||||
};
|
||||
}
|
||||
];
|
||||
};
|
||||
}
|
||||
Loading…
Reference in a new issue