telemetry: added tempo service
This commit is contained in:
parent
a54dcef036
commit
4786f0562b
|
@ -73,6 +73,17 @@
|
||||||
"bind"
|
"bind"
|
||||||
];
|
];
|
||||||
};
|
};
|
||||||
|
|
||||||
|
"/nas/telemetry/tempo" = lib.mkIf config.profile.services.telemetry.tempo.enable {
|
||||||
|
device = "/var/lib/tempo";
|
||||||
|
fsType = "auto";
|
||||||
|
options = [
|
||||||
|
"defaults"
|
||||||
|
"nofail"
|
||||||
|
"nobootwait"
|
||||||
|
"bind"
|
||||||
|
];
|
||||||
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
swapDevices = [ ];
|
swapDevices = [ ];
|
||||||
|
|
|
@ -9,6 +9,12 @@ loki:
|
||||||
username: ENC[AES256_GCM,data:MRwky3O8LGS/4w==,iv:CUHjGRNc8NU5FqhqvpqbATmVE3Kg9Z0jMBFlzsAwON0=,tag:uZlxw9skd0VNLfZTJ/6ZSQ==,type:str]
|
username: ENC[AES256_GCM,data:MRwky3O8LGS/4w==,iv:CUHjGRNc8NU5FqhqvpqbATmVE3Kg9Z0jMBFlzsAwON0=,tag:uZlxw9skd0VNLfZTJ/6ZSQ==,type:str]
|
||||||
#ENC[AES256_GCM,data:t+u4g7nvpq2U27CHgmu1xi2Ppwv7cJf1s6Et,iv:LtwoPoxsQn1MujHRoD6SqDLm8uN4uBpuIVmn23DDgjI=,tag:PKOHF5vcQzHszpp1sfuU0Q==,type:comment]
|
#ENC[AES256_GCM,data:t+u4g7nvpq2U27CHgmu1xi2Ppwv7cJf1s6Et,iv:LtwoPoxsQn1MujHRoD6SqDLm8uN4uBpuIVmn23DDgjI=,tag:PKOHF5vcQzHszpp1sfuU0Q==,type:comment]
|
||||||
password: ENC[AES256_GCM,data:GcQ6x8ewxInmAcQwhhwJgXMHxd/ygkscsp2vg7PILEeaOv1heBX0fTHb0sRyAfhLxwKDH84LtGb37656,iv:UEQ7dgqNEKisalpPXFffsVzn6kXDt9DmJP6ec3LOHRE=,tag:CyPPqG2Vf8eeSVzAASintQ==,type:str]
|
password: ENC[AES256_GCM,data:GcQ6x8ewxInmAcQwhhwJgXMHxd/ygkscsp2vg7PILEeaOv1heBX0fTHb0sRyAfhLxwKDH84LtGb37656,iv:UEQ7dgqNEKisalpPXFffsVzn6kXDt9DmJP6ec3LOHRE=,tag:CyPPqG2Vf8eeSVzAASintQ==,type:str]
|
||||||
|
tempo:
|
||||||
|
caddy:
|
||||||
|
basic_auth:
|
||||||
|
username: ENC[AES256_GCM,data:B+8IkRh/MQ1f8g==,iv:0pmiHofPm+SvavQ1UsxOZcjdkWFuPpOs6cejTfkYH98=,tag:GiPPWqIKxhcxmnp709qyRg==,type:str]
|
||||||
|
#ENC[AES256_GCM,data:7/lgFtPiBjVhWba7cByCXNP2kmtdXTjqCEq6,iv:BAwTjTxXFy3vSxNaPRD9NsraMznrPFcjAQwtO0joOE8=,tag:7HqKeGXpGvT+Du3NoZskGQ==,type:comment]
|
||||||
|
password: ENC[AES256_GCM,data:tx/6SEqOxsStAKov6rZ83WJbO6CckvOw+tFkGSE0Co6ppSJB8CW4KiNGsUj9K6p+NjHalR1+48S32Bik,iv:MIJH6y2m3m+rks4+vwJAi6FrHqE8wv8aF30yvAHVTPM=,tag:uq/yUvoP/7KO0Z0jBEfYhw==,type:str]
|
||||||
sops:
|
sops:
|
||||||
kms: []
|
kms: []
|
||||||
gcp_kms: []
|
gcp_kms: []
|
||||||
|
@ -24,8 +30,8 @@ sops:
|
||||||
bzYzSEQrRkN6WVVmN254a2NCcUxGVFkK4aHv8tiFiNEnd7I5LB0Jd/4upkEEEXis
|
bzYzSEQrRkN6WVVmN254a2NCcUxGVFkK4aHv8tiFiNEnd7I5LB0Jd/4upkEEEXis
|
||||||
9A5hdTn20EqL62QuHeYRav1TRu42dp+R4iZAlVl9cRzThkzZKJdHlg==
|
9A5hdTn20EqL62QuHeYRav1TRu42dp+R4iZAlVl9cRzThkzZKJdHlg==
|
||||||
-----END AGE ENCRYPTED FILE-----
|
-----END AGE ENCRYPTED FILE-----
|
||||||
lastmodified: "2024-08-29T05:57:46Z"
|
lastmodified: "2024-09-02T09:51:05Z"
|
||||||
mac: ENC[AES256_GCM,data:JGrZUe8PLjlcHULvVa8Yi8ORAW5bMKOMxSGbJ2UFji9byRGu+JHaU7gdF45lrR5XKxJZLmZesWI9fRjsnlEd9WDTEroiwFo965mYFcdmbrJb37BIRO6Thy6C77GXMNcOVW2hBgcVDckNIbAk3qgvVG2QOZ5VKwxPLVQPsfWfCFs=,iv:Do05RY+cgahdv8/Nk6RIOxBA6x28GxyErrgSQRoIR80=,tag:HoqhdhZmcS3QxXGfZyxfFw==,type:str]
|
mac: ENC[AES256_GCM,data:VsbdOVWBk49kiSS+3WXzJ209UENXkGqq4lRr0ETczp5PZAYbrUPzVlBrrqtvyxXMWZI+Q/J2vpoJmEQypyhwodtDp1NMbbiu5Nh0z5GQ1XVo6PBVUyIpCEaKkusWKzWmcWEP7HAvRzNdMEmlaRIaT07x0Ea8xLq79AeA0PZDT6A=,iv:JRIWfCZnO/9O458se670g5ei4xqXk9nEOTEV82bfGoc=,tag:LgX+9xgy5qPQ6EWhQCcFoA==,type:str]
|
||||||
pgp: []
|
pgp: []
|
||||||
unencrypted_suffix: _unencrypted
|
unencrypted_suffix: _unencrypted
|
||||||
version: 3.8.1
|
version: 3.8.1
|
||||||
|
|
|
@ -3,5 +3,6 @@
|
||||||
imports = [
|
imports = [
|
||||||
./grafana.nix
|
./grafana.nix
|
||||||
./loki.nix
|
./loki.nix
|
||||||
|
./tempo.nix
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
|
@ -3,10 +3,10 @@ let
|
||||||
cfg = config.profile.services.telemetry.loki;
|
cfg = config.profile.services.telemetry.loki;
|
||||||
inherit (lib) mkIf;
|
inherit (lib) mkIf;
|
||||||
lokiDomain = "loki.tigor.web.id";
|
lokiDomain = "loki.tigor.web.id";
|
||||||
|
server = config.services.loki.configuration.server;
|
||||||
in
|
in
|
||||||
{
|
{
|
||||||
config = mkIf cfg.enable {
|
config = mkIf cfg.enable {
|
||||||
|
|
||||||
sops =
|
sops =
|
||||||
let
|
let
|
||||||
usernameKey = "loki/caddy/basic_auth/username";
|
usernameKey = "loki/caddy/basic_auth/username";
|
||||||
|
@ -36,7 +36,7 @@ in
|
||||||
basicauth {
|
basicauth {
|
||||||
{$LOKI_USERNAME} {$LOKI_PASSWORD}
|
{$LOKI_USERNAME} {$LOKI_PASSWORD}
|
||||||
}
|
}
|
||||||
reverse_proxy ${config.services.loki.configuration.server.http_listen_address}:${toString config.services.loki.configuration.server.http_listen_port}
|
reverse_proxy ${server.http_listen_address}:${toString server.http_listen_port}
|
||||||
'';
|
'';
|
||||||
|
|
||||||
services.loki =
|
services.loki =
|
||||||
|
@ -90,17 +90,14 @@ in
|
||||||
{
|
{
|
||||||
name = "Loki";
|
name = "Loki";
|
||||||
type = "loki";
|
type = "loki";
|
||||||
|
uid = "loki";
|
||||||
access = "proxy";
|
access = "proxy";
|
||||||
url = "http://${config.services.loki.configuration.server.http_listen_address}:${toString config.services.loki.configuration.server.http_listen_port}";
|
url = "http://${server.http_listen_address}:${toString server.http_listen_port}";
|
||||||
basicAuth = true;
|
basicAuth = false;
|
||||||
basicAuthUser = "$__file{${config.sops.secrets."loki/caddy/basic_auth/username".path}}";
|
|
||||||
jsonData = {
|
jsonData = {
|
||||||
timeout = 60;
|
timeout = 60;
|
||||||
maxLines = 1000;
|
maxLines = 1000;
|
||||||
};
|
};
|
||||||
secureJsonData = {
|
|
||||||
basicAuthPassword = "$__file{${config.sops.secrets."loki/caddy/basic_auth/password".path}}";
|
|
||||||
};
|
|
||||||
}
|
}
|
||||||
];
|
];
|
||||||
};
|
};
|
||||||
|
|
104
system/services/telemetry/tempo.nix
Normal file
104
system/services/telemetry/tempo.nix
Normal file
|
@ -0,0 +1,104 @@
|
||||||
|
{ config, lib, ... }:
|
||||||
|
let
|
||||||
|
cfg = config.profile.services.telemetry.tempo;
|
||||||
|
inherit (lib) mkIf;
|
||||||
|
domain = "tempo.tigor.web.id";
|
||||||
|
basic_auth = {
|
||||||
|
username = "tempo/caddy/basic_auth/username";
|
||||||
|
password = "tempo/caddy/basic_auth/password";
|
||||||
|
template = "tempo/caddy/basic_auth";
|
||||||
|
};
|
||||||
|
server = config.services.tempo.settings.server;
|
||||||
|
in
|
||||||
|
{
|
||||||
|
config = mkIf cfg.enable {
|
||||||
|
sops = {
|
||||||
|
secrets =
|
||||||
|
let
|
||||||
|
opts = { sopsFile = ../../../secrets/telemetry.yaml; owner = "grafana"; };
|
||||||
|
in
|
||||||
|
{
|
||||||
|
${basic_auth.username} = opts;
|
||||||
|
${basic_auth.password} = opts;
|
||||||
|
};
|
||||||
|
templates = {
|
||||||
|
${basic_auth.template}.content = /*sh*/ ''
|
||||||
|
TEMPO_USERNAME=${config.sops.placeholder.${basic_auth.username}}
|
||||||
|
TEMPO_PASSWORD=${config.sops.placeholder.${basic_auth.password}}
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
systemd.services."caddy".serviceConfig = {
|
||||||
|
EnvironmentFile = [ config.sops.templates.${basic_auth.template}.path ];
|
||||||
|
};
|
||||||
|
|
||||||
|
services.caddy.virtualHosts.${domain}.extraConfig = ''
|
||||||
|
basicauth {
|
||||||
|
{$TEMPO_USERNAME} {$TEMPO_PASSWORD}
|
||||||
|
}
|
||||||
|
|
||||||
|
reverse_proxy ${server.http_listen_address}:${toString server.http_listen_port}
|
||||||
|
'';
|
||||||
|
|
||||||
|
services.tempo = {
|
||||||
|
enable = true;
|
||||||
|
settings = {
|
||||||
|
server = {
|
||||||
|
http_listen_address = "0.0.0.0";
|
||||||
|
http_listen_port = 3200;
|
||||||
|
grpc_listen_port = 9096;
|
||||||
|
};
|
||||||
|
distributor = {
|
||||||
|
receivers = {
|
||||||
|
otlp = {
|
||||||
|
protocols = {
|
||||||
|
http = { };
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
storage.trace = {
|
||||||
|
backend = "local";
|
||||||
|
local.path = "/var/lib/tempo/traces";
|
||||||
|
wal.path = "/var/lib/tempo/wal";
|
||||||
|
};
|
||||||
|
ingester = {
|
||||||
|
lifecycler.ring.replication_factor = 1;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
services.grafana.provision.datasources.settings.datasources = [
|
||||||
|
{
|
||||||
|
name = "Tempo";
|
||||||
|
type = "tempo";
|
||||||
|
access = "proxy";
|
||||||
|
url = "http://${server.http_listen_address}:${toString server.http_listen_port}";
|
||||||
|
basicAuth = false;
|
||||||
|
jsonData = {
|
||||||
|
nodeGraph.enabled = true;
|
||||||
|
search.hide = false;
|
||||||
|
traceQuery = {
|
||||||
|
timeShiftEnabled = true;
|
||||||
|
spanStartTimeShift = "1h";
|
||||||
|
spanEndTimeShift = "1h";
|
||||||
|
};
|
||||||
|
spanBar = {
|
||||||
|
type = "Tag";
|
||||||
|
tag = "http.path";
|
||||||
|
};
|
||||||
|
tracesToLogsV2 = mkIf config.profile.services.telemetry.loki.enable {
|
||||||
|
datasourceUid = "loki";
|
||||||
|
spanStartTimeShift = "-1h";
|
||||||
|
spanEndTimeShift = "1h";
|
||||||
|
tags = [ "job" "instance" "pod" "namespace" ];
|
||||||
|
filterByTraceID = false;
|
||||||
|
filterBySpanID = false;
|
||||||
|
customQuery = true;
|
||||||
|
query = ''method="$''${__span.tags.method}"'';
|
||||||
|
};
|
||||||
|
};
|
||||||
|
}
|
||||||
|
];
|
||||||
|
};
|
||||||
|
}
|
Loading…
Reference in a new issue