telemetry: added tempo service

hardware-configuration/homeserver.nix

@@ -73,6 +73,17 @@
           "bind"
         ];
       };
+
+      "/nas/telemetry/tempo" = lib.mkIf config.profile.services.telemetry.tempo.enable {
+        device = "/var/lib/tempo";
+        fsType = "auto";
+        options = [
+          "defaults"
+          "nofail"
+          "nobootwait"
+          "bind"
+        ];
+      };
     };
 
     swapDevices = [ ];

secrets/telemetry.yaml

@@ -9,6 +9,12 @@ loki:
             username: ENC[AES256_GCM,data:MRwky3O8LGS/4w==,iv:CUHjGRNc8NU5FqhqvpqbATmVE3Kg9Z0jMBFlzsAwON0=,tag:uZlxw9skd0VNLfZTJ/6ZSQ==,type:str]
             #ENC[AES256_GCM,data:t+u4g7nvpq2U27CHgmu1xi2Ppwv7cJf1s6Et,iv:LtwoPoxsQn1MujHRoD6SqDLm8uN4uBpuIVmn23DDgjI=,tag:PKOHF5vcQzHszpp1sfuU0Q==,type:comment]
             password: ENC[AES256_GCM,data:GcQ6x8ewxInmAcQwhhwJgXMHxd/ygkscsp2vg7PILEeaOv1heBX0fTHb0sRyAfhLxwKDH84LtGb37656,iv:UEQ7dgqNEKisalpPXFffsVzn6kXDt9DmJP6ec3LOHRE=,tag:CyPPqG2Vf8eeSVzAASintQ==,type:str]
+tempo:
+    caddy:
+        basic_auth:
+            username: ENC[AES256_GCM,data:B+8IkRh/MQ1f8g==,iv:0pmiHofPm+SvavQ1UsxOZcjdkWFuPpOs6cejTfkYH98=,tag:GiPPWqIKxhcxmnp709qyRg==,type:str]
+            #ENC[AES256_GCM,data:7/lgFtPiBjVhWba7cByCXNP2kmtdXTjqCEq6,iv:BAwTjTxXFy3vSxNaPRD9NsraMznrPFcjAQwtO0joOE8=,tag:7HqKeGXpGvT+Du3NoZskGQ==,type:comment]
+            password: ENC[AES256_GCM,data:tx/6SEqOxsStAKov6rZ83WJbO6CckvOw+tFkGSE0Co6ppSJB8CW4KiNGsUj9K6p+NjHalR1+48S32Bik,iv:MIJH6y2m3m+rks4+vwJAi6FrHqE8wv8aF30yvAHVTPM=,tag:uq/yUvoP/7KO0Z0jBEfYhw==,type:str]
 sops:
     kms: []
     gcp_kms: []
@@ -24,8 +30,8 @@ sops:
             bzYzSEQrRkN6WVVmN254a2NCcUxGVFkK4aHv8tiFiNEnd7I5LB0Jd/4upkEEEXis
             9A5hdTn20EqL62QuHeYRav1TRu42dp+R4iZAlVl9cRzThkzZKJdHlg==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2024-08-29T05:57:46Z"
+    lastmodified: "2024-09-02T09:51:05Z"
-    mac: ENC[AES256_GCM,data:JGrZUe8PLjlcHULvVa8Yi8ORAW5bMKOMxSGbJ2UFji9byRGu+JHaU7gdF45lrR5XKxJZLmZesWI9fRjsnlEd9WDTEroiwFo965mYFcdmbrJb37BIRO6Thy6C77GXMNcOVW2hBgcVDckNIbAk3qgvVG2QOZ5VKwxPLVQPsfWfCFs=,iv:Do05RY+cgahdv8/Nk6RIOxBA6x28GxyErrgSQRoIR80=,tag:HoqhdhZmcS3QxXGfZyxfFw==,type:str]
+    mac: ENC[AES256_GCM,data:VsbdOVWBk49kiSS+3WXzJ209UENXkGqq4lRr0ETczp5PZAYbrUPzVlBrrqtvyxXMWZI+Q/J2vpoJmEQypyhwodtDp1NMbbiu5Nh0z5GQ1XVo6PBVUyIpCEaKkusWKzWmcWEP7HAvRzNdMEmlaRIaT07x0Ea8xLq79AeA0PZDT6A=,iv:JRIWfCZnO/9O458se670g5ei4xqXk9nEOTEV82bfGoc=,tag:LgX+9xgy5qPQ6EWhQCcFoA==,type:str]
     pgp: []
     unencrypted_suffix: _unencrypted
     version: 3.8.1

system/services/telemetry/default.nix

@@ -3,5 +3,6 @@
   imports = [
     ./grafana.nix
     ./loki.nix
+    ./tempo.nix
   ];
 }

system/services/telemetry/loki.nix

@@ -3,10 +3,10 @@ let
   cfg = config.profile.services.telemetry.loki;
   inherit (lib) mkIf;
   lokiDomain = "loki.tigor.web.id";
+  server = config.services.loki.configuration.server;
 in
 {
   config = mkIf cfg.enable {
-
     sops =
       let
         usernameKey = "loki/caddy/basic_auth/username";
@@ -36,7 +36,7 @@ in
       basicauth {
         {$LOKI_USERNAME} {$LOKI_PASSWORD}
       }
-      reverse_proxy ${config.services.loki.configuration.server.http_listen_address}:${toString config.services.loki.configuration.server.http_listen_port}
+      reverse_proxy ${server.http_listen_address}:${toString server.http_listen_port}
     '';
 
     services.loki =
@@ -90,17 +90,14 @@ in
       {
         name = "Loki";
         type = "loki";
+        uid = "loki";
         access = "proxy";
-        url = "http://${config.services.loki.configuration.server.http_listen_address}:${toString config.services.loki.configuration.server.http_listen_port}";
+        url = "http://${server.http_listen_address}:${toString server.http_listen_port}";
-        basicAuth = true;
+        basicAuth = false;
-        basicAuthUser = "$__file{${config.sops.secrets."loki/caddy/basic_auth/username".path}}";
         jsonData = {
           timeout = 60;
           maxLines = 1000;
         };
-        secureJsonData = {
-          basicAuthPassword = "$__file{${config.sops.secrets."loki/caddy/basic_auth/password".path}}";
-        };
       }
     ];
   };

system/services/telemetry/tempo.nix

@@ -0,0 +1,104 @@
+{ config, lib, ... }:
+let
+  cfg = config.profile.services.telemetry.tempo;
+  inherit (lib) mkIf;
+  domain = "tempo.tigor.web.id";
+  basic_auth = {
+    username = "tempo/caddy/basic_auth/username";
+    password = "tempo/caddy/basic_auth/password";
+    template = "tempo/caddy/basic_auth";
+  };
+  server = config.services.tempo.settings.server;
+in
+{
+  config = mkIf cfg.enable {
+    sops = {
+      secrets =
+        let
+          opts = { sopsFile = ../../../secrets/telemetry.yaml; owner = "grafana"; };
+        in
+        {
+          ${basic_auth.username} = opts;
+          ${basic_auth.password} = opts;
+        };
+      templates = {
+        ${basic_auth.template}.content = /*sh*/ ''
+          TEMPO_USERNAME=${config.sops.placeholder.${basic_auth.username}}
+          TEMPO_PASSWORD=${config.sops.placeholder.${basic_auth.password}}
+        '';
+      };
+    };
+
+    systemd.services."caddy".serviceConfig = {
+      EnvironmentFile = [ config.sops.templates.${basic_auth.template}.path ];
+    };
+
+    services.caddy.virtualHosts.${domain}.extraConfig = ''
+      basicauth {
+          {$TEMPO_USERNAME} {$TEMPO_PASSWORD}
+      }
+
+      reverse_proxy ${server.http_listen_address}:${toString server.http_listen_port}
+    '';
+
+    services.tempo = {
+      enable = true;
+      settings = {
+        server = {
+          http_listen_address = "0.0.0.0";
+          http_listen_port = 3200;
+          grpc_listen_port = 9096;
+        };
+        distributor = {
+          receivers = {
+            otlp = {
+              protocols = {
+                http = { };
+              };
+            };
+          };
+        };
+        storage.trace = {
+          backend = "local";
+          local.path = "/var/lib/tempo/traces";
+          wal.path = "/var/lib/tempo/wal";
+        };
+        ingester = {
+          lifecycler.ring.replication_factor = 1;
+        };
+      };
+    };
+    services.grafana.provision.datasources.settings.datasources = [
+      {
+        name = "Tempo";
+        type = "tempo";
+        access = "proxy";
+        url = "http://${server.http_listen_address}:${toString server.http_listen_port}";
+        basicAuth = false;
+        jsonData = {
+          nodeGraph.enabled = true;
+          search.hide = false;
+          traceQuery = {
+            timeShiftEnabled = true;
+            spanStartTimeShift = "1h";
+            spanEndTimeShift = "1h";
+          };
+          spanBar = {
+            type = "Tag";
+            tag = "http.path";
+          };
+          tracesToLogsV2 = mkIf config.profile.services.telemetry.loki.enable {
+            datasourceUid = "loki";
+            spanStartTimeShift = "-1h";
+            spanEndTimeShift = "1h";
+            tags = [ "job" "instance" "pod" "namespace" ];
+            filterByTraceID = false;
+            filterBySpanID = false;
+            customQuery = true;
+            query = ''method="$''${__span.tags.method}"'';
+          };
+        };
+      }
+    ];
+  };
+}