Container logs

.ci/docker/netbird/docker-compose.yml

@@ -2,6 +2,9 @@ version: "3.9"
 services:
   signal:
     image: netbirdio/signal:${NETBIRD_SIGNAL_VERSION}
+    labels:
+      logging: "promtail"
+      logging_job: "netbird-signal"
     volumes:
       - netbird-signal:/var/lib/netbird
     networks:
@@ -16,6 +19,9 @@ services:
       dockerfile: ./.ci/docker/netbird/Dockerfile_management
       args:
         NETBIRD_MANAGEMENT_VERSION: ${NETBIRD_MANAGEMENT_VERSION}
+    labels:
+      logging: "promtail"
+      logging_job: "netbird-management"
     volumes:
       - netbird-mgmt:/var/lib/netbird
       - ./management.json:/etc/netbird/management.json
@@ -51,6 +57,9 @@ services:
       - "--accesslog"
     environment:
       - "LEGO_CA_CERTIFICATES=/opendut-tls/insecure-development-ca.pem"
+    labels:
+      logging: "promtail"
+      logging_job: "netbird-traefik"
 #    ports:
 #      - "127.0.0.1:8082:8080"
 #      - "80:80"
@@ -75,6 +84,9 @@ services:
         - PUSER=$PUSER
         - PGROUP=$PGROUP
     user: $DOCKER_USER # ${UID}:${GID} run with user privileges
+    labels:
+      logging: "promtail"
+      logging_job: "netbird-management-init"
     volumes:
       #- ./management/:/management
       - netbird-management-keys:/management
@@ -95,6 +107,9 @@ services:
     # https://hub.docker.com/r/coturn/coturn/tags
     image: coturn/coturn:4.6.2
     hostname: netbird-coturn
+    labels:
+      logging: "promtail"
+      logging_job: "netbird-coturn"
     volumes:
       - ./turnserver.conf:/etc/turnserver.conf:ro
     command:
@@ -120,6 +135,9 @@ services:
       - AUTH_SUPPORTED_SCOPES=openid profile email offline_access api
       - AUTH_REDIRECT_URI=
       - AUTH_SILENT_REDIRECT_URI=
+    labels:
+      logging: "promtail"
+      logging_job: "netbird-dashboard"
     networks:
       opendutnet:
         ipv4_address: 192.168.32.213

.ci/docker/telemetry/docker-compose.yml

@@ -39,6 +39,19 @@ services:
     networks:
       - opendutnet
 
+ promtail:
+   image: grafana/promtail:latest
+   container_name: promtail
+   volumes:
+     - ./promtail.yml:/etc/promtail/docker-config.yaml
+     - /var/lib/docker/containers:/var/lib/docker/containers:ro
+     - /var/run/docker.sock:/var/run/docker.sock
+   command: -config.file=/etc/promtail/docker-config.yaml
+   depends_on:
+     - loki
+   networks:
+     - opendutnet
+
  otel-collector:
    container_name: otel-collector
    image: otel/opentelemetry-collector-contrib:0.97.0

.ci/docker/telemetry/grafana/dashboards/OpenDuT Logs-1712149939103.json → .ci/docker/telemetry/grafana/dashboards/OpenDuT-Logs-1713948751058.json

@@ -18,7 +18,6 @@
   "editable": true,
   "fiscalYearStartMonth": 0,
   "graphTooltip": 0,
-  "id": 13,
   "links": [],
   "liveNow": false,
   "panels": [
@@ -116,8 +115,7 @@
               "viz": false
             }
           },
-          "mappings": [],
-          "unitScale": true
+          "mappings": []
         },
         "overrides": [
           {
@@ -179,6 +177,51 @@
                 }
               }
             ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "WARN"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "yellow",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "DEFAULT"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#848380",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "WARNING"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "yellow",
+                  "mode": "fixed"
+                }
+              }
+            ]
           }
         ]
       },
@@ -356,6 +399,6 @@
   "timezone": "",
   "title": "OpenDuT Logs",
   "uid": "e4c47d24-47ae-45c3-b0c9-3199b385f804",
-  "version": 2,
+  "version": 1,
   "weekStart": ""
 }

.ci/docker/telemetry/promtail.yml

@@ -0,0 +1,67 @@
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+positions:
+  filename: /tmp/positions.yaml
+
+clients:
+  - url: http://loki:3100/loki/api/v1/push
+
+scrape_configs:
+  - job_name: docker_logs_scrape
+    docker_sd_configs:
+      - host: unix:///var/run/docker.sock
+        refresh_interval: 5s
+        filters:
+          - name: label
+            values: ["logging=promtail"]
+    relabel_configs:
+      - source_labels: ['__meta_docker_container_name']
+        regex: '/(.*)'
+        target_label: 'container'
+      - source_labels: ['__meta_docker_container_log_stream']
+        target_label: 'logstream'
+      - source_labels: ['__meta_docker_container_label_logging_job']
+        target_label: 'job'
+    pipeline_stages:
+      - match:
+          docker: { }
+          selector: '{job=~"netbird-traefik|netbird-coturn|netbird-dashboard|netbird-management|netbird-management-init|netbird-signal"}'
+          stages:
+            - static_labels:
+                level: DEFAULT
+            - regex:
+                expression: '.*level=(?P<level>[a-zA-Z]+).*'
+            - template:
+                source: level
+                template: '{{ ToUpper .Value }}'
+            - labels:
+                level:
+            - regex:
+                expression: '.*\([0-9]+\): (?P<level>[A-Z]+):.*'
+            - template:
+                source: level
+                template: '{{ ToUpper .Value }}'
+            - labels:
+                level:
+            - regex:
+                expression: '.*[0-9]{2}:[0-9]{2}:[0-9]{2},[0-9]+ (?P<level>[A-Z]+) .*'
+            - template:
+                source: level
+                template: '{{ ToUpper .Value }}'
+            - labels:
+                level:
+            - regex:
+                expression: '.*[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z (?P<level>[A-Z]+) .*'
+            - template:
+                source: level
+                template: '{{ ToUpper .Value }}'
+            - labels:
+                level:
+      - match:
+          docker: { }
+          selector: '{level="WARNING"}'
+          stages:
+            - static_labels:
+                level: WARN

opendut-carl/src/lib.rs

@@ -56,8 +56,9 @@ pub async fn create_with_logging(settings_override: config::Config) -> Result<()
 
     let service_instance_id = format!("carl-{}", Uuid::new_v4());
 
+    let file_logging = None;
     let logging_config = LoggingConfig::load(&settings.config, service_instance_id)?;
-    let mut shutdown = logging::initialize_with_config(logging_config.clone())?;
+    let mut shutdown = logging::initialize_with_config(logging_config.clone(), file_logging)?;
 
     if let logging::OpenTelemetryConfig::Enabled { cpu_collection_interval_ms, .. } = logging_config.opentelemetry {
         logging::initialize_metrics_collection(cpu_collection_interval_ms);

opendut-edgar/src/service/start.rs

@@ -63,8 +63,9 @@ pub async fn create_with_logging(settings_override: config::Config) -> anyhow::R
 
     let service_instance_id = self_id.to_string();
 
+    let file_logging = None;
     let logging_config = LoggingConfig::load(&settings.config, service_instance_id)?;
-    let mut shutdown = logging::initialize_with_config(logging_config.clone())?;
+    let mut shutdown = logging::initialize_with_config(logging_config.clone(), file_logging)?;
 
     if let logging::OpenTelemetryConfig::Enabled { cpu_collection_interval_ms, .. } = logging_config.opentelemetry {
         logging::initialize_metrics_collection(cpu_collection_interval_ms);   

opendut-edgar/src/setup/start.rs

@@ -117,16 +117,15 @@ pub async fn unmanaged(
 
 pub fn init_logging() -> anyhow::Result<()> {
 
-    let log_file = std::env::current_exe().unwrap()
-        .parent().unwrap()
-        .join("setup.log");
+    let mut log_file = std::env::current_exe().unwrap();
+    log_file.set_file_name("setup.log");
+    let file_logging = Some(log_file);
 
     let config = logging::LoggingConfig {
-        file_logging: Some(log_file),
         logging_stdout: false,
         opentelemetry: OpenTelemetryConfig::Disabled,
     };
-    let _ = logging::initialize_with_config(config)?;
+    let _ = logging::initialize_with_config(config, file_logging)?;
 
     Ok(())
 }

opendut-util/src/logging.rs

@@ -39,11 +39,11 @@ pub enum Error {
     EndpointConfigurationMissing,
 }
 
-pub fn initialize() -> Result<ShutdownHandle, Error> {
-    initialize_with_config(LoggingConfig::default())
+pub fn initialize_with_defaults() -> Result<ShutdownHandle, Error> {
+    initialize_with_config(LoggingConfig::default(), None)
 }
 
-pub fn initialize_with_config(config: LoggingConfig) -> Result<ShutdownHandle, Error> {
+pub fn initialize_with_config(config: LoggingConfig, file_logging: Option<PathBuf>) -> Result<ShutdownHandle, Error> {
 
     global::set_text_map_propagator(TraceContextPropagator::new());
 
@@ -62,7 +62,7 @@ pub fn initialize_with_config(config: LoggingConfig) -> Result<ShutdownHandle, E
         };
 
     let file_logging_layer =
-        if let Some(log_file) = config.file_logging {
+        if let Some(log_file) = file_logging {
 
             let log_file = File::create(&log_file)
                 .unwrap_or_else(|cause| panic!("Failed to open log file at '{}': {cause}", log_file.display()));
@@ -211,7 +211,6 @@ pub fn initialize_metrics_collection(cpu_collection_interval_ms: Duration) {
 
 #[derive(Default, Clone)]
 pub struct LoggingConfig {
-    pub file_logging: Option<PathBuf>,
     pub logging_stdout: bool,
     pub opentelemetry: OpenTelemetryConfig,
 }
@@ -229,7 +228,6 @@ pub enum OpenTelemetryConfig {
 }
 impl LoggingConfig {
     pub fn load(config: &config::Config, service_instance_id: String) -> Result<Self, LoggingConfigError> {
-        let file_logging = None; //TODO load from config
         let logging_stdout = config.get_bool("opentelemetry.logging.stdout")?;
 
         let opentelemetry_enabled = config.get_bool("opentelemetry.enabled")?;
@@ -295,7 +293,6 @@ impl LoggingConfig {
         };
 
         Ok(LoggingConfig {
-            file_logging,
             logging_stdout,
             opentelemetry,
         })

tests/src/register.rs

@@ -11,7 +11,7 @@ use crate::util;
 
 #[tokio::test(flavor = "multi_thread")]
 async fn register_edgar_carl() -> Result<()> {
-    let _ = logging::initialize()?;
+    let _ = logging::initialize_with_defaults()?;
 
     let carl_port = util::select_free_port();
     info!("Running test with CARL port {carl_port}.");