Merge pull request #57 from srl-labs/feat/tail-nos-container-logs

feat: tee node logs to pod logs

launcher/clabernetes.go

@@ -16,6 +16,7 @@ import (
 
 const (
 	maxDockerLaunchAttempts = 10
+	containerCheckInterval  = 5 * time.Second
 )
 
 // StartClabernetes is a function that starts the clabernetes launcher.
@@ -38,15 +39,28 @@ func StartClabernetes() {
 		),
 	)
 
-	ctx, _ := clabernetesutil.SignalHandledContext(clabernetesLogger.Criticalf)
+	containerlabLogger := logManager.MustRegisterAndGetLogger(
+		"containerlab",
+		clabernetesconstants.Info,
+	)
+
+	nodeLogger := logManager.MustRegisterAndGetLogger(
+		"node",
+		clabernetesconstants.Info,
+	)
+
+	ctx, cancel := clabernetesutil.SignalHandledContext(clabernetesLogger.Criticalf)
 
 	clabernetesInstance = &clabernetes{
-		ctx: ctx,
+		ctx:    ctx,
+		cancel: cancel,
 		appName: clabernetesutil.GetEnvStrOrDefault(
 			clabernetesconstants.AppNameEnvVar,
 			clabernetesconstants.AppNameDefault,
 		),
-		logger: clabernetesLogger,
+		logger:             clabernetesLogger,
+		containerlabLogger: containerlabLogger,
+		nodeLogger:         nodeLogger,
 	}
 
 	clabernetesInstance.startup()
@@ -55,11 +69,16 @@ func StartClabernetes() {
 var clabernetesInstance *clabernetes //nolint:gochecknoglobals
 
 type clabernetes struct {
-	ctx context.Context
+	ctx    context.Context
+	cancel context.CancelFunc
 
 	appName string
 
-	logger claberneteslogging.Instance
+	logger             claberneteslogging.Instance
+	containerlabLogger claberneteslogging.Instance
+	nodeLogger         claberneteslogging.Instance
+
+	containerIDs []string
 }
 
 func (c *clabernetes) startup() {
@@ -70,9 +89,13 @@ func (c *clabernetes) startup() {
 	c.setup()
 	c.launch()
 
+	go c.watch()
+
 	c.logger.Info("running for forever or until sigint...")
 
 	<-c.ctx.Done()
+
+	claberneteslogging.GetManager().Flush()
 }
 
 func (c *clabernetes) setup() {
@@ -115,13 +138,26 @@ func (c *clabernetes) setup() {
 func (c *clabernetes) launch() {
 	c.logger.Debug("launching containerlab...")
 
-	err := c.runClab()
+	err := c.runContainerlab()
 	if err != nil {
 		c.logger.Criticalf("failed launching containerlab, err: %s", err)
 
 		clabernetesutil.Panic(err.Error())
 	}
 
+	c.containerIDs = c.getContainerIDs()
+
+	if len(c.containerIDs) > 0 {
+		c.logger.Debugf("found container ids %q", c.containerIDs)
+
+		c.tailContainerLogs()
+	} else {
+		c.logger.Warn(
+			"failed determining container ids, will continue but may not be in a working " +
+				"state and no container logs will be captured",
+		)
+	}
+
 	c.logger.Info("containerlab started, setting up any required tunnels...")
 
 	tunnelBytes, err := os.ReadFile("tunnels.yaml")
@@ -141,7 +177,7 @@ func (c *clabernetes) launch() {
 	}
 
 	for _, tunnel := range tunnelObj {
-		err = c.runClabVxlanTools(
+		err = c.runContainerlabVxlanTools(
 			tunnel.LocalNodeName,
 			tunnel.LocalLinkName,
 			tunnel.RemoteName,
@@ -159,3 +195,27 @@ func (c *clabernetes) launch() {
 		}
 	}
 }
+
+func (c *clabernetes) watch() {
+	if len(c.containerIDs) == 0 {
+		return
+	}
+
+	ticker := time.NewTicker(containerCheckInterval)
+
+	for range ticker.C {
+		currentContainerIDs := c.getContainerIDs()
+
+		if len(currentContainerIDs) != len(c.containerIDs) {
+			c.logger.Criticalf(
+				"expected %d running containers, but got %d, sending done signal",
+				len(c.containerIDs),
+				len(currentContainerIDs),
+			)
+
+			c.cancel()
+
+			return
+		}
+	}
+}

launcher/containerlab.go

@@ -11,13 +11,13 @@ import (
 	claberneteserrors "github.com/srl-labs/clabernetes/errors"
 )
 
-func (c *clabernetes) runClab() error {
-	clabLogFile, err := os.Create("clab.log")
+func (c *clabernetes) runContainerlab() error {
+	containerlabLogFile, err := os.Create("containerlab.log")
 	if err != nil {
 		return err
 	}
 
-	clabOutWriter := io.MultiWriter(c.logger, clabLogFile)
+	containerlabOutWriter := io.MultiWriter(c.containerlabLogger, containerlabLogFile)
 
 	args := []string{
 		"deploy",
@@ -31,8 +31,8 @@ func (c *clabernetes) runClab() error {
 
 	cmd := exec.Command("containerlab", args...)
 
-	cmd.Stdout = clabOutWriter
-	cmd.Stderr = clabOutWriter
+	cmd.Stdout = containerlabOutWriter
+	cmd.Stderr = containerlabOutWriter
 
 	err = cmd.Run()
 	if err != nil {
@@ -42,7 +42,7 @@ func (c *clabernetes) runClab() error {
 	return nil
 }
 
-func (c *clabernetes) runClabVxlanTools(
+func (c *clabernetes) runContainerlabVxlanTools(
 	localNodeName, cntLink, vxlanRemote string,
 	vxlanID int,
 ) error {

launcher/docker.go

@@ -3,6 +3,7 @@ package launcher
 import (
 	"bytes"
 	"fmt"
+	"io"
 	"os"
 	"os/exec"
 	"strings"
@@ -103,3 +104,66 @@ func (c *clabernetes) startDocker() error {
 		attempts++
 	}
 }
+
+func (c *clabernetes) getContainerIDs() []string {
+	// return all the container ids running in the pod
+	psCmd := exec.Command("docker", "ps", "--quiet")
+
+	output, err := psCmd.Output()
+	if err != nil {
+		c.logger.Warnf(
+			"failed determining container ids will continue but will not log container output,"+
+				" err: %s",
+			err,
+		)
+
+		return nil
+	}
+
+	containerIDLines := strings.Split(string(output), "\n")
+
+	var containerIDs []string
+
+	for _, line := range containerIDLines {
+		trimmedLine := strings.TrimSpace(line)
+
+		if trimmedLine != "" {
+			containerIDs = append(containerIDs, trimmedLine)
+		}
+	}
+
+	return containerIDs
+}
+
+func (c *clabernetes) tailContainerLogs() {
+	nodeLogFile, err := os.Create("node.log")
+	if err != nil {
+		c.logger.Warnf("failed creating node log file, err: %s", err)
+
+		return
+	}
+
+	nodeOutWriter := io.MultiWriter(c.nodeLogger, nodeLogFile)
+
+	for _, containerID := range c.containerIDs {
+		go func(containerID string, nodeOutWriter io.Writer) {
+			args := []string{
+				"logs",
+				"-f",
+				containerID,
+			}
+
+			cmd := exec.Command("docker", args...) //nolint:gosec
+
+			cmd.Stdout = nodeOutWriter
+			cmd.Stderr = nodeOutWriter
+
+			err = cmd.Run()
+			if err != nil {
+				c.logger.Warnf(
+					"tailing node logs for container id %q failed, err: %s", containerID, err,
+				)
+			}
+		}(containerID, nodeOutWriter)
+	}
+}