feat: crash pod if containers die

launcher/clabernetes.go

@@ -16,6 +16,7 @@ import (
 
 const (
 	maxDockerLaunchAttempts = 10
+	containerCheckInterval  = 5 * time.Second
 )
 
 // StartClabernetes is a function that starts the clabernetes launcher.
@@ -48,10 +49,11 @@ func StartClabernetes() {
 		clabernetesconstants.Info,
 	)
 
-	ctx, _ := clabernetesutil.SignalHandledContext(clabernetesLogger.Criticalf)
+	ctx, cancel := clabernetesutil.SignalHandledContext(clabernetesLogger.Criticalf)
 
 	clabernetesInstance = &clabernetes{
-		ctx: ctx,
+		ctx:    ctx,
+		cancel: cancel,
 		appName: clabernetesutil.GetEnvStrOrDefault(
 			clabernetesconstants.AppNameEnvVar,
 			clabernetesconstants.AppNameDefault,
@@ -67,13 +69,16 @@ func StartClabernetes() {
 var clabernetesInstance *clabernetes //nolint:gochecknoglobals
 
 type clabernetes struct {
-	ctx context.Context
+	ctx    context.Context
+	cancel context.CancelFunc
 
 	appName string
 
 	logger             claberneteslogging.Instance
 	containerlabLogger claberneteslogging.Instance
 	nodeLogger         claberneteslogging.Instance
+
+	containerIDs []string
 }
 
 func (c *clabernetes) startup() {
@@ -84,9 +89,13 @@ func (c *clabernetes) startup() {
 	c.setup()
 	c.launch()
 
+	go c.watch()
+
 	c.logger.Info("running for forever or until sigint...")
 
 	<-c.ctx.Done()
+
+	claberneteslogging.GetManager().Flush()
 }
 
 func (c *clabernetes) setup() {
@@ -136,8 +145,18 @@ func (c *clabernetes) launch() {
 		clabernetesutil.Panic(err.Error())
 	}
 
-	containerIDs := c.getContainerIDs()
-	c.tailContainerLogs(containerIDs)
+	c.containerIDs = c.getContainerIDs()
+
+	if len(c.containerIDs) > 0 {
+		c.logger.Debugf("found container ids %q", c.containerIDs)
+
+		c.tailContainerLogs()
+	} else {
+		c.logger.Warn(
+			"failed determining container ids, will continue but may not be in a working " +
+				"state and no container logs will be captured",
+		)
+	}
 
 	c.logger.Info("containerlab started, setting up any required tunnels...")
 
@@ -176,3 +195,27 @@ func (c *clabernetes) launch() {
 		}
 	}
 }
+
+func (c *clabernetes) watch() {
+	if len(c.containerIDs) == 0 {
+		return
+	}
+
+	ticker := time.NewTicker(containerCheckInterval)
+
+	for range ticker.C {
+		currentContainerIDs := c.getContainerIDs()
+
+		if len(currentContainerIDs) != len(c.containerIDs) {
+			c.logger.Criticalf(
+				"expected %d running containers, but got %d, sending done signal",
+				len(c.containerIDs),
+				len(currentContainerIDs),
+			)
+
+			c.cancel()
+
+			return
+		}
+	}
+}

launcher/docker.go

@@ -120,14 +120,22 @@ func (c *clabernetes) getContainerIDs() []string {
 		return nil
 	}
 
-	containerIDs := strings.Split(strings.TrimSpace(string(output)), "\n")
+	containerIDLines := strings.Split(string(output), "\n")
 
-	c.logger.Debugf("found container ids %q", containerIDs)
+	var containerIDs []string
+
+	for _, line := range containerIDLines {
+		trimmedLine := strings.TrimSpace(line)
+
+		if trimmedLine != "" {
+			containerIDs = append(containerIDs, trimmedLine)
+		}
+	}
 
 	return containerIDs
 }
 
-func (c *clabernetes) tailContainerLogs(containerIDs []string) {
+func (c *clabernetes) tailContainerLogs() {
 	nodeLogFile, err := os.Create("node.log")
 	if err != nil {
 		c.logger.Warnf("failed creating node log file, err: %s", err)
@@ -137,7 +145,7 @@ func (c *clabernetes) tailContainerLogs(containerIDs []string) {
 
 	nodeOutWriter := io.MultiWriter(c.nodeLogger, nodeLogFile)
 
-	for _, containerID := range containerIDs {
+	for _, containerID := range c.containerIDs {
 		go func(containerID string, nodeOutWriter io.Writer) {
 			args := []string{
 				"logs",