implement direct to kernel boot

.github/workflows/build-all.yml

@@ -9,25 +9,46 @@ on:
       - "*"
 
 jobs:
-  build-pi3:
+  build-pi3-direct:
     uses: ./.github/workflows/build.yml
     with:
-      host: pi3
+      host: pi3-direct
     permissions:
       contents: write
       id-token: write
-  build-pi4:
+  build-pi3-uboot:
     uses: ./.github/workflows/build.yml
     with:
-      host: pi4
+      host: pi3-uboot
+    permissions:
+      contents: write
+      id-token: write
+  build-pi4-direct:
+    uses: ./.github/workflows/build.yml
+    with:
+      host: pi4-direct
+    permissions:
+      contents: write
+      id-token: write
+  build-pi4-uboot:
+    uses: ./.github/workflows/build.yml
+    with:
+      host: pi4-uboot
     permissions:
       contents: write
       id-token: write
   # disabled until pi5 works, so the builds arent all red
-  # build-pi5:
+  # build-pi5-direct:
+  #   uses: ./.github/workflows/build.yml
+  #   with:
+  #     host: pi5-direct
+  #   permissions:
+  #     contents: write
+  #     id-token: write
+  # build-pi5-uboot:
   #   uses: ./.github/workflows/build.yml
   #   with:
-  #     host: pi5
+  #     host: pi5-uboot
   #   permissions:
   #     contents: write
   #     id-token: write

.gitignore

@@ -1,3 +1,4 @@
 # nix
 result
 result-*
+nixos.img

README.md

@@ -10,6 +10,8 @@ While [nix-community/raspberry-pi-nix] has wide support for creating images whic
 Raspberry Pis, the approach of using `sd-image.nix` restricts the ability to test the machine
 end to end (including bootloader) using `qemu-vm.nix`, which is what `testers.runNixOSTest` uses.
 
+This repository doesn't yet perform any tests with `testers.runNixOSTest`, but that will come soon.
+
 ## Getting started
 
 ### Prebuilt image
@@ -30,6 +32,8 @@ It isn't intended to serve as a long lived OS for your board.
 
 ### Roll your own
 
+For something more sustainable, make your own `nixosConfiguration` with the following:
+
 ```
 # flake.nix
 inputs.nixos-hardware.url = "github:NixOS/nixos-hardware/master";
@@ -49,14 +53,16 @@ boot.pi-loader.enable = true;
 ```
 
 You cannot use `boot.loader.generic-extlinux-compatible`. This module disables it because it has its
-own modified version at `boot.loader.generic-extlinux-compatible-pi-loader`. You can set options
-there instead, but please see the upcoming work section for why this might be removed in the future.
+own modified version at `boot.loader.generic-extlinux-compatible-pi-loader`. You can customise options
+there instead.
 
 ### Building an image file
 
+To put this onto a Raspberry Pi, have nix create a disk image then burn the image to an SD/USB.
+
 ```
 # configuration.nix
-system.build.image = (import "${toString modulesPath}/../lib/make-disk-image.nix" {
+system.build.image = (inputs.nix-pi-loader.nixosModules.make-disk-image {
   inherit lib config pkgs;
   format = "raw";
   partitionTableType = "legacy+boot";

hosts/pi3-direct/configuration.nix

@@ -0,0 +1,8 @@
+{ config, flake, lib, modulesPath, pkgs, ... }: with lib; {
+  imports = [
+    flake.nixosModules.host-common
+
+    flake.inputs.nixos-hardware.nixosModules.raspberry-pi-3
+  ];
+  boot.pi-loader.bootMode = "direct";
+}

hosts/pi3/configuration.nix → hosts/pi3-uboot/configuration.nix

@@ -4,4 +4,5 @@
 
     flake.inputs.nixos-hardware.nixosModules.raspberry-pi-3
   ];
+  boot.pi-loader.bootMode = "uboot";
 }

hosts/pi4-direct/configuration.nix

@@ -0,0 +1,8 @@
+{ config, flake, lib, modulesPath, pkgs, ... }: with lib; {
+  imports = [
+    flake.nixosModules.host-common
+
+    flake.inputs.nixos-hardware.nixosModules.raspberry-pi-4
+  ];
+  boot.pi-loader.bootMode = "direct";
+}

hosts/pi4/configuration.nix → hosts/pi4-uboot/configuration.nix

@@ -4,4 +4,5 @@
 
     flake.inputs.nixos-hardware.nixosModules.raspberry-pi-4
   ];
+  boot.pi-loader.bootMode = "uboot";
 }

hosts/pi5-direct/configuration.nix

@@ -0,0 +1,8 @@
+{ config, flake, lib, modulesPath, pkgs, ... }: with lib; {
+  imports = [
+    flake.nixosModules.host-common
+
+    flake.inputs.nixos-hardware.nixosModules.raspberry-pi-5
+  ];
+  boot.pi-loader.bootMode = "direct";
+}

hosts/pi5/configuration.nix → hosts/pi5-uboot/configuration.nix

@@ -4,4 +4,5 @@
 
     flake.inputs.nixos-hardware.nixosModules.raspberry-pi-5
   ];
+  boot.pi-loader.bootMode = "uboot";
 }

modules/nixos/atomic-copy-clobber/atomic-copy-clobber.sh

@@ -0,0 +1,18 @@
+#! @bash@/bin/sh -e
+
+# copy+paste of copyToKernelsDir https://github.com/NixOS/nixpkgs/blob/904ecf0b4e055dc465f5ae6574be2af8cc25dec3/nixos/modules/system/boot/loader/generic-extlinux-compatible/extlinux-conf-builder.sh#L53
+# but without the check which skips the copy if the destination exists
+
+shopt -s nullglob
+
+export PATH=/empty
+for i in @path@; do PATH=$PATH:$i/bin; done
+
+src=$(readlink -f "$1")
+dst="$2"
+
+# Create $dst atomically to prevent partially copied files
+# if this script is ever interrupted.
+dstTmp=$dst.tmp.$$
+cp -r $src $dstTmp
+mv $dstTmp $dst

modules/nixos/atomic-copy-clobber/default.nix

@@ -0,0 +1,8 @@
+{ pkgs }:
+
+pkgs.substituteAll {
+  src = ./atomic-copy-clobber.sh;
+  isExecutable = true;
+  path = [pkgs.coreutils pkgs.gnused pkgs.gnugrep];
+  inherit (pkgs) bash;
+}

modules/nixos/atomic-copy-safe/atomic-copy-safe.sh

@@ -0,0 +1,20 @@
+#! @bash@/bin/sh -e
+
+# copy+paste of copyToKernelsDir https://github.com/NixOS/nixpkgs/blob/904ecf0b4e055dc465f5ae6574be2af8cc25dec3/nixos/modules/system/boot/loader/generic-extlinux-compatible/extlinux-conf-builder.sh#L53
+
+shopt -s nullglob
+
+export PATH=/empty
+for i in @path@; do PATH=$PATH:$i/bin; done
+
+src=$(readlink -f "$1")
+dst="$2"
+
+# Don't copy the file if $dst already exists.
+# Also create $dst atomically to prevent partially copied files
+# if this script is ever interrupted.
+if ! test -e $dst; then
+	dstTmp=$dst.tmp.$$
+	cp -r $src $dstTmp
+	mv $dstTmp $dst
+fi

modules/nixos/default/atomic-copy.nix → modules/nixos/atomic-copy-safe/default.nix

@@ -1,7 +1,7 @@
 { pkgs }:
 
 pkgs.substituteAll {
-  src = ./atomic-copy.sh;
+  src = ./atomic-copy-safe.sh;
   isExecutable = true;
   path = [pkgs.coreutils pkgs.gnused pkgs.gnugrep];
   inherit (pkgs) bash;

modules/nixos/default/default.nix

@@ -4,6 +4,13 @@
 #
 { config, lib, pkgs, ... }: with lib; let
   cfg = config.boot.pi-loader;
+
+  # used for direct-to-kernel boot only: emulate cleanName()
+  # https://github.com/NixOS/nixpkgs/blob/904ecf0b4e055dc465f5ae6574be2af8cc25dec3/nixos/modules/system/boot/loader/generic-extlinux-compatible/extlinux-conf-builder.sh#L47
+  kernelStorePath = "${config.system.build.kernel}/${config.system.boot.loader.kernelFile}";
+  initrdStorePath = "${config.system.build.initialRamdisk}/${config.system.boot.loader.initrdFile}";
+  kernelBootPath = "nixos/${builtins.replaceStrings [ "/nix/store/" "/" ] [ "" "-" ] kernelStorePath}";
+  initrdBootPath = "nixos/${builtins.replaceStrings [ "/nix/store/" "/" ] [ "" "-" ] initrdStorePath}";
 in {
   imports = [
     ../generic-extlinux-compatible
@@ -29,27 +36,57 @@ in {
       type = types.str;
       default = "/boot";
     };
+    bootMode = mkOption {
+      description = "Select whether to boot direct to kernel or use uboot. Direct to kernel will not allow generation selection";
+      type = types.enum [ "uboot" "direct" ];
+      default = "uboot";
+    };
+    rootPartition = mkOption {
+      description = "Kernel parameter (`root=`) used only for direct to kernel boot to identify the root partition.";
+      default = "LABEL=nixos";
+    };
     configTxt = mkOption {
       type = types.attrs;
       default = {
         pi3 = {
-          kernel = "u-boot-rpi3.bin";
-        };
+          direct = {
+            kernel = kernelBootPath;
+            ramfsfile = initrdBootPath;
+            ramfsaddr = -1;
+          };
+          uboot = {
+            kernel = "u-boot-rpi3.bin";
+          };
+        }.${cfg.bootMode};
         pi02 = {
-          kernel = "u-boot-rpi3.bin";
-        };
+          direct = {
+            kernel = kernelBootPath;
+            ramfsfile = initrdBootPath;
+            ramfsaddr = -1;
+          };
+          uboot = {
+            kernel = "u-boot-rpi3.bin";
+          };
+        }.${cfg.bootMode};
         pi4 = {
-          kernel = "u-boot-rpi4.bin";
-          enable_gic = 1;
-          armstub = "armstub8-gic.bin";
-
-          # Otherwise the resolution will be weird in most cases, compared to
-          # what the pi3 firmware does by default.
-          disable_overscan = 1;
+            # Otherwise the resolution will be weird in most cases, compared to
+            # what the pi3 firmware does by default.
+            disable_overscan = 1;
 
-          # Supported in newer board revisions
-          arm_boost = 1;
-        };
+            # Supported in newer board revisions
+            arm_boost = 1;
+        } // ({
+          direct = {
+            kernel = kernelBootPath;
+            ramfsfile = initrdBootPath;
+            ramfsaddr = -1;
+          };
+          uboot = {
+            kernel = "u-boot-rpi4.bin";
+            enable_gic = 1;
+            armstub = "armstub8-gic.bin";
+          };
+        }.${cfg.bootMode});
         cm4 = {
           # Enable host mode on the 2711 built-in XHCI USB controller.
           # This line should be removed if the legacy DWC2 controller is required
@@ -77,28 +114,53 @@ in {
     boot.loader.grub.enable = false;
     boot.loader.generic-extlinux-compatible.enable = false;
     boot.loader.generic-extlinux-compatible-pi-loader.enable = true;
+    boot.kernelParams = [
+      # todo
+    ] ++ ({
+      direct = [
+          "root=${cfg.rootPartition}"
+          "rootfstype=ext4"
+          "rootwait"
+          "init=/nix/var/nix/profiles/system/init"
+      ];
+      uboot = [];
+    }.${cfg.bootMode});
     boot.loader.generic-extlinux-compatible-pi-loader.extraCommandsAfter = let
-      atomicCopy = import ./atomic-copy.nix { inherit pkgs; };
+      atomicCopySafe = import ../atomic-copy-safe { inherit pkgs; };
+      atomicCopyClobber = import ../atomic-copy-clobber { inherit pkgs; };
       configTxt = (pkgs.formats.ini {}).generate "config.txt" cfg.configTxt;
-      setupRaspiBoot = pkgs.writeShellScript "cp-pi-loaders.sh" ''
-        # Add generic files
-        cd ${pkgs.raspberrypifw}/share/raspberrypi/boot
-        ${atomicCopy} bootcode.bin ${cfg.firmwareDir}/bootcode.bin
-        ${atomicCopy} overlays     ${cfg.firmwareDir}/overlays
-        ${pkgs.findutils}/bin/find . -type f -name 'fixup*.dat' -exec ${atomicCopy} {} ${cfg.firmwareDir}/{} \;
-        ${pkgs.findutils}/bin/find . -type f -name 'start*.elf' -exec ${atomicCopy} {} ${cfg.firmwareDir}/{} \;
-        ${pkgs.findutils}/bin/find . -type f -name '*.dtb'      -exec ${atomicCopy} {} ${cfg.firmwareDir}/{} \;
+      cmdLineTxt = pkgs.writeTextFile {
+        name = "cmdline.txt";
+        text = ''
+          ${lib.strings.concatStringsSep " " config.boot.kernelParams}
+        '';
+      };
+      setupRaspiBoot = pkgs.writeShellScript "cp-pi-loaders.sh" (''
+          # Add generic files
+          cd ${pkgs.raspberrypifw}/share/raspberrypi/boot
+          ${atomicCopySafe} bootcode.bin                                                     ${cfg.firmwareDir}/bootcode.bin
+          ${atomicCopySafe} overlays                                                         ${cfg.firmwareDir}/overlays
+          ${pkgs.findutils}/bin/find . -type f -name 'fixup*.dat' -exec ${atomicCopySafe} {} ${cfg.firmwareDir}/{} \;
+          ${pkgs.findutils}/bin/find . -type f -name 'start*.elf' -exec ${atomicCopySafe} {} ${cfg.firmwareDir}/{} \;
+          ${pkgs.findutils}/bin/find . -type f -name '*.dtb'      -exec ${atomicCopySafe} {} ${cfg.firmwareDir}/{} \;
 
-        # Add the config
-        ${atomicCopy} ${configTxt} ${cfg.firmwareDir}/config.txt
+          # Add pi3 specific files
+          ${atomicCopySafe} ${pkgs.ubootRaspberryPi3_64bit}/u-boot.bin                       ${cfg.firmwareDir}/u-boot-rpi3.bin
 
-        # Add pi3 specific files
-        ${atomicCopy} ${pkgs.ubootRaspberryPi3_64bit}/u-boot.bin                            ${cfg.firmwareDir}/u-boot-rpi3.bin
+          # Add pi4 specific files
+          ${atomicCopySafe} ${pkgs.ubootRaspberryPi4_64bit}/u-boot.bin                       ${cfg.firmwareDir}/u-boot-rpi4.bin
+          ${atomicCopySafe} ${pkgs.raspberrypi-armstubs}/armstub8-gic.bin                    ${cfg.firmwareDir}/armstub8-gic.bin
 
-        # Add pi4 specific files
-        ${atomicCopy} ${pkgs.ubootRaspberryPi4_64bit}/u-boot.bin                        ${cfg.firmwareDir}/u-boot-rpi4.bin
-        ${atomicCopy} ${pkgs.raspberrypi-armstubs}/armstub8-gic.bin                     ${cfg.firmwareDir}/armstub8-gic.bin
-      '';
+          # Add config.txt
+          ${atomicCopyClobber} ${configTxt}                                                  ${cfg.firmwareDir}/config.txt
+        '' + {
+          direct = ''
+            # Add cmdline.txt
+            ${atomicCopyClobber} ${cmdLineTxt}                                                 ${cfg.firmwareDir}/cmdline.txt
+          '';
+          uboot = "";
+        }.${cfg.bootMode}
+      );
     in [ (toString setupRaspiBoot) ];
   };
 }

modules/nixos/host-common.nix

@@ -27,10 +27,6 @@
   });
 
   boot.consoleLogLevel = mkDefault 7;
-  # The serial ports listed here are:
-  # - ttyS0: for Tegra (Jetson TX1)
-  # - ttyAMA0: for QEMU's -machine virt
-  boot.kernelParams = ["console=ttyS0,115200n8" "console=ttyAMA0,115200n8" "console=tty0"];
 
   fileSystems = {
     "/boot" = {