[manuf] enable alerts during individualization

This updates the individualization firmware to enable alerts and capture crashdump information if requested by the host. This may be used to debug alert escalations that may occur during various manufacturing operations such as OTP programming. This feature is piped through the orchestrator script to enable turning it on or off as needed during bringup / debugging efforts. Signed-off-by: Tim Trippel <[email protected]>
lowRISC · Jan 17, 2025 · f0c4fca · f0c4fca
1 parent 1015507
commit f0c4fca
Show file tree

Hide file tree

Showing 8 changed files with 143 additions and 9 deletions.
diff --git a/sw/device/lib/testing/json/provisioning_data.h b/sw/device/lib/testing/json/provisioning_data.h
@@ -56,6 +56,7 @@ UJSON_SERDE_STRUCT(ManufCpTestData, \
  */
 // clang-format off
 #define STRUCT_MANUF_FT_INDIVIDUALIZE_DATA(field, string) \
+    field(enable_alerts, bool) \
     field(use_ext_clk, bool) \
     field(ft_device_id, uint32_t, 4)
 UJSON_SERDE_STRUCT(ManufFtIndividualizeData, \

diff --git a/sw/device/silicon_creator/manuf/base/BUILD b/sw/device/silicon_creator/manuf/base/BUILD
@@ -193,10 +193,12 @@ opentitan_test(
         linker_script = "//sw/device/silicon_creator/manuf/lib:sram_program_linker_script",
         deps = [
             ":flash_info_permissions",
+            "//hw/top_earlgrey:alert_handler_c_regs",
             "//hw/top_earlgrey/sw/autogen:top_earlgrey",
             "//sw/device/lib/arch:device",
             "//sw/device/lib/base:abs_mmio",
             "//sw/device/lib/base:macros",
+            "//sw/device/lib/dif:alert_handler",
             "//sw/device/lib/dif:clkmgr",
             "//sw/device/lib/dif:flash_ctrl",
             "//sw/device/lib/dif:otp_ctrl",
@@ -209,9 +211,12 @@ opentitan_test(
             "//sw/device/lib/testing:pinmux_testutils",
             "//sw/device/lib/testing/test_framework:check",
             "//sw/device/lib/testing/test_framework:ottf_console",
+            "//sw/device/lib/testing/test_framework:ottf_isrs",
             "//sw/device/lib/testing/test_framework:ottf_test_config",
             "//sw/device/lib/testing/test_framework:status",
             "//sw/device/lib/testing/test_framework:ujson_ottf",
+            "//sw/device/silicon_creator/lib/drivers:ibex",
+            "//sw/device/silicon_creator/lib/drivers:rstmgr",
             "//sw/device/silicon_creator/manuf/lib:flash_info_fields",
             "//sw/device/silicon_creator/manuf/lib:individualize",
             "//sw/device/silicon_creator/manuf/lib:otp_fields",

diff --git a/sw/device/silicon_creator/manuf/base/sram_ft_individualize.c b/sw/device/silicon_creator/manuf/base/sram_ft_individualize.c
@@ -6,6 +6,7 @@
 
 #include "sw/device/lib/arch/device.h"
 #include "sw/device/lib/base/abs_mmio.h"
+#include "sw/device/lib/dif/dif_alert_handler.h"
 #include "sw/device/lib/dif/dif_clkmgr.h"
 #include "sw/device/lib/dif/dif_flash_ctrl.h"
 #include "sw/device/lib/dif/dif_otp_ctrl.h"
@@ -20,18 +21,22 @@
 #include "sw/device/lib/testing/test_framework/ottf_console.h"
 #include "sw/device/lib/testing/test_framework/ottf_test_config.h"
 #include "sw/device/lib/testing/test_framework/ujson_ottf.h"
+#include "sw/device/silicon_creator/lib/drivers/ibex.h"
+#include "sw/device/silicon_creator/lib/drivers/rstmgr.h"
 #include "sw/device/silicon_creator/manuf/base/flash_info_permissions.h"
 #include "sw/device/silicon_creator/manuf/lib/flash_info_fields.h"
 #include "sw/device/silicon_creator/manuf/lib/individualize.h"
 #include "sw/device/silicon_creator/manuf/lib/individualize_sw_cfg.h"
 #include "sw/device/silicon_creator/manuf/lib/otp_fields.h"
 
+#include "alert_handler_regs.h"  // Generated.
 #include "hw/top_earlgrey/sw/autogen/top_earlgrey.h"
 
 OTTF_DEFINE_TEST_CONFIG(.console.type = kOttfConsoleSpiDevice,
                         .console.base_addr = TOP_EARLGREY_SPI_DEVICE_BASE_ADDR,
                         .console.test_may_clobber = false, );
 
+static dif_alert_handler_t alert_handler;
 static dif_clkmgr_t clkmgr;
 static dif_flash_ctrl_state_t flash_ctrl_state;
 static dif_otp_ctrl_t otp_ctrl;
@@ -45,10 +50,25 @@ static uint32_t ast_cfg_data[kFlashInfoAstCalibrationDataSizeIn32BitWords];
 // This is used to delay further action when the switch happens.
 static const int kSettleDelayMicros = 200;
 
+static size_t alert_nmi_count = 0;
+
+/**
+ * Handle NMI's from the alert escalation mechanism.
+ *
+ * @param exc_info Execution info.
+ */
+void ottf_external_nmi_handler(uint32_t *exc_info) {
+  LOG_INFO("Processing Alert NMI %d ...", alert_nmi_count++);
+  ibex_clear_nmi(kIbexNmiSourceAlert);
+}
+
 /**
  * Initializes all DIF handles used in this SRAM program.
  */
 static status_t peripheral_handles_init(void) {
+  TRY(dif_alert_handler_init(
+      mmio_region_from_addr(TOP_EARLGREY_ALERT_HANDLER_BASE_ADDR),
+      &alert_handler));
   TRY(dif_clkmgr_init(mmio_region_from_addr(TOP_EARLGREY_CLKMGR_AON_BASE_ADDR),
                       &clkmgr));
   TRY(dif_flash_ctrl_init_state(
@@ -105,6 +125,61 @@ static bool did_extclk_settle(const dif_clkmgr_t *clkmgr) {
   return status;
 }
 
+static status_t configure_all_alerts(void) {
+  // Enable capturing both alert and CPU crashdump info collection.
+  rstmgr_alert_info_enable();
+  rstmgr_cpu_info_enable();
+
+  // Enable all alerts and configure them in class A.
+  for (size_t i = 0; i < ALERT_HANDLER_PARAM_N_ALERTS; ++i) {
+    TRY(dif_alert_handler_configure_alert(
+        &alert_handler, (dif_alert_handler_alert_t)i, kDifAlertHandlerClassA,
+        /*enabled=*/kDifToggleEnabled,
+        /*locked=*/kDifToggleDisabled));
+  }
+
+  // Enable all local alerts and configure them in class A.
+  for (size_t i = 0; i < ALERT_HANDLER_PARAM_N_LOC_ALERT; ++i) {
+    TRY(dif_alert_handler_configure_local_alert(
+        &alert_handler, (dif_alert_handler_local_alert_t)i,
+        kDifAlertHandlerClassA,
+        /*enabled=*/kDifToggleEnabled,
+        /*locked=*/kDifToggleDisabled));
+  }
+
+  // Configure class A alert escalation behavior.
+  const dif_alert_handler_escalation_phase_t kEscPhases[2] = {
+      {
+          .phase = kDifAlertHandlerClassStatePhase0,
+          .signal = 0,  // NMI
+          .duration_cycles = 1000000,
+      },
+      {
+          .phase = kDifAlertHandlerClassStatePhase1,
+          .signal = 3,  // SW Reset
+          .duration_cycles = 1000,
+      },
+  };
+  dif_alert_handler_class_config_t alert_class_a_config = {
+      .auto_lock_accumulation_counter = kDifToggleDisabled,
+      .accumulator_threshold = 0,  // A single alert will trigger escalation.
+      .irq_deadline_cycles = 0,    // Disabled.
+      .escalation_phases = kEscPhases,
+      .escalation_phases_len = ARRAYSIZE(kEscPhases),
+      .crashdump_escalation_phase = kDifAlertHandlerClassStatePhase0,
+
+  };
+  TRY(dif_alert_handler_configure_class(&alert_handler, kDifAlertHandlerClassA,
+                                        alert_class_a_config,
+                                        /*enabled=*/kDifToggleEnabled,
+                                        /*locked=*/kDifToggleDisabled));
+
+  // Enable NMIs from alert escalation.
+  ibex_enable_nmi(kIbexNmiSourceAlert);
+
+  return OK_STATUS();
+}
+
 /**
  * Provision OTP {CreatorSw,OwnerSw,Hw}Cfg and RotCreatorAuth{Codesign,State}
  * partitions.
@@ -117,6 +192,12 @@ static status_t provision(ujson_t *uj) {
   LOG_INFO("Waiting for FT SRAM provisioning data ...");
   TRY(ujson_deserialize_manuf_ft_individualize_data_t(uj, &in_data));
 
+  // Enable all alerts (and alert NMIs) if requested.
+  if (in_data.enable_alerts) {
+    TRY(configure_all_alerts());
+    LOG_INFO("All alerts and alert NMI enabled.");
+  }
+
   // Enable external clock on silicon platforms if requested.
   if (kDeviceType == kDeviceSilicon && in_data.use_ext_clk) {
     CHECK_DIF_OK(dif_clkmgr_external_clock_set_enabled(&clkmgr,
@@ -149,6 +230,15 @@ bool test_main(void) {
   ottf_console_init();
   ujson_t uj = ujson_ottf_console();
 
+  // Abort if reset reason is due to an escalation.
+  if (rstmgr_reason_get() == kRstmgrReasonEscalation) {
+    LOG_INFO("Reset due to alert escalation ... aborting.");
+    abort();
+  }
+
+  // Clear the reset reasons.
+  rstmgr_reason_clear(UINT8_MAX);
+
   // Read and log flash and AST data to console (for manual verification
   // purposes), and perform provisioning operations.
   CHECK_STATUS_OK(read_and_print_flash_and_ast_data());

diff --git a/sw/host/provisioning/ft/src/main.rs b/sw/host/provisioning/ft/src/main.rs
@@ -40,6 +40,9 @@ pub struct ManufFtProvisioningDataInput {
     #[arg(long)]
     pub ft_device_id: String,
 
+    #[arg(long)]
+    pub enable_alerts_during_individualize: bool,
+
     #[arg(long)]
     pub use_ext_clk_during_individualize: bool,
 
@@ -156,6 +159,7 @@ fn main() -> Result<()> {
     // The FT device ID is sent to the DUT in little endian order.
     ft_device_id.reverse();
     let ft_individualize_data_in = ManufFtIndividualizeData {
+        enable_alerts: opts.provisioning_data.enable_alerts_during_individualize,
         use_ext_clk: opts.provisioning_data.use_ext_clk_during_individualize,
         ft_device_id,
     };

diff --git a/sw/host/provisioning/ft_lib/src/lib.rs b/sw/host/provisioning/ft_lib/src/lib.rs
@@ -6,9 +6,10 @@ use sha2::{Digest, Sha256};
 use std::collections::HashMap;
 use std::collections::HashSet;
 use std::path::PathBuf;
+use std::thread;
 use std::time::{Duration, Instant};
 
-use anyhow::{bail, Result};
+use anyhow::{anyhow, bail, Result};
 use arrayvec::ArrayVec;
 use zerocopy::AsBytes;
 
@@ -18,6 +19,9 @@ use opentitanlib::app::TransportWrapper;
 use opentitanlib::console::spi::SpiConsoleDevice;
 use opentitanlib::dif::lc_ctrl::{DifLcCtrlState, LcCtrlReg};
 use opentitanlib::io::jtag::{JtagParams, JtagTap};
+use opentitanlib::test_utils::crashdump::{
+    read_alert_crashdump_data, read_cpu_crashdump_data, read_reset_reason,
+};
 use opentitanlib::test_utils::init::InitializeTest;
 use opentitanlib::test_utils::lc_transition::trigger_lc_transition;
 use opentitanlib::test_utils::load_sram_program::{
@@ -112,13 +116,30 @@ pub fn run_sram_ft_individualize(
     // Inject provisioning data into the device.
     ft_individualize_data_in.send(spi_console)?;
 
-    // Wait for provisioning operations to complete.
-    let _ = UartConsole::wait_for(spi_console, r"FT SRAM provisioning done.", timeout)?;
-
-    jtag.disconnect()?;
-    transport.pin_strapping("PINMUX_TAP_RISCV")?.remove()?;
-
-    Ok(())
+    // Wait for provisioning operations to complete. If we see at least 10 NMIs, we know it is time
+    // to capture crashdump information.
+    let console_text = UartConsole::wait_for(
+        spi_console,
+        r"FT SRAM provisioning done.|Processing Alert NMI 10 ...",
+        timeout,
+    )?;
+    match console_text[0].as_str() {
+        "FT SRAM provisioning done." => {
+            jtag.disconnect()?;
+            transport.pin_strapping("PINMUX_TAP_RISCV")?.remove()?;
+            Ok(())
+        }
+        "Processing Alert NMI 10 ..." => {
+            transport.pin_strapping("PINMUX_TAP_RISCV")?.remove()?;
+            jtag.disconnect()?;
+            thread::sleep(Duration::from_millis(10000));
+            read_reset_reason(transport, jtag_params)?;
+            read_cpu_crashdump_data(transport, jtag_params)?;
+            read_alert_crashdump_data(transport, jtag_params)?;
+            Ok(())
+        }
+        _ => Err(anyhow!("Unexpected console_text: {:?}", console_text)),
+    }
 }
 
 pub fn test_exit(

diff --git a/sw/host/provisioning/orchestrator/src/orchestrator.py b/sw/host/provisioning/orchestrator/src/orchestrator.py
@@ -105,6 +105,12 @@ def main(args_in):
         action="store_true",
         help="If set, the FPGA bitsream will not be cleared before CP.",
     )
+    parser.add_argument(
+        "--enable-alerts",
+        action="store_true",
+        default=False,
+        help="Enable all alerts during FT individualize step.",
+    )
     parser.add_argument(
         "--use-ext-clk",
         action="store_true",
@@ -167,6 +173,7 @@ def main(args_in):
                 test_exit_token=args.test_exit_token,
                 fpga=args.fpga,
                 fpga_dont_clear_bitstream=args.fpga_dont_clear_bitstream,
+                enable_alerts=args.enable_alerts,
                 use_ext_clk=args.use_ext_clk,
                 require_confirmation=not args.non_interactive)
     dut.run_cp()

diff --git a/sw/host/provisioning/orchestrator/src/ot_dut.py b/sw/host/provisioning/orchestrator/src/ot_dut.py
@@ -51,6 +51,7 @@ class OtDut():
     test_exit_token: str
     fpga: str
     fpga_dont_clear_bitstream: bool
+    enable_alerts: bool
     use_ext_clk: bool
     require_confirmation: bool = True
 
@@ -271,7 +272,11 @@ def run_ft(self) -> None:
             --token-encrypt-key-der-file={self.sku_config.token_encrypt_key} \
             """
 
-            # Enable external clock if requested.
+            # Enable alerts during individualization if requested.
+            if self.enable_alerts:
+                cmd += " --enable-alerts-during-individualize"
+
+            # Enable external clock during individualization if requested.
             if self.use_ext_clk:
                 cmd += " --use-ext-clk-during-individualize"
 

diff --git a/sw/host/provisioning/orchestrator/tests/e2e.sh b/sw/host/provisioning/orchestrator/tests/e2e.sh
@@ -28,6 +28,7 @@ $PYTHON ${ORCHESTRATOR_PATH} \
   --test-unlock-token="0x11111111_11111111_11111111_11111111" \
   --test-exit-token="0x22222222_22222222_22222222_22222222" \
   --fpga=hyper310 \
+  --enable-alerts \
   --use-ext-clk \
   --non-interactive \
   --db-path=$TEST_TMPDIR/registry.sqlite