From f0c4fca86e949f2da8b15d856582c75e0e3608e5 Mon Sep 17 00:00:00 2001 From: Tim Trippel Date: Thu, 16 Jan 2025 12:12:57 -0800 Subject: [PATCH] [manuf] enable alerts during individualization This updates the individualization firmware to enable alerts and capture crashdump information if requested by the host. This may be used to debug alert escalations that may occur during various manufacturing operations such as OTP programming. This feature is piped through the orchestrator script to enable turning it on or off as needed during bringup / debugging efforts. Signed-off-by: Tim Trippel --- .../lib/testing/json/provisioning_data.h | 1 + sw/device/silicon_creator/manuf/base/BUILD | 5 ++ .../manuf/base/sram_ft_individualize.c | 90 +++++++++++++++++++ sw/host/provisioning/ft/src/main.rs | 4 + sw/host/provisioning/ft_lib/src/lib.rs | 37 ++++++-- .../orchestrator/src/orchestrator.py | 7 ++ .../provisioning/orchestrator/src/ot_dut.py | 7 +- .../provisioning/orchestrator/tests/e2e.sh | 1 + 8 files changed, 143 insertions(+), 9 deletions(-) diff --git a/sw/device/lib/testing/json/provisioning_data.h b/sw/device/lib/testing/json/provisioning_data.h index ba3a6312beb10..c4f585f402bc3 100644 --- a/sw/device/lib/testing/json/provisioning_data.h +++ b/sw/device/lib/testing/json/provisioning_data.h @@ -56,6 +56,7 @@ UJSON_SERDE_STRUCT(ManufCpTestData, \ */ // clang-format off #define STRUCT_MANUF_FT_INDIVIDUALIZE_DATA(field, string) \ + field(enable_alerts, bool) \ field(use_ext_clk, bool) \ field(ft_device_id, uint32_t, 4) UJSON_SERDE_STRUCT(ManufFtIndividualizeData, \ diff --git a/sw/device/silicon_creator/manuf/base/BUILD b/sw/device/silicon_creator/manuf/base/BUILD index bd771b1ea724c..5078a0b4a09ca 100644 --- a/sw/device/silicon_creator/manuf/base/BUILD +++ b/sw/device/silicon_creator/manuf/base/BUILD @@ -193,10 +193,12 @@ opentitan_test( linker_script = "//sw/device/silicon_creator/manuf/lib:sram_program_linker_script", deps = [ ":flash_info_permissions", + "//hw/top_earlgrey:alert_handler_c_regs", "//hw/top_earlgrey/sw/autogen:top_earlgrey", "//sw/device/lib/arch:device", "//sw/device/lib/base:abs_mmio", "//sw/device/lib/base:macros", + "//sw/device/lib/dif:alert_handler", "//sw/device/lib/dif:clkmgr", "//sw/device/lib/dif:flash_ctrl", "//sw/device/lib/dif:otp_ctrl", @@ -209,9 +211,12 @@ opentitan_test( "//sw/device/lib/testing:pinmux_testutils", "//sw/device/lib/testing/test_framework:check", "//sw/device/lib/testing/test_framework:ottf_console", + "//sw/device/lib/testing/test_framework:ottf_isrs", "//sw/device/lib/testing/test_framework:ottf_test_config", "//sw/device/lib/testing/test_framework:status", "//sw/device/lib/testing/test_framework:ujson_ottf", + "//sw/device/silicon_creator/lib/drivers:ibex", + "//sw/device/silicon_creator/lib/drivers:rstmgr", "//sw/device/silicon_creator/manuf/lib:flash_info_fields", "//sw/device/silicon_creator/manuf/lib:individualize", "//sw/device/silicon_creator/manuf/lib:otp_fields", diff --git a/sw/device/silicon_creator/manuf/base/sram_ft_individualize.c b/sw/device/silicon_creator/manuf/base/sram_ft_individualize.c index 1c61be2b57994..e087a8b944485 100644 --- a/sw/device/silicon_creator/manuf/base/sram_ft_individualize.c +++ b/sw/device/silicon_creator/manuf/base/sram_ft_individualize.c @@ -6,6 +6,7 @@ #include "sw/device/lib/arch/device.h" #include "sw/device/lib/base/abs_mmio.h" +#include "sw/device/lib/dif/dif_alert_handler.h" #include "sw/device/lib/dif/dif_clkmgr.h" #include "sw/device/lib/dif/dif_flash_ctrl.h" #include "sw/device/lib/dif/dif_otp_ctrl.h" @@ -20,18 +21,22 @@ #include "sw/device/lib/testing/test_framework/ottf_console.h" #include "sw/device/lib/testing/test_framework/ottf_test_config.h" #include "sw/device/lib/testing/test_framework/ujson_ottf.h" +#include "sw/device/silicon_creator/lib/drivers/ibex.h" +#include "sw/device/silicon_creator/lib/drivers/rstmgr.h" #include "sw/device/silicon_creator/manuf/base/flash_info_permissions.h" #include "sw/device/silicon_creator/manuf/lib/flash_info_fields.h" #include "sw/device/silicon_creator/manuf/lib/individualize.h" #include "sw/device/silicon_creator/manuf/lib/individualize_sw_cfg.h" #include "sw/device/silicon_creator/manuf/lib/otp_fields.h" +#include "alert_handler_regs.h" // Generated. #include "hw/top_earlgrey/sw/autogen/top_earlgrey.h" OTTF_DEFINE_TEST_CONFIG(.console.type = kOttfConsoleSpiDevice, .console.base_addr = TOP_EARLGREY_SPI_DEVICE_BASE_ADDR, .console.test_may_clobber = false, ); +static dif_alert_handler_t alert_handler; static dif_clkmgr_t clkmgr; static dif_flash_ctrl_state_t flash_ctrl_state; static dif_otp_ctrl_t otp_ctrl; @@ -45,10 +50,25 @@ static uint32_t ast_cfg_data[kFlashInfoAstCalibrationDataSizeIn32BitWords]; // This is used to delay further action when the switch happens. static const int kSettleDelayMicros = 200; +static size_t alert_nmi_count = 0; + +/** + * Handle NMI's from the alert escalation mechanism. + * + * @param exc_info Execution info. + */ +void ottf_external_nmi_handler(uint32_t *exc_info) { + LOG_INFO("Processing Alert NMI %d ...", alert_nmi_count++); + ibex_clear_nmi(kIbexNmiSourceAlert); +} + /** * Initializes all DIF handles used in this SRAM program. */ static status_t peripheral_handles_init(void) { + TRY(dif_alert_handler_init( + mmio_region_from_addr(TOP_EARLGREY_ALERT_HANDLER_BASE_ADDR), + &alert_handler)); TRY(dif_clkmgr_init(mmio_region_from_addr(TOP_EARLGREY_CLKMGR_AON_BASE_ADDR), &clkmgr)); TRY(dif_flash_ctrl_init_state( @@ -105,6 +125,61 @@ static bool did_extclk_settle(const dif_clkmgr_t *clkmgr) { return status; } +static status_t configure_all_alerts(void) { + // Enable capturing both alert and CPU crashdump info collection. + rstmgr_alert_info_enable(); + rstmgr_cpu_info_enable(); + + // Enable all alerts and configure them in class A. + for (size_t i = 0; i < ALERT_HANDLER_PARAM_N_ALERTS; ++i) { + TRY(dif_alert_handler_configure_alert( + &alert_handler, (dif_alert_handler_alert_t)i, kDifAlertHandlerClassA, + /*enabled=*/kDifToggleEnabled, + /*locked=*/kDifToggleDisabled)); + } + + // Enable all local alerts and configure them in class A. + for (size_t i = 0; i < ALERT_HANDLER_PARAM_N_LOC_ALERT; ++i) { + TRY(dif_alert_handler_configure_local_alert( + &alert_handler, (dif_alert_handler_local_alert_t)i, + kDifAlertHandlerClassA, + /*enabled=*/kDifToggleEnabled, + /*locked=*/kDifToggleDisabled)); + } + + // Configure class A alert escalation behavior. + const dif_alert_handler_escalation_phase_t kEscPhases[2] = { + { + .phase = kDifAlertHandlerClassStatePhase0, + .signal = 0, // NMI + .duration_cycles = 1000000, + }, + { + .phase = kDifAlertHandlerClassStatePhase1, + .signal = 3, // SW Reset + .duration_cycles = 1000, + }, + }; + dif_alert_handler_class_config_t alert_class_a_config = { + .auto_lock_accumulation_counter = kDifToggleDisabled, + .accumulator_threshold = 0, // A single alert will trigger escalation. + .irq_deadline_cycles = 0, // Disabled. + .escalation_phases = kEscPhases, + .escalation_phases_len = ARRAYSIZE(kEscPhases), + .crashdump_escalation_phase = kDifAlertHandlerClassStatePhase0, + + }; + TRY(dif_alert_handler_configure_class(&alert_handler, kDifAlertHandlerClassA, + alert_class_a_config, + /*enabled=*/kDifToggleEnabled, + /*locked=*/kDifToggleDisabled)); + + // Enable NMIs from alert escalation. + ibex_enable_nmi(kIbexNmiSourceAlert); + + return OK_STATUS(); +} + /** * Provision OTP {CreatorSw,OwnerSw,Hw}Cfg and RotCreatorAuth{Codesign,State} * partitions. @@ -117,6 +192,12 @@ static status_t provision(ujson_t *uj) { LOG_INFO("Waiting for FT SRAM provisioning data ..."); TRY(ujson_deserialize_manuf_ft_individualize_data_t(uj, &in_data)); + // Enable all alerts (and alert NMIs) if requested. + if (in_data.enable_alerts) { + TRY(configure_all_alerts()); + LOG_INFO("All alerts and alert NMI enabled."); + } + // Enable external clock on silicon platforms if requested. if (kDeviceType == kDeviceSilicon && in_data.use_ext_clk) { CHECK_DIF_OK(dif_clkmgr_external_clock_set_enabled(&clkmgr, @@ -149,6 +230,15 @@ bool test_main(void) { ottf_console_init(); ujson_t uj = ujson_ottf_console(); + // Abort if reset reason is due to an escalation. + if (rstmgr_reason_get() == kRstmgrReasonEscalation) { + LOG_INFO("Reset due to alert escalation ... aborting."); + abort(); + } + + // Clear the reset reasons. + rstmgr_reason_clear(UINT8_MAX); + // Read and log flash and AST data to console (for manual verification // purposes), and perform provisioning operations. CHECK_STATUS_OK(read_and_print_flash_and_ast_data()); diff --git a/sw/host/provisioning/ft/src/main.rs b/sw/host/provisioning/ft/src/main.rs index 6c7ca3545b21e..767911383b9b6 100644 --- a/sw/host/provisioning/ft/src/main.rs +++ b/sw/host/provisioning/ft/src/main.rs @@ -40,6 +40,9 @@ pub struct ManufFtProvisioningDataInput { #[arg(long)] pub ft_device_id: String, + #[arg(long)] + pub enable_alerts_during_individualize: bool, + #[arg(long)] pub use_ext_clk_during_individualize: bool, @@ -156,6 +159,7 @@ fn main() -> Result<()> { // The FT device ID is sent to the DUT in little endian order. ft_device_id.reverse(); let ft_individualize_data_in = ManufFtIndividualizeData { + enable_alerts: opts.provisioning_data.enable_alerts_during_individualize, use_ext_clk: opts.provisioning_data.use_ext_clk_during_individualize, ft_device_id, }; diff --git a/sw/host/provisioning/ft_lib/src/lib.rs b/sw/host/provisioning/ft_lib/src/lib.rs index a0e38ce20bf8b..3e3c70bd3e215 100644 --- a/sw/host/provisioning/ft_lib/src/lib.rs +++ b/sw/host/provisioning/ft_lib/src/lib.rs @@ -6,9 +6,10 @@ use sha2::{Digest, Sha256}; use std::collections::HashMap; use std::collections::HashSet; use std::path::PathBuf; +use std::thread; use std::time::{Duration, Instant}; -use anyhow::{bail, Result}; +use anyhow::{anyhow, bail, Result}; use arrayvec::ArrayVec; use zerocopy::AsBytes; @@ -18,6 +19,9 @@ use opentitanlib::app::TransportWrapper; use opentitanlib::console::spi::SpiConsoleDevice; use opentitanlib::dif::lc_ctrl::{DifLcCtrlState, LcCtrlReg}; use opentitanlib::io::jtag::{JtagParams, JtagTap}; +use opentitanlib::test_utils::crashdump::{ + read_alert_crashdump_data, read_cpu_crashdump_data, read_reset_reason, +}; use opentitanlib::test_utils::init::InitializeTest; use opentitanlib::test_utils::lc_transition::trigger_lc_transition; use opentitanlib::test_utils::load_sram_program::{ @@ -112,13 +116,30 @@ pub fn run_sram_ft_individualize( // Inject provisioning data into the device. ft_individualize_data_in.send(spi_console)?; - // Wait for provisioning operations to complete. - let _ = UartConsole::wait_for(spi_console, r"FT SRAM provisioning done.", timeout)?; - - jtag.disconnect()?; - transport.pin_strapping("PINMUX_TAP_RISCV")?.remove()?; - - Ok(()) + // Wait for provisioning operations to complete. If we see at least 10 NMIs, we know it is time + // to capture crashdump information. + let console_text = UartConsole::wait_for( + spi_console, + r"FT SRAM provisioning done.|Processing Alert NMI 10 ...", + timeout, + )?; + match console_text[0].as_str() { + "FT SRAM provisioning done." => { + jtag.disconnect()?; + transport.pin_strapping("PINMUX_TAP_RISCV")?.remove()?; + Ok(()) + } + "Processing Alert NMI 10 ..." => { + transport.pin_strapping("PINMUX_TAP_RISCV")?.remove()?; + jtag.disconnect()?; + thread::sleep(Duration::from_millis(10000)); + read_reset_reason(transport, jtag_params)?; + read_cpu_crashdump_data(transport, jtag_params)?; + read_alert_crashdump_data(transport, jtag_params)?; + Ok(()) + } + _ => Err(anyhow!("Unexpected console_text: {:?}", console_text)), + } } pub fn test_exit( diff --git a/sw/host/provisioning/orchestrator/src/orchestrator.py b/sw/host/provisioning/orchestrator/src/orchestrator.py index 942e47b0c4a99..253f9621aa179 100644 --- a/sw/host/provisioning/orchestrator/src/orchestrator.py +++ b/sw/host/provisioning/orchestrator/src/orchestrator.py @@ -105,6 +105,12 @@ def main(args_in): action="store_true", help="If set, the FPGA bitsream will not be cleared before CP.", ) + parser.add_argument( + "--enable-alerts", + action="store_true", + default=False, + help="Enable all alerts during FT individualize step.", + ) parser.add_argument( "--use-ext-clk", action="store_true", @@ -167,6 +173,7 @@ def main(args_in): test_exit_token=args.test_exit_token, fpga=args.fpga, fpga_dont_clear_bitstream=args.fpga_dont_clear_bitstream, + enable_alerts=args.enable_alerts, use_ext_clk=args.use_ext_clk, require_confirmation=not args.non_interactive) dut.run_cp() diff --git a/sw/host/provisioning/orchestrator/src/ot_dut.py b/sw/host/provisioning/orchestrator/src/ot_dut.py index 374eb9d7dc7f2..1362e31ad2a33 100644 --- a/sw/host/provisioning/orchestrator/src/ot_dut.py +++ b/sw/host/provisioning/orchestrator/src/ot_dut.py @@ -51,6 +51,7 @@ class OtDut(): test_exit_token: str fpga: str fpga_dont_clear_bitstream: bool + enable_alerts: bool use_ext_clk: bool require_confirmation: bool = True @@ -271,7 +272,11 @@ def run_ft(self) -> None: --token-encrypt-key-der-file={self.sku_config.token_encrypt_key} \ """ - # Enable external clock if requested. + # Enable alerts during individualization if requested. + if self.enable_alerts: + cmd += " --enable-alerts-during-individualize" + + # Enable external clock during individualization if requested. if self.use_ext_clk: cmd += " --use-ext-clk-during-individualize" diff --git a/sw/host/provisioning/orchestrator/tests/e2e.sh b/sw/host/provisioning/orchestrator/tests/e2e.sh index 9bc5f7107894d..0eb2302ba32e7 100755 --- a/sw/host/provisioning/orchestrator/tests/e2e.sh +++ b/sw/host/provisioning/orchestrator/tests/e2e.sh @@ -28,6 +28,7 @@ $PYTHON ${ORCHESTRATOR_PATH} \ --test-unlock-token="0x11111111_11111111_11111111_11111111" \ --test-exit-token="0x22222222_22222222_22222222_22222222" \ --fpga=hyper310 \ + --enable-alerts \ --use-ext-clk \ --non-interactive \ --db-path=$TEST_TMPDIR/registry.sqlite