whnvmtool hex file support

Author: bigbrett

docs/src/6-Utilities.md

@@ -16,9 +16,9 @@ This chapter describes the auxiliary tools that ship alongside the wolfHSM clien
 
 ## NVM Provisioning Tool
 
-The NVM provisioning tool (`tools/whnvmtool/`) is a host-side utility that builds a pre-populated wolfHSM NVM image from a configuration file. It is intended for device provisioning: rather than having the server populate its NVM at runtime, the integrator describes the desired initial contents — a set of NVM objects and keys, each with its metadata ID, access permissions, flags, label, and a path to the binary payload — and the tool produces a single image file that can be programmed into the device's flash at manufacture or used in place to back a `whNvmFlash` provider in simulation. Currently the tool targets the `whNvmFlash` provider; the generated image is binary, and can be converted to Intel HEX with the standard `objcopy` workflow for use with automated programmers.
+The NVM provisioning tool (`tools/whnvmtool/`) is a host-side utility that builds a pre-populated wolfHSM NVM image from a configuration file. It is intended for device provisioning: rather than having the server populate its NVM at runtime, the integrator describes the desired initial contents — a set of NVM objects and keys, each with its metadata ID, access permissions, flags, label, and a path to the binary payload — and the tool produces a single image file that can be programmed into the device's flash. Currently the tool targets the `whNvmFlash` provider. By default, the tool creates a contiguous binary image, but can also emit an Intel HEX file (`--hex`, with `--hex-base` and `--hex-align`) for use with automated programmers. In hex mode, the HEX file contains only the bytes actually programmed while building the image, leaving erased regions absent so flash programmers do not program the erased value.
 
-Because the on-flash layout depends on build-time configuration, the tool must be compiled against the same wolfHSM version as the target server and with a matching `WOLFHSM_CFG_NVM_OBJECT_COUNT` and `WOLFHSM_CFG_NVM_FLASH_CRC16` setting, and the `--size` argument must match the server's `whNvmFlash` partition size. For the full configuration file schema, command-line options, hex conversion recipe, and test workflow, see [`tools/whnvmtool/README.md`](https://github.com/wolfSSL/wolfHSM/blob/main/tools/whnvmtool/README.md).
+Because the on-flash layout depends on build-time configuration, the tool must be compiled against the same wolfHSM version as the target server and with a matching `WOLFHSM_CFG_NVM_OBJECT_COUNT` and `WOLFHSM_CFG_NVM_FLASH_CRC16` setting, and the `--size` argument must match the server's `whNvmFlash` partition size. For the full configuration file schema, command-line options, Intel HEX output details, and test workflow, see [`tools/whnvmtool/README.md`](https://github.com/wolfSSL/wolfHSM/blob/main/tools/whnvmtool/README.md).
 
 ## Benchmark Suite
 

tools/whnvmtool/Makefile

@@ -111,10 +111,21 @@ all: $(OUT)
 $(OUT): $(SRC)
 	$(CC) $(CFLAGS) $(CFLAGS_EXTRA) $(SRC) -o $(OUT) $(LDFLAGS)
 
-# Generate the test NVM image
+# Generate the test NVM images: the main test image (erased byte 0x00) with
+# a hex file whose base address forces a record split at a 64KB boundary,
+# a default-erased-byte image whose hex file uses a high base address and
+# alignment, and a merge image whose alignment is large enough that the
+# expanded directory and data ranges overlap and must be merged into one.
+# The hex parameters must match the expectations in test/test_whnvmtool.c
 test-gen: $(OUT)
-	rm -f whNvmImage.bin whNvmImage.hex
-	./$(OUT) --test test/nvminit/test.nvminit --invert-erased-byte
+	rm -f whNvmImage.bin whNvmImage.hex whNvmImageFF.bin whNvmImageFF.hex \
+		whNvmImageMerge.bin whNvmImageMerge.hex
+	./$(OUT) --test test/nvminit/test.nvminit --invert-erased-byte \
+		--hex --hex-base 0xFFF8
+	./$(OUT) --image=whNvmImageFF.bin --hex=whNvmImageFF.hex \
+		--hex-base 0x0800FFF0 --hex-align 16 test/nvminit/test.nvminit
+	./$(OUT) --image=whNvmImageMerge.bin --hex=whNvmImageMerge.hex \
+		--hex-base 0x0800F800 --hex-align 0x800 test/nvminit/test.nvminit
 
 # Run the negative tests, where invalid configs and key files must fail the tool.
 # Order-only prereq on test-gen: every whnvmtool instance binds the same TCP
@@ -145,7 +156,8 @@ scan:
 	            --status-bugs $(MAKE) analyze 2> $(WOLFHSM_DIR)/scan_out/$(SCAN_LOG)
 
 clean: clean-test
-	rm -f whNvmImage.bin whNvmImage.hex
+	rm -f whNvmImage.bin whNvmImage.hex whNvmImageFF.bin whNvmImageFF.hex
+	rm -f whNvmImageMerge.bin whNvmImageMerge.hex
 	rm -f $(OUT)
 
 clean-test:

tools/whnvmtool/README.md

@@ -11,14 +11,17 @@ Currently, `whnvmtool` only supports the `whNvmFlash` provider.
 ## Usage
 
 ```
-./whnvmtool [--test] [--image[=<file>]] [--size <size>] [--invert-erased-byte] <config-file>
+./whnvmtool [--test] [--image[=<file>]] [--size <size>] [--invert-erased-byte] [--hex[=<file>]] [--hex-base <addr>] [--hex-align <n>] <config-file>
 ```
 
 where:
 
 - `--image[=<file>]`: Specifies the output NVM image file. If not provided, defaults to `whNvmImage.bin`.
 - `--size <size>`: Sets the partition size for the NVM image. Can be specified in decimal or hexadecimal (with '0x' prefix).
 - `--invert-erased-byte`: Inverts the erased byte value (default is 0xFF, this option changes it to 0x00).
+- `--hex[=<file>]`: Additionally generates an Intel HEX file containing only the programmed bytes of the image (see [Generating an Intel HEX File](#generating-an-intel-hex-file)). If no file is given, defaults to `whNvmImage.hex`.
+- `--hex-base <addr>`: Base address added to every record address in the Intel HEX file, corresponding to the address of the NVM partition in the device's memory map. Defaults to 0x0. Requires `--hex`.
+- `--hex-align <n>`: Expands each Intel HEX record range to `n`-byte boundaries, for devices that program flash in multi-byte units. Defaults to 1. Requires `--hex`, and `--hex-base` must be a multiple of `n`.
 - `--test`: Enables test mode. In this mode, the tool generates an intermediate file (`nvm_metadata.txt`) containing comma separated metadata ID/file path pairs, associating each object ID with the file path containing the object's original data. This option is used by the `whnvmtool` tests to verify the contents of the generated NVM image, and is not required for normal operation.
 
 ## Configuration File Schema
@@ -137,24 +140,25 @@ The generated NVM image is a binary file that can be used to initialize an insta
 2. The partition size specified for the NVM image must match that of the server's `whNvmFlash` provider
 3. If using a real flash implementation, the binary NVM image must be programmed to the correct address
 
-### Generating a Hex File
+### Generating an Intel HEX File
 
-Users may find it useful to generate a hex file to program the NVM image into device memory. This can be accomplished by using the `objcopy` utility to convert the generated NVM image to a hex file, making sure to specify the correct offset into the image for the start of the NVM partition. For example:
+Passing `--hex[=<file>]` generates an Intel HEX version of the NVM image alongside the binary, ready to be loaded by a flash programmer:
 
 ```
-objcopy -I binary -O ihex --change-address <offset> <input-file> <output-file>
+./whnvmtool --hex --hex-base 0x08040000 my.nvminit
 ```
 
-where:
+The HEX file contains only the bytes actually programmed while building the image: the partition state, the used directory entries, and the object data.
+
+- `--hex-base <addr>` sets the base address of the NVM partition in the device's memory map. Record addresses in the HEX file are the image offsets plus this value. Without it, records start at address 0x0.
+- `--hex-align <n>` expands each record range outward to `n`-byte boundaries, padding with the surrounding image bytes. Use this when the target flash programs in multi-byte units (for example ECC words), so every emitted range covers whole program units. The base address must be a multiple of `n` for this to hold in the device address space, so the tool rejects a `--hex-base` that is not. The default of 1 emits exactly the programmed ranges, which are already multiples of the 8-byte wolfHSM flash unit.
 
-- `<offset>` is the offset to the base address that will be applied to the generated hex file. Without this option, the offset is 0x0, so automated programming tools will attempt to load the hex file starting at address 0x0, which is likely not the desired behavior. The `<offset>` parameter should correspond to the base address of the NVM partition used by wolfHSM in the device's address space.
-- `<input-file>` is the NVM image file generated by `whnvmtool`
-- `<output-file>` is the name of the output hex file
+Because only the bytes programmed during the current run are tracked, `--hex` requires building a fresh image: the tool exits with an error if the `--image` file already exists.
 
 ## Testing
 
 Tests for `whnvmtool` can be run by invoking `make check` or `make test`. This will perform the following steps:
 
-1. Invoke `whnvmtool` to generate an NVM image using an example configuration file, using the `--test` option to export the ID/data file pairs to a file
-2. Run the negative test script `test/test_invalid_input.sh`, which verifies that invalid configuration files (out-of-range clientId, keyId, SHE slot, counter, or flags, or the volatile `RAM_KEY` slot) and bad key files (wrong size or missing) cause the tool to exit with an error instead of producing an incomplete image
-3. Build and run a test program `test/test_whnvmtool.c`, which loads the generated NVM image and verifies the contents of the objects and keys using the exported ID/data file pairs. The compatibility of the generated image is verified by loading the image into two `whNvmFlash` providers: the POSIX port file-based NVM flash file simulator (`port/posix/posix_flash_file.c`) and the RAM-based NVM flash simulator (`src/wh_flash_ramsim.c`).
+1. Invoke `whnvmtool` to generate NVM images using an example configuration file, using the `--test` option to export the ID/data file pairs to a file and the `--hex` options to generate Intel HEX output for both erased byte values, plus a third image whose alignment is large enough that its expanded record ranges overlap and must be merged
+2. Run the negative test script `test/test_invalid_input.sh`, which verifies that invalid configuration files (out-of-range clientId, keyId, SHE slot, counter, or flags, or the volatile `RAM_KEY` slot), bad key files (wrong size or missing), and invalid hex option combinations cause the tool to exit with an error instead of producing an incomplete image
+3. Build and run a test program `test/test_whnvmtool.c`, which loads the generated NVM image and verifies the contents of the objects and keys using the exported ID/data file pairs. The compatibility of the generated image is verified by loading the image into two `whNvmFlash` providers: the POSIX port file-based NVM flash file simulator (`port/posix/posix_flash_file.c`) and the RAM-based NVM flash simulator (`src/wh_flash_ramsim.c`). The test program also parses the generated Intel HEX files and verifies that overlaying their records on an erased image reproduces the binary image exactly, that no record touches a region left erased (such as the inactive partition), that bytes programmed with the erased value are still present, that record ranges honor the requested base address and alignment, and that no two records cover the same byte.

tools/whnvmtool/test/test_invalid_input.sh

@@ -1,8 +1,8 @@
 #!/bin/sh
 #
-# Negative tests for whnvmtool. Each case runs the tool on an invalid config
-# or key file and requires a nonzero exit plus an error message, stopping at
-# the first unexpected result.
+# Negative tests for whnvmtool. Each case runs the tool on an invalid config,
+# key file, or command line and requires a nonzero exit plus an error message,
+# stopping at the first unexpected result.
 #
 # Usage: test_invalid_input.sh [path-to-whnvmtool]
 
@@ -24,15 +24,18 @@ head -c 16 /dev/zero > "$TMP/key16.bin"
 head -c 15 /dev/zero > "$TMP/key15.bin"
 head -c 65536 /dev/zero > "$TMP/big.bin"
 
-# Runs the tool on config $2. $1 is "pass" (require exit 0) or "fail"
+# Runs the tool on config $2, passing any further arguments after the
+# description through to the tool. $1 is "pass" (require exit 0) or "fail"
 # (require nonzero exit and an error message on stderr).
 run_case() {
     expect=$1
     cfg=$2
     desc=$3
+    shift 3
 
-    rm -f "$TMP/img.bin"
-    if "$TOOL" --image="$TMP/img.bin" "$cfg" >"$TMP/out.log" 2>"$TMP/err.log"
+    rm -f "$TMP/img.bin" "$TMP/img.hex"
+    if "$TOOL" --image="$TMP/img.bin" "$@" "$cfg" \
+        >"$TMP/out.log" 2>"$TMP/err.log"
     then
         rc=0
     else
@@ -109,4 +112,44 @@ run_case fail "$TMP/cfg" "missing key file fails the run"
 } > "$TMP/cfg"
 run_case fail "$TMP/cfg" "bad entry fails the run despite valid entries after it"
 
+# Hex output option validation
+echo "key 1 1 0 0 \"k\" $TMP/key16.bin" > "$TMP/cfg"
+run_case pass "$TMP/cfg" "--hex with base and align accepted" \
+    --hex="$TMP/img.hex" --hex-base 0x1000 --hex-align 8
+if [ ! -s "$TMP/img.hex" ]; then
+    echo "FAIL: --hex did not produce a hex file"
+    exit 1
+fi
+
+run_case fail "$TMP/cfg" "--hex-base without --hex rejected" --hex-base 0x1000
+run_case fail "$TMP/cfg" "--hex-align without --hex rejected" --hex-align 8
+run_case fail "$TMP/cfg" "--hex-align 0 rejected" \
+    --hex="$TMP/img.hex" --hex-align 0
+run_case fail "$TMP/cfg" "malformed hex base rejected" \
+    --hex="$TMP/img.hex" --hex-base zzz
+run_case fail "$TMP/cfg" "malformed hex alignment rejected" \
+    --hex="$TMP/img.hex" --hex-align zzz
+run_case fail "$TMP/cfg" "hex base overflowing the 32-bit space rejected" \
+    --hex="$TMP/img.hex" --hex-base 0xFFFFF000
+run_case fail "$TMP/cfg" "hex base not a multiple of hex alignment rejected" \
+    --hex="$TMP/img.hex" --hex-base 0x1003 --hex-align 8
+run_case fail "$TMP/cfg" "hex file equal to image file rejected" \
+    --hex="$TMP/img.bin"
+
+# A pre-existing image must be rejected with --hex: only bytes programmed
+# during the current run are tracked, so the hex file would be incomplete
+head -c 16 /dev/zero > "$TMP/img.bin"
+if "$TOOL" --image="$TMP/img.bin" --hex="$TMP/img.hex" "$TMP/cfg" \
+    >"$TMP/out.log" 2>"$TMP/err.log"
+then
+    echo "FAIL: pre-existing image accepted with --hex"
+    exit 1
+fi
+if ! grep -q "Error" "$TMP/err.log"; then
+    echo "FAIL: pre-existing image case exited without an error message"
+    cat "$TMP/err.log"
+    exit 1
+fi
+echo "PASS: pre-existing image with --hex rejected"
+
 echo "All whnvmtool negative tests passed"