Merge branch 'master' into client-pin-features

.github/workflows/cargo_fmt.yml

@@ -5,6 +5,7 @@ on:
       - 'examples/*.rs'
       - 'libraries/**/*.rs'
       - 'src/**/*.rs'
+      - 'tools/**/*.rs'
       - 'patches/**'
       - '**/Cargo.toml'
       - '.cargo/config'
@@ -46,3 +47,9 @@ jobs:
         with:
           command: fmt
           args: --manifest-path libraries/crypto/Cargo.toml --all -- --check
+
+      - name: Cargo format tools/heapviz
+        uses: actions-rs/cargo@v1
+        with:
+          command: fmt
+          args: --manifest-path tools/heapviz/Cargo.toml --all -- --check

.github/workflows/heapviz_test.yml

@@ -0,0 +1,34 @@
+---
+name: Heapviz tool tests
+on:
+  push:
+    paths:
+      - 'tools/heapviz/**'
+  pull_request:
+    types: [opened, synchronize, reopened]
+
+jobs:
+  heapviz_test:
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install ncurses
+        run: sudo apt-get install libncurses-dev
+
+      - name: Check heapviz tool
+        uses: actions-rs/cargo@v1
+        with:
+          command: check
+          args: --manifest-path tools/heapviz/Cargo.toml
+
+      - name: Unit testing of heapviz tool (debug mode)
+        uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: --manifest-path tools/heapviz/Cargo.toml
+
+      - name: Unit testing of heapviz tool (release mode)
+        uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: --manifest-path tools/heapviz/Cargo.toml --release

.github/workflows/python.yml

@@ -22,7 +22,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip setuptools wheel
-          pip install 'tockloader~=1.4' pylint
+          pip install 'tockloader==1.4' pylint
       - name: Register matcher
         run: echo ::add-matcher::./.github/python_matcher.json
       - name: Test code with pylint

README.md

@@ -122,6 +122,70 @@ All the required files can be downloaded from
 file, allowing you to easily make the modifications you need to further
 customize it.
 
+## Development and testing
+
+### Printing panic messages to the console
+
+By default, libtock-rs blinks some LEDs when the userspace application panicks.
+This is not always convenient as the panic message is lost. In order to enable
+a custom panic handler that first writes the panic message via Tock's console
+driver, before faulting the app, you can use the `--panic-console` flag of the
+`deploy.py` script.
+
+```shell
+# Example on Nordic nRF52840-DK board
+./deploy.py --board=nrf52840dk --opensk --panic-console
+```
+
+### Debugging memory allocations
+
+You may want to track memory allocations to understand the heap usage of
+OpenSK. This can be useful if you plan to port it to a board with fewer
+available RAM for example. To do so, you can enable the `--debug-allocations`
+flag of the `deploy.py` script. This enables a custom (userspace) allocator
+that prints a message to the console for each allocation and deallocation
+operation.
+
+The additional output looks like the following.
+
+```
+# Allocation of 256 byte(s), aligned on 1 byte(s). The allocated address is
+# 0x2002401c. After this operation, 2 pointers have been allocated, totalling
+# 384 bytes (the total heap usage may be larger, due to alignment and
+# fragmentation of allocations within the heap).
+alloc[256, 1] = 0x2002401c (2 ptrs, 384 bytes)
+# Deallocation of 64 byte(s), aligned on 1 byte(s), from address 0x2002410c.
+# After this operation, 1 pointers are allocated, totalling 512 bytes.
+dealloc[64, 1] = 0x2002410c (1 ptrs, 512 bytes)
+```
+
+A tool is provided to analyze such reports, in `tools/heapviz`. This tool
+parses the console output, identifies the lines corresponding to (de)allocation
+operations, and first computes some statistics:
+
+- Address range used by the heap over this run of the program,
+- Peak heap usage (how many useful bytes are allocated),
+- Peak heap consumption (how many bytes are used by the heap, including
+  unavailable bytes between allocated blocks, due to alignment constraints and
+  memory fragmentation),
+- Fragmentation overhead (difference between heap consumption and usage).
+
+Then, the `heapviz` tool displays an animated "movie" of the allocated bytes in
+heap memory. Each frame in this "movie" shows bytes that are currently
+allocated, that were allocated but are now freed, and that have never been
+allocated. A new frame is generated for each (de)allocation operation. This tool
+uses the `ncurses` library, that you may have to install beforehand.
+
+You can control the tool with the following parameters:
+- `--logfile` (required) to provide the file which contains the console output
+  to parse,
+- `--fps` (optional) to customize the number of frames per second in the movie
+  animation.
+
+```shell
+cargo run --manifest-path tools/heapviz/Cargo.toml -- --logfile console.log --fps 50
+```
+
 ## Contributing
 
 See [Contributing.md](docs/contributing.md).

reproduce_hashes.sh

@@ -35,3 +35,6 @@ done
 echo "Computing SHA-256 sum of the TAB file..."
 ./third_party/tock/tools/sha256sum/target/debug/sha256sum target/tab/ctap2.tab >> reproducible/binaries.sha256sum
 tar -rvf reproducible/reproduced.tar target/tab/ctap2.tab
+
+tar -rvf reproducible/reproduced.tar reproducible/elf2tab.txt
+tar -rvf reproducible/reproduced.tar reproducible/binaries.sha256sum

run_desktop_tests.sh

@@ -23,9 +23,16 @@ cd ../..
 cd libraries/crypto
 cargo fmt --all -- --check
 cd ../..
+cd tools/heapviz
+cargo fmt --all -- --check
+cd ../..
 
 echo "Building sha256sum tool..."
 cargo build --manifest-path third_party/tock/tools/sha256sum/Cargo.toml
+echo "Checking that heapviz tool builds properly..."
+cargo build --manifest-path tools/heapviz/Cargo.toml
+echo "Testing heapviz tool..."
+cargo test --manifest-path tools/heapviz/Cargo.toml
 
 echo "Checking that CTAP2 builds properly..."
 cargo check --release --target=thumbv7em-none-eabi

setup.sh

@@ -86,7 +86,7 @@ source tools/gen_key_materials.sh
 generate_crypto_materials N
 
 rustup install $(head -n 1 rust-toolchain)
-pip3 install --user --upgrade 'tockloader~=1.4' six intelhex
+pip3 install --user --upgrade 'tockloader==1.4' six intelhex
 rustup target add thumbv7em-none-eabi
 
 # Install dependency to create applications.

tools/heapviz/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "heapviz"
+version = "0.1.0"
+authors = [
+  "Guillaume Endignoux <guillaumee@google.com>",
+]
+license = "Apache-2.0"
+edition = "2018"
+
+[dependencies]
+clap = "2.33.1"
+lazy_static = "1.4.0"
+ncurses = "5.99.0"
+regex = "1"

tools/heapviz/src/main.rs

@@ -0,0 +1,217 @@
+// Copyright 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use clap::{App, Arg};
+use lazy_static::lazy_static;
+use regex::Regex;
+use std::fs::File;
+use std::io::{BufRead, BufReader, Read, Write};
+use std::thread::sleep;
+use std::time::Duration;
+
+/// Configuration, built from CLI parameters.
+struct Config {
+    /// Handle to the log file containing allocation operations.
+    logfile: File,
+    /// Number of allocation operations to show per second.
+    fps: u64,
+}
+
+fn parse_cli() -> Config {
+    let matches = App::new("Heap visualizer")
+                          .version("0.1")
+                          .author("Guillaume Endignoux <guillaumee@google.com>")
+                          .about("Tool to visualize heap usage of libtock-rs applications")
+                          .arg(Arg::with_name("logfile")
+                               .short("f")
+                               .long("logfile")
+                               .value_name("FILE")
+                               .help("Log file containing allocation info (deploy OpenSK with --debug-allocations to obtain it)")
+                               .takes_value(true)
+                               .required(true))
+                          .arg(Arg::with_name("fps")
+                               .long("fps")
+                               .value_name("FPS")
+                               .help("Number of allocation operations to show per second")
+                               .takes_value(true)
+                               .default_value("20"))
+                          .get_matches();
+
+    let logpath = matches.value_of("logfile").unwrap();
+    let fps = matches
+        .value_of("fps")
+        .unwrap()
+        .parse::<u64>()
+        .expect("The --fps parameter must be an integer");
+    let logfile = File::open(logpath).expect("Couldn't open --logfile for reading");
+
+    Config { logfile, fps }
+}
+
+/// An allocation or deallocation event.
+#[cfg_attr(test, derive(Debug, PartialEq))]
+struct Event {
+    /// Whether this even is an allocation (true) or a deallocation (false).
+    is_alloc: bool,
+    /// The start address of the (de)allocated block, in bytes.
+    start: usize,
+    /// The length of the (de)allocated block, in bytes.
+    len: usize,
+}
+
+fn parse_event(line: &str) -> Option<Event> {
+    // The following regex matches lines looking like the following from OpenSK's output. Such lines
+    // are printed to the console when the `--debug-allocations` feature is enabled in the deploy
+    // script.
+    //
+    // ```
+    // alloc[256, 1] = 0x2002401c (2 ptrs, 384 bytes)
+    // dealloc[64, 1] = 0x2002410c (1 ptrs, 512 bytes)
+    // ```
+    //
+    // The two integers between square brackets after the (de)alloc keywords represent the length
+    // and alignement of the allocated block, respectively. The integer serialized in hexadecimal
+    // after the equal sign represents the starting address of the allocated block. The two
+    // integers within parentheses represent statistics about the total number of allocated blocks
+    // and the total number of allocated bytes after the (de)allocation operation, respectively.
+    //
+    // This regex captures three elements, in this order.
+    // - The keyword to know whether this operation is an allocation or a deallocation.
+    // - The length of the allocated block.
+    // - The starting address of the allocated block.
+    lazy_static! {
+        static ref RE: Regex =
+            Regex::new(r"^(alloc|dealloc)\[(\d+), \d+\] = 0x([0-9a-f]+) \(\d+ ptrs, \d+ bytes\)$")
+                .unwrap();
+    }
+
+    RE.captures(line).map(|caps| {
+        let typ = caps.get(1).unwrap().as_str();
+        let len = caps.get(2).unwrap().as_str().parse::<usize>().unwrap();
+        let start = usize::from_str_radix(&caps.get(3).unwrap().as_str(), 16).unwrap();
+        Event {
+            is_alloc: typ == "alloc",
+            start,
+            len,
+        }
+    })
+}
+
+fn main() {
+    let config = parse_cli();
+
+    let mut events = Vec::new();
+    for line in BufReader::new(config.logfile).lines() {
+        if let Some(event) = parse_event(&line.unwrap()) {
+            events.push(event);
+        }
+    }
+
+    let count_alloc = events.iter().filter(|e| e.is_alloc).count();
+    let count_dealloc = events.len() - count_alloc;
+    let start = events.iter().map(|e| e.start).min().unwrap_or(0);
+    let end = events.iter().map(|e| e.start + e.len).max().unwrap_or(0);
+    let mut usage = 0;
+    let peak = events
+        .iter()
+        .map(|e| {
+            if e.is_alloc {
+                usage += e.len;
+            } else {
+                usage -= e.len;
+            }
+            usage
+        })
+        .max()
+        .unwrap_or(0);
+    let len = end - start;
+    println!(
+        "Observed {} allocations and {} deallocations",
+        count_alloc, count_dealloc
+    );
+    println!("Start address: {:08x}", start);
+    println!("End address: {:08x}", end);
+    println!("Peak usage: {0} = {0:08x} bytes", peak);
+    println!("Peak consumption: {0} = {0:08x} bytes", len);
+    println!("Fragmentation overhead: {0} = {0:08x} bytes", len - peak);
+
+    print!("\nPress ENTER to start the visualization...");
+    std::io::stdout().flush().unwrap();
+    // Wait for ENTER, by reading a single byte and discarding it.
+    let _ = std::io::stdin().lock().read(&mut [0u8]).unwrap();
+
+    let window = ncurses::initscr();
+    ncurses::cbreak();
+    ncurses::noecho();
+    ncurses::intrflush(window, false);
+    ncurses::curs_set(ncurses::CURSOR_VISIBILITY::CURSOR_INVISIBLE);
+
+    let width = ncurses::getmaxx(window) as usize;
+
+    for e in events.iter() {
+        let position = e.start - start;
+        ncurses::wmove(window, (position / width) as i32, (position % width) as i32);
+
+        let mut s = Vec::with_capacity(e.len);
+        if e.is_alloc {
+            s.resize(e.len, b'#');
+        } else {
+            s.resize(e.len, b'.');
+        }
+        ncurses::addstr(std::str::from_utf8(s.as_slice()).unwrap());
+        ncurses::refresh();
+        sleep(Duration::from_nanos(1_000_000_000 / config.fps));
+    }
+
+    ncurses::endwin();
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_parse_event_alloc() {
+        assert_eq!(
+            parse_event("alloc[256, 1] = 0x2002401c (2 ptrs, 384 bytes)"),
+            Some(Event {
+                is_alloc: true,
+                start: 0x2002401c,
+                len: 256,
+            })
+        );
+    }
+
+    #[test]
+    fn test_parse_event_dealloc() {
+        assert_eq!(
+            parse_event("dealloc[64, 1] = 0x2002410c (1 ptrs, 512 bytes)"),
+            Some(Event {
+                is_alloc: false,
+                start: 0x2002410c,
+                len: 64,
+            })
+        );
+    }
+
+    #[test]
+    fn test_parse_event_none() {
+        assert_eq!(
+            parse_event(
+                "NRF52 HW INFO: Variant: AAD0, Part: N52840, Package: QI, Ram: K256, Flash: K1024"
+            ),
+            None
+        );
+    }
+}