From 4903232c4258a7b68fb19ffbd154db1c6e28a1c7 Mon Sep 17 00:00:00 2001 From: Guillaume Endignoux Date: Tue, 28 Apr 2020 13:51:03 +0200 Subject: [PATCH 01/17] Add a --verbose-build option to the deploy script. --- deploy.py | 42 +++++++++++++++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/deploy.py b/deploy.py index bb1c056..c198ba1 100755 --- a/deploy.py +++ b/deploy.py @@ -269,6 +269,19 @@ class OpenSKInstaller: port=None, ) + def checked_command(self, cmd, env=None, cwd=None): + stdout = None if self.args.verbose_build else subprocess.DEVNULL + try: + subprocess.run( + cmd, + stdout=stdout, + timeout=None, + check=True, + env=env, + cwd=cwd) + except subprocess.CalledProcessError as e: + fatal("Failed to execute {}: {}".format(cmd[0], str(e))) + def checked_command_output(self, cmd, env=None, cwd=None): cmd_output = "" try: @@ -300,9 +313,9 @@ class OpenSKInstaller: target_toolchain[1] in current_version): info("Updating rust toolchain to {}".format("-".join(target_toolchain))) # Need to update - self.checked_command_output( + self.checked_command( ["rustup", "install", target_toolchain_fullstring]) - self.checked_command_output( + self.checked_command( ["rustup", "target", "add", SUPPORTED_BOARDS[self.args.board].arch]) info("Rust toolchain up-to-date") @@ -312,7 +325,11 @@ class OpenSKInstaller: out_directory = os.path.join("third_party", "tock", "target", props.arch, "release") os.makedirs(out_directory, exist_ok=True) - self.checked_command_output(["make"], cwd=props.path) + + env = os.environ.copy() + if self.args.verbose_build: + env["V"] = "1" + self.checked_command(["make"], cwd=props.path, env=env) def build_example(self): info("Building example {}".format(self.args.application)) @@ -347,7 +364,9 @@ class OpenSKInstaller: ] if is_example: command.extend(["--example", self.args.application]) - self.checked_command_output(command, env=env) + if self.args.verbose_build: + command.extend(["--verbose"]) + self.checked_command(command, env=env) app_path = os.path.join("target", props.arch, "release") if is_example: app_path = os.path.join(app_path, "examples") @@ -383,6 +402,8 @@ class OpenSKInstaller: elf2tab_args = [ "elf2tab", package_parameter, self.args.application, "-o", tab_filename ] + if self.args.verbose_build: + elf2tab_args.extend(["--verbose"]) for arch, app_file in binaries.items(): dest_file = os.path.join(self.tab_folder, "{}.elf".format(arch)) shutil.copyfile(app_file, dest_file) @@ -392,7 +413,7 @@ class OpenSKInstaller: "--stack={}".format(STACK_SIZE), "--app-heap={}".format(APP_HEAP_SIZE), "--kernel-heap=1024", "--protected-region-size=64" ]) - self.checked_command_output(elf2tab_args) + self.checked_command(elf2tab_args) def install_tab_file(self, tab_filename): assert self.args.application @@ -616,14 +637,14 @@ class OpenSKInstaller: if self.args.programmer == "pyocd": info("Flashing HEX file") - self.checked_command_output([ + self.checked_command([ "pyocd", "flash", "--target={}".format(board_props.pyocd_target), "--format=hex", "--erase=auto", dest_file ]) if self.args.programmer == "nordicdfu": info("Creating DFU package") dfu_pkg_file = "target/{}_dfu.zip".format(self.args.board) - self.checked_command_output([ + self.checked_command([ "nrfutil", "pkg", "generate", "--hw-version=52", "--sd-req=0", "--application-version=1", "--application={}".format(dest_file), dfu_pkg_file @@ -711,6 +732,13 @@ if __name__ == "__main__": help=("Only compiles and flash the application/example. " "Otherwise TockOS will also be bundled and flashed."), ) + main_parser.add_argument( + "--verbose-build", + action="store_true", + default=False, + dest="verbose_build", + help=("Build everything in verbose mode."), + ) main_parser.add_argument( "--panic-console", From ec2c3fb201ae46c8f90b851938b04f5dc68c7a79 Mon Sep 17 00:00:00 2001 From: Guillaume Endignoux Date: Tue, 28 Apr 2020 13:56:17 +0200 Subject: [PATCH 02/17] Formatting. --- deploy.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/deploy.py b/deploy.py index c198ba1..79a54b1 100755 --- a/deploy.py +++ b/deploy.py @@ -313,8 +313,7 @@ class OpenSKInstaller: target_toolchain[1] in current_version): info("Updating rust toolchain to {}".format("-".join(target_toolchain))) # Need to update - self.checked_command( - ["rustup", "install", target_toolchain_fullstring]) + self.checked_command(["rustup", "install", target_toolchain_fullstring]) self.checked_command( ["rustup", "target", "add", SUPPORTED_BOARDS[self.args.board].arch]) info("Rust toolchain up-to-date") From 463a289dd0031e22767c46bc16f6fd95c64b08f3 Mon Sep 17 00:00:00 2001 From: Guillaume Endignoux Date: Tue, 28 Apr 2020 14:02:24 +0200 Subject: [PATCH 03/17] Verbose rustup. --- deploy.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/deploy.py b/deploy.py index 79a54b1..b782baa 100755 --- a/deploy.py +++ b/deploy.py @@ -313,9 +313,17 @@ class OpenSKInstaller: target_toolchain[1] in current_version): info("Updating rust toolchain to {}".format("-".join(target_toolchain))) # Need to update - self.checked_command(["rustup", "install", target_toolchain_fullstring]) - self.checked_command( - ["rustup", "target", "add", SUPPORTED_BOARDS[self.args.board].arch]) + rustup_install = ["rustup"] + if self.args.verbose_build: + rustup_install.extend(["--verbose"]) + rustup_install.extend(["install", target_toolchain_fullstring]) + self.checked_command(rustup_install) + + rustup_target = ["rustup"] + if self.args.verbose_build: + rustup_target.extend(["--verbose"]) + rustup_target.extend(["target", "add", SUPPORTED_BOARDS[self.args.board].arch]) + self.checked_command(rustup_target) info("Rust toolchain up-to-date") def build_tockos(self): From 7d47de2b3b4169b3b1754cb95c62c81a5cad918d Mon Sep 17 00:00:00 2001 From: Guillaume Endignoux Date: Tue, 28 Apr 2020 14:05:36 +0200 Subject: [PATCH 04/17] Formatting. --- deploy.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/deploy.py b/deploy.py index b782baa..3bd8a2f 100755 --- a/deploy.py +++ b/deploy.py @@ -273,12 +273,7 @@ class OpenSKInstaller: stdout = None if self.args.verbose_build else subprocess.DEVNULL try: subprocess.run( - cmd, - stdout=stdout, - timeout=None, - check=True, - env=env, - cwd=cwd) + cmd, stdout=stdout, timeout=None, check=True, env=env, cwd=cwd) except subprocess.CalledProcessError as e: fatal("Failed to execute {}: {}".format(cmd[0], str(e))) @@ -322,7 +317,8 @@ class OpenSKInstaller: rustup_target = ["rustup"] if self.args.verbose_build: rustup_target.extend(["--verbose"]) - rustup_target.extend(["target", "add", SUPPORTED_BOARDS[self.args.board].arch]) + rustup_target.extend( + ["target", "add", SUPPORTED_BOARDS[self.args.board].arch]) self.checked_command(rustup_target) info("Rust toolchain up-to-date") From dc088479db30a8612a10e24349713758079af3c8 Mon Sep 17 00:00:00 2001 From: Guillaume Endignoux Date: Tue, 28 Apr 2020 17:57:10 +0200 Subject: [PATCH 05/17] Fix review comments. --- deploy.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/deploy.py b/deploy.py index 3bd8a2f..077a385 100755 --- a/deploy.py +++ b/deploy.py @@ -310,13 +310,13 @@ class OpenSKInstaller: # Need to update rustup_install = ["rustup"] if self.args.verbose_build: - rustup_install.extend(["--verbose"]) + rustup_install.extend("--verbose") rustup_install.extend(["install", target_toolchain_fullstring]) self.checked_command(rustup_install) rustup_target = ["rustup"] if self.args.verbose_build: - rustup_target.extend(["--verbose"]) + rustup_target.extend("--verbose") rustup_target.extend( ["target", "add", SUPPORTED_BOARDS[self.args.board].arch]) self.checked_command(rustup_target) @@ -368,7 +368,7 @@ class OpenSKInstaller: if is_example: command.extend(["--example", self.args.application]) if self.args.verbose_build: - command.extend(["--verbose"]) + command.extend("--verbose") self.checked_command(command, env=env) app_path = os.path.join("target", props.arch, "release") if is_example: @@ -406,7 +406,7 @@ class OpenSKInstaller: "elf2tab", package_parameter, self.args.application, "-o", tab_filename ] if self.args.verbose_build: - elf2tab_args.extend(["--verbose"]) + elf2tab_args.extend("--verbose") for arch, app_file in binaries.items(): dest_file = os.path.join(self.tab_folder, "{}.elf".format(arch)) shutil.copyfile(app_file, dest_file) @@ -698,7 +698,7 @@ if __name__ == "__main__": choices=("boards", "programmers"), default=None, dest="listing", - help=("List supported boards or programmers, 1 per line and then exit."), + help="List supported boards or programmers, 1 per line and then exit.", ) action_group.add_argument( "--board", @@ -740,7 +740,7 @@ if __name__ == "__main__": action="store_true", default=False, dest="verbose_build", - help=("Build everything in verbose mode."), + help="Build everything in verbose mode.", ) main_parser.add_argument( From e838356f3bd67baacf9ec6df7b06c4f88ad90903 Mon Sep 17 00:00:00 2001 From: Guillaume Endignoux Date: Tue, 28 Apr 2020 18:00:36 +0200 Subject: [PATCH 06/17] Revert review suggestion. No, we don't want to run: Command '['rustup', '-', '-', 'v', 'e', 'r', 'b', 'o', 's', 'e', 'install', 'nightly-2020-02-03']' --- deploy.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/deploy.py b/deploy.py index 077a385..b8ba61c 100755 --- a/deploy.py +++ b/deploy.py @@ -310,13 +310,13 @@ class OpenSKInstaller: # Need to update rustup_install = ["rustup"] if self.args.verbose_build: - rustup_install.extend("--verbose") + rustup_install.extend(["--verbose"]) rustup_install.extend(["install", target_toolchain_fullstring]) self.checked_command(rustup_install) rustup_target = ["rustup"] if self.args.verbose_build: - rustup_target.extend("--verbose") + rustup_target.extend(["--verbose"]) rustup_target.extend( ["target", "add", SUPPORTED_BOARDS[self.args.board].arch]) self.checked_command(rustup_target) @@ -368,7 +368,7 @@ class OpenSKInstaller: if is_example: command.extend(["--example", self.args.application]) if self.args.verbose_build: - command.extend("--verbose") + command.extend(["--verbose"]) self.checked_command(command, env=env) app_path = os.path.join("target", props.arch, "release") if is_example: @@ -406,7 +406,7 @@ class OpenSKInstaller: "elf2tab", package_parameter, self.args.application, "-o", tab_filename ] if self.args.verbose_build: - elf2tab_args.extend("--verbose") + elf2tab_args.extend(["--verbose"]) for arch, app_file in binaries.items(): dest_file = os.path.join(self.tab_folder, "{}.elf".format(arch)) shutil.copyfile(app_file, dest_file) From c780d76613afdf9d0a393b426e782cd3cd88cb9b Mon Sep 17 00:00:00 2001 From: Guillaume Endignoux Date: Tue, 28 Apr 2020 18:04:41 +0200 Subject: [PATCH 07/17] Append. --- deploy.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/deploy.py b/deploy.py index b8ba61c..2848924 100755 --- a/deploy.py +++ b/deploy.py @@ -310,13 +310,13 @@ class OpenSKInstaller: # Need to update rustup_install = ["rustup"] if self.args.verbose_build: - rustup_install.extend(["--verbose"]) + rustup_install.append("--verbose") rustup_install.extend(["install", target_toolchain_fullstring]) self.checked_command(rustup_install) rustup_target = ["rustup"] if self.args.verbose_build: - rustup_target.extend(["--verbose"]) + rustup_target.append("--verbose") rustup_target.extend( ["target", "add", SUPPORTED_BOARDS[self.args.board].arch]) self.checked_command(rustup_target) @@ -368,7 +368,7 @@ class OpenSKInstaller: if is_example: command.extend(["--example", self.args.application]) if self.args.verbose_build: - command.extend(["--verbose"]) + command.append("--verbose") self.checked_command(command, env=env) app_path = os.path.join("target", props.arch, "release") if is_example: @@ -406,7 +406,7 @@ class OpenSKInstaller: "elf2tab", package_parameter, self.args.application, "-o", tab_filename ] if self.args.verbose_build: - elf2tab_args.extend(["--verbose"]) + elf2tab_args.append("--verbose") for arch, app_file in binaries.items(): dest_file = os.path.join(self.tab_folder, "{}.elf".format(arch)) shutil.copyfile(app_file, dest_file) From 32f00908884351c48d2dc1c14c7b84d2e08f70bf Mon Sep 17 00:00:00 2001 From: Jean-Michel Picod Date: Tue, 28 Apr 2020 18:24:46 +0200 Subject: [PATCH 08/17] Try to fix yapf matcher --- .github/python_matcher.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/python_matcher.json b/.github/python_matcher.json index ac38ab7..6532ac3 100644 --- a/.github/python_matcher.json +++ b/.github/python_matcher.json @@ -4,7 +4,7 @@ "owner": "yapf-diff", "pattern": [ { - "regexp": "^[+-]{3}\\s*([^\\s]*)\\s*\\((original|reformatted)\\)$", + "regexp": "^(?:---|\\+\\+\\+)\\s*([^\\s]*)\\s*\\((?:original|reformatted)\\)$", "file": 1 }, { @@ -13,7 +13,7 @@ "column": 2 }, { - "regexp": "^(\\s|\\+[^+]|\\-[^-]).*$", + "regexp": "^((?:\\s|\\+[^+]|\\-[^-]).*)$", "loop": true, "message": 1 } From a6114964971666aca91161769149d0b2873eef98 Mon Sep 17 00:00:00 2001 From: Jean-Michel Picod Date: Tue, 28 Apr 2020 18:27:05 +0200 Subject: [PATCH 09/17] Force format error to test yapf matcher --- deploy.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/deploy.py b/deploy.py index bb1c056..6b8377d 100755 --- a/deploy.py +++ b/deploy.py @@ -302,8 +302,7 @@ class OpenSKInstaller: # Need to update self.checked_command_output( ["rustup", "install", target_toolchain_fullstring]) - self.checked_command_output( - ["rustup", "target", "add", SUPPORTED_BOARDS[self.args.board].arch]) + self.checked_command_output(["rustup", "target", "add", SUPPORTED_BOARDS[self.args.board].arch]) info("Rust toolchain up-to-date") def build_tockos(self): From 332d7bc2ea775dccdd481a150e403e674d8efa23 Mon Sep 17 00:00:00 2001 From: Jean-Michel Picod Date: Tue, 28 Apr 2020 18:36:04 +0200 Subject: [PATCH 10/17] Update yapf matcher. Extracting the diff is not very useful because messages are not concatenated. The line/column info is also not useful because the diff includes some context lines. --- .github/python_matcher.json | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/.github/python_matcher.json b/.github/python_matcher.json index 6532ac3..d66fea8 100644 --- a/.github/python_matcher.json +++ b/.github/python_matcher.json @@ -5,17 +5,8 @@ "pattern": [ { "regexp": "^(?:---|\\+\\+\\+)\\s*([^\\s]*)\\s*\\((?:original|reformatted)\\)$", - "file": 1 - }, - { - "regexp": "^@@\\s*-(\\d+),(\\d+)\\s*\\+(\\d+),(\\d+)\\s*@@$", - "line": 1, - "column": 2 - }, - { - "regexp": "^((?:\\s|\\+[^+]|\\-[^-]).*)$", - "loop": true, - "message": 1 + "file": 1, + "message": "This file needs formating." } ] }, From 2e35d0074a67ec56112e5362bd829a542bc1334a Mon Sep 17 00:00:00 2001 From: Jean-Michel Picod Date: Wed, 29 Apr 2020 11:05:28 +0200 Subject: [PATCH 11/17] Introduce formatting error to test yapf --- deploy.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/deploy.py b/deploy.py index 6b8377d..c480656 100755 --- a/deploy.py +++ b/deploy.py @@ -302,7 +302,8 @@ class OpenSKInstaller: # Need to update self.checked_command_output( ["rustup", "install", target_toolchain_fullstring]) - self.checked_command_output(["rustup", "target", "add", SUPPORTED_BOARDS[self.args.board].arch]) + self.checked_command_output( + ["rustup", "target", "add", SUPPORTED_BOARDS[self.args.board].arch]) info("Rust toolchain up-to-date") def build_tockos(self): @@ -314,7 +315,8 @@ class OpenSKInstaller: self.checked_command_output(["make"], cwd=props.path) def build_example(self): - info("Building example {}".format(self.args.application)) + info( + "Building example {}".format(self.args.application)) self._build_app_or_example(is_example=True) def build_opensk(self): From 674c4c1b9a3d19e43e13c63c627fa763ee77705e Mon Sep 17 00:00:00 2001 From: Jean-Michel Picod Date: Wed, 29 Apr 2020 11:38:33 +0200 Subject: [PATCH 12/17] Fixing yapf matcher --- .github/python_matcher.json | 13 ++++++++++--- deploy.py | 6 +++--- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/.github/python_matcher.json b/.github/python_matcher.json index d66fea8..2778f15 100644 --- a/.github/python_matcher.json +++ b/.github/python_matcher.json @@ -4,9 +4,16 @@ "owner": "yapf-diff", "pattern": [ { - "regexp": "^(?:---|\\+\\+\\+)\\s*([^\\s]*)\\s*\\((?:original|reformatted)\\)$", - "file": 1, - "message": "This file needs formating." + "regexp": "^---\\s*([^\\s]*)\\s*\\(original\\)$", + "file": 1 + }, + { + "regexp": "^\\+\\+\\+\\s*([^\\s]*)\\s*\\(reformatted\\)$", + "message": 2 + }, + { + "regexp": "^@@\\s*-(\\d+),(\\d+)\\s*\\+(\\d+),(\\d+)\\s*@@$", + "line": 1 } ] }, diff --git a/deploy.py b/deploy.py index c480656..16276ca 100755 --- a/deploy.py +++ b/deploy.py @@ -315,8 +315,7 @@ class OpenSKInstaller: self.checked_command_output(["make"], cwd=props.path) def build_example(self): - info( - "Building example {}".format(self.args.application)) + info("Building example {}".format(self.args.application)) self._build_app_or_example(is_example=True) def build_opensk(self): @@ -343,7 +342,8 @@ class OpenSKInstaller: env["RUSTFLAGS"] = " ".join(rust_flags) command = [ - "cargo", "build", "--release", "--target={}".format(props.arch), + "cargo", + "build", "--release", "--target={}".format(props.arch), "--features={}".format(",".join(self.args.features)) ] if is_example: From 3e19c7512f12b24217ff5de8730eb74beb2dc93a Mon Sep 17 00:00:00 2001 From: Jean-Michel Picod Date: Wed, 29 Apr 2020 11:40:14 +0200 Subject: [PATCH 13/17] Fix missing parentheses in regexp --- .github/python_matcher.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/python_matcher.json b/.github/python_matcher.json index 2778f15..4c9a945 100644 --- a/.github/python_matcher.json +++ b/.github/python_matcher.json @@ -8,7 +8,7 @@ "file": 1 }, { - "regexp": "^\\+\\+\\+\\s*([^\\s]*)\\s*\\(reformatted\\)$", + "regexp": "^\\+\\+\\+\\s*([^\\s]*)\\s*\\((reformatted)\\)$", "message": 2 }, { From 8e182b9de91ff94898c7555aa126f271c0c473a3 Mon Sep 17 00:00:00 2001 From: Jean-Michel Picod Date: Wed, 29 Apr 2020 11:40:48 +0200 Subject: [PATCH 14/17] Introduce formatting mistake --- deploy.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy.py b/deploy.py index 16276ca..93dbe7b 100755 --- a/deploy.py +++ b/deploy.py @@ -342,8 +342,8 @@ class OpenSKInstaller: env["RUSTFLAGS"] = " ".join(rust_flags) command = [ - "cargo", - "build", "--release", "--target={}".format(props.arch), + "cargo", "build", + "--release", "--target={}".format(props.arch), "--features={}".format(",".join(self.args.features)) ] if is_example: From 8211df81d5cbf5b4c0902622be2675626a151132 Mon Sep 17 00:00:00 2001 From: Jean-Michel Picod Date: Wed, 29 Apr 2020 11:46:52 +0200 Subject: [PATCH 15/17] Adjust regexp --- .github/python_matcher.json | 2 +- deploy.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/python_matcher.json b/.github/python_matcher.json index 4c9a945..6c99675 100644 --- a/.github/python_matcher.json +++ b/.github/python_matcher.json @@ -8,7 +8,7 @@ "file": 1 }, { - "regexp": "^\\+\\+\\+\\s*([^\\s]*)\\s*\\((reformatted)\\)$", + "regexp": "^\\+\\+\\+\\s*([^\\s]*)\\s*\\((.*)\\)$", "message": 2 }, { diff --git a/deploy.py b/deploy.py index 93dbe7b..16276ca 100755 --- a/deploy.py +++ b/deploy.py @@ -342,8 +342,8 @@ class OpenSKInstaller: env["RUSTFLAGS"] = " ".join(rust_flags) command = [ - "cargo", "build", - "--release", "--target={}".format(props.arch), + "cargo", + "build", "--release", "--target={}".format(props.arch), "--features={}".format(",".join(self.args.features)) ] if is_example: From ca15ea43235f06bc58a70833c28e8bb461938425 Mon Sep 17 00:00:00 2001 From: Jean-Michel Picod Date: Wed, 29 Apr 2020 11:48:12 +0200 Subject: [PATCH 16/17] Fix formatting --- deploy.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/deploy.py b/deploy.py index 16276ca..bb1c056 100755 --- a/deploy.py +++ b/deploy.py @@ -342,8 +342,7 @@ class OpenSKInstaller: env["RUSTFLAGS"] = " ".join(rust_flags) command = [ - "cargo", - "build", "--release", "--target={}".format(props.arch), + "cargo", "build", "--release", "--target={}".format(props.arch), "--features={}".format(",".join(self.args.features)) ] if is_example: From 999e3313063cf8a298a7bc9831310c7d0f097314 Mon Sep 17 00:00:00 2001 From: Jean-Michel Picod Date: Wed, 29 Apr 2020 13:33:35 +0200 Subject: [PATCH 17/17] Add missing paramter that got introduced in tockloader 1.4 --- deploy.py | 1 + 1 file changed, 1 insertion(+) diff --git a/deploy.py b/deploy.py index 2848924..5fb65f7 100755 --- a/deploy.py +++ b/deploy.py @@ -251,6 +251,7 @@ class OpenSKInstaller: self.tockloader_default_args = argparse.Namespace( arch=board.arch, board=self.args.board, + bundle_apps=False, debug=False, force=False, jlink_cmd="JLinkExe",