From d69b40f2cfb6ded211e4e5edb0570e207a924e4f Mon Sep 17 00:00:00 2001 From: Jean-Michel Picod Date: Thu, 27 Feb 2020 18:39:44 +0100 Subject: [PATCH 01/28] Add custom python problem matcher for pylint and yapf --- .github/python_matcher.json | 47 ++++++++++++++++++++++++++++++++++++ .github/workflows/python.yml | 12 ++++++--- 2 files changed, 56 insertions(+), 3 deletions(-) create mode 100644 .github/python_matcher.json diff --git a/.github/python_matcher.json b/.github/python_matcher.json new file mode 100644 index 0000000..35abaf1 --- /dev/null +++ b/.github/python_matcher.json @@ -0,0 +1,47 @@ +{ + "problemMatcher": [ + { + "owner": "yapf-diff", + "pattern": [ + { + "regexp": "^[+-]{3}\\s*([^\\s]*)\\s*\\((original|reformatted)\\)$", + "file": 1 + }, + { + "regexp": "^@@\\s*-(\\d+),(\\d+)\\s*\\+(\\d+),(\\d+)\\s*@@$", + "line": 1, + "column": 2 + }, + { + "regexp": "^(\\s|\\+[^+]|\\-[^-]).*$", + "loop": true, + "message": 1 + } + ] + }, + { + "owner": "pylint", + "pattern": [ + { + "regexp": "^PYLINT:\\(.*)/.*$", + "fromPath": 1 + }, + { + "regexp": "^\\*{13}\\s*Module\\s+(.*)$", + "file": 1 + }, + { + "regexp": "^([CEFIRW]\\d{4}):\\s*(\\d+)\\s*:\\s*(.*)$", + "code": 1, + "line": 2, + "message": 3 + }, + { + "regexp": "^([^CEFIRW*]*)$", + "message": 1, + "loop": true + } + ] + } + ] +} diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index dc739b8..7c0e104 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -26,7 +26,10 @@ jobs: - name: Test code with pylint run: | pip install pylint - pylint --rcfile=.pylintrc --score=n `find . ! -path "./third_party/*" -type f -name '*.py'` + ./setup.sh + echo ::add-matcher::./.github/python_matcher.json + find . -type f -name '*.py' -exec echo PYLINT:\{\} \; -exec pylint --rcfile=.pylintrc --score=n \{\} \; + #find . ! -path "./third_party/*" -type f -name '*.py' -exec echo PYLINT:\{\} \; -exec pylint --rcfile=.pylintrc --score=n \{\} \; yapf: runs-on: ubuntu-18.04 @@ -39,6 +42,9 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip setuptools wheel - pip install yapf + pip install yapf tockloader - name: Test code formatting with yapf - run: yapf --style=chromium --recursive --exclude third_party --diff . + run: | + ./setup.sh + echo ::add-matcher::./.github/python_matcher.json + yapf --style=chromium --recursive --exclude third_party --diff . From 28afe7f86f217edff0d553f0e331e57a50c01b05 Mon Sep 17 00:00:00 2001 From: Jean-Michel Picod Date: Thu, 27 Feb 2020 18:43:02 +0100 Subject: [PATCH 02/28] Fix yaml bad indent --- .github/workflows/python.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 7c0e104..9e97b0e 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -45,6 +45,6 @@ jobs: pip install yapf tockloader - name: Test code formatting with yapf run: | - ./setup.sh - echo ::add-matcher::./.github/python_matcher.json - yapf --style=chromium --recursive --exclude third_party --diff . + ./setup.sh + echo ::add-matcher::./.github/python_matcher.json + yapf --style=chromium --recursive --exclude third_party --diff . From 4e4b4a33570d7c10d022a8ae2036dea507bfb9d5 Mon Sep 17 00:00:00 2001 From: Jean-Michel Picod Date: Thu, 27 Feb 2020 18:45:53 +0100 Subject: [PATCH 03/28] Add mistake in deploy.py to test matchers --- deploy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy.py b/deploy.py index 1dc3fc8..5419cdb 100755 --- a/deploy.py +++ b/deploy.py @@ -32,7 +32,7 @@ from tockloader import tab, tbfh, tockloader # This structure allows us in the future to also support out-of-tree boards. SUPPORTED_BOARDS = { "nrf52840_dk": "third_party/tock/boards/nordic/nrf52840dk", - "nrf52840_dongle": "third_party/tock/boards/nordic/nrf52840_dongle" + "nrf52840_dongle":"third_party/tock/boards/nordic/nrf52840_dongle" } # The STACK_SIZE value below must match the one used in the linker script From 507f4a9f893c2532356aa07e743334a394e8e2b1 Mon Sep 17 00:00:00 2001 From: Jean-Michel Picod Date: Thu, 27 Feb 2020 18:47:21 +0100 Subject: [PATCH 04/28] Temp disable path restriction --- .github/workflows/python.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 9e97b0e..3ac7d6d 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -1,10 +1,10 @@ name: pylint on: push: - paths: - - '**/*.py' - - '.pylintrc' - - '!third_party/**' +# paths: +# - '**/*.py' +# - '.pylintrc' +# - '!third_party/**' pull_request: types: [opened, synchronize, reopened] jobs: From 816d2020be7e8b9a4be0352035fabb0e8b7f867d Mon Sep 17 00:00:00 2001 From: Jean-Michel Picod Date: Thu, 27 Feb 2020 18:51:03 +0100 Subject: [PATCH 05/28] Fix invalid regexp --- .github/python_matcher.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/python_matcher.json b/.github/python_matcher.json index 35abaf1..1606b09 100644 --- a/.github/python_matcher.json +++ b/.github/python_matcher.json @@ -23,7 +23,7 @@ "owner": "pylint", "pattern": [ { - "regexp": "^PYLINT:\\(.*)/.*$", + "regexp": "^PYLINT:(.*)/.*$", "fromPath": 1 }, { From 051a78ab980e29c8a9ce15fa35a053ae93d99bd4 Mon Sep 17 00:00:00 2001 From: Jean-Michel Picod Date: Thu, 27 Feb 2020 19:09:57 +0100 Subject: [PATCH 06/28] Fix yapf matcher --- .github/python_matcher.json | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/python_matcher.json b/.github/python_matcher.json index 1606b09..ac38ab7 100644 --- a/.github/python_matcher.json +++ b/.github/python_matcher.json @@ -35,11 +35,6 @@ "code": 1, "line": 2, "message": 3 - }, - { - "regexp": "^([^CEFIRW*]*)$", - "message": 1, - "loop": true } ] } From 7a86f80e2bedcd34f715e670e9f809c1727aede2 Mon Sep 17 00:00:00 2001 From: Jean-Michel Picod Date: Thu, 27 Feb 2020 19:16:45 +0100 Subject: [PATCH 07/28] Now try to trigger pylint matcher --- .github/workflows/python.yml | 8 ++------ deploy.py | 4 ++-- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 3ac7d6d..19103c2 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -22,14 +22,11 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip setuptools wheel - pip install tockloader + pip install tockloader pylint - name: Test code with pylint run: | - pip install pylint - ./setup.sh echo ::add-matcher::./.github/python_matcher.json - find . -type f -name '*.py' -exec echo PYLINT:\{\} \; -exec pylint --rcfile=.pylintrc --score=n \{\} \; - #find . ! -path "./third_party/*" -type f -name '*.py' -exec echo PYLINT:\{\} \; -exec pylint --rcfile=.pylintrc --score=n \{\} \; + find . ! -path "./third_party/*" -type f -name '*.py' -exec echo PYLINT:\{\} \; -exec pylint --rcfile=.pylintrc --score=n \{\} \; yapf: runs-on: ubuntu-18.04 @@ -45,6 +42,5 @@ jobs: pip install yapf tockloader - name: Test code formatting with yapf run: | - ./setup.sh echo ::add-matcher::./.github/python_matcher.json yapf --style=chromium --recursive --exclude third_party --diff . diff --git a/deploy.py b/deploy.py index 5419cdb..da835d5 100755 --- a/deploy.py +++ b/deploy.py @@ -18,6 +18,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import colorama import argparse import copy import os @@ -25,14 +26,13 @@ import shutil import subprocess import sys -import colorama from tockloader.exceptions import TockLoaderException from tockloader import tab, tbfh, tockloader # This structure allows us in the future to also support out-of-tree boards. SUPPORTED_BOARDS = { "nrf52840_dk": "third_party/tock/boards/nordic/nrf52840dk", - "nrf52840_dongle":"third_party/tock/boards/nordic/nrf52840_dongle" + "nrf52840_dongle": "third_party/tock/boards/nordic/nrf52840_dongle" } # The STACK_SIZE value below must match the one used in the linker script From db3b23cf22c13754eb1675c3e50da2f3a524baf3 Mon Sep 17 00:00:00 2001 From: Jean-Michel Picod Date: Thu, 27 Feb 2020 19:22:06 +0100 Subject: [PATCH 08/28] Still trying to trigger pylint --- .github/workflows/python.yml | 3 ++- deploy.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 19103c2..1edd79d 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -26,7 +26,8 @@ jobs: - name: Test code with pylint run: | echo ::add-matcher::./.github/python_matcher.json - find . ! -path "./third_party/*" -type f -name '*.py' -exec echo PYLINT:\{\} \; -exec pylint --rcfile=.pylintrc --score=n \{\} \; + find . -type f -name '*.py' -exec echo PYLINT:\{\} \; -exec pylint --rcfile=.pylintrc --score=n \{\} \; +# find . ! -path "./third_party/*" -type f -name '*.py' -exec echo PYLINT:\{\} \; -exec pylint --rcfile=.pylintrc --score=n \{\} \; yapf: runs-on: ubuntu-18.04 diff --git a/deploy.py b/deploy.py index da835d5..1dc3fc8 100755 --- a/deploy.py +++ b/deploy.py @@ -18,7 +18,6 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -import colorama import argparse import copy import os @@ -26,6 +25,7 @@ import shutil import subprocess import sys +import colorama from tockloader.exceptions import TockLoaderException from tockloader import tab, tbfh, tockloader From fa19a803666c76a4b06545f01093920b5f60db1d Mon Sep 17 00:00:00 2001 From: Jean-Michel Picod Date: Thu, 27 Feb 2020 20:07:29 +0100 Subject: [PATCH 09/28] Try with a custom action --- .github/workflows/python.yml | 11 ++++++---- github_actions/pylint/Dockerfile | 10 +++++++++ github_actions/pylint/action.yml | 20 ++++++++++++++++++ github_actions/pylint/entrypoint.sh | 32 +++++++++++++++++++++++++++++ 4 files changed, 69 insertions(+), 4 deletions(-) create mode 100644 github_actions/pylint/Dockerfile create mode 100644 github_actions/pylint/action.yml create mode 100644 github_actions/pylint/entrypoint.sh diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 1edd79d..d66067f 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -23,11 +23,14 @@ jobs: run: | python -m pip install --upgrade pip setuptools wheel pip install tockloader pylint + - name: Register matcher + run: echo ::add-matcher::./.github/python_matcher.json - name: Test code with pylint - run: | - echo ::add-matcher::./.github/python_matcher.json - find . -type f -name '*.py' -exec echo PYLINT:\{\} \; -exec pylint --rcfile=.pylintrc --score=n \{\} \; -# find . ! -path "./third_party/*" -type f -name '*.py' -exec echo PYLINT:\{\} \; -exec pylint --rcfile=.pylintrc --score=n \{\} \; + uses: ./github_actions/pylint + with: + config-file: .pylintrc + files: '**/*.py' + exclude-path: third_party yapf: runs-on: ubuntu-18.04 diff --git a/github_actions/pylint/Dockerfile b/github_actions/pylint/Dockerfile new file mode 100644 index 0000000..63fbb9a --- /dev/null +++ b/github_actions/pylint/Dockerfile @@ -0,0 +1,10 @@ +FROM python:3.7-alpine + +RUN apk add --no-cache bash build-base gcc +RUN pip install --upgrade pip +RUN pip install pylint +RUN python --version ; pip --version ; pylint --version + +COPY entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh +ENTRYPOINT ["/entrypoint.sh"] diff --git a/github_actions/pylint/action.yml b/github_actions/pylint/action.yml new file mode 100644 index 0000000..7ca6dda --- /dev/null +++ b/github_actions/pylint/action.yml @@ -0,0 +1,20 @@ +--- +name: 'Pylint' +description: 'Runs pylint across multiple files/modules' +author: 'Jean-Michel Picod ' +inputs: + config-file: + description: pylintrc configuration file + required: false + files: + description: files, directories, or globs + required: true + ignore-files: + description: files to ignore/exclude + required: false + exclude-path: + description: paths to ignore/exclude + required: false +runs: + using: docker + image: Dockerfile diff --git a/github_actions/pylint/entrypoint.sh b/github_actions/pylint/entrypoint.sh new file mode 100644 index 0000000..f99938b --- /dev/null +++ b/github_actions/pylint/entrypoint.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +env + +PYLINT_CMD=pylint --score=n${INPUT_CONFIG_FILE:+ --rcfile=${INPUT_CONFIG_FILE}} +EXCLUDE_PATH=${INPUT_EXCLUDE_PATH:-} +EXCLUDE_FILES=${INPUT_EXCLUDE_PATH:-} + +SUCCESS=0 +for file in ${FILES} +do + fname=$(basename $file) + directory=$(dirname $file) + if [[ "$directory" =~ "^${EXCLUDE_PATH}" ]] + then + echo "Ignoring file '$file' (reason: matching exclude-path parameter)" + continue + fi + if [[ "$fname" =~ "${EXCLUDE_FILES}" ]] + then + echo "Ignoring file '$file' (reason: matching exclude-files parameter)" + continue + fi + # Just to trigger the custom matcher + echo PYLINT:$file + if ! $PYLINT_CMD $file + then + SUCCESS=1 + fi +done + +exit $SUCCESS From 47c523e959f0dcc2cdc9534862cb961e2cc6cdf0 Mon Sep 17 00:00:00 2001 From: Jean-Michel Picod Date: Thu, 27 Feb 2020 20:11:29 +0100 Subject: [PATCH 10/28] Fix entrypoint --- github_actions/pylint/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/github_actions/pylint/entrypoint.sh b/github_actions/pylint/entrypoint.sh index f99938b..da48ac5 100644 --- a/github_actions/pylint/entrypoint.sh +++ b/github_actions/pylint/entrypoint.sh @@ -2,7 +2,7 @@ env -PYLINT_CMD=pylint --score=n${INPUT_CONFIG_FILE:+ --rcfile=${INPUT_CONFIG_FILE}} +PYLINT_CMD="pylint --score=n${INPUT_CONFIG_FILE:+ --rcfile=${INPUT_CONFIG_FILE}}" EXCLUDE_PATH=${INPUT_EXCLUDE_PATH:-} EXCLUDE_FILES=${INPUT_EXCLUDE_PATH:-} From 19056abbf31712afbbe5e72cf198c9fb4652504e Mon Sep 17 00:00:00 2001 From: Jean-Michel Picod Date: Fri, 28 Feb 2020 15:52:09 +0100 Subject: [PATCH 11/28] Replace custom action by shell script --- .github/workflows/python.yml | 6 +----- github_actions/pylint/Dockerfile | 10 --------- github_actions/pylint/action.yml | 20 ------------------ github_actions/pylint/entrypoint.sh | 32 ----------------------------- tools/run_pylint.sh | 18 ++++++++++++++++ 5 files changed, 19 insertions(+), 67 deletions(-) delete mode 100644 github_actions/pylint/Dockerfile delete mode 100644 github_actions/pylint/action.yml delete mode 100644 github_actions/pylint/entrypoint.sh create mode 100755 tools/run_pylint.sh diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index d66067f..82c0624 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -26,11 +26,7 @@ jobs: - name: Register matcher run: echo ::add-matcher::./.github/python_matcher.json - name: Test code with pylint - uses: ./github_actions/pylint - with: - config-file: .pylintrc - files: '**/*.py' - exclude-path: third_party + runs: ./tools/run_pylint.sh yapf: runs-on: ubuntu-18.04 diff --git a/github_actions/pylint/Dockerfile b/github_actions/pylint/Dockerfile deleted file mode 100644 index 63fbb9a..0000000 --- a/github_actions/pylint/Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -FROM python:3.7-alpine - -RUN apk add --no-cache bash build-base gcc -RUN pip install --upgrade pip -RUN pip install pylint -RUN python --version ; pip --version ; pylint --version - -COPY entrypoint.sh /entrypoint.sh -RUN chmod +x /entrypoint.sh -ENTRYPOINT ["/entrypoint.sh"] diff --git a/github_actions/pylint/action.yml b/github_actions/pylint/action.yml deleted file mode 100644 index 7ca6dda..0000000 --- a/github_actions/pylint/action.yml +++ /dev/null @@ -1,20 +0,0 @@ ---- -name: 'Pylint' -description: 'Runs pylint across multiple files/modules' -author: 'Jean-Michel Picod ' -inputs: - config-file: - description: pylintrc configuration file - required: false - files: - description: files, directories, or globs - required: true - ignore-files: - description: files to ignore/exclude - required: false - exclude-path: - description: paths to ignore/exclude - required: false -runs: - using: docker - image: Dockerfile diff --git a/github_actions/pylint/entrypoint.sh b/github_actions/pylint/entrypoint.sh deleted file mode 100644 index da48ac5..0000000 --- a/github_actions/pylint/entrypoint.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash - -env - -PYLINT_CMD="pylint --score=n${INPUT_CONFIG_FILE:+ --rcfile=${INPUT_CONFIG_FILE}}" -EXCLUDE_PATH=${INPUT_EXCLUDE_PATH:-} -EXCLUDE_FILES=${INPUT_EXCLUDE_PATH:-} - -SUCCESS=0 -for file in ${FILES} -do - fname=$(basename $file) - directory=$(dirname $file) - if [[ "$directory" =~ "^${EXCLUDE_PATH}" ]] - then - echo "Ignoring file '$file' (reason: matching exclude-path parameter)" - continue - fi - if [[ "$fname" =~ "${EXCLUDE_FILES}" ]] - then - echo "Ignoring file '$file' (reason: matching exclude-files parameter)" - continue - fi - # Just to trigger the custom matcher - echo PYLINT:$file - if ! $PYLINT_CMD $file - then - SUCCESS=1 - fi -done - -exit $SUCCESS diff --git a/tools/run_pylint.sh b/tools/run_pylint.sh new file mode 100755 index 0000000..cdb7919 --- /dev/null +++ b/tools/run_pylint.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +SUCCESS=0 + +# Ensure we are at the project root directory +cd $(readlink -f $(dirname $0))/.. + +for file in `find . ! -path "./third_party/*" -type f -name '*.py'` +do + # Output header for our custom matcher on Github workflow + echo "PYLINT:${file}" + if ! pylint --rcfile=.pylintrc --score=n "$file" + then + SUCCESS=1 + fi +done + +exit $SUCCESS From 7dedf4044091f91fbccb730961d6fca70b182039 Mon Sep 17 00:00:00 2001 From: Jean-Michel Picod Date: Fri, 28 Feb 2020 15:53:44 +0100 Subject: [PATCH 12/28] Fix typo in YAML workflow --- .github/workflows/python.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 82c0624..aa9070b 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -26,7 +26,7 @@ jobs: - name: Register matcher run: echo ::add-matcher::./.github/python_matcher.json - name: Test code with pylint - runs: ./tools/run_pylint.sh + run: ./tools/run_pylint.sh yapf: runs-on: ubuntu-18.04 From ac37d7dd8a900691b0dcc55709c37127162edd47 Mon Sep 17 00:00:00 2001 From: Jean-Michel Picod Date: Fri, 28 Feb 2020 15:55:06 +0100 Subject: [PATCH 13/28] Restore path guards --- .github/workflows/python.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index aa9070b..f6051a3 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -1,10 +1,10 @@ name: pylint on: push: -# paths: -# - '**/*.py' -# - '.pylintrc' -# - '!third_party/**' + paths: + - '**/*.py' + - '.pylintrc' + - '!third_party/**' pull_request: types: [opened, synchronize, reopened] jobs: From 9de3817ade6643822e77402410a6d38ce0de7897 Mon Sep 17 00:00:00 2001 From: Jean-Michel Picod Date: Fri, 28 Feb 2020 15:57:48 +0100 Subject: [PATCH 14/28] Add missing license header --- tools/run_pylint.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tools/run_pylint.sh b/tools/run_pylint.sh index cdb7919..2953e4f 100755 --- a/tools/run_pylint.sh +++ b/tools/run_pylint.sh @@ -1,4 +1,17 @@ #!/usr/bin/env bash +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. SUCCESS=0 From 68f9eb6b09af9833b6641039f24e90ce9ffab8ac Mon Sep 17 00:00:00 2001 From: Jean-Michel Picod Date: Fri, 28 Feb 2020 15:58:32 +0100 Subject: [PATCH 15/28] Enables cargo audit on pull request --- .github/workflows/cargo_audit.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/cargo_audit.yml b/.github/workflows/cargo_audit.yml index 94b00f6..1a1be78 100644 --- a/.github/workflows/cargo_audit.yml +++ b/.github/workflows/cargo_audit.yml @@ -2,6 +2,9 @@ name: Security audit on: schedule: - cron: '0 0 * * *' + pull_request: + types: [opened, synchronize, reopened] + jobs: audit: runs-on: ubuntu-18.04 From cc35ce5067181ecd75c702981f6ef2cc8b60a393 Mon Sep 17 00:00:00 2001 From: Jean-Michel Picod Date: Fri, 28 Feb 2020 16:03:04 +0100 Subject: [PATCH 16/28] Create workflows for unit testing --- .github/workflows/cbor_test.yml | 38 +++++++++++++++++++++++ .github/workflows/crypto_test.yml | 40 ++++++++++++++++++++++++ .github/workflows/opensk_build.yml | 30 ++++++++++++++++++ .github/workflows/opensk_test.yml | 50 ++++++++++++++++++++++++++++++ 4 files changed, 158 insertions(+) create mode 100644 .github/workflows/cbor_test.yml create mode 100644 .github/workflows/crypto_test.yml create mode 100644 .github/workflows/opensk_build.yml create mode 100644 .github/workflows/opensk_test.yml diff --git a/.github/workflows/cbor_test.yml b/.github/workflows/cbor_test.yml new file mode 100644 index 0000000..37d08f8 --- /dev/null +++ b/.github/workflows/cbor_test.yml @@ -0,0 +1,38 @@ +--- +name: CBOR tests +on: + push: + paths: + - 'libraries/cbor/**/*' + pull_request: + types: [opened, synchronize, reopened] + +jobs: + cbor_test: + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v2 + - uses: actions-rs/toolchain@v1 + with: + toolchain: nightly + target: thumbv7em-none-eabi + override: true + - uses: actions/setup-python@v1 + with: + python-version: 3.7 + - name: Install Python dependencies + run: python -m pip install --upgrade pip setuptools wheel + - name: Set up OpenSK + run: ./setup.sh + + - name: Unit testing of CBOR library (release mode) + uses: actions-rs/cargo@v1 + with: + command: test + args: --manifest-path libraries/cbor/Cargo.toml --release --features std + + - name: Unit testing of CBOR library (debug mode) + uses: actions-rs/cargo@v1 + with: + command: test + args: --manifest-path libraries/cbor/Cargo.toml --features std diff --git a/.github/workflows/crypto_test.yml b/.github/workflows/crypto_test.yml new file mode 100644 index 0000000..da4ce37 --- /dev/null +++ b/.github/workflows/crypto_test.yml @@ -0,0 +1,40 @@ +--- +name: CBOR tests +on: + push: + paths: + - 'libraries/crypto/**/*' + pull_request: + types: [opened, synchronize, reopened] + +jobs: + crypto_test: + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v2 + - uses: actions-rs/toolchain@v1 + with: + toolchain: nightly + target: thumbv7em-none-eabi + override: true + - uses: actions/setup-python@v1 + with: + python-version: 3.7 + - name: Install Python dependencies + run: python -m pip install --upgrade pip setuptools wheel + - name: Set up OpenSK + run: ./setup.sh + + - run: echo "::set-env name=RUSTFLAGS::-C target-feature=+aes" + + - name: Unit testing of crypto library (release mode) + uses: actions-rs/cargo@v1 + with: + command: test + args: --manifest-path libraries/crypto/Cargo.toml --release --features std,derive_debug + + - name: Unit testing of crypto library (debug mode) + uses: actions-rs/cargo@v1 + with: + command: test + args: --manifest-path libraries/crypto/Cargo.toml --features std,derive_debug diff --git a/.github/workflows/opensk_build.yml b/.github/workflows/opensk_build.yml new file mode 100644 index 0000000..91fe551 --- /dev/null +++ b/.github/workflows/opensk_build.yml @@ -0,0 +1,30 @@ +--- +name: OpenSK build +on: + push: + pull_request: + types: [opened, synchronize, reopened] + +jobs: + build: + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v2 + - uses: actions-rs/toolchain@v1 + with: + toolchain: nightly + target: thumbv7em-none-eabi + override: true + - uses: actions/setup-python@v1 + with: + python-version: 3.7 + - name: Install Python dependencies + run: python -m pip install --upgrade pip setuptools wheel + - name: Set up OpenSK + run: ./setup.sh + + - name: Building OpenSK + uses: actions-rs/cargo@v1 + with: + command: build + args: --release --target=thumbv7em-none-eabi --features with_ctap1 diff --git a/.github/workflows/opensk_test.yml b/.github/workflows/opensk_test.yml new file mode 100644 index 0000000..1b40727 --- /dev/null +++ b/.github/workflows/opensk_test.yml @@ -0,0 +1,50 @@ +--- +name: OpenSK tests +on: + push: + paths: + - 'src/**/*.rs' + pull_request: + types: [opened, synchronize, reopened] + +jobs: + ctap2_test: + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v2 + - uses: actions-rs/toolchain@v1 + with: + toolchain: nightly + target: thumbv7em-none-eabi + override: true + - uses: actions/setup-python@v1 + with: + python-version: 3.7 + - name: Install Python dependencies + run: python -m pip install --upgrade pip setuptools wheel + - name: Set up OpenSK + run: ./setup.sh + + - name: Unit testing of CTAP2 (release mode) + uses: actions-rs/cargo@v1 + with: + command: test + args: --release --features std + + - name: Unit testing of CTAP2 (debug mode) + uses: actions-rs/cargo@v1 + with: + command: test + args: --features std + + - name: Unit testing of CTAP2 (release mode + CTAP1) + uses: actions-rs/cargo@v1 + with: + command: test + args: --features std,with_ctap1 + + - name: Unit testing of CTAP2 (debug mode + CTAP1) + uses: actions-rs/cargo@v1 + with: + command: test + args: --features std,with_ctap1 From ae1242a55d9bb416d1b9db9e3e2a4dfe9d69331f Mon Sep 17 00:00:00 2001 From: Jean-Michel Picod Date: Fri, 28 Feb 2020 16:11:23 +0100 Subject: [PATCH 17/28] Temp enable all unit test to verify workflows --- .github/workflows/cbor_test.yml | 4 ++-- .github/workflows/crypto_test.yml | 4 ++-- .github/workflows/opensk_build.yml | 2 +- .github/workflows/opensk_test.yml | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/cbor_test.yml b/.github/workflows/cbor_test.yml index 37d08f8..5f38f57 100644 --- a/.github/workflows/cbor_test.yml +++ b/.github/workflows/cbor_test.yml @@ -2,8 +2,8 @@ name: CBOR tests on: push: - paths: - - 'libraries/cbor/**/*' +# paths: +# - 'libraries/cbor/**/*' pull_request: types: [opened, synchronize, reopened] diff --git a/.github/workflows/crypto_test.yml b/.github/workflows/crypto_test.yml index da4ce37..594fcad 100644 --- a/.github/workflows/crypto_test.yml +++ b/.github/workflows/crypto_test.yml @@ -2,8 +2,8 @@ name: CBOR tests on: push: - paths: - - 'libraries/crypto/**/*' +# paths: +# - 'libraries/crypto/**/*' pull_request: types: [opened, synchronize, reopened] diff --git a/.github/workflows/opensk_build.yml b/.github/workflows/opensk_build.yml index 91fe551..ac82c73 100644 --- a/.github/workflows/opensk_build.yml +++ b/.github/workflows/opensk_build.yml @@ -6,7 +6,7 @@ on: types: [opened, synchronize, reopened] jobs: - build: + build_ctap2: runs-on: ubuntu-18.04 steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/opensk_test.yml b/.github/workflows/opensk_test.yml index 1b40727..77be4de 100644 --- a/.github/workflows/opensk_test.yml +++ b/.github/workflows/opensk_test.yml @@ -2,8 +2,8 @@ name: OpenSK tests on: push: - paths: - - 'src/**/*.rs' +# paths: +# - 'src/**/*.rs' pull_request: types: [opened, synchronize, reopened] From c8ac2baca3ed8c0a9068e057a3de0a252c0f0a01 Mon Sep 17 00:00:00 2001 From: Jean-Michel Picod Date: Fri, 28 Feb 2020 16:24:21 +0100 Subject: [PATCH 18/28] Fix wrong workflow name --- .github/workflows/crypto_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/crypto_test.yml b/.github/workflows/crypto_test.yml index 594fcad..d8a2208 100644 --- a/.github/workflows/crypto_test.yml +++ b/.github/workflows/crypto_test.yml @@ -1,5 +1,5 @@ --- -name: CBOR tests +name: Crypto library tests on: push: # paths: From 578742034e4dd7d1e26f0b495a9f0a2588c337a4 Mon Sep 17 00:00:00 2001 From: Jean-Michel Picod Date: Fri, 28 Feb 2020 16:32:20 +0100 Subject: [PATCH 19/28] Remove toolchain parameter for workflow It seems the github action will pick up the rust-toolchain file and if we specify it as an input, it will have a higher priority --- .github/workflows/cargo_audit.yml | 2 -- .github/workflows/cargo_check.yml | 2 -- .github/workflows/cargo_fmt.yml | 2 -- .github/workflows/cbor_test.yml | 2 -- .github/workflows/crypto_test.yml | 2 -- .github/workflows/opensk_build.yml | 2 -- .github/workflows/opensk_test.yml | 2 -- 7 files changed, 14 deletions(-) diff --git a/.github/workflows/cargo_audit.yml b/.github/workflows/cargo_audit.yml index 1a1be78..b3f9897 100644 --- a/.github/workflows/cargo_audit.yml +++ b/.github/workflows/cargo_audit.yml @@ -12,9 +12,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions-rs/toolchain@v1 with: - toolchain: nightly target: thumbv7em-none-eabi - override: true - uses: actions/setup-python@v1 with: python-version: 3.7 diff --git a/.github/workflows/cargo_check.yml b/.github/workflows/cargo_check.yml index 9697b37..36110a9 100644 --- a/.github/workflows/cargo_check.yml +++ b/.github/workflows/cargo_check.yml @@ -19,9 +19,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions-rs/toolchain@v1 with: - toolchain: nightly target: thumbv7em-none-eabi - override: true - uses: actions/setup-python@v1 with: python-version: 3.7 diff --git a/.github/workflows/cargo_fmt.yml b/.github/workflows/cargo_fmt.yml index 324b475..00589ca 100644 --- a/.github/workflows/cargo_fmt.yml +++ b/.github/workflows/cargo_fmt.yml @@ -19,9 +19,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions-rs/toolchain@v1 with: - toolchain: nightly target: thumbv7em-none-eabi - override: true - uses: actions/setup-python@v1 with: python-version: 3.7 diff --git a/.github/workflows/cbor_test.yml b/.github/workflows/cbor_test.yml index 5f38f57..e1c73aa 100644 --- a/.github/workflows/cbor_test.yml +++ b/.github/workflows/cbor_test.yml @@ -14,9 +14,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions-rs/toolchain@v1 with: - toolchain: nightly target: thumbv7em-none-eabi - override: true - uses: actions/setup-python@v1 with: python-version: 3.7 diff --git a/.github/workflows/crypto_test.yml b/.github/workflows/crypto_test.yml index d8a2208..d79fd1f 100644 --- a/.github/workflows/crypto_test.yml +++ b/.github/workflows/crypto_test.yml @@ -14,9 +14,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions-rs/toolchain@v1 with: - toolchain: nightly target: thumbv7em-none-eabi - override: true - uses: actions/setup-python@v1 with: python-version: 3.7 diff --git a/.github/workflows/opensk_build.yml b/.github/workflows/opensk_build.yml index ac82c73..407a603 100644 --- a/.github/workflows/opensk_build.yml +++ b/.github/workflows/opensk_build.yml @@ -12,9 +12,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions-rs/toolchain@v1 with: - toolchain: nightly target: thumbv7em-none-eabi - override: true - uses: actions/setup-python@v1 with: python-version: 3.7 diff --git a/.github/workflows/opensk_test.yml b/.github/workflows/opensk_test.yml index 77be4de..3e34ce5 100644 --- a/.github/workflows/opensk_test.yml +++ b/.github/workflows/opensk_test.yml @@ -14,9 +14,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions-rs/toolchain@v1 with: - toolchain: nightly target: thumbv7em-none-eabi - override: true - uses: actions/setup-python@v1 with: python-version: 3.7 From f91538506e5ae7bb8b0e95ae355ba268e24bc99e Mon Sep 17 00:00:00 2001 From: Jean-Michel Picod Date: Fri, 28 Feb 2020 16:38:42 +0100 Subject: [PATCH 20/28] Restore path guards --- .github/workflows/cbor_test.yml | 4 ++-- .github/workflows/crypto_test.yml | 4 ++-- .github/workflows/opensk_test.yml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cbor_test.yml b/.github/workflows/cbor_test.yml index e1c73aa..a007752 100644 --- a/.github/workflows/cbor_test.yml +++ b/.github/workflows/cbor_test.yml @@ -2,8 +2,8 @@ name: CBOR tests on: push: -# paths: -# - 'libraries/cbor/**/*' + paths: + - 'libraries/cbor/**/*' pull_request: types: [opened, synchronize, reopened] diff --git a/.github/workflows/crypto_test.yml b/.github/workflows/crypto_test.yml index d79fd1f..c8f4544 100644 --- a/.github/workflows/crypto_test.yml +++ b/.github/workflows/crypto_test.yml @@ -2,8 +2,8 @@ name: Crypto library tests on: push: -# paths: -# - 'libraries/crypto/**/*' + paths: + - 'libraries/crypto/**/*' pull_request: types: [opened, synchronize, reopened] diff --git a/.github/workflows/opensk_test.yml b/.github/workflows/opensk_test.yml index 3e34ce5..1571ec7 100644 --- a/.github/workflows/opensk_test.yml +++ b/.github/workflows/opensk_test.yml @@ -2,8 +2,8 @@ name: OpenSK tests on: push: -# paths: -# - 'src/**/*.rs' + paths: + - 'src/**/*.rs' pull_request: types: [opened, synchronize, reopened] From ac42e86d376485abe889e1a1a517401f494cacf0 Mon Sep 17 00:00:00 2001 From: Jean-Michel Picod Date: Fri, 28 Feb 2020 16:54:16 +0100 Subject: [PATCH 21/28] Also ensure we build on OS X --- .github/workflows/opensk_build.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/opensk_build.yml b/.github/workflows/opensk_build.yml index 407a603..3cd5636 100644 --- a/.github/workflows/opensk_build.yml +++ b/.github/workflows/opensk_build.yml @@ -7,7 +7,10 @@ on: jobs: build_ctap2: - runs-on: ubuntu-18.04 + strategy: + matriX: + os: [ubuntu-18.04, macos-10.15] + runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 - uses: actions-rs/toolchain@v1 From 40632af1863894b3b68766bcc7cc95516442dc65 Mon Sep 17 00:00:00 2001 From: Jean-Michel Picod Date: Fri, 28 Feb 2020 16:55:18 +0100 Subject: [PATCH 22/28] Remove cargo audit on pull request as it fails --- .github/workflows/cargo_audit.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/cargo_audit.yml b/.github/workflows/cargo_audit.yml index b3f9897..32338d6 100644 --- a/.github/workflows/cargo_audit.yml +++ b/.github/workflows/cargo_audit.yml @@ -2,8 +2,6 @@ name: Security audit on: schedule: - cron: '0 0 * * *' - pull_request: - types: [opened, synchronize, reopened] jobs: audit: From e30ef9c984ef7f99c191a2b92dc0a18a54a7985e Mon Sep 17 00:00:00 2001 From: Jean-Michel Picod Date: Fri, 28 Feb 2020 16:56:15 +0100 Subject: [PATCH 23/28] Fix typo --- .github/workflows/opensk_build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/opensk_build.yml b/.github/workflows/opensk_build.yml index 3cd5636..a99438a 100644 --- a/.github/workflows/opensk_build.yml +++ b/.github/workflows/opensk_build.yml @@ -8,7 +8,7 @@ on: jobs: build_ctap2: strategy: - matriX: + matrix: os: [ubuntu-18.04, macos-10.15] runs-on: ${{ matrix.os }} steps: From 421f8194af4f15caf627a16e79f906b2b1efff30 Mon Sep 17 00:00:00 2001 From: Jean-Michel Picod Date: Fri, 28 Feb 2020 17:10:28 +0100 Subject: [PATCH 24/28] Gate also crypto unit tests in pull-request by paths These tests take 10 min. Let's avoid running them when not needed. --- .github/workflows/crypto_test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/crypto_test.yml b/.github/workflows/crypto_test.yml index c8f4544..790bee2 100644 --- a/.github/workflows/crypto_test.yml +++ b/.github/workflows/crypto_test.yml @@ -6,6 +6,8 @@ on: - 'libraries/crypto/**/*' pull_request: types: [opened, synchronize, reopened] + paths: + - 'libraries/crypto/**/*' jobs: crypto_test: From 73de02d1413819309bec95ba069cde56bedf443f Mon Sep 17 00:00:00 2001 From: Jean-Michel Picod Date: Mon, 2 Mar 2020 14:24:10 +0100 Subject: [PATCH 25/28] Try using strategy.matrix to run cargo check --- .github/workflows/opensk_test.yml | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/.github/workflows/opensk_test.yml b/.github/workflows/opensk_test.yml index 1571ec7..02c80d9 100644 --- a/.github/workflows/opensk_test.yml +++ b/.github/workflows/opensk_test.yml @@ -9,7 +9,12 @@ on: jobs: ctap2_test: + name: CTAP2 unit tests runs-on: ubuntu-18.04 + strategy: + matrix: + cargo_features: [std std,with_ctap1] + steps: - uses: actions/checkout@v2 - uses: actions-rs/toolchain@v1 @@ -27,22 +32,10 @@ jobs: uses: actions-rs/cargo@v1 with: command: test - args: --release --features std + args: --release --features ${{ cargo_features }} - name: Unit testing of CTAP2 (debug mode) uses: actions-rs/cargo@v1 with: command: test - args: --features std - - - name: Unit testing of CTAP2 (release mode + CTAP1) - uses: actions-rs/cargo@v1 - with: - command: test - args: --features std,with_ctap1 - - - name: Unit testing of CTAP2 (debug mode + CTAP1) - uses: actions-rs/cargo@v1 - with: - command: test - args: --features std,with_ctap1 + args: --features ${{ cargo_features }} From 72ac346d57e46705dda54a2e6d4a4a937aa32d8c Mon Sep 17 00:00:00 2001 From: Jean-Michel Picod Date: Mon, 2 Mar 2020 14:27:51 +0100 Subject: [PATCH 26/28] Revert strategy.matrix and fix missing release mode --- .github/workflows/opensk_test.yml | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/.github/workflows/opensk_test.yml b/.github/workflows/opensk_test.yml index 02c80d9..6b17763 100644 --- a/.github/workflows/opensk_test.yml +++ b/.github/workflows/opensk_test.yml @@ -11,9 +11,6 @@ jobs: ctap2_test: name: CTAP2 unit tests runs-on: ubuntu-18.04 - strategy: - matrix: - cargo_features: [std std,with_ctap1] steps: - uses: actions/checkout@v2 @@ -32,10 +29,23 @@ jobs: uses: actions-rs/cargo@v1 with: command: test - args: --release --features ${{ cargo_features }} + args: --release --features std - name: Unit testing of CTAP2 (debug mode) uses: actions-rs/cargo@v1 with: command: test - args: --features ${{ cargo_features }} + args: --features std + + - name: Unit testing of CTAP2 (release mode + CTAP1) + uses: actions-rs/cargo@v1 + with: + command: test + args: --release --features std,with_ctap1 + + - name: Unit testing of CTAP2 (debug mode + CTAP1) + uses: actions-rs/cargo@v1 + with: + command: test + args: --features std,with_ctap1 + From f9f27d7f897e46563922c6b20aef449299e3a94d Mon Sep 17 00:00:00 2001 From: Jean-Michel Picod Date: Mon, 2 Mar 2020 14:28:16 +0100 Subject: [PATCH 27/28] Try to avoid running cargo audit on forks (it fails) --- .github/workflows/cargo_audit.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/cargo_audit.yml b/.github/workflows/cargo_audit.yml index 32338d6..584dc20 100644 --- a/.github/workflows/cargo_audit.yml +++ b/.github/workflows/cargo_audit.yml @@ -6,6 +6,7 @@ on: jobs: audit: runs-on: ubuntu-18.04 + if: env.GITHUB_HEAD_REF == 0 steps: - uses: actions/checkout@v2 - uses: actions-rs/toolchain@v1 From 1c87efd27af94d065dc581b117bc93d0b7d752cc Mon Sep 17 00:00:00 2001 From: Jean-Michel Picod Date: Mon, 2 Mar 2020 15:13:19 +0100 Subject: [PATCH 28/28] Fix clippy warnings. Because of libtock-rs we can't run clippy on our codebase. But it could run at least on `build.rs`. --- build.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/build.rs b/build.rs index 70844b0..c8c9007 100644 --- a/build.rs +++ b/build.rs @@ -35,9 +35,8 @@ fn main() { // Load the OpenSSL PEM ECC key let ecc_data = include_bytes!("crypto_data/opensk.key"); - let pkey = ec::EcKey::private_key_from_pem(ecc_data) - .ok() - .expect("Failed to load OpenSK private key file"); + let pkey = + ec::EcKey::private_key_from_pem(ecc_data).expect("Failed to load OpenSK private key file"); // Check key validity pkey.check_key().unwrap(); @@ -70,9 +69,7 @@ fn main() { // Convert the PEM certificate to DER and extract the serial for AAGUID let input_pem_cert = include_bytes!("crypto_data/opensk_cert.pem"); - let cert = x509::X509::from_pem(input_pem_cert) - .ok() - .expect("Failed to load OpenSK certificate"); + let cert = x509::X509::from_pem(input_pem_cert).expect("Failed to load OpenSK certificate"); // Do some sanity check on the certificate assert!(cert