From 15233dba3eba0b71bf9a6ae2bf2c9184ae440342 Mon Sep 17 00:00:00 2001 From: Jean-Michel Picod Date: Tue, 7 Jun 2022 20:39:22 +0200 Subject: [PATCH] Fix pylint configuration, script and matcher (#491) * Fix pylint configuration, script and matcher * Makes yapf happy * Fix one-liner docstring * Fix one-liner docstring --- .pylintrc | 545 ++++++++++++++++++++++++++------------ fuzz/make_corpus.py | 34 +-- tools/configure.py | 3 +- tools/deploy_partition.py | 1 + tools/run_pylint.sh | 14 +- 5 files changed, 397 insertions(+), 200 deletions(-) diff --git a/.pylintrc b/.pylintrc index b0a913e..501e9f3 100644 --- a/.pylintrc +++ b/.pylintrc @@ -1,168 +1,266 @@ -# File taken from Tensor2Tensor project -# https://github.com/tensorflow/tensor2tensor/blob/master/pylintrc +# This Pylint rcfile contains a best-effort configuration to uphold the +# best-practices and style described in the Google Python style guide: +# https://google.github.io/styleguide/pyguide.html +# +# Its canonical open-source location is: +# https://google.github.io/styleguide/pylintrc -[MASTER] +[MAIN] + +# Files or directories to be skipped. They should be base names, not paths. +ignore=third_party + +# Files or directories matching the regex patterns are skipped. The regex +# matches against base names, not paths. +ignore-patterns= # Pickle collected data for later comparisons. persistent=no -# Set the cache size for astng objects. -cache-size=500 - -# Ignore Py3 files -ignore=get_references_web.py,get_references_web_single_group.py - +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. load-plugins= - pylint.extensions.bad_builtin, - pylint.extensions.docparams, - pylint.extensions.docstyle, - pylint.extensions.redefined_variable_type, - pylint.extensions.overlapping_exceptions, + +# Use multiple processes to speed up Pylint. +jobs=4 + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED +confidence= + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +#enable= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once).You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use"--disable=all --enable=classes +# --disable=W" +disable=abstract-method, + apply-builtin, + arguments-differ, + attribute-defined-outside-init, + backtick, + bad-option-value, + basestring-builtin, + buffer-builtin, + c-extension-no-member, + consider-using-enumerate, + cmp-builtin, + cmp-method, + coerce-builtin, + coerce-method, + delslice-method, + div-method, + duplicate-code, + eq-without-hash, + execfile-builtin, + file-builtin, + filter-builtin-not-iterating, + fixme, + getslice-method, + global-statement, + hex-method, + idiv-method, + implicit-str-concat-in-sequence, + import-error, + import-self, + import-star-module-level, + inconsistent-return-statements, + input-builtin, + intern-builtin, + invalid-str-codec, + locally-disabled, + long-builtin, + long-suffix, + map-builtin-not-iterating, + misplaced-comparison-constant, + missing-function-docstring, + metaclass-assignment, + next-method-called, + next-method-defined, + no-absolute-import, + no-else-break, + no-else-continue, + no-else-raise, + no-else-return, + no-init, # added + no-member, + no-name-in-module, + no-self-use, + nonzero-method, + oct-method, + old-division, + old-ne-operator, + old-octal-literal, + old-raise-syntax, + parameter-unpacking, + print-statement, + raising-string, + range-builtin-not-iterating, + raw_input-builtin, + rdiv-method, + reduce-builtin, + relative-import, + reload-builtin, + round-builtin, + setslice-method, + signature-differs, + standarderror-builtin, + suppressed-message, + sys-max-int, + too-few-public-methods, + too-many-ancestors, + too-many-arguments, + too-many-boolean-expressions, + too-many-branches, + too-many-instance-attributes, + too-many-locals, + too-many-nested-blocks, + too-many-public-methods, + too-many-return-statements, + too-many-statements, + trailing-newlines, + unichr-builtin, + unicode-builtin, + unnecessary-pass, + unpacking-in-except, + unrecognized-option, + useless-else-on-loop, + useless-object-inheritance, + useless-suppression, + using-cmp-argument, + wrong-import-order, + xrange-builtin, + zip-builtin-not-iterating, + [REPORTS] -# Set the output format. -# output-format=sorted-text +# Set the output format. Available formats are text, parseable, colorized, msvs +# (visual studio) and html. You can also give a reporter class, eg +# mypackage.mymodule.MyReporterClass. +output-format=text # Put messages in a separate file for each module / package specified on the # command line instead of printing them on stdout. Reports (if any) will be -# written in a file name "pylint_global.[txt|html]". +# written in a file name "pylint_global.[txt|html]". This option is deprecated +# and it will be removed in Pylint 2.0. files-output=no -# Tells whether to display a full report or only the messages. +# Tells whether to display a full report or only the messages reports=no -# Disable the report(s) with the given id(s). -disable-report=R0001,R0002,R0003,R0004,R0101,R0102,R0201,R0202,R0220,R0401,R0402,R0701,R0801,R0901,R0902,R0903,R0904,R0911,R0912,R0913,R0914,R0915,R0921,R0922,R0923 +# Python expression which should return a note less than 10 (10 is the highest +# note). You have access to the variables errors warning, statement which +# respectively contain the number of errors / warnings messages and the total +# number of statements analyzed. This is used by the global evaluation report +# (RP0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) -# Error message template (continued on second line) -msg-template={msg_id}:{line:3} {obj}: {msg} [{symbol}] +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details +#msg-template= -# We don't need evaluation score -score=no - -[MESSAGES CONTROL] -# List of checkers and warnings to enable. -enable=indexing-exception,old-raise-syntax - -# List of checkers and warnings to disable. -disable=design,similarities,no-self-use,attribute-defined-outside-init,locally-disabled,star-args,pointless-except,bad-option-value,global-statement,fixme,suppressed-message,useless-suppression,locally-enabled,file-ignored,multiple-imports,c-extension-no-member,trailing-newlines,unsubscriptable-object,misplaced-comparison-constant,no-member,abstract-method,no-else-return,missing-docstring,wrong-import-order,protected-access,inconsistent-return-statements,invalid-unary-operand-type,import-error,no-name-in-module,arguments-differ,not-context-manager,unused-argument [BASIC] -# Required attributes for module, separated by a comma -required-attributes= - -# Regular expression which should only match the name -# of functions or classes which do not require a docstring. -no-docstring-rgx=(__.*__|main) - -# Min length in lines of a function that requires a docstring. -docstring-min-length=10 - -# Regular expression which should only match correct module names. The -# leading underscore is sanctioned for private modules by Google's style -# guide. -# -# There are exceptions to the basic rule (_?[a-z][a-z0-9_]*) to cover -# requirements of Python's module system. -module-rgx=^(_?[a-z][a-z0-9_]*)|__init__$ - -# Regular expression which should only match correct module level names -const-rgx=^(_?[A-Z][A-Z0-9_]*|__[a-z0-9_]+__|_?[a-z][a-z0-9_]*)$ - -# Regular expression which should only match correct class attribute -class-attribute-rgx=^(_?[A-Z][A-Z0-9_]*|__[a-z0-9_]+__|_?[a-z][a-z0-9_]*)$ - -# Regular expression which should only match correct class names -class-rgx=^_?[A-Z][a-zA-Z0-9]*$ - -# Regular expression which should only match correct function names. -# 'camel_case' and 'snake_case' group names are used for consistency of naming -# styles across functions and methods. -function-rgx=^(?:(?PsetUp|tearDown|setUpModule|tearDownModule)|(?P_?[A-Z][a-zA-Z0-9]*)|(?P_?[a-z][a-z0-9_]*))$ - - -# Regular expression which should only match correct method names. -# 'camel_case' and 'snake_case' group names are used for consistency of naming -# styles across functions and methods. 'exempt' indicates a name which is -# consistent with all naming styles. -method-rgx=(?x) - ^(?:(?P_[a-z0-9_]+__|runTest|setUp|tearDown|setUpTestCase - |tearDownTestCase|setupSelf|tearDownClass|setUpClass - |(test|assert)_*[A-Z0-9][a-zA-Z0-9_]*|next) - |(?P_{0,2}[A-Z][a-zA-Z0-9_]*) - |(?P_{0,2}[a-z][a-z0-9_]*))$ - - -# Regular expression which should only match correct instance attribute names -attr-rgx=^_{0,2}[a-z][a-z0-9_]*$ - -# Regular expression which should only match correct argument names -argument-rgx=^[a-z][a-z0-9_]*$ - -# Regular expression which should only match correct variable names -variable-rgx=^[a-z][a-z0-9_]*$ - -# Regular expression which should only match correct list comprehension / -# generator expression variable names -inlinevar-rgx=^[a-z][a-z0-9_]*$ - # Good variable names which should always be accepted, separated by a comma good-names=main,_ # Bad variable names which should always be refused, separated by a comma bad-names= -# List of builtins function names that should not be used, separated by a comma -bad-functions=input,apply,reduce +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= -# List of decorators that define properties, such as abc.abstractproperty. -property-classes=abc.abstractproperty +# Include a hint for the correct naming format with invalid-name +include-naming-hint=no + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +property-classes=abc.abstractproperty,cached_property.cached_property,cached_property.threaded_cached_property,cached_property.cached_property_with_ttl,cached_property.threaded_cached_property_with_ttl + +# Regular expression matching correct function names +function-rgx=^(?:(?PsetUp|tearDown|setUpModule|tearDownModule)|(?P_?[A-Z][a-zA-Z0-9]*)|(?P_?[a-z][a-z0-9_]*))$ + +# Regular expression matching correct variable names +variable-rgx=^[a-z][a-z0-9_]*$ + +# Regular expression matching correct constant names +const-rgx=^(_?[A-Z][A-Z0-9_]*|__[a-z0-9_]+__|_?[a-z][a-z0-9_]*)$ + +# Regular expression matching correct attribute names +attr-rgx=^_{0,2}[a-z][a-z0-9_]*$ + +# Regular expression matching correct argument names +argument-rgx=^[a-z][a-z0-9_]*$ + +# Regular expression matching correct class attribute names +class-attribute-rgx=^(_?[A-Z][A-Z0-9_]*|__[a-z0-9_]+__|_?[a-z][a-z0-9_]*)$ + +# Regular expression matching correct inline iteration names +inlinevar-rgx=^[a-z][a-z0-9_]*$ + +# Regular expression matching correct class names +class-rgx=^_?[A-Z][a-zA-Z0-9]*$ + +# Regular expression matching correct module names +module-rgx=^(_?[a-z][a-z0-9_]*|__init__)$ + +# Regular expression matching correct method names +method-rgx=(?x)^(?:(?P_[a-z0-9_]+__|runTest|setUp|tearDown|setUpTestCase|tearDownTestCase|setupSelf|tearDownClass|setUpClass|(test|assert)_*[A-Z0-9][a-zA-Z0-9_]*|next)|(?P_{0,2}[A-Z][a-zA-Z0-9_]*)|(?P_{0,2}[a-z][a-z0-9_]*))$ + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=(__.*__|main|test.*|.*test|.*Test)$ + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=10 [TYPECHECK] +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager,contextlib2.contextmanager + # Tells whether missing members accessed in mixin class should be ignored. A # mixin class is detected if its name ends with "mixin" (case insensitive). ignore-mixin-members=yes -# List of decorators that create context managers from functions, such as -# contextlib.contextmanager. -contextmanager-decorators=contextlib.contextmanager,contextlib2.contextmanager +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis. It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local -[VARIABLES] - -# Tells whether we should check for unused import in __init__ files. -init-import=no - -# A regular expression matching names used for dummy variables (i.e. not used). -dummy-variables-rgx=^\*{0,2}(_$|unused_|dummy_) - -# List of additional names supposed to be defined in builtins. Remember that -# you should avoid to define new builtins when possible. -additional-builtins= - - -[CLASSES] - -# List of method names used to declare (i.e. assign) instance attributes. -defining-attr-methods=__init__,__new__,setUp - -# "class_" is also a valid for the first argument to a class method. -valid-classmethod-first-arg=cls,class_ - - -[EXCEPTIONS] - -overgeneral-exceptions=StandardError,Exception,BaseException - - -[IMPORTS] - -# Deprecated modules which should not be used, separated by a comma -deprecated-modules=regsub,TERMIOS,Bastion,rexec,sets +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= [FORMAT] @@ -170,66 +268,175 @@ deprecated-modules=regsub,TERMIOS,Bastion,rexec,sets # Maximum number of characters on a single line. max-line-length=80 +# TODO(https://github.com/PyCQA/pylint/issues/3352): Direct pylint to exempt +# lines made too long by directives to pytype. + # Regexp for a line that is allowed to be longer than the limit. -# This "ignore" regex is today composed of several independent parts: -# (1) Long import lines -# (2) URLs in comments or pydocs. Detecting URLs by regex is a hard problem and -# no amount of tweaking will make a perfect regex AFAICT. This one is a good -# compromise. -# (3) Constant string literals at the start of files don't need to be broken -# across lines. Allowing long paths and urls to be on a single -# line. Also requires that the string not be a triplequoted string. -ignore-long-lines=(?x) - (^\s*(import|from)\s - |^\s*(\#\ )??$ - |^[a-zA-Z_][a-zA-Z0-9_]*\s*=\s*("[^"]\S+"|'[^']\S+') - ) +ignore-long-lines=(?x)( + ^\s*(\#\ )??$| + ^\s*(from\s+\S+\s+)?import\s+.+$) + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=yes + +# List of optional constructs for which whitespace checking is disabled. `dict- +# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. +# `trailing-comma` allows a space between comma and closing bracket: (a, ). +# `empty-line` allows space-only lines. +no-space-check= # Maximum number of lines in a module max-module-lines=99999 -# String used as indentation unit. We differ from PEP8's normal 4 spaces. +# String used as indentation unit. The internal Google style guide mandates 2 +# spaces. Google's externaly-published style guide says 4, consistent with +# PEP 8. Here, we use 2 spaces, for conformity with many open-sourced Google +# projects (like TensorFlow). indent-string=' ' -# Do not warn about multiple statements on a single line for constructs like -# if test: stmt -single-line-if-stmt=y +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 -# Make sure : in dicts and trailing commas are checked for whitespace. -no-space-check= - - -[LOGGING] - -# Add logging modules. -logging-modules=logging,absl.logging +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= [MISCELLANEOUS] # List of note tags to take in consideration, separated by a comma. -notes= +notes=TODO -# Maximum line length for lambdas -short-func-length=1 +[STRING] -# List of module members that should be marked as deprecated. -# All of the string functions are listed in 4.1.4 Deprecated string functions -# in the Python 2.4 docs. -deprecated-members=string.atof,string.atoi,string.atol,string.capitalize,string.expandtabs,string.find,string.rfind,string.index,string.rindex,string.count,string.lower,string.split,string.rsplit,string.splitfields,string.join,string.joinfields,string.lstrip,string.rstrip,string.strip,string.swapcase,string.translate,string.upper,string.ljust,string.rjust,string.center,string.zfill,string.replace,sys.exitfunc,sys.maxint +# This flag controls whether inconsistent-quotes generates a warning when the +# character used as a quote delimiter is used inconsistently within a module. +check-quote-consistency=yes -# List of exceptions that do not need to be mentioned in the Raises section of -# a docstring. -ignore-exceptions=AssertionError,NotImplementedError,StopIteration,TypeError +[VARIABLES] + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# A regular expression matching the name of dummy variables (i.e. expectedly +# not used). +dummy-variables-rgx=^\*{0,2}(_$|unused_|dummy_) + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid to define new builtins when possible. +additional-builtins= + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_,_cb + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six,six.moves,past.builtins,future.builtins,functools -# Number of spaces of indent required when the last token on the preceding line -# is an open (, [, or {. -indent-after-paren=4 +[LOGGING] -# Set the linting for string quotes -string-quote=double -triple-quote=double -docstring-quote=double +# Logging modules to check that the string format arguments are in logging +# function parameter format +logging-modules=logging,absl.logging,tensorflow.io.logging + + +[SIMILARITIES] + +# Minimum lines number of a similarity. +min-similarity-lines=4 + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + +# Ignore imports when computing similarities. +ignore-imports=no + + +[SPELLING] + +# Spelling dictionary name. Available dictionaries: none. To make it working +# install python-enchant package. +spelling-dict= + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to indicated private dictionary in +# --spelling-private-dict-file option instead of raising a message. +spelling-store-unknown-words=no + + +[IMPORTS] + +# Deprecated modules which should not be used, separated by a comma +deprecated-modules=regsub, + TERMIOS, + Bastion, + rexec, + sets + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled) +import-graph= + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled) +ext-import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled) +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant, absl + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + + +[CLASSES] + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__, + __new__, + setUp + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict, + _fields, + _replace, + _source, + _make + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls, + class_ + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "Exception" +overgeneral-exceptions=StandardError, + Exception, + BaseException diff --git a/fuzz/make_corpus.py b/fuzz/make_corpus.py index a6bd9ff..776fa54 100644 --- a/fuzz/make_corpus.py +++ b/fuzz/make_corpus.py @@ -1,24 +1,24 @@ +"""Creates a directory containing seed inputs from a json file having + the following structure: + [ + { + "hex": "a901a1182a182a02a3626964781a6d616b655f6261645f7...", + "cbor": "{1: h'42', 2: {\"id\": \"make.example.com\", ...", + "description": "make credential parameters" + }, + ... + ] + + Usage: + - pass the resulting corpus directory path as the first argument + - pass the json file path to make the corpus from as the second argument + Example: + python make_corpus.py ./corpus ./corpus_file.json +""" import argparse import json import os.path -# Creates a directory containing seed inputs from a json file having -# the following structure: -# [ -# { -# "hex": "a901a1182a182a02a3626964781a6d616b655f6261645f7...", -# "cbor": "{1: h'42', 2: {\"id\": \"make.example.com\", ...", -# "description": "make credential parameters" -# }, -# ... -# ] -# -# Usage: -# - pass the resulting corpus directory path as the first argument -# - pass the json file path to make the corpus from as the second argument -# Example: -# python make_corpus.py ./corpus ./corpus_file.json - # Creates a corpus directory to the given path from the given json file. def make_corpus(corpus_dir, corpus_json): diff --git a/tools/configure.py b/tools/configure.py index 52c4728..0d90630 100755 --- a/tools/configure.py +++ b/tools/configure.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # Lint as: python3 +"""Tools that implements vendor-specific CTAP2 commands to configure OpenSK.""" from __future__ import absolute_import from __future__ import division @@ -121,7 +122,7 @@ def main(args): cert.public_bytes(serialization.Encoding.DER), 2: priv_key.private_numbers().private_value.to_bytes( - length=32, byteorder='big', signed=False) + length=32, byteorder="big", signed=False) } devices = get_opensk_devices(args.batch) diff --git a/tools/deploy_partition.py b/tools/deploy_partition.py index 8ed6312..e524bd6 100755 --- a/tools/deploy_partition.py +++ b/tools/deploy_partition.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # Lint as: python3 +"""Tool that is part of firmware upgrabability in OpenSK.""" from __future__ import absolute_import from __future__ import division diff --git a/tools/run_pylint.sh b/tools/run_pylint.sh index 2953e4f..58a7cb0 100755 --- a/tools/run_pylint.sh +++ b/tools/run_pylint.sh @@ -13,19 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -SUCCESS=0 - # Ensure we are at the project root directory cd $(readlink -f $(dirname $0))/.. -for file in `find . ! -path "./third_party/*" -type f -name '*.py'` -do - # Output header for our custom matcher on Github workflow - echo "PYLINT:${file}" - if ! pylint --rcfile=.pylintrc --score=n "$file" - then - SUCCESS=1 - fi -done - -exit $SUCCESS +pylint --score=n `git ls-files --deduplicate --exclude-standard --full-name '*.py'`