From patchwork Tue May 14 12:05:00 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Luca Vizzarro X-Patchwork-Id: 140060 X-Patchwork-Delegate: thomas@monjalon.net Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 4FE434402B; Tue, 14 May 2024 14:05:17 +0200 (CEST) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 104C54064C; Tue, 14 May 2024 14:05:13 +0200 (CEST) Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by mails.dpdk.org (Postfix) with ESMTP id 19A9A40156 for ; Tue, 14 May 2024 14:05:11 +0200 (CEST) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 8876C1007; Tue, 14 May 2024 05:05:35 -0700 (PDT) Received: from localhost.localdomain (unknown [10.57.4.224]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id 606E33F641; Tue, 14 May 2024 05:05:09 -0700 (PDT) From: Luca Vizzarro To: dev@dpdk.org Cc: =?utf-8?q?Juraj_Linke=C5=A1?= , Jeremy Spewock , Luca Vizzarro , Paul Szczepanek Subject: [PATCH v4 1/3] dts: update mypy static checker Date: Tue, 14 May 2024 13:05:00 +0100 Message-Id: <20240514120502.1955468-2-luca.vizzarro@arm.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20240514120502.1955468-1-luca.vizzarro@arm.com> References: <20240122182611.1904974-1-luca.vizzarro@arm.com> <20240514120502.1955468-1-luca.vizzarro@arm.com> MIME-Version: 1.0 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Update the mypy static checker to the latest version and fix all the reported errors. Bugzilla ID: 1433 Signed-off-by: Luca Vizzarro Reviewed-by: Paul Szczepanek --- dts/framework/config/__init__.py | 2 +- dts/framework/logger.py | 4 +- .../interactive_remote_session.py | 4 +- .../remote_session/interactive_shell.py | 2 +- dts/framework/remote_session/ssh_session.py | 6 +- dts/framework/runner.py | 14 +-- dts/framework/test_result.py | 4 +- dts/framework/test_suite.py | 6 +- dts/framework/testbed_model/tg_node.py | 2 +- .../capturing_traffic_generator.py | 4 +- .../testbed_model/traffic_generator/scapy.py | 6 +- .../traffic_generator/traffic_generator.py | 2 +- dts/framework/utils.py | 2 +- dts/poetry.lock | 86 +++++++++++-------- dts/pyproject.toml | 2 +- dts/tests/TestSuite_os_udp.py | 4 +- dts/tests/TestSuite_pmd_buffer_scatter.py | 8 +- 17 files changed, 87 insertions(+), 71 deletions(-) diff --git a/dts/framework/config/__init__.py b/dts/framework/config/__init__.py index 4cb5c74059..6b2ad2b16d 100644 --- a/dts/framework/config/__init__.py +++ b/dts/framework/config/__init__.py @@ -40,7 +40,7 @@ from pathlib import Path from typing import Union -import warlock # type: ignore[import] +import warlock # type: ignore[import-untyped] import yaml from framework.config.types import ( diff --git a/dts/framework/logger.py b/dts/framework/logger.py index fc6c50c983..5b88725481 100644 --- a/dts/framework/logger.py +++ b/dts/framework/logger.py @@ -79,7 +79,7 @@ def makeRecord(self, *args, **kwargs) -> logging.LogRecord: record: The generated record with the stage information. """ record = super().makeRecord(*args, **kwargs) - record.stage = DTSLogger._stage # type: ignore[attr-defined] + record.stage = DTSLogger._stage return record def add_dts_root_logger_handlers(self, verbose: bool, output_dir: str) -> None: @@ -178,7 +178,7 @@ def _remove_extra_file_handlers(self) -> None: self._extra_file_handlers = [] -def get_dts_logger(name: str = None) -> DTSLogger: +def get_dts_logger(name: str | None = None) -> DTSLogger: """Return a DTS logger instance identified by `name`. Args: diff --git a/dts/framework/remote_session/interactive_remote_session.py b/dts/framework/remote_session/interactive_remote_session.py index c50790db79..97194e6af8 100644 --- a/dts/framework/remote_session/interactive_remote_session.py +++ b/dts/framework/remote_session/interactive_remote_session.py @@ -6,8 +6,8 @@ import socket import traceback -from paramiko import AutoAddPolicy, SSHClient, Transport # type: ignore[import] -from paramiko.ssh_exception import ( # type: ignore[import] +from paramiko import AutoAddPolicy, SSHClient, Transport # type: ignore[import-untyped] +from paramiko.ssh_exception import ( # type: ignore[import-untyped] AuthenticationException, BadHostKeyException, NoValidConnectionsError, diff --git a/dts/framework/remote_session/interactive_shell.py b/dts/framework/remote_session/interactive_shell.py index 5cfe202e15..074a541279 100644 --- a/dts/framework/remote_session/interactive_shell.py +++ b/dts/framework/remote_session/interactive_shell.py @@ -18,7 +18,7 @@ from pathlib import PurePath from typing import Callable, ClassVar -from paramiko import Channel, SSHClient, channel # type: ignore[import] +from paramiko import Channel, SSHClient, channel # type: ignore[import-untyped] from framework.logger import DTSLogger from framework.settings import SETTINGS diff --git a/dts/framework/remote_session/ssh_session.py b/dts/framework/remote_session/ssh_session.py index 782220092c..216bd25aed 100644 --- a/dts/framework/remote_session/ssh_session.py +++ b/dts/framework/remote_session/ssh_session.py @@ -7,13 +7,13 @@ import traceback from pathlib import PurePath -from fabric import Connection # type: ignore[import] -from invoke.exceptions import ( # type: ignore[import] +from fabric import Connection # type: ignore[import-untyped] +from invoke.exceptions import ( # type: ignore[import-untyped] CommandTimedOut, ThreadException, UnexpectedExit, ) -from paramiko.ssh_exception import ( # type: ignore[import] +from paramiko.ssh_exception import ( # type: ignore[import-untyped] AuthenticationException, BadHostKeyException, NoValidConnectionsError, diff --git a/dts/framework/runner.py b/dts/framework/runner.py index db8e3ba96b..d74f1871db 100644 --- a/dts/framework/runner.py +++ b/dts/framework/runner.py @@ -23,7 +23,7 @@ import re import sys from pathlib import Path -from types import MethodType +from types import FunctionType from typing import Iterable, Sequence from .config import ( @@ -132,8 +132,8 @@ def run(self): the :option:`--test-suite` command line argument or the :envvar:`DTS_TESTCASES` environment variable. """ - sut_nodes: dict[str, SutNode] = {} - tg_nodes: dict[str, TGNode] = {} + sut_nodes = {} + tg_nodes = {} try: # check the python version of the server that runs dts self._check_dts_python_version() @@ -305,7 +305,7 @@ def is_test_suite(object) -> bool: def _filter_test_cases( self, test_suite_class: type[TestSuite], test_cases_to_run: Sequence[str] - ) -> tuple[list[MethodType], list[MethodType]]: + ) -> tuple[list[FunctionType], list[FunctionType]]: """Filter `test_cases_to_run` from `test_suite_class`. There are two rounds of filtering if `test_cases_to_run` is not empty. @@ -593,7 +593,7 @@ def _run_test_suite( def _execute_test_suite( self, test_suite: TestSuite, - test_cases: Iterable[MethodType], + test_cases: Iterable[FunctionType], test_suite_result: TestSuiteResult, ) -> None: """Execute all `test_cases` in `test_suite`. @@ -626,7 +626,7 @@ def _execute_test_suite( def _run_test_case( self, test_suite: TestSuite, - test_case_method: MethodType, + test_case_method: FunctionType, test_case_result: TestCaseResult, ) -> None: """Setup, execute and teardown `test_case_method` from `test_suite`. @@ -672,7 +672,7 @@ def _run_test_case( def _execute_test_case( self, test_suite: TestSuite, - test_case_method: MethodType, + test_case_method: FunctionType, test_case_result: TestCaseResult, ) -> None: """Execute `test_case_method` from `test_suite`, record the result and handle failures. diff --git a/dts/framework/test_result.py b/dts/framework/test_result.py index 28f84fd793..d8d0fe2b2b 100644 --- a/dts/framework/test_result.py +++ b/dts/framework/test_result.py @@ -27,7 +27,7 @@ from collections.abc import MutableSequence from dataclasses import dataclass from enum import Enum, auto -from types import MethodType +from types import FunctionType from typing import Union from .config import ( @@ -63,7 +63,7 @@ class is to hold a subset of test cases (which could be all test cases) because """ test_suite_class: type[TestSuite] - test_cases: list[MethodType] + test_cases: list[FunctionType] def create_config(self) -> TestSuiteConfig: """Generate a :class:`TestSuiteConfig` from the stored test suite with test cases. diff --git a/dts/framework/test_suite.py b/dts/framework/test_suite.py index 9c3b516002..8768f756a6 100644 --- a/dts/framework/test_suite.py +++ b/dts/framework/test_suite.py @@ -16,9 +16,9 @@ from ipaddress import IPv4Interface, IPv6Interface, ip_interface from typing import ClassVar, Union -from scapy.layers.inet import IP # type: ignore[import] -from scapy.layers.l2 import Ether # type: ignore[import] -from scapy.packet import Packet, Padding # type: ignore[import] +from scapy.layers.inet import IP # type: ignore[import-untyped] +from scapy.layers.l2 import Ether # type: ignore[import-untyped] +from scapy.packet import Packet, Padding # type: ignore[import-untyped] from .exception import TestCaseVerifyError from .logger import DTSLogger, get_dts_logger diff --git a/dts/framework/testbed_model/tg_node.py b/dts/framework/testbed_model/tg_node.py index d3206e87e0..164f790383 100644 --- a/dts/framework/testbed_model/tg_node.py +++ b/dts/framework/testbed_model/tg_node.py @@ -9,7 +9,7 @@ A TG node is where the TG runs. """ -from scapy.packet import Packet # type: ignore[import] +from scapy.packet import Packet # type: ignore[import-untyped] from framework.config import TGNodeConfiguration diff --git a/dts/framework/testbed_model/traffic_generator/capturing_traffic_generator.py b/dts/framework/testbed_model/traffic_generator/capturing_traffic_generator.py index e5a1560e90..c8380b7d57 100644 --- a/dts/framework/testbed_model/traffic_generator/capturing_traffic_generator.py +++ b/dts/framework/testbed_model/traffic_generator/capturing_traffic_generator.py @@ -13,8 +13,8 @@ from abc import abstractmethod from dataclasses import dataclass -import scapy.utils # type: ignore[import] -from scapy.packet import Packet # type: ignore[import] +import scapy.utils # type: ignore[import-untyped] +from scapy.packet import Packet # type: ignore[import-untyped] from framework.settings import SETTINGS from framework.testbed_model.port import Port diff --git a/dts/framework/testbed_model/traffic_generator/scapy.py b/dts/framework/testbed_model/traffic_generator/scapy.py index df3069d516..ed5467d825 100644 --- a/dts/framework/testbed_model/traffic_generator/scapy.py +++ b/dts/framework/testbed_model/traffic_generator/scapy.py @@ -20,9 +20,9 @@ import xmlrpc.client from xmlrpc.server import SimpleXMLRPCServer -import scapy.all # type: ignore[import] -from scapy.layers.l2 import Ether # type: ignore[import] -from scapy.packet import Packet # type: ignore[import] +import scapy.all # type: ignore[import-untyped] +from scapy.layers.l2 import Ether # type: ignore[import-untyped] +from scapy.packet import Packet # type: ignore[import-untyped] from framework.config import OS, ScapyTrafficGeneratorConfig from framework.remote_session import PythonShell diff --git a/dts/framework/testbed_model/traffic_generator/traffic_generator.py b/dts/framework/testbed_model/traffic_generator/traffic_generator.py index d86d7fb532..4ce1148706 100644 --- a/dts/framework/testbed_model/traffic_generator/traffic_generator.py +++ b/dts/framework/testbed_model/traffic_generator/traffic_generator.py @@ -10,7 +10,7 @@ from abc import ABC, abstractmethod -from scapy.packet import Packet # type: ignore[import] +from scapy.packet import Packet # type: ignore[import-untyped] from framework.config import TrafficGeneratorConfig from framework.logger import DTSLogger, get_dts_logger diff --git a/dts/framework/utils.py b/dts/framework/utils.py index cc5e458cc8..74a11f1aaf 100644 --- a/dts/framework/utils.py +++ b/dts/framework/utils.py @@ -21,7 +21,7 @@ from pathlib import Path from subprocess import SubprocessError -from scapy.packet import Packet # type: ignore[import] +from scapy.packet import Packet # type: ignore[import-untyped] from .exception import ConfigurationError diff --git a/dts/poetry.lock b/dts/poetry.lock index a734fa71f0..df9cecb7e0 100644 --- a/dts/poetry.lock +++ b/dts/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. [[package]] name = "attrs" @@ -353,44 +353,49 @@ files = [ [[package]] name = "mypy" -version = "0.961" +version = "1.10.0" description = "Optional static typing for Python" optional = false -python-versions = ">=3.6" +python-versions = ">=3.8" files = [ - {file = "mypy-0.961-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:697540876638ce349b01b6786bc6094ccdaba88af446a9abb967293ce6eaa2b0"}, - {file = "mypy-0.961-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b117650592e1782819829605a193360a08aa99f1fc23d1d71e1a75a142dc7e15"}, - {file = "mypy-0.961-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:bdd5ca340beffb8c44cb9dc26697628d1b88c6bddf5c2f6eb308c46f269bb6f3"}, - {file = "mypy-0.961-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:3e09f1f983a71d0672bbc97ae33ee3709d10c779beb613febc36805a6e28bb4e"}, - {file = "mypy-0.961-cp310-cp310-win_amd64.whl", hash = "sha256:e999229b9f3198c0c880d5e269f9f8129c8862451ce53a011326cad38b9ccd24"}, - {file = "mypy-0.961-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:b24be97351084b11582fef18d79004b3e4db572219deee0212078f7cf6352723"}, - {file = "mypy-0.961-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f4a21d01fc0ba4e31d82f0fff195682e29f9401a8bdb7173891070eb260aeb3b"}, - {file = "mypy-0.961-cp36-cp36m-win_amd64.whl", hash = "sha256:439c726a3b3da7ca84a0199a8ab444cd8896d95012c4a6c4a0d808e3147abf5d"}, - {file = "mypy-0.961-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5a0b53747f713f490affdceef835d8f0cb7285187a6a44c33821b6d1f46ed813"}, - {file = "mypy-0.961-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:0e9f70df36405c25cc530a86eeda1e0867863d9471fe76d1273c783df3d35c2e"}, - {file = "mypy-0.961-cp37-cp37m-win_amd64.whl", hash = "sha256:b88f784e9e35dcaa075519096dc947a388319cb86811b6af621e3523980f1c8a"}, - {file = "mypy-0.961-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:d5aaf1edaa7692490f72bdb9fbd941fbf2e201713523bdb3f4038be0af8846c6"}, - {file = "mypy-0.961-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9f5f5a74085d9a81a1f9c78081d60a0040c3efb3f28e5c9912b900adf59a16e6"}, - {file = "mypy-0.961-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:f4b794db44168a4fc886e3450201365c9526a522c46ba089b55e1f11c163750d"}, - {file = "mypy-0.961-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:64759a273d590040a592e0f4186539858c948302c653c2eac840c7a3cd29e51b"}, - {file = "mypy-0.961-cp38-cp38-win_amd64.whl", hash = "sha256:63e85a03770ebf403291ec50097954cc5caf2a9205c888ce3a61bd3f82e17569"}, - {file = "mypy-0.961-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:5f1332964963d4832a94bebc10f13d3279be3ce8f6c64da563d6ee6e2eeda932"}, - {file = "mypy-0.961-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:006be38474216b833eca29ff6b73e143386f352e10e9c2fbe76aa8549e5554f5"}, - {file = "mypy-0.961-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9940e6916ed9371809b35b2154baf1f684acba935cd09928952310fbddaba648"}, - {file = "mypy-0.961-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:a5ea0875a049de1b63b972456542f04643daf320d27dc592d7c3d9cd5d9bf950"}, - {file = "mypy-0.961-cp39-cp39-win_amd64.whl", hash = "sha256:1ece702f29270ec6af25db8cf6185c04c02311c6bb21a69f423d40e527b75c56"}, - {file = "mypy-0.961-py3-none-any.whl", hash = "sha256:03c6cc893e7563e7b2949b969e63f02c000b32502a1b4d1314cabe391aa87d66"}, - {file = "mypy-0.961.tar.gz", hash = "sha256:f730d56cb924d371c26b8eaddeea3cc07d78ff51c521c6d04899ac6904b75492"}, + {file = "mypy-1.10.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:da1cbf08fb3b851ab3b9523a884c232774008267b1f83371ace57f412fe308c2"}, + {file = "mypy-1.10.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:12b6bfc1b1a66095ab413160a6e520e1dc076a28f3e22f7fb25ba3b000b4ef99"}, + {file = "mypy-1.10.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9e36fb078cce9904c7989b9693e41cb9711e0600139ce3970c6ef814b6ebc2b2"}, + {file = "mypy-1.10.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2b0695d605ddcd3eb2f736cd8b4e388288c21e7de85001e9f85df9187f2b50f9"}, + {file = "mypy-1.10.0-cp310-cp310-win_amd64.whl", hash = "sha256:cd777b780312ddb135bceb9bc8722a73ec95e042f911cc279e2ec3c667076051"}, + {file = "mypy-1.10.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3be66771aa5c97602f382230165b856c231d1277c511c9a8dd058be4784472e1"}, + {file = "mypy-1.10.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8b2cbaca148d0754a54d44121b5825ae71868c7592a53b7292eeb0f3fdae95ee"}, + {file = "mypy-1.10.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ec404a7cbe9fc0e92cb0e67f55ce0c025014e26d33e54d9e506a0f2d07fe5de"}, + {file = "mypy-1.10.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e22e1527dc3d4aa94311d246b59e47f6455b8729f4968765ac1eacf9a4760bc7"}, + {file = "mypy-1.10.0-cp311-cp311-win_amd64.whl", hash = "sha256:a87dbfa85971e8d59c9cc1fcf534efe664d8949e4c0b6b44e8ca548e746a8d53"}, + {file = "mypy-1.10.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:a781f6ad4bab20eef8b65174a57e5203f4be627b46291f4589879bf4e257b97b"}, + {file = "mypy-1.10.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b808e12113505b97d9023b0b5e0c0705a90571c6feefc6f215c1df9381256e30"}, + {file = "mypy-1.10.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f55583b12156c399dce2df7d16f8a5095291354f1e839c252ec6c0611e86e2e"}, + {file = "mypy-1.10.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4cf18f9d0efa1b16478c4c129eabec36148032575391095f73cae2e722fcf9d5"}, + {file = "mypy-1.10.0-cp312-cp312-win_amd64.whl", hash = "sha256:bc6ac273b23c6b82da3bb25f4136c4fd42665f17f2cd850771cb600bdd2ebeda"}, + {file = "mypy-1.10.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9fd50226364cd2737351c79807775136b0abe084433b55b2e29181a4c3c878c0"}, + {file = "mypy-1.10.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:f90cff89eea89273727d8783fef5d4a934be2fdca11b47def50cf5d311aff727"}, + {file = "mypy-1.10.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fcfc70599efde5c67862a07a1aaf50e55bce629ace26bb19dc17cece5dd31ca4"}, + {file = "mypy-1.10.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:075cbf81f3e134eadaf247de187bd604748171d6b79736fa9b6c9685b4083061"}, + {file = "mypy-1.10.0-cp38-cp38-win_amd64.whl", hash = "sha256:3f298531bca95ff615b6e9f2fc0333aae27fa48052903a0ac90215021cdcfa4f"}, + {file = "mypy-1.10.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:fa7ef5244615a2523b56c034becde4e9e3f9b034854c93639adb667ec9ec2976"}, + {file = "mypy-1.10.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3236a4c8f535a0631f85f5fcdffba71c7feeef76a6002fcba7c1a8e57c8be1ec"}, + {file = "mypy-1.10.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4a2b5cdbb5dd35aa08ea9114436e0d79aceb2f38e32c21684dcf8e24e1e92821"}, + {file = "mypy-1.10.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:92f93b21c0fe73dc00abf91022234c79d793318b8a96faac147cd579c1671746"}, + {file = "mypy-1.10.0-cp39-cp39-win_amd64.whl", hash = "sha256:28d0e038361b45f099cc086d9dd99c15ff14d0188f44ac883010e172ce86c38a"}, + {file = "mypy-1.10.0-py3-none-any.whl", hash = "sha256:f8c083976eb530019175aabadb60921e73b4f45736760826aa1689dda8208aee"}, + {file = "mypy-1.10.0.tar.gz", hash = "sha256:3d087fcbec056c4ee34974da493a826ce316947485cef3901f511848e687c131"}, ] [package.dependencies] -mypy-extensions = ">=0.4.3" +mypy-extensions = ">=1.0.0" tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} -typing-extensions = ">=3.10" +typing-extensions = ">=4.1.0" [package.extras] dmypy = ["psutil (>=4.0)"] -python2 = ["typed-ast (>=1.4.0,<2)"] +install-types = ["pip"] +mypyc = ["setuptools (>=50)"] reports = ["lxml"] [[package]] @@ -580,6 +585,7 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -587,8 +593,16 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -605,6 +619,7 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -612,6 +627,7 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -810,13 +826,13 @@ files = [ [[package]] name = "typing-extensions" -version = "4.7.1" -description = "Backported and Experimental Type Hints for Python 3.7+" +version = "4.11.0" +description = "Backported and Experimental Type Hints for Python 3.8+" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "typing_extensions-4.7.1-py3-none-any.whl", hash = "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36"}, - {file = "typing_extensions-4.7.1.tar.gz", hash = "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2"}, + {file = "typing_extensions-4.11.0-py3-none-any.whl", hash = "sha256:c1f94d72897edaf4ce775bb7558d5b79d8126906a14ea5ed1635921406c0387a"}, + {file = "typing_extensions-4.11.0.tar.gz", hash = "sha256:83f085bd5ca59c80295fc2a82ab5dac679cbe02b9f33f7d83af68e241bea51b0"}, ] [[package]] @@ -837,4 +853,4 @@ jsonschema = ">=4,<5" [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "3501e97b3dadc19fe8ae179fe21b1edd2488001da9a8e86ff2bca0b86b99b89b" +content-hash = "1572cb14f0bf88cddc6b6b225312ad468d01916b32067d17a7776af76c6d466c" diff --git a/dts/pyproject.toml b/dts/pyproject.toml index a81e46fc07..05c91ef9be 100644 --- a/dts/pyproject.toml +++ b/dts/pyproject.toml @@ -28,7 +28,7 @@ scapy = "^2.5.0" pydocstyle = "6.1.1" [tool.poetry.group.dev.dependencies] -mypy = "^0.961" +mypy = "^1.10.0" black = "^22.6.0" isort = "^5.10.1" pylama = "^8.4.1" diff --git a/dts/tests/TestSuite_os_udp.py b/dts/tests/TestSuite_os_udp.py index b4784dd95e..a78bd74139 100644 --- a/dts/tests/TestSuite_os_udp.py +++ b/dts/tests/TestSuite_os_udp.py @@ -7,8 +7,8 @@ Send a packet to the SUT node, verify it comes back on the second port on the TG node. """ -from scapy.layers.inet import IP, UDP # type: ignore[import] -from scapy.layers.l2 import Ether # type: ignore[import] +from scapy.layers.inet import IP, UDP # type: ignore[import-untyped] +from scapy.layers.l2 import Ether # type: ignore[import-untyped] from framework.test_suite import TestSuite diff --git a/dts/tests/TestSuite_pmd_buffer_scatter.py b/dts/tests/TestSuite_pmd_buffer_scatter.py index 3701c47408..a020682e8d 100644 --- a/dts/tests/TestSuite_pmd_buffer_scatter.py +++ b/dts/tests/TestSuite_pmd_buffer_scatter.py @@ -17,10 +17,10 @@ import struct -from scapy.layers.inet import IP # type: ignore[import] -from scapy.layers.l2 import Ether # type: ignore[import] -from scapy.packet import Raw # type: ignore[import] -from scapy.utils import hexstr # type: ignore[import] +from scapy.layers.inet import IP # type: ignore[import-untyped] +from scapy.layers.l2 import Ether # type: ignore[import-untyped] +from scapy.packet import Raw # type: ignore[import-untyped] +from scapy.utils import hexstr # type: ignore[import-untyped] from framework.remote_session.testpmd_shell import TestPmdForwardingModes, TestPmdShell from framework.test_suite import TestSuite From patchwork Tue May 14 12:05:01 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Luca Vizzarro X-Patchwork-Id: 140061 X-Patchwork-Delegate: thomas@monjalon.net Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 39EEC4402B; Tue, 14 May 2024 14:05:25 +0200 (CEST) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 504DA4067D; Tue, 14 May 2024 14:05:14 +0200 (CEST) Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by mails.dpdk.org (Postfix) with ESMTP id 4A3D2402F2 for ; Tue, 14 May 2024 14:05:12 +0200 (CEST) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id BE0661042; Tue, 14 May 2024 05:05:36 -0700 (PDT) Received: from localhost.localdomain (unknown [10.57.4.224]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id EC6AB3F641; Tue, 14 May 2024 05:05:10 -0700 (PDT) From: Luca Vizzarro To: dev@dpdk.org Cc: =?utf-8?q?Juraj_Linke=C5=A1?= , Jeremy Spewock , Luca Vizzarro , Paul Szczepanek Subject: [PATCH v4 2/3] dts: clean up config types Date: Tue, 14 May 2024 13:05:01 +0100 Message-Id: <20240514120502.1955468-3-luca.vizzarro@arm.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20240514120502.1955468-1-luca.vizzarro@arm.com> References: <20240122182611.1904974-1-luca.vizzarro@arm.com> <20240514120502.1955468-1-luca.vizzarro@arm.com> MIME-Version: 1.0 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Clean up types used with the configuration classes, and use Self from the newly added typing_extensions module. Methods that instantiate their own class should be @classmethod instead of @staticmethod. Bugzilla ID: 1433 Signed-off-by: Luca Vizzarro Reviewed-by: Paul Szczepanek --- dts/framework/config/__init__.py | 47 ++++++++++--------- .../traffic_generator/__init__.py | 10 ++-- dts/poetry.lock | 2 +- dts/pyproject.toml | 1 + 4 files changed, 32 insertions(+), 28 deletions(-) diff --git a/dts/framework/config/__init__.py b/dts/framework/config/__init__.py index 6b2ad2b16d..41b2fbc94d 100644 --- a/dts/framework/config/__init__.py +++ b/dts/framework/config/__init__.py @@ -42,6 +42,7 @@ import warlock # type: ignore[import-untyped] import yaml +from typing_extensions import Self from framework.config.types import ( BuildTargetConfigDict, @@ -156,8 +157,8 @@ class PortConfig: peer_node: str peer_pci: str - @staticmethod - def from_dict(node: str, d: PortConfigDict) -> "PortConfig": + @classmethod + def from_dict(cls, node: str, d: PortConfigDict) -> Self: """A convenience method that creates the object from fewer inputs. Args: @@ -167,7 +168,7 @@ def from_dict(node: str, d: PortConfigDict) -> "PortConfig": Returns: The port configuration instance. """ - return PortConfig(node=node, **d) + return cls(node=node, **d) @dataclass(slots=True, frozen=True) @@ -183,7 +184,7 @@ class TrafficGeneratorConfig: traffic_generator_type: TrafficGeneratorType @staticmethod - def from_dict(d: TrafficGeneratorConfigDict) -> "ScapyTrafficGeneratorConfig": + def from_dict(d: TrafficGeneratorConfigDict) -> "TrafficGeneratorConfig": """A convenience method that produces traffic generator config of the proper type. Args: @@ -312,7 +313,7 @@ class TGNodeConfiguration(NodeConfiguration): traffic_generator: The configuration of the traffic generator present on the TG node. """ - traffic_generator: ScapyTrafficGeneratorConfig + traffic_generator: TrafficGeneratorConfig @dataclass(slots=True, frozen=True) @@ -356,8 +357,8 @@ class BuildTargetConfiguration: compiler_wrapper: str name: str - @staticmethod - def from_dict(d: BuildTargetConfigDict) -> "BuildTargetConfiguration": + @classmethod + def from_dict(cls, d: BuildTargetConfigDict) -> Self: r"""A convenience method that processes the inputs before creating an instance. `arch`, `os`, `cpu` and `compiler` are converted to :class:`Enum`\s and @@ -369,7 +370,7 @@ def from_dict(d: BuildTargetConfigDict) -> "BuildTargetConfiguration": Returns: The build target configuration instance. """ - return BuildTargetConfiguration( + return cls( arch=Architecture(d["arch"]), os=OS(d["os"]), cpu=CPUType(d["cpu"]), @@ -407,10 +408,11 @@ class TestSuiteConfig: test_suite: str test_cases: list[str] - @staticmethod + @classmethod def from_dict( + cls, entry: str | TestSuiteConfigDict, - ) -> "TestSuiteConfig": + ) -> Self: """Create an instance from two different types. Args: @@ -420,9 +422,9 @@ def from_dict( The test suite configuration instance. """ if isinstance(entry, str): - return TestSuiteConfig(test_suite=entry, test_cases=[]) + return cls(test_suite=entry, test_cases=[]) elif isinstance(entry, dict): - return TestSuiteConfig(test_suite=entry["suite"], test_cases=entry["cases"]) + return cls(test_suite=entry["suite"], test_cases=entry["cases"]) else: raise TypeError(f"{type(entry)} is not valid for a test suite config.") @@ -454,11 +456,12 @@ class ExecutionConfiguration: traffic_generator_node: TGNodeConfiguration vdevs: list[str] - @staticmethod + @classmethod def from_dict( + cls, d: ExecutionConfigDict, - node_map: dict[str, Union[SutNodeConfiguration | TGNodeConfiguration]], - ) -> "ExecutionConfiguration": + node_map: dict[str, SutNodeConfiguration | TGNodeConfiguration], + ) -> Self: """A convenience method that processes the inputs before creating an instance. The build target and the test suite config are transformed into their respective objects. @@ -494,7 +497,7 @@ def from_dict( vdevs = ( d["system_under_test_node"]["vdevs"] if "vdevs" in d["system_under_test_node"] else [] ) - return ExecutionConfiguration( + return cls( build_targets=build_targets, perf=d["perf"], func=d["func"], @@ -505,7 +508,7 @@ def from_dict( vdevs=vdevs, ) - def copy_and_modify(self, **kwargs) -> "ExecutionConfiguration": + def copy_and_modify(self, **kwargs) -> Self: """Create a shallow copy with any of the fields modified. The only new data are those passed to this method. @@ -525,7 +528,7 @@ def copy_and_modify(self, **kwargs) -> "ExecutionConfiguration": else: new_config[field.name] = getattr(self, field.name) - return ExecutionConfiguration(**new_config) + return type(self)(**new_config) @dataclass(slots=True, frozen=True) @@ -541,8 +544,8 @@ class Configuration: executions: list[ExecutionConfiguration] - @staticmethod - def from_dict(d: ConfigurationDict) -> "Configuration": + @classmethod + def from_dict(cls, d: ConfigurationDict) -> Self: """A convenience method that processes the inputs before creating an instance. Build target and test suite config are transformed into their respective objects. @@ -555,7 +558,7 @@ def from_dict(d: ConfigurationDict) -> "Configuration": Returns: The whole configuration instance. """ - nodes: list[Union[SutNodeConfiguration | TGNodeConfiguration]] = list( + nodes: list[SutNodeConfiguration | TGNodeConfiguration] = list( map(NodeConfiguration.from_dict, d["nodes"]) ) assert len(nodes) > 0, "There must be a node to test" @@ -567,7 +570,7 @@ def from_dict(d: ConfigurationDict) -> "Configuration": map(ExecutionConfiguration.from_dict, d["executions"], [node_map for _ in d]) ) - return Configuration(executions=executions) + return cls(executions=executions) def load_config(config_file_path: Path) -> Configuration: diff --git a/dts/framework/testbed_model/traffic_generator/__init__.py b/dts/framework/testbed_model/traffic_generator/__init__.py index 0eaf0355cd..03e57a77fc 100644 --- a/dts/framework/testbed_model/traffic_generator/__init__.py +++ b/dts/framework/testbed_model/traffic_generator/__init__.py @@ -16,7 +16,7 @@ # pylama:ignore=W0611 -from framework.config import ScapyTrafficGeneratorConfig, TrafficGeneratorType +from framework.config import ScapyTrafficGeneratorConfig, TrafficGeneratorConfig from framework.exception import ConfigurationError from framework.testbed_model.node import Node @@ -28,7 +28,7 @@ def create_traffic_generator( - tg_node: Node, traffic_generator_config: ScapyTrafficGeneratorConfig + tg_node: Node, traffic_generator_config: TrafficGeneratorConfig ) -> CapturingTrafficGenerator: """The factory function for creating traffic generator objects from the test run configuration. @@ -39,10 +39,10 @@ def create_traffic_generator( Returns: A traffic generator capable of capturing received packets. """ - match traffic_generator_config.traffic_generator_type: - case TrafficGeneratorType.SCAPY: + match traffic_generator_config: + case ScapyTrafficGeneratorConfig(): return ScapyTrafficGenerator(tg_node, traffic_generator_config) case _: raise ConfigurationError( - "Unknown traffic generator: {traffic_generator_config.traffic_generator_type}" + f"Unknown traffic generator: {traffic_generator_config.traffic_generator_type}" ) diff --git a/dts/poetry.lock b/dts/poetry.lock index df9cecb7e0..5f8fa03933 100644 --- a/dts/poetry.lock +++ b/dts/poetry.lock @@ -853,4 +853,4 @@ jsonschema = ">=4,<5" [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "1572cb14f0bf88cddc6b6b225312ad468d01916b32067d17a7776af76c6d466c" +content-hash = "4af4dd49c59e5bd6ed99e8c19c6756aaf00125339d26cfad2ef98551dc765f8b" diff --git a/dts/pyproject.toml b/dts/pyproject.toml index 05c91ef9be..a160d2fa02 100644 --- a/dts/pyproject.toml +++ b/dts/pyproject.toml @@ -26,6 +26,7 @@ types-PyYAML = "^6.0.8" fabric = "^2.7.1" scapy = "^2.5.0" pydocstyle = "6.1.1" +typing-extensions = "^4.11.0" [tool.poetry.group.dev.dependencies] mypy = "^1.10.0" From patchwork Tue May 14 12:05:02 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Luca Vizzarro X-Patchwork-Id: 140062 X-Patchwork-Delegate: thomas@monjalon.net Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id B5FC04402B; Tue, 14 May 2024 14:05:35 +0200 (CEST) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 28E07406B7; Tue, 14 May 2024 14:05:16 +0200 (CEST) Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by mails.dpdk.org (Postfix) with ESMTP id 7BCFA4067D for ; Tue, 14 May 2024 14:05:13 +0200 (CEST) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 0B8B8139F; Tue, 14 May 2024 05:05:38 -0700 (PDT) Received: from localhost.localdomain (unknown [10.57.4.224]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id 2C7F23F641; Tue, 14 May 2024 05:05:12 -0700 (PDT) From: Luca Vizzarro To: dev@dpdk.org Cc: =?utf-8?q?Juraj_Linke=C5=A1?= , Jeremy Spewock , Luca Vizzarro Subject: [PATCH v4 3/3] dts: rework arguments framework Date: Tue, 14 May 2024 13:05:02 +0100 Message-Id: <20240514120502.1955468-4-luca.vizzarro@arm.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20240514120502.1955468-1-luca.vizzarro@arm.com> References: <20240122182611.1904974-1-luca.vizzarro@arm.com> <20240514120502.1955468-1-luca.vizzarro@arm.com> MIME-Version: 1.0 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org The existing argument handling in the code relies on basic argparse functionality and a custom argparse action to integrate environment variables. This commit improves the current handling by augmenting argparse. This rework implements the following improvements: - There are duplicate expressions scattered throughout the code. To improve readability and maintainability, these are refactored into list/dict comprehensions or factory functions. - Instead of relying solely on argument flags, error messages now accurately reference environment variables when applicable, enhancing user experience. For instance: error: environment variable DTS_DPDK_TARBALL: Invalid file - Knowing the number of environment variables and arguments set allow for a useful help page display when none are provided. - A display of which environment variables have been detected and their corresponding values in the help page, aiding user awareness. Signed-off-by: Luca Vizzarro --- doc/guides/tools/dts.rst | 53 +++++--- dts/framework/settings.py | 280 +++++++++++++++++++++++++++----------- 2 files changed, 235 insertions(+), 98 deletions(-) diff --git a/doc/guides/tools/dts.rst b/doc/guides/tools/dts.rst index 47b218b2c6..6993443389 100644 --- a/doc/guides/tools/dts.rst +++ b/doc/guides/tools/dts.rst @@ -215,30 +215,41 @@ DTS is run with ``main.py`` located in the ``dts`` directory after entering Poet .. code-block:: console (dts-py3.10) $ ./main.py --help - usage: main.py [-h] [--config-file CONFIG_FILE] [--output-dir OUTPUT_DIR] [-t TIMEOUT] [-v] [-s] [--tarball TARBALL] [--compile-timeout COMPILE_TIMEOUT] [--test-suite TEST_SUITE [TEST_CASES ...]] [--re-run RE_RUN] + usage: main.py [-h] [--config-file FILE_PATH] [--output-dir DIR_PATH] [-t SECONDS] [-v] [-s] [--tarball FILE_PATH] + [--compile-timeout SECONDS] [--test-suite TEST_SUITE [TEST_CASES ...]] [--re-run N_TIMES] - Run DPDK test suites. All options may be specified with the environment variables provided in brackets. Command line arguments have higher priority. + Run DPDK test suites. All options may be specified with the environment variables provided in brackets. Command + line arguments have higher priority. options: - -h, --help show this help message and exit - --config-file CONFIG_FILE - [DTS_CFG_FILE] The configuration file that describes the test cases, SUTs and targets. (default: ./conf.yaml) - --output-dir OUTPUT_DIR, --output OUTPUT_DIR - [DTS_OUTPUT_DIR] Output directory where DTS logs and results are saved. (default: output) - -t TIMEOUT, --timeout TIMEOUT - [DTS_TIMEOUT] The default timeout for all DTS operations except for compiling DPDK. (default: 15) - -v, --verbose [DTS_VERBOSE] Specify to enable verbose output, logging all messages to the console. (default: False) - -s, --skip-setup [DTS_SKIP_SETUP] Specify to skip all setup steps on SUT and TG nodes. (default: False) - --tarball TARBALL, --snapshot TARBALL, --git-ref TARBALL - [DTS_DPDK_TARBALL] Path to DPDK source code tarball or a git commit ID, tag ID or tree ID to test. To test local changes, first commit them, then use the commit ID with this option. (default: dpdk.tar.xz) - --compile-timeout COMPILE_TIMEOUT - [DTS_COMPILE_TIMEOUT] The timeout for compiling DPDK. (default: 1200) - --test-suite TEST_SUITE [TEST_CASES ...] - [DTS_TEST_SUITES] A list containing a test suite with test cases. The first parameter is the test suite name, and the rest are test case names, which are optional. May be specified multiple times. To specify multiple test suites in the environment - variable, join the lists with a comma. Examples: --test-suite suite case case --test-suite suite case ... | DTS_TEST_SUITES='suite case case, suite case, ...' | --test-suite suite --test-suite suite case ... | DTS_TEST_SUITES='suite, suite case, ...' - (default: []) - --re-run RE_RUN, --re_run RE_RUN - [DTS_RERUN] Re-run each test case the specified number of times if a test failure occurs. (default: 0) + -h, --help show this help message and exit + --config-file FILE_PATH + [DTS_CFG_FILE] The configuration file that describes the test cases, SUTs and targets. + (default: conf.yaml) + --output-dir DIR_PATH, --output DIR_PATH + [DTS_OUTPUT_DIR] Output directory where DTS logs and results are saved. (default: output) + -t SECONDS, --timeout SECONDS + [DTS_TIMEOUT] The default timeout for all DTS operations except for compiling DPDK. + (default: 15) + -v, --verbose [DTS_VERBOSE] Specify to enable verbose output, logging all messages to the console. + (default: False) + -s, --skip-setup [DTS_SKIP_SETUP] Specify to skip all setup steps on SUT and TG nodes. (default: False) + --tarball FILE_PATH, --snapshot FILE_PATH, --git-ref FILE_PATH + [DTS_DPDK_TARBALL] Path to DPDK source code tarball or a git commit ID,tag ID or tree ID to + test. To test local changes, first commit them, then use the commit ID with this option. + (default: dpdk.tar.xz) + --compile-timeout SECONDS + [DTS_COMPILE_TIMEOUT] The timeout for compiling DPDK. (default: 1200) + --test-suite TEST_SUITE [TEST_CASES ...] + [DTS_TEST_SUITES] A list containing a test suite with test cases. The first parameter is + the test suite name, and the rest are test case names, which are optional. May be specified + multiple times. To specify multiple test suites in the environment variable, join the lists + with a comma. Examples: --test-suite SUITE1 CASE1 CASE2 --test-suite SUITE2 CASE1 ... | + DTS_TEST_SUITES='SUITE1 CASE1 CASE2, SUITE2 CASE1, ...' | --test-suite SUITE1 --test-suite SUITE2 + CASE1 ... | DTS_TEST_SUITES='SUITE1, SUITE2 CASE1, ...' (default: []) + --re-run N_TIMES, --re_run N_TIMES + [DTS_RERUN] Re-run each test case the specified number of times if a test failure occurs. + (default: 0) The brackets contain the names of environment variables that set the same thing. diff --git a/dts/framework/settings.py b/dts/framework/settings.py index 688e8679a7..b19f274f9d 100644 --- a/dts/framework/settings.py +++ b/dts/framework/settings.py @@ -2,6 +2,7 @@ # Copyright(c) 2010-2021 Intel Corporation # Copyright(c) 2022-2023 PANTHEON.tech s.r.o. # Copyright(c) 2022 University of New Hampshire +# Copyright(c) 2024 Arm Limited """Environment variables and command line arguments parsing. @@ -72,9 +73,15 @@ import argparse import os +import sys +from argparse import ( + Action, + ArgumentDefaultsHelpFormatter, + _get_action_name, +) from dataclasses import dataclass, field from pathlib import Path -from typing import Any +from typing import Callable, ParamSpec from .config import TestSuiteConfig from .utils import DPDKGitTarball @@ -109,104 +116,224 @@ class Settings: SETTINGS: Settings = Settings() +P = ParamSpec("P") -def _get_parser() -> argparse.ArgumentParser: - """Create the argument parser for DTS. +#: Attribute name representing the env variable name to augment :class:`~argparse.Action` with. +_ENV_VAR_NAME_ATTR = "env_var_name" +#: Attribute name representing the value origin to augment :class:`~argparse.Action` with. +_IS_FROM_ENV_ATTR = "is_from_env" - Command line options take precedence over environment variables, which in turn take precedence - over default values. +#: The prefix to be added to all of the environment variables. +ENV_PREFIX = "DTS_" + + +def is_action_in_args(action: Action) -> bool: + """Check if the action is invoked in the command line arguments.""" + for option in action.option_strings: + if option in sys.argv: + return True + return False + + +def make_env_var_name(action: Action, env_var_name: str | None) -> str: + """Make and assign an environment variable name to the given action.""" + env_var_name = f"{ENV_PREFIX}{env_var_name or action.dest.upper()}" + setattr(action, _ENV_VAR_NAME_ATTR, env_var_name) + return env_var_name + + +def get_env_var_name(action: Action) -> str | None: + """Get the environment variable name of the given action.""" + return getattr(action, _ENV_VAR_NAME_ATTR, None) + + +def set_is_from_env(action: Action) -> None: + """Make the environment the given action's value origin.""" + setattr(action, _IS_FROM_ENV_ATTR, True) - Returns: - argparse.ArgumentParser: The configured argument parser with defined options. - """ - def env_arg(env_var: str, default: Any) -> Any: - """A helper function augmenting the argparse with environment variables. +def is_from_env(action: Action) -> bool: + """Check if the given action's value originated from the environment.""" + return getattr(action, _IS_FROM_ENV_ATTR, False) - If the supplied environment variable is defined, then the default value - of the argument is modified. This satisfies the priority order of - command line argument > environment variable > default value. - Args: - env_var: Environment variable name. - default: Default value. +def augment_add_argument_with_env( + add_argument_fn: Callable[P, Action], +): + """Augment any :class:`~argparse._ActionsContainer.add_argument` with environment variables.""" - Returns: - Environment variable or default value. + def _add_argument( + *args: P.args, + env_var_name: str | None = None, + **kwargs: P.kwargs, + ) -> Action: + """Add an argument with an environment variable to the parser.""" + action = add_argument_fn(*args, **kwargs) + env_var_name = make_env_var_name(action, env_var_name) + + if not is_action_in_args(action): + env_var_value = os.environ.get(env_var_name) + if env_var_value: + set_is_from_env(action) + sys.argv[1:0] = [action.format_usage(), env_var_value] + + return action + + return _add_argument + + +class ArgumentParser(argparse.ArgumentParser): + """ArgumentParser with a custom error message. + + This custom version of ArgumentParser changes the error message to + accurately reflect its origin if an environment variable is used + as an argument. + + Instead of printing usage on every error, it prints instructions + on how to do it. + """ + + def find_action( + self, action_name: str, filter_fn: Callable[[Action], bool] | None = None + ) -> Action | None: + """Find and return an action by its name. + + Arguments: + action_name: the name of the action to find. + filter_fn: a filter function to use in the search. """ - return os.environ.get(env_var) or default + it = (action for action in self._actions if action_name == _get_action_name(action)) + action = next(it, None) + + if action is not None and filter_fn is not None: + return action if filter_fn(action) else None + + return action - parser = argparse.ArgumentParser( + def error(self, message): + """Augments :meth:`~argparse.ArgumentParser.error` with environment variable awareness.""" + for action in self._actions: + if is_from_env(action): + action_name = _get_action_name(action) + env_var_name = get_env_var_name(action) + env_var_value = os.environ.get(env_var_name) + + message = message.replace( + f"argument {action_name}", + f"environment variable {env_var_name} (value: {env_var_value})", + ) + + print(f"{self.prog}: error: {message}\n", file=sys.stderr) + self.exit(2, "For help and usage, " "run the command with the --help flag.\n") + + +class EnvVarHelpFormatter(ArgumentDefaultsHelpFormatter): + """Custom formatter to add environment variables in the help page.""" + + def _get_help_string(self, action): + """Overrides :meth:`ArgumentDefaultsHelpFormatter._get_help_string`.""" + help = super()._get_help_string(action) + + env_var_name = get_env_var_name(action) + if env_var_name is not None: + help = f"[{env_var_name}] {help}" + + env_var_value = os.environ.get(env_var_name) + if env_var_value is not None: + help += f" (env value: {env_var_value})" + + return help + + +def _get_parser() -> ArgumentParser: + """Create the argument parser for DTS. + + Command line options take precedence over environment variables, which in turn take precedence + over default values. + + Returns: + ArgumentParser: The configured argument parser with defined options. + """ + parser = ArgumentParser( description="Run DPDK test suites. All options may be specified with the environment " "variables provided in brackets. Command line arguments have higher priority.", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, + formatter_class=EnvVarHelpFormatter, + allow_abbrev=False, ) - parser.add_argument( + add_argument_to_parser_with_env = augment_add_argument_with_env(parser.add_argument) + + add_argument_to_parser_with_env( "--config-file", - default=env_arg("DTS_CFG_FILE", SETTINGS.config_file_path), + default=SETTINGS.config_file_path, type=Path, - help="[DTS_CFG_FILE] The configuration file that describes the test cases, " - "SUTs and targets.", + help="The configuration file that describes the test cases, SUTs and targets.", + metavar="FILE_PATH", + env_var_name="CFG_FILE", ) - parser.add_argument( + add_argument_to_parser_with_env( "--output-dir", "--output", - default=env_arg("DTS_OUTPUT_DIR", SETTINGS.output_dir), - help="[DTS_OUTPUT_DIR] Output directory where DTS logs and results are saved.", + default=SETTINGS.output_dir, + help="Output directory where DTS logs and results are saved.", + metavar="DIR_PATH", ) - parser.add_argument( + add_argument_to_parser_with_env( "-t", "--timeout", - default=env_arg("DTS_TIMEOUT", SETTINGS.timeout), + default=SETTINGS.timeout, type=float, - help="[DTS_TIMEOUT] The default timeout for all DTS operations except for compiling DPDK.", + help="The default timeout for all DTS operations except for compiling DPDK.", + metavar="SECONDS", ) - parser.add_argument( + add_argument_to_parser_with_env( "-v", "--verbose", action="store_true", - default=env_arg("DTS_VERBOSE", SETTINGS.verbose), - help="[DTS_VERBOSE] Specify to enable verbose output, logging all messages " - "to the console.", + default=SETTINGS.verbose, + help="Specify to enable verbose output, logging all messages to the console.", ) - parser.add_argument( + add_argument_to_parser_with_env( "-s", "--skip-setup", action="store_true", - default=env_arg("DTS_SKIP_SETUP", SETTINGS.skip_setup), - help="[DTS_SKIP_SETUP] Specify to skip all setup steps on SUT and TG nodes.", + default=SETTINGS.skip_setup, + help="Specify to skip all setup steps on SUT and TG nodes.", ) - parser.add_argument( + add_argument_to_parser_with_env( "--tarball", "--snapshot", "--git-ref", - default=env_arg("DTS_DPDK_TARBALL", SETTINGS.dpdk_tarball_path), + default=SETTINGS.dpdk_tarball_path, type=Path, - help="[DTS_DPDK_TARBALL] Path to DPDK source code tarball or a git commit ID, " + help="Path to DPDK source code tarball or a git commit ID, " "tag ID or tree ID to test. To test local changes, first commit them, " "then use the commit ID with this option.", + metavar="FILE_PATH", + dest="dpdk_tarball_path", + env_var_name="DPDK_TARBALL", ) - parser.add_argument( + add_argument_to_parser_with_env( "--compile-timeout", - default=env_arg("DTS_COMPILE_TIMEOUT", SETTINGS.compile_timeout), + default=SETTINGS.compile_timeout, type=float, - help="[DTS_COMPILE_TIMEOUT] The timeout for compiling DPDK.", + help="The timeout for compiling DPDK.", + metavar="SECONDS", ) - parser.add_argument( + add_argument_to_parser_with_env( "--test-suite", action="append", nargs="+", metavar=("TEST_SUITE", "TEST_CASES"), - default=env_arg("DTS_TEST_SUITES", SETTINGS.test_suites), - help="[DTS_TEST_SUITES] A list containing a test suite with test cases. " + default=SETTINGS.test_suites, + help="A list containing a test suite with test cases. " "The first parameter is the test suite name, and the rest are test case names, " "which are optional. May be specified multiple times. To specify multiple test suites in " "the environment variable, join the lists with a comma. " @@ -215,21 +342,23 @@ def env_arg(env_var: str, default: Any) -> Any: "DTS_TEST_SUITES='suite case case, suite case, ...' | " "--test-suite suite --test-suite suite case ... | " "DTS_TEST_SUITES='suite, suite case, ...'", + dest="test_suites", ) - parser.add_argument( + add_argument_to_parser_with_env( "--re-run", "--re_run", - default=env_arg("DTS_RERUN", SETTINGS.re_run), + default=SETTINGS.re_run, type=int, - help="[DTS_RERUN] Re-run each test case the specified number of times " - "if a test failure occurs.", + help="Re-run each test case the specified number of times if a test failure occurs.", + env_var_name="RERUN", + metavar="N_TIMES", ) return parser -def _process_test_suites(args: str | list[list[str]]) -> list[TestSuiteConfig]: +def _process_test_suites(parser: ArgumentParser, args: list[list[str]]) -> list[TestSuiteConfig]: """Process the given argument to a list of :class:`TestSuiteConfig` to execute. Args: @@ -240,17 +369,12 @@ def _process_test_suites(args: str | list[list[str]]) -> list[TestSuiteConfig]: Returns: A list of test suite configurations to execute. """ - if isinstance(args, str): - # Environment variable in the form of "suite case case, suite case, suite, ..." - args = [suite_with_cases.split() for suite_with_cases in args.split(",")] + test_suites = parser.find_action("test_suites", is_from_env) + if test_suites is not None: + # Environment variable in the form of "SUITE1 CASE1 CASE2, SUITE2 CASE1, SUITE3, ..." + args = [suite_with_cases.split() for suite_with_cases in args[0][0].split(",")] - test_suites_to_run = [] - for suite_with_cases in args: - test_suites_to_run.append( - TestSuiteConfig(test_suite=suite_with_cases[0], test_cases=suite_with_cases[1:]) - ) - - return test_suites_to_run + return [TestSuiteConfig(test_suite, test_cases) for [test_suite, *test_cases] in args] def get_settings() -> Settings: @@ -261,19 +385,21 @@ def get_settings() -> Settings: Returns: The new settings object. """ - parsed_args = _get_parser().parse_args() - return Settings( - config_file_path=parsed_args.config_file, - output_dir=parsed_args.output_dir, - timeout=parsed_args.timeout, - verbose=parsed_args.verbose, - skip_setup=parsed_args.skip_setup, - dpdk_tarball_path=Path( - Path(DPDKGitTarball(parsed_args.tarball, parsed_args.output_dir)) - if not os.path.exists(parsed_args.tarball) - else Path(parsed_args.tarball) - ), - compile_timeout=parsed_args.compile_timeout, - test_suites=_process_test_suites(parsed_args.test_suite), - re_run=parsed_args.re_run, + parser = _get_parser() + + if len(sys.argv) == 1: + parser.print_help() + sys.exit(1) + + args = parser.parse_args() + + args.dpdk_tarball_path = Path( + Path(DPDKGitTarball(args.dpdk_tarball_path, args.output_dir)) + if not os.path.exists(args.dpdk_tarball_path) + else Path(args.dpdk_tarball_path) ) + + args.test_suites = _process_test_suites(parser, args.test_suites) + + kwargs = {k: v for k, v in vars(args).items() if hasattr(SETTINGS, k)} + return Settings(**kwargs)