new file mode 100755
@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2022 University of New Hampshire
+#
+
+function get_devtools_dir() {
+ dirname "$0"
+}
+
+function main() {
+ DEVTOOLS_DIR="$(get_devtools_dir)"
+ ERRORS=0
+
+ echo "Formatting:"
+ env "$DEVTOOLS_DIR/python-format.sh" -c
+ ERRORS=$(( ERRORS + $? ))
+
+ echo -ne "\n\n"
+ echo "Linting:"
+ env "$DEVTOOLS_DIR/python-lint.sh"
+ ERRORS=$(( ERRORS + $?))
+
+ exit $ERRORS
+}
+
+function usage() {
+ echo "Runs all of the dts devtools scripts."
+ echo "$0 usage:" && grep -P " \w+\)\ #" "$0"
+ exit 0
+}
+
+# There shouldn't be any arguments
+while getopts "" arg; do
+ case $arg in
+ *)
+ esac
+done
+
+main
new file mode 100755
@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2022 University of New Hampshire
+#
+
+function format() {
+ echo "Formatting code with black."
+ black .
+ echo "Sorting imports with isort."
+ isort .
+}
+
+function main() {
+ format
+ exit 0
+}
+
+function check_formatting() {
+ git update-index --refresh
+ retval=$?
+ if [[ $retval -ne 0 ]]
+ then
+ echo 'The "needs update" files have been reformatted.'\
+ 'Please update your commit.'
+ fi
+ exit $retval
+}
+
+function usage() {
+ echo "Automatically formats dts."
+ echo "$0 usage:" && grep -P " \w+\)\ #" $0
+ exit 0
+}
+
+while getopts "h,c,d:" arg; do
+ case $arg in
+ h) # Display this message
+ usage
+ ;;
+
+# Unlike most of these other scripts, format has an argument to control the
+# non-zero exit code. This is to allow you to set it as your IDE's formatting
+# script, since many IDEs are not compatible with formatting scripts which
+# consider changing anything as a failure condition.
+ c) # Exit with a non-zero exit code if any files were not properly formatted.
+ format
+ check_formatting
+ ;;
+ *)
+ esac
+done
+
+echo "Running formatting"
+main
new file mode 100755
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2022 University of New Hampshire
+#
+
+function main() {
+ echo "Running static analysis (linting) using pylama."
+ pylama .
+ exit $?
+}
+
+function usage() {
+ echo "Runs pylama, the linter for DTS."
+ echo "Exits with a non-zero exit code if there were errors."
+ exit 1
+}
+
+# There shouldn't be any arguments
+while getopts "" arg; do
+ case $arg in
+ *)
+ usage
+ esac
+done
+
+main
new file mode 100644
@@ -0,0 +1,30 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
+// https://github.com/microsoft/vscode-dev-containers/tree/v0.241.1/containers/docker-existing-dockerfile
+{
+ "name": "Existing Dockerfile",
+
+ // Sets the run context to one level up instead of the .devcontainer folder.
+ "context": "..",
+
+ // Update the 'dockerFile' property if you aren't using the standard 'Dockerfile' filename.
+ "dockerFile": "../Dockerfile",
+
+ // Use 'forwardPorts' to make a list of ports inside the container available locally.
+ // "forwardPorts": [],
+
+ // Uncomment the next line to run commands after the container is created - for example installing curl.
+ "postCreateCommand": "poetry install",
+
+ "extensions": [
+ "ms-python.vscode-pylance",
+ ]
+
+ // Uncomment when using a ptrace-based debugger like C++, Go, and Rust
+ // "runArgs": [ "--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined" ],
+
+ // Uncomment to use the Docker CLI from inside the container. See https://aka.ms/vscode-remote/samples/docker-from-docker.
+ // "mounts": [ "source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind" ],
+
+ // Uncomment to connect as a non-root user if you've added one. See https://aka.ms/vscode-remote/containers/non-root.
+ // "remoteUser": "vscode"
+}
new file mode 100644
@@ -0,0 +1,39 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2022 University of New Hampshire
+#
+
+# There are two Docker images defined in this Dockerfile.
+# One is to be used in CI for automated testing.
+# The other provides a DTS development environment, simplifying Python dependency management.
+
+FROM ubuntu:22.04 AS base
+
+RUN apt-get -y update && apt-get -y upgrade && \
+ apt-get -y install --no-install-recommends \
+ python3 \
+ python3-pip \
+ python3-pexpect \
+ python3-poetry \
+ python3-cachecontrol \
+ openssh-client
+WORKDIR /dpdk/dts
+
+
+FROM base AS runner
+
+# This image is intended to be used as the base for automated systems.
+# It bakes DTS into the image during the build.
+
+COPY . /dpdk/dts
+RUN poetry install --no-dev
+
+CMD ["poetry", "run", "python", "main.py"]
+
+FROM base AS dev
+
+# This image is intended to be used as DTS development environment. It doesn't need C compilation
+# capabilities, only Python dependencies. Once a container mounting DTS using this image is running,
+# the dependencies should be installed using Poetry.
+
+RUN apt-get -y install --no-install-recommends \
+ vim emacs git
@@ -1,3 +1,36 @@
+DTS environment
+===============
+This README contains helpful steps for setting up your own DTS development or execution
+environment. As DTS is written purely in Python, we only need to download pre-built
+Python packages, obviating the need for build tools. This in turn means that both the
+DTS development and execution environments are the same. DTS environment, DTS
+development environment and DTS execution environment are used interchangeably, as
+they're referring to the same thing.
+
+# DTS definitions
+Before talking about environment setup itself, it's useful to sort out some basic
+definitions:
+* **DTS node**: A generic description of any element/server DTS connects to.
+* **DTS execution environment**: An environment containing Python with packages needed
+ to run DTS.
+* **DTS execution environment node**: A node where at least one DTS execution
+ environment is present. This is the node where we run DTS and from which DTS connects
+ to other nodes.
+* **System under test**: An SUT is the combination of DPDK and the hardware we're
+ testing in conjunction with DPDK (NICs, crypto and other devices).
+* **System under test node**: A node where at least one SUT is present.
+* **Traffic generator**: A TG is either software or hardware capable of sending packets.
+* **Traffic generator node**: A node where at least one TG is present. In case of
+ hardware traffic generators, the TG and the node are literally the same.
+
+In most cases, referring to an execution environment, SUT, TG or the node they're
+running on interchangeably (e.g. using SUT and SUT node interchangeably) doesn't cause
+confusion. There could theoretically be more than of these running on the same node and
+in that case it's useful to have stricter definitions. An example would be two different
+traffic generators (such as Trex and Scapy) running on the same node. A different
+example would be a node containing both a DTS execution environment and a traffic
+generator, in which case it's both a DTS execution environment node and a TG node.
+
# [Poetry](https://python-poetry.org/docs/)
The typical style of python dependency management, requirements.txt, has a few issues.
The advantages of Poetry include specifying what python version is required and forcing
@@ -11,3 +44,111 @@ in the **[tool.poetry.dependencies]** section. Poetry doesn't install Python, so
need to satisfy this requirement if your Python is not up-to-date. A tool such as
[Pyenv](https://github.com/pyenv/pyenv) is a good way to get Python, though not the only
one. However, DTS includes a development environment in the form of a Docker image.
+
+# DTS Environment
+The execution and development environments for DTS are the same, a
+[Docker](https://docs.docker.com/) container defined by our [Dockerfile](./Dockerfile).
+Using a container for the development environment helps with a few things.
+
+1. It helps enforce the boundary between the DTS environment and the TG/SUT, something
+ which caused issues in the past.
+2. It makes creating containers to run DTS inside automated tooling much easier, since
+ they can be based off of a known-working environment that will be updated as DTS is.
+3. It abstracts DTS from the server it is running on. This means that the bare-metal os
+ can be whatever corporate policy or your personal preferences dictate, and DTS does
+ not have to try to support all distros that are supported by DPDK CI.
+4. It makes automated testing for DTS easier, since new dependencies can be sent in with
+ the patches.
+5. It fixes the issue of undocumented dependencies, where some test suites require
+ python libraries that are not installed.
+6. Allows everyone to use the same python version easily, even if they are using a
+ distribution or Windows with out-of-date packages.
+7. Allows you to run the tester on Windows while developing via Docker for Windows.
+
+## Tips for setting up a development environment
+
+### Getting a docker shell
+These commands will give you a bash shell inside the container with all the python
+dependencies installed. This will place you inside a python virtual environment. DTS is
+mounted via a volume, which is essentially a symlink from the host to the container.
+This enables you to edit and run inside the container and then delete the container when
+you are done, keeping your work.
+
+```shell
+docker build --target dev -t dpdk-dts .
+docker run -v $(pwd)/..:/dpdk -it dpdk-dts bash
+$ poetry install
+$ poetry shell
+```
+
+### Vim/Emacs
+Any editor in the ubuntu repos should be easy to use, with vim and emacs already
+installed. You can add your normal config files as a volume, enabling you to use your
+preferred settings.
+
+```shell
+docker run -v ${HOME}/.vimrc:/root/.vimrc -v $(pwd)/..:/dpdk -it dpdk-dts bash
+```
+
+### Visual Studio Code
+VSCode has first-class support for developing with containers. You may need to run the
+non-docker setup commands in the integrated terminal. DTS contains a .devcontainer
+config, so if you open the folder in vscode it should prompt you to use the dev
+container assuming you have the plugin installed. Please refer to
+[VS Development Containers Docs](https://code.visualstudio.com/docs/remote/containers)
+to set it all up.
+
+### Other
+Searching for '$IDE dev containers' will probably lead you in the right direction.
+
+DTS Devtools
+============
+
+# Running the scripts
+These scripts should be run in the [dts](.) directory. You can install their
+dependencies directly, but all the scripts are designed to run in the DTS container
+(specified by [Dockerfile](./Dockerfile)). The .git directory for dpdk must be present
+inside the Dockerfile, meaning you may need to mount the repository as a volume, as
+outlined earlier.
+
+# Script Descriptions
+
+### [../devtools/python-checkpatch.sh](../devtools/python-checkpatch.sh)
+This script runs all the scripts below that provide information on code quality and
+correctness, exiting with a non-zero exit code if any of the scripts below found any
+issues.
+
+### [../devtools/python-format.sh](../devtools/python-format.sh)
+By default, this script will format all the python code according to the DTS code style
+standards. It will not change the semantics of any code, but fixes many issues around
+whitespace, comment formatting and line length automatically.
+
+This script uses two tools to accomplish this:
+
+* [isort](https://pycqa.github.io/isort/): which alphabetically sorts python imports
+within blocks.
+* [black](https://github.com/psf/black): This tool does most of the actual formatting,
+and works similarly to clang-format.
+
+### [../devtools/python-lint.sh](../devtools/python-lint.sh)
+This script runs [pylama](https://github.com/klen/pylama), which runs a collection of
+python linters and aggregates output. It will run these tools over the repository:
+
+* pycodestyle
+* pylint
+* mccabe
+* mypy
+
+Some lints are disabled due to conflicts with the automatic formatters.
+
+Mypy is not running in strict mode since scapy, an important dependency for packet
+manipulation, inspection and construction, does not have python type annotations. In
+strict mode, this makes mypy fail even an empty file that imports scapy.
+
+# Adding additional scripts
+The shebang MUST be "#!/usr/bin/env bash". Many developers will be working inside a
+python virtual environment, where environment variables are changed to allow multiple
+python versions to coexist on the same system.
+
+If the script provides feedback on code quality or correctness, or can reasonably be
+made to do so, it should be added to dts-checkpatch.sh.