Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/135830/?format=api
{ "id": 135830, "url": "https://patches.dpdk.org/api/patches/135830/?format=api", "web_url": "https://patches.dpdk.org/project/ci/patch/20240110145715.28157-2-ahassick@iol.unh.edu/", "project": { "id": 5, "url": "https://patches.dpdk.org/api/projects/5/?format=api", "name": "CI", "link_name": "ci", "list_id": "ci.dpdk.org", "list_email": "ci@dpdk.org", "web_url": "", "scm_url": "git://dpdk.org/tools/dpdk-ci", "webscm_url": "https://git.dpdk.org/tools/dpdk-ci/", "list_archive_url": "https://inbox.dpdk.org/ci", "list_archive_url_format": "https://inbox.dpdk.org/ci/{}", "commit_url_format": "" }, "msgid": "<20240110145715.28157-2-ahassick@iol.unh.edu>", "list_archive_url": "https://inbox.dpdk.org/ci/20240110145715.28157-2-ahassick@iol.unh.edu", "date": "2024-01-10T14:57:14", "name": "[1/2] tools: Add script to create artifacts", "commit_ref": null, "pull_url": null, "state": "new", "archived": false, "hash": "c65d0797a2115f440027e7142391f8e306a7d3a2", "submitter": { "id": 3127, "url": "https://patches.dpdk.org/api/people/3127/?format=api", "name": "Adam Hassick", "email": "ahassick@iol.unh.edu" }, "delegate": null, "mbox": "https://patches.dpdk.org/project/ci/patch/20240110145715.28157-2-ahassick@iol.unh.edu/mbox/", "series": [ { "id": 30776, "url": "https://patches.dpdk.org/api/series/30776/?format=api", "web_url": "https://patches.dpdk.org/project/ci/list/?series=30776", "date": "2024-01-10T14:57:13", "name": "Add a script to create series artifacts", "version": 1, "mbox": "https://patches.dpdk.org/series/30776/mbox/" } ], "comments": "https://patches.dpdk.org/api/patches/135830/comments/", "check": "pending", "checks": "https://patches.dpdk.org/api/patches/135830/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<ci-bounces@dpdk.org>", "X-Original-To": "patchwork@inbox.dpdk.org", "Delivered-To": "patchwork@inbox.dpdk.org", "Received": [ "from mails.dpdk.org (mails.dpdk.org [217.70.189.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id B4B1E43883;\n\tWed, 10 Jan 2024 15:58:31 +0100 (CET)", "from mails.dpdk.org (localhost [127.0.0.1])\n\tby mails.dpdk.org (Postfix) with ESMTP id AFB1B40269;\n\tWed, 10 Jan 2024 15:58:31 +0100 (CET)", "from mail-qv1-f44.google.com (mail-qv1-f44.google.com\n [209.85.219.44]) by mails.dpdk.org (Postfix) with ESMTP id 91E3B4021E\n for <ci@dpdk.org>; Wed, 10 Jan 2024 15:58:30 +0100 (CET)", "by mail-qv1-f44.google.com with SMTP id\n 6a1803df08f44-67fb9df3699so30322356d6.2\n for <ci@dpdk.org>; Wed, 10 Jan 2024 06:58:30 -0800 (PST)", "from pogmachine2.loudonlune.net ([216.212.51.182])\n by smtp.gmail.com with ESMTPSA id\n da7-20020a05621408c700b00680b1a92322sm1755378qvb.77.2024.01.10.06.58.29\n (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256);\n Wed, 10 Jan 2024 06:58:29 -0800 (PST)" ], "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed;\n d=iol.unh.edu; s=unh-iol; t=1704898710; x=1705503510; darn=dpdk.org;\n h=content-transfer-encoding:mime-version:references:in-reply-to\n :message-id:date:subject:cc:to:from:from:to:cc:subject:date\n :message-id:reply-to;\n bh=YQOvGQL+AW4DsRXJCSD8qBdX9tLjw7aoOI/7UQeFREQ=;\n b=jEXvShmqaEWwEaDdcPrzR9OaL5LrmWdcTbsnYfH8FK7vzAW7+d7M80CBXQd9kh4ytL\n DtGaKcHx9rG2b+QSz5DDbsmGyaX+gHwiNyF+wycMjq6d11jZSmGbgEW77ce2z0nXTMeu\n 79y1x0Mcg5MUPEBZtmfQJurk6MBBdkeklf8KE=", "X-Google-DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed;\n d=1e100.net; s=20230601; t=1704898710; x=1705503510;\n h=content-transfer-encoding:mime-version:references:in-reply-to\n :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc\n :subject:date:message-id:reply-to;\n bh=YQOvGQL+AW4DsRXJCSD8qBdX9tLjw7aoOI/7UQeFREQ=;\n b=oDOG6r3wck0W/W9oydTWvHUW1AwdLSb/pXn+Ol+CjvIrT/YF12z0NVlF/bXnNzEQLp\n pmloVeGJSNMB+klNHhS59gpj8qnhwrK8jTSjJFBImX7q+++Pp+MD7tvPyrGCIwoRCM1t\n fOA+km9xtnAVK4GtrP4dFszO9USyCGGnKgBy505JO5ujb2rQpgasQ7LcYWVijQnLEYiy\n Splva25daa3XeeBcMBadW/hpxdiNp4VPBypOP8HH5ePgflyiAdXr1XG1NfBVdpB/1CJ5\n gqQ+JGrph+knxEFxhpxRmyPPGYFc7JFO6zem79O36UTw+rlXkLEAmG77fO2bzlw9eY6e\n vTHg==", "X-Gm-Message-State": "AOJu0YzTNUbRAUQyIAwRbWXJK5sNn9LA25DWoicFet08KbLh9K6cdNti\n U/VjcVSGpfr3lMRtqpLlz3TTveq2sONtrNzw4KdLwQAdAhJin1hSADD5RQ8OZKB8Xs4e458eYtr\n KyUSY6aw6TNRtvQYBCetCl101tcC10pMiNHuRLmsV7gJftLIJK4h49HUx1X0Kytt2RfvVI3qH", "X-Google-Smtp-Source": "\n AGHT+IHVqOM/U/eKRTM97bIi9D7Ghqqbi9hpKMWHY8jMHQ8HT8vr3O79o5qQijiqDwazgWHvr2bO3A==", "X-Received": "by 2002:ad4:5c8c:0:b0:680:a1c:9501 with SMTP id\n o12-20020ad45c8c000000b006800a1c9501mr1770432qvh.65.1704898709733;\n Wed, 10 Jan 2024 06:58:29 -0800 (PST)", "From": "Adam Hassick <ahassick@iol.unh.edu>", "To": "ci@dpdk.org", "Cc": "aconole@redhat.com, alialnu@nvidia.com,\n Adam Hassick <ahassick@iol.unh.edu>", "Subject": "[PATCH 1/2] tools: Add script to create artifacts", "Date": "Wed, 10 Jan 2024 09:57:14 -0500", "Message-ID": "<20240110145715.28157-2-ahassick@iol.unh.edu>", "X-Mailer": "git-send-email 2.43.0", "In-Reply-To": "<20240110145715.28157-1-ahassick@iol.unh.edu>", "References": "<20240110145715.28157-1-ahassick@iol.unh.edu>", "MIME-Version": "1.0", "Content-Transfer-Encoding": "8bit", "X-BeenThere": "ci@dpdk.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "DPDK CI discussions <ci.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/ci>,\n <mailto:ci-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/ci/>", "List-Post": "<mailto:ci@dpdk.org>", "List-Help": "<mailto:ci-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/ci>,\n <mailto:ci-request@dpdk.org?subject=subscribe>", "Errors-To": "ci-bounces@dpdk.org" }, "content": "This script takes in a URL to a series on Patchwork and emits a\ntarball which may be used for running tests.\n\nSigned-off-by: Adam Hassick <ahassick@iol.unh.edu>\n---\n tools/create_series_artifact.py | 453 ++++++++++++++++++++++++++++++++\n 1 file changed, 453 insertions(+)\n create mode 100755 tools/create_series_artifact.py", "diff": "diff --git a/tools/create_series_artifact.py b/tools/create_series_artifact.py\nnew file mode 100755\nindex 0000000..3049aaa\n--- /dev/null\n+++ b/tools/create_series_artifact.py\n@@ -0,0 +1,453 @@\n+#!/usr/bin/env python3\n+\n+import argparse\n+import os\n+import subprocess\n+import requests\n+import pathlib\n+import yaml\n+import pygit2\n+import requests\n+import shutil\n+\n+from dataclasses import dataclass\n+from typing import Optional, Dict, Tuple, Any, List\n+\n+\n+@dataclass\n+class CreateSeriesParameters(object):\n+ pw_server: str\n+ pw_project: str\n+ pw_token: str\n+ git_user: str\n+ git_email: str\n+ series_url: str\n+ patch_ids: List[int]\n+ labels: List[str]\n+ apply_error: Optional[Tuple[str, str]]\n+ config: Dict\n+ series: Dict\n+ pw_mcli_script: pathlib.Path\n+ patch_parser_script: pathlib.Path\n+ patch_parser_cfg: pathlib.Path\n+ lzma: bool\n+ output_tarball: pathlib.Path\n+ output_properties: pathlib.Path\n+ no_depends: bool\n+ docs_only: bool\n+\n+ def __get_tags(self) -> List[str]:\n+ series_filename = f\"{self.series['id']}.patch\"\n+\n+ # Pull down the patch series as a single file.\n+ pulldown_result = subprocess.run(\n+ [\n+ \"git\",\n+ \"pw\",\n+ \"--server\", # Pass in the pw server we wish to download from.\n+ self.pw_server,\n+ \"series\",\n+ \"download\",\n+ \"--combined\", # Specifies that we want the series in one patch file.\n+ str(self.series[\"id\"]),\n+ series_filename,\n+ ],\n+ stdout=subprocess.DEVNULL,\n+ stderr=subprocess.DEVNULL,\n+ )\n+\n+ # Assert that this succeeds.\n+ pulldown_result.check_returncode()\n+\n+ # Call the patch parser script to obtain the tags\n+ parse_result = subprocess.run(\n+ [self.patch_parser_script, self.patch_parser_cfg, series_filename],\n+ stdout=subprocess.PIPE,\n+ stderr=subprocess.PIPE,\n+ )\n+\n+ # Assert that patch parser succeeded.\n+ parse_result.check_returncode()\n+\n+ # Return the output\n+ return parse_result.stdout.decode().splitlines()\n+\n+ def __init__(self):\n+ parser = argparse.ArgumentParser(\n+ formatter_class=argparse.RawDescriptionHelpFormatter,\n+ description=\"\"\"This script will create an artifact given a URL to a Patchwork series.\n+Much of the information provided is acquired by this script through the use of a configuration file.\n+This configuration file can be found with the other script configs in the config directory of the CI repo.\n+\n+The configuration file is used to aggregate:\n+ - Git credentials\n+ - Patchwork configuration and the user token (user token is optional)\n+ - The URL of the DPDK Git mirror\n+ - Locations of dependency scripts and their configuration files\n+\n+More detail and examples can be found in the default configuration file.\n+This default file is located at \"config/artifacts.yml\" in the dpdk-ci repository.\n+\"\"\",\n+ )\n+ parser.add_argument(\n+ \"config\",\n+ type=argparse.FileType(),\n+ help=\"The config file to load. Must be a path to a YAML file.\",\n+ )\n+ parser.add_argument(\n+ \"series_url\", type=str, help=\"The URL to a Patchwork series.\"\n+ )\n+ parser.add_argument(\n+ \"-t\",\n+ \"--pw-token\",\n+ dest=\"pw_token\",\n+ type=str,\n+ help=\"The Patchwork token\",\n+ )\n+ parser.add_argument(\n+ \"-l\",\n+ \"--lzma\",\n+ action=\"store_true\",\n+ help=\"Use LZMA compression rather than GNU zip compression.\",\n+ )\n+ parser.add_argument(\n+ \"-nd\",\n+ \"--no-depends\",\n+ action=\"store_true\",\n+ help=\"Do not use the Depends-on label.\",\n+ )\n+\n+ args = parser.parse_args()\n+\n+ # Collect basic arguments.\n+ self.series_url = args.series_url\n+ self.no_depends = args.no_depends\n+ self.lzma = args.lzma\n+\n+ # Read the configuration file.\n+ with args.config as config_file:\n+ self.config = yaml.safe_load(config_file)\n+\n+ self.pw_server = self.config[\"patchwork\"][\"server\"]\n+ self.pw_project = self.config[\"patchwork\"][\"project\"]\n+\n+ if args.pw_token:\n+ self.pw_token = args.pw_token\n+ else:\n+ self.pw_token = self.config[\"patchwork\"].get(\"token\")\n+\n+ if not self.pw_token:\n+ print(\"Failed to obtain the Patchworks token.\")\n+ exit(1)\n+\n+ self.pw_mcli_script = pathlib.Path(\n+ self.config[\"pw_maintainers_cli\"][\"path\"]\n+ ).absolute()\n+ self.git_user = self.config[\"git\"][\"user\"]\n+ self.git_email = self.config[\"git\"][\"email\"]\n+\n+ self.patch_parser_script = pathlib.Path(\n+ self.config[\"patch_parser\"][\"path\"]\n+ ).absolute()\n+ self.patch_parser_cfg = pathlib.Path(\n+ self.config[\"patch_parser\"][\"config\"]\n+ ).absolute()\n+\n+ if self.lzma:\n+ tarball_name = \"dpdk.tar.xz\"\n+ else:\n+ tarball_name = \"dpdk.tar.gz\"\n+\n+ self.output_tarball = pathlib.Path(tarball_name)\n+ self.output_properties = pathlib.Path(f\"{tarball_name}.properties\")\n+\n+ # Pull the series JSON down.\n+ resp = requests.get(self.series_url)\n+ resp.raise_for_status()\n+ self.series = resp.json()\n+\n+ # Get the labels using the patch parser.\n+ self.labels = self.__get_tags()\n+\n+ # See if this is a documentation-only patch.\n+ self.docs_only = len(self.labels) == 1 and self.labels[0] == \"documentation\"\n+\n+ # Get the patch ids in this patch series.\n+ self.patch_ids = list(map(lambda x: int(x[\"id\"]), self.series[\"patches\"]))\n+ self.patch_ids.sort()\n+\n+\n+class ProjectTree(object):\n+ artifact_path: pathlib.Path\n+ tree: str\n+ commit_id: str\n+ path: pathlib.Path\n+ log_file_path: pathlib.Path\n+ props_file_path: pathlib.Path\n+ data: CreateSeriesParameters\n+ repo: pygit2.Repository\n+ log_buf: List[str]\n+ properties: Dict[str, Any]\n+\n+ def log(self, msg: str):\n+ print(msg)\n+ self.log_buf.append(msg)\n+\n+ def write_log(self):\n+ with open(self.log_file_path, \"w\") as log_file:\n+ log_file.write(\"\\n\".join([msg for msg in self.log_buf]))\n+\n+ def write_properties(self):\n+ with open(self.props_file_path, \"w\") as prop_file:\n+ for key, value in self.properties.items():\n+ prop_file.write(f\"{key}={value}\\n\")\n+\n+ def move_logs(self):\n+ shutil.move(self.log_file_path, pathlib.Path(os.getcwd(), \"log.txt\"))\n+ shutil.move(\n+ self.props_file_path, pathlib.Path(os.getcwd(), self.data.output_properties)\n+ )\n+\n+ def __init__(self, data: CreateSeriesParameters):\n+ self.data = data\n+ self.path = pathlib.Path(os.curdir, \"dpdk\").absolute()\n+ self.log_buf = []\n+ self.log_file_path = pathlib.Path(self.path, \"log.txt\")\n+ self.props_file_path = pathlib.Path(self.path, data.output_properties)\n+ self.tree = \"main\"\n+ self.properties = {}\n+ self.artifact_path = data.output_tarball\n+\n+ # Set the range of patch IDs this series (aka patchset) covers.\n+ self.properties[\"patchset_range\"] = f\"{data.patch_ids[0]}-{data.patch_ids[-1]}\"\n+\n+ # Set the tags using tags obtained by the params class\n+ self.properties[\"tags\"] = \" \".join(data.labels)\n+\n+ # Record whether this patch is only documentation\n+ self.properties[\"is_docs_only\"] = str(data.docs_only)\n+\n+ if not self.path.exists():\n+ # Find the URL to clone from based on the tree name.\n+ repo = self.data.config[\"repo_url\"]\n+\n+ self.log(f\"Cloning the DPDK mirror at: {repo}\")\n+\n+ # Pull down the git repo we found.\n+ repo = pygit2.clone_repository(repo, self.path)\n+ else:\n+ # Fetch any changes.\n+ repo = pygit2.Repository(self.path)\n+\n+ self.log(f\"Fetching the remote for tree: {self.tree}\")\n+\n+ origin: pygit2.Remote = repo.remotes[\"origin\"]\n+\n+ progress = origin.fetch()\n+\n+ self.log(\n+ f\"Received objects: {progress.received_objects} of {progress.total_objects}\"\n+ )\n+\n+ self.log(\"Cleaning repository state...\")\n+\n+ repo.state_cleanup()\n+\n+ # Initially, check out to main.\n+ self.repo = repo\n+ self.checkout(\"main\")\n+\n+ self.log(f\"Done: {self.tree} commit {self.commit_id}\")\n+\n+ def checkout(self, branch: str) -> Optional[str]:\n+ \"\"\"\n+ Check out to some branch.\n+ Returns true if successful, false otherwise.\n+ \"\"\"\n+ if branch not in self.repo.branches:\n+ return None\n+\n+ git_branch = self.repo.branches[branch]\n+ self.log(f\"Trying to checkout branch: {git_branch.branch_name}\")\n+ reference = self.repo.resolve_refish(git_branch.branch_name)\n+ self.commit_id = str(reference[0].id)\n+ self.repo.checkout(reference[1])\n+ self.tree = branch\n+\n+ return branch\n+\n+ def guess_git_tree(self) -> Optional[str]:\n+ \"\"\"\n+ Run pw_maintainers_cli to guess the git tree of the patch series we are applying.\n+ Returns None if the pw_maintainers_cli failed.\n+ \"\"\"\n+\n+ if \"id\" not in self.data.series:\n+ raise Exception(\"ID was not found in the series JSON\")\n+\n+ result = subprocess.run(\n+ [\n+ self.data.pw_mcli_script,\n+ \"--type\",\n+ \"series\",\n+ \"--pw-server\",\n+ self.data.pw_server,\n+ \"--pw-project\",\n+ self.data.pw_project,\n+ \"list-trees\",\n+ str(self.data.series[\"id\"]),\n+ ],\n+ stdout=subprocess.PIPE,\n+ cwd=self.path,\n+ env={\n+ \"MAINTAINERS_FILE_PATH\": \"MAINTAINERS\",\n+ \"PW_TOKEN\": self.data.pw_token,\n+ },\n+ )\n+\n+ if result.returncode == 0:\n+ branch = result.stdout.decode().strip()\n+\n+ if branch in [\"main\", \"dpdk\"]:\n+ branch = \"main\"\n+ else:\n+ return None\n+\n+ if branch[0:5] == \"dpdk-\":\n+ branch = branch[5 : len(branch)]\n+\n+ return self.checkout(branch)\n+\n+ def set_properties(self):\n+ self.properties[\"tree\"] = self.tree\n+ self.properties[\"applied_commit_id\"] = self.commit_id\n+\n+ def apply_patch_series(self) -> bool:\n+ # Run git-pw to apply the series.\n+\n+ # Configure the tree to point at the given patchwork server and project\n+ self.repo.config[\"pw.server\"] = self.data.pw_server\n+ self.repo.config[\"pw.project\"] = self.data.pw_project\n+ self.repo.config[\"user.email\"] = self.data.git_email\n+ self.repo.config[\"user.name\"] = self.data.git_user\n+\n+ result = subprocess.run(\n+ [\"git\", \"pw\", \"series\", \"apply\", str(self.data.series[\"id\"])],\n+ cwd=self.path,\n+ stdout=subprocess.PIPE,\n+ stderr=subprocess.PIPE,\n+ )\n+\n+ # Write the log from the apply process to disk.\n+ self.log(f\"Applying patch...\")\n+ self.log(result.stdout.decode())\n+ self.log(result.stderr.decode())\n+\n+ # Store whether there was an error, and return the flag.\n+ error = result.returncode != 0\n+ self.properties[\"apply_error\"] = error\n+ return not error\n+\n+ def test_build(self) -> bool:\n+ ninja_result: Optional[subprocess.CompletedProcess] = None\n+ meson_result: subprocess.CompletedProcess = subprocess.run(\n+ [\"meson\", \"setup\", \"build\"],\n+ cwd=self.path,\n+ stdout=subprocess.PIPE,\n+ stderr=subprocess.PIPE,\n+ )\n+\n+ build_error = meson_result.returncode != 0\n+\n+ self.log(\"Running test build...\")\n+ self.log(meson_result.stdout.decode())\n+\n+ if not build_error:\n+ ninja_result = subprocess.run(\n+ [\"ninja\", \"-C\", \"build\"],\n+ cwd=self.path,\n+ stdout=subprocess.PIPE,\n+ stderr=subprocess.PIPE,\n+ )\n+\n+ build_error = build_error or ninja_result.returncode != 0\n+ shutil.rmtree(pathlib.Path(self.path, \"build\"))\n+\n+ self.log(ninja_result.stdout.decode())\n+ self.log(ninja_result.stderr.decode())\n+\n+ self.log(meson_result.stderr.decode())\n+\n+ if build_error:\n+ self.log(\"Test build failed.\")\n+\n+ self.properties[\"build_error\"] = build_error\n+ return not build_error\n+\n+ def create_tarball(self):\n+ # Copy the logs into the artifact tarball.\n+ self.write_log()\n+ self.write_properties()\n+\n+ tar_args = [\"tar\"]\n+\n+ if self.data.lzma:\n+ tar_args.append(\"--lzma\")\n+ else:\n+ tar_args.append(\"-z\")\n+\n+ tar_args.extend([\"-cf\", self.artifact_path, \"-C\", self.path, \".\"])\n+\n+ result = subprocess.run(\n+ tar_args,\n+ stdout=subprocess.DEVNULL,\n+ stderr=subprocess.DEVNULL,\n+ )\n+\n+ if result.returncode != 0:\n+ return False\n+\n+ print(\"Successfully created artifact:\", self.artifact_path)\n+\n+ # Move the log file out of the working directory.\n+ self.move_logs()\n+\n+ return True\n+\n+\n+def try_to_apply(tree: ProjectTree) -> bool:\n+ tree.set_properties()\n+ return tree.apply_patch_series() and tree.test_build() and tree.create_tarball()\n+\n+\n+def main() -> int:\n+ data = CreateSeriesParameters()\n+\n+ # Get the main branch.\n+ # We solve the chicken and egg problem of pw_maintainers_cli needing\n+ # information in the repository by always pulling down the main tree.\n+ tree = ProjectTree(data)\n+\n+ # Try to guess the Git tree for this patchset.\n+ guessed_tree = tree.guess_git_tree()\n+\n+ # Try to apply this patch.\n+ if not (\n+ try_to_apply(tree) # First, try to apply on the guessed tree.\n+ or guessed_tree != \"main\" # If that fails, and the guessed tree was not main\n+ and tree.checkout(\"main\") # Checkout to main, then\n+ and try_to_apply(tree) # Try to apply on main\n+ ):\n+ tree.write_log()\n+ tree.write_properties()\n+ tree.move_logs()\n+\n+ print(\"FAILURE\")\n+\n+ return 1\n+\n+ return 0\n+\n+\n+if __name__ == \"__main__\":\n+ exit(main())\n", "prefixes": [ "1/2" ] }