Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/130211/?format=api
https://patches.dpdk.org/api/patches/130211/?format=api", "web_url": "https://patches.dpdk.org/project/ci/patch/20230811200018.5650-4-ahassick@iol.unh.edu/", "project": { "id": 5, "url": "https://patches.dpdk.org/api/projects/5/?format=api", "name": "CI", "link_name": "ci", "list_id": "ci.dpdk.org", "list_email": "ci@dpdk.org", "web_url": "", "scm_url": "git://dpdk.org/tools/dpdk-ci", "webscm_url": "https://git.dpdk.org/tools/dpdk-ci/", "list_archive_url": "https://inbox.dpdk.org/ci", "list_archive_url_format": "https://inbox.dpdk.org/ci/{}", "commit_url_format": "" }, "msgid": "<20230811200018.5650-4-ahassick@iol.unh.edu>", "list_archive_url": "https://inbox.dpdk.org/ci/20230811200018.5650-4-ahassick@iol.unh.edu", "date": "2023-08-11T20:00:15", "name": "[v9,3/6] containers/builder: Dockerfile creation script", "commit_ref": null, "pull_url": null, "state": "new", "archived": false, "hash": "4df36beae64bdcac98eb98cb4b5cec6ad0e182df", "submitter": { "id": 3127, "url": "https://patches.dpdk.org/api/people/3127/?format=api", "name": "Adam Hassick", "email": "ahassick@iol.unh.edu" }, "delegate": null, "mbox": "https://patches.dpdk.org/project/ci/patch/20230811200018.5650-4-ahassick@iol.unh.edu/mbox/", "series": [ { "id": 29199, "url": "https://patches.dpdk.org/api/series/29199/?format=api", "web_url": "https://patches.dpdk.org/project/ci/list/?series=29199", "date": "2023-08-11T20:00:12", "name": "Community Lab Containers and Builder Engine", "version": 9, "mbox": "https://patches.dpdk.org/series/29199/mbox/" } ], "comments": "https://patches.dpdk.org/api/patches/130211/comments/", "check": "pending", "checks": "https://patches.dpdk.org/api/patches/130211/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<ci-bounces@dpdk.org>", "X-Original-To": "patchwork@inbox.dpdk.org", "Delivered-To": "patchwork@inbox.dpdk.org", "Received": [ "from mails.dpdk.org (mails.dpdk.org [217.70.189.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id 9B7FF43039;\n\tFri, 11 Aug 2023 22:03:04 +0200 (CEST)", "from mails.dpdk.org (localhost [127.0.0.1])\n\tby mails.dpdk.org (Postfix) with ESMTP id 75A8643259;\n\tFri, 11 Aug 2023 22:03:04 +0200 (CEST)", "from mail-qt1-f193.google.com (mail-qt1-f193.google.com\n [209.85.160.193])\n by mails.dpdk.org (Postfix) with ESMTP id A9AA340144\n for <ci@dpdk.org>; Fri, 11 Aug 2023 22:03:03 +0200 (CEST)", "by mail-qt1-f193.google.com with SMTP id\n d75a77b69052e-40ffc70e740so15121251cf.0\n for <ci@dpdk.org>; Fri, 11 Aug 2023 13:03:03 -0700 (PDT)", "from pogmachine2.loudonlune.net ([216.212.51.182])\n by smtp.gmail.com with ESMTPSA id\n f19-20020ac84993000000b004053bcffe49sm1399874qtq.9.2023.08.11.13.03.02\n (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256);\n Fri, 11 Aug 2023 13:03:02 -0700 (PDT)" ], "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed;\n d=iol.unh.edu; s=unh-iol; t=1691784183; x=1692388983;\n h=content-transfer-encoding:mime-version:references:in-reply-to\n :message-id:date:subject:cc:to:from:from:to:cc:subject:date\n :message-id:reply-to;\n bh=w0Ik2REP1HyG3EAdnzsUT2znHudKIif3irmAoBb6Gds=;\n b=enj0Wuuwv6BwzIdq1yWzSaihUtBhusnklu4ZFbc4DHgiLnj2DcHXslvlWAuS0E/TjO\n TC4M+PdECjjlQ1jqs4OX8fWRZDczYFdS9ojOFQ0iA60pRlFSmc/IxcTAQ2sdzqg7C/bX\n Jb51hhUOM+7HOosKq2llfBSl09klk38EFurEY=", "X-Google-DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed;\n d=1e100.net; s=20221208; t=1691784183; x=1692388983;\n h=content-transfer-encoding:mime-version:references:in-reply-to\n :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc\n :subject:date:message-id:reply-to;\n bh=w0Ik2REP1HyG3EAdnzsUT2znHudKIif3irmAoBb6Gds=;\n b=A0fcsMPLy4Igumw43BHlcmQMJRzDLzsd6Q+3BWSLg4UTtrHxf2vzhKxROOieRO6F/L\n co7SvjZ1iXxDgmzGIhX8NL1v40E2r1bUwvZGu1lwlQamtGleb0nT0SqmTVNfpIkkKJ8H\n UT579EQufuk1FB5LEM2Elt6MVoaZahCT+fVBfDT3+BzZVp50eVCMky9W6X3RZGvN2qPy\n IGN2KjmnCNJWs4A+7NQFSFtrdehlCChj+/TUL9oDod9ew7dE0NPJcnIJrcrfi2VF3mL4\n p0y1roqjJdSn9uncTfyy48kaIZOekZgtgDfhG1TIBH2T8ZJvwFYC3GQ+8dcu7beDO1oZ\n ukxw==", "X-Gm-Message-State": "AOJu0YznyxK3jqyMdbNk/1eggs/AUK5t/qBY1MYWKvyDRBTthMW59w5J\n JcuG9aeYyvo3i2QfvakmHBzwCp0cM03VYi3jqz9Tvf/fxRcgl+Nm53HfFQw9jW12xjoGkWzU7+9\n X5HQf9QRitlBwol7IerCIA2MAbg9D2aAdUobzUZbSShsJ9MywSX69fRjNeQnkw9jKMOexyw==", "X-Google-Smtp-Source": "\n AGHT+IFS7JdUfG7EjPeoyrOHae3je7BFXX91QHfrC29eJFyxJoagc2tZ/Kt8uc6DNrY9QVu7xL90jw==", "X-Received": "by 2002:a05:622a:15c2:b0:40f:5510:d74d with SMTP id\n d2-20020a05622a15c200b0040f5510d74dmr3868590qty.13.1691784182765;\n Fri, 11 Aug 2023 13:03:02 -0700 (PDT)", "From": "Adam Hassick <ahassick@iol.unh.edu>", "To": "ci@dpdk.org", "Cc": "aconole@redhat.com, alialnu@nvidia.com,\n Owen Hilyard <ohilyard@iol.unh.edu>, Adam Hassick <ahassick@iol.unh.edu>", "Subject": "[PATCH v9 3/6] containers/builder: Dockerfile creation script", "Date": "Fri, 11 Aug 2023 16:00:15 -0400", "Message-ID": "<20230811200018.5650-4-ahassick@iol.unh.edu>", "X-Mailer": "git-send-email 2.41.0", "In-Reply-To": "<20230811200018.5650-1-ahassick@iol.unh.edu>", "References": "<20230811200018.5650-1-ahassick@iol.unh.edu>", "MIME-Version": "1.0", "Content-Transfer-Encoding": "8bit", "X-BeenThere": "ci@dpdk.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "DPDK CI discussions <ci.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/ci>,\n <mailto:ci-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/ci/>", "List-Post": "<mailto:ci@dpdk.org>", "List-Help": "<mailto:ci-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/ci>,\n <mailto:ci-request@dpdk.org?subject=subscribe>", "Errors-To": "ci-bounces@dpdk.org" }, "content": "From: Owen Hilyard <ohilyard@iol.unh.edu>\n\nThis script will template out all of the Dockerfiles based on the\ndefinitions provided in the inventory using the jinja2 templating\nlibrary.\n\nSigned-off-by: Owen Hilyard <ohilyard@iol.unh.edu>\nSigned-off-by: Adam Hassick <ahassick@iol.unh.edu>\n---\n containers/template_engine/make_dockerfile.py | 371 ++++++++++++++++++\n 1 file changed, 371 insertions(+)\n create mode 100755 containers/template_engine/make_dockerfile.py", "diff": "diff --git a/containers/template_engine/make_dockerfile.py b/containers/template_engine/make_dockerfile.py\nnew file mode 100755\nindex 0000000..9f5fc7f\n--- /dev/null\n+++ b/containers/template_engine/make_dockerfile.py\n@@ -0,0 +1,371 @@\n+#!/usr/bin/env python3\n+# SPDX-License-Identifier: BSD-3-Clause\n+# Copyright (c) 2023 University of New Hampshire\n+import argparse\n+import json\n+import logging\n+import os\n+import re\n+from dataclasses import dataclass\n+from datetime import datetime\n+import platform\n+from typing import Any, Dict, List, Optional\n+\n+import jsonschema\n+import yaml\n+from jinja2 import Environment, FileSystemLoader\n+\n+\n+@dataclass(frozen=True)\n+class Options:\n+ on_rhel: bool\n+ fail_on_unbuildable: bool\n+ has_coverity: bool\n+ build_libabigail: bool\n+ build_abi: bool\n+ output_dir: str\n+ registry_hostname: str\n+ host_arch_only: bool\n+ omit_latest: bool\n+ is_builder: bool\n+ date_override: Optional[str]\n+ ninja_workers: Optional[int]\n+\n+\n+def _get_arg_parser() -> argparse.ArgumentParser:\n+ parser = argparse.ArgumentParser(description=\"Makes the dockerfile\")\n+ parser.add_argument(\"--output-dir\", required=True)\n+ parser.add_argument(\n+ \"--rhel\",\n+ action=\"store_true\",\n+ help=\"Overwrite the check for running on RHEL\",\n+ default=False,\n+ )\n+ parser.add_argument(\n+ \"--fail-on-unbuildable\",\n+ action=\"store_true\",\n+ help=\"If any container would not be possible to build, fail and exit with a non-zero exit code.\",\n+ default=False,\n+ )\n+ parser.add_argument(\n+ \"--build-abi\",\n+ action=\"store_true\",\n+ help=\"Whether to build the ABI references into the image. Disabled by default due to producing 10+ GB images. Implies '--build-libabigail'.\",\n+ )\n+ parser.add_argument(\n+ \"--build-libabigail\",\n+ action=\"store_true\",\n+ help=\"Whether to build libabigail from source for distros that do not package it. Implied by '--build-abi'\",\n+ )\n+ parser.add_argument(\n+ \"--host-arch-only\",\n+ action=\"store_true\",\n+ help=\"Only build containers for the architecture of the host system\",\n+ )\n+ parser.add_argument(\n+ \"--omit-latest\",\n+ action=\"store_true\",\n+ help='Whether to include the \"latest\" tag in the generated makefile.',\n+ )\n+ parser.add_argument(\n+ \"--builder-mode\",\n+ action=\"store_true\",\n+ help='Specifies that the makefile is being templated for a builder. This implicitly sets \"--host-arch-only\" to true and disables making the manifests.',\n+ default=False,\n+ )\n+ parser.add_argument(\n+ \"--date\",\n+ type=str,\n+ help=\"Overrides generation of the timestamp and uses the provided string instead.\",\n+ )\n+ parser.add_argument(\n+ \"--ninja-workers\",\n+ type=int,\n+ help=\"Specifies a number of ninja workers to limit builds to. Uses the ninja default when not given.\",\n+ )\n+ parser.add_argument(\n+ \"--coverity\",\n+ action=\"store_true\",\n+ help=\"Whether the Coverity Scan binaries are available for building the Coverity containers.\",\n+ default=False,\n+ )\n+ return parser\n+\n+\n+def parse_args() -> Options:\n+ parser = _get_arg_parser()\n+ args = parser.parse_args()\n+\n+ registry_hostname = (\n+ os.environ.get(\"DPDK_CI_CONTAINERS_REGISTRY_HOSTNAME\") or \"localhost\"\n+ )\n+\n+ # In order to to build the ABIs, libabigail must be built from source on\n+ # some platforms\n+ build_libabigail: bool = args.build_libabigail or args.build_abi\n+\n+ opts = Options(\n+ on_rhel=args.rhel,\n+ fail_on_unbuildable=args.fail_on_unbuildable,\n+ build_libabigail=build_libabigail,\n+ build_abi=args.build_abi,\n+ output_dir=args.output_dir,\n+ registry_hostname=registry_hostname,\n+ host_arch_only=args.host_arch_only or args.builder_mode,\n+ omit_latest=args.omit_latest,\n+ is_builder=args.builder_mode,\n+ date_override=args.date,\n+ ninja_workers=args.ninja_workers,\n+ has_coverity=args.coverity,\n+ )\n+\n+ logging.info(f\"make_dockerfile.py options: {opts}\")\n+ return opts\n+\n+\n+def running_on_RHEL(options: Options) -> bool:\n+ \"\"\"\n+ RHEL containers can only be built on RHEL, so disable them and emit a\n+ warning if not on RHEL.\n+ \"\"\"\n+ redhat_release_path = \"/etc/redhat-release\"\n+\n+ if os.path.exists(redhat_release_path):\n+ with open(redhat_release_path) as f:\n+ first_line = f.readline()\n+ on_rhel = \"Red Hat Enterprise Linux\" in first_line\n+ if on_rhel:\n+ logging.info(\"Running on RHEL, allowing RHEL containers\")\n+ return True\n+\n+ logging.warning(\"Not on RHEL, disabling RHEL containers\")\n+ assert options is not None, \"Internal state error, OPTIONS should not be None\"\n+\n+ if options.on_rhel:\n+ logging.info(\"Override enabled, enabling RHEL containers\")\n+\n+ return options.on_rhel\n+\n+\n+def get_path_to_parent_directory() -> str:\n+ return os.path.dirname(__file__)\n+\n+\n+def get_raw_inventory():\n+ parent_dir = get_path_to_parent_directory()\n+\n+ schema_path = os.path.join(parent_dir, \"inventory_schema.json\")\n+ inventory_path = os.path.join(parent_dir, \"inventory.yaml\")\n+\n+ inventory: Dict[str, Any]\n+ with open(inventory_path, \"r\") as f:\n+ inventory = yaml.safe_load(f)\n+\n+ schema: Dict[str, Any]\n+ with open(schema_path, \"r\") as f:\n+ schema = json.load(f)\n+\n+ jsonschema.validate(instance=inventory, schema=schema)\n+ return inventory\n+\n+\n+def apply_group_config_to_target(\n+ target: Dict[str, Any],\n+ raw_inventory: Dict[str, Any],\n+ on_rhel: bool,\n+ fail_on_unbuildable: bool,\n+) -> Optional[Dict[str, Any]]:\n+ groups_for_target: List[Dict[str, Any]] = []\n+ groups: List[Dict[str, Any]] = raw_inventory[\"dockerfiles\"][\"groups\"]\n+ group = groups[target[\"group\"]]\n+\n+ target_primary_group = target[\"group\"]\n+\n+ assert isinstance(\n+ target_primary_group, str\n+ ), \"Target group name was \\\n+ not a string\"\n+\n+ requires_rhel = \"rhel\" in target_primary_group.lower()\n+\n+ if requires_rhel and not on_rhel:\n+ logging.warning(\n+ f\"Disabling target {target['name']}, because it must be built on RHEL.\"\n+ )\n+ if fail_on_unbuildable:\n+ raise AssertionError(\n+ f\"Not on RHEL and target {target['name']} must be built on RHEL\"\n+ )\n+\n+ return None\n+\n+ while group[\"parent\"] != \"NONE\":\n+ groups_for_target.append(group)\n+ group = groups[group[\"parent\"]]\n+\n+ groups_for_target.append(group) # add the \"all\" group\n+ groups_for_target.reverse() # reverse it so overrides work\n+\n+ target_packages: List[str] = target.get(\"packages\") or []\n+\n+ for group in groups_for_target:\n+ target_packages = [*target_packages, *(group.get(\"packages\") or [])]\n+ target = dict(target, **group)\n+\n+ target[\"packages\"] = target_packages\n+\n+ return target\n+\n+\n+def apply_defaults_to_target(target: Dict[str, Any]) -> Dict[str, Any]:\n+ def default_if_unset(\n+ target: Dict[str, Any], key: str, value: Any\n+ ) -> Dict[str, Any]:\n+ if key not in target:\n+ target[key] = value\n+\n+ return target\n+\n+ target = default_if_unset(target, \"requires_coverity\", False)\n+ target = default_if_unset(target, \"force_disable_abi\", False)\n+ target = default_if_unset(\n+ target, \"minimum_dpdk_version\", dict(major=0, minor=0, revision=0)\n+ )\n+ target = default_if_unset(target, \"extra_information\", {})\n+\n+ return target\n+\n+\n+def get_host_arch() -> str:\n+ machine: str = platform.machine()\n+ match machine:\n+ case \"aarch64\" | \"armv8b\" | \"armv8l\":\n+ return \"linux/arm64\"\n+ case \"ppc64le\":\n+ return \"linux/ppc64le\"\n+ case \"x86_64\" | \"x64\" | \"amd64\":\n+ return \"linux/amd64\"\n+ case arch:\n+ raise ValueError(f\"Unknown arch {arch}\")\n+\n+\n+def process_target(\n+ target: Dict[str, Any],\n+ raw_inventory: Dict[str, Any],\n+ has_coverity: bool,\n+ on_rhel: bool,\n+ fail_on_unbuildable: bool,\n+ host_arch_only: bool,\n+ build_timestamp: str,\n+) -> Optional[Dict[str, Any]]:\n+ target = apply_defaults_to_target(target)\n+ # Copy the platforms, for building the manifest list.\n+\n+ # Write the build timestamp.\n+ target[\"extra_information\"].update({\"build_timestamp\": build_timestamp})\n+\n+ if (not has_coverity) and target[\"requires_coverity\"]:\n+ print(\n+ f\"Disabling {target['name']}. Target requires Coverity, and it is not enabled.\"\n+ )\n+ return None\n+\n+ if host_arch_only:\n+ host_arch = get_host_arch()\n+ if host_arch in target[\"platforms\"]:\n+ target[\"platforms\"] = [host_arch]\n+ else:\n+ return None\n+\n+ return apply_group_config_to_target(\n+ target, raw_inventory, on_rhel, fail_on_unbuildable\n+ )\n+\n+\n+def get_processed_inventory(options: Options, build_timestamp: str) -> Dict[str, Any]:\n+ raw_inventory: Dict[str, Any] = get_raw_inventory()\n+ on_rhel = running_on_RHEL(options)\n+ targets = raw_inventory[\"dockerfiles\"][\"targets\"]\n+ targets = [\n+ process_target(\n+ target,\n+ raw_inventory,\n+ options.has_coverity,\n+ on_rhel,\n+ options.fail_on_unbuildable,\n+ options.host_arch_only,\n+ build_timestamp,\n+ )\n+ for target in targets\n+ ]\n+ # remove disabled options\n+ targets = [target for target in targets if target is not None]\n+ raw_inventory[\"dockerfiles\"][\"targets\"] = targets\n+\n+ return raw_inventory\n+\n+\n+def main():\n+ options: Options = parse_args()\n+\n+ env = Environment(\n+ loader=FileSystemLoader(\"templates\"),\n+ )\n+\n+ build_timestamp = datetime.now().strftime(\"%Y-%m-%d-%H-%M-%S\")\n+\n+ inventory = get_processed_inventory(options, build_timestamp)\n+\n+ if options.date_override:\n+ timestamp = options.date_override\n+ else:\n+ timestamp = datetime.now().strftime(\"%Y-%m-%d\")\n+\n+ for target in inventory[\"dockerfiles\"][\"targets\"]:\n+ template = env.get_template(f\"containers/{target['group']}.dockerfile.j2\")\n+ dockerfile_location = os.path.join(\n+ options.output_dir, target[\"name\"] + \".dockerfile\"\n+ )\n+\n+ tags: list[str] = target.get(\"extra_tags\") or []\n+\n+ tags.insert(0, \"$R/$N:$T\")\n+ if not options.omit_latest:\n+ tags.insert(0, \"$R/$N:latest\")\n+ else:\n+ tags = list(filter(lambda x: re.match(\"^.*:latest$\", x) is None, tags))\n+\n+ target[\"tags\"] = tags\n+\n+ rendered_dockerfile = template.render(\n+ timestamp=timestamp,\n+ target=target,\n+ build_libabigail=options.build_libabigail,\n+ build_abi=options.build_abi,\n+ build_timestamp=build_timestamp,\n+ registry_hostname=options.registry_hostname,\n+ ninja_workers=options.ninja_workers,\n+ **inventory,\n+ )\n+ with open(dockerfile_location, \"w\") as output_file:\n+ output_file.write(rendered_dockerfile)\n+\n+ makefile_template = env.get_template(f\"containers.makefile.j2\")\n+ rendered_makefile = makefile_template.render(\n+ timestamp=timestamp,\n+ build_libabigail=options.build_libabigail,\n+ build_abi=options.build_abi,\n+ host_arch_only=options.host_arch_only,\n+ registry_hostname=options.registry_hostname,\n+ is_builder=options.is_builder,\n+ **inventory,\n+ )\n+ makefile_output_path = os.path.join(options.output_dir, \"Makefile\")\n+ with open(makefile_output_path, \"w\") as f:\n+ f.write(rendered_makefile)\n+\n+\n+if __name__ == \"__main__\":\n+ logging.basicConfig()\n+ logging.root.setLevel(0) # log everything\n+ main()\n", "prefixes": [ "v9", "3/6" ] }{ "id": 130211, "url": "