Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/129481/?format=api
https://patches.dpdk.org/api/patches/129481/?format=api", "web_url": "https://patches.dpdk.org/project/ci/patch/20230711202124.1636317-4-ahassick@iol.unh.edu/", "project": { "id": 5, "url": "https://patches.dpdk.org/api/projects/5/?format=api", "name": "CI", "link_name": "ci", "list_id": "ci.dpdk.org", "list_email": "ci@dpdk.org", "web_url": "", "scm_url": "git://dpdk.org/tools/dpdk-ci", "webscm_url": "https://git.dpdk.org/tools/dpdk-ci/", "list_archive_url": "https://inbox.dpdk.org/ci", "list_archive_url_format": "https://inbox.dpdk.org/ci/{}", "commit_url_format": "" }, "msgid": "<20230711202124.1636317-4-ahassick@iol.unh.edu>", "list_archive_url": "https://inbox.dpdk.org/ci/20230711202124.1636317-4-ahassick@iol.unh.edu", "date": "2023-07-11T20:21:21", "name": "[v7,3/6] containers/builder: Dockerfile creation script", "commit_ref": null, "pull_url": null, "state": "new", "archived": false, "hash": "d3167ca1ebd5f8e71fbf4479cd42dbbc0bcf216a", "submitter": { "id": 3127, "url": "https://patches.dpdk.org/api/people/3127/?format=api", "name": "Adam Hassick", "email": "ahassick@iol.unh.edu" }, "delegate": null, "mbox": "https://patches.dpdk.org/project/ci/patch/20230711202124.1636317-4-ahassick@iol.unh.edu/mbox/", "series": [ { "id": 28912, "url": "https://patches.dpdk.org/api/series/28912/?format=api", "web_url": "https://patches.dpdk.org/project/ci/list/?series=28912", "date": "2023-07-11T20:21:18", "name": "Community Lab Containers and Builder Engine", "version": 7, "mbox": "https://patches.dpdk.org/series/28912/mbox/" } ], "comments": "https://patches.dpdk.org/api/patches/129481/comments/", "check": "pending", "checks": "https://patches.dpdk.org/api/patches/129481/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<ci-bounces@dpdk.org>", "X-Original-To": "patchwork@inbox.dpdk.org", "Delivered-To": "patchwork@inbox.dpdk.org", "Received": [ "from mails.dpdk.org (mails.dpdk.org [217.70.189.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id AFBD842E4A;\n\tTue, 11 Jul 2023 22:22:28 +0200 (CEST)", "from mails.dpdk.org (localhost [127.0.0.1])\n\tby mails.dpdk.org (Postfix) with ESMTP id ABB3940A7D;\n\tTue, 11 Jul 2023 22:22:28 +0200 (CEST)", "from mail-qt1-f169.google.com (mail-qt1-f169.google.com\n [209.85.160.169])\n by mails.dpdk.org (Postfix) with ESMTP id 155A14003C\n for <ci@dpdk.org>; Tue, 11 Jul 2023 22:22:27 +0200 (CEST)", "by mail-qt1-f169.google.com with SMTP id\n d75a77b69052e-4036d80caaeso38382441cf.2\n for <ci@dpdk.org>; Tue, 11 Jul 2023 13:22:27 -0700 (PDT)", "from ahassick-Desktop.iol.unh.edu\n ([2606:4100:3880:1220:714e:bdff:bc40:f34f])\n by smtp.gmail.com with ESMTPSA id\n s19-20020ac85293000000b004035c1062f8sm1470604qtn.10.2023.07.11.13.22.25\n (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256);\n Tue, 11 Jul 2023 13:22:26 -0700 (PDT)" ], "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed;\n d=iol.unh.edu; s=unh-iol; t=1689106946; x=1691698946;\n h=content-transfer-encoding:mime-version:references:in-reply-to\n :message-id:date:subject:cc:to:from:from:to:cc:subject:date\n :message-id:reply-to;\n bh=R6hhnvIaluFLKbPQH1MTOMcY34O5nxl+UZGl8zsqC70=;\n b=FXmfhtsyfm5slaVkO+12wQ/d7WI7i9iki1xCk0c84VTQbtyAv/3I9WpuUkGv8bnoDM\n o3YA9hslr0/JMDzGwUfKBll4+1Al3wO9CKV7S9Pi229qFg/uavjZws3449gdr7bjbfUv\n HHDkQuYJLHlW7648r6wfG95fhZOi7Dvz1z5jY=", "X-Google-DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed;\n d=1e100.net; s=20221208; t=1689106946; x=1691698946;\n h=content-transfer-encoding:mime-version:references:in-reply-to\n :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc\n :subject:date:message-id:reply-to;\n bh=R6hhnvIaluFLKbPQH1MTOMcY34O5nxl+UZGl8zsqC70=;\n b=PV1roXyIKEShh5gL4NDR3CRh+gk/Kz0PRazfmhdvMiOqHwekTlhAe8hSSGMYUoZgod\n zirHKkf9gl/4n+8t97cpVSi3Qwt4Urmg1meJpxZ7j0l0DqCgKZ9YiNkeUCkvAaSGF8He\n kxcEyjSrMF5Nd7HMApdUAE02qCxsV44Lw6LJ6aUImIrdfGE4t2Qn3eisTJaf8I2f9fk5\n Hec7VTHhuljyQ3REco8D7cocwTdI6iWC2R2ElKgcr01HYuduqrL8O0b4+8Fi5yTtp2qo\n mltbODph/fx5HKOr0HzHqsWWIwPJyok+hgWqS5pwLW9iPAO6KOlD9qP/yGuABc6fKbda\n 3olw==", "X-Gm-Message-State": "ABy/qLa4IoJgNytrm8QTBHgjRUW4NaKABKevyel1Vc7MrzJ/sSCcwNMI\n /OuicPNpL77n1bf/jj3yH+tDF/jZzc3smazoH6uvDZnTiIjzxhnsWlW7fB9t2NFzxw6bUHvqE2J\n QkNLj1qifDBn7tIX8Di499CcU1G8f6W1DRDLU+rnsi8N7975gkI2uiKxcw940bhbm", "X-Google-Smtp-Source": "\n APBJJlGAazTfZ3LRW/xJBAFBUbLJozyySjKPqKiDuFSnvh5S8198XLIf7vIpj23e9y2c6r8BuJX9PA==", "X-Received": "by 2002:ac8:5ad6:0:b0:403:31f9:3ed8 with SMTP id\n d22-20020ac85ad6000000b0040331f93ed8mr19918746qtd.45.1689106946304;\n Tue, 11 Jul 2023 13:22:26 -0700 (PDT)", "From": "Adam Hassick <ahassick@iol.unh.edu>", "To": "ci@dpdk.org", "Cc": "alialnu@nvidia.com, aconole@redhat.com,\n Adam Hassick <ahassick@iol.unh.edu>", "Subject": "[PATCH v7 3/6] containers/builder: Dockerfile creation script", "Date": "Tue, 11 Jul 2023 16:21:21 -0400", "Message-Id": "<20230711202124.1636317-4-ahassick@iol.unh.edu>", "X-Mailer": "git-send-email 2.34.1", "In-Reply-To": "<20230711202124.1636317-1-ahassick@iol.unh.edu>", "References": "<20230711202124.1636317-1-ahassick@iol.unh.edu>", "MIME-Version": "1.0", "Content-Transfer-Encoding": "8bit", "X-BeenThere": "ci@dpdk.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "DPDK CI discussions <ci.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/ci>,\n <mailto:ci-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/ci/>", "List-Post": "<mailto:ci@dpdk.org>", "List-Help": "<mailto:ci-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/ci>,\n <mailto:ci-request@dpdk.org?subject=subscribe>", "Errors-To": "ci-bounces@dpdk.org" }, "content": "From: Owen Hilyard <ohilyard at iol.unh.edu>\n\nThis script will template out all of the Dockerfiles based on the\ndefinitions provided in the inventory using the jinja2 templating\nlibrary.\n\nSigned-off-by: Owen Hilyard <ohilyard at iol.unh.edu>\nSigned-off-by: Adam Hassick <ahassick@iol.unh.edu>\n---\n containers/template_engine/make_dockerfile.py | 358 ++++++++++++++++++\n 1 file changed, 358 insertions(+)\n create mode 100755 containers/template_engine/make_dockerfile.py", "diff": "diff --git a/containers/template_engine/make_dockerfile.py b/containers/template_engine/make_dockerfile.py\nnew file mode 100755\nindex 0000000..60269a0\n--- /dev/null\n+++ b/containers/template_engine/make_dockerfile.py\n@@ -0,0 +1,358 @@\n+#!/usr/bin/env python3\n+# SPDX-License-Identifier: BSD-3-Clause\n+# Copyright (c) 2022 University of New Hampshire\n+import argparse\n+import json\n+import logging\n+import os\n+import re\n+from dataclasses import dataclass\n+from datetime import datetime\n+import platform\n+from typing import Any, Dict, List, Optional\n+\n+import jsonschema\n+import yaml\n+from jinja2 import Environment, FileSystemLoader, select_autoescape\n+\n+\n+@dataclass(frozen=True)\n+class Options:\n+ on_rhel: bool\n+ fail_on_unbuildable: bool\n+ has_coverity: bool\n+ build_libabigail: bool\n+ build_abi: bool\n+ output_dir: str\n+ registry_hostname: str\n+ host_arch_only: bool\n+ omit_latest: bool\n+ is_builder: bool\n+ date_override: Optional[str]\n+ ninja_workers: Optional[int]\n+\n+\n+def _get_arg_parser() -> argparse.ArgumentParser:\n+ parser = argparse.ArgumentParser(description=\"Makes the dockerfile\")\n+ parser.add_argument(\"--output-dir\", required=True)\n+ parser.add_argument(\n+ \"--rhel\",\n+ action=\"store_true\",\n+ help=\"Overwrite the check for running on RHEL\",\n+ default=False,\n+ )\n+ parser.add_argument(\n+ \"--fail-on-unbuildable\",\n+ action=\"store_true\",\n+ help=\"If any container would not be possible to build, fail and exit with a non-zero exit code.\",\n+ default=False,\n+ )\n+ parser.add_argument(\n+ \"--build-abi\",\n+ action=\"store_true\",\n+ help=\"Whether to build the ABI references into the image. Disabled by \\\n+ default due to producing 10+ GB images. \\\n+ Implies '--build-libabigail'.\",\n+ )\n+ parser.add_argument(\n+ \"--build-libabigail\",\n+ action=\"store_true\",\n+ help=\"Whether to build libabigail from source for distros that do not \\\n+ package it. Implied by '--build-abi'\",\n+ )\n+ parser.add_argument(\n+ \"--host-arch-only\",\n+ action=\"store_true\",\n+ help=\"Only build containers for the architecture of the host system\",\n+ )\n+ parser.add_argument(\n+ \"--omit-latest\",\n+ action=\"store_true\",\n+ help=\"Whether to include the \\\"latest\\\" tag in the generated makefile.\"\n+ )\n+ parser.add_argument(\n+ \"--builder-mode\",\n+ action=\"store_true\",\n+ help=\"Specifies that the makefile is being templated for a builder. \\\n+ This implicitly sets \\\"--host-arch-only\\\" to true and disables making the manifests.\",\n+ default=False\n+ )\n+ parser.add_argument(\n+ \"--date\",\n+ type=str,\n+ help=\"Overrides generation of the timestamp and uses the provided string instead.\"\n+ )\n+ parser.add_argument(\n+ \"--ninja-workers\",\n+ type=int,\n+ help=\"Specifies a number of ninja workers to limit builds to. Uses the ninja default when not given.\"\n+ )\n+ parser.add_argument(\n+ \"--coverity\",\n+ action=\"store_true\",\n+ help=\"Whether the Coverity Scan binaries are available for building the Coverity containers.\",\n+ default=False\n+ )\n+ return parser\n+\n+\n+def parse_args() -> Options:\n+ parser = _get_arg_parser()\n+ args = parser.parse_args()\n+\n+ registry_hostname = (\n+ os.environ.get(\"DPDK_CI_CONTAINERS_REGISTRY_HOSTNAME\") or \"localhost\"\n+ )\n+\n+ # In order to to build the ABIs, libabigail must be built from source on\n+ # some platforms\n+ build_libabigail: bool = args.build_libabigail or args.build_abi\n+\n+ opts = Options(\n+ on_rhel=args.rhel,\n+ fail_on_unbuildable=args.fail_on_unbuildable,\n+ build_libabigail=build_libabigail,\n+ build_abi=args.build_abi,\n+ output_dir=args.output_dir,\n+ registry_hostname=registry_hostname,\n+ host_arch_only=args.host_arch_only or args.builder_mode,\n+ omit_latest=args.omit_latest,\n+ is_builder=args.builder_mode,\n+ date_override=args.date,\n+ ninja_workers=args.ninja_workers,\n+ has_coverity=args.coverity\n+ )\n+\n+ logging.info(f\"make_dockerfile.py options: {opts}\")\n+ return opts\n+\n+\n+def running_on_RHEL(options: Options) -> bool:\n+ \"\"\"\n+ RHEL containers can only be built on RHEL, so disable them and emit a\n+ warning if not on RHEL.\n+ \"\"\"\n+ redhat_release_path = \"/etc/redhat-release\"\n+\n+ if os.path.exists(redhat_release_path):\n+ with open(redhat_release_path) as f:\n+ first_line = f.readline()\n+ on_rhel = \"Red Hat Enterprise Linux\" in first_line\n+ if on_rhel:\n+ logging.info(\"Running on RHEL, allowing RHEL containers\")\n+ return True\n+\n+ logging.warning(\"Not on RHEL, disabling RHEL containers\")\n+ assert options is not None, \"Internal state error, OPTIONS should not be None\"\n+\n+ if options.on_rhel:\n+ logging.info(\"Override enabled, enabling RHEL containers\")\n+\n+ return options.on_rhel\n+\n+\n+def get_path_to_parent_directory() -> str:\n+ return os.path.dirname(__file__)\n+\n+\n+def get_raw_inventory():\n+ parent_dir = get_path_to_parent_directory()\n+\n+ schema_path = os.path.join(parent_dir, \"inventory_schema.json\")\n+ inventory_path = os.path.join(parent_dir, \"inventory.yaml\")\n+\n+ inventory: Dict[str, Any]\n+ with open(inventory_path, \"r\") as f:\n+ inventory = yaml.safe_load(f)\n+\n+ schema: Dict[str, Any]\n+ with open(schema_path, \"r\") as f:\n+ schema = json.load(f)\n+\n+ jsonschema.validate(instance=inventory, schema=schema)\n+ return inventory\n+\n+\n+def apply_group_config_to_target(\n+ target: Dict[str, Any],\n+ raw_inventory: Dict[str, Any],\n+ on_rhel: bool,\n+ fail_on_unbuildable: bool,\n+) -> Optional[Dict[str, Any]]:\n+ groups_for_target: List[Dict[str, Any]] = []\n+ groups: List[Dict[str, Any]] = raw_inventory[\"dockerfiles\"][\"groups\"]\n+ group = groups[target[\"group\"]]\n+\n+ target_primary_group = target[\"group\"]\n+\n+ assert isinstance(target_primary_group, str), \"Target group name was not a string\"\n+\n+ requires_rhel = \"rhel\" in target_primary_group.lower()\n+\n+ if requires_rhel and not on_rhel:\n+ logging.warning(\n+ f\"Disabling target {target['name']}, because it must be built on RHEL.\"\n+ )\n+ if fail_on_unbuildable:\n+ raise AssertionError(\n+ f\"Not on RHEL and target {target['name']} must be built on RHEL\"\n+ )\n+\n+ return None\n+\n+ while group[\"parent\"] != \"NONE\":\n+ groups_for_target.append(group)\n+ group = groups[group[\"parent\"]]\n+\n+ groups_for_target.append(group) # add the \"all\" group\n+ groups_for_target.reverse() # reverse it so overrides work\n+\n+ target_packages: List[str] = target.get(\"packages\") or []\n+\n+ for group in groups_for_target:\n+ target_packages = [*target_packages, *(group.get(\"packages\") or [])]\n+ target = dict(target, **group)\n+\n+ target[\"packages\"] = target_packages\n+\n+ return target\n+\n+def apply_defaults_to_target(target: Dict[str, Any]) -> Dict[str, Any]:\n+ def default_if_unset(target: Dict[str, Any], key: str, value: Any) -> Dict[str, Any]:\n+ if key not in target:\n+ target[key] = value\n+\n+ return target\n+\n+ target = default_if_unset(target, \"requires_coverity\", False)\n+ target = default_if_unset(target, \"force_disable_abi\", False)\n+ target = default_if_unset(target, \"minimum_dpdk_version\", dict(major=0, minor=0, revision=0))\n+ target = default_if_unset(target, \"extra_information\", {})\n+\n+ return target\n+\n+def get_host_arch() -> str:\n+ machine: str = platform.machine()\n+ match machine:\n+ case \"aarch64\" | \"armv8b\" | \"armv8l\":\n+ return \"linux/arm64\"\n+ case \"ppc64le\":\n+ return \"linux/ppc64le\"\n+ case \"x86_64\" | \"x64\" | \"amd64\":\n+ return \"linux/amd64\"\n+ case arch:\n+ raise ValueError(f\"Unknown arch {arch}\")\n+\n+def process_target(\n+ target: Dict[str, Any],\n+ raw_inventory: Dict[str, Any],\n+ has_coverity: bool,\n+ on_rhel: bool,\n+ fail_on_unbuildable: bool,\n+ host_arch_only: bool,\n+ build_timestamp: str\n+) -> Optional[Dict[str, Any]]:\n+ target = apply_defaults_to_target(target)\n+ # Copy the platforms, for building the manifest list.\n+\n+ # Write the build timestamp.\n+ target[\"extra_information\"].update({\n+ \"build_timestamp\": build_timestamp\n+ })\n+\n+ if (not has_coverity) and target[\"requires_coverity\"]:\n+ print(f\"Disabling {target['name']}. Target requires Coverity, and it is not enabled.\")\n+ return None\n+\n+ if host_arch_only:\n+ host_arch = get_host_arch()\n+ if host_arch in target[\"platforms\"]:\n+ target[\"platforms\"] = [host_arch]\n+ else:\n+ return None\n+\n+ return apply_group_config_to_target(\n+ target, raw_inventory, on_rhel, fail_on_unbuildable\n+ )\n+\n+def get_processed_inventory(options: Options, build_timestamp: str) -> Dict[str, Any]:\n+ raw_inventory: Dict[str, Any] = get_raw_inventory()\n+ on_rhel = running_on_RHEL(options)\n+ targets = raw_inventory[\"dockerfiles\"][\"targets\"]\n+ targets = [\n+ process_target(\n+ target, raw_inventory, options.has_coverity, on_rhel, options.fail_on_unbuildable, options.host_arch_only, build_timestamp\n+ )\n+ for target in targets\n+ ]\n+ # remove disabled options\n+ targets = [target for target in targets if target is not None]\n+ raw_inventory[\"dockerfiles\"][\"targets\"] = targets\n+\n+ return raw_inventory\n+\n+\n+def main():\n+ options: Options = parse_args()\n+\n+ env = Environment(\n+ loader=FileSystemLoader(\"templates\"),\n+ )\n+\n+ build_timestamp = datetime.now().strftime(\"%Y-%m-%d-%H-%M-%S\")\n+\n+ inventory = get_processed_inventory(options, build_timestamp)\n+\n+ if options.date_override:\n+ timestamp = options.date_override\n+ else:\n+ timestamp = datetime.now().strftime(\"%Y-%m-%d\")\n+\n+ for target in inventory[\"dockerfiles\"][\"targets\"]:\n+ template = env.get_template(f\"containers/{target['group']}.dockerfile.j2\")\n+ dockerfile_location = os.path.join(\n+ options.output_dir, target[\"name\"] + \".dockerfile\"\n+ )\n+\n+ tags: list[str] = target.get(\"extra_tags\") or []\n+\n+ tags.insert(0, \"$R/$N:$T\")\n+ if not options.omit_latest:\n+ tags.insert(0, \"$R/$N:latest\")\n+ else:\n+ tags = list(filter(lambda x: re.match('^.*:latest$', x) is None, tags))\n+\n+ target[\"tags\"] = tags\n+\n+ rendered_dockerfile = template.render(\n+ timestamp=timestamp,\n+ target=target,\n+ build_libabigail=options.build_libabigail,\n+ build_abi=options.build_abi,\n+ build_timestamp=build_timestamp,\n+ registry_hostname=options.registry_hostname,\n+ ninja_workers=options.ninja_workers,\n+ **inventory,\n+ )\n+ with open(dockerfile_location, \"w\") as output_file:\n+ output_file.write(rendered_dockerfile)\n+\n+ makefile_template = env.get_template(f\"containers.makefile.j2\")\n+ rendered_makefile = makefile_template.render(\n+ timestamp=timestamp,\n+ build_libabigail=options.build_libabigail,\n+ build_abi=options.build_abi,\n+ host_arch_only=options.host_arch_only,\n+ registry_hostname=options.registry_hostname,\n+ is_builder=options.is_builder,\n+ **inventory,\n+ )\n+ makefile_output_path = os.path.join(options.output_dir, \"Makefile\")\n+ with open(makefile_output_path, \"w\") as f:\n+ f.write(rendered_makefile)\n+\n+\n+if __name__ == \"__main__\":\n+ logging.basicConfig()\n+ logging.root.setLevel(0) # log everything\n+ main()\n", "prefixes": [ "v7", "3/6" ] }{ "id": 129481, "url": "