Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/136040/?format=api
{ "id": 136040, "url": "https://patches.dpdk.org/api/patches/136040/?format=api", "web_url": "https://patches.dpdk.org/project/ci/patch/20240122172635.3641078-3-aconole@redhat.com/", "project": { "id": 5, "url": "https://patches.dpdk.org/api/projects/5/?format=api", "name": "CI", "link_name": "ci", "list_id": "ci.dpdk.org", "list_email": "ci@dpdk.org", "web_url": "", "scm_url": "git://dpdk.org/tools/dpdk-ci", "webscm_url": "https://git.dpdk.org/tools/dpdk-ci/", "list_archive_url": "https://inbox.dpdk.org/ci", "list_archive_url_format": "https://inbox.dpdk.org/ci/{}", "commit_url_format": "" }, "msgid": "<20240122172635.3641078-3-aconole@redhat.com>", "list_archive_url": "https://inbox.dpdk.org/ci/20240122172635.3641078-3-aconole@redhat.com", "date": "2024-01-22T17:26:35", "name": "[2/2] post_pw: Store submitted checks locally as well", "commit_ref": null, "pull_url": null, "state": "new", "archived": false, "hash": "abc77c162b3e57ede59c0d147330f91040f97999", "submitter": { "id": 332, "url": "https://patches.dpdk.org/api/people/332/?format=api", "name": "Aaron Conole", "email": "aconole@redhat.com" }, "delegate": null, "mbox": "https://patches.dpdk.org/project/ci/patch/20240122172635.3641078-3-aconole@redhat.com/mbox/", "series": [ { "id": 30878, "url": "https://patches.dpdk.org/api/series/30878/?format=api", "web_url": "https://patches.dpdk.org/project/ci/list/?series=30878", "date": "2024-01-22T17:26:34", "name": "Reduced checks API usage", "version": 1, "mbox": "https://patches.dpdk.org/series/30878/mbox/" } ], "comments": "https://patches.dpdk.org/api/patches/136040/comments/", "check": "pending", "checks": "https://patches.dpdk.org/api/patches/136040/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<ci-bounces@dpdk.org>", "X-Original-To": "patchwork@inbox.dpdk.org", "Delivered-To": "patchwork@inbox.dpdk.org", "Received": [ "from mails.dpdk.org (mails.dpdk.org [217.70.189.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id 65E604399D;\n\tMon, 22 Jan 2024 18:26:40 +0100 (CET)", "from mails.dpdk.org (localhost [127.0.0.1])\n\tby mails.dpdk.org (Postfix) with ESMTP id 3320C40DDA;\n\tMon, 22 Jan 2024 18:26:40 +0100 (CET)", "from us-smtp-delivery-124.mimecast.com\n (us-smtp-delivery-124.mimecast.com [170.10.133.124])\n by mails.dpdk.org (Postfix) with ESMTP id D7AF840298\n for <ci@dpdk.org>; Mon, 22 Jan 2024 18:26:38 +0100 (CET)", "from mimecast-mx02.redhat.com (mimecast-mx02.redhat.com\n [66.187.233.88]) by relay.mimecast.com with ESMTP with STARTTLS\n (version=TLSv1.3, cipher=TLS_AES_256_GCM_SHA384) id\n us-mta-600-a96rdh4gMBSR1dCVEp6yFA-1; Mon, 22 Jan 2024 12:26:36 -0500", "from smtp.corp.redhat.com (int-mx03.intmail.prod.int.rdu2.redhat.com\n [10.11.54.3])\n (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits)\n key-exchange X25519 server-signature RSA-PSS (2048 bits) server-digest\n SHA256)\n (No client certificate requested)\n by mimecast-mx02.redhat.com (Postfix) with ESMTPS id 64643185A782;\n Mon, 22 Jan 2024 17:26:36 +0000 (UTC)", "from RHTPC1VM0NT.redhat.com (unknown [10.22.33.141])\n by smtp.corp.redhat.com (Postfix) with ESMTP id 2D48B1121306;\n Mon, 22 Jan 2024 17:26:36 +0000 (UTC)" ], "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed; d=redhat.com;\n s=mimecast20190719; t=1705944398;\n h=from:from:reply-to:subject:subject:date:date:message-id:message-id:\n to:to:cc:cc:mime-version:mime-version:content-type:content-type:\n content-transfer-encoding:content-transfer-encoding:\n in-reply-to:in-reply-to:references:references;\n bh=jcJ9oW6QlaOEMu7V7u2CCIyJdmz/TMjmr8F5eMsMm5Q=;\n b=YfTuvs1TohE7pLTDKhAHTs39BlSnb3LAYHSqWpRp3Ho9qU+L1cjKKNHqHOstPeP5ijQEAu\n Qc4KcIde7OdAKkv6kr0m6tXIkJXEoVvnevZLEph4+op41Rs8d+Fb1W0MXPH4/gWrzf6Fo/\n toiRID3YxTCh9LAWneQubItpNbJdLFI=", "X-MC-Unique": "a96rdh4gMBSR1dCVEp6yFA-1", "From": "Aaron Conole <aconole@redhat.com>", "To": "ci@dpdk.org", "Cc": "Michael Santana <msantana@redhat.com>, Ilya Maximets <i.maximets@ovn.org>,\n Jeremy Kerr <jk@ozlabs.org>", "Subject": "[PATCH 2/2] post_pw: Store submitted checks locally as well", "Date": "Mon, 22 Jan 2024 12:26:35 -0500", "Message-ID": "<20240122172635.3641078-3-aconole@redhat.com>", "In-Reply-To": "<20240122172635.3641078-1-aconole@redhat.com>", "References": "<20240122172635.3641078-1-aconole@redhat.com>", "MIME-Version": "1.0", "X-Scanned-By": "MIMEDefang 3.4.1 on 10.11.54.3", "X-Mimecast-Spam-Score": "0", "X-Mimecast-Originator": "redhat.com", "Content-Transfer-Encoding": "8bit", "Content-Type": "text/plain; charset=\"US-ASCII\"; x-default=true", "X-BeenThere": "ci@dpdk.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "DPDK CI discussions <ci.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/ci>,\n <mailto:ci-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/ci/>", "List-Post": "<mailto:ci@dpdk.org>", "List-Help": "<mailto:ci-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/ci>,\n <mailto:ci-request@dpdk.org?subject=subscribe>", "Errors-To": "ci-bounces@dpdk.org" }, "content": "Jeremy Kerr reports that our PW checks reporting submitted 43000 API calls\nin just a single day. That is alarmingly unacceptable. We can store the\nURLs we've already submitted and then just skip over any additional\nprocessing at least on the PW side.\n\nThis patch does two things to try and mitigate this issue:\n\n1. Store each patch ID and URL in the series DB to show that we reported\n the check. This means we don't need to poll patchwork for check status\n\n2. Store the last modified time of the reports mailing list. This means\n we only poll the mailing list when a new email has surely landed.\n\nSigned-off-by: Aaron Conole <aconole@redhat.com>\n---\n post_pw.sh | 35 ++++++++++++++++++++++++++++++++++-\n series_db_lib.sh | 25 +++++++++++++++++++++++++\n 2 files changed, 59 insertions(+), 1 deletion(-)", "diff": "diff --git a/post_pw.sh b/post_pw.sh\nindex fe2f41c..3e3a493 100755\n--- a/post_pw.sh\n+++ b/post_pw.sh\n@@ -20,6 +20,7 @@\n # License for the specific language governing permissions and limitations\n # under the License.\n \n+[ -f \"$(dirname $0)/series_db_lib.sh\" ] && source \"$(dirname $0)/series_db_lib.sh\" || exit 1\n [ -f \"${HOME}/.mail_patchwork_sync.rc\" ] && source \"${HOME}/.mail_patchwork_sync.rc\"\n \n # Patchwork instance to update with new reports from mailing list\n@@ -75,6 +76,13 @@ send_post() {\n if [ -z \"$context\" -o -z \"$state\" -o -z \"$description\" -o -z \"$patch_id\" ]; then\n echo \"Skpping \\\"$link\\\" due to missing context, state, description,\" \\\n \"or patch_id\" 1>&2\n+ # Just don't want to even bother seeing these \"bad\" patches as well.\n+ add_check_scanned_url \"$patch_id\" \"$target_url\"\n+ return 0\n+ fi\n+\n+ if check_id_exists \"$patch_id\" \"$target_url\" ; then\n+ echo \"Skipping \\\"$link\\\" - already reported.\" 1>&2\n return 0\n fi\n \n@@ -84,6 +92,7 @@ send_post() {\n \"$api_url\")\"\n if [ $? -ne 0 ]; then\n echo \"Failed to get proper server response on link ${api_url}\" 1>&2\n+ # Don't store these as processed in case the server has a temporary issue.\n return 0\n fi\n \n@@ -95,6 +104,9 @@ send_post() {\n jq -e \"[.[].target_url] | contains([\\\"$mail_url\\\"])\" >/dev/null\n then\n echo \"Report ${target_url} already pushed to patchwork. Skipping.\" 1>&2\n+ # Somehow this was not stored (for example, first time we apply the tracking\n+ # feature). Store it now.\n+ add_check_scanned_url \"$patch_id\" \"$target_url\"\n return 0\n fi\n \n@@ -114,12 +126,31 @@ send_post() {\n if [ $? -ne 0 ]; then\n echo -e \"Failed to push retults based on report ${link} to the\"\\\n \"patchwork instance ${pw_instance} using the following REST\"\\\n- \"API Endpoint ${api_url} with the following data:\\n$data\\n\"\n+ \"API Endpoint ${api_url} with the following data:\\n$data\\n\" 1>&2\n return 0\n fi\n+\n+ add_check_scanned_url \"$patch_id\" \"$target_url\"\n }\n \n+# Collect the date. NOTE: this needs some accomodate to catch the month change-overs\n year_month=\"$(date +\"%Y-%B\")\"\n+\n+# Get the last modified time\n+report_last_mod=$(curl -A \"pw-post\" -sSf \"${mail_archive}${year_month}/thread.html\" | grep Last-Modified)\n+\n+mailing_list_save_file=$(echo \".post_pw_${mail_archive}${year_month}\" | sed -e \"s@/@_@g\" -e \"s@:@_@g\" -e \"s,@,_,g\")\n+\n+if [ -e \"${HOME}/${mailing_list_save_file}\" ]; then\n+ last_read_date=$(cat \"${HOME}/${mailing_list_save_file}\")\n+ if [ \"$last_read_date\" == \"$report_last_mod\" ]; then\n+ echo \"Last modified times match. Skipping list parsing.\"\n+ exit 0\n+ fi\n+else\n+ touch \"${HOME}/${mailing_list_save_file}\"\n+fi\n+\n reports=\"$(curl -A \"pw-post\" -sSf \"${mail_archive}${year_month}/thread.html\" | \\\n grep -i 'HREF=' | sed -e 's@[0-9]*<LI><A HREF=\"@\\|@' -e 's@\">@\\|@')\"\n if [ $? -ne 0 ]; then\n@@ -132,3 +163,5 @@ echo \"$reports\" | while IFS='|' read -r blank link title; do\n send_post \"${mail_archive}${year_month}/$link\"\n fi\n done\n+\n+echo \"$last_read_date\" > \"${HOME}/${mailing_list_save_file}\"\ndiff --git a/series_db_lib.sh b/series_db_lib.sh\nindex c5f42e0..0635469 100644\n--- a/series_db_lib.sh\n+++ b/series_db_lib.sh\n@@ -130,6 +130,17 @@ recheck_sync INTEGER\n EOF\n run_db_command \"INSERT INTO series_schema_version(id) values (8);\"\n fi\n+\n+ run_db_command \"select * from series_schema_version;\" | egrep '^9$' > /dev/null 2>&1\n+ if [ $? -eq 1 ]; then\n+ sqlite3 ${HOME}/.series-db <<EOF\n+CREATE TABLE check_id_scanned (\n+check_patch_id INTEGER,\n+check_url STRING\n+)\n+EOF\n+ run_db_command \"INSERT INTO series_schema_version(id) values (9);\"\n+ fi\n }\n \n function series_db_exists() {\n@@ -468,3 +479,17 @@ function set_recheck_request_state() {\n \n echo \"UPDATE recheck_requests set recheck_sync=$recheck_state where patchwork_instance=\\\"$recheck_instance\\\" and patchwork_project=\\\"$recheck_project\\\" and recheck_requested_by=\\\"$recheck_requested_by\\\" and recheck_series=\\\"$recheck_series\\\";\" | series_db_execute\n }\n+\n+function add_check_scanned_url() {\n+ local patch_id=\"$1\"\n+ local url=\"$2\"\n+\n+ echo \"INSERT into check_id_scanned (check_patch_id, check_url) values (${patch_id}, \\\"$url\\\");\" | series_db_execute\n+}\n+\n+function check_id_exists() {\n+ local patch_id=\"$1\"\n+ local url=\"$2\"\n+\n+ echo \"select * from check_id_scanned where check_patch_id=$patch_id and check_url=\\\"$url\\\";\" | series_db_execute | grep \"$url\" >/dev/null 2>&1\n+}\n", "prefixes": [ "2/2" ] }