From patchwork Mon Jan 22 23:40:33 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Aaron Conole X-Patchwork-Id: 136048 Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 6C6F7439A0; Tue, 23 Jan 2024 00:40:45 +0100 (CET) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 4E228406A2; Tue, 23 Jan 2024 00:40:45 +0100 (CET) Received: from us-smtp-delivery-124.mimecast.com (us-smtp-delivery-124.mimecast.com [170.10.133.124]) by mails.dpdk.org (Postfix) with ESMTP id 85AD8402DC for ; Tue, 23 Jan 2024 00:40:43 +0100 (CET) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=redhat.com; s=mimecast20190719; t=1705966843; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version:content-type:content-type: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=MePjdnrl0MWSI4QGnI/23uMdpPjGMAhFZZKpL+6A6tY=; b=XmOSMtVXLhj/3ughuMS0j10+L8fQEixR1RBvqsPu5g1qTLIWjJCdQ5jmJZtWzVNuq8hCzm x0Wkmw1w37jUvlx83h14487nlYK6UOAbeqHw3eGaynqAWJXtAcMJ2InQkK/BzuSWY9x1Te OxdtmcNXFoLD+FLzG3e2pCAlBe5+iVE= Received: from mimecast-mx02.redhat.com (mx-ext.redhat.com [66.187.233.73]) by relay.mimecast.com with ESMTP with STARTTLS (version=TLSv1.3, cipher=TLS_AES_256_GCM_SHA384) id us-mta-112-xyIv9R44MTmBDIHrIFQn2g-1; Mon, 22 Jan 2024 18:40:39 -0500 X-MC-Unique: xyIv9R44MTmBDIHrIFQn2g-1 Received: from smtp.corp.redhat.com (int-mx03.intmail.prod.int.rdu2.redhat.com [10.11.54.3]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits) server-digest SHA256) (No client certificate requested) by mimecast-mx02.redhat.com (Postfix) with ESMTPS id A6EF52815E25; Mon, 22 Jan 2024 23:40:38 +0000 (UTC) Received: from RHTPC1VM0NT.redhat.com (unknown [10.22.33.141]) by smtp.corp.redhat.com (Postfix) with ESMTP id 71F4F1121306; Mon, 22 Jan 2024 23:40:38 +0000 (UTC) From: Aaron Conole To: ci@dpdk.org Cc: Michael Santana , Ilya Maximets , Jeremy Kerr Subject: [PATCH v3 1/2] treewide: Add a User Agent for CURL requests Date: Mon, 22 Jan 2024 18:40:33 -0500 Message-ID: <20240122234034.3883647-2-aconole@redhat.com> In-Reply-To: <20240122234034.3883647-1-aconole@redhat.com> References: <20240122234034.3883647-1-aconole@redhat.com> MIME-Version: 1.0 X-Scanned-By: MIMEDefang 3.4.1 on 10.11.54.3 X-Mimecast-Spam-Score: 0 X-Mimecast-Originator: redhat.com X-BeenThere: ci@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK CI discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: ci-bounces@dpdk.org This will help identifying robot usage across PW instances. Acked-by: Michael Santana Signed-off-by: Aaron Conole --- v2: Added the pw-ci prefix to all curl commands v3: no change ci_mon | 2 +- github_get_logs.sh | 8 ++++---- github_mon | 2 +- github_restart | 6 ++++-- jenkins_lib.sh | 5 +++-- post_pw.sh | 8 ++++---- pw_mon | 14 +++++++------- 7 files changed, 24 insertions(+), 21 deletions(-) diff --git a/ci_mon b/ci_mon index 2e30da4..94ec1a3 100755 --- a/ci_mon +++ b/ci_mon @@ -159,7 +159,7 @@ for I in travis github obs dummy; do author=$(echo $SERIES_LINE | cut -d\| -f4) email=$(echo $SERIES_LINE | cut -d\| -f5) - PATCHDATA=$(curl -s $url) + PATCHDATA=$(curl -A "(pw-ci) ci-mon-${PROJECT}" -s $url) patch_id="$(get_patch_id_by_series_id_and_sha "$series_id" "$shasum" "$pw_chk_instance")" if [ "X$patch_id" == "X" ]; then diff --git a/github_get_logs.sh b/github_get_logs.sh index 9f44840..9ebbdc2 100755 --- a/github_get_logs.sh +++ b/github_get_logs.sh @@ -74,14 +74,14 @@ print_errored_logs_for_commit () { run_id="$(echo "$run_meta" | jq -r ".id")" # Get the real logs url, download logs, and unzip logs - logs_url="$(curl -s -S -H "${AUTH}" -H "${APP}" "${redirect_url}" -I | \ + logs_url="$(curl -A "(pw-ci) github-logs-crawler" -s -S -H "${AUTH}" -H "${APP}" "${redirect_url}" -I | \ grep -i 'Location: '| sed 's/Location: //i' | tr -d '\r')" - curl -s -S "$logs_url" -o "build_logs_series_${series_id}.zip" + curl -A "(pw-ci) github-logs-crawler" -s -S "$logs_url" -o "build_logs_series_${series_id}.zip" unzip -o -q "build_logs_series_${series_id}.zip" -d "build_logs_series_${series_id}" # Get the names of the failed jobs and the steps that failed tmp_url="$GITHUB_API/repos/$repo_name/actions/runs/${run_id}/jobs" - jobs_results="$(curl -s -S -H "${AUTH}" -H "${APP}" "${tmp_url}")" + jobs_results="$(curl -A "(pw-ci) github-logs-crawler" -s -S -H "${AUTH}" -H "${APP}" "${tmp_url}")" jobs_results="$(echo "$jobs_results" | jq "[.jobs[] | \ select(.conclusion==\"failure\") | {name, failed_step: .steps[] | \ select(.conclusion==\"failure\") | {name, conclusion, number}}]")" @@ -135,7 +135,7 @@ repo_name=$(echo "$repo_name" | sed -e 's@%2F@/@g' -e 's,%40,@,g') # Get GHA runs tmp_url="$GITHUB_API/repos/$repo_name/actions/runs?per_page=9000" -all_runs="$(curl -s -S -H "${AUTH}" -H "${APP}" "${tmp_url}")" +all_runs="$(curl -A "(pw-ci) github-logs-crawler" -s -S -H "${AUTH}" -H "${APP}" "${tmp_url}")" runs=$(echo $all_runs | jq -rc ".workflow_runs[] | select(.head_branch == \"series_$series_id\")") not_found="$(echo "$runs" | jq -rc ".message")" if [ "$not_found" == "Not Found" ]; then diff --git a/github_mon b/github_mon index 8670ce8..d1e54a8 100755 --- a/github_mon +++ b/github_mon @@ -117,7 +117,7 @@ while IFS="|" read -r series_id patch_id patch_url patch_name sha patchwork_inst tmp_url="$GITHUB_API/repos/$repo_name/actions/runs?per_page=9000" if [ "$tmp_url" != "$prev_url" ]; then prev_url="$tmp_url" - all_runs="$(curl -s -S -H "${AUTH}" -H "${APP}" "${tmp_url}")" + all_runs="$(curl -A "(pw-ci) github-mon-${pw_project}" -s -S -H "${AUTH}" -H "${APP}" "${tmp_url}")" fi runs=$(echo "$all_runs" | jq -rc ".workflow_runs[] | select(.head_branch == \"series_$series_id\")") diff --git a/github_restart b/github_restart index d5d63b9..f4e9ebd 100755 --- a/github_restart +++ b/github_restart @@ -114,7 +114,8 @@ if [ "X$runid" == "X" ]; then fi comma="" - for job in $(curl -s -S -H "${AUTH}" -H "${APP}" \ + for job in $(curl -A "(pw-ci) github-restart-${pw_project}" -s -S -H "${AUTH}" \ + -H "${APP}" \ "https://api.github.com/repos/${reponame}/actions/runs?head_sha=${sha}" | \ jq -rc ".workflow_runs[] ${workflow_select} .id") do @@ -126,7 +127,8 @@ fi echo -n "{\"results\":[" comma="" for job in $(echo "$runid" | sed 's/,/ /'); do - result=$(curl -s -X POST -L -S -H "${AUTH}" -H "${APP}" \ + result=$(curl -A "(pw-ci) github-restart-${pw_project}" -s -X POST -L -S \ + -H "${AUTH}" -H "${APP}" \ "https://api.github.com/repos/${reponame}/actions/runs/$job/rerun") msg=$(echo "$result" | jq -rc '.message') diff --git a/jenkins_lib.sh b/jenkins_lib.sh index fc7cb1d..e3826c8 100644 --- a/jenkins_lib.sh +++ b/jenkins_lib.sh @@ -41,12 +41,12 @@ if [ "X$jenkins_token" != "X" ]; then jenkins_credentials="$jenkins_user:$jenkins_token" fi -jenkins_crumb_value=$(curl -s "http://${jenkins_credentials}@${jenkins_url}/crumbIssuer/api/xml?xpath=concat(//crumbRequestField,\":\",//crumb)") +jenkins_crumb_value=$(curl -A "(pw-ci) jenkins-sub-${PROJECT}" -s "http://${jenkins_credentials}@${jenkins_url}/crumbIssuer/api/xml?xpath=concat(//crumbRequestField,\":\",//crumb)") function jenkins_check_for_job() { local jenkins_job=jenkins_${pw_project}_job - curl -s -f -X GET \ + curl -A "(pw-ci) jenkins-sub-${PROJECT}" -s -f -X GET \ "http://${jenkins_credentials}@${jenkins_url}/job/${!jenkins_job}/config.xml" >/dev/null } @@ -86,6 +86,7 @@ function jenkins_submit_series() { }' curl -s -f -X POST \ + -A "(pw-ci) jenkins-sub-${PROJECT}" \ -H "${jenkins_crumb_value}" \ --data token="${!jenkins_job_token}" \ --data-urlencode json="$json_data" \ diff --git a/post_pw.sh b/post_pw.sh index e98e8ab..9163ea1 100755 --- a/post_pw.sh +++ b/post_pw.sh @@ -53,7 +53,7 @@ send_post() { return 0 fi - report="$(curl -sSf "${link}")" + report="$(curl -A "(pw-ci) pw-post" -sSf "${link}")" if [ $? -ne 0 ]; then echo "Failed to get proper server response on link ${link}" 1>&2 return 0 @@ -79,7 +79,7 @@ send_post() { fi # Get reports from patch - checks="$(curl -sSf -X GET \ + checks="$(curl -A "(pw-ci) pw-post" -sSf -X GET \ --header "Content-Type: application/json" \ "$api_url")" if [ $? -ne 0 ]; then @@ -105,7 +105,7 @@ send_post() { \"description\": \"$description\"\ }" - curl -sSf -X POST \ + curl -A "(pw-ci) pw-post" -sSf -X POST \ -H "Authorization: Token ${token}" \ --header "Content-Type: application/json" \ --data "$data" \ @@ -120,7 +120,7 @@ send_post() { } year_month="$(date +"%Y-%B")" -reports="$(curl -sSf "${mail_archive}${year_month}/thread.html" | \ +reports="$(curl -A "(pw-ci) pw-post" -sSf "${mail_archive}${year_month}/thread.html" | \ grep -i 'HREF=' | sed -e 's@[0-9]*
  • @\|@')" if [ $? -ne 0 ]; then echo "Failed to get proper server response on link ${reports}" 1>&2 diff --git a/pw_mon b/pw_mon index ee77256..068fe1a 100755 --- a/pw_mon +++ b/pw_mon @@ -146,7 +146,7 @@ function check_new_series() { GET_URL="http://${INSTANCE}/api/series/?project=${PROJECT}&since=${SINCE}" - response=$(curl -s "$userpw" "$GET_URL") + response=$(curl -A "(pw-ci) pw-mon-${PROJECT}" -s "$userpw" "$GET_URL") series_info=$(echo "$response" | jq -rc '.[] | (.id|tostring) + ";" + .url + ";" + .submitter.name + ";" + .submitter.email + ";" + (.received_all|tostring)') @@ -162,7 +162,7 @@ function check_new_series() { function check_completed_series() { get_uncompleted_jobs_as_line "$pw_instance" "$pw_project" | while IFS=\| read -r id url submitter_name submitter_email; do echo "Checking on series: $id" - local RESPONSE=$(curl -s "$userpw" "$url" | jq -rc '.received_all') + local RESPONSE=$(curl -A "(pw-ci) pw-mon-${PROJECT}" -s "$userpw" "$url" | jq -rc '.received_all') if [ "$RESPONSE" = "true" ]; then echo "Setting series $id to completed" series_id_set_complete "$pw_instance" "$id" @@ -183,8 +183,8 @@ function check_superseded_series() { local pw_instance="$1" series_get_active_branches "$pw_instance" | while IFS=\| read -r series_id project url repo branchname; do # first query the patch states - local last_patch_url=$(curl -s "$userpw" "$url" | jq -rc '.patches[-1].url') - local patch_state=$(curl -s "$userpw" "$last_patch_url" | jq -rc '.state') + local last_patch_url=$(curl -A "(pw-ci) pw-mon-${PROJECT}" -s "$userpw" "$url" | jq -rc '.patches[-1].url') + local patch_state=$(curl -A "(pw-ci) pw-mon-${PROJECT}" -s "$userpw" "$last_patch_url" | jq -rc '.state') # now check to see if the patch should even be reported... if [ "$patch_state" = "superseded" -o "$patch_state" = "rejected" -o "$patch_state" = "accepted" \ @@ -214,7 +214,7 @@ function check_patch_for_retest_request() { local pw_project="$2" local patch_url="$3" - local patch_json=$(curl -s "$userpw" "$patch_url") + local patch_json=$(curl -A "(pw-ci) pw-mon-${PROJECT}" -s "$userpw" "$patch_url") local patch_comments_url=$(echo "$patch_json" | jq -rc '.comments') local patch_id=$(echo "$patch_json" | jq -rc '.id') local series_id=$(echo "$patch_json" | jq -rc '.series[].id') @@ -231,7 +231,7 @@ function check_patch_for_retest_request() { fi if [ "Xnull" != "X$patch_comments_url" ]; then - local comments_json=$(curl -s "$userpw" "$patch_comments_url") + local comments_json=$(curl -A "(pw-ci) pw-mon-${PROJECT}" -s "$userpw" "$patch_comments_url") local seq_end=$(echo "$comments_json" | jq -rc 'length') if [ "$seq_end" -a $seq_end -gt 0 ]; then seq_end=$((seq_end-1)) @@ -250,7 +250,7 @@ function check_series_needs_retest() { local pw_instance="$1" local pw_project="$2" - local series_list=$(curl -s "$userpw" "http://${pw_instance}/api/series/?project=${pw_project}&state=new&state=rfc&state=under-review&archived=false&order=-id") + local series_list=$(curl -A "(pw-ci) pw-mon-${PROJECT}" -s "$userpw" "http://${pw_instance}/api/series/?project=${pw_project}&state=new&state=rfc&state=under-review&archived=false&order=-id") local n=$(echo "$series_list" | jq -rc 'length') if [ "Xnull" == "X$n" -o "X" == "X$n" ]; then