From a659c02bbd8a8b43e489f8085ed055c99c6f346c Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Wed, 23 Nov 2022 00:58:43 -0600 Subject: [PATCH] Added a bot-generated PR-comment with a simple status table The littlefs CI is actually in a nice state that generates a lot of information about PRs (code/stack/struct changes, line/branch coverage changes, benchmark changes), but GitHub's UI has changed overtime to make CI statuses harder to find for some reason. This bot comment should hopefully make this information easy to find without creating too much noise in the discussion. If not, this can always be changed later. --- .github/workflows/post-release.yml | 4 + .github/workflows/release.yml | 38 +++--- .github/workflows/status.yml | 47 ++++++- .github/workflows/test.yml | 196 +++++++++++++++++++++++++++-- 4 files changed, 261 insertions(+), 24 deletions(-) diff --git a/.github/workflows/post-release.yml b/.github/workflows/post-release.yml index 48e8f981..6524124d 100644 --- a/.github/workflows/post-release.yml +++ b/.github/workflows/post-release.yml @@ -4,6 +4,10 @@ on: branches: [master] types: [released] +defaults: + run: + shell: bash -euv -o pipefail {0} + jobs: post-release: runs-on: ubuntu-22.04 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ff03d473..b2ead2e4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -5,6 +5,10 @@ on: branches: [master] types: [completed] +defaults: + run: + shell: bash -euv -o pipefail {0} + jobs: release: runs-on: ubuntu-22.04 @@ -82,14 +86,14 @@ jobs: echo "LFS_PREV_VERSION=$LFS_PREV_VERSION" >> $GITHUB_ENV # try to find results from tests - - name: collect-table + - name: create-table run: | # previous results to compare against? [ -n "$LFS_PREV_VERSION" ] && curl -sS \ - "$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/` - `status/$LFS_PREV_VERSION?per_page=100" \ + "$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/status/$LFS_PREV_VERSION` + `?per_page=100" \ | jq -re 'select(.sha != env.GITHUB_SHA) | .statuses[]' \ - >> prev-statuses.json \ + >> prev-status.json \ || true # build table for GitHub @@ -113,7 +117,7 @@ jobs: export PREV="$(jq -re ' select(.context == "'"sizes (thumb${c:+, $c}) / $s"'").description | capture("(?[0-9∞]+)").prev' \ - prev-statuses.json || echo 0)" + prev-status.json || echo 0)" ./scripts/summary.py $f --max=stack_limit -Y \ | awk ' NR==2 {$1=0; printf "%s B",$NF} @@ -138,11 +142,12 @@ jobs: [ -e $f ] && table[$i,$j]=$( \ export PREV="$(jq -re ' select(.context == "'"cov / $s"'").description - | capture("(?[0-9\\.]+)").prev' \ - prev-statuses.json || echo 0)" + | capture("(?[0-9]+)/(?[0-9]+)") + | 100*((.prev_a|tonumber) / (.prev_b|tonumber))' \ + prev-status.json || echo 0)" ./scripts/cov.py -u $f -f$s -Y \ | awk -F '[ /%]+' -v s=$s ' - NR==2 {$1=0; printf "%.1f%% of %d %s",$4,$3,s} + NR==2 {$1=0; printf "%d/%d %s",$2,$3,s} NR==2 && ENVIRON["PREV"]+0 != 0 { printf " (%+.1f%%)",$4-ENVIRON["PREV"] }' \ @@ -163,7 +168,7 @@ jobs: export PREV="$(jq -re ' select(.context == "'"bench / $s"'").description | capture("(?[0-9]+)").prev' \ - prev-statuses.json || echo 0)" + prev-status.json || echo 0)" ./scripts/summary.py $f -f$s=bench_$s -Y \ | awk ' NR==2 {$1=0; printf "%s B",$NF} @@ -184,7 +189,7 @@ jobs: do echo -n " " >> table.txt [[ i -eq 2 && j -eq 5 ]] && echo -n "**Benchmarks**" >> table.txt - echo -n "${table[$i,$j]}" >> table.txt + echo -n "${table[$i,$j]:-}" >> table.txt echo -n " |" >> table.txt done echo >> table.txt @@ -193,7 +198,7 @@ jobs: cat table.txt # find changes from history - - name: collect-changes + - name: create-changes run: | [ -n "$LFS_PREV_VERSION" ] || exit 0 # use explicit link to github commit so that release notes can @@ -235,15 +240,18 @@ jobs: run: | # create release and patch version tag (vN.N.N) # only draft if not a patch release - [ -e table.txt ] && export TABLES="$(cat table.txt)" - [ -e changes.txt ] && export CHANGES="$(cat changes.txt)" + [ -e table.txt ] && cat table.txt >> release.txt + echo >> release.txt + [ -e changes.txt ] && cat changes.txt >> release.txt + cat release.txt + curl -sS -X POST -H "authorization: token ${{secrets.BOT_TOKEN}}" \ "$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/releases" \ - -d "$(jq -n '{ + -d "$(jq -n --rawfile release release.txt '{ tag_name: env.LFS_VERSION, name: env.LFS_VERSION | rtrimstr(".0"), target_commitish: "${{github.event.workflow_run.head_sha}}", draft: env.LFS_VERSION | endswith(".0"), - body: [env.TABLES, env.CHANGES | select(.)] | join("\n\n") + body: $release, }' | tee /dev/stderr)" diff --git a/.github/workflows/status.yml b/.github/workflows/status.yml index a970db83..8bd3990c 100644 --- a/.github/workflows/status.yml +++ b/.github/workflows/status.yml @@ -4,11 +4,15 @@ on: workflows: [test] types: [completed] +defaults: + run: + shell: bash -euv -o pipefail {0} + jobs: + # forward custom statuses status: runs-on: ubuntu-22.04 steps: - # custom statuses? - uses: dawidd6/action-download-artifact@v2 continue-on-error: true with: @@ -53,3 +57,44 @@ jobs: target_url: env.TARGET_URL, }' | tee /dev/stderr)" done + + # forward custom pr-comments + comment: + runs-on: ubuntu-22.04 + + # only run on success (we don't want garbage comments!) + if: ${{github.event.workflow_run.conclusion == 'success'}} + + steps: + # generated comment? + - uses: dawidd6/action-download-artifact@v2 + continue-on-error: true + with: + workflow: ${{github.event.workflow_run.name}} + run_id: ${{github.event.workflow_run.id}} + name: comment + path: comment + - name: update-comment + continue-on-error: true + run: | + ls comment + for s in $(shopt -s nullglob ; echo comment/*.json) + do + export NUMBER="$(jq -er '.number' $s)" + export BODY="$(jq -er '.body' $s)" + + # check that the comment was from the most recent commit on the + # pull request + [ "$(curl -sS -H "authorization: token ${{secrets.BOT_TOKEN}}" \ + "$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/pulls/$NUMBER" \ + | jq -er '.head.sha')" \ + == ${{github.event.workflow_run.head_sha}} ] || continue + + # update comment + curl -sS -X POST -H "authorization: token ${{secrets.BOT_TOKEN}}" \ + "$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/issues/` + `$NUMBER/comments" \ + -d "$(jq -n '{ + body: env.BODY, + }' | tee /dev/stderr)" + done diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bc51488c..664246a3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -245,8 +245,8 @@ jobs: | sed -n 's/[^-.]*-\([^.]*\)\..*csv/, \1/p')) / $(echo $f \ | sed -n 's/[^.]*\.\(.*\)\.csv/\1/p')" export PREV="$(curl -sS \ - "$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/status/` - `master?per_page=100" \ + "$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/status/master` + `?per_page=100" \ | jq -re 'select(.sha != env.GITHUB_SHA) | .statuses[] | select(.context == env.CONTEXT).description | capture("(?[0-9∞]+)").prev' \ @@ -289,15 +289,16 @@ jobs: export STEP="cov" export CONTEXT="cov / $s" export PREV="$(curl -sS \ - "$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/status/` - `master?per_page=100" \ + "$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/status/master` + `?per_page=100" \ | jq -re 'select(.sha != env.GITHUB_SHA) | .statuses[] | select(.context == env.CONTEXT).description - | capture("(?[0-9\\.]+)").prev' \ + | capture("(?[0-9]+)/(?[0-9]+)") + | 100*((.prev_a|tonumber) / (.prev_b|tonumber))' \ || echo 0)" export DESCRIPTION="$(./scripts/cov.py -u $f -f$s -Y \ | awk -F '[ /%]+' -v s=$s ' - NR==2 {$1=0; printf "%.1f%% of %d %s",$4,$3,s} + NR==2 {$1=0; printf "%d/%d %s",$2,$3,s} NR==2 && ENVIRON["PREV"]+0 != 0 { printf " (%+.1f%%)",$4-ENVIRON["PREV"] }')" @@ -435,8 +436,8 @@ jobs: export STEP="bench" export CONTEXT="bench / $s" export PREV="$(curl -sS \ - "$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/status/` - `master?per_page=100" \ + "$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/status/master` + `?per_page=100" \ | jq -re 'select(.sha != env.GITHUB_SHA) | .statuses[] | select(.context == env.CONTEXT).description | capture("(?[0-9]+)").prev' \ @@ -584,3 +585,182 @@ jobs: make -B test-runner make -B test + # status related tasks that run after tests + status: + runs-on: ubuntu-22.04 + needs: [test, bench] + steps: + - uses: actions/checkout@v2 + if: ${{github.event_name == 'pull_request'}} + - name: install + if: ${{github.event_name == 'pull_request'}} + run: | + # need a few things + sudo apt-get install -qq gcc python3 python3-pip + pip3 install toml + gcc --version + python3 --version + - uses: actions/download-artifact@v2 + if: ${{github.event_name == 'pull_request'}} + continue-on-error: true + with: + name: sizes + path: sizes + - uses: actions/download-artifact@v2 + if: ${{github.event_name == 'pull_request'}} + continue-on-error: true + with: + name: cov + path: cov + - uses: actions/download-artifact@v2 + if: ${{github.event_name == 'pull_request'}} + continue-on-error: true + with: + name: bench + path: bench + + # try to find results from tests + - name: create-table + if: ${{github.event_name == 'pull_request'}} + run: | + # compare against pull-request target + curl -sS \ + "$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/status/` + `${{github.event.pull_request.base.ref}}` + `?per_page=100" \ + | jq -re 'select(.sha != env.GITHUB_SHA) | .statuses[]' \ + >> prev-status.json \ + || true + + # build table for GitHub + declare -A table + + # sizes table + i=0 + j=0 + for c in "" readonly threadsafe migrate error-asserts + do + # per-config results + c_or_default=${c:-default} + c_camel=${c_or_default^} + table[$i,$j]=$c_camel + ((j+=1)) + + for s in code stack struct + do + f=sizes/thumb${c:+-$c}.$s.csv + [ -e $f ] && table[$i,$j]=$( \ + export PREV="$(jq -re ' + select(.context == "'"sizes (thumb${c:+, $c}) / $s"'").description + | capture("(?[0-9∞]+)").prev' \ + prev-status.json || echo 0)" + ./scripts/summary.py $f --max=stack_limit -Y \ + | awk ' + NR==2 {$1=0; printf "%s B",$NF} + NR==2 && ENVIRON["PREV"]+0 != 0 { + printf " (%+.1f%%)",100*($NF-ENVIRON["PREV"])/ENVIRON["PREV"] + }' \ + | sed -e 's/ /\ /g') + ((j+=1)) + done + ((j=0, i+=1)) + done + + # coverage table + i=0 + j=4 + for s in lines branches + do + table[$i,$j]=${s^} + ((j+=1)) + + f=cov/cov.csv + [ -e $f ] && table[$i,$j]=$( \ + export PREV="$(jq -re ' + select(.context == "'"cov / $s"'").description + | capture("(?[0-9]+)/(?[0-9]+)") + | 100*((.prev_a|tonumber) / (.prev_b|tonumber))' \ + prev-status.json || echo 0)" + ./scripts/cov.py -u $f -f$s -Y \ + | awk -F '[ /%]+' -v s=$s ' + NR==2 {$1=0; printf "%d/%d %s",$2,$3,s} + NR==2 && ENVIRON["PREV"]+0 != 0 { + printf " (%+.1f%%)",$4-ENVIRON["PREV"] + }' \ + | sed -e 's/ /\ /g') + ((j=4, i+=1)) + done + + # benchmark table + i=3 + j=4 + for s in readed proged erased + do + table[$i,$j]=${s^} + ((j+=1)) + + f=bench/bench.csv + [ -e $f ] && table[$i,$j]=$( \ + export PREV="$(jq -re ' + select(.context == "'"bench / $s"'").description + | capture("(?[0-9]+)").prev' \ + prev-status.json || echo 0)" + ./scripts/summary.py $f -f$s=bench_$s -Y \ + | awk ' + NR==2 {$1=0; printf "%s B",$NF} + NR==2 && ENVIRON["PREV"]+0 != 0 { + printf " (%+.1f%%)",100*($NF-ENVIRON["PREV"])/ENVIRON["PREV"] + }' \ + | sed -e 's/ /\ /g') + ((j=4, i+=1)) + done + + # build the actual table + echo "| | Code | Stack | Structs | | Coverage |" >> table.txt + echo "|:--|-----:|------:|--------:|:--|---------:|" >> table.txt + for ((i=0; i<6; i++)) + do + echo -n "|" >> table.txt + for ((j=0; j<6; j++)) + do + echo -n " " >> table.txt + [[ i -eq 2 && j -eq 5 ]] && echo -n "**Benchmarks**" >> table.txt + echo -n "${table[$i,$j]:-}" >> table.txt + echo -n " |" >> table.txt + done + echo >> table.txt + done + + cat table.txt + + # create a bot comment for successful runs on pull requests + - name: create-comment + if: ${{github.event_name == 'pull_request'}} + run: | + touch comment.txt + echo "
" >> comment.txt + echo "" >> comment.txt + echo "Tests passed ✓, ` + `Code: $(awk 'NR==3 {print $4}' table.txt || true), ` + `Stack: $(awk 'NR==3 {print $6}' table.txt || true), ` + `Structs: $(awk 'NR==3 {print $8}' table.txt || true)" \ + >> comment.txt + echo "" >> comment.txt + echo >> comment.txt + [ -e table.txt ] && cat table.txt >> comment.txt + echo >> comment.txt + echo "
" >> comment.txt + cat comment.txt + + mkdir -p comment + jq -n --rawfile comment comment.txt '{ + number: ${{github.event.number}}, + body: $comment, + }' | tee comment/comment.json + - name: upload-comment + uses: actions/upload-artifact@v2 + with: + name: comment + path: comment + retention-days: 1 +