529e5fa6c2
## Summary Generate a timing diff across the whole ecosystem and deploy it to CloudFlare pages. The timing information is collected already, we just need to create and upload the HTML report. The timing results are just based on a single run. No statistical analysis across multiple runs or similar is performed. This means that results can be noisy, as can be seen on this PR, where we see slowdowns up to 1.26× and speedups down to 0.89×, even though the change should be neutral. Across all projects, these random events cancel out and we see an average factor of 1.01×. So I think this feature can still be interesting, given that it comes "for free". We just need to keep in mind that it will be noisy, and shouldn't read too much into these results. ## Test Plan CI run on this PR (see the new *timing results* link).
154 lines
5.0 KiB
YAML
154 lines
5.0 KiB
YAML
name: ty ecosystem-analyzer
|
|
|
|
permissions: {}
|
|
|
|
on:
|
|
pull_request:
|
|
types: [labeled]
|
|
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.event.pull_request.number || github.sha }}
|
|
cancel-in-progress: true
|
|
|
|
env:
|
|
CARGO_INCREMENTAL: 0
|
|
CARGO_NET_RETRY: 10
|
|
CARGO_TERM_COLOR: always
|
|
RUSTUP_MAX_RETRIES: 10
|
|
RUST_BACKTRACE: 1
|
|
REF_NAME: ${{ github.ref_name }}
|
|
CF_API_TOKEN_EXISTS: ${{ secrets.CF_API_TOKEN != '' }}
|
|
|
|
jobs:
|
|
ty-ecosystem-analyzer:
|
|
name: Compute diagnostic diff
|
|
runs-on: depot-ubuntu-22.04-32
|
|
timeout-minutes: 20
|
|
if: contains(github.event.label.name, 'ecosystem-analyzer')
|
|
steps:
|
|
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
|
with:
|
|
path: ruff
|
|
fetch-depth: 0
|
|
persist-credentials: false
|
|
|
|
- name: Install the latest version of uv
|
|
uses: astral-sh/setup-uv@b75a909f75acd358c2196fb9a5f1299a9a8868a4 # v6.7.0
|
|
|
|
- uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0
|
|
with:
|
|
workspaces: "ruff"
|
|
|
|
- name: Install Rust toolchain
|
|
run: rustup show
|
|
|
|
- name: Compute diagnostic diff
|
|
shell: bash
|
|
run: |
|
|
cd ruff
|
|
|
|
echo "Enabling configuration overloads (see .github/mypy-primer-ty.toml)"
|
|
mkdir -p ~/.config/ty
|
|
cp .github/mypy-primer-ty.toml ~/.config/ty/ty.toml
|
|
|
|
echo "new commit"
|
|
git checkout -b new_commit "$GITHUB_SHA"
|
|
git rev-list --format=%s --max-count=1 new_commit
|
|
cp crates/ty_python_semantic/resources/primer/good.txt projects_new.txt
|
|
|
|
echo "old commit (merge base)"
|
|
MERGE_BASE="$(git merge-base "$GITHUB_SHA" "origin/$GITHUB_BASE_REF")"
|
|
git checkout -b old_commit "$MERGE_BASE"
|
|
git rev-list --format=%s --max-count=1 old_commit
|
|
cp crates/ty_python_semantic/resources/primer/good.txt projects_old.txt
|
|
|
|
cd ..
|
|
|
|
uv tool install "git+https://github.com/astral-sh/ecosystem-analyzer@fc0f612798710b0dd69bb7528bc9b361dc60bd43"
|
|
|
|
ecosystem-analyzer \
|
|
--repository ruff \
|
|
diff \
|
|
--profile=release \
|
|
--projects-old ruff/projects_old.txt \
|
|
--projects-new ruff/projects_new.txt \
|
|
--old old_commit \
|
|
--new new_commit \
|
|
--output-old diagnostics-old.json \
|
|
--output-new diagnostics-new.json
|
|
|
|
mkdir dist
|
|
|
|
ecosystem-analyzer \
|
|
generate-diff \
|
|
diagnostics-old.json \
|
|
diagnostics-new.json \
|
|
--old-name "main (merge base)" \
|
|
--new-name "$REF_NAME" \
|
|
--output-html dist/diff.html
|
|
|
|
ecosystem-analyzer \
|
|
generate-diff-statistics \
|
|
diagnostics-old.json \
|
|
diagnostics-new.json \
|
|
--old-name "main (merge base)" \
|
|
--new-name "$REF_NAME" \
|
|
--output diff-statistics.md
|
|
|
|
ecosystem-analyzer \
|
|
generate-timing-diff \
|
|
diagnostics-old.json \
|
|
diagnostics-new.json \
|
|
--old-name "main (merge base)" \
|
|
--new-name "$REF_NAME" \
|
|
--output-html dist/timing.html
|
|
|
|
echo '## `ecosystem-analyzer` results' > comment.md
|
|
echo >> comment.md
|
|
cat diff-statistics.md >> comment.md
|
|
|
|
cat diff-statistics.md >> "$GITHUB_STEP_SUMMARY"
|
|
|
|
echo ${{ github.event.number }} > pr-number
|
|
|
|
- name: "Deploy to Cloudflare Pages"
|
|
if: ${{ env.CF_API_TOKEN_EXISTS == 'true' }}
|
|
id: deploy
|
|
uses: cloudflare/wrangler-action@da0e0dfe58b7a431659754fdf3f186c529afbe65 # v3.14.1
|
|
with:
|
|
apiToken: ${{ secrets.CF_API_TOKEN }}
|
|
accountId: ${{ secrets.CF_ACCOUNT_ID }}
|
|
command: pages deploy dist --project-name=ty-ecosystem --branch ${{ github.head_ref }} --commit-hash ${GITHUB_SHA}
|
|
|
|
- name: "Append deployment URL"
|
|
if: ${{ env.CF_API_TOKEN_EXISTS == 'true' }}
|
|
env:
|
|
DEPLOYMENT_URL: ${{ steps.deploy.outputs.pages-deployment-alias-url }}
|
|
run: |
|
|
echo >> comment.md
|
|
echo "**[Full report with detailed diff]($DEPLOYMENT_URL/diff)** ([timing results]($DEPLOYMENT_URL/timing))" >> comment.md
|
|
|
|
- name: Upload comment
|
|
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
|
with:
|
|
name: comment.md
|
|
path: comment.md
|
|
|
|
- name: Upload pr-number
|
|
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
|
with:
|
|
name: pr-number
|
|
path: pr-number
|
|
|
|
- name: Upload diagnostics diff
|
|
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
|
with:
|
|
name: diff.html
|
|
path: dist/diff.html
|
|
|
|
- name: Upload timing diff
|
|
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
|
with:
|
|
name: timing.html
|
|
path: dist/timing.html
|