Auto merge of #29201 - delan:ci-mac-wpt, r=<try>

bors-servo · web-flow · commit 8c8fabdfb6fc · 2023-01-04T20:10:38.000-05:00
CI: run WPT tests on macOS This patch uncomments and fixes the mac-wpt job in the main workflow, with a few changes: * we downgrade python3 and pip3 to 3.9, because inspect.getargspec was [removed in 3.11](https://docs.python.org/3/whatsnew/3.11.html#removed) * we should apparently move to [inspect.getfullargspec](https://docs.python.org/3/library/inspect.html#inspect.getfullargspec) * we download the real Servo artifact, rather than the fake Servo for testing We also align the rest of the job (other than `--timeout-multiplier`) to be as close as possible to linux-wpt, based on the assumption that linux-wpt is more up to date. This includes: * setting strategy.fail-fast to false * adding a failure() condition to the “Archive logs” step * adding the chunk id to log filenames There are some failing WPT tests, so for now the job is not required to succeed. Example run: <https://github.com/delan/servo/actions/runs/3837530905> --- - [ ] ~~`./mach build -d` does not report any errors~~ - [ ] ~~`./mach test-tidy` does not report any errors~~ - [x] These changes fix #28284 (GitHub issue number if applicable)  - [ ] There are tests for these changes OR - [x] These changes do not require tests because they affect the CI configuration
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -144,59 +144,62 @@ jobs:
           name: release-binary-macos
           path: target.tar.gz
 
-  # mac-wpt:
-  #   #needs: build-mac
-  #   runs-on: macos-10.12
-  #   env:
-  #     max_chunk_id: 20
-  #   strategy:
-  #     matrix:
-  #       chunk_id: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]
-  #   steps:
-  #     - uses: actions/checkout@v3
-  #       with:
-  #         fetch-depth: 2
-  #
-  #     #- name: Download release binary
-  #     #  uses: actions/download-artifact@v3
-  #     #  with:
-  #     #    name: release-binary-macos
-  #
-  #     - name: Fake build
-  #       run: |
-  #         wget https://joshmatthews.net/release-binary-macos.zip
-  #         unzip release-binary-macos.zip
-  #
-  #     - name: Prep test environment
-  #       run: |
-  #         brew install gnu-tar
-  #         gtar -xzf target.tar.gz
-  #         python3 -m pip install --upgrade pip virtualenv
-  #         brew bundle install --verbose --no-upgrade --file=etc/taskcluster/macos/Brewfile
-  #     - name: Smoketest
-  #       run: python3 ./mach smoketest
-  #     - name: Run tests
-  #       run: |
-  #         python3 ./mach test-wpt \
-  #           --release --processes=3 --timeout-multiplier=8 \
-  #           --total-chunks=${{ env.max_chunk_id }} --this-chunk=${{ matrix.chunk_id }} \
-  #           --log-raw=test-wpt.log \
-  #           --log-servojson=wpt-jsonsummary.log \
-  #           --always-succeed | cat
-  #         python3 ./mach filter-intermittents wpt-jsonsummary.log \
-  #           --log-intermittents=intermittents.log \
-  #           --log-filteredsummary=filtered-wpt-summary.log \
-  #           --tracker-api=default --reporter-api=default
-  #
-  #     - name: Archive logs
-  #       uses: actions/upload-artifact@v3
-  #       with:
-  #         name: wpt${{ matrix.chunk_id }}-logs-macos
-  #         path: |
-  #           test-wpt.log
-  #           wpt-jsonsummary.log
-  #           filtered-wpt-summary.log
-  #           intermittents.log
+  mac-wpt:
+    name: Mac WPT Tests
+    runs-on: macos-12
+    needs: ["build-mac"]
+    env:
+      max_chunk_id: 20
+    strategy:
+      fail-fast: false
+      matrix:
+        chunk_id: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 2
+      - uses: actions/download-artifact@v3
+        with:
+          name: release-binary-macos
+      # TODO: Remove this step when the compatibility issue between mozjs and
+      #       Homebrew's Python 3.10 formula (servo/rust-mozjs#559) is fixed
+      - name: Select Python 3.9
+        run: |
+          brew install python@3.9
+          cd $(dirname $(which python3.9))
+          rm -f python3 pip3
+          ln -s python3.9 python3
+          ln -s pip3.9 pip3
+      - name: Prep test environment
+        run: |
+          gtar -xzf target.tar.gz
+          python3 -m pip install --upgrade pip virtualenv
+          brew bundle install --verbose --no-upgrade --file=etc/taskcluster/macos/Brewfile
+      - name: Smoketest
+        run: python3 ./mach smoketest
+      - name: Run tests
+        run: |
+          python3 ./mach test-wpt \
+            --release --processes $(nproc) --timeout-multiplier 8 \
+            --total-chunks ${{ env.max_chunk_id }} --this-chunk ${{ matrix.chunk_id }} \
+            --log-raw test-wpt.${{ matrix.chunk_id }}.log \
+            --log-servojson wpt-jsonsummary.${{ matrix.chunk_id }}.log \
+            --always-succeed
+          python3 ./mach filter-intermittents wpt-jsonsummary.${{ matrix.chunk_id }}.log \
+            --log-intermittents=intermittents.${{ matrix.chunk_id }}.log \
+            --log-filteredsummary=filtered-wpt-summary.${{ matrix.chunk_id }}.log \
+            --tracker-api=default --reporter-api=default
+
+      - name: Archive logs
+        uses: actions/upload-artifact@v3
+        if: ${{ failure() }}
+        with:
+          name: wpt${{ matrix.chunk_id }}-logs-macos
+          path: |
+            test-wpt.${{ matrix.chunk_id }}.log
+            wpt-jsonsummary.${{ matrix.chunk_id }}.log
+            filtered-wpt-summary.${{ matrix.chunk_id }}.log
+            intermittents.${{ matrix.chunk_id }}.log
 
   build-linux:
     name: Build (Linux)
@@ -290,6 +293,7 @@ jobs:
       - "build-linux"
       - "build-mac"
       - "linux-wpt"
+      # - "mac-wpt"
 
     steps:
       - name: Mark the job as successful
