Consolidate Build & Test CI (#32929)

* Consolidate Build & Test CI

* Add missing workflow dependency

* Fix artifact name clash

* Fix playwright config

* Fix playwright_ew job

* Fix ed tests

* Fix playwright tags

* Iterate

* Fix file reads

* Fix sample-files paths

* Fix PW_TAG

* Fix blob report paths

* Delint

* Fix build-and-test.yaml

* Iterate

* Fix consentHomeserver.ts

* Simplify

* Iterate

* Delint

* Iterate

* Iterate

* Iterate

* Specify shell

* Simplify

* Delete apps/web/playwright/sample-files/index.ts

* Discard changes to apps/web/playwright/sample-files/index.ts

* Exclude playwright-common from coverage gate

* Attempt to speed up arm64 desktop test

* Revert "Attempt to speed up arm64 desktop test"

This reverts commit 8fa8ff0c78.

* Iterate

* Fix cache key

* Accept python or python3 as per node-gyp

* Accept python or python3 as per node-gypd

* Exclude apps/desktop/hak from coverage gate

.github/workflows/build-and-test-netlify.yaml

@@ -5,7 +5,7 @@ on:
     # Privilege escalation necessary to publish to Netlify
     # 🚨 We must not execute any checked out code here.
     workflow_run: # zizmor: ignore[dangerous-triggers]
-        workflows: ["End to End Tests"]
+        workflows: ["Build & Test"]
         types:
             - completed
 

.github/workflows/build-and-test.yaml

@@ -1,7 +1,13 @@
-# Produce a build of element-web with this version of react-sdk
-# and any matching branches of element-web and js-sdk, output it
-# as an artifact and run end-to-end tests.
-name: End to End Tests
+# builds Element Web
+# runs Playwright tests against the built Element Web
+# builds Element Desktop using the built Element Web
+#
+# Tries to use a matching js-sdk branch for the build.
+#
+# Produces a `webapp` artifact
+# Produces multiple Desktop artifacts
+# Produces multiple Playwright report artifacts
+name: Build & Test
 on:
     # CRON to run all Projects at 6am UTC
     schedule:
@@ -10,7 +16,8 @@ on:
     merge_group:
         types: [checks_requested]
     push:
-        branches: [develop, master]
+        # We do not build on push to develop as the merge_group check handles that
+        branches: [staging, master]
     repository_dispatch:
         types: [element-web-notify]
 
@@ -35,15 +42,15 @@ concurrency:
 env:
     # fetchdep.sh needs to know our PR number
     PR_NUMBER: ${{ github.event.pull_request.number }}
-    # Use 6 runners in the default case, but 4 when running on a schedule where we run all 5 projects (20 runners total)
-    NUM_RUNNERS: ${{ github.event_name == 'schedule' && 4 || 6 }}
+    # Use 4 runners in the default case, but only 1 when running on a schedule where we run all 5 projects
+    NUM_RUNNERS: ${{ github.event_name == 'schedule' && 1 || 4 }}
     NX_DEFAULT_OUTPUT_STYLE: stream-without-prefixes
 
 permissions: {} # No permissions required
 
 jobs:
-    build:
-        name: "Build Element-Web"
+    build_ew:
+        name: "Build Element Web"
         runs-on: ubuntu-24.04
         if: inputs.skip != true
         outputs:
@@ -94,9 +101,9 @@ jobs:
                       const matrix = Array.from({ length: numRunners }, (_, i) => i + 1);
                       core.setOutput("matrix", JSON.stringify(matrix));
 
-    playwright:
-        name: "Run Tests [${{ matrix.project }}] ${{ matrix.runner }}/${{ needs.build.outputs.num-runners }}"
-        needs: build
+    playwright_ew:
+        name: "Run Tests [${{ matrix.project }}] ${{ matrix.runner }}/${{ needs.build_ew.outputs.num-runners }}"
+        needs: build_ew
         if: inputs.skip != true
         runs-on: ubuntu-24.04
         permissions:
@@ -107,7 +114,7 @@ jobs:
             fail-fast: false
             matrix:
                 # Run multiple instances in parallel to speed up the tests
-                runner: ${{ fromJSON(needs.build.outputs.runners-matrix) }}
+                runner: ${{ fromJSON(needs.build_ew.outputs.runners-matrix) }}
                 project:
                     - Chrome
                     - Firefox
@@ -179,29 +186,85 @@ jobs:
                       --project="${{ matrix.project }}" \
                       ${{ (github.event_name == 'pull_request' && matrix.runAllTests == false ) && '--grep-invert @mergequeue' || '' }}
               env:
-                  SHARD: ${{ format('{0}/{1}', matrix.runner, needs.build.outputs.num-runners) }}
+                  SHARD: ${{ format('{0}/{1}', matrix.runner, needs.build_ew.outputs.num-runners) }}
 
             - name: Upload blob report to GitHub Actions Artifacts
               if: always()
               uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
               with:
-                  name: all-blob-reports-${{ matrix.project }}-${{ matrix.runner }}
+                  name: blob-report-${{ matrix.project }}-${{ matrix.runner }}
                   path: apps/web/blob-report
                   retention-days: 1
+                  if-no-files-found: error
 
     downstream-modules:
         name: Downstream Playwright tests [element-modules]
-        needs: build
+        needs: build_ew
         if: inputs.skip != true && github.event_name == 'merge_group'
         uses: element-hq/element-modules/.github/workflows/reusable-playwright-tests.yml@main # zizmor: ignore[unpinned-uses]
         with:
             webapp-artifact: webapp
 
+    prepare_ed:
+        name: "Prepare Element Desktop"
+        uses: ./.github/workflows/build_desktop_prepare.yaml
+        needs: build_ew
+        if: inputs.skip != true
+        permissions:
+            contents: read
+        with:
+            config: ${{ (github.event.pull_request.base.ref || github.ref_name) == 'develop' && 'element.io/nightly' || 'element.io/release' }}
+            version: ${{ (github.event.pull_request.base.ref || github.ref_name) == 'develop' && 'develop' || '' }}
+            webapp-artifact: webapp
+
+    build_ed_windows:
+        needs: prepare_ed
+        name: "Desktop Windows"
+        uses: ./.github/workflows/build_desktop_windows.yaml
+        if: inputs.skip != true
+        strategy:
+            matrix:
+                arch: [x64, ia32, arm64]
+        with:
+            arch: ${{ matrix.arch }}
+            blob_report: true
+
+    build_ed_linux:
+        needs: prepare_ed
+        name: "Desktop Linux"
+        uses: ./.github/workflows/build_desktop_linux.yaml
+        if: inputs.skip != true
+        strategy:
+            matrix:
+                sqlcipher: [system, static]
+                arch: [amd64, arm64]
+                runAllTests:
+                    - ${{ github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'X-Run-All-Tests') }}
+                # We ship static sqlcipher builds, so delegate testing the system builds to the merge queue
+                exclude:
+                    - runAllTests: false
+                      sqlcipher: system
+        with:
+            sqlcipher: ${{ matrix.sqlcipher }}
+            arch: ${{ matrix.arch }}
+            blob_report: true
+
+    build_ed_macos:
+        needs: prepare_ed
+        name: "Desktop macOS"
+        uses: ./.github/workflows/build_desktop_macos.yaml
+        if: inputs.skip != true
+        with:
+            blob_report: true
+
     complete:
         name: end-to-end-tests
         needs:
-            - playwright
+            - playwright_ew
             - downstream-modules
+            - build_ed_windows
+            - build_ed_linux
+            - build_ed_macos
         if: always()
         runs-on: ubuntu-24.04
         steps:
@@ -227,18 +290,20 @@ jobs:
               if: inputs.skip != true
               uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
               with:
-                  pattern: all-blob-reports-*
-                  path: apps/web/all-blob-reports
+                  pattern: blob-report-*
+                  path: all-blob-reports
                   merge-multiple: true
 
             - name: Merge into HTML Report
               if: inputs.skip != true
-              working-directory: apps/web
-              run: pnpm playwright merge-reports --reporter=html,./playwright/flaky-reporter.ts,@element-hq/element-web-playwright-common/lib/stale-screenshot-reporter.js ./all-blob-reports
+              run: |
+                  pnpm playwright merge-reports \
+                      --config=playwright-merge.config.ts \
+                      ./all-blob-reports
               env:
                   # Only pass creds to the flaky-reporter on main branch runs
                   GITHUB_TOKEN: ${{ github.ref_name == 'develop' && secrets.ELEMENT_BOT_TOKEN || '' }}
-                  PLAYWRIGHT_HTML_TITLE: ${{ case(github.event_name == 'pull_request', format('EW Playwright Report PR-{0}', env.PR_NUMBER), 'EW Playwright Report') }}
+                  PLAYWRIGHT_HTML_TITLE: ${{ case(github.event_name == 'pull_request', format('Playwright Report PR-{0}', env.PR_NUMBER), 'Playwright Report') }}
 
             # Upload the HTML report even if one of our reporters fails, this can happen when stale screenshots are detected
             - name: Upload HTML report
@@ -246,7 +311,7 @@ jobs:
               uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
               with:
                   name: html-report
-                  path: apps/web/playwright-report
+                  path: playwright-report
                   retention-days: 14
                   if-no-files-found: error
 

.github/workflows/build_desktop_and_test.yaml

@@ -1,89 +0,0 @@
-name: Build and Test
-on:
-    pull_request: {}
-    push:
-        branches: [develop, staging, master]
-concurrency:
-    group: ${{ github.workflow }}-${{ github.ref }}
-    cancel-in-progress: true
-permissions: {} # No permissions required
-jobs:
-    fetch:
-        uses: ./.github/workflows/build_desktop_prepare.yaml
-        permissions:
-            contents: read
-        with:
-            config: ${{ (github.event.pull_request.base.ref || github.ref_name) == 'develop' && 'element.io/nightly' || 'element.io/release' }}
-            version: ${{ (github.event.pull_request.base.ref || github.ref_name) == 'develop' && 'develop' || '' }}
-            branch-matching: true
-
-    windows:
-        needs: fetch
-        name: Windows
-        uses: ./.github/workflows/build_desktop_windows.yaml
-        strategy:
-            matrix:
-                arch: [x64, ia32, arm64]
-        with:
-            arch: ${{ matrix.arch }}
-            blob_report: true
-
-    linux:
-        needs: fetch
-        name: "Linux (${{ matrix.arch }}) (sqlcipher: ${{ matrix.sqlcipher }})"
-        uses: ./.github/workflows/build_desktop_linux.yaml
-        strategy:
-            matrix:
-                sqlcipher: [system, static]
-                arch: [amd64, arm64]
-        with:
-            sqlcipher: ${{ matrix.sqlcipher }}
-            arch: ${{ matrix.arch }}
-            blob_report: true
-
-    macos:
-        needs: fetch
-        name: macOS
-        uses: ./.github/workflows/build_desktop_macos.yaml
-        with:
-            blob_report: true
-
-    tests-done:
-        needs: [windows, linux, macos]
-        runs-on: ubuntu-24.04
-        if: ${{ !cancelled() }}
-        steps:
-            - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
-              with:
-                  persist-credentials: false
-
-            - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5
-            - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6
-              with:
-                  cache: "pnpm"
-                  node-version: "lts/*"
-
-            - name: Install dependencies
-              run: pnpm install --frozen-lockfile
-
-            - name: Download blob reports from GitHub Actions Artifacts
-              uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
-              with:
-                  pattern: blob-report-*
-                  path: apps/desktop/all-blob-reports
-                  merge-multiple: true
-
-            - name: Merge into HTML Report
-              working-directory: apps/desktop
-              run: pnpm playwright merge-reports -c ./playwright.config.ts --reporter=html ./all-blob-reports
-
-            - name: Upload HTML report
-              if: always()
-              uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
-              with:
-                  name: html-report
-                  path: apps/desktop/playwright-report
-                  retention-days: 14
-
-            - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')
-              run: exit 1

.github/workflows/build_desktop_linux.yaml

@@ -28,7 +28,7 @@ on:
                 type: string
                 required: false
                 description: |
-                    The name of the prepare artifact to use, defaults to 'webapp'.
+                    The name of the prepare artifact to use, defaults to 'desktop-prepare'.
                     The artifact must contain the following:
                     + webapp.asar - the asar archive of the webapp to embed in the desktop app
                     + electronVersion - the version of electron to use for cache keying
@@ -38,7 +38,7 @@ on:
 
                     The artifact can also contain any additional files which will be applied as overrides to the checkout root before building,
                     for example icons in the `build/` directory to override the app icons.
-                default: "webapp"
+                default: "desktop-prepare"
             test:
                 type: boolean
                 required: false
@@ -73,20 +73,8 @@ jobs:
         # https://github.com/matrix-org/seshat/issues/135
         runs-on: ${{ inputs.runs-on || (inputs.arch == 'arm64' && 'ubuntu-22.04-arm' || 'ubuntu-22.04') }}
         env:
-            HAK_DOCKER_IMAGE: ghcr.io/element-hq/element-web/desktop-build-env
+            HAK_DOCKER_IMAGE: ghcr.io/element-hq/element-web/desktop-build-env:${{ case(github.event_name == 'push', inputs.ref || github.ref_name, github.event_name == 'release', 'staging', 'develop') }}
         steps:
-            - name: Resolve docker image tag for push
-              if: github.event_name == 'push'
-              run: echo "HAK_DOCKER_IMAGE=$HAK_DOCKER_IMAGE:$REF" >> $GITHUB_ENV
-              env:
-                  REF: ${{ inputs.ref || github.ref_name }}
-            - name: Resolve docker image tag for release
-              if: github.event_name == 'release'
-              run: echo "HAK_DOCKER_IMAGE=$HAK_DOCKER_IMAGE:staging" >> $GITHUB_ENV
-            - name: Resolve docker image tag for other triggers
-              if: github.event_name != 'push' && github.event_name != 'release'
-              run: echo "HAK_DOCKER_IMAGE=$HAK_DOCKER_IMAGE:develop" >> $GITHUB_ENV
-
             - uses: nbucic/variable-mapper@0673f6891a0619ba7c002ecfed0f9f4f39017b6f
               id: config
               with:
@@ -95,11 +83,9 @@ jobs:
                   map: |
                       {
                         "amd64": {
-                          "target": "x86_64-unknown-linux-gnu",
                           "arch": "x86-64"
                         },
                         "arm64": {
-                          "target": "aarch64-unknown-linux-gnu",
                           "arch": "aarch64",
                           "build-args": "--arm64"
                         }
@@ -120,7 +106,7 @@ jobs:
               id: cache
               uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5
               with:
-                  key: ${{ runner.os }}-${{ github.ref_name }}-${{ inputs.sqlcipher }}-${{ inputs.arch }}-${{ hashFiles('hakHash', 'electronVersion', 'dockerbuild/*') }}
+                  key: ${{ runner.os }}-${{ github.ref_name }}-${{ inputs.sqlcipher }}-${{ inputs.arch }}-${{ hashFiles('apps/desktop/hakHash', 'apps/desktop/electronVersion', 'apps/desktop/dockerbuild/*') }}
                   path: |
                       apps/desktop/.hak
 
@@ -135,7 +121,7 @@ jobs:
 
             - name: Install Deps
               working-directory: apps/desktop
-              run: pnpm install --frozen-lockfile
+              run: "pnpm install --frozen-lockfile --filter element-desktop"
 
             - name: "Get modified files"
               id: changed_files

.github/workflows/build_desktop_macos.yaml

@@ -37,7 +37,7 @@ on:
                 type: string
                 required: false
                 description: |
-                    The name of the prepare artifact to use, defaults to 'webapp'.
+                    The name of the prepare artifact to use, defaults to 'desktop-prepare'.
                     The artifact must contain the following:
                     + webapp.asar - the asar archive of the webapp to embed in the desktop app
                     + electronVersion - the version of electron to use for cache keying
@@ -46,7 +46,7 @@ on:
 
                     The artifact can also contain any additional files which will be applied as overrides to the checkout root before building,
                     for example icons in the `build/` directory to override the app icons.
-                default: "webapp"
+                default: "desktop-prepare"
             test:
                 type: boolean
                 required: false
@@ -92,7 +92,7 @@ jobs:
               id: cache
               uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5
               with:
-                  key: ${{ runner.os }}-${{ hashFiles('hakHash', 'electronVersion') }}
+                  key: ${{ runner.os }}-${{ hashFiles('apps/desktop/hakHash', 'apps/desktop/electronVersion') }}
                   path: |
                       apps/desktop/.hak
 
@@ -121,7 +121,7 @@ jobs:
 
             - name: Install Deps
               working-directory: apps/desktop
-              run: "pnpm install --frozen-lockfile"
+              run: "pnpm install --frozen-lockfile --filter element-desktop"
 
             - name: Build Natives
               if: steps.cache.outputs.cache-hit != 'true'

.github/workflows/build_desktop_prepare.yaml

@@ -20,11 +20,10 @@ on:
                 required: false
                 default: false
                 description: "Whether the build should be deployed to production"
-            branch-matching:
-                type: boolean
+            webapp-artifact:
+                type: string
                 required: false
-                default: false
-                description: "Whether the branch name should be matched to find the element-web commit"
+                description: "Name of the webapp artifact that should be used, will fetch a relevant build if omitted"
         secrets:
             # Required if `nightly` is set
             CF_R2_ACCESS_KEY_ID:
@@ -66,28 +65,26 @@ jobs:
 
             - name: Install Deps
               working-directory: apps/desktop
-              run: "pnpm install --frozen-lockfile"
+              run: "pnpm install --frozen-lockfile --filter element-desktop"
 
-            - name: Fetch Element Web (matching branch)
-              id: branch-matching
-              if: inputs.branch-matching
+            - name: Fetch Element Web (from artifact)
+              if: inputs.webapp-artifact != ''
+              uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
+              with:
+                  name: ${{ inputs.webapp-artifact }}
+                  path: apps/desktop/webapp
+
+            - name: Build webapp.asar (from artifact)
+              if: inputs.webapp-artifact != ''
               working-directory: apps/desktop
-              continue-on-error: true
               run: |
-                  scripts/branch-match.sh
-                  cp "$CONFIG_DIR/config.json" element-web/
-                  pnpm --cwd element-web install --frozen-lockfile
-                  pnpm --cwd element-web run build
-                  mv element-web/webapp .
+                  cp -f "$CONFIG_DIR/config.json" webapp/config.json
                   pnpm run asar-webapp
               env:
-                  # These must be set for branch-match.sh to get the right branch
-                  REPOSITORY: ${{ github.repository }}
-                  PR_NUMBER: ${{ github.event.pull_request.number }}
                   CONFIG_DIR: ${{ inputs.config }}
 
             - name: Fetch Element Web (${{ inputs.version }})
-              if: steps.branch-matching.outcome == 'failure' || steps.branch-matching.outcome == 'skipped'
+              if: inputs.webapp-artifact == ''
               working-directory: apps/desktop
               run: pnpm run fetch --noverify -d ${CONFIG} ${VERSION}
               env:
@@ -189,7 +186,7 @@ jobs:
 
             - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
               with:
-                  name: webapp
+                  name: desktop-prepare
                   retention-days: 1
                   path: |
                       apps/desktop/webapp.asar

.github/workflows/build_desktop_test.yaml

@@ -48,8 +48,7 @@ jobs:
                   cache: "pnpm"
 
             - name: Install Deps
-              working-directory: apps/desktop
-              run: "pnpm install --frozen-lockfile"
+              run: "pnpm install --frozen-lockfile --filter element-desktop"
 
             - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
               with:
@@ -85,12 +84,19 @@ jobs:
                   EXECUTABLE: ${{ steps.executable.outputs.path }}
 
             - name: Run tests
-              uses: coactions/setup-xvfb@6b00cf1889f4e1d5a48635647013c0508128ee1a
               timeout-minutes: 20
-              with:
-                  run: pnpm -C apps/desktop test --project=${{ inputs.project }} ${{ runner.os != 'Linux' && '--ignore-snapshots' || '' }} ${{ inputs.blob_report == false && '--reporter=html' || '' }} ${{ inputs.args }}
+              shell: bash
+              working-directory: apps/desktop
+              run: |
+                  $PREFIX pnpm playwright test \
+                      ${{ runner.os != 'Linux' && '--ignore-snapshots' || '' }} \
+                      ${{ inputs.blob_report == false && '--reporter=html' || '' }} \
+                      $ARGS
               env:
+                  PREFIX: ${{ runner.os == 'Linux' && 'xvfb-run' || '' }}
+                  PW_TAG: ${{ inputs.project }}
                   ELEMENT_DESKTOP_EXECUTABLE: ${{ steps.executable.outputs.path }}
+                  ARGS: ${{ inputs.args }}
 
             - name: Upload blob report
               if: always() && inputs.blob_report
@@ -99,6 +105,7 @@ jobs:
                   name: blob-report-${{ inputs.artifact }}
                   path: apps/desktop/blob-report
                   retention-days: 1
+                  if-no-files-found: error
 
             - name: Upload HTML report
               if: always() && inputs.blob_report == false
@@ -107,3 +114,4 @@ jobs:
                   name: ${{ inputs.artifact }}-test
                   path: apps/desktop/playwright-report
                   retention-days: 14
+                  if-no-files-found: error

.github/workflows/build_desktop_windows.yaml

@@ -42,7 +42,7 @@ on:
                 type: string
                 required: false
                 description: |
-                    The name of the prepare artifact to use, defaults to 'webapp'.
+                    The name of the prepare artifact to use, defaults to 'desktop-prepare'.
                     The artifact must contain the following:
                     + webapp.asar - the asar archive of the webapp to embed in the desktop app
                     + electronVersion - the version of electron to use for cache keying
@@ -52,7 +52,7 @@ on:
 
                     The artifact can also contain any additional files which will be applied as overrides to the checkout root before building,
                     for example icons in the `build/` directory to override the app icons.
-                default: "webapp"
+                default: "desktop-prepare"
             test:
                 type: boolean
                 required: false
@@ -123,7 +123,7 @@ jobs:
               id: cache
               uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5
               with:
-                  key: ${{ runner.os }}-${{ inputs.arch }}-${{ hashFiles('hakHash', 'electronVersion') }}
+                  key: ${{ runner.os }}-${{ inputs.arch }}-${{ hashFiles('apps/desktop/hakHash', 'apps/desktop/electronVersion') }}
                   path: |
                       apps/desktop/.hak
 
@@ -160,7 +160,7 @@ jobs:
 
             - name: Install Deps
               working-directory: apps/desktop
-              run: "pnpm install --frozen-lockfile"
+              run: "pnpm install --frozen-lockfile --filter element-desktop"
 
             - name: Insert config snippet
               if: steps.config.outputs.extra_config != ''