diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 0b490ae416..2971032018 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -15,6 +15,10 @@ on:
 jobs:
   test:
     runs-on: ubuntu-20.04
+    outputs:
+      arm-boards: ${{ steps.set-matrix.outputs.arm-boards }}
+      riscv-boards: ${{ steps.set-matrix.outputs.riscv-boards }}
+      xtensa-boards: ${{ steps.set-matrix.outputs.xtensa-boards }}
     steps:
     - name: Dump GitHub context
       env:
@@ -43,9 +47,6 @@ jobs:
       run: |
         gcc --version
         python3 --version
-    - name: New boards check
-      run: python3 -u ci_new_boards_check.py
-      working-directory: tools
     - name: Duplicate USB VID/PID Check
       run: python3 -u -m tools.ci_check_duplicate_usb_vid_pid
     - name: Build and Validate Stubs
@@ -139,6 +140,29 @@ jobs:
         # setup.py sdist was run by 'make stubs'
         [ -z "$TWINE_USERNAME" ] || echo "Uploading dev release to PyPi"
         [ -z "$TWINE_USERNAME" ] || twine upload circuitpython-stubs/dist/*
+    - uses: dorny/paths-filter@v2
+      id: filter
+      with:
+        # Enable listing of files matching each filter.
+        # Paths to files will be available in `${FILTER_NAME}_files` output variable.
+        # Paths will be formatted as JSON array
+        list-files: json
+
+        # Compare against this branch. (Ignored for PRs.)
+        base: ${{ github.ref }}
+
+        # In this example all changed files are passed to the following action to do
+        # some custom processing.
+        filters: |
+          changed:
+            - '**'
+    - name: "Set boards to build"
+      id: set-matrix
+      working-directory: tools
+      env:
+        CHANGED_FILES: ${{ steps.filter.outputs.changed_files }}
+      run: |
+        python3 -u ci_changed_board_list.py
 
   mpy-cross-mac:
     runs-on: macos-10.15
@@ -203,205 +227,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        board:
-        - "8086_commander"
-        - "ADM_B_NRF52840_1"
-        - "TG-Watch"
-        - "adafruit_feather_rp2040"
-        - "adafruit_itsybitsy_rp2040"
-        - "adafruit_led_glasses_nrf52840"
-        - "adafruit_macropad_rp2040"
-        - "adafruit_neokey_trinkey_m0"
-        - "adafruit_proxlight_trinkey_m0"
-        - "adafruit_qt2040_trinkey"
-        - "adafruit_qtpy_rp2040"
-        - "adafruit_rotary_trinkey_m0"
-        - "adafruit_slide_trinkey_m0"
-        - "aloriumtech_evo_m51"
-        - "aramcon2_badge"
-        - "aramcon_badge_2019"
-        - "arduino_mkr1300"
-        - "arduino_mkrzero"
-        - "arduino_nano_33_ble"
-        - "arduino_nano_33_iot"
-        - "arduino_nano_rp2040_connect"
-        - "arduino_zero"
-        - "bast_pro_mini_m0"
-        - "bastble"
-        - "bdmicro_vina_d21"
-        - "bdmicro_vina_d51"
-        - "bdmicro_vina_d51_pcb7"
-        - "bless_dev_board_multi_sensor"
-        - "blm_badge"
-        - "bluemicro840"
-        - "capablerobot_usbhub"
-        - "catwan_usbstick"
-        - "circuitbrains_basic_m0"
-        - "circuitbrains_deluxe_m4"
-        - "circuitplayground_bluefruit"
-        - "circuitplayground_express"
-        - "circuitplayground_express_crickit"
-        - "circuitplayground_express_displayio"
-        - "clue_nrf52840_express"
-        - "cp32-m4"
-        - "cp_sapling_m0"
-        - "cp_sapling_m0_revb"
-        - "cp_sapling_m0_spiflash"
-        - "cytron_maker_pi_rp2040"
-        - "datalore_ip_m4"
-        - "datum_distance"
-        - "datum_imu"
-        - "datum_light"
-        - "datum_weather"
-        - "dynalora_usb"
-        - "dynossat_edu_eps"
-        - "dynossat_edu_obc"
-        - "electronut_labs_blip"
-        - "electronut_labs_papyr"
-        - "escornabot_makech"
-        - "espruino_pico"
-        - "espruino_wifi"
-        - "feather_bluefruit_sense"
-        - "feather_m0_adalogger"
-        - "feather_m0_basic"
-        - "feather_m0_express"
-        - "feather_m0_express_crickit"
-        - "feather_m0_rfm69"
-        - "feather_m0_rfm9x"
-        - "feather_m0_supersized"
-        - "feather_m4_can"
-        - "feather_m4_express"
-        - "feather_m7_1011"
-        - "feather_mimxrt1011"
-        - "feather_mimxrt1062"
-        - "feather_nrf52840_express"
-        - "feather_stm32f405_express"
-        - "fluff_m0"
-        - "gemma_m0"
-        - "grandcentral_m4_express"
-        - "hallowing_m0_express"
-        - "hallowing_m4_express"
-        - "hiibot_bluefi"
-        - "huntercat_nfc"
-        - "ikigaisense_vita"
-        - "imxrt1010_evk"
-        - "imxrt1020_evk"
-        - "imxrt1060_evk"
-        - "itsybitsy_m0_express"
-        - "itsybitsy_m4_express"
-        - "itsybitsy_nrf52840_express"
-        - "jpconstantineau_encoderpad_rp2040"
-        - "kicksat-sprite"
-        - "loc_ber_m4_base_board"
-        - "makerdiary_m60_keyboard"
-        - "makerdiary_nrf52840_m2_devkit"
-        - "makerdiary_nrf52840_mdk"
-        - "makerdiary_nrf52840_mdk_usb_dongle"
-        - "matrixportal_m4"
-        - "meowbit_v121"
-        - "meowmeow"
-        - "metro_m0_express"
-        - "metro_m4_airlift_lite"
-        - "metro_m4_express"
-        - "metro_m7_1011"
-        - "metro_nrf52840_express"
-        - "microbit_v2"
-        - "mini_sam_m4"
-        - "monster_m4sk"
-        - "ndgarage_ndbit6"
-        - "ndgarage_ndbit6_v2"
-        - "neopixel_trinkey_m0"
-        - "nfc_copy_cat"
-        - "nice_nano"
-        - "nucleo_f746zg"
-        - "nucleo_f767zi"
-        - "nucleo_h743zi_2"
-        - "ohs2020_badge"
-        - "openbook_m4"
-        - "openmv_h7"
-        - "particle_argon"
-        - "particle_boron"
-        - "particle_xenon"
-        - "pca10056"
-        - "pca10059"
-        - "pca10100"
-        - "pewpew10"
-        - "pewpew_m4"
-        - "picoplanet"
-        - "pimoroni_interstate75"
-        - "pimoroni_keybow2040"
-        - "pimoroni_pga2040"
-        - "pimoroni_picolipo_16mb"
-        - "pimoroni_picolipo_4mb"
-        - "pimoroni_picosystem"
-        - "pimoroni_plasma2040"
-        - "pimoroni_tiny2040"
-        - "pitaya_go"
-        - "pyb_nano_v2"
-        - "pybadge"
-        - "pyboard_v11"
-        - "pycubed"
-        - "pycubed_mram"
-        - "pygamer"
-        - "pyportal"
-        - "pyportal_titano"
-        - "pyruler"
-        - "qtpy_m0"
-        - "qtpy_m0_haxpress"
-        - "raspberry_pi_pico"
-        - "raytac_mdbt50q-db-40"
-        - "raytac_mdbt50q-rx"
-        - "robohatmm1_m4"
-        - "sam32"
-        - "same54_xplained"
-        - "seeeduino_wio_terminal"
-        - "seeeduino_xiao"
-        - "sensebox_mcu"
-        - "serpente"
-        - "shirtty"
-        - "silicognition-m4-shim"
-        - "simmel"
-        - "snekboard"
-        - "sparkfun_lumidrive"
-        - "sparkfun_micromod_rp2040"
-        - "sparkfun_nrf52840_micromod"
-        - "sparkfun_nrf52840_mini"
-        - "sparkfun_pro_micro_rp2040"
-        - "sparkfun_qwiic_micro_no_flash"
-        - "sparkfun_qwiic_micro_with_flash"
-        - "sparkfun_redboard_turbo"
-        - "sparkfun_samd21_dev"
-        - "sparkfun_samd21_mini"
-        - "sparkfun_samd51_micromod"
-        - "sparkfun_samd51_thing_plus"
-        - "sparkfun_stm32f405_micromod"
-        - "sparkfun_thing_plus_rp2040"
-        - "spresense"
-        - "stackrduino_m0_pro"
-        - "stm32f411ce_blackpill"
-        - "stm32f411ce_blackpill_with_flash"
-        - "stm32f411ve_discovery"
-        - "stm32f412zg_discovery"
-        - "stm32f4_discovery"
-        - "stm32f746g_discovery"
-        - "stringcar_m0_express"
-        - "teensy40"
-        - "teensy41"
-        - "teknikio_bluebird"
-        - "thunderpack_v11"
-        - "thunderpack_v12"
-        - "tinkeringtech_scoutmakes_azul"
-        - "trellis_m4_express"
-        - "trinket_m0"
-        - "trinket_m0_haxpress"
-        - "uartlogger2"
-        - "uchip"
-        - "ugame10"
-        - "warmbit_bluepixel"
-        - "winterbloom_big_honking_button"
-        - "winterbloom_sol"
-        - "xinabox_cc03"
-        - "xinabox_cs11"
+        board: ${{ fromJSON(needs.test.outputs.arm-boards) }}
 
     steps:
     - name: Set up Python 3.8
@@ -451,8 +277,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        board:
-        - "fomu"
+        board: ${{ fromJSON(needs.test.outputs.riscv-boards) }}
 
     steps:
     - name: Set up Python 3.8
@@ -501,41 +326,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        board:
-        - "adafruit_feather_esp32s2_nopsram"
-        - "adafruit_feather_esp32s2_tftback_nopsram"
-        - "adafruit_funhouse"
-        - "adafruit_magtag_2.9_grayscale"
-        - "adafruit_metro_esp32s2"
-        - "ai_thinker_esp_12k_nodemcu"
-        - "artisense_rd00"
-        - "atmegazero_esp32s2"
-        - "crumpspace_crumps2"
-        - "electroniccats_bastwifi"
-        - "espressif_hmi_devkit_1"
-        - "espressif_kaluga_1"
-        - "espressif_kaluga_1.3"
-        - "espressif_saola_1_wroom"
-        - "espressif_saola_1_wrover"
-        - "franzininho_wifi_wroom"
-        - "franzininho_wifi_wrover"
-        - "gravitech_cucumber_m"
-        - "gravitech_cucumber_ms"
-        - "gravitech_cucumber_r"
-        - "gravitech_cucumber_rs"
-        - "lilygo_ttgo_t8_s2_st7789"
-        - "lolin_s2_mini"
-        - "microdev_micro_s2"
-        - "morpheans_morphesp-240"
-        - "muselab_nanoesp32_s2_wroom"
-        - "muselab_nanoesp32_s2_wrover"
-        - "odt_pixelwing_esp32_s2"
-        - "targett_module_clip_wroom"
-        - "targett_module_clip_wrover"
-        - "unexpectedmaker_feathers2"
-        - "unexpectedmaker_feathers2_neo"
-        - "unexpectedmaker_feathers2_prerelease"
-        - "unexpectedmaker_tinys2"
+        board: ${{ fromJSON(needs.test.outputs.xtensa-boards) }}
 
     steps:
     - name: Set up Python 3.8
diff --git a/tools/ci_changed_board_list.py b/tools/ci_changed_board_list.py
new file mode 100644
index 0000000000..221b1ecaa3
--- /dev/null
+++ b/tools/ci_changed_board_list.py
@@ -0,0 +1,94 @@
+#! /usr/bin/env python3
+
+"""This script is used in GitHub Actions to determine what boards are built
+  based on what files were changed. The base commit varies depending on the
+  event that triggered run. Pull request runs will compare to the base branch
+  while pushes will compare to the current ref. We override this for the
+  adafruit/circuitpython repo so we build all boards for pushes.
+  """
+
+# SPDX-FileCopyrightText: 2021 Scott Shawcroft
+#
+# SPDX-License-Identifier: MIT
+
+import sys
+import os
+import json
+import yaml
+import re
+
+import build_board_info
+
+PORT_TO_ARCH = {
+    "atmel-samd": "arm",
+    "cxd56": "arm",
+    "esp32s2": "xtensa",
+    "litex": "riscv",
+    "mimxrt10xx": "arm",
+    "nrf": "arm",
+    "raspberrypi": "arm",
+    "stm": "arm",
+}
+
+changed_files = json.loads(os.environ["CHANGED_FILES"])
+print("changed_files")
+print(changed_files)
+
+# Get boards in json format
+boards_info_json = build_board_info.get_board_mapping()
+all_board_ids = set()
+port_to_boards = {}
+board_to_port = {}
+for board_id in boards_info_json:
+    info = boards_info_json[board_id]
+    if info.get("alias", False):
+        continue
+    all_board_ids.add(board_id)
+    port = info["port"]
+    if port not in port_to_boards:
+        port_to_boards[port] = set()
+    port_to_boards[port].add(board_id)
+    board_to_port[board_id] = port
+
+# Build all boards if this is a push to an adafruit/circuitpython branch because we save the artifacts.
+boards_to_build = all_board_ids
+if not changed_files or (
+    os.environ.get("GITHUB_EVENT_NAME", "") == "push"
+    and os.environ.get("GITHUB_REPOSITORY", "") == "adafruit/circuitpython"
+):
+    print("Building all boards because of adafruit/circuitpython branch")
+else:
+    print("Adding boards to build based on changed files")
+    boards_to_build = set()
+    board_pattern = re.compile(r"ports/\w+/boards/(\w+)/")
+    port_pattern = re.compile(r"ports/(\w+)/")
+    for p in changed_files:
+        # See if it is board specific
+        board_matches = board_pattern.search(p)
+        if board_matches:
+            board = board_matches.group(1)
+            boards_to_build.add(board)
+            continue
+
+        # See if it is port specific
+        port_matches = port_pattern.search(p)
+        if port_matches:
+            port = port_matches.group(1)
+            boards_to_build.update(port_to_boards[port])
+            continue
+
+        # Otherwise build it all
+        boards_to_build = all_board_ids
+        break
+
+# Split boards by architecture.
+print("Building boards:")
+arch_to_boards = {"arm": [], "riscv": [], "xtensa": []}
+for board in boards_to_build:
+    print(" ", board)
+    arch = PORT_TO_ARCH[board_to_port[board]]
+    arch_to_boards[arch].append(board)
+
+# Set the step outputs for each architecture
+for arch in arch_to_boards:
+    print("::set-output name=" + arch + "-boards::" + json.dumps(sorted(arch_to_boards[arch])))
diff --git a/tools/ci_new_boards_check.py b/tools/ci_new_boards_check.py
deleted file mode 100644
index 3b41c8d67b..0000000000
--- a/tools/ci_new_boards_check.py
+++ /dev/null
@@ -1,57 +0,0 @@
-#! /usr/bin/env python3
-
-# SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors)
-#
-# SPDX-License-Identifier: MIT
-
-import sys
-import os
-import json
-import yaml
-
-import build_board_info
-
-workflow_file = ".github/workflows/build.yml"
-
-# Get boards in json format
-boards_info_json = build_board_info.get_board_mapping()
-
-# Get all the boards out of the json format
-info_boards = [
-    board for board in boards_info_json.keys() if not boards_info_json[board].get("alias", False)
-]
-
-# We need to know the path of the workflow file
-base_path = os.path.dirname(__file__)
-yml_path = os.path.abspath(os.path.join(base_path, "..", workflow_file))
-
-# Loading board list based on build jobs in the workflow file.
-ci_boards = []
-with open(yml_path, "r") as f:
-    workflow = yaml.safe_load(f)
-
-ok = True
-for job in workflow["jobs"]:
-    if not job.startswith("build"):
-        continue
-    job_boards = workflow["jobs"][job]["strategy"]["matrix"]["board"]
-    if job_boards != sorted(job_boards):
-        print('Boards for job "{}" not sorted. Must be:'.format(job))
-        print('        - "' + '"\n        - "'.join(sorted(job_boards)) + '"')
-        ok = False
-    ci_boards.extend(job_boards)
-
-# All the travis_boards elements must be on info_boards
-info_boards.sort()
-ci_boards.sort()
-
-missing_boards = set(info_boards) - set(ci_boards)
-
-if missing_boards:
-    ok = False
-    print("Boards missing in {}:".format(workflow_file))
-    for board in missing_boards:
-        print(board)
-
-if not ok:
-    sys.exit(1)