Skip to content

Commit 1a3c07d

Browse files
DenisVieriu97Siddharth Kotapati
authored andcommitted
Add CI scripts for MPS delegate (pytorch#8)
* Add CI scripts * Fix lint * Fix lint
1 parent 4e9eadc commit 1a3c07d

4 files changed

Lines changed: 269 additions & 1 deletion

File tree

‎.github/workflows/pull-mps.yml‎

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
name: pull_mps
2+
3+
on:
4+
pull_request:
5+
push:
6+
branches:
7+
- main
8+
workflow_dispatch:
9+
10+
concurrency:
11+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}-${{ github.event_name == 'schedule' }}
12+
cancel-in-progress: true
13+
14+
jobs:
15+
gather-models-mps:
16+
runs-on: macos-executorch
17+
outputs:
18+
models: ${{ steps.gather-models-mps.outputs.models }}
19+
mps_models: ${{ steps.gather-models-mps.outputs.mps_models }}
20+
steps:
21+
- uses: actions/checkout@v4
22+
with:
23+
submodules: 'recursive'
24+
- name: Extract the list of models to test
25+
id: gather-models-mps
26+
run: |
27+
set -eux
28+
29+
# Conda setup
30+
if conda info --envs | grep -w -q "env-${{ runner.name }}"; then echo "env-${{ runner.name }} already exists"; else conda create -y -n "env-${{ runner.name }}" python=3.10; fi
31+
CONDA_ENV="~/miniconda3/envs/env-${{ runner.name }}"
32+
if [[ -n "$CONDA_ENV" ]]; then
33+
# Use binaries under conda environment
34+
export PATH="$CONDA_ENV/bin":$PATH
35+
fi
36+
37+
conda init zsh
38+
conda init
39+
source ~/miniconda3/bin/activate
40+
conda deactivate
41+
conda activate "$CONDA_ENV"
42+
conda install -y cmake
43+
44+
source .ci/scripts/utils.sh
45+
# This is a simple Python script but as it tries to import executorch.examples.models,
46+
# it requires a whole bunch of ExecuTorch dependencies on the Docker image
47+
PYTHON_EXECUTABLE=python3 bash install_requirements.sh
48+
# Discover all the tests from test_mps.py
49+
python -m unittest backends.apple.mps.test.test_mps -k test_mps_backend_mm -q &>/dev/null
50+
51+
PYTHONPATH="${PWD}" python -m backends.apple.mps.ci.scripts.gather_test_models_mps
52+
# test-mps-models:
53+
# name: test-mps-models
54+
# runs-on: macos-executorch
55+
# needs: gather-models-mps
56+
# strategy:
57+
# matrix: ${{ fromJSON(needs.gather-models-mps.outputs.models) }}
58+
# fail-fast: false
59+
# steps:
60+
# - uses: actions/checkout@v4
61+
# with:
62+
# submodules: 'recursive'
63+
# - name: Run test ${{ matrix.test }}
64+
# if: always()
65+
# run: |
66+
# set -eux
67+
68+
# # Conda setup
69+
# if conda info --envs | grep -w -q "env-${{ runner.name }}"; then echo "env-${{ runner.name }} already exists"; else conda create -y -n "env-${{ runner.name }}" python=3.10; fi
70+
# CONDA_ENV="~/miniconda3/envs/env-${{ runner.name }}"
71+
# PYTHON_PATH="$CONDA_ENV/bin/python3"
72+
# if [[ -n "$CONDA_ENV" ]]; then
73+
# # Use binaries under conda environment
74+
# export PATH="$CONDA_ENV/bin":$PATH
75+
# fi
76+
77+
# conda init zsh
78+
# conda init
79+
# source ~/miniconda3/bin/activate
80+
# conda deactivate
81+
# conda activate "$CONDA_ENV"
82+
83+
# # Print out some information about the test environment
84+
# which conda
85+
# conda --version
86+
# which python3
87+
# python3 --version
88+
# which python
89+
# python --version
90+
91+
# source .ci/scripts/utils.sh
92+
# conda install -y cmake
93+
94+
# WORKSPACE=$(pwd)
95+
# pushd "${WORKSPACE}"
96+
# MODEL_NAME=${{ matrix.model }}
97+
# BUILD_TOOL=${{ matrix.build-tool }}
98+
99+
# # Setup MacOS dependencies as there is no Docker support on MacOS atm
100+
# PYTHON_EXECUTABLE=python3 bash install_requirements.sh --pybind mps
101+
# # Build and test ExecuTorch
102+
# PYTHON_EXECUTABLE=python3 bash backends/apple/mps/ci/scripts/test-mps.sh "${MODEL_NAME}" "${BUILD_TOOL}" false
103+
# popd
104+
test-mps:
105+
name: test-mps
106+
runs-on: macos-executorch
107+
needs: gather-models-mps
108+
strategy:
109+
matrix: ${{ fromJSON(needs.gather-models-mps.outputs.mps_models) }}
110+
fail-fast: false
111+
steps:
112+
- uses: actions/checkout@v4
113+
with:
114+
submodules: 'recursive'
115+
- name: Run test ${{ matrix.test }}
116+
if: always()
117+
run: |
118+
set -eux
119+
120+
# Conda setup
121+
if conda info --envs | grep -w -q "env-${{ runner.name }}"; then echo "env-${{ runner.name }} already exists"; else conda create -y -n "env-${{ runner.name }}" python=3.10; fi
122+
CONDA_ENV="~/miniconda3/envs/env-${{ runner.name }}"
123+
PYTHON_PATH="$CONDA_ENV/bin/python3"
124+
if [[ -n "$CONDA_ENV" ]]; then
125+
# Use binaries under conda environment
126+
export PATH="$CONDA_ENV/bin":$PATH
127+
fi
128+
129+
conda init zsh
130+
conda init
131+
source ~/miniconda3/bin/activate
132+
conda deactivate
133+
conda activate "$CONDA_ENV"
134+
135+
# Print out some information about the test environment
136+
which conda
137+
conda --version
138+
which python3
139+
python3 --version
140+
which python
141+
python --version
142+
143+
source .ci/scripts/utils.sh
144+
conda install -y cmake
145+
PYTHON_EXECUTABLE=python3 bash install_requirements.sh --pybind mps
146+
147+
MODEL_NAME=${{ matrix.model }}
148+
BUILD_TOOL=${{ matrix.build-tool }}
149+
# Setup MacOS dependencies as there is no Docker support on MacOS atm
150+
PYTHON_EXECUTABLE=python3 bash backends/apple/mps/install_requirements.sh
151+
# Build and test ExecuTorch
152+
python3 -m unittest "${MODEL_NAME}"
Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
#
2+
# Copyright (c) 2023 Apple Inc. All rights reserved.
3+
# Provided subject to the LICENSE file in the top level directory.
4+
#
5+
6+
import json
7+
import os
8+
import unittest
9+
from typing import Any
10+
11+
from examples.models import MODEL_NAME_TO_MODEL
12+
13+
MPS_TEST_SUITE_PATH = "backends/apple/mps/test"
14+
MPS_SUITE = unittest.defaultTestLoader.discover(MPS_TEST_SUITE_PATH)
15+
16+
BUILD_TOOLS = {
17+
"cmake": {"macos-14"},
18+
}
19+
DEFAULT_RUNNERS = {
20+
"macos-14": "macos-executorch",
21+
}
22+
23+
24+
def parse_args() -> Any:
25+
from argparse import ArgumentParser
26+
27+
parser = ArgumentParser("Gather all models to test on CI for macOS MPS delegate")
28+
parser.add_argument(
29+
"--target-os",
30+
type=str,
31+
choices=["macos-14"],
32+
default="macos-14",
33+
help="the target OS",
34+
)
35+
return parser.parse_args()
36+
37+
38+
def set_output(name: str, val: Any) -> None:
39+
"""
40+
Set the GitHb output so that it can be accessed by other jobs
41+
"""
42+
print(f"Setting {val} to GitHub output")
43+
44+
if os.getenv("GITHUB_OUTPUT"):
45+
with open(str(os.getenv("GITHUB_OUTPUT")), "a") as env:
46+
print(f"{name}={val}", file=env)
47+
else:
48+
print(f"::set-output name={name}::{val}")
49+
50+
51+
def gather_mps_test_list(suite, mps_test_list):
52+
if hasattr(suite, "__iter__"):
53+
for x in suite:
54+
gather_mps_test_list(x, mps_test_list)
55+
else:
56+
mps_test_list.append(suite)
57+
58+
59+
def gather_mps_tests(suite):
60+
mps_test_list = []
61+
gather_mps_test_list(suite, mps_test_list)
62+
return mps_test_list
63+
64+
65+
def export_models_for_ci() -> None:
66+
"""
67+
This gathers all the example models that we want to test on GitHub OSS CI
68+
"""
69+
args = parse_args()
70+
target_os = args.target_os
71+
72+
# This is the JSON syntax for configuration matrix used by GitHub
73+
# https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs
74+
models = {"include": []}
75+
for name in MODEL_NAME_TO_MODEL.keys():
76+
for build_tool in BUILD_TOOLS.keys():
77+
if target_os not in BUILD_TOOLS[build_tool]:
78+
continue
79+
80+
record = {
81+
"build-tool": build_tool,
82+
"model": name,
83+
"runner": DEFAULT_RUNNERS.get(target_os),
84+
}
85+
86+
models["include"].append(record)
87+
88+
set_output("models", json.dumps(models))
89+
90+
# Gather the list of all mps tests
91+
mps_models = {"include": []}
92+
mps_test_list = gather_mps_tests(MPS_SUITE)
93+
for testcase in mps_test_list:
94+
if "test_mps" not in str(testcase.__class__):
95+
continue
96+
for build_tool in BUILD_TOOLS.keys():
97+
if target_os not in BUILD_TOOLS[build_tool]:
98+
continue
99+
100+
start_path = ".".join(
101+
[MPS_TEST_SUITE_PATH.replace("/", "."), testcase.__module__]
102+
)
103+
cmd = ".".join([start_path, testcase.__class__.__name__])
104+
record = {
105+
"build-tool": build_tool,
106+
"model": cmd,
107+
"runner": DEFAULT_RUNNERS.get(target_os),
108+
}
109+
110+
if record not in mps_models["include"]:
111+
mps_models["include"].append(record)
112+
113+
set_output("mps_models", json.dumps(mps_models))
114+
115+
116+
if __name__ == "__main__":
117+
export_models_for_ci()

‎backends/apple/mps/test/test_mps_utils.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@
2222
from executorch.exir.backend.backend_details import CompileSpec
2323
from executorch.exir.capture._config import ExecutorchBackendConfig
2424
from executorch.exir.tracer import Value
25-
from executorch.extension.pytree import tree_flatten
2625
from executorch.sdk import BundledProgram
2726
from executorch.sdk.bundled_program.config import MethodTestCase, MethodTestSuite
2827
from executorch.sdk.bundled_program.serialize import (

0 commit comments

Comments
 (0)