Skip to content

Commit 70c9d2e

Browse files
mikaylagawareckimikaylagawarecki
committed
Update on "Allow NJT by default for weights_only torch.load"
[ghstack-poisoned]
2 parents 42cbdd2 + b634c51 commit 70c9d2e

File tree

172 files changed

+7415
-2916
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

172 files changed

+7415
-2916
lines changed

‎.buckconfig.oss‎

Lines changed: 0 additions & 23 deletions
This file was deleted.

‎.ci/docker/ci_commit_pins/executorch.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
8ab3385f4187bad56c66dba29152e34c158f368a
1+
289e84edde3e17c6276a3c57c07daf09a88f8bc0

‎.ci/docker/manywheel/Dockerfile_s390x‎

Lines changed: 75 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,20 @@
1-
FROM --platform=linux/s390x docker.io/ubuntu:24.04 as base
1+
FROM quay.io/pypa/manylinux_2_28_s390x as base
22

33
# Language variables
44
ENV LC_ALL=C.UTF-8
55
ENV LANG=C.UTF-8
66
ENV LANGUAGE=C.UTF-8
77

8+
ARG DEVTOOLSET_VERSION=13
89
# Installed needed OS packages. This is to support all
910
# the binary builds (torch, vision, audio, text, data)
10-
RUN apt update ; apt upgrade -y
11-
RUN apt install -y \
12-
build-essential \
11+
RUN yum -y install epel-release
12+
RUN yum -y update
13+
RUN yum install -y \
14+
sudo \
1315
autoconf \
1416
automake \
17+
bison \
1518
bzip2 \
1619
curl \
1720
diffutils \
@@ -24,34 +27,49 @@ RUN apt install -y \
2427
util-linux \
2528
wget \
2629
which \
27-
xz-utils \
30+
xz \
31+
yasm \
2832
less \
2933
zstd \
34+
libgomp \
35+
gcc-toolset-${DEVTOOLSET_VERSION}-gcc \
36+
gcc-toolset-${DEVTOOLSET_VERSION}-gcc-c++ \
37+
gcc-toolset-${DEVTOOLSET_VERSION}-binutils \
38+
gcc-toolset-${DEVTOOLSET_VERSION}-gcc-gfortran \
3039
cmake \
31-
python3 \
32-
python3-dev \
33-
python3-setuptools \
34-
python3-yaml \
35-
python3-typing-extensions \
36-
libblas-dev \
37-
libopenblas-dev \
38-
liblapack-dev \
39-
libatlas-base-dev
40+
rust \
41+
cargo \
42+
llvm-devel \
43+
libzstd-devel \
44+
python3.12-devel \
45+
python3.12-setuptools \
46+
python3.12-pip \
47+
python3-virtualenv \
48+
python3.12-pyyaml \
49+
python3.12-numpy \
50+
python3.12-wheel \
51+
python3.12-cryptography \
52+
blas-devel \
53+
openblas-devel \
54+
lapack-devel \
55+
atlas-devel \
56+
libjpeg-devel \
57+
libxslt-devel \
58+
libxml2-devel \
59+
openssl-devel \
60+
valgrind
61+
62+
ENV PATH=/opt/rh/gcc-toolset-${DEVTOOLSET_VERSION}/root/usr/bin:$PATH
63+
ENV LD_LIBRARY_PATH=/opt/rh/gcc-toolset-${DEVTOOLSET_VERSION}/root/usr/lib64:/opt/rh/gcc-toolset-${DEVTOOLSET_VERSION}/root/usr/lib:$LD_LIBRARY_PATH
4064

4165
# git236+ would refuse to run git commands in repos owned by other users
4266
# Which causes version check to fail, as pytorch repo is bind-mounted into the image
4367
# Override this behaviour by treating every folder as safe
4468
# For more details see https://github.com/pytorch/pytorch/issues/78659#issuecomment-1144107327
4569
RUN git config --global --add safe.directory "*"
4670

47-
FROM base as openssl
48-
# Install openssl (this must precede `build python` step)
49-
# (In order to have a proper SSL module, Python is compiled
50-
# against a recent openssl [see env vars above], which is linked
51-
# statically. We delete openssl afterwards.)
52-
ADD ./common/install_openssl.sh install_openssl.sh
53-
RUN bash ./install_openssl.sh && rm install_openssl.sh
54-
ENV SSL_CERT_FILE=/opt/_internal/certs.pem
71+
# installed python doesn't have development parts. Rebuild it from scratch
72+
RUN /bin/rm -rf /opt/_internal /opt/python /usr/local/*/*
5573

5674
# EPEL for cmake
5775
FROM base as patchelf
@@ -64,10 +82,43 @@ FROM patchelf as python
6482
# build python
6583
COPY manywheel/build_scripts /build_scripts
6684
ADD ./common/install_cpython.sh /build_scripts/install_cpython.sh
85+
ENV SSL_CERT_FILE=
6786
RUN bash build_scripts/build.sh && rm -r build_scripts
6887

69-
FROM openssl as final
88+
FROM base as final
7089
COPY --from=python /opt/python /opt/python
7190
COPY --from=python /opt/_internal /opt/_internal
72-
COPY --from=python /opt/python/cp39-cp39/bin/auditwheel /usr/local/bin/auditwheel
91+
COPY --from=python /opt/python/cp39-cp39/bin/auditwheel /usr/local/bin/auditwheel
7392
COPY --from=patchelf /usr/local/bin/patchelf /usr/local/bin/patchelf
93+
94+
RUN alternatives --set python /usr/bin/python3.12
95+
RUN alternatives --set python3 /usr/bin/python3.12
96+
97+
RUN pip-3.12 install typing_extensions
98+
99+
ENTRYPOINT []
100+
CMD ["/bin/bash"]
101+
102+
# install test dependencies:
103+
# - grpcio requires system openssl, bundled crypto fails to build
104+
# - ml_dtypes 0.4.0 requires some fixes provided in later commits to build
105+
RUN dnf install -y \
106+
protobuf-devel \
107+
protobuf-c-devel \
108+
protobuf-lite-devel \
109+
wget \
110+
patch
111+
112+
RUN env GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=True pip3 install grpcio==1.65.4
113+
RUN cd ~ && \
114+
git clone https://github.com/jax-ml/ml_dtypes && \
115+
cd ml_dtypes && \
116+
git checkout v0.4.0 && \
117+
git submodule update --init --recursive && \
118+
wget https://github.com/jax-ml/ml_dtypes/commit/b969f76914d6b30676721bc92bf0f6021a0d1321.patch && \
119+
wget https://github.com/jax-ml/ml_dtypes/commit/d4e6d035ecda073eab8bcf60f4eef572ee7087e6.patch && \
120+
patch -p1 < b969f76914d6b30676721bc92bf0f6021a0d1321.patch && \
121+
patch -p1 < d4e6d035ecda073eab8bcf60f4eef572ee7087e6.patch && \
122+
python3 setup.py bdist_wheel && \
123+
pip3 install dist/*.whl && \
124+
rm -rf ml_dtypes

‎.ci/docker/manywheel/build.sh‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ case ${GPU_ARCH_TYPE} in
6161
cpu-s390x)
6262
TARGET=final
6363
DOCKER_TAG=cpu-s390x
64-
GPU_IMAGE=redhat/ubi9
64+
GPU_IMAGE=s390x/almalinux:8
6565
DOCKER_GPU_BUILD_ARG=""
6666
MANY_LINUX_VERSION="s390x"
6767
;;
@@ -125,11 +125,13 @@ fi
125125
(
126126
set -x
127127

128-
# TODO: Remove LimitNOFILE=1048576 patch once https://github.com/pytorch/test-infra/issues/5712
129-
# is resolved. This patch is required in order to fix timing out of Docker build on Amazon Linux 2023.
130-
sudo sed -i s/LimitNOFILE=infinity/LimitNOFILE=1048576/ /usr/lib/systemd/system/docker.service
131-
sudo systemctl daemon-reload
132-
sudo systemctl restart docker
128+
if [ "$(uname -m)" != "s390x" ]; then
129+
# TODO: Remove LimitNOFILE=1048576 patch once https://github.com/pytorch/test-infra/issues/5712
130+
# is resolved. This patch is required in order to fix timing out of Docker build on Amazon Linux 2023.
131+
sudo sed -i s/LimitNOFILE=infinity/LimitNOFILE=1048576/ /usr/lib/systemd/system/docker.service
132+
sudo systemctl daemon-reload
133+
sudo systemctl restart docker
134+
fi
133135

134136
DOCKER_BUILDKIT=1 docker build \
135137
${DOCKER_GPU_BUILD_ARG} \

‎.ci/docker/manywheel/build_scripts/build.sh‎

Lines changed: 24 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -16,38 +16,28 @@ CURL_HASH=cf34fe0b07b800f1c01a499a6e8b2af548f6d0e044dca4a29d88a4bee146d131
1616
AUTOCONF_ROOT=autoconf-2.69
1717
AUTOCONF_HASH=954bd69b391edc12d6a4a51a2dd1476543da5c6bbf05a95b59dc0dd6fd4c2969
1818

19-
# Get build utilities
20-
MY_DIR=$(dirname "${BASH_SOURCE[0]}")
21-
source $MY_DIR/build_utils.sh
19+
# Dependencies for compiling Python that we want to remove from
20+
# the final image after compiling Python
21+
PYTHON_COMPILE_DEPS="zlib-devel bzip2-devel ncurses-devel sqlite-devel readline-devel tk-devel gdbm-devel libpcap-devel xz-devel libffi-devel"
2222

2323
if [ "$(uname -m)" != "s390x" ] ; then
24-
# Dependencies for compiling Python that we want to remove from
25-
# the final image after compiling Python
26-
PYTHON_COMPILE_DEPS="zlib-devel bzip2-devel ncurses-devel sqlite-devel readline-devel tk-devel gdbm-devel db4-devel libpcap-devel xz-devel libffi-devel"
27-
28-
# Libraries that are allowed as part of the manylinux1 profile
29-
MANYLINUX1_DEPS="glibc-devel libstdc++-devel glib2-devel libX11-devel libXext-devel libXrender-devel mesa-libGL-devel libICE-devel libSM-devel ncurses-devel"
30-
31-
# Development tools and libraries
32-
yum -y install bzip2 make git patch unzip bison yasm diffutils \
33-
automake which file cmake28 \
34-
kernel-devel-`uname -r` \
35-
${PYTHON_COMPILE_DEPS}
24+
PYTHON_COMPILE_DEPS="${PYTHON_COMPILE_DEPS} db4-devel"
3625
else
37-
# Dependencies for compiling Python that we want to remove from
38-
# the final image after compiling Python
39-
PYTHON_COMPILE_DEPS="zlib1g-dev libbz2-dev libncurses-dev libsqlite3-dev libdb-dev libpcap-dev liblzma-dev libffi-dev"
40-
41-
# Libraries that are allowed as part of the manylinux1 profile
42-
MANYLINUX1_DEPS="libglib2.0-dev libX11-dev libncurses-dev"
43-
44-
# Development tools and libraries
45-
apt install -y bzip2 make git patch unzip diffutils \
46-
automake which file cmake \
47-
linux-headers-virtual \
48-
${PYTHON_COMPILE_DEPS}
26+
PYTHON_COMPILE_DEPS="${PYTHON_COMPILE_DEPS} libdb-devel"
4927
fi
5028

29+
# Libraries that are allowed as part of the manylinux1 profile
30+
MANYLINUX1_DEPS="glibc-devel libstdc++-devel glib2-devel libX11-devel libXext-devel libXrender-devel mesa-libGL-devel libICE-devel libSM-devel ncurses-devel"
31+
32+
# Get build utilities
33+
MY_DIR=$(dirname "${BASH_SOURCE[0]}")
34+
source $MY_DIR/build_utils.sh
35+
36+
# Development tools and libraries
37+
yum -y install bzip2 make git patch unzip bison yasm diffutils \
38+
automake which file \
39+
${PYTHON_COMPILE_DEPS}
40+
5141
# Install newest autoconf
5242
build_autoconf $AUTOCONF_ROOT $AUTOCONF_HASH
5343
autoconf --version
@@ -92,16 +82,13 @@ ln -s $PY39_BIN/auditwheel /usr/local/bin/auditwheel
9282

9383
# Clean up development headers and other unnecessary stuff for
9484
# final image
95-
if [ "$(uname -m)" != "s390x" ] ; then
96-
yum -y erase wireless-tools gtk2 libX11 hicolor-icon-theme \
97-
avahi freetype bitstream-vera-fonts \
98-
${PYTHON_COMPILE_DEPS} || true > /dev/null 2>&1
99-
yum -y install ${MANYLINUX1_DEPS}
100-
yum -y clean all > /dev/null 2>&1
101-
yum list installed
102-
else
103-
apt purge -y ${PYTHON_COMPILE_DEPS} || true > /dev/null 2>&1
104-
fi
85+
yum -y erase wireless-tools gtk2 libX11 hicolor-icon-theme \
86+
avahi freetype bitstream-vera-fonts \
87+
${PYTHON_COMPILE_DEPS} || true > /dev/null 2>&1
88+
yum -y install ${MANYLINUX1_DEPS}
89+
yum -y clean all > /dev/null 2>&1
90+
yum list installed
91+
10592
# we don't need libpython*.a, and they're many megabytes
10693
find /opt/_internal -name '*.a' -print0 | xargs -0 rm -f
10794
# Strip what we can -- and ignore errors, because this just attempts to strip

‎.ci/pytorch/test.sh‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1354,10 +1354,11 @@ test_executorch() {
13541354
export EXECUTORCH_BUILD_PYBIND=ON
13551355
export CMAKE_ARGS="-DEXECUTORCH_BUILD_XNNPACK=ON -DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON"
13561356

1357+
# For llama3
1358+
bash examples/models/llama3_2_vision/install_requirements.sh
13571359
# NB: We need to rebuild ExecuTorch runner here because it depends on PyTorch
13581360
# from the PR
1359-
# shellcheck disable=SC1091
1360-
source .ci/scripts/setup-linux.sh cmake
1361+
bash .ci/scripts/setup-linux.sh cmake
13611362

13621363
echo "Run ExecuTorch unit tests"
13631364
pytest -v -n auto

‎.clang-format‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -101,16 +101,9 @@ SpacesInParentheses: false
101101
SpacesInSquareBrackets: false
102102
Standard: c++17
103103
StatementMacros:
104-
- C10_DEFINE_bool
105-
- C10_DEFINE_int
106-
- C10_DEFINE_int32
107-
- C10_DEFINE_int64
108-
- C10_DEFINE_string
109104
- PyObject_HEAD
110105
- PyObject_VAR_HEAD
111106
- PyException_HEAD
112-
- DEFINE_BINARY
113-
114107
TabWidth: 8
115108
UseTab: Never
116109
---

‎.github/scripts/generate_binary_build_matrix.py‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -371,15 +371,23 @@ def generate_wheels_matrix(
371371
# TODO: Enable python 3.13 on rocm, aarch64, windows
372372
if (
373373
gpu_arch_type == "rocm"
374-
or os not in ["linux", "linux-s390x", "linux-aarch64", "macos-arm64"]
374+
or os
375+
not in [
376+
"linux",
377+
"linux-s390x",
378+
"linux-aarch64",
379+
"macos-arm64",
380+
"windows",
381+
]
375382
) and python_version in ["3.13", "3.13t"]:
376383
continue
377384

378-
# TODO: Enable python 3.13t on xpu and cpu-s390x or MacOS
385+
# TODO: Enable python 3.13t on xpu and cpu-s390x or MacOS or Windows
379386
if (
380387
gpu_arch_type in ["xpu", "cpu-s390x"]
381388
or os == "macos-arm64"
382389
or os == "linux-aarch64"
390+
or os == "windows"
383391
) and python_version == "3.13t":
384392
continue
385393

‎.github/scripts/s390x-ci/self-hosted-builder/actions-runner.Dockerfile‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,3 +69,6 @@ RUN curl -L https://github.com/actions/runner/releases/download/v2.317.0/actions
6969

7070
ENTRYPOINT ["/usr/bin/entrypoint"]
7171
CMD ["/usr/bin/actions-runner"]
72+
73+
# podman requires additional settings to use docker.io by default
74+
RUN mkdir -pv .config/containers ; echo 'unqualified-search-registries = ["docker.io"]' > .config/containers/registries.conf

‎.github/scripts/s390x-ci/self-hosted-builder/[email protected]‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,10 @@ Type=simple
99
Restart=always
1010
ExecStartPre=-/usr/bin/docker rm --force actions-runner.%i
1111
ExecStartPre=-/usr/local/bin/gh_token_generator.sh /etc/actions-runner/%i/appid.env /etc/actions-runner/%i/installid.env /etc/actions-runner/%i/key_private.pem /etc/actions-runner/%i/ghtoken.env
12+
ExecStartPre=-/usr/local/bin/gh_cat_token.sh /etc/actions-runner/%i/ghtoken.env /etc/actions-runner/%i/ghtoken.socket
1213
ExecStart=/usr/bin/docker run \
1314
--env-file=/etc/actions-runner/%i/env \
14-
--env-file=/etc/actions-runner/%i/ghtoken.env \
15+
--volume /etc/actions-runner/%i/ghtoken.socket:/run/runner_secret \
1516
--init \
1617
--interactive \
1718
--name=actions-runner.%i \
@@ -21,6 +22,7 @@ ExecStart=/usr/bin/docker run \
2122
ExecStop=/bin/sh -c "docker exec actions-runner.%i kill -INT -- -1"
2223
ExecStop=/bin/sh -c "docker wait actions-runner.%i"
2324
ExecStop=/bin/sh -c "docker rm actions-runner.%i"
25+
ExecStop=/usr/bin/env rm -f /etc/actions-runner/%i/ghtoken.env /etc/actions-runner/%i/ghtoken.socket
2426

2527
[Install]
2628
WantedBy=multi-user.target

0 commit comments

Comments
 (0)