Skip to content

Commit 56a3db6

Browse files
author
Michael Andreas Dagitses
committed
Update on "extract aten_core_srcs function into shared build structure"
This captures some overlap between Bazel and fbcode. Differential Revision: [D36780883](https://our.internmc.facebook.com/intern/diff/D36780883/) **NOTE FOR REVIEWERS**: This PR has internal Facebook specific changes or comments, please review them on [Phabricator](https://our.internmc.facebook.com/intern/diff/D36780883/)! [ghstack-poisoned]
2 parents 2c9a88a + 997e8d9 commit 56a3db6

File tree

302 files changed

+9415
-4176
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

302 files changed

+9415
-4176
lines changed

‎.buckconfig.oss‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,11 @@
33

44
[buildfile]
55
name = BUCK.oss
6+
includes = //tools/build_defs/select.bzl
67

78
[repositories]
89
bazel_skylib = third_party/bazel-skylib/
10+
ovr_config = .
911

1012
[download]
1113
in_build = true

‎.circleci/docker/build.sh‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,8 @@ elif [[ "$image" == *-bionic* ]]; then
5454
UBUNTU_VERSION=18.04
5555
elif [[ "$image" == *-focal* ]]; then
5656
UBUNTU_VERSION=20.04
57+
elif [[ "$image" == *-jammy* ]]; then
58+
UBUNTU_VERSION=22.04
5759
elif [[ "$image" == *ubuntu* ]]; then
5860
extract_version_from_image_name ubuntu UBUNTU_VERSION
5961
elif [[ "$image" == *centos* ]]; then
@@ -70,7 +72,8 @@ else
7072
fi
7173

7274
DOCKERFILE="${OS}/Dockerfile"
73-
if [[ "$image" == *cuda* ]]; then
75+
# When using ubuntu - 22.04, start from Ubuntu docker image, instead of nvidia/cuda docker image.
76+
if [[ "$image" == *cuda* && "$UBUNTU_VERSION" != "22.04" ]]; then
7477
DOCKERFILE="${OS}-cuda/Dockerfile"
7578
elif [[ "$image" == *rocm* ]]; then
7679
DOCKERFILE="${OS}-rocm/Dockerfile"
@@ -249,6 +252,15 @@ case "$image" in
249252
VISION=yes
250253
KATEX=yes
251254
;;
255+
pytorch-linux-jammy-cuda11.6-cudnn8-py3.8-clang12)
256+
ANACONDA_PYTHON_VERSION=3.8
257+
CUDA_VERSION=11.6
258+
CUDNN_VERSION=8
259+
CLANG_VERSION=12
260+
PROTOBUF=yes
261+
DB=yes
262+
VISION=yes
263+
;;
252264
*)
253265
# Catch-all for builds that are not hardcoded.
254266
PROTOBUF=yes

‎.circleci/docker/common/install_base.sh‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,20 @@ install_ubuntu() {
1515
elif [[ "$UBUNTU_VERSION" == "20.04"* ]]; then
1616
cmake3="cmake=3.16*"
1717
maybe_libiomp_dev=""
18+
elif [[ "$UBUNTU_VERSION" == "22.04"* ]]; then
19+
cmake3="cmake=3.22*"
20+
maybe_libiomp_dev=""
1821
else
1922
cmake3="cmake=3.5*"
2023
maybe_libiomp_dev="libiomp-dev"
2124
fi
2225

26+
if [[ "$CLANG_VERSION" == 12 ]]; then
27+
libomp_dev="libomp-12-dev"
28+
else
29+
libomp_dev=""
30+
fi
31+
2332
# TODO: Remove this once nvidia package repos are back online
2433
# Comment out nvidia repositories to prevent them from getting apt-get updated, see https://github.com/pytorch/pytorch/issues/74968
2534
# shellcheck disable=SC2046
@@ -51,6 +60,7 @@ install_ubuntu() {
5160
libjpeg-dev \
5261
libasound2-dev \
5362
libsndfile-dev \
63+
${libomp_dev} \
5464
software-properties-common \
5565
wget \
5666
sudo \
@@ -60,6 +70,20 @@ install_ubuntu() {
6070
# see: https://github.com/pytorch/pytorch/issues/65931
6171
apt-get install -y libgnutls30
6272

73+
# cuda-toolkit does not work with gcc-11.2.0 which is default in Ubunutu 22.04
74+
# see: https://github.com/NVlabs/instant-ngp/issues/119
75+
if [[ "$UBUNTU_VERSION" == "22.04"* ]]; then
76+
apt-get install -y g++-10
77+
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 30
78+
update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 30
79+
update-alternatives --install /usr/bin/gcov gcov /usr/bin/gcov-10 30
80+
81+
# https://www.spinics.net/lists/libreoffice/msg07549.html
82+
sudo rm -rf /usr/lib/gcc/x86_64-linux-gnu/11
83+
wget https://github.com/gcc-mirror/gcc/commit/2b2d97fc545635a0f6aa9c9ee3b017394bc494bf.patch -O noexecpt.patch
84+
sudo patch /usr/include/c++/10/bits/range_access.h noexecpt.patch
85+
fi
86+
6387
# Cleanup package manager
6488
apt-get autoclean && apt-get clean
6589
rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*

‎.circleci/docker/common/install_cache.sh‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,9 @@ set -ex
55
install_ubuntu() {
66
echo "Preparing to build sccache from source"
77
apt-get update
8-
apt-get install -y cargo pkg-config libssl-dev
8+
# libssl-dev will not work as it is upgraded to libssl3 in Ubuntu-22.04.
9+
# Instead use lib and headers from OpenSSL1.1 installed in `install_openssl.sh``
10+
apt-get install -y cargo
911
echo "Checking out sccache repo"
1012
git clone https://github.com/pytorch/sccache
1113
cd sccache

‎.circleci/docker/common/install_openssl.sh‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,5 +10,7 @@ cd "${OPENSSL}"
1010
./config --prefix=/opt/openssl -d '-Wl,--enable-new-dtags,-rpath,$(LIBRPATH)'
1111
# NOTE: openssl install errors out when built with the -j option
1212
make -j6; make install_sw
13+
# Link the ssl libraries to the /usr/lib folder.
14+
sudo ln -s /opt/openssl/lib/lib* /usr/lib
1315
cd ..
1416
rm -rf "${OPENSSL}"

‎.circleci/docker/ubuntu-cuda/Dockerfile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ ENV INSTALLED_VISION ${VISION}
6565
ADD ./common/install_openssl.sh install_openssl.sh
6666
ENV OPENSSL_ROOT_DIR /opt/openssl
6767
RUN bash ./install_openssl.sh
68+
ENV OPENSSL_DIR /opt/openssl
6869

6970
# (optional) Install non-default CMake version
7071
ARG CMAKE_VERSION

‎.circleci/docker/ubuntu/Dockerfile‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,15 @@ ARG UBUNTU_VERSION
66

77
ENV DEBIAN_FRONTEND noninteractive
88

9+
ARG CLANG_VERSION
10+
911
# Install common dependencies (so that this step can be cached separately)
1012
ARG EC2
1113
ADD ./common/install_base.sh install_base.sh
1214
RUN bash ./install_base.sh && rm install_base.sh
1315

1416
# Install clang
1517
ARG LLVMDEV
16-
ARG CLANG_VERSION
1718
ADD ./common/install_clang.sh install_clang.sh
1819
RUN bash ./install_clang.sh && rm install_clang.sh
1920

@@ -50,6 +51,13 @@ RUN bash ./install_gcc.sh && rm install_gcc.sh
5051
ADD ./common/install_lcov.sh install_lcov.sh
5152
RUN bash ./install_lcov.sh && rm install_lcov.sh
5253

54+
# Install cuda and cudnn
55+
ARG CUDA_VERSION
56+
RUN wget -q https://raw.githubusercontent.com/pytorch/builder/main/common/install_cuda.sh -O install_cuda.sh
57+
RUN bash ./install_cuda.sh ${CUDA_VERSION} && rm install_cuda.sh
58+
ENV DESIRED_CUDA ${CUDA_VERSION}
59+
ENV PATH /usr/local/nvidia/bin:/usr/local/cuda/bin:$PATH
60+
5361
# (optional) Install protobuf for ONNX
5462
ARG PROTOBUF
5563
ADD ./common/install_protobuf.sh install_protobuf.sh
@@ -111,6 +119,8 @@ RUN rm install_ninja.sh
111119
ADD ./common/install_openssl.sh install_openssl.sh
112120
RUN bash ./install_openssl.sh
113121
ENV OPENSSL_ROOT_DIR /opt/openssl
122+
ENV OPENSSL_DIR /opt/openssl
123+
RUN rm install_openssl.sh
114124

115125
# Install ccache/sccache (do this last, so we get priority in PATH)
116126
ADD ./common/install_cache.sh install_cache.sh
@@ -122,12 +132,22 @@ ADD ./common/install_jni.sh install_jni.sh
122132
ADD ./java/jni.h jni.h
123133
RUN bash ./install_jni.sh && rm install_jni.sh
124134

135+
# Install Open MPI for CUDA
136+
ADD ./common/install_openmpi.sh install_openmpi.sh
137+
RUN if [ -n "${CUDA_VERSION}" ]; then bash install_openmpi.sh; fi
138+
RUN rm install_openmpi.sh
139+
125140
# Include BUILD_ENVIRONMENT environment variable in image
126141
ARG BUILD_ENVIRONMENT
127142
ENV BUILD_ENVIRONMENT ${BUILD_ENVIRONMENT}
128143

129144
# Install LLVM dev version (Defined in the pytorch/builder github repository)
130145
COPY --from=pytorch/llvm:9.0.1 /opt/llvm /opt/llvm
131146

147+
# AWS specific CUDA build guidance
148+
ENV TORCH_CUDA_ARCH_LIST Maxwell
149+
ENV TORCH_NVCC_FLAGS "-Xfatbin -compress-all"
150+
ENV CUDA_PATH /usr/local/cuda
151+
132152
USER jenkins
133153
CMD ["bash"]

‎.github/actions/teardown-rocm/action.yml‎

Lines changed: 0 additions & 25 deletions
This file was deleted.

‎.github/ci_commit_pins/torchdynamo.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
dbb83776c5de185a7a8e2dbafa41d4dc9552d2f9
1+
2d43de8576e47a1d7f287011f14a55e93b568428

‎.github/ci_commit_pins/xla.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
de45c7c503f403be2c85066013b6a860f04f1152
1+
02346af955653a179b896eef5475e569ab8d4229

0 commit comments

Comments
 (0)