setup.py
2
# Licensed to the Apache Software Foundation (ASF) under one or more
3
# contributor license agreements. See the NOTICE file distributed with
4
# this work for additional information regarding copyright ownership.
5
# The ASF licenses this file to You under the Apache License, Version 2.0
6
# (the "License"); you may not use this file except in compliance with
7
# the License. You may obtain a copy of the License at
9
# http://www.apache.org/licenses/LICENSE-2.0
10
#
11
# Unless required by applicable law or agreed to in writing, software
12
# distributed under the License is distributed on an "AS IS" BASIS,
13
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
# See the License for the specific language governing permissions and
15
# limitations under the License.
18
"""Apache Beam SDK for Python setup file."""
19
import glob
20
import logging
21
import os
22
import shutil
23
import subprocess
25
import warnings
26
# Pylint and isort disagree here.
27
# pylint: disable=ungrouped-imports
28
from importlib.metadata import PackageNotFoundError
29
from importlib.metadata import distribution
30
from pathlib import Path
32
# pylint: disable=ungrouped-imports
33
import setuptools
34
from packaging.version import parse
35
from setuptools import Command
37
# pylint: disable=wrong-import-order
38
# It is recommended to import setuptools prior to importing distutils to avoid
39
# using legacy behavior from distutils.
40
# https://setuptools.readthedocs.io/en/latest/history.html#v48-0-0
41
from distutils.errors import DistutilsError # isort:skip
44
def to_filename(name: str) -> str:
45
return name.replace('-', '_')
46
47
48
def normalize_path(filename):
49
return os.path.normcase(os.path.realpath(os.path.normpath(filename)))
50
51
52
class mypy(Command):
53
user_options = []
54
55
def initialize_options(self):
56
"""Abstract method that is required to be overwritten"""
57
58
def finalize_options(self):
59
"""Abstract method that is required to be overwritten"""
60
61
def get_project_path(self):
62
self.run_command('egg_info')
63
64
# Build extensions in-place
65
self.reinitialize_command('build_ext', inplace=1)
66
self.run_command('build_ext')
67
68
ei_cmd = self.get_finalized_command("egg_info")
69
70
project_path = normalize_path(ei_cmd.egg_base)
71
return os.path.join(project_path, to_filename(ei_cmd.egg_name))
72
73
def run(self):
74
args = ['mypy', self.get_project_path()]
75
result = subprocess.call(args)
76
if result != 0:
77
raise DistutilsError("mypy exited with status %d" % result)
78
79
80
def get_version():
81
global_names = {}
82
exec( # pylint: disable=exec-used
83
open(os.path.join(
84
os.path.dirname(os.path.abspath(__file__)),
85
'apache_beam/version.py')
86
).read(),
87
global_names
88
)
89
return global_names['__version__']
90
92
PACKAGE_NAME = 'apache-beam'
93
PACKAGE_VERSION = get_version()
94
PACKAGE_DESCRIPTION = 'Apache Beam SDK for Python'
95
PACKAGE_URL = 'https://beam.apache.org'
96
PACKAGE_DOWNLOAD_URL = 'https://pypi.python.org/pypi/apache-beam'
97
PACKAGE_AUTHOR = 'Apache Software Foundation'
98
PACKAGE_EMAIL = '[email protected]'
99
PACKAGE_KEYWORDS = 'apache beam'
100
101
RECOMMENDED_MIN_PIP_VERSION = '19.3.0'
103
_PIP_VERSION = distribution('pip').version
104
if parse(_PIP_VERSION) < parse(RECOMMENDED_MIN_PIP_VERSION):
105
warnings.warn(
106
"You are using version {0} of pip. " \
107
"However, the recommended min version is {1}.".format(
108
_PIP_VERSION, RECOMMENDED_MIN_PIP_VERSION
109
)
110
)
111
except PackageNotFoundError:
112
# Do nothing if pip is not found. This can happen when using `Poetry` or
113
# `pipenv` package managers.
114
pass
116
REQUIRED_CYTHON_VERSION = '3.0.0'
117
try:
118
_CYTHON_VERSION = distribution('cython').version
119
if parse(_CYTHON_VERSION) < parse(REQUIRED_CYTHON_VERSION):
120
warnings.warn(
121
"You are using version {0} of cython. " \
122
"However, version {1} is recommended.".format(
123
_CYTHON_VERSION, REQUIRED_CYTHON_VERSION
124
)
125
)
126
except PackageNotFoundError:
127
# do nothing if Cython is not installed
128
pass
130
try:
131
# pylint: disable=wrong-import-position
132
from Cython.Build import cythonize as cythonize0
134
def cythonize(*args, **kwargs):
135
import numpy
136
extensions = cythonize0(*args, **kwargs)
137
for e in extensions:
138
e.include_dirs.append(numpy.get_include())
139
return extensions
140
except ImportError:
141
cythonize = lambda *args, **kwargs: []
142
143
# [BEAM-8181] pyarrow cannot be installed on 32-bit Windows platforms.
144
if sys.platform == 'win32' and sys.maxsize <= 2**32:
145
pyarrow_dependency = ['']
147
pyarrow_dependency = [
148
'pyarrow>=6.0.1,<24.0.0',
149
# NOTE(https://github.com/apache/beam/issues/29392): We can remove this
150
# once Beam increases the pyarrow lower bound to a version that fixes CVE.
151
# (lower bound >= 14.0.1)
152
'pyarrow-hotfix<1'
153
]
154
155
# Exclude pandas<=1.4.2 since it doesn't work with numpy 1.24.x.
156
# Exclude 1.5.0 and 1.5.1 because of
157
# https://github.com/pandas-dev/pandas/issues/45725
158
# must update the below "docs" and "test" for extras_require
159
dataframe_dependency = [
160
'pandas>=1.4.3,!=1.5.0,!=1.5.1,<2.3',
163
milvus_dependency = ['pymilvus>=2.5.10,<3.0.0']
164
165
ml_base = [
166
'embeddings>=0.0.4', # 0.0.3 crashes setuptools
167
'google-adk',
168
'onnxruntime',
169
'langchain',
170
'sentence-transformers>=2.2.2',
171
'skl2onnx',
172
'pyod>=0.7.6', # 0.7.5 crashes setuptools
173
'tensorflow',
174
# tensorflow transitive dep, lower versions not compatible with Python3.10+
175
'absl-py>=0.12.0',
176
'tensorflow-hub',
177
'tf2onnx',
178
'torch',
179
'transformers',
180
]
181
183
def find_by_ext(root_dir, ext):
184
for root, _, files in os.walk(root_dir):
185
for file in files:
186
if file.endswith(ext):
187
yield os.path.realpath(os.path.join(root, file))
190
# We must generate protos after setup_requires are installed.
191
def generate_protos_first():
193
# Pyproject toml build happens in isolated environemnts. In those envs,
194
# gen_protos is unable to get imported. so we run a subprocess call.
195
cwd = os.path.abspath(os.path.dirname(__file__))
196
# when pip install <>.tar.gz gets called, if gen_protos.py is not available
197
# in the sdist,then the proto files would have already been generated. So we
198
# skip proto generation in that case.
199
if not os.path.exists(os.path.join(cwd, 'gen_protos.py')):
200
# make sure we already generated protos
201
pb2_files = list(
202
find_by_ext(
203
os.path.join(cwd, 'apache_beam', 'portability', 'api'),
204
'_pb2.py'))
205
if not pb2_files:
206
raise RuntimeError(
207
'protobuf files are not generated. '
208
'Please generate pb2 files')
209
210
warnings.warn('Skipping proto generation as they are already generated.')
211
return
212
out = subprocess.run(
213
[sys.executable, os.path.join(cwd, 'gen_protos.py'), '--no-force'],
214
capture_output=True,
215
check=True)
216
print(out.stdout)
217
except subprocess.CalledProcessError as err:
218
raise RuntimeError('Could not generate protos due to error: %s', err.stderr)
219
220
221
def copy_tests_from_docs():
222
python_root = os.path.abspath(os.path.dirname(__file__))
223
docs_src = os.path.normpath(
224
os.path.join(
225
python_root, '../../website/www/site/content/en/documentation/sdks'))
226
docs_dest = os.path.normpath(
227
os.path.join(python_root, 'apache_beam/yaml/docs'))
228
if os.path.exists(docs_src):
229
shutil.rmtree(docs_dest, ignore_errors=True)
230
os.mkdir(docs_dest)
231
for path in glob.glob(os.path.join(docs_src, 'yaml*.md')):
232
shutil.copy(path, docs_dest)
233
else:
234
warnings.warn(
235
f'Could not locate yaml docs source directory {docs_src}. '
236
f'Skipping copying tests from docs.')
239
def generate_external_transform_wrappers():
240
try:
241
sdk_dir = os.path.abspath(os.path.dirname(__file__))
242
script_exists = os.path.exists(
243
os.path.join(sdk_dir, 'gen_xlang_wrappers.py'))
244
config_exists = os.path.exists(
245
os.path.join(
246
os.path.dirname(sdk_dir), 'standard_external_transforms.yaml'))
247
# we need both the script and the standard transforms config file.
248
# at build time, we don't have access to apache_beam to discover and
249
# retrieve external transforms, so the config file has to already exist
250
if not script_exists or not config_exists:
251
generated_transforms_dir = os.path.join(
252
sdk_dir, 'apache_beam', 'transforms', 'xlang')
253
254
# if exists, this directory will have at least its __init__.py file
255
if (not os.path.exists(generated_transforms_dir) or
256
len(os.listdir(generated_transforms_dir)) <= 1):
257
message = 'External transform wrappers have not been generated '
258
if not script_exists:
259
message += 'and the generation script `gen_xlang_wrappers.py`'
260
if not config_exists:
261
message += 'and the standard external transforms config'
262
message += ' could not be found'
263
raise RuntimeError(message)
264
else:
265
logging.info(
266
'Skipping external transform wrapper generation as they '
267
'are already generated.')
268
return
269
subprocess.run([
270
sys.executable,
271
os.path.join(sdk_dir, 'gen_xlang_wrappers.py'),
272
'--cleanup',
273
'--transforms-config-source',
274
os.path.join(
275
os.path.dirname(sdk_dir), 'standard_external_transforms.yaml')
276
],
277
capture_output=True,
278
check=True)
279
except subprocess.CalledProcessError as err:
280
raise RuntimeError(
281
'Could not generate external transform wrappers due to '
282
'error: %s',
283
err.stderr)
286
def get_portability_package_data():
287
files = []
288
portability_dir = Path(__file__).parent / 'apache_beam' / \
289
'portability' / 'api'
290
for ext in ['*.pyi', '*.yaml']:
291
files.extend(
292
str(p.relative_to(portability_dir.parent.parent))
293
for p in portability_dir.rglob(ext))
294
295
return files
298
python_requires = '>=3.10'
300
if sys.version_info.major == 3 and sys.version_info.minor >= 15:
301
warnings.warn(
302
'This version of Apache Beam has not been sufficiently tested on '
303
'Python %s.%s. You may encounter bugs or missing features.' %
304
(sys.version_info.major, sys.version_info.minor))
306
if __name__ == '__main__':
307
# In order to find the tree of proto packages, the directory
308
# structure must exist before the call to setuptools.find_packages()
309
# executes below.
310
generate_protos_first()
312
generate_external_transform_wrappers()
313
314
# These data files live elsewhere in the full Beam repository.
315
copy_tests_from_docs()
316
317
# generate cythonize extensions only if we are building a wheel or
318
# building an extension or running in editable mode.
319
cythonize_cmds = ('bdist_wheel', 'build_ext', 'editable_wheel')
320
if any(cmd in sys.argv for cmd in cythonize_cmds):
321
extensions = cythonize([
322
'apache_beam/**/*.pyx',
323
'apache_beam/coders/coder_impl.py',
324
'apache_beam/metrics/cells.py',
325
'apache_beam/metrics/execution.py',
326
'apache_beam/runners/common.py',
327
'apache_beam/runners/worker/logger.py',
328
'apache_beam/runners/worker/opcounters.py',
329
'apache_beam/runners/worker/operations.py',
330
'apache_beam/transforms/cy_combiners.py',
331
'apache_beam/transforms/stats.py',
332
'apache_beam/utils/counters.py',
333
'apache_beam/utils/windowed_value.py',
334
])
335
else:
336
extensions = []
338
try:
339
long_description = ((Path(__file__).parent /
340
"README.md").read_text(encoding='utf-8'))
341
except FileNotFoundError:
342
long_description = (
343
'Apache Beam is a unified programming model for both batch and '
344
'streaming data processing, enabling efficient execution across '
345
'diverse distributed execution engines and providing extensibility '
346
'points for connecting to different technologies and user '
347
'communities.')
349
# Keep all dependencies inlined in the setup call, otherwise Dependabot won't
350
# be able to parse it.
351
setuptools.setup(
352
name=PACKAGE_NAME,
353
version=PACKAGE_VERSION,
354
description=PACKAGE_DESCRIPTION,
355
long_description=long_description,
356
long_description_content_type='text/markdown',
357
url=PACKAGE_URL,
358
download_url=PACKAGE_DOWNLOAD_URL,
359
author=PACKAGE_AUTHOR,
360
author_email=PACKAGE_EMAIL,
361
packages=setuptools.find_packages(),
362
package_data={
363
'apache_beam': [
364
'*/*.pyx',
365
'*/*/*.pyx',
366
'*/*.pxd',
367
'*/*/*.pxd',
368
'*/*.h',
369
'*/*/*.h',
370
'testing/data/*.yaml',
371
'yaml/*.yaml',
372
'yaml/docs/*.md',
373
*get_portability_package_data()
376
ext_modules=extensions,
377
install_requires=[
378
'cryptography>=39.0.0,<48.0.0',
379
'envoy-data-plane>=1.0.3,<2; python_version >= "3.11"',
380
# Newer version only work on Python 3.11. Versions 0.3 <= ver < 1.x
381
# conflict with other GCP dependencies.
382
'envoy-data-plane<0.3.0; python_version < "3.11"',
383
'fastavro>=0.23.6,<2',
384
'fasteners>=0.3,<1.0',
385
'grpcio>=1.33.1,<2,!=1.48.0,!=1.59.*,!=1.60.*,!=1.61.*,!=1.62.0,!=1.62.1,!=1.66.*,!=1.67.*,!=1.68.*,!=1.69.*,!=1.70.*', # pylint: disable=line-too-long
386
'httplib2>=0.8,<0.32.0',
387
'jsonpickle>=3.0.0,<4.0.0',
388
# numpy can have breaking changes in minor versions.
389
# Use a strict upper bound.
390
'numpy>=1.14.3,<2.5.0', # Update pyproject.toml as well.
391
'objsize>=0.6.1,<0.8.0',
392
'packaging>=22.0',
393
'pillow>=12.1.1,<13',
394
'pymongo>=3.8.0,<5.0.0',
395
'proto-plus>=1.7.1,<2',
396
# 1. Use a tighter upper bound in protobuf dependency to make sure
397
# the minor version at job submission
398
# does not exceed the minor version at runtime.
399
# To avoid depending on an old dependency, update the minor version on
400
# every Beam release, see: https://github.com/apache/beam/issues/25590
401
402
# 2. Allow latest protobuf 3 version as a courtesy to some customers.
403
#
404
# 3. Exclude protobuf 4 versions that leak memory, see:
405
# https://github.com/apache/beam/issues/28246
406
'protobuf>=3.20.3,<7.0.0.dev0,!=4.0.*,!=4.21.*,!=4.22.0,!=4.23.*,!=4.24.*', # pylint: disable=line-too-long
407
'python-dateutil>=2.8.0,<3',
408
'pytz>=2018.3',
409
'requests>=2.32.4,<3.0.0',
410
'sortedcontainers>=2.4.0',
411
'typing-extensions>=3.7.0',
412
'zstandard>=0.18.0,<1',
413
'pyyaml>=3.12,<7.0.0',
414
'beartype>=0.21.0,<0.23.0',
415
# Dynamic dependencies must be specified in a separate list, otherwise
416
# Dependabot won't be able to parse the main list. Any dynamic
417
# dependencies will not receive updates from Dependabot.
418
] + pyarrow_dependency,
419
python_requires=python_requires,
420
# BEAM-8840: Do NOT use tests_require or setup_requires.
421
extras_require={
422
'dill': [
423
# Dill doesn't have forwards-compatibility guarantees within minor
424
# version. Pickles created with a new version of dill may not
425
# unpickle using older version of dill. It is best to use the same
426
# version of dill on client and server, therefore list of allowed
427
# versions is very narrow.
428
# See: https://github.com/uqfoundation/dill/issues/341.
429
'dill>=0.3.1.1,<0.3.2',
431
'docs': [
432
'jinja2>=3.0,<3.2',
433
'Sphinx>=7.0.0,<8.0',
434
'docstring-parser>=0.15,<1.0',
435
'docutils>=0.18.1',
436
'markdown',
437
'pandas<2.3.0',
438
'openai',
439
'virtualenv-clone>=0.5,<1.0',
441
'test': [
442
'cloud-sql-python-connector[pg8000]>=1.0.0,<2.0.0',
443
'docstring-parser>=0.15,<1.0',
444
'freezegun>=0.3.12',
445
'jinja2>=3.0,<3.2',
446
'joblib>=1.0.1',
447
'mock>=1.0.1,<6.0.0',
448
'pandas<2.3.0',
449
'parameterized>=0.7.1,<0.10.0',
450
'pyhamcrest>=1.9,!=1.10.0,<3.0.0',
451
'requests_mock>=1.7,<2.0',
452
'tenacity>=8.0.0,<9',
453
'pytest>=7.1.2,<9.0',
454
'pytest-xdist>=2.5.0,<4',
455
'pytest-timeout>=2.1.0,<3',
456
'scikit-learn>=0.20.0,<1.8.0',
457
'sqlalchemy>=1.3,<3.0',
458
'psycopg2-binary>=2.8.5,<3.0',
459
'testcontainers[mysql,kafka,milvus]>=4.0.0,<5.0.0',
460
'cryptography>=41.0.2',
461
# TODO(https://github.com/apache/beam/issues/36951): need to
462
# further investigate the cause
463
'hypothesis>5.0.0,<6.148.4',
464
'virtualenv-clone>=0.5,<1.0',
465
'python-tds>=1.16.1',
466
'sqlalchemy-pytds>=1.0.2',
467
'pg8000>=1.31.5',
468
"PyMySQL>=1.1.0",
469
'oracledb>=3.1.1'
471
'gcp': [
472
'cachetools>=3.1.0,<7',
473
'google-api-core>=2.0.0,<3',
474
'google-apitools>=0.5.31,<0.5.32; python_version < "3.13"',
475
'google-apitools>=0.5.35; python_version >= "3.13"',
476
# NOTE: Maintainers, please do not require google-auth>=2.x.x
477
# Until this issue is closed
478
# https://github.com/googleapis/google-cloud-python/issues/10566
479
'google-auth>=1.18.0,<3',
480
'google-auth-httplib2>=0.1.0,<0.3.0',
481
'google-cloud-datastore>=2.0.0,<3',
482
'google-cloud-pubsub>=2.1.0,<3',
483
'google-cloud-storage>=2.18.2,<3',
484
# GCP packages required by tests
485
'google-cloud-bigquery>=2.0.0,<4',
486
'google-cloud-bigquery-storage>=2.6.3,<3',
487
'google-cloud-core>=2.0.0,<3',
488
'google-cloud-bigtable>=2.19.0,<3',
489
'google-cloud-build>=3.35.0,<4',
490
'google-cloud-spanner>=3.0.0,<4',
491
# GCP Packages required by ML functionality
492
'google-cloud-dlp>=3.0.0,<4',
493
'google-cloud-kms>=3.0.0,<4',
494
'google-cloud-language>=2.0,<3',
495
'google-cloud-secret-manager>=2.0,<3',
496
'google-cloud-videointelligence>=2.0,<3',
497
'google-cloud-vision>=2,<4',
498
'google-cloud-recommendations-ai>=0.1.0,<0.11.0',
499
'google-cloud-aiplatform>=1.26.0, < 2.0',
500
'cloud-sql-python-connector>=1.18.2,<2.0.0',
501
'python-tds>=1.16.1',
502
'pg8000>=1.31.5',
503
"PyMySQL>=1.1.0",
504
# Authentication for Google Artifact Registry when using
505
# --extra-index-url or --index-url in requirements.txt in
506
# Dataflow, which allows installing python packages from private
507
# Python repositories in GAR.
508
'keyrings.google-artifactregistry-auth',
509
'orjson>=3.9.7,<4',
510
'regex>=2020.6.8',
511
],
512
'interactive': [
513
'facets-overview>=1.1.0,<2',
514
'google-cloud-dataproc>=5.0.0,<6',
515
'ipython>=7,<9',
516
'ipykernel>=6,<7',
517
'ipywidgets>=8,<9',
518
# Skip version 6.1.13 due to
519
# https://github.com/jupyter/jupyter_client/issues/637
520
'jupyter-client>=6.1.11,!=6.1.13,<8.2.1',
521
'pydot>=1.2.0,<2',
522
'timeloop>=1.0.2,<2',
523
'nbformat>=5.0.5,<6',
524
'nbconvert>=6.2.0,<8',
525
] + dataframe_dependency,
526
'interactive_test': [
527
# headless chrome based integration tests
528
'needle>=0.5.0,<1',
529
'chromedriver-binary>=117,<118',
530
# use a fixed major version of PIL for different python versions
531
'pillow>=7.1.1,<10',
532
# urllib 2.x is a breaking change for the headless chrome tests
533
'urllib3<2,>=1.21.1'
535
# Optional dependencies to unit-test ML functionality.
536
# We don't expect users to install this extra. Users should install
537
# necessary dependencies individually, or we should create targeted
538
# extras. Keeping the bounds open as much as possible so that we
539
# can find out early when Beam doesn't work with new versions.
540
'ml_test': [
541
'datatable',
542
# tensorflow-transform requires dill, but doesn't set dill as a
543
# hard requirement in setup.py.
544
'dill',
545
# match tft extra.
546
'tensorflow_transform>=1.14.0,<1.15.0',
547
# TFT->TFX-BSL require pandas 1.x, which is not compatible
548
# with numpy 2.x
549
'numpy<2',
550
# To help with dependency resolution in test suite. Revise once
551
# https://github.com/apache/beam/issues/37854 is fixed
552
'protobuf<4; python_version<"3.11"'
553
# Comment out xgboost as it is breaking presubmit python ml
554
# tests due to tag check introduced since pip 24.2
555
# https://github.com/apache/beam/issues/31285
556
# 'xgboost<2.0', # https://github.com/apache/beam/issues/31252
557
] + ml_base,
558
'p310_ml_test': [
559
'datatable',
560
] + ml_base,
561
'p312_ml_test': [
562
'datatable',
563
] + ml_base,
564
# maintainer: milvus tests only run with this extension. Make sure it
565
# is covered by docker-in-docker test when changing py version
566
'p313_ml_test': ml_base + milvus_dependency,
567
'aws': ['boto3>=1.9,<2'],
568
'azure': [
569
'azure-storage-blob>=12.3.2,<13',
570
'azure-core>=1.7.0,<2',
571
'azure-identity>=1.12.0,<2',
573
'dataframe': dataframe_dependency,
574
'dask': [
575
'distributed >= 2024.4.2',
576
'dask >= 2024.4.2',
577
# For development, 'distributed >= 2023.12.1' should work with
578
# the above dask PR, however it can't be installed as part of
579
# a single `pip` call, since distributed releases are pinned to
580
# specific dask releases. As a workaround, distributed can be
581
# installed first, and then `.[dask]` installed second, with the
582
# `--update` / `-U` flag to replace the dask release brought in
583
# by distributed.
585
'hadoop': ['hdfs>=2.1.0,<3.0.0'],
586
'yaml': [
587
'docstring-parser>=0.15,<1.0',
588
'jinja2>=3.0,<3.2',
589
'virtualenv-clone>=0.5,<1.0',
590
# https://github.com/PiotrDabkowski/Js2Py/issues/317
591
'js2py>=0.74,<1; python_version<"3.12"',
592
'jsonschema>=4.0.0,<5.0.0',
593
] + dataframe_dependency,
594
# Keep the following dependencies in line with what we test against
595
# in https://github.com/apache/beam/blob/master/sdks/python/tox.ini
596
# For more info, see
597
# https://docs.google.com/document/d/1c84Gc-cZRCfrU8f7kWGsNR2o8oSRjCM-dGHO9KvPWPw/edit?usp=sharing
598
'torch': ['torch>=1.9.0,<2.8.0'],
599
'tensorflow': [
600
'tensorflow>=2.12rc1,<2.21',
601
# tensorflow transitive dep
602
'absl-py>=0.12.0'
603
],
604
'transformers': [
605
'transformers>=4.28.0,<4.56.0',
606
'tensorflow>=2.12.0',
607
'torch>=1.9.0'
609
'ml_cpu': [
610
'tensorflow>=2.12.0',
611
'torch==2.8.0+cpu',
612
'transformers>=4.28.0,<4.56.0',
613
# tensorflow transient dep
614
'absl-py>=0.12.0'
616
'redis': ['redis>=5.0.0,<6'],
618
'tensorflow_transform>=1.14.0,<1.15.0',
619
# TFT->TFX-BSL require pandas 1.x, which is not compatible
620
# with numpy 2.x
621
'numpy<2',
622
# tensorflow-transform requires dill, but doesn't set dill as a
623
# hard requirement in setup.py.
624
'dill'
625
],
626
'tfrecord': ['crcmod>=1.7,<2.0'],
627
'onnx': [
628
'onnxruntime==1.13.1',
629
'torch==1.13.1',
630
'tensorflow==2.11.0',
631
'tf2onnx==1.13.0',
632
'skl2onnx==1.13',
633
'transformers==4.25.1',
634
# tensorflow transient dep
635
'absl-py>=0.12.0'
637
'xgboost': ['xgboost>=1.6.0,<2.1.3', 'datatable==1.0.0'],
638
'tensorflow-hub': ['tensorflow-hub>=0.14.0,<0.16.0'],
639
'milvus': milvus_dependency,
640
'vllm': ['openai==1.107.1', 'vllm==0.10.1.1', 'triton==3.3.1']
641
},
642
zip_safe=False,
643
# PyPI package information.
644
classifiers=[
645
'Intended Audience :: End Users/Desktop',
646
'License :: OSI Approved :: Apache Software License',
647
'Operating System :: POSIX :: Linux',
648
'Programming Language :: Python :: 3.10',
649
'Programming Language :: Python :: 3.11',
650
'Programming Language :: Python :: 3.12',
651
'Programming Language :: Python :: 3.13',
652
'Programming Language :: Python :: 3.14',
653
# When updating version classifiers, also update version warnings
654
# above and in apache_beam/__init__.py.
655
'Topic :: Software Development :: Libraries',
656
'Topic :: Software Development :: Libraries :: Python Modules',
657
],
658
license='Apache License, Version 2.0',
659
keywords=PACKAGE_KEYWORDS,
660
cmdclass={
661
'mypy': mypy,