Skip to content

Commit 0e7fb07

Browse files
committed
python module: add an automatic byte-compilation step
For all source `*.py` files installed via either py.install_sources() or an `install_dir: py.get_install_dir()`, produce `*.pyc` files at install time. Controllable via a module option.
1 parent 4a25308 commit 0e7fb07

8 files changed

Lines changed: 149 additions & 7 deletions

File tree

‎docs/markdown/Builtin-options.md‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -351,6 +351,7 @@ install prefix. For example: if the install prefix is `/usr` and the
351351

352352
| Option | Default value | Possible values | Description |
353353
| ------ | ------------- | ----------------- | ----------- |
354+
| bytecompile | 0 | integer from -1 to 2 | What bytecode optimization level to use (Since 1.2.0) |
354355
| install_env | prefix | {auto,prefix,system,venv} | Which python environment to install to (Since 0.62.0) |
355356
| platlibdir | | Directory path | Directory for site-specific, platform-specific files (Since 0.60.0) |
356357
| purelibdir | | Directory path | Directory for site-specific, non-platform-specific files (Since 0.60.0) |
@@ -373,3 +374,13 @@ installation is a virtualenv, and use `venv` or `system` as appropriate (but
373374
never `prefix`). This option is mutually exclusive with the `platlibdir`/`purelibdir`.
374375

375376
For backwards compatibility purposes, the default `install_env` is `prefix`.
377+
378+
*Since 1.2.0* The `python.bytecompile` option can be used to enable compiling
379+
python bytecode. Bytecode has 3 optimization levels:
380+
381+
- 0, bytecode without optimizations
382+
- 1, bytecode with some optimizations
383+
- 2, bytecode with some more optimizations
384+
385+
To this, Meson adds level `-1`, which is to not attempt to compile bytecode at
386+
all.
Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
## Python module can now compile bytecode
2+
3+
A new builtin option is available: `-Dpython.bytecompile=2`. It can be used to
4+
compile bytecode for all pure python files installed via the python module.

‎mesonbuild/coredata.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1266,6 +1266,8 @@ def add_to_argparse(self, name: str, parser: argparse.ArgumentParser, help_suffi
12661266
BuiltinOption(UserBooleanOption, 'Generate pkgconfig files as relocatable', False)),
12671267

12681268
# Python module
1269+
(OptionKey('bytecompile', module='python'),
1270+
BuiltinOption(UserIntegerOption, 'Whether to compile bytecode', (-1, 2, 0))),
12691271
(OptionKey('install_env', module='python'),
12701272
BuiltinOption(UserComboOption, 'Which python environment to install to', 'prefix', choices=['auto', 'prefix', 'system', 'venv'])),
12711273
(OptionKey('platlibdir', module='python'),

‎mesonbuild/modules/python.py‎

Lines changed: 61 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,7 @@
1313
# limitations under the License.
1414
from __future__ import annotations
1515

16-
import copy
17-
import os
18-
import shutil
16+
import copy, json, os, shutil
1917
import typing as T
2018

2119
from . import ExtensionModule, ModuleInfo
@@ -41,7 +39,7 @@
4139
from typing_extensions import TypedDict
4240

4341
from . import ModuleState
44-
from ..build import SharedModule, Data
42+
from ..build import Build, SharedModule, Data
4543
from ..dependencies import Dependency
4644
from ..interpreter import Interpreter
4745
from ..interpreter.kwargs import ExtractRequired
@@ -66,6 +64,12 @@ class FindInstallationKw(ExtractRequired):
6664

6765

6866
class PythonExternalProgram(BasicPythonExternalProgram):
67+
68+
# This is a ClassVar instead of an instance bool, because although an
69+
# installation is cached, we actually copy it, modify attributes such as pure,
70+
# and return a temporary one rather than the cached object.
71+
run_bytecompile: T.ClassVar[T.Dict[str, bool]] = {}
72+
6973
def sanity(self, state: T.Optional['ModuleState'] = None) -> bool:
7074
ret = super().sanity()
7175
if ret:
@@ -216,6 +220,7 @@ def dependency_method(self, args: T.List['TYPE_var'], kwargs: 'TYPE_kwargs') ->
216220
)
217221
def install_sources_method(self, args: T.Tuple[T.List[T.Union[str, mesonlib.File]]],
218222
kwargs: 'PyInstallKw') -> 'Data':
223+
self.held_object.run_bytecompile[self.version] = True
219224
tag = kwargs['install_tag'] or 'python-runtime'
220225
pure = kwargs['pure'] if kwargs['pure'] is not None else self.pure
221226
install_dir = self._get_install_dir_impl(pure, kwargs['subdir'])
@@ -229,6 +234,7 @@ def install_sources_method(self, args: T.Tuple[T.List[T.Union[str, mesonlib.File
229234
@noPosargs
230235
@typed_kwargs('python_installation.install_dir', _PURE_KW, _SUBDIR_KW)
231236
def get_install_dir_method(self, args: T.List['TYPE_var'], kwargs: 'PyInstallKw') -> str:
237+
self.held_object.run_bytecompile[self.version] = True
232238
pure = kwargs['pure'] if kwargs['pure'] is not None else self.pure
233239
return self._get_install_dir_impl(pure, kwargs['subdir'])
234240

@@ -297,6 +303,56 @@ def __init__(self, interpreter: 'Interpreter') -> None:
297303
'find_installation': self.find_installation,
298304
})
299305

306+
def _get_install_scripts(self) -> T.List[mesonlib.ExecutableSerialisation]:
307+
backend = self.interpreter.backend
308+
ret = []
309+
optlevel = self.interpreter.environment.coredata.get_option(mesonlib.OptionKey('bytecompile', module='python'))
310+
if optlevel == -1:
311+
return ret
312+
if not any(PythonExternalProgram.run_bytecompile.values()):
313+
return ret
314+
315+
installdata = backend.create_install_data()
316+
py_files = []
317+
318+
def should_append(f, isdir: bool = False):
319+
# This uses the install_plan decorated names to see if the original source was propagated via
320+
# install_sources() or get_install_dir().
321+
return f.startswith(('{py_platlib}', '{py_purelib}')) and (f.endswith('.py') or isdir)
322+
323+
for t in installdata.targets:
324+
if should_append(t.out_name):
325+
py_files.append(os.path.join(installdata.prefix, t.outdir, os.path.basename(t.fname)))
326+
for d in installdata.data:
327+
if should_append(d.install_path_name):
328+
py_files.append(os.path.join(installdata.prefix, d.install_path))
329+
for d in installdata.install_subdirs:
330+
if should_append(d.install_path_name, True):
331+
py_files.append(os.path.join(installdata.prefix, d.install_path))
332+
333+
import importlib.resources
334+
pycompile = os.path.join(self.interpreter.environment.get_scratch_dir(), 'pycompile.py')
335+
with open(pycompile, 'wb') as f:
336+
f.write(importlib.resources.read_binary('mesonbuild.scripts', 'pycompile.py'))
337+
338+
for i in self.installations.values():
339+
if isinstance(i, PythonExternalProgram) and i.run_bytecompile[i.info['version']]:
340+
i = T.cast(PythonExternalProgram, i)
341+
manifest = f'python-{i.info["version"]}-installed.json'
342+
manifest_json = []
343+
for f in py_files:
344+
if f.startswith((os.path.join(installdata.prefix, i.platlib), os.path.join(installdata.prefix, i.purelib))):
345+
manifest_json.append(f)
346+
with open(os.path.join(self.interpreter.environment.get_scratch_dir(), manifest), 'w', encoding='utf-8') as f:
347+
json.dump(manifest_json, f)
348+
cmd = i.command + [pycompile, manifest, str(optlevel)]
349+
script = backend.get_executable_serialisation(cmd, verbose=True)
350+
ret.append(script)
351+
return ret
352+
353+
def postconf_hook(self, b: Build) -> None:
354+
b.install_scripts.extend(self._get_install_scripts())
355+
300356
# https://www.python.org/dev/peps/pep-0397/
301357
@staticmethod
302358
def _get_win_pythonpath(name_or_path: str) -> T.Optional[str]:
@@ -421,6 +477,7 @@ def find_installation(self, state: 'ModuleState', args: T.Tuple[T.Optional[str]]
421477
else:
422478
python = copy.copy(python)
423479
python.pure = kwargs['pure']
480+
python.run_bytecompile.setdefault(python.info['version'], False)
424481
return python
425482

426483
raise mesonlib.MesonBugException('Unreachable code was reached (PythonModule.find_installation).')

‎mesonbuild/scripts/pycompile.py‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
# Copyright 2016 The Meson development team
2+
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
# ignore all lints for this file, since it is run by python2 as well
16+
17+
# type: ignore
18+
# pylint: disable=deprecated-module
19+
20+
import json, os, subprocess, sys
21+
from compileall import compile_file
22+
23+
destdir = os.environ.get('DESTDIR')
24+
quiet = int(os.environ.get('MESON_INSTALL_QUIET', 0))
25+
26+
def destdir_join(d1, d2):
27+
if not d1:
28+
return d2
29+
# c:\destdir + c:\prefix must produce c:\destdir\prefix
30+
parts = os.path.splitdrive(d2)
31+
return d1 + parts[1]
32+
33+
def compileall(files):
34+
for f in files:
35+
if destdir is not None:
36+
ddir = os.path.dirname(f)
37+
fullpath = destdir_join(destdir, f)
38+
else:
39+
ddir = None
40+
fullpath = f
41+
42+
if os.path.isdir(fullpath):
43+
for root, _, files in os.walk(fullpath):
44+
ddir = os.path.dirname(os.path.splitdrive(f)[0] + root[len(destdir):])
45+
for dirf in files:
46+
if dirf.endswith('.py'):
47+
fullpath = os.path.join(root, dirf)
48+
compile_file(fullpath, ddir, force=True, quiet=quiet)
49+
else:
50+
compile_file(fullpath, ddir, force=True, quiet=quiet)
51+
52+
def run(manifest):
53+
data_file = os.path.join(os.path.dirname(__file__), manifest)
54+
with open(data_file, 'rb') as f:
55+
dat = json.load(f)
56+
compileall(dat)
57+
58+
if __name__ == '__main__':
59+
manifest = sys.argv[1]
60+
run(manifest)
61+
if len(sys.argv) > 2:
62+
optlevel = int(sys.argv[2])
63+
# python2 only needs one or the other
64+
if optlevel == 1 or (sys.version_info >= (3,) and optlevel > 0):
65+
subprocess.check_call([sys.executable, '-O'] + sys.argv[:2])
66+
if optlevel == 2:
67+
subprocess.check_call([sys.executable, '-OO'] + sys.argv[:2])

‎test cases/common/252 install data structured/meson.build‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
project('install structured data')
1+
project('install structured data', default_options: ['python.bytecompile=-1'])
22

33
install_data(
44
'dir1/file1',

‎test cases/python/2 extmodule/meson.build‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
project('Python extension module', 'c',
2-
default_options : ['buildtype=release', 'werror=true'])
2+
default_options : ['buildtype=release', 'werror=true', 'python.bytecompile=-1'])
33
# Because Windows Python ships only with optimized libs,
44
# we must build this project the same way.
55

‎test cases/python/7 install path/meson.build‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
project('install path',
22
default_options: [
3+
'python.bytecompile=-1',
34
'python.purelibdir=/pure',
4-
'python.platlibdir=/plat'
5+
'python.platlibdir=/plat',
56
]
67
)
78

0 commit comments

Comments
 (0)