Skip to content

Commit f6127fc

Browse files
authored
ARROW-17423: [CI][C++] Fix building CUDA docker images (#13896)
* Update the CUDA runtime version as CUDA 9.1 images are not available anymore * Fix passing child command arguments to "docker run" Checked locally under a Ubuntu 20.04 host with: ``` UBUNTU=18.04 archery --debug docker run ubuntu-cuda-cpp UBUNTU=20.04 archery --debug docker run ubuntu-cuda-cpp ``` Authored-by: Antoine Pitrou <[email protected]> Signed-off-by: Antoine Pitrou <[email protected]>
1 parent 050876c commit f6127fc

12 files changed

Lines changed: 41 additions & 27 deletions

File tree

‎.env‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ UBUNTU=20.04
5454

5555
# Default versions for various dependencies
5656
CLANG_TOOLS=12
57-
CUDA=9.1
57+
CUDA=11.0.3
5858
DASK=latest
5959
DOTNET=6.0
6060
GCC_VERSION=""

‎dev/archery/archery/docker/core.py‎

Lines changed: 6 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -16,14 +16,15 @@
1616
# under the License.
1717

1818
import os
19-
import re
19+
import shlex
2020
import subprocess
2121
from io import StringIO
2222

2323
from dotenv import dotenv_values
2424
from ruamel.yaml import YAML
2525

2626
from ..utils.command import Command, default_bin
27+
from ..utils.source import arrow_path
2728
from ..compat import _ensure_path
2829

2930

@@ -42,12 +43,6 @@ def flatten(node, parents=None):
4243
raise TypeError(node)
4344

4445

45-
def _sanitize_command(cmd):
46-
if isinstance(cmd, list):
47-
cmd = " ".join(cmd)
48-
return re.sub(r"\s+", " ", cmd)
49-
50-
5146
_arch_short_mapping = {
5247
'arm64v8': 'arm64',
5348
}
@@ -294,7 +289,7 @@ def _build(service, use_cache):
294289

295290
args.extend([
296291
'--output', 'type=docker',
297-
'-f', service['build']['dockerfile'],
292+
'-f', arrow_path(service['build']['dockerfile']),
298293
'-t', service['image'],
299294
service['build'].get('context', '.')
300295
])
@@ -306,7 +301,7 @@ def _build(service, use_cache):
306301
for img in cache_from:
307302
args.append('--cache-from="{}"'.format(img))
308303
args.extend([
309-
'-f', service['build']['dockerfile'],
304+
'-f', arrow_path(service['build']['dockerfile']),
310305
'-t', service['image'],
311306
service['build'].get('context', '.')
312307
])
@@ -381,10 +376,9 @@ def run(self, service_name, command=None, *, env=None, volumes=None,
381376
if command is not None:
382377
args.append(command)
383378
else:
384-
# replace whitespaces from the preformatted compose command
385-
cmd = _sanitize_command(service.get('command', ''))
379+
cmd = service.get('command', '')
386380
if cmd:
387-
args.append(cmd)
381+
args.extend(shlex.split(cmd))
388382

389383
# execute as a plain docker cli command
390384
self._execute_docker('run', '--rm', *args)

‎dev/archery/archery/docker/tests/test_docker.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -509,7 +509,7 @@ def test_image_with_gpu(arrow_compose_path):
509509
"-e", "OTHER_ENV=2",
510510
"-v", "/host:/container:rw",
511511
"org/ubuntu-cuda",
512-
'/bin/bash -c "echo 1 > /tmp/dummy && cat /tmp/dummy"'
512+
"/bin/bash", "-c", "echo 1 > /tmp/dummy && cat /tmp/dummy",
513513
]
514514
]
515515
with assert_docker_calls(compose, expected_calls):

‎dev/archery/archery/integration/runner.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,8 +35,8 @@
3535
from .tester_java import JavaTester
3636
from .tester_js import JSTester
3737
from .tester_csharp import CSharpTester
38-
from .util import (ARROW_ROOT_DEFAULT, guid, SKIP_ARROW, SKIP_FLIGHT,
39-
printer)
38+
from .util import guid, SKIP_ARROW, SKIP_FLIGHT, printer
39+
from ..utils.source import ARROW_ROOT_DEFAULT
4040
from . import datagen
4141

4242

‎dev/archery/archery/integration/tester_cpp.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,8 @@
2020
import subprocess
2121

2222
from .tester import Tester
23-
from .util import run_cmd, ARROW_ROOT_DEFAULT, log
23+
from .util import run_cmd, log
24+
from ..utils.source import ARROW_ROOT_DEFAULT
2425

2526

2627
_EXE_PATH = os.environ.get(

‎dev/archery/archery/integration/tester_csharp.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,8 @@
1818
import os
1919

2020
from .tester import Tester
21-
from .util import run_cmd, ARROW_ROOT_DEFAULT, log
21+
from .util import run_cmd, log
22+
from ..utils.source import ARROW_ROOT_DEFAULT
2223

2324

2425
_EXE_PATH = os.path.join(

‎dev/archery/archery/integration/tester_java.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,8 @@
2020
import subprocess
2121

2222
from .tester import Tester
23-
from .util import run_cmd, ARROW_ROOT_DEFAULT, log
23+
from .util import run_cmd, log
24+
from ..utils.source import ARROW_ROOT_DEFAULT
2425

2526

2627
def load_version_from_pom():

‎dev/archery/archery/integration/tester_js.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,9 @@
1818
import os
1919

2020
from .tester import Tester
21-
from .util import run_cmd, ARROW_ROOT_DEFAULT, log
21+
from .util import run_cmd, log
22+
from ..utils.source import ARROW_ROOT_DEFAULT
23+
2224

2325
_EXE_PATH = os.path.join(ARROW_ROOT_DEFAULT, 'js/bin')
2426
_VALIDATE = os.path.join(_EXE_PATH, 'integration.js')

‎dev/archery/archery/integration/tester_rust.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,8 @@
2020
import subprocess
2121

2222
from .tester import Tester
23-
from .util import run_cmd, ARROW_ROOT_DEFAULT, log
23+
from .util import run_cmd, log
24+
from ..utils.source import ARROW_ROOT_DEFAULT
2425

2526

2627
_EXE_PATH = os.path.join(ARROW_ROOT_DEFAULT, "rust/target/debug")

‎dev/archery/archery/integration/util.py‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717

1818
import contextlib
1919
import io
20-
import os
2120
import random
2221
import socket
2322
import subprocess
@@ -36,11 +35,6 @@ def guid():
3635
SKIP_ARROW = 'arrow'
3736
SKIP_FLIGHT = 'flight'
3837

39-
ARROW_ROOT_DEFAULT = os.environ.get(
40-
'ARROW_ROOT',
41-
os.path.abspath(__file__).rsplit("/", 5)[0]
42-
)
43-
4438

4539
class _Printer:
4640
"""

0 commit comments

Comments
 (0)