Skip to content

Commit e92c792

Browse files
committed
Reverting some changes
diff --git c/cpp/src/arrow/config.cc i/cpp/src/arrow/config.cc index b2f7a38..8ff5a3d 100644 --- c/cpp/src/arrow/config.cc +++ i/cpp/src/arrow/config.cc @@ -64,6 +64,10 @@ std::string MakeSimdLevelString(QueryFlagFunction&& query_flag) { } } +// TODO(apacheGH-48593): Remove when libc++ supports std::chrono timezone +// apache#48593 +std::optional<std::string> timezone_db_path; + }; // namespace const BuildInfo& GetBuildInfo() { return kBuildInfo; } @@ -75,6 +79,12 @@ RuntimeInfo GetRuntimeInfo() { MakeSimdLevelString([&](int64_t flags) { return cpu_info->IsSupported(flags); }); info.detected_simd_level = MakeSimdLevelString([&](int64_t flags) { return cpu_info->IsDetected(flags); }); + info.using_os_timezone_db = USE_OS_TZDB; +#if !USE_OS_TZDB + info.timezone_db_path = timezone_db_path; +#else + info.timezone_db_path = std::optional<std::string>(); +#endif return info; } @@ -89,7 +99,12 @@ Status Initialize(const GlobalOptions& options) noexcept { } catch (const std::runtime_error& e) { return Status::IOError(e.what()); } -#endif + timezone_db_path = options.timezone_db_path.value(); +#else + return Status::Invalid( + "Arrow was set to use OS timezone database at compile time, " + "so a downloaded database cannot be provided at runtime."); +#endif // !USE_OS_TZDB } return Status::OK(); } diff --git c/cpp/src/arrow/config.h i/cpp/src/arrow/config.h index 9fb1710..54f4235 100644 --- c/cpp/src/arrow/config.h +++ i/cpp/src/arrow/config.h @@ -64,6 +64,17 @@ struct RuntimeInfo { /// The SIMD level available on the OS and CPU std::string detected_simd_level; + + /// Whether using the OS-based timezone database + /// This is set at compile-time. + // ARROW_DEPRECATED("Deprecated in 23.0.0") + bool using_os_timezone_db; + + /// The path to the timezone database; by default None. + /// This is only used on some Windows builds where + /// std::chrono timezone support is not available. + // ARROW_DEPRECATED("Deprecated in 23.0.0") + std::optional<std::string> timezone_db_path; }; /// \brief Get runtime build info. @@ -82,8 +93,10 @@ RuntimeInfo GetRuntimeInfo(); // TODO(apacheGH-48743): Remove when RTools upgrades to GCC with std::chrono timezone support // apache#48743 struct GlobalOptions { - /// Path to text timezone database. This is only used on Windows MinGW - /// builds where std::chrono timezone support is not available. + /// The path to the timezone database; by default None. + /// This is only used on some Windows builds where + /// std::chrono timezone support is not available. + // ARROW_DEPRECATED("Deprecated in 23.0.0") std::optional<std::string> timezone_db_path; }; diff --git c/cpp/src/arrow/public_api_test.cc i/cpp/src/arrow/public_api_test.cc index 0b66089..25ee485 100644 --- c/cpp/src/arrow/public_api_test.cc +++ i/cpp/src/arrow/public_api_test.cc @@ -122,4 +122,48 @@ TEST(Misc, BuildInfo) { ASSERT_THAT(info.full_so_version, ::testing::HasSubstr(info.so_version)); } +// TODO(apacheGH-48593): Remove when libc++ supports std::chrono timezone +// apache#48593 +TEST(Misc, SetTimezoneConfig) { +#ifndef _WIN32 + GTEST_SKIP() << "Can only set the Timezone database on Windows"; +#elif !defined(ARROW_FILESYSTEM) + GTEST_SKIP() << "Need filesystem support to test timezone config."; +#else + auto fs = std::make_shared<arrow::fs::LocalFileSystem>(); + + std::optional<std::string> tzdata_result = GetTestTimezoneDatabaseRoot(); + std::string tzdata_dir; + if (tzdata_result.has_value()) { + tzdata_dir = tzdata_result.value(); + } else { + auto home_raw = std::getenv("USERPROFILE"); + std::string home = home_raw == nullptr ? "~" : std::string(home_raw); + ASSERT_OK_AND_ASSIGN(tzdata_dir, fs->NormalizePath(home + "\\Downloads\\tzdata")); + } + ASSERT_OK_AND_ASSIGN(tzdata_dir, fs->NormalizePath(tzdata_dir)); + ASSERT_OK_AND_ASSIGN(auto tzdata_path, + arrow::internal::PlatformFilename::FromString(tzdata_dir)); + + if (!arrow::internal::FileExists(tzdata_path).ValueOr(false)) { + GTEST_SKIP() << "Couldn't find timezone database in expected dir: " << tzdata_dir; + } + // Create a tmp directory + ASSERT_OK_AND_ASSIGN(auto tempdir, arrow::internal::TemporaryDir::Make("tzdata")); + + // Validate that setting tzdb to that dir fails + arrow::GlobalOptions options = {std::make_optional(tempdir->path().ToString())}; + ASSERT_NOT_OK(arrow::Initialize(options)); + + // Copy tzdb data from ~/Downloads + auto selector = arrow::fs::FileSelector(); + selector.base_dir = tzdata_dir; + selector.recursive = true; + ASSERT_OK(arrow::fs::CopyFiles(fs, selector, fs, tempdir->path().ToString())); + + // Validate that tzdb is working + ASSERT_OK(arrow::Initialize(options)); +#endif +} + } // namespace arrow diff --git c/cpp/src/arrow/testing/util.cc i/cpp/src/arrow/testing/util.cc index 8846347..204d61f 100644 --- c/cpp/src/arrow/testing/util.cc +++ i/cpp/src/arrow/testing/util.cc @@ -122,6 +122,29 @@ Status GetTestResourceRoot(std::string* out) { return Status::OK(); } +// TODO(apacheGH-48593): Remove when libc++ supports std::chrono timezone +// apache#48593 +std::optional<std::string> GetTestTimezoneDatabaseRoot() { + const char* c_root = std::getenv("ARROW_TIMEZONE_DATABASE"); + if (!c_root) { + return std::optional<std::string>(); + } + return std::make_optional(std::string(c_root)); +} + +// TODO(apacheGH-48593): Remove when libc++ supports std::chrono timezone +// apache#48593 +Status InitTestTimezoneDatabase() { + auto maybe_tzdata = GetTestTimezoneDatabaseRoot(); + // If missing, timezone database will default to %USERPROFILE%\Downloads\tzdata + if (!maybe_tzdata.has_value()) return Status::OK(); + + auto tzdata_path = std::string(maybe_tzdata.value()); + arrow::GlobalOptions options = {std::make_optional(tzdata_path)}; + ARROW_RETURN_NOT_OK(arrow::Initialize(options)); + return Status::OK(); +} + int GetListenPort() { // Get a new available port number by binding a socket to an ephemeral port // and then closing it. Since ephemeral port allocation tends to avoid diff --git c/cpp/src/arrow/testing/util.h i/cpp/src/arrow/testing/util.h index 98b1bdb..3304889 100644 --- c/cpp/src/arrow/testing/util.h +++ i/cpp/src/arrow/testing/util.h @@ -112,6 +112,17 @@ UnionTypeFactories() { // Status ARROW_TESTING_EXPORT Status GetTestResourceRoot(std::string*); +// TODO(apacheGH-48593): Remove when libc++ supports std::chrono timezone +// apache#48593 +// Return the value of the ARROW_TIMEZONE_DATABASE environment variable +ARROW_TESTING_EXPORT std::optional<std::string> GetTestTimezoneDatabaseRoot(); + +// TODO(apacheGH-48593): Remove when libc++ supports std::chrono timezone +// apache#48593 +// Set the Timezone database based on the ARROW_TIMEZONE_DATABASE env variable +// This is only relevant on Windows, since other OSs have compatible databases built-in +ARROW_TESTING_EXPORT Status InitTestTimezoneDatabase(); + // Get a TCP port number to listen on. This is a different number every time, // as reusing the same port across tests can produce spurious bind errors on // Windows. diff --git c/docs/source/cpp/build_system.rst i/docs/source/cpp/build_system.rst index b124060..d3c47ce 100644 --- c/docs/source/cpp/build_system.rst +++ i/docs/source/cpp/build_system.rst @@ -228,3 +228,29 @@ can control the source of each dependency and whether it is statically or dynamically linked. See :doc:`/developers/cpp/building` for instructions. Or alternatively, use Arrow from a package manager such as Conda or vcpkg which will manage consistent versions of Arrow and its dependencies. + + +.. _download-timezone-database: + +Runtime Dependencies +==================== + +.. TODO(apacheGH-48593): Remove when libc++ supports std::chrono timezone + apache#48593 + +While Arrow uses the OS-provided timezone database on Linux and macOS, it +requires a user-provided database on Windows. You must download and extract the +text version of the IANA timezone database and add the Windows timezone mapping +XML. To download, you can use the following batch script: + +.. literalinclude:: ../../../ci/appveyor-cpp-setup.bat + :language: batch + :start-after: @Rem (Doc section: Download timezone database) + :end-before: @Rem (Doc section: Download timezone database) + +By default, the timezone database will be detected at ``%USERPROFILE%\Downloads\tzdata``, +but you can set a custom path at runtime in :struct:`arrow::ArrowGlobalOptions`:: + + arrow::GlobalOptions options; + options.timezone_db_path = "path/to/tzdata"; + ARROW_RETURN_NOT_OK(arrow::Initialize(options)); diff --git c/docs/source/developers/cpp/windows.rst i/docs/source/developers/cpp/windows.rst index b4d8f19..a28ff07 100644 --- c/docs/source/developers/cpp/windows.rst +++ i/docs/source/developers/cpp/windows.rst @@ -381,6 +381,18 @@ be defined, and similarly for ``-DARROW_FLIGHT_SQL=ON``. ARROW_FLIGHT_STATIC ARROW_FLIGHT_SQL_STATIC) +Downloading the Timezone Database +================================= + +.. TODO(apacheGH-48593): Remove when libc++ supports std::chrono timezone + apache#48593 + +To run some of the compute unit tests on Windows, the IANA timezone database +and the Windows timezone mapping need to be downloaded first. See +:ref:`download-timezone-database` for download instructions. To set a non-default +path for the timezone database while running the unit tests, set the +``ARROW_TIMEZONE_DATABASE`` environment variable. + Replicating Appveyor Builds =========================== diff --git c/docs/source/python/install.rst i/docs/source/python/install.rst index aaf4304..44d3fbc 100644 --- c/docs/source/python/install.rst +++ i/docs/source/python/install.rst @@ -80,6 +80,42 @@ Optional dependencies Additional packages PyArrow is compatible with are :ref:`fsspec <filesystem-fsspec>` and **pytz**, **dateutil** or **tzdata** package for timezones. +tzdata on Windows +^^^^^^^^^^^^^^^^^ + +.. TODO(apacheGH-48593): Remove when libc++ supports std::chrono timezone + apache#48593 + +While Arrow uses the OS-provided timezone database on Linux and macOS, it requires a +user-provided database on Windows. To download and extract the text version of +the IANA timezone database follow the instructions in the C++ +:ref:`download-timezone-database` or use pyarrow utility function +``pyarrow.util.download_tzdata_on_windows()`` that does the same. + +By default, the timezone database will be detected at ``%USERPROFILE%\Downloads\tzdata``. +If the database has been downloaded in a different location, you will need to set +a custom path to the database from Python: + +.. code-block:: python + + >>> import pyarrow as pa + >>> pa.set_timezone_db_path("custom_path") + +You may encounter problems writing datetime data to an ORC file if you install +pyarrow with pip. One possible solution to fix this problem: + + 1. Install tzdata with ``pip install tzdata`` + 2. Set the environment variable ``TZDIR = path\to\.venv\Lib\site-packages\tzdata\`` + +You can find where ``tzdata`` is installed with the following python +command: + +.. code-block:: python + + >>> import tzdata + >>> print(tzdata.__file__) + path\to\.venv\Lib\site-packages\tzdata\__init__.py + .. _python-conda-differences: diff --git c/python/pyarrow/__init__.py i/python/pyarrow/__init__.py index d031812..18a40d8 100644 --- c/python/pyarrow/__init__.py +++ i/python/pyarrow/__init__.py @@ -56,7 +56,7 @@ except ImportError: except ImportError: __version__ = None -from pyarrow.lib import (BuildInfo, CppBuildInfo, RuntimeInfo, +from pyarrow.lib import (BuildInfo, CppBuildInfo, RuntimeInfo, set_timezone_db_path, MonthDayNano, VersionInfo, build_info, cpp_build_info, cpp_version, cpp_version_info, runtime_info, cpu_count, set_cpu_count, enable_signal_handlers, diff --git c/python/pyarrow/config.pxi i/python/pyarrow/config.pxi index 4fdaaf0..c8ecf20 100644 --- c/python/pyarrow/config.pxi +++ i/python/pyarrow/config.pxi @@ -96,3 +96,23 @@ build_info = _build_info() cpp_build_info = build_info.cpp_build_info cpp_version = build_info.cpp_build_info.version cpp_version_info = build_info.cpp_build_info.version_info + + +# TODO(apacheGH-48593): Remove when libc++ supports std::chrono timezone +# apache#48593 +def set_timezone_db_path(path): + """ + Configure the path to text timezone database on Windows. + + Parameters + ---------- + path : str + Path to text timezone database. + """ + cdef: + CGlobalOptions options + + if path is not None: + options.timezone_db_path = <c_string>tobytes(path) + + check_status(Initialize(options)) diff --git c/python/pyarrow/includes/libarrow.pxd i/python/pyarrow/includes/libarrow.pxd index 897ead1..617a4b7 100644 --- c/python/pyarrow/includes/libarrow.pxd +++ i/python/pyarrow/includes/libarrow.pxd @@ -90,6 +90,13 @@ cdef extern from "arrow/config.h" namespace "arrow" nogil: CRuntimeInfo GetRuntimeInfo() + # TODO(apacheGH-48593): Remove when libc++ supports std::chrono timezone + # apache#48593 + cdef cppclass CGlobalOptions" arrow::GlobalOptions": + optional[c_string] timezone_db_path + + CStatus Initialize(const CGlobalOptions& options) + cdef extern from "arrow/util/future.h" namespace "arrow" nogil: cdef cppclass CFuture_Void" arrow::Future<>": diff --git c/python/pyarrow/tests/strategies.py i/python/pyarrow/tests/strategies.py index f18d58b..3c31650 100644 --- c/python/pyarrow/tests/strategies.py +++ i/python/pyarrow/tests/strategies.py @@ -327,7 +327,7 @@ def arrays(draw, type, size=None, nullable=True): value = st.dates() elif pa.types.is_timestamp(ty): if zoneinfo is None: - pytest.skip('no module named zoneinfo') + pytest.skip('no module named zoneinfo (or tzdata on Windows)') if ty.tz is None: pytest.skip('requires timezone not None') min_int64 = -(2**63) diff --git c/python/pyarrow/tests/test_compute.py i/python/pyarrow/tests/test_compute.py index e585810..3321f14 100644 --- c/python/pyarrow/tests/test_compute.py +++ i/python/pyarrow/tests/test_compute.py @@ -203,14 +203,14 @@ def test_option_class_equality(request): first_week_is_fully_in_year=False), pc.ZeroFillOptions(4, "0"), ] - # Timezone database might not be installed on Emscripten + # Timezone database might not be installed on Windows or Emscripten if request.config.pyarrow.is_enabled["timezone_data"]: options.append(pc.AssumeTimezoneOptions("Europe/Ljubljana")) classes = {type(option) for option in options} for cls in exported_option_classes: - # Timezone database might not be installed on Emscripten + # Timezone database might not be installed on Windows or Emscripten if ( cls not in classes and (request.config.pyarrow.is_enabled["timezone_data"]) @@ -2381,6 +2381,7 @@ def test_strftime(): for fmt in formats: options = pc.StrftimeOptions(fmt) result = pc.strftime(tsa, options=options) + # cast to the same type as result to ignore string vs large_string expected = pa.array(ts.strftime(fmt)).cast(result.type) if sys.platform == "win32" and fmt == "%Z": # TODO(apacheGH-48743): On Windows, std::chrono returns GMT @@ -2564,7 +2565,7 @@ def test_extract_datetime_components(request): # Test timezone aware timestamp array if not request.config.pyarrow.is_enabled["timezone_data"]: - pytest.skip('Timezone database is not available') + pytest.skip('Timezone database is not installed on Windows') else: for timezone in timezones: _check_datetime_components(timestamps, timezone) diff --git c/python/pyarrow/tests/test_misc.py i/python/pyarrow/tests/test_misc.py index fb73d65..af01723 100644 --- c/python/pyarrow/tests/test_misc.py +++ i/python/pyarrow/tests/test_misc.py @@ -22,6 +22,7 @@ import sys import pytest import pyarrow as pa +from pyarrow.lib import ArrowInvalid def test_get_include(): @@ -137,6 +138,19 @@ def test_import_at_shutdown(): subprocess.check_call([sys.executable, "-c", code]) +# TODO(apacheGH-48593): Remove when libc++ supports std::chrono timezone +# apache#48593 [email protected](sys.platform == "win32", + reason="Path to timezone database is not configurable " + "on non-Windows platforms") +def test_set_timezone_db_path_non_windows(): + # set_timezone_db_path raises an error on non-Windows platforms + with pytest.raises(ArrowInvalid, + match="Arrow was set to use OS timezone " + "database at compile time"): + pa.set_timezone_db_path("path") + + @pytest.mark.parametrize('klass', [ pa.Field, pa.Schema, diff --git c/python/pyarrow/tests/test_util.py i/python/pyarrow/tests/test_util.py index 9fccb76..31cb740 100644 --- c/python/pyarrow/tests/test_util.py +++ i/python/pyarrow/tests/test_util.py @@ -16,14 +16,17 @@ # under the License. import gc +import os import signal +import shutil import sys import textwrap import weakref import pytest -from pyarrow.util import doc, _break_traceback_cycle_from_frame +from pyarrow.util import (doc, _break_traceback_cycle_from_frame, + download_tzdata_on_windows) from pyarrow.tests.util import disabled_gc @@ -207,3 +210,22 @@ def test_signal_refcycle(): assert wr() is not None _break_traceback_cycle_from_frame(sys._getframe(0)) assert wr() is None + + +# TODO(apacheGH-48593): Remove when libc++ supports std::chrono timezone +# apache#48593 [email protected](sys.platform != "win32", + reason="Timezone database is already provided.") +def test_download_tzdata_on_windows(): + tzdata_path = os.path.expandvars(r"%USERPROFILE%\Downloads\tzdata") + + # Download timezone database and remove data in case it already exists + if (os.path.exists(tzdata_path)): + shutil.rmtree(tzdata_path) + download_tzdata_on_windows() + + # Inspect the folder + assert os.path.exists(tzdata_path) + assert os.path.exists(os.path.join(tzdata_path, "windowsZones.xml")) + assert os.path.exists(os.path.join(tzdata_path, "europe")) + assert 'version' in os.listdir(tzdata_path) diff --git c/python/pyarrow/util.py i/python/pyarrow/util.py index a95826e..6d35e58 100644 --- c/python/pyarrow/util.py +++ i/python/pyarrow/util.py @@ -244,7 +244,8 @@ def _download_requests(url, out_path): with open(out_path, 'wb') as f: f.write(response.content) - +# TODO(apacheGH-48593): Remove when libc++ supports std::chrono timezone +# apache#48593 def download_tzdata_on_windows(): r""" Download and extract latest IANA timezone database into the
1 parent 2c3fd96 commit e92c792

16 files changed

Lines changed: 255 additions & 10 deletions

File tree

‎cpp/src/arrow/config.cc‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,10 @@ std::string MakeSimdLevelString(QueryFlagFunction&& query_flag) {
6464
}
6565
}
6666

67+
// TODO(GH-48593): Remove when libc++ supports std::chrono timezone
68+
// https://github.com/apache/arrow/issues/48593
69+
std::optional<std::string> timezone_db_path;
70+
6771
}; // namespace
6872

6973
const BuildInfo& GetBuildInfo() { return kBuildInfo; }
@@ -75,6 +79,12 @@ RuntimeInfo GetRuntimeInfo() {
7579
MakeSimdLevelString([&](int64_t flags) { return cpu_info->IsSupported(flags); });
7680
info.detected_simd_level =
7781
MakeSimdLevelString([&](int64_t flags) { return cpu_info->IsDetected(flags); });
82+
info.using_os_timezone_db = USE_OS_TZDB;
83+
#if !USE_OS_TZDB
84+
info.timezone_db_path = timezone_db_path;
85+
#else
86+
info.timezone_db_path = std::optional<std::string>();
87+
#endif
7888
return info;
7989
}
8090

@@ -89,7 +99,12 @@ Status Initialize(const GlobalOptions& options) noexcept {
8999
} catch (const std::runtime_error& e) {
90100
return Status::IOError(e.what());
91101
}
92-
#endif
102+
timezone_db_path = options.timezone_db_path.value();
103+
#else
104+
return Status::Invalid(
105+
"Arrow was set to use OS timezone database at compile time, "
106+
"so a downloaded database cannot be provided at runtime.");
107+
#endif // !USE_OS_TZDB
93108
}
94109
return Status::OK();
95110
}

‎cpp/src/arrow/config.h‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,17 @@ struct RuntimeInfo {
6464

6565
/// The SIMD level available on the OS and CPU
6666
std::string detected_simd_level;
67+
68+
/// Whether using the OS-based timezone database
69+
/// This is set at compile-time.
70+
// ARROW_DEPRECATED("Deprecated in 23.0.0")
71+
bool using_os_timezone_db;
72+
73+
/// The path to the timezone database; by default None.
74+
/// This is only used on some Windows builds where
75+
/// std::chrono timezone support is not available.
76+
// ARROW_DEPRECATED("Deprecated in 23.0.0")
77+
std::optional<std::string> timezone_db_path;
6778
};
6879

6980
/// \brief Get runtime build info.
@@ -82,8 +93,10 @@ RuntimeInfo GetRuntimeInfo();
8293
// TODO(GH-48743): Remove when RTools upgrades to GCC with std::chrono timezone support
8394
// https://github.com/apache/arrow/issues/48743
8495
struct GlobalOptions {
85-
/// Path to text timezone database. This is only used on Windows MinGW
86-
/// builds where std::chrono timezone support is not available.
96+
/// The path to the timezone database; by default None.
97+
/// This is only used on some Windows builds where
98+
/// std::chrono timezone support is not available.
99+
// ARROW_DEPRECATED("Deprecated in 23.0.0")
87100
std::optional<std::string> timezone_db_path;
88101
};
89102

‎cpp/src/arrow/public_api_test.cc‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,4 +122,48 @@ TEST(Misc, BuildInfo) {
122122
ASSERT_THAT(info.full_so_version, ::testing::HasSubstr(info.so_version));
123123
}
124124

125+
// TODO(GH-48593): Remove when libc++ supports std::chrono timezone
126+
// https://github.com/apache/arrow/issues/48593
127+
TEST(Misc, SetTimezoneConfig) {
128+
#ifndef _WIN32
129+
GTEST_SKIP() << "Can only set the Timezone database on Windows";
130+
#elif !defined(ARROW_FILESYSTEM)
131+
GTEST_SKIP() << "Need filesystem support to test timezone config.";
132+
#else
133+
auto fs = std::make_shared<arrow::fs::LocalFileSystem>();
134+
135+
std::optional<std::string> tzdata_result = GetTestTimezoneDatabaseRoot();
136+
std::string tzdata_dir;
137+
if (tzdata_result.has_value()) {
138+
tzdata_dir = tzdata_result.value();
139+
} else {
140+
auto home_raw = std::getenv("USERPROFILE");
141+
std::string home = home_raw == nullptr ? "~" : std::string(home_raw);
142+
ASSERT_OK_AND_ASSIGN(tzdata_dir, fs->NormalizePath(home + "\\Downloads\\tzdata"));
143+
}
144+
ASSERT_OK_AND_ASSIGN(tzdata_dir, fs->NormalizePath(tzdata_dir));
145+
ASSERT_OK_AND_ASSIGN(auto tzdata_path,
146+
arrow::internal::PlatformFilename::FromString(tzdata_dir));
147+
148+
if (!arrow::internal::FileExists(tzdata_path).ValueOr(false)) {
149+
GTEST_SKIP() << "Couldn't find timezone database in expected dir: " << tzdata_dir;
150+
}
151+
// Create a tmp directory
152+
ASSERT_OK_AND_ASSIGN(auto tempdir, arrow::internal::TemporaryDir::Make("tzdata"));
153+
154+
// Validate that setting tzdb to that dir fails
155+
arrow::GlobalOptions options = {std::make_optional(tempdir->path().ToString())};
156+
ASSERT_NOT_OK(arrow::Initialize(options));
157+
158+
// Copy tzdb data from ~/Downloads
159+
auto selector = arrow::fs::FileSelector();
160+
selector.base_dir = tzdata_dir;
161+
selector.recursive = true;
162+
ASSERT_OK(arrow::fs::CopyFiles(fs, selector, fs, tempdir->path().ToString()));
163+
164+
// Validate that tzdb is working
165+
ASSERT_OK(arrow::Initialize(options));
166+
#endif
167+
}
168+
125169
} // namespace arrow

‎cpp/src/arrow/testing/util.cc‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,29 @@ Status GetTestResourceRoot(std::string* out) {
122122
return Status::OK();
123123
}
124124

125+
// TODO(GH-48593): Remove when libc++ supports std::chrono timezone
126+
// https://github.com/apache/arrow/issues/48593
127+
std::optional<std::string> GetTestTimezoneDatabaseRoot() {
128+
const char* c_root = std::getenv("ARROW_TIMEZONE_DATABASE");
129+
if (!c_root) {
130+
return std::optional<std::string>();
131+
}
132+
return std::make_optional(std::string(c_root));
133+
}
134+
135+
// TODO(GH-48593): Remove when libc++ supports std::chrono timezone
136+
// https://github.com/apache/arrow/issues/48593
137+
Status InitTestTimezoneDatabase() {
138+
auto maybe_tzdata = GetTestTimezoneDatabaseRoot();
139+
// If missing, timezone database will default to %USERPROFILE%\Downloads\tzdata
140+
if (!maybe_tzdata.has_value()) return Status::OK();
141+
142+
auto tzdata_path = std::string(maybe_tzdata.value());
143+
arrow::GlobalOptions options = {std::make_optional(tzdata_path)};
144+
ARROW_RETURN_NOT_OK(arrow::Initialize(options));
145+
return Status::OK();
146+
}
147+
125148
int GetListenPort() {
126149
// Get a new available port number by binding a socket to an ephemeral port
127150
// and then closing it. Since ephemeral port allocation tends to avoid

‎cpp/src/arrow/testing/util.h‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,17 @@ UnionTypeFactories() {
112112
// Status
113113
ARROW_TESTING_EXPORT Status GetTestResourceRoot(std::string*);
114114

115+
// TODO(GH-48593): Remove when libc++ supports std::chrono timezone
116+
// https://github.com/apache/arrow/issues/48593
117+
// Return the value of the ARROW_TIMEZONE_DATABASE environment variable
118+
ARROW_TESTING_EXPORT std::optional<std::string> GetTestTimezoneDatabaseRoot();
119+
120+
// TODO(GH-48593): Remove when libc++ supports std::chrono timezone
121+
// https://github.com/apache/arrow/issues/48593
122+
// Set the Timezone database based on the ARROW_TIMEZONE_DATABASE env variable
123+
// This is only relevant on Windows, since other OSs have compatible databases built-in
124+
ARROW_TESTING_EXPORT Status InitTestTimezoneDatabase();
125+
115126
// Get a TCP port number to listen on. This is a different number every time,
116127
// as reusing the same port across tests can produce spurious bind errors on
117128
// Windows.

‎docs/source/cpp/build_system.rst‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -228,3 +228,29 @@ can control the source of each dependency and whether it is statically or
228228
dynamically linked. See :doc:`/developers/cpp/building` for instructions. Or
229229
alternatively, use Arrow from a package manager such as Conda or vcpkg which
230230
will manage consistent versions of Arrow and its dependencies.
231+
232+
233+
.. _download-timezone-database:
234+
235+
Runtime Dependencies
236+
====================
237+
238+
.. TODO(GH-48593): Remove when libc++ supports std::chrono timezone
239+
https://github.com/apache/arrow/issues/48593
240+
241+
While Arrow uses the OS-provided timezone database on Linux and macOS, it
242+
requires a user-provided database on Windows. You must download and extract the
243+
text version of the IANA timezone database and add the Windows timezone mapping
244+
XML. To download, you can use the following batch script:
245+
246+
.. literalinclude:: ../../../ci/appveyor-cpp-setup.bat
247+
:language: batch
248+
:start-after: @rem (Doc section: Download timezone database)
249+
:end-before: @rem (Doc section: Download timezone database)
250+
251+
By default, the timezone database will be detected at ``%USERPROFILE%\Downloads\tzdata``,
252+
but you can set a custom path at runtime in :struct:`arrow::ArrowGlobalOptions`::
253+
254+
arrow::GlobalOptions options;
255+
options.timezone_db_path = "path/to/tzdata";
256+
ARROW_RETURN_NOT_OK(arrow::Initialize(options));

‎docs/source/developers/cpp/windows.rst‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -381,6 +381,18 @@ be defined, and similarly for ``-DARROW_FLIGHT_SQL=ON``.
381381
ARROW_FLIGHT_STATIC
382382
ARROW_FLIGHT_SQL_STATIC)
383383
384+
Downloading the Timezone Database
385+
=================================
386+
387+
.. TODO(GH-48593): Remove when libc++ supports std::chrono timezone
388+
https://github.com/apache/arrow/issues/48593
389+
390+
To run some of the compute unit tests on Windows, the IANA timezone database
391+
and the Windows timezone mapping need to be downloaded first. See
392+
:ref:`download-timezone-database` for download instructions. To set a non-default
393+
path for the timezone database while running the unit tests, set the
394+
``ARROW_TIMEZONE_DATABASE`` environment variable.
395+
384396
Replicating Appveyor Builds
385397
===========================
386398

‎docs/source/python/install.rst‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,42 @@ Optional dependencies
8080
Additional packages PyArrow is compatible with are :ref:`fsspec <filesystem-fsspec>`
8181
and **pytz**, **dateutil** or **tzdata** package for timezones.
8282

83+
tzdata on Windows
84+
^^^^^^^^^^^^^^^^^
85+
86+
.. TODO(GH-48593): Remove when libc++ supports std::chrono timezone
87+
https://github.com/apache/arrow/issues/48593
88+
89+
While Arrow uses the OS-provided timezone database on Linux and macOS, it requires a
90+
user-provided database on Windows. To download and extract the text version of
91+
the IANA timezone database follow the instructions in the C++
92+
:ref:`download-timezone-database` or use pyarrow utility function
93+
``pyarrow.util.download_tzdata_on_windows()`` that does the same.
94+
95+
By default, the timezone database will be detected at ``%USERPROFILE%\Downloads\tzdata``.
96+
If the database has been downloaded in a different location, you will need to set
97+
a custom path to the database from Python:
98+
99+
.. code-block:: python
100+
101+
>>> import pyarrow as pa
102+
>>> pa.set_timezone_db_path("custom_path")
103+
104+
You may encounter problems writing datetime data to an ORC file if you install
105+
pyarrow with pip. One possible solution to fix this problem:
106+
107+
1. Install tzdata with ``pip install tzdata``
108+
2. Set the environment variable ``TZDIR = path\to\.venv\Lib\site-packages\tzdata\``
109+
110+
You can find where ``tzdata`` is installed with the following python
111+
command:
112+
113+
.. code-block:: python
114+
115+
>>> import tzdata
116+
>>> print(tzdata.__file__)
117+
path\to\.venv\Lib\site-packages\tzdata\__init__.py
118+
83119
84120
.. _python-conda-differences:
85121

‎python/pyarrow/__init__.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ def parse_git(root, **kwargs):
5656
except ImportError:
5757
__version__ = None
5858

59-
from pyarrow.lib import (BuildInfo, CppBuildInfo, RuntimeInfo,
59+
from pyarrow.lib import (BuildInfo, CppBuildInfo, RuntimeInfo, set_timezone_db_path,
6060
MonthDayNano, VersionInfo, build_info, cpp_build_info,
6161
cpp_version, cpp_version_info, runtime_info,
6262
cpu_count, set_cpu_count, enable_signal_handlers,

‎python/pyarrow/config.pxi‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,3 +96,23 @@ build_info = _build_info()
9696
cpp_build_info = build_info.cpp_build_info
9797
cpp_version = build_info.cpp_build_info.version
9898
cpp_version_info = build_info.cpp_build_info.version_info
99+
100+
101+
# TODO(GH-48593): Remove when libc++ supports std::chrono timezone
102+
# https://github.com/apache/arrow/issues/48593
103+
def set_timezone_db_path(path):
104+
"""
105+
Configure the path to text timezone database on Windows.
106+
107+
Parameters
108+
----------
109+
path : str
110+
Path to text timezone database.
111+
"""
112+
cdef:
113+
CGlobalOptions options
114+
115+
if path is not None:
116+
options.timezone_db_path = <c_string>tobytes(path)
117+
118+
check_status(Initialize(options))

0 commit comments

Comments
 (0)