Skip to content

Commit 9f0736e

Browse files
Afloat16ahojnnes
andauthored
Restore the Flickr image downloader on Python 3 (#4977)
The dataset downloader imports `urllib2` and `urlparse`, so loading the script on Python 3 raises `ModuleNotFoundError` before the CLI can parse any options. Use the standard `urllib.request` and `urllib.parse` modules and close the HTTP response after parsing the XML page. The regression test reads a Flickr-shaped XML response and checks the image URL and local filename passed to the retriever, including a URL with a query string. Network access is mocked. Verified `python python/util/flickr_downloader.py --help` and `pytest python/util/flickr_downloader_test.py` on Python 3.12. Ruff lint and format checks pass. --------- Co-authored-by: Johannes Schönberger <[email protected]>
1 parent b5ff5c3 commit 9f0736e

2 files changed

Lines changed: 72 additions & 13 deletions

File tree

‎python/util/flickr_downloader.py‎

Lines changed: 6 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -6,13 +6,11 @@
66
import os
77
import time
88
import urllib.error
9+
import urllib.parse
910
import urllib.request
1011
import xml.etree.ElementTree as ElementTree
1112
from typing import Final
1213

13-
import urllib2
14-
import urlparse
15-
1614
PER_PAGE: Final[int] = 500
1715
SORT: Final[str] = "date-posted-desc"
1816
URL: Final[str] = (
@@ -56,29 +54,26 @@ def compose_url(
5654
def parse_page(
5755
page: int, api_key: str, text: str, min_date: float, max_date: float
5856
) -> tuple[dict[str, str], tuple[dict[str, str], ...]]:
59-
f: urllib2.urlopen | None = None
6057
for _ in range(MAX_PAGE_REQUESTS):
6158
try:
62-
f = urllib2.urlopen(
59+
with urllib.request.urlopen(
6360
compose_url(page, api_key, text, min_date, max_date),
6461
timeout=MAX_PAGE_TIMEOUT,
65-
)
62+
) as response:
63+
root = ElementTree.parse(response).getroot()
6664
except TimeoutError:
6765
continue
6866
else:
6967
break
7068

71-
if f is None:
69+
else:
7270
return {
7371
"pages": "0",
7472
"total": "0",
7573
"page": "0",
7674
"perpage": "0",
7775
}, tuple()
7876

79-
response = f.read()
80-
root = ElementTree.fromstring(response)
81-
8277
if root.attrib["stat"] != "ok":
8378
raise OSError
8479

@@ -106,9 +101,7 @@ def __call__(self, photo: dict[str, str]) -> None:
106101
break
107102

108103
if url is not None:
109-
# Note that the following statement may fail in Python 3. urlparse
110-
# may need to be replaced with urllib.parse.
111-
url_filename = urlparse.urlparse(url).path
104+
url_filename = urllib.parse.urlparse(url).path
112105
image_ext = os.path.splitext(url_filename)[1]
113106

114107
image_name = f"{photo['id']}_{photo['secret']}{image_ext}"
Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
# SPDX-License-Identifier: BSD-3-Clause
2+
3+
import io
4+
import runpy
5+
import urllib.request
6+
from pathlib import Path
7+
from unittest.mock import patch
8+
9+
10+
def _photo_page_response(url: str) -> io.BytesIO:
11+
return io.BytesIO(
12+
f'<rsp stat="ok"><photos pages="1"><photo id="12" secret="abc" '
13+
f'url_l="{url}"/></photos></rsp>'.encode()
14+
)
15+
16+
17+
def test_download_photo_from_page(tmp_path: Path) -> None:
18+
downloader = runpy.run_path(
19+
str(Path(__file__).with_name("flickr_downloader.py"))
20+
)
21+
url = "https://example.org/image.jpg?size=large"
22+
response = _photo_page_response(url)
23+
with patch.object(urllib.request, "urlopen", return_value=response):
24+
metadata, photos = downloader["parse_page"](1, "key", "scene", 0, 1)
25+
assert metadata["pages"] == "1"
26+
assert len(photos) == 1
27+
assert response.closed
28+
with patch.object(urllib.request, "urlretrieve") as retrieve:
29+
downloader["PhotoDownloader"](str(tmp_path))(photos[0])
30+
retrieve.assert_called_once_with(url, str(tmp_path / "12_abc.jpg"))
31+
32+
33+
def test_parse_page_retries_after_timeout() -> None:
34+
downloader = runpy.run_path(
35+
str(Path(__file__).with_name("flickr_downloader.py"))
36+
)
37+
url = "https://example.org/image.jpg?size=large"
38+
response = _photo_page_response(url)
39+
with patch.object(
40+
urllib.request,
41+
"urlopen",
42+
side_effect=[TimeoutError, response],
43+
) as urlopen_mock:
44+
metadata, photos = downloader["parse_page"](1, "key", "scene", 0, 1)
45+
assert metadata["pages"] == "1"
46+
assert len(photos) == 1
47+
assert response.closed
48+
assert urlopen_mock.call_count == 2
49+
50+
51+
def test_parse_page_returns_empty_after_repeated_timeouts() -> None:
52+
downloader = runpy.run_path(
53+
str(Path(__file__).with_name("flickr_downloader.py"))
54+
)
55+
with patch.object(
56+
urllib.request, "urlopen", side_effect=TimeoutError
57+
) as urlopen_mock:
58+
metadata, photos = downloader["parse_page"](1, "key", "scene", 0, 1)
59+
assert metadata == {
60+
"pages": "0",
61+
"total": "0",
62+
"page": "0",
63+
"perpage": "0",
64+
}
65+
assert photos == ()
66+
assert urlopen_mock.call_count == downloader["MAX_PAGE_REQUESTS"]

0 commit comments

Comments
 (0)