Skip to content

Commit 00e4fae

Browse files
authored
Preserve S3 object keys and close downloaded bodies (#1686)
* Quote Python 3.10 in the CI matrix * Accept typed HTTP headers in authentication validation Flask 3 supplies Werkzeug Headers to the API routes. Include that existing input alongside the dictionary used by gRPC in the validate annotation. Keep the optional header type under TYPE_CHECKING. Verified the six mypy errors on the unchanged base. With this annotation, all 144 source files pass mypy 1.14.1 under both Flask 3.1.3 and Flask 2.0.2. Pylint scores 10.00/10, the existing token-validation tests pass in both environments, and dictionary, Headers and EnvironHeaders checks cover valid, absent, invalid and disabled authentication. * Preserve S3 object keys and close downloaded bodies
1 parent ac0c1bf commit 00e4fae

2 files changed

Lines changed: 60 additions & 5 deletions

File tree

‎deepface/modules/filestore.py‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import hashlib
66
import posixpath
77
from abc import ABC, abstractmethod
8+
from contextlib import closing
89
from typing import Any, List, Optional, cast
910
from urllib.parse import urlparse, unquote, quote
1011

@@ -124,14 +125,17 @@ def __init__(self, uri: str) -> None:
124125
raise ValueError(f"Bucket name is missing in {uri}")
125126

126127
self.bucket = parsed.netloc
127-
self.prefix = parsed.path.strip("/")
128+
self.prefix = self.__key(uri).rstrip("/")
128129
self.client = boto3.client("s3")
129130

130131
def __key(self, path: str) -> str:
131-
parsed = urlparse(path)
132-
if parsed.scheme != "s3" or parsed.netloc != self.bucket:
132+
# S3 identities use raw object keys. A query or fragment delimiter after
133+
# the bucket, or another leading slash, belongs to the key.
134+
scheme, _, location = path.partition("://")
135+
bucket, _, key = location.partition("/")
136+
if scheme.lower() != "s3" or bucket != self.bucket:
133137
raise ValueError(f"{path} is not in bucket {self.bucket}")
134-
return parsed.path.lstrip("/")
138+
return key
135139

136140
def validate(self) -> None:
137141
try:
@@ -171,7 +175,8 @@ def exists(self, path: str) -> bool:
171175

172176
def read_bytes(self, path: str) -> bytes:
173177
obj = self.client.get_object(Bucket=self.bucket, Key=self.__key(path))
174-
return cast(bytes, obj["Body"].read())
178+
with closing(obj["Body"]) as body:
179+
return cast(bytes, body.read())
175180

176181
def write_bytes(self, path: str, data: bytes) -> None:
177182
self.client.put_object(Bucket=self.bucket, Key=self.__key(path), Body=data)

‎tests/unit/test_s3_object_paths.py‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
import io
2+
import sys
3+
from unittest.mock import Mock
4+
5+
import pytest
6+
7+
from deepface.modules.filestore import S3FileStore
8+
9+
10+
@pytest.fixture
11+
def s3(monkeypatch):
12+
client = Mock()
13+
monkeypatch.setitem(sys.modules, "boto3", Mock(client=Mock(return_value=client)))
14+
return client
15+
16+
17+
@pytest.mark.parametrize("key", ["a?b/c#d%20.jpg", "a#b/c?.png", "/gallery/face.jpg"])
18+
def test_listed_s3_keys_round_trip(s3, key):
19+
s3.get_paginator.return_value.paginate.return_value = [{"Contents": [{"Key": key}]}]
20+
s3.get_object.return_value = {"Body": io.BytesIO(b"pixels")}
21+
store = S3FileStore("s3://faces")
22+
23+
(path,) = store.list_images()
24+
assert store.read_bytes(path) == b"pixels"
25+
s3.get_object.assert_called_once_with(Bucket="faces", Key=key)
26+
27+
28+
def test_prefix_and_joined_path_keep_reserved_key_characters(s3):
29+
store = S3FileStore("s3://faces/gallery#1?x")
30+
store.write_bytes(store.join("a%20?#.jpg"), b"data")
31+
s3.put_object.assert_called_once_with(
32+
Bucket="faces", Key="gallery#1?x/a%20?#.jpg", Body=b"data"
33+
)
34+
35+
36+
@pytest.mark.parametrize("fail", [False, True])
37+
def test_response_stream_is_closed_after_read(s3, fail):
38+
body = Mock()
39+
error = OSError("interrupted")
40+
body.read.side_effect = error if fail else None
41+
body.read.return_value = b"pixels"
42+
s3.get_object.return_value = {"Body": body}
43+
store = S3FileStore("s3://faces")
44+
if fail:
45+
with pytest.raises(OSError) as result:
46+
store.read_bytes(store.join("photo.jpg"))
47+
assert result.value is error
48+
else:
49+
assert store.read_bytes(store.join("photo.jpg")) == b"pixels"
50+
body.close.assert_called_once_with()

0 commit comments

Comments
 (0)