Skip to content

Commit bf8c55e

Browse files
cospectrumcospectrum
authored
api fixes (#19)
* check not found from gitlab * test_bucket_listv2_encoding_basic * test_bucket_list_encoding_basic * specify safe chars for url escape * next marker
1 parent 5bcca44 commit bf8c55e

14 files changed

Lines changed: 151 additions & 62 deletions

File tree

‎README.md‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -78,10 +78,11 @@ class MyCustomStore(ObjectStore):
7878
self,
7979
bucket_name: BucketName,
8080
*,
81-
prefix: Key | None = None,
81+
prefix: str | None = None,
8282
delimiter: str | None = None,
8383
max_keys: MaxKeys = 1000,
84-
marker: Key | None = None,
84+
marker: str | None = None,
85+
encoding_type: str | None = None,
8586
) -> ListObjectsInfo: ...
8687
async def list_objects_v2(
8788
self,
@@ -91,7 +92,7 @@ class MyCustomStore(ObjectStore):
9192
delimiter: str | None = None,
9293
encoding_type: str | None = None,
9394
max_keys: MaxKeys = 1000,
94-
prefix: Key | None = None,
95+
prefix: str | None = None,
9596
start_after: Key | None = None,
9697
) -> ListObjectsV2Info: ...
9798
```
@@ -149,3 +150,4 @@ uv run mypy .
149150
## License
150151

151152
Apache 2.0 – see the [LICENSE](./LICENSE) file for details.
153+

‎examples/custom_store.pyi‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,10 +25,11 @@ class MyCustomStore(ObjectStore):
2525
self,
2626
bucket_name: BucketName,
2727
*,
28-
prefix: Key | None = None,
28+
prefix: str | None = None,
2929
delimiter: str | None = None,
3030
max_keys: MaxKeys = 1000,
31-
marker: Key | None = None,
31+
marker: str | None = None,
32+
encoding_type: str | None = None,
3233
) -> ListObjectsInfo: ...
3334
async def list_objects_v2(
3435
self,
@@ -38,6 +39,6 @@ class MyCustomStore(ObjectStore):
3839
delimiter: str | None = None,
3940
encoding_type: str | None = None,
4041
max_keys: MaxKeys = 1000,
41-
prefix: Key | None = None,
42+
prefix: str | None = None,
4243
start_after: Key | None = None,
4344
) -> ListObjectsV2Info: ...

‎src/boxdrive/handlers.py‎

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
import logging
44
from typing import Annotated, Literal
55

6-
from fastapi import APIRouter, Depends, Header, Query, Request, Response, status
6+
from fastapi import APIRouter, Depends, Header, Query, Request, Response
77
from fastapi.responses import StreamingResponse
88

99
from . import (
@@ -33,18 +33,26 @@ async def list_buckets(s3: S3Dep) -> XMLResponse:
3333
@router.get("/{bucket}")
3434
async def list_objects(
3535
bucket: BucketName,
36-
prefix: Key | None = Query(None),
36+
prefix: str | None = Query(None),
3737
delimiter: str | None = Query(None),
3838
max_keys: MaxKeys = Query(constants.MAX_KEYS, alias="max-keys"),
39-
marker: Key | None = Query(None),
39+
marker: str | None = Query(None),
4040
continuation_token: Key | None = Query(None, alias="continuation-token"),
4141
start_after: Key | None = Query(None, alias="start-after"),
4242
list_type: Literal["1", "2"] = Query("1", alias="list-type"),
43+
encoding_type: Literal["url"] | None = Query(None, alias="encoding-type"),
4344
*,
4445
s3: S3Dep,
4546
) -> XMLResponse:
4647
if list_type == "1":
47-
objects = await s3.list_objects(bucket, prefix=prefix, delimiter=delimiter, max_keys=max_keys, marker=marker)
48+
objects = await s3.list_objects(
49+
bucket,
50+
prefix=prefix,
51+
delimiter=delimiter,
52+
max_keys=max_keys,
53+
marker=marker,
54+
encoding_type=encoding_type,
55+
)
4856
else:
4957
objects = await s3.list_objects_v2(
5058
bucket,
@@ -53,6 +61,7 @@ async def list_objects(
5361
max_keys=max_keys,
5462
continuation_token=continuation_token,
5563
start_after=start_after,
64+
encoding_type=encoding_type,
5665
)
5766
return XMLResponse(objects)
5867

@@ -98,9 +107,8 @@ async def put_object(
98107

99108

100109
@router.delete("/{bucket}/{key:path}")
101-
async def delete_object(bucket: BucketName, key: Key, s3: S3Dep) -> XMLResponse:
102-
await s3.delete_object(bucket, key)
103-
return XMLResponse(status_code=status.HTTP_204_NO_CONTENT)
110+
async def delete_object(bucket: BucketName, key: Key, s3: S3Dep) -> Response:
111+
return await s3.delete_object(bucket, key)
104112

105113

106114
@router.put("/{bucket}")

‎src/boxdrive/middleware.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,9 +41,9 @@ async def dispatch(self, request: Request, call_next: RequestResponseEndpoint) -
4141
logger.info(
4242
"Response info: %s",
4343
{
44+
"status_code": status_code,
4445
"method": method,
4546
"path": path,
46-
"status_code": status_code,
4747
"process_time": f"{process_time:.3f}s",
4848
"content_length": content_length,
4949
},

‎src/boxdrive/s3.py‎

Lines changed: 51 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -2,22 +2,25 @@
22
import os
33
from collections.abc import AsyncIterator
44

5-
from fastapi import HTTPException, Response
5+
from fastapi import HTTPException, Response, status
66
from fastapi.responses import StreamingResponse
7+
from opentelemetry import trace
78

89
from boxdrive.schemas import BaseListObjectsInfo
910

1011
from . import constants, exceptions
1112
from .schemas import BucketName, ContentType, Key, MaxKeys, xml
1213
from .store import ObjectStore
1314

15+
tracer = trace.get_tracer(__name__)
1416
logger = logging.getLogger(__name__)
1517

1618

1719
class S3:
1820
def __init__(self, store: ObjectStore):
1921
self.store = store
2022

23+
@tracer.start_as_current_span("list_buckets")
2124
async def list_buckets(self) -> xml.ListAllMyBucketsResult:
2225
buckets = await self.store.list_buckets()
2326
buckets_xml = [
@@ -27,57 +30,66 @@ async def list_buckets(self) -> xml.ListAllMyBucketsResult:
2730
buckets_model = xml.Buckets(buckets=buckets_xml)
2831
return xml.ListAllMyBucketsResult(owner=owner, buckets=buckets_model)
2932

30-
async def list_objects_v2(
33+
@tracer.start_as_current_span("list_objects")
34+
async def list_objects(
3135
self,
3236
bucket: BucketName,
33-
prefix: Key | None = None,
37+
prefix: str | None = None,
3438
delimiter: str | None = None,
3539
max_keys: MaxKeys = constants.MAX_KEYS,
36-
continuation_token: Key | None = None,
37-
start_after: Key | None = None,
40+
marker: Key | None = None,
41+
encoding_type: str | None = None,
3842
) -> xml.ListBucketResult:
39-
objects_info = await self.store.list_objects_v2(
40-
bucket,
41-
prefix=prefix,
42-
delimiter=delimiter,
43-
max_keys=max_keys,
44-
continuation_token=continuation_token,
45-
start_after=start_after,
43+
objects_info = await self.store.list_objects(
44+
bucket, prefix=prefix, delimiter=delimiter, max_keys=max_keys, marker=marker, encoding_type=encoding_type
4645
)
4746
return self._build_list_bucket_result(
4847
bucket,
49-
objects_info,
48+
next_marker=objects_info.next_marker,
49+
objects_info=objects_info,
5050
prefix=prefix,
5151
delimiter=delimiter,
5252
max_keys=max_keys,
5353
)
5454

55-
async def list_objects(
55+
@tracer.start_as_current_span("list_objects_v2")
56+
async def list_objects_v2(
5657
self,
5758
bucket: BucketName,
58-
prefix: Key | None = None,
59+
prefix: str | None = None,
5960
delimiter: str | None = None,
6061
max_keys: MaxKeys = constants.MAX_KEYS,
61-
marker: Key | None = None,
62+
continuation_token: Key | None = None,
63+
start_after: Key | None = None,
64+
encoding_type: str | None = None,
6265
) -> xml.ListBucketResult:
63-
objects_info = await self.store.list_objects(
64-
bucket, prefix=prefix, delimiter=delimiter, max_keys=max_keys, marker=marker
66+
objects_info = await self.store.list_objects_v2(
67+
bucket,
68+
prefix=prefix,
69+
delimiter=delimiter,
70+
max_keys=max_keys,
71+
continuation_token=continuation_token,
72+
start_after=start_after,
73+
encoding_type=encoding_type,
6574
)
6675
return self._build_list_bucket_result(
6776
bucket,
68-
objects_info,
77+
objects_info=objects_info,
6978
prefix=prefix,
7079
delimiter=delimiter,
7180
max_keys=max_keys,
7281
)
7382

83+
# TODO: exclude None NextMarker from response
7484
def _build_list_bucket_result(
7585
self,
7686
bucket: BucketName,
87+
*,
7788
objects_info: BaseListObjectsInfo,
78-
prefix: Key | None = None,
89+
prefix: str | None = None,
7990
delimiter: str | None = None,
8091
max_keys: MaxKeys = constants.MAX_KEYS,
92+
next_marker: str = "",
8193
) -> xml.ListBucketResult:
8294
objects: list[xml.Content] = []
8395
for obj in objects_info.objects:
@@ -99,10 +111,12 @@ def _build_list_bucket_result(
99111
key_count=len(objects) + len(objects_info.common_prefixes),
100112
is_truncated=objects_info.is_truncated,
101113
delimiter=delimiter,
114+
next_marker=next_marker or None,
102115
contents=objects,
103116
common_prefixes=[xml.CommonPrefix(prefix=prefix) for prefix in objects_info.common_prefixes],
104117
)
105118

119+
@tracer.start_as_current_span("get_object")
106120
async def get_object(
107121
self,
108122
bucket: BucketName,
@@ -153,10 +167,11 @@ async def generate() -> AsyncIterator[bytes]:
153167
status_code=status_code,
154168
)
155169

170+
@tracer.start_as_current_span("head_object")
156171
async def head_object(self, bucket: BucketName, key: Key) -> Response:
157172
metadata = await self.store.head_object(bucket, key)
158173
return Response(
159-
status_code=200,
174+
status_code=status.HTTP_200_OK,
160175
headers={
161176
"Content-Length": str(metadata.size),
162177
"ETag": f'"{metadata.etag}"',
@@ -166,6 +181,7 @@ async def head_object(self, bucket: BucketName, key: Key) -> Response:
166181
},
167182
)
168183

184+
@tracer.start_as_current_span("put_object")
169185
async def put_object(
170186
self,
171187
bucket: BucketName,
@@ -175,27 +191,32 @@ async def put_object(
175191
) -> Response:
176192
final_content_type = content_type or constants.DEFAULT_CONTENT_TYPE
177193
result_etag = await self.store.put_object(bucket, key, content, final_content_type)
178-
return Response(status_code=200, headers={"ETag": f'"{result_etag}"', "Content-Length": "0"})
194+
return Response(status_code=status.HTTP_200_OK, headers={"ETag": f'"{result_etag}"'})
179195

180-
async def delete_object(self, bucket: BucketName, key: Key) -> None:
196+
@tracer.start_as_current_span("delete_object")
197+
async def delete_object(self, bucket: BucketName, key: Key) -> Response:
181198
try:
182199
await self.store.delete_object(bucket, key)
183200
except exceptions.NoSuchBucket:
184201
logger.info("Bucket %s not found", bucket)
185202
except exceptions.NoSuchKey:
186203
logger.info("Object %s not found in bucket %s", key, bucket)
187-
return None
204+
return Response(
205+
status_code=status.HTTP_204_NO_CONTENT,
206+
headers={
207+
"content-length": "0",
208+
},
209+
)
188210

211+
@tracer.start_as_current_span("create_bucket")
189212
async def create_bucket(self, bucket: BucketName) -> Response:
190-
try:
191-
await self.store.create_bucket(bucket)
192-
except exceptions.BucketAlreadyExists:
193-
raise HTTPException(status_code=409, detail="Bucket already exists")
194-
return Response(status_code=200, headers={"Location": f"/{bucket}"})
213+
await self.store.create_bucket(bucket)
214+
return Response(status_code=status.HTTP_200_OK, headers={"Location": f"/{bucket}"})
195215

216+
@tracer.start_as_current_span("delete_bucket")
196217
async def delete_bucket(self, bucket: BucketName) -> Response:
197218
try:
198219
await self.store.delete_bucket(bucket)
199220
except exceptions.NoSuchBucket:
200221
logger.info("Bucket %s not found", bucket)
201-
return Response(status_code=204)
222+
return Response(status_code=status.HTTP_204_NO_CONTENT)

‎src/boxdrive/schemas/store.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,7 @@ class BaseListObjectsInfo(BaseModel):
104104

105105

106106
class ListObjectsInfo(BaseListObjectsInfo):
107-
pass
107+
next_marker: str = ""
108108

109109

110110
class ListObjectsV2Info(BaseListObjectsInfo):

‎src/boxdrive/schemas/xml.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,3 +66,4 @@ class ListBucketResult(BaseXmlModel):
6666
delimiter: str | None = element(tag="Delimiter", default=None)
6767
contents: list[Content] = element(tag="Contents")
6868
common_prefixes: list[CommonPrefix] = element(tag="CommonPrefixes")
69+
next_marker: str | None = element(tag="NextMarker", default=None)

‎src/boxdrive/store.py‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -39,10 +39,11 @@ async def list_objects(
3939
self,
4040
bucket_name: BucketName,
4141
*,
42-
prefix: Key | None = None,
42+
prefix: str | None = None,
4343
delimiter: str | None = None,
4444
max_keys: MaxKeys = constants.MAX_KEYS,
45-
marker: Key | None = None,
45+
marker: str | None = None,
46+
encoding_type: str | None = None,
4647
) -> ListObjectsInfo:
4748
"""List objects in a bucket."""
4849
pass
@@ -56,7 +57,7 @@ async def list_objects_v2(
5657
delimiter: str | None = None,
5758
encoding_type: str | None = None,
5859
max_keys: MaxKeys = constants.MAX_KEYS,
59-
prefix: Key | None = None,
60+
prefix: str | None = None,
6061
start_after: Key | None = None,
6162
) -> ListObjectsV2Info:
6263
"""List objects in a bucket."""

0 commit comments

Comments
 (0)