Skip to content

Commit

Permalink
feature: Allow to upload at named paths, when uploading at named URLs
Browse files Browse the repository at this point in the history
  • Loading branch information
playpauseandstop committed Mar 12, 2020
1 parent ab7d833 commit 4da922c
Show file tree
Hide file tree
Showing 5 changed files with 93 additions and 44 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.rst
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
1.0.0a1 (In Development)
========================

- Allow to upload on named upload paths, when using named upload URLs
- Ensure named upload URLs (e.g. ``/user/{username}/uploads``) works as well
- Ensure package is typed by adding ``py.typed``

Expand Down
83 changes: 57 additions & 26 deletions aiohttp_tus/data.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
from typing import Any, Callable, Tuple

import attr
from aiohttp import web

from .annotations import DictStrAny

Expand All @@ -19,18 +20,19 @@ class TusConfig:
json_dumps: Callable[[Any], str] = json.dumps
json_loads: Callable[[str], Any] = json.loads

@property
def metadata_path(self) -> Path:
metadata_path = self.upload_path / ".metadata"
def resolve_metadata_path(self, match_info: web.UrlMappingMatchInfo) -> Path:
metadata_path = self.resolve_upload_path(match_info) / ".metadata"
metadata_path.mkdir(mode=self.mkdir_mode, parents=True, exist_ok=True)
return metadata_path

@property
def resources_path(self) -> Path:
resources_path = self.upload_path / ".resources"
def resolve_resources_path(self, match_info: web.UrlMappingMatchInfo) -> Path:
resources_path = self.resolve_upload_path(match_info) / ".resources"
resources_path.mkdir(mode=self.mkdir_mode, parents=True, exist_ok=True)
return resources_path

def resolve_upload_path(self, match_info: web.UrlMappingMatchInfo) -> Path:
return Path(str(self.upload_path.absolute()).format(**match_info))


@attr.dataclass(frozen=True, slots=True)
class Resource:
Expand All @@ -41,24 +43,41 @@ class Resource:

uid: str = attr.Factory(lambda: str(uuid.uuid4()))

def complete(self, *, config: TusConfig) -> Path:
resource_path = get_resource_path(config=config, uid=self.uid)
file_path = get_file_path(config=config, file_name=self.file_name)
def complete(
self, *, config: TusConfig, match_info: web.UrlMappingMatchInfo
) -> Path:
resource_path = get_resource_path(
config=config, match_info=match_info, uid=self.uid
)
file_path = get_file_path(
config=config, match_info=match_info, file_name=self.file_name
)

shutil.move(resource_path, file_path)
self.delete_metadata(config=config)
self.delete_metadata(config=config, match_info=match_info)

return file_path

def delete(self, *, config: TusConfig) -> bool:
return delete_path(get_resource_path(config=config, uid=self.uid))
def delete(self, *, config: TusConfig, match_info: web.UrlMappingMatchInfo) -> bool:
return delete_path(
get_resource_path(config=config, match_info=match_info, uid=self.uid)
)

def delete_metadata(self, *, config: TusConfig) -> int:
return delete_path(get_resource_metadata_path(config=config, uid=self.uid))
def delete_metadata(
self, *, config: TusConfig, match_info: web.UrlMappingMatchInfo
) -> int:
return delete_path(
get_resource_metadata_path(
config=config, match_info=match_info, uid=self.uid
)
)

@classmethod
def from_metadata(cls, *, config: TusConfig, uid: str) -> "Resource":
path = get_resource_metadata_path(config=config, uid=uid)
def from_metadata(
cls, *, config: TusConfig, match_info: web.UrlMappingMatchInfo
) -> "Resource":
uid = match_info["resource_uid"]
path = get_resource_metadata_path(config=config, match_info=match_info, uid=uid)
data = config.json_loads(path.read_text())
return cls(
uid=data["uid"],
Expand All @@ -68,15 +87,21 @@ def from_metadata(cls, *, config: TusConfig, uid: str) -> "Resource":
metadata_header=data["metadata_header"],
)

def save(self, *, config: TusConfig, chunk: bytes) -> Tuple[Path, int]:
path = get_resource_path(config=config, uid=self.uid)
def save(
self, *, config: TusConfig, match_info: web.UrlMappingMatchInfo, chunk: bytes
) -> Tuple[Path, int]:
path = get_resource_path(config=config, match_info=match_info, uid=self.uid)
with open(path, "wb+") as handler:
handler.seek(self.offset)
chunk_size = handler.write(chunk)
return (path, chunk_size)

def save_metadata(self, *, config: TusConfig) -> Tuple[Path, DictStrAny]:
path = get_resource_metadata_path(config=config, uid=self.uid)
def save_metadata(
self, *, config: TusConfig, match_info: web.UrlMappingMatchInfo
) -> Tuple[Path, DictStrAny]:
path = get_resource_metadata_path(
config=config, match_info=match_info, uid=self.uid
)

data = attr.asdict(self)
path.write_text(config.json_dumps(data))
Expand All @@ -91,13 +116,19 @@ def delete_path(path: Path) -> bool:
return False


def get_file_path(*, config: TusConfig, file_name: str) -> Path:
return config.upload_path / file_name
def get_file_path(
*, config: TusConfig, match_info: web.UrlMappingMatchInfo, file_name: str
) -> Path:
return config.resolve_upload_path(match_info) / file_name


def get_resource_path(*, config: TusConfig, uid: str) -> Path:
return config.resources_path / uid
def get_resource_path(
*, config: TusConfig, match_info: web.UrlMappingMatchInfo, uid: str
) -> Path:
return config.resolve_resources_path(match_info) / uid


def get_resource_metadata_path(*, config: TusConfig, uid: str) -> Path:
return config.metadata_path / f"{uid}.json"
def get_resource_metadata_path(
*, config: TusConfig, match_info: web.UrlMappingMatchInfo, uid: str
) -> Path:
return config.resolve_metadata_path(match_info) / f"{uid}.json"
7 changes: 4 additions & 3 deletions aiohttp_tus/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,7 @@

def get_resource(request: web.Request) -> Resource:
return Resource.from_metadata(
config=request.config_dict[APP_TUS_CONFIG_KEY],
uid=request.match_info["resource_uid"],
config=request.config_dict[APP_TUS_CONFIG_KEY], match_info=request.match_info,
)


Expand All @@ -34,7 +33,9 @@ def get_resource_or_410(request: web.Request) -> Resource:
try:
resource = get_resource(request)
if not get_resource_path(
config=request.config_dict[APP_TUS_CONFIG_KEY], uid=resource.uid
config=request.config_dict[APP_TUS_CONFIG_KEY],
match_info=request.match_info,
uid=resource.uid,
).exists():
raise IOError(f"{resource.uid} does not exist")
except IOError:
Expand Down
17 changes: 10 additions & 7 deletions aiohttp_tus/views.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,9 @@ async def delete_resource(request: web.Request) -> web.Response:

# Remove resource file and its metadata
config: TusConfig = request.config_dict[constants.APP_TUS_CONFIG_KEY]
resource.delete(config=config)
resource.delete_metadata(config=config)
match_info = request.match_info
resource.delete(config=config, match_info=match_info)
resource.delete_metadata(config=config, match_info=match_info)

return web.Response(status=204, headers=constants.BASE_HEADERS)

Expand Down Expand Up @@ -76,9 +77,10 @@ async def start_upload(request: web.Request) -> web.Response:
)

# Save resource and its metadata
match_info = request.match_info
try:
resource.save(config=config, chunk=b"\0")
resource.save_metadata(config=config)
resource.save(config=config, match_info=match_info, chunk=b"\0")
resource.save_metadata(config=config, match_info=match_info)
# In case if file system is not able to store given files - abort the upload
except IOError:
logger.error(
Expand Down Expand Up @@ -154,17 +156,18 @@ async def upload_resource(request: web.Request) -> web.Response:

# Save current chunk to the resource
config: TusConfig = request.config_dict[constants.APP_TUS_CONFIG_KEY]
resource.save(config=config, chunk=await request.read())
match_info = request.match_info
resource.save(config=config, match_info=match_info, chunk=await request.read())

# If this is a final chunk - complete upload
chunk_size = int(request.headers.get(constants.HEADER_CONTENT_LENGTH) or 0)
next_offset = resource.offset + chunk_size
if next_offset == resource.file_size:
resource.complete(config=config)
resource.complete(config=config, match_info=match_info)
# But if it is not - store new metadata
else:
next_resource = attr.evolve(resource, offset=next_offset)
next_resource.save_metadata(config=config)
next_resource.save_metadata(config=config, match_info=match_info)

# Return upload headers
return web.Response(
Expand Down
29 changes: 21 additions & 8 deletions tests/test_tus.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,32 +22,45 @@
@pytest.fixture
def aiohttp_test_client(aiohttp_client):
@asynccontextmanager
async def factory(*, upload_url: str) -> TestClient:
async def factory(*, upload_url: str, upload_suffix: str = None) -> TestClient:
with tempfile.TemporaryDirectory(prefix="aiohttp_tus") as temp_path:
base_path = Path(temp_path)
app = setup_tus(
web.Application(), upload_path=Path(temp_path), upload_url=upload_url
web.Application(),
upload_path=base_path / upload_suffix if upload_suffix else base_path,
upload_url=upload_url,
)
yield await aiohttp_client(app)

return factory


@pytest.mark.parametrize(
"upload_url, tus_upload_url",
"upload_url, upload_suffix, tus_upload_url, match_info",
(
("/uploads", "/uploads"),
(r"/user/{username}/uploads", "/user/playpauseandtop/uploads"),
("/uploads", None, "/uploads", {}),
(r"/user/{username}/uploads", None, "/user/playpauseanddtop/uploads", {}),
(
r"/user/{username}/uploads",
r"{username}",
"/user/playpauseandstop/uploads",
{"username": "playpauseandstop"},
),
),
)
async def test_upload(aiohttp_test_client, loop, upload_url, tus_upload_url):
async with aiohttp_test_client(upload_url=upload_url) as client:
async def test_upload(
aiohttp_test_client, loop, upload_url, upload_suffix, tus_upload_url, match_info,
):
async with aiohttp_test_client(
upload_url=upload_url, upload_suffix=upload_suffix
) as client:
upload_url = f"http://{client.host}:{client.port}{tus_upload_url}"
test_upload_path = TEST_DATA_PATH / "hello.txt"

with open(test_upload_path, "rb") as handler:
await loop.run_in_executor(None, tus.upload, handler, upload_url)

config: TusConfig = client.app[APP_TUS_CONFIG_KEY]
expected_upload_path = config.upload_path / "hello.txt"
expected_upload_path = config.resolve_upload_path(match_info) / "hello.txt"
assert expected_upload_path.exists()
assert expected_upload_path.read_bytes() == test_upload_path.read_bytes()

0 comments on commit 4da922c

Please sign in to comment.