Skip to content

Commit

Permalink
Improve exception handling (#230)
Browse files Browse the repository at this point in the history
This improves exception chaining and fixes a case where we'd get a
BufferError after an exception. The bad case is if you have a finally
block that writes to the file after an exception, e.g. if you use a gzip
file in a context manager it will write a trailer to the underlying file
on close.
  • Loading branch information
hauntsaninja authored Nov 8, 2023
1 parent 36e6f95 commit 1f27848
Show file tree
Hide file tree
Showing 2 changed files with 19 additions and 8 deletions.
7 changes: 6 additions & 1 deletion blobfile/_azure.py
Original file line number Diff line number Diff line change
Expand Up @@ -1193,7 +1193,12 @@ def _upload_chunk(self, chunk: memoryview, finalize: bool) -> None:
# from the 400 status code)
retry_codes=(400,) + DEFAULT_RETRY_CODES,
)
execute_api_request(self._conf, req)
try:
execute_api_request(self._conf, req)
except Exception:
del chunk, data, req.data, req
raise

self._block_index += 1
if self._block_index >= BLOCK_COUNT_LIMIT:
raise Error(
Expand Down
20 changes: 13 additions & 7 deletions blobfile/_common.py
Original file line number Diff line number Diff line change
Expand Up @@ -599,7 +599,7 @@ def execute_request(conf: Config, build_req: Callable[[], Request]) -> "urllib3.
else:
raise RequestFailure.create_from_request_response(
"host does not exist", request=req, response=fake_resp
)
) from e

err = RequestFailure.create_from_request_response(
message=f"request failed with exception {e}",
Expand Down Expand Up @@ -730,9 +730,12 @@ def _upload_buf(self, buf: memoryview, finalize: bool = False) -> int:
size = (len(buf) // self._chunk_size) * self._chunk_size
assert size > 0

chunk = buf[:size]
self._upload_chunk(chunk, finalize)
self._offset += len(chunk)
try:
chunk = buf[:size]
self._upload_chunk(chunk, finalize)
self._offset += len(chunk)
finally:
del chunk, buf # pyright: ignore[reportUnboundVariable]
return size

def close(self) -> None:
Expand Down Expand Up @@ -761,9 +764,12 @@ def write(self, b: bytes) -> int: # type: ignore
else:
self._buf += b
if len(self._buf) >= self._chunk_size:
mv = memoryview(self._buf)
size = self._upload_buf(mv)
self._buf = bytearray(mv[size:])
try:
mv = memoryview(self._buf)
size = self._upload_buf(mv)
self._buf = bytearray(mv[size:])
finally:
del mv # pyright: ignore[reportUnboundVariable]
assert len(self._buf) < self._chunk_size
return len(b)

Expand Down

0 comments on commit 1f27848

Please sign in to comment.