Merge pull request #24 from datarevenue-berlin/remove-recursive

Add recursive removal to local fs, expose it in s3 fs too

drfs/filesystems/local.py

@@ -38,9 +38,15 @@ def makedirs(self, *args, **kwargs):
         os.makedirs(*args, **kwargs)
 
     @allow_pathlib
-    def remove(self, path):
-        """Remove a file."""
-        os.remove(path)
+    def remove(self, path, recursive=False):
+        """Remove a file or a directory which may be non-empty."""
+        try:
+            os.remove(path)
+        except IsADirectoryError:
+            if not recursive:
+                self.rmdir(path)
+            else:
+                shutil.rmtree(path)
 
     @return_pathlib
     @allow_pathlib

drfs/filesystems/s3.py

@@ -19,6 +19,9 @@ def makedirs(self, *args, **kwargs):
     def rmdir(self, path, **kwargs):
         pass
 
+    def rm(self, path, recursive=False, **kwargs):
+        return self.fs.rm(path, recursive=recursive, **kwargs)
+
     def put(self, filename, path, **kwargs):
         from drfs.path import asstr
         filename, path = asstr(filename), asstr(path)

drfs/tests/filesystems/test_local.py

@@ -61,3 +61,25 @@ def test_walk_scheme_local(tmpdir):
     assert all(isinstance(item, Path) for item in res)
     assert any('deep_test' in str(item) for item in res)
     assert not any(str(item).rstrip('/').endswith('dir1') for item in res)
+
+
+def test_remove(tmpdir):
+    fs = LocalFileSystem()
+    empty_dir = tmpdir / 'empty'
+    fs.makedirs(empty_dir)
+    dir1 = tmpdir / 'dir1'
+    file1 = dir1 / 'deep_test.txt'
+    fs.touch(file1)
+
+    assert fs.exists(empty_dir)
+    fs.remove(empty_dir)
+    assert not fs.exists(empty_dir)
+
+    with pytest.raises(OSError):
+        fs.remove(dir1)
+
+    assert fs.exists(file1)
+    fs.remove(dir1, recursive=True)
+    assert not fs.exists(file1)
+    assert not fs.exists(dir1)
+

drfs/tests/filesystems/test_s3.py

@@ -38,3 +38,15 @@ def test_walk_scheme_s3(s3):
     assert all(str(item).startswith('s3://') for item in res)
     assert any('deep_test' in str(item) for item in res)
     assert not any(str(item).rstrip('/').endswith('dir1') for item in res)
+
+
+def test_remove_recursive(s3):
+    fs = S3FileSystem()
+    fs.touch("s3://test-bucket/dir1/deep_test.txt")
+    fs.touch("s3://test-bucket/dir1/subdir/deeper_test.txt")
+
+    fs.remove("s3://test-bucket/dir1", recursive=True)
+
+    assert not fs.exists("s3://test-bucket/dir1")
+    assert not fs.exists("s3://test-bucket/dir1/deep_test.txt")
+    assert not fs.exists("s3://test-bucket/dir1/subdir/deeper_test.txt")