feat(api): expose download percentage

Author: xyb

README.md

@@ -56,6 +56,10 @@ The creation API will return the task object:
   "largest_file": null,
   "largest_file_size": null,
   "is_downloading": False,
+  "sample_download_percent": 0.0,
+  "sample_downloaded_files": 0,
+  "download_percent": 0.0,
+  "downloaded_size": 0,
   "current_progessing_stage": "waiting_assign",
   "done": false,
   "failed": false,

pytest.ini

@@ -1,4 +1,4 @@
 [pytest]
 DJANGO_SETTINGS_MODULE = baidupcsleecher.settings
 python_files = tests.py test_*.py *_tests.py
-addopts = -ra --doctest-modules --last-failed --durations=3 --cov --cov-report term-missing --no-cov-on-fail --disable-socket
+addopts = -ra --doctest-modules --last-failed --durations=3 --cov --cov-report term-missing --no-cov-on-fail --disable-socket -vv

task/models.py

@@ -152,8 +152,11 @@ def list_remote_files(self, files_only=True):
     def remote_files(self):
         return self.list_remote_files(files_only=True)
 
-    def list_local_files(self):
-        data_path = self.data_path
+    def list_local_files(self, samples_only=False):
+        if samples_only:
+            data_path = self.sample_data_path
+        else:
+            data_path = self.data_path
         for root, dirs, files in walk(data_path):
             for file in files:
                 filepath = join(root, file)
@@ -164,6 +167,10 @@ def list_local_files(self):
     def local_files(self):
         return list(self.list_local_files())
 
+    @property
+    def local_sample_files(self) -> list:
+        return list(self.list_local_files(samples_only=True))
+
     @property
     def total_files(self):
         return len([f for f in self.load_files() if f["is_file"]])
@@ -380,3 +387,23 @@ def delete_files(self):
     def erase(self):
         self.delete_files()
         self.delete()
+
+    @property
+    def sample_downloaded_files(self) -> int:
+        return len(list(self.list_local_files()))
+
+    @property
+    def sample_download_percent(self) -> float:
+        if self.total_files == 0:
+            return 0.0
+        return 100.0 * self.sample_downloaded_files / self.total_files
+
+    @property
+    def downloaded_size(self) -> int:
+        return self.local_size
+
+    @property
+    def download_percent(self) -> float:
+        if self.total_files == 0:
+            return 0.0
+        return 100.0 * self.downloaded_size / self.total_size

task/serializers.py

@@ -30,6 +30,10 @@ class Meta:
             "largest_file",
             "largest_file_size",
             "is_downloading",
+            "sample_download_percent",
+            "sample_downloaded_files",
+            "download_percent",
+            "downloaded_size",
             "current_progressing_stage",
             "done",
             "failed",

task/tests/test_api.py

@@ -15,7 +15,7 @@
 
 def touch_file(path: Path):
     path.parent.mkdir(parents=True, exist_ok=True)
-    open(path, "w").write("")
+    open(path, "w").write("hello" * 1024)
 
 
 def touch_task_files(task: Task):
@@ -72,6 +72,10 @@ def test_create_task(self):
         assert data["full_download_now"] is False
         assert data["current_progressing_stage"] == "waiting_assign"
         assert data["is_downloading"] is False
+        assert data["download_percent"] == 0
+        assert data["sample_download_percent"] == 0
+        assert data["downloaded_size"] == 0
+        assert data["sample_downloaded_files"] == 0
         assert set(data.keys()) == {
             "callback",
             "captcha_required",
@@ -80,6 +84,8 @@ def test_create_task(self):
             "created_at",
             "current_progressing_stage",
             "done",
+            "download_percent",
+            "downloaded_size",
             "failed",
             "file_listed_at",
             "finished_at",
@@ -93,7 +99,9 @@ def test_create_task(self):
             "path",
             "recoverable",
             "retry_times",
+            "sample_download_percent",
             "sample_downloaded_at",
+            "sample_downloaded_files",
             "sample_path",
             "shared_id",
             "shared_link",
@@ -365,3 +373,16 @@ def test_purge_nothing(self):
 
         assert response.json() == {"done": True}
         assert sorted(list_files(settings.DATA_DIR)) == files
+
+    def test_download_percent(self):
+        touch_task_files(self.task)
+
+        url = reverse("task-detail", args=[self.task.id])
+
+        response = self.client.get(url)
+
+        data = response.json()
+        assert f'{data["download_percent"]:0.3f}' == "0.095"
+        assert f'{data["sample_download_percent"]:0.2f}' == "100.00"
+        assert data["downloaded_size"] == 10240
+        assert data["sample_downloaded_files"] == 2