Renommer automatiquement les images

app/settings.py

@@ -268,8 +268,9 @@
 S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY")
 S3_SECRET_KEY = os.getenv("S3_SECRET_KEY")
 
-QUESTION_FOLDER_NAME = "questions"
-QUIZ_FOLDER_NAME = "quizs"
+PREFIX_FOLDER_NAME = "test/" if DEBUG else ""
+QUESTION_FOLDER_NAME = f"{PREFIX_FOLDER_NAME}questions"
+QUIZ_FOLDER_NAME = f"{PREFIX_FOLDER_NAME}quizs"
 
 STORAGE_UPLOAD_KINDS = {
     "default": {
@@ -349,11 +350,14 @@
 
 SHELL_PLUS = "ipython"
 SHELL_PLUS_IMPORTS = [
+    "import requests",
     "import csv, json, yaml",
     "from datetime import datetime, date, timedelta",
     "from core import constants",
     "from core.utils import utilities, notion, github, sendinblue, s3",
-    "from stats import utilities as utilities_stats",
+    "from users import constants as user_constants",
+    "from stats import constants as stats_constants",
+    "from stats import utilities as stats_utilities",
 ]
 
 

core/management/commands/export_stats_to_github.py

@@ -8,7 +8,7 @@
 
 from core.models import Configuration
 from core.utils import github, utilities
-from stats import utilities as utilities_stats
+from stats import utilities as stats_utilities
 
 
 class Command(BaseCommand):
@@ -40,12 +40,12 @@ def handle(self, *args, **options):
             # data/stats.yaml
             start_time = time.time()
             stats_dict = {
-                **utilities_stats.question_stats(),
-                **utilities_stats.quiz_stats(),
-                **utilities_stats.answer_stats(),
-                **utilities_stats.category_stats(),
-                **utilities_stats.tag_stats(),
-                **utilities_stats.contribution_stats(),
+                **stats_utilities.question_stats(),
+                **stats_utilities.quiz_stats(),
+                **stats_utilities.answer_stats(),
+                **stats_utilities.category_stats(),
+                **stats_utilities.tag_stats(),
+                **stats_utilities.contribution_stats(),
             }
             stats_yaml = yaml.safe_dump(stats_dict, allow_unicode=True, sort_keys=False)
             stats_element = github.create_file_element(file_path="data/stats.yaml", file_content=stats_yaml)
@@ -55,7 +55,7 @@ def handle(self, *args, **options):
             #####################################
             # data/difficulty-levels.yaml
             start_time = time.time()
-            difficulty_levels_list = utilities_stats.difficulty_aggregate()
+            difficulty_levels_list = stats_utilities.difficulty_aggregate()
             difficulty_levels_yaml = yaml.safe_dump(difficulty_levels_list, allow_unicode=True, sort_keys=False)
             difficulty_levels_element = github.create_file_element(
                 file_path="data/difficulty-levels.yaml",
@@ -67,7 +67,7 @@ def handle(self, *args, **options):
             # #####################################
             # # data/authors.yaml
             # start_time = time.time()
-            # authors_list = utilities_stats.author_aggregate()
+            # authors_list = stats_utilities.author_aggregate()
             # authors_yaml = yaml.safe_dump(authors_list, allow_unicode=True, sort_keys=False)
             # authors_element = github.create_file_element(file_path="data/authors.yaml", file_content=authors_yaml)
 
@@ -76,7 +76,7 @@ def handle(self, *args, **options):
             #####################################
             # data/languages.yaml
             start_time = time.time()
-            languages_list = utilities_stats.language_aggregate()
+            languages_list = stats_utilities.language_aggregate()
             languages_yaml = yaml.safe_dump(languages_list, allow_unicode=True, sort_keys=False)
             languages_element = github.create_file_element(
                 file_path="data/languages.yaml", file_content=languages_yaml
@@ -87,7 +87,7 @@ def handle(self, *args, **options):
             # #####################################
             # # data/quiz-stats.yaml
             # start_time = time.time()
-            # quiz_detail_stats_list = utilities_stats.quiz_detail_stats()
+            # quiz_detail_stats_list = stats_utilities.quiz_detail_stats()
             # quiz_detail_stats_yaml = yaml.safe_dump(quiz_detail_stats_list, allow_unicode=True, sort_keys=False)
             # quiz_stats_element = github.create_file_element(
             #     file_path="data/quiz-stats.yaml", file_content=quiz_detail_stats_yaml

core/management/commands/transfer_images_to_s3.py → core/management/commands/update_images_in_s3.py

@@ -1,5 +1,4 @@
 import os
-import uuid
 
 import requests
 from django.conf import settings
@@ -16,15 +15,23 @@
 
 class Command(BaseCommand):
     """
-    One-shot command to migrate images to the new S3 destination
-    Example (Question id=15): image.png --> questions/000015-7gh5.png
+    Goal:
+    - Migrate images to the new S3 destination (initial usage)
+    - Update their naming (current usage)
 
-    Usage: python manage.py transfer_images_to_s3
+    Examples:
+    - transfer example (Question id=15): https://example.com/image.png --> questions/000015-7gh5.png
+    - naming example (Question id=15): questions/g5k78wx0.png --> questions/000015-7gh5.png
+
+    Usage: python manage.py update_images_in_s3
     """
 
     def handle(self, *args, **options):
-        self.transfer_question_images()
-        self.transfer_quiz_images()
+        if not settings.DEBUG:
+            self.transfer_question_images()
+            self.transfer_quiz_images()
+        else:
+            print("DEBUG mode. Exiting. This script can only be run in production")
 
     def transfer_question_images(self):
         for question in Question.objects.all():
@@ -36,17 +43,14 @@ def transfer_question_images(self):
                 print(image_url)
                 try:
                     image_filename = image_url.split("/")[-1].split("?")[0]
-                    image_extension = image_filename.split(".")[1]
                     print(image_filename)
                     # download image
-                    response = requests.get(image_url, allow_redirects=True)
+                    response = requests.get(image_url, allow_redirects=True, timeout=10)
                     # store image
                     open(image_filename, "wb").write(response.content)
                     # s3 upload
                     bucket = s3.get_bucket(settings.S3_BUCKET_NAME)
-                    s3_image_filename = (
-                        f"{str(question.id).zfill(6)}-{str(uuid.uuid4())[:4]}.{image_extension.lower()}"
-                    )
+                    s3_image_filename = s3.create_image_name(question.id, image_filename)
                     image_key = (
                         f"{settings.STORAGE_UPLOAD_KINDS['question_answer_image']['key_path']}/{s3_image_filename}"
                     )
@@ -79,15 +83,14 @@ def transfer_quiz_images(self):
                 print(image_url)
                 try:
                     image_filename = image_url.split("/")[-1].split("?")[0]
-                    image_extension = image_filename.split(".")[1]
                     print(image_filename)
                     # download image
-                    response = requests.get(image_url, allow_redirects=True)
+                    response = requests.get(image_url, allow_redirects=True, timeout=10)
                     # store image
                     open(image_filename, "wb").write(response.content)
                     # s3 upload
                     bucket = s3.get_bucket(settings.S3_BUCKET_NAME)
-                    s3_image_filename = f"{str(quiz.id).zfill(6)}-{str(uuid.uuid4())[:4]}.{image_extension.lower()}"
+                    s3_image_filename = s3.create_image_name(quiz.id, image_filename)
                     image_key = (
                         f"{settings.STORAGE_UPLOAD_KINDS['quiz_image_background']['key_path']}/{s3_image_filename}"
                     )
@@ -107,5 +110,6 @@ def transfer_quiz_images(self):
                     Quiz.objects.filter(id=quiz.id).update(image_background_url=s3_image_url)
                     # delete local image
                     os.remove(image_filename)
+                    # TODO: delete
                 except:  # noqa
                     print("Error")

core/tests/test_s3.py

@@ -0,0 +1,16 @@
+from django.test import TestCase
+
+from core.utils import s3
+
+
+class UtilitiesTest(TestCase):
+    def test_create_image_name(self):
+        INSTANCE_IMAGES = [
+            (5, "test.png", "000005-UUID.png"),
+            (15, "another-test.jpeg", "000015-UUID.jpeg"),
+            (150, "https://example.com/another-test.gif", "000150-UUID.gif"),
+            (1500, "https://example.com/another-test.webp?height=100", "001500-UUID.webp"),
+        ]
+        for item in INSTANCE_IMAGES:
+            image_name = s3.create_image_name(item[0], item[1])
+            self.assertTrue(image_name[:4] in item[2])

core/utils/s3.py

@@ -1,4 +1,5 @@
 import json
+import uuid
 
 import boto3
 from botocore.client import Config
@@ -183,3 +184,19 @@ def get_bucket_policy(bucket):
 
 def update_bucket_policy(bucket, policy_configuration=DEFAULT_POLICY_CONFIGURATION):
     client.put_bucket_policy(Bucket=bucket.name, Policy=json.dumps(policy_configuration))
+
+
+def create_image_name(instance_id, image_filename):
+    """
+    Image naming convention
+
+    Examples:
+    - (Question id=15): test.jpeg --> 000015-7gh5.png
+    - (Question id=15): questions/g5k78wx0.png --> 000015-7gh5.png
+    - (Question id=15): https://example.com/image.png --> 000015-7gh5.png
+    """
+    if "/" in image_filename:
+        # an image_url was given...
+        image_filename = image_filename.split("/")[-1].split("?")[0]
+    image_extension = image_filename.split(".")[1]
+    return f"{str(instance_id).zfill(6)}-{str(uuid.uuid4())[:4]}.{image_extension.lower()}"