feat(Postgress): Switch to PSQL

AnalyticsForSpotify/settings.py

@@ -39,9 +39,10 @@
 
 ROOT_URLCONF = 'AnalyticsForSpotify.urls'
 
+ # django.db.backends.mysql
 DATABASES = {
     'default': {
-        'ENGINE': 'django.db.backends.mysql',
+        'ENGINE': 'django.db.backends.postgresql',
         'NAME': os.environ.get('DATABASE'),
         'USER': os.environ.get('DB_USER'),
         'PASSWORD': os.environ.get('DB_PASSWORD'),

container/podman-compose.yaml

@@ -25,31 +25,45 @@
 version: "3.4.4"
 
 services:
-  mariadb:
-    container_name: mariadb
-    hostname: mariadb
-    image: docker.io/mariadb:10.10
+  postgres:
+    container_name: postgres
+    hostname: postgres
+    image: docker.io/postgres:17
     network_mode: host
     ports:
-      - 3306:3306
+      - 5432:5432
     environment:
-      MYSQL_ROOT_PASSWORD: "spotify"
-      MARIADB_DATABASE: "spotify"
+      POSTGRES_USER: "spotify"
+      POSTGRES_PASSWORD: "spotify"
+      POSTGRES_DB: "spotify"
     volumes:
-      - /mnt/mariadb:/var/lib/mysql
+      - /mnt/postgres:/var/lib/postgresql/data:z
 
-  phpMyAdmin:
-    container_name: phpMyAdmin
-    hostname: phpMyAdmin
-    image: docker.io/bitnami/phpmyadmin:latest
-    network_mode: host
-    ports:
-      - "8081:8081"
-    depends_on:
-      - mariadb
-    environment:
-      DATABASE_HOST: "10.0.0.19" # TODO FIX
-      APACHE_HTTP_PORT_NUMBER: "8081"
+  # mariadb:
+  #   container_name: mariadb
+  #   hostname: mariadb
+  #   image: docker.io/mariadb:10.10
+  #   network_mode: host
+  #   ports:
+  #     - 3306:3306
+  #   environment:
+  #     MYSQL_ROOT_PASSWORD: "spotify"
+  #     MARIADB_DATABASE: "spotify"
+  #   volumes:
+  #     - /mnt/mariadb:/var/lib/mysql
+
+  # phpMyAdmin:
+  #   container_name: phpMyAdmin
+  #   hostname: phpMyAdmin
+  #   image: docker.io/bitnami/phpmyadmin:latest
+  #   network_mode: host
+  #   ports:
+  #     - "8081:8081"
+  #   depends_on:
+  #     - mariadb
+  #   environment:
+  #     DATABASE_HOST: "10.0.0.19" # TODO FIX
+  #     APACHE_HTTP_PORT_NUMBER: "8081"
 
   # analytics-for-spotify:
   #   container_name: analytics-for-spotify

kubernetes/base/kustomization.yaml

@@ -7,3 +7,4 @@ resources:
   - secret.yaml
   - application.yaml
   - push-secret.yaml
+  - postgres.yaml

kubernetes/base/postgres.yaml

@@ -0,0 +1,162 @@
+# PSQL 15 Public Scheme Tweak
+# \c spotify
+# GRANT CREATE ON SCHEMA public TO spotify;
+apiVersion: postgres-operator.crunchydata.com/v1beta1
+kind: PostgresCluster
+metadata:
+  name: spotify
+  namespace: analytics-for-spotify
+  annotations:
+    argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
+  labels:
+    app.kubernetes.io/instance: spotify
+spec:
+  backups:
+    pgbackrest:
+      configuration:
+        - secret:
+            name: truenas-s3-backup
+      global:
+        repo1-retention-full: "14"
+        repo1-retention-full-type: count
+        repo2-path: /spotify
+        repo2-retention-full: "14"
+        repo2-retention-full-type: count
+        repo2-s3-uri-style: path
+      repoHost:
+        resources:
+          limits:
+            memory: "32Mi"
+            cpu: "45m"
+          requests:
+            memory: "16Mi"
+            cpu: "5m"
+      sidecars:
+        pgbackrest:
+          resources:
+            limits:
+              memory: "32Mi"
+              cpu: "100m"
+            requests:
+              memory: "16Mi"
+              cpu: "5m"
+        pgbackrestConfig:
+          resources:
+            limits:
+              memory: "32Mi"
+              cpu: "45m"
+            requests:
+              memory: "16Mi"
+              cpu: "5m"
+      repos:
+        - name: repo1
+          schedules:
+            full: "0 7 * * *"
+          volume:
+            volumeClaimSpec:
+              storageClassName: rook-ceph-block
+              accessModes:
+                - ReadWriteOnce
+              resources:
+                requests:
+                  storage: 1Gi
+        - name: repo2
+          s3:
+            bucket: postgres
+            endpoint: "truenas.arthurvardevanyan.com:9000"
+            region: homelab
+          schedules:
+            full: "0 6 * * *"
+  instances:
+    - dataVolumeClaimSpec:
+        storageClassName: rook-ceph-block
+        accessModes:
+          - ReadWriteOnce
+        resources:
+          requests:
+            storage: 2Gi
+      name: ""
+      replicas: 2
+      resources:
+        requests:
+          memory: "128Mi"
+          cpu: "10m"
+        limits:
+          memory: "512Mi"
+          cpu: "250m"
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            - labelSelector:
+                matchLabels:
+                  postgres-operator.crunchydata.com/cluster: spotify
+                  postgres-operator.crunchydata.com/instance-set: "00"
+              topologyKey: "kubernetes.io/hostname"
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              podAffinityTerm:
+                labelSelector:
+                  matchLabels:
+                    postgres-operator.crunchydata.com/cluster: spotify
+                    postgres-operator.crunchydata.com/instance-set: "00"
+                topologyKey: topology.kubernetes.io/zone
+  patroni:
+    dynamicConfiguration:
+      postgresql:
+        pg_hba:
+          - "host all all 0.0.0.0/0 trust"
+  monitoring:
+    pgmonitor:
+      exporter:
+        resources:
+          limits:
+            memory: "32Mi"
+            cpu: "10m"
+          requests:
+            memory: "16Mi"
+            cpu: "5m"
+  # userInterface:
+  #   pgAdmin:
+  #     dataVolumeClaimSpec:
+  #       accessModes:
+  #         - ReadWriteOnce
+  #       resources:
+  #         requests:
+  #           storage: 25Mi
+  #       storageClassName: rook-ceph-block
+  #     replicas: 1
+  #     resources:
+  #       limits:
+  #         cpu: 150m
+  #         memory: 256Mi
+  #       requests:
+  #         cpu: 25m
+  #         memory: 128Mi
+  openshift: true
+  port: 5432
+  postgresVersion: 17
+# ---
+# apiVersion: postgres-operator.crunchydata.com/v1beta1
+# kind: PGUpgrade
+# metadata:
+#   name: spotify-upgrade
+#   namespace: postgres
+# spec:
+#   fromPostgresVersion: 14
+#   postgresClusterName: spotify
+#   toPostgresVersion: 15
+
+# kubectl delete ResourceQuota -n postgres --all
+# kubectl delete LimitRange -n postgres --all
+
+# kubectl patch postgrescluster spotify -n postgres --type=merge \
+#     -p '{"spec":{"shutdown":true}}'
+
+# kubectl annotate postgrescluster spotify -n postgres  \
+#     postgres-operator.crunchydata.com/allow-upgrade=spotify-upgrade
+
+# kubectl patch postgrescluster spotify -n postgres --type "json" -p \
+# '[{"op":"replace","path":"/spec/postgresVersion","value":15}]'
+
+# kubectl patch postgrescluster spotify -n postgres --type=merge \
+#     -p '{"spec":{"shutdown":false}}'

kubernetes/base/secret.yaml

@@ -77,3 +77,24 @@ spec:
       remoteRef:
         key: spotify/spotify
         property: redirect_url
+---
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: truenas-s3-backup
+  namespace: analytics-for-spotify
+  annotations:
+    argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
+spec:
+  refreshInterval: "1h"
+  secretStoreRef:
+    name: vault
+    kind: ClusterSecretStore
+  target:
+    name: truenas-s3-backup
+    #creationPolicy: "Merge"
+  data:
+    - secretKey: s3.conf
+      remoteRef: # checkov:skip=CKV_SECRET_6 PlaceHolder Values
+        key: homelab/postgres
+        property: truenas_s3_backup

kubernetes/components/deployment/deployment.yaml

@@ -115,7 +115,7 @@ spec:
               scheme: HTTPS
             initialDelaySeconds: 15
             timeoutSeconds: 1
-            periodSeconds: 60
+            periodSeconds: 45
             successThreshold: 1
             failureThreshold: 3
           readinessProbe:
@@ -125,7 +125,7 @@ spec:
               scheme: HTTPS
             initialDelaySeconds: 15
             timeoutSeconds: 1
-            periodSeconds: 60
+            periodSeconds: 45
             successThreshold: 1
             failureThreshold: 3
           startupProbe:

kubernetes/components/knative/service.yaml

@@ -52,7 +52,7 @@ spec:
               port: 8080
               scheme: HTTP
             initialDelaySeconds: 15
-            periodSeconds: 60
+            periodSeconds: 45
             successThreshold: 1
             timeoutSeconds: 1
           name: analytics-for-spotify
@@ -67,7 +67,7 @@ spec:
               port: 8080
               scheme: HTTP
             initialDelaySeconds: 15
-            periodSeconds: 60
+            periodSeconds: 45
             successThreshold: 1
             timeoutSeconds: 1
           resources:

requirements.txt

@@ -5,8 +5,12 @@ coverage==7.6.9
 Django==5.1.4
 django-cors-headers==4.6.0
 idna==3.10
-mysqlclient==2.2.6
 requests==2.32.3
 sqlparse==0.5.3
 typing_extensions==4.12.2
 urllib3==2.2.3
+# MariaDB / MYSQL
+# mysqlclient==2.2.6
+# PostgreSQL
+psycopg==3.2.3
+psycopg-binary==3.2.3

tekton/base/unit-test.yaml

@@ -8,8 +8,10 @@ spec:
     - name: data
 
   sidecars:
-    - image: mariadb:10.11.7
-      name: mariadb
+    # - image: mariadb:10.11.7
+    #   name: mariadb
+    - image: docker.io/library/postgres:17.2@sha256:f95eacea382cb05066e978a24f7336cc82bf10c78d96ac6f02fd0bea34bd223f
+      name: postgres
       securityContext:
         runAsNonRoot: true
         runAsUser: 65532
@@ -32,17 +34,25 @@ spec:
       ports:
         - containerPort: 3306
       env:
-        - name: MARIADB_ROOT_PASSWORD
+        # - name: MARIADB_ROOT_PASSWORD
+        #   value: spotify
+        # - name: MARIADB_DATABASE
+        #   value: spotify
+        - name: POSTGRES_USER
           value: spotify
-        - name: MARIADB_DATABASE
+        - name: POSTGRES_PASSWORD
+          value: spotify
+        - name: POSTGRES_DB
           value: spotify
       volumeMounts:
-        - name: mysql
-          mountPath: /var/lib/mysql/
+        - name: psql
+          mountPath: /var/lib/postgresql/data
+        # - name: mysql
+        #   mountPath: /var/lib/mysql/
         - name: tmp
           mountPath: /tmp
-        - name: run-mysqld
-          mountPath: /run/mysqld
+        # - name: run-mysqld
+        #   mountPath: /run/mysqld
   steps:
     - name: unit-test
       image: "registry.arthurvardevanyan.com/apps/analytics-for-spotify:base"
@@ -131,15 +141,18 @@ spec:
         }
         retry 5 test
   volumes:
-    - name: mysql
+    - name: psql
       emptyDir:
         sizeLimit: 200Mi
+    # - name: mysql
+    #   emptyDir:
+    #     sizeLimit: 200Mi
     - name: tmp
       emptyDir:
         sizeLimit: 1Mi
-    - name: run-mysqld
-      emptyDir:
-        sizeLimit: 1Mi
+    # - name: run-mysqld
+    #   emptyDir:
+    #     sizeLimit: 1Mi
     - name: cache
       persistentVolumeClaim:
         claimName: cache