diff --git a/AnalyticsForSpotify/settings.py b/AnalyticsForSpotify/settings.py index 74f78b7..793d49c 100644 --- a/AnalyticsForSpotify/settings.py +++ b/AnalyticsForSpotify/settings.py @@ -39,9 +39,10 @@ ROOT_URLCONF = 'AnalyticsForSpotify.urls' + # django.db.backends.mysql DATABASES = { 'default': { - 'ENGINE': 'django.db.backends.mysql', + 'ENGINE': 'django.db.backends.postgresql', 'NAME': os.environ.get('DATABASE'), 'USER': os.environ.get('DB_USER'), 'PASSWORD': os.environ.get('DB_PASSWORD'), diff --git a/container/podman-compose.yaml b/container/podman-compose.yaml index fb543be..f3a618a 100644 --- a/container/podman-compose.yaml +++ b/container/podman-compose.yaml @@ -25,31 +25,45 @@ version: "3.4.4" services: - mariadb: - container_name: mariadb - hostname: mariadb - image: docker.io/mariadb:10.10 + postgres: + container_name: postgres + hostname: postgres + image: docker.io/postgres:17 network_mode: host ports: - - 3306:3306 + - 5432:5432 environment: - MYSQL_ROOT_PASSWORD: "spotify" - MARIADB_DATABASE: "spotify" + POSTGRES_USER: "spotify" + POSTGRES_PASSWORD: "spotify" + POSTGRES_DB: "spotify" volumes: - - /mnt/mariadb:/var/lib/mysql + - /mnt/postgres:/var/lib/postgresql/data:z - phpMyAdmin: - container_name: phpMyAdmin - hostname: phpMyAdmin - image: docker.io/bitnami/phpmyadmin:latest - network_mode: host - ports: - - "8081:8081" - depends_on: - - mariadb - environment: - DATABASE_HOST: "10.0.0.19" # TODO FIX - APACHE_HTTP_PORT_NUMBER: "8081" + # mariadb: + # container_name: mariadb + # hostname: mariadb + # image: docker.io/mariadb:10.10 + # network_mode: host + # ports: + # - 3306:3306 + # environment: + # MYSQL_ROOT_PASSWORD: "spotify" + # MARIADB_DATABASE: "spotify" + # volumes: + # - /mnt/mariadb:/var/lib/mysql + + # phpMyAdmin: + # container_name: phpMyAdmin + # hostname: phpMyAdmin + # image: docker.io/bitnami/phpmyadmin:latest + # network_mode: host + # ports: + # - "8081:8081" + # depends_on: + # - mariadb + # environment: + # DATABASE_HOST: "10.0.0.19" # TODO FIX + # APACHE_HTTP_PORT_NUMBER: "8081" # analytics-for-spotify: # container_name: analytics-for-spotify diff --git a/kubernetes/base/kustomization.yaml b/kubernetes/base/kustomization.yaml index 75828f1..7368211 100644 --- a/kubernetes/base/kustomization.yaml +++ b/kubernetes/base/kustomization.yaml @@ -7,3 +7,4 @@ resources: - secret.yaml - application.yaml - push-secret.yaml + - postgres.yaml diff --git a/kubernetes/base/postgres.yaml b/kubernetes/base/postgres.yaml new file mode 100644 index 0000000..1308c8a --- /dev/null +++ b/kubernetes/base/postgres.yaml @@ -0,0 +1,162 @@ +# PSQL 15 Public Scheme Tweak +# \c spotify +# GRANT CREATE ON SCHEMA public TO spotify; +apiVersion: postgres-operator.crunchydata.com/v1beta1 +kind: PostgresCluster +metadata: + name: spotify + namespace: analytics-for-spotify + annotations: + argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true + labels: + app.kubernetes.io/instance: spotify +spec: + backups: + pgbackrest: + configuration: + - secret: + name: truenas-s3-backup + global: + repo1-retention-full: "14" + repo1-retention-full-type: count + repo2-path: /spotify + repo2-retention-full: "14" + repo2-retention-full-type: count + repo2-s3-uri-style: path + repoHost: + resources: + limits: + memory: "32Mi" + cpu: "45m" + requests: + memory: "16Mi" + cpu: "5m" + sidecars: + pgbackrest: + resources: + limits: + memory: "32Mi" + cpu: "100m" + requests: + memory: "16Mi" + cpu: "5m" + pgbackrestConfig: + resources: + limits: + memory: "32Mi" + cpu: "45m" + requests: + memory: "16Mi" + cpu: "5m" + repos: + - name: repo1 + schedules: + full: "0 7 * * *" + volume: + volumeClaimSpec: + storageClassName: rook-ceph-block + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + - name: repo2 + s3: + bucket: postgres + endpoint: "truenas.arthurvardevanyan.com:9000" + region: homelab + schedules: + full: "0 6 * * *" + instances: + - dataVolumeClaimSpec: + storageClassName: rook-ceph-block + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi + name: "" + replicas: 2 + resources: + requests: + memory: "128Mi" + cpu: "10m" + limits: + memory: "512Mi" + cpu: "250m" + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + postgres-operator.crunchydata.com/cluster: spotify + postgres-operator.crunchydata.com/instance-set: "00" + topologyKey: "kubernetes.io/hostname" + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchLabels: + postgres-operator.crunchydata.com/cluster: spotify + postgres-operator.crunchydata.com/instance-set: "00" + topologyKey: topology.kubernetes.io/zone + patroni: + dynamicConfiguration: + postgresql: + pg_hba: + - "host all all 0.0.0.0/0 trust" + monitoring: + pgmonitor: + exporter: + resources: + limits: + memory: "32Mi" + cpu: "10m" + requests: + memory: "16Mi" + cpu: "5m" + # userInterface: + # pgAdmin: + # dataVolumeClaimSpec: + # accessModes: + # - ReadWriteOnce + # resources: + # requests: + # storage: 25Mi + # storageClassName: rook-ceph-block + # replicas: 1 + # resources: + # limits: + # cpu: 150m + # memory: 256Mi + # requests: + # cpu: 25m + # memory: 128Mi + openshift: true + port: 5432 + postgresVersion: 17 +# --- +# apiVersion: postgres-operator.crunchydata.com/v1beta1 +# kind: PGUpgrade +# metadata: +# name: spotify-upgrade +# namespace: postgres +# spec: +# fromPostgresVersion: 14 +# postgresClusterName: spotify +# toPostgresVersion: 15 + +# kubectl delete ResourceQuota -n postgres --all +# kubectl delete LimitRange -n postgres --all + +# kubectl patch postgrescluster spotify -n postgres --type=merge \ +# -p '{"spec":{"shutdown":true}}' + +# kubectl annotate postgrescluster spotify -n postgres \ +# postgres-operator.crunchydata.com/allow-upgrade=spotify-upgrade + +# kubectl patch postgrescluster spotify -n postgres --type "json" -p \ +# '[{"op":"replace","path":"/spec/postgresVersion","value":15}]' + +# kubectl patch postgrescluster spotify -n postgres --type=merge \ +# -p '{"spec":{"shutdown":false}}' diff --git a/kubernetes/base/secret.yaml b/kubernetes/base/secret.yaml index 9a1803c..c472c41 100644 --- a/kubernetes/base/secret.yaml +++ b/kubernetes/base/secret.yaml @@ -77,3 +77,24 @@ spec: remoteRef: key: spotify/spotify property: redirect_url +--- +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: truenas-s3-backup + namespace: analytics-for-spotify + annotations: + argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true +spec: + refreshInterval: "1h" + secretStoreRef: + name: vault + kind: ClusterSecretStore + target: + name: truenas-s3-backup + #creationPolicy: "Merge" + data: + - secretKey: s3.conf + remoteRef: # checkov:skip=CKV_SECRET_6 PlaceHolder Values + key: homelab/postgres + property: truenas_s3_backup diff --git a/kubernetes/components/deployment/deployment.yaml b/kubernetes/components/deployment/deployment.yaml index e694289..1818609 100644 --- a/kubernetes/components/deployment/deployment.yaml +++ b/kubernetes/components/deployment/deployment.yaml @@ -115,7 +115,7 @@ spec: scheme: HTTPS initialDelaySeconds: 15 timeoutSeconds: 1 - periodSeconds: 60 + periodSeconds: 45 successThreshold: 1 failureThreshold: 3 readinessProbe: @@ -125,7 +125,7 @@ spec: scheme: HTTPS initialDelaySeconds: 15 timeoutSeconds: 1 - periodSeconds: 60 + periodSeconds: 45 successThreshold: 1 failureThreshold: 3 startupProbe: diff --git a/kubernetes/components/knative/service.yaml b/kubernetes/components/knative/service.yaml index d7d93cc..7a18db0 100644 --- a/kubernetes/components/knative/service.yaml +++ b/kubernetes/components/knative/service.yaml @@ -52,7 +52,7 @@ spec: port: 8080 scheme: HTTP initialDelaySeconds: 15 - periodSeconds: 60 + periodSeconds: 45 successThreshold: 1 timeoutSeconds: 1 name: analytics-for-spotify @@ -67,7 +67,7 @@ spec: port: 8080 scheme: HTTP initialDelaySeconds: 15 - periodSeconds: 60 + periodSeconds: 45 successThreshold: 1 timeoutSeconds: 1 resources: diff --git a/requirements.txt b/requirements.txt index b58b90c..d2c88c1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,8 +5,12 @@ coverage==7.6.9 Django==5.1.4 django-cors-headers==4.6.0 idna==3.10 -mysqlclient==2.2.6 requests==2.32.3 sqlparse==0.5.3 typing_extensions==4.12.2 urllib3==2.2.3 +# MariaDB / MYSQL +# mysqlclient==2.2.6 +# PostgreSQL +psycopg==3.2.3 +psycopg-binary==3.2.3 diff --git a/tekton/base/unit-test.yaml b/tekton/base/unit-test.yaml index 3c260bf..4deb4bd 100644 --- a/tekton/base/unit-test.yaml +++ b/tekton/base/unit-test.yaml @@ -8,8 +8,10 @@ spec: - name: data sidecars: - - image: mariadb:10.11.7 - name: mariadb + # - image: mariadb:10.11.7 + # name: mariadb + - image: docker.io/library/postgres:17.2@sha256:f95eacea382cb05066e978a24f7336cc82bf10c78d96ac6f02fd0bea34bd223f + name: postgres securityContext: runAsNonRoot: true runAsUser: 65532 @@ -32,17 +34,29 @@ spec: ports: - containerPort: 3306 env: - - name: MARIADB_ROOT_PASSWORD + # - name: MARIADB_ROOT_PASSWORD + # value: spotify + # - name: MARIADB_DATABASE + # value: spotify + - name: POSTGRES_USER value: spotify - - name: MARIADB_DATABASE + - name: POSTGRES_PASSWORD value: spotify + - name: POSTGRES_DB + value: spotify + - name: PGDATA + value: /var/lib/postgresql/data/pgdata volumeMounts: - - name: mysql - mountPath: /var/lib/mysql/ + - name: psql + mountPath: /var/lib/postgresql/data + # - name: mysql + # mountPath: /var/lib/mysql/ - name: tmp mountPath: /tmp - - name: run-mysqld - mountPath: /run/mysqld + # - name: run-mysqld + # mountPath: /run/mysqld + - name: run-psql + mountPath: "/var/run/postgresql" steps: - name: unit-test image: "registry.arthurvardevanyan.com/apps/analytics-for-spotify:base" @@ -73,7 +87,7 @@ spec: - name: DATABASE value: spotify - name: DB_USER - value: root + value: spotify #root - name: DB_PASSWORD value: spotify - name: TEST @@ -131,13 +145,19 @@ spec: } retry 5 test volumes: - - name: mysql + - name: psql emptyDir: sizeLimit: 200Mi + # - name: mysql + # emptyDir: + # sizeLimit: 200Mi - name: tmp emptyDir: sizeLimit: 1Mi - - name: run-mysqld + # - name: run-mysqld + # emptyDir: + # sizeLimit: 1Mi + - name: run-psql emptyDir: sizeLimit: 1Mi - name: cache