Project TS

.github/workflows/chartpress.yml

@@ -60,12 +60,12 @@ jobs:
     #   run: ./scripts/cideploy --dev
 
     # Uncomment this section to create a staging stack
-    # - name: Deploy -- Staging
-    #   env:
-    #     postgresPassword: ${{ secrets.TERRAFORM_STAGING_POSTGRES_PASSWORD }}
-    #     auth0BaseUrl: ${{ secrets.STAGING_AUTH0_ISSUER_BASE_URL }}
-    #   if: github.ref == 'refs/heads/develop'
-    #   run: ./scripts/cideploy --staging
+    - name: Deploy -- Staging
+      env:
+        postgresPassword: ${{ secrets.TERRAFORM_STAGING_POSTGRES_PASSWORD }}
+        auth0BaseUrl: ${{ secrets.STAGING_AUTH0_ISSUER_BASE_URL }}
+      if: github.ref == 'refs/heads/project-ts-staging-deploy'
+      run: ./scripts/cideploy --staging
 
     - name: Deploy -- Production
       env:

LICENSE.md

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2021 Development Seed
+Copyright (c) 2023 Development Seed
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
@@ -18,4 +18,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
+SOFTWARE.

deployment/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:18.04
+FROM ubuntu:22.04
 
 RUN apt-get update
 RUN apt-get install -y wget unzip curl gnupg \
@@ -21,6 +21,6 @@ RUN mv terraform /usr/local/bin
 RUN pip3 install -U pip
 
 # Install azure client
-RUN pip3 install azure-cli
+RUN pip3 install azure-cli==2.51.0
 
 WORKDIR /opt/src

deployment/helm/pearl-helm/templates/api/deployment.yaml

@@ -76,9 +76,9 @@ spec:
             - name: GpuImageTag
               value: {{ .Values.gpu.image.tag }}
             - name: GpuCount
-              value: {{ .Values.api.GpuCount | quote }}
+              value: {{ .Values.api.gpuCount | quote }}
             - name: CpuCount
-              value: {{ .Values.api.CpuCount | quote }}
+              value: {{ .Values.api.cpuCount | quote }}
             - name: Deployment
               value: {{ template "lulc-helm.fullname" . }}
             - name: TileUrl
@@ -91,4 +91,16 @@ spec:
               value: ws://{{ template "lulc-helm.fullname" . }}-socket
             - name: AUTH0_ISSUER_BASE_URL
               value: {{ .Values.api.auth0BaseUrl }}
+            - name: LiveInferenceSize
+              value: {{ .Values.api.liveInferenceSize | quote }}
+            - name: MaxInferenceSize
+              value: {{ .Values.api.maxInferenceSize | quote }}
+            - name: InstanceRequestCPU
+              value: {{ .Values.api.instanceRequestCPU | quote }}
+            - name: InstanceRequestMemory
+              value: {{ .Values.api.instanceRequestMemory | quote }}
+            - name: InstanceLimitCPU
+              value: {{ .Values.api.instanceLimitCPU | quote }}
+            - name: InstanceLimitMemory
+              value: {{ .Values.api.instanceLimitMemory | quote }}
 {{- end }}

deployment/helm/pearl-helm/templates/redis/deployment.yaml

@@ -30,6 +30,7 @@ spec:
                   operator: NotIn
                   values:
                     - "gpunodepool"
+                    - "cpunodepool"
       containers:
         - name: {{ .Chart.Name }}-redis
           image: "redis:latest"

deployment/helm/pearl-helm/templates/socket/deployment.yaml

@@ -31,6 +31,7 @@ spec:
                   values:
                     - "gpunodepool"
                     - "tiler"
+                    - "cpunodepool"
       containers:
         - name: {{ .Chart.Name }}-socket
           image: "{{ .Values.socket.image.name }}:{{ .Values.socket.image.tag }}"

deployment/helm/pearl-helm/templates/tiles/scaledobject.yaml

@@ -6,8 +6,8 @@ metadata:
 spec:
   scaleTargetRef:
     name: {{ template "lulc-helm.fullname" . }}-tiles
-  minReplicaCount: 2
-  maxReplicaCount: 5
+  minReplicaCount: {{ .Values.tiles.minReplicaCount }}
+  maxReplicaCount: {{ .Values.tiles.minReplicaCount }}
   cooldownPeriod: 300
   pollingInterval: 20
   triggers:

deployment/helm/pearl-helm/values.yaml

@@ -7,17 +7,21 @@ api:
   replicaCount: 1
   containerPort: 2000
   gpuCount: 0
-  cpuCount: 2
+  cpuCount: 3
   auth0BaseUrl: replace_manually
   signingSecret: replace_manually
   postgresUrl: replace_manually
+  liveInferenceSize: 100000000
+  maxInferenceSize: 500000000
+  instanceRequestCPU: '1500m'
+  instanceRequestMemory: '4Gi'
   image:
     name: 'chartpress_replace'
     tag: 'chartpress_replace'
   resources:
     requests:
-      cpu: "0.5"
-      memory: 1.5Gi
+      cpu: "0.3"
+      memory: 1Gi
     limits:
       cpu: "4"
       memory: 12Gi
@@ -31,8 +35,8 @@ socket:
     tag: 'chartpress_replace'
   resources:
     requests:
-      cpu: "0.5"
-      memory: 1.5Gi
+      cpu: "0.2"
+      memory: 400Mi
     limits:
       cpu: "4"
       memory: 8Gi
@@ -45,6 +49,8 @@ tiles:
   debug: false
   mosaicBackends: sqlite:///
   mosaicHost: /tmp/data/mosaics.db
+  minReplicaCount: 2
+  maxReplicaCount: 5
   image:
     name: 'chartpress_replace'
     tag: 'chartpress_replace'
@@ -64,7 +70,7 @@ redis:
   enabled: true
   resources:
     requests:
-      cpu: "0.3"
+      cpu: "0.2"
       memory: 600Mi
     limits:
       cpu: "0.3"

deployment/terraform/resources/aks.tf

@@ -6,7 +6,7 @@ resource "azurerm_kubernetes_cluster" "lulc" {
   location            = azurerm_resource_group.lulc.location
   resource_group_name = azurerm_resource_group.lulc.name
   dns_prefix          = "${local.prefix}-cluster"
-  kubernetes_version  = "1.20.13"
+  kubernetes_version  = "1.25.11"
 
   default_node_pool {
     name           = "nodepool1"
@@ -47,11 +47,11 @@ resource "azurerm_kubernetes_cluster_node_pool" "tiler" {
 resource "azurerm_kubernetes_cluster_node_pool" "cpunodepool" {
   name                  = "cpunodepool"
   kubernetes_cluster_id = azurerm_kubernetes_cluster.lulc.id
-  vm_size               = "Standard_DS2_v2"
+  vm_size               = "Standard_B2ms"
   vnet_subnet_id = azurerm_subnet.aks.id
   enable_auto_scaling   = true
-  min_count             = 0
-  max_count             = 2
+  min_count             = 1
+  max_count             = 3
   tags = {
     Environment = var.environment
     ManagedBy   = "AI4E"
@@ -66,7 +66,7 @@ resource "azurerm_kubernetes_cluster_node_pool" "gpunodepool" {
   vnet_subnet_id = azurerm_subnet.aks.id
   enable_auto_scaling   = true
   min_count             = 0
-  max_count             = 2
+  max_count             = 1
   tags = {
     Environment = var.environment
     ManagedBy   = "AI4E"

deployment/terraform/resources/database.tf

@@ -6,7 +6,7 @@ resource "azurerm_postgresql_flexible_server" "lulc" {
   administrator_login    = "lulc"
   administrator_password = var.postgres_password
   storage_mb             = 32768
-  sku_name               = "GP_Standard_D4s_v3"
+  sku_name               = "B_Standard_B1ms"
   private_dns_zone_id    = azurerm_private_dns_zone.lulc.id
   delegated_subnet_id    = azurerm_subnet.postgres.id
   depends_on = [azurerm_private_dns_zone_virtual_network_link.lulc]

deployment/terraform/resources/helm-ingress-nginx.tf

@@ -18,4 +18,8 @@ resource "helm_release" "lulc-ingress-nginx" {
     value = azurerm_public_ip.lulc.ip_address
   }
 
+  set {
+    name = "controller.service.externalTrafficPolicy"
+    value = "Local"
+  }
 }

deployment/terraform/resources/providers.tf

@@ -6,7 +6,7 @@ terraform {
   required_providers {
     azurerm = {
       source  = "hashicorp/azurerm"
-      version = "=3.4.0"
+      version = "=3.84.0"
     }
     helm = {
       source = "hashicorp/helm"

docker-compose.yml

@@ -4,12 +4,12 @@ services:
     api:
         platform: linux/amd64
         build: ./services/api/
-        command: sh -c "sleep 5; npm run dev"
+        command: sh -c "sleep 20 && npx knex migrate:latest && sleep 5 && npm run dev"
         restart: always
         links:
             - postgis
         depends_on:
-            - migration
+            - postgis
         ports:
             - "2000:2000"
         environment:
@@ -35,6 +35,9 @@ services:
     socket:
         platform: linux/amd64
         build: ./services/socket/
+        command: sh -c "sleep 30; npm run dev"
+        depends_on:
+            - api
         ports:
             - "1999:1999"
         environment:
@@ -59,17 +62,3 @@ services:
         image: redis:latest
         ports:
             - 6379:6379
-
-    migration:
-        platform: linux/amd64
-        build: ./services/api
-        command: sh -c "sleep 20; npx knex migrate:latest"
-        links:
-            - postgis
-        ports:
-            - "2001:2001"
-        depends_on:
-            - postgis
-        environment:
-            - PORT=2001
-            - Postgres=postgres://docker:docker@postgis:5432/gis

helpers/create_sentinel_mosaic.py

@@ -0,0 +1,28 @@
+import httpx
+
+# Search against the Planetary Computer STAC API
+endpoint = "https://planetarycomputer.microsoft.com"
+
+# Define your temporal range
+daterange = {"interval": ["2022-01-12T10:25:31Z", "2023-01-30T23:59:59Z"]}
+
+# ##### Register Search query
+search_request = {
+    "filter": {
+        "op": "and",
+        "args": [
+            {"op": "anyinteracts", "args": [{"property": "datetime"}, daterange]},
+            {"op": "=", "args": [{"property": "collection"}, "sentinel-2-l2a"]},
+            {"op": "<=", "args": [{"property": "eo:cloud_cover"}, 50]},
+            {"op": "<=", "args": [{"property": "s2:snow_ice_percentage"}, 50]},
+        ],
+    },
+    "filter-lang": "cql2-json",
+}
+
+response = httpx.post(
+    f"{endpoint}/api/data/v1/mosaic/register",
+    json=search_request,
+).json()
+searchid = response["searchid"]
+print('searchid', searchid)

helpers/requirements.txt

@@ -0,0 +1 @@
+httpx==0.23.3