wip: almost fully deployed scylladb (some issue with IP addrs remains)

comrade-coop · Sep 28, 2024 · cddc280 · cddc280
1 parent 5a44179
commit cddc280
Show file tree

Hide file tree

Showing 9 changed files with 643 additions and 9 deletions.
diff --git a/test/e2e/scylladb/.gitignore b/test/e2e/scylladb/.gitignore
@@ -0,0 +1,2 @@
+deployments/
+nebula-config/
diff --git a/test/e2e/scylladb/docker-compose.yml b/test/e2e/scylladb/docker-compose.yml
@@ -5,8 +5,8 @@ services:
     command: [
       '--memory', '2G',
       '--smp', '2',
-      '--seeds', '172.123.0.2,172.123.0.3',
-      # '--seeds', '172.123.0.2,172.123.0.3,172.123.0.4',
+      '--seeds', '172.16.123.2,172.16.123.3',
+      # '--seeds', '172.16.123.2,172.16.123.3,172.16.123.4',
       '--overprovisioned', '1',
       # '--developer-mode', '0',
     ]
@@ -16,15 +16,15 @@ services:
       - scyllat1:/var/lib/scylla
     networks:
       scyllanet:
-        ipv4_address: 172.123.0.2
+        ipv4_address: 172.16.123.2
   scyllat2:
     container_name: scyllat2
     image: scylladb/scylla
     command: [
       '--memory', '2G',
       '--smp', '2',
-      '--seeds', '172.123.0.2,172.123.0.3',
-      # '--seeds', '172.123.0.2,172.123.0.3,172.123.0.4',
+      '--seeds', '172.16.123.2,172.16.123.3',
+      # '--seeds', '172.16.123.2,172.16.123.3,172.16.123.4',
       '--overprovisioned', '1',
       # '--developer-mode', '0',
     ]
@@ -34,14 +34,14 @@ services:
       - scyllat2:/var/lib/scylla
     networks:
       scyllanet:
-        ipv4_address: 172.123.0.3
+        ipv4_address: 172.16.123.3
   # scyllat3:
   #   container_name: scyllat3
   #   image: scylladb/scylla
   #   command: [
   #     '--memory', '2G',
   #     '--smp', '2',
-  #     '--seeds', '172.123.0.2,172.123.0.3,172.123.0.4',
+  #     '--seeds', '172.16.123.2,172.16.123.3,172.16.123.4',
   #     '--overprovisioned', '1',
   #     # '--developer-mode', '0',
   #   ]
@@ -51,13 +51,13 @@ services:
   #     - scyllat3:/var/lib/scylla
   #   networks:
   #     scyllanet:
-  #       ipv4_address: 172.123.0.4
+  #       ipv4_address: 172.16.123.4
 networks:
   scyllanet:
     # driver: bridge
     ipam:
       config:
-        - subnet: 172.123.0.0/16
+        - subnet: 172.16.123.0/8
 volumes:
   scyllat1:
   scyllat2:

diff --git a/test/e2e/scylladb/manifest-lighthouse.yaml b/test/e2e/scylladb/manifest-lighthouse.yaml
@@ -0,0 +1,45 @@
+containers:
+  - name: nebula
+    image:
+      url: docker.io/nebulaoss/nebula:latest
+    ports:
+      - containerPort: '4242'
+        hostUdpPort: 0
+        name: nebula
+    resourceRequests:
+      - amountMillis: '10'
+        resource: cpu
+      - amountMebi: '100'
+        resource: memory
+    volumes:
+      - name: nebula-yaml
+        mountPath: /config/config.yml
+      - name: ca-crt
+        mountPath: /config/ca.crt
+      - name: host-key
+        mountPath: /config/host.key
+      - name: host-crt
+        mountPath: /config/host.crt
+    capabilities:
+      - CAP_NET_ADMIN
+volumes:
+  - name: nebula-yaml
+    type: VOLUME_SECRET
+    secret:
+      file: 'nebula-conf-lighthouse.yaml'
+  - name: ca-crt
+    type: VOLUME_SECRET
+    secret:
+      file: 'nebula-config/ca.crt'
+  - name: host-key
+    type: VOLUME_SECRET
+    secret:
+      file: 'nebula-config/lighthouse.key'
+  - name: host-crt
+    type: VOLUME_SECRET
+    secret:
+      file: 'nebula-config/lighthouse.crt'
+replicas:
+  min: 1
+  max: 1
+
diff --git a/test/e2e/scylladb/manifest-node-tmpl.yaml b/test/e2e/scylladb/manifest-node-tmpl.yaml
@@ -0,0 +1,81 @@
+containers:
+  - name: scylla
+    image:
+      url: docker.io/scylladb/scylla:latest
+    command: [
+      '--memory', '1536M',
+      '--smp', '1',
+      '--seeds', '127.0.0.1:',
+      '--overprovisioned', '1',
+      # '--developer-mode', '0',
+    ]
+    resourceRequests:
+      - amountMillis: '900'
+        resource: cpu
+      - amountMebi: '1536'
+        resource: memory
+    volumes:
+      - name: scylla-yaml
+        mountPath: /etc/scylla/scylla.yaml
+      - name: cassandra-rackdc
+        mountPath: /etc/scylla/cassandra-rackdc.properties
+      - name: scylla-db
+        mountPath: /var/lib/scylla
+  - name: nebula
+    image:
+      url: docker.io/nebulaoss/nebula:latest
+    ports:
+      - containerPort: '4242'
+        hostUdpPort: $SERVICEPORT$
+        name: nebula
+    resourceRequests:
+      - amountMillis: '10'
+        resource: cpu
+      - amountMebi: '100'
+        resource: memory
+    volumes:
+      - name: nebula-yaml
+        mountPath: /config/config.yml
+      - name: ca-crt
+        mountPath: /config/ca.crt
+      - name: host-key
+        mountPath: /config/host.key
+      - name: host-crt
+        mountPath: /config/host.crt
+    capabilities:
+      - CAP_NET_ADMIN
+volumes:
+  - name: scylla-db
+    type: VOLUME_FILESYSTEM
+    filesystem:
+      resourceRequests:
+        - amountGibi: '1'
+          resource: storage
+  - name: scylla-yaml
+    type: VOLUME_SECRET
+    secret:
+      file: 'scylla.yaml'
+  - name: cassandra-rackdc
+    type: VOLUME_SECRET
+    secret:
+      file: 'scylla.yaml'
+  - name: nebula-yaml
+    type: VOLUME_SECRET
+    secret:
+      file: 'nebula-config/config.yml'
+  - name: ca-crt
+    type: VOLUME_SECRET
+    secret:
+      file: 'nebula-config/ca.crt'
+  - name: host-key
+    type: VOLUME_SECRET
+    secret:
+      file: 'nebula-config/host.key'
+  - name: host-crt
+    type: VOLUME_SECRET
+    secret:
+      file: 'nebula-config/host.crt'
+replicas:
+  min: 1
+  max: 1
+
diff --git a/test/e2e/scylladb/manifest-node.yaml b/test/e2e/scylladb/manifest-node.yaml
@@ -0,0 +1,81 @@
+containers:
+  - name: scylla
+    image:
+      url: docker.io/scylladb/scylla:latest
+    command: [
+      '--memory', '1536M',
+      '--smp', '1',
+      '--seeds', '127.0.0.1:',
+      '--overprovisioned', '1',
+      # '--developer-mode', '0',
+    ]
+    resourceRequests:
+      - amountMillis: '900'
+        resource: cpu
+      - amountMebi: '1536'
+        resource: memory
+    volumes:
+      - name: scylla-yaml
+        mountPath: /etc/scylla/scylla.yaml
+      - name: cassandra-rackdc
+        mountPath: /etc/scylla/cassandra-rackdc.properties
+      - name: scylla-db
+        mountPath: /var/lib/scylla
+  - name: nebula
+    image:
+      url: docker.io/nebulaoss/nebula:latest
+    ports:
+      - containerPort: '4242'
+        hostUdpPort: 30002
+        name: nebula
+    resourceRequests:
+      - amountMillis: '10'
+        resource: cpu
+      - amountMebi: '100'
+        resource: memory
+    volumes:
+      - name: nebula-yaml
+        mountPath: /config/config.yml
+      - name: ca-crt
+        mountPath: /config/ca.crt
+      - name: host-key
+        mountPath: /config/host.key
+      - name: host-crt
+        mountPath: /config/host.crt
+    capabilities:
+      - CAP_NET_ADMIN
+volumes:
+  - name: scylla-db
+    type: VOLUME_FILESYSTEM
+    filesystem:
+      resourceRequests:
+        - amountGibi: '1'
+          resource: storage
+  - name: scylla-yaml
+    type: VOLUME_SECRET
+    secret:
+      file: 'scylla.yaml'
+  - name: cassandra-rackdc
+    type: VOLUME_SECRET
+    secret:
+      file: 'scylla.yaml'
+  - name: nebula-yaml
+    type: VOLUME_SECRET
+    secret:
+      file: 'nebula-config/config.yml'
+  - name: ca-crt
+    type: VOLUME_SECRET
+    secret:
+      file: 'nebula-config/ca.crt'
+  - name: host-key
+    type: VOLUME_SECRET
+    secret:
+      file: 'nebula-config/host.key'
+  - name: host-crt
+    type: VOLUME_SECRET
+    secret:
+      file: 'nebula-config/host.crt'
+replicas:
+  min: 1
+  max: 1
+
diff --git a/test/e2e/scylladb/nebula-conf-lighthouse.yaml b/test/e2e/scylladb/nebula-conf-lighthouse.yaml
@@ -0,0 +1,84 @@
+# This is the nebula example configuration file. You must edit, at a minimum, the static_host_map, lighthouse, and firewall sections
+# https://raw.githubusercontent.com/slackhq/nebula/master/examples/config.yml
+
+pki:
+  # The CAs that are accepted by this node. Must contain one or more certificates created by 'nebula-cert ca'
+  ca: /config/ca.crt
+  cert: /config/host.crt
+  key: /config/host.key
+
+lighthouse:
+  # am_lighthouse is used to enable lighthouse functionality for a node. This should ONLY be true on nodes
+  # you have configured to be lighthouses in your network
+  am_lighthouse: true
+  # interval is the number of seconds between updates from this node to a lighthouse.
+  # during updates, a node sends information about its current IP addresses to each node.
+  interval: 60
+
+# Port Nebula will be listening on. The default here is 4242. For a lighthouse node, the port should be defined,
+# however using port 0 will dynamically assign a port and is recommended for roaming nodes.
+listen:
+  # To listen on both any ipv4 and ipv6 use "::"
+  host: 0.0.0.0
+  port: 4242
+
+punchy:
+  # Continues to punch inbound/outbound at a regular interval to avoid expiration of firewall nat mappings
+  punch: false
+
+# Configure the private interface. Note: addr is baked into the nebula certificate
+tun:
+  # When tun is disabled, a lighthouse can be started without a local tun interface (and therefore without root)
+  disabled: false
+  # Name of the device. If not set, a default will be chosen by the OS.
+  # For macOS: if set, must be in the form `utun[0-9]+`.
+  # For NetBSD: Required to be set, must be in the form `tun[0-9]+`
+  dev: nebula1
+  # Toggles forwarding of local broadcast packets, the address of which depends on the ip/mask encoded in pki.cert
+  drop_local_broadcast: false
+  # Toggles forwarding of multicast packets
+  drop_multicast: false
+  # Sets the transmit queue length, if you notice lots of transmit drops on the tun it may help to raise this number. Default is 500
+  tx_queue: 500
+  # Default MTU for every packet, safe setting is (and the default) 1300 for internet based traffic
+  mtu: 1300
+  # Route based MTU overrides, you have known vpn ip paths that can support larger MTUs you can increase/decrease them here
+  routes:
+  - route: 172.16.123.0/8
+    mtu: 1300
+
+# Configure logging level
+logging:
+  # panic, fatal, error, warning, info, or debug. Default is info and is reloadable.
+  #NOTE: Debug mode can log remotely controlled/untrusted data which can quickly fill a disk in some
+  # scenarios. Debug logging is also CPU intensive and will decrease performance overall.
+  # Only enable debug logging while actively investigating an issue.
+  level: info
+  format: text
+  disable_timestamp: true
+
+# Nebula security group configuration
+firewall:
+  # Action to take when a packet is not allowed by the firewall rules.
+  # Can be one of:
+  #   `drop` (default): silently drop the packet.
+  #   `reject`: send a reject reply.
+  #     - For TCP, this will be a RST "Connection Reset" packet.
+  #     - For other protocols, this will be an ICMP port unreachable packet.
+  outbound_action: reject
+  inbound_action: reject
+
+  conntrack:
+    tcp_timeout: 12m
+    udp_timeout: 3m
+    default_timeout: 10m
+
+  outbound:
+    - port: any
+      proto: any
+      host: any
+
+  inbound:
+    - port: any
+      proto: any
+      host: any