Implement new and improved terraform-base deployment

ops/marketplace-tf/backend.tf

@@ -0,0 +1,6 @@
+terraform {
+  backend "gcs" {
+    bucket = "bacalhau-infra-state"
+    prefix = "terraform"
+  }
+}

ops/marketplace-tf/main.tf

@@ -0,0 +1,99 @@
+provider "google" {
+  project = var.gcp_project_id
+  region  = var.gcp_region
+  zone = var.gcp_zone
+}
+
+module "gcp_network" {
+  source      = "./modules/gcp/network"
+  region      = var.gcp_region
+  subnet_cidr = "10.0.0.0/16" // Example CIDR, adjust as needed
+}
+
+module "requester_instance" {
+  source = "./modules/gcp/compute_instances/requester"
+  cloud_init_content = ""
+
+  aws_credentials = local.aws_credentials
+  build_config = local.build_config
+  token_config = local.token_config
+  gcp_config = local.gcp_config
+
+  disk_config = {
+    boot_size = var.bacalhau_boot_disk_size
+    repo_size = var.bacalhau_repo_disk_size
+  }
+
+  requester_static_ip = module.gcp_network.requester_ip
+  requester_instance_type = var.requester_machine_type
+
+  bacalhau_accept_networked_jobs = var.bacalhau_accept_networked_jobs
+  bacalhau_otel_collector_endpoint = var.bacalhau_otel_collector_endpoint
+
+}
+
+module "compute_instance" {
+  source = "./modules/gcp/compute_instances/compute"
+  cloud_init_content = ""
+
+  aws_credentials = local.aws_credentials
+  build_config = local.build_config
+  token_config = local.token_config
+  gcp_config = local.gcp_config
+
+  disk_config = {
+    boot_size = var.bacalhau_boot_disk_size
+    repo_size = var.bacalhau_repo_disk_size
+    local_size = var.bacalhau_local_disk_size
+  }
+
+  // This creates an implicit dependency, meaning Terraform will create the requester_instance before the compute_instance.
+  requester_ip = module.requester_instance.requester_private_ips[0]
+  compute_instance_count = var.compute_count
+  compute_instance_type = var.compute_machine_type
+
+  bacalhau_accept_networked_jobs = var.bacalhau_accept_networked_jobs
+  bacalhau_otel_collector_endpoint = var.bacalhau_otel_collector_endpoint
+}
+
+locals {
+  token_config = {
+    requester_api_token = var.bacalhau_requester_api_token != "" ? var.bacalhau_requester_api_token : random_string.bacalhau_requester_api_token.result
+    compute_api_token = var.bacalhau_compute_api_token != "" ? var.bacalhau_compute_api_token : random_string.bacalhau_compute_api_token.result
+  }
+  build_config = {
+    install_version = var.bacalhau_install_version
+    install_branch = var.bacalhau_install_branch
+    install_commit = var.bacalhau_install_commit
+  }
+  aws_credentials = {
+    access_key_id = var.aws_access_key_id
+    secret_access_key = var.aws_secret_access_key
+  }
+  gcp_config = {
+    network = module.gcp_network.vpc_network_name
+    subnetwork = module.gcp_network.subnetwork_name
+    zone = var.gcp_zone
+    boot_image = var.gcp_boot_image
+  }
+}
+
+resource "random_string" "bacalhau_requester_api_token" {
+  length  = 32
+  special = false
+  # Only generate a new random string if no bacalhau_client_access_token is provided
+  keepers = {
+    token = var.bacalhau_requester_api_token == "" ? "generate" : "provided"
+  }
+}
+
+resource "random_string" "bacalhau_compute_api_token" {
+  length  = 32
+  special = false
+  # Only generate a new random string if no bacalhau_client_access_token is provided
+  keepers = {
+    token = var.bacalhau_compute_api_token == "" ? "generate" : "provided"
+  }
+}
+
+

ops/marketplace-tf/modules/cloud-init/cloud-init.yml

@@ -0,0 +1,56 @@
+write_files:
+  # bacalhau install-script
+  - path: /etc/install-bacalhau.sh
+    encoding: b64
+    owner: root:root
+    permissions: "0600"
+    content: |
+      ${bacalhau_install_script_file}
+
+  # bacalhau config
+  - path: /etc/config.yaml
+    encoding: b64
+    owner: root:root
+    permissions: "0600"
+    content: |
+      ${bacalhau_config_file}
+
+  # bacalhau service file
+  - path: /etc/systemd/system/bacalhau.service
+    encoding: b64
+    owner: root:root
+    permissions: "0600"
+    content: |
+      ${bacalhau_service_file}
+
+  # authn policy file
+  - path: /etc/authn_policy.rego
+    encoding: b64
+    owner: root:root
+    permissions: "0600"
+    content: |
+      ${bacalhau_authn_policy_file}
+
+  # authz policy file
+  - path: /etc/authz_policy.rego
+    encoding: b64
+    owner: root:root
+    permissions: "0600"
+    content: |
+      ${bacalhau_authz_policy_file}
+
+  # otel config file
+  - path: /etc/otel-collector.yaml
+    encoding: b64
+    owner: root:root
+    permissions: "0600"
+    content: |
+      ${otel_config_file}
+
+  # otel service file
+  - path: /etc/systemd/system/otel.service
+    encoding: b64
+    owner: root:root
+    permissions: "0600"
+    content: |
+      ${otel_service_file}

ops/marketplace-tf/modules/gcp/compute_instances/compute/main.tf

@@ -0,0 +1,170 @@
+// define compute instance(s)
+resource "google_compute_instance" "compute" {
+  count = var.compute_instance_count
+  name         = "bacalhau-compute-${count.index + 1}"
+  machine_type = var.compute_instance_type
+  zone         = var.gcp_config.zone
+
+  metadata = {
+    user-data = data.cloudinit_config.compute_cloud_init.rendered
+  }
+
+  metadata_startup_script = local.bacalhau_start_script
+
+  boot_disk {
+    initialize_params {
+      image = var.gcp_config.boot_image
+      size = var.disk_config.boot_size
+    }
+  }
+
+  lifecycle {
+    ignore_changes = [attached_disk]
+  }
+  allow_stopping_for_update = true
+
+  network_interface {
+    network = var.gcp_config.network
+    subnetwork = var.gcp_config.subnetwork
+    access_config {
+      // Ephemeral public IP will be assigned
+    }
+  }
+}
+
+// define disk(s) to contain the bacalhau repo for instance(s)
+resource "google_compute_disk" "bacalhau_repo_disks" {
+  count = var.compute_instance_count
+  name  = "bacalhau-repo-disk-compute-${count.index + 1}"
+  type  = "pd-standard"
+  zone  = var.gcp_config.zone
+  size  = var.disk_config.repo_size
+}
+
+// attach the disk(s) to instance(s)
+resource "google_compute_attached_disk" "attach_bacalhau_repo_disks" {
+  count = var.compute_instance_count
+  disk     = google_compute_disk.bacalhau_repo_disks[count.index].self_link
+  instance = google_compute_instance.compute[count.index].self_link
+  device_name = "bacalhau-repo"
+}
+
+// define disk(s) to contain the bacalhau repo for instance(s)
+resource "google_compute_disk" "bacalhau_local_disks" {
+  count = var.compute_instance_count
+  name  = "bacalhau-local-disk-compute-${count.index + 1}"
+  type  = "pd-standard"
+  zone  = var.gcp_config.zone
+  size  = var.disk_config.local_size
+}
+
+// attach the disk(s) to instance(s)
+resource "google_compute_attached_disk" "attach_bacalhau_local_disks" {
+  count = var.compute_instance_count
+  disk     = google_compute_disk.bacalhau_local_disks[count.index].self_link
+  instance = google_compute_instance.compute[count.index].self_link
+  device_name = "bacalhau-local"
+}
+
+locals {
+  //
+  // templating the bacalhau install script file
+
+  // service env vars
+  bacalhau_env_vars = {
+    LOG_LEVEL                   = "debug"
+    BACALHAU_NODE_LOGGINGMODE   = "default"
+    BACALHAU_DIR                = "/data"
+    BACALHAU_ENVIRONMENT        = "local"
+    // TODO make this a variable
+    OTEL_EXPORTER_OTLP_ENDPOINT = "http://localhost:4318"
+    AWS_ACCESS_KEY_ID           = var.aws_credentials.access_key_id
+    AWS_SECRET_ACCESS_KEY       = var.aws_credentials.secret_access_key
+    # Add more variables here as needed
+  }
+  # Convert the map to the required string format for the systemd service file
+  env_vars_string = join("\n", [for k, v in local.bacalhau_env_vars : "Environment=\"${k}=${v}\""])
+
+  // service bacalhau arguments
+  bacalhau_args = ""
+
+  bacalhau_service_content = templatefile("${path.module}/../../../instance_files/bacalhau.service", {
+    env_vars = local.env_vars_string
+    args = local.bacalhau_args,
+  })
+
+  //
+  // templating the bacalhau config file
+  //
+  compute_config_content = templatefile("${path.module}/../../../instance_files/compute_config.yaml", {
+    requester_ip = var.requester_ip
+    bacalhau_accept_networked_jobs = var.bacalhau_accept_networked_jobs
+    compute_api_token = var.token_config.compute_api_token
+  })
+
+  //
+  // templating the bacalhau start script
+  //
+
+  // inject custom bacalhau install based on variables.
+  // I am sorry reader, terraform requires this be one line
+  bacalhau_install_cmd_content = var.build_config.install_version  != "" ? "release ${var.build_config.install_version}" : var.build_config.install_branch  != "" ? "branch ${var.build_config.install_branch}" : var.build_config.install_commit  != "" ?"commit ${var.build_config.install_commit}" : ""
+  bacalhau_start_script = templatefile("${path.module}/../../../instance_files/start.sh", {
+    node_type = "compute"
+    bacalhau_version_cmd = local.bacalhau_install_cmd_content
+    // Add more arguments as needed
+  })
+
+  bacalhau_install_script_content = file("${path.module}/../../../instance_files/install-bacalhau.sh")
+
+  //
+  // templating otel config file
+  //
+  otel_config_content = templatefile("${path.module}/../../../instance_files/otel-collector.yaml", {
+    bacalhau_otel_collector_endpoint = var.bacalhau_otel_collector_endpoint
+    // add more arguments as needed
+  })
+
+  //
+  // templating otel service file
+  //
+  otel_service_content = templatefile("${path.module}/../../../instance_files/otel.service", {
+    // add more arguments as needed
+  })
+
+  //
+  // templating rego
+  //
+
+  // authn
+  bacalhau_authn_policy_content = templatefile("${path.module}/../../../instance_files/authn_policy.rego", {
+    bacalhau_secret_user_access_token = var.token_config.requester_api_token
+  })
+  // authz
+  bacalhau_authz_policy_content = templatefile("${path.module}/../../../instance_files/authz_policy.rego", {
+    // add more arguments as needed
+  })
+}
+
+
+data "cloudinit_config" "compute_cloud_init" {
+  gzip        = false
+  base64_encode = false
+
+  // provide parameters to cloud-init like files and arguments to scripts in the above part.
+  part {
+    filename     = "cloud-config.yaml"
+    content_type = "text/cloud-config"
+
+    content = templatefile("${path.module}/../../../cloud-init/cloud-init.yml", {
+      bacalhau_install_script_file: base64encode(local.bacalhau_install_script_content)
+      bacalhau_config_file        : base64encode(local.compute_config_content)
+      bacalhau_service_file       : base64encode(local.bacalhau_service_content)
+      bacalhau_authn_policy_file  : base64encode(local.bacalhau_authn_policy_content)
+      bacalhau_authz_policy_file  : base64encode(local.bacalhau_authz_policy_content)
+      otel_config_file            : base64encode(local.otel_config_content)
+      otel_service_file           : base64encode(local.otel_service_content)
+      requester_ip                : var.requester_ip
+    })
+  }
+}

ops/marketplace-tf/modules/gcp/compute_instances/compute/outputs.tf

@@ -0,0 +1,8 @@
+output "compute_private_ips" {
+  value = [for instance in google_compute_instance.compute : instance.network_interface[0].network_ip]
+}
+
+output "compute_public_ips" {
+  value = google_compute_instance.compute.*.network_interface.0.access_config.0.nat_ip
+}
+