🤖 Daggerize & deploy to Fly.io ✈️

TL;DR the app is currently running as https://changelog-nightly-2023-10-10.fly.dev/

This adds everything needed to run this app on Fly.io:
- [x] a Dagger pipeline captured as Go code
- [x] GitHub Actions workflow that runs the Dagger pipeline
- [x] `nginx.conf` used to serve the static files
- [x] `supercronic` to run the `crontab` (now versioned in this repo!)
  - includes Sentry.io cron integration via `SENTRY_DSN`
- [x] `Procfile` support so that the Fly app runs both nginx & supercronic
  - hi `foreman`, old friend!
- [x] 1Password service account integration

A good command to start with is `dagger run go run . build`

Use the `--debug` flag to build a local image. Requires
`OP_SERVICE_ACCOUNT_TOKEN` to be set. FWIW:
https://developer.1password.com/docs/service-accounts/get-started

The local image will be exported to `tmp/image.tar`. Test it locally by
running:

  docker load -i tmp/image.tar
  docker run --rm -p 8081:80 -it <sha256:...>

To see all available options, run: `dagger run go run .`

This was done part of thechangelog/changelog.com#480

Follow-ups:
- Remove Buffer
- Update Ruby to a supported version
  - https://endoflife.date/ruby + https://hub.docker.com/_/ruby/tags
- Extract Daggerverse modules

Signed-off-by: Gerhard Lazu <[email protected]>

.github/workflows/ship_it.yml

@@ -0,0 +1,33 @@
+name: "Ship It!"
+
+concurrency:
+  # There should only be able one running job per repository / branch combo.
+  # We do not want multiple deploys running in parallel.
+  group: ${{ github.repository }}-${{ github.ref_name }}
+
+on:
+  push:
+    branches:
+      - 'master'
+      - 'daggerize'
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  dagger:
+      runs-on: ubuntu-latest
+      steps:
+        - name: "Checkout code..."
+          uses: actions/checkout@v3
+
+        - name: "Setup Go..."
+          uses: actions/setup-go@v4
+          with:
+            go-version: "1.20"
+
+        - name: "Ship it!"
+          env:
+            FLY_API_TOKEN: "${{ secrets.FLY_API_TOKEN }}"
+            OP_SERVICE_ACCOUNT_TOKEN: "${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}"
+          run: |
+            go run . cicd --app "${{ vars.APP }}"

.tool-versions

@@ -1 +1,2 @@
 ruby 2.3.3
+flyctl 0.1.107

Gemfile

@@ -14,6 +14,7 @@ gem "sqlite3"
 gem "rest-client"
 gem "obscenity"
 gem "whatlanguage"
+gem "foreman"
 
 group :test do
   gem "rspec"

Gemfile.lock

@@ -33,6 +33,7 @@ GEM
     extlib (0.9.16)
     faraday (0.10.1)
       multipart-post (>= 1.2, < 3)
+    foreman (0.87.2)
     google-api-client (0.8.6)
       activesupport (>= 3.2)
       addressable (~> 2.3)
@@ -124,6 +125,7 @@ DEPENDENCIES
   bigquery
   createsend
   dotenv
+  foreman
   gemoji!
   hashie
   obscenity
@@ -140,4 +142,4 @@ RUBY VERSION
    ruby 2.3.3p222
 
 BUNDLED WITH
-   1.13.6
+   1.14.6

Procfile

@@ -0,0 +1,2 @@
+web: nginx
+cron: supercronic -debug crontab

Rakefile

@@ -5,6 +5,7 @@ require "json"
 require "dotenv/tasks"
 require "createsend"
 require "pry"
+require "uri"
 
 require_relative "lib/core_ext/date"
 require_relative "lib/core_ext/integer"
@@ -15,11 +16,11 @@ require_relative "lib/repo"
 require_relative "lib/template"
 require_relative "lib/buffer"
 
-DATE      = Date.parse(ENV["DATE"]) rescue Date.today
+DATE      = Date.parse(ENV.fetch("DATE")) rescue Date.today
 DIST_DIR  = "dist"
-ISSUE_DIR = "#{DIST_DIR}/#{DATE.path}"
-ISSUE_URL = "http://nightly.changelog.com/#{DATE.path}"
-DATA_FILE = "#{ISSUE_DIR}/data.json"
+ISSUE_DIR = File.join(DIST_DIR, DATE.path)
+ISSUE_URL = URI.join(ENV.fetch("URL", "https://nightly.changelog.com/"), DATE.path)
+DATA_FILE = File.join(ISSUE_DIR, "data.json")
 THEMES    = %w(night day)
 MAX_REPOS = 15
 
@@ -108,8 +109,8 @@ namespace :issue do
   task buffer: [:data] do
     json = JSON.load File.read DATA_FILE
     issue = Issue.new DATE, json
-    gotime = Buffer.new ENV["BUFFER_GO_TIME"], %w(Go), "#golang"
-    jsparty = Buffer.new ENV["BUFFER_JS_PARTY"], %w(CSS JavaScript JSX PureScript TypeScript Vue)
+    gotime = Buffer.new ENV.fetch("BUFFER_GO_TIME"), %w(Go), "#golang"
+    jsparty = Buffer.new ENV.fetch("BUFFER_JS_PARTY"), %w(CSS JavaScript JSX PureScript TypeScript Vue)
 
     [gotime, jsparty].each do |buffer|
       buffer.injest issue.top_new
@@ -155,17 +156,17 @@ namespace :issue do
     json = JSON.load File.read DATA_FILE
     next unless json["top_new"].any? || json["top_all"].any?
 
-    auth = {api_key: ENV["CAMPAIGN_MONITOR_KEY"]}
+    auth = {api_key: ENV.fetch("CAMPAIGN_MONITOR_KEY")}
 
-    CreateSend::List.new(auth, ENV["CAMPAIGN_MONITOR_LIST"]).segments.each do |segment|
+    CreateSend::List.new(auth, ENV.fetch("CAMPAIGN_MONITOR_LIST")).segments.each do |segment|
       theme_name = segment.Title.downcase
       theme_id = segment.SegmentID
 
       next unless THEMES.include? theme_name
 
       campaign_id = CreateSend::Campaign.create(
         auth,
-        ENV["CAMPAIGN_MONITOR_ID"], # client id
+        ENV.fetch("CAMPAIGN_MONITOR_ID"), # client id
         "The Hottest Repos on GitHub - #{DATE.day_month_abbrev}", # subject
         "Nightly – #{DATE} (#{theme_name} theme)", # campaign name
         "Changelog Nightly", # from name

crontab

@@ -0,0 +1,2 @@
+# generate/deliver Nightly at 9:59pm CDT (2:59am UTC)
+59 2 * * * rake generate deliver

env.op

@@ -0,0 +1,6 @@
+export GITHUB_TOKEN={{ op://nightly/app/GITHUB_TOKEN }}
+export BQ_CLIENT_ID={{ op://nightly/app/BQ_CLIENT_ID }}
+export BQ_SERVICE_EMAIL={{ op://nightly/app//BQ_SERVICE_EMAIL }}
+export CAMPAIGN_MONITOR_ID={{ op://nightly/app/CAMPAIGN_MONITOR_ID }}
+export CAMPAIGN_MONITOR_KEY={{ op://nightly/app/CAMPAIGN_MONITOR_KEY }}
+export CAMPAIGN_MONITOR_LIST={{ op://nightly/app/CAMPAIGN_MONITOR_LIST_TEST }}

fly.toml

@@ -0,0 +1,22 @@
+# https://fly.io/docs/reference/configuration/
+app = "changelog-nightly-2023-10-10"
+primary_region = "ord"
+
+[env]
+		# used by supercronic - https://changelog-media.sentry.io/settings/projects/changelog-com/keys/
+		SENTRY_DSN = "https://[email protected]/5668962"
+		DB_DIR = "/app/dist"
+
+[mounts]
+	source = "changelog_nightly_2023_10_10"
+	destination = "/app/dist"
+
+[http_service]
+	internal_port = 80
+	force_https = true
+
+[[http_service.checks]]
+	method = "GET"
+	path = "/health"
+	interval = "5s"
+	timeout = "4s"

flyio.go

@@ -0,0 +1,176 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"time"
+
+	"dagger.io/dagger"
+)
+
+type Flyio struct {
+	app               string
+	deployWait        string
+	publishedImageRef string
+	org               string
+	pipeline          *Pipeline
+	region            string
+	registry          string
+	token             *dagger.Secret
+	version           string
+	volume            string
+	volumeSize        string
+}
+
+func newFlyio(p *Pipeline) *Flyio {
+	token := os.Getenv("FLY_API_TOKEN")
+	if token == "" {
+		panic("FLY_API_TOKEN env var must be set")
+	}
+
+	f := &Flyio{
+		app:        p.app,
+		deployWait: "180",
+		org:        "changelog",
+		pipeline:   p,
+		region:     "ord",
+		registry:   "registry.fly.io",
+		token:      p.dag.SetSecret("FLY_API_TOKEN", token),
+		version:    p.tools.Flyctl(),
+		volumeSize: "2",
+	}
+
+	f.volume = strings.ReplaceAll(f.app, "-", "_")
+
+	return f
+}
+
+func (f *Flyio) Cli() *dagger.Container {
+	flyctl := f.pipeline.Container().Pipeline("flyctl").
+		From(fmt.Sprintf("flyio/flyctl:v%s", f.version)).
+		File("/flyctl")
+
+	// we need Alpine so that we can run shell scripts that set secrets in secure way
+	container := f.pipeline.Container().Pipeline("fly.io").
+		From(fmt.Sprintf("alpine:%s", f.pipeline.tools.Alpine())).
+		WithFile("/usr/local/bin/flyctl", flyctl, dagger.ContainerWithFileOpts{Permissions: 755}).
+		WithExec([]string{"flyctl", "version"}).
+		WithSecretVariable("FLY_API_TOKEN", f.token).
+		WithEnvVariable("RUN_AT", time.Now().String()).
+		WithNewFile("fly.toml", dagger.ContainerWithNewFileOpts{
+			Contents: f.Config(),
+		})
+
+	_, err := container.File("fly.toml").Export(f.pipeline.ctx, "fly.toml")
+	if err != nil {
+		panic(err)
+	}
+
+	return container
+}
+
+func (f *Flyio) Config() string {
+	return fmt.Sprintf(`# https://fly.io/docs/reference/configuration/
+app = "%s"
+primary_region = "%s"
+
+[env]
+		# used by supercronic - https://changelog-media.sentry.io/settings/projects/changelog-com/keys/
+		SENTRY_DSN = "https://[email protected]/5668962"
+		DB_DIR = "/app/dist"
+
+[mounts]
+	source = "%s"
+	destination = "/app/dist"
+
+[http_service]
+	internal_port = 80
+	force_https = true
+
+[[http_service.checks]]
+	method = "GET"
+	path = "/health"
+	interval = "5s"
+	timeout = "4s"`, f.app, f.region, f.volume)
+}
+
+func (f *Flyio) App() *Flyio {
+	cli := f.Cli()
+
+	_, err := cli.
+		WithExec([]string{"flyctl", "status"}).
+		Sync(f.pipeline.ctx)
+	if err != nil {
+		_, err = cli.
+			WithExec([]string{"flyctl", "apps", "create", f.app, "--org", f.org}).
+			WithExec([]string{"flyctl", "volume", "create", f.volume, "--yes", "--region", f.region, "--size", f.volumeSize}).
+			Sync(f.pipeline.ctx)
+		if err != nil {
+			panic(err)
+		}
+	}
+
+	return f
+}
+
+func (f *Flyio) ImageRef() string {
+	gitSHA := os.Getenv("GITHUB_SHA")
+	if gitSHA == "" {
+		gitSHA = "dev"
+	}
+
+	return fmt.Sprintf("%s/%s:%s", f.registry, f.app, gitSHA)
+}
+
+func (f *Flyio) Publish() *Flyio {
+	var err error
+
+	f.publishedImageRef, err = f.pipeline.workspace.
+		Pipeline("publish").
+		WithRegistryAuth(f.registry, "x", f.token).
+		Publish(f.pipeline.ctx, f.ImageRef())
+	if err != nil {
+		panic(err)
+	}
+
+	return f
+}
+
+func (f *Flyio) Secrets(secrets map[string]string) *Flyio {
+	cli := f.Cli().Pipeline("secrets")
+	var envs []string
+	for name, secret := range secrets {
+		cli = cli.WithSecretVariable(name, f.pipeline.dag.SetSecret(name, secret))
+		envs = append(envs, fmt.Sprintf(`%s="$%s"`, name, name))
+	}
+
+	_, err := cli.WithNewFile("/flyctl-set-secrets-and-keep-hidden.sh", dagger.ContainerWithNewFileOpts{
+		Contents: fmt.Sprintf(`#!/bin/sh
+flyctl secrets set %s --app %s --stage`, strings.Join(envs, " "), f.app),
+		Permissions: 755,
+	}).
+		WithExec([]string{"/flyctl-set-secrets-and-keep-hidden.sh"}).
+		Sync(f.pipeline.ctx)
+	if err != nil {
+		panic(err)
+	}
+
+	return f
+}
+
+func (f *Flyio) Deploy() *Flyio {
+	_, err := f.Cli().Pipeline("deploy").
+		WithExec([]string{
+			"flyctl", "deploy", "--now",
+			"--app", f.app,
+			"--image", f.publishedImageRef,
+			"--wait-timeout", f.deployWait,
+		}).
+		Sync(f.pipeline.ctx)
+	if err != nil {
+		panic(err)
+	}
+
+	return f
+}

go.mod

@@ -0,0 +1,21 @@
+module github.com/thechangelog/nightly
+
+go 1.20
+
+require (
+	dagger.io/dagger v0.8.8
+	github.com/urfave/cli/v2 v2.25.7
+)
+
+require (
+	github.com/99designs/gqlgen v0.17.31 // indirect
+	github.com/Khan/genqlient v0.6.0 // indirect
+	github.com/adrg/xdg v0.4.0 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/vektah/gqlparser/v2 v2.5.6 // indirect
+	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
+	golang.org/x/sync v0.3.0 // indirect
+	golang.org/x/sys v0.12.0 // indirect
+)