From 7a85daffbf3e9d9e54da17df82e7cf92b8acd4e4 Mon Sep 17 00:00:00 2001 From: Gerhard Lazu Date: Sat, 30 Sep 2023 18:35:10 +0100 Subject: [PATCH] =?UTF-8?q?=F0=9F=A4=96=20Daggerize=20&=20deploy=20to=20Fl?= =?UTF-8?q?y.io=20=E2=9C=88=EF=B8=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TL;DR the app is currently running as https://changelog-nightly-2023-10-10.fly.dev/ This adds everything needed to run this app on Fly.io: - [x] a Dagger pipeline captured as Go code - [x] GitHub Actions workflow that runs the Dagger pipeline - [x] `nginx.conf` used to serve the static files - [x] `supercronic` to run the `crontab` (now versioned in this repo!) - includes Sentry.io cron integration via `SENTRY_DSN` - [x] `Procfile` support so that the Fly app runs both nginx & supercronic - hi `foreman`, old friend! - [x] 1Password service account integration A good command to start with is `dagger run go run . build` Use the `--debug` flag to build a local image. Requires `OP_SERVICE_ACCOUNT_TOKEN` to be set. FWIW: https://developer.1password.com/docs/service-accounts/get-started The local image will be exported to `tmp/image.tar`. Test it locally by running: docker load -i tmp/image.tar docker run --rm -p 8081:80 -it To see all available options, run: `dagger run go run .` This was done part of https://github.com/thechangelog/changelog.com/discussions/480 Follow-ups: - Remove Buffer - Update Ruby to a supported version - https://endoflife.date/ruby + https://hub.docker.com/_/ruby/tags - Extract Daggerverse modules Signed-off-by: Gerhard Lazu --- .github/workflows/ship_it.yml | 32 +++++ .tool-versions | 1 + Gemfile | 1 + Gemfile.lock | 4 +- Procfile | 2 + Rakefile | 19 +-- crontab | 2 + env.op | 6 + fly.toml | 22 ++++ flyio.go | 176 +++++++++++++++++++++++++++ go.mod | 21 ++++ go.sum | 50 ++++++++ lib/bq_client.rb | 8 +- lib/db.rb | 2 +- main.go | 109 +++++++++++++++++ nginx.conf | 37 ++++++ pipeline.go | 217 ++++++++++++++++++++++++++++++++++ versions.go | 63 ++++++++++ 18 files changed, 757 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/ship_it.yml create mode 100644 Procfile create mode 100644 crontab create mode 100644 env.op create mode 100644 fly.toml create mode 100644 flyio.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 main.go create mode 100644 nginx.conf create mode 100644 pipeline.go create mode 100644 versions.go diff --git a/.github/workflows/ship_it.yml b/.github/workflows/ship_it.yml new file mode 100644 index 0000000..470f680 --- /dev/null +++ b/.github/workflows/ship_it.yml @@ -0,0 +1,32 @@ +name: "Ship It!" + +concurrency: + # There should only be able one running job per repository / branch combo. + # We do not want multiple deploys running in parallel. + group: ${{ github.repository }}-${{ github.ref_name }} + +on: + push: + branches: + - 'master' + - 'daggerize' + workflow_dispatch: + +jobs: + dagger: + runs-on: ubuntu-latest + steps: + - name: "Checkout code..." + uses: actions/checkout@v3 + + - name: "Setup Go..." + uses: actions/setup-go@v4 + with: + go-version: "1.20" + + - name: "Ship it!" + env: + FLY_API_TOKEN: "${{ secrets.FLY_API_TOKEN }}" + OP_SERVICE_ACCOUNT_TOKEN: "${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}" + run: | + go run . cicd --app "${{ vars.APP }}" diff --git a/.tool-versions b/.tool-versions index 69ed28b..e51da7d 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1 +1,2 @@ ruby 2.3.3 +flyctl 0.1.107 diff --git a/Gemfile b/Gemfile index 88d75cb..8e7558b 100644 --- a/Gemfile +++ b/Gemfile @@ -14,6 +14,7 @@ gem "sqlite3" gem "rest-client" gem "obscenity" gem "whatlanguage" +gem "foreman" group :test do gem "rspec" diff --git a/Gemfile.lock b/Gemfile.lock index a03151a..9746bd1 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -33,6 +33,7 @@ GEM extlib (0.9.16) faraday (0.10.1) multipart-post (>= 1.2, < 3) + foreman (0.87.2) google-api-client (0.8.6) activesupport (>= 3.2) addressable (~> 2.3) @@ -124,6 +125,7 @@ DEPENDENCIES bigquery createsend dotenv + foreman gemoji! hashie obscenity @@ -140,4 +142,4 @@ RUBY VERSION ruby 2.3.3p222 BUNDLED WITH - 1.13.6 + 1.14.6 diff --git a/Procfile b/Procfile new file mode 100644 index 0000000..4e171f7 --- /dev/null +++ b/Procfile @@ -0,0 +1,2 @@ +web: nginx +cron: supercronic -debug crontab diff --git a/Rakefile b/Rakefile index 543a516..ff3bdff 100644 --- a/Rakefile +++ b/Rakefile @@ -5,6 +5,7 @@ require "json" require "dotenv/tasks" require "createsend" require "pry" +require "uri" require_relative "lib/core_ext/date" require_relative "lib/core_ext/integer" @@ -15,11 +16,11 @@ require_relative "lib/repo" require_relative "lib/template" require_relative "lib/buffer" -DATE = Date.parse(ENV["DATE"]) rescue Date.today +DATE = Date.parse(ENV.fetch("DATE")) rescue Date.today DIST_DIR = "dist" -ISSUE_DIR = "#{DIST_DIR}/#{DATE.path}" -ISSUE_URL = "http://nightly.changelog.com/#{DATE.path}" -DATA_FILE = "#{ISSUE_DIR}/data.json" +ISSUE_DIR = File.join(DIST_DIR, DATE.path) +ISSUE_URL = URI.join(ENV.fetch("URL", "https://nightly.changelog.com/"), DATE.path) +DATA_FILE = File.join(ISSUE_DIR, "data.json") THEMES = %w(night day) MAX_REPOS = 15 @@ -108,8 +109,8 @@ namespace :issue do task buffer: [:data] do json = JSON.load File.read DATA_FILE issue = Issue.new DATE, json - gotime = Buffer.new ENV["BUFFER_GO_TIME"], %w(Go), "#golang" - jsparty = Buffer.new ENV["BUFFER_JS_PARTY"], %w(CSS JavaScript JSX PureScript TypeScript Vue) + gotime = Buffer.new ENV.fetch("BUFFER_GO_TIME"), %w(Go), "#golang" + jsparty = Buffer.new ENV.fetch("BUFFER_JS_PARTY"), %w(CSS JavaScript JSX PureScript TypeScript Vue) [gotime, jsparty].each do |buffer| buffer.injest issue.top_new @@ -155,9 +156,9 @@ namespace :issue do json = JSON.load File.read DATA_FILE next unless json["top_new"].any? || json["top_all"].any? - auth = {api_key: ENV["CAMPAIGN_MONITOR_KEY"]} + auth = {api_key: ENV.fetch("CAMPAIGN_MONITOR_KEY")} - CreateSend::List.new(auth, ENV["CAMPAIGN_MONITOR_LIST"]).segments.each do |segment| + CreateSend::List.new(auth, ENV.fetch("CAMPAIGN_MONITOR_LIST")).segments.each do |segment| theme_name = segment.Title.downcase theme_id = segment.SegmentID @@ -165,7 +166,7 @@ namespace :issue do campaign_id = CreateSend::Campaign.create( auth, - ENV["CAMPAIGN_MONITOR_ID"], # client id + ENV.fetch("CAMPAIGN_MONITOR_ID"), # client id "The Hottest Repos on GitHub - #{DATE.day_month_abbrev}", # subject "Nightly – #{DATE} (#{theme_name} theme)", # campaign name "Changelog Nightly", # from name diff --git a/crontab b/crontab new file mode 100644 index 0000000..79f30c3 --- /dev/null +++ b/crontab @@ -0,0 +1,2 @@ +# generate/deliver Nightly at 9:59pm CDT (2:59am UTC) +59 2 * * * rake generate issue:deliver diff --git a/env.op b/env.op new file mode 100644 index 0000000..9725b88 --- /dev/null +++ b/env.op @@ -0,0 +1,6 @@ +export GITHUB_TOKEN={{ op://nightly/app/GITHUB_TOKEN }} +export BQ_CLIENT_ID={{ op://nightly/app/BQ_CLIENT_ID }} +export BQ_SERVICE_EMAIL={{ op://nightly/app//BQ_SERVICE_EMAIL }} +export CAMPAIGN_MONITOR_ID={{ op://nightly/app/CAMPAIGN_MONITOR_ID }} +export CAMPAIGN_MONITOR_KEY={{ op://nightly/app/CAMPAIGN_MONITOR_KEY }} +export CAMPAIGN_MONITOR_LIST={{ op://nightly/app/CAMPAIGN_MONITOR_LIST_PROD }} diff --git a/fly.toml b/fly.toml new file mode 100644 index 0000000..46a8cc4 --- /dev/null +++ b/fly.toml @@ -0,0 +1,22 @@ +# https://fly.io/docs/reference/configuration/ +app = "changelog-nightly-2023-10-10" +primary_region = "ord" + +[env] + # used by supercronic - https://changelog-media.sentry.io/settings/projects/changelog-com/keys/ + SENTRY_DSN = "https://2b1aed8f16f5404cb2bc79b855f2f92d@o546963.ingest.sentry.io/5668962" + DB_DIR = "/app/dist" + +[mounts] + source = "changelog_nightly_2023_10_10" + destination = "/app/dist" + +[http_service] + internal_port = 80 + force_https = true + +[[http_service.checks]] + method = "GET" + path = "/health" + interval = "5s" + timeout = "4s" \ No newline at end of file diff --git a/flyio.go b/flyio.go new file mode 100644 index 0000000..5788d32 --- /dev/null +++ b/flyio.go @@ -0,0 +1,176 @@ +package main + +import ( + "fmt" + "os" + "strings" + "time" + + "dagger.io/dagger" +) + +type Flyio struct { + app string + deployWait string + publishedImageRef string + org string + pipeline *Pipeline + region string + registry string + token *dagger.Secret + version string + volume string + volumeSize string +} + +func newFlyio(p *Pipeline) *Flyio { + token := os.Getenv("FLY_API_TOKEN") + if token == "" { + panic("FLY_API_TOKEN env var must be set") + } + + f := &Flyio{ + app: p.app, + deployWait: "180", + org: "changelog", + pipeline: p, + region: "ord", + registry: "registry.fly.io", + token: p.dag.SetSecret("FLY_API_TOKEN", token), + version: p.tools.Flyctl(), + volumeSize: "2", + } + + f.volume = strings.ReplaceAll(f.app, "-", "_") + + return f +} + +func (f *Flyio) Cli() *dagger.Container { + flyctl := f.pipeline.Container().Pipeline("flyctl"). + From(fmt.Sprintf("flyio/flyctl:v%s", f.version)). + File("/flyctl") + + // we need Alpine so that we can run shell scripts that set secrets in secure way + container := f.pipeline.Container().Pipeline("fly.io"). + From(fmt.Sprintf("alpine:%s", f.pipeline.tools.Alpine())). + WithFile("/usr/local/bin/flyctl", flyctl, dagger.ContainerWithFileOpts{Permissions: 755}). + WithExec([]string{"flyctl", "version"}). + WithSecretVariable("FLY_API_TOKEN", f.token). + WithEnvVariable("RUN_AT", time.Now().String()). + WithNewFile("fly.toml", dagger.ContainerWithNewFileOpts{ + Contents: f.Config(), + }) + + _, err := container.File("fly.toml").Export(f.pipeline.ctx, "fly.toml") + if err != nil { + panic(err) + } + + return container +} + +func (f *Flyio) Config() string { + return fmt.Sprintf(`# https://fly.io/docs/reference/configuration/ +app = "%s" +primary_region = "%s" + +[env] + # used by supercronic - https://changelog-media.sentry.io/settings/projects/changelog-com/keys/ + SENTRY_DSN = "https://2b1aed8f16f5404cb2bc79b855f2f92d@o546963.ingest.sentry.io/5668962" + DB_DIR = "/app/dist" + +[mounts] + source = "%s" + destination = "/app/dist" + +[http_service] + internal_port = 80 + force_https = true + +[[http_service.checks]] + method = "GET" + path = "/health" + interval = "5s" + timeout = "4s"`, f.app, f.region, f.volume) +} + +func (f *Flyio) App() *Flyio { + cli := f.Cli() + + _, err := cli. + WithExec([]string{"flyctl", "status"}). + Sync(f.pipeline.ctx) + if err != nil { + _, err = cli. + WithExec([]string{"flyctl", "apps", "create", f.app, "--org", f.org}). + WithExec([]string{"flyctl", "volume", "create", f.volume, "--yes", "--region", f.region, "--size", f.volumeSize}). + Sync(f.pipeline.ctx) + if err != nil { + panic(err) + } + } + + return f +} + +func (f *Flyio) ImageRef() string { + gitSHA := os.Getenv("GITHUB_SHA") + if gitSHA == "" { + gitSHA = "dev" + } + + return fmt.Sprintf("%s/%s:%s", f.registry, f.app, gitSHA) +} + +func (f *Flyio) Publish() *Flyio { + var err error + + f.publishedImageRef, err = f.pipeline.workspace. + Pipeline("publish"). + WithRegistryAuth(f.registry, "x", f.token). + Publish(f.pipeline.ctx, f.ImageRef()) + if err != nil { + panic(err) + } + + return f +} + +func (f *Flyio) Secrets(secrets map[string]string) *Flyio { + cli := f.Cli().Pipeline("secrets") + var envs []string + for name, secret := range secrets { + cli = cli.WithSecretVariable(name, f.pipeline.dag.SetSecret(name, secret)) + envs = append(envs, fmt.Sprintf(`%s="$%s"`, name, name)) + } + + _, err := cli.WithNewFile("/flyctl-set-secrets-and-keep-hidden.sh", dagger.ContainerWithNewFileOpts{ + Contents: fmt.Sprintf(`#!/bin/sh +flyctl secrets set %s --app %s --stage`, strings.Join(envs, " "), f.app), + Permissions: 755, + }). + WithExec([]string{"/flyctl-set-secrets-and-keep-hidden.sh"}). + Sync(f.pipeline.ctx) + if err != nil { + panic(err) + } + + return f +} + +func (f *Flyio) Deploy() *Flyio { + _, err := f.Cli().Pipeline("deploy"). + WithExec([]string{ + "flyctl", "deploy", "--now", + "--app", f.app, + "--image", f.publishedImageRef, + "--wait-timeout", f.deployWait, + }). + Sync(f.pipeline.ctx) + if err != nil { + panic(err) + } + + return f +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..f5e992c --- /dev/null +++ b/go.mod @@ -0,0 +1,21 @@ +module github.com/thechangelog/nightly + +go 1.20 + +require ( + dagger.io/dagger v0.8.8 + github.com/urfave/cli/v2 v2.25.7 +) + +require ( + github.com/99designs/gqlgen v0.17.31 // indirect + github.com/Khan/genqlient v0.6.0 // indirect + github.com/adrg/xdg v0.4.0 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/vektah/gqlparser/v2 v2.5.6 // indirect + github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect + golang.org/x/sync v0.3.0 // indirect + golang.org/x/sys v0.12.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..b8d30fd --- /dev/null +++ b/go.sum @@ -0,0 +1,50 @@ +dagger.io/dagger v0.8.8 h1:vzQVbRJ2xg+uKmcg3Y/FHn1Ry513JVOACn0gDBFFEVA= +dagger.io/dagger v0.8.8/go.mod h1:d6yRd1k37fs5fCOlLKAkSB//qB26+z23UiikIb0z+uU= +github.com/99designs/gqlgen v0.17.31 h1:VncSQ82VxieHkea8tz11p7h/zSbvHSxSDZfywqWt158= +github.com/99designs/gqlgen v0.17.31/go.mod h1:i4rEatMrzzu6RXaHydq1nmEPZkb3bKQsnxNRHS4DQB4= +github.com/Khan/genqlient v0.6.0 h1:Bwb1170ekuNIVIwTJEqvO8y7RxBxXu639VJOkKSrwAk= +github.com/Khan/genqlient v0.6.0/go.mod h1:rvChwWVTqXhiapdhLDV4bp9tz/Xvtewwkon4DpWWCRM= +github.com/adrg/xdg v0.4.0 h1:RzRqFcjH4nE5C6oTAxhBtoE2IRyjBSa62SCbyPidvls= +github.com/adrg/xdg v0.4.0/go.mod h1:N6ag73EX4wyxeaoeHctc1mas01KZgsj5tYiAIwqJE/E= +github.com/agnivade/levenshtein v1.1.1/go.mod h1:veldBMzWxcCG2ZvUTKD2kJNRdCk5hVbJomOvKkmgYbo= +github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNgfBlViaCIJKLlCJ6/fmUseuG0wVQ= +github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= +github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE= +github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= +github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= +github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs= +github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= +github.com/vektah/gqlparser/v2 v2.5.6 h1:Ou14T0N1s191eRMZ1gARVqohcbe1e8FrcONScsq8cRU= +github.com/vektah/gqlparser/v2 v2.5.6/go.mod h1:z8xXUff237NntSuH8mLFijZ+1tjV1swDbpDqjJmk6ME= +github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= +github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/lib/bq_client.rb b/lib/bq_client.rb index 0721d2b..c4f74d1 100644 --- a/lib/bq_client.rb +++ b/lib/bq_client.rb @@ -6,10 +6,10 @@ class BqClient def initialize day @day = day @bq = BigQuery::Client.new({ - "client_id" => ENV["BQ_CLIENT_ID"], - "service_email" => ENV["BQ_SERVICE_EMAIL"], - "key" => ENV["BQ_KEY"], - "project_id" => ENV["BQ_PROJECT_ID"] + "client_id" => ENV.fetch("BQ_CLIENT_ID"), + "service_email" => ENV.fetch("BQ_SERVICE_EMAIL"), + "key" => ENV.fetch("BQ_KEY", "bq-key.p12"), + "project_id" => ENV.fetch("BQ_PROJECT_ID", "changelog-nightly") }) end diff --git a/lib/db.rb b/lib/db.rb index 870b213..a3d0ec2 100644 --- a/lib/db.rb +++ b/lib/db.rb @@ -2,7 +2,7 @@ require "sqlite3" module DB - @gh = Sequel.sqlite "github.db" + @gh = Sequel.sqlite(File.join(ENV.fetch("DB_DIR", "."), "github.db")) def self.create unless @gh.table_exists? :listings diff --git a/main.go b/main.go new file mode 100644 index 0000000..3b05bb4 --- /dev/null +++ b/main.go @@ -0,0 +1,109 @@ +package main + +import ( + "context" + "log" + "os" + "time" + + "dagger.io/dagger" + "github.com/urfave/cli/v2" +) + +func main() { + ctx := context.Background() + dag, err := dagger.Connect(ctx, dagger.WithLogOutput(os.Stderr)) + if err != nil { + panic(err) + } + defer dag.Close() + + app := &cli.App{ + Name: "nightly", + Usage: "Changelog Nightly CI/CD pipeline commands", + Version: "v2023.10.10", + Compiled: time.Now(), + Authors: []*cli.Author{ + { + Name: "Jerod Santo", + Email: "jerod@changelog.com", + }, + { + Name: "Gerhard Lazu", + Email: "gerhard@changelog.com", + }, + }, + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "nocache", + Aliases: []string{"n"}, + Usage: "Bust Dagger ops cache", + EnvVars: []string{"NOCACHE"}, + }, + &cli.BoolFlag{ + Name: "debug", + Aliases: []string{"d"}, + Usage: "Debug command", + EnvVars: []string{"DEBUG"}, + }, + &cli.StringFlag{ + Name: "platform", + Aliases: []string{"p"}, + Usage: "Runtime platform", + Value: "linux/amd64", + EnvVars: []string{"PLATFORM"}, + }, + }, + Commands: []*cli.Command{ + { + Name: "build", + Aliases: []string{"b"}, + Usage: "Builds container image", + Action: func(cCtx *cli.Context) error { + newPipeline(ctx, cCtx, dag). + Build() + + return nil + }, + }, + { + Name: "test", + Aliases: []string{"t"}, + Usage: "Runs tests", + Action: func(cCtx *cli.Context) error { + newPipeline(ctx, cCtx, dag). + Build(). + Test() + + return nil + }, + }, + { + Name: "cicd", + Usage: "Runs the entire CI/CD pipeline", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "app", + Aliases: []string{"a"}, + Usage: "Fly.io app name", + EnvVars: []string{"APP"}, + Required: true, + }, + }, + Action: func(cCtx *cli.Context) error { + newPipeline(ctx, cCtx, dag). + Build(). + Test(). + Prod(). + Deploy() + + return nil + }, + }, + }, + } + + if err := app.Run(os.Args); err != nil { + log.Fatal(err) + } +} diff --git a/nginx.conf b/nginx.conf new file mode 100644 index 0000000..8a3cea0 --- /dev/null +++ b/nginx.conf @@ -0,0 +1,37 @@ +daemon off; +user nginx; +worker_processes auto; + +error_log /dev/stderr warn; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + +http { + include mime.types; + default_type text/plain; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" "$http_x_forwarded_for"'; + access_log /dev/stdout main; + + sendfile on; + + server_tokens off; + + server { + listen 80; + listen [::]:80; + server_name localhost; + + location / { + root /app/dist; + try_files $uri $uri/index.html $uri.html =404; + } + + location /health { + return 204; + } + } +} diff --git a/pipeline.go b/pipeline.go new file mode 100644 index 0000000..e386273 --- /dev/null +++ b/pipeline.go @@ -0,0 +1,217 @@ +package main + +import ( + "context" + "fmt" + "os" + "strings" + "time" + + "dagger.io/dagger" + "github.com/urfave/cli/v2" +) + +type Pipeline struct { + app string + ctx context.Context + dag *dagger.Client + debug bool + nocache bool + platform dagger.Platform + workspace *dagger.Container + tools *Versions +} + +func newPipeline(ctx context.Context, cCtx *cli.Context, dag *dagger.Client) *Pipeline { + p := &Pipeline{ + app: cCtx.String("app"), + ctx: ctx, + platform: dagger.Platform(cCtx.String("platform")), + debug: cCtx.Bool("debug"), + nocache: cCtx.Bool("nocache"), + dag: dag, + tools: currentToolVersions(), + } + + p.workspace = p.Container() + + return p +} + +func (p *Pipeline) OK() *Pipeline { + var err error + p.workspace, err = p.workspace.Sync(p.ctx) + if err != nil { + panic(err) + } + return p +} + +func (p *Pipeline) platformKebab() string { + return strings.ReplaceAll(string(p.platform), "/", "-") +} + +func (p *Pipeline) platformSnake() string { + return strings.ReplaceAll(string(p.platform), "/", "_") +} + +func (p *Pipeline) Container() *dagger.Container { + return p.dag.Container(dagger.ContainerOpts{ + Platform: p.platform, + }) +} + +func (p *Pipeline) Build() *Pipeline { + p.workspace = p.workspace.Pipeline("container image"). + From(fmt.Sprintf("ruby:%s-alpine", p.tools.Ruby())). + WithExec([]string{"ruby", "--version"}). + WithExec([]string{"apk", "update"}). + WithExec([]string{"apk", "add", "git", "build-base", "sqlite-dev", "bash"}) + + if p.nocache { + p.workspace = p.workspace.WithEnvVariable("DAGGER_CACHE_BUSTED_AT", time.Now().String()) + } + + app := p.dag.Host().Directory(".", dagger.HostDirectoryOpts{ + Include: []string{ + "images", + "lib", + "styles", + "views", + "Gemfile", + "Gemfile.lock", + "LICENSE", + "Procfile", + "Rakefile", + "env.op", + }}) + + pathWithBundleBin := "/usr/local/bundle/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + + p.workspace = p.workspace. + WithDirectory("/app", app). + WithWorkdir("/app"). + WithExec([]string{"bundle", "install", "--frozen", "--without=test"}). + WithEnvVariable("PATH", pathWithBundleBin). + WithNewFile("/etc/profile.d/append-bundle-bin-to-path.sh", dagger.ContainerWithNewFileOpts{ + Contents: fmt.Sprintf("export PATH=%s", pathWithBundleBin), + }). + WithExec([]string{"rake", "-T"}). + WithExec([]string{"foreman", "check"}). + WithEntrypoint(nil). + WithDefaultArgs() + + p.workspace = p.workspace. + WithExec([]string{"apk", "add", "nginx"}). + WithFile("/etc/nginx/nginx.conf", p.dag.Host().File("nginx.conf")). + WithExec([]string{"nginx", "-t"}) + + p.workspace = p.workspace. + WithFile("/usr/local/bin/supercronic", + p.supercronic(), dagger.ContainerWithFileOpts{Permissions: 555}). + WithFile("/app/crontab", + p.dag.Host().File("crontab")). + WithExec([]string{"supercronic", "-test", "crontab"}) + + p.workspace = p.workspace. + WithFile("/usr/local/bin/op", + p.op(), dagger.ContainerWithFileOpts{Permissions: 555}). + WithExec([]string{"op", "--version"}) + + if p.debug { + token := os.Getenv("OP_SERVICE_ACCOUNT_TOKEN") + if token == "" { + panic("OP_SERVICE_ACCOUNT_TOKEN env var must be set") + } + + p.workspace = p.workspace.Pipeline("generate with local config"). + WithSecretVariable("OP_SERVICE_ACCOUNT_TOKEN", p.dag.SetSecret("OP_SERVICE_ACCOUNT_TOKEN", token)). + WithExec([]string{"op", "inject", "--in-file", "env.op", "--out-file", ".env"}). + WithExec([]string{"op", "read", "--out-file", "bq-key.p12", "op://nightly/app/bq-key.p12"}). + WithExec([]string{"op", "read", "--out-file", "github.db", "--force", "op://nightly/app/github.db"}). + WithExec([]string{"apk", "add", "tmux", "vim", "htop", "strace"}). + WithExec([]string{"bash", "-c", `DATE=2023-10-10 rake generate`}). + WithEntrypoint([]string{"tmux"}) + + _, err := p.workspace.Pipeline("export tmp/image.tar"). + Export(p.ctx, "tmp/image.tar") + if err != nil { + panic(err) + } + } + + return p.OK() +} + +func (p *Pipeline) Test() *Pipeline { + if p.nocache { + p.workspace = p.workspace.WithEnvVariable("DAGGER_CACHE_BUSTED_AT", time.Now().String()) + } + + p.workspace = p.workspace. + WithExec([]string{"bundle", "install", "--frozen", "--with=test"}). + WithDirectory("/app/spec", p.dag.Host().Directory("spec")). + WithExec([]string{"rspec"}) + + return p.OK() +} + +func (p *Pipeline) Prod() *Pipeline { + if p.nocache { + p.workspace = p.workspace.WithEnvVariable("DAGGER_CACHE_BUSTED_AT", time.Now().String()) + } + + p.workspace = p.workspace.WithNewFile("/entrypoint.sh", dagger.ContainerWithNewFileOpts{ + Contents: `#!/bin/bash +set -ex +op inject --in-file env.op --out-file .env +op read --out-file bq-key.p12 --force op://nightly/app/bq-key.p12 +foreman start`, + Permissions: 555, + }). + WithEntrypoint([]string{"/entrypoint.sh"}) + + return p.OK() +} + +func (p *Pipeline) Deploy() *Pipeline { + token := os.Getenv("OP_SERVICE_ACCOUNT_TOKEN") + if token == "" { + panic("OP_SERVICE_ACCOUNT_TOKEN env var must be set") + } + + secretEnvs := map[string]string{ + "OP_SERVICE_ACCOUNT_TOKEN": token, + } + + newFlyio(p). + App(). + Publish(). + Secrets(secretEnvs). + Deploy() + + return p +} + +func (p *Pipeline) op() *dagger.File { + file := fmt.Sprintf("op_%s_v%s.zip", p.platformSnake(), p.tools._1Password()) + url := fmt.Sprintf("https://cache.agilebits.com/dist/1P/op2/pkg/v%s/%s", p.tools._1Password(), file) + + // https://hub.docker.com/layers/library/alpine/3.18.4/images/sha256-48d9183eb12a05c99bcc0bf44a003607b8e941e1d4f41f9ad12bdcc4b5672f86 + return p.Container().From("alpine@sha256:48d9183eb12a05c99bcc0bf44a003607b8e941e1d4f41f9ad12bdcc4b5672f86"). + WithFile(file, p.dag.HTTP(url)). + WithExec([]string{"unzip", file}). + WithExec([]string{"mv", "op", "/usr/local/bin/op"}). + WithExec([]string{"op", "--version"}). + File("/usr/local/bin/op") +} + +func (p *Pipeline) supercronic() *dagger.File { + return p.dag.HTTP( + fmt.Sprintf( + "https://github.com/aptible/supercronic/releases/download/v%s/supercronic-%s", + p.tools.Supercronic(), + p.platformKebab(), + ), + ) +} diff --git a/versions.go b/versions.go new file mode 100644 index 0000000..73724a1 --- /dev/null +++ b/versions.go @@ -0,0 +1,63 @@ +package main + +import ( + "bufio" + "os" + "path/filepath" + "strings" +) + +type Versions struct { + toolVersions map[string]string +} + +// https://www.ruby-lang.org/en/downloads/releases/ || asdf list all ruby +func (v *Versions) Ruby() string { + return v.toolVersions["ruby"] +} + +// https://hub.docker.com/r/flyio/flyctl/tags +func (v *Versions) Flyctl() string { + return v.toolVersions["flyctl"] +} + +// https://github.com/aptible/supercronic/releases +func (v *Versions) Supercronic() string { + return "0.2.26" +} + +// https://app-updates.agilebits.com/product_history/CLI2 +func (v *Versions) _1Password() string { + return "2.21.0" +} + +// https://hub.docker.com/_/alpine/tags +func (v *Versions) Alpine() string { + return "3.18.4@sha256:48d9183eb12a05c99bcc0bf44a003607b8e941e1d4f41f9ad12bdcc4b5672f86" +} + +func currentToolVersions() *Versions { + return &Versions{ + toolVersions: toolVersions(), + } +} + +func toolVersions() map[string]string { + wd, err := os.Getwd() + if err != nil { + panic(err) + } + versions, err := os.Open(filepath.Join(wd, ".tool-versions")) + if err != nil { + panic(err) + } + toolVersions := make(map[string]string) + scanner := bufio.NewScanner(versions) + for scanner.Scan() { + line := scanner.Text() + toolAndVersion := strings.Split(line, " ") + toolVersions[toolAndVersion[0]] = toolAndVersion[1] + } + + return toolVersions +}