From 3f3ab701c78128e46d7bd7f7f21019526accda63 Mon Sep 17 00:00:00 2001 From: Kor Nielsen Date: Fri, 16 Aug 2024 22:44:39 -0700 Subject: [PATCH] ci-tools: Add rtool subcommand to download artifact This is useful to keep the runner images up-to-date. --- ci-tools/github-runner/artifact.go | 67 +++++++++++++++++++++++ ci-tools/github-runner/cmd/rtool/main.go | 68 +++++++++++++++++++++++- 2 files changed, 134 insertions(+), 1 deletion(-) create mode 100644 ci-tools/github-runner/artifact.go diff --git a/ci-tools/github-runner/artifact.go b/ci-tools/github-runner/artifact.go new file mode 100644 index 0000000000..03f9ce6670 --- /dev/null +++ b/ci-tools/github-runner/artifact.go @@ -0,0 +1,67 @@ +// Licensed under the Apache-2.0 license + +package runner + +import ( + "context" + "errors" + "io" + "log" + "net/http" + "os" + + "github.com/google/go-github/v53/github" +) + +func findArtifact(artifacts *github.ArtifactList, name string) (*github.Artifact, error) { + for _, artifact := range artifacts.Artifacts { + if artifact.GetName() == name { + return artifact, nil + } + } + return nil, errors.New("could not find artifact") +} + +func DownloadArtifact(ctx context.Context, client *github.Client, workflowFilename string, artifactName string) error { + repo := "caliptra-sw" + workflow, _, err := client.Actions.GetWorkflowByFileName(ctx, githubOrg, repo, workflowFilename) + if err != nil { + return err + } + runs, _, err := client.Actions.ListWorkflowRunsByID(ctx, githubOrg, repo, workflow.GetID(), &github.ListWorkflowRunsOptions{ + Branch: "main", + Status: "completed", + }) + if err != nil { + return err + } + if len(runs.WorkflowRuns) == 0 { + return errors.New("no workflow runs") + } + artifacts, _, err := client.Actions.ListWorkflowRunArtifacts(ctx, githubOrg, repo, runs.WorkflowRuns[0].GetID(), &github.ListOptions{}) + if err != nil { + return err + } + artifact, err := findArtifact(artifacts, artifactName) + if err != nil { + return err + } + url, _, err := client.Actions.DownloadArtifact(ctx, githubOrg, repo, artifact.GetID(), true) + if err != nil { + return err + } + resp, err := http.Get(url.String()) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + log.Fatalf("Response failed with status code: %d and body %s\n", resp.StatusCode, body) + } + _, err = io.Copy(os.Stdout, resp.Body) + if err != nil { + return errors.New("unable to copy response body") + } + return nil +} diff --git a/ci-tools/github-runner/cmd/rtool/main.go b/ci-tools/github-runner/cmd/rtool/main.go index c50a6e82c9..a3c8e5d2a4 100644 --- a/ci-tools/github-runner/cmd/rtool/main.go +++ b/ci-tools/github-runner/cmd/rtool/main.go @@ -16,7 +16,55 @@ import ( ) func usage() { - fmt.Println("Usage: this_cmd [launch|serve|build_image|cleanup|jitconfig]") + fmt.Println(`Usage: rtool [launch|serve|build_image|cleanup|jitconfig|download_artifact] [...] + + download_artifact + + Download an artifact from Github. A cronjob on the fpga_boss machine + can use this command to download the latest output of the "Build FPGA SD + image" workflow, which fpga_boss will flash the FPGAs with between every + job. + + launch + + Launch a GCE VM of the specified machine-type and give it new just-in-time + GHA runner creds. This is typically used for testing logic used by the GCF + locally. + + jitconfig + + Register a new GHA just-in-time runner and return the jitconfig. This can + be used with "fpga_boss serve" to get a fresh jitconfig for an FPGA if you + have app credentials. + + receive_jitconfig + + Pull a jitconfig from the GCP queue; may block if none is available. + + publish_jitconfig + + Generate a new jitconfig and put it into the GCP queue; for testing only. + + serve + + Launch a webserver locally to serve the cloud functions. Can be useful for testing. + + cleanup + + Cleanup old or stuck GCE VMs. Mostly used for testing GCF logic. + + build_image + + Launch a GCE VM to build a fresh GitHub Runner image with all the latest + security fixes. Mostly used for testing GCF logic. + + Common arguments: + + app_id: The Github app_id to use. This is typically 379559 (the [Caliptra] + GHA-runners-on-GCP app) + + installation_id: The installation id of the app +`) } func main() { @@ -45,6 +93,24 @@ func main() { if err != nil { log.Fatal(err) } + } else if os.Args[1] == "download_artifact" { + appId, err := strconv.ParseInt(os.Args[2], 10, 64) + if err != nil { + log.Fatal(err) + } + installationId, err := strconv.ParseInt(os.Args[3], 10, 64) + if err != nil { + log.Fatal(err) + } + client, err := hello.GithubClient(appId, installationId) + if err != nil { + log.Fatal(err) + } + err = hello.DownloadArtifact(ctx, client, os.Args[4], os.Args[5]) + if err != nil { + log.Fatal(err) + } + return } else if os.Args[1] == "launch" || os.Args[1] == "jitconfig" || os.Args[1] == "publish_jitconfig" { if len(os.Args) <= 4 { log.Fatalf("usage: this_cmd launch ")