-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add Dockerfile and add WIP scraper to serverless
- Loading branch information
1 parent
2c6616e
commit 6d648a3
Showing
6 changed files
with
76 additions
and
111 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| FROM public.ecr.aws/lambda/go:1 as lambda | ||
|
|
||
| # Copy over go source code | ||
| COPY scraper/main.go /src/main.go | ||
| COPY go.mod /src/go.mod | ||
| COPY go.sum /src/go.sum | ||
|
|
||
| WORKDIR /src | ||
|
|
||
| # Install packages | ||
| RUN yum install go xz atk cups-libs gtk3 libXcomposite alsa-lib tar \ | ||
| libXcursor libXdamage libXext libXi libXrandr libXScrnSaver \ | ||
| libXtst pango at-spi2-atk libXt xorg-x11-server-Xvfb \ | ||
| xorg-x11-xauth dbus-glib dbus-glib-devel unzip bzip2 -y -q | ||
|
|
||
| # Build go lambda function | ||
| RUN GOARCH=amd64 GOOS=linux go build -ldflags="-s -w" -o ${LAMBDA_TASK_ROOT} /src/main.go | ||
|
|
||
| # Install chrome | ||
| RUN mkdir -p /opt/chrome/ | ||
| RUN curl -Lo "/opt/chrome/chrome-linux.zip" "https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Linux_x64%2F1299153%2Fchrome-linux.zip?generation=1715336417866122&alt=media" | ||
| RUN unzip -q "/opt/chrome/chrome-linux.zip" -d "/opt/chrome/" | ||
| RUN mv /opt/chrome/chrome-linux/* /opt/chrome/ | ||
| RUN rm -rf /opt/chrome/chrome-linux "/opt/chrome/chrome-linux.zip" | ||
|
|
||
| CMD [ "app" ] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,39 +1,18 @@ | ||
| module github.com/aurora-dot/watcher-lambda | ||
|
|
||
| go 1.22.2 | ||
| go 1.22 | ||
|
|
||
| require github.com/aws/aws-lambda-go v1.47.0 | ||
|
|
||
| require ( | ||
| github.com/aws/aws-lambda-go v1.47.0 // indirect | ||
| github.com/aws/aws-sdk-go v1.51.23 // indirect | ||
| github.com/blmayer/awslambdarpc v1.1.0 // indirect | ||
| github.com/cilium/ebpf v0.11.0 // indirect | ||
| github.com/cosiner/argv v0.1.0 // indirect | ||
| github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect | ||
| github.com/derekparker/trie v0.0.0-20230829180723-39f4de51ef7d // indirect | ||
| github.com/go-delve/delve v1.22.1 // indirect | ||
| github.com/go-delve/liner v1.2.3-0.20231231155935-4726ab1d7f62 // indirect | ||
| github.com/go-rod/rod v0.115.0 // indirect | ||
| github.com/google/go-dap v0.11.0 // indirect | ||
| github.com/hashicorp/golang-lru v1.0.2 // indirect | ||
| github.com/inconshreveable/mousetrap v1.1.0 // indirect | ||
| github.com/jmespath/go-jmespath v0.4.0 // indirect | ||
| github.com/mattn/go-colorable v0.1.13 // indirect | ||
| github.com/mattn/go-isatty v0.0.20 // indirect | ||
| github.com/mattn/go-runewidth v0.0.13 // indirect | ||
| github.com/rivo/uniseg v0.2.0 // indirect | ||
| github.com/russross/blackfriday/v2 v2.1.0 // indirect | ||
| github.com/sirupsen/logrus v1.9.3 // indirect | ||
| github.com/spf13/cobra v1.7.0 // indirect | ||
| github.com/spf13/pflag v1.0.5 // indirect | ||
| github.com/ysmood/fetchup v0.2.3 // indirect | ||
| github.com/ysmood/goob v0.4.0 // indirect | ||
| github.com/ysmood/got v0.34.1 // indirect | ||
| github.com/ysmood/gson v0.7.3 // indirect | ||
| github.com/ysmood/leakless v0.8.0 // indirect | ||
| go.starlark.net v0.0.0-20231101134539-556fd59b42f6 // indirect | ||
| golang.org/x/arch v0.6.0 // indirect | ||
| golang.org/x/exp v0.0.0-20230224173230-c95f2b4c22f2 // indirect | ||
| golang.org/x/sys v0.13.0 // indirect | ||
| gopkg.in/yaml.v2 v2.4.0 // indirect | ||
| gopkg.in/yaml.v3 v3.0.1 // indirect | ||
| ) | ||
|
|
||
| require ( | ||
| github.com/go-rod/rod v0.116.0 | ||
| github.com/stretchr/testify v1.8.4 // indirect | ||
| ) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,13 +1,31 @@ | ||
| package main | ||
|
|
||
| // import ( | ||
| // "github.com/aws/aws-lambda-go/lambda" | ||
| // ) | ||
| import ( | ||
| "context" | ||
|
|
||
| // func scrape(websiteUrl string, xpathString string) (string, string) { | ||
| "github.com/aws/aws-lambda-go/lambda" | ||
| "github.com/go-rod/rod" | ||
| "github.com/go-rod/rod/lib/launcher" | ||
| ) | ||
|
|
||
| // } | ||
| type MyEvent struct { | ||
| URL string `json:"URL"` | ||
| XPATH string `json:"XPATH"` | ||
| } | ||
|
|
||
| // func main() { | ||
| // lambda.Start(scrape) | ||
| // } | ||
| type MyResponse struct { | ||
| Content string `json:"content"` | ||
| Hash string `json:"hash"` | ||
| } | ||
|
|
||
| func scrape(ctx context.Context, event *MyEvent) (*MyResponse, error) { | ||
| u := launcher.New().Bin("/opt/chrome/chrome").MustLaunch() | ||
| page := rod.New().ControlURL(u).MustConnect().MustPage("https://www.wikipedia.org/") | ||
| page.MustWaitStable().MustScreenshot("a.png") | ||
|
|
||
| return &MyResponse{Content: "Test", Hash: "Test"}, nil | ||
| } | ||
|
|
||
| func main() { | ||
| lambda.Start(scrape) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters