Scrape campaign finance data #302

	name: Scrape campaign finance data

	on:
	schedule:
	- cron: '15 2 * * *'

	workflow_dispatch:

	jobs:
	scrape:
	runs-on: ubuntu-latest
	strategy:
	matrix:
	target: [
	data/processed/disclosures.xlsx,
	'data/processed/candidate_committees.csv data/processed/candidate_committee_filings.csv',
	'data/processed/pac_committees.csv data/processed/pac_committee_filings.csv',
	data/processed/independent_expenditures.xlsx,
	data/processed/lobbyist.xlsx,
	data/processed/lobbyist_employer.xlsx
	]

	steps:
	- uses: actions/checkout@v4

	- name: Setup requirements
	run: pip install -r requirements.txt

	- name: Make ${{ matrix.target }}
	run: make ${{ matrix.target }}

	- name: Upload ${{ matrix.target }} to S3
	env:
	S3BUCKET: ${{ secrets.S3BUCKET }}
	AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
	AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
	run: \|
	for file in ${{ matrix.target }}; do aws s3 cp $file $S3BUCKET --acl public-read; done

	import:
	needs: scrape
	runs-on: ubuntu-latest

	steps:
	- name: Trigger Openness Project ETL
	uses: peter-evans/repository-dispatch@v3
	with:
	token: ${{ secrets.PAT }}
	repository: datamade/openness-project-nmid
	event-type: nightly-scrape-done

Provide feedback