Add rate limiting in development

README.md

@@ -42,6 +42,14 @@ Run it!
 
  $ yarn run dev
 
+To slow? Run it with your own ratelimit config
+
+ $ MAX_CONCURRENT=20; MIN_TIME=100; yarn run dev
+
+For running in production, turn off the ratelimiter
+
+ $ NO_RATELIMIT=true && yarn run build
+
 ### License
 
 This project is licensed under the MIT license. Please see [`LICENSE.md`](LICENSE.md) for the full text.

gatsby-node.js

@@ -1,40 +1,57 @@
-const { writeFile, existsSync, mkdirSync } = require('fs')
+const { writeFile, mkdir } = require('fs').promises
+const { existsSync } = require('fs')
 const axios = require('axios')
 const path = require('path')
 const readline = require('readline')
+const Bottleneck = require('bottleneck')
+
 const regions = require('./src/regions.js')
 
+const ratelimitConfig = {}
+if (!process.env.NO_RATELIMIT) {
+ ratelimitConfig.maxConcurrent = process.env.MAX_CONCURRENT || 20,
+ ratelimitConfig.minTime = process.env.MIN_TIME || 100
+}
+
+const limiter = new Bottleneck(ratelimitConfig)
+
+const axiosGet = limiter.wrap(axios.get)
+
+let startTime = Date.now()
+const logMessage = (...args) => {
+ readline.clearLine(process.stdout)
+ readline.cursorTo(process.stdout, 0)
+ const elapsedTime = ((Date.now() - startTime).toFixed(2) / 1000)
+ console.log(' ', ...args, `– ${elapsedTime} s`)
+ startTime = Date.now()
+}
+
 const imageFolder = 'static/images/'
+const dataFolder = 'data/'
 
-const downloadImage = (event, image, type = 'event') =>
- new Promise((resolve, reject) => {
- if (!image) resolve(null)
- axios
- .get(`https://api.hackclub.com${image.file_path}`, {
- responseType: 'arraybuffer',
- })
- .then(res => {
- let extension = ''
- switch (res.headers['content-type']) {
- case 'image/jpeg':
- extension = '.jpg'
- break
- case 'image/png':
- extension = '.png'
- break
- default:
- throw `Invalid content-type: ${res.headers['content-type']}`
- }
- let updatedAt = Date.parse(image.updated_at)
- const fileName = `${type}_${image.type}_${event.id}.${updatedAt}${extension}`
- writeFile(imageFolder + fileName, res.data, 'binary', err => {
- if (err) throw err
- resolve(`images/${fileName}`)
- })
- })
- .catch(err => reject(err))
+const downloadImage = async (event, image, type = 'event') => {
+ if (!image) {return null}
+ const res = await axiosGet(`https://api.hackclub.com${image.file_path}`, {
+ responseType: 'arraybuffer'
  })
 
+ let extension
+ switch(res.headers['content-type']) {
+ case 'image/jpeg':
+ extension = '.jpg'
+ break
+ case 'image/png':
+ extension = '.png'
+ break
+ default:
+ throw `Invalid content-type: ${res.headers['content-type']}`
+ }
+ const updatedAt = Date.parse(image.updated_at)
+ const filename = `${type}_${image.type}_${event.id}.${updatedAt}${extension}`
+ await writeFile(imageFolder + filename, res.data, 'binary')
+ return 'images/' + filename
+}
+
 const processEvent = async event => ({
  ...event,
  id: event.id.toString(),
@@ -49,139 +66,114 @@ const processGroup = async group => ({
  logo: await downloadImage(group, group.logo, 'group')
 })
 
-exports.onPreBootstrap = () => {
- let startTime = Date.now()
- const logMessage = (msg) => {
- readline.clearLine(process.stdout)
- readline.cursorTo(process.stdout, 0)
- const elapsedTime = ((Date.now() - startTime).toFixed(2) / 1000)
- console.log(` ${msg} – ${elapsedTime} s`)
- startTime = Date.now()
+const getSubscriberInfo = async () => {
+ const statsJson = await axiosGet('https://api.hackclub.com/v1/event_email_subscribers/stats')
+ logMessage('Fetched subscriber stats')
+
+ await writeFile(dataFolder + 'stats.json', JSON.stringify(statsJson.data))
+ logMessage('Subscriber stats written to file')
+}
+
+const getEventInfo = async () => {
+ const eventsJson = await axiosGet('https://api.hackclub.com/v1/events')
+ logMessage('Fetched events json')
+
+ const eventsData = await Promise.all(eventsJson.data.map(processEvent))
+ logMessage('Fetched event images')
+
+ await writeFile(dataFolder + 'events.json', JSON.stringify(eventsData))
+ logMessage('Event data written to file')
+}
+
+const getGroupInfo = async () => {
+ const groupsJson = await axiosGet('https://api.hackclub.com/v1/events/groups')
+ logMessage('Fetched groups json')
+
+ const groupsData = await Promise.all(groupsJson.data.map(processGroup))
+ logMessage('Fetched group images')
+
+ await writeFile(dataFolder + 'groups.json', JSON.stringify(groupsData))
+ logMessage('Group data written to file')
+}
+
+const ensureFolderExists = async foldername => {
+ if (!existsSync(foldername)) {
+ await mkdir(foldername)
+ logMessage('Created folder at', foldername)
  }
+}
 
- // Download & process events
- return axios
- .get('https://api.hackclub.com/v1/events')
- .then(eventsRes => {
- logMessage('Fetched events data')
- return axios.get('https://api.hackclub.com/v1/events/groups').then(groupsRes => {
- logMessage('Fetched groups data')
-
- if (!existsSync(imageFolder)) {
- mkdirSync(imageFolder)
- logMessage('Created image folder')
- }
- const groupsPromiseArray = groupsRes.data.map(group => processGroup(group))
- return Promise.all(groupsPromiseArray).then(groupsData => {
- const eventsPromiseArray = eventsRes.data.map(event => processEvent(event))
- return Promise.all(eventsPromiseArray)
- .then(eventsData => {
- logMessage('Mapped through event data')
- const writeGroups = new Promise((resolve, reject) => {
- writeFile('data/groups.json', JSON.stringify(groupsData), err => {
- if (err) return reject(err)
-
- logMessage('Group data written to file')
- resolve()
- })
- })
-
- const writeEvents = new Promise((resolve, reject) => {
- writeFile('data/events.json', JSON.stringify(eventsData), err => {
- if (err) return reject(err)
-
- logMessage('Event data written to file')
- resolve()
- })
- })
- return Promise.all([writeEvents, writeGroups])
- })
- })
- })
- })
- // Download & process event stats
- .then(() => (
- axios.get('https://api.hackclub.com/v1/event_email_subscribers/stats')
- ))
- .then(res => (
- new Promise((resolve, reject) => {
- writeFile('data/stats.json', JSON.stringify(res.data), err => {
- if (err) return reject(err)
-
- logMessage('Event stats written to file')
- resolve()
- })
- })
- ))
+exports.onPreBootstrap = async () => {
+ await Promise.all([
+ ensureFolderExists(dataFolder),
+ ensureFolderExists(imageFolder)
+ ])
+
+ await Promise.all([
+ getSubscriberInfo(),
+ getEventInfo(),
+ getGroupInfo(),
+ ])
 }
 
-exports.createPages = ({ graphql, actions }) => {
+exports.createPages = async ({ graphql, actions }) => {
+ logMessage('Creating pages')
  const { createPage } = actions
 
- return new Promise((resolve, reject) => {
- const component = path.resolve('src/templates/region.js')
- resolve(
- graphql(`
- {
- allEventsJson {
- edges {
- node {
- id
- startHumanized: start(formatString: "MMMM D")
- endHumanized: end(formatString: "D")
- start
- end
- startYear: start(formatString: "YYYY")
- parsed_city
- parsed_state
- parsed_state_code
- parsed_country
- parsed_country_code
- name
- website: website_redirect
- latitude
- longitude
- banner
- logo
- mlh: mlh_associated
- }
+ const component = path.resolve('src/templates/region.js')
+ const query = await graphql(`
+ {
+ allEventsJson {
+ edges {
+ node {
+ id
+ startHumanized: start(formatString: "MMMM D")
+ endHumanized: end(formatString: "D")
+ start
+ end
+ startYear: start(formatString: "YYYY")
+ parsed_city
+ parsed_state
+ parsed_state_code
+ parsed_country
+ parsed_country_code
+ name
+ website: website_redirect
+ latitude
+ longitude
+ banner
+ logo
+ mlh: mlh_associated
  }
  }
- dataJson {
- cities
- countries
- }
  }
- `).then(result => {
- if (result.errors) {
- console.error(result.errors)
- reject(result.errors)
+ dataJson {
+ cities
+ countries
  }
+ }
+ `)
+ logMessage('Ran graphql query')
+ if (query.errors) {
+ console.error(query.errors)
+ throw query.errors
+ }
 
- regions.map(region => {
- const events = result.data.allEventsJson.edges.filter(edge =>
- region.filter(edge.node)
- )
- const emailStats = result.data.dataJson
- if (events.length > 3) {
- createPage({
- path: region.path,
- component,
- context: {
- region,
- events,
- emailStats
- },
- })
- }
- })
- })
+ await Promise.all(regions.map(async region => {
+ const events = query.data.allEventsJson.edges.filter(edge =>
+ region.filter(edge.node)
  )
- locations.map(location => {
- createPage({
- path: location,
+ const emailStats = query.data.dataJson
+ if (events.length > 3) {
+ await createPage({
+ path: region.path,
  component,
+ context: {
+ region,
+ events,
+ emailStats
+ },
  })
- })
- })
+ }
+ }))
 }

package.json

@@ -7,6 +7,7 @@
  "@hackclub/design-system": "0.0.1-10",
  "axios": "^0.19.0",
  "babel-plugin-styled-components": "^1.10.6",
+ "bottleneck": "^2.19.5",
  "formik": "^2.0.8",
  "gatsby": "^2.18.16",
  "gatsby-plugin-canonical-urls": "^2.1.18",

yarn.lock

@@ -2595,6 +2595,11 @@ boolbase@^1.0.0, boolbase@~1.0.0:
  resolved "https://registry.yarnpkg.com/boolbase/-/boolbase-1.0.0.tgz#68dff5fbe60c51eb37725ea9e3ed310dcc1e776e"
  integrity sha1-aN/1++YMUes3cl6p4+0xDcwed24=
 
+bottleneck@^2.19.5:
+ version "2.19.5"
+ resolved "https://registry.yarnpkg.com/bottleneck/-/bottleneck-2.19.5.tgz#5df0b90f59fd47656ebe63c78a98419205cadd91"
+ integrity sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw==
+
 [email protected]:
  version "1.3.0"
  resolved "https://registry.yarnpkg.com/boxen/-/boxen-1.3.0.tgz#55c6c39a8ba58d9c61ad22cd877532deb665a20b"