From a6aa834a697a236f8df2fceba34315b6f158f83e Mon Sep 17 00:00:00 2001 From: Max Wofford Date: Fri, 3 Jan 2020 09:02:32 -0500 Subject: [PATCH 1/4] Example of bottlenecked requests on build --- gatsby-node.js | 27 ++++++++++++++++++++------- package.json | 1 + yarn.lock | 5 +++++ 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/gatsby-node.js b/gatsby-node.js index 1af9589..aa04d19 100644 --- a/gatsby-node.js +++ b/gatsby-node.js @@ -3,14 +3,22 @@ const axios = require('axios') const path = require('path') const readline = require('readline') const regions = require('./src/regions.js') +const Bottleneck = require('bottleneck') + +const limiter = new Bottleneck({ + maxConcurrent: 10, + minTime: 100, +}) + +const axiosGet = limiter.wrap(axios.get) const imageFolder = 'static/images/' const downloadImage = (event, image, type = 'event') => new Promise((resolve, reject) => { if (!image) resolve(null) - axios - .get(`https://api.hackclub.com${image.file_path}`, { + console.log(`downloading ${image.file_path}`) + axiosGet(`https://api.hackclub.com${image.file_path}`, { responseType: 'arraybuffer', }) .then(res => { @@ -32,7 +40,11 @@ const downloadImage = (event, image, type = 'event') => resolve(`images/${fileName}`) }) }) - .catch(err => reject(err)) + .catch(err => { + console.log(`failure while downloading ${image.file_path}`) + console.error(err) + reject(err) + }) }) const processEvent = async event => ({ @@ -60,11 +72,10 @@ exports.onPreBootstrap = () => { } // Download & process events - return axios - .get('https://api.hackclub.com/v1/events') + return axiosGet('https://api.hackclub.com/v1/events') .then(eventsRes => { logMessage('Fetched events data') - return axios.get('https://api.hackclub.com/v1/events/groups').then(groupsRes => { + return axiosGet('https://api.hackclub.com/v1/events/groups').then(groupsRes => { logMessage('Fetched groups data') if (!existsSync(imageFolder)) { @@ -72,8 +83,10 @@ exports.onPreBootstrap = () => { logMessage('Created image folder') } const groupsPromiseArray = groupsRes.data.map(group => processGroup(group)) + logMessage(`starting to get groupsePromiseArray ${groupsPromiseArray.length}`) return Promise.all(groupsPromiseArray).then(groupsData => { const eventsPromiseArray = eventsRes.data.map(event => processEvent(event)) + logMessage('starting to get eventsPromiseArray') return Promise.all(eventsPromiseArray) .then(eventsData => { logMessage('Mapped through event data') @@ -101,7 +114,7 @@ exports.onPreBootstrap = () => { }) // Download & process event stats .then(() => ( - axios.get('https://api.hackclub.com/v1/event_email_subscribers/stats') + axiosGet('https://api.hackclub.com/v1/event_email_subscribers/stats') )) .then(res => ( new Promise((resolve, reject) => { diff --git a/package.json b/package.json index e6f43ea..234e4a4 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,7 @@ "@hackclub/design-system": "0.0.1-10", "axios": "^0.19.0", "babel-plugin-styled-components": "^1.10.6", + "bottleneck": "^2.19.5", "formik": "^2.0.8", "gatsby": "^2.18.16", "gatsby-plugin-canonical-urls": "^2.1.18", diff --git a/yarn.lock b/yarn.lock index e1ae65a..24988fe 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2595,6 +2595,11 @@ boolbase@^1.0.0, boolbase@~1.0.0: resolved "https://registry.yarnpkg.com/boolbase/-/boolbase-1.0.0.tgz#68dff5fbe60c51eb37725ea9e3ed310dcc1e776e" integrity sha1-aN/1++YMUes3cl6p4+0xDcwed24= +bottleneck@^2.19.5: + version "2.19.5" + resolved "https://registry.yarnpkg.com/bottleneck/-/bottleneck-2.19.5.tgz#5df0b90f59fd47656ebe63c78a98419205cadd91" + integrity sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw== + boxen@1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/boxen/-/boxen-1.3.0.tgz#55c6c39a8ba58d9c61ad22cd877532deb665a20b" From ee08f982d8c35dde8e7a887f14c29befe9b0b70d Mon Sep 17 00:00:00 2001 From: Max Wofford Date: Fri, 3 Jan 2020 10:51:54 -0500 Subject: [PATCH 2/4] Add ratelimiting for development --- README.md | 8 ++ gatsby-node.js | 297 +++++++++++++++++++++++-------------------------- 2 files changed, 146 insertions(+), 159 deletions(-) diff --git a/README.md b/README.md index efb2bf2..7388753 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,14 @@ Run it! $ yarn run dev +To slow? Run it with your own ratelimit config + + $ MAX_CONCURRENT=20; MIN_TIME=100; yarn run dev + +For running in production, turn off the ratelimiter + + $ NO_RATELIMIT=true && yarn run build + ### License This project is licensed under the MIT license. Please see [`LICENSE.md`](LICENSE.md) for the full text. diff --git a/gatsby-node.js b/gatsby-node.js index aa04d19..3939718 100644 --- a/gatsby-node.js +++ b/gatsby-node.js @@ -1,52 +1,57 @@ -const { writeFile, existsSync, mkdirSync } = require('fs') +const { writeFile, mkdir } = require('fs').promises +const { existsSync } = require('fs') const axios = require('axios') const path = require('path') const readline = require('readline') -const regions = require('./src/regions.js') const Bottleneck = require('bottleneck') -const limiter = new Bottleneck({ - maxConcurrent: 10, - minTime: 100, -}) +const regions = require('./src/regions.js') + +const ratelimitConfig = {} +if (!process.env.NO_RATELIMIT) { + ratelimitConfig.maxConcurrent = process.env.MAX_CONCURRENT || 20, + ratelimitConfig.minTime = process.env.MIN_TIME || 100 +} + +const limiter = new Bottleneck(ratelimitConfig) const axiosGet = limiter.wrap(axios.get) +let startTime = Date.now() +const logMessage = (...args) => { + readline.clearLine(process.stdout) + readline.cursorTo(process.stdout, 0) + const elapsedTime = ((Date.now() - startTime).toFixed(2) / 1000) + console.log(' ', ...args, `– ${elapsedTime} s`) + startTime = Date.now() +} + const imageFolder = 'static/images/' +const dataFolder = 'data/' -const downloadImage = (event, image, type = 'event') => - new Promise((resolve, reject) => { - if (!image) resolve(null) - console.log(`downloading ${image.file_path}`) - axiosGet(`https://api.hackclub.com${image.file_path}`, { - responseType: 'arraybuffer', - }) - .then(res => { - let extension = '' - switch (res.headers['content-type']) { - case 'image/jpeg': - extension = '.jpg' - break - case 'image/png': - extension = '.png' - break - default: - throw `Invalid content-type: ${res.headers['content-type']}` - } - let updatedAt = Date.parse(image.updated_at) - const fileName = `${type}_${image.type}_${event.id}.${updatedAt}${extension}` - writeFile(imageFolder + fileName, res.data, 'binary', err => { - if (err) throw err - resolve(`images/${fileName}`) - }) - }) - .catch(err => { - console.log(`failure while downloading ${image.file_path}`) - console.error(err) - reject(err) - }) +const downloadImage = async (event, image, type = 'event') => { + if (!image) {return null} + const res = await axiosGet(`https://api.hackclub.com${image.file_path}`, { + responseType: 'arraybuffer' }) + let extension + switch(res.headers['content-type']) { + case 'image/jpeg': + extension = '.jpg' + break + case 'image/png': + extension = '.png' + break + default: + throw `Invalid content-type: ${res.headers['content-type']}` + } + const updatedAt = Date.parse(image.updated_at) + const filename = `${imageFolder}${type}_${image.type}_${event.id}.${updatedAt}${extension}` + await writeFile(filename, res.data, 'binary') + return filename +} + const processEvent = async event => ({ ...event, id: event.id.toString(), @@ -61,140 +66,114 @@ const processGroup = async group => ({ logo: await downloadImage(group, group.logo, 'group') }) -exports.onPreBootstrap = () => { - let startTime = Date.now() - const logMessage = (msg) => { - readline.clearLine(process.stdout) - readline.cursorTo(process.stdout, 0) - const elapsedTime = ((Date.now() - startTime).toFixed(2) / 1000) - console.log(` ${msg} – ${elapsedTime} s`) - startTime = Date.now() - } +const getSubscriberInfo = async () => { + const statsJson = await axiosGet('https://api.hackclub.com/v1/event_email_subscribers/stats') + logMessage('Fetched subscriber stats') - // Download & process events - return axiosGet('https://api.hackclub.com/v1/events') - .then(eventsRes => { - logMessage('Fetched events data') - return axiosGet('https://api.hackclub.com/v1/events/groups').then(groupsRes => { - logMessage('Fetched groups data') + await writeFile(dataFolder + 'stats.json', JSON.stringify(statsJson.data)) + logMessage('Subscriber stats written to file') +} - if (!existsSync(imageFolder)) { - mkdirSync(imageFolder) - logMessage('Created image folder') - } - const groupsPromiseArray = groupsRes.data.map(group => processGroup(group)) - logMessage(`starting to get groupsePromiseArray ${groupsPromiseArray.length}`) - return Promise.all(groupsPromiseArray).then(groupsData => { - const eventsPromiseArray = eventsRes.data.map(event => processEvent(event)) - logMessage('starting to get eventsPromiseArray') - return Promise.all(eventsPromiseArray) - .then(eventsData => { - logMessage('Mapped through event data') - const writeGroups = new Promise((resolve, reject) => { - writeFile('data/groups.json', JSON.stringify(groupsData), err => { - if (err) return reject(err) - - logMessage('Group data written to file') - resolve() - }) - }) - - const writeEvents = new Promise((resolve, reject) => { - writeFile('data/events.json', JSON.stringify(eventsData), err => { - if (err) return reject(err) - - logMessage('Event data written to file') - resolve() - }) - }) - return Promise.all([writeEvents, writeGroups]) - }) - }) - }) - }) - // Download & process event stats - .then(() => ( - axiosGet('https://api.hackclub.com/v1/event_email_subscribers/stats') - )) - .then(res => ( - new Promise((resolve, reject) => { - writeFile('data/stats.json', JSON.stringify(res.data), err => { - if (err) return reject(err) - - logMessage('Event stats written to file') - resolve() - }) - }) - )) +const getEventInfo = async () => { + const eventsJson = await axiosGet('https://api.hackclub.com/v1/events') + logMessage('Fetched events json') + + const eventsData = await Promise.all(eventsJson.data.map(processEvent)) + logMessage('Fetched event images') + + await writeFile(dataFolder + 'events.json', JSON.stringify(eventsData)) + logMessage('Event data written to file') +} + +const getGroupInfo = async () => { + const groupsJson = await axiosGet('https://api.hackclub.com/v1/events/groups') + logMessage('Fetched groups json') + + const groupsData = await Promise.all(groupsJson.data.map(processGroup)) + logMessage('Fetched group images') + + await writeFile(dataFolder + 'groups.json', JSON.stringify(groupsData)) + logMessage('Group data written to file') +} + +const ensureFolderExists = async foldername => { + if (!existsSync(foldername)) { + await mkdir(foldername) + logMessage('Created folder at', foldername) + } } -exports.createPages = ({ graphql, actions }) => { +exports.onPreBootstrap = async () => { + await Promise.all([ + ensureFolderExists(dataFolder), + ensureFolderExists(imageFolder) + ]) + + await Promise.all([ + getSubscriberInfo(), + getEventInfo(), + getGroupInfo(), + ]) +} + +exports.createPages = async ({ graphql, actions }) => { + logMessage('Creating pages') const { createPage } = actions - return new Promise((resolve, reject) => { - const component = path.resolve('src/templates/region.js') - resolve( - graphql(` - { - allEventsJson { - edges { - node { - id - startHumanized: start(formatString: "MMMM D") - endHumanized: end(formatString: "D") - start - end - startYear: start(formatString: "YYYY") - parsed_city - parsed_state - parsed_state_code - parsed_country - parsed_country_code - name - website: website_redirect - latitude - longitude - banner - logo - mlh: mlh_associated - } + const component = path.resolve('src/templates/region.js') + const query = await graphql(` + { + allEventsJson { + edges { + node { + id + startHumanized: start(formatString: "MMMM D") + endHumanized: end(formatString: "D") + start + end + startYear: start(formatString: "YYYY") + parsed_city + parsed_state + parsed_state_code + parsed_country + parsed_country_code + name + website: website_redirect + latitude + longitude + banner + logo + mlh: mlh_associated } } - dataJson { - cities - countries - } } - `).then(result => { - if (result.errors) { - console.error(result.errors) - reject(result.errors) + dataJson { + cities + countries } + } + `) + logMessage('Ran graphql query') + if (query.errors) { + console.error(query.errors) + throw query.errors + } - regions.map(region => { - const events = result.data.allEventsJson.edges.filter(edge => - region.filter(edge.node) - ) - const emailStats = result.data.dataJson - if (events.length > 3) { - createPage({ - path: region.path, - component, - context: { - region, - events, - emailStats - }, - }) - } - }) - }) + await Promise.all(regions.map(async region => { + const events = query.data.allEventsJson.edges.filter(edge => + region.filter(edge.node) ) - locations.map(location => { - createPage({ - path: location, + const emailStats = query.data.dataJson + if (events.length > 3) { + await createPage({ + path: region.path, component, + context: { + region, + events, + emailStats + }, }) - }) - }) + } + })) } From e5d064685c0c856577873b215efe2b9f926c1417 Mon Sep 17 00:00:00 2001 From: Max Wofford Date: Fri, 3 Jan 2020 11:03:53 -0500 Subject: [PATCH 3/4] Fix misnamed image files --- gatsby-node.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gatsby-node.js b/gatsby-node.js index 3939718..9f8d568 100644 --- a/gatsby-node.js +++ b/gatsby-node.js @@ -47,8 +47,8 @@ const downloadImage = async (event, image, type = 'event') => { throw `Invalid content-type: ${res.headers['content-type']}` } const updatedAt = Date.parse(image.updated_at) - const filename = `${imageFolder}${type}_${image.type}_${event.id}.${updatedAt}${extension}` - await writeFile(filename, res.data, 'binary') + const filename = `${type}_${image.type}_${event.id}.${updatedAt}${extension}` + await writeFile(imageFolder + filename, res.data, 'binary') return filename } From 52ac2cfd4e834442c347b9024379bdf3f008a6eb Mon Sep 17 00:00:00 2001 From: Max Wofford Date: Fri, 3 Jan 2020 11:39:34 -0500 Subject: [PATCH 4/4] Fix image filepath resolution in build --- gatsby-node.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gatsby-node.js b/gatsby-node.js index 9f8d568..4bcb769 100644 --- a/gatsby-node.js +++ b/gatsby-node.js @@ -49,7 +49,7 @@ const downloadImage = async (event, image, type = 'event') => { const updatedAt = Date.parse(image.updated_at) const filename = `${type}_${image.type}_${event.id}.${updatedAt}${extension}` await writeFile(imageFolder + filename, res.data, 'binary') - return filename + return 'images/' + filename } const processEvent = async event => ({