diff --git a/README.md b/README.md index efb2bf2..7388753 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,14 @@ Run it! $ yarn run dev +To slow? Run it with your own ratelimit config + + $ MAX_CONCURRENT=20; MIN_TIME=100; yarn run dev + +For running in production, turn off the ratelimiter + + $ NO_RATELIMIT=true && yarn run build + ### License This project is licensed under the MIT license. Please see [`LICENSE.md`](LICENSE.md) for the full text. diff --git a/gatsby-node.js b/gatsby-node.js index 1af9589..4bcb769 100644 --- a/gatsby-node.js +++ b/gatsby-node.js @@ -1,40 +1,57 @@ -const { writeFile, existsSync, mkdirSync } = require('fs') +const { writeFile, mkdir } = require('fs').promises +const { existsSync } = require('fs') const axios = require('axios') const path = require('path') const readline = require('readline') +const Bottleneck = require('bottleneck') + const regions = require('./src/regions.js') +const ratelimitConfig = {} +if (!process.env.NO_RATELIMIT) { + ratelimitConfig.maxConcurrent = process.env.MAX_CONCURRENT || 20, + ratelimitConfig.minTime = process.env.MIN_TIME || 100 +} + +const limiter = new Bottleneck(ratelimitConfig) + +const axiosGet = limiter.wrap(axios.get) + +let startTime = Date.now() +const logMessage = (...args) => { + readline.clearLine(process.stdout) + readline.cursorTo(process.stdout, 0) + const elapsedTime = ((Date.now() - startTime).toFixed(2) / 1000) + console.log(' ', ...args, `– ${elapsedTime} s`) + startTime = Date.now() +} + const imageFolder = 'static/images/' +const dataFolder = 'data/' -const downloadImage = (event, image, type = 'event') => - new Promise((resolve, reject) => { - if (!image) resolve(null) - axios - .get(`https://api.hackclub.com${image.file_path}`, { - responseType: 'arraybuffer', - }) - .then(res => { - let extension = '' - switch (res.headers['content-type']) { - case 'image/jpeg': - extension = '.jpg' - break - case 'image/png': - extension = '.png' - break - default: - throw `Invalid content-type: ${res.headers['content-type']}` - } - let updatedAt = Date.parse(image.updated_at) - const fileName = `${type}_${image.type}_${event.id}.${updatedAt}${extension}` - writeFile(imageFolder + fileName, res.data, 'binary', err => { - if (err) throw err - resolve(`images/${fileName}`) - }) - }) - .catch(err => reject(err)) +const downloadImage = async (event, image, type = 'event') => { + if (!image) {return null} + const res = await axiosGet(`https://api.hackclub.com${image.file_path}`, { + responseType: 'arraybuffer' }) + let extension + switch(res.headers['content-type']) { + case 'image/jpeg': + extension = '.jpg' + break + case 'image/png': + extension = '.png' + break + default: + throw `Invalid content-type: ${res.headers['content-type']}` + } + const updatedAt = Date.parse(image.updated_at) + const filename = `${type}_${image.type}_${event.id}.${updatedAt}${extension}` + await writeFile(imageFolder + filename, res.data, 'binary') + return 'images/' + filename +} + const processEvent = async event => ({ ...event, id: event.id.toString(), @@ -49,139 +66,114 @@ const processGroup = async group => ({ logo: await downloadImage(group, group.logo, 'group') }) -exports.onPreBootstrap = () => { - let startTime = Date.now() - const logMessage = (msg) => { - readline.clearLine(process.stdout) - readline.cursorTo(process.stdout, 0) - const elapsedTime = ((Date.now() - startTime).toFixed(2) / 1000) - console.log(` ${msg} – ${elapsedTime} s`) - startTime = Date.now() +const getSubscriberInfo = async () => { + const statsJson = await axiosGet('https://api.hackclub.com/v1/event_email_subscribers/stats') + logMessage('Fetched subscriber stats') + + await writeFile(dataFolder + 'stats.json', JSON.stringify(statsJson.data)) + logMessage('Subscriber stats written to file') +} + +const getEventInfo = async () => { + const eventsJson = await axiosGet('https://api.hackclub.com/v1/events') + logMessage('Fetched events json') + + const eventsData = await Promise.all(eventsJson.data.map(processEvent)) + logMessage('Fetched event images') + + await writeFile(dataFolder + 'events.json', JSON.stringify(eventsData)) + logMessage('Event data written to file') +} + +const getGroupInfo = async () => { + const groupsJson = await axiosGet('https://api.hackclub.com/v1/events/groups') + logMessage('Fetched groups json') + + const groupsData = await Promise.all(groupsJson.data.map(processGroup)) + logMessage('Fetched group images') + + await writeFile(dataFolder + 'groups.json', JSON.stringify(groupsData)) + logMessage('Group data written to file') +} + +const ensureFolderExists = async foldername => { + if (!existsSync(foldername)) { + await mkdir(foldername) + logMessage('Created folder at', foldername) } +} - // Download & process events - return axios - .get('https://api.hackclub.com/v1/events') - .then(eventsRes => { - logMessage('Fetched events data') - return axios.get('https://api.hackclub.com/v1/events/groups').then(groupsRes => { - logMessage('Fetched groups data') - - if (!existsSync(imageFolder)) { - mkdirSync(imageFolder) - logMessage('Created image folder') - } - const groupsPromiseArray = groupsRes.data.map(group => processGroup(group)) - return Promise.all(groupsPromiseArray).then(groupsData => { - const eventsPromiseArray = eventsRes.data.map(event => processEvent(event)) - return Promise.all(eventsPromiseArray) - .then(eventsData => { - logMessage('Mapped through event data') - const writeGroups = new Promise((resolve, reject) => { - writeFile('data/groups.json', JSON.stringify(groupsData), err => { - if (err) return reject(err) - - logMessage('Group data written to file') - resolve() - }) - }) - - const writeEvents = new Promise((resolve, reject) => { - writeFile('data/events.json', JSON.stringify(eventsData), err => { - if (err) return reject(err) - - logMessage('Event data written to file') - resolve() - }) - }) - return Promise.all([writeEvents, writeGroups]) - }) - }) - }) - }) - // Download & process event stats - .then(() => ( - axios.get('https://api.hackclub.com/v1/event_email_subscribers/stats') - )) - .then(res => ( - new Promise((resolve, reject) => { - writeFile('data/stats.json', JSON.stringify(res.data), err => { - if (err) return reject(err) - - logMessage('Event stats written to file') - resolve() - }) - }) - )) +exports.onPreBootstrap = async () => { + await Promise.all([ + ensureFolderExists(dataFolder), + ensureFolderExists(imageFolder) + ]) + + await Promise.all([ + getSubscriberInfo(), + getEventInfo(), + getGroupInfo(), + ]) } -exports.createPages = ({ graphql, actions }) => { +exports.createPages = async ({ graphql, actions }) => { + logMessage('Creating pages') const { createPage } = actions - return new Promise((resolve, reject) => { - const component = path.resolve('src/templates/region.js') - resolve( - graphql(` - { - allEventsJson { - edges { - node { - id - startHumanized: start(formatString: "MMMM D") - endHumanized: end(formatString: "D") - start - end - startYear: start(formatString: "YYYY") - parsed_city - parsed_state - parsed_state_code - parsed_country - parsed_country_code - name - website: website_redirect - latitude - longitude - banner - logo - mlh: mlh_associated - } + const component = path.resolve('src/templates/region.js') + const query = await graphql(` + { + allEventsJson { + edges { + node { + id + startHumanized: start(formatString: "MMMM D") + endHumanized: end(formatString: "D") + start + end + startYear: start(formatString: "YYYY") + parsed_city + parsed_state + parsed_state_code + parsed_country + parsed_country_code + name + website: website_redirect + latitude + longitude + banner + logo + mlh: mlh_associated } } - dataJson { - cities - countries - } } - `).then(result => { - if (result.errors) { - console.error(result.errors) - reject(result.errors) + dataJson { + cities + countries } + } + `) + logMessage('Ran graphql query') + if (query.errors) { + console.error(query.errors) + throw query.errors + } - regions.map(region => { - const events = result.data.allEventsJson.edges.filter(edge => - region.filter(edge.node) - ) - const emailStats = result.data.dataJson - if (events.length > 3) { - createPage({ - path: region.path, - component, - context: { - region, - events, - emailStats - }, - }) - } - }) - }) + await Promise.all(regions.map(async region => { + const events = query.data.allEventsJson.edges.filter(edge => + region.filter(edge.node) ) - locations.map(location => { - createPage({ - path: location, + const emailStats = query.data.dataJson + if (events.length > 3) { + await createPage({ + path: region.path, component, + context: { + region, + events, + emailStats + }, }) - }) - }) + } + })) } diff --git a/package.json b/package.json index e6f43ea..234e4a4 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,7 @@ "@hackclub/design-system": "0.0.1-10", "axios": "^0.19.0", "babel-plugin-styled-components": "^1.10.6", + "bottleneck": "^2.19.5", "formik": "^2.0.8", "gatsby": "^2.18.16", "gatsby-plugin-canonical-urls": "^2.1.18", diff --git a/yarn.lock b/yarn.lock index e1ae65a..24988fe 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2595,6 +2595,11 @@ boolbase@^1.0.0, boolbase@~1.0.0: resolved "https://registry.yarnpkg.com/boolbase/-/boolbase-1.0.0.tgz#68dff5fbe60c51eb37725ea9e3ed310dcc1e776e" integrity sha1-aN/1++YMUes3cl6p4+0xDcwed24= +bottleneck@^2.19.5: + version "2.19.5" + resolved "https://registry.yarnpkg.com/bottleneck/-/bottleneck-2.19.5.tgz#5df0b90f59fd47656ebe63c78a98419205cadd91" + integrity sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw== + boxen@1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/boxen/-/boxen-1.3.0.tgz#55c6c39a8ba58d9c61ad22cd877532deb665a20b"