Skip to content

Commit

Permalink
Add User-Agent header to outgoing HTTP requests (#19)
Browse files Browse the repository at this point in the history
Excludes requests made by the AWS SDK to read/write data in S3, but
includes all other HTTP requests made during operation, e.g. when
getting changeset metadata from the OSM API, or when querying the
Overpass server.
  • Loading branch information
jake-low authored Aug 27, 2024
1 parent f12a0a0 commit af3e39e
Show file tree
Hide file tree
Showing 6 changed files with 330 additions and 348 deletions.
12 changes: 7 additions & 5 deletions lib/get-changesets.js
Original file line number Diff line number Diff line change
Expand Up @@ -149,12 +149,12 @@ const queryOverpass = async (changesetId, data, bbox) => {

try {
console.log(`Trying primary overpass for changeset ${changesetId}: ${primaryUrl}`);
return await request(primaryUrl);
return await request(primaryUrl).then(res => res.text());
} catch (e) {
console.log('Primary overpass error: ', e);
console.log(`Primary overpass failed, trying secondary for changeset ${changesetId}: ${secondaryUrl}`);
try {
return await request(secondaryUrl);
return await request(secondaryUrl).then(res => res.text());
} catch (e) {
throw new Error(`Primary and secondary Overpass failed for changeset ${changesetId}: ${e}`);
}
Expand All @@ -163,7 +163,7 @@ const queryOverpass = async (changesetId, data, bbox) => {

const getChangesetMetadata = async (changesetId) => {
try {
const body = await request(`${OSM_CHANGESET_API}/${changesetId}`);
const body = await request(`${OSM_CHANGESET_API}/${changesetId}`).then(res => res.text());
const changesetData = parseChangesetXml(body);

const meta = changesetData.osm[0].changeset[0];
Expand All @@ -188,15 +188,17 @@ const checkLatest = async (max) => {
const primaryUrl = `${OVERPASS_PRIMARY_URL}/api/augmented_diff_status`;
const secondaryUrl = `${OVERPASS_SECONDARY_URL}/api/augmented_diff_status`;

const getStatus = async (url) => parseInt(await request(url).then(res => res.text()), 10);

try {
const primaryLatest = parseInt(await request(primaryUrl), 10);
const primaryLatest = await getStatus(primaryUrl);
if(max > primaryLatest) throw new Error('Required state is not in Primary overpass yet');
return primaryLatest;
} catch (e) {
console.log(`Primary overpass (${primaryUrl}) failed: ${e}`);
console.log(`Trying secondary ${secondaryUrl}...`);
try {
const secondaryLatest = parseInt(await request(secondaryUrl), 10);
const secondaryLatest = await getStatus(secondaryUrl);
if(max > secondaryLatest) {
throw new Error('Required state is not in Secondary overpass yet');
}
Expand Down
10 changes: 8 additions & 2 deletions lib/tagChanges.js
Original file line number Diff line number Diff line change
Expand Up @@ -7,15 +7,21 @@ const { request } = require('../util/request');
const postTagChanges = async (changeset) => {
if (!process.env.OsmchaAdminToken) {
console.log('OSMCha API Token is not configured.')
return;
}

try {
const changesetId = changeset.metadata.id;
const url = `${OSMCHA_URL}/changesets/${changesetId}/tag-changes/`;
const tagDiff = createTagDiff(changeset);
if (tagDiff) {
await request(url, 'POST', JSON.stringify(tagDiff), `Token ${process.env.OsmchaAdminToken}`);
const body = Buffer.from(JSON.stringify(tagDiff), "utf8");
const headers = {
'Authorization': `Token ${process.env.OsmchaAdminToken}`,
'Content-Type': 'application/json',
'Content-Length': body.length,
};

await request(url, { method: 'POST', headers, body });
console.log(`Posted ${changesetId} tag changes to OSMCHA API`);
}
} catch (e) {
Expand Down
4 changes: 2 additions & 2 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
"lodash": "^4.17.15",
"minimist": "^1.2.0",
"moment": "^2.25.3",
"node-fetch": "^1.7.3",
"redis": "^4.6.13"
},
"devDependencies": {
Expand All @@ -39,8 +40,7 @@
"eslint-plugin-import": "^2.2.0",
"eslint-plugin-node": "^4.2.2",
"eslint-plugin-promise": "^3.5.0",
"eslint-plugin-standard": "^2.2.0",
"node-fetch": "^1.6.3",
"eslint-plugin-standard": "^3.0.0",
"pako": "^1.0.5",
"tape": "^5.0.0"
}
Expand Down
6 changes: 3 additions & 3 deletions util/get-states.js
Original file line number Diff line number Diff line change
Expand Up @@ -32,11 +32,11 @@ const getStateForMinute = (minute) => {

const getOverpassTimestamp = async () => {
try {
const timestamp = await request(`${OVERPASS_PRIMARY_URL}/api/timestamp`);
const timestamp = await request(`${OVERPASS_PRIMARY_URL}/api/timestamp`).then(res => res.text());
return timestamp;
} catch (e) {
const overpassTimestamp = await request(`${OVERPASS_SECONDARY_URL}/api/timestamp`);
return overpassTimestamp;
const timestamp = await request(`${OVERPASS_SECONDARY_URL}/api/timestamp`).then(res => res.text());
return timestamp;
}
}

Expand Down
62 changes: 18 additions & 44 deletions util/request.js
Original file line number Diff line number Diff line change
@@ -1,49 +1,23 @@
const https = require('https');
const fetch = require('node-fetch');
const { version } = require("../package.json");

const request = async (url, method = 'GET', postData, token) => {
const [h, ...rest] = url.split('://')[1].split('/');
const path = `/${rest.join('/')}`;
const [host, port] = h.split(':');
async function request(url, options = {}) {
if (!options.headers) {
options.headers = {}
}

const params = {
method,
host,
port: port || url.startsWith('https://') ? 443 : 80,
path: path || '/',
};
options.headers['User-Agent'] = `OSMCha osm-adiff-service ${version}`

if (token && postData) params.headers = {
Authorization: token,
'Content-Type': 'application/json',
'Content-Length' : Buffer.byteLength(postData, 'utf8')
};
let res = await fetch(url, options);

if (res.ok) {
return res;
} else {
let error = new Error(`HTTP ${res.status} ${res.statusText}`);
error.status = res.status;
error.statusText = res.statusText;
throw error;
}
}

return new Promise((resolve, reject) => {
const req = https.request(params, res => {
if (res.statusCode < 200 || res.statusCode >= 303) {
return reject(new Error(`Status Code: ${res.statusCode}`));
}

const data = [];

res.on('data', chunk => {
data.push(chunk);
});

res.on('end', () => resolve(Buffer.concat(data).toString()));
});

req.on('error', reject);

if (postData) {
req.write(postData);
}

req.end();
});
};

module.exports = {
request,
};
module.exports = { request };
Loading

0 comments on commit af3e39e

Please sign in to comment.