From 614440d814ccaf5598d90d73a903be7384fa670d Mon Sep 17 00:00:00 2001 From: amirhossein-haerian Date: Thu, 20 Jun 2024 13:14:55 +0200 Subject: [PATCH] fix(KUI-1369): noIndexMiddleware and noIndexMiddleware.test has been added to make sure that URL with aap*. is not indexed by google --- package-lock.json | 127 ++++++++++++++++++++++++- package.json | 1 + server/server.js | 6 ++ server/utils/noIndexMiddleware.js | 28 ++++++ server/utils/noIndexMiddleware.test.js | 50 ++++++++++ 5 files changed, 209 insertions(+), 3 deletions(-) create mode 100644 server/utils/noIndexMiddleware.js create mode 100644 server/utils/noIndexMiddleware.test.js diff --git a/package-lock.json b/package-lock.json index 37ccf1ef..dc70784b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -70,6 +70,7 @@ "jest-environment-jsdom": "29.7.0", "lint-staged": "^15.2.5", "mini-css-extract-plugin": "^2.9.0", + "node-mocks-http": "^1.14.1", "nodemon": "^3.1.3", "null-loader": "^4.0.1", "path": "^0.12.7", @@ -3998,6 +3999,25 @@ "@babel/types": "^7.20.7" } }, + "node_modules/@types/body-parser": { + "version": "1.19.5", + "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.5.tgz", + "integrity": "sha512-fB3Zu92ucau0iQ0JMCFQE7b/dv8Ot07NI3KaZIkIUNXq82k4eBAqUaneXfleGY9JWskeS9y+u0nXMyspcuQrCg==", + "devOptional": true, + "dependencies": { + "@types/connect": "*", + "@types/node": "*" + } + }, + "node_modules/@types/connect": { + "version": "3.4.38", + "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz", + "integrity": "sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==", + "devOptional": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/d3-array": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/@types/d3-array/-/d3-array-3.2.1.tgz", @@ -4078,6 +4098,30 @@ "integrity": "sha512-/kYRxGDLWzHOB7q+wtSUQlFrtcdUccpfy+X+9iMBpHK8QLLhx2wIPYuS5DYtR9Wa/YlZAbIovy7qVdB1Aq6Lyw==", "dev": true }, + "node_modules/@types/express": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.21.tgz", + "integrity": "sha512-ejlPM315qwLpaQlQDTjPdsUFSc6ZsP4AN6AlWnogPjQ7CVi7PYF3YVz+CY3jE2pwYf7E/7HlDAN0rV2GxTG0HQ==", + "devOptional": true, + "dependencies": { + "@types/body-parser": "*", + "@types/express-serve-static-core": "^4.17.33", + "@types/qs": "*", + "@types/serve-static": "*" + } + }, + "node_modules/@types/express-serve-static-core": { + "version": "4.19.5", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.19.5.tgz", + "integrity": "sha512-y6W03tvrACO72aijJ5uF02FRq5cgDR9lUxddQ8vyF+GvmjJQqbzDcJngEjURc+ZsG31VI3hODNZJ2URj86pzmg==", + "devOptional": true, + "dependencies": { + "@types/node": "*", + "@types/qs": "*", + "@types/range-parser": "*", + "@types/send": "*" + } + }, "node_modules/@types/graceful-fs": { "version": "4.1.9", "resolved": "https://registry.npmjs.org/@types/graceful-fs/-/graceful-fs-4.1.9.tgz", @@ -4087,6 +4131,12 @@ "@types/node": "*" } }, + "node_modules/@types/http-errors": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@types/http-errors/-/http-errors-2.0.4.tgz", + "integrity": "sha512-D0CFMMtydbJAegzOyHjtiKPLlvnm3iTZyZRSZoLq2mRhDdmLfIWOCYPfQJ4cu2erKghU++QvjcUjp/5h7hESpA==", + "devOptional": true + }, "node_modules/@types/http-proxy": { "version": "1.17.14", "resolved": "https://registry.npmjs.org/@types/http-proxy/-/http-proxy-1.17.14.tgz", @@ -4184,10 +4234,16 @@ "integrity": "sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==", "dev": true }, + "node_modules/@types/mime": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz", + "integrity": "sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==", + "devOptional": true + }, "node_modules/@types/node": { - "version": "20.10.4", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.10.4.tgz", - "integrity": "sha512-D08YG6rr8X90YB56tSIuBaddy/UXAA9RKJoFvrsnogAum/0pmjkgi4+2nx96A330FmioegBWmEYQ+syqCFaveg==", + "version": "20.14.6", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.14.6.tgz", + "integrity": "sha512-JbA0XIJPL1IiNnU7PFxDXyfAwcwVVrOoqyzzyQTyMeVhBzkJVMSkC1LlVsRQ2lpqiY4n6Bb9oCS6lzDKVQxbZw==", "dependencies": { "undici-types": "~5.26.4" } @@ -4200,6 +4256,18 @@ "optional": true, "peer": true }, + "node_modules/@types/qs": { + "version": "6.9.15", + "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.15.tgz", + "integrity": "sha512-uXHQKES6DQKKCLh441Xv/dwxOq1TVS3JPUMlEqoEglvlhR6Mxnlew/Xq/LRVHpLyk7iK3zODe1qYHIMltO7XGg==", + "devOptional": true + }, + "node_modules/@types/range-parser": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.7.tgz", + "integrity": "sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==", + "devOptional": true + }, "node_modules/@types/react": { "version": "18.2.42", "resolved": "https://registry.npmjs.org/@types/react/-/react-18.2.42.tgz", @@ -4238,6 +4306,27 @@ "integrity": "sha512-dn1l8LaMea/IjDoHNd9J52uBbInB796CDffS6VdIxvqYCPSG0V0DzHp76GpaWnlhg88uYyPbXCDIowa86ybd5A==", "dev": true }, + "node_modules/@types/send": { + "version": "0.17.4", + "resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.4.tgz", + "integrity": "sha512-x2EM6TJOybec7c52BX0ZspPodMsQUd5L6PRwOunVyVUhXiBSKf3AezDL8Dgvgt5o0UfKNfuA0eMLr2wLT4AiBA==", + "devOptional": true, + "dependencies": { + "@types/mime": "^1", + "@types/node": "*" + } + }, + "node_modules/@types/serve-static": { + "version": "1.15.7", + "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.7.tgz", + "integrity": "sha512-W8Ym+h8nhuRwaKPaDw34QUkwsGi6Rc4yYqvKFo5rm2FUEhCFbzVWrxXUxuKK8TASjWsysJY0nsmNCGhCOIsrOw==", + "devOptional": true, + "dependencies": { + "@types/http-errors": "*", + "@types/node": "*", + "@types/send": "*" + } + }, "node_modules/@types/shimmer": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/@types/shimmer/-/shimmer-1.0.5.tgz", @@ -13481,6 +13570,38 @@ "integrity": "sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==", "dev": true }, + "node_modules/node-mocks-http": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/node-mocks-http/-/node-mocks-http-1.14.1.tgz", + "integrity": "sha512-mfXuCGonz0A7uG1FEjnypjm34xegeN5+HI6xeGhYKecfgaZhjsmYoLE9LEFmT+53G1n8IuagPZmVnEL/xNsFaA==", + "dev": true, + "dependencies": { + "@types/express": "^4.17.21", + "@types/node": "^20.10.6", + "accepts": "^1.3.7", + "content-disposition": "^0.5.3", + "depd": "^1.1.0", + "fresh": "^0.5.2", + "merge-descriptors": "^1.0.1", + "methods": "^1.1.2", + "mime": "^1.3.4", + "parseurl": "^1.3.3", + "range-parser": "^1.2.0", + "type-is": "^1.6.18" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/node-mocks-http/node_modules/depd": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", + "integrity": "sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/node-releases": { "version": "2.0.14", "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.14.tgz", diff --git a/package.json b/package.json index bc2c53de..27960b78 100644 --- a/package.json +++ b/package.json @@ -87,6 +87,7 @@ "jest-environment-jsdom": "29.7.0", "lint-staged": "^15.2.5", "mini-css-extract-plugin": "^2.9.0", + "node-mocks-http": "^1.14.1", "nodemon": "^3.1.3", "null-loader": "^4.0.1", "path": "^0.12.7", diff --git a/server/server.js b/server/server.js index cdc4544f..f74a96db 100644 --- a/server/server.js +++ b/server/server.js @@ -160,6 +160,12 @@ server.use( }) ) +/* ******************************** + * ******* No index middleware **** + * ******************************** + */ +server.use(require('./utils/noIndexMiddleware.js')) + /* ********************************** * ******* APPLICATION ROUTES ******* * ********************************** diff --git a/server/utils/noIndexMiddleware.js b/server/utils/noIndexMiddleware.js new file mode 100644 index 00000000..2011d71a --- /dev/null +++ b/server/utils/noIndexMiddleware.js @@ -0,0 +1,28 @@ +const { server: serverConfig } = require('../configuration') + +/** + * Middleware that adds "noindex" robots header if "server host" and the request + * host ("forwarded host") doesn't match. This is done to prevent Google from + * indexing app.kth.se/* when we want the user to use www.kth.se/* + * + * The "server host" (SERVER_HOST_URL) is the primary host we expected the + * app to be hosted on (ie https://www.kth.se). + * + * The orginal host of the request is replaced with an Azure host + * (*.azurewebsites.net) before reaching our app so we look at the "forwarded + * host" ("x-forwarded-host" header) . + * + * Default, if x-forwarded-host is missing, is to do nothing. + */ +const noIndexMiddleware = function (req, res, next) { + const forwardedHost = req.header('x-forwarded-host') + if (forwardedHost) { + const serverHostUrl = new URL(serverConfig.hostUrl) + const serverHost = serverHostUrl.host + if (serverHost !== forwardedHost) { + res.set('x-robots-tag', 'noindex') + } + } + next() +} +module.exports = noIndexMiddleware diff --git a/server/utils/noIndexMiddleware.test.js b/server/utils/noIndexMiddleware.test.js new file mode 100644 index 00000000..d4888c44 --- /dev/null +++ b/server/utils/noIndexMiddleware.test.js @@ -0,0 +1,50 @@ +const httpMocks = require('node-mocks-http') + +const noIndexMiddleware = require('./noIndexMiddleware') + +const mockedServerHost = 'mockedserverhost.example.com' +jest.mock('../configuration', () => ({ + server: { hostUrl: 'https://mockedserverhost.example.com' }, +})) + +describe('noIndexMiddleware', () => { + it('should not set "X-Robots-Tag" header if "X-Forwarded-Host" is missing', () => { + const next = jest.fn() + const { req, res } = httpMocks.createMocks({ + headers: {}, + }) + + noIndexMiddleware(req, res, next) + + expect(res.getHeader('X-Robots-Tag')).toBeUndefined() + expect(next).toHaveBeenCalledTimes(1) + }) + + it('should not set "X-Robots-Tag" header if "X-Forwarded-Host" match SERVER_HOST_URL', () => { + const next = jest.fn() + const { req, res } = httpMocks.createMocks({ + headers: { + 'X-Forwarded-Host': mockedServerHost, + }, + }) + + noIndexMiddleware(req, res, next) + + expect(res.getHeader('X-Robots-Tag')).toBeUndefined() + expect(next).toHaveBeenCalledTimes(1) + }) + + it('should set "X-Robots-Tag" header if "X-Forwarded-Host" does not match SERVER_HOST_URL', () => { + const next = jest.fn() + const { req, res } = httpMocks.createMocks({ + headers: { + 'X-Forwarded-Host': 'anotherhost.example.com', + }, + }) + + noIndexMiddleware(req, res, next) + + expect(res.getHeader('X-Robots-Tag')).toBe('noindex') + expect(next).toHaveBeenCalledTimes(1) + }) +})