[UE] remove UE instrumentation in pass-through converter (#1201)

- fetch html from the franklin delivery servlet
hlxsites · Jul 8, 2024 · 89e758f · 89e758f
1 parent 75e26f2
commit 89e758f
Show file tree

Hide file tree

Showing 2 changed files with 152 additions and 6 deletions.
diff --git a/tools/actions/convert/src/index.js b/tools/actions/convert/src/index.js
@@ -15,12 +15,90 @@
 
 // eslint-disable-next-line import/no-extraneous-dependencies
 import { JSDOM } from 'jsdom';
-import { pipe, fetchContent, toRuntime } from 'crosswalk-converter';
+import { pipe, toRuntime } from 'crosswalk-converter';
 import transform from '../../../importer/import.js';
 import converterCfg from '../../../../converter.yaml';
 import mappingCfg from '../../../../paths.yaml';
 import createPipeline from './utils.js';
 
+const mediaTypes = {
+  'application/atom+xml': false,
+  'application/base64': true,
+  'application/excel': true,
+  'application/font-woff': true,
+  'application/gnutar': true,
+  'application/java-archive': true,
+  'application/javascript': false,
+  'application/json': false, // we treat JSON as binary, since its encoding is not variable but defined by RFC4627
+  'application/json-patch+json': false, // we treat JSON as binary, since its encoding is not variable but defined by RFC4627
+  'application/lha': true,
+  'application/lzx': true,
+  'application/mspowerpoint': true,
+  'application/msword': true,
+  'application/octet-stream': true,
+  'application/pdf': true,
+  'application/postscript': true,
+  'application/rss+xml': false,
+  'application/soap+xml': false,
+  'application/vnd.api+json': true, // we treat JSON as binary, since its encoding is not variable but defined by RFC4627
+  'application/vnd.google-earth.kml+xml': false,
+  'application/vnd.google-earth.kmz': true,
+  'application/vnd.ms-fontobject': true,
+  'application/vnd.oasis.opendocument.chart': true,
+  'application/vnd.oasis.opendocument.database': true,
+  'application/vnd.oasis.opendocument.formula': true,
+  'application/vnd.oasis.opendocument.graphics': true,
+  'application/vnd.oasis.opendocument.image': true,
+  'application/vnd.oasis.opendocument.presentation': true,
+  'application/vnd.oasis.opendocument.spreadsheet': true,
+  'application/vnd.oasis.opendocument.text': true,
+  'application/vnd.oasis.opendocument.text-master': true,
+  'application/vnd.oasis.opendocument.text-web': true,
+  'application/vnd.openxmlformats-officedocument.presentationml.presentation': true,
+  'application/vnd.openxmlformats-officedocument.presentationml.slide': true,
+  'application/vnd.openxmlformats-officedocument.presentationml.slideshow': true,
+  'application/vnd.openxmlformats-officedocument.presentationml.template': true,
+  'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': true,
+  'application/vnd.openxmlformats-officedocument.spreadsheetml.template': true,
+  'application/vnd.openxmlformats-officedocument.wordprocessingml.document': true,
+  'application/vnd.openxmlformats-officedocument.wordprocessingml.template': true,
+  'application/x-7z-compressed': true,
+  'application/x-ace-compressed': true,
+  'application/x-apple-diskimage': true,
+  'application/x-arc-compressed': true,
+  'application/x-bzip': true,
+  'application/x-bzip2': true,
+  'application/x-chrome-extension': true,
+  'application/x-compress': true,
+  'application/x-compressed': true,
+  'application/x-debian-package': true,
+  'application/x-dvi': true,
+  'application/x-font-truetype': true,
+  'application/x-font-opentype': true,
+  'application/x-gtar': true,
+  'application/x-gzip': true,
+  'application/x-latex': true,
+  'application/x-rar-compressed': true,
+  'application/x-redhat-package-manager': true,
+  'application/x-shockwave-flash': true,
+  'application/x-tar': true,
+  'application/x-tex': true,
+  'application/x-texinfo': true,
+  'application/x-vrml': false,
+  'application/x-www-form-urlencoded': false,
+  'application/x-x509-ca-cert': true,
+  'application/x-xpinstall': true,
+  'application/xhtml+xml': false,
+  'application/xml-dtd': false,
+  'application/xml': false,
+  'application/zip': true,
+};
+
+export function isBinary(contentType) {
+  if (contentType.startsWith('text/') || contentType.startsWith('message/')) return false;
+  if (contentType.startsWith('audio/') || contentType.startsWith('image/') || contentType.startsWith('video/')) return true;
+  return mediaTypes[contentType];
+}
 function skipConverter(path) {
   // TODO: remove the logic for test pages (with -jck1 in the path)
   if (!path) return false;
@@ -112,13 +190,75 @@ async function rewriteImages(state) {
   return state;
 }
 
+function mapPathToFranklinDeliveryServlet(host, path) {
+  // host: https://ref--repo--owner.hlx.live
+  // mapped path: /bin/franklin.delivery/owner/repo/ref/path
+  const [ref, repo, owner] = host.split('://')[1].split('.')[0].split('--');
+  return `/bin/franklin.delivery/${owner}/${repo}/${ref}${path}`;
+}
+
+function defaultAppendSuffix(mappedPath, suffix) {
+  // if suffix is defined and the mapped path has no extension, add suffix
+  if (suffix && !mappedPath.includes('.') && !mappedPath.endsWith('/')) {
+    /* eslint-disable no-param-reassign */
+    mappedPath += suffix;
+  }
+  return mappedPath;
+}
+
+async function fetchContentWithFranklinDeliveryServlet(state, params, opts) {
+  // input URL: https://ref--repo--owner/path
+  // output URL: https://author-pXYZ-eXYZ.adobeaemcloud.com/bin/franklin.delivery/owner/repo/ref/path
+  const { path, queryString } = state;
+  const { authorization, loginToken } = params;
+  const { origin, suffix, internalHost } = converterCfg || {};
+  const appendSuffix = opts.appendSuffix || defaultAppendSuffix;
+
+  if (!origin) {
+    throw new Error('\'origin\' not set in converter.yaml');
+  }
+
+  if (!internalHost) {
+    throw new Error('\'internalHost\' not set in converter.yaml');
+  }
+
+  let mappedPath = mapPathToFranklinDeliveryServlet(internalHost, path);
+  mappedPath = appendSuffix(mappedPath, suffix);
+  const originUrl = new URL(mappedPath, origin);
+  if (queryString) {
+    originUrl.search = queryString;
+  }
+
+  const requestHeaders = { 'cache-control': 'no-cache' };
+  if (authorization) {
+    requestHeaders.authorization = authorization;
+  } else if (loginToken) {
+    requestHeaders.cookie = loginToken;
+  }
+
+  const resp = await fetch(originUrl, { headers: { ...requestHeaders } });
+
+  if (!resp.ok) {
+    return { ...state, error: { code: resp.status, message: resp.statusText } };
+  }
+
+  const [contentType] = (resp.headers.get('content-type') || 'text/html').split(';');
+  const contentLength = resp.headers.get('content-length') || -1;
+  // for binaries return the readable stream
+  const blob = isBinary(contentType) ? resp.body : await resp.text();
+
+  return {
+    ...state, originUrl, blob, contentType, contentLength,
+  };
+}
+
 export async function main(params) {
   // eslint-disable-next-line no-underscore-dangle
   const path = params.__ow_path;
   const silent = params.silent === 'true';
   const pipeline = skipConverter(path)
     ? pipe()
-      .use(fetchContent)
+      .use(fetchContentWithFranklinDeliveryServlet)
       .use(rewriteImages)
       .use(rewriteLinks)
     : createPipeline();

diff --git a/tools/actions/convert/test/pass-through-converter.test.js b/tools/actions/convert/test/pass-through-converter.test.js
@@ -9,20 +9,26 @@ describe('Pass through Converter', () => {
   // eslint-disable-next-line no-undef
   const passThroughFolder = path.resolve(__testdir, 'pass-through');
   converterCfg.origin = 'https://author-dummy.adobeaemcloud.com';
+  const passThroughPath = '/ls/us/en/products/centrifuges/topics-jck1/microcentrifuge-benefits';
+  const convertedPath = '/ls/us/en/products/centrifuges';
   const inputHtml = fs.readFileSync(path.resolve(passThroughFolder, 'simple.html'), { encoding: 'utf-8' });
   const convertedHtml = fs.readFileSync(path.resolve(passThroughFolder, 'simple-converted.html'), { encoding: 'utf-8' });
   const passThroughHtml = fs.readFileSync(path.resolve(passThroughFolder, 'simple-pass-through.html'), { encoding: 'utf-8' });
 
+  nock(converterCfg.origin)
+    .get(`${convertedPath}.html?wcmmode=disabled`)
+    .reply(200, inputHtml, { 'content-type': 'text/html' });
+
+  nock(converterCfg.origin)
+    .get(`/bin/franklin.delivery/hlxsites/danaher-ls-aem/main${passThroughPath}`)
+    .reply(200, inputHtml, { 'content-type': 'text/html' });
+
   async function runTest(requestPath, outputHtml) {
     const params = {
       __ow_path: requestPath,
       silent: 'true',
     };
 
-    nock(converterCfg.origin)
-      .get(`${requestPath}.html?wcmmode=disabled`)
-      .reply(200, inputHtml, { 'content-type': 'text/html' });
-
     const result = await main(params);
     assert.equal(result.body.trim(), outputHtml.replaceAll('\r\n', '\n').trim(), 'result is not as expected');
   }