feat: better resolution for github urls

lib/transforms/code/index.js

@@ -4,7 +4,7 @@ const remoteRequest = require('../../utils/remoteRequest')
 const { isLocalPath } = require('../../utils/fs')
 const { deepLog } = require('../../utils/logs')
 const { getLineCount, getTextBetweenLines } = require('../../utils/text')
-const { resolveGithubContents, resolveGithubLink } = require('./resolve-github-file')
+const { resolveGithubContents, isGithubLink } = require('./resolve-github-file')
 
 const GITHUB_LINK = /https:\/\/github\.com\/([^/\s]*)\/([^/\s]*)\/blob\//
 const GIST_LINK = /https:\/\/gist\.github\.com\/([^/\s]*)\/([^/\s]*)(\/)?/
@@ -31,13 +31,6 @@ const GIST_LINK = /https:\/\/gist\.github\.com\/([^/\s]*)\/([^/\s]*)(\/)?/
  */
 
 
-function resolveAccessToken(accessToken) {
-  if (typeof accessToken === 'string' && accessToken.match(/process\.env\./)) {
-    return process.env[accessToken.replace('process.env.', '')]
-  }
-  return accessToken || process.env.GITHUB_ACCESS_TOKEN
-}
-
 // TODO code sections 
 // https://github.com/linear/linear/blob/94af540244864fbe466fb933256278e04e87513e/docs/transforms/code-section.js
 // https://github.com/linear/linear/blob/bc39d23af232f9fdbe7df458b0aaa9554ca83c57/packages/sdk/src/_tests/readme.test.ts#L133-L140
@@ -96,10 +89,10 @@ module.exports = async function CODE(api) {
       remoteContent = remoteRequest(src)
     }
     if (!remoteContent) {
-      if (resolveGithubLink(src)) {
-        remoteContent = await resolveGithubContents({ 
+      if (isGithubLink(src)) {
+        remoteContent = await resolveGithubContents({
           repoFilePath: src,
-          accessToken: resolveAccessToken(accessToken),
+          accessToken,
           // debug: true
         })
       }

lib/transforms/code/resolve-github-file.js

@@ -1,5 +1,4 @@
 const https = require('https')
-const safe = require('safe-await')
 const { exec } = require('child_process')
 const { getTextBetweenLines } = require('../../utils/text')
 
@@ -8,51 +7,53 @@ const VALID_FILE_REGEX = /^[^;]*$/
 const GITHUB_LINK_REGEX = /^(?:https:\/\/)?github\.com\/([^/\s]*)\/([^/\s]*)\/blob\/([^/\s]*)\/([^\s]*)/
 const GITHUB_RAW_LINK_REGEX = /^(?:https:\/\/)?raw\.githubusercontent\.com\/([^/\s]*)\/([^/\s]*)\/([^/\s]*)\/([^\s]*)/
 
-function resolveGithubLink(repoFilePath) {
-  const matches = repoFilePath.match(GITHUB_LINK_REGEX)
-  if (matches) {
-    const [_match, repoOwner, repoName, branchOrRef, filePath ] = matches
-    const [ filePathStart, hash ] = filePath.split('#')
-    const result = {
-      repoOwner,
-      repoName,
-      filePath: filePathStart,
-    }
-    if (isGitHash(branchOrRef)) {
-      result.ref = branchOrRef
-    } else {
-      result.branch = branchOrRef
-    }
-    if (hash) {
-      const range = parseLineRange(`#${hash}`)
-      if (range) {
-        result.range = range
-      }
-    }
-    return result
+function isGithubLink(str = '') {
+  return isGithubRepoLink(str) || isGithubRawLink(str)
+}
+
+function isGithubRepoLink(str = '') {
+  return GITHUB_LINK_REGEX.test(str)
+}
+
+function isGithubRawLink(str = '') {
+  return GITHUB_RAW_LINK_REGEX.test(str)
+}
+
+function convertLinkToRaw(link) {
+  if (!isGithubRepoLink(link)) return link
+  return link.replace(GITHUB_LINK_REGEX, 'https://raw.githubusercontent.com/$1/$2/$3/$4')
+}
+
+function resolveGithubDetails(repoFilePath) {
+  let parts
+  if (isGithubRepoLink(repoFilePath)) {
+    parts = repoFilePath.match(GITHUB_LINK_REGEX)
   }
-  const rawMatches = repoFilePath.match(GITHUB_RAW_LINK_REGEX)
-  if (rawMatches) {
-    const [_match, repoOwner, repoName, branchOrRef, filePath ] = rawMatches
-    const [ filePathStart, hash ] = filePath.split('#')
-    const result = {
-      repoOwner,
-      repoName,
-      filePath: filePathStart,
-    }
-    if (isGitHash(branchOrRef)) {
-      result.ref = branchOrRef
-    } else {
-      result.branch = branchOrRef
-    }
-    if (hash) {
-      const range = parseLineRange(`#${hash}`)
-      if (range) {
-        result.range = range
-      }
+  if (isGithubRawLink(repoFilePath)) {
+    parts = repoFilePath.match(GITHUB_RAW_LINK_REGEX)
+  }
+  if (!parts) {
+    return
+  }
+  const [ _match, repoOwner, repoName, branchOrRef, filePath ] = parts
+  const [ filePathStart, hash ] = filePath.split('#')
+  const result = {
+    repoOwner,
+    repoName,
+    filePath: filePathStart,
+  }
+  if (isGitHash(branchOrRef)) {
+    result.ref = branchOrRef
+  } else {
+    result.branch = branchOrRef
+  }
+  if (hash) {
+    const range = parseLineRange(`#${hash}`)
+    if (range) {
+      result.range = range
     }
-    return result
   }
+  return result
 }
 
 /**
@@ -70,35 +71,60 @@ async function resolveGithubContents({
   accessToken,
   debug = false
 }) {
+  const token = resolveAccessToken(accessToken)
   const logger = (debug) ? console.log : () => {}
-  const linkInfo = resolveGithubLink(repoFilePath)
-  logger('github link info', linkInfo)
-  if (!linkInfo) {
+  const githubDetails = resolveGithubDetails(repoFilePath)
+  if (!githubDetails) {
     throw new Error(`Invalid github link. "${repoFilePath}" is not a valid github link`)
   }
-  const githubFetcher = (accessToken) ? getGitHubFileContents : getGitHubFileContentsCli
-  logger('github fetcher', githubFetcher.name)
+
+  logger(`Github File Details "${repoFilePath}"`, githubDetails)
+
   const payload = {
-    ...linkInfo,
-    accessToken
+    ...githubDetails,
+    accessToken: token
   }
-  let [err, fileContent] = await safe(githubFetcher(payload))
-  if (fileContent) {
-    logger(`${githubFetcher.name} resolved contents`, fileContent)
+
+  let errs = []
+
+  /* Try raw request first */
+  try {
+    const fileContent = await getGitHubFileContentsRaw(payload)
+    logger(`✅  GitHub file resolved via raw GET`)
+    return returnCode(fileContent, githubDetails.range)
+  } catch (err) {
+    logger('❌  Unable to resolve GitHub raw content')
+    errs.push(err)
   }
-
-  if (!fileContent) {
-    [err, fileContent] = await safe(getGitHubFileContents(payload))
+
+  /* Then try Github CLI or GitHub API */
+  const githubFetcher = (!token) ? getGitHubFileContentsCli : getGitHubFileContentsApi
+  try {
+    const fileContent = await githubFetcher(payload)
+    logger(`✅  GitHub file resolved via ${githubFetcher.name}`)
+    return returnCode(fileContent, githubDetails.range)
+  } catch (err) {
+    logger(`❌ Unable to resolve GitHub file via ${githubFetcher.name}`)
+    errs.push(err)
   }
 
-  if (fileContent) {
-    if (linkInfo.range) {
-      const [startLine, endLine] = linkInfo.range
-      return getTextBetweenLines(fileContent, startLine, endLine)
-    }
-    return fileContent
+  /* Then try API */
+  try {
+    const fileContent = await getGitHubFileContentsApi(payload)
+    logger(`✅  GitHub file resolved via ${getGitHubFileContentsApi.name}`)
+    return returnCode(fileContent, githubDetails.range)
+  } catch (err) {
+    logger(`❌ Unable to resolve GitHub file via ${githubFetcher.name}`)
+    errs.push(err)
   }
-  throw new Error(`Failed to fetch file. ${err.message}`)
+
+  throw new Error(`Failed to fetch GitHub file "${repoFilePath}". \n${errs.forEach(err => err.message)}`)
+}
+
+function returnCode(fileContent, lines) {
+  if (!lines) return fileContent
+  const [startLine, endLine] =lines
+  return getTextBetweenLines(fileContent, startLine, endLine)
 }
 
 /**
@@ -159,7 +185,7 @@ async function getGitHubFileContentsCli(options) {
  * @returns {Promise<string>} A promise that resolves with the decoded contents of the file.
  * @throws {Error} If the file retrieval fails or the response status code is not 200.
  */
-function getGitHubFileContents(options) {
+function getGitHubFileContentsApi(options) {
   validateInputs(options)
 
   const {
@@ -192,7 +218,9 @@ function getGitHubFileContents(options) {
         ...(accessToken) ? { 'Authorization': `token ${accessToken}` } : {},
       }
     }
-
+    /*
+    console.log('getGitHubFileContentsApi options', options)
+    /** */
     const req = https.request(options, (res) => {
       let data = ''
       res.on('data', (chunk) => {
@@ -217,6 +245,54 @@ function getGitHubFileContents(options) {
   })
 }
 
+function getGitHubFileContentsRaw(options) {
+  validateInputs(options)
+
+  const {
+    repoOwner,
+    repoName,
+    filePath,
+    branch,
+    accessToken
+  } = options
+
+  const [ _filePath, hash ] = filePath.split('#')
+  return new Promise((resolve, reject) => {
+    const options = {
+      hostname: 'raw.githubusercontent.com',
+      path: `/${repoOwner}/${repoName}/${branch}/${_filePath}`,
+      method: 'GET',
+      headers: {
+        'User-Agent': 'Node.js',
+        ...(accessToken) ? { 'Authorization': `token ${accessToken}` } : {},
+      }
+    }
+    /*
+    console.log('getGitHubFileContentsRaw options', options)
+    /** */
+    const req = https.request(options, (res) => {
+      let data = ''
+      res.on('data', chunk => {
+        data += chunk
+      })
+
+      res.on('end', () => {
+        if (res.statusCode === 200) {
+          resolve(data)
+        } else {
+          reject(new Error(`Failed to fetch file. Status code: ${res.statusCode}`))
+        }
+      })
+    })
+
+    req.on('error', error => {
+      reject(error)
+    })
+
+    req.end()
+  })
+}
+
 /**
  * Validates the inputs for a repository operation.
  *
@@ -252,6 +328,13 @@ function validateInputs({
   }
 }
 
+function resolveAccessToken(accessToken) {
+  if (typeof accessToken === 'string' && accessToken.match(/process\.env\./)) {
+    return process.env[accessToken.replace('process.env.', '')]
+  }
+  return accessToken || process.env.GITHUB_ACCESS_TOKEN
+}
+
 function decode(fileContent) {
   return Buffer.from(fileContent, 'base64').toString('utf-8')
 }
@@ -271,6 +354,9 @@ function parseLineRange(lineRangeString) {
 }
 
 module.exports = {
-  resolveGithubLink,
+  isGithubLink,
+  isGithubRawLink,
+  getGitHubFileContentsRaw,
+  resolveGithubDetails,
   resolveGithubContents,
 }

lib/transforms/code/resolve-github-file.test.js

@@ -1,17 +1,32 @@
-const { resolveGithubContents } = require('./resolve-github-file')
+const { resolveGithubContents, getGitHubFileContentsRaw } = require('./resolve-github-file')
 
 let repoFilePath
-// repoFilePath = 'https://github.com/DavidWells/markdown-magic/blob/master/package.json'
+repoFilePath = 'https://github.com/DavidWells/markdown-magic/blob/master/package.json'
 // repoFilePath = 'https://github.com/DavidWells/notes/blob/master/cognito.md'
 // repoFilePath = 'github.com/DavidWells/notes/blob/master/cognito.md#L1-L5'
 repoFilePath = 'github.com/DavidWells/notes/blob/master/cognito.md'
 // repoFilePath = 'https://raw.githubusercontent.com/DavidWells/notes/master/cognito.md'
 // repoFilePath = 'raw.githubusercontent.com/DavidWells/notes/master/cognito.md'
 // repoFilePath = 'https://github.com/reapit/foundations/blob/53b2be65ea69d5f1338dbea6e5028c7599d78cf7/packages/connect-session/src/browser/index.ts#L125-L163'
+
+/*
 resolveGithubContents({
   repoFilePath,
   debug: true,
+  //accessToken: process.env.GITHUB_LAST_EDITED_TOKEN
+})
+  .then(console.log)
+  .catch(console.error);
+/** */
+
+/*
+getGitHubFileContentsRaw({
+  repoOwner: 'DavidWells', 
+  repoName: 'notes',
+  filePath: 'cognito.md', 
+  branch: 'master', 
   accessToken: process.env.GITHUB_LAST_EDITED_TOKEN
 })
   .then(console.log)
-  .catch(console.error);
+  .catch(console.error);
+/** */

lib/utils/text.js

@@ -60,6 +60,14 @@ function replaceTextBetweenChars(str = '', start, end, newStr) {
   return str.substring(0, start) + newStr + str.substring(end)
 }
 
+/**
+ * Retrieves the text content between the specified start and end lines.
+ *
+ * @param {string} content - The content to extract text from.
+ * @param {number} startLine - The line number where the extraction should start.
+ * @param {number} endLine - The line number where the extraction should end.
+ * @returns {string|undefined} - The extracted text content, or undefined if both startLine and endLine are not defined.
+ */
 function getTextBetweenLines(content, startLine, endLine) {
   const startDefined = typeof startLine !== 'undefined'
   const endDefined = typeof endLine !== 'undefined'