index.js

const puppeteer = require('puppeteer')
const cheerio = require('cheerio')
const fs = require('fs-extra')
const path = require('path')
const url = require('url')
const axios = require('axios')

/**
 * Global Variables
 */
const visitedUrlSet = new Set()
const queuedUrlSet = new Set()
let completedItemCount = 0
const supportedAssetExtensions = [
  '.jpg', '.jpeg', '.png', '.gif', '.svg', '.webp', '.ico',
  '.css', '.js', '.woff', '.woff2', '.ttf', '.otf', '.eot',
  '.mp4', '.webm', '.ogg', '.mp3', '.wav'
]
const assetRegexPattern = new RegExp('([a-zA-Z0-9_\\-\\/.]+(' + supportedAssetExtensions.map(ext => ext.replace('.', '\\.')).join('|') + '))', 'gi')

/**
 * Print Progress Status
 * Params: queueLength, completedCount
 */
function printProgressStatus(queueLength, completedCount) {
  const progressPercent = ((completedCount / (queueLength + completedCount)) * 100).toFixed(1)
  process.stdout.write(`\r[${completedCount}/${queueLength + completedCount}] | Queue Left: ${queueLength} | Progress: ${progressPercent}%     `)
}

/**
 * Check Local File Validity
 * Params: localFilePath
 */
async function isLocalFileValid(localFilePath) {
  try {
    const fileStat = await fs.stat(localFilePath)
    if (fileStat.isFile() && fileStat.size > 0) {
      return true
    } else {
      return false
    }
  } catch {
    return false
  }
}

/**
 * Find All Asset Links By Regex
 * Params: textContent, baseUrl
 */
function findAllAssetLinksByRegex(textContent, baseUrl) {
  const foundAssets = []
  let matchedResult
  while ((matchedResult = assetRegexPattern.exec(textContent)) !== null) {
    let assetLink = matchedResult[1]
    if (assetLink.startsWith('data:')) { continue }
    if (assetLink.startsWith('http') || assetLink.startsWith('//')) {
      assetLink = assetLink.startsWith('//') ? 'https:' + assetLink : assetLink
    } else {
      assetLink = url.resolve(baseUrl, assetLink)
    }
    if (!(queuedUrlSet.has(assetLink))) {
      foundAssets.push(assetLink)
    }
  }
  return foundAssets
}

/**
 * Parse CSS File And Queue Asset
 * Params: cssFilePath, cssFileUrl, outputFolderPath, assetQueueList
 */
async function parseCssFileAndQueueAsset(cssFilePath, cssFileUrl, outputFolderPath, assetQueueList) {
  let cssContent
  try {
    cssContent = await fs.readFile(cssFilePath, 'utf-8')
  } catch { return }
  const cssBaseDir = cssFileUrl.endsWith('/') ? cssFileUrl : cssFileUrl.substring(0, cssFileUrl.lastIndexOf('/') + 1)
  const urlRegex = /url\((['"]?)([^'")]+)\1\)/g
  let matchedResult
  while ((matchedResult = urlRegex.exec(cssContent)) !== null) {
    let cssAssetPath = matchedResult[2]
    if (cssAssetPath.startsWith('data:')) { continue }
    let cssAssetUrl = cssAssetPath.startsWith('http') || cssAssetPath.startsWith('//')
      ? (cssAssetPath.startsWith('//') ? 'https:' + cssAssetPath : cssAssetPath)
      : url.resolve(cssBaseDir, cssAssetPath)
    if (
      cssAssetUrl.match(/\.(woff2?|ttf|otf|eot|svg|png|jpg|jpeg|gif|webp|ico|css|js)$/i)
      && !(queuedUrlSet.has(cssAssetUrl))
    ) {
      console.log('[CSS url()]', cssAssetUrl)
      assetQueueList.push({ url: cssAssetUrl, type: 'asset' })
      queuedUrlSet.add(cssAssetUrl)
    }
  }
  const importUrlRegex = /@import\s+url\((['"]?)([^'")]+)\1\)/g
  while ((matchedResult = importUrlRegex.exec(cssContent)) !== null) {
    let importAssetPath = matchedResult[2]
    if (importAssetPath.startsWith('data:')) { continue }
    let importAssetUrl = importAssetPath.startsWith('http') || importAssetPath.startsWith('//')
      ? (importAssetPath.startsWith('//') ? 'https:' + importAssetPath : importAssetPath)
      : url.resolve(cssBaseDir, importAssetPath)
    if (
      importAssetUrl.match(/\.css$/i)
      && !(queuedUrlSet.has(importAssetUrl))
    ) {
      console.log('[CSS @import]', importAssetUrl)
      assetQueueList.push({ url: importAssetUrl, type: 'asset' })
      queuedUrlSet.add(importAssetUrl)
    }
  }
  const allAssetLinks = findAllAssetLinksByRegex(cssContent, cssBaseDir)
  for (const regexAssetUrl of allAssetLinks) {
    if (
      regexAssetUrl.match(/\.(woff2?|ttf|otf|eot|svg|png|jpg|jpeg|gif|webp|ico|css|js)$/i)
      && !(queuedUrlSet.has(regexAssetUrl))
    ) {
      console.log('[CSS regex]', regexAssetUrl)
      assetQueueList.push({ url: regexAssetUrl, type: 'asset' })
      queuedUrlSet.add(regexAssetUrl)
    }
  }
}

/**
 * Download With Retry
 * Params: fn, retries, delayMs
 */
async function downloadWithRetry(fn, retries = 3, delayMs = 2000) {
  for (let retryAttempt = 1; retryAttempt <= retries; retryAttempt++) {
    try {
      return await fn()
    } catch {
      if (retryAttempt === retries) { throw new Error('Retry Failed') }
      await new Promise(resolve => setTimeout(resolve, delayMs))
    }
  }
}

/**
 * Start Cloning Process
 * Params: rootWebsiteUrl, outputFolderPath
 */
async function startCloningProcess(rootWebsiteUrl, outputFolderPath) {
  const browserInstance = await puppeteer.launch({ headless: true, args: ['--no-sandbox', '--disable-setuid-sandbox'] })
  const browserPage = await browserInstance.newPage()
  const mainQueueList = []
  mainQueueList.push({ url: rootWebsiteUrl, type: 'page' })
  queuedUrlSet.add(rootWebsiteUrl)

  while (mainQueueList.length > 0) {
    const currentQueueItem = mainQueueList.shift()
    if (currentQueueItem.type === 'page') {
      await downloadWithRetry(() => downloadHtmlPage(currentQueueItem.url, rootWebsiteUrl, outputFolderPath, mainQueueList, browserPage))
    } else {
      await downloadWithRetry(() => downloadAssetFile(currentQueueItem.url, outputFolderPath, mainQueueList))
    }
    completedItemCount++
    printProgressStatus(mainQueueList.length, completedItemCount)
  }
  await browserInstance.close()
  process.stdout.write('\nDone!\n')
}

/**
 * Download HTML Page
 * Params: pageUrl, rootWebsiteUrl, outputFolderPath, queueList, browserPage
 */
async function downloadHtmlPage(pageUrl, rootWebsiteUrl, outputFolderPath, queueList, browserPage) {
  if (visitedUrlSet.has(pageUrl)) { return }
  visitedUrlSet.add(pageUrl)
  console.log(`\n[Page] Downloading: ${pageUrl}`)
  try {
    await browserPage.goto(pageUrl, { waitUntil: 'networkidle2', timeout: 60000 })
  } catch { return }
  const pageContent = await browserPage.content()
  const $ = cheerio.load(pageContent)
  const htmlBaseDir = pageUrl.endsWith('/') ? pageUrl : pageUrl.substring(0, pageUrl.lastIndexOf('/') + 1)
  const htmlAssetTags = [
    ['link', 'href'],
    ['script', 'src'],
    ['img', 'src'],
    ['video', 'src'],
    ['audio', 'src'],
    ['source', 'src'],
    ['iframe', 'src'],
    ['source', 'srcset']
  ]
  for (const [tagName, attrName] of htmlAssetTags) {
    $(`${tagName}[${attrName}]`).each((_, tagEl) => {
      let assetSourceUrl = $(tagEl).attr(attrName)
      if (!assetSourceUrl) { return }
      if (assetSourceUrl.startsWith('http') || assetSourceUrl.startsWith('//')) {
        assetSourceUrl = assetSourceUrl.startsWith('//') ? 'https:' + assetSourceUrl : assetSourceUrl
      } else {
        assetSourceUrl = url.resolve(htmlBaseDir, assetSourceUrl)
      }
      if (
        assetSourceUrl.match(/\.(woff2?|ttf|otf|eot|svg|png|jpg|jpeg|gif|webp|ico|css|js)$/i)
        && !(queuedUrlSet.has(assetSourceUrl))
      ) {
        console.log('[TAG]', assetSourceUrl)
        queueList.push({ url: assetSourceUrl, type: 'asset' })
        queuedUrlSet.add(assetSourceUrl)
      }
    })
  }
  $('style').each((_, styleTag) => {
    const cssTextContent = $(styleTag).html()
    const cssUrlRegex = /url\((['"]?)([^'")]+)\1\)/g
    let matchedCss
    while ((matchedCss = cssUrlRegex.exec(cssTextContent)) !== null) {
      let styleAssetPath = matchedCss[2]
      if (styleAssetPath.startsWith('data:')) { continue }
      let styleAssetUrl = styleAssetPath.startsWith('http') || styleAssetPath.startsWith('//')
        ? (styleAssetPath.startsWith('//') ? 'https:' + styleAssetPath : styleAssetPath)
        : url.resolve(htmlBaseDir, styleAssetPath)
      if (
        styleAssetUrl.match(/\.(woff2?|ttf|otf|eot|svg|png|jpg|jpeg|gif|webp|ico|css|js)$/i)
        && !(queuedUrlSet.has(styleAssetUrl))
      ) {
        console.log('[STYLE url()]', styleAssetUrl)
        queueList.push({ url: styleAssetUrl, type: 'asset' })
        queuedUrlSet.add(styleAssetUrl)
      }
    }
  })
  const foundAssetsByRegex = findAllAssetLinksByRegex(pageContent, pageUrl)
  for (const regexAssetUrl of foundAssetsByRegex) {
    if (
      regexAssetUrl.match(/\.(woff2?|ttf|otf|eot|svg|png|jpg|jpeg|gif|webp|ico|css|js)$/i)
      && !(queuedUrlSet.has(regexAssetUrl))
    ) {
      console.log('[HTML regex]', regexAssetUrl)
      queueList.push({ url: regexAssetUrl, type: 'asset' })
      queuedUrlSet.add(regexAssetUrl)
    }
  }
  await saveClonedFile(pageUrl, rootWebsiteUrl, outputFolderPath, pageContent)

  $('a[href]').each((_, anchorTag) => {
    let hrefLink = $(anchorTag).attr('href')
    if (
      hrefLink &&
      !(hrefLink.startsWith('http')) &&
      !(hrefLink.startsWith('//')) &&
      !(hrefLink.startsWith('mailto:')) &&
      !(hrefLink.startsWith('#'))
    ) {
      hrefLink = url.resolve(htmlBaseDir, hrefLink)
      if (!(queuedUrlSet.has(hrefLink))) {
        queueList.push({ url: hrefLink, type: 'page' })
        queuedUrlSet.add(hrefLink)
      }
    } else if (
      hrefLink &&
      hrefLink.startsWith(rootWebsiteUrl) &&
      !(queuedUrlSet.has(hrefLink))
    ) {
      queueList.push({ url: hrefLink, type: 'page' })
      queuedUrlSet.add(hrefLink)
    }
  })
}

/**
 * Download Asset File
 * Params: assetUrl, outputFolderPath, assetQueueList
 */
async function downloadAssetFile(assetUrl, outputFolderPath, assetQueueList) {
  if (visitedUrlSet.has(assetUrl)) { return }
  visitedUrlSet.add(assetUrl)
  console.log(`\n[Asset] Downloading: ${assetUrl}`)
  try {
    const parsedAssetUrl = new url.URL(assetUrl)
    let assetFilePath = parsedAssetUrl.pathname
    if (assetFilePath.endsWith('/')) { assetFilePath += 'index.html' }
    if (assetFilePath === '') { assetFilePath = '/index.html' }
    const localAssetPath = path.join(outputFolderPath, assetFilePath)
    if (await isLocalFileValid(localAssetPath)) {
      if (assetUrl.endsWith('.css')) {
        await parseCssFileAndQueueAsset(localAssetPath, assetUrl, outputFolderPath, assetQueueList)
      }
      return
    }
    const response = await axios.get(assetUrl, { responseType: 'arraybuffer', timeout: 60000, headers: { 'User-Agent': 'Mozilla/5.0' } })
    await saveClonedFile(assetUrl, '', outputFolderPath, response.data, true)
    if (assetUrl.endsWith('.css')) {
      await parseCssFileAndQueueAsset(localAssetPath, assetUrl, outputFolderPath, assetQueueList)
    }
  } catch {
    return
  }
}

/**
 * Save Cloned File
 * Params: fileUrl, rootWebsiteUrl, outputFolderPath, fileContent, isBinary
 */
async function saveClonedFile(fileUrl, rootWebsiteUrl, outputFolderPath, fileContent, isBinary) {
  const parsedFileUrl = new url.URL(fileUrl)
  let outputFilePath = parsedFileUrl.pathname
  if (outputFilePath.endsWith('/')) { outputFilePath += 'index.html' }
  if (outputFilePath === '') { outputFilePath = '/index.html' }
  const localOutputFilePath = path.join(outputFolderPath, outputFilePath)
  await fs.mkdir(path.dirname(localOutputFilePath), { recursive: true })
  if (!isBinary && (outputFilePath.endsWith('.html') || outputFilePath.endsWith('.css'))) {
    fileContent = fileContent.replace(
      /(url\(['"]?[^)'"]+\.(woff2?|woff|ttf|otf|eot|svg|png|jpe?g|gif|webp|ico|css|js))([?#][^)'"]*)?(['"]?\))/gmi,
      function (_, a, b, c, d) {
        return a + (d ? d : ')')
      }
    )
    fileContent = fileContent.replace(
      /((?:src|href)=["'][^"']+\.(?:woff2?|woff|ttf|otf|eot|svg|png|jpe?g|gif|webp|ico|css|js))([?#][^"']*)/gi,
      '$1'
    )
  }
  if (isBinary) {
    await fs.writeFile(localOutputFilePath, fileContent)
  } else {
    await fs.writeFile(localOutputFilePath, fileContent, 'utf-8')
  }
}

/**
 * CLI Entrypoint
 */
const inputUserUrl = process.argv[2]
const outputTargetFolder = process.argv[3] || './output'
if (!(inputUserUrl)) {
  console.log('Missing Url')
  process.exit(1)
}
const normalizedRootUrl = inputUserUrl.endsWith('/') ? inputUserUrl : inputUserUrl + '/'
startCloningProcess(normalizedRootUrl, outputTargetFolder)