Case study: Archiving web sites with wget and puppeteer

#!/usr/bin/env bash

DOMAIN=$1
START_URL=$2

wget \
  --mirror \
  --page-requisites \
  --html-extension \
  --convert-links \
  --domains $DOMAIN \
  --rejected-log=$DOMAIN-rejected.log \
  --directory-prefix=site-$DOMAIN \
     $START_URL

const http = require("http");
const puppeteer = require("puppeteer");
const fetch = require("node-fetch");

let browser;
const server = http.createServer(async (req, res) => {
  // We look at the proxy request and figure out the url
  // the client (wget) wants to fetch.
  const { url = "/" } = req;
  try {
    // Make a network request, and figure out
    // what lies beyond the url. It could be a web page,
    // an image, a video or something else.
    const fetchResult = await fetch(url);
    const headers = fetchResult.headers.raw();
    console.log("Processing", { url });
    console.log({ headers });
    const { "content-type": contentType = [] } = headers;
    res.writeHead(fetchResult.status, {
      ["content-type"]: contentType
    });
    if (contentType[0] === "text/html") {
      // If the content is a web page we fire up puppeteer
      // to help us lazy load all images. See parseWithPuppeteer
      // function.
      const text = await fetchResult.text();
      const modifiedText = await parseWithPuppeteer({ text });
      res.write(modifiedText);
      res.end();
    } else {
      // For everything else we just send the fetched contents
      // back to wget.
      fetchResult.body.pipe(res);
    }
  } catch (err) {
    // If something goes wrong we need to respond with proper
    // status code so that wget can move on instead of waiting
    // endlessly for a response.
    console.log("Failed to parse url", { url, err });
    res.status = 500;
    res.end();
  }
});

const parseWithPuppeteer = async ({ text }) => {
  const page = await browser.newPage();
  await page.setViewport({
    width: 2560,
    height: 10000
  });
  // Take the html text and insert it into the browser,
  // this will run its javascript and load any resources.
  // We tell puppeteer to wait until the network is idle.
  await page.setContent(text, {
    waitUntil: "networkidle0"
  });
  // Finally! This is the meat and potatoes where our
  // proxy can have puppeteer make sweeping changes to the
  // html. This code is tailored to the lazy loading functionality
  // found on the iris.no web site.
  await page.evaluate(() => {
    // Manually lazy load all images on the page by rewriting
    // the html so that invisible references to images becomes
    // visible to the browser and thus downloadable.
    document.querySelectorAll("[data-src]").forEach(el => {
      el.setAttribute("src", el.getAttribute("data-src"));
      el.removeAttribute("data-src");
    });
    // Remove .lazyload classes to prevent the page's javascript
    // from trying to lazy load these images when we view the
    // archived web site in the future.
    document.querySelectorAll(".lazyload").forEach(el => {
      el.classList.remove("lazyload", "lazyfade");
    });
    // Scroll to the bottom of the page to trigger lazy loading
    // just in case if there was something we missed out on.
    window.scrollTo(0, document.body.scrollHeight);
  });
  // Wait a bit to give the site a chance to load any images,
  // before pulling the changed html out of the browser and
  // passing it on.
  await page.waitFor(100);
  const content = await page.content();
  await page.close();
  return content;
};

// The initialisation code that opens a hidden browser ready
// to parse html and starts the web server.
(async () => {
  browser = await puppeteer.launch({ headless: true });
  const port = 8080;
  server.listen(port);
  console.log(`Node.js web server at port ${port} is running..`);
})();

// Ensure that we close the hidden browser when the process exits.
process.on("SIGINT", () => {
  browser.close();
});

const puppeteer = require('puppeteer')

// Spin up a browser in the background
let browser
const setupBrowser = async () => {
  puppeteer
    .launch({ headless: true, ignoreHTTPSErrors: true })
    .then(builtBrowser => {
      browser = builtBrowser
      browser.on('disconnected', setupBrowser)
    })
}
setupBrowser()

process.on('SIGINT', () => {
  browser.close()
})

// Build an Anyproxy module following its specification.
module.exports = {
  // introduction
  summary: 'Puppeteer rules for anyproxy',
  // intercept before making send request to target server
  *beforeSendRequest(requestDetail) {
    const {
      url,
      requestOptions: { path, hostname },
      protocol,
    } = requestDetail

    // If the url contains an at-symbol we 404 it. This
    // avoids confusing wget. When crawling I encountered
    // email urls that returned the uni.no site leading wget
    // to start traversing the site from there as if the
    // email was a new domain. This had to be mitigated.
    if (new RegExp(`/https?:\/\/.*@${hostname}`).test(url)) {
      return {
        response: {
          statusCode: 404,
          header: { 'content-type': 'text/html' },
          body: '',
        },
      }
    }

    // Avoid errors from the faulty certificate on cms.uni.no. I
    // first restricted the crawling to uni.no, but then discovered that
    // the whole site was available on cms.uni.no. And I would much rather
    // restrict the crawling to the cms. subdomain to increase chances of it
    // completing. This wasn't straightforward since the https certificate
    // was only setup for uni.no. So, any https requests to cms.uni.no would
    // throw a certificate error. This was finally fixed by pre-modifying
    // any https requests to http.
    if (hostname === 'cms.uni.no' && protocol === 'https') {
      return { protocol: 'http' }
    }

    return null
  },
  // deal response before send to client
  *beforeSendResponse(requestDetail, responseDetail) {
    const { url } = requestDetail
    const { 'Content-Type': contentType = '' } = responseDetail.response.header
    if (contentType.split(';')[0] === 'text/html') {
      const html = responseDetail.response.body.toString('utf8')
      return browser
        .newPage()
        .then(async page => {
          await page.setViewport({
            width: 2560,
            height: 10000,
          })
          await page.setContent(html)
          await page.evaluate(() => {
            // Remove list of year filters on uni.no. No need
            // to have wget following various filters because the
            // default is to display all the content we wish to archive.
            document.querySelectorAll('.widget.years').forEach(widget => {
              Array.from(widget.getElementsByTagName('ul')).forEach(ul => {
                Array.from(ul.children).forEach(li => {
                  if (li.textContent.indexOf('Alle år') !== -1) {
                    // do nothing
                  } else if (li.textContent.indexOf('All years') !== -1) {
                    // do nothing
                  } else {
                    li.remove()
                  }
                })
              })
            })
            window.scrollTo(0, document.body.scrollHeight)
          })
          const newResponse = responseDetail.response
          newResponse.body = await page.content()
          await page.close()
          return {
            response: newResponse,
          }
        })
        .catch(err => {
          console.log('Failed to render url', url, err)
        })
    }
    return null
  },
  // if deal https request
  // *beforeDealHttpsRequest(requestDetail) {
  //   /* ... */
  // },
  // error happened when dealing requests
  *onError(requestDetail, error) {
    /* ... */
  },
  // error happened when connect to https server
  *onConnectError(requestDetail, error) {
    /* ... */
  },
}

#!/usr/bin/env bash

# Departments was tricky to archive because they had a lot of cross-links.
# I solved that by restricting what areas it was allowed to jump to.
DOMAIN=cms.uni.no
PROXY_PORT=9001
wget \
  --recursive \
  --level=inf \
  --timestamping \
  --page-requisites \
  --html-extension \
  --convert-links \
  --no-check-certificate \
  --include /nb/avdelinger,/nb/news,/nb/uni-cipr,(... several more urls ...) \
  -e use_proxy=on \
  -e http_proxy=http://127.0.0.1:$PROXY_PORT \
  -e https_proxy=http://127.0.0.1:$PROXY_PORT \
  --domains $DOMAIN \
  --rejected-log=$DOMAIN-rejected.log \
  --directory-prefix=site-$DOMAIN \
     http://cms.uni.no/nb/avdelinger/