Merry Christmas 2020

.eslintrc.js

@@ -0,0 +1,36 @@
+module.exports = {
+  env: {
+    es6: true,
+    node: true,
+  },
+  extends: ['eslint:recommended'],
+  parserOptions: {
+    sourceType: 'module',
+    ecmaVersion: 9
+  },
+
+  plugins: [],
+  rules: {
+    'array-bracket-spacing': ['error', 'never'],
+    indent: ['warn', 2],
+    'linebreak-style': ['error', 'unix'],
+    'lines-between-class-members': ['warn', 'always'],
+    semi: ['error', 'always'],
+    'new-cap': 'off',
+    'no-console': 'off',
+    'no-debugger': 'off',
+    'no-mixed-spaces-and-tabs': 2,
+    'no-use-before-define': [2, 'nofunc'],
+    'no-unreachable': ['warn'],
+    'no-unused-vars': ['warn'],
+    'no-extra-parens': ['off'],
+    'no-mixed-operators': ['off'],
+    quotes: [2, 'single', 'avoid-escape'],
+    'block-scoped-var': 2,
+    'brace-style': [2, '1tbs', { allowSingleLine: true }],
+    'computed-property-spacing': [2, 'never'],
+    'keyword-spacing': 2,
+    'space-unary-ops': 2,
+    'max-len': ['warn', { 'code': 140 }]
+  }
+};

.gitignore

@@ -0,0 +1,6 @@
+node_modules
+log
+main.db
+feed.xml
+sections
+*.swp

.jshintrc

@@ -0,0 +1,6 @@
+{
+  "esversion": 8,
+  "asi": true,
+  "laxbreak": true,
+  "predef": [ "-Promise" ]
+}

.repl.js

@@ -0,0 +1,20 @@
+require('dotenv').config()
+
+function reload(module){
+  delete require.cache[require.resolve(module)]
+  return require(module)
+}
+
+global.reload = reload
+global.rl     = reload
+global.cl     = console.log
+
+global.axios     = require('axios').default
+global.cheerio   = require('cheerio')
+global.url       = require('urlite')
+global.RSSParser = require('rss-parser')
+global.Database  = require('better-sqlite3')
+global.luxon     = require('luxon')
+global.DateTime  = global.luxon.DateTime
+global.ds        = require('./index')
+global.x         = {}

README.md

@@ -0,0 +1,20 @@
+# Daily Stormer Utilities
+
+This is a little project to help me get out of my funk.
+
+## Goals
+
+* Crawl all dailystormer articles.
+* Save crawl results as JSON files.
+* Be able to run the crawler again to get new articles.
+* This implies that I can keep track of what I've already crawled.
+* Make it possible to do a full text search on those articles.
+
+## Questions
+
+### Crawling
+
+The recursive crawl to get old articles and the crawl to update should be different.  The updater
+can assume a big crawl has already been done, and just pull articles down from the RSS feed.  However,
+the recursive crawl doesn't have that luxury, because it must find past articles and handle newly
+discovered tags and categories as new crawl targets.

bin/articles-update

@@ -0,0 +1,24 @@
+#!/usr/bin/env node
+// Load RSS feed and insert new article data into the database
+require('dotenv').config()
+const program = require('commander')
+const Database = require('better-sqlite3')
+const Bluebird = require('bluebird')
+const ds = require('../index')
+
+const db = new Database('./main.db', {})
+const feedURL = 'https://dailystormer.su/feed/'
+
+async function main() {
+  const feed = await ds.getArticlesFromFeed(feedURL)
+  Bluebird.each(feed, (async (item) => {
+    const res = await ds.insertArticle(db, item)
+    if (res.success) {
+      console.warn(`+ "${item.title}"`)
+    } else {
+      console.warn(`= "${item.title}"`)
+    }
+  }))
+}
+
+main()

bin/download-all

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+sections=(featured-stories world us society insight jewish-problem race-war)
+for s in ${sections[@]} ; do
+  touch log/$s
+  bin/scan -t section $s
+done

bin/queue

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+// Tail a file, and run bin/scan on new categories that have been appended.

bin/repl

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+node -r ./.repl "$@"

bin/scan

@@ -0,0 +1,37 @@
+#!/usr/bin/env node
+// scan a category (or tag) for articles
+require('dotenv').config()
+const program = require('commander')
+const Database = require('better-sqlite3')
+const Bluebird = require('bluebird')
+const ds = require('../index')
+
+const db = new Database('./main.db', {})
+const baseURL = 'https://dailystormer.su'
+
+async function main() {
+  program.option('-t, --taxonomy <TYPE>', `"section" or "tag"`, 'section')
+  program.parse(process.argv)
+  //console.log(program)
+  const category = program.args[0]
+  if (!category) {
+    console.warn('category required')
+    process.exit(1)
+  }
+  const taxonomyBaseURL = `${baseURL}/${program.taxonomy}`
+  const partialArticles = await ds.scanCategory(taxonomyBaseURL, category)
+  const ps = partialArticles // partialArticles.slice(0, 3)
+  //console.log(ps)
+  Bluebird.each(ps, (async (art, i, length) => {
+    const more = await ds.getArticle(art.link)
+    const article = Object.assign({}, art, more)
+    const res = await ds.insertArticle(db, article)
+    if (res.success) {
+      console.log(`+ [${i+1}/${length}] "${art.title}"`)
+    } else {
+      console.log(`= [${i+1}/${length}] "${art.title}"`)
+    }
+  }))
+}
+
+main()

index.js

@@ -0,0 +1,265 @@
+const axios     = require('axios').default
+const cheerio   = require('cheerio')
+const flatten   = require('lodash.flatten')
+const RSSParser = require('rss-parser')
+const Bluebird  = require('bluebird')
+const url       = require('urlite')
+const luxon     = require('luxon')
+const DateTime  = luxon.DateTime
+
+const parser = new RSSParser()
+
+async function getFeed(feedURL='https://dailystormer.su/feed/') {
+  const res = await axios.get(feedURL)
+  const feed = await parser.parseString(res.data)
+  return feed
+}
+
+/**
+ * Fetch the RSS feed and return its parsed contents.
+ * @param {String} feedURL - URL of the feed
+ * @returns {Object} parsed RSS feed
+ */
+async function getArticlesFromFeed(feedURL='https://dailystormer.su/feed/') {
+  const feed = await getFeed(feedURL);
+  return feed.items.map((a) => {
+    const slug = url.parse(a.link).pathname
+    const article = {
+      slug,
+      title: a.title,
+      author: a.creator,
+      content: a['content:encoded'],
+      categories: a.categories.map((cat) => cat.toLowerCase()),
+      tags: [],
+      published_date: a.isoDate,
+      //published_date: DateTime.fromRFC2822(a.pubDate).toISO()
+    }
+    return article
+  })
+}
+
+async function scanCategory(baseURL, category, options={}) {
+  const categoryURL = `${baseURL}/${category}`
+  console.log(categoryURL)
+  // scan first page of category to find out how many pages we have.
+  const res = await axios.get(categoryURL)
+  const $ = cheerio.load(res.data)
+  const p = parseInt($('.pagination .pages').text().split(/\s/)[3])
+  const pages = []
+  for (let i = 1; i <= p; i++) {
+    pages.push(i)
+  }
+  const scans = await Bluebird.map(pages, (n) => { return scanCategoryPage(baseURL, category, n) }, { concurrency: 4 })
+  const flatScans = flatten(scans)
+  return flatScans
+}
+
+async function scanCategoryPage(baseURL, category, page) {
+  var categoryURL
+  if (page === 1) {
+    categoryURL = `${baseURL}/${category}`
+  } else {
+    categoryURL = `${baseURL}/${category}/page/${page}`
+  }
+  const res = await axios.get(categoryURL)
+  console.log(categoryURL)
+  const $ = cheerio.load(res.data)
+  const articles = $('article.item-list').map((i, article) => {
+    const h2_a = $(article).find('h2.post-box-title a')
+    const link = h2_a.attr('href')
+    const slug = url.parse(link).pathname
+    const title = h2_a.text().trim()
+    const author = $(article).find('p.post-meta .post-meta-author').text().trim()
+    const publishedDateText = $(article).find('p.post-meta .tie-date').text().trim()
+    const publishedDate = DateTime.fromFormat(publishedDateText, 'LLLL d, yyyy', { zone: 'UTC' }).toISO()
+    return {
+      link,
+      slug,
+      title,
+      author,
+      published_date: publishedDate
+    }
+  }).toArray()
+  return articles
+}
+
+function unhyphen(word) {
+  const ws = word.split(/-/)
+  return ws.slice(1).map((s) => s.toLowerCase()).join(" ")
+}
+
+function extractTaxonomyFromArticle($article, type) {
+  const cs = $article.attr('class')
+    .split(/\s+/)
+    .filter((word) => word.match(new RegExp(`^${type}`)))
+  const ts = cs.map(unhyphen)
+  return ts
+}
+
+async function getArticle(url) {
+  const res = await axios.get(url)
+  const $ = cheerio.load(res.data)
+  const $article = $('article')
+  // get tags
+  const tags = extractTaxonomyFromArticle($article, 'tag')
+  // get categories
+  const categories = extractTaxonomyFromArticle($article, 'category')
+  // get content
+  const content = $article.find('div.entry').html()
+  return {
+    tags,
+    categories,
+    content
+  }
+}
+
+async function getCategory(db, name) {
+  const category = await db.prepare('SELECT id, name FROM category WHERE name = ? LIMIT 1').get(name)
+  return category
+}
+
+async function assocCategory(db, articleId, name) {
+  try {
+    var categoryId
+    const category = await getCategory(db, name)
+    if (!category) {
+      const res = await insertCategory(db, name)
+      categoryId = res.lastInsertRowid
+    } else {
+      categoryId = category.id
+    }
+    const res2 = await db.prepare('INSERT INTO article__category (article_id, category_id) VALUES (?, ?)')
+      .run(articleId, categoryId)
+  } catch (e) {
+    return { success: false, error: e }
+  }
+  return { success: true }
+}
+
+async function getTag(db, name) {
+  const tag = await db.prepare('SELECT id, name FROM tag WHERE name = ? LIMIT 1').get(name)
+  return tag
+}
+
+async function assocTag(db, articleId, name) {
+  try {
+    var tagId
+    const tag = await getTag(db, name)
+    if (!tag) {
+      const res = await insertTag(db, name)
+      tagId = res.lastInsertRowid
+    } else {
+      tagId = tag.id
+    }
+    const res2 = await db.prepare('INSERT INTO article__tag (article_id, tag_id) VALUES (?, ?)')
+      .run(articleId, tagId)
+  } catch (e) {
+    return { success: false, error: e }
+  }
+  return { success: true }
+}
+
+async function insertCategory(db, name) {
+  try {
+    const res = db.prepare(`INSERT INTO category (name) VALUES (?)`).run(name) // not async apparently
+    res.success = true
+    return res
+  } catch (e) {
+    return {
+      success: false,
+      error: e
+    }
+  }
+}
+
+async function insertTag(db, name) {
+  try {
+    const res = db.prepare(`INSERT INTO tag (name) VALUES (?)`).run(name) // not async apparently
+    res.success = true
+    return res
+  } catch (e) {
+    return {
+      success: false,
+      error: e
+    }
+  }
+}
+
+async function insertArticle(db, article) {
+  try {
+    // TODO wrap in a transaction?
+
+    // insert into article
+    const insert = db.prepare('INSERT INTO article (slug, title, author, content, published_date) VALUES (@slug, @title, @author, @content, @published_date)')
+    const res = await insert.run(article)
+    const articleId = res.lastInsertRowid
+    // insert into article_search for full-text search
+    const insertArticleSearch = db.prepare('INSERT INTO article_search (author, title, content, slug) VALUES (@author, @title, @content, @slug)')
+    const articleForSearch = Object.assign({}, article)
+    articleForSearch.content = cheerio.load(article.content).text()
+    await insertArticleSearch.run(articleForSearch)
+    // associate categories
+    article.categories.forEach((async (name) => {
+      await assocCategory(db, articleId, name)
+    }))
+    // associate tags
+    if (article.tags) {
+      article.tags.forEach((async (name) => {
+        await assocTag(db, articleId, name)
+      }))
+    }
+  }
+  catch (e) {
+    return { success: false, error: e }
+  }
+  return { success: true }
+}
+
+/**
+ * Perform a full-text search of the db.
+ * @param {Database} db - articles database
+ * @param {String} query - search query in SQLite3 fts5 syntax
+ */
+async function search(db, query, limit, offset) {
+  const stmt = db.prepare(`
+    SELECT rank, author, title, slug,
+      snippet(article_search, 2, '<b>', '</b>', '...', 32) as snippet
+    FROM article_search
+    WHERE article_search MATCH @query
+    ORDER BY rank
+    LIMIT @limit OFFSET @offset
+  `)
+  const res = stmt.all({ query, limit, offset })
+  return res
+}
+
+async function searchCount(db, query) {
+  const stmt = db.prepare(`SELECT count(*) as c FROM article_search WHERE article_search MATCH @query`)
+  const res = stmt.get({ query })
+  return res.c
+}
+
+// TODO write a function to insert an article into the database
+// TODO write a function to insert a category into the database
+// TODO write a function to associate an article with categories in a database
+// TODO write a function to scan a category for articles
+// TODO write a function to scrape an article into an insertable object
+
+module.exports = {
+  getFeed,
+  getArticlesFromFeed,
+  scanCategory,
+  scanCategoryPage,
+  unhyphen,
+  extractTaxonomyFromArticle,
+  getArticle,
+  getCategory,
+  assocCategory,
+  getTag,
+  assocTag,
+  insertCategory,
+  insertTag,
+  insertArticle,
+  search,
+  searchCount
+}

package-lock.json

@@ -0,0 +1,747 @@
+{
+  "name": "ds",
+  "version": "1.0.0",
+  "lockfileVersion": 1,
+  "requires": true,
+  "dependencies": {
+    "@types/node": {
+      "version": "14.14.13",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-14.14.13.tgz",
+      "integrity": "sha512-vbxr0VZ8exFMMAjCW8rJwaya0dMCDyYW2ZRdTyjtrCvJoENMpdUHOT/eTzvgyA5ZnqRZ/sI0NwqAxNHKYokLJQ=="
+    },
+    "ansi-regex": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz",
+      "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8="
+    },
+    "aproba": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz",
+      "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw=="
+    },
+    "are-we-there-yet": {
+      "version": "1.1.5",
+      "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.5.tgz",
+      "integrity": "sha512-5hYdAkZlcG8tOLujVDTgCT+uPX0VnpAH28gWsLfzpXYm7wP6mp5Q/gYyR7YQ0cKVJcXJnl3j2kpBan13PtQf6w==",
+      "requires": {
+        "delegates": "^1.0.0",
+        "readable-stream": "^2.0.6"
+      },
+      "dependencies": {
+        "readable-stream": {
+          "version": "2.3.7",
+          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz",
+          "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==",
+          "requires": {
+            "core-util-is": "~1.0.0",
+            "inherits": "~2.0.3",
+            "isarray": "~1.0.0",
+            "process-nextick-args": "~2.0.0",
+            "safe-buffer": "~5.1.1",
+            "string_decoder": "~1.1.1",
+            "util-deprecate": "~1.0.1"
+          }
+        },
+        "safe-buffer": {
+          "version": "5.1.2",
+          "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
+          "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
+        },
+        "string_decoder": {
+          "version": "1.1.1",
+          "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
+          "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
+          "requires": {
+            "safe-buffer": "~5.1.0"
+          }
+        }
+      }
+    },
+    "atomic-sleep": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz",
+      "integrity": "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ=="
+    },
+    "axios": {
+      "version": "0.21.0",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.0.tgz",
+      "integrity": "sha512-fmkJBknJKoZwem3/IKSSLpkdNXZeBu5Q7GA/aRsr2btgrptmSCxi2oFjZHqGdK9DoTil9PIHlPIZw2EcRJXRvw==",
+      "requires": {
+        "follow-redirects": "^1.10.0"
+      }
+    },
+    "base64-js": {
+      "version": "1.5.1",
+      "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
+      "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA=="
+    },
+    "better-sqlite3": {
+      "version": "7.1.1",
+      "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-7.1.1.tgz",
+      "integrity": "sha512-AkvGGyhAVZhRBOul2WT+5CB2EuveM3ZkebEKe1wxMqDZUy1XB/1RBgM66t0ybHC4DIni8+pr7NaLqEX87NUTwg==",
+      "requires": {
+        "bindings": "^1.5.0",
+        "prebuild-install": "^5.3.3",
+        "tar": "4.4.10"
+      }
+    },
+    "bindings": {
+      "version": "1.5.0",
+      "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz",
+      "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==",
+      "requires": {
+        "file-uri-to-path": "1.0.0"
+      }
+    },
+    "bl": {
+      "version": "4.0.3",
+      "resolved": "https://registry.npmjs.org/bl/-/bl-4.0.3.tgz",
+      "integrity": "sha512-fs4G6/Hu4/EE+F75J8DuN/0IpQqNjAdC7aEQv7Qt8MHGUH7Ckv2MwTEEeN9QehD0pfIDkMI1bkHYkKy7xHyKIg==",
+      "requires": {
+        "buffer": "^5.5.0",
+        "inherits": "^2.0.4",
+        "readable-stream": "^3.4.0"
+      }
+    },
+    "bluebird": {
+      "version": "3.7.2",
+      "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz",
+      "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg=="
+    },
+    "boolbase": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz",
+      "integrity": "sha1-aN/1++YMUes3cl6p4+0xDcwed24="
+    },
+    "buffer": {
+      "version": "5.7.1",
+      "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
+      "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==",
+      "requires": {
+        "base64-js": "^1.3.1",
+        "ieee754": "^1.1.13"
+      }
+    },
+    "cheerio": {
+      "version": "1.0.0-rc.3",
+      "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.3.tgz",
+      "integrity": "sha512-0td5ijfUPuubwLUu0OBoe98gZj8C/AA+RW3v67GPlGOrvxWjZmBXiBCRU+I8VEiNyJzjth40POfHiz2RB3gImA==",
+      "requires": {
+        "css-select": "~1.2.0",
+        "dom-serializer": "~0.1.1",
+        "entities": "~1.1.1",
+        "htmlparser2": "^3.9.1",
+        "lodash": "^4.15.0",
+        "parse5": "^3.0.1"
+      }
+    },
+    "chownr": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz",
+      "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg=="
+    },
+    "code-point-at": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz",
+      "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c="
+    },
+    "commander": {
+      "version": "6.2.1",
+      "resolved": "https://registry.npmjs.org/commander/-/commander-6.2.1.tgz",
+      "integrity": "sha512-U7VdrJFnJgo4xjrHpTzu0yrHPGImdsmD95ZlgYSEajAn2JKzDhDTPG9kBTefmObL2w/ngeZnilk+OV9CG3d7UA=="
+    },
+    "console-control-strings": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz",
+      "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4="
+    },
+    "core-util-is": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
+      "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac="
+    },
+    "css-select": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz",
+      "integrity": "sha1-KzoRBTnFNV8c2NMUYj6HCxIeyFg=",
+      "requires": {
+        "boolbase": "~1.0.0",
+        "css-what": "2.1",
+        "domutils": "1.5.1",
+        "nth-check": "~1.0.1"
+      }
+    },
+    "css-what": {
+      "version": "2.1.3",
+      "resolved": "https://registry.npmjs.org/css-what/-/css-what-2.1.3.tgz",
+      "integrity": "sha512-a+EPoD+uZiNfh+5fxw2nO9QwFa6nJe2Or35fGY6Ipw1R3R4AGz1d1TEZrCegvw2YTmZ0jXirGYlzxxpYSHwpEg=="
+    },
+    "decompress-response": {
+      "version": "4.2.1",
+      "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-4.2.1.tgz",
+      "integrity": "sha512-jOSne2qbyE+/r8G1VU+G/82LBs2Fs4LAsTiLSHOCOMZQl2OKZ6i8i4IyHemTe+/yIXOtTcRQMzPcgyhoFlqPkw==",
+      "requires": {
+        "mimic-response": "^2.0.0"
+      }
+    },
+    "deep-extend": {
+      "version": "0.6.0",
+      "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz",
+      "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA=="
+    },
+    "delegates": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz",
+      "integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o="
+    },
+    "detect-libc": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz",
+      "integrity": "sha1-+hN8S9aY7fVc1c0CrFWfkaTEups="
+    },
+    "dom-serializer": {
+      "version": "0.1.1",
+      "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.1.1.tgz",
+      "integrity": "sha512-l0IU0pPzLWSHBcieZbpOKgkIn3ts3vAh7ZuFyXNwJxJXk/c4Gwj9xaTJwIDVQCXawWD0qb3IzMGH5rglQaO0XA==",
+      "requires": {
+        "domelementtype": "^1.3.0",
+        "entities": "^1.1.1"
+      }
+    },
+    "domelementtype": {
+      "version": "1.3.1",
+      "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.1.tgz",
+      "integrity": "sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w=="
+    },
+    "domhandler": {
+      "version": "2.4.2",
+      "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-2.4.2.tgz",
+      "integrity": "sha512-JiK04h0Ht5u/80fdLMCEmV4zkNh2BcoMFBmZ/91WtYZ8qVXSKjiw7fXMgFPnHcSZgOo3XdinHvmnDUeMf5R4wA==",
+      "requires": {
+        "domelementtype": "1"
+      }
+    },
+    "domutils": {
+      "version": "1.5.1",
+      "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.5.1.tgz",
+      "integrity": "sha1-3NhIiib1Y9YQeeSMn3t+Mjc2gs8=",
+      "requires": {
+        "dom-serializer": "0",
+        "domelementtype": "1"
+      }
+    },
+    "dotenv": {
+      "version": "8.2.0",
+      "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-8.2.0.tgz",
+      "integrity": "sha512-8sJ78ElpbDJBHNeBzUbUVLsqKdccaa/BXF1uPTw3GrvQTBgrQrtObr2mUrE38vzYd8cEv+m/JBfDLioYcfXoaw=="
+    },
+    "end-of-stream": {
+      "version": "1.4.4",
+      "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz",
+      "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==",
+      "requires": {
+        "once": "^1.4.0"
+      }
+    },
+    "entities": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz",
+      "integrity": "sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w=="
+    },
+    "expand-template": {
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz",
+      "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg=="
+    },
+    "fast-redact": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/fast-redact/-/fast-redact-3.0.0.tgz",
+      "integrity": "sha512-a/S/Hp6aoIjx7EmugtzLqXmcNsyFszqbt6qQ99BdG61QjBZF6shNis0BYR6TsZOQ1twYc0FN2Xdhwwbv6+KD0w=="
+    },
+    "fast-safe-stringify": {
+      "version": "2.0.7",
+      "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.0.7.tgz",
+      "integrity": "sha512-Utm6CdzT+6xsDk2m8S6uL8VHxNwI6Jub+e9NYTcAms28T84pTa25GJQV9j0CY0N1rM8hK4x6grpF2BQf+2qwVA=="
+    },
+    "file-uri-to-path": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz",
+      "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw=="
+    },
+    "flatstr": {
+      "version": "1.0.12",
+      "resolved": "https://registry.npmjs.org/flatstr/-/flatstr-1.0.12.tgz",
+      "integrity": "sha512-4zPxDyhCyiN2wIAtSLI6gc82/EjqZc1onI4Mz/l0pWrAlsSfYH/2ZIcU+e3oA2wDwbzIWNKwa23F8rh6+DRWkw=="
+    },
+    "follow-redirects": {
+      "version": "1.13.1",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.13.1.tgz",
+      "integrity": "sha512-SSG5xmZh1mkPGyKzjZP8zLjltIfpW32Y5QpdNJyjcfGxK3qo3NDDkZOZSFiGn1A6SclQxY9GzEwAHQ3dmYRWpg=="
+    },
+    "fs-constants": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz",
+      "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow=="
+    },
+    "fs-minipass": {
+      "version": "1.2.7",
+      "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-1.2.7.tgz",
+      "integrity": "sha512-GWSSJGFy4e9GUeCcbIkED+bgAoFyj7XF1mV8rma3QW4NIqX9Kyx79N/PF61H5udOV3aY1IaMLs6pGbH71nlCTA==",
+      "requires": {
+        "minipass": "^2.6.0"
+      }
+    },
+    "gauge": {
+      "version": "2.7.4",
+      "resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz",
+      "integrity": "sha1-LANAXHU4w51+s3sxcCLjJfsBi/c=",
+      "requires": {
+        "aproba": "^1.0.3",
+        "console-control-strings": "^1.0.0",
+        "has-unicode": "^2.0.0",
+        "object-assign": "^4.1.0",
+        "signal-exit": "^3.0.0",
+        "string-width": "^1.0.1",
+        "strip-ansi": "^3.0.1",
+        "wide-align": "^1.1.0"
+      }
+    },
+    "github-from-package": {
+      "version": "0.0.0",
+      "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz",
+      "integrity": "sha1-l/tdlr/eiXMxPyDoKI75oWf6ZM4="
+    },
+    "has-unicode": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz",
+      "integrity": "sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk="
+    },
+    "htmlparser2": {
+      "version": "3.10.1",
+      "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.10.1.tgz",
+      "integrity": "sha512-IgieNijUMbkDovyoKObU1DUhm1iwNYE/fuifEoEHfd1oZKZDaONBSkal7Y01shxsM49R4XaMdGez3WnF9UfiCQ==",
+      "requires": {
+        "domelementtype": "^1.3.1",
+        "domhandler": "^2.3.0",
+        "domutils": "^1.5.1",
+        "entities": "^1.1.1",
+        "inherits": "^2.0.1",
+        "readable-stream": "^3.1.1"
+      }
+    },
+    "ieee754": {
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz",
+      "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA=="
+    },
+    "inherits": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
+      "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
+    },
+    "ini": {
+      "version": "1.3.8",
+      "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz",
+      "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew=="
+    },
+    "is-fullwidth-code-point": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz",
+      "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=",
+      "requires": {
+        "number-is-nan": "^1.0.0"
+      }
+    },
+    "isarray": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
+      "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE="
+    },
+    "lodash": {
+      "version": "4.17.20",
+      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz",
+      "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA=="
+    },
+    "lodash.flatten": {
+      "version": "4.4.0",
+      "resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz",
+      "integrity": "sha1-8xwiIlqWMtK7+OSt2+8kCqdlph8="
+    },
+    "luxon": {
+      "version": "1.25.0",
+      "resolved": "https://registry.npmjs.org/luxon/-/luxon-1.25.0.tgz",
+      "integrity": "sha512-hEgLurSH8kQRjY6i4YLey+mcKVAWXbDNlZRmM6AgWDJ1cY3atl8Ztf5wEY7VBReFbmGnwQPz7KYJblL8B2k0jQ=="
+    },
+    "mimic-response": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-2.1.0.tgz",
+      "integrity": "sha512-wXqjST+SLt7R009ySCglWBCFpjUygmCIfD790/kVbiGmUgfYGuB14PiTd5DwVxSV4NcYHjzMkoj5LjQZwTQLEA=="
+    },
+    "minimist": {
+      "version": "1.2.5",
+      "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz",
+      "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw=="
+    },
+    "minipass": {
+      "version": "2.9.0",
+      "resolved": "https://registry.npmjs.org/minipass/-/minipass-2.9.0.tgz",
+      "integrity": "sha512-wxfUjg9WebH+CUDX/CdbRlh5SmfZiy/hpkxaRI16Y9W56Pa75sWgd/rvFilSgrauD9NyFymP/+JFV3KwzIsJeg==",
+      "requires": {
+        "safe-buffer": "^5.1.2",
+        "yallist": "^3.0.0"
+      }
+    },
+    "minizlib": {
+      "version": "1.3.3",
+      "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-1.3.3.tgz",
+      "integrity": "sha512-6ZYMOEnmVsdCeTJVE0W9ZD+pVnE8h9Hma/iOwwRDsdQoePpoX56/8B6z3P9VNwppJuBKNRuFDRNRqRWexT9G9Q==",
+      "requires": {
+        "minipass": "^2.9.0"
+      }
+    },
+    "mkdirp": {
+      "version": "0.5.5",
+      "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz",
+      "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==",
+      "requires": {
+        "minimist": "^1.2.5"
+      }
+    },
+    "mkdirp-classic": {
+      "version": "0.5.3",
+      "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz",
+      "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A=="
+    },
+    "napi-build-utils": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz",
+      "integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg=="
+    },
+    "node-abi": {
+      "version": "2.19.3",
+      "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-2.19.3.tgz",
+      "integrity": "sha512-9xZrlyfvKhWme2EXFKQhZRp1yNWT/uI1luYPr3sFl+H4keYY4xR+1jO7mvTTijIsHf1M+QDe9uWuKeEpLInIlg==",
+      "requires": {
+        "semver": "^5.4.1"
+      }
+    },
+    "noop-logger": {
+      "version": "0.1.1",
+      "resolved": "https://registry.npmjs.org/noop-logger/-/noop-logger-0.1.1.tgz",
+      "integrity": "sha1-lKKxYzxPExdVMAfYlm/Q6EG2pMI="
+    },
+    "npmlog": {
+      "version": "4.1.2",
+      "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-4.1.2.tgz",
+      "integrity": "sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==",
+      "requires": {
+        "are-we-there-yet": "~1.1.2",
+        "console-control-strings": "~1.1.0",
+        "gauge": "~2.7.3",
+        "set-blocking": "~2.0.0"
+      }
+    },
+    "nth-check": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-1.0.2.tgz",
+      "integrity": "sha512-WeBOdju8SnzPN5vTUJYxYUxLeXpCaVP5i5e0LF8fg7WORF2Wd7wFX/pk0tYZk7s8T+J7VLy0Da6J1+wCT0AtHg==",
+      "requires": {
+        "boolbase": "~1.0.0"
+      }
+    },
+    "number-is-nan": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz",
+      "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0="
+    },
+    "object-assign": {
+      "version": "4.1.1",
+      "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
+      "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM="
+    },
+    "once": {
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
+      "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
+      "requires": {
+        "wrappy": "1"
+      }
+    },
+    "parse5": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/parse5/-/parse5-3.0.3.tgz",
+      "integrity": "sha512-rgO9Zg5LLLkfJF9E6CCmXlSE4UVceloys8JrFqCcHloC3usd/kJCyPDwH2SOlzix2j3xaP9sUX3e8+kvkuleAA==",
+      "requires": {
+        "@types/node": "*"
+      }
+    },
+    "pino": {
+      "version": "6.8.0",
+      "resolved": "https://registry.npmjs.org/pino/-/pino-6.8.0.tgz",
+      "integrity": "sha512-nxq+6Jr7m0cMjYFBoTRw3bco14omZ/SQCheAHz9GVwdkbUrzKhgT+gSI/ql2Mnsca0QQKgpB/ACWhjxE4JsX3Q==",
+      "requires": {
+        "fast-redact": "^3.0.0",
+        "fast-safe-stringify": "^2.0.7",
+        "flatstr": "^1.0.12",
+        "pino-std-serializers": "^2.4.2",
+        "quick-format-unescaped": "^4.0.1",
+        "sonic-boom": "^1.0.2"
+      }
+    },
+    "pino-std-serializers": {
+      "version": "2.5.0",
+      "resolved": "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-2.5.0.tgz",
+      "integrity": "sha512-wXqbqSrIhE58TdrxxlfLwU9eDhrzppQDvGhBEr1gYbzzM4KKo3Y63gSjiDXRKLVS2UOXdPNR2v+KnQgNrs+xUg=="
+    },
+    "prebuild-install": {
+      "version": "5.3.6",
+      "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-5.3.6.tgz",
+      "integrity": "sha512-s8Aai8++QQGi4sSbs/M1Qku62PFK49Jm1CbgXklGz4nmHveDq0wzJkg7Na5QbnO1uNH8K7iqx2EQ/mV0MZEmOg==",
+      "requires": {
+        "detect-libc": "^1.0.3",
+        "expand-template": "^2.0.3",
+        "github-from-package": "0.0.0",
+        "minimist": "^1.2.3",
+        "mkdirp-classic": "^0.5.3",
+        "napi-build-utils": "^1.0.1",
+        "node-abi": "^2.7.0",
+        "noop-logger": "^0.1.1",
+        "npmlog": "^4.0.1",
+        "pump": "^3.0.0",
+        "rc": "^1.2.7",
+        "simple-get": "^3.0.3",
+        "tar-fs": "^2.0.0",
+        "tunnel-agent": "^0.6.0",
+        "which-pm-runs": "^1.0.0"
+      }
+    },
+    "process-nextick-args": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
+      "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
+    },
+    "pump": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz",
+      "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==",
+      "requires": {
+        "end-of-stream": "^1.1.0",
+        "once": "^1.3.1"
+      }
+    },
+    "quick-format-unescaped": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/quick-format-unescaped/-/quick-format-unescaped-4.0.1.tgz",
+      "integrity": "sha512-RyYpQ6Q5/drsJyOhrWHYMWTedvjTIat+FTwv0K4yoUxzvekw2aRHMQJLlnvt8UantkZg2++bEzD9EdxXqkWf4A=="
+    },
+    "rc": {
+      "version": "1.2.8",
+      "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz",
+      "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==",
+      "requires": {
+        "deep-extend": "^0.6.0",
+        "ini": "~1.3.0",
+        "minimist": "^1.2.0",
+        "strip-json-comments": "~2.0.1"
+      }
+    },
+    "readable-stream": {
+      "version": "3.6.0",
+      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz",
+      "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==",
+      "requires": {
+        "inherits": "^2.0.3",
+        "string_decoder": "^1.1.1",
+        "util-deprecate": "^1.0.1"
+      }
+    },
+    "rss-parser": {
+      "version": "3.10.0",
+      "resolved": "https://registry.npmjs.org/rss-parser/-/rss-parser-3.10.0.tgz",
+      "integrity": "sha512-TC6FNvEmdFeaW6r/60MSJT7cp4d95X4M9As+mvNtxRx7YXHxpV95syMnWZthZSeD1BRN7SEKdq6c3nxMLQRopw==",
+      "requires": {
+        "entities": "^2.0.3",
+        "xml2js": "^0.4.19"
+      },
+      "dependencies": {
+        "entities": {
+          "version": "2.1.0",
+          "resolved": "https://registry.npmjs.org/entities/-/entities-2.1.0.tgz",
+          "integrity": "sha512-hCx1oky9PFrJ611mf0ifBLBRW8lUUVRlFolb5gWRfIELabBlbp9xZvrqZLZAs+NxFnbfQoeGd8wDkygjg7U85w=="
+        }
+      }
+    },
+    "safe-buffer": {
+      "version": "5.2.1",
+      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
+      "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="
+    },
+    "sax": {
+      "version": "1.2.4",
+      "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz",
+      "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw=="
+    },
+    "semver": {
+      "version": "5.7.1",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz",
+      "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ=="
+    },
+    "set-blocking": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz",
+      "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc="
+    },
+    "signal-exit": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.3.tgz",
+      "integrity": "sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA=="
+    },
+    "simple-concat": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz",
+      "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q=="
+    },
+    "simple-get": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-3.1.0.tgz",
+      "integrity": "sha512-bCR6cP+aTdScaQCnQKbPKtJOKDp/hj9EDLJo3Nw4y1QksqaovlW/bnptB6/c1e+qmNIDHRK+oXFDdEqBT8WzUA==",
+      "requires": {
+        "decompress-response": "^4.2.0",
+        "once": "^1.3.1",
+        "simple-concat": "^1.0.0"
+      }
+    },
+    "sonic-boom": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-1.3.0.tgz",
+      "integrity": "sha512-4nX6OYvOYr6R76xfQKi6cZpTO3YSWe/vd+QdIfoH0lBy0MnPkeAbb2rRWgmgADkXUeCKPwO1FZAKlAVWAadELw==",
+      "requires": {
+        "atomic-sleep": "^1.0.0",
+        "flatstr": "^1.0.12"
+      }
+    },
+    "string-width": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz",
+      "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=",
+      "requires": {
+        "code-point-at": "^1.0.0",
+        "is-fullwidth-code-point": "^1.0.0",
+        "strip-ansi": "^3.0.0"
+      }
+    },
+    "string_decoder": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz",
+      "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==",
+      "requires": {
+        "safe-buffer": "~5.2.0"
+      }
+    },
+    "strip-ansi": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz",
+      "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=",
+      "requires": {
+        "ansi-regex": "^2.0.0"
+      }
+    },
+    "strip-json-comments": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz",
+      "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo="
+    },
+    "tar": {
+      "version": "4.4.10",
+      "resolved": "https://registry.npmjs.org/tar/-/tar-4.4.10.tgz",
+      "integrity": "sha512-g2SVs5QIxvo6OLp0GudTqEf05maawKUxXru104iaayWA09551tFCTI8f1Asb4lPfkBr91k07iL4c11XO3/b0tA==",
+      "requires": {
+        "chownr": "^1.1.1",
+        "fs-minipass": "^1.2.5",
+        "minipass": "^2.3.5",
+        "minizlib": "^1.2.1",
+        "mkdirp": "^0.5.0",
+        "safe-buffer": "^5.1.2",
+        "yallist": "^3.0.3"
+      }
+    },
+    "tar-fs": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz",
+      "integrity": "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==",
+      "requires": {
+        "chownr": "^1.1.1",
+        "mkdirp-classic": "^0.5.2",
+        "pump": "^3.0.0",
+        "tar-stream": "^2.1.4"
+      }
+    },
+    "tar-stream": {
+      "version": "2.1.4",
+      "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.1.4.tgz",
+      "integrity": "sha512-o3pS2zlG4gxr67GmFYBLlq+dM8gyRGUOvsrHclSkvtVtQbjV0s/+ZE8OpICbaj8clrX3tjeHngYGP7rweaBnuw==",
+      "requires": {
+        "bl": "^4.0.3",
+        "end-of-stream": "^1.4.1",
+        "fs-constants": "^1.0.0",
+        "inherits": "^2.0.3",
+        "readable-stream": "^3.1.1"
+      }
+    },
+    "tunnel-agent": {
+      "version": "0.6.0",
+      "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz",
+      "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=",
+      "requires": {
+        "safe-buffer": "^5.0.1"
+      }
+    },
+    "urlite": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/urlite/-/urlite-3.0.0.tgz",
+      "integrity": "sha512-Ffgts2pTfFFybsHm1uk8s01jBpIBIU8/ZBrTVFpsM5eA1AYVsEfBYkiSahyJcuCEdLvfOkjq5vXznlcI1HhfRw=="
+    },
+    "util-deprecate": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
+      "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8="
+    },
+    "which-pm-runs": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/which-pm-runs/-/which-pm-runs-1.0.0.tgz",
+      "integrity": "sha1-Zws6+8VS4LVd9rd4DKdGFfI60cs="
+    },
+    "wide-align": {
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.3.tgz",
+      "integrity": "sha512-QGkOQc8XL6Bt5PwnsExKBPuMKBxnGxWWW3fU55Xt4feHozMUhdUMaBCk290qpm/wG5u/RSKzwdAC4i51YigihA==",
+      "requires": {
+        "string-width": "^1.0.2 || 2"
+      }
+    },
+    "wrappy": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
+      "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8="
+    },
+    "xml2js": {
+      "version": "0.4.23",
+      "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.23.tgz",
+      "integrity": "sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==",
+      "requires": {
+        "sax": ">=0.6.0",
+        "xmlbuilder": "~11.0.0"
+      }
+    },
+    "xmlbuilder": {
+      "version": "11.0.1",
+      "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz",
+      "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA=="
+    },
+    "yallist": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz",
+      "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g=="
+    }
+  }
+}

package.json

@@ -0,0 +1,24 @@
+{
+  "name": "ds",
+  "version": "1.0.0",
+  "description": "Daily Stormer Utilities",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "",
+  "license": "MIT",
+  "dependencies": {
+    "axios": "^0.21.0",
+    "better-sqlite3": "^7.1.1",
+    "bluebird": "^3.7.2",
+    "cheerio": "^1.0.0-rc.3",
+    "commander": "^6.2.1",
+    "dotenv": "^8.2.0",
+    "lodash.flatten": "^4.4.0",
+    "luxon": "^1.25.0",
+    "pino": "^6.8.0",
+    "rss-parser": "^3.10.0",
+    "urlite": "^3.0.0"
+  }
+}

sql/ds.sql

@@ -0,0 +1,35 @@
+CREATE VIRTUAL TABLE article_search USING fts5(author, title, content, slug);
+
+CREATE TABLE article (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  slug VARCHAR(256) NOT NULL,
+  title VARCHAR(256) NOT NULL,
+  author VARCHAR(256) NOT NULL,
+  content text,
+  published_date DATETIME NOT NULL,
+  UNIQUE(slug)
+);
+
+CREATE TABLE category (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  name VARCHAR(128) NOT NULL,
+  UNIQUE(name)
+);
+
+CREATE TABLE tag (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  name VARCHAR(128) NOT NULL,
+  UNIQUE(name)
+);
+
+CREATE TABLE article__category (
+  article_id INTEGER,
+  category_id INTEGER,
+  UNIQUE(category_id, article_id)
+);
+
+CREATE TABLE article__tag (
+  article_id INTEGER,
+  tag_id INTEGER,
+  UNIQUE(tag_id, article_id)
+);