38 lines
1.1 KiB
JavaScript
Executable File
38 lines
1.1 KiB
JavaScript
Executable File
#!/usr/bin/env node
|
|
// scan a category (or tag) for articles
|
|
require('dotenv').config()
|
|
const program = require('commander')
|
|
const Database = require('better-sqlite3')
|
|
const Bluebird = require('bluebird')
|
|
const ds = require('../index')
|
|
|
|
const db = new Database('./main.db', {})
|
|
const baseURL = 'https://dailystormer.su'
|
|
|
|
async function main() {
|
|
program.option('-t, --taxonomy <TYPE>', `"section" or "tag"`, 'section')
|
|
program.parse(process.argv)
|
|
//console.log(program)
|
|
const category = program.args[0]
|
|
if (!category) {
|
|
console.warn('category required')
|
|
process.exit(1)
|
|
}
|
|
const taxonomyBaseURL = `${baseURL}/${program.taxonomy}`
|
|
const partialArticles = await ds.scanCategory(taxonomyBaseURL, category)
|
|
const ps = partialArticles // partialArticles.slice(0, 3)
|
|
//console.log(ps)
|
|
Bluebird.each(ps, (async (art, i, length) => {
|
|
const more = await ds.getArticle(art.link)
|
|
const article = Object.assign({}, art, more)
|
|
const res = await ds.insertArticle(db, article)
|
|
if (res.success) {
|
|
console.log(`+ [${i+1}/${length}] "${art.title}"`)
|
|
} else {
|
|
console.log(`= [${i+1}/${length}] "${art.title}"`)
|
|
}
|
|
}))
|
|
}
|
|
|
|
main()
|