From f80a820a1994cbe9ecd7eeab13428a20e839bdfd Mon Sep 17 00:00:00 2001 From: Perception Date: Wed, 6 Jan 2021 01:15:17 -0800 Subject: [PATCH] Refine info on downloading articles --- README.md | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 8c6afc3..23dc94f 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ In the root of the project, do the following. # Install node modules npm i -# Initialize database +# Initialize database from scratch sqlite3 main.db < sql/ds.sql sqlite3 main.db < sql/ds.001.sql sqlite3 main.db < sql/ds.002.sql @@ -19,9 +19,28 @@ sqlite3 main.db < sql/ds.002.sql wget http://dss.coincidencedetector.com/archive/dss-latest.db.gz gzcat dss-latest.db.gz > main.db -# Scrape articles; This may take a few days if done from scratch. -bin/download-all - # Run the server micro-dev -p 1984 site.js ``` + +## Downloading Articles + +```sh +# Download all articles from scratch: +# This takes a few days and may need babysitting in case of crashes. +bin/download-all + +# Dowloading the last 140 articles per category: +# If you have a recent snapshot, running this once is how you should get it up to date. +P=1 bin/download-all +``` + +## Cron + +To update the database with new articles, replace the value of `DSS_ROOT` with the path +to where DSS is on your system, and place the following in your crontab. + +```sh +DSS_ROOT=/path/to/dss + 55 * * * * cd $DSS_ROOT && bin/articles-update +```