Add robots.txt

Disallow all crawlers.

site.js

@@ -3,6 +3,11 @@ require('dotenv').config()
 const { send } = require('micro')
 const files = require('serve-handler')
 
+const robotsTxt =
+`User-agent: *
+Disallow: /
+`
+
 const staticPaths = [ '/css', '/images', '/archive' ]
 
 function isStaticPath(req) {
@@ -26,5 +31,6 @@ module.exports = async function(req, res) {
   if (isStaticPath(req)) return await files(req, res, { public: 'routes' })
   let matched = match(req)
   if (matched) return await matched(req, res)
+  if (req.url === '/robots.txt') return send(res, 200, robotsTxt)
   return send(res, 404, { error: 'Not found' })
 }