Initial commit.

.gitignore

@@ -0,0 +1,67 @@
+##
+# Vim swap/working files.
+*.sw[opa]
+
+##
+# SublimeText files.
+*.sublime-project
+*.sublime-workspace
+
+##
+# IDEA IntelliJ files.
+*.idea
+*.iml
+
+##
+# Visual Studio Code files.
+.vscode
+
+##
+# Mac OS-X miscellany.
+.DS_Store
+.AppleDouble
+.LSOverride
+._*
+
+##
+# Windows image file caches
+Thumbs.db
+ehthumbs.db
+
+# Folder config file
+Desktop.ini
+
+##
+# Compiled Object files, Static and Dynamic libs (Shared Objects).
+*.o
+*.a
+*.so
+
+##
+# Folders.
+_obj
+_test
+
+##
+# Architecture specific extensions/prefixes.
+*.[568vq]
+[568vq].out
+
+*.cgo1.go
+*.cgo2.c
+_cgo_defun.c
+_cgo_gotypes.go
+_cgo_export.*
+
+_testmain.go
+
+*.exe
+*.test
+*.prof
+
+##
+# Certificate files.
+id_[rd]sa
+*.pem
+*.crt
+*.key

.travis.yml

@@ -0,0 +1,18 @@
+language: go
+
+go:
+  - tip
+  - "1.10"
+  - 1.9
+  - 1.8
+  - 1.7
+  - 1.6
+  - 1.5
+
+script:
+  - go test ./...
+
+notifications:
+  email:
+    on_success: change
+    on_failure: always

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2018 Jay Taylor
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,25 @@
+# archiveis
+
+[![Documentation](https://godoc.org/github.com/jaytaylor/archiveis?status.svg)](https://godoc.org/github.com/jaytaylor/archiveis)
+[![Build Status](https://travis-ci.org/jaytaylor/archiveis.svg?branch=master)](https://travis-ci.org/jaytaylor/archiveis)
+[![Report Card](https://goreportcard.com/badge/github.com/jaytaylor/archiveis)](https://goreportcard.com/report/github.com/jaytaylor/archiveis)
+
+### About
+
+archiveis is a golang package for archiving web pages via [archive.is](https://archive.is).
+
+Please be mindful and responsible and go easy on them, we want archive.is to last forever!
+
+Created by [Jay Taylor](https://jaytaylor.com/).
+
+### Requirements
+
+* Go version 1.5 or newer
+
+### Running the test suite
+
+    go test ./...
+
+#### License
+
+Permissive MIT license, see the [LICENSE](LICENSE) file for more information.

archiveis.go

@@ -0,0 +1,104 @@
+package archiveis
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"net/url"
+	"regexp"
+	"strings"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/gigawattio/errorlib"
+	"github.com/parnurzeal/gorequest"
+)
+
+const (
+	baseURL   = "https://archive.is"
+	userAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.162 Safari/537.36"
+)
+
+var jsLocationExpr = regexp.MustCompile(`document\.location\.replace\(["']([^"']+)`)
+
+// Capture archives the provided URL using the archive.is service.
+func Capture(u string) (string, error) {
+	submitID, err := newSubmitID()
+	if err != nil {
+		return "", err
+	}
+
+	// return id, nil
+
+	content := fmt.Sprintf("submitid=%v&url=%v", url.QueryEscape(submitID), url.QueryEscape(u))
+	fmt.Printf("content=%v\n", content)
+	resp, body, errs := newRequest().Post(baseURL+"/submit/").Send(content).Set("content-type", "application/x-www-form-urlencoded").EndBytes()
+	if err := errorlib.Merge(errs); err != nil {
+		return "", err
+	}
+	if resp.StatusCode/100 != 2 {
+		return "", fmt.Errorf("form submit received unhappy response status-code=%v", resp.StatusCode)
+	}
+
+	doc, err := goquery.NewDocumentFromReader(bytes.NewBuffer(body))
+	if err != nil {
+		return "", fmt.Errorf("constructing goquery doc from submission response: %s", err)
+	}
+
+	if script := doc.Find("script").First(); script != nil {
+		js := strings.Trim(script.Text(), "\r\n\t ")
+		if match := jsLocationExpr.FindStringSubmatch(js); len(match) > 1 {
+			return match[1], nil
+		}
+	}
+
+	fmt.Printf("body: %+v\n", string(body))
+	fmt.Printf("headers: %+v\n", resp.Header)
+	fmt.Printf("trailers: %+v\n", resp.Trailer)
+
+	input := doc.Find("input[name=id]").First()
+	if input == nil {
+		return "", errors.New("page archive ID not found in submission response content")
+	}
+	id, exists := input.Attr("value")
+	if !exists {
+		return "", errors.New("no page archive ID value available")
+	}
+
+	final := fmt.Sprintf("%v/%v", baseURL, id)
+	return final, nil
+}
+
+// newSubmitID gets the index page and extracts the form submission identifier.
+func newSubmitID() (string, error) {
+	resp, body, errs := newRequest().Get(baseURL).EndBytes()
+	if err := errorlib.Merge(errs); err != nil {
+		return "", err
+	}
+	if resp.StatusCode/100 != 2 {
+		return "", fmt.Errorf("index retrieval received unhappy response status-code=%v", resp.StatusCode)
+	}
+
+	doc, err := goquery.NewDocumentFromReader(bytes.NewBuffer(body))
+	if err != nil {
+		return "", fmt.Errorf("constructing goquery doc from index: %s", err)
+	}
+
+	input := doc.Find("input[name=submitid]").First()
+	if input == nil {
+		return "", errors.New("no submitid element found")
+	}
+	id, exists := input.Attr("value")
+	if !exists {
+		return "", errors.New("no submitid value available")
+	}
+	return id, nil
+}
+
+func newRequest() *gorequest.SuperAgent {
+	r := gorequest.New().
+		Set("host", strings.Split(baseURL, "://")[1]).
+		Set("user-agent", userAgent).
+		Set("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8").
+		Set("referer", baseURL+"/")
+	return r
+}

archiveis_test.go

@@ -0,0 +1,27 @@
+package archiveis
+
+import (
+	"fmt"
+	"testing"
+	"time"
+)
+
+const page = "https://yro.slashdot.org/story/18/03/21/2112247/russia-secretly-helped-venezuela-launch-a-cryptocurrency-to-evade-us-sanctions#comments"
+
+func TestCapture1(t *testing.T) {
+	// Link which has been submitted before.
+	url, err := Capture(page)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Logf("Resolved URL=%q", url)
+}
+
+func TestCapture2(t *testing.T) {
+	// Link which has likely not been submitted before.
+	url, err := Capture(fmt.Sprintf("%v?%v", page, time.Now().Unix()))
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Logf("Resolved URL=%q", url)
+}