commit 57c2c3a99a69e71c8852e6885262d003230560fd
parent 70ca38eedeae4b8efba8976d1c16106816b6d308
Author: David Eisinger <[email protected]>
Date: Mon, 6 Oct 2025 13:24:24 -0400
Remove w3m dependency
Diffstat:
5 files changed, 191 insertions(+), 39 deletions(-)
diff --git a/README.md b/README.md
@@ -1,6 +1,6 @@
# marky
-A simple web API for bookmarking websites, written in Go. It stores bookmarks in a local SQLite database at `~/.marky/bookmarks.sql` and exposes HTTP endpoints to create and list bookmarks. On creation, Marky archives a plaintext rendering of the page using `w3m` and stores it alongside the bookmark.
+A simple web API for bookmarking websites, written in Go. It stores bookmarks in a local SQLite database at `~/.marky/bookmarks.sql` and exposes HTTP endpoints to create and list bookmarks. On creation, Marky archives a plaintext rendering of the page using a pure-Go HTML-to-text conversion and stores it alongside the bookmark.
- Language: Go
- Storage: SQLite (via `modernc.org/sqlite`, pure Go)
@@ -10,7 +10,7 @@ A simple web API for bookmarking websites, written in Go. It stores bookmarks in
- Go 1.24+ (tested with 1.24.5)
- Make (optional, for the provided Makefile)
-- `w3m` in PATH for plaintext archiving
+ (No external tools required for archiving.)
## Build
@@ -43,7 +43,6 @@ Create a new bookmark.
- `400 Bad Request` for invalid payload
- `415 Unsupported Media Type` if not JSON
- `502 Bad Gateway` if archiver fails to fetch content
- - `503 Service Unavailable` if `w3m` is not installed/available
- `500 Internal Server Error` on unexpected errors
- Response body:
```
@@ -113,6 +112,7 @@ curl -s http://localhost:8080/bookmarks/1/archive
- Code entrypoint: `cmd/marky.go`
- Database and helpers: `internal/`
- Migrations: `internal/migrations/*.sql`
+- Archiver: `internal/archiver.go` (pure-Go HTML-to-text)
---
diff --git a/cmd/marky.go b/cmd/marky.go
@@ -85,18 +85,14 @@ func main() {
return
}
- // Always archive synchronously during creation
- if !dbpkg.HasW3M() {
- http.Error(w, "w3m not available on server", http.StatusServiceUnavailable)
- return
- }
+ // Always archive synchronously during creation (pure-Go archiver)
if _, err := url.ParseRequestURI(req.URL); err != nil {
http.Error(w, "invalid url", http.StatusBadRequest)
return
}
ctx, cancel := context.WithTimeout(r.Context(), 45*time.Second)
defer cancel()
- txt, err := dbpkg.ArchiveWithW3M(ctx, req.URL)
+ txt, err := dbpkg.ArchiveToText(ctx, req.URL)
if err != nil {
http.Error(w, "failed to archive URL", http.StatusBadGateway)
return
@@ -203,11 +199,7 @@ func main() {
}
ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second)
defer cancel()
- if !dbpkg.HasW3M() {
- http.Error(w, "w3m not available on server", http.StatusServiceUnavailable)
- return
- }
- txt, err := dbpkg.ArchiveWithW3M(ctx, b.URL)
+ txt, err := dbpkg.ArchiveToText(ctx, b.URL)
if err != nil {
http.Error(w, "failed to fetch archive", http.StatusBadGateway)
return
diff --git a/go.mod b/go.mod
@@ -4,6 +4,8 @@ go 1.24.5
require modernc.org/sqlite v1.33.0
+require golang.org/x/net v0.44.0
+
require (
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/google/uuid v1.6.0 // indirect
@@ -11,7 +13,7 @@ require (
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/ncruces/go-strftime v0.1.9 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
- golang.org/x/sys v0.22.0 // indirect
+ golang.org/x/sys v0.36.0 // indirect
modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect
modernc.org/mathutil v1.6.0 // indirect
modernc.org/memory v1.8.0 // indirect
diff --git a/go.sum b/go.sum
@@ -16,9 +16,11 @@ github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0=
golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/net v0.44.0 h1:evd8IRDyfNBMBTTY5XRF1vaZlD+EmWx6x8PkhR04H/I=
+golang.org/x/net v0.44.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
-golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k=
+golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/tools v0.15.0 h1:zdAyfUGbYmuVokhzVmghFl2ZJh5QhcfebBgmVPFYA+8=
golang.org/x/tools v0.15.0/go.mod h1:hpksKq4dtpQWS1uQ61JkdqWM3LscIS6Slf+VVkm+wQk=
modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE=
diff --git a/internal/archiver.go b/internal/archiver.go
@@ -3,37 +3,193 @@ package internal
import (
"bytes"
"context"
- "errors"
"fmt"
- "os/exec"
+ "io"
+ "net/http"
+ neturl "net/url"
"strings"
+ "time"
+
+ "golang.org/x/net/html"
)
-// HasW3M reports whether the 'w3m' binary is available in PATH.
-func HasW3M() bool {
- _, err := exec.LookPath("w3m")
- return err == nil
+// ArchiveToText fetches the given URL and returns a plaintext rendering.
+// It attempts to extract the main content via Readability and then converts
+// the resulting HTML to text; falls back to full document if extraction fails.
+func ArchiveToText(ctx context.Context, rawURL string) (string, error) {
+ if _, err := neturl.ParseRequestURI(rawURL); err != nil {
+ return "", fmt.Errorf("invalid url: %w", err)
+ }
+
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
+ if err != nil {
+ return "", fmt.Errorf("build request: %w", err)
+ }
+ req.Header.Set("User-Agent", "marky-archiver/1.0 (+https://example.invalid)")
+
+ client := &http.Client{Timeout: 30 * time.Second}
+ resp, err := client.Do(req)
+ if err != nil {
+ return "", fmt.Errorf("fetch: %w", err)
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+ return "", fmt.Errorf("unexpected status: %s", resp.Status)
+ }
+
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return "", fmt.Errorf("read body: %w", err)
+ }
+
+ // Convert full HTML document to plaintext with a basic parser.
+ txt := htmlToText(string(body))
+ return strings.TrimSpace(txt), nil
}
-// ArchiveWithW3M runs w3m to retrieve a plaintext rendering of the given URL.
-// It returns the plain text output.
-func ArchiveWithW3M(ctx context.Context, url string) (string, error) {
- path, err := exec.LookPath("w3m")
+// htmlToText converts HTML to a readable plaintext representation.
+// It removes script/style, collapses whitespace, and inserts newlines around block elements.
+type liState struct{ hasContent bool }
+
+func htmlToText(s string) string {
+ doc, err := html.Parse(strings.NewReader(s))
if err != nil {
- return "", errors.New("w3m not found in PATH")
+ return strings.TrimSpace(stripTagsFallback(s))
+ }
+ var b bytes.Buffer
+ var walk func(n *html.Node, linkHref string, li *liState)
+ walk = func(n *html.Node, linkHref string, li *liState) {
+ if n.Type == html.ElementNode {
+ tag := strings.ToLower(n.Data)
+ switch tag {
+ case "script", "style":
+ return
+ case "br":
+ ensureNL(&b, 1)
+ case "p", "div", "section", "article", "header", "footer", "h1", "h2", "h3", "h4", "h5", "h6", "pre", "blockquote":
+ // Ensure blank line before block paragraphs/headers
+ ensureNL(&b, 2)
+ case "ul", "ol":
+ ensureNL(&b, 1)
+ case "li":
+ ensureNL(&b, 1)
+ }
+ if tag == "a" {
+ // capture href
+ if href := getAttr(n, "href"); href != "" {
+ linkHref = href
+ }
+ }
+ }
+
+ if n.Type == html.TextNode {
+ // Only emit bullet when the first non-empty text appears within this <li>
+ trimmed := strings.TrimSpace(n.Data)
+ if li != nil && !li.hasContent && trimmed != "" {
+ ensureNL(&b, 1)
+ b.WriteString("- ")
+ li.hasContent = true
+ }
+ writeCollapsed(&b, n.Data)
+ }
+
+ // children
+ startLen := b.Len()
+ // If we are at an <li>, propagate a fresh liState to children
+ var childLI *liState
+ if n.Type == html.ElementNode && strings.EqualFold(n.Data, "li") {
+ childLI = &liState{}
+ } else {
+ childLI = li
+ }
+ for c := n.FirstChild; c != nil; c = c.NextSibling {
+ walk(c, linkHref, childLI)
+ }
+
+ if n.Type == html.ElementNode {
+ tag := strings.ToLower(n.Data)
+ // If this was an anchor with text and href, append inline reference
+ if tag == "a" && linkHref != "" && b.Len() > startLen {
+ b.WriteString(" [")
+ b.WriteString(linkHref)
+ b.WriteString("]")
+ }
+ switch tag {
+ case "p", "div", "section", "article", "header", "footer", "h1", "h2", "h3", "h4", "h5", "h6", "pre", "blockquote":
+ ensureNL(&b, 2) // blank line after paragraphs/headers
+ case "li":
+ ensureNL(&b, 1)
+ case "tr":
+ ensureNL(&b, 1)
+ }
+ }
+ }
+ walk(doc, "", nil)
+ return b.String()
+}
+
+func getAttr(n *html.Node, key string) string {
+ for _, a := range n.Attr {
+ if strings.EqualFold(a.Key, key) {
+ return a.Val
+ }
+ }
+ return ""
+}
+
+func ensureNL(b *bytes.Buffer, n int) {
+ // Ensure exactly at least n trailing newlines
+ count := 0
+ bs := b.Bytes()
+ for i := len(bs) - 1; i >= 0 && bs[i] == '\n'; i-- {
+ count++
}
- // w3m -dump -T text/html -o display_link_number=1 <url>
- cmd := exec.CommandContext(ctx, path, "-dump", "-T", "text/html", "-o", "display_link_number=1", url)
- var out, stderr bytes.Buffer
- cmd.Stdout = &out
- cmd.Stderr = &stderr
- if err := cmd.Run(); err != nil {
- se := strings.TrimSpace(stderr.String())
- if se == "" {
- se = err.Error()
+ for count < n {
+ b.WriteByte('\n')
+ count++
+ }
+}
+
+func writeCollapsed(b *bytes.Buffer, s string) {
+ // collapse all whitespace runs to single spaces; trim leading spaces at line start
+ space := false
+ for _, r := range s {
+ if r == '\n' || r == '\r' || r == '\t' || r == ' ' || r == '\f' {
+ space = true
+ continue
}
- return "", fmt.Errorf("w3m failed: %s", se)
+ if space {
+ if last := lastByte(b); last != 0 && last != '\n' && last != ' ' {
+ b.WriteByte(' ')
+ }
+ space = false
+ }
+ b.WriteRune(r)
+ }
+}
+
+func lastByte(b *bytes.Buffer) byte {
+ if b.Len() == 0 {
+ return 0
}
- return out.String(), nil
+ return b.Bytes()[b.Len()-1]
}
+func stripTagsFallback(s string) string {
+ // naive fallback: remove angle-bracketed tags
+ out := make([]rune, 0, len(s))
+ inTag := false
+ for _, r := range s {
+ switch r {
+ case '<':
+ inTag = true
+ case '>':
+ inTag = false
+ default:
+ if !inTag {
+ out = append(out, r)
+ }
+ }
+ }
+ return string(out)
+}
