commit 70ca38eedeae4b8efba8976d1c16106816b6d308
parent b86e0a1385d0b007f43c90d34103cd1b05c9e8c6
Author: David Eisinger <[email protected]>
Date: Mon, 6 Oct 2025 13:12:38 -0400
Archiving
Diffstat:
5 files changed, 250 insertions(+), 10 deletions(-)
diff --git a/README.md b/README.md
@@ -1,6 +1,6 @@
# marky
-A simple web API for bookmarking websites, written in Go. It stores bookmarks in a local SQLite database at `~/.marky/bookmarks.sql` and exposes HTTP endpoints to create and list bookmarks.
+A simple web API for bookmarking websites, written in Go. It stores bookmarks in a local SQLite database at `~/.marky/bookmarks.sql` and exposes HTTP endpoints to create and list bookmarks. On creation, Marky archives a plaintext rendering of the page using `w3m` and stores it alongside the bookmark.
- Language: Go
- Storage: SQLite (via `modernc.org/sqlite`, pure Go)
@@ -10,6 +10,7 @@ A simple web API for bookmarking websites, written in Go. It stores bookmarks in
- Go 1.24+ (tested with 1.24.5)
- Make (optional, for the provided Makefile)
+- `w3m` in PATH for plaintext archiving
## Build
@@ -38,9 +39,11 @@ Create a new bookmark.
}
```
- Responses:
- - `201 Created` with JSON body of the created bookmark
+ - `201 Created` with JSON body of the created bookmark (archive stored synchronously)
- `400 Bad Request` for invalid payload
- `415 Unsupported Media Type` if not JSON
+ - `502 Bad Gateway` if archiver fails to fetch content
+ - `503 Service Unavailable` if `w3m` is not installed/available
- `500 Internal Server Error` on unexpected errors
- Response body:
```
@@ -89,6 +92,22 @@ Table: `bookmarks`
Migrations are embedded and applied automatically on startup.
+An `archives` table stores the plaintext dump per bookmark and is kept in sync on creation.
+
+### GET /bookmarks/{id}/archive
+
+Return the archived plaintext for the given bookmark ID.
+
+- Responses:
+ - `200 OK` with `text/plain; charset=utf-8`
+ - `404 Not Found` if no archive exists for the bookmark
+ - `500 Internal Server Error` on unexpected errors
+
+curl example:
+```
+curl -s http://localhost:8080/bookmarks/1/archive
+```
+
## Development
- Code entrypoint: `cmd/marky.go`
diff --git a/cmd/marky.go b/cmd/marky.go
@@ -3,12 +3,13 @@ package main
import (
"context"
"encoding/json"
- "errors"
"log"
"net/http"
+ "net/url"
"os"
"os/signal"
"strconv"
+ "strings"
"syscall"
"time"
@@ -84,14 +85,25 @@ func main() {
return
}
- created, err := db.CreateBookmark(r.Context(), req.URL, req.Title, req.Description)
+ // Always archive synchronously during creation
+ if !dbpkg.HasW3M() {
+ http.Error(w, "w3m not available on server", http.StatusServiceUnavailable)
+ return
+ }
+ if _, err := url.ParseRequestURI(req.URL); err != nil {
+ http.Error(w, "invalid url", http.StatusBadRequest)
+ return
+ }
+ ctx, cancel := context.WithTimeout(r.Context(), 45*time.Second)
+ defer cancel()
+ txt, err := dbpkg.ArchiveWithW3M(ctx, req.URL)
if err != nil {
- // Map validation errors to 400, others 500
- if errors.Is(err, os.ErrInvalid) {
- http.Error(w, err.Error(), http.StatusBadRequest)
- return
- }
- log.Printf("create bookmark error: %v", err)
+ http.Error(w, "failed to archive URL", http.StatusBadGateway)
+ return
+ }
+ created, err := db.CreateBookmarkWithArchive(r.Context(), req.URL, req.Title, req.Description, txt)
+ if err != nil {
+ log.Printf("create bookmark with archive error: %v", err)
http.Error(w, "internal server error", http.StatusInternalServerError)
return
}
@@ -145,6 +157,82 @@ func main() {
}
})
+ // Archive endpoints: GET /bookmarks/{id}/archive
+ mux.HandleFunc("/bookmarks/", func(w http.ResponseWriter, r *http.Request) {
+ // Only handle the /bookmarks/{id}/archive path here
+ // Basic parsing without full router
+ path := r.URL.Path
+ // Expect suffix "/archive"
+ if !strings.HasSuffix(path, "/archive") {
+ http.NotFound(w, r)
+ return
+ }
+ // Trim prefix and suffix to get the ID segment
+ // "/bookmarks/{id}/archive" -> "{id}"
+ trimmed := strings.TrimSuffix(strings.TrimPrefix(path, "/bookmarks/"), "/archive")
+ if trimmed == "" || strings.Contains(trimmed, "/") {
+ http.NotFound(w, r)
+ return
+ }
+ id64, err := strconv.ParseInt(trimmed, 10, 64)
+ if err != nil || id64 <= 0 {
+ http.Error(w, "invalid id", http.StatusBadRequest)
+ return
+ }
+
+ switch r.Method {
+ case http.MethodGet:
+ // If not present and fetch=1, attempt on-demand fetch via w3m and persist
+ content, _, ok, err := db.GetArchive(r.Context(), id64)
+ if err != nil {
+ log.Printf("get archive error: %v", err)
+ http.Error(w, "internal server error", http.StatusInternalServerError)
+ return
+ }
+ if !ok {
+ if r.URL.Query().Get("fetch") == "1" || r.URL.Query().Get("fetch") == "true" {
+ b, err := db.GetBookmark(r.Context(), id64)
+ if err != nil {
+ log.Printf("lookup bookmark error: %v", err)
+ http.Error(w, "internal server error", http.StatusInternalServerError)
+ return
+ }
+ if b == nil {
+ http.NotFound(w, r)
+ return
+ }
+ ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second)
+ defer cancel()
+ if !dbpkg.HasW3M() {
+ http.Error(w, "w3m not available on server", http.StatusServiceUnavailable)
+ return
+ }
+ txt, err := dbpkg.ArchiveWithW3M(ctx, b.URL)
+ if err != nil {
+ http.Error(w, "failed to fetch archive", http.StatusBadGateway)
+ return
+ }
+ if err := db.SaveArchive(ctx, id64, txt); err != nil {
+ log.Printf("save archive error: %v", err)
+ http.Error(w, "internal server error", http.StatusInternalServerError)
+ return
+ }
+ content = txt
+ ok = true
+ }
+ }
+ if !ok {
+ http.NotFound(w, r)
+ return
+ }
+ w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+ _, _ = w.Write([]byte(content))
+ default:
+ w.Header().Set("Allow", http.MethodGet)
+ http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+ }
+ })
+
srv := &http.Server{
Addr: ":8080",
Handler: logging(mux),
diff --git a/internal/archiver.go b/internal/archiver.go
@@ -0,0 +1,39 @@
+package internal
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "fmt"
+ "os/exec"
+ "strings"
+)
+
+// HasW3M reports whether the 'w3m' binary is available in PATH.
+func HasW3M() bool {
+ _, err := exec.LookPath("w3m")
+ return err == nil
+}
+
+// ArchiveWithW3M runs w3m to retrieve a plaintext rendering of the given URL.
+// It returns the plain text output.
+func ArchiveWithW3M(ctx context.Context, url string) (string, error) {
+ path, err := exec.LookPath("w3m")
+ if err != nil {
+ return "", errors.New("w3m not found in PATH")
+ }
+ // w3m -dump -T text/html -o display_link_number=1 <url>
+ cmd := exec.CommandContext(ctx, path, "-dump", "-T", "text/html", "-o", "display_link_number=1", url)
+ var out, stderr bytes.Buffer
+ cmd.Stdout = &out
+ cmd.Stderr = &stderr
+ if err := cmd.Run(); err != nil {
+ se := strings.TrimSpace(stderr.String())
+ if se == "" {
+ se = err.Error()
+ }
+ return "", fmt.Errorf("w3m failed: %s", se)
+ }
+ return out.String(), nil
+}
+
diff --git a/internal/database.go b/internal/database.go
@@ -292,3 +292,87 @@ func nullString(ns sql.NullString) any {
return nil
}
+// CreateBookmarkWithArchive creates a bookmark and saves its plaintext archive
+// within a single transaction to ensure consistency.
+func (db *DB) CreateBookmarkWithArchive(ctx context.Context, url, title string, description *string, archiveContent string) (*Bookmark, error) {
+ if strings.TrimSpace(url) == "" || strings.TrimSpace(title) == "" {
+ return nil, fmt.Errorf("url and title are required")
+ }
+ now := time.Now().UTC()
+ var desc sql.NullString
+ if description != nil {
+ desc = sql.NullString{String: *description, Valid: true}
+ }
+
+ tx, err := db.sql.BeginTx(ctx, nil)
+ if err != nil {
+ return nil, fmt.Errorf("begin tx: %w", err)
+ }
+ res, err := tx.ExecContext(ctx, `
+ INSERT INTO bookmarks(url, title, description, created_at, updated_at)
+ VALUES(?, ?, ?, ?, ?)
+ `, url, title, nullString(desc), now.Format(time.RFC3339Nano), now.Format(time.RFC3339Nano))
+ if err != nil {
+ _ = tx.Rollback()
+ return nil, fmt.Errorf("insert bookmark: %w", err)
+ }
+ id, err := res.LastInsertId()
+ if err != nil {
+ _ = tx.Rollback()
+ return nil, fmt.Errorf("last insert id: %w", err)
+ }
+ if _, err := tx.ExecContext(ctx, `
+ INSERT INTO archives(bookmark_id, content, archived_at)
+ VALUES(?, ?, ?)
+ `, id, archiveContent, now.Format(time.RFC3339Nano)); err != nil {
+ _ = tx.Rollback()
+ return nil, fmt.Errorf("insert archive: %w", err)
+ }
+ if err := tx.Commit(); err != nil {
+ return nil, fmt.Errorf("commit: %w", err)
+ }
+ return &Bookmark{
+ ID: id,
+ URL: url,
+ Title: title,
+ Description: desc,
+ CreatedAt: now,
+ UpdatedAt: now,
+ }, nil
+}
+
+// SaveArchive upserts the plaintext archive content for a bookmark.
+func (db *DB) SaveArchive(ctx context.Context, bookmarkID int64, content string) error {
+ now := time.Now().UTC().Format(time.RFC3339Nano)
+ _, err := db.sql.ExecContext(ctx, `
+ INSERT INTO archives(bookmark_id, content, archived_at)
+ VALUES(?, ?, ?)
+ ON CONFLICT(bookmark_id) DO UPDATE SET
+ content=excluded.content,
+ archived_at=excluded.archived_at
+ `, bookmarkID, content, now)
+ if err != nil {
+ return fmt.Errorf("save archive: %w", err)
+ }
+ return nil
+}
+
+// GetArchive returns archived plaintext and timestamp for a bookmark.
+// ok is false if no archive exists.
+func (db *DB) GetArchive(ctx context.Context, bookmarkID int64) (content string, archivedAt time.Time, ok bool, err error) {
+ var ts string
+ err = db.sql.QueryRowContext(ctx, `
+ SELECT content, archived_at FROM archives WHERE bookmark_id = ?
+ `, bookmarkID).Scan(&content, &ts)
+ if err != nil {
+ if errors.Is(err, sql.ErrNoRows) {
+ return "", time.Time{}, false, nil
+ }
+ return "", time.Time{}, false, fmt.Errorf("get archive: %w", err)
+ }
+ at, parseErr := time.Parse(time.RFC3339Nano, ts)
+ if parseErr != nil {
+ return content, time.Time{}, true, fmt.Errorf("parse archived_at: %w", parseErr)
+ }
+ return content, at, true, nil
+}
diff --git a/internal/migrations/0002_archives.sql b/internal/migrations/0002_archives.sql
@@ -0,0 +1,10 @@
+-- Archive table to store plaintext dumps per bookmark
+CREATE TABLE IF NOT EXISTS archives (
+ bookmark_id INTEGER PRIMARY KEY,
+ content TEXT NOT NULL,
+ archived_at TEXT NOT NULL,
+ FOREIGN KEY (bookmark_id) REFERENCES bookmarks(id) ON DELETE CASCADE
+);
+
+CREATE INDEX IF NOT EXISTS idx_archives_archived_at ON archives(archived_at);
+
