commit 97c0eb383058b979acbb5dcf7a17340fded79297
parent cc4249f287ba6c604f11e99f79bc03f4264125c3
Author: David Eisinger <[email protected]>
Date: Mon, 6 Oct 2025 13:55:56 -0400
title fetching
Diffstat:
3 files changed, 69 insertions(+), 17 deletions(-)
diff --git a/README.md b/README.md
@@ -33,9 +33,9 @@ Create a new bookmark.
- Request body:
```
{
- "url": "https://example.com",
- "title": "Example",
- "description": "Optional description" // optional
+ "url": "https://example.com", // required
+ "title": "Example", // optional; fetched from page <title> if missing
+ "description": "Optional description" // optional
}
```
- Responses:
diff --git a/cmd/marky.go b/cmd/marky.go
@@ -80,8 +80,8 @@ func main() {
http.Error(w, "invalid JSON payload", http.StatusBadRequest)
return
}
- if req.URL == "" || req.Title == "" {
- http.Error(w, "url and title are required", http.StatusBadRequest)
+ if req.URL == "" {
+ http.Error(w, "url is required", http.StatusBadRequest)
return
}
@@ -92,11 +92,19 @@ func main() {
}
ctx, cancel := context.WithTimeout(r.Context(), 45*time.Second)
defer cancel()
- txt, err := dbpkg.ArchiveToText(ctx, req.URL)
+ fetchedTitle, txt, err := dbpkg.TitleAndText(ctx, req.URL)
if err != nil {
http.Error(w, "failed to archive URL", http.StatusBadGateway)
return
}
+ title := req.Title
+ if strings.TrimSpace(title) == "" {
+ if strings.TrimSpace(fetchedTitle) != "" {
+ title = fetchedTitle
+ } else {
+ title = req.URL
+ }
+ }
// If URL exists, re-archive and touch updated_at; else create
if existing, err := db.GetBookmarkByURL(r.Context(), req.URL); err != nil {
log.Printf("lookup by url error: %v", err)
@@ -126,7 +134,7 @@ func main() {
_ = json.NewEncoder(w).Encode(toResp(updated))
return
} else {
- created, err := db.CreateBookmarkWithArchive(r.Context(), req.URL, req.Title, req.Description, txt)
+ created, err := db.CreateBookmarkWithArchive(r.Context(), req.URL, title, req.Description, txt)
if err != nil {
log.Printf("create bookmark with archive error: %v", err)
http.Error(w, "internal server error", http.StatusInternalServerError)
diff --git a/internal/archiver.go b/internal/archiver.go
@@ -17,34 +17,47 @@ import (
// It attempts to extract the main content via Readability and then converts
// the resulting HTML to text; falls back to full document if extraction fails.
func ArchiveToText(ctx context.Context, rawURL string) (string, error) {
+ _, txt, err := TitleAndText(ctx, rawURL)
+ return txt, err
+}
+
+// TitleAndText fetches the URL and returns the page <title> (if any) and a
+// plaintext rendering of the document.
+func TitleAndText(ctx context.Context, rawURL string) (string, string, error) {
if _, err := neturl.ParseRequestURI(rawURL); err != nil {
- return "", fmt.Errorf("invalid url: %w", err)
+ return "", "", fmt.Errorf("invalid url: %w", err)
}
+ body, err := fetchHTML(ctx, rawURL)
+ if err != nil {
+ return "", "", err
+ }
+ htmlStr := string(body)
+ title := extractTitle(htmlStr)
+ txt := strings.TrimSpace(htmlToText(htmlStr))
+ return title, txt, nil
+}
+func fetchHTML(ctx context.Context, rawURL string) ([]byte, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
if err != nil {
- return "", fmt.Errorf("build request: %w", err)
+ return nil, fmt.Errorf("build request: %w", err)
}
req.Header.Set("User-Agent", "marky-archiver/1.0 (+https://example.invalid)")
client := &http.Client{Timeout: 30 * time.Second}
resp, err := client.Do(req)
if err != nil {
- return "", fmt.Errorf("fetch: %w", err)
+ return nil, fmt.Errorf("fetch: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
- return "", fmt.Errorf("unexpected status: %s", resp.Status)
+ return nil, fmt.Errorf("unexpected status: %s", resp.Status)
}
-
body, err := io.ReadAll(resp.Body)
if err != nil {
- return "", fmt.Errorf("read body: %w", err)
+ return nil, fmt.Errorf("read body: %w", err)
}
-
- // Convert full HTML document to plaintext with a basic parser.
- txt := htmlToText(string(body))
- return strings.TrimSpace(txt), nil
+ return body, nil
}
// htmlToText converts HTML to a readable plaintext representation.
@@ -128,6 +141,37 @@ func htmlToText(s string) string {
return b.String()
}
+// extractTitle parses the HTML and returns the document <title> text if found.
+func extractTitle(s string) string {
+ doc, err := html.Parse(strings.NewReader(s))
+ if err != nil {
+ return ""
+ }
+ var title string
+ var walk func(*html.Node)
+ walk = func(n *html.Node) {
+ if n.Type == html.ElementNode && strings.EqualFold(n.Data, "title") {
+ // Collect text nodes under <title>
+ var b strings.Builder
+ for c := n.FirstChild; c != nil; c = c.NextSibling {
+ if c.Type == html.TextNode {
+ b.WriteString(c.Data)
+ }
+ }
+ title = strings.TrimSpace(b.String())
+ return
+ }
+ for c := n.FirstChild; c != nil; c = c.NextSibling {
+ if title != "" {
+ break
+ }
+ walk(c)
+ }
+ }
+ walk(doc)
+ return title
+}
+
func getAttr(n *html.Node, key string) string {
for _, a := range n.Attr {
if strings.EqualFold(a.Key, key) {
