marky

archiver.go (8160B)

---

 package internal
 
 import (
 	"bytes"
 	"context"
 	"fmt"
 	"io"
 	"net/http"
 	neturl "net/url"
 	"os/exec"
 	"strings"
 	"time"
 
 	"golang.org/x/net/html"
 )
 
 // ArchiveToText fetches the given URL and returns a plaintext rendering.
 // It attempts to extract the main content via Readability and then converts
 // the resulting HTML to text; falls back to full document if extraction fails.
 func ArchiveToText(ctx context.Context, rawURL string) (string, error) {
 	_, txt, err := TitleAndText(ctx, rawURL)
 	return txt, err
 }
 
 // TitleAndText fetches the URL and returns the page <title> (if any) and a
 // plaintext rendering of the document.
 func TitleAndText(ctx context.Context, rawURL string) (string, string, error) {
 	if _, err := neturl.ParseRequestURI(rawURL); err != nil {
 		return "", "", fmt.Errorf("invalid url: %w", err)
 	}
 	body, err := fetchHTML(ctx, rawURL)
 	if err != nil {
 		return "", "", err
 	}
 	htmlStr := string(body)
 	title := extractTitle(htmlStr)
 	// Prefer external w3m if available for robust text rendering; fall back to built-in.
 	var txt string
 	if path, err := exec.LookPath("w3m"); err == nil && path != "" {
 		if out, err := w3mDump(ctx, rawURL); err == nil && strings.TrimSpace(out) != "" {
 			txt = strings.TrimSpace(out)
 		} else {
 			txt = strings.TrimSpace(htmlToText(htmlStr))
 		}
 	} else {
 		txt = strings.TrimSpace(htmlToText(htmlStr))
 	}
 	return title, txt, nil
 }
 
 // IsHTMLURL attempts a HEAD request and inspects Content-Type to determine if
 // the resource is HTML. If Content-Type is empty or HEAD fails, we assume HTML
 // to avoid false negatives on servers that mishandle HEAD.
 func IsHTMLURL(ctx context.Context, rawURL string) bool {
 	req, err := http.NewRequestWithContext(ctx, http.MethodHead, rawURL, nil)
 	if err != nil {
 		return true
 	}
 	req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
 	req.Header.Set("Accept-Language", "en-US,en;q=0.9")
 	client := &http.Client{Timeout: 15 * time.Second}
 	resp, err := client.Do(req)
 	if err != nil {
 		return true
 	}
 	defer resp.Body.Close()
 	// A denial or other error response describes the error document, not the
 	// resource. Treat it as inconclusive so the archive path can still try its
 	// renderer fallback (for example, w3m may be allowed by the origin).
 	if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
 		return true
 	}
 	ct := strings.ToLower(resp.Header.Get("Content-Type"))
 	if ct == "" {
 		return true
 	}
 	if strings.Contains(ct, "text/html") || strings.Contains(ct, "+html") || strings.Contains(ct, "application/xhtml+xml") {
 		return true
 	}
 	return false
 }
 
 // w3mDump runs `w3m -dump` to render the given URL to plaintext.
 func w3mDump(ctx context.Context, url string) (string, error) {
 	// Force UTF-8 output to avoid mojibake on systems with non-UTF locales.
 	cmd := exec.CommandContext(ctx, "w3m", "-dump", "-T", "text/html", "-O", "UTF-8", "-o", "display_link_number=1", url)
 	// Keep environment as-is; rely on user PATH. Capture stdout only.
 	b, err := cmd.Output()
 	if err != nil {
 		return "", fmt.Errorf("w3m dump: %w", err)
 	}
 	return string(b), nil
 }
 
 func fetchHTML(ctx context.Context, rawURL string) ([]byte, error) {
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
 	if err != nil {
 		return nil, fmt.Errorf("build request: %w", err)
 	}
 	req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
 	req.Header.Set("Accept-Language", "en-US,en;q=0.9")
 
 	client := &http.Client{Timeout: 30 * time.Second}
 	resp, err := client.Do(req)
 	if err != nil {
 		return nil, fmt.Errorf("fetch: %w", err)
 	}
 	defer resp.Body.Close()
 	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
 		return nil, fmt.Errorf("unexpected status: %s", resp.Status)
 	}
 	body, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return nil, fmt.Errorf("read body: %w", err)
 	}
 	return body, nil
 }
 
 // htmlToText converts HTML to a readable plaintext representation.
 // It removes script/style, collapses whitespace, and inserts newlines around block elements.
 type liState struct{ hasContent bool }
 
 func htmlToText(s string) string {
 	doc, err := html.Parse(strings.NewReader(s))
 	if err != nil {
 		return strings.TrimSpace(stripTagsFallback(s))
 	}
 	var b bytes.Buffer
 	var walk func(n *html.Node, linkHref string, li *liState)
 	walk = func(n *html.Node, linkHref string, li *liState) {
 		if n.Type == html.ElementNode {
 			tag := strings.ToLower(n.Data)
 			switch tag {
 			case "script", "style":
 				return
 			case "br":
 				ensureNL(&b, 1)
 			case "p", "div", "section", "article", "header", "footer", "h1", "h2", "h3", "h4", "h5", "h6", "pre", "blockquote":
 				// Ensure blank line before block paragraphs/headers
 				ensureNL(&b, 2)
 			case "ul", "ol":
 				ensureNL(&b, 1)
 			case "li":
 				ensureNL(&b, 1)
 			}
 			if tag == "a" {
 				// capture href
 				if href := getAttr(n, "href"); href != "" {
 					linkHref = href
 				}
 			}
 		}
 
 		if n.Type == html.TextNode {
 			// Only emit bullet when the first non-empty text appears within this <li>
 			trimmed := strings.TrimSpace(n.Data)
 			if li != nil && !li.hasContent && trimmed != "" {
 				ensureNL(&b, 1)
 				b.WriteString("- ")
 				li.hasContent = true
 			}
 			writeCollapsed(&b, n.Data)
 		}
 
 		// children
 		startLen := b.Len()
 		// If we are at an <li>, propagate a fresh liState to children
 		var childLI *liState
 		if n.Type == html.ElementNode && strings.EqualFold(n.Data, "li") {
 			childLI = &liState{}
 		} else {
 			childLI = li
 		}
 		for c := n.FirstChild; c != nil; c = c.NextSibling {
 			walk(c, linkHref, childLI)
 		}
 
 		if n.Type == html.ElementNode {
 			tag := strings.ToLower(n.Data)
 			// If this was an anchor with text and href, append inline reference
 			if tag == "a" && linkHref != "" && b.Len() > startLen {
 				b.WriteString(" [")
 				b.WriteString(linkHref)
 				b.WriteString("]")
 			}
 			switch tag {
 			case "p", "div", "section", "article", "header", "footer", "h1", "h2", "h3", "h4", "h5", "h6", "pre", "blockquote":
 				ensureNL(&b, 2) // blank line after paragraphs/headers
 			case "li":
 				ensureNL(&b, 1)
 			case "tr":
 				ensureNL(&b, 1)
 			}
 		}
 	}
 	walk(doc, "", nil)
 	return b.String()
 }
 
 // extractTitle parses the HTML and returns the document <title> text if found.
 func extractTitle(s string) string {
 	doc, err := html.Parse(strings.NewReader(s))
 	if err != nil {
 		return ""
 	}
 	var title string
 	var walk func(*html.Node)
 	walk = func(n *html.Node) {
 		if n.Type == html.ElementNode && strings.EqualFold(n.Data, "title") {
 			// Collect text nodes under <title>
 			var b strings.Builder
 			for c := n.FirstChild; c != nil; c = c.NextSibling {
 				if c.Type == html.TextNode {
 					b.WriteString(c.Data)
 				}
 			}
 			title = strings.TrimSpace(b.String())
 			return
 		}
 		for c := n.FirstChild; c != nil; c = c.NextSibling {
 			if title != "" {
 				break
 			}
 			walk(c)
 		}
 	}
 	walk(doc)
 	return title
 }
 
 func getAttr(n *html.Node, key string) string {
 	for _, a := range n.Attr {
 		if strings.EqualFold(a.Key, key) {
 			return a.Val
 		}
 	}
 	return ""
 }
 
 func ensureNL(b *bytes.Buffer, n int) {
 	// Ensure exactly at least n trailing newlines
 	count := 0
 	bs := b.Bytes()
 	for i := len(bs) - 1; i >= 0 && bs[i] == '\n'; i-- {
 		count++
 	}
 	for count < n {
 		b.WriteByte('\n')
 		count++
 	}
 }
 
 func writeCollapsed(b *bytes.Buffer, s string) {
 	// collapse all whitespace runs to single spaces; trim leading spaces at line start
 	space := false
 	for _, r := range s {
 		if r == '\n' || r == '\r' || r == '\t' || r == ' ' || r == '\f' {
 			space = true
 			continue
 		}
 		if space {
 			if last := lastByte(b); last != 0 && last != '\n' && last != ' ' {
 				b.WriteByte(' ')
 			}
 			space = false
 		}
 		b.WriteRune(r)
 	}
 }
 
 func lastByte(b *bytes.Buffer) byte {
 	if b.Len() == 0 {
 		return 0
 	}
 	return b.Bytes()[b.Len()-1]
 }
 
 func stripTagsFallback(s string) string {
 	// naive fallback: remove angle-bracketed tags
 	out := make([]rune, 0, len(s))
 	inTag := false
 	for _, r := range s {
 		switch r {
 		case '<':
 			inTag = true
 		case '>':
 			inTag = false
 		default:
 			if !inTag {
 				out = append(out, r)
 			}
 		}
 	}
 	return string(out)
 }