commit 618b9ef38b861f92769a5ab15e484d03710b6be3
parent 16e579a2e440dd6fe9b404365dc69f600c63c089
Author: David Eisinger <[email protected]>
Date: Fri, 10 Oct 2025 21:01:49 -0400
Fix bookshop.org error
Diffstat:
3 files changed, 21 insertions(+), 2 deletions(-)
diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,4 @@
bin
coverage.out
dist
+scripts
diff --git a/internal/api.go b/internal/api.go
@@ -10,6 +10,7 @@ import (
"net/http"
"net/url"
"os"
+ "os/exec"
"path"
"strconv"
"strings"
@@ -505,6 +506,16 @@ func NewHandlerWithConfig(db *DB, cfg HandlerConfig) (http.Handler, error) {
if isHTML {
fetchedTitle, txt, err = TitleAndText(ctx, req.URL)
if err != nil {
+ // Fallback: if HTML fetch fails, try w3m directly to get plaintext
+ if path, err2 := exec.LookPath("w3m"); err2 == nil && path != "" {
+ if out, err3 := w3mDump(ctx, req.URL); err3 == nil && strings.TrimSpace(out) != "" {
+ txt = strings.TrimSpace(out)
+ fetchedTitle = ""
+ err = nil
+ }
+ }
+ }
+ if err != nil {
http.Error(w, "failed to archive URL", http.StatusBadGateway)
return
}
diff --git a/internal/archiver.go b/internal/archiver.go
@@ -48,6 +48,9 @@ func TitleAndText(ctx context.Context, rawURL string) (string, string, error) {
return title, txt, nil
}
+// browserUA is a conservative desktop browser User-Agent that passes common bot checks.
+const browserUA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:128.0) Gecko/20100101 Firefox/128.0"
+
// IsHTMLURL attempts a HEAD request and inspects Content-Type to determine if
// the resource is HTML. If Content-Type is empty or HEAD fails, we assume HTML
// to avoid false negatives on servers that mishandle HEAD.
@@ -56,7 +59,9 @@ func IsHTMLURL(ctx context.Context, rawURL string) bool {
if err != nil {
return true
}
- req.Header.Set("User-Agent", "marky-archiver/1.0 (+https://example.invalid)")
+ req.Header.Set("User-Agent", browserUA)
+ req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
+ req.Header.Set("Accept-Language", "en-US,en;q=0.9")
client := &http.Client{Timeout: 15 * time.Second}
resp, err := client.Do(req)
if err != nil {
@@ -89,7 +94,9 @@ func fetchHTML(ctx context.Context, rawURL string) ([]byte, error) {
if err != nil {
return nil, fmt.Errorf("build request: %w", err)
}
- req.Header.Set("User-Agent", "marky-archiver/1.0 (+https://example.invalid)")
+ req.Header.Set("User-Agent", browserUA)
+ req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
+ req.Header.Set("Accept-Language", "en-US,en;q=0.9")
client := &http.Client{Timeout: 30 * time.Second}
resp, err := client.Do(req)
