marky

commit fdaf61bd91edf8b5ac90448203c7ad3b92b9a8c6
parent f4c3b098dbf4621e09ef7fd414b8adf5901d6e5a
Author: David Eisinger <[email protected]>
Date:   Thu, 16 Apr 2026 10:08:24 -0400

Remove fake browser UA

Causing issues on brainbaking.com

Diffstat:
M
internal/archiver.go
 | 
5
-----

1 file changed, 0 insertions(+), 5 deletions(-)
diff --git a/internal/archiver.go b/internal/archiver.go
@@ -48,9 +48,6 @@ func TitleAndText(ctx context.Context, rawURL string) (string, string, error) {
 	return title, txt, nil
 }
 
-// browserUA is a conservative desktop browser User-Agent that passes common bot checks.
-const browserUA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:128.0) Gecko/20100101 Firefox/128.0"
-
 // IsHTMLURL attempts a HEAD request and inspects Content-Type to determine if
 // the resource is HTML. If Content-Type is empty or HEAD fails, we assume HTML
 // to avoid false negatives on servers that mishandle HEAD.
@@ -59,7 +56,6 @@ func IsHTMLURL(ctx context.Context, rawURL string) bool {
 	if err != nil {
 		return true
 	}
-	req.Header.Set("User-Agent", browserUA)
 	req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
 	req.Header.Set("Accept-Language", "en-US,en;q=0.9")
 	client := &http.Client{Timeout: 15 * time.Second}
@@ -95,7 +91,6 @@ func fetchHTML(ctx context.Context, rawURL string) ([]byte, error) {
 	if err != nil {
 		return nil, fmt.Errorf("build request: %w", err)
 	}
-	req.Header.Set("User-Agent", browserUA)
 	req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
 	req.Header.Set("Accept-Language", "en-US,en;q=0.9")