davideisinger.com

My personal website
Log | Files | Refs | README

archive (3385B)

      1 #!/usr/bin/env ruby
      2 
      3 require "bundler/setup"
      4 require "uri"
      5 require "digest"
      6 require "time"
      7 require "optparse"
      8 require "nokogiri"
      9 require "ferrum"
     10 
     11 def absolutize_url(url, base_url)
     12   return url if url.nil? || url.empty?
     13   return url if url.match?(/\A(?:data|javascript|mailto|tel|about):/i)
     14 
     15   URI.join(base_url, url).to_s
     16 rescue URI::InvalidURIError
     17   url
     18 end
     19 
     20 def absolutize_srcset(srcset, base_url)
     21   srcset.split(",").map do |entry|
     22     parts = entry.strip.split(/\s+/, 2)
     23     next if parts.empty?
     24 
     25     src = absolutize_url(parts[0], base_url)
     26     descriptor = parts[1]
     27 
     28     [src, descriptor].compact.join(" ")
     29   end.compact.join(", ")
     30 end
     31 
     32 def absolutize_links!(doc, base_url)
     33   %w[href src poster].each do |attr|
     34     doc.css("[#{attr}]").each do |node|
     35       node[attr] = absolutize_url(node[attr].to_s.strip, base_url)
     36     end
     37   end
     38 
     39   doc.css("[srcset]").each do |node|
     40     node["srcset"] = absolutize_srcset(node["srcset"].to_s.strip, base_url)
     41   end
     42 end
     43 
     44 def text_from_html(html)
     45   IO.popen(["w3m", "-dump", "-T", "text/html", "-o", "display_link_number=1"], "r+") do |io|
     46     io.write(html)
     47     io.close_write
     48     io.read
     49   end
     50 end
     51 
     52 options = {
     53   manual: false,
     54   browser_path: nil
     55 }
     56 
     57 OptionParser.new do |parser|
     58   parser.banner = "Usage: bin/archive [--manual] [--browser-path PATH] URL [URL ...]"
     59 
     60   parser.on("--manual", "Open a visible browser window so you can complete anti-bot challenges") do
     61     options[:manual] = true
     62   end
     63 
     64   parser.on("--browser-path PATH", "Path to the browser binary to launch") do |path|
     65     options[:browser_path] = path
     66   end
     67 end.parse!
     68 
     69 urls = ARGV
     70 clipboard = ""
     71 
     72 unless urls.any?
     73   warn "Please supply one or more URLs"
     74   exit 1
     75 end
     76 
     77 puts "references:"
     78 
     79 if options[:manual] && options[:browser_path].nil?
     80   brave_path = "/Applications/Brave Browser.app/Contents/MacOS/Brave Browser"
     81   options[:browser_path] = brave_path if File.exist?(brave_path)
     82 end
     83 
     84 browser = Ferrum::Browser.new(
     85   headless: !options[:manual],
     86   timeout: 30,
     87   process_timeout: options[:manual] ? 60 : 30,
     88   browser_path: options[:browser_path],
     89   browser_options: { "no-sandbox": nil }
     90 )
     91 page = browser.create_page
     92 
     93 begin
     94   urls.each do |url|
     95     begin
     96       page.goto(url)
     97 
     98       if options[:manual]
     99         warn "Manual mode: finish any challenge in the browser window for #{url}"
    100         warn "Press Enter here once the page is fully loaded."
    101         $stdin.gets
    102         warn "Capturing page..."
    103       else
    104         page.network.wait_for_idle(timeout: 10)
    105       end
    106 
    107       html = page.body.encode("UTF-8", invalid: :replace, undef: :replace, replace: "")
    108       doc = Nokogiri::HTML(html)
    109       absolutize_links!(doc, url)
    110 
    111       title = doc.at("title")&.text&.strip
    112       raise "No title found" if title.to_s.empty?
    113 
    114       text_content = text_from_html(doc.to_html)
    115     rescue => ex
    116       warn "Archive error (#{ex}; #{url})"
    117       exit 1
    118     end
    119 
    120     hash = Digest::MD5.base64digest(url + text_content)
    121       .scan(/[a-z0-9]/i)
    122       .first(6)
    123       .join
    124       .downcase
    125 
    126     filename = "#{URI.parse(url).host.gsub(".", "-")}-#{hash}.txt"
    127 
    128     File.write("static/archive/#{filename}", text_content)
    129 
    130     yaml = <<~STR
    131       - title: "#{title}"
    132         url: #{url}
    133         date: #{Time.now.utc.iso8601}
    134         file: #{filename}
    135     STR
    136 
    137     puts yaml
    138 
    139     clipboard += yaml
    140   end
    141 ensure
    142   browser.quit
    143 end
    144 
    145 IO.popen("pbcopy", "w") { |pb| pb.write(clipboard) }