davideisinger.com

My personal website
Log | Files | Refs | README

archive (3361B)

      1 #!/usr/bin/env ruby
      2 
      3 require "uri"
      4 require "digest"
      5 require "time"
      6 require "optparse"
      7 require "nokogiri"
      8 require "ferrum"
      9 
     10 def absolutize_url(url, base_url)
     11   return url if url.nil? || url.empty?
     12   return url if url.match?(/\A(?:data|javascript|mailto|tel|about):/i)
     13 
     14   URI.join(base_url, url).to_s
     15 rescue URI::InvalidURIError
     16   url
     17 end
     18 
     19 def absolutize_srcset(srcset, base_url)
     20   srcset.split(",").map do |entry|
     21     parts = entry.strip.split(/\s+/, 2)
     22     next if parts.empty?
     23 
     24     src = absolutize_url(parts[0], base_url)
     25     descriptor = parts[1]
     26 
     27     [src, descriptor].compact.join(" ")
     28   end.compact.join(", ")
     29 end
     30 
     31 def absolutize_links!(doc, base_url)
     32   %w[href src poster].each do |attr|
     33     doc.css("[#{attr}]").each do |node|
     34       node[attr] = absolutize_url(node[attr].to_s.strip, base_url)
     35     end
     36   end
     37 
     38   doc.css("[srcset]").each do |node|
     39     node["srcset"] = absolutize_srcset(node["srcset"].to_s.strip, base_url)
     40   end
     41 end
     42 
     43 def text_from_html(html)
     44   IO.popen(["w3m", "-dump", "-T", "text/html", "-o", "display_link_number=1"], "r+") do |io|
     45     io.write(html)
     46     io.close_write
     47     io.read
     48   end
     49 end
     50 
     51 options = {
     52   manual: false,
     53   browser_path: nil
     54 }
     55 
     56 OptionParser.new do |parser|
     57   parser.banner = "Usage: bin/archive [--manual] [--browser-path PATH] URL [URL ...]"
     58 
     59   parser.on("--manual", "Open a visible browser window so you can complete anti-bot challenges") do
     60     options[:manual] = true
     61   end
     62 
     63   parser.on("--browser-path PATH", "Path to the browser binary to launch") do |path|
     64     options[:browser_path] = path
     65   end
     66 end.parse!
     67 
     68 urls = ARGV
     69 clipboard = ""
     70 
     71 unless urls.any?
     72   warn "Please supply one or more URLs"
     73   exit 1
     74 end
     75 
     76 puts "references:"
     77 
     78 if options[:manual] && options[:browser_path].nil?
     79   brave_path = "/Applications/Brave Browser.app/Contents/MacOS/Brave Browser"
     80   options[:browser_path] = brave_path if File.exist?(brave_path)
     81 end
     82 
     83 browser = Ferrum::Browser.new(
     84   headless: !options[:manual],
     85   timeout: 30,
     86   process_timeout: options[:manual] ? 60 : 30,
     87   browser_path: options[:browser_path],
     88   browser_options: { "no-sandbox": nil }
     89 )
     90 page = browser.create_page
     91 
     92 begin
     93   urls.each do |url|
     94     begin
     95       page.goto(url)
     96 
     97       if options[:manual]
     98         warn "Manual mode: finish any challenge in the browser window for #{url}"
     99         warn "Press Enter here once the page is fully loaded."
    100         $stdin.gets
    101         warn "Capturing page..."
    102       else
    103         page.network.wait_for_idle(timeout: 10)
    104       end
    105 
    106       html = page.body.encode("UTF-8", invalid: :replace, undef: :replace, replace: "")
    107       doc = Nokogiri::HTML(html)
    108       absolutize_links!(doc, url)
    109 
    110       title = doc.at("title")&.text&.strip
    111       raise "No title found" if title.to_s.empty?
    112 
    113       text_content = text_from_html(doc.to_html)
    114     rescue => ex
    115       warn "Archive error (#{ex}; #{url})"
    116       exit 1
    117     end
    118 
    119     hash = Digest::MD5.base64digest(url + text_content)
    120       .scan(/[a-z0-9]/i)
    121       .first(6)
    122       .join
    123       .downcase
    124 
    125     filename = "#{URI.parse(url).host.gsub(".", "-")}-#{hash}.txt"
    126 
    127     File.write("static/archive/#{filename}", text_content)
    128 
    129     yaml = <<~STR
    130       - title: "#{title}"
    131         url: #{url}
    132         date: #{Time.now.utc.iso8601}
    133         file: #{filename}
    134     STR
    135 
    136     puts yaml
    137 
    138     clipboard += yaml
    139   end
    140 ensure
    141   browser.quit
    142 end
    143 
    144 IO.popen("pbcopy", "w") { |pb| pb.write(clipboard) }