commit b7b58cf3abd2702a625811ae71ef82f47272605a parent f5e5481eb6c9fdd1855667afad451c2788e24072 Author: David Eisinger <[email protected]> Date: Sat, 29 Apr 2023 20:47:05 -0400 Move web archives into dedicated directory Diffstat:
9 files changed, 48 insertions(+), 48 deletions(-)
diff --git a/bin/archive b/bin/archive @@ -0,0 +1,47 @@ +#!/usr/bin/env ruby + +require "uri" +require "digest" +require "time" + +*urls = ARGV + +unless urls.any? + warn "Please supply one or more URLs" + exit 1 +end + +puts "references:" + +urls.each do |url| + page_content = `curl -s #{url}` + text_content = `lynx -dump --display_charset=utf-8 #{url}` + + begin + title = page_content + .scan(/<title[^>]*>(.*?)<\/title>/mi) + .first + .first + .strip + rescue => ex + warn "Title error (#{ex}; #{url})" + exit 1 + end + + hash = Digest::MD5.base64digest(url + text_content) + .scan(/[a-z0-9]/i) + .first(6) + .join + .downcase + + filename = "#{URI.parse(url).host.gsub(".", "-")}-#{hash}.txt" + + File.write("static/archive/#{filename}", text_content) + + puts <<~STR + - title: "#{title}" + url: #{url} + date: #{Time.now.utc.iso8601} + file: #{filename} + STR +end diff --git a/bin/backup b/bin/backup @@ -1,47 +0,0 @@ -#!/usr/bin/env ruby - -require "uri" -require "digest" -require "time" - -path, *urls = ARGV - -unless path && urls.any? - warn "Please supply a path and url" - exit 1 -end - -puts "references:" - -urls.each do |url| - page_content = `curl -s #{url}` - text_content = `lynx -dump --display_charset=utf-8 #{url}` - - begin - title = page_content - .scan(/<title[^>]*>(.*?)<\/title>/mi) - .first - .first - .strip - rescue => ex - warn "Title error (#{ex}; #{url})" - exit 1 - end - - hash = Digest::MD5.base64digest(url + text_content) - .scan(/[a-z0-9]/i) - .first(6) - .join - .downcase - - filename = "#{URI.parse(url).host.gsub(".", "-")}-#{hash}.txt" - - File.write("#{path}/#{filename}", text_content) - - puts <<~STR - - title: "#{title}" - url: #{url} - date: #{Time.now.utc.iso8601} - file: #{filename} - STR -end diff --git a/content/journal/dispatch-2-april-2023/blog-testdouble-com-ft5qbw.txt b/static/archive/blog-testdouble-com-ft5qbw.txt diff --git a/content/journal/dispatch-1-march-2023/craigmod-com-xfi1ge.txt b/static/archive/craigmod-com-xfi1ge.txt diff --git a/content/journal/dispatch-2-april-2023/jeffhuang-com-njdbjn.txt b/static/archive/jeffhuang-com-njdbjn.txt diff --git a/content/journal/dispatch-1-march-2023/www-baldurbjarnason-com-uaaq9d.txt b/static/archive/www-baldurbjarnason-com-uaaq9d.txt diff --git a/content/journal/dispatch-1-march-2023/www-nytimes-com-gvijqf.txt b/static/archive/www-nytimes-com-gvijqf.txt diff --git a/content/journal/dispatch-2-april-2023/www-viget-com-97ckk4.txt b/static/archive/www-viget-com-97ckk4.txt diff --git a/themes/v2/layouts/journal/single.html b/themes/v2/layouts/journal/single.html @@ -27,7 +27,7 @@ {{ range . }} <li> “<a href="{{ .url }}">{{ .title }}</a>”; - <a href="{{ .file }}"> + <a href="/archive/{{ .file }}"> backed up {{ .date | time.Format "2006-01-02 15:04:05 MST" }} </a> </li>