quickly build a static timeline of your mastodon followers' latest posts so you can decide whether to follow them
1#!/usr/bin/env ruby
2#
3# Copyright (c) 2012-2024 joshua stein <jcs@jcs.org>
4#
5# Permission to use, copy, modify, and distribute this software for any
6# purpose with or without fee is hereby granted, provided that the above
7# copyright notice and this permission notice appear in all copies.
8#
9# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
10# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
11# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
12# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
13# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
14# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
15# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
16#
17
18require "cgi"
19require "uri"
20require "net/https"
21require "socket"
22require "ipaddr"
23require "securerandom"
24require "stringio"
25
26require "active_support/hash_with_indifferent_access"
27
28class CaseInsensitiveHash < HashWithIndifferentAccess
29 def [](key)
30 super convert_key(key)
31 end
32
33protected
34 def convert_key(key)
35 key.respond_to?(:downcase) ? key.downcase : key
36 end
37end
38
39module Net
40 class HTTP
41 attr_accessor :address, :custom_conn_address, :skip_close
42
43 def start # :yield: http
44 if block_given? && !skip_close
45 begin
46 do_start
47 return yield(self)
48 ensure
49 do_finish
50 end
51 end
52 do_start
53 self
54 end
55
56 private
57 def conn_address
58 if self.custom_conn_address.to_s != ""
59 self.custom_conn_address
60 else
61 address
62 end
63 end
64 end
65end
66
67class SpongeResponse
68 attr_reader :from_uri
69
70 def initialize(net_http_res, from_uri = nil)
71 @res = net_http_res
72 @from_uri = from_uri
73 end
74
75 def inspect
76 "<#{self.class} from #{self.from_uri.to_s}: status=#{self.status} " <<
77 "body=#{self.body ? self.body.to_s[0, 100] : nil}>"
78 end
79
80 def body
81 @res.body
82 end
83
84 def status
85 @res.code.to_i
86 end
87
88 def headers
89 return @headers if @headers
90
91 @headers = CaseInsensitiveHash.new(@res.to_hash)
92 @headers.each do |k,v|
93 @headers[k] = v[0]
94 end
95 @headers
96 end
97
98 def json
99 @json ||= JSON.parse(@res.body)
100 end
101
102 def ok?
103 (200 .. 299).include?(status)
104 end
105
106 def to_s
107 @res.body
108 end
109end
110
111class Sponge
112 MAX_TIME = 60
113 MAX_DNS_TIME = 10
114 MAX_KEEP_ALIVE_TIME = 30
115
116 @@KEEP_ALIVES = {}
117
118 attr_accessor :debug, :follow_redirection, :use_custom_resolver,
119 :keep_alive, :timeout, :use_private_keepalives, :resolve_cache,
120 :avoid_badnets, :local_ip, :user_agent
121
122 # rfc3330
123 BAD_NETS = [
124 "0.0.0.0/8",
125 "10.0.0.0/8",
126 "127.0.0.0/8",
127 "169.254.0.0/16",
128 "172.16.0.0/12",
129 "192.0.2.0/24",
130 "192.88.99.0/24",
131 "192.168.0.0/16",
132 "198.18.0.0/15",
133 "224.0.0.0/4",
134 "240.0.0.0/4"
135 ]
136
137 # old api
138 def self.fetch(uri, headers = {}, limit = 10)
139 s = Sponge.new
140 s.fetch(uri, "get", nil, nil, headers, {}, limit)
141 end
142
143 def initialize
144 @cookies = {}
145 @follow_redirection = true
146 @use_custom_resolver = true
147 @keep_alive = false
148 @timeout = MAX_TIME
149 @use_private_keepalives = false
150 @resolve_cache = {}
151 @local_ip = nil
152 @json = nil
153 @user_agent = "sponge/1.0"
154
155 @avoid_badnets = true
156 begin
157 if defined?(Rails) && Rails.env.development?
158 @avoid_badnets = false
159 end
160 rescue
161 end
162
163 @KEEP_ALIVES = {}
164 end
165
166 def close_stale_keep_alives
167 [ @KEEP_ALIVES, @@KEEP_ALIVES ].each do |ka|
168 ka.keys.each do |h|
169 if Time.now - ka[h][:last] > MAX_KEEP_ALIVE_TIME
170 begin
171 ka[h][:obj].finish
172 rescue IOError
173 end
174 ka.delete(h)
175 end
176 end
177 end
178 end
179
180 def find_keep_alive_for(host)
181 where = @@KEEP_ALIVES
182 if self.use_private_keepalives
183 where = @KEEP_ALIVES
184 end
185
186 if !where[host]
187 return nil
188 end
189
190 return where[host][:obj]
191 end
192
193 def save_keep_alive(host, obj)
194 where = @@KEEP_ALIVES
195 if self.use_private_keepalives
196 where = @KEEP_ALIVES
197 end
198
199 if obj == nil
200 if where[host]
201 begin
202 where[host][:obj].finish
203 rescue IOError
204 end
205 where.delete(host)
206 end
207 else
208 where[host] = { :last => Time.now, :obj => obj }
209 end
210 end
211
212 def set_cookie(from_host, cookie_line)
213 cookie = { "domain" => from_host }
214
215 cookie_line.split(/; ?/).each do |chunk|
216 pieces = chunk.split("=")
217
218 cookie[pieces[0]] = pieces[1]
219 if pieces[0].match(/^(path|domain|httponly)$/i)
220 cookie[pieces[0]] = pieces[1]
221 else
222 cookie["name"] = pieces[0]
223 cookie["value"] = pieces[1]
224 end
225 end
226
227 dputs "setting cookie #{cookie["name"]} on domain #{cookie["domain"]} " +
228 "to #{cookie["value"].inspect}"
229
230 if !@cookies[cookie["domain"]]
231 @cookies[cookie["domain"]] = {}
232 end
233
234 if cookie["value"].to_s == ""
235 @cookies[cookie["domain"]][cookie["name"]] ?
236 @cookies[cookie["domain"]][cookie["name"]].delete : nil
237 else
238 @cookies[cookie["domain"]][cookie["name"]] = cookie["value"]
239 end
240 end
241
242 def cookies(host)
243 cooks = @cookies[host] || {}
244
245 # check for domain cookies
246 @cookies.keys.each do |dom|
247 if dom.length < host.length &&
248 dom == host[host.length - dom.length .. host.length - 1]
249 dputs "adding domain keys from #{dom}"
250 cooks = cooks.merge @cookies[dom]
251 end
252 end
253
254 if cooks
255 return cooks.map{|k,v| "#{k}=#{v};" }.join(" ")
256 else
257 return ""
258 end
259 end
260
261 def fetch(uri, method = :get, fields = nil, raw_post_data = nil,
262 headers = {}, attachments = {}, limit = 10)
263 if limit <= 0
264 raise ArgumentError, "HTTP redirection too deep"
265 end
266
267 if !uri.is_a?(URI)
268 uri = URI.parse(uri)
269 end
270 host = nil
271 ip = nil
272 method = method.to_s.downcase.to_sym
273 @json = nil
274
275 if self.keep_alive && (host = self.find_keep_alive_for(uri.host))
276 dputs "using cached keep-alive connection to #{uri.host}"
277 else
278 if @use_custom_resolver
279 # we'll manually resolve the ip so we can verify it's not local
280 tip = nil
281 ips = @resolve_cache[uri.host]
282 if !ips || !ips.any?
283 begin
284 Timeout.timeout(MAX_DNS_TIME) do
285 ips = [ Addrinfo.ip(uri.host).ip_address ]
286
287 if !ips.any?
288 raise
289 end
290
291 @resolve_cache[uri.host] = ips
292 end
293 rescue Timeout::Error
294 raise "couldn't resolve #{uri.host} (DNS timeout)"
295 rescue SocketError, StandardError => e
296 raise "couldn't resolve #{uri.host} (#{e.inspect}) " <<
297 "(#{ips.inspect}) {#{tip.inspect})"
298 end
299 end
300
301 # pick a random one
302 tip = ips[rand(ips.length)]
303 ip = IPAddr.new(tip)
304
305 if !ip
306 raise "couldn't resolve #{uri.host}"
307 end
308
309 if @avoid_badnets &&
310 BAD_NETS.select{|n| IPAddr.new(n).include?(ip) }.any?
311 raise "refusing to talk to IP #{ip.to_s}"
312 end
313
314 host = Net::HTTP.new(ip.to_s, uri.port)
315
316 if uri.scheme == "https"
317 # openssl needs to know the hostname, so we'll override conn_address
318 # to connect to our ip
319 host.address = uri.host
320 host.custom_conn_address = ip.to_s
321 end
322 else
323 host = Net::HTTP.new(uri.host, uri.port)
324 end
325
326 if host.respond_to?(:local_host) && self.local_ip
327 host.local_host = self.local_ip
328 end
329
330 if self.debug
331 host.set_debug_output STDOUT
332 end
333
334 if uri.scheme == "https"
335 host.use_ssl = true
336 host.verify_mode = OpenSSL::SSL::VERIFY_NONE
337 end
338 end
339
340 # convert post params into query params for get requests
341 if method == :get
342 if raw_post_data
343 uri.query = URI.encode(raw_post_data)
344 if !headers["Content-Type"]
345 headers["Content-Type"] = "application/x-www-form-urlencoded"
346 end
347 elsif fields && fields.any?
348 uri.query = encode_fields(fields)
349 end
350 end
351
352 if method != :get
353 if raw_post_data && attachments.any?
354 raise "can't do raw POST data and attachments"
355 end
356
357 if attachments.any?
358 boundary = "----------#{SecureRandom.hex}"
359
360 headers["Content-Type"] = "multipart/form-data; boundary=#{boundary}"
361
362 post_data = fields.map{|k,v|
363 "--#{boundary}\r\n" +
364 "Content-Disposition: form-data; name=\"#{k}\"\r\n" +
365 "\r\n" +
366 v.to_s +
367 "\r\n"
368 }.join
369
370 post_data = post_data.force_encoding("binary")
371
372 attachments.each do |k,v|
373 if !v.is_a?(Hash)
374 raise "attachment #{k} is not a hash"
375 elsif !v.include?(:data)
376 raise "attachment #{k} has no :data"
377 end
378
379 post_data << ("--#{boundary}\r\n" <<
380 "Content-Disposition: form-data; name=\"#{k}\"; filename=\"" <<
381 "#{v[:filename]}\"\r\n" <<
382 "Content-Type: #{v[:content_type]}\r\n" <<
383 "\r\n").force_encoding("binary")
384
385 post_data << v[:data].force_encoding("binary")
386 post_data << "\r\n".force_encoding("binary")
387 end
388
389 post_data << ("--#{boundary}--\r\n").force_encoding("binary")
390
391 post_data = post_data.force_encoding("binary")
392 elsif raw_post_data
393 post_data = raw_post_data
394 if !headers["Content-Type"]
395 headers["Content-Type"] = "application/x-www-form-urlencoded"
396 end
397 elsif fields && fields.any?
398 post_data = encode_fields(fields)
399 else
400 post_data = ""
401 end
402
403 headers["Content-Length"] = post_data.bytesize.to_s
404 end
405
406 if uri.path.to_s == ""
407 uri.path = "/"
408 end
409
410 uri.path = uri.path.gsub(/^\/\/+/, "/")
411
412 cooks = cookies(uri.host).to_s
413
414 dputs "fetching #{uri} (#{ip.to_s}) " + (uri.user ? "with http auth " +
415 uri.user + "/" + ("*" * uri.password.length) + " " : "") +
416 "by #{method} with cookies #{cooks}" +
417 (attachments.any? ? " with #{attachments.length} attachment(s)" : "")
418
419 hs = {
420 "Host" => uri.host,
421 "User-Agent" => self.user_agent,
422 }
423
424 if cooks != ""
425 hs["Cookie"] = cooks
426 end
427
428 headers = hs.merge(headers || {})
429
430 if self.keep_alive
431 headers["Connection"] = "keep-alive"
432 host.skip_close = true
433 end
434
435 if uri.user
436 headers["Authorization"] = "Basic " +
437 ["#{uri.user}:#{uri.password}"].pack("m").delete("\r\n")
438 end
439
440 res = nil
441 begin
442 path = uri.path
443 if uri.query.to_s != ""
444 path += "?" + uri.query
445 end
446
447 Timeout.timeout(@timeout) do
448 req = case method
449 when :delete
450 Net::HTTP::Delete.new(path, headers)
451 when :get
452 Net::HTTP::Get.new(path, headers)
453 when :head
454 Net::HTTP::Head.new(path, headers)
455 when :options
456 Net::HTTP::Options.new(path, headers)
457 when :post
458 Net::HTTP::Post.new(path, headers)
459 when :put
460 Net::HTTP::Put.new(path, headers)
461 else
462 raise "unsupported method #{method}"
463 end
464
465 if post_data
466 req.body = post_data
467 end
468
469 res = host.request(req)
470 end
471 rescue EOFError, Errno::EBADF => e
472 if self.keep_alive && self.find_keep_alive_for(uri.host)
473 # tried to re-use a dead connection, retry again from the start
474 self.save_keep_alive(uri.host, nil)
475 dputs "got eof using dead keep-alive socket, retrying"
476 return fetch(uri, method, fields, raw_post_data, headers, attachments,
477 limit - 1)
478 else
479 raise e
480 end
481 end
482
483 if res.get_fields("Set-Cookie")
484 res.get_fields("Set-Cookie").each do |cook|
485 set_cookie(uri.host, cook)
486 end
487 end
488
489 if self.keep_alive
490 self.save_keep_alive(uri.host, host)
491 end
492
493 self.close_stale_keep_alives
494
495 case res
496 when Net::HTTPRedirection
497 if @follow_redirection
498 # follow
499 newuri = URI.parse(res["location"])
500 if newuri.host
501 dputs "following redirection to " + res["location"]
502 else
503 # relative path
504 newuri.host = uri.host
505 newuri.scheme = uri.scheme
506 newuri.port = uri.port
507 newuri.path = "/#{newuri.path}"
508
509 dputs "following relative redirection to " + newuri.to_s
510 end
511
512 fetch(newuri.to_s, "get", nil, nil, {}, {}, limit - 1)
513 else
514 dputs "not following redirection (disabled)"
515 return SpongeResponse.new(res, uri)
516 end
517 else
518 return SpongeResponse.new(res, uri)
519 end
520 end
521
522 def get(uri, params = {}, headers = {})
523 fetch(uri, :get, params, nil, headers)
524 end
525
526 def post(uri, fields, headers = {})
527 fetch(uri, :post, fields, nil, headers)
528 end
529
530private
531 def dputs(string)
532 if self.debug
533 puts string
534 end
535 end
536
537 def encode_fields(fields)
538 e = []
539 fields.each do |k,v|
540 if v.is_a?(Hash)
541 # :user => { :name => "hi", :age => "1" }
542 # becomes
543 # user[hame]=hi and user[age]=1
544 v.each do |vk,vv|
545 e.push "#{CGI.escape("#{k}[#{vk}]")}=#{CGI.escape(vv.to_s)}"
546 end
547 elsif v.is_a?(Array)
548 # :user => [ "one", "two" ]
549 # becomes
550 # user[]=one and user[]=two
551 v.each do |vv|
552 e.push "#{CGI.escape("#{k}[]")}=#{CGI.escape(vv.to_s)}"
553 end
554 else
555 e.push "#{CGI.escape(k.to_s)}=#{CGI.escape(v.to_s)}"
556 end
557 end
558
559 e.join("&")
560 end
561end