Don't forget to lycansubscribe 🐺
process items from the firehose in the worker
+32
-8
+24
-6
app/import_worker.rb
+24
-6
app/import_worker.rb
···
3
3
require_relative 'init'
4
4
require_relative 'import_manager'
5
5
require_relative 'models/import'
6
+
require_relative 'post_downloader'
6
7
require_relative 'reports/basic_report'
7
8
8
9
class ImportWorker
···
30
31
def run
31
32
@user_threads = []
32
33
34
+
@firehose_thread = Thread.new { process_firehose_items }
35
+
@downloader = PostDownloader.new
36
+
33
37
loop do
34
38
@user_threads.delete_if { |t| !t.alive? }
35
39
36
-
ActiveRecord::Base.transaction do
37
-
if user = User.with_unfinished_imports.where.not(id: @user_threads.map(&:user_id)).first
38
-
collections = user.imports.unfinished.map(&:collection)
39
-
thread = UserThread.new(user, collections, @verbose)
40
-
@user_threads << thread
41
-
end
40
+
if user = User.with_unfinished_imports.where.not(id: @user_threads.map(&:user_id)).first
41
+
collections = user.imports.unfinished.map(&:collection)
42
+
thread = UserThread.new(user, collections, @verbose)
43
+
@user_threads << thread
42
44
end
43
45
44
46
# possible future enhancement: use LISTEN/UNLISTEN/NOTIFY and wait_for_notify
45
47
sleep 5
48
+
end
49
+
end
50
+
51
+
def process_firehose_items
52
+
loop do
53
+
items = [Like, Repost, Pin, Quote]
54
+
.map { |type| type.in_queue(:firehose).order('time').limit(25) }
55
+
.flatten
56
+
.sort_by(&:time)
57
+
.first(25)
58
+
59
+
if items.length > 0
60
+
@downloader.process_items(items)
61
+
else
62
+
sleep 5
63
+
end
46
64
end
47
65
end
48
66
end
+8
-2
app/post_downloader.rb
+8
-2
app/post_downloader.rb
···
31
31
32
32
@report&.update(queue: { length: queue.length })
33
33
34
+
process_items(items)
35
+
end
36
+
end
37
+
38
+
def process_items(items)
39
+
#
34
40
existing_posts = Post.where(rkey: items.map { |x| AT_URI(x.post_uri).rkey }).to_a
35
41
36
42
items.dup.each do |item|
···
40
46
end
41
47
end
42
48
43
-
next if items.empty?
49
+
return if items.empty?
44
50
45
51
begin
46
52
response = @sky.get_request('app.bsky.feed.getPosts', { uris: items.map(&:post_uri).uniq })
···
67
73
rescue StandardError => e
68
74
puts "Error in PostDownloader: #{e.class}: #{e}"
69
75
end
70
-
end
76
+
#
71
77
end
72
78
73
79
def save_post(post_uri, record)