Don't forget to lycansubscribe 🐺
improved handling of invalid records (hi retr0id 👋)
+80
-28
+12
app/firehose_client.rb
+12
app/firehose_client.rb
···
173
173
elsif op.action == :delete
174
174
@current_user.likes.where(rkey: op.rkey).delete_all
175
175
end
176
+
rescue StandardError => e
177
+
log "Error in #process_like (#{msg.seq}, #{op.uri}): #{e}"
178
+
log e.backtrace.reject { |x| x.include?('/ruby/') }
179
+
sleep 5 if e.is_a?(ActiveRecord::ConnectionFailed)
176
180
end
177
181
178
182
def process_repost(msg, op)
···
184
188
elsif op.action == :delete
185
189
@current_user.reposts.where(rkey: op.rkey).delete_all
186
190
end
191
+
rescue StandardError => e
192
+
log "Error in #process_repost (#{msg.seq}, #{op.uri}): #{e}"
193
+
log e.backtrace.reject { |x| x.include?('/ruby/') }
194
+
sleep 5 if e.is_a?(ActiveRecord::ConnectionFailed)
187
195
end
188
196
189
197
def process_post(msg, op)
···
204
212
post.destroy
205
213
end
206
214
end
215
+
rescue StandardError => e
216
+
log "Error in #process_post (#{msg.seq}, #{op.uri}): #{e}"
217
+
log e.backtrace.reject { |x| x.include?('/ruby/') }
218
+
sleep 5 if e.is_a?(ActiveRecord::ConnectionFailed)
207
219
end
208
220
209
221
def log(text)
+8
app/importers/base_importer.rb
+8
app/importers/base_importer.rb
···
1
1
require 'didkit'
2
2
require 'minisky'
3
+
require 'time'
3
4
4
5
require_relative '../at_uri'
6
+
require_relative '../errors'
5
7
require_relative '../models/post'
6
8
require_relative '../models/user'
7
9
···
42
44
43
45
def import_items
44
46
raise NotImplementedError
47
+
end
48
+
49
+
def created_at(record)
50
+
Time.parse(record['createdAt'])
51
+
rescue StandardError
52
+
raise InvalidRecordError
45
53
end
46
54
end
+10
-7
app/importers/likes_importer.rb
+10
-7
app/importers/likes_importer.rb
···
1
-
require 'time'
2
1
require_relative 'base_importer'
3
2
4
3
class LikesImporter < BaseImporter
···
11
10
12
11
records = response['records']
13
12
cursor = response['cursor']
14
-
15
-
@imported_count += records.length
16
-
@report&.update(importers: { importer_name => { :imported_items => @imported_count }})
17
-
@report&.update(importers: { importer_name => { :oldest_date => Time.parse(records.last['value']['createdAt']) }}) unless records.empty?
13
+
oldest_date = nil
18
14
19
15
records.each do |record|
20
16
begin
21
17
like = @user.likes.import_from_record(record['uri'], record['value'], queue: :import)
18
+
19
+
record_date = like&.time || created_at(record['value'])
20
+
oldest_date = [oldest_date, record_date].compact.min
22
21
23
22
if like && like.pending? && @item_queue
24
23
@item_queue.push(like)
25
24
@report&.update(queue: { length: @item_queue.length })
26
25
end
27
-
rescue StandardError => e
26
+
rescue InvalidRecordError => e
28
27
puts "Error in LikesImporter: #{record['uri']}: #{e}"
29
28
end
30
29
end
31
30
31
+
@imported_count += records.length
32
+
@report&.update(importers: { importer_name => { :imported_items => @imported_count }})
33
+
@report&.update(importers: { importer_name => { :oldest_date => oldest_date }}) if oldest_date
34
+
32
35
params[:cursor] = cursor
33
36
@import.update!(cursor: cursor)
34
37
35
38
break if !cursor
36
-
break if @time_limit && records.any? { |x| Time.parse(x['value']['createdAt']) < @time_limit }
39
+
break if @time_limit && oldest_date && oldest_date < @time_limit
37
40
end
38
41
end
39
42
end
+10
-7
app/importers/posts_importer.rb
+10
-7
app/importers/posts_importer.rb
···
1
-
require 'time'
2
1
require_relative 'base_importer'
3
2
4
3
class PostsImporter < BaseImporter
···
11
10
12
11
records = response['records']
13
12
cursor = response['cursor']
14
-
15
-
@imported_count += records.length
16
-
@report&.update(importers: { importer_name => { :imported_items => @imported_count }})
17
-
@report&.update(importers: { importer_name => { :oldest_date => Time.parse(records.last['value']['createdAt']) }}) unless records.empty?
13
+
oldest_date = nil
18
14
19
15
records.each do |record|
20
16
begin
21
17
quote = @user.quotes.import_from_record(record['uri'], record['value'], queue: :import)
22
18
pin = @user.pins.import_from_record(record['uri'], record['value'], queue: :import)
19
+
20
+
record_date = quote&.time || pin&.time || created_at(record['value'])
21
+
oldest_date = [oldest_date, record_date].compact.min
23
22
24
23
if @item_queue
25
24
if quote && quote.pending?
···
32
31
33
32
@report&.update(queue: { length: @item_queue.length })
34
33
end
35
-
rescue StandardError => e
34
+
rescue InvalidRecordError => e
36
35
puts "Error in PostsImporter: #{record['uri']}: #{e}"
37
36
end
38
37
end
39
38
39
+
@imported_count += records.length
40
+
@report&.update(importers: { importer_name => { :imported_items => @imported_count }})
41
+
@report&.update(importers: { importer_name => { :oldest_date => oldest_date }}) if oldest_date
42
+
40
43
params[:cursor] = cursor
41
44
@import.update!(cursor: cursor)
42
45
43
46
break if !cursor
44
-
break if @time_limit && records.any? { |x| Time.parse(x['value']['createdAt']) < @time_limit }
47
+
break if @time_limit && oldest_date && oldest_date < @time_limit
45
48
end
46
49
end
47
50
end
+10
-7
app/importers/reposts_importer.rb
+10
-7
app/importers/reposts_importer.rb
···
1
-
require 'time'
2
1
require_relative 'base_importer'
3
2
4
3
class RepostsImporter < BaseImporter
···
11
10
12
11
records = response['records']
13
12
cursor = response['cursor']
14
-
15
-
@imported_count += records.length
16
-
@report&.update(importers: { importer_name => { :imported_items => @imported_count }})
17
-
@report&.update(importers: { importer_name => { :oldest_date => Time.parse(records.last['value']['createdAt']) }}) unless records.empty?
13
+
oldest_date = nil
18
14
19
15
records.each do |record|
20
16
begin
21
17
repost = @user.reposts.import_from_record(record['uri'], record['value'], queue: :import)
18
+
19
+
record_date = repost&.time || created_at(record['value'])
20
+
oldest_date = [oldest_date, record_date].compact.min
22
21
23
22
if repost && repost.pending? && @item_queue
24
23
@item_queue.push(repost)
25
24
@report&.update(queue: { length: @item_queue.length })
26
25
end
27
-
rescue StandardError => e
26
+
rescue InvalidRecordError => e
28
27
puts "Error in RepostsImporter: #{record['uri']}: #{e}"
29
28
end
30
29
end
31
30
31
+
@imported_count += records.length
32
+
@report&.update(importers: { importer_name => { :imported_items => @imported_count }})
33
+
@report&.update(importers: { importer_name => { :oldest_date => oldest_date }}) if oldest_date
34
+
32
35
params[:cursor] = cursor
33
36
@import.update!(cursor: cursor)
34
37
35
38
break if !cursor
36
-
break if @time_limit && records.any? { |x| Time.parse(x['value']['createdAt']) < @time_limit }
39
+
break if @time_limit && oldest_date && oldest_date < @time_limit
37
40
end
38
41
end
39
42
end
+9
-1
app/models/user_importable.rb
+9
-1
app/models/user_importable.rb
···
1
+
require_relative '../errors'
2
+
1
3
module UserImportable
2
4
def import_from_record(item_uri, record, **args)
3
-
item = self.new_from_record(item_uri, record)
5
+
item = try_build_from_record(item_uri, record)
4
6
return nil if item.nil? || already_imported?(item)
5
7
6
8
item.import_item!(args)
9
+
end
10
+
11
+
def try_build_from_record(item_uri, record)
12
+
self.new_from_record(item_uri, record)
13
+
rescue StandardError
14
+
raise InvalidRecordError
7
15
end
8
16
9
17
def already_imported?(item)
+19
-6
app/post_downloader.rb
+19
-6
app/post_downloader.rb
···
63
63
puts "Invalid post #{item.post_uri}: #{post.errors.full_messages.join("; ")}"
64
64
invalidate_item(item)
65
65
end
66
-
rescue StandardError => e
66
+
rescue InvalidRecordError => e
67
67
puts "Error in PostDownloader: #{item.post_uri}: #{e.class}: #{e}"
68
+
69
+
item = items.detect { |x| x.post_uri == data['uri'] }
70
+
item.update!(queue: nil)
71
+
items.delete(item)
68
72
end
69
73
end
70
74
···
77
81
def save_post(post_uri, record)
78
82
did, _, rkey = AT_URI(post_uri)
79
83
80
-
text = record.delete('text')
81
-
created = record.delete('createdAt')
82
-
record.delete('$type')
83
-
84
84
author = User.find_or_create_by!(did: did)
85
85
86
86
if post = Post.find_by(user: author, rkey: rkey)
87
87
return post
88
+
else
89
+
post = build_post(author, rkey, record)
90
+
post.save
91
+
post
88
92
end
93
+
end
89
94
90
-
Post.create(
95
+
def build_post(author, rkey, record)
96
+
text = record.delete('text')
97
+
created = record.delete('createdAt')
98
+
99
+
record.delete('$type')
100
+
101
+
Post.new(
91
102
user: author,
92
103
rkey: rkey,
93
104
time: Time.parse(created),
94
105
text: text,
95
106
data: JSON.generate(record)
96
107
)
108
+
rescue StandardError
109
+
raise InvalidRecordError
97
110
end
98
111
99
112
def update_item(item, post)