this repo has no description
feeds: add timestamp parsing, use in rapidfire.py
+41
-2
+39
feeds/__init__.py
+39
feeds/__init__.py
···
1
+
from datetime import datetime, timezone, timedelta
2
+
1
3
class BaseFeed:
2
4
def process_commit(self, commit):
3
5
raise NotImplementedError
···
7
9
8
10
def run_tasks_minute(self):
9
11
pass
12
+
13
+
def parse_timestamp(self, timestamp):
14
+
# https://atproto.com/specs/lexicon#datetime
15
+
formats = {
16
+
# preferred
17
+
'1985-04-12T23:20:50.123Z': '%Y-%m-%dT%H:%M:%S.%f%z',
18
+
# '1985-04-12T23:20:50.123456Z': '%Y-%m-%dT%H:%M:%S.%f%z',
19
+
# '1985-04-12T23:20:50.120Z': '%Y-%m-%dT%H:%M:%S.%f%z',
20
+
# '1985-04-12T23:20:50.120000Z': '%Y-%m-%dT%H:%M:%S.%f%z',
21
+
22
+
# supported
23
+
# '1985-04-12T23:20:50.12345678912345Z': '',
24
+
'1985-04-12T23:20:50Z': '%Y-%m-%dT%H:%M:%S%z',
25
+
# '1985-04-12T23:20:50.0Z': '%Y-%m-%dT%H:%M:%S.%f%z',
26
+
# '1985-04-12T23:20:50.123+00:00': '%Y-%m-%dT%H:%M:%S.%f%z',
27
+
# '1985-04-12T23:20:50.123-07:00': '%Y-%m-%dT%H:%M:%S.%f%z',
28
+
}
29
+
30
+
for format in formats.values():
31
+
try:
32
+
ts = datetime.strptime(timestamp, format)
33
+
except ValueError
34
+
continue
35
+
else:
36
+
return ts
37
+
38
+
return datetime.now(timezone.utc)
39
+
40
+
def safe_timestamp(self, timestamp):
41
+
parsed = self.parse_timestamp(timestamp)
42
+
utc_now = datetime.now(timezone.utc)
43
+
if parsed.timestamp() <= 0:
44
+
return utc_now
45
+
elif parsed - timedelta(minutes=2) < utc_now:
46
+
return parsed
47
+
elif parsed > utc_now:
48
+
return utc_now
10
49
11
50
class FeedManager:
12
51
def __init__(self):
+2
-2
feeds/rapidfire.py
+2
-2
feeds/rapidfire.py
···
50
50
repo = commit['repo']
51
51
path = op['path']
52
52
post_uri = f'at://{repo}/{path}'
53
-
ts = record['createdAt']
53
+
ts = self.safe_timestamp(record['createdAt']).timestamp()
54
54
55
55
with self.db_cnx:
56
56
langs = record.get('langs') or ['']
···
65
65
66
66
with self.db_cnx:
67
67
self.db_cnx.execute(
68
-
"delete from posts where strftime('%s', create_ts) < strftime('%s', 'now', '-15 minutes')"
68
+
"delete from posts where create_ts < unixepoch('now', '-15 minutes')"
69
69
)
70
70
71
71
self.db_cnx.pragma('wal_checkpoint(TRUNCATE)')