this repo has no description
feeds/rapidfire.py: update for new framework
+24
-22
+1
-1
feedgen.py
+1
-1
feedgen.py
+22
-20
feeds/rapidfire.py
+22
-20
feeds/rapidfire.py
···
1
1
import os
2
2
import sys
3
-
import sqlite3
3
+
import apsw
4
4
5
5
from . import BaseFeed
6
6
···
10
10
FEED_URI = 'at://did:plc:4nsduwlpivpuur4mqkbfvm6a/app.bsky.feed.generator/rapidfire'
11
11
12
12
def __init__(self):
13
+
db_fname = ''
13
14
if os.path.isdir('/dev/shm/'):
14
15
os.makedirs('/dev/shm/feedgens/', exist_ok=True)
15
-
self.db_cnx = sqlite3.connect('/dev/shm/feedgens/rapidfire.db')
16
+
db_fname = '/dev/shm/feedgens/rapidfire.db'
16
17
else:
17
-
self.db_cnx = sqlite3.connect('db/rapidfire.db')
18
+
db_fname = 'db/rapidfire.db'
19
+
20
+
self.db_cnx = apsw.Connection(db_fname)
21
+
self.db_cnx.pragma('journal_mode', 'WAL')
22
+
self.db_cnx.pragma('synchronous', 'OFF')
23
+
self.db_cnx.pragma('wal_autocheckpoint', '0')
18
24
19
25
with self.db_cnx:
20
-
self.db_cnx.executescript(
21
-
"pragma journal_mode = WAL;"
22
-
"pragma synchronous = OFF;"
23
-
"pragma wal_autocheckpoint = 0;"
24
-
"create table if not exists posts (uri text, create_ts timestamp, lang text);"
25
-
"create index if not exists create_ts_idx on posts(create_ts);"
26
-
)
27
-
28
-
self.checkpoint = 0
26
+
self.db_cnx.execute("""
27
+
create table if not exists posts (uri text, create_ts timestamp, lang text);
28
+
create index if not exists create_ts_idx on posts(create_ts);
29
+
""")
29
30
30
31
def process_commit(self, commit):
31
32
op = commit['op']
···
48
49
path = op['path']
49
50
post_uri = f'at://{repo}/{path}'
50
51
ts = record['createdAt']
51
-
self.checkpoint += 1
52
52
53
53
with self.db_cnx:
54
54
langs = record.get('langs') or ['']
···
58
58
dict(uri=post_uri, ts=ts, lang=lang)
59
59
)
60
60
61
-
self.db_cnx.execute(
62
-
"delete from posts where strftime('%s', create_ts) < strftime('%s', 'now', '-15 minutes')"
63
-
)
61
+
def run_tasks_minute(self):
62
+
sys.stdout.write('rapidfire: running minute tasks\n')
63
+
sys.stdout.flush()
64
+
65
+
with self.db_cnx:
66
+
self.db_cnx.execute(
67
+
"delete from posts where strftime('%s', create_ts) < strftime('%s', 'now', '-15 minutes')"
68
+
)
64
69
65
-
if self.checkpoint % 100 == 0:
66
-
sys.stdout.write('rapidfire: checkpoint\n')
67
-
sys.stdout.flush()
68
-
self.db_cnx.execute("pragma wal_checkpoint(TRUNCATE)")
70
+
self.db_cnx.pragma('wal_checkpoint(TRUNCATE)')
69
71
70
72
def serve_feed(self, limit, offset, langs):
71
73
if '*' in langs: