this repo has no description
Improved --since= for mentions-timeline, refs #39
+16
-12
+15
-12
twitter_to_sqlite/cli.py
+15
-12
twitter_to_sqlite/cli.py
···
389
389
table="mentions_tweets",
390
390
api_url="https://api.twitter.com/1.1/statuses/mentions_timeline.json",
391
391
sleep=10,
392
+
since_type="mentions",
392
393
)
393
394
394
395
395
-
def _shared_timeline(db_path, auth, since, since_id, table, api_url, sleep=1):
396
+
def _shared_timeline(
397
+
db_path, auth, since, since_id, table, api_url, sleep=1, since_type=None
398
+
):
396
399
auth = json.load(open(auth))
397
400
session = utils.session_for_auth(auth)
398
401
db = utils.open_database(db_path)
399
402
profile = utils.get_profile(db, session)
400
403
expected_length = 800
401
-
if since and db[table].exists:
402
-
# Set since_id to highest value for this timeline
403
-
try:
404
-
since_id = db.conn.execute(
405
-
"select max(tweet) from {} where user = ?".format(table),
406
-
[profile["id"]],
407
-
).fetchall()[0][0]
408
-
expected_length = None
409
-
except IndexError:
410
-
pass
404
+
since_key = profile["id"]
411
405
412
406
with click.progressbar(
413
-
utils.fetch_timeline(session, api_url, db, sleep=sleep, since_id=since_id),
407
+
utils.fetch_timeline(
408
+
session,
409
+
api_url,
410
+
db,
411
+
sleep=sleep,
412
+
since=since,
413
+
since_id=since_id,
414
+
since_type=since_type,
415
+
since_key=since_key,
416
+
),
414
417
length=expected_length,
415
418
label="Importing tweets",
416
419
show_pos=True,
+1
twitter_to_sqlite/utils.py
+1
twitter_to_sqlite/utils.py
···
175
175
max_seen_id = max(t["id"] for t in tweets)
176
176
if last_since_id is not None:
177
177
max_seen_id = max((last_since_id, max_seen_id))
178
+
last_since_id = max_seen_id
178
179
db["since_ids"].insert(
179
180
{"type": since_type_id, "key": since_key, "since_id": max_seen_id,},
180
181
replace=True,