this repo has no description
New feature: track history of various counts, closes #40
+55
+55
twitter_to_sqlite/utils.py
+55
twitter_to_sqlite/utils.py
···
21
21
"mentions": 3,
22
22
"search": 4,
23
23
}
24
+
COUNT_HISTORY_TYPES = {
25
+
"followers": 1,
26
+
"friends": 2,
27
+
"listed": 3,
28
+
# Don't track these - they're uninteresting and really noisy in terms
29
+
# of writing new rows to the count_history table:
30
+
# "favourites": 4,
31
+
# "statuses": 5,
32
+
}
24
33
25
34
source_re = re.compile('<a href="(?P<url>.*?)".*?>(?P<name>.*?)</a>')
26
35
···
340
349
foreign_keys=(("type", "since_id_types", "id"),),
341
350
)
342
351
352
+
# Tables for recording history of user follower counts etc
353
+
if "count_history" not in table_names:
354
+
db["count_history_types"].create({"id": int, "name": str,}, pk="id")
355
+
db["count_history_types"].insert_all(
356
+
[{"id": id, "name": name} for name, id in COUNT_HISTORY_TYPES.items()]
357
+
)
358
+
db["count_history"].create(
359
+
{"type": int, "user": int, "datetime": str, "count": int},
360
+
pk=("type", "user", "datetime"),
361
+
foreign_keys=(
362
+
("type", "count_history_types", "id"),
363
+
("user", "users", "id"),
364
+
),
365
+
)
366
+
343
367
344
368
def save_tweets(db, tweets, favorited_by=None):
345
369
ensure_tables(db)
···
363
387
if nested:
364
388
save_tweets(db, nested)
365
389
db["users"].insert(user, pk="id", alter=True, replace=True)
390
+
save_user_counts(db, user)
366
391
table = db["tweets"].insert(tweet, pk="id", alter=True, replace=True)
367
392
if favorited_by is not None:
368
393
db["favorited_by"].insert(
···
384
409
for user in users:
385
410
transform_user(user)
386
411
db["users"].insert_all(users, pk="id", alter=True, replace=True)
412
+
for user in users:
413
+
save_user_counts(db, user)
387
414
if followed_id or follower_id:
388
415
first_seen = datetime.datetime.utcnow().isoformat()
389
416
db["following"].insert_all(
···
595
622
m = source_re.match(source)
596
623
details = m.groupdict()
597
624
return db["sources"].insert(details, hash_id="id", replace=True).last_pk
625
+
626
+
627
+
def save_user_counts(db, user):
628
+
for type_name, type_id in COUNT_HISTORY_TYPES.items():
629
+
previous_count = None
630
+
try:
631
+
previous_count = db.conn.execute(
632
+
"""
633
+
select count from count_history
634
+
where type = ? and user = ?
635
+
order by datetime desc limit 1
636
+
""",
637
+
[type_id, user["id"]],
638
+
).fetchall()[0][0]
639
+
except IndexError:
640
+
pass
641
+
current_count = user["{}_count".format(type_name)]
642
+
if current_count != previous_count:
643
+
db["count_history"].insert(
644
+
{
645
+
"type": type_id,
646
+
"user": user["id"],
647
+
"datetime": datetime.datetime.utcnow().isoformat().split(".")[0]
648
+
+ "+00:00",
649
+
"count": current_count,
650
+
},
651
+
replace=True,
652
+
)