this repo has no description
0
fork

Configure Feed

Select the types of activity you want to include in your feed.

New feature: track history of various counts, closes #40

+55
+55
twitter_to_sqlite/utils.py
··· 21 21 "mentions": 3, 22 22 "search": 4, 23 23 } 24 + COUNT_HISTORY_TYPES = { 25 + "followers": 1, 26 + "friends": 2, 27 + "listed": 3, 28 + # Don't track these - they're uninteresting and really noisy in terms 29 + # of writing new rows to the count_history table: 30 + # "favourites": 4, 31 + # "statuses": 5, 32 + } 24 33 25 34 source_re = re.compile('<a href="(?P<url>.*?)".*?>(?P<name>.*?)</a>') 26 35 ··· 340 349 foreign_keys=(("type", "since_id_types", "id"),), 341 350 ) 342 351 352 + # Tables for recording history of user follower counts etc 353 + if "count_history" not in table_names: 354 + db["count_history_types"].create({"id": int, "name": str,}, pk="id") 355 + db["count_history_types"].insert_all( 356 + [{"id": id, "name": name} for name, id in COUNT_HISTORY_TYPES.items()] 357 + ) 358 + db["count_history"].create( 359 + {"type": int, "user": int, "datetime": str, "count": int}, 360 + pk=("type", "user", "datetime"), 361 + foreign_keys=( 362 + ("type", "count_history_types", "id"), 363 + ("user", "users", "id"), 364 + ), 365 + ) 366 + 343 367 344 368 def save_tweets(db, tweets, favorited_by=None): 345 369 ensure_tables(db) ··· 363 387 if nested: 364 388 save_tweets(db, nested) 365 389 db["users"].insert(user, pk="id", alter=True, replace=True) 390 + save_user_counts(db, user) 366 391 table = db["tweets"].insert(tweet, pk="id", alter=True, replace=True) 367 392 if favorited_by is not None: 368 393 db["favorited_by"].insert( ··· 384 409 for user in users: 385 410 transform_user(user) 386 411 db["users"].insert_all(users, pk="id", alter=True, replace=True) 412 + for user in users: 413 + save_user_counts(db, user) 387 414 if followed_id or follower_id: 388 415 first_seen = datetime.datetime.utcnow().isoformat() 389 416 db["following"].insert_all( ··· 595 622 m = source_re.match(source) 596 623 details = m.groupdict() 597 624 return db["sources"].insert(details, hash_id="id", replace=True).last_pk 625 + 626 + 627 + def save_user_counts(db, user): 628 + for type_name, type_id in COUNT_HISTORY_TYPES.items(): 629 + previous_count = None 630 + try: 631 + previous_count = db.conn.execute( 632 + """ 633 + select count from count_history 634 + where type = ? and user = ? 635 + order by datetime desc limit 1 636 + """, 637 + [type_id, user["id"]], 638 + ).fetchall()[0][0] 639 + except IndexError: 640 + pass 641 + current_count = user["{}_count".format(type_name)] 642 + if current_count != previous_count: 643 + db["count_history"].insert( 644 + { 645 + "type": type_id, 646 + "user": user["id"], 647 + "datetime": datetime.datetime.utcnow().isoformat().split(".")[0] 648 + + "+00:00", 649 + "count": current_count, 650 + }, 651 + replace=True, 652 + )