this repo has no description
0
fork

Configure Feed

Select the types of activity you want to include in your feed.

friends command, refs #31

+49 -16
+40 -8
twitter_to_sqlite/cli.py
··· 101 101 @click.option("--silent", is_flag=True, help="Disable progress bar") 102 102 def followers(db_path, auth, user_id, screen_name, silent): 103 103 "Save followers for specified user (defaults to authenticated user)" 104 + _shared_friends_followers(db_path, auth, user_id, screen_name, silent, "followers") 105 + 106 + 107 + def _shared_friends_followers(db_path, auth, user_id, screen_name, silent, noun): 108 + assert noun in ("friends", "followers") 104 109 auth = json.load(open(auth)) 105 110 session = utils.session_for_auth(auth) 106 111 db = utils.open_database(db_path) ··· 112 117 screen_name = profile["screen_name"] 113 118 user_id = profile["id"] 114 119 120 + save_users_kwargs = {} 121 + if noun == "followers": 122 + save_users_kwargs["followed_id"] = user_id 123 + elif noun == "friends": 124 + save_users_kwargs["follower_id"] = user_id 125 + 115 126 def go(update): 116 - for followers_chunk in utils.fetch_follower_chunks( 117 - session, user_id, screen_name 127 + for users_chunk in utils.fetch_user_list_chunks( 128 + session, user_id, screen_name, noun=noun 118 129 ): 119 - fetched.extend(followers_chunk) 120 - utils.save_users(db, followers_chunk, followed_id=user_id) 121 - update(len(followers_chunk)) 130 + fetched.extend(users_chunk) 131 + utils.save_users(db, users_chunk, **save_users_kwargs) 132 + update(len(users_chunk)) 122 133 123 134 if not silent: 124 - count = profile["followers_count"] 135 + count = profile["{}_count".format(noun)] 125 136 with click.progressbar( 126 137 length=count, 127 - label="Importing {:,} followers for @{}".format(count, screen_name), 138 + label="Importing {:,} {}s for @{}".format(count, noun, screen_name), 128 139 ) as bar: 129 140 go(bar.update) 130 141 else: 131 142 go(lambda x: None) 132 - # open("/tmp/all.json", "w").write(json.dumps(fetched, indent=4)) 143 + 144 + 145 + @cli.command() 146 + @click.argument( 147 + "db_path", 148 + type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), 149 + required=True, 150 + ) 151 + @click.option( 152 + "-a", 153 + "--auth", 154 + type=click.Path(file_okay=True, dir_okay=False, allow_dash=True, exists=True), 155 + default="auth.json", 156 + help="Path to auth.json token file", 157 + ) 158 + @click.option("--user_id", help="Numeric user ID") 159 + @click.option("--screen_name", help="Screen name") 160 + @click.option("--silent", is_flag=True, help="Disable progress bar") 161 + def friends(db_path, auth, user_id, screen_name, silent): 162 + "Save friends for specified user (defaults to authenticated user)" 163 + _shared_friends_followers(db_path, auth, user_id, screen_name, silent, "friends") 164 + 133 165 134 166 135 167 @cli.command()
+9 -8
twitter_to_sqlite/utils.py
··· 52 52 ) 53 53 54 54 55 - def fetch_follower_chunks(session, user_id, screen_name, sleep=61): 55 + def fetch_user_list_chunks(session, user_id, screen_name, sleep=61, noun="followers"): 56 56 cursor = -1 57 57 users = [] 58 58 while cursor: 59 - headers, body = fetch_followers(session, cursor, user_id, screen_name) 59 + headers, body = fetch_user_list(session, cursor, user_id, screen_name, noun) 60 60 yield body["users"] 61 61 cursor = body["next_cursor"] 62 62 if not cursor: ··· 64 64 time.sleep(sleep) # Rate limit = 15 per 15 minutes! 65 65 66 66 67 - def fetch_followers(session, cursor, user_id, screen_name): 67 + def fetch_user_list(session, cursor, user_id, screen_name, noun="followers"): 68 68 args = user_args(user_id, screen_name) 69 69 args.update({"count": 200, "cursor": cursor}) 70 70 r = session.get( 71 - "https://api.twitter.com/1.1/followers/list.json?" 71 + "https://api.twitter.com/1.1/{}/list.json?".format(noun) 72 72 + urllib.parse.urlencode(args) 73 73 ) 74 74 return r.headers, r.json() ··· 301 301 table.m2m("media", media, pk="id") 302 302 303 303 304 - def save_users(db, users, followed_id=None): 304 + def save_users(db, users, followed_id=None, follower_id=None): 305 + assert not (followed_id and follower_id) 305 306 ensure_tables(db) 306 307 for user in users: 307 308 transform_user(user) 308 309 db["users"].upsert_all(users, pk="id", alter=True) 309 - if followed_id: 310 + if followed_id or follower_id: 310 311 first_seen = datetime.datetime.utcnow().isoformat() 311 312 db["following"].insert_all( 312 313 ( 313 314 { 314 - "followed_id": followed_id, 315 - "follower_id": user["id"], 315 + "followed_id": followed_id or user["id"], 316 + "follower_id": follower_id or user["id"], 316 317 "first_seen": first_seen, 317 318 } 318 319 for user in users