this repo has no description
friends command, refs #31
+49
-16
+40
-8
twitter_to_sqlite/cli.py
+40
-8
twitter_to_sqlite/cli.py
···
101
101
@click.option("--silent", is_flag=True, help="Disable progress bar")
102
102
def followers(db_path, auth, user_id, screen_name, silent):
103
103
"Save followers for specified user (defaults to authenticated user)"
104
+
_shared_friends_followers(db_path, auth, user_id, screen_name, silent, "followers")
105
+
106
+
107
+
def _shared_friends_followers(db_path, auth, user_id, screen_name, silent, noun):
108
+
assert noun in ("friends", "followers")
104
109
auth = json.load(open(auth))
105
110
session = utils.session_for_auth(auth)
106
111
db = utils.open_database(db_path)
···
112
117
screen_name = profile["screen_name"]
113
118
user_id = profile["id"]
114
119
120
+
save_users_kwargs = {}
121
+
if noun == "followers":
122
+
save_users_kwargs["followed_id"] = user_id
123
+
elif noun == "friends":
124
+
save_users_kwargs["follower_id"] = user_id
125
+
115
126
def go(update):
116
-
for followers_chunk in utils.fetch_follower_chunks(
117
-
session, user_id, screen_name
127
+
for users_chunk in utils.fetch_user_list_chunks(
128
+
session, user_id, screen_name, noun=noun
118
129
):
119
-
fetched.extend(followers_chunk)
120
-
utils.save_users(db, followers_chunk, followed_id=user_id)
121
-
update(len(followers_chunk))
130
+
fetched.extend(users_chunk)
131
+
utils.save_users(db, users_chunk, **save_users_kwargs)
132
+
update(len(users_chunk))
122
133
123
134
if not silent:
124
-
count = profile["followers_count"]
135
+
count = profile["{}_count".format(noun)]
125
136
with click.progressbar(
126
137
length=count,
127
-
label="Importing {:,} followers for @{}".format(count, screen_name),
138
+
label="Importing {:,} {}s for @{}".format(count, noun, screen_name),
128
139
) as bar:
129
140
go(bar.update)
130
141
else:
131
142
go(lambda x: None)
132
-
# open("/tmp/all.json", "w").write(json.dumps(fetched, indent=4))
143
+
144
+
145
+
@cli.command()
146
+
@click.argument(
147
+
"db_path",
148
+
type=click.Path(file_okay=True, dir_okay=False, allow_dash=False),
149
+
required=True,
150
+
)
151
+
@click.option(
152
+
"-a",
153
+
"--auth",
154
+
type=click.Path(file_okay=True, dir_okay=False, allow_dash=True, exists=True),
155
+
default="auth.json",
156
+
help="Path to auth.json token file",
157
+
)
158
+
@click.option("--user_id", help="Numeric user ID")
159
+
@click.option("--screen_name", help="Screen name")
160
+
@click.option("--silent", is_flag=True, help="Disable progress bar")
161
+
def friends(db_path, auth, user_id, screen_name, silent):
162
+
"Save friends for specified user (defaults to authenticated user)"
163
+
_shared_friends_followers(db_path, auth, user_id, screen_name, silent, "friends")
164
+
133
165
134
166
135
167
@cli.command()
+9
-8
twitter_to_sqlite/utils.py
+9
-8
twitter_to_sqlite/utils.py
···
52
52
)
53
53
54
54
55
-
def fetch_follower_chunks(session, user_id, screen_name, sleep=61):
55
+
def fetch_user_list_chunks(session, user_id, screen_name, sleep=61, noun="followers"):
56
56
cursor = -1
57
57
users = []
58
58
while cursor:
59
-
headers, body = fetch_followers(session, cursor, user_id, screen_name)
59
+
headers, body = fetch_user_list(session, cursor, user_id, screen_name, noun)
60
60
yield body["users"]
61
61
cursor = body["next_cursor"]
62
62
if not cursor:
···
64
64
time.sleep(sleep) # Rate limit = 15 per 15 minutes!
65
65
66
66
67
-
def fetch_followers(session, cursor, user_id, screen_name):
67
+
def fetch_user_list(session, cursor, user_id, screen_name, noun="followers"):
68
68
args = user_args(user_id, screen_name)
69
69
args.update({"count": 200, "cursor": cursor})
70
70
r = session.get(
71
-
"https://api.twitter.com/1.1/followers/list.json?"
71
+
"https://api.twitter.com/1.1/{}/list.json?".format(noun)
72
72
+ urllib.parse.urlencode(args)
73
73
)
74
74
return r.headers, r.json()
···
301
301
table.m2m("media", media, pk="id")
302
302
303
303
304
-
def save_users(db, users, followed_id=None):
304
+
def save_users(db, users, followed_id=None, follower_id=None):
305
+
assert not (followed_id and follower_id)
305
306
ensure_tables(db)
306
307
for user in users:
307
308
transform_user(user)
308
309
db["users"].upsert_all(users, pk="id", alter=True)
309
-
if followed_id:
310
+
if followed_id or follower_id:
310
311
first_seen = datetime.datetime.utcnow().isoformat()
311
312
db["following"].insert_all(
312
313
(
313
314
{
314
-
"followed_id": followed_id,
315
-
"follower_id": user["id"],
315
+
"followed_id": followed_id or user["id"],
316
+
"follower_id": follower_id or user["id"],
316
317
"first_seen": first_seen,
317
318
}
318
319
for user in users