this repo has no description
user-timeline now takes --sql/--attach/--ids and multiple identifiers
Refs #35, refs #8. Still needs documentation.
+86
-34
+78
-30
twitter_to_sqlite/cli.py
+78
-30
twitter_to_sqlite/cli.py
···
199
199
type=click.Path(file_okay=True, dir_okay=False, allow_dash=False),
200
200
required=True,
201
201
)
202
+
@add_identifier_options
202
203
@click.option(
203
204
"-a",
204
205
"--auth",
···
206
207
default="auth.json",
207
208
help="Path to auth.json token file",
208
209
)
210
+
@click.option("--ids", is_flag=True, help="Treat input as user IDs, not screen names")
209
211
@click.option("--stop_after", type=int, help="Only pull this number of recent tweets")
210
-
@click.option("--user_id", help="Numeric user ID")
211
-
@click.option("--screen_name", help="Screen name")
212
+
@click.option("--user_id", help="Numeric user ID", hidden=True)
213
+
@click.option("--screen_name", help="Screen name", hidden=True)
212
214
@click.option(
213
215
"--since",
214
216
is_flag=True,
···
218
220
@click.option(
219
221
"--since_id", type=str, default=False, help="Pull tweets since this Tweet ID"
220
222
)
221
-
def user_timeline(db_path, auth, stop_after, user_id, screen_name, since, since_id):
223
+
def user_timeline(
224
+
db_path,
225
+
identifiers,
226
+
attach,
227
+
sql,
228
+
auth,
229
+
ids,
230
+
stop_after,
231
+
user_id,
232
+
screen_name,
233
+
since,
234
+
since_id,
235
+
):
222
236
"Save tweets posted by specified user"
223
237
if since and since_id:
224
238
raise click.ClickException("Use either --since or --since_id, not both")
239
+
225
240
auth = json.load(open(auth))
226
241
session = utils.session_for_auth(auth)
227
242
db = utils.open_database(db_path)
228
-
profile = utils.get_profile(db, session, user_id, screen_name)
229
-
expected_length = profile["statuses_count"]
243
+
identifiers = utils.resolve_identifiers(db, identifiers, attach, sql)
230
244
231
-
if since or since_id:
232
-
expected_length = None
245
+
# Backwards compatible support for old --user_id and --screen_name options
246
+
if screen_name:
247
+
if ids:
248
+
raise click.ClickException("Cannot use --screen_name with --ids")
249
+
identifiers.append(screen_name)
233
250
234
-
if since and db["tweets"].exists:
235
-
try:
236
-
since_id = db.conn.execute(
237
-
"select max(id) from tweets where user = ?", [profile["id"]]
238
-
).fetchall()[0][0]
239
-
except IndexError:
240
-
pass
251
+
if user_id:
252
+
if not identifiers:
253
+
identifiers = [user_id]
254
+
else:
255
+
if not ids:
256
+
raise click.ClickException("Use --user_id with --ids")
257
+
identifiers.append(user_id)
241
258
242
-
with click.progressbar(
243
-
utils.fetch_user_timeline(
244
-
session, user_id, screen_name, stop_after, since_id=since_id
245
-
),
246
-
length=expected_length,
247
-
label="Importing tweets",
248
-
show_pos=True,
249
-
) as bar:
250
-
# Save them 100 at a time
251
-
chunk = []
252
-
for tweet in bar:
253
-
chunk.append(tweet)
254
-
if len(chunk) >= 100:
259
+
# If identifiers is empty, fetch the authenticated user
260
+
fetch_profiles = True
261
+
if not identifiers:
262
+
fetch_profiles = False
263
+
profile = utils.get_profile(db, session, user_id, screen_name)
264
+
identifiers = [profile.screen_name]
265
+
ids = False
266
+
267
+
for identifier in identifiers:
268
+
kwargs = {}
269
+
if ids:
270
+
kwargs["user_id"] = identifier
271
+
else:
272
+
kwargs["screen_name"] = identifier
273
+
if fetch_profiles:
274
+
profile = utils.get_profile(db, session, **kwargs)
275
+
else:
276
+
profile = db["users"].get(profile["id"])
277
+
expected_length = profile["statuses_count"]
278
+
279
+
if since or since_id:
280
+
expected_length = None
281
+
282
+
if since and db["tweets"].exists:
283
+
try:
284
+
since_id = db.conn.execute(
285
+
"select max(id) from tweets where user = ?", [profile["id"]]
286
+
).fetchall()[0][0]
287
+
except IndexError:
288
+
pass
289
+
290
+
with click.progressbar(
291
+
utils.fetch_user_timeline(
292
+
session, stop_after=stop_after, since_id=since_id, **kwargs
293
+
),
294
+
length=expected_length,
295
+
label=profile["screen_name"],
296
+
show_pos=True,
297
+
) as bar:
298
+
# Save them 100 at a time
299
+
chunk = []
300
+
for tweet in bar:
301
+
chunk.append(tweet)
302
+
if len(chunk) >= 100:
303
+
utils.save_tweets(db, chunk)
304
+
chunk = []
305
+
if chunk:
255
306
utils.save_tweets(db, chunk)
256
-
chunk = []
257
-
if chunk:
258
-
utils.save_tweets(db, chunk)
259
307
260
308
261
309
@cli.command(name="home-timeline")
+8
-4
twitter_to_sqlite/utils.py
+8
-4
twitter_to_sqlite/utils.py
···
52
52
)
53
53
54
54
55
-
def fetch_user_list_chunks(session, user_id, screen_name, sleep=61, noun="followers"):
55
+
def fetch_user_list_chunks(
56
+
session, user_id=None, screen_name=None, sleep=61, noun="followers"
57
+
):
56
58
cursor = -1
57
59
users = []
58
60
while cursor:
···
64
66
time.sleep(sleep) # Rate limit = 15 per 15 minutes!
65
67
66
68
67
-
def fetch_user_list(session, cursor, user_id, screen_name, noun="followers"):
69
+
def fetch_user_list(session, cursor, user_id=None, screen_name=None, noun="followers"):
68
70
args = user_args(user_id, screen_name)
69
71
args.update({"count": 200, "cursor": cursor})
70
72
r = session.get(
···
133
135
time.sleep(sleep)
134
136
135
137
136
-
def fetch_user_timeline(session, user_id, screen_name, stop_after=None, since_id=None):
138
+
def fetch_user_timeline(
139
+
session, user_id=None, screen_name=None, stop_after=None, since_id=None
140
+
):
137
141
args = user_args(user_id, screen_name)
138
142
if since_id:
139
143
args["since_id"] = since_id
···
146
150
)
147
151
148
152
149
-
def fetch_favorites(session, user_id, screen_name, stop_after=None):
153
+
def fetch_favorites(session, user_id=None, screen_name=None, stop_after=None):
150
154
args = user_args(user_id, screen_name)
151
155
# Rate limit 75/15 mins = 5/minute = every 12 seconds
152
156
sleep = 12