this repo has no description
users-lookup command, closes #7
+58
-5
+10
README.md
+10
README.md
···
53
53
54
54
Twitter's API only returns up to around 3,200 tweets for most user accounts, but you may find that it returns all available tweets for your own user account.
55
55
56
+
## Retrieve accounts in bulk
57
+
58
+
If you have a list of Twitter screen names (or user IDs) you can bulk fetch their fully inflated Twitter profiles using the `users-lookup` command:
59
+
60
+
$ twitter-to-sqlite users-lookup users.db simonw cleopaws
61
+
62
+
You can pass user IDs instead usincg the `--ids` option:
63
+
64
+
$ twitter-to-sqlite users-lookup users.db 12497 3166449535 --ids
65
+
56
66
## Retrieving Twitter followers
57
67
58
68
The `followers` command retrieves details of every follower of the specified account. You can use it to retrieve your own followers, or you can pass a screen_name to pull the followers for another account.
+25
-5
twitter_to_sqlite/cli.py
+25
-5
twitter_to_sqlite/cli.py
···
149
149
default="auth.json",
150
150
help="Path to auth.json token file",
151
151
)
152
-
@click.option(
153
-
"--stop_after",
154
-
type=int,
155
-
help="Only pull this number of recent tweets",
156
-
)
152
+
@click.option("--stop_after", type=int, help="Only pull this number of recent tweets")
157
153
@click.option("--user_id", help="Numeric user ID")
158
154
@click.option("--screen_name", help="Screen name")
159
155
def user_timeline(db_path, auth, stop_after, user_id, screen_name):
···
177
173
chunk = []
178
174
if chunk:
179
175
utils.save_tweets(db, chunk)
176
+
177
+
178
+
@cli.command(name="users-lookup")
179
+
@click.argument(
180
+
"db_path",
181
+
type=click.Path(file_okay=True, dir_okay=False, allow_dash=False),
182
+
required=True,
183
+
)
184
+
@click.argument("screen_name", nargs=-1)
185
+
@click.option(
186
+
"-a",
187
+
"--auth",
188
+
type=click.Path(file_okay=True, dir_okay=False, allow_dash=True, exists=True),
189
+
default="auth.json",
190
+
help="Path to auth.json token file",
191
+
)
192
+
@click.option("--ids", is_flag=True, help="Treat input as user IDs, not screen names")
193
+
def users_lookup(db_path, screen_name, auth, ids):
194
+
"Fetch user accounts"
195
+
auth = json.load(open(auth))
196
+
session = utils.session_for_auth(auth)
197
+
db = sqlite_utils.Database(db_path)
198
+
for batch in utils.fetch_user_batches(session, screen_name, ids):
199
+
utils.save_users(db, batch)
+23
twitter_to_sqlite/utils.py
+23
twitter_to_sqlite/utils.py
···
229
229
),
230
230
ignore=True,
231
231
)
232
+
233
+
234
+
def fetch_user_batches(session, ids_or_screen_names, use_ids=False, sleep=1):
235
+
# Yields lists of up to 70 users (tried 100 but got this error:
236
+
# # {'code': 18, 'message': 'Too many terms specified in query.'} )
237
+
batches = []
238
+
batch = []
239
+
for id in ids_or_screen_names:
240
+
batch.append(id)
241
+
if len(batch) == 70:
242
+
batches.append(batch)
243
+
batch = []
244
+
if batch:
245
+
batches.append(batch)
246
+
url = "https://api.twitter.com/1.1/users/lookup.json"
247
+
for batch in batches:
248
+
if use_ids:
249
+
args = {"user_id": ",".join(map(str, batch))}
250
+
else:
251
+
args = {"screen_name": ",".join(batch)}
252
+
users = session.get(url, params=args).json()
253
+
yield users
254
+
time.sleep(sleep)