this repo has no description
Added list-members subcommand
+74
+12
README.md
+12
README.md
···
79
79
80
80
See [Analyzing my Twitter followers with Datasette](https://simonwillison.net/2018/Jan/28/analyzing-my-twitter-followers/) for the original inspiration for this command.
81
81
82
+
## Retrieving Twitter followers
83
+
84
+
The `list-members` command can be used to retrieve details of one or more Twitter lists, including all of their members.
85
+
86
+
$ twitter-to-sqlite list-members members.db simonw/the-good-place
87
+
88
+
You can pass multiple `screen_name/list_slug` identifiers.
89
+
90
+
If you know the numeric IDs of the lists instead, you can use `--ids`:
91
+
92
+
$ twitter-to-sqlite list-members members.db 927913322841653248
93
+
82
94
## Design notes
83
95
84
96
* Tweet IDs are stored as integers, to afford sorting by ID in a sensible way
+26
twitter_to_sqlite/cli.py
+26
twitter_to_sqlite/cli.py
···
217
217
identifiers = utils.resolve_identifiers(db, identifiers, attach, sql)
218
218
for batch in utils.fetch_user_batches(session, identifiers, ids):
219
219
utils.save_users(db, batch)
220
+
221
+
222
+
@cli.command(name="list-members")
223
+
@click.argument(
224
+
"db_path",
225
+
type=click.Path(file_okay=True, dir_okay=False, allow_dash=False),
226
+
required=True,
227
+
)
228
+
@click.argument("identifiers", type=str, nargs=-1)
229
+
@click.option(
230
+
"-a",
231
+
"--auth",
232
+
type=click.Path(file_okay=True, dir_okay=False, allow_dash=True, exists=True),
233
+
default="auth.json",
234
+
help="Path to auth.json token file",
235
+
)
236
+
@click.option(
237
+
"--ids", is_flag=True, help="Treat input as list IDs, not user/slug strings"
238
+
)
239
+
def list_members(db_path, identifiers, auth, ids):
240
+
"Fetch lists - accepts one or more screen_name/list_slug identifiers"
241
+
auth = json.load(open(auth))
242
+
session = utils.session_for_auth(auth)
243
+
db = sqlite_utils.Database(db_path)
244
+
for identifier in identifiers:
245
+
utils.fetch_and_save_list(db, session, identifier, ids)
+36
twitter_to_sqlite/utils.py
+36
twitter_to_sqlite/utils.py
···
273
273
else:
274
274
sql_identifiers = []
275
275
return list(identifiers) + sql_identifiers
276
+
277
+
278
+
def fetch_and_save_list(db, session, identifier, identifier_is_id=False):
279
+
show_url = "https://api.twitter.com/1.1/lists/show.json"
280
+
args = {}
281
+
if identifier_is_id:
282
+
args["list_id"] = identifier
283
+
else:
284
+
screen_name, slug = identifier.split("/")
285
+
args.update({"owner_screen_name": screen_name, "slug": slug})
286
+
# First fetch the list details
287
+
data = session.get(show_url, params=args).json()
288
+
list_id = data["id"]
289
+
del data["id_str"]
290
+
user = data.pop("user")
291
+
save_users(db, [user])
292
+
data["user"] = user["id"]
293
+
data["created_at"] = parser.parse(data["created_at"])
294
+
db["lists"].upsert(data, pk="id", foreign_keys=("user",))
295
+
# Now fetch the members
296
+
url = "https://api.twitter.com/1.1/lists/members.json"
297
+
cursor = -1
298
+
while cursor:
299
+
args.update({"count": 5000, "cursor": cursor})
300
+
body = session.get(url, params=args).json()
301
+
users = body["users"]
302
+
save_users(db, users)
303
+
db["list_members"].upsert_all(
304
+
({"list": list_id, "user": user["id"]} for user in users),
305
+
pk=("list", "user"),
306
+
foreign_keys=("list", "user"),
307
+
)
308
+
cursor = body["next_cursor"]
309
+
if not cursor:
310
+
break
311
+
time.sleep(1) # Rate limit = 900 per 15 minutes