this repo has no description
--attach and --sql for users-lookup, refs #8
+44
-3
+23
-3
twitter_to_sqlite/cli.py
+23
-3
twitter_to_sqlite/cli.py
···
5
5
from twitter_to_sqlite import utils
6
6
7
7
8
+
def add_identifier_options(subcommand):
9
+
for decorator in reversed(
10
+
(
11
+
click.argument("identifiers", type=str, nargs=-1),
12
+
click.option(
13
+
"--attach",
14
+
type=click.Path(
15
+
file_okay=True, dir_okay=False, allow_dash=False, exists=True
16
+
),
17
+
multiple=True,
18
+
help="Additional database file to attach",
19
+
),
20
+
click.option("--sql", help="SQL query to fetch identifiers to use"),
21
+
)
22
+
):
23
+
subcommand = decorator(subcommand)
24
+
return subcommand
25
+
26
+
8
27
@click.group()
9
28
@click.version_option()
10
29
def cli():
···
181
200
type=click.Path(file_okay=True, dir_okay=False, allow_dash=False),
182
201
required=True,
183
202
)
184
-
@click.argument("screen_name", nargs=-1)
203
+
@add_identifier_options
185
204
@click.option(
186
205
"-a",
187
206
"--auth",
···
190
209
help="Path to auth.json token file",
191
210
)
192
211
@click.option("--ids", is_flag=True, help="Treat input as user IDs, not screen names")
193
-
def users_lookup(db_path, screen_name, auth, ids):
212
+
def users_lookup(db_path, identifiers, attach, sql, auth, ids):
194
213
"Fetch user accounts"
195
214
auth = json.load(open(auth))
196
215
session = utils.session_for_auth(auth)
197
216
db = sqlite_utils.Database(db_path)
198
-
for batch in utils.fetch_user_batches(session, screen_name, ids):
217
+
identifiers = utils.resolve_identifiers(db, identifiers, attach, sql)
218
+
for batch in utils.fetch_user_batches(session, identifiers, ids):
199
219
utils.save_users(db, batch)
+21
twitter_to_sqlite/utils.py
+21
twitter_to_sqlite/utils.py
···
2
2
from dateutil import parser
3
3
import datetime
4
4
import time
5
+
import pathlib
5
6
import json
6
7
import urllib.parse
7
8
···
252
253
users = session.get(url, params=args).json()
253
254
yield users
254
255
time.sleep(sleep)
256
+
257
+
258
+
def resolve_identifiers(db, identifiers, attach, sql):
259
+
if sql:
260
+
if attach:
261
+
for filepath in attach:
262
+
if ":" in filepath:
263
+
alias, filepath = filepath.split(":", 1)
264
+
else:
265
+
alias = filepath.split("/")[-1].split(".")[0]
266
+
attach_sql = """
267
+
ATTACH DATABASE '{}' AS [{}];
268
+
""".format(
269
+
str(pathlib.Path(filepath).resolve()), alias
270
+
)
271
+
db.conn.execute(attach_sql)
272
+
sql_identifiers = [r[0] for r in db.conn.execute(sql).fetchall()]
273
+
else:
274
+
sql_identifiers = []
275
+
return list(identifiers) + sql_identifiers