this repo has no description
0
fork

Configure Feed

Select the types of activity you want to include in your feed.

--attach and --sql for users-lookup, refs #8

+44 -3
+23 -3
twitter_to_sqlite/cli.py
··· 5 5 from twitter_to_sqlite import utils 6 6 7 7 8 + def add_identifier_options(subcommand): 9 + for decorator in reversed( 10 + ( 11 + click.argument("identifiers", type=str, nargs=-1), 12 + click.option( 13 + "--attach", 14 + type=click.Path( 15 + file_okay=True, dir_okay=False, allow_dash=False, exists=True 16 + ), 17 + multiple=True, 18 + help="Additional database file to attach", 19 + ), 20 + click.option("--sql", help="SQL query to fetch identifiers to use"), 21 + ) 22 + ): 23 + subcommand = decorator(subcommand) 24 + return subcommand 25 + 26 + 8 27 @click.group() 9 28 @click.version_option() 10 29 def cli(): ··· 181 200 type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), 182 201 required=True, 183 202 ) 184 - @click.argument("screen_name", nargs=-1) 203 + @add_identifier_options 185 204 @click.option( 186 205 "-a", 187 206 "--auth", ··· 190 209 help="Path to auth.json token file", 191 210 ) 192 211 @click.option("--ids", is_flag=True, help="Treat input as user IDs, not screen names") 193 - def users_lookup(db_path, screen_name, auth, ids): 212 + def users_lookup(db_path, identifiers, attach, sql, auth, ids): 194 213 "Fetch user accounts" 195 214 auth = json.load(open(auth)) 196 215 session = utils.session_for_auth(auth) 197 216 db = sqlite_utils.Database(db_path) 198 - for batch in utils.fetch_user_batches(session, screen_name, ids): 217 + identifiers = utils.resolve_identifiers(db, identifiers, attach, sql) 218 + for batch in utils.fetch_user_batches(session, identifiers, ids): 199 219 utils.save_users(db, batch)
+21
twitter_to_sqlite/utils.py
··· 2 2 from dateutil import parser 3 3 import datetime 4 4 import time 5 + import pathlib 5 6 import json 6 7 import urllib.parse 7 8 ··· 252 253 users = session.get(url, params=args).json() 253 254 yield users 254 255 time.sleep(sleep) 256 + 257 + 258 + def resolve_identifiers(db, identifiers, attach, sql): 259 + if sql: 260 + if attach: 261 + for filepath in attach: 262 + if ":" in filepath: 263 + alias, filepath = filepath.split(":", 1) 264 + else: 265 + alias = filepath.split("/")[-1].split(".")[0] 266 + attach_sql = """ 267 + ATTACH DATABASE '{}' AS [{}]; 268 + """.format( 269 + str(pathlib.Path(filepath).resolve()), alias 270 + ) 271 + db.conn.execute(attach_sql) 272 + sql_identifiers = [r[0] for r in db.conn.execute(sql).fetchall()] 273 + else: 274 + sql_identifiers = [] 275 + return list(identifiers) + sql_identifiers