+9 -18

twitter_to_sqlite/cli.py

reviewed

··· 205 205 db = utils.open_database(db_path) 206 206 profile = utils.get_profile(db, session, user_id, screen_name) 207 207 with click.progressbar( 208 208 - utils.fetch_favorites(session, user_id, screen_name, stop_after), 208 208 + utils.fetch_favorites(session, db, user_id, screen_name, stop_after), 209 209 label="Importing favorites", 210 210 show_pos=True, 211 211 ) as bar: ··· 253 253 since_id, 254 254 ): 255 255 "Save tweets posted by specified user" 256 256 - if since and since_id: 257 257 - raise click.ClickException("Use either --since or --since_id, not both") 258 258 - 259 256 auth = json.load(open(auth)) 260 257 session = utils.session_for_auth(auth) 261 258 db = utils.open_database(db_path) ··· 302 299 if since or since_id: 303 300 expected_length = None 304 301 305 305 - if since and db["tweets"].exists: 306 306 - try: 307 307 - since_id = db.conn.execute( 308 308 - "select max(id) from tweets where user = ?", [profile["id"]] 309 309 - ).fetchall()[0][0] 310 310 - except IndexError: 311 311 - pass 312 312 - 313 302 with click.progressbar( 314 303 utils.fetch_user_timeline( 315 315 - session, stop_after=stop_after, since_id=since_id, **kwargs 304 304 + session, 305 305 + db, 306 306 + stop_after=stop_after, 307 307 + since_id=since_id, 308 308 + since=since, 309 309 + **kwargs 316 310 ), 317 311 length=expected_length, 318 312 label=format_string.format(profile["screen_name"]), ··· 399 393 400 394 401 395 def _shared_timeline(db_path, auth, since, since_id, table, api_url, sleep=1): 402 402 - if since and since_id: 403 403 - raise click.ClickException("Use either --since or --since_id, not both") 404 396 auth = json.load(open(auth)) 405 397 session = utils.session_for_auth(auth) 406 398 db = utils.open_database(db_path) ··· 418 410 pass 419 411 420 412 with click.progressbar( 421 421 - utils.fetch_timeline(session, api_url, sleep=sleep, since_id=since_id), 413 413 + utils.fetch_timeline(session, api_url, db, sleep=sleep, since_id=since_id), 422 414 length=expected_length, 423 415 label="Importing tweets", 424 416 show_pos=True, ··· 777 769 https://developer.twitter.com/en/docs/tweets/search/api-reference/get-search-tweets 778 770 """ 779 771 since_id = kwargs.pop("since_id", None) 780 780 - if since and since_id: 781 781 - raise click.ClickException("Use either --since or --since_id, not both") 782 772 stop_after = kwargs.pop("stop_after", None) 783 773 auth = json.load(open(auth)) 784 774 session = utils.session_for_auth(auth) ··· 810 800 tweets = utils.fetch_timeline( 811 801 session, 812 802 "https://api.twitter.com/1.1/search/tweets.json", 803 803 + db, 813 804 search_args, 814 805 sleep=6, 815 806 key="statuses",

+57 -5

twitter_to_sqlite/utils.py

reviewed

··· 1 1 + import click 1 2 import datetime 2 3 import html 3 4 import json ··· 99 100 100 101 101 102 def fetch_timeline( 102 102 - session, url, args=None, sleep=1, stop_after=None, key=None, since_id=None 103 103 + session, 104 104 + url, 105 105 + db, 106 106 + args=None, 107 107 + sleep=1, 108 108 + stop_after=None, 109 109 + key=None, 110 110 + since_id=None, 111 111 + since=False, 112 112 + since_type=None, 113 113 + since_key=None, 103 114 ): 104 115 # See https://developer.twitter.com/en/docs/tweets/timelines/guides/working-with-timelines 116 116 + if since and since_id: 117 117 + raise click.ClickException("Use either --since or --since_id, not both") 118 118 + 119 119 + since_type_id = None 120 120 + last_since_id = None 121 121 + if since_type is not None: 122 122 + assert since_key is not None 123 123 + since_type_id = SINCE_ID_TYPES[since_type] 124 124 + # Figure out the last since_id in case we need it 125 125 + try: 126 126 + last_since_id = db.conn.execute( 127 127 + """ 128 128 + select since_id from since_ids 129 129 + where type = ? and key = ? 130 130 + """, 131 131 + [since_type_id, since_key], 132 132 + ).fetchall()[0][0] 133 133 + except IndexError: 134 134 + pass 135 135 + 136 136 + if since: 137 137 + # Load since_id from database 138 138 + since_id = last_since_id 139 139 + 105 140 args = dict(args or {}) 106 141 args["count"] = 200 107 142 if stop_after is not None: ··· 137 172 for tweet in tweets: 138 173 yield tweet 139 174 min_seen_id = min(t["id"] for t in tweets) 175 175 + max_seen_id = max(t["id"] for t in tweets) 176 176 + if last_since_id is not None: 177 177 + max_seen_id = max((last_since_id, max_seen_id)) 178 178 + db["since_ids"].insert( 179 179 + {"type": since_type_id, "key": since_key, "since_id": max_seen_id,}, 180 180 + replace=True, 181 181 + ) 140 182 if stop_after is not None: 141 183 break 142 184 time.sleep(sleep) 143 185 144 186 145 187 def fetch_user_timeline( 146 146 - session, user_id=None, screen_name=None, stop_after=None, since_id=None 188 188 + session, 189 189 + db, 190 190 + user_id=None, 191 191 + screen_name=None, 192 192 + stop_after=None, 193 193 + since_id=None, 194 194 + since=False, 147 195 ): 148 196 args = user_args(user_id, screen_name) 149 149 - if since_id: 150 150 - args["since_id"] = since_id 151 197 yield from fetch_timeline( 152 198 session, 153 199 "https://api.twitter.com/1.1/statuses/user_timeline.json", 200 200 + db, 154 201 args, 155 202 sleep=1, 156 203 stop_after=stop_after, 204 204 + since_id=since_id, 205 205 + since_type="user", 206 206 + since_key="id:{}".format(user_id) if user_id else screen_name, 207 207 + since=since, 157 208 ) 158 209 159 210 160 160 - def fetch_favorites(session, user_id=None, screen_name=None, stop_after=None): 211 211 + def fetch_favorites(session, db, user_id=None, screen_name=None, stop_after=None): 161 212 args = user_args(user_id, screen_name) 162 213 # Rate limit 75/15 mins = 5/minute = every 12 seconds 163 214 sleep = 12 164 215 yield from fetch_timeline( 165 216 session, 166 217 "https://api.twitter.com/1.1/favorites/list.json", 218 218 + db, 167 219 args, 168 220 sleep=sleep, 169 221 stop_after=stop_after,