this repo has no description
Added --since_id and --since to user-timeline, refs #20
+34
-5
+2
README.md
+2
README.md
···
53
53
54
54
Twitter's API only returns up to around 3,200 tweets for most user accounts, but you may find that it returns all available tweets for your own user account.
55
55
56
+
You can use `--since` to retrieve every tweet since the last time you imported for that user, or `--since_id=xxx` to retrieve every tweet since a specific tweet ID.
57
+
56
58
## Retrieve accounts in bulk
57
59
58
60
If you have a list of Twitter screen names (or user IDs) you can bulk fetch their fully inflated Twitter profiles using the `users-lookup` command:
+29
-4
twitter_to_sqlite/cli.py
+29
-4
twitter_to_sqlite/cli.py
···
178
178
@click.option("--stop_after", type=int, help="Only pull this number of recent tweets")
179
179
@click.option("--user_id", help="Numeric user ID")
180
180
@click.option("--screen_name", help="Screen name")
181
-
def user_timeline(db_path, auth, stop_after, user_id, screen_name):
181
+
@click.option(
182
+
"--since",
183
+
is_flag=True,
184
+
default=False,
185
+
help="Pull tweets since last retrieved tweet",
186
+
)
187
+
@click.option(
188
+
"--since_id", type=str, default=False, help="Pull tweets since this Tweet ID"
189
+
)
190
+
def user_timeline(db_path, auth, stop_after, user_id, screen_name, since, since_id):
182
191
"Save tweets posted by specified user"
192
+
if since and since_id:
193
+
raise click.ClickException("Use either --since or --since_id, not both")
183
194
auth = json.load(open(auth))
184
195
session = utils.session_for_auth(auth)
185
196
profile = utils.get_profile(session, user_id, screen_name)
186
197
db = sqlite_utils.Database(db_path)
198
+
expected_length = profile["statuses_count"]
199
+
200
+
if since or since_id:
201
+
expected_length = None
202
+
203
+
if since:
204
+
try:
205
+
since_id = db.conn.execute(
206
+
"select max(id) from tweets where user = ?", [profile["id"]]
207
+
).fetchall()[0][0]
208
+
print("since = ", since_id)
209
+
except IndexError:
210
+
pass
211
+
187
212
with click.progressbar(
188
-
utils.fetch_user_timeline(session, user_id, screen_name, stop_after),
189
-
length=profile["statuses_count"],
213
+
utils.fetch_user_timeline(session, user_id, screen_name, stop_after, since_id=since_id),
214
+
length=expected_length,
190
215
label="Importing tweets",
191
216
show_pos=True,
192
217
) as bar:
···
226
251
def home_timeline(db_path, auth, since, since_id):
227
252
"Save tweets from timeline for authenticated user"
228
253
if since and since_id:
229
-
raise click.ClickException("Use either --since or --since_id, not both ")
254
+
raise click.ClickException("Use either --since or --since_id, not both")
230
255
auth = json.load(open(auth))
231
256
session = utils.session_for_auth(auth)
232
257
profile = utils.get_profile(session)
+3
-1
twitter_to_sqlite/utils.py
+3
-1
twitter_to_sqlite/utils.py
···
94
94
time.sleep(sleep)
95
95
96
96
97
-
def fetch_user_timeline(session, user_id, screen_name, stop_after=None):
97
+
def fetch_user_timeline(session, user_id, screen_name, stop_after=None, since_id=None):
98
98
args = user_args(user_id, screen_name)
99
+
if since_id:
100
+
args["since_id"] = since_id
99
101
yield from fetch_timeline(
100
102
session,
101
103
"https://api.twitter.com/1.1/statuses/user_timeline.json",