this repo has no description
home-timeline command, closes #18
+68
-1
+15
README.md
+15
README.md
···
128
128
129
129
The underlying Twitter APIs have a rate limit of 15 requests every 15 minutes - though they do return up to 5,000 IDs in each call. By default both of these subcommands will wait for 61 seconds between API calls in order to stay within the rate limit - you can adjust this behaviour down to just one second delay if you know you will not be making many calls using `--sleep=1`.
130
130
131
+
## Retrieving tweets from your home timeline
132
+
133
+
The `home-timeline` command retrieves up to 800 tweets from the home timeline of the authenticated user - generally this means tweets from people you follow.
134
+
135
+
$ twitter-to-sqlite home-timeline twitter.db
136
+
Importing timeline [#################--------] 591/800 00:01:14
137
+
138
+
The tweets are stored in the `tweets` table, and a record is added to the `timeline_tweets` table noting that this tweet came in due to being spotted in the timeline of your user.
139
+
140
+
You can then view your timeline in Datasette using the following URL:
141
+
142
+
`/tweets/tweets?_where=id+in+(select+tweet+from+[timeline_tweets])&_sort_desc=id&_facet=user`
143
+
144
+
This will filter your tweets table to just tweets that appear in your timeline, ordered by most recent first and use faceting to show you which users are responsible for the most tweets.
145
+
131
146
## Providing input from a SQL query with --sql and --attach
132
147
133
148
This option is available for some subcommands - run `twitter-to-sqlite command-name --help` to check.
+43
twitter_to_sqlite/cli.py
+43
twitter_to_sqlite/cli.py
···
198
198
utils.save_tweets(db, chunk)
199
199
200
200
201
+
@cli.command(name="home-timeline")
202
+
@click.argument(
203
+
"db_path",
204
+
type=click.Path(file_okay=True, dir_okay=False, allow_dash=False),
205
+
required=True,
206
+
)
207
+
@click.option(
208
+
"-a",
209
+
"--auth",
210
+
type=click.Path(file_okay=True, dir_okay=False, allow_dash=True, exists=True),
211
+
default="auth.json",
212
+
help="Path to auth.json token file",
213
+
)
214
+
def home_timeline(db_path, auth):
215
+
"Save tweets from timeline for authenticated user"
216
+
auth = json.load(open(auth))
217
+
session = utils.session_for_auth(auth)
218
+
profile = utils.get_profile(session)
219
+
db = sqlite_utils.Database(db_path)
220
+
with click.progressbar(
221
+
utils.fetch_home_timeline(session),
222
+
length=800,
223
+
label="Importing timeline",
224
+
show_pos=True,
225
+
) as bar:
226
+
# Save them 100 at a time
227
+
def save_chunk(db, chunk):
228
+
utils.save_tweets(db, chunk)
229
+
# Record who's timeline they came from
230
+
db["timeline_tweets"].upsert_all([{
231
+
"user": profile["id"],
232
+
"tweet": tweet["id"]
233
+
} for tweet in chunk], pk=("user", "tweet"), foreign_keys=("user", "tweet"))
234
+
chunk = []
235
+
for tweet in bar:
236
+
chunk.append(tweet)
237
+
if len(chunk) >= 100:
238
+
save_chunk(db, chunk)
239
+
chunk = []
240
+
if chunk:
241
+
save_chunk(db, chunk)
242
+
243
+
201
244
@cli.command(name="users-lookup")
202
245
@click.argument(
203
246
"db_path",
+10
-1
twitter_to_sqlite/utils.py
+10
-1
twitter_to_sqlite/utils.py
···
40
40
return r.headers, r.json()
41
41
42
42
43
-
def get_profile(session, user_id, screen_name):
43
+
def get_profile(session, user_id=None, screen_name=None):
44
44
if not (user_id or screen_name):
45
45
return session.get(
46
46
"https://api.twitter.com/1.1/account/verify_credentials.json"
···
82
82
args,
83
83
sleep=1,
84
84
stop_after=stop_after,
85
+
)
86
+
87
+
88
+
def fetch_home_timeline(session):
89
+
yield from fetch_timeline(
90
+
session,
91
+
"https://api.twitter.com/1.1/statuses/home_timeline.json",
92
+
{},
93
+
sleep=1,
85
94
)
86
95
87
96