this repo has no description
Fix #34 by upgrading sqlite-utils
+18
-15
+1
-1
setup.py
+1
-1
setup.py
+2
-2
twitter_to_sqlite/archive.py
+2
-2
twitter_to_sqlite/archive.py
···
219
219
if table_name in existing_tables:
220
220
db[table_name].drop()
221
221
if pk is not None:
222
-
db[table_name].upsert_all(rows, pk=pk)
222
+
db[table_name].insert_all(rows, pk=pk, replace=True)
223
223
else:
224
-
db[table_name].upsert_all(rows, hash_id="pk")
224
+
db[table_name].insert_all(rows, hash_id="pk", replace=True)
+4
-3
twitter_to_sqlite/cli.py
+4
-3
twitter_to_sqlite/cli.py
···
163
163
_shared_friends_followers(db_path, auth, user_id, screen_name, silent, "friends")
164
164
165
165
166
-
167
166
@cli.command()
168
167
@click.argument(
169
168
"db_path",
···
357
356
def save_chunk(db, chunk):
358
357
utils.save_tweets(db, chunk)
359
358
# Record who's timeline they came from
360
-
db[table].upsert_all(
359
+
db[table].insert_all(
361
360
[{"user": profile["id"], "tweet": tweet["id"]} for tweet in chunk],
362
361
pk=("user", "tweet"),
363
362
foreign_keys=("user", "tweet"),
363
+
replace=True,
364
364
)
365
365
366
366
chunk = []
···
756
756
def save_chunk(db, search_run_id, chunk):
757
757
utils.save_tweets(db, chunk)
758
758
# Record which search run produced them
759
-
db["search_runs_tweets"].upsert_all(
759
+
db["search_runs_tweets"].insert_all(
760
760
[{"search_run": search_run_id, "tweet": tweet["id"]} for tweet in chunk],
761
761
pk=("search_run", "tweet"),
762
762
foreign_keys=("search_run", "tweet"),
763
+
replace=True,
763
764
)
764
765
765
766
search_run_id = None
+11
-9
twitter_to_sqlite/utils.py
+11
-9
twitter_to_sqlite/utils.py
···
274
274
tweet["user"] = user["id"]
275
275
tweet["source"] = extract_and_save_source(db, tweet["source"])
276
276
if tweet.get("place"):
277
-
db["places"].upsert(tweet["place"], pk="id", alter=True)
277
+
db["places"].insert(tweet["place"], pk="id", alter=True, replace=True)
278
278
tweet["place"] = tweet["place"]["id"]
279
279
# extended_entities contains media
280
280
extended_entities = tweet.pop("extended_entities", None)
···
286
286
tweet[tweet_key] = tweet[tweet_key]["id"]
287
287
if nested:
288
288
save_tweets(db, nested)
289
-
db["users"].upsert(user, pk="id", alter=True)
290
-
table = db["tweets"].upsert(tweet, pk="id", alter=True)
289
+
db["users"].insert(user, pk="id", alter=True, replace=True)
290
+
table = db["tweets"].insert(tweet, pk="id", alter=True, replace=True)
291
291
if favorited_by is not None:
292
-
db["favorited_by"].upsert(
292
+
db["favorited_by"].insert(
293
293
{"tweet": tweet["id"], "user": favorited_by},
294
294
pk=("user", "tweet"),
295
295
foreign_keys=("tweet", "user"),
296
+
replace=True,
296
297
)
297
298
if extended_entities and extended_entities.get("media"):
298
299
for media in extended_entities["media"]:
299
300
# TODO: Remove this line when .m2m() grows alter=True
300
-
db["media"].upsert(media, pk="id", alter=True)
301
+
db["media"].insert(media, pk="id", alter=True, replace=True)
301
302
table.m2m("media", media, pk="id")
302
303
303
304
···
306
307
ensure_tables(db)
307
308
for user in users:
308
309
transform_user(user)
309
-
db["users"].upsert_all(users, pk="id", alter=True)
310
+
db["users"].insert_all(users, pk="id", alter=True, replace=True)
310
311
if followed_id or follower_id:
311
312
first_seen = datetime.datetime.utcnow().isoformat()
312
313
db["following"].insert_all(
···
400
401
save_users(db, [user])
401
402
data["user"] = user["id"]
402
403
data["created_at"] = parser.parse(data["created_at"])
403
-
db["lists"].upsert(data, pk="id", foreign_keys=("user",))
404
+
db["lists"].insert(data, pk="id", foreign_keys=("user",), replace=True)
404
405
# Now fetch the members
405
406
url = "https://api.twitter.com/1.1/lists/members.json"
406
407
cursor = -1
···
409
410
body = session.get(url, params=args).json()
410
411
users = body["users"]
411
412
save_users(db, users)
412
-
db["list_members"].upsert_all(
413
+
db["list_members"].insert_all(
413
414
({"list": list_id, "user": user["id"]} for user in users),
414
415
pk=("list", "user"),
415
416
foreign_keys=("list", "user"),
417
+
replace=True,
416
418
)
417
419
cursor = body["next_cursor"]
418
420
if not cursor:
···
516
518
def extract_and_save_source(db, source):
517
519
m = source_re.match(source)
518
520
details = m.groupdict()
519
-
return db["sources"].upsert(details, hash_id="id").last_pk
521
+
return db["sources"].insert(details, hash_id="id", replace=True).last_pk