this repo has no description
Store unescaped full_text of Tweet, closes #21
+5
-4
+3
-3
tests/test_save_tweets.py
+3
-3
tests/test_save_tweets.py
···
178
178
"id": 861696799362478100,
179
179
"user": 14148390,
180
180
"created_at": "2017-05-08T21:38:21+00:00",
181
-
"full_text": "If you use Photos (mac) & Live Photos, run this command to generate a lovely sound collage of where you’ve been https://gist.github.com/bwhitman/5be2f905556a25145dbac74fe4080739",
181
+
"full_text": "If you use Photos (mac) & Live Photos, run this command to generate a lovely sound collage of where you’ve been https://gist.github.com/bwhitman/5be2f905556a25145dbac74fe4080739",
182
182
"retweeted_status": None,
183
183
"quoted_status": None,
184
184
"place": None,
···
253
253
"id": 1169242008432644000,
254
254
"user": 22737278,
255
255
"created_at": "2019-09-04T13:33:12+00:00",
256
-
"full_text": "My new post: an explainer on “carbon capture & utilization” (CCU). CO2 captured from waste gases or the ambient air can be used to make valuable products. Could CCU help the carbon capture industry scale up? https://www.vox.com/energy-and-environment/2019/9/4/20829431/climate-change-carbon-capture-utilization-sequestration-ccu-ccs?utm_campaign=drvox&utm_content=chorus&utm_medium=social&utm_source=twitter",
256
+
"full_text": "My new post: an explainer on “carbon capture & utilization” (CCU). CO2 captured from waste gases or the ambient air can be used to make valuable products. Could CCU help the carbon capture industry scale up? https://www.vox.com/energy-and-environment/2019/9/4/20829431/climate-change-carbon-capture-utilization-sequestration-ccu-ccs?utm_campaign=drvox&utm_content=chorus&utm_medium=social&utm_source=twitter",
257
257
"retweeted_status": None,
258
258
"quoted_status": None,
259
259
"place": None,
···
278
278
"id": 1169246717864136700,
279
279
"user": 12497,
280
280
"created_at": "2019-09-04T13:51:55+00:00",
281
-
"full_text": "RT @drvox: My new post: an explainer on “carbon capture & utilization” (CCU). CO2 captured from waste gases or the ambient air can be used…",
281
+
"full_text": "RT @drvox: My new post: an explainer on “carbon capture & utilization” (CCU). CO2 captured from waste gases or the ambient air can be used…",
282
282
"retweeted_status": 1169242008432644000,
283
283
"quoted_status": None,
284
284
"place": None,
+2
-1
twitter_to_sqlite/utils.py
+2
-1
twitter_to_sqlite/utils.py
···
1
1
import datetime
2
+
import html
2
3
import json
3
4
import pathlib
4
5
import time
···
134
135
135
136
136
137
def transform_tweet(tweet):
137
-
tweet["full_text"] = expand_entities(tweet["full_text"], tweet.pop("entities"))
138
+
tweet["full_text"] = html.unescape(expand_entities(tweet["full_text"], tweet.pop("entities")))
138
139
to_remove = [k for k in tweet if k.endswith("_str")] + [
139
140
"quoted_status_id",
140
141
"quoted_status_permalink",