41 lines
1.3 KiB
Python
41 lines
1.3 KiB
Python
|
import datetime
|
||
|
|
|
||
|
|
import pandas as pd
|
||
|
|
import json
|
||
|
|
|
||
|
|
from utils.documents_to_database import documents_to_database
|
||
|
|
from utils.convert_encoding_meta import convert_encoding_meta
|
||
|
|
|
||
|
|
# In[ ]:
|
||
|
|
instagram_data_path = 'data/Instagram/threads/threads_and_replies.json'
|
||
|
|
with open(instagram_data_path, "r", encoding="raw-unicode-escape") as posts:
|
||
|
|
post_comments_1 = json.loads(convert_encoding_meta(posts.read()))
|
||
|
|
|
||
|
|
# In[ ]:
|
||
|
|
threads_comments = []
|
||
|
|
for post in post_comments_1['text_post_app_text_posts']:
|
||
|
|
for element in post['media']:
|
||
|
|
threads_comments.append({"texte": element['title'],
|
||
|
|
'datepublication': datetime.datetime.fromtimestamp(
|
||
|
|
timestamp=element['creation_timestamp']).isoformat(),
|
||
|
|
"chemin": instagram_data_path,
|
||
|
|
"index": "rs_instagram_threads",
|
||
|
|
"type": "posts",
|
||
|
|
"network": "Instagram"})
|
||
|
|
|
||
|
|
# In[ ]:
|
||
|
|
ig_comments_df = pd.DataFrame(threads_comments)
|
||
|
|
|
||
|
|
# In[ ]:
|
||
|
|
ig_comments_df.fillna(value="", inplace=True)
|
||
|
|
|
||
|
|
# In[ ]:
|
||
|
|
ig_comments_df.drop_duplicates(subset=['texte', 'datepublication'], inplace=True)
|
||
|
|
|
||
|
|
# In[ ]:
|
||
|
|
# Filter empty texte
|
||
|
|
ig_comments_df = ig_comments_df[~ig_comments_df['texte'].str.strip('\n').str.strip().eq('')]
|
||
|
|
|
||
|
|
# In[ ]:
|
||
|
|
documents_to_database(ig_comments_df)
|