libere-tes-chaine-de-mots/import_data/26_importation_threads.py
2024-10-02 21:53:37 -04:00

49 lines
1.6 KiB
Python

import pandas as pd
import json
import os
from pathlib import Path
from utils.documents_to_database import documents_to_database
from utils.convert_encoding_meta import convert_encoding_meta
#%% In[ ]:
# Get the current file's directory
try:
# This will work when running as a script
script_dir = Path(__file__).parent.parent
except NameError:
# This will work in interactive environments
script_dir = Path().absolute()
project_root = script_dir
instagram_data_path = os.path.join(project_root, 'import_data', 'data', 'Instagram', 'threads', 'threads_and_replies.json')
with open(instagram_data_path, "r", encoding="raw-unicode-escape") as posts:
post_comments_1 = json.loads(convert_encoding_meta(posts.read()))
#%% In[ ]:
threads_comments = []
for post in post_comments_1['text_post_app_text_posts']:
for element in post['media']:
threads_comments.append({"texte": element['title'],
'creation_timestamp': element['creation_timestamp'],
"chemin": instagram_data_path,
"index": "rs_instagram_threads",
"type": "posts",
"network": "Instagram"})
#%% In[ ]:
ig_comments_df = pd.DataFrame(threads_comments)
#%% In[ ]:
ig_comments_df.fillna(value="", inplace=True)
#%% In[ ]:
ig_comments_df.drop_duplicates(subset=['texte', 'creation_timestamp'], inplace=True)
#%% In[ ]:
# Filter empty texte
ig_comments_df = ig_comments_df[~ig_comments_df['texte'].str.strip('\n').str.strip().eq('')]
#%% In[ ]:
documents_to_database(ig_comments_df)