diff --git a/import_data/24_importation_instagram_post_comments.py b/import_data/24_importation_instagram_post_comments.py index d847a83..5292924 100644 --- a/import_data/24_importation_instagram_post_comments.py +++ b/import_data/24_importation_instagram_post_comments.py @@ -3,7 +3,6 @@ import datetime import pandas as pd import json -from utils.get_ids import get_idtypedocument, get_idreseausocial from utils.documents_to_database import documents_to_database from utils.convert_encoding_meta import convert_encoding_meta diff --git a/import_data/26_importation_threads.py b/import_data/26_importation_threads.py new file mode 100644 index 0000000..34f899d --- /dev/null +++ b/import_data/26_importation_threads.py @@ -0,0 +1,40 @@ +import datetime + +import pandas as pd +import json + +from utils.documents_to_database import documents_to_database +from utils.convert_encoding_meta import convert_encoding_meta + +# In[ ]: +instagram_data_path = 'data/Instagram/threads/threads_and_replies.json' +with open(instagram_data_path, "r", encoding="raw-unicode-escape") as posts: + post_comments_1 = json.loads(convert_encoding_meta(posts.read())) + +# In[ ]: +threads_comments = [] +for post in post_comments_1['text_post_app_text_posts']: + for element in post['media']: + threads_comments.append({"texte": element['title'], + 'datepublication': datetime.datetime.fromtimestamp( + timestamp=element['creation_timestamp']).isoformat(), + "chemin": instagram_data_path, + "index": "rs_instagram_threads", + "type": "posts", + "network": "Instagram"}) + +# In[ ]: +ig_comments_df = pd.DataFrame(threads_comments) + +# In[ ]: +ig_comments_df.fillna(value="", inplace=True) + +# In[ ]: +ig_comments_df.drop_duplicates(subset=['texte', 'datepublication'], inplace=True) + +# In[ ]: +# Filter empty texte +ig_comments_df = ig_comments_df[~ig_comments_df['texte'].str.strip('\n').str.strip().eq('')] + +# In[ ]: +documents_to_database(ig_comments_df)