Ajout de Threads dans les données importées

import_data/24_importation_instagram_post_comments.py

@@ -3,7 +3,6 @@ import datetime
 import pandas as pd
 import json
 
-from utils.get_ids import get_idtypedocument, get_idreseausocial
 from utils.documents_to_database import documents_to_database
 from utils.convert_encoding_meta import convert_encoding_meta
 

import_data/26_importation_threads.py

@@ -0,0 +1,40 @@
+import datetime
+
+import pandas as pd
+import json
+
+from utils.documents_to_database import documents_to_database
+from utils.convert_encoding_meta import convert_encoding_meta
+
+# In[ ]:
+instagram_data_path = 'data/Instagram/threads/threads_and_replies.json'
+with open(instagram_data_path, "r", encoding="raw-unicode-escape") as posts:
+    post_comments_1 = json.loads(convert_encoding_meta(posts.read()))
+
+# In[ ]:
+threads_comments = []
+for post in post_comments_1['text_post_app_text_posts']:
+    for element in post['media']:
+        threads_comments.append({"texte": element['title'],
+                                 'datepublication': datetime.datetime.fromtimestamp(
+                                     timestamp=element['creation_timestamp']).isoformat(),
+                                 "chemin": instagram_data_path,
+                                 "index": "rs_instagram_threads",
+                                 "type": "posts",
+                                 "network": "Instagram"})
+
+# In[ ]:
+ig_comments_df = pd.DataFrame(threads_comments)
+
+# In[ ]:
+ig_comments_df.fillna(value="", inplace=True)
+
+# In[ ]:
+ig_comments_df.drop_duplicates(subset=['texte', 'datepublication'], inplace=True)
+
+# In[ ]:
+# Filter empty texte
+ig_comments_df = ig_comments_df[~ig_comments_df['texte'].str.strip('\n').str.strip().eq('')]
+
+# In[ ]:
+documents_to_database(ig_comments_df)