From f4acc324513b600d344218817e465ad16db2dafe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fran=C3=A7ois=20Pelletier?= <francois@jevalide.ca>
Date: Tue, 23 Jul 2024 00:07:24 -0400
Subject: [PATCH] =?UTF-8?q?Ajout=20de=20Threads=20dans=20les=20donn=C3=A9e?=
 =?UTF-8?q?s=20import=C3=A9es?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../24_importation_instagram_post_comments.py |  1 -
 import_data/26_importation_threads.py         | 40 +++++++++++++++++++
 2 files changed, 40 insertions(+), 1 deletion(-)
 create mode 100644 import_data/26_importation_threads.py

diff --git a/import_data/24_importation_instagram_post_comments.py b/import_data/24_importation_instagram_post_comments.py
index d847a83..5292924 100644
--- a/import_data/24_importation_instagram_post_comments.py
+++ b/import_data/24_importation_instagram_post_comments.py
@@ -3,7 +3,6 @@ import datetime
 import pandas as pd
 import json
 
-from utils.get_ids import get_idtypedocument, get_idreseausocial
 from utils.documents_to_database import documents_to_database
 from utils.convert_encoding_meta import convert_encoding_meta
 
diff --git a/import_data/26_importation_threads.py b/import_data/26_importation_threads.py
new file mode 100644
index 0000000..34f899d
--- /dev/null
+++ b/import_data/26_importation_threads.py
@@ -0,0 +1,40 @@
+import datetime
+
+import pandas as pd
+import json
+
+from utils.documents_to_database import documents_to_database
+from utils.convert_encoding_meta import convert_encoding_meta
+
+# In[ ]:
+instagram_data_path = 'data/Instagram/threads/threads_and_replies.json'
+with open(instagram_data_path, "r", encoding="raw-unicode-escape") as posts:
+    post_comments_1 = json.loads(convert_encoding_meta(posts.read()))
+
+# In[ ]:
+threads_comments = []
+for post in post_comments_1['text_post_app_text_posts']:
+    for element in post['media']:
+        threads_comments.append({"texte": element['title'],
+                                 'datepublication': datetime.datetime.fromtimestamp(
+                                     timestamp=element['creation_timestamp']).isoformat(),
+                                 "chemin": instagram_data_path,
+                                 "index": "rs_instagram_threads",
+                                 "type": "posts",
+                                 "network": "Instagram"})
+
+# In[ ]:
+ig_comments_df = pd.DataFrame(threads_comments)
+
+# In[ ]:
+ig_comments_df.fillna(value="", inplace=True)
+
+# In[ ]:
+ig_comments_df.drop_duplicates(subset=['texte', 'datepublication'], inplace=True)
+
+# In[ ]:
+# Filter empty texte
+ig_comments_df = ig_comments_df[~ig_comments_df['texte'].str.strip('\n').str.strip().eq('')]
+
+# In[ ]:
+documents_to_database(ig_comments_df)