44 lines
1.5 KiB
Python
44 lines
1.5 KiB
Python
|
import datetime
|
||
|
|
|
||
|
|
import pandas as pd
|
||
|
|
import json
|
||
|
|
|
||
|
|
from utils.get_ids import get_idtypedocument, get_idreseausocial
|
||
|
|
from utils.documents_to_database import documents_to_database
|
||
|
|
from utils.convert_encoding_meta import convert_encoding_meta
|
||
|
|
|
||
|
|
# In[ ]:
|
||
|
|
fb_data_path = ['data/Facebook/comments_and_reactions/comments.json']
|
||
|
|
with open(fb_data_path[0], "r", encoding="raw-unicode-escape") as posts:
|
||
|
|
comments_json = json.loads(convert_encoding_meta(posts.read()))
|
||
|
|
|
||
|
|
# In[ ]:
|
||
|
|
facebook_comments = []
|
||
|
|
for comment in comments_json['comments_v2']:
|
||
|
|
if comment.get('data'):
|
||
|
|
for data_item in comment['data']:
|
||
|
|
if data_item.get('comment'):
|
||
|
|
comment = data_item['comment']
|
||
|
|
facebook_comments.append({"network": "Facebook",
|
||
|
|
"type": "comments",
|
||
|
|
"index": "rs_facebook_comments_and_reactions",
|
||
|
|
"chemin": fb_data_path[0],
|
||
|
|
"texte": comment["comment"],
|
||
|
|
"creation_timestamp": comment["timestamp"]})
|
||
|
|
|
||
|
|
# In[ ]:
|
||
|
|
facebook_comments_df = pd.DataFrame(facebook_comments)
|
||
|
|
|
||
|
|
# In[ ]:
|
||
|
|
facebook_comments_df['datepublication'] = facebook_comments_df['creation_timestamp'].apply(
|
||
|
|
lambda x: datetime.datetime.fromtimestamp(x).isoformat())
|
||
|
|
|
||
|
|
# In[ ]:
|
||
|
|
facebook_comments_df.fillna(value="", inplace=True)
|
||
|
|
|
||
|
|
# In[ ]:
|
||
|
|
del facebook_comments_df['creation_timestamp']
|
||
|
|
|
||
|
|
# In[ ]:
|
||
|
|
documents_to_database(facebook_comments_df)
|