import datetime

import pandas as pd
import json

from utils.get_ids import get_idtypedocument, get_idreseausocial
from utils.documents_to_database import documents_to_database
from utils.convert_encoding_meta import convert_encoding_meta

# In[ ]:
instagram_data_path = 'data/Instagram/comments/post_comments_1.json'
with open(instagram_data_path, "r", encoding="raw-unicode-escape") as posts:
    post_comments_1 = json.loads(convert_encoding_meta(posts.read()))

# In[ ]:
ig_comments = []
for comment in post_comments_1:
    ig_comments.append({"texte": comment['string_map_data']['Comment']['value'],
                        'datepublication': datetime.datetime.fromtimestamp(
                            timestamp=comment['string_map_data']['Time']['timestamp']).isoformat(),
                        "chemin": instagram_data_path,
                        "index": "rs_instagram_comments",
                        "type": "comments",
                        "network": "Instagram"})

# In[ ]:
ig_comments_df = pd.DataFrame(ig_comments)

# In[ ]:
ig_comments_df.fillna(value="", inplace=True)

# In[ ]:
ig_comments_df.drop_duplicates(subset=['texte', 'datepublication'], inplace=True)

# In[ ]:
# Filter empty texte
ig_comments_df = ig_comments_df[~ig_comments_df['texte'].str.strip('\n').str.strip().eq('')]

# In[ ]:
documents_to_database(ig_comments_df)
