2024-07-19 00:04:51 +00:00
|
|
|
import pandas as pd
|
|
|
|
import datetime
|
|
|
|
|
2024-10-03 01:53:37 +00:00
|
|
|
import os
|
|
|
|
from pathlib import Path
|
|
|
|
|
2024-07-19 00:04:51 +00:00
|
|
|
from utils.documents_to_database import documents_to_database
|
|
|
|
|
2024-10-03 01:53:37 +00:00
|
|
|
#%% In[ ]:
|
|
|
|
# Get the current file's directory
|
|
|
|
try:
|
|
|
|
# This will work when running as a script
|
|
|
|
script_dir = Path(__file__).parent.parent
|
|
|
|
except NameError:
|
|
|
|
# This will work in interactive environments
|
|
|
|
script_dir = Path().absolute()
|
|
|
|
|
|
|
|
project_root = script_dir
|
|
|
|
linkedin_data_path = os.path.join(project_root, 'import_data', 'data', 'LinkedIn', 'shares', 'Shares.csv')
|
|
|
|
|
2024-07-19 00:04:51 +00:00
|
|
|
raw_shares = pd.read_csv(linkedin_data_path)
|
|
|
|
|
2024-10-03 01:53:37 +00:00
|
|
|
#%% In[ ]:
|
2024-07-19 00:04:51 +00:00
|
|
|
raw_shares['index'] = "rs_linkedin_shares"
|
|
|
|
raw_shares['type'] = "posts"
|
|
|
|
raw_shares['network'] = "LinkedIn"
|
|
|
|
raw_shares['chemin'] = linkedin_data_path
|
|
|
|
|
2024-10-03 01:53:37 +00:00
|
|
|
#%% In[ ]:
|
|
|
|
raw_shares["creation_timestamp"] = raw_shares["Date"].apply(
|
|
|
|
lambda x: int(datetime.datetime.fromisoformat(x).timestamp())
|
|
|
|
)
|
2024-07-19 00:04:51 +00:00
|
|
|
del raw_shares["Date"]
|
|
|
|
|
2024-10-03 01:53:37 +00:00
|
|
|
#%% In[ ]:
|
2024-07-19 00:04:51 +00:00
|
|
|
raw_shares.rename(columns={"ShareLink": "uri", "ShareCommentary": "texte"}, inplace=True)
|
|
|
|
|
2024-10-03 01:53:37 +00:00
|
|
|
#%% In[ ]:
|
2024-07-19 00:04:51 +00:00
|
|
|
raw_shares["texte"] = raw_shares["texte"].apply(lambda x: str(x))
|
|
|
|
|
2024-10-03 01:53:37 +00:00
|
|
|
#%% In[ ]:
|
2024-07-19 00:04:51 +00:00
|
|
|
del raw_shares["SharedUrl"]
|
|
|
|
del raw_shares["MediaUrl"]
|
|
|
|
del raw_shares["Visibility"]
|
|
|
|
|
2024-10-03 01:53:37 +00:00
|
|
|
#%% In[ ]:
|
2024-07-19 00:04:51 +00:00
|
|
|
raw_shares.fillna(value="", inplace=True)
|
|
|
|
|
2024-10-03 01:53:37 +00:00
|
|
|
#%% In[ ]:
|
|
|
|
raw_shares.drop_duplicates(subset=['texte', 'creation_timestamp'], inplace=True)
|
2024-07-19 00:04:51 +00:00
|
|
|
|
2024-10-03 01:53:37 +00:00
|
|
|
#%% In[ ]:
|
2024-07-19 00:04:51 +00:00
|
|
|
# Filter empty texte
|
|
|
|
raw_shares = raw_shares[~raw_shares['texte'].str.strip('\n').str.strip().eq('')]
|
|
|
|
|
2024-10-03 01:53:37 +00:00
|
|
|
#%% In[ ]:
|
2024-07-19 00:04:51 +00:00
|
|
|
documents_to_database(raw_shares)
|
|
|
|
|