libere-tes-chaine-de-mots/import_data/41_importation_wordpress.py

import datetime
import re

import xmltodict
import pandas as pd
import markdownify

import os
from pathlib import Path

from utils.documents_to_database import documents_to_database

#%% In[ ]:
# Get the current file's directory
try:
    # This will work when running as a script
    script_dir = Path(__file__).parent.parent
except NameError:
    # This will work in interactive environments
    script_dir = Path().absolute()

project_root = script_dir
wordpress_xml_path = os.path.join(project_root, 'import_data', 'data', 'Wordpress', 'jevalideca', 'wordpress.xml')

with open(wordpress_xml_path, "r") as xml_file:
    wordpress_xml = xml_file.read()

#%% In[ ]:
wordpress_dict = xmltodict.parse(wordpress_xml)

#%% In[ ]:
items_df = pd.DataFrame(wordpress_dict['rss']['channel']['item'])

#%% In[ ]:
items_df_filter = items_df[
    (items_df['wp:post_type'].isin(['page', 'post'])) & (items_df['wp:status'] == 'publish')].copy()

#%% In[ ]:
items_df_filter['creation_timestamp'] = items_df_filter['wp:post_date'].apply(
    lambda x: int(datetime.datetime.fromisoformat(x).timestamp()))


#%% In[ ]:
def wp_to_markdown(x):
    try:
        md_text = re.sub(r'\n+', ' ', markdownify.markdownify(x, heading_style='ATX')).strip()
    except Exception as e:
        print(e)
        md_text = str()
        pass
    return md_text


#%% In[ ]:
items_df_filter['texte'] = items_df_filter['content:encoded'].apply(lambda x: wp_to_markdown(x))

#%% In[ ]:
items_df_filter.rename(columns={"link": "uri", "wp:post_type": "type"}, inplace=True)

#%% In[ ]:
items_df_filter['index'] = "rs_wordpress_jevalideca"
items_df_filter['network'] = "Wordpress"
items_df_filter['chemin'] = wordpress_xml_path

#%% In[ ]:
items_df_filter.fillna(value="", inplace=True)

#%% In[ ]:
documents_to_database(items_df_filter[['title',
                                       'uri',
                                       'type',
                                       'creation_timestamp',
                                       'texte',
                                       'index',
                                       'network',
                                       'chemin']])