libere-tes-chaine-de-mots/import_data/41_importation_wordpress.py

import datetime
import re

import xmltodict
import pandas as pd
import markdownify

from utils.documents_to_database import documents_to_database

# In[ ]:
wordpress_xml_path = "data/Wordpress/jevalideca/wordpress.xml"
with open(wordpress_xml_path, "r") as xml_file:
    wordpress_xml = xml_file.read()

# In[ ]:
wordpress_dict = xmltodict.parse(wordpress_xml)

# In[ ]:
items_df = pd.DataFrame(wordpress_dict['rss']['channel']['item'])

# In[ ]:
items_df_filter = items_df[
    (items_df['wp:post_type'].isin(['page', 'post'])) & (items_df['wp:status'] == 'publish')].copy()

# In[ ]:
items_df_filter['datepublication'] = items_df_filter['wp:post_date'].apply(
    lambda x: str(datetime.datetime.fromisoformat(x).isoformat()))


# In[ ]:
def wp_to_markdown(x):
    try:
        md_text = re.sub(r'\n+', ' ', markdownify.markdownify(x, heading_style='ATX')).strip()
    except Exception as e:
        print(e)
        md_text = str()
        pass
    return md_text


# In[ ]:
items_df_filter['texte'] = items_df_filter['content:encoded'].apply(lambda x: wp_to_markdown(x))

# In[ ]:
items_df_filter.rename(columns={"link": "uri", "wp:post_type": "type"}, inplace=True)

# In[ ]:
items_df_filter['index'] = "rs_wordpress_jevalideca"
items_df_filter['network'] = "Wordpress"
items_df_filter['chemin'] = wordpress_xml_path

# In[ ]:
items_df_filter.fillna(value="", inplace=True)

# In[ ]:
documents_to_database(items_df_filter[['title',
                                       'uri',
                                       'type',
                                       'datepublication',
                                       'texte',
                                       'index',
                                       'network',
                                       'chemin']])