66 lines
1.8 KiB
Python
66 lines
1.8 KiB
Python
|
import datetime
|
||
|
|
import re
|
||
|
|
|
||
|
|
import xmltodict
|
||
|
|
import pandas as pd
|
||
|
|
import markdownify
|
||
|
|
|
||
|
|
import utils.config
|
||
|
|
from utils.get_ids import get_idreseausocial, get_idtypedocument
|
||
|
|
from utils.documents_to_database import documents_to_database
|
||
|
|
|
||
|
|
# In[ ]:
|
||
|
|
wordpress_xml_path = "import_data/data/Wordpress/jevalideca/wordpress.xml"
|
||
|
|
with open(wordpress_xml_path, "r") as xml_file:
|
||
|
|
wordpress_xml = xml_file.read()
|
||
|
|
|
||
|
|
# In[ ]:
|
||
|
|
wordpress_dict = xmltodict.parse(wordpress_xml)
|
||
|
|
|
||
|
|
# In[ ]:
|
||
|
|
items_df = pd.DataFrame(wordpress_dict['rss']['channel']['item'])
|
||
|
|
|
||
|
|
# In[ ]:
|
||
|
|
items_df_filter = items_df[
|
||
|
|
(items_df['wp:post_type'].isin(['page', 'post'])) & (items_df['wp:status'] == 'publish')].copy()
|
||
|
|
|
||
|
|
# In[ ]:
|
||
|
|
items_df_filter['datepublication'] = items_df_filter['wp:post_date'].apply(
|
||
|
|
lambda x: str(datetime.datetime.fromisoformat(x).isoformat()))
|
||
|
|
|
||
|
|
|
||
|
|
# In[ ]:
|
||
|
|
def wp_to_markdown(x):
|
||
|
|
try:
|
||
|
|
md_text = re.sub(r'\n+', ' ', markdownify.markdownify(x, heading_style='ATX')).strip()
|
||
|
|
except Exception as e:
|
||
|
|
print(e)
|
||
|
|
md_text = str()
|
||
|
|
pass
|
||
|
|
return md_text
|
||
|
|
|
||
|
|
|
||
|
|
# In[ ]:
|
||
|
|
items_df_filter['texte'] = items_df_filter['content:encoded'].apply(lambda x: wp_to_markdown(x))
|
||
|
|
|
||
|
|
# In[ ]:
|
||
|
|
items_df_filter.rename(columns={"link": "uri", "wp:post_type": "type"}, inplace=True)
|
||
|
|
|
||
|
|
# In[ ]:
|
||
|
|
items_df_filter['index'] = "rs_wordpress_jevalideca"
|
||
|
|
items_df_filter['network'] = "Wordpress"
|
||
|
|
items_df_filter['chemin'] = wordpress_xml_path
|
||
|
|
|
||
|
|
# In[ ]:
|
||
|
|
items_df_filter.fillna(value="", inplace=True)
|
||
|
|
|
||
|
|
# In[ ]:
|
||
|
|
documents_to_database(items_df_filter[['title',
|
||
|
|
'uri',
|
||
|
|
'type',
|
||
|
|
'datepublication',
|
||
|
|
'texte',
|
||
|
|
'index',
|
||
|
|
'network',
|
||
|
|
'chemin']])
|