import os
import re
import pandas as pd
import requests
from urllib.parse import unquote
import newspaper

def get_comments(file_path):
    df = pd.read_excel(file_path, skiprows=5, 
                       names=['comment_id',
                              'nested_id',
                              'name',
                              'id',
                              'date',
                              'likes',
                              'comment',
                              'source'])
    df["comment_id"] = df["comment_id"].mask(pd.isnull,df["nested_id"].mask(pd.isnull,"0-0").apply(lambda x: re.match(r"([0-9]+)\-([0-9]+)",x).group(1)))
    df["nested_id"] = df["nested_id"].mask(pd.isnull,"0-0").apply(lambda x: re.match(r"([0-9]+)\-([0-9]+)",x).group(2))
    del df["source"]
    return df

def get_text_article(file_path):
    url = pd.read_excel(file_path,skiprows=1,nrows=1,header=None,names=['source','url']+['']*6)['url'][0]
    request_url = requests.get(url)
    html_content = str(request_url.content)
    link_urls = re.findall(r'http[s]?://l\.facebook(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+',html_content)
    u_link_urls = [unquote(unquote(u.replace("https://l.facebook.com/l.php?u=",""))) for u in link_urls]
    article = newspaper.Article('')
    try:
        url_article = [re.search("(.*)&amp;.*",u).group(1) for u in u_link_urls][-3]
        request_url_article = requests.get(url_article)
        article.set_html(request_url_article.content)
    except:
        article.set_html(html_content)
    try:
        article.parse()
        text_article = article.text.replace("\n\n"," ")
    except:
        text_article = ""
    return text_article

def getListOfFiles(dirName):
    # create a list of file and sub directories 
    # names in the given directory 
    listOfFile = os.listdir(dirName)
    allFiles = list()
    # Iterate over all the entries
    for entry in listOfFile:
        # Create full path
        fullPath = os.path.join(dirName, entry)
        # If entry is a directory then get the list of files in this directory 
        if os.path.isdir(fullPath):
            allFiles = allFiles + getListOfFiles(fullPath)
        else:
            allFiles.append(fullPath)
    return allFiles