#!/usr/bin/env python
# coding: utf-8
import datetime

import pandas as pd
import scrapy
from scrapy.crawler import CrawlerProcess
from scrapy.item import Item, Field
import html2text
import re
from dateutil.parser import parse


class GoodReadsBookItem(Item):
    bookTitle = Field()
    bookSeries = Field()
    authorURL = Field()
    authorName = Field()
    description = Field()
    ratingValue = Field()
    ratingCount = Field()
    reviewCount = Field()
    bookFormat = Field()
    numberOfPages = Field()
    datePublished = Field()
    isbn = Field()


class GoodReadsBookSpider(scrapy.Spider):
    name = "goodreadsbookspider"

    def start_requests(self):
        urls = pd.read_json("scraped_data/goodreadslist.json").href
        for url in urls:
            yield scrapy.Request(url=url,
                                 callback=self.parse,
                                 dont_filter=True
                                 )

    def clean_string(self, strExtract):
        if strExtract is None:
            x = ""
        else:
            x = strExtract.strip('\n ')
        return x

    def split_published_date(self, strPublisher):
        try:
            datepublished_str = re.search(
                r'^.*Published\s(\w+\s\d+\w*\s\d+)\sby.*',
                strPublisher).group(1)
            datepublished = parse(datepublished_str).isoformat()
        except:
            datepublished = datetime.datetime.now()
        return datepublished

    def parse(self, response):
        metacol = response.css('#metacol')

        converter = html2text.HTML2Text()
        converter.ignore_links = True

        data = GoodReadsBookItem(
            {
                'bookTitle': self.clean_string(metacol.css(
                    'h1#bookTitle::text').extract_first()),
                'bookSeries': self.clean_string(metacol.css(
                    'h2#bookSeries::text').extract_first()),
                'authorURL': self.clean_string(metacol.css(
                    '.authorName').attrib["href"]),
                'authorName': self.clean_string(metacol.css(
                    '.authorName > span:nth-child(1)::text')
                                                .extract_first()),
                'description': converter.handle(metacol.css(
                    '#descriptionContainer').extract_first()),
                'ratingValue': self.clean_string(metacol.css(
                    '#bookMeta > span:nth-child(2)::text')
                                                 .extract_first()),
                'ratingCount': self.clean_string(metacol.css(
                    'a.gr-hyperlink:nth-child(7) > '
                    'meta:nth-child(1)')
                                                 .attrib[
                                                     'content']),
                'reviewCount': self.clean_string(metacol.css(
                    'a.gr-hyperlink:nth-child(9) > '
                    'meta:nth-child(1)')
                                                 .attrib[
                                                     'content']),
                'bookFormat': self.clean_string(metacol.css(
                    '#details > div:nth-child(1) >'
                    ' span:nth-child(1)::text').extract_first()),
                'numberOfPages': self.clean_string(metacol.css(
                    '#details > div:nth-child(1) >'
                    ' span:nth-child(2)::text').extract_first())
                    .replace(
                    r' pages', ''),
                'datePublished': self.split_published_date(
                    self.clean_string(metacol.css(
                        '#details > div:nth-child(2)')
                                      .extract_first())),
                'isbn': self.clean_string(metacol.css(
                    'div.infoBoxRowItem:nth-child(2) >'
                    ' span:nth-child(1) > span:nth-child(1)::text')
                                          .extract_first())
            }
        )

        if data['bookTitle'] != "":
            yield data
        else:
            pass


process = CrawlerProcess({
    'AUTOTHROTTLE_ENABLED': 'True',
    'AUTOTHROTTLE_START_DELAY': 1.0,
    'HTTPCACHE_ENABLED': False,
    'COOKIES_ENABLED': False,
    'DUPEFILTER_CLASS': 'scrapy.dupefilters.BaseDupeFilter',
    'FEEDS': {
        'goodreadsbooks_new.json': {
            'format': 'json',
            'encoding': 'utf8',
            'store_empty': False,
            'fields': None,
            'indent': 4,
            'item_export_kwargs': {
                'export_empty_fields': True,
            }
        }
    }
})

process.crawl(GoodReadsBookSpider)
process.start()
process.stop()
