#!/usr/bin/env python
# coding: utf-8


import scrapy
from scrapy.crawler import CrawlerProcess
from scrapy.item import Item, Field


class GoodReadsListItem(Item):
    title = Field()
    href = Field()


class GoodReadsListSpider(scrapy.Spider):
    name = "goodreadslistspider"

    def start_requests(self):
        urls = [
            "https://www.goodreads.com/list/show/"
            "168413.Banned_Books_According_to_Krause_Part_1_",
            "https://www.goodreads.com/list/show/"
            "168429.Banned_Books_According_to_Krause_Part_7",
            "https://www.goodreads.com/list/show/"
            "168430.Banned_Books_According_to_Krause_Part_8",
            "https://www.goodreads.com/list/show/"
            "168425.Banned_Books_According_to_Krause_Part_4",
            "https://www.goodreads.com/list/show/"
            "168420.Banned_Books_According_to_Krause_Part_2",
            "https://www.goodreads.com/list/show/"
            "168424.Banned_Books_According_to_Krause_Part_3",
            "https://www.goodreads.com/list/show/"
            "168428.Banned_Books_According_to_Krause_Part_6",
            "https://www.goodreads.com/list/show/"
            "168426.Banned_Books_According_to_Krause_Part_5",
            "https://www.goodreads.com/list/show/"
            "168432.Banned_Books_According_to_Krause_Part_9"
            ]
        for url in urls:
            yield scrapy.Request(url=url, callback=self.parse)

    def parse(self, response):
        page = response.url.split("/")
        anchors = response.css("a.bookTitle")
        for anchor in anchors:
            data = GoodReadsListItem({
                'title': anchor.css('span::text').get(),
                'href': "https://goodreads.com" + anchor.attrib[
                    'href']
            })
            yield data


# Il faut désactiver le cache et les cookies
# sinon le site web a un comportement imprévisible
# et ne retourne pas les bonnes données

process = CrawlerProcess({
    'AUTOTHROTTLE_ENABLED': 'True',
    'AUTOTHROTTLE_START_DELAY': 1.0,
    'HTTPCACHE_ENABLED': False,
    'COOKIES_ENABLED': False,
    'FEEDS': {
        'scraped_data/goodreadslist.json': {
            'format': 'json',
            'encoding': 'utf8',
            'store_empty': False,
            'fields': None,
            'indent': 4,
            'item_export_kwargs': {
                'export_empty_fields': True,
            }
        }
    }
})

process.crawl(GoodReadsListSpider)
process.start()
process.stop()
