#!/usr/bin/env python
# coding: utf-8

import scrapy
from scrapy.crawler import CrawlerProcess
from scrapy.exporters import JsonLinesItemExporter
from scrapy.item import Item, Field

class DiversityTermItem(Item):
    term = Field()
    definition = Field()

class DiversityTermItemSpider(scrapy.Spider):
    name = "diversitytermitemspider"
    def start_requests(self):
        urls = ["https://blog.ongig.com/diversity-and-inclusion/diversity-terms/"]
        for url in urls:
            yield scrapy.Request(url=url, callback=self.parse)
    
    def parse(self, response):
        words = response.css(".post_content > p")
        for word in words:
            try:
                data = DiversityTermItem({
                    'term': word.css('strong::Text').get().strip('\xa0— '),
                    'definition': word.get().replace(word.css('strong').get(),'').strip('\xa0— ')
                })
                yield data
            except:
                continue

process = CrawlerProcess({
    'AUTOTHROTTLE_ENABLED': 'True',
    'AUTOTHROTTLE_START_DELAY': 1.0,
    'HTTPCACHE_ENABLED': False,
    'COOKIES_ENABLED': False,
    'FEEDS': {
        'scraped_data/ongig-diversity-terms.json':{
            'format': 'json',
        'encoding': 'utf8',
        'store_empty': False,
        'fields': None,
        'indent': 4,
        'item_export_kwargs': {
           'export_empty_fields': True,
        }
        }
    }
})

process.crawl(DiversityTermItemSpider)
process.start()
process.stop()

