import scrapy

"""
You need to set 
USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'

ROBOTSTXT_OBEY = False
in the setting.py
"""
class CHI17Spider(scrapy.Spider):
    name = "acmCHI17"
    start_urls = [
        'https://dl.acm.org/results.cfm?query=chi%202017&start=0&filtered=acmdlPublicationTitle%2Eraw%3DCHI%20%2717%3A%20Proceedings%20of%20the%202017%20CHI%20Conference%20on%20Human%20Factors%20in%20Computing%20Systems&within=owners%2Eowner%3DHOSTED&dte=&bfr=&srt=_score',
    ]

    def parse_paper(self, response):
        # here we find if the badge is in the detail page
        tag = ''
        if 'Best Paper' in response.text:
            tag = 'Best Paper'
        if 'Honorable Mention' in response.text:
            tag = 'Honorable Mention'
        if len(tag) > 0:
            title = response.css('#divmain div.large-text h1::text').extract_first().strip()
            print (tag+',', title)

    def parse(self, response):
        # here we parse the search result page
        # for each result we go to the detail page
        for detail in response.css('div.details div.title a::attr(href)'):
            yield response.follow(detail, self.parse_paper)

        log = response.css("div.pagelogic")[0]
        current_page = int(log.css('span strong::text').extract()[0])
        # here we go the next page of the serach result
        for a in log.css('span a'):
            try:
                if int(a.css('::text').extract_first()) > current_page:
                    yield response.follow(a.css("::attr(href)").extract_first(), self.parse)
            except ValueError:
                continue