import scrapy
from scrapy.crawler import CrawlerProcess
import re


"""
feature categories:
1. PLANETS/POINTS in the SIGNS (e.g. Sun in Aries)
    - there is usually 1 (but up to 2) feature/s per planet/point. e.g. sun may be close 
    to 2 different signs, so that the birth chart indicates the planet/point being in 2 
    signs
    
2. PLANETS/POINTS in the HOUSES (e.g. Moon in 1st House)
    - there are 1 or 2 features per planet/point.(Same explanation as above).

3. SIGNS on the HOUSE CUSPS (e.g. Aries on 1st House Cusp)

4. ASPECTS between two planets/points(for now the website seems to only have sun-related)
    - e.g. Sun conjunct Mercury

5. DOMINANCE/WEAKNESS OF ELEMENTS
    - e.g. Dominance of FIRE, Weakness of Water

6. DOMINANCE/WEAKNESS OF QUALITIES
    - 3 QUALITIES: Mutable, Fixed, Cardinal
    - same as for elements: the feature is based on a DOMINANCE or WEAKNESS of the QUALITY

each feature is potentially associated with a description/interpretation of the person who
bears this feature.

BIG starting question: how do I structure the corpus?
Maybe?:
FEATURE_CAT | FEATURE | DESC
===============================
int         | Str     | Str

So, we start w a given birth chart (generated by inputting birth info to the website),
then from the birth chart we extract features. then we select rows based on the features.
and compare the various descriptions for consistency/conflict.

"""


class AstroSpider(scrapy.Spider):
    name = 'astro_spider'
    # start_urls = ['https://astrolibrary.org/interpretations/']
    # start_urls = ['https://astrolibrary.org/interpretations/category/planets-in-signs/']
    start_urls = ['https://astrolibrary.org/interpretations/sun/']
    desc_links = []
    data = []

    # find category links from /interpretations
    def parse2(self, response):
        INTERPS_SELECTOR = '.interps-list-main'
        LINK_SELECTOR = 'a ::attr(href)'
        for interp in response.css(INTERPS_SELECTOR):
            next_page = interp.css(LINK_SELECTOR).extract_first()
            self.main_links.append(next_page)
            if next_page:
                yield scrapy.Request(
                    response.urljoin(next_page),
                    callback=self.page2
                )
                print(next_page)

    # first find links to descriptions from /category/planets-in-signs
    def parse1(self, response):
        LIST_SELECTOR = '.listnice'
        LI_SELECTOR = 'li'
        LINK_SELECTOR = 'a ::attr(href)'

        for nice in response.css(LIST_SELECTOR):
            for li in nice.css(LI_SELECTOR):
                next_page = li.css(LINK_SELECTOR).extract_first()

                if next_page:
                    # todo: exclude links that have the substring 'category'
                    # we want links that bring us to the page of descriptions on a pair
                    # of features.
                    self.desc_links.append(next_page)
                    yield scrapy.Request(
                        response.urljoin(next_page),
                        callback=self.nothing
                    )
                    print(next_page)

    def parse(self, response):
        # find <div id="ris">
        # split descriptions with <h2> tag
        # for each <h2> tag, extract feature name.
        # use each <h2> tag, extract descriptions from <p> tags, making sure to remove
        # google ads.
        selected = response.xpath('//div[@id="ris"]').extract()
        x = re.split("<h2", selected[0])
        print(x)
        # I've gotten the all the h2 and p tags in this long string. But i need to split
        # it up.

    def parse_pof(self, response):
        # use this method if the preceding URL contained "part-of-fortune". This is due to
        # the fact that the descriptions for part-of-fortune are organized and formatted
        # differently.
        pass

    def nothing(self, response):
        pass


def main():
    process = CrawlerProcess(settings={
        'FEED_FORMAT': 'csv',
        'FEED_URI': 'items.csv'
    })

    process.crawl(AstroSpider)
    process.start()  # the script will block here until the crawling is finished
    # print(AstroSpider.data)
    # todo: remove final links that have the substring 'category'


if __name__ == '__main__':
    main()
aae1c8f8c9ee4d798fd0a6cbc12d66d5b64ac9c8
<mask token>


class AstroSpider(scrapy.Spider):
    <mask token>
    <mask token>
    <mask token>
    <mask token>

    def parse2(self, response):
        INTERPS_SELECTOR = '.interps-list-main'
        LINK_SELECTOR = 'a ::attr(href)'
        for interp in response.css(INTERPS_SELECTOR):
            next_page = interp.css(LINK_SELECTOR).extract_first()
            self.main_links.append(next_page)
            if next_page:
                yield scrapy.Request(response.urljoin(next_page), callback=
                    self.page2)
                print(next_page)
    <mask token>

    def parse(self, response):
        selected = response.xpath('//div[@id="ris"]').extract()
        x = re.split('<h2', selected[0])
        print(x)
    <mask token>

    def nothing(self, response):
        pass


<mask token>

<mask token>


class AstroSpider(scrapy.Spider):
    <mask token>
    <mask token>
    <mask token>
    <mask token>

    def parse2(self, response):
        INTERPS_SELECTOR = '.interps-list-main'
        LINK_SELECTOR = 'a ::attr(href)'
        for interp in response.css(INTERPS_SELECTOR):
            next_page = interp.css(LINK_SELECTOR).extract_first()
            self.main_links.append(next_page)
            if next_page:
                yield scrapy.Request(response.urljoin(next_page), callback=
                    self.page2)
                print(next_page)

    def parse1(self, response):
        LIST_SELECTOR = '.listnice'
        LI_SELECTOR = 'li'
        LINK_SELECTOR = 'a ::attr(href)'
        for nice in response.css(LIST_SELECTOR):
            for li in nice.css(LI_SELECTOR):
                next_page = li.css(LINK_SELECTOR).extract_first()
                if next_page:
                    self.desc_links.append(next_page)
                    yield scrapy.Request(response.urljoin(next_page),
                        callback=self.nothing)
                    print(next_page)

    def parse(self, response):
        selected = response.xpath('//div[@id="ris"]').extract()
        x = re.split('<h2', selected[0])
        print(x)

    def parse_pof(self, response):
        pass

    def nothing(self, response):
        pass


<mask token>

<mask token>


class AstroSpider(scrapy.Spider):
    name = 'astro_spider'
    start_urls = ['https://astrolibrary.org/interpretations/sun/']
    desc_links = []
    data = []

    def parse2(self, response):
        INTERPS_SELECTOR = '.interps-list-main'
        LINK_SELECTOR = 'a ::attr(href)'
        for interp in response.css(INTERPS_SELECTOR):
            next_page = interp.css(LINK_SELECTOR).extract_first()
            self.main_links.append(next_page)
            if next_page:
                yield scrapy.Request(response.urljoin(next_page), callback=
                    self.page2)
                print(next_page)

    def parse1(self, response):
        LIST_SELECTOR = '.listnice'
        LI_SELECTOR = 'li'
        LINK_SELECTOR = 'a ::attr(href)'
        for nice in response.css(LIST_SELECTOR):
            for li in nice.css(LI_SELECTOR):
                next_page = li.css(LINK_SELECTOR).extract_first()
                if next_page:
                    self.desc_links.append(next_page)
                    yield scrapy.Request(response.urljoin(next_page),
                        callback=self.nothing)
                    print(next_page)

    def parse(self, response):
        selected = response.xpath('//div[@id="ris"]').extract()
        x = re.split('<h2', selected[0])
        print(x)

    def parse_pof(self, response):
        pass

    def nothing(self, response):
        pass


def main():
    process = CrawlerProcess(settings={'FEED_FORMAT': 'csv', 'FEED_URI':
        'items.csv'})
    process.crawl(AstroSpider)
    process.start()


<mask token>

<mask token>


class AstroSpider(scrapy.Spider):
    name = 'astro_spider'
    start_urls = ['https://astrolibrary.org/interpretations/sun/']
    desc_links = []
    data = []

    def parse2(self, response):
        INTERPS_SELECTOR = '.interps-list-main'
        LINK_SELECTOR = 'a ::attr(href)'
        for interp in response.css(INTERPS_SELECTOR):
            next_page = interp.css(LINK_SELECTOR).extract_first()
            self.main_links.append(next_page)
            if next_page:
                yield scrapy.Request(response.urljoin(next_page), callback=
                    self.page2)
                print(next_page)

    def parse1(self, response):
        LIST_SELECTOR = '.listnice'
        LI_SELECTOR = 'li'
        LINK_SELECTOR = 'a ::attr(href)'
        for nice in response.css(LIST_SELECTOR):
            for li in nice.css(LI_SELECTOR):
                next_page = li.css(LINK_SELECTOR).extract_first()
                if next_page:
                    self.desc_links.append(next_page)
                    yield scrapy.Request(response.urljoin(next_page),
                        callback=self.nothing)
                    print(next_page)

    def parse(self, response):
        selected = response.xpath('//div[@id="ris"]').extract()
        x = re.split('<h2', selected[0])
        print(x)

    def parse_pof(self, response):
        pass

    def nothing(self, response):
        pass


def main():
    process = CrawlerProcess(settings={'FEED_FORMAT': 'csv', 'FEED_URI':
        'items.csv'})
    process.crawl(AstroSpider)
    process.start()


if __name__ == '__main__':
    main()

import scrapy
from scrapy.crawler import CrawlerProcess
import re


"""
feature categories:
1. PLANETS/POINTS in the SIGNS (e.g. Sun in Aries)
    - there is usually 1 (but up to 2) feature/s per planet/point. e.g. sun may be close 
    to 2 different signs, so that the birth chart indicates the planet/point being in 2 
    signs
    
2. PLANETS/POINTS in the HOUSES (e.g. Moon in 1st House)
    - there are 1 or 2 features per planet/point.(Same explanation as above).

3. SIGNS on the HOUSE CUSPS (e.g. Aries on 1st House Cusp)

4. ASPECTS between two planets/points(for now the website seems to only have sun-related)
    - e.g. Sun conjunct Mercury

5. DOMINANCE/WEAKNESS OF ELEMENTS
    - e.g. Dominance of FIRE, Weakness of Water

6. DOMINANCE/WEAKNESS OF QUALITIES
    - 3 QUALITIES: Mutable, Fixed, Cardinal
    - same as for elements: the feature is based on a DOMINANCE or WEAKNESS of the QUALITY

each feature is potentially associated with a description/interpretation of the person who
bears this feature.

BIG starting question: how do I structure the corpus?
Maybe?:
FEATURE_CAT | FEATURE | DESC
===============================
int         | Str     | Str

So, we start w a given birth chart (generated by inputting birth info to the website),
then from the birth chart we extract features. then we select rows based on the features.
and compare the various descriptions for consistency/conflict.

"""


class AstroSpider(scrapy.Spider):
    name = 'astro_spider'
    # start_urls = ['https://astrolibrary.org/interpretations/']
    # start_urls = ['https://astrolibrary.org/interpretations/category/planets-in-signs/']
    start_urls = ['https://astrolibrary.org/interpretations/sun/']
    desc_links = []
    data = []

    # find category links from /interpretations
    def parse2(self, response):
        INTERPS_SELECTOR = '.interps-list-main'
        LINK_SELECTOR = 'a ::attr(href)'
        for interp in response.css(INTERPS_SELECTOR):
            next_page = interp.css(LINK_SELECTOR).extract_first()
            self.main_links.append(next_page)
            if next_page:
                yield scrapy.Request(
                    response.urljoin(next_page),
                    callback=self.page2
                )
                print(next_page)

    # first find links to descriptions from /category/planets-in-signs
    def parse1(self, response):
        LIST_SELECTOR = '.listnice'
        LI_SELECTOR = 'li'
        LINK_SELECTOR = 'a ::attr(href)'

        for nice in response.css(LIST_SELECTOR):
            for li in nice.css(LI_SELECTOR):
                next_page = li.css(LINK_SELECTOR).extract_first()

                if next_page:
                    # todo: exclude links that have the substring 'category'
                    # we want links that bring us to the page of descriptions on a pair
                    # of features.
                    self.desc_links.append(next_page)
                    yield scrapy.Request(
                        response.urljoin(next_page),
                        callback=self.nothing
                    )
                    print(next_page)

    def parse(self, response):
        # find <div id="ris">
        # split descriptions with <h2> tag
        # for each <h2> tag, extract feature name.
        # use each <h2> tag, extract descriptions from <p> tags, making sure to remove
        # google ads.
        selected = response.xpath('//div[@id="ris"]').extract()
        x = re.split("<h2", selected[0])
        print(x)
        # I've gotten the all the h2 and p tags in this long string. But i need to split
        # it up.

    def parse_pof(self, response):
        # use this method if the preceding URL contained "part-of-fortune". This is due to
        # the fact that the descriptions for part-of-fortune are organized and formatted
        # differently.
        pass

    def nothing(self, response):
        pass


def main():
    process = CrawlerProcess(settings={
        'FEED_FORMAT': 'csv',
        'FEED_URI': 'items.csv'
    })

    process.crawl(AstroSpider)
    process.start()  # the script will block here until the crawling is finished
    # print(AstroSpider.data)
    # todo: remove final links that have the substring 'category'


if __name__ == '__main__':
    main()