#! python3
# -*- coding: utf-8 -*-

# ncodeWebScraper.py - Given a WebNovel, scan through each chapter for a keyword
# and return the chapters with the keyword

# Issues: Cannot get this to work with window consoles
# Not sure if this applies to other OSs
# Tested to work in IDLE environment

import bs4 as bs
import requests
import re
from multiprocessing.dummy import Pool as ThreadPool

# Returns the url if the term is found within its text


def term_in_chapter(args):
    searchTerm, chapterURL = args
    pRegex = re.compile(r'L\d+')
    r = requests.get(chapterURL, headers={'user-agent': 'Mozilla/5.0'})
    soup = bs.BeautifulSoup(r.content, 'lxml')
    for paragraph in soup.find_all('p'):
        if paragraph.br:
            continue
        if 'id' in paragraph.attrs:
            if pRegex.search(paragraph.get('id')):
                if searchTerm in paragraph.text:
                    return chapterURL
    return None


# Get User Inputs
print('Enter WebNovel ID:')
wnID = input()
print('WebNovel ID is: {}'.format(wnID))

print('Enter Search Term:')
searchTerm = input()
print('Search Term is: {}'.format(searchTerm))

# Init Variables
base = 'https://ncode.syosetu.com'
wnURL = base + '/' + wnID
chstr = '/' + re.escape(wnID) + r'/\d+'
chapterRegex = re.compile(chstr)
chapters = []

# Get Chapter Links
r = requests.get(wnURL, headers={'user-agent': 'Mozilla/5.0'})
soup = bs.BeautifulSoup(r.content, 'lxml')
for link in soup.find_all('a'):
    if chapterRegex.search(link.get('href')):
        chapters.append(base + link.get('href'))

# Generate args for threads
args = ((searchTerm, x) for x in chapters)

# Create Threads
pool = ThreadPool(8)
results = pool.map(term_in_chapter, args)

# Close Threads
pool.close()
pool.join()

# Print Results
print('Found Chapters: ')
for chapter in results:
    if chapter:
        print(chapter)
print('End')

1c572fa8122c6fe45e7c3c5ba5d7abd61ee07784
<mask token>


def term_in_chapter(args):
    searchTerm, chapterURL = args
    pRegex = re.compile('L\\d+')
    r = requests.get(chapterURL, headers={'user-agent': 'Mozilla/5.0'})
    soup = bs.BeautifulSoup(r.content, 'lxml')
    for paragraph in soup.find_all('p'):
        if paragraph.br:
            continue
        if 'id' in paragraph.attrs:
            if pRegex.search(paragraph.get('id')):
                if searchTerm in paragraph.text:
                    return chapterURL
    return None


<mask token>

<mask token>


def term_in_chapter(args):
    searchTerm, chapterURL = args
    pRegex = re.compile('L\\d+')
    r = requests.get(chapterURL, headers={'user-agent': 'Mozilla/5.0'})
    soup = bs.BeautifulSoup(r.content, 'lxml')
    for paragraph in soup.find_all('p'):
        if paragraph.br:
            continue
        if 'id' in paragraph.attrs:
            if pRegex.search(paragraph.get('id')):
                if searchTerm in paragraph.text:
                    return chapterURL
    return None


print('Enter WebNovel ID:')
<mask token>
print('WebNovel ID is: {}'.format(wnID))
print('Enter Search Term:')
<mask token>
print('Search Term is: {}'.format(searchTerm))
<mask token>
for link in soup.find_all('a'):
    if chapterRegex.search(link.get('href')):
        chapters.append(base + link.get('href'))
<mask token>
pool.close()
pool.join()
print('Found Chapters: ')
for chapter in results:
    if chapter:
        print(chapter)
print('End')

<mask token>


def term_in_chapter(args):
    searchTerm, chapterURL = args
    pRegex = re.compile('L\\d+')
    r = requests.get(chapterURL, headers={'user-agent': 'Mozilla/5.0'})
    soup = bs.BeautifulSoup(r.content, 'lxml')
    for paragraph in soup.find_all('p'):
        if paragraph.br:
            continue
        if 'id' in paragraph.attrs:
            if pRegex.search(paragraph.get('id')):
                if searchTerm in paragraph.text:
                    return chapterURL
    return None


print('Enter WebNovel ID:')
wnID = input()
print('WebNovel ID is: {}'.format(wnID))
print('Enter Search Term:')
searchTerm = input()
print('Search Term is: {}'.format(searchTerm))
base = 'https://ncode.syosetu.com'
wnURL = base + '/' + wnID
chstr = '/' + re.escape(wnID) + '/\\d+'
chapterRegex = re.compile(chstr)
chapters = []
r = requests.get(wnURL, headers={'user-agent': 'Mozilla/5.0'})
soup = bs.BeautifulSoup(r.content, 'lxml')
for link in soup.find_all('a'):
    if chapterRegex.search(link.get('href')):
        chapters.append(base + link.get('href'))
args = ((searchTerm, x) for x in chapters)
pool = ThreadPool(8)
results = pool.map(term_in_chapter, args)
pool.close()
pool.join()
print('Found Chapters: ')
for chapter in results:
    if chapter:
        print(chapter)
print('End')

import bs4 as bs
import requests
import re
from multiprocessing.dummy import Pool as ThreadPool


def term_in_chapter(args):
    searchTerm, chapterURL = args
    pRegex = re.compile('L\\d+')
    r = requests.get(chapterURL, headers={'user-agent': 'Mozilla/5.0'})
    soup = bs.BeautifulSoup(r.content, 'lxml')
    for paragraph in soup.find_all('p'):
        if paragraph.br:
            continue
        if 'id' in paragraph.attrs:
            if pRegex.search(paragraph.get('id')):
                if searchTerm in paragraph.text:
                    return chapterURL
    return None


print('Enter WebNovel ID:')
wnID = input()
print('WebNovel ID is: {}'.format(wnID))
print('Enter Search Term:')
searchTerm = input()
print('Search Term is: {}'.format(searchTerm))
base = 'https://ncode.syosetu.com'
wnURL = base + '/' + wnID
chstr = '/' + re.escape(wnID) + '/\\d+'
chapterRegex = re.compile(chstr)
chapters = []
r = requests.get(wnURL, headers={'user-agent': 'Mozilla/5.0'})
soup = bs.BeautifulSoup(r.content, 'lxml')
for link in soup.find_all('a'):
    if chapterRegex.search(link.get('href')):
        chapters.append(base + link.get('href'))
args = ((searchTerm, x) for x in chapters)
pool = ThreadPool(8)
results = pool.map(term_in_chapter, args)
pool.close()
pool.join()
print('Found Chapters: ')
for chapter in results:
    if chapter:
        print(chapter)
print('End')

#! python3
# -*- coding: utf-8 -*-

# ncodeWebScraper.py - Given a WebNovel, scan through each chapter for a keyword
# and return the chapters with the keyword

# Issues: Cannot get this to work with window consoles
# Not sure if this applies to other OSs
# Tested to work in IDLE environment

import bs4 as bs
import requests
import re
from multiprocessing.dummy import Pool as ThreadPool

# Returns the url if the term is found within its text


def term_in_chapter(args):
    searchTerm, chapterURL = args
    pRegex = re.compile(r'L\d+')
    r = requests.get(chapterURL, headers={'user-agent': 'Mozilla/5.0'})
    soup = bs.BeautifulSoup(r.content, 'lxml')
    for paragraph in soup.find_all('p'):
        if paragraph.br:
            continue
        if 'id' in paragraph.attrs:
            if pRegex.search(paragraph.get('id')):
                if searchTerm in paragraph.text:
                    return chapterURL
    return None


# Get User Inputs
print('Enter WebNovel ID:')
wnID = input()
print('WebNovel ID is: {}'.format(wnID))

print('Enter Search Term:')
searchTerm = input()
print('Search Term is: {}'.format(searchTerm))

# Init Variables
base = 'https://ncode.syosetu.com'
wnURL = base + '/' + wnID
chstr = '/' + re.escape(wnID) + r'/\d+'
chapterRegex = re.compile(chstr)
chapters = []

# Get Chapter Links
r = requests.get(wnURL, headers={'user-agent': 'Mozilla/5.0'})
soup = bs.BeautifulSoup(r.content, 'lxml')
for link in soup.find_all('a'):
    if chapterRegex.search(link.get('href')):
        chapters.append(base + link.get('href'))

# Generate args for threads
args = ((searchTerm, x) for x in chapters)

# Create Threads
pool = ThreadPool(8)
results = pool.map(term_in_chapter, args)

# Close Threads
pool.close()
pool.join()

# Print Results
print('Found Chapters: ')
for chapter in results:
    if chapter:
        print(chapter)
print('End')
