class Bug_pa(object):

    def __init__(self, RootLink):

        """ 初始化 """

        self.rootLink = RootLink

    def scarpylink(self):

        """ 爬取书籍每个章节的 url """

        try:

            Link = self.rootLink[0]

            data = []

            res = requests.get(url=Link, headers=header)

            res.encoding = 'gbk'

            """ 解析内容 """

            res = str(res.text)

            soup = BeautifulSoup(res, 'lxml')

            # 获取书名

            bookname = soup.select_one('#info > h1').text

            # 获取章节链接

            ran = 1

            for htmls in soup.select('#list > dl > dd > a'):

                if ran > 12:

                    htmls = str(htmls)

                    links = re_link.findall(htmls)

                    data.append(Link + ''.join(links))

                else:

                    ran += 1

            if data:

                print(' 章节链接获取成功 ')

                return {'bookname': bookname,

                        'links': data}

            else:

                return []

        except Exception:

            print(' 出错 ')

            return []

    def scarpytext(self, url):

        """ 爬取书籍每个章节的内容 """

        try:

            texts = []

            page = requests.get(url=url, headers=header)

            page.encoding = 'gbk'

            soup = BeautifulSoup(page.text, 'lxml')

            selctname = soup.select_one('.bookname > h1').text

            print('外汇跟单gendan5.com 章节标题获取成功 ')

            for text_list in soup.select('#content'):

                for tex in text_list:    # 需要获取详细文本

                    tex = str(tex)

                    tex = tex.replace('
', '').replace('\xa0\xa0\xa0\xa0', '')

                    texts.append(tex)

            if texts:

                print(' 章节文本获取成功！ ')

                return {'name': selctname,

                        'text': texts}

        except Exception:

            print(' 文本函数出错 ')

            return []

    def save(self, bookname, name, booktext):

        """ 保存小说 """

        try:

            # exists: 判断括号中的文件路径是否存在

            # 如果文件夹不存在 , 就以书名创建一个文件夹

            print(bookname)

            print(name)

            if not os.path.exists(r'G:\book\-'+str(bookname)):

                os.mkdir(r'G:\book\-'+str(bookname))

            # with open(r'G:\book\book' + str(bookname) + str(name) + '.txt', 'a') as fp:

            with open(r'G:\book\-' + str(bookname) + '\-' + str(name) + '.txt', 'a') as fp:

                fp.write(name + '\n')

                for txt in booktext:

                    fp.write(txt + '\n')

            print(' 保存成功 ')

        except Exception:

            print(' 保存函数出错了 ')

            return []

    def rand_time(self):

        """ 启动随机延时 """

        i = random.uniform(0, 30)

        time.sleep(i)

    def main(self):

        """ 主函数 """

        try:

            # 爬取书的章节链接

            linkinfo = self.scarpylink()

            i = 1

            for link in linkinfo['links']:

                txt = self.scarpytext(link)

                if txt:

                    if self.save(linkinfo['bookname'], str(i) + '-' + txt['name'], txt['text'] ):

                        print(' 储存 ok', txt['name'])

                else:

                    print(' 储存失败 ', txt['name'])

                i += 1

        except Exception:

            print(' 主函数有问题 ')

def scrapyRootLink(url):

    """ 获取分类下的 url """

    try:

        links = []

        res = requests.get(url=url, headers=header)

        res.encoding = 'gbk'

        date = BeautifulSoup(res.text, 'lxml')

        for lin in date.select('.s2 > :link'):

            lin = str(lin)

            link = re_link.findall(lin)

            if link:

                links.append(link)

        if links:

            print(' 书籍链接获取成功 ')

            return links

        else:

            print('scrapyRootLink 有问题 ')

            return []

    except Exception as e:

        print(e)

        return []

if __name__ == '__main__':

    rootlinks = []

    for i in range(1, 2):

        rlink = 'https://www.qqxsw.co/top/allvisit/%s.html' %(i)

        rootlinks.append(rlink)

    for rootlink in rootlinks:

        for alink in scrapyRootLink(rootlink):

            shu = Bug_pa(alink)

            shu.main()

            shu.rand_time()     # 每爬取一页线程随机睡眠 0-30 秒