#请求网页
import requests
#操作excel
import openpyxl
#调用多线程
import threading
#解析网页
from bs4 import BeautifulSoup as bs

#目标地址，一共有863页
urls = ['http://www.allitebooks.org/page/{}/'.format(i) for i in range(1,864)]
#设置请求头
headers = {
    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.13 Safari/537.36'
    }
#创建表格
wb = openpyxl.Workbook()
ws = wb.active
ws.title = 'books'
#添加第一列
ws.append(['BookName','DownloadLinks'])
threading_list = []
#调整列宽
for column in ['A','B']:
    ws.column_dimensions[column].width = 13.0

def run(urls,i):
    #当urls不为空的时候循环执行
    while True:
        #为空时跳出循环
        if len(urls) == 0:
            break
        try:
            #请求网页,获取urls列表中的第一条，并删除该条
            res = requests.get(url = urls.pop(0),headers = headers)
            res.encoding = res.apparent_encoding
            #解析网页并找到该页所有书目
            bf = bs(res.text,'lxml')
            book_list = bf.find_all('h2',class_ = "entry-title")
            #逐条清洗数据
            for book in book_list:
                #获取书名
                book_title = book.find('a').text
                #获取书籍详细页面地址
                book_url = book.find('a').get('href')
                #请求详细页面
                res = requests.get(url = book_url, headers = headers)
                res.encoding = res.apparent_encoding
                #解析获得电子书下载地址
                bz = bs(res.text,'lxml')
                download_links = bz.find('span',class_ = "download-links").find('a').get('href')
                #保存合格的数据（即下载地址后缀为pdf的数据）
                second_name = download_links.split('.')
                if second_name[-1] == 'pdf':
                    ws.append([book_title, download_links])
            print("线程-->%d-->已获取一页" % i)
        except:
            pass

#创建五个线程
for i in range(5):
    t = threading.Thread(target = run,args = (urls,i,))
    t.start()
    #将线程添加到线程列表
    threading_list.append(t)
#让主线程等待所有子线程执行完毕
for t in threading_list:
    t.join()
#保存工作表
wb.save('allitebooks.xlsx')