python实现爬虫抓取小说功能示例【抓取金庸小说】

# -*- coding: utf-8 -*-from bs4 import BeautifulSoupfrom urllib import requestimport reimport os,time#访问url，返回html页面def get_html(url):  req = request.Request(url)  req.add_header('User-Agent','Mozilla/5.0')  response = request.urlopen(url)  html = response.read()  return html#从列表页获取小说书名和链接def get_books(url):#根据列表页，返回此页的{书名:链接}的字典  html = get_html(url)  soup = BeautifulSoup(html,'lxml')  fixed_html = soup.prettify()  books = soup.find_all('div',attrs={'class':'bbox'})  book_dict = {}  for book in books:    book_name = book.h3.a.string    book_url = book.h3.a.get('href')    book_dict[book_name] = book_url  return book_dict#根据书名链接，获取具体的章节{名称:链接} 的字典def get_parts(url):  html = get_html(url)  soup = BeautifulSoup(html,'lxml')  fixed_html = soup.prettify()  part_urls = soup.find_all('a')  host = "http://www.xiaoshuotxt.org"  part_dict = {}  for p in part_urls:    p_url = str(p.get('href'))    if re.search(r'/d{5}.html',p_url) and ("xiaoshuotxt" not in p_url):      part_dict[p.string] = host + p_url  return part_dict#根据章节的url获取具体的章节内容def get_txt(url):  html = get_html(url)  soup = BeautifulSoup(html,'lxml')  fixed_html = soup.prettify()  title = soup.h1.string #获取文章标题  content = soup.find('div',attrs={'class':'zw'})  txt = BeautifulSoup.get_text(content) #正文内容  return txtif __name__ == "__main__":  root_dir= r'e:/books'  #url = 'http://www.xiaoshuotxt.org/mingzhu/index_2.html' #第2页的小说  url = "http://www.xiaoshuotxt.org/writer/58" #金庸的小说  books = get_books(url)  for book_name,book_url in books.items():    os.mkdir(os.path.join(root_dir,book_name))    part_dict = get_parts(book_url)    print(book_name,"共：",len(part_dict),"章节")    for part_name,part_url in part_dict.items():      print("正在保存:",part_name)      f1 = open(r'e:/books/%s/%s.txt'%(book_name,part_name),'w',encoding='utf-8')#以utf-8编码创建文件      part_txt = get_txt(part_url)      f1.write(str(part_txt))      f1.close()      time.sleep(2)