Python学习-简单抓取页面

2019-11-14 17:34:18

字体：大中小

来源：转载

供稿：网友

最近想做一个小web应用，就是把豆瓣读书和亚马逊等写有书评的网站上关于某本书的打分记录下来，这样自己买书的时候当作参考。

这篇日志这是以豆瓣网为例，只讨论简单的功能。

向服务器发送查询请求

这很好处理，找到网站的搜索框，然后填入相关信息，提交后查看url即可。

这里以豆瓣为例，当我在http://book.douban.com页面的搜索框中输入现代操作系统后得到下面的url：

http://book.douban.com/subject_search?search_text=%E7%8E%B0%E4%BB%A3%E6%93%8D%E4%BD%9C%E7%B3%BB%E7%BB%9F&cat=1001

这样就知道如何向服务器提交查询请求了，注意search_text后面的一串字符只是编码不同（。。。）。

利用Urllib2和Urllib库发送和获取HTTP页面

详见下面代码：

book_name = '现代操作系统'douban_book = 'http://book.douban.com/subject_search?'search = [('search_text','现代操作系统'),('cat','1001')]getbook = douban_book + urllib.urlencode(search)content = urllib2.urlopen(getbook).read()

利用SGMLParser库解析HTTP文本

第一步，利用浏览器自带的查看页面信息的工具，查看页面布局。
根据布局，思考解析的方法。这一步很主要，决定了第三步的效率
编写代码。基本上就是重写SGMLParser子类的方法。

详细代码

代码写的很乱，一些语法还不是很熟悉。我是以写代码来学习Python的，什么不懂就查什么。

# -*- coding: utf-8 -*-import urllib2import urllibfrom sgmllib import SGMLParserclass BookInfo(SGMLParser):    def reset(self):        SGMLParser.reset(self)        # 标记对应的标签        self.is_subject = 0        self.is_subject_info = 0        self.is_subject_h2 = 0        self.is_subject_pub = 0        self.is_subject_star = 0                self.temp = {} # 一个字典，保存暂时的信息        self.info = [] # 一个列表，保存所有的信息        # li标签开始出现    def start_li(self,attrs):        if 'subject-item' in [v for k, v in attrs if k == 'class']:            self.is_subject = 1    # li标签结束    def end_li(self):        self.is_subject = 0        def start_h2(self,attrs):        if self.is_subject == 1 and '' in [v for k,v in attrs if k == 'class']:            self.is_subject_h2 = 1        def end_h2(self):        self.is_subject_h2 = 0    def start_div(self,attrs):        attr = ''        for k,v in attrs:            if k == 'class':                attr = v                break        if attr == 'info' and self.is_subject == 1:            self.is_subject_info = 1        elif attr == 'pub' and self.is_subject_info == 1:            self.is_subject_pub = 1        elif attr == 'star clearfix' and self.is_subject_info == 1:            self.is_subject_star = 1        else:            pass    def end_div(self):        if self.is_subject_star == 0:            if self.is_subject_pub == 0:                self.is_subject_info = 0                self.info.append(self.temp)                self.temp = {}            else:                self.is_subject_pub = 0        else:            self.is_subject_star = 0        def handle_data(self,data):        if self.is_subject_h2:            string = data.strip()            if len(string):                if 'name' in self.temp:                    self.temp['name'] = self.temp['name'] + string                else:                    self.temp['name'] = string                #PRint string        elif self.is_subject_pub:                string = data.strip()                if len(string):                    if 'pub' in self.temp:                        self.temp['pub'] = self.temp['pub']+string                    else:                        self.temp['pub'] = string        elif self.is_subject_star:                string = data.strip()                if len(string):                    if 'star' in self.temp:                        self.temp['star'] = self.temp['star'] + string                    else:                        self.temp['star'] = string                    #print string        else:            pass                        book_name = '现代操作系统'douban_book = 'http://book.douban.com/subject_search?'search = [('search_text','现代操作系统'),('cat','1001')]getbook = douban_book + urllib.urlencode(search)print getbookcontent = urllib2.urlopen(getbook).read()fobj = open('book.txt','w')fileobj = open('books.txt','w')book = BookInfo()book.feed(content)for books in book.info:    for item in books:        print '*************************************************'        print '书名：%s' % books['name']        if 'pub' in books:            print '出版信息：%s' %  books['pub']        if 'star' in books:            print '评价：%s' % books['star']        breakfobj.write(content)fobj.close()fileobj.close()

输出结果

这只是开头的第一步，以后的日子里不断的学习和实践。。。

Bug修复和改进

上面的代码其实还是有问题的，只是没用被发现。当标记第一个 div 标签的确是没用问题，但是当出现第二个div标签时，如果第二个是第一个的子元素，那么当处理第二个子标签的/div 闭合标签的时候就会出错。

一个小小的改进。这个程序严格要求输入的是正确的书名，这样处理的结果才是正确的。如果不是完全正确的书名，我的代码量就成集合倍增加了。在豆瓣读书中，评价书小于特定的数目时，是没有评论的（代表这个版次的书一般是很久的，上个世纪的书了），那么就没有参考价值了。

下面是修改后的代码：

# -*- coding: utf-8 -*-import urllib2import urllibfrom sgmllib import SGMLParserclass BookInfo(SGMLParser):    def reset(self):        SGMLParser.reset(self)        # 标记对应的标签        self.is_subject = 0        self.is_subject_info = 0        self.is_subject_h2 = 0        self.is_subject_pub = 0        self.is_subject_star = 0        self.is_subject_rating = 0                self.temp = {} # 一个字典，保存暂时的信息        self.info = [] # 一个列表，保存所有的信息        # li标签开始出现    def start_li(self,attrs):        if 'subject-item' in [v for k, v in attrs if k == 'class']:            self.is_subject = 1    # li标签结束    def end_li(self):        self.is_subject = 0        def start_h2(self,attrs):        if self.is_subject == 1 and '' in [v for k,v in attrs if k == 'class']:            self.is_subject_h2 = 1        def end_h2(self):        self.is_subject_h2 = 0    def start_div(self,attrs):        attr = ''        for k,v in attrs:            if k == 'class':                attr = v                break        if attr == 'info' and self.is_subject == 1:            self.is_subject_info = 1        elif attr == 'pub' and self.is_subject_info == 1:            self.is_subject_pub = 1        elif attr == 'star clearfix' and self.is_subject_info == 1:            self.is_subject_star = 1        else:            pass    def end_div(self):        if self.is_subject_info:            if self.is_subject_pub:                if self.is_subject_star:                    self.is_subject_star = 0                    self.is_subject_rating = 0                else:                    self.is_subject_pub = 0            elif self.is_subject_star:                self.is_subject_star = 0                self.is_subject_rating = 0                if len(self.temp) == 3:                    self.info.append(self.temp)                self.temp = {}            else:                self.is_subject_info = 0    def start_span(self,attrs):        if self.is_subject_star and 'allstar45' in [v for k,v in attrs if k == 'class']:            print [v for k,v in attrs if k == 'class']            self.is_subject_rating = 1        def handle_data(self,data):        if self.is_subject_h2:            string = data.strip()            if len(string):                if 'name' in self.temp:                    self.temp['name'] = self.temp['name'] + string                else:                    self.temp['name'] = string                    if string != book_name:                        self.temp = {}                #print string        elif self.is_subject_pub:                string = data.strip()                if len(string):                    if 'pub' in self.temp:                        self.temp['pub'] = self.temp['pub']+string                    else:                        self.temp['pub'] = string        elif self.is_subject_star:                string = data.strip()                if len(string) and self.is_subject_rating:                    if 'star' in self.temp:                        self.temp['star'] = self.temp['star'] + string                    else:                        self.temp['star'] = string                    print string        else:            pass                        book_name = '现代操作系统'douban_book = 'http://book.douban.com/subject_search?'search = [('search_text','现代操作系统'),('cat','1001')]getbook = douban_book + urllib.urlencode(search)print getbookcontent = urllib2.urlopen(getbook).read()fobj = open('book.txt','w')fileobj = open('books.txt','w')book = BookInfo()book.feed(content)for books in book.info:    for item in books:        print '*************************************************'        print '书名：%s' % books['name']        if 'pub' in books:            print '出版信息：%s' %  books['pub']        if 'star' in books:            print '评价：%s' % books['star']        breakfobj.write(content)fobj.close()fileobj.close()

下面是输出结果：