Python爬虫 scrapy框架爬取某招聘网存入mongodb解析

scrapy startproject zhaoping

cd zhaopingscrapy genspider hr zhaopingwang.com

  title = scrapy.Field()  position = scrapy.Field()  publish_date = scrapy.Field()

from pymongo import MongoClientmongoclient = MongoClient(host='192.168.226.150',port=27017)collection = mongoclient['zhaoping']['hr']class TencentPipeline(object):  def process_item(self, item, spider):    print(item)    # 需要转换为 dict    collection.insert(dict(item))    return item

def parse(self, response):    # 不要第一个 和最后一个    tr_list = response.xpath("//table[@class='tablelist']/tr")[1:-1]    for tr in tr_list:      item = TencentItem()      # xpath 从1 开始数起      item["title"] = tr.xpath("./td[1]/a/text()").extract_first()      item["position"] = tr.xpath("./td[2]/text()").extract_first()      item["publish_date"] = tr.xpath("./td[5]/text()").extract_first()      yield item    next_url = response.xpath("//a[@id='next']/@href").extract_first()    # 构造url    if next_url != "javascript:;":      print(next_url)      next_url = "https://hr.tencent.com/" + next_url      yield scrapy.Request(url=next_url,callback=self.parse,)