使用python爬虫获取黄金价格的核心代码

#!/usr/bin/env python# -*- coding: utf-8 -*-"""获取每天黄金价格@author: yufei@site: http://www.antuan.com2017-05-11"""import reimport urllib2,urllibimport randomimport threadingimport timeimport sqlite3import sysfrom __builtin__ import exit#Some User Agentshds=[{'User-Agent':'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6'},/  {'User-Agent':'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.12 Safari/535.11'},/  {'User-Agent':'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Trident/6.0)'},/  {'User-Agent':'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:34.0) Gecko/20100101 Firefox/34.0'},/  {'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/44.0.2403.89 Chrome/44.0.2403.89 Safari/537.36'},/  {'User-Agent':'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50'},/  {'User-Agent':'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50'},/  {'User-Agent':'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0'},/  {'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1'},/  {'User-Agent':'Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1'},/  {'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11'},/  {'User-Agent':'Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11'},/  {'User-Agent':'Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11'}]'''create table GoldPricelist(  [id]  integer PRIMARY KEY autoincrement,  [name]  varchar default 0,  [price]  varchar default 0,  [time] datetime default (datetime('now', 'localtime')));INSERT INTO OilPrice(id,name,price,time) VALUES(NULL,sss, 300, 20180404);'''def sqliteinto(Pricelist):  con = sqlite3.connect('F:/ID/python/sqlite3/Gold.db')  cur = con.cursor()  print Pricelist  sql = '''INSERT INTO GoldPrices (id,name,price,time) VALUES(NULL,?,?,?)'''  cur.execute(sql,Pricelist)  con.commit()  cur.close()  con.close()def getPrice():  url='http://www.dyhjw.com/matter_gold/'  req = urllib2.Request(url=url,headers=hds[random.randint(0,len(hds)-1)])  res = urllib2.urlopen(req)  res = res.read()  #获取的块  re_set = re.compile(r'<dl class="main_bname">(.*?)</div>',re.S)  re_get = re.findall(re_set,res)  #获取价格详情  p = re.compile('target="_blank">(.*?)<span class="zd">-</span>/n ',re.S)  Pricelist = re.findall(p,re_get[0])  for i in range(0,len(Pricelist)):    p = re.compile('(.*?)</a>/n              <span class="jg">(.*)</span>',re.S)    Price = re.findall(p,Pricelist[i])    gname = Price[0][0]    gprice = Price[0][1]    timep = re.compile(r"(/d+)-(/d+)-.*")    nowtime = time.strftime('%Y-%m-%d',time.localtime(time.time()))    datas = []    datas.append(gname.decode('utf8'))    datas.append(gprice)    datas.append(nowtime)    datas = tuple(datas)    sqliteinto(datas)if __name__=="__main__":  getPrice()