python爬虫中多线程的使用详解

#多线程实战栗子（糗百）#用一个队列Queue对象，#先产生所有url，put进队列；#开启多线程，把queue队列作为参数传入#主函数中读取urlimport requestsfrom queue import Queueimport re,os,threading,time# 构造所有ip地址并添加进queue队列headers = {  'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36'}urlQueue = Queue()[urlQueue.put('http://www.qiumeimei.com/image/page/{}'.format(i)) for i in range(1,14)]def get_image(urlQueue):  while True:    try:      # 不阻塞的读取队列数据      url = urlQueue.get_nowait()      # i = urlQueue.qsize()    except Exception as e:      break    print('Current Thread Name %s, Url: %s ' % (threading.currentThread().name, url))    try:      res = requests.get(url, headers=headers)      url_infos = re.findall('data-lazy-src="(.*?)"', res.text, re.S)      for url_info in url_infos:        if os.path.exists(img_path + url_info[-20:]):          print('图片已存在')        else:          image = requests.get(url_info, headers=headers)          with open(img_path + url_info[-20:], 'wb') as fp:            time.sleep(1)            fp.write(image.content)          print('正在下载:' + url_info)    except Exception as e:      print(e)if __name__ == '__main__':  startTime = time.time()  # 定义图片存储路径  img_path = './img/'  if not os.path.exists(img_path):    os.mkdir(img_path)  threads = []  # 可以调节线程数， 进而控制抓取速度  threadNum = 4  for i in range(0, threadNum):    t = threading.Thread(target=get_image, args=(urlQueue,))    threads.append(t)  for t in threads:    t.start()  for t in threads:    # 多线程多join的情况下，依次执行各线程的join方法, 这样可以确保主线程最后退出， 且各个线程间没有阻塞    t.join()  endTime = time.time()  print('Done, Time cost: %s ' % (endTime - startTime))