python爬虫selenium和phantomJs使用方法解析

pip install selenium

#使用下面的方法，查找指定的元素进行操作即可find_element_by_id 根据id找节点find_elements_by_name 根据name找find_elements_by_xpath 根据xpath查找find_elements_by_tag_name 根据标签名找find_elements_by_class_name 根据class名字查找# 通过id定位:dr.find_element_by_id("kw")# 通过name定位:dr.find_element_by_name("wd")# 通过class name定位:dr.find_element_by_class_name("s_ipt")# 通过tag name定位:dr.find_element_by_tag_name("input")# 通过xpath定位，xpath定位有N种写法，这里列几个常用写法:dr.find_element_by_xpath("//*[@id='kw']")dr.find_element_by_xpath("//*[@name='wd']")dr.find_element_by_xpath("//input[@class='s_ipt']")dr.find_element_by_xpath("/html/body/form/span/input")dr.find_element_by_xpath("//span[@class='soutu-btn']/input")dr.find_element_by_xpath("//form[@id='form']/span/input")dr.find_element_by_xpath("//input[@id='kw' and @name='wd']")# 通过css定位，css定位有N种写法，这里列几个常用写法:dr.find_element_by_css_selector("#kw")dr.find_element_by_css_selector("[name=wd]")dr.find_element_by_css_selector(".s_ipt")dr.find_element_by_css_selector("html > body > form > span > input")dr.find_element_by_css_selector("span.soutu-btn> input#kw")dr.find_element_by_css_selector("form#form > span > input")

<a class="mnav" href="http://news.baidu.com" rel="external nofollow" name="tj_trnews">新闻</a><a class="mnav" href="http://www.hao123.com" rel="external nofollow" name="tj_trhao123">hao123</a>

# 通过link text定位:dr.find_element_by_link_text("新闻")dr.find_element_by_link_text("hao123")# 通过partial link text定位:dr.find_element_by_partial_link_text("新")dr.find_element_by_partial_link_text("hao")dr.find_element_by_partial_link_text("123")

from selenium import webdriverfrom time import sleep# 创建浏览器对象 驱动为浏览器的路径bro = webdriver.Chrome("./chromedriver.exe")url = "https://www.baidu.com"7 # 发送请求bro.get(url)# 让百度指定词条的搜索text = bro.find_element_by_id('kw')# 发送关键字text.send_keys('python')# 点击搜索按钮button = bro.find_element_by_id('su')button.click()# 关闭浏览器bro.quit()

from selenium import webdriverfrom time import sleepbro = webdriver.PhantomJS('E:/BaiduNetdiskDownload/爬虫课件/5. 动态数据加载爬取/phantomjs2.1.1windows/bin/phantomjs.exe')url = "https://www.baidu.com"# 发送请求bro.get(url)bro.save_screenshot('./1.png')# 让百度指定词条的搜索text = bro.find_element_by_id('kw')# 发送关键字text.send_keys('python')bro.save_screenshot('./2.png')# 点击搜索按钮button = bro.find_element_by_id('su')button.click()sleep(3)bro.save_screenshot('./3.png')# 关闭浏览器bro.quit()

from selenium import webdriverfrom time import sleep3 url = 'https://movie.douban.com/typerank?type_name=%E5%96%9C%E5%89%A7&type=24&interval_id=100:90&action='bro = webdriver.PhantomJS('E:/BaiduNetdiskDownload/爬虫课件/5. 动态数据加载爬取/phantomjs2.1.1windows/bin/phantomjs.exe')bro.get(url)sleep(1)bro.save_screenshot("./1.png")js = 'window.scrollTo(0,document.body.scrollHeight)'# 执行js代码 滚动条bro.execute_script(js)sleep(1)bro.save_screenshot('./2.png')# 获取页面数据page_source = bro.page_sourceprint(page_source)