java利用url实现网页内容的抓取

import java.io.BufferedReader;import java.io.IOException;import java.io.InputStreamReader;import java.net.HttpURLConnection;import java.net.URL;import java.util.*;/** * Created by chunmiao on 17-3-10. */public class ReadBaiduSearch { //储存返回结果 private LinkedHashMap<String,String> mapOfBaike; //获取搜索信息 public LinkedHashMap<String,String> getInfomationOfBaike(String infomationWords) throws IOException {  mapOfBaike = getResult(infomationWords);  return mapOfBaike; } //通过网络链接获取信息 private static LinkedHashMap<String, String> getResult(String keywords) throws IOException {  //搜索的url  String keyUrl = "http://baike.baidu.com/search?word=" + keywords;  //搜索词条的节点  String startNode = "<dl class=/"search-list/">";  //词条的链接关键字  String keyOfHref = "href=/"";  //词条的标题关键字  String keyOfTitle = "target=/"_blank/">";  String endNode = "</dl>";  boolean isNode = false;  String title;  String href;  String rLine;  LinkedHashMap<String,String> keyMap = new LinkedHashMap<String,String>();  //开始网络请求  URL url = new URL(keyUrl);  HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection();  InputStreamReader inputStreamReader = new InputStreamReader(urlConnection.getInputStream(),"utf-8");  BufferedReader bufferedReader = new BufferedReader(inputStreamReader);  //读取网页内容  while ((rLine = bufferedReader.readLine()) != null){   //判断目标节点是否出现   if(rLine.contains(startNode)){    isNode = true;   }   //若目标节点出现，则开始抓取数据   if (isNode){    //若目标结束节点出现，则结束读取，节省读取时间    if (rLine.contains(endNode)) {     //关闭读取流     bufferedReader.close();     inputStreamReader.close();     break;    }    //若值为空则不读取    if (((title = getName(rLine,keyOfTitle)) != "") && ((href = getHref(rLine,keyOfHref)) != "")){     keyMap.put(title,href);    }   }  }  return keyMap; } //获取词条对应的url private static String getHref(String rLine,String keyOfHref){  String baikeUrl = "http://baike.baidu.com";  String result = "";  if(rLine.contains(keyOfHref)){   //获取url   for (int j = rLine.indexOf(keyOfHref) + keyOfHref.length();j < rLine.length()&&(rLine.charAt(j) != '/"');j ++){    result += rLine.charAt(j);   }   //获取的url中可能不含baikeUrl,如果没有则在头部添加一个   if(!result.contains(baikeUrl)){    result = baikeUrl + result;   }  }  return result; } //获取词条对应的名称 private static String getName(String rLine,String keyOfTitle){  String result = "";  //获取标题内容  if(rLine.contains(keyOfTitle)){   result = rLine.substring(rLine.indexOf(keyOfTitle) + keyOfTitle.length(),rLine.length());   //将标题中的内容含有的标签去掉   result = result.replaceAll("<em>|</em>|</a>|<a>","");  }  return result; }}