分享一个简单的java爬虫框架

package DownloadPackage;import java.io.BufferedReader;import java.io.IOException;import java.io.InputStreamReader;import org.apache.http.HttpEntity;import org.apache.http.HttpResponse;import org.apache.http.impl.client.CloseableHttpClient;import org.apache.http.impl.client.HttpClients;/* * 根据一个url下载一个html页面 */public class HtmlDownloader {	RequestSet requestset = null;	public HtmlDownloader(RequestSet requestset){		this.requestset = requestset;	}	public String downloadhtml(String url){		String html = null;		//创建一个客户端		//创建一个读取流从entity读取html		BufferedReader reader = null;		CloseableHttpClient httpclient = HttpClients.createDefault();		HttpResponse response = null;		try {			response = httpclient.execute(requestset.getMethod(url));			HttpEntity entity = response.getEntity();			reader = new BufferedReader(new InputStreamReader(entity.getContent()));			StringBuilder sb = new StringBuilder();			while((html = reader.readLine()) != null){				sb.append(html);			}			html = sb.toString();			System.out.println("一个html页面获取成功");		}		catch (IOException e) {			System.out.println(url+"连接失败");		}		finally{			if(reader != null){				try {					reader.close();					httpclient.close();				}				catch (IOException e) {					// TODO Auto-generated catch block					e.printStackTrace();				}			}		}		return html;	}}

package DownloadPackage;import java.util.LinkedList;import org.jsoup.Jsoup;import org.jsoup.nodes.Document;import org.jsoup.nodes.Element;import org.jsoup.select.Elements;public class UrlGet {	public LinkedList<String> geturls(String html){		LinkedList<String> urls = new LinkedList<String>();		Document doc = Jsoup.parse(html);		Elements links = doc.getElementsByTag("a");		for (Element link:links){			String url = link.attr("href");			urls.add(url);		}		return urls;	}}

package ChoosePackage;public interface ResourseChooser {	public Boolean isNeed(String url);	public Boolean isResourse(String url);	public String process(String url);}

package DownloadPackage;import org.apache.http.client.methods.HttpGet;/* * 一个用于获得Request请求的接口 * 实现getMethod方法获取Get方法 */public interface RequestSet {	public HttpGet getMethod(String url);}Saveutil接口用于自定义保存方式,需要实现save方法package SaveUtil;/* * 数据储存的工具接口,必须实现保存方法 */public interface SaveUtil {	public void save(String url,String html);}

package Spider;import java.io.BufferedWriter;import java.io.File;import java.io.FileWriter;import java.io.IOException;import java.util.HashSet;import java.util.Iterator;import java.util.LinkedList;import org.apache.http.client.config.RequestConfig;import org.apache.http.client.methods.HttpGet;import ChoosePackage.MyResourseChooser;import ChoosePackage.ResourseChooser;import DownloadPackage.HtmlDownloader;import DownloadPackage.RequestSet;import DownloadPackage.UrlGet;import SaveUtil.MySaveUtil;import SaveUtil.SaveUtil;/* * 用于爬取资源的类 */public class Spider{	public static void main(String[] args) {		new Spider("http://www.bilibili.net").spiderstart();	}	//种子url	String seed = null;	//用于保存数据的类,需要自己实现	private SaveUtil saveutil = null;	//html下载类	private HtmlDownloader downloader = null;	//url下载类	private UrlGet urldownloader = null;	//资源选择工具	private ResourseChooser resoursechooser = null;	//用于保存未下载的网页	LinkedList<String> unvisited = new LinkedList<String>();	//用于保存已下载的网页	HashSet<String> visited = new HashSet<String>();	//自定义储存方式,请求方式,资源筛选方式的构造方法	public Spider(SaveUtil saveutil,RequestSet request,ResourseChooser resoursechooser,String seed){		this.saveutil = saveutil;		this.downloader = new HtmlDownloader(request);		this.urldownloader = new UrlGet();		this.resoursechooser = resoursechooser;		this.seed = seed;		unvisited.add(seed);	}	//自定义储存方式,资源筛选方式的构造方法	public Spider(SaveUtil saveutil,ResourseChooser resoursechooser,String seed){		this.resoursechooser = resoursechooser;		this.downloader = new HtmlDownloader(new getRequest());		this.saveutil = saveutil;		this.urldownloader = new UrlGet();		this.seed = seed;		unvisited.add(seed);	}	//自定义储存方式,请求的构造方法	public Spider(SaveUtil saveutil,RequestSet requestset,String seed){		this.saveutil = saveutil;		this.downloader = new HtmlDownloader(requestset);		this.resoursechooser = new MyResourseChooser();		this.urldownloader = new UrlGet();		this.seed = seed;		unvisited.add(seed);	}	//自定义储存方式的构造方法	public Spider(SaveUtil saveutil,String seed){		this.saveutil = saveutil;		this.downloader = new HtmlDownloader(new getRequest());		this.resoursechooser = (new MyResourseChooser());		this.urldownloader = new UrlGet();		this.seed = seed;		unvisited.add(seed);	}	//默认的爬虫构造方法	public Spider(String seed){		this.saveutil = new MySaveUtil();		this.downloader = new HtmlDownloader(new getRequest());		this.resoursechooser = (new MyResourseChooser());		this.urldownloader = new UrlGet();		this.seed = seed;		unvisited.add(seed);	}	//开始爬取的方法	private void spiderstart(){		String html = null;		while(!unvisited.isEmpty()){			String url = unvisited.poll();			System.out.println("开始获取"+url);			if(resoursechooser.isNeed(url)){				try{					html = downloader.downloadhtml(url);				}				catch(RuntimeException e){					System.out.println(url+"连接获取失败");					continue;				}				visited.add(url);				LinkedList<String> urls = new LinkedList<String>();				try{					urls = urldownloader.geturls(html);				}				catch(RuntimeException e){					System.out.println(url+"的html页面为空");					continue;				}				Iterator<String> it = urls.iterator();				while(it.hasNext()){					String newurl = it.next();					if(resoursechooser.isNeed(newurl)&&!visited.contains(newurl)&&!unvisited.contains(newurl)){						newurl = resoursechooser.process(newurl);						unvisited.add(newurl);						System.out.println(newurl+"加入页面");					}				}				System.out.println("获取了"+url+"上的所有url");				if(resoursechooser.isResourse(url)){					saveutil.save(url,html);				}			}		}	}	//默认资源筛选类	private class MyResourseChooser implements ResourseChooser{		@Override		public Boolean isNeed(String url) {			// TODO Auto-generated method stub			if(!url.startsWith("/")&&!url.startsWith("http")){				return false;			}			return true;		}		@Override		public Boolean isResourse(String url) {			// TODO Auto-generated method stub			return true;		}		@Override		public String process(String url) {			// TODO Auto-generated method stub			if(!url.startsWith("http")){				url = seed+url;			}			return url;		}	}	public class getRequest implements RequestSet{		public HttpGet getMethod(String url) {			// TODO Auto-generated method stub			//创建一个get请求方法			HttpGet getmethod = new HttpGet(url);			//HttpHost proxy = new HttpHost("124.88.67.81",80);这里不设置代理IP			//设置请求超时时间等 			RequestConfig responseconfig = RequestConfig.custom().setConnectionRequestTimeout(10000).setConnectTimeout(10000).setSocketTimeout(10000).build();			//设置请求头,主要是user-agent			getmethod.addHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36");			//设置请求参数			getmethod.setConfig(responseconfig);			return getmethod;		}	}	//默认的存储类	public class MySaveUtil implements SaveUtil{		@Override		public void save(String url, String html) {			// TODO Auto-generated method stub			String filename = getfilename(url);			BufferedWriter writer = null;			try{				writer = new BufferedWriter(new FileWriter(filename));				writer.write(html);				writer.flush();				System.out.println("文件写入成功");			}			catch(IOException e){				System.out.println("文件写入失败");			}			finally{				try {					if(writer != null)					writer.close();				}				catch (IOException e) {					// TODO Auto-generated catch block					System.out.println("流关闭失败");				}			}		}		private String getfilename(String url){			String fileparentpath = "f://html";			File file = new File(fileparentpath);			if(!file.exists()){				file.mkdir();			}			int last = url.lastIndexOf(".");			int first = url.indexOf(".");			url = url.substring(first,last);			url = url.replaceAll("//.", "");			url = url.replaceAll("/", "");			return fileparentpath+"/"+url+".txt";		}	}}