分享一个简单的java爬虫框架

分享一个简单的java爬虫框架

反复给网站编写不同的爬虫逻辑太麻烦了,自己实现了一个小框架

可以自定义的部分有:

请求方式(默认为Getuser-agent为谷歌浏览器的设置),可以通过实现RequestSet接口来自定义请求方式

储存方式(默认储存在f盘的html文件夹下),可以通过SaveUtil接口来自定义保存方式

需要保存的资源(默认为整个html页面)

筛选方式(默认所有url都符合要求),通过实现ResourseChooser接口来自定义需要保存的url和资源页面

实现的部分有:

html页面的下载方式,通过HttpClient实现html页面的下载

html页面的解析部分,通过jsoup实现html页面的解析

HtmlDownloader类,用于根据一个url下载一个html页面

package DownloadPackage;

import java.io.BufferedReader;

import java.io.IOException;

import java.io.InputStreamReader;

import org.apache.http.HttpEntity;

import org.apache.http.HttpResponse;

import org.apache.http.impl.client.CloseableHttpClient;

import org.apache.http.impl.client.HttpClients;

/*

* 根据一个url下载一个html页面

*/

public class HtmlDownloader {

RequestSet requestset = null;

public HtmlDownloader(RequestSet requestset){

this.requestset = requestset;

}

public String downloadhtml(String url){

String html = null;

//创建一个客户端

//创建一个读取流从entity读取html

BufferedReader reader = null;

CloseableHttpClient httpclient = HttpClients.createDefault();

HttpResponse response = null;

try {

response = httpclient.execute(requestset.getMethod(url));

HttpEntity entity = response.getEntity();

reader = new BufferedReader(new InputStreamReader(entity.getContent()));

StringBuilder sb = new StringBuilder();

while((html = reader.readLine()) != null){

sb.append(html);

}

html = sb.toString();

System.out.println("一个html页面获取成功");

}

catch (IOException e) {

System.out.println(url+"连接失败");

}

finally{

if(reader != null){

try {

reader.close();

httpclient.close();

}

catch (IOException e) {

// TODO Auto-generated catch block

e.printStackTrace();

}

}

}

return html;

}

}

UrlGet类,用于根据一个html页面获得所有的url连接

package DownloadPackage;

import java.util.LinkedList;

import org.jsoup.Jsoup;

import org.jsoup.nodes.Document;

import org.jsoup.nodes.Element;

import org.jsoup.select.Elements;

public class UrlGet {

public LinkedList geturls(String html){

LinkedList urls = new LinkedList();

Document doc = Jsoup.parse(html);

Elements links = doc.getElementsByTag("a");

for (Element link:links){

String url = link.attr("href");

urls.add(url);

}

return urls;

}

}

资源选择接口,需要实现三个方法,第一是isNeed方法,判断url是否为需要的,第二个是isResourse方法,判断url页面是不是需要的资源页面,第三个是process方法,

有时网页上的url是我们需要的但是格式不对,对url进行加工

package ChoosePackage;

public interface ResourseChooser {

public Boolean isNeed(String url);

public Boolean isResourse(String url);

public String process(String url);

}

RequsetSet类,用于自定义请求方法的接口,实现getMethod方法获取请求方法

package DownloadPackage;

import org.apache.http.client.methods.HttpGet;

/*

* 一个用于获得Request请求的接口

* 实现getMethod方法获取Get方法

*/

public interface RequestSet {

public HttpGet getMethod(String url);

}

Saveutil接口用于自定义保存方式,需要实现save方法

package SaveUtil;

/*

* 数据储存的工具接口,必须实现保存方法

*/

public interface SaveUtil {

public void save(String url,String html);

}

Spider类,有五中构造方法,可以实现多种自定义操作,其中实现了上述自定义接口的默认实现类

package Spider;

import java.io.BufferedWriter;

import java.io.File;

import java.io.FileWriter;

import java.io.IOException;

import java.util.HashSet;

import java.util.Iterator;

import java.util.LinkedList;

import org.apache.http.client.config.RequestConfig;

import org.apache.http.client.methods.HttpGet;

import ChoosePackage.MyResourseChooser;

import ChoosePackage.ResourseChooser;

import DownloadPackage.HtmlDownloader;

import DownloadPackage.RequestSet;

import DownloadPackage.UrlGet;

import SaveUtil.MySaveUtil;

import SaveUtil.SaveUtil;

/*

* 用于爬取资源的类

*/

public class Spider{

public static void main(String[] args) {

new Spider("http://bilibili.net").spiderstart();

}

//种子url

String seed = null;

//用于保存数据的类,需要自己实现

private SaveUtil saveutil = null;

//html下载类

private HtmlDownloader downloader = null;

//url下载类

private UrlGet urldownloader = null;

//资源选择工具

private ResourseChooser resoursechooser = null;

//用于保存未下载的网页

LinkedList unvisited = new LinkedList();

//用于保存已下载的网页

HashSet visited = new HashSet();

//自定义储存方式,请求方式,资源筛选方式的构造方法

public Spider(SaveUtil saveutil,RequestSet request,ResourseChooser resoursechooser,String seed){

this.saveutil = saveutil;

this.downloader = new HtmlDownloader(request);

this.urldownloader = new UrlGet();

thhttp://is.resoursechooser = resoursechooser;

this.seed = seed;

unvisited.add(seed);

}

//自定义储存方式,资源筛选方式的构造方法

public Spider(SaveUtil saveutil,ResourseChooser resoursechooser,String seed){

this.resoursechooser = resoursechooser;

this.downloader = new HtmlDownloader(new getRequest());

this.saveutil = saveutil;

this.urldownloader = new UrlGet();

this.seed = seed;

unvisited.add(seed);

}

//自定义储存方式,请求的构造方法

public Spider(SaveUtil saveutil,RequestSet requestset,String seed){

this.saveutil = saveutil;

this.downloader = new HtmlDownloader(requestset);

this.resoursechooser = new MyResourseChooser();

this.urldownloader = new UrlGet();

this.seed = seed;

unvisited.add(seed);

}

//自定义储存方式的构造方法

public Spider(SaveUtil saveutil,String seed){

this.saveutil = saveutil;

this.downloader = new HtmlDownloader(new getRequest());

this.resoursechooser = (new MyResourseChooser());

this.urldownloader = new UrlGet();

this.seed = seed;

unvisited.add(seed);

}

//默认的爬虫构造方法

public Spider(String seed){

this.saveutil = new MySaveUtil();

this.downloader = new HtmlDownloader(new getRequest());

this.resoursechooser = (new MyResourseChooser());

this.urldownloader = new UrlGet();

this.seed = seed;

unvisited.add(seed);

}

//开始爬取的方法

private void spiderstart(){

String html = null;

while(!unvisited.isEmpty()){

String url = unvisited.poll();

System.out.println("开始获取"+url);

if(resoursechooser.isNeed(url)){

try{

html = downloader.downloadhtml(url);

}

catch(RuntimeException e){

System.out.println(url+"连接获取失败");

continue;

}

visited.add(url);

LinkedList urls = new LinkedList();

try{

urls = urldownloader.geturls(html);

}

catch(RuntimeException e){

System.out.println(url+"的html页面为空");

continue;

}

Iterator it = urls.iterator();

while(it.hasNext()){

String newurl = it.next();

if(resoursechooser.isNeed(newurl)&&!visited.contains(newurl)&&!unvisited.contains(newurl)){

newurl = resoursechooser.process(newurl);

unvisited.add(newurl);

System.out.println(newurl+"加入页面");

}

}

System.out.println("获取了"+url+"上的所有url");

if(resoursechooser.isResourse(url)){

saveutil.save(url,html);

}

}

}

}

//默认资源筛选类

private class MyResourseChooser implements ResourseChooser{

@Override

public Boolean isNeed(String url) {

// TODO Auto-generated method stub

if(!url.startsWith("/")&&!url.startsWith("http")){

return false;

}

return true;

}

@Override

public Boolean isResourse(String url) {

// TODO Auto-generated method stub

return true;

}

@Override

public String process(String url) {

// TODO Auto-generated method stub

if(!url.startsWith("http")){

url = seed+url;

}

return url;

}

}

public class getRequest implements RequestSet{

public HttpGet getMethod(String url) {

// TODO Auto-generated method stub

//创建一个get请求方法

HttpGet getmethod = new HttpGet(url);

//HttpHost proxy = new HttpHost("124.88.67.81",80);这里不设置代理IP

//设置请求超时时间等

RequestConfig responseconfig = RequestConfig.custom().setConnectionRequestTimeout(10000).setConnectTimeout(10000).setSocketTimeout(10000).build();

//设置请求头,主要是user-agent

getmethod.addHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36");

//设置请求参数

getmethod.setConfig(responseconfig);

return getmethod;

}

}

//默认的存储类

public class MySaveUtil implements SaveUtil{

@Override

public void save(String url, String html) {

// TODO Auto-generated method stub

String filename = getfilename(url);

BufferedWriter writer = null;

try{

writer = new BufferedWriter(new FileWriter(filename));

writer.write(html);

writer.flush();

System.out.println("文件写入成功");

}

catch(IOException e){

System.out.println("文件写入失败");

}

finally{

try {

if(writer != null)

writer.close();

}

catch (IOException e) {

// TODO Auto-generated catch block

System.out.println("流关闭失败");

}

}

}

private String getfilename(String url){

String fileparentpath = "f://html";

File file = new File(fileparentpath);

if(!file.exists()){

file.mkdir();

}

int last = url.lastIndexOf(".");

int first = url.indexOf(".");

url = url.substring(first,last);

url = url.replaceAll("\\.", "");

url = url.replaceAll("/", "");

return fileparentpath+"/"+url+".txt";

}

}

}

总结

以上就是本文关于分享一个简单的java爬虫框架的全部内容，希望对大家有所帮助。感兴趣的朋友可以继续参阅本站：python爬虫实例爬取网站搞笑段子、Java线程之锁对象Lock-同步问题更完美的处理方式代码实例、Java编程几个循环实例代码分享等，有什么问题可以随时留言，会及时回复大家的。感谢朋友们对本站的支持！

版权声明：本文内容由网络用户投稿，版权归原作者所有，本站不拥有其著作权，亦不承担相应法律责任。如果您发现本站中有涉嫌抄袭或描述失实的内容，请联系我们jiasou666@gmail.com 处理，核实后本网站将在24小时内删除侵权内容。