Java-HttpURLConnection爬虫程序-0913.doc

Android总结 孙沛林Java-HttpURLConnection抓取网络数据(2016-9-13)项目:JavaSpiderDemo 环境:MyEclipse8.5 导包源码JavaMyConn.java import java.io.BufferedReader;import java.io.InputStreamReader;import .*;import org.jsoup.Jsoup;import org.jsoup.nodes.Document;import org.jsoup.nodes.Element;import org.jsoup.select.Elements;/* * 独立的解析一篇CSDN博客 * 提取出:标题,分类,内容 * author 孙沛林 * */public class MyConn extends Thread public MyConn()public MyConn(String surl)this.surl=surl;private String surl;/ 文章的网址Overridepublic void run() getHTML();/ 客户端的浏览器类型public static String UserAgent = Mozilla/5.0 (jsoup),/ PC端的浏览器/ 以下都是手机端的浏览器 Mozilla/5.0 (Linux; U; Android 2.2; en-us; Nexus One Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.2, Mozilla/5.0 (iPad; U; CPU OS 3_2_2 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Version/4.0.4 Mobile/7B500 Safari/531.21.11, Mozilla/5.0 (SymbianOS/9.4; Series60/5.0 NokiaN97-1/20.0.019; Profile/MIDP-2.1 Configuration/CLDC-1.1) AppleWebKit/525 (KHTML, like Gecko) BrowserNG/7.1.18121, //yjflinchong Nokia5700AP23.01/SymbianOS/9.1 Series60/3.0, UCWEB7/28/998, NOKIA5700/UCWEB7/28/977, Openwave/UCWEB7/28/978, Mozilla/4.0 (compatible; MSIE 6.0; ) Opera/UCWEB7/28/989 ;/* * 根据一个网址,获取该页面的HTML源码 * param surl */public void getHTML()try / 1. 创建URL对象URL url = new URL(surl);/ 2. 获取HttpUrlConnection对象HttpURLConnection conn = (HttpURLConnection) url.openConnection();/ 2.5 伪装成浏览器conn.setRequestProperty( User-Agent, / 浏览器的类型 UserAgent1);/ 3. 获取服务器的响应码int responseCode = conn.getResponseCode();System.out.println(responseCode=+responseCode);/ 4. 判断响应码是否正常,HttpURLConnection.HTTP_OK=200if (responseCode = HttpURLConnection.HTTP_OK) StringBuffer sb = new StringBuffer();/ HTML的容器String readLine;/ 每一行的临时存放/5. 处理响应流，必须与服务器响应流输出的编码一致UTF-8BufferedReader responseReader = new BufferedReader( new InputStreamReader( conn.getInputStream(), UTF-8);/ 6. 循环读取流中的行while (readLine = responseReader.readLine() != null) sb.append(readLine).append(n);/ while/ 7. 关闭流responseReader.close();/System.out.println(*HTML*n+sb.toString();/输入HTMLparseHTML(sb.toString(); catch (Exception e) / TODO Auto-generated catch blocke.printStackTrace(); / 解析文章详细页面 private void parseHTML(String html) StringBuffer text = new StringBuffer();/ HTML的容器 Document doc = Jsoup.parse(html); / 1文章 title String atitle = doc.getElementsByClass(article_t).first().text(); text.append(n-文章 title-n); text.append(atitle); / 2文章 分类 Elements cats = doc.getElementsByClass(category_p); / 循环遍历 text.append(n-文章 分类-n); for(Element cat : cats) text.append(cat.text(); / 3文章 content String acontent = doc.getElementsByClass(article_c).first().text(); /text.append(acontent); text.append(n=文章 content=n); text.append(acontent); / 数据的插入 /insert(atitle,cats.text(),acontent); System.out.println(n=解析的结果=n +text.toString(); ListPage.java import java.io.BufferedReader;import java.io.InputStreamReader;import .*;import org.jsoup.Jsoup;import org.jsoup.nodes.Document;import org.jsoup.nodes.Element;import org.jsoup.select.Elements;/* * 解析CSDN博客专栏(包含很多文章列表) * 提取出:文章标题和链接 * author 孙沛林 * */public class ListPage extends Thread public ListPage()public ListPage(String surl)this.surl=surl;private String surl;/ 文章的网址Overridepublic void run() getHTML();/ 客户端的浏览器类型public static String UserAgent = Mozilla/5.0 (jsoup),/ PC端的浏览器/ 以下都是手机端的浏览器 Mozilla/5.0 (Linux; U; Android 2.2; en-us; Nexus One Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.2, Mozilla/5.0 (iPad; U; CPU OS 3_2_2 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Version/4.0.4 Mobile/7B500 Safari/531.21.11, Mozilla/5.0 (SymbianOS/9.4; Series60/5.0 NokiaN97-1/20.0.019; Profile/MIDP-2.1 Configuration/CLDC-1.1) AppleWebKit/525 (KHTML, like Gecko) BrowserNG/7.1.18121, //yjflinchong Nokia5700AP23.01/SymbianOS/9.1 Series60/3.0, UCWEB7/28/998, NOKIA5700/UCWEB7/28/977, Openwave/UCWEB7/28/978, Mozilla/4.0 (compatible; MSIE 6.0; ) Opera/UCWEB7/28/989 ;/* * 根据一个网址,获取该页面的HTML源码 * param surl */public void getHTML()try / 1. 创建URL对象URL url = new URL(surl);/ 2. 获取HttpUrlConnection对象HttpURLConnection conn = (HttpURLConnection) url.openConnection();/ 2.5 伪装成浏览器conn.setRequestProperty( User-Agent, / 浏览器的类型 UserAgent1);/ 3. 获取服务器的响应码int responseCode = conn.getResponseCode();System.out.println(responseCode=+responseCode);/ 4. 判断响应码是否正常,HttpURLConnection.HTTP_OK=200if (responseCode = HttpURLConnection.HTTP_OK) StringBuffer sb = new StringBuffer();/ HTML的容器String readLine;/ 每一行的临时存放/5. 处理响应流，必须与服务器响应流输出的编码一致UTF-8BufferedReader responseReader = new BufferedReader( new InputStreamReader( conn.getInputStream(), UTF-8);/ 6. 循环读取流中的行while (readLine = responseReader.readLine() != null) sb.append(readLine).append(n);/ while/ 7. 关闭流responseReader.close();/System.out.println(*HTML*n+sb.toString();/输入HTMLparseHTML(sb.toString(); catch (Exception e) / TODO Auto-generated catch blocke.printStackTrace(); / 解析文章详细页面 private void parseHTML(String html) StringBuffer text = new StringBuffer();/ HTML的容器 Document doc = Jsoup.parse(html); Elements items = doc.getElementsByTag(h4); for (Element item : items) / 找到链接 Element link = item.getElementsByTag(a).first(); if(link!=null) String href = link.attr(href); String title = link.text(); text.append(href); text.append(title); text.append(n-n); /getHTML(href); new MyConn(href).start();/ 启动线程- 解析一篇文章 Main.javaimport java.util.*;/* * java爬虫程序 * author 钧 * */public class Main /* * param args */public static void main(String args) / 循环解析多个URL/List list = new ArrayList();/list.add(/withiter/article/details/16802241);/list.add(