htmlparser实现从网页上抓取数据

1、package parser;import java.io.BufferedReader;import java.io.BufferedWriter;import java.io.FileWriter;import java.io.IOException;import java.io.InputStream;import java.io.InputStreamReader;import .MalformedURLException;import .URL;/* * 基本能实现网页抓取，不过要手动输入URL 将整个html内容保存到指定文件 * * author

2、chenguoyong * */public class ScrubSelectedWeb private final static String CRLF = System.getProperty(line.separator);/* * param args */public static void main(String args) try URL ur = new URL(99:8083/injs100/);InputStream instr = ur.openStream();String s, str;BufferedReader in = ne

3、w BufferedReader(new InputStreamReader(instr);StringBuffer sb = new StringBuffer();BufferedWriter out = new BufferedWriter(new FileWriter(D:/outPut.txt);while (s = in.readLine() != null) sb.append(s + CRLF);System.out.println(sb);str = new String(sb);out.write(str);out.close();in.close(); catch (Mal

4、formedURLException e) e.printStackTrace(); catch (IOException e) e.printStackTrace();基本能实现网页抓取，不过要手动输入URL，此外没有重构。只是一个简单的思路。1.htmlparser 使用htmlparser是一个纯的java写的html解析的库，htmlparser不依赖于其它的java库，htmlparser主要用于改造 或提取html。htmlparser能超高速解析html，而且不会出错。毫不夸张地说，htmlparser就是目前最好的html解 析和分析的工具。无论你是想抓取网页数据还是改造htm

5、l的内容，用了htmlparser绝对会忍不住称赞。由于htmlparser 结构设计精良，所以扩展htmlparser 非常便利。Htmlparser中文论坛./thread.php?fid=6Constructor SummaryParser() Parser(URLConnectionconnection) Construct a parser using the provided URLConnection.Method:staticParser createParser(Stringhtml, Stringcharset) Creates the

6、 parser on an input string.void visitAllNodesWith(NodeVisitorvisitor) Apply the given visitor to the current page.HtmlPage(Parserparser)NodeListgetBody() TableTaggetTables() StringgetTitle() voidsetTitle(Stringtitle) voidvisitTag(Tagtag) Called for each Tag visited.Constructor SummaryNodeList() Node

7、List(Nodenode) Create a one element node list.NodeList extractAllNodesThatMatch(NodeFilterfilter) Filter the list with the given filter non-recursively. NodeList extractAllNodesThatMatch(NodeFilterfilter, booleanrecursive) Filter the list with the given filter.Node elementAt(inti)1. html代码里面所有的链接地址和

8、链接名称package parser;import org.htmlparser.Parser;import org.htmlparser.Node;import org.htmlparser.NodeFilter;import org.htmlparser.Parser;import org.htmlparser.filters.TagNameFilter;import org.htmlparser.tags.LinkTag;import org.htmlparser.tags.TableTag;import org.htmlparser.util.NodeList;import org.h

9、tmlparser.util.ParserException;import org.htmlparser.visitors.HtmlPage;/* * htmlparser取得一段html代码里面所有的链接地址和链接名称 * * author chenguoyong * */public class Testhtmlparser /* * param args */public static void main(String args) String htmlcode = AAA+ 连接1+ 连接2;/ 创建Parser对象根据传给字符串和指定的编码Parser parser = Parser

10、.createParser(htmlcode, GBK);/ 创建HtmlPage对象HtmlPage(Parser parser)HtmlPage page = new HtmlPage(parser);try / HtmlPage extends visitor,Apply the given visitor to the current/ page.parser.visitAllNodesWith(page); catch (ParserException e1) e1 = null;/ 所有的节点NodeList nodelist = page.getBody();/ 建立一个节点fi

11、lter用于过滤节点NodeFilter filter = new TagNameFilter(A);/ 得到所有过滤后，想要的节点nodelist = nodelist.extractAllNodesThatMatch(filter, true);for (int i = 0; i nodelist.size(); i+) LinkTag link = (LinkTag) nodelist.elementAt(i);/ 链接地址System.out.println(link.getAttribute(href) + n);/ 链接名称System.out.println(link.getSt

12、ringText();结果如下：/u/14/0ff402ef-c382-499a-8213-ba6b2f.html连接1连接22. 使用HtmlParser抓去网页内容package parser;import org.htmlparser.Parser;import org.htmlparser.beans.StringBean;import org.htmlparser.filters.NodeClassFilter;import org.htmlparser.parserapplications.String

13、Extractor;import org.htmlparser.tags.BodyTag;import org.htmlparser.util.NodeList;import org.htmlparser.util.ParserException;/* * 使用HtmlParser抓去网页内容: 要抓去页面的内容最方便的方法就是使用StringBean. 里面有几个控制页面内容的几个参数. * 在后面的代码中会有说明. Htmlparser包中还有一个示例StringExtractor 里面有个直接得到内容的方法, * 其中也是使用了StringBean . 另外直接解析Parser的每个标签

14、也可以的. * * author chenguoyong * */public class GetContent public void getContentUsingStringBean(String url) StringBean sb = new StringBean();sb.setLinks(true); / 是否显示web页面的连接(Links)/ 为了取得页面的整洁美观一般设置上面两项为true , 如果要保持页面的原有格式, 如代码页面的空格缩进 可以设置为falsesb.setCollapse(true); / 如果是true的话把一系列空白字符用一个字符替代.sb.setR

15、eplaceNonBreakingSpaces(true);/ If true regular spacesb.setURL(/51AOP/archive/2006/07/19/59064.html);System.out.println(The Content is :n + sb.getStrings();public void getContentUsingStringExtractor(String url, boolean link) / StringExtractor内部机制和上面的一样.做了一下包装StringExtractor se

16、= new StringExtractor(url);String text = null;try text = se.extractStrings(link);System.out.println(The content is :n + text); catch (ParserException e) e.printStackTrace();public void getContentUsingParser(String url) NodeList nl;try Parser p = new Parser(url);nl = p.parse(new NodeClassFilter(BodyT

17、ag.class);BodyTag bt = (BodyTag) nl.elementAt(0);System.out.println(bt.toPlainTextString(); / 保留原来的内容格式. 包含js代码 catch (ParserException e) e.printStackTrace();/* * param args */public static void main(String args) String url = /51AOP/archive/2006/07/19/59064.html;/new GetContent

18、().getContentUsingParser(url);/-new GetContent().getContentUsingStringBean(url);3.将整个html内容保存到指定文件package parser;import java.io.BufferedReader;import java.io.BufferedWriter;import java.io.FileWriter;import java.io.IOException;import java.io.InputStream;import java.io.InputStreamReader;import java.ne

19、t.MalformedURLException;import .URL;/* * 基本能实现网页抓取，不过要手动输入URL 将整个html内容保存到指定文件 * * author chenguoyong * */public class ScrubSelectedWeb private final static String CRLF = System.getProperty(line.separator);/* * param args */public static void main(String args) try URL ur = new URL(http:/www.

20、/);InputStream instr = ur.openStream();String s, str;BufferedReader in = new BufferedReader(new InputStreamReader(instr);StringBuffer sb = new StringBuffer();BufferedWriter out = new BufferedWriter(new FileWriter(D:/outPut.txt);while (s = in.readLine() != null) sb.append(s + CRLF);System.ou

21、t.println(sb);str = new String(sb);out.write(str);out.close();in.close(); catch (MalformedURLException e) e.printStackTrace(); catch (IOException e) e.printStackTrace();4利用htmlparser提取网页纯文本的例子package parser;import org.htmlparser.Node;import org.htmlparser.NodeFilter;import org.htmlparser.Parser;impo

22、rt org.htmlparser.filters.TagNameFilter;import org.htmlparser.tags.TableTag;import org.htmlparser.util.NodeList;/* * 标题:利用htmlparser提取网页纯文本的例子 */public class TestHTMLParser2 /* * 读取目标html内容 * */public static void testHtml() try String sCurrentLine;String sTotalString;sCurrentLine = ;sTotalString = ;

23、java.io.InputStream l_urlStream;.URL l_url = new .URL(99:8083/injs100/);.HttpURLConnection l_connection = (.HttpURLConnection) l_url.openConnection();l_connection.connect();l_urlStream = l_connection.getInputStream();java.io.BufferedReader l_reader =

24、 new java.io.BufferedReader(new java.io.InputStreamReader(l_urlStream);while (sCurrentLine = l_reader.readLine() != null) sTotalString += sCurrentLine + rn;String testText = extractText(sTotalString); catch (Exception e) e.printStackTrace(); /* * 抽取纯文本信息 * param inputHtml：html文本 * return * throws Ex

25、ception */public static String extractText(String inputHtml) throws Exception StringBuffer text = new StringBuffer();Parser parser = Parser.createParser(new String(inputHtml.getBytes(),GBK), GBK);/ 遍历所有的节点NodeList nodes = parser.extractAllNodesThatMatch(new NodeFilter() public boolean accept(Node no

26、de) return true;);System.out.println(nodes.size();for (int i = 0; i nodes.size(); i+) Node nodet = nodes.elementAt(i);/字符串的代表性节点:节点的描述text.append(new String(nodet.toPlainTextString().getBytes(GBK)+ rn);return text.toString(); /* * 读取文件的方式/utl 来分析内容. filePath也可以是一个Url. * param resource :文件/Url * thro

27、ws Exception */public static void test5(String resource) throws Exception Parser myParser = new Parser(resource);myParser.setEncoding(GBK);String filterStr = table;NodeFilter filter = new TagNameFilter(filterStr);NodeList nodeList = myParser.extractAllNodesThatMatch(filter);/*for(int i=0;inodeList.s

28、ize();i+)TableTag tabletag = (TableTag) nodeList.elementAt(i);/标签名称System.out.println(tabletag.getTagName();System.out.println(tabletag.getText();*/TableTag tabletag = (TableTag) nodeList.elementAt(1);public static void main(String args) throws Exception test5(99:8083/injs100/);/te

29、stHtml();5.html解析tablepackage parser;import org.apache.log4j.Logger;import org.htmlparser.NodeFilter;import org.htmlparser.Parser;import org.htmlparser.filters.NodeClassFilter;import org.htmlparser.filters.OrFilter;import org.htmlparser.filters.TagNameFilter;import org.htmlparser.tags.TableColumn;im

30、port org.htmlparser.tags.TableRow;import org.htmlparser.tags.TableTag;import org.htmlparser.util.NodeList;import org.htmlparser.util.ParserException;import junit.framework.TestCase;public class ParserTestCase extends TestCase private static final Logger logger = Logger.getLogger(ParserTestCase.class

31、);public ParserTestCase(String name) super(name);/* * 测试对 * * * * 的解析 */public void testTable() Parser myParser;NodeList nodeList = null;myParser = Parser.createParser( + + 1-111-121-13+ 1-211-221-23+ 1-311-321-33+ + 2-112-122-13+ 2-212-222-23+ 2-312-322-33+ , GBK);NodeFilter tableFilter = new NodeC

32、lassFilter(TableTag.class);OrFilter lastFilter = new OrFilter();lastFilter.setPredicates(new NodeFilter tableFilter );try nodeList = myParser.parse(lastFilter);for (int i = 0; i = nodeList.size(); i+) if (nodeList.elementAt(i) instanceof TableTag) TableTag tag = (TableTag) nodeList.elementAt(i);Tabl

33、eRow rows = tag.getRows();for (int j = 0; j rows.length; j+) TableRow tr = (TableRow) rowsj;System.out.println(tr.getAttribute(id);if (tr.getAttribute(id).equalsIgnoreCase(tro1) TableColumn td = tr.getColumns();for (int k = 0; k td.length; k+) / logger.fatal( +/ tdk.toPlainTextString();System.out.pr

34、intln(+ tdk.toPlainTextString(); catch (ParserException e) e.printStackTrace();/* * 得到目标数据 * * param url:目标url * throws Exception */public static void getDatabyUrl(String url) throws Exception Parser myParser = new Parser(url);NodeList nodeList = null;myParser.setEncoding(gb2312);NodeFilter tableFil

35、ter = new NodeClassFilter(TableTag.class);OrFilter lastFilter = new OrFilter();lastFilter.setPredicates(new NodeFilter tableFilter );try nodeList = myParser.parse(lastFilter);/ 可以从数据table的size:19-21开始到结束for (int i = 15; i = nodeList.size(); i+) if (nodeList.elementAt(i) instanceof TableTag) TableTag

36、 tag = (TableTag) nodeList.elementAt(i);TableRow rows = tag.getRows();for (int j = 0; j rows.length; j+) TableRow tr = (TableRow) rowsj;if (tr.getAttribute(id) != null& tr.getAttribute(id).equalsIgnoreCase(tr02) TableColumn td = tr.getColumns();/ 对不起，没有你要查询的记录！if (td.length = 1) System.out.println(对

37、不起，没有你要查询的记录); else for (int k = 0; k td.length; k+) System.out.println(内容：+ tdk.toPlainTextString().trim(); catch (ParserException e) e.printStackTrace();/* * 测试已经得出有数据时table:22个，没有数据时table:19个 * * param args */public static void main(String args) try / getDatabyUrl(/user/queryton

38、ebytype.do?field=tonecode&condition=&type=1006&pkValue=);getDatabyUrl(/user/querytonebytype.do?field=tonecode&condition=&type=1006&pkValue=); catch (Exception e) e.printStackTrace();6.html解析常用package com.jscud.test;import java.io.BufferedReader;import java.io.File;import java.io.Fi

39、leInputStream;import java.io.InputStreamReader;import org.htmlparser.Node;import org.htmlparser.NodeFilter;import org.htmlparser.Parser;import org.htmlparser.filters.NodeClassFilter;import org.htmlparser.filters.OrFilter;import org.htmlparser.nodes.TextNode;import org.htmlparser.tags.LinkTag;import

40、org.htmlparser.util.NodeList;import org.htmlparser.util.ParserException;import org.htmlparser.visitors.HtmlPage;import org.htmlparser.visitors.TextExtractingVisitor;import com.jscud.util.LogMan; /一个日志记录类/* 演示了Html Parse的应用.* * author scud (/)*/public class Pars

41、eHtmlTestpublic static void main(String args) throws ExceptionString aFile = e:/jscud/temp/test.htm;String content = readTextFile(aFile, GBK);test1(content);System.out.println(=);test2(content);System.out.println(=);test3(content);System.out.println(=);test4(content);System.out.println(=);test5(aFil

42、e);System.out.println(=);/访问外部资源,相对慢test5( (/); System.out.println(=);/* 读取文件的方式来分析内容.* filePath也可以是一个Url.* * param resource 文件/Url*/public static void test5(String resource) throws ExceptionParser myParser = new Parser(resource);/设置编码myParser.setEncoding(GBK);H

43、tmlPage visitor = new HtmlPage(myParser);myParser.visitAllNodesWith(visitor);String textInPage = visitor.getTitle();System.out.println(textInPage);/* 按页面方式处理.对一个标准的Html页面,推荐使用此种方式.*/public static void test4(String content) throws ExceptionParser myParser;myParser = Parser.createParser(content, GBK);

44、HtmlPage visitor = new HtmlPage(myParser);myParser.visitAllNodesWith(visitor);String textInPage = visitor.getTitle();System.out.println(textInPage);/* 利用Visitor模式解析html页面.* 小优点:翻译了等符号 * 缺点:好多空格,无法提取link* */public static void test3(String content) throws ExceptionParser myParser;myParser = Parser.cre

45、ateParser(content, GBK);TextExtractingVisitor visitor = new TextExtractingVisitor();myParser.visitAllNodesWith(visitor);String textInPage = visitor.getExtractedText();System.out.println(textInPage);/* 得到普通文本和链接的内容.* * 使用了过滤条件.*/public static void test2(String content) throws ParserExceptionParser my

46、Parser;NodeList nodeList = null;myParser = Parser.createParser(content, GBK);NodeFilter textFilter = new NodeClassFilter(TextNode.class);NodeFilter linkFilter = new NodeClassFilter(LinkTag.class);/暂时不处理 meta/NodeFilter metaFilter = new NodeClassFilter(MetaTag.class);OrFilter lastFilter = new OrFilte

47、r();lastFilter.setPredicates(new NodeFilter textFilter, linkFilter );nodeList = myParser.parse(lastFilter);Node nodes = nodeList.toNodeArray();for (int i = 0; i nodes.length; i+)Node anode = (Node) nodesi;String line = ;if (anode instanceof TextNode)TextNode textnode = (TextNode) anode;/line = textnode.toPlainTextString().trim();line = textnode.getText();else if (anode instanceof LinkTag)LinkTag linknode = (LinkTag) anode;line = linknode.getLink();/todo () 过滤jsp标签:可以自己实现这个函数/line = StringFunc.replace(l