网络爬虫之获取图片到本地.doc

/* * Created on Aug 26, 2011 2:41:26 PM * * HtmlSourceGetter.java * * NOTICE OF PROPRIETARY RIGHTS * * This program is a confidential trade secret and the property of author. Use, examination, * reproduction, disassembly, decompiling, transfer and/or disclosure to others of * all or any part of this software program are strictly prohibited except by express * written agreement with author. * * - * Modification History * Date Author Version Description * Aug 26, 2011 Cross 1.0 New * - */package com.cross.tools;import java.io.BufferedInputStream;import java.io.File;import java.io.FileOutputStream;import java.io.IOException;import java.io.OutputStream;import .HttpURLConnection;import .URL;import org.htmlparser.Node;import org.htmlparser.NodeFilter;import org.htmlparser.Parser;import org.htmlparser.filters.NodeClassFilter;import org.htmlparser.tags.ImageTag;import org.htmlparser.tags.LinkTag;import org.htmlparser.util.NodeList;import org.htmlparser.util.ParserException;import org.htmlparser.util.SimpleNodeIterator;public class HtmlSourceGetter private static HttpURLConnection con = null; private static BufferedInputStream bis = null; private static OutputStream out = null; public static void getSource(String url) public static void parseHTML(String url, String keyword) private static void processNodeList(NodeList list, String keyword) public static void extractLinks(String url) try Parser parser = new Parser(url); parser.setEncoding(UTF-8); / frame filter NodeFilter frameFilter = new NodeFilter() Override public boolean accept(Node node) if(node.getText().startsWith(frame src=) return true; return false; ; / image filter; NodeFilter imageFilter = new NodeClassFilter(ImageTag.class); / href filter; NodeFilter linkFilter = new NodeClassFilter(LinkTag.class); / link or image filter/ OrFilter orFilter = new OrFilter(new NodeClassFilter(LinkTag.class),new NodeClassFilter(ImageTag.class); / link or image or frame filter/ OrFilter allFilter = new OrFilter(orFilter,frameFilter); NodeList nodeList = parser.extractAllNodesThatMatch(imageFilter); for (int i = 0; i nodeList.size(); i+) Node tag = nodeList.elementAt(i); / tag/ if(tag instanceof LinkTag) / LinkTag link = (LinkTag)tag;/ String linkURL = link.getLink();/ String linkText = link.getLinkText();/ System.out.println(linkURL:+linkURL);/ System.out.println(linkText:+linkText);/ / tag/ else if(tag instanceof ImageTag) ImageTag image = (ImageTag)tag; String imageURL = image.getImageURL(); String imageText = image.getText(); System.out.println(imageURL:+imageURL); System.out.println(imageText:+imageText); con = (HttpURLConnection)(new URL(imageURL).openConnection(); con.connect(); bis = new BufferedInputStream(con.getInputStream(); out = new FileOutputStream(new File(c:/cross/ + i + _ +System.currentTimeMillis() +imageURL.substring(imageURL.lastIndexOf(.); byte buf = new byte1024; int size = 0; while(size = bis.read(buf) != -1) out.write(buf, 0, size); / out.flush(); / else / tag eg:/ String frame = tag.getText();/ String frameURL = frame.split()1;/ System.out.println(frameURL:+frameURL);/ / catch (Exception e) System.err.println(e.getStackTrace(); finally try out.close(); bis.close(); con.disconnect(); catch (IOException e) e.printStackTrace(); public static void main(String args) / HtmlSourceGetter.parseHTML(http:/localhost:8080/test/, );/