网络爬虫源代码

1、网络爬虫源代码 public class Spider implements Runnable private ArrayList urls; /URL列表 private HashMap indexedURLs; /已经检索过的URL列表 private int threads ; /初始化线程数 public static void main(String argv) throws Exception if(argv0 = null) System.out.println(Missing required argument: Sit URL); return ; Spider Spider

2、 = new Spider(argv0); Spider.go(); public Spider(String strURL) urls = new ArrayList(); threads = 10; urls.add(strURL); threadList = new ArrayList(); indexedURLs = new HashMap(); if (urls.size() = 0) throw new IllegalArgumentException(Missing required argument: -u start url); if (threads 1) (Invalid

3、 number of threads: + threads); public void go(String strURL) throws Exception / index each entry point URL long start = System.currentTimeMillis(); for (int i = 0; i ; 0) Thread child = (Thread)threadList.remove(0); child.join(); long elapsed = System.currentTimeMillis() - start; public void run()

4、String url; try while (url = dequeueURL() != null) indexURL(url); catch(Exception e) (e.getMessage(); /检测URL列表容器中有没有URL没有被解析,如果有则返回URL由线程继续执行 public synchronized String dequeueURL() throws Exception while (true) if (urls.size() ; 0) return (String)urls.remove(0); else threads-; if (thread

5、s ; 0) wait(); threads+; else notifyAll(); return null; /* * 添加URL和当前URL的级数，并唤醒睡眠线程 */ public synchronized void enqueueURL(String url,int level) if (indexedURLs.get(url) = null) urls.add(url); indexedURLs.put(url, new Integer(level); notifyAll(); /* * 通过URL解析出网页内容并解析出页面上的URL * param url 页面链接 * throw

6、s java.lang.Exception */ private void indexURL(String url) throws Exception boolean flag = true ; /判断网页链接的级别，系统默认为三级 int level = 1 ; if (indexedURLs.get(url) = null) indexedURLs.put(url, new Integer(level); else level = (Integer)indexedURLs.get(url).intValue(); /只检测到页面的第二级 if(level ; 2 ) return ; level+ ; String strBody = null ; try /解析页面内容 strBody = loadURL(url); catch(Exception e) return ; if (strBody != null) String urlGroups = null ; try /解析出页面所以URL urlGroups = parseURLs(summary); catch(Exception e) (e.getMessage(); if(urlGroups = null) urlGr