java开发项目集锦(附源码)

新浪天气预报新闻java抓去程序package .weather1;import java.io.BufferedReader;import java.io.ByteArrayOutputStream;import java.io.File;import java.io.FileWriter;import java.io.IOException;import java.io.InputStream;import java.io.InputStreamReader;import java.io.PrintWriter;import .URL;import .URLConnection;import java.util.regex.Matcher;import java.util.regex.Pattern;import mons.logging.Log;import mons.logging.LogFactory;import .update.Getdata;/* * 正则方式抓取新浪天气新闻上的新闻 * 地址/weather/news/index.html * param args */public class Newlist private static final Log log = LogFactory.getLog(Newlist.class); /* * 测试 * param args */ public static void main(String args) Newlist n=new Newlist(); String k=n.getNewList(); for (int i=0;ik.length;i+) System.out.println(ki.replace(href=, href=newinfo2.jsp?url=); String m=n.getNewinfo(news/2008/1119/35261.html); for (int l=0;lm.length;l+) System.out.println(ml); /* * 由url地址获得新闻内容string * 新闻中的图片下载到本地，文中新闻地址改成本地地址 * param url * return */ public String getNewinfo(String url) String URL=/+url; /30是指取30段满足给出的正则条件的字符串，如果只找出10个，那数组后面的全为null String s = analysis(.*?) , getContent(URL) , 30); for (int i=0;i , content , 50); String s = analysis(.*?) , content , 50); return s; private String analysis(String pattern, String match , int i) Pattern sp = Ppile(pattern); Matcher matcher = sp.matcher(match); String content = new Stringi; for (int i1 = 0; matcher.find(); i1+) contenti1 = matcher.group(1); /下面一段是为了剔除为空的串 int l=0; for (int k=0;kcontent.length;k+) if (contentk=null) l=k; break; String content2; if (l!=0) content2=new Stringl; for (int n=0;n 0) outputstream.write(str_b,0,i); all_content = outputstream.toString(); / System.out.println(all_content); catch (Exception e) e.printStackTrace(); log.error(获取网页内容出错); finally uc = null; / return new String(all_content.getBytes(ISO8859-1); System.out.println(all_content.length(); return all_content; 现在的问题是:图片下载不全，我用后面两种getContent方法下图片，下来的图片大小都和文件头里获得的Content-Length，也就是图片的实际大小不符，预览不了。 而且反复测试，两种方法每次下来的东西大小是固定的，所以重复下载没有用？ 测试toString后length大小比图片实际的小，而生成的图片比图片数据大。下载后存储过程中图片数据增加了！ 图片数据流toString过程中数据大小发生了改变，还原不回来。其它新闻内容没有问题。估计是图片的编码格式等的问题。在图片数据流读过来时直接生成图片就可以了。public int saveImage (String strUrl) URLConnection uc = null; try URL url = new URL(strUrl); uc = url.openConnection(); uc.setRequestProperty(User-Agent, Mozilla/4.0 (compatible; MSIE 5.0; Windows XP; DigExt); /uc.setReadTimeout(30000); /获取图片长度 /System.out.println(Content-Length: +uc.getContentLength(); /获取文件头信息 /System.out.println(Header+uc.getHeaderFields().toString(); if (uc = null) return 0; InputStream ins = uc.getInputStream(); byte str_b = new byte1024; int byteRead=0; String images=strUrl.split(/); String imagename=imagesimages.length-1; File fwl = new File(imagename); FileOutputStream fos= new FileOutputStream(fwl); while (byteRead=ins.read(str_b) 0) fos.write(str_b,0,byteRead); ; fos.flush(); fos.close(); catch (Exception e) e.printStackTrace(); log.error(获取网页内容出错); finally uc = null; return 1; 方法二：首先把搜索后的页面用流读取出来，再写个正则，去除不要的内容，再把最后的结果存成xml格式文件、或者直接存入数据库，用的时候再调用本代码只是显示html页的源码内容，如果需要抽取内容请自行改写public static String regex()中的正则式 package rssTest; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; import .HttpURLConnection; import .MalformedURLException; import .URL; import .URLConnection; import java.util.ArrayList; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; public class MyRSS /* * 获取搜索结果的html源码 * */ public static String getHtmlSource(String url) StringBuffer codeBuffer = null; BufferedReader in=null; try URLConnection uc = new URL(url).openConnection(); /* * 为了限制客户端不通过网页直接读取网页内容,就限制只能从浏览器提交请求. * 但是我们可以通过修改http头的User-Agent来伪装,这个代码就是这个作用 * */ uc.setRequestProperty(User-Agent, Mozilla/4.0 (compatible; MSIE 5.0; Windows XP; DigExt); / 读取url流内容 in = new BufferedReader(new InputStreamReader(uc .getInputStream(), gb2312); codeBuffer = new StringBuffer(); String tempCode = ; / 把buffer内的值读取出来,保存到code中 while (tempCode = in.readLine() != null) codeBuffer.append(tempCode).append(n); in.close(); catch (MalformedURLException e) e.printStackTrace(); catch (IOException e) e.printStackTrace(); return codeBuffer.toString(); /* * 正则表达式 * */ public static String regex() String googleRegex = (.*?)href=(.*?)(.*?)(.*?)(.*?)(.*?); return googleRegex; /* * 测试用 * 在google中检索关键字，并抽取自己想要的内容 * * */ public static List GetNews() List newsList = new ArrayList(); String allHtmlSource = MyRSS .getHtmlSource(/search?complete=1&hl=zh-CN&newwindow=1&client=aff-os- maxthon&hs=SUZ&q=%E8%A7%81%E9%BE%99%E5%8D%B8%E7%94%B2&meta=&aq=f); Pattern pattern = Ppile(regex(); Matcher matcher = pattern.matcher(allHtmlSource); while (matcher.find() String urlLink = matcher.group(2); String title = matcher.group(4); title = title.replaceAll(, ); title = title.replaceAll(, ); title = title.replaceAll(., ); String content = matcher.group(6); content = content.replaceAll(, ); content = content.replaceAll(, ); content = content.replaceAll(., ); newsList.add(urlLink); newsList.add(title); newsList.add(content); return newsList; /* * main方法 * */ public static void main(String args) System.out .println(MyRSS .getHtmlSource(/news/zckb/index.html); 方法三：jsp自动抓取新闻 自动抓取新闻package com.news.spider;import java.io.File;import java.io.FileFilter;import java.text.SimpleDateFormat;import java.util.ArrayList;import java.util.Calendar;import java.util.Date;import java.util.List;import java.util.regex.Matcher;import java.util.regex.Pattern;import com.db.DBAccess;public class SpiderNewsServer public static void main(String args) throws Exception /设置抓取信息的首页面 String endPointUrl = /zixun/; /获得当前时间 Calendar calendar=Calendar.getInstance(); SimpleDateFormat sdf=new SimpleDateFormat(yyyy-MM-dd); String DateNews = sdf.format(calendar.getTime(); /* * 抓取二级URl 开始 * url匹配类型：/article/ */ List listNewsType = new ArrayList(); /取入口页面html WebHtml webHtml = new WebHtml(); String htmlDocuemtnt1 = webHtml.getWebHtml(endPointUrl); if(htmlDocuemtnt1 = null | htmlDocuemtnt1.length() = 0) return; String strTemp1 = /article/; String strTemp2 = ; int stopIndex=0; int startIndex=0; int dd=0; while(true) dd+; startIndex = htmlDocuemtnt1.indexOf(strTemp1, stopIndex); System.out.println(=+startIndex); stopIndex= htmlDocuemtnt1.indexOf(strTemp2, startIndex); System.out.println(=-+stopIndex); if(startIndex!=-1 & stopIndex!=-1) String companyType=htmlDocuemtnt1.substring(startIndex,stopIndex); System.out.println(-+companyType); System.out.println(-+companyType.indexOf(); companyType=companyType.substring(0,companyType.indexOf(); System.out.println(#-+companyType); listNewsType.add(companyType); if(dd10) break; if(stopIndex=-1 | startIndex=-1) break; System.out.println(listCompanyType=+listNewsType.size(); /* * 抓取二级URl 结束 */ /* * 抓取页面内容 开始 */ String title=; String hometext=; String bodytext=; String keywords=; String counter = 221; String cdate= ; int begainIndex=0;/检索字符串的起点索引 int endIndex=0;/检索字符串的终点索引 String begainStr;/检索开始字符串 String endStr;/检索结束字符串 for (int rows = 1; rows 0) WebHtml newsListHtml = new WebHtml(); String htmlDocuemtntCom = newsListHtml.getWebHtml(strNewsDetail); System.out.println($-+htmlDocuemtntCom); if(htmlDocuemtntCom = null | htmlDocuemtntCom.length() = 0) return; /截取时间 int dateBegainIndex = htmlDocuemtntCom.indexOf(时间：); System.out.println(%-+dateBegainIndex); String newTime = htmlDocuemtntCom.substring(dateBegainIndex,dateBegainIndex+20); System.out.println(-+newTime); String newTimeM = newTime.substring(newTime.lastIndexOf(-)+1,newTime.lastIndexOf(-)+3); String dateM = DateNews.substring(DateNews.lastIndexOf(-)+1); System.out.println(-+newTimeM); System.out.println(-+dateM); if(newTimeM = dateM | newTimeM.equals(dateM) /检索新闻标题 begainStr=; endStr=时间：; begainIndex=htmlDocuemtntCom.indexOf(begainStr,0); System.out.println(&-+begainIndex); endIndex=htmlDocuemtntCom.indexOf(endStr,0); System.out.println(&-+endIndex); if(begainIndex!=-1 & endIndex!=-1) title = htmlDocuemtntCom.substring(begainIndex,endIndex).trim(); title = title.substring(title.indexOf()+4,title.indexOf(); title = title.replace(, ); title = title.replace(;, ); title = title.replace( , ); /检索新闻内容 begainStr=; endStr=; begainIndex=htmlDocuemtntCom.indexOf(begainStr,0); endIndex=htmlDocuemtntCom.indexOf(endStr,0); if(begainIndex!=-1 & endIndex!=-1) bodytext = htmlDocuemtntCom.substring(begainIndex,endIndex).trim(); if(bodytext.indexOf()0 & bodytext.indexOf()bodytext.indexOf() & bodytext.indexOf()0) bodytext = bodytext.substring(bodytext.indexOf()+3,bodytext.indexOf(); bodytext=bodytext.replace( , ); bodytext=bodytext.replace(, ); bodytext=bodytext.replace(n, ); bodytext=bodytext.replace(, ); bodytext=bodytext.replace(;, ); /简介 if(bodytext.length()40) hometext = bodytext.substring(0,40)+.; else hometext = bodytext+.; /浏览量 String str = String.valueOf(Math.random(); counter = str.substring(str.lastIndexOf(.)+1,5); Calendar cal = Calendar.getInstance(); cal.setTime(new Date(); cdate = cal.getTimeInMillis()+; cdate = cdate.substring(0,10);