如何构建搜索引擎

1、如何构建搜索引擎思路：网页采集（网络爬虫） 网页分析 网页索引 网页检索 索引创建1. 网络爬虫从指定的网址或IP下载网页，保存到本地临时文件。package com.yjb.a;import java.io.BufferedReader;import java.io.File;import java.io.FileWriter;import java.io.IOException;import java.io.InputStreamReader;import java.io.PrintWriter;import .Socket;import .UnknownHostException;pub

2、lic class WebCrawler private static String Text_File_Path = "D:Project_2010chhtmlsrc.html"public static void main(String args) tryFile file = new File(Text_File_Path);/ 文件可以没有，但文件夹一定要有FileWriter fpWriter = new FileWriter(file);Socket webclient = new Socket("",80);PrintWriter resu

3、lt = new PrintWriter(webclient.getOutputStream(),true);BufferedReader receiver = new BufferedReader(new InputStreamReader(webclient.getInputStream();result.println("GET / HTTP/1.1");result.println("Host:");result.println("Connection:Close");result.println();boolean bRet

4、 = true;StringBuffer sb = new StringBuffer(8096);while(bRet)if(receiver.ready()int idx = 0;while(idx != -1)idx = receiver.read();if(idx = '<')break;while(idx != -1)sb.append(char)idx);idx = receiver.read();bRet = false;System.out.println(sb.toString(); fpWriter.write(sb.toString(); webcli

5、ent.close(); fpWriter.close();catch(UnknownHostException e)System.out.println(" 无法访问指定主机 ");System.exit(1);catch(IOException e)System.out.println(" 下载失败，请检查输入的地址是否正确 "); System.exit(1);2. 网页分析程序读取本地临时文件，通过筛选器筛选，保留与关键字匹配的内容，删掉无 用内容。package com.yjb.a;import java.io.File;import java

6、.io.FileReader;import java.io.FileWriter;import java.io.IOException;import .UnknownHostException;public class WebParserFilter private static String src_File_Path = "D:Project_2010chhtmlsrc.html" private static String dst_File_Path = "D:Project_2010chpuresrcfilter.txt" public stat

7、ic void main(String args) tryParserFilter();catch(IOException e)System.out.println(" 下载失败，请检查输入的地址是否正确 "); System.exit(1);public static void ParserFilter() throws IOExceptiontryinf j H o八 boo-ean bf-ag Hfrup boo-ean bconCDnf HfrupSiring Buffer sBuffer H new Siring Buffer(8096*2)八 char-1 eB

8、uffer H new char-8096*2一八 char= dssuffer H new char-8096*2一八 inf nCounf H 0-Fi-e srcfi-e H new Fi-e(srclFi-elpafh)八 Fi-eReader fpReader H new Fi-eReader(srcfi-e)八 Fi-e dsffi-e H new Fi-e-dsflFi-elpafh)八 Fi-ewriCDr fpwriCDr H new Fi-ewriCDr(dsffi-e)八 nCounf H fpReader.read(CBUffer)八 for(inf i H 0= A

9、ncounf.+)宀 if(bconCDnf HH fa-sex ifoBuffer 三 HH -V-)宀 bconCDnf Hfrupose 宀 continue-)e-se 宀 ifoBuffer 三 HH A)宀bconCDnf H fa-spcontinue-ose ifoBuffer三 HH 三-CBUffesHH - - - CBU=fter三 HH - -CBUffemTH-50/ 回<福continue-ose ifoBuffer三 HH g-QOQOCBUfferT二 HH -n-QOQOCBUffer=+21 HH -b-奚 CBUffer=+3 一 HH -s-奚

10、CBUfferT4 一 HH -p-奚 CBUffer=+5 一 HHi H i + 5八continue-ds60ufferu+上 H eBuffer 三八bf-ag H frupfor(inf 3 H A j-m +)宀if(ds60uffer-ml AH -N-QOQOds60uffer-m;u -A-)=(dssufferm AH -N-QOQOds60uffer-ml AH -a-)=(dssufferm AH 9QOQOds5uffer-m;Ho-)宀if(bflag HH fa-sexSBUffer.appendz)八s B uffe r a p pe nd (dssuffe r

11、m)八elseif(dstBufferm = ' 、 ' | dstBufferm = '|' | dstBufferm = '"' | dstBufferm = ':'| dstBufferm = '' | dstBufferm = '.')sBuffer.append(' ');continue;if(bflag = true) sBuffer.append(' ');sBuffer.append(dstBufferm); sBuffer.append(

12、' ');bflag = false;System.out.println(sBuffer.toString(); fpWriter.write(sBuffer.toString();fpReader.close();fpWriter.close();catch(UnknownHostException e) System.out.println(" 无法指定主机 ");System.exit(1);catch (IOException e) throw e;3. 网页索引根据筛选器筛选过的文件来创建链表格式的索引。package com.yjb.a;/ 网

13、页索引程序public class InfoItem publicint fileIDpublicint offsetInfoItem next ;public InfoItem() fileID= 0;offset = 0; next = nullpublic InfoItem( fileID = id;int id, int pos)offset = pos; n ext =n ull ;public voidset_ next(l nfoltem param)n ext = param;public int get_id()return fileID ;public int get_po

14、s()return offset ;publicIn foItem get_ next()returnnext ;4. 网页检索程序是在索引的基础上来检索的，它将返回你要检索的信息。 package com.yjb.a;import java.util.Hashtable;/网页检索程序public class Wordln dex static Hashtable keywordIdx ;static Stri ngFileList = "北京师范大学“，"北师大附属实验小学”，"北师大第二附属中学“;public static voidmain( Stri n

15、g args) try in dex ();search ("北“); catch (Exception e)System. out .println(”索引检索失败，请检查")；System. exit (1);public static void search(Stri ng keyword)throws Excepti onIn foItem item;System. out .println("search:-begin-");if ( null= keywordldx )return ; try item = (I nfoItem)keywor

16、dIdx.get(keyword);/获取关键字while (item !=n ull )System.out.pr in tl n("search:File n umber:"+item.get_id();System.out.pr in tl n("search:File offset:"+item.get_pos();System.out.pr in tl n("search:File Conten t:"+FileList item.get_pos();item = item.get_ next(); catch (Excep

17、tion e)throw e;System. out .println("search:-end-");throwsExcepti onpublic static voidin dex()In foItem item,item2;System. out .println("in dex:-begi n-");keywordIdx = new Hashtable();try System. out .println("index:Hash Table initial Size:"keywordIdx .size();for ( int

18、i = 0;i < 3;i +)int len = FileListi .len gth();for (int j = 0;j < len;j +)item = new In foltem(i,i);Stri ng key =FileList i.substri ng(j,j+1);System. out .print(key);if (! keywordIdx.containsKey(key)keywordIdx .put(key, item); else item2 = (In foItem)keywordIdx .get(key);item.set_ next(item2);keywordIdx .put(key, item);System. out .println( "" );System. out .println( "index:Hash Table f