编译原理之词法分析器--java语言版

1、java词法分析器 使用java开发，并且用来分析java源文件2003年1月12日1. 开发工具：rational rose2002 jedition，borland jbuilder6 professional2. 开发步骤：1) 基于状态转换图的编译器原理如下： 2)在rose中建立分析器模型框架,根据分析器的状态转换图算法以及算法构造。词法分析器的框架结构如下图所示： (分析器软件包) （词法分析器的控制器结构，包括预编译器，扫描程序，保留字表和单词的类型种别码表以及词法分析器的引导程序和控制程序） （词法分析器的扫描缓冲区和输入缓冲区结构以及获得缓冲区的缓冲工厂）3)使用rose正向

2、工程产生java框架代码，在jbuilder中进行编辑实现功能代码，生成最终的代码，进行test和debug，最后形成最终的目标程序。具体的实现请参考源代码。编辑和测试如下图所示： （开发环境） （运行结果，详细结果附在后面）3. 源代码：/lisence head/*java accidence analyser*author yellowicq*all copyright reserved*version 1.0*/lisence1) 词法分析器引导文件：main.javapackage jaccidenceanalyse;import javax.xml.parsers.*;import

3、 org.w3c.dom.*;public class main /* * param args * return void * roseuid 3d9bae4702ad */ public static void main(string args) /读取配置文件,得到系统属性 string cfgstring = new string4; try cfgstring = main.loadaacfg("d:aacfg.xml"); catch (exception e) e.printstacktrace(system.err); /设置待读文件名/ /保留字表文件 s

4、tring reservefilename = cfgstring0; /类型种别码表文件 string classfilename = cfgstring1; /需要分析的源文件 string sourcefilename = cfgstring2; /输出文件 string outputfilename = cfgstring3;/ /创建词法分析器 accidenceanalyser aa = new accidenceanalyser(); aa.setfilespath(reservefilename, classfilename, sourcefilename, outputfil

5、ename); /建立所需要的文件对象 /初始化词法分析器 aa.initaa(); /初始化关键字表 aa.keywordtable.initkeywordtable(); /初始化类型种别码表 aa.classidentity.initclassidentitytable(); /开始进行词法分析 aa.startaa(); /分析完毕 /读取配置文件 private static string loadaacfg(string name) throws exception string cfgstring = new string4; /*解析xml配置文件*/ try /*创建文档工厂

6、*/ documentbuilderfactory factory = documentbuilderfactory.newinstance(); /*创建文档解析器*/ documentbuilder builder = factory.newdocumentbuilder(); /*解析配置文件*/ document doc = builder.parse(name); /*规范化文档*/ doc.normalize(); /*查找接点表*/ nodelist nlists = doc.getelementsbytagname("filepath"); for (int

7、 i = 0; i < nlists.getlength(); i+) element item = (element) nlists.item(i); /取得需要的配置属性 /*/ cfgstring0 = item.getelementsbytagname("reservefilename").item(0). getfirstchild().getnodevalue().trim(); /*/ cfgstring1 = item.getelementsbytagname("classfilename").item(0).getfirstchi

8、ld().getnodevalue().trim(); /*/ cfgstring2 = item.getelementsbytagname("sourcefilename").item(0). getfirstchild().getnodevalue().trim(); /*/ cfgstring3 = item.getelementsbytagname("outputfilename").item(0). getfirstchild().getnodevalue().trim(); /*/ catch (exception e) e.printsta

9、cktrace(); throw new exception("error加载配置文件 " + name + " 错误!"); /返回属性数组 return cfgstring; 2） 词法分析器主程序：accidenceanalyser.java/source file: d:jaccidenceanalyseaccidenceanalyser.javapackage jaccidenceanalyse;import java.io.*;import java.util.*;import jaccidenceanalyse.buffer.*;publi

10、c class accidenceanalyser private java.io.file sourcefile; private java.io.file reservefile; private java.io.file classfile; private java.io.file outputfile; public pretreatment pretreatment; public keywordtable keywordtable; public classidentity classidentity; public scaner scaner; public concretes

11、canbufferfactory csbfactory; /* * roseuid 3d9bb93303d0 */ public accidenceanalyser() system.out.println("infor已经建立词法分析器!"); /* * roseuid 3d9baef9029f */ public void initaa() /创建缓冲工厂 this.csbfactory = new concretescanbufferfactory(); /创建字符串扫描对象 scaner = new scaner(this); /创建pre处理对象 pretreat

12、ment = new pretreatment(sourcefile, this); /创建关键字表对象 keywordtable = new keywordtable(reservefile); /创建对象种别码表对象 classidentity = new classidentity(classfile); system.out.println("infor已经初始化词法分析器!"); /* * roseuid 3d9baf12022d */ public void setfilespath(string reservefilename, string classfil

13、ename, string sourcefilename, string outputfilename) /创建文件对象 sourcefile = new java.io.file(sourcefilename); /创建文件对象 reservefile = new java.io.file(reservefilename); /创建文件对象 classfile = new java.io.file(classfilename); /创建文件对象 outputfile = new java.io.file(outputfilename); /如果文件已经存在,先删除,然后建立新文件 if (o

14、utputfile.exists() outputfile.delete(); try outputfile.createnewfile(); catch (exception e) e.printstacktrace(system.err); try /创建文件随机读取对象 java.io.randomaccessfile routputfile = new java.io.randomaccessfile(this. outputfile, "rw"); /提示信息 routputfile.write("/n". getbytes(); routpu

15、tfile.write( ("/jaccidenceanalyser version " + getversion() + " design by yellowicq/n").getbytes(); routputfile.write("/java词法分析器/n".getbytes(); routputfile.write("/使用java语言开发/n". getbytes(); routputfile.write("/n". getbytes(); routputfile.write(&quo

16、t;词法分析结果如下:n".getbytes(); /关闭文件流 routputfile.close(); catch (exception e) e.printstacktrace(system.err); /* * roseuid 3d9bafab0089 */ public void startaa() /从预处理开始词法分析 this.pretreatment.startpretreatment(); /* * roseuid 3d9bb0b40383 */ public void outputaccidence(string outputstring) /把分析出来的单词写

17、入文件 outputstring = "n第" + this.pretreatment.filerow + "行n" + outputstring; try /创建文件随机读取对象 java.io.randomaccessfile routputfile = new java.io.randomaccessfile(this. outputfile, "rw"); /移动指针到文件末尾 routputfile.seek(routputfile.length(); /start appending! routputfile.write(

18、outputstring.getbytes(); /关闭文件流 routputfile.close(); catch (exception e) e.printstacktrace(system.err); /将分析的单词结果输出到终端 system.out.print(outputstring); /* * roseuid 3d9bb0ce02c2 */ public void controlthread() /控制扫描器启动扫描 scaner.controlthread(); /获得版本号 public string getversion() return "1.0"

19、3） 预处理子程序：pretreatment.java /source file: d:jaccidenceanalysepretreatment.javapackage jaccidenceanalyse;import jaccidenceanalyse.buffer.*;import java.io.*;public class pretreatment private string tmpstring; private string outputstring; private int buffer_size = 100; private accidenceanalyser aa; pub

20、lic inputbuffer inputbuffer; /输入缓冲区-共享 private java.io.file sourcefile; /文件对象 private java.io.randomaccessfile randomafile; /随机文件对象 public static int filerow = 0; /* * roseuid 3dab7c530399 */ public pretreatment(file sourcefile, accidenceanalyser aa) try this.sourcefile = sourcefile; this.randomafil

21、e = new java.io.randomaccessfile(this.sourcefile, "r"); catch (filenotfoundexception e) e.printstacktrace(system.err); this.aa = aa; inputbuffer = aa.csbfactory.createinputbuffer(buffer_size); system.out.println("infor预处理器已经创建!"); /* * roseuid 3d9bafe20331 */ public void putsourc

22、etoinbuffer(string tmpstring) this.inputbuffer.data = tmpstring.tochararray(); /* * roseuid 3d9bb0400169 */ public void putfintoscbuffer(string filtratedstring) aa.scaner.scanbuffer.data = filtratedstring.tochararray(); /* * roseuid 3d9bb05e00a4 */ public void controlthread() int intlength; int resc

23、ounter = 0; string tmpstring; string filtratedstring; system.out.println("infor开始单词分析/"); try if (sourcefile.exists() /文件存在 /读文件内容到缓冲区 while ( (tmpstring = this.randomafile.readline() != null) +filerow; /分割符 system.out.println(".begin row " + this.filerow + "."); /开始这一行

24、分析 system.out.println("infor正在处理行: " + string.valueof(filerow); /放入输入缓冲区 this.putsourcetoinbuffer(tmpstring); /处理字符串 filtratedstring = this.filtratesource(this.inputbuffer.data); system.out.println("infor已过滤句子: " + filtratedstring); /放入扫描缓冲区 this.putfintoscbuffer(filtratedstring)

25、; aa.controlthread(); system.out.println( "infor分析完毕/"); else /文件不存在 system.err.println("error源文件不存在!"); catch (exception e) e.printstacktrace(system.err); /* * roseuid 3d9bb07d0239 */ public string filtratesource(char data) string filtratedstring = string.valueof(data).trim(); r

26、eturn filtratedstring; /* * roseuid 3d9bb9350315 */ public void startpretreatment() this.controlthread(); 4） 扫描子程序：scaner.java/source file: d:jaccidenceanalysescaner.javapackage jaccidenceanalyse;import jaccidenceanalyse.buffer.*;public class scaner public scanbuffer scanbuffer; /扫描缓冲区-共享 private st

27、ring finalaccidence; private accidenceanalyser aa; private int buffer_size = 100; private string todelstring; private int senlength = 0; private char sentencechar = new char1000; private string token; private char char; private int index = 0; private string identity = "identity" private st

28、ring digit = "digit" private string word_error_inf = "在此行发现不能识别的单词,此行分析终止!" private boolean astate = true; /* * roseuid 3d9bb9370213 */ public scaner(accidenceanalyser aa) this.aa = aa; initbuffer(); this.finalaccidence = "" system.out.println("infor扫描处理器已经创建!"

29、;); /* * roseuid 3d9bb2860329 */ public string readfrombuffer(char data) string todelstring = string.valueof(data); return todelstring; /* * param tmpstring * return string * roseuid 3d9bb2d5008d */ public string scan(string todelstring) sentencechar = todelstring.tochararray(); this.senlength = sen

30、tencechar.length; int i = 0; /分析单词 while (this.index <= this.senlength) /state0: this.token = "" this.char = getbc(sentencechar); if (this.char = '') break; /''表示这一行结束 /进入状态判断 switch (this.char) /judge lettercase 'a':case 'b':case 'c':case 'd&

31、#39;:case 'e':case 'f':case 'g':case 'h':case 'i':case 'j':case 'k':case 'l':case 'm':case 'n':case 'o':case 'p':case 'q':case 'r':case 's':case 't':case 'u':case &

32、#39;v':case 'w':case 'x':case 'y':case 'z':case 'a':case 'b':case 'c':case 'd':case 'e':case 'f':case 'g':case 'h':case 'i':case 'j':case 'k':case 'l':case 'm':

33、case 'n':case 'o':case 'p':case 'q':case 'r':case 's':case 't':case 'u':case 'v':case 'w':case 'x':case 'y':case 'z': /do this.token = this.contact(token, char); /state1 char = this.getchar(senten

34、cechar); while (this.isletter(char) | this.isdigit(char) this.token = this.contact(this.token, char); char = this.getchar(sentencechar); this.retract(); /state2 if (aa.keywordtable.iskeyword(token) this.finalaccidence = this.finalaccidence + "保留字 " + this.returnaword(token) + "n"

35、 else this.finalaccidence = this.finalaccidence + "标识符 " + this.returnaword(token) + "种别码 " + string.valueof(aa.classidentity.findkey(identity) + "n" /clear up token this.token = "" break; /judge ditital case '0':case '1':case '2':case

36、'3':case '4':case '5':case '6':case '7':case '8': case '9': /do this.token = this.contact(token, char); /state3 char = this.getchar(sentencechar); while (this.isdigit(char) this.token = this.contact(token, char); char = this.getchar(sentencecha

37、r); this.retract(); /state4 this.finalaccidence = this.finalaccidence + "数字 " + this.returnaword(token) + "种别码 " + string.valueof(aa.classidentity.findkey(digit) + "n" /clear up token this.token = "" break; case '=': /state5 this.token = this.contact(t

38、oken, char); this.finalaccidence = this.finalaccidence + "等号 " + this.returnaword(token) + "种别码 " + string.valueof(aa.classidentity.findkey(string.valueof(char) + "n" /clear up token this.token = "" break; case '+': /state6 this.token = this.contact(to

39、ken, char);this.finalaccidence = this.finalaccidence + "加号 " + this.returnaword(token) + "种别码 " + string.valueof(aa.classidentity.findkey(string.valueof(char) + "n" /clear up token this.token = "" break; case '*': /state7 this.token = this.contact(toke

40、n, char); char = this.getchar(sentencechar); /state8 if (char = '*') this.token = this.contact(token, char); this.finalaccidence = this.finalaccidence + "乘方 " + this.returnaword(token) + "种别码 " + string.valueof(aa.classidentity.findkey(string.valueof(char) + "n"

41、 /state9 else this.finalaccidence = this.finalaccidence + "乘号 " + this.returnaword(token) + "种别码 " + string.valueof(aa.classidentity.findkey(string.valueof(char) + "n" /clear up token this.token = "" break; case ',': /state10 this.token = this.contact(

42、token, char); this.finalaccidence = this.finalaccidence + "逗号 " + this.returnaword(token) + "种别码 " + string.valueof(aa.classidentity.findkey(string.valueof(char) + "n" /clear up token this.token = "" break; case '(': /state11 this.token = this.contact(

43、token, char); this.finalaccidence = this.finalaccidence + "左括号 " + this.returnaword(token) + "种别码 " + string.valueof(aa.classidentity.findkey(string.valueof(char) + "n" /clear up token this.token = "" break; case ')': /state12 this.token = this.contact

44、(token, char); this.finalaccidence = this.finalaccidence + "右括号 " +this.returnaword(token) + "种别码 " + string.valueof(aa.classidentity.findkey(string.valueof(char) + "n" /clear up token this.token = "" break; case '': /state13 this.token = this.contact(

45、token, char); this.finalaccidence = this.finalaccidence + "左大括号 " + this.returnaword(token) + "种别码 " + string.valueof(aa.classidentity.findkey(string.valueof(char) + "n" /clear up token this.token = "" break; case '': /state14 this.token = this.contact

46、(token, char); this.finalaccidence = this.finalaccidence + "右大括号 " + this.returnaword(token) + "种别码 " + string.valueof(aa.classidentity.findkey(string.valueof(char) + "n" /clear up token this.token = "" break; case '': /state15 this.token = this.contac

47、t(token, char); this.finalaccidence = this.finalaccidence + "左中括号 " + this.returnaword(token) + "种别码 " + string.valueof(aa.classidentity.findkey(string.valueof(char) + "n" /clear up token this.token = "" break; case '': /state16 this.token = this.contact(token, char); this.finalaccidence = this.finalaccidence + "右中括号 " + this.returnaword(token) + "种别码 " + string.valueof(aa.classidentity.findkey(string.valueof(char) + "n" /clear up t