任务7实训：MapReduce编程

任务七实训：MapReduce编程一、实验目的1.1掌握MapReduce的编程方法二、实验内容2.1在idea中新建一个Maven工程2.2在pom文件中添加依赖添加依赖<dependency><groupId>org.apache.hadoop</groupId><artifactId>hadoop-client</artifactId><version>2.9.2</version></dependency>2.3新建一个class命名为WordCount，importorg.apache.hadoop.conf.Configuration;

importorg.apache.hadoop.fs.Path;

importorg.apache.hadoop.io.IntWritable;

importorg.apache.hadoop.io.Text;

importorg.apache.hadoop.mapreduce.Job;

importorg.apache.hadoop.mapreduce.Mapper;

importorg.apache.hadoop.mapreduce.Reducer;

importorg.apache.hadoop.mapreduce.lib.input.FileInputFormat;

importorg.apache.hadoop.mapreduce.lib.input.TextInputFormat;

importorg.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

importorg.apache.hadoop.mapreduce.lib.output.TextOutputFormat;

importjava.io.IOException;

importjava.util.StringTokenizer;publicclassWordCount{

publicstaticclassMyMapperextendsMapper<Object,Text,Text,IntWritable>{

privateTextword=newText();

privatefinalstaticIntWritableone=newIntWritable(1);

@Override

protectedvoidmap(Objectkey,Textvalue,Contextcontext)throwsIOException,InterruptedException{

StringTokenizeritr=newStringTokenizer(value.toString());

while(itr.hasMoreTokens()){

word.set(itr.nextToken());

context.write(word,one);

}

}

}

publicstaticclassMyReducerextendsReducer<Text,IntWritable,Text,IntWritable>{

privateIntWritableresult=newIntWritable();

@Override

protectedvoidreduce(Textkey,Iterable<IntWritable>values,Contextcontext)throwsIOException,InterruptedException{

intsum=0;

for(IntWritableval:values){

sum+=val.get();

}

result.set(sum);

context.write(key,result);

}

}

publicstaticvoidmain(String[]args)throwsIOException,InterruptedException,ClassNotFoundException{

Configurationconf=newConfiguration();

Jobjob=Job.getInstance(conf);

job.setJarByClass(WordCount.class);

job.setJobName("WordCount");

job.setInputFormatClass(TextInputFormat.class);

job.setOutputFormatClass(TextOutputFormat.class);

FileInputFormat.addInputPath(job,newPath(args[0]));

FileOutputFormat.setOutputPath(job,newPath(args[1]));

job.setMapperClass(MyMapper.class);

job.setReducerClass(MyReducer.class);

job.setMapOutputKeyClass(Text.class);

job.setMapOutputValueClass(IntWritable.class);

job.setOutputKeyClass(Text.class);

job.setOutputValueClass(IntWritable.class);

job.waitForCompletion(true);

}

}2.4.配置hadoop环境变量Windows本地解压Hadoop压缩包，然后像配置JDK环境变量一样在系统环境变量里配置HADOOP_HOME和path：%HADOOP_HOME\bin%环境变量。注意：hadoop安装目录尽量不要包含空格或者中文字符。2.5.下载hadoop对应版本补丁MapReduce程序在Linux、mac系统中可以直接运行，但无法直接在Windows下运行，因为MapReduce代码在Windows下运行存在bug，需要添加hadoop补丁才可以。Hadoop补丁下载地址：/cdarlint/winutilsHadoop补丁文件为hadoop.dll和winutils.exe。2.6补丁添加hadoop安装目录hadoop补丁文件下载之后，解压，然后将相应版本中的hadoop.dll和winutils.exe补丁文件，拷贝到{HADOOP_HOME}/bin目录下（可能需要重启IDEA）2.6补丁添加windowsc盘目录如果1.3步骤操作完之后，还无法运行MapReduce程序，则需要将hadoop.dll和winutils.exe补丁文件，拷贝