Mapreduce_Distinct数据去重

MapReduce中数据去重

输入如下的数据，统计其中的地址信息，并对输出的地址信息进行去重
实现方法：Map阶段输出的信息K2为想要去重的内容，利用Reduce阶段的聚合特点，对K2进行聚合，去重。在两阶段中，V2，V3，V4为Null

distinct.csv

John,30,New York
Jane,25,Los Angeles
Tom,33,New York
Doe,45,Chicago
Sam,26,Los Angeles

main

package com.hadoop;import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.streaming.io.InputWriter;public class Main {public static void main(String[] args) throws Exception {Job job =Job.getInstance(new Configuration());job.setJarByClass(Main.class);job.setMapperClass(Map.class);job.setMapOutputKeyClass(Text.class);//k2job.setMapOutputValueClass(NullWritable.class);//v2job.setReducerClass(Reduce.class);job.setOutputKeyClass(Text.class);//k4job.setOutputValueClass(NullWritable.class);//v4FileInputFormat.setInputPaths(job,new Path(args[0]));FileOutputFormat.setOutputPath(job,new Path(args[1]));job.waitForCompletion(true);}
}

package com.hadoop;import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;import java.io.IOException;//Los Angeles
public class Map extends Mapper<LongWritable, Text,Text, NullWritable> {//Jane,25,Los Angeles@Overrideprotected void map(LongWritable k1, Text v1,Context context)throws IOException, InterruptedException {String data =v1.toString();String words[]=data.split(",");context.write(new Text(words[2]),NullWritable.get());}
}

Reduce

package com.hadoop;import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;import java.io.IOException;public class Reduce extends Reducer<Text, NullWritable,Text,NullWritable> {@Overrideprotected void reduce(Text k3, Iterable<NullWritable> v3,Context context)throws IOException, InterruptedException {context.write(k3,NullWritable.get());}
}

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"><modelVersion>4.0.0</modelVersion><groupId>com.hadoop</groupId><artifactId>Mapreduce_Distinct</artifactId><version>1.0-SNAPSHOT</version><name>Mapreduce_Distinct</name><description>wunaiieq</description><properties><maven.compiler.source>8</maven.compiler.source><maven.compiler.target>8</maven.compiler.target><project.build.sourceEncoding>UTF-8</project.build.sourceEncoding><!--版本控制--><hadoop.version>2.7.3</hadoop.version></properties><dependencies><dependency><groupId>org.apache.hadoop</groupId><artifactId>hadoop-common</artifactId><version>${hadoop.version}</version></dependency><dependency><groupId>org.apache.hadoop</groupId><artifactId>hadoop-hdfs</artifactId><version>${hadoop.version}</version></dependency><dependency><groupId>org.apache.hadoop</groupId><artifactId>hadoop-mapreduce-client-core</artifactId><version>${hadoop.version}</version></dependency><dependency><groupId>org.apache.hadoop</groupId><artifactId>hadoop-client</artifactId><version>${hadoop.version}</version></dependency><dependency><groupId>org.apache.hadoop</groupId><artifactId>hadoop-yarn-api</artifactId><version>${hadoop.version}</version></dependency><dependency><groupId>org.apache.hadoop</groupId><artifactId>hadoop-streaming</artifactId><version>${hadoop.version}</version></dependency></dependencies><!--构建配置--><build><plugins><plugin><!--声明--><groupId>org.apache.maven.plugins</groupId><artifactId>maven-assembly-plugin</artifactId><version>3.3.0</version><!--具体配置--><configuration><archive><manifest><!--jar包的执行入口--><mainClass>com.hadoop.Main</mainClass></manifest></archive><descriptorRefs><!--描述符，此处为预定义的，表示创建一个包含项目所有依赖的可执行 JAR 文件;允许自定义生成jar文件内容--><descriptorRef>jar-with-dependencies</descriptorRef></descriptorRefs></configuration><!--执行配置--><executions><execution><!--执行配置ID，可修改--><id>make-assembly</id><!--执行的生命周期--><phase>package</phase><goals><!--执行的目标，single表示创建一个分发包--><goal>single</goal></goals></execution></executions></plugin></plugins></build>
</project>