Hadoop 高级程序设计（二）---自定义输入输出格式，hadoop输入输出
Hadoop 高级程序设计（二）---自定义输入输出格式，hadoop输入输出
Hadoop提供了较为丰富的数据输入输出格式，可以满足很多的设计实现，但是在某些时候需要自定义输入输出格式。
数据的输入格式用于描述MapReduce作业的数据输入规范，MapReduce框架依靠数据输入格式完后输入规范检查（比如输入文件目录的检查），对数据文件进行输入分块（InputSpilt）以及提供从输入分快中将数据逐行的读出，并转换为Map过程的输入键值对等功能。Hadoop提供了很多的输入格式，TextInputFormat和KeyValueInputFormat,对于每个输入格式都有与之对应的RecordReader,LineRecordReader和KeyValueLineRecordReader。用户需要自定义输入格式，主要实现InputFormat中的createRecordReader()和getSplit()方法，而在RecordReader中实现getCurrentKey().....
例如：
package com.rpc.nefu;
import java.io.IOException;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.util.LineReader;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
//自定义的输入格式需要 继承FileInputFormat接口
public class ZInputFormat extends FileInputFormat<IntWritable,IntWritable>{
@Override //实现RecordReader
public RecordReader<IntWritable, IntWritable> createRecordReader(
InputSplit split, TaskAttemptContext context)
throws IOException, InterruptedException {
return new ZRecordReader();
}
//自定义的数据类型
public static class ZRecordReader extends RecordReader<IntWritable,IntWritable>
{
//data
private LineReader in; //输入流
private boolean more = true;//提示后续还有没有数据
private IntWritable key = null;
private IntWritable value = null;
//这三个保存当前读取到位置（即文件中的位置）
private long start;
private long end;
private long pos;
//private Log LOG = LogFactory.getLog(ZRecordReader.class);//日志写入系统，可加可不加
@Override
public void initialize(InputSplit split, TaskAttemptContext context)
throws IOException, InterruptedException {
// 初始化函数
FileSplit inputsplit = (FileSplit)split;
start = inputsplit.getStart(); //得到此分片开始位置
end = start + inputsplit.getLength();//结束此分片位置
final Path file = inputsplit.getPath();
// 打开文件
FileSystem fs = file.getFileSystem(context.getConfiguration());
FSDataInputStream fileIn = fs.open(inputsplit.getPath());
//将文件指针移动到当前分片，因为每次默认打开文件时，其指针指向开头
fileIn.seek(start);
in = new LineReader(fileIn, context.getConfiguration());
if (start != 0)
{
System.out.println("4");
//如果这不是第一个分片，那么假设第一个分片是0——4，那么，第4个位置已经被读取，则需要跳过4，否则会产生读入错误，因为你回头又去读之前读过的地方
start += in.readLine(new Text(), 0, maxBytesToConsume(start));
}
pos = start;
}
private int maxBytesToConsume(long pos)
{
return (int) Math.min(Integer.MAX_VALUE, end - pos);
}
@Override
public boolean nextKeyValue() throws IOException,
InterruptedException {
//下一组值
//tips:以后在这种函数中最好不要有输出，费时
//LOG.info("正在读取下一个，嘿嘿");
if(null == key)
{
key = new IntWritable();
}
if(null == value)
{
value = new IntWritable();
}
Text nowline = new Text();//保存当前行的内容
int readsize = in.readLine(nowline);
//更新当前读取到位置
pos += readsize;
//如果pos的值大于等于end，说明此分片已经读取完毕
if(pos >= end)
{
more = false;
return false;
}
if(0 == readsize)
{
key = null;
value = null;
more = false;//说明此时已经读取到文件末尾，则more为false
return false;
}
String[] keyandvalue = nowline.toString().split(",");
//排除第一行
if(keyandvalue[0].endsWith("\"CITING\""))
{
readsize = in.readLine(nowline);
//更新当前读取到位置
pos += readsize;
if(0 == readsize)
{
more = false;//说明此时已经读取到文件末尾，则more为false
return false;
}
//重新划分
keyandvalue = nowline.toString().split(",");
}
//得到key和value
//LOG.info("key is :" + key +"value is" + value);
key.set(Integer.parseInt(keyandvalue[0]));
value.set(Integer.parseInt(keyandvalue[1]));
return true;
}
@Override
public IntWritable getCurrentKey() throws IOException,
InterruptedException {
//得到当前的Key
return key;
}
@Override
public IntWritable getCurrentValue() throws IOException,
InterruptedException {
//得到当前的value
return value;
}
@Override
public float getProgress() throws IOException, InterruptedException {
//计算对于当前片的处理进度
if( false == more || end == start)
{
return 0f;
}
else
{
return Math.min(1.0f, (pos - start)/(end - start));
}
}
@Override
public void close() throws IOException {
//关闭此输入流
if(null != in)
{
in.close();
}
}
}
}
package reverseIndex;
import java.io.IOException;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
import org.apache.hadoop.mapreduce.lib.input.LineRecordReader;
public class FileNameLocInputFormat extends FileInputFormat<Text, Text>{
@Override
public org.apache.hadoop.mapreduce.RecordReader<Text, Text> createRecordReader(
org.apache.hadoop.mapreduce.InputSplit split, TaskAttemptContext context)
throws IOException, InterruptedException {
// TODO Auto-generated method stub
return new FileNameLocRecordReader();
}
public static class FileNameLocRecordReader extends RecordReader<Text,Text>{
String FileName;
LineRecordReader line = new LineRecordReader();
/**
* ......
*/
@Override
public Text getCurrentKey() throws IOException, InterruptedException {
// TODO Auto-generated method stub
return new Text("("+FileName+"@"+line.getCurrentKey()+")");
}
@Override
public Text getCurrentValue() throws IOException, InterruptedException {
// TODO Auto-generated method stub
return line.getCurrentValue();
}
@Override
public void initialize(InputSplit split, TaskAttemptContext arg1)
throws IOException, InterruptedException {
// TODO Auto-generated method stub
line.initialize(split, arg1);
FileSplit inputsplit = (FileSplit)split;
FileName = (inputsplit).getPath().getName();
}
@Override
public void close() throws IOException {
// TODO Auto-generated method stub
}
@Override
public float getProgress() throws IOException, InterruptedException {
// TODO Auto-generated method stub
return 0;
}
@Override
public boolean nextKeyValue() throws IOException, InterruptedException {
// TODO Auto-generated method stub
return false;
}
}
}
Hadoop中也内置了很多的输出格式与RecordWriter.输出格式完成输出规范检查，作业结果数据输出。
自定义的输出格式：
public static class AlphaOutputFormat extends multiformat<Text, IntWritable>{
@Override
protected String generateFileNameForKeyValue(Text key,
IntWritable value, Configuration conf) {
// TODO Auto-generated method stub
char c = key.toString().toLowerCase().charAt(0);
if( c>='a' && c<='z'){
return c+".txt";
}else{
return "other.txt";
}
}
}//设置输出格式 job.setOutputFormatClass(AlphaOutputFormat.class);
package com.rpc.nefu;
import java.io.DataOutputStream;
import java.io.IOException;
import java.util.HashMap;
import java.util.Iterator;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.GzipCodec;
import org.apache.hadoop.mapreduce.OutputCommitter;
import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.ReflectionUtils;
public abstract class multiformat<K extends WritableComparable<?>, V extends Writable>
extends FileOutputFormat<K, V> {
private MultiRecordWriter writer = null;
public RecordWriter<K, V> getRecordWriter(TaskAttemptContext job) throws IOException,
InterruptedException {
if (writer == null) {
writer = new MultiRecordWriter(job, getTaskOutputPath(job));
}
return writer;
}
private Path getTaskOutputPath(TaskAttemptContext conf) throws IOException {
Path workPath = null;
OutputCommitter committer = super.getOutputCommitter(conf);
if (committer instanceof FileOutputCommitter) {
workPath = ((FileOutputCommitter) committer).getWorkPath();
} else {
Path outputPath = super.getOutputPath(conf);
if (outputPath == null) {
throw new IOException("Undefined job output-path");
}
workPath = outputPath;
}
return workPath;
}
/**通过key, value, conf来确定输出文件名（含扩展名）*/
protected abstract String generateFileNameForKeyValue(K key, V value, Configuration conf);
public class MultiRecordWriter extends RecordWriter<K, V> {
/**RecordWriter的缓存*/
private HashMap<String, RecordWriter<K, V>> recordWriters = null;
private TaskAttemptContext job = null;
/**输出目录*/
private Path workPath = null;
public MultiRecordWriter(TaskAttemptContext job, Path workPath) {
super();
this.job = job;
this.workPath = workPath;
recordWriters = new HashMap<String, RecordWriter<K, V>>();
}
@Override
public void close(TaskAttemptContext context) throws IOException, InterruptedException {
Iterator<RecordWriter<K, V>> values = this.recordWriters.values().iterator();
while (values.hasNext()) {
values.next().close(context);
}
this.recordWriters.clear();
}
@Override
public void write(K key, V value) throws IOException, InterruptedException {
//得到输出文件名
String baseName = generateFileNameForKeyValue(key, value, job.getConfiguration());
RecordWriter<K, V> rw = this.recordWriters.get(baseName);
if (rw == null) {
rw = getBaseRecordWriter(job, baseName);
this.recordWriters.put(baseName, rw);
}
rw.write(key, value);
}
// ${mapred.out.dir}/_temporary/_${taskid}/${nameWithExtension}
private RecordWriter<K, V> getBaseRecordWriter(TaskAttemptContext job, String baseName)
throws IOException, InterruptedException {
Configuration conf = job.getConfiguration();
boolean isCompressed = getCompressOutput(job);
String keyValueSeparator = ",";
RecordWriter<K, V> recordWriter = null;
if (isCompressed) {
Class<? extends CompressionCodec> codecClass = getOutputCompressorClass(job,
GzipCodec.class);
CompressionCodec codec = ReflectionUtils.newInstance(codecClass, conf);
Path file = new Path(workPath, baseName + codec.getDefaultExtension());
FSDataOutputStream fileOut = file.getFileSystem(conf).create(file, false);
recordWriter = new lineRecordWrite<K, V>(new DataOutputStream(codec
.createOutputStream(fileOut)), keyValueSeparator);
} else {
Path file = new Path(workPath, baseName);
FSDataOutputStream fileOut = file.getFileSystem(conf).create(file, false);
recordWriter = new lineRecordWrite<K, V>(fileOut, keyValueSeparator);
}
return recordWriter;
}
}
}
得看你源代码了
我不是高手，但我可以告诉你我怎么学习。①选择一个Hadoop的版本，然后阅读文档了解Hadoop：What's Hadoop, Why Hadoop exists；②安装Hadoop，三种方式都试下；③在Hadoop文档里面有Hadoop Command的资料，I.hdfs command，II.job command，尽量试试这两方面的命令；④Hadoop Files，看看Hadoop文件的概念，关注它的分布式特点，然后看看Reduce函数输出的文件；⑤自己写WordCount与Advanced WordCount；⑥写HDFS io，这个例子在《Hadoop In Action》里面有，讲得也不错。如copy，sequenceFile等；⑦写Sort程序；⑧写MRBench程序（这个网上有很多例子），了解MRBench是什么；⑨使用RandomTextWriter；10.模仿SequenceFileInputFormat、SequenceFileOutputFormat、SequenceFileRecordReader写自己的；11.yahoo有一个Hadoop的教程，英文版的，里面的内容很好；12.《hadoop权威指南》当参考书，自己实战了
评论暂时关闭