| Version 1 (modified by waue, 17 years ago) (diff) |
|---|
錯誤sample code 如下
/*
* NCHC Hbase with map reduce sample code
* DemoHBaseSlink.java
*/
package tw.org.nchc.demo;
import java.io.IOException;
import java.util.Iterator;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapred.TableReduce;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.MapWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.lib.IdentityMapper;
import org.apache.hadoop.mapred.lib.IdentityReducer;
/**
* This sample code will put the indicate data to Hbase.
* 1. put test.txt in t1 directory which content is
---------------
name locate years
waue taiwan 1981
shellon taiwan 1981
---------------
* 2. hadoop_root/$ bin/hadoop dfs -put t1 t1
* 3. hbase_root/$ bin/hbase shell
* 4. hql > create table t1_table("person");
* 5. run this code, and we will let database as that
t1_table -> person
----------------
| name | locate | years |
----------------
| waue | taiwan | 1981 |
----------------
| shellon | taiwan | 1981 |
**/
public class DemoHBaseSink {
private static class ReduceClass extends TableReduce<LongWritable, Text> {
// this is the column we're going to be writing
private static final Text col1 = new Text("person:name");
private static final Text col2 = new Text("person:locate");
private static final Text col3 = new Text("person:years");
// this map holds the columns per row
private MapWritable map = new MapWritable();
public void reduce(LongWritable key, Iterator<Text> values,
OutputCollector<Text, MapWritable> output, Reporter reporter)
throws IOException {
// contents must be ImmutableBytesWritable
String[] str = (values.next().getBytes().toString()).split(" ");
byte bl[] = str[0].getBytes();
byte bn[] = str[1].getBytes();
byte by[] = str[2].getBytes();
ImmutableBytesWritable bytes1 = new ImmutableBytesWritable( bl);
ImmutableBytesWritable bytes2 = new ImmutableBytesWritable( bn );
ImmutableBytesWritable bytes3 = new ImmutableBytesWritable( by );
// ImmutableBytesWritable bytes3 = new ImmutableBytesWritable(values.next().getBytes());
// populate the current row
map.clear();
map.put(col1, bytes1);
map.put(col2, bytes2);
map.put(col3, bytes3);
// add the row with the key as the row id
output.collect(new Text(key.toString()), map);
}
}
private DemoHBaseSink() {
}
/**
* Runs the demo.
*/
public static void main(String[] args) throws IOException {
// which path of input files in Hadoop file system
String file_path = "/user/waue/t1";
int mapTasks = 1;
int reduceTasks = 1;
JobConf conf = new JobConf(DemoHBaseSink.class);
conf.setJobName("DemoPersonBase");
// must initialize the TableReduce before running job
TableReduce.initJob("t1_table", ReduceClass.class, conf);
conf.setNumMapTasks(mapTasks);
conf.setNumReduceTasks(reduceTasks);
conf.setInputPath(new Path(file_path));
conf.setMapperClass(IdentityMapper.class);
conf.setCombinerClass(IdentityReducer.class);
conf.setReducerClass(ReduceClass.class);
JobClient.runJob(conf);
}
}
錯誤訊息
08/06/04 18:03:21 INFO mapred.FileInputFormat: Total input paths to process : 1 08/06/04 18:03:22 INFO mapred.JobClient: Running job: job_200805291341_0019 08/06/04 18:03:23 INFO mapred.JobClient: map 0% reduce 0% 08/06/04 18:03:28 INFO mapred.JobClient: map 100% reduce 0% 08/06/04 18:03:36 INFO mapred.JobClient: Task Id : task_200805291341_0019_r_000000_0, Status : FAILED java.lang.ArrayIndexOutOfBoundsException: 1 at tw.org.nchc.demo.DemoHBaseSink$ReduceClass.reduce(DemoHBaseSink.java:63) at tw.org.nchc.demo.DemoHBaseSink$ReduceClass.reduce(DemoHBaseSink.java:1) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:333) at org.apache.hadoop.mapred.TaskTracker$Child.main(TaskTracker.java:2084) 08/06/04 18:03:40 INFO mapred.JobClient: Task Id : task_200805291341_0019_r_000000_1, Status : FAILED java.lang.ArrayIndexOutOfBoundsException: 1 at tw.org.nchc.demo.DemoHBaseSink$ReduceClass.reduce(DemoHBaseSink.java:63) at tw.org.nchc.demo.DemoHBaseSink$ReduceClass.reduce(DemoHBaseSink.java:1) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:333) at org.apache.hadoop.mapred.TaskTracker$Child.main(TaskTracker.java:2084) 08/06/04 18:03:45 INFO mapred.JobClient: Task Id : task_200805291341_0019_r_000000_2, Status : FAILED java.lang.ArrayIndexOutOfBoundsException: 1 at tw.org.nchc.demo.DemoHBaseSink$ReduceClass.reduce(DemoHBaseSink.java:63) at tw.org.nchc.demo.DemoHBaseSink$ReduceClass.reduce(DemoHBaseSink.java:1) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:333) at org.apache.hadoop.mapred.TaskTracker$Child.main(TaskTracker.java:2084) 08/06/04 18:03:49 INFO mapred.JobClient: map 100% reduce 100% Exception in thread "main" java.io.IOException: Job failed! at org.apache.hadoop.mapred.JobClient.runJob(JobClient.java:892) at tw.org.nchc.demo.DemoHBaseSink.main(DemoHBaseSink.java:108) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.util.RunJar.main(RunJar.java:155)
