hdfs mapreduce导入到habase 中发生RetriesExhaustedWithDetailsException

我的mapreduce读三类文件,进行解析,在map任务中导入到hbase
代码如下:
package com.su;


import org.apache.hadoop.conf.Configuration;


import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableOutputFormat;

import java.io.*;

import java.net.URI;
import java.util.HashMap;
import java.util.Map;


import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;

import org.apache.hadoop.mapreduce.filecache.DistributedCache;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.util.GenericOptionsParser;


import java.io.IOException;

/**
* Created by suhuadong on 2015/8/30.
*/
public class HdfsToHbase {

public static class HdfsToHbaseMapper
extends Mapper<Object, Text, ImmutableBytesWritable, Put>{
private Map<String,String> idMovieMap;
private Map<String,String> idUserMap;
/**
* 获取movieMap
* @return
*/
public void getMovieMap(){

FileReader reader = null;
BufferedReader br = null;
Map<String,String> idMovie = new HashMap<String, String>();
try {
reader = new FileReader("movie");
br = new BufferedReader(reader);
String s1 = null;
while ((s1 = br.readLine()) != null)
{

String cols = s1.split("::");
idMovie.put(cols[0],s1);

}
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e){
e.printStackTrace();
}
finally {
try{
if(br != null){
br.close();
}
if(reader != null){
reader.close();
}

}catch (IOException e){
e.printStackTrace();
}


}
this.idMovieMap = idMovie;
}

/**
* 获取userMap
* @return
*/
public void getUserMap(){
FileReader reader = null;
BufferedReader br = null;
Map<String,String> idUserMap = new HashMap<String, String>();

try {

reader = new FileReader("user");
br = new BufferedReader(reader);
String s1 = null;
while ((s1 = br.readLine()) != null)
{

String cols = s1.split("::");
idUserMap.put(cols[0], s1);
}
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e){
e.printStackTrace();
}
finally {
try{
if(br != null){
br.close();
}
if(reader != null){
reader.close();
}
}catch (IOException e){
e.printStackTrace();
}


}
this.idUserMap = idUserMap;
}
protected void setup(Context context) throws IOException, InterruptedException {
getUserMap();
getMovieMap();

}
public void map(Object key, Text value, Context context) throws IOException, InterruptedException {
System.out.println("ratting line:" + value.toString());
String ratingArr = value.toString().split("::");
if(ratingArr.length != 4){
return;
}
String userId = ratingArr[0];
String movieId = ratingArr[1];
String score = ratingArr[2];
if(this.idMovieMap.get(movieId) == null){
return;
}
String movieArr = this.idMovieMap.get(movieId).split("::");
String movieName = movieArr[1];
String movieClassify = movieArr[2];

if(this.idUserMap.get(userId) == null){
return;
}
String userArr = this.idUserMap.get(userId).split("::");
String sex = userArr[1];
String age = userArr[2];
String profession = userArr[3];
String zipCode = userArr[4];
long ts = 2872005692000l - System.currentTimeMillis();
Put put = new Put(userId.getBytes());

put.add("movie".getBytes(), "moveName".getBytes(), ts, movieName.getBytes());
put.add("movie".getBytes(), "movieClassify".getBytes(), ts, movieClassify.getBytes());
put.add("movie".getBytes(), "score".getBytes(), ts, score.getBytes());

put.add("user".getBytes(), "sex".getBytes(), ts, sex.getBytes());
put.add("user".getBytes(), "age".getBytes(), ts, age.getBytes());
put.add("user".getBytes(), "profession".getBytes(), ts,profession.getBytes());
put.add("user".getBytes(), "zipCode".getBytes(), ts,zipCode.getBytes());

context.write(new ImmutableBytesWritable(userId.getBytes()),put);
}
}

public static void main(String args) throws Exception {
Configuration conf = new Configuration();
String otherArgs = new GenericOptionsParser(conf,args).getRemainingArgs();
Path usersPath = new Path(otherArgs[0]);
Path moviePath = new Path(otherArgs[1]);
String userWithLine = usersPath.toUri().toString() + "#" + "user";
String movieWithLine = moviePath.toUri().toString() + "#" + "movie";

DistributedCache.createSymlink(conf);
DistributedCache.addCacheFile(new URI(userWithLine),conf);
DistributedCache.addCacheFile(new URI(movieWithLine),conf);
conf.addResource("hbase-site.xml");
Job job = Job.getInstance(conf, "word count");
job.setJarByClass(HdfsToHbase.class);



//job.addCacheFile(usersPath.toUri());
//job.addCacheFile(moviePath.toUri());

job.setJarByClass(HdfsToHbase.class);
job.setMapperClass(HdfsToHbaseMapper.class);
job.setMapOutputKeyClass(ImmutableBytesWritable.class);
job.setMapOutputValueClass(Put.class);
job.setNumReduceTasks(0);
job.setOutputFormatClass(TableOutputFormat.class);
job.getConfiguration().set(TableOutputFormat.OUTPUT_TABLE, "user_movie");

FileInputFormat.addInputPath(job, new Path(otherArgs[2]));
System.exit(job.waitForCompletion(true) ? 0 : 1);
}
}



运行方式如下: 
sudo -u hdfs hadoop jar movielens-hbase-1.0-SNAPSHOT.jar /user/suhuadong/moivelens/user/users.dat /user/suhuadong/moivelens/movie/movies.dat /user/suhuadong/moivelens/ratting/
 
发生的异常如下:
[root@SU-3 target]# sudo -u hdfs hadoop jar movielens-hbase-1.0-SNAPSHOT.jar /user/suhuadong/moivelens/user/users.dat /user/suhuadong/moivelens/movie/movies.dat /user/suhuadong/moivelens/ratting
15/09/01 19:43:04 INFO client.RMProxy: Connecting to ResourceManager at SU-1/10.116.107.179:8032
15/09/01 19:43:09 INFO input.FileInputFormat: Total input paths to process : 2
15/09/01 19:43:09 INFO lzo.GPLNativeCodeLoader: Loaded native gpl library
15/09/01 19:43:09 INFO lzo.LzoCodec: Successfully loaded & initialized native-lzo library [hadoop-lzo rev 39cf0c71a251a79c50555810ca660450d9682140]
15/09/01 19:43:09 INFO mapreduce.JobSubmitter: number of splits:2
15/09/01 19:43:10 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_1441014928183_0041
15/09/01 19:43:10 INFO impl.YarnClientImpl: Submitted application application_1441014928183_0041
15/09/01 19:43:10 INFO mapreduce.Job: The url to track the job: http://SU-1:8088/proxy/applica ... 0041/
15/09/01 19:43:10 INFO mapreduce.Job: Running job: job_1441014928183_0041
15/09/01 19:43:24 INFO mapreduce.Job: Job job_1441014928183_0041 running in uber mode : false
15/09/01 19:43:24 INFO mapreduce.Job:  map 0% reduce 0%
15/09/01 19:43:36 INFO mapreduce.Job: Task Id : attempt_1441014928183_0041_m_000001_0, Status : FAILED
Error: java.io.IOException: java.lang.reflect.InvocationTargetException
        at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:240)
        at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:218)
        at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:119)
        at org.apache.hadoop.hbase.mapreduce.TableOutputFormat$TableRecordWriter.<init>(TableOutputFormat.java:97)
        at org.apache.hadoop.hbase.mapreduce.TableOutputFormat.getRecordWriter(TableOutputFormat.java:144)
        at org.apache.hadoop.mapred.MapTask$NewDirectOutputCollector.<init>(MapTask.java:647)
        at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:767)
        at org.apache.hadoop.mapred.MapTask.run(MapTask.java:341)
        at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:163)
        at java.security.AccessController.doPrivileged(Native Method)
        at javax.security.auth.Subject.doAs(Subject.java:415)
        at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1671)
        at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:158)
Caused by: java.lang.reflect.InvocationTargetException
        at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
        at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:57)
        at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
        at java.lang.reflect.Constructor.newInstance(Constructor.java:526)
        at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:238)
        ... 12 more
Caused by: java.lang.ExceptionInInitializerError
        at org.apache.hadoop.hbase.ClusterId.parseFrom(ClusterId.java:64)
        at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:75)
        at org.apache.hadoop.hbase.client.ZooKeeperRegistry.getClusterId(ZooKeeperRegistry.java:86)
        at org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.retrieveClusterId(ConnectionManager.java:850)
        at org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.<init>(ConnectionManager.java:635)
        ... 17 more
Caused by: java.lang.RuntimeException: Failed to create local dir /usr/local/hbase/var/local/jars, DynamicClassLoader failed to init
        at org.apache.hadoop.hbase.util.DynamicClassLoader.<init>(DynamicClassLoader.java:94)
        at org.apache.hadoop.hbase.protobuf.ProtobufUtil.<clinit>(ProtobufUtil.java:242)
        ... 22 more

15/09/01 19:43:46 INFO mapreduce.Job:  map 1% reduce 0%
15/09/01 19:43:52 INFO mapreduce.Job:  map 2% reduce 0%
15/09/01 19:43:55 INFO mapreduce.Job:  map 3% reduce 0%
15/09/01 19:43:58 INFO mapreduce.Job:  map 5% reduce 0%
15/09/01 19:44:01 INFO mapreduce.Job:  map 6% reduce 0%
15/09/01 19:44:04 INFO mapreduce.Job:  map 7% reduce 0%
15/09/01 19:44:07 INFO mapreduce.Job:  map 8% reduce 0%
15/09/01 19:44:10 INFO mapreduce.Job:  map 9% reduce 0%
15/09/01 19:44:13 INFO mapreduce.Job:  map 10% reduce 0%
15/09/01 19:44:16 INFO mapreduce.Job:  map 11% reduce 0%
15/09/01 19:44:19 INFO mapreduce.Job:  map 12% reduce 0%
15/09/01 19:44:20 INFO mapreduce.Job:  map 13% reduce 0%
15/09/01 19:44:22 INFO mapreduce.Job:  map 14% reduce 0%
15/09/01 19:44:25 INFO mapreduce.Job:  map 15% reduce 0%
15/09/01 19:44:27 INFO mapreduce.Job:  map 16% reduce 0%
15/09/01 19:44:28 INFO mapreduce.Job:  map 17% reduce 0%
15/09/01 19:44:31 INFO mapreduce.Job:  map 18% reduce 0%
15/09/01 19:44:33 INFO mapreduce.Job:  map 19% reduce 0%
15/09/01 19:44:37 INFO mapreduce.Job:  map 20% reduce 0%
15/09/01 19:44:40 INFO mapreduce.Job:  map 21% reduce 0%
15/09/01 19:44:44 INFO mapreduce.Job:  map 22% reduce 0%
15/09/01 19:44:56 INFO mapreduce.Job:  map 23% reduce 0%
15/09/01 19:44:58 INFO mapreduce.Job:  map 24% reduce 0%
15/09/01 19:45:01 INFO mapreduce.Job:  map 26% reduce 0%
15/09/01 19:45:03 INFO mapreduce.Job:  map 27% reduce 0%
15/09/01 19:45:04 INFO mapreduce.Job:  map 28% reduce 0%
15/09/01 19:45:06 INFO mapreduce.Job:  map 29% reduce 0%
15/09/01 19:45:07 INFO mapreduce.Job:  map 31% reduce 0%
15/09/01 19:45:09 INFO mapreduce.Job:  map 32% reduce 0%
15/09/01 19:45:10 INFO mapreduce.Job:  map 34% reduce 0%
15/09/01 19:45:12 INFO mapreduce.Job:  map 35% reduce 0%
15/09/01 19:45:13 INFO mapreduce.Job:  map 36% reduce 0%
15/09/01 19:45:24 INFO mapreduce.Job: Task Id : attempt_1441014928183_0041_m_000001_1, Status : FAILED
Error: org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException: Failed 3138 actions: ConnectException: 3138 times, 
        at org.apache.hadoop.hbase.client.AsyncProcess$BatchErrors.makeException(AsyncProcess.java:227)
        at org.apache.hadoop.hbase.client.AsyncProcess$BatchErrors.access$1700(AsyncProcess.java:207)
        at org.apache.hadoop.hbase.client.AsyncProcess.waitForAllPreviousOpsAndReset(AsyncProcess.java:1658)
        at org.apache.hadoop.hbase.client.BufferedMutatorImpl.backgroundFlushCommits(BufferedMutatorImpl.java:208)
        at org.apache.hadoop.hbase.client.BufferedMutatorImpl.doMutate(BufferedMutatorImpl.java:141)
        at org.apache.hadoop.hbase.client.BufferedMutatorImpl.mutate(BufferedMutatorImpl.java:98)
        at org.apache.hadoop.hbase.mapreduce.TableOutputFormat$TableRecordWriter.write(TableOutputFormat.java:129)
        at org.apache.hadoop.hbase.mapreduce.TableOutputFormat$TableRecordWriter.write(TableOutputFormat.java:85)
        at org.apache.hadoop.mapred.MapTask$NewDirectOutputCollector.write(MapTask.java:658)
        at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89)
        at org.apache.hadoop.mapreduce.lib.map.WrappedMapper$Context.write(WrappedMapper.java:112)
        at com.su.HdfsToHbase$HdfsToHbaseMapper.map(HdfsToHbase.java:164)
        at com.su.HdfsToHbase$HdfsToHbaseMapper.map(HdfsToHbase.java:35)
        at org.apache.hadoop.mapreduce.Mapper.run(Mapper.java:145)
        at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:787)
        at org.apache.hadoop.mapred.MapTask.run(MapTask.java:341)
        at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:163)
        at java.security.AccessController.doPrivileged(Native Method)
        at javax.security.auth.Subject.doAs(Subject.java:415)
        at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1671)
        at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:158)

15/09/01 19:45:24 INFO mapreduce.Job: Task Id : attempt_1441014928183_0041_m_000000_0, Status : FAILED
Error: org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException: Failed 3137 actions: ConnectException: 3137 times, 
        at org.apache.hadoop.hbase.client.AsyncProcess$BatchErrors.makeException(AsyncProcess.java:227)
        at org.apache.hadoop.hbase.client.AsyncProcess$BatchErrors.access$1700(AsyncProcess.java:207)
        at org.apache.hadoop.hbase.client.AsyncProcess.waitForAllPreviousOpsAndReset(AsyncProcess.java:1658)
        at org.apache.hadoop.hbase.client.BufferedMutatorImpl.backgroundFlushCommits(BufferedMutatorImpl.java:208)
        at org.apache.hadoop.hbase.client.BufferedMutatorImpl.doMutate(BufferedMutatorImpl.java:141)
        at org.apache.hadoop.hbase.client.BufferedMutatorImpl.mutate(BufferedMutatorImpl.java:98)
        at org.apache.hadoop.hbase.mapreduce.TableOutputFormat$TableRecordWriter.write(TableOutputFormat.java:129)
        at org.apache.hadoop.hbase.mapreduce.TableOutputFormat$TableRecordWriter.write(TableOutputFormat.java:85)
        at org.apache.hadoop.mapred.MapTask$NewDirectOutputCollector.write(MapTask.java:658)
        at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89)
        at org.apache.hadoop.mapreduce.lib.map.WrappedMapper$Context.write(WrappedMapper.java:112)
        at com.su.HdfsToHbase$HdfsToHbaseMapper.map(HdfsToHbase.java:164)
        at com.su.HdfsToHbase$HdfsToHbaseMapper.map(HdfsToHbase.java:35)
        at org.apache.hadoop.mapreduce.Mapper.run(Mapper.java:145)
        at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:787)
        at org.apache.hadoop.mapred.MapTask.run(MapTask.java:341)
        at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:163)
        at java.security.AccessController.doPrivileged(Native Method)
        at javax.security.auth.Subject.doAs(Subject.java:415)
        at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1671)
        at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:158)

15/09/01 19:45:25 INFO mapreduce.Job:  map 0% reduce 0%
15/09/01 19:45:35 INFO mapreduce.Job: Task Id : attempt_1441014928183_0041_m_000001_2, Status : FAILED
Error: java.io.IOException: java.lang.reflect.InvocationTargetException
        at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:240)
        at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:218)
        at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:119)
        at org.apache.hadoop.hbase.mapreduce.TableOutputFormat$TableRecordWriter.<init>(TableOutputFormat.java:97)
        at org.apache.hadoop.hbase.mapreduce.TableOutputFormat.getRecordWriter(TableOutputFormat.java:144)
        at org.apache.hadoop.mapred.MapTask$NewDirectOutputCollector.<init>(MapTask.java:647)
        at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:767)
        at org.apache.hadoop.mapred.MapTask.run(MapTask.java:341)
        at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:163)
        at java.security.AccessController.doPrivileged(Native Method)
        at javax.security.auth.Subject.doAs(Subject.java:415)
        at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1671)
        at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:158)
Caused by: java.lang.reflect.InvocationTargetException
        at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
        at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:57)
        at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
        at java.lang.reflect.Constructor.newInstance(Constructor.java:526)
        at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:238)
        ... 12 more
Caused by: java.lang.ExceptionInInitializerError
        at org.apache.hadoop.hbase.ClusterId.parseFrom(ClusterId.java:64)
        at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:75)
        at org.apache.hadoop.hbase.client.ZooKeeperRegistry.getClusterId(ZooKeeperRegistry.java:86)
        at org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.retrieveClusterId(ConnectionManager.java:850)
        at org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.<init>(ConnectionManager.java:635)
        ... 17 more
Caused by: java.lang.RuntimeException: Failed to create local dir /usr/local/hbase/var/local/jars, DynamicClassLoader failed to init
        at org.apache.hadoop.hbase.util.DynamicClassLoader.<init>(DynamicClassLoader.java:94)
        at org.apache.hadoop.hbase.protobuf.ProtobufUtil.<clinit>(ProtobufUtil.java:242)
        ... 22 more

Container killed by the ApplicationMaster.
Container killed on request. Exit code is 143
Container exited with a non-zero exit code 143

15/09/01 19:45:53 INFO mapreduce.Job:  map 1% reduce 0%
15/09/01 19:45:56 INFO mapreduce.Job:  map 2% reduce 0%
15/09/01 19:45:59 INFO mapreduce.Job:  map 3% reduce 0%
15/09/01 19:46:02 INFO mapreduce.Job:  map 4% reduce 0%
15/09/01 19:46:04 INFO mapreduce.Job:  map 5% reduce 0%
15/09/01 19:46:05 INFO mapreduce.Job:  map 6% reduce 0%
15/09/01 19:46:08 INFO mapreduce.Job:  map 7% reduce 0%
15/09/01 19:46:10 INFO mapreduce.Job:  map 8% reduce 0%
15/09/01 19:46:11 INFO mapreduce.Job:  map 9% reduce 0%
15/09/01 19:46:14 INFO mapreduce.Job:  map 10% reduce 0%
15/09/01 19:46:17 INFO mapreduce.Job:  map 11% reduce 0%
15/09/01 19:46:18 INFO mapreduce.Job:  map 12% reduce 0%
15/09/01 19:46:21 INFO mapreduce.Job:  map 13% reduce 0%
15/09/01 19:46:24 INFO mapreduce.Job:  map 15% reduce 0%
15/09/01 19:46:27 INFO mapreduce.Job:  map 16% reduce 0%
15/09/01 19:46:30 INFO mapreduce.Job:  map 17% reduce 0%
15/09/01 19:46:31 INFO mapreduce.Job:  map 18% reduce 0%
15/09/01 19:46:34 INFO mapreduce.Job:  map 19% reduce 0%
15/09/01 19:46:36 INFO mapreduce.Job:  map 20% reduce 0%
15/09/01 19:46:37 INFO mapreduce.Job:  map 21% reduce 0%
15/09/01 19:46:41 INFO mapreduce.Job:  map 22% reduce 0%
15/09/01 19:46:45 INFO mapreduce.Job:  map 23% reduce 0%
15/09/01 19:46:48 INFO mapreduce.Job:  map 24% reduce 0%
15/09/01 19:46:51 INFO mapreduce.Job:  map 26% reduce 0%
15/09/01 19:46:55 INFO mapreduce.Job:  map 27% reduce 0%
15/09/01 19:46:58 INFO mapreduce.Job:  map 28% reduce 0%
15/09/01 19:47:01 INFO mapreduce.Job:  map 29% reduce 0%
15/09/01 19:47:03 INFO mapreduce.Job:  map 30% reduce 0%
15/09/01 19:47:04 INFO mapreduce.Job:  map 31% reduce 0%
15/09/01 19:47:07 INFO mapreduce.Job:  map 33% reduce 0%
15/09/01 19:47:10 INFO mapreduce.Job:  map 36% reduce 0%
15/09/01 19:47:13 INFO mapreduce.Job:  map 38% reduce 0%
15/09/01 19:47:16 INFO mapreduce.Job:  map 40% reduce 0%
15/09/01 19:47:19 INFO mapreduce.Job:  map 43% reduce 0%
15/09/01 19:47:22 INFO mapreduce.Job:  map 44% reduce 0%
15/09/01 19:47:23 INFO mapreduce.Job:  map 45% reduce 0%
15/09/01 19:47:24 INFO mapreduce.Job:  map 47% reduce 0%
15/09/01 19:47:27 INFO mapreduce.Job:  map 49% reduce 0%
15/09/01 19:47:29 INFO mapreduce.Job:  map 50% reduce 0%
15/09/01 19:47:31 INFO mapreduce.Job:  map 51% reduce 0%
15/09/01 19:47:32 INFO mapreduce.Job:  map 52% reduce 0%
15/09/01 19:47:34 INFO mapreduce.Job:  map 53% reduce 0%
15/09/01 19:47:35 INFO mapreduce.Job:  map 54% reduce 0%
15/09/01 19:47:37 INFO mapreduce.Job:  map 55% reduce 0%
15/09/01 19:47:40 INFO mapreduce.Job:  map 57% reduce 0%
15/09/01 19:47:43 INFO mapreduce.Job:  map 59% reduce 0%
15/09/01 19:47:46 INFO mapreduce.Job:  map 61% reduce 0%
15/09/01 19:47:49 INFO mapreduce.Job:  map 62% reduce 0%
15/09/01 19:47:52 INFO mapreduce.Job:  map 63% reduce 0%
/n15/09/01 19:47:54 INFO mapreduce.Job:  map 64% reduce 0%
15/09/01 19:47:57 INFO mapreduce.Job:  map 65% reduce 0%
15/09/01 19:48:01 INFO mapreduce.Job:  map 66% reduce 0%
15/09/01 19:48:04 INFO mapreduce.Job:  map 67% reduce 0%
15/09/01 19:48:07 INFO mapreduce.Job:  map 68% reduce 0%
15/09/01 19:48:12 INFO mapreduce.Job:  map 69% reduce 0%
15/09/01 19:48:17 INFO mapreduce.Job:  map 70% reduce 0%
15/09/01 19:48:23 INFO mapreduce.Job:  map 71% reduce 0%
15/09/01 19:48:29 INFO mapreduce.Job:  map 72% reduce 0%
15/09/01 19:48:35 INFO mapreduce.Job:  map 73% reduce 0%
15/09/01 19:48:45 INFO mapreduce.Job:  map 74% reduce 0%
15/09/01 19:49:01 INFO mapreduce.Job:  map 75% reduce 0%
15/09/01 19:50:27 INFO mapreduce.Job: Task Id : attempt_1441014928183_0041_m_000000_1, Status : FAILED
Error: org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException: Failed 1567 actions: IOException: 1567 times, 
        at org.apache.hadoop.hbase.client.AsyncProcess$BatchErrors.makeException(AsyncProcess.java:227)
        at org.apache.hadoop.hbase.client.AsyncProcess$BatchErrors.access$1700(AsyncProcess.java:207)
        at org.apache.hadoop.hbase.client.AsyncProcess.waitForAllPreviousOpsAndReset(AsyncProcess.java:1658)
        at org.apache.hadoop.hbase.client.BufferedMutatorImpl.backgroundFlushCommits(BufferedMutatorImpl.java:208)
        at org.apache.hadoop.hbase.client.BufferedMutatorImpl.doMutate(BufferedMutatorImpl.java:141)
        at org.apache.hadoop.hbase.client.BufferedMutatorImpl.mutate(BufferedMutatorImpl.java:98)
        at org.apache.hadoop.hbase.mapreduce.TableOutputFormat$TableRecordWriter.write(TableOutputFormat.java:129)
        at org.apache.hadoop.hbase.mapreduce.TableOutputFormat$TableRecordWriter.write(TableOutputFormat.java:85)
        at org.apache.hadoop.mapred.MapTask$NewDirectOutputCollector.write(MapTask.java:658)
        at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89)
        at org.apache.hadoop.mapreduce.lib.map.WrappedMapper$Context.write(WrappedMapper.java:112)
        at com.su.HdfsToHbase$HdfsToHbaseMapper.map(HdfsToHbase.java:164)
        at com.su.HdfsToHbase$HdfsToHbaseMapper.map(HdfsToHbase.java:35)
        at org.apache.hadoop.mapreduce.Mapper.run(Mapper.java:145)
        at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:787)
        at org.apache.hadoop.mapred.MapTask.run(MapTask.java:341)
        at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:163)
        at java.security.AccessController.doPrivileged(Native Method)
        at javax.security.auth.Subject.doAs(Subject.java:415)
        at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1671)
        at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:158)

15/09/01 19:50:28 INFO mapreduce.Job:  map 46% reduce 0%
15/09/01 19:50:53 INFO mapreduce.Job:  map 47% reduce 0%
/n15/09/01 19:53:26 INFO mapreduce.Job:  map 100% reduce 0%
15/09/01 19:53:26 INFO mapreduce.Job: Job job_1441014928183_0041 failed with state FAILED due to: Task failed task_1441014928183_0041_m_000001
Job failed as tasks failed. failedMaps:1 failedReduces:0

15/09/01 19:53:27 INFO mapreduce.Job: Counters: 10
        Job Counters 
                Failed map tasks=6
                Killed map tasks=1
                Launched map tasks=7
                Other local map tasks=5
                Data-local map tasks=2
                Total time spent by all maps in occupied slots (ms)=4768664
                Total time spent by all reduces in occupied slots (ms)=0
                Total time spent by all map tasks (ms)=1192166
                Total vcore-seconds taken by all map tasks=1192166
                Total megabyte-seconds taken by all map tasks=238433200



 

 

 
两个错误:
1:Error: org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException: Failed 1567 actions: IOException: 1567 times, 
:2:Error: java.io.IOException: java.lang.reflect.InvocationTargetException
        at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:240)
        at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:218)
        at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:119)
        at org.apache.hadoop.hbase.mapreduce.TableOutputFormat$TableRecordWriter.<init>(TableOutputFormat.java:97)
        at org.apache.hadoop.hbase.mapreduce.TableOutputFormat.getRecordWriter(TableOutputFormat.java:144)
        at org.apache.hadoop.mapred.MapTask$NewDirectOutputCollector.<init>(MapTask.java:647)
        at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:767)
        at org.apache.hadoop.mapred.MapTask.run(MapTask.java:341)
        at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:163)
        at java.security.AccessController.doPrivileged(Native Method)
        at javax.security.auth.Subject.doAs(Subject.java:415)
        at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1671)
        at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:158)
Caused by: java.lang.reflect.InvocationTargetException
        at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
        at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:57)
        at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
        at java.lang.reflect.Constructor.newInstance(Constructor.java:526)
        at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:238)
        ... 12 more
Caused by: java.lang.ExceptionInInitializerError
        at org.apache.hadoop.hbase.ClusterId.parseFrom(ClusterId.java:64)
        at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:75)
        at org.apache.hadoop.hbase.client.ZooKeeperRegistry.getClusterId(ZooKeeperRegistry.java:86)
        at org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.retrieveClusterId(ConnectionManager.java:850)
        at org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.<init>(ConnectionManager.java:635)
        ... 17 more
Caused by: java.lang.RuntimeException: Failed to create local dir /usr/local/hbase/var/local/jars, DynamicClassLoader failed to init
        at org.apache.hadoop.hbase.util.DynamicClassLoader.<init>(DynamicClassLoader.java:94)
        at org.apache.hadoop.hbase.protobuf.ProtobufUtil.<clinit>(ProtobufUtil.java:242)
        ... 22 more



scan 表的时候:
hbase(main):001:0> list
TABLE                                                                                                                                                                  
user_movie                                                                                                                                                             
1 row(s) in 0.5680 seconds

=> ["user_movie"]
hbase(main):002:0> scan 'user_movie'
ROW                                        COLUMN+CELL                                                                                                                 

ERROR: Connection refused

Here is some help for this command:
Scan a table; pass table name and optionally a dictionary of scanner
specifications.  Scanner specifications may include one or more of:
TIMERANGE, FILTER, LIMIT, STARTROW, STOPROW, TIMESTAMP, MAXLENGTH,
or COLUMNS, CACHE or RAW, VERSIONS

If no columns are specified, all columns will be scanned.
To scan all members of a column family, leave the qualifier empty as in
'col_family:'.

The filter can be specified in two ways:
1. Using a filterString - more information on this is available in the
Filter Language document attached to the HBASE-4176 JIRA
2. Using the entire package name of the filter.

Some examples:

  hbase> scan 'hbase:meta'
  hbase> scan 'hbase:meta', {COLUMNS => 'info:regioninfo'}
  hbase> scan 'ns1:t1', {COLUMNS => ['c1', 'c2'], LIMIT => 10, STARTROW => 'xyz'}
  hbase> scan 't1', {COLUMNS => ['c1', 'c2'], LIMIT => 10, STARTROW => 'xyz'}
  hbase> scan 't1', {COLUMNS => 'c1', TIMERANGE => [1303668804, 1303668904]}
  hbase> scan 't1', {REVERSED => true}
  hbase> scan 't1', {FILTER => "(PrefixFilter ('row2') AND
    (QualifierFilter (>=, 'binary:xyz'))) AND (TimestampsFilter ( 123, 456))"}
  hbase> scan 't1', {FILTER =>
 

mopishv0 - 高级开发工程师@美团

赞同来自: suhuadong fish

Master里面的是处理rs挂掉后的处理,split hlog。 挂掉的原因要去看具体的rs日志。

mopishv0 - 高级开发工程师@美团

赞同来自: suhuadong

看你的第二个log依然是rs有问题,可以观察下rs的页面handler是否都卡住了,大量离线数据写入推荐使用bulkload,否则memstore达到upperlimit、storefile数量超限都会引发写入阻塞

wangwensheng - 大数据工程师@腾讯

赞同来自:

看下抛异常时hbase master的日志。

suhuadong

赞同来自:

hbase 已经用不了了,我在恢复他

suhuadong

赞同来自:

我重新跑了一遍,发现SU-4这台机器的 [root@SU-4 logs]# tail -f hbase-hbase-regionserver-SU-4.log  2015-09-01 22:23:22,902 ERROR [B.defaultRpcServer.handler=29,queue=2,port=60020] ipc.RpcServer: Unexpected throwable object  java.lang.OutOfMemoryError: Java heap space 2015-09-01 22:23:25,622 ERROR [B.defaultRpcServer.handler=2,queue=2,port=60020] ipc.RpcServer: Unexpected throwable object  java.lang.OutOfMemoryError: Java heap space 2015-09-01 22:23:29,381 ERROR [B.defaultRpcServer.handler=19,queue=1,port=60020] ipc.RpcServer: Unexpected throwable object  java.lang.OutOfMemoryError: Java heap space 2015-09-01 22:26:35,005 INFO  [LruBlockCacheStatsExecutor] hfile.LruBlockCache: totalSize=102.79 KB, freeSize=96.60 MB, max=96.70 MB, blockCount=0, accesses=0, hits=0, hitRatio=0, cachingAccesses=0, cachingHits=0, cachingHitsRatio=0,evictions=89, evicted=0, evictedPerRun=0.0 2015-09-01 22:31:35,005 INFO  [LruBlockCacheStatsExecutor] hfile.LruBlockCache: totalSize=102.79 KB, freeSize=96.60 MB, max=96.70 MB, blockCount=0, accesses=0, hits=0, hitRatio=0, cachingAccesses=0, cachingHits=0, cachingHitsRatio=0,evictions=119, evicted=0, evictedPerRun=0.0 2015-09-01 22:36:35,005 INFO  [LruBlockCacheStatsExecutor] hfile.LruBlockCache: totalSize=102.79 KB, freeSize=96.60 MB, max=96.70 MB, blockCount=0, accesses=0, hits=0, hitRatio=0, cachingAccesses=0, cachingHits=0, cachingHitsRatio=0,evictions=149, evicted=0, evictedPerRun=0.0 2015-09-01 22:41:35,005 INFO  [LruBlockCacheStatsExecutor] hfile.LruBlockCache: totalSize=102.79 KB, freeSize=96.60 MB, max=96.70 MB, blockCount=0, accesses=0, hits=0, hitRatio=0, cachingAccesses=0, cachingHits=0, cachingHitsRatio=0,evictions=179, evicted=0, evictedPerRun=0.0 2015-09-01 22:44:00,266 ERROR [B.defaultRpcServer.handler=22,queue=1,port=60020] ipc.RpcServer: Unexpected throwable object  java.lang.OutOfMemoryError: Java heap space 2015-09-01 22:44:03,383 ERROR [B.defaultRpcServer.handler=24,queue=0,port=60020] ipc.RpcServer: Unexpected throwable object  java.lang.OutOfMemoryError: Java heap space 2015-09-01 22:44:07,606 ERROR [B.defaultRpcServer.handler=27,queue=0,port=60020] ipc.RpcServer: Unexpected throwable object  java.lang.OutOfMemoryError: Java heap space 2015-09-01 22:44:09,908 ERROR [B.defaultRpcServer.handler=0,queue=0,port=60020] ipc.RpcServer: Unexpected throwable object  java.lang.OutOfMemoryError: Java heap space 2015-09-01 22:44:11,573 ERROR [B.defaultRpcServer.handler=5,queue=2,port=60020] ipc.RpcServer: Unexpected throwable object  java.lang.OutOfMemoryError: Java heap space 2015-09-01 22:44:13,189 ERROR [B.defaultRpcServer.handler=25,queue=1,port=60020] ipc.RpcServer: Unexpected throwable object  java.lang.OutOfMemoryError: Java heap space 2015-09-01 22:44:14,796 ERROR [B.defaultRpcServer.handler=3,queue=0,port=60020] ipc.RpcServer: Unexpected throwable object  java.lang.OutOfMemoryError: Java heap space 2015-09-01 22:44:16,655 ERROR [B.defaultRpcServer.handler=28,queue=1,port=60020] ipc.RpcServer: Unexpected throwable object  java.lang.OutOfMemoryError: Java heap space 2015-09-01 22:44:18,356 ERROR [B.defaultRpcServer.handler=1,queue=1,port=60020] ipc.RpcServer: Unexpected throwable object  java.lang.OutOfMemoryError: Java heap space 2015-09-01 22:44:19,075 ERROR [B.defaultRpcServer.handler=8,queue=2,port=60020] ipc.RpcServer: Unexpected throwable object  java.lang.OutOfMemoryError: Java heap space 2015-09-01 22:44:20,840 ERROR [B.defaultRpcServer.handler=4,queue=1,port=60020] ipc.RpcServer: Unexpected throwable object  java.lang.OutOfMemoryError: Java heap space 2015-09-01 22:44:23,540 ERROR [B.defaultRpcServer.handler=11,queue=2,port=60020] ipc.RpcServer: Unexpected throwable object  java.lang.OutOfMemoryError: Java heap space 2015-09-01 22:44:27,421 ERROR [B.defaultRpcServer.handler=6,queue=0,port=60020] ipc.RpcServer: Unexpected throwable object  java.lang.OutOfMemoryError: Java heap space 2015-09-01 22:44:36,786 ERROR [B.defaultRpcServer.handler=7,queue=1,port=60020] ipc.RpcServer: Unexpected throwable object  java.lang.OutOfMemoryError: Java heap space 2015-09-01 22:44:38,833 ERROR [B.defaultRpcServer.handler=9,queue=0,port=60020] ipc.RpcServer: Unexpected throwable object  java.lang.OutOfMemoryError: Java heap space 2015-09-01 22:44:42,226 ERROR [B.defaultRpcServer.handler=14,queue=2,port=60020] ipc.RpcServer: Unexpected throwable object  java.lang.OutOfMemoryError: Java heap space 2015-09-01 22:44:43,000 ERROR [B.defaultRpcServer.handler=17,queue=2,port=60020] ipc.RpcServer: Unexpected throwable object  java.lang.OutOfMemoryError: Java heap space 2015-09-01 22:44:44,812 ERROR [B.defaultRpcServer.handler=12,queue=0,port=60020] ipc.RpcServer: Unexpected throwable object  java.lang.OutOfMemoryError: Java heap space 2015-09-01 22:44:47,625 ERROR [B.defaultRpcServer.handler=20,queue=2,port=60020] ipc.RpcServer: Unexpected throwable object  java.lang.OutOfMemoryError: Java heap space 2015-09-01 22:44:50,588 ERROR [B.defaultRpcServer.handler=23,queue=2,port=60020] ipc.RpcServer: Unexpected throwable object  java.lang.OutOfMemoryError: Java heap space 2015-09-01 22:44:51,461 ERROR [B.defaultRpcServer.handler=10,queue=1,port=60020] ipc.RpcServer: Unexpected throwable object  java.lang.OutOfMemoryError: Java heap space 2015-09-01 22:44:52,472 ERROR [B.defaultRpcServer.handler=26,queue=2,port=60020] ipc.RpcServer: Unexpected throwable object  java.lang.OutOfMemoryError: Java heap space 2015-09-01 22:44:55,654 ERROR [B.defaultRpcServer.handler=13,queue=1,port=60020] ipc.RpcServer: Unexpected throwable object  java.lang.OutOfMemoryError: Java heap space 2015-09-01 22:44:56,459 ERROR [B.defaultRpcServer.handler=29,queue=2,port=60020] ipc.RpcServer: Unexpected throwable object  java.lang.OutOfMemoryError: Java heap space 2015-09-01 22:44:58,262 ERROR [B.defaultRpcServer.handler=16,queue=1,port=60020] ipc.RpcServer: Unexpected throwable object  java.lang.OutOfMemoryError: Java heap space 2015-09-01 22:45:01,074 ERROR [B.defaultRpcServer.handler=19,queue=1,port=60020] ipc.RpcServer: Unexpected throwable object  java.lang.OutOfMemoryError: Java heap space 2015-09-01 22:45:04,895 ERROR [B.defaultRpcServer.handler=22,queue=1,port=60020] ipc.RpcServer: Unexpected throwable object  java.lang.OutOfMemoryError: Java heap space 2015-09-01 22:45:11,573 ERROR [B.defaultRpcServer.handler=25,queue=1,port=60020] ipc.RpcServer: Unexpected throwable object  java.lang.OutOfMemoryError: Java heap space 2015-09-01 22:45:13,704 ERROR [B.defaultRpcServer.handler=28,queue=1,port=60020] ipc.RpcServer: Unexpected throwable object  java.lang.OutOfMemoryError: Java heap space 2015-09-01 22:45:17,436 ERROR [B.defaultRpcServer.handler=1,queue=1,port=60020] ipc.RpcServer: Unexpected throwable object  java.lang.OutOfMemoryError: Java heap space 2015-09-01 22:45:18,331 ERROR [B.defaultRpcServer.handler=2,queue=2,port=60020] ipc.RpcServer: Unexpected throwable object  java.lang.OutOfMemoryError: Java heap space 2015-09-01 22:45:20,240 ERROR [B.defaultRpcServer.handler=5,queue=2,port=60020] ipc.RpcServer: Unexpected throwable object  java.lang.OutOfMemoryError: Java heap space   regionServer 的堆内存不够了

suhuadong

赞同来自:

我在hbase-env.sh中调大了堆内存 重跑任务,在regionserver.log没有java.lang.OutOfMemoryError: Java heap space错误了,但是还有 Error: org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException   附件是hmaster的日志,跑madpreduce的时间为:15/09/01 23:17:39 到15/09/01 23:33:57   在跑完mapreduce后,我在hbase shell ,scan 表的时候,出现connection refused,查看master的日志,如下 2015-09-01 23:26:43,595 WARN  [CatalogJanitor-SU-4:60000] master.CatalogJanitor: Failed scan of catalog table org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=31, exceptions: Tue Sep 01 23:26:43 CST 2015, null, java.net.SocketTimeoutException: callTimeout=60000, callDuration=78334: row '' on table 'hbase:meta' at region=hbase:meta,,1.1588230740, hostname=SU-3,60020,1441120636962, seqNum=0         at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:270)         at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:203)         at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:57)         at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithoutRetries(RpcRetryingCaller.java:200)         at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:294)         at org.apache.hadoop.hbase.client.ClientScanner.nextScanner(ClientScanner.java:269)         at org.apache.hadoop.hbase.client.ClientScanner.initializeScannerInConstruction(ClientScanner.java:141)         at org.apache.hadoop.hbase.client.ClientScanner.<init>(ClientScanner.java:136)         at org.apache.hadoop.hbase.client.HTable.getScanner(HTable.java:886)         at org.apache.hadoop.hbase.client.MetaScanner.metaScan(MetaScanner.java:187)         at org.apache.hadoop.hbase.client.MetaScanner.metaScan(MetaScanner.java:89)         at org.apache.hadoop.hbase.master.CatalogJanitor.getMergedRegionsAndSplitParents(CatalogJanitor.java:169)         at org.apache.hadoop.hbase.master.CatalogJanitor.getMergedRegionsAndSplitParents(CatalogJanitor.java:121)         at org.apache.hadoop.hbase.master.CatalogJanitor.scan(CatalogJanitor.java:222)         at org.apache.hadoop.hbase.master.CatalogJanitor.chore(CatalogJanitor.java:103)         at org.apache.hadoop.hbase.Chore.run(Chore.java:87)         at java.lang.Thread.run(Thread.java:744) Caused by: java.net.SocketTimeoutException: callTimeout=60000, callDuration=78334: row '' on table 'hbase:meta' at region=hbase:meta,,1.1588230740, hostname=SU-3,60020,1441120636962, seqNum=0         at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(RpcRetryingCaller.java:159)         at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:64)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)         ... 1 more Caused by: java.net.ConnectException: Connection refused         at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)         at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:735)         at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)         at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:530)         at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:494)         at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.setupConnection(RpcClientImpl.java:404)         at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.setupIOstreams(RpcClientImpl.java:710)         at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.writeRequest(RpcClientImpl.java:881)         at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.tracedWriteRequest(RpcClientImpl.java:850)         at org.apache.hadoop.hbase.ipc.RpcClientImpl.call(RpcClientImpl.java:1174)         at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:216)         at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:300)         at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$BlockingStub.scan(ClientProtos.java:31751)         at org.apache.hadoop.hbase.client.ScannerCallable.openScanner(ScannerCallable.java:337)         at org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:192)         at org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:62)         at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithoutRetries(RpcRetryingCaller.java:200)         at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:316)         at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:290)         at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(RpcRetryingCaller.java:126)         ... 4 more

suhuadong

赞同来自:

在跑mapreduce 中从hmaster日志中,我看到master将SU-3 regionserver shutdown了 2015-09-02 11:16:36,791 INFO  [MASTER_SERVER_OPERATIONS-SU-4:60000-4] handler.ServerShutdownHandler: Finished processing of shutdown of SU-3,60020,1441157466488 2015-09-02 11:16:36,796 INFO  [AM.ZK.Worker-pool2-t55] master.RegionStates: Transition {7d518b107bd99dc086f693feb69b5e8f state=PENDING_OPEN, ts=1441163796614, server=SU-2,60020,1441163776450} to {7d518b107bd99dc086f693feb69b5e8f state=OPENING, ts=1441163796796, server=SU-2,60020,1441163776450}   并且后面有下线日志 对于SU-3 regionserver: 2015-09-02 11:16:45,166 INFO  [AM.ZK.Worker-pool2-t58] master.RegionStates: Offlined 7d518b107bd99dc086f693feb69b5e8f from SU-3,60020,1441157466488 对于SU-2节点 2015-09-02 11:19:07,512 INFO  [AM.ZK.Worker-pool2-t69] master.RegionStates: Offlined 7d518b107bd99dc086f693feb69b5e8f from SU-2,60020,1441163776450   对于SU-4节点: 2015-09-02 11:21:15,297 INFO  [main-EventThread] zookeeper.RegionServerTracker: RegionServer ephemeral node deleted, processing expiration [SU-4,60020,1441163839578] 2015-09-02 11:21:15,314 INFO  [MASTER_SERVER_OPERATIONS-SU-4:60000-0] handler.ServerShutdownHandler: Splitting logs for SU-4,60020,1441163839578 before assignment; region count=1 2015-09-02 11:21:15,320 INFO  [MASTER_SERVER_OPERATIONS-SU-4:60000-0] master.SplitLogManager: dead splitlog workers [SU-4,60020,1441163839578] 2015-09-02 11:21:15,322 INFO  [MASTER_SERVER_OPERATIONS-SU-4:60000-0] master.SplitLogManager: started splitting 1 logs in [hdfs://SU-1:8020/hbase/WALs/SU-4,60020,1441163839578-splitting] for [SU-4,60020,1441163839578]        

suhuadong

赞同来自:

hmaster建表到mapreduce任务出现异常期间的日志: 2015-09-02 11:59:10,953 INFO  [MASTER_TABLE_OPERATIONS-SU-4:60000-0] handler.CreateTableHandler: Create table user_movie 2015-09-02 11:59:11,152 INFO  [RegionOpenAndInitThread-user_movie-1] regionserver.HRegion: creating HRegion user_movie HTD == 'user_movie', {NAME => 'movie', DATA_BLOCK_ENCODING => 'NONE', BLOOMFILTER => 'ROW', REPLICATION_SCOPE => '0', VERSIONS => '1', COMPRESSION => 'NONE', MIN_VERSIONS => '0', TTL => 'FOREVER', KEEP_DELETED_CELLS => 'FALSE', BLOCKSIZE => '65536', IN_MEMORY => 'false', BLOCKCACHE => 'true'}, {NAME => 'user', DATA_BLOCK_ENCODING => 'NONE', BLOOMFILTER => 'ROW', REPLICATION_SCOPE => '0', VERSIONS => '1', COMPRESSION => 'NONE', MIN_VERSIONS => '0', TTL => 'FOREVER', KEEP_DELETED_CELLS => 'FALSE', BLOCKSIZE => '65536', IN_MEMORY => 'false', BLOCKCACHE => 'true'} RootDir = hdfs://SU-1:8020/hbase/.tmp Table name == user_movie 2015-09-02 11:59:11,204 INFO  [RegionOpenAndInitThread-user_movie-1] regionserver.HRegion: Closed user_movie,,1441166350920.f9adbfa60b93a96c20e19a24b98d23d5. 2015-09-02 11:59:11,217 INFO  [MASTER_TABLE_OPERATIONS-SU-4:60000-0] hbase.MetaTableAccessor: Added 2 2015-09-02 11:59:11,244 INFO  [MASTER_TABLE_OPERATIONS-SU-4:60000-0] master.AssignmentManager: Assigning 1 region(s) to SU-4,60020,1441166195390 2015-09-02 11:59:11,250 INFO  [MASTER_TABLE_OPERATIONS-SU-4:60000-0] master.RegionStates: Transition {f9adbfa60b93a96c20e19a24b98d23d5 state=OFFLINE, ts=1441166351244, server=null} to {f9adbfa60b93a96c20e19a24b98d23d5 state=PENDING_OPEN, ts=1441166351250, server=SU-4,60020,1441166195390} 2015-09-02 11:59:11,260 WARN  [MASTER_TABLE_OPERATIONS-SU-4:60000-0] zookeeper.ZKTableStateManager: Moving table user_movie state from ENABLING to ENABLED 2015-09-02 11:59:11,266 INFO  [MASTER_TABLE_OPERATIONS-SU-4:60000-0] handler.CreateTableHandler: Table, user_movie, creation successful 2015-09-02 11:59:11,271 INFO  [AM.ZK.Worker-pool2-t17] master.RegionStates: Transition {f9adbfa60b93a96c20e19a24b98d23d5 state=PENDING_OPEN, ts=1441166351250, server=SU-4,60020,1441166195390} to {f9adbfa60b93a96c20e19a24b98d23d5 state=OPENING, ts=1441166351271, server=SU-4,60020,1441166195390} 2015-09-02 11:59:11,326 INFO  [AM.ZK.Worker-pool2-t18] master.RegionStates: Transition {f9adbfa60b93a96c20e19a24b98d23d5 state=OPENING, ts=1441166351271, server=SU-4,60020,1441166195390} to {f9adbfa60b93a96c20e19a24b98d23d5 state=OPEN, ts=1441166351326, server=SU-4,60020,1441166195390} 2015-09-02 12:00:40,838 INFO  [main-EventThread] zookeeper.RegionServerTracker: RegionServer ephemeral node deleted, processing expiration [SU-4,60020,1441166195390] 2015-09-02 12:00:40,847 INFO  [MASTER_META_SERVER_OPERATIONS-SU-4:60000-0] handler.MetaServerShutdownHandler: Splitting hbase:meta logs for SU-4,60020,1441166195390 2015-09-02 12:00:40,857 INFO  [MASTER_META_SERVER_OPERATIONS-SU-4:60000-0] master.SplitLogManager: dead splitlog workers [SU-4,60020,1441166195390] 2015-09-02 12:00:40,862 INFO  [MASTER_META_SERVER_OPERATIONS-SU-4:60000-0] master.SplitLogManager: started splitting 1 logs in [hdfs://SU-1:8020/hbase/WALs/SU-4,60020,1441166195390-splitting] for [SU-4,60020,1441166195390] 2015-09-02 12:00:40,933 INFO  [SU-4,60000,1441166179441.splitLogManagerTimeoutMonitor] master.SplitLogManager: total tasks = 1 unassigned = 1 tasks={/hbase/splitWAL/WALs%2FSU-4%2C60020%2C1441166195390-splitting%2FSU-4%252C60020%252C1441166195390..meta.1441166202430.meta=last_update = -1 last_version = -1 cur_worker_name = null status = in_progress incarnation = 0 resubmits = 0 batch = installed = 1 done = 0 error = 0} 2015-09-02 12:00:40,951 INFO  [main-EventThread] coordination.SplitLogManagerCoordination: task /hbase/splitWAL/WALs%2FSU-4%2C60020%2C1441166195390-splitting%2FSU-4%252C60020%252C1441166195390..meta.1441166202430.meta acquired by SU-3,60020,1441166207007 2015-09-02 12:00:45,546 INFO  [main-EventThread] coordination.SplitLogManagerCoordination: task /hbase/splitWAL/WALs%2FSU-4%2C60020%2C1441166195390-splitting%2FSU-4%252C60020%252C1441166195390..meta.1441166202430.meta entered state: DONE SU-3,60020,1441166207007 2015-09-02 12:00:45,576 INFO  [main-EventThread] wal.WALSplitter: Archived processed log hdfs://SU-1:8020/hbase/WALs/SU-4,60020,1441166195390-splitting/SU-4%2C60020%2C1441166195390..meta.1441166202430.meta to hdfs://SU-1:8020/hbase/oldWALs/SU-4%2C60020%2C1441166195390..meta.1441166202430.meta 2015-09-02 12:00:45,578 INFO  [main-EventThread] coordination.SplitLogManagerCoordination: Done splitting /hbase/splitWAL/WALs%2FSU-4%2C60020%2C1441166195390-splitting%2FSU-4%252C60020%252C1441166195390..meta.1441166202430.meta 2015-09-02 12:00:45,596 WARN  [MASTER_META_SERVER_OPERATIONS-SU-4:60000-0] master.SplitLogManager: returning success without actually splitting and deleting all the log files in path hdfs://SU-1:8020/hbase/WALs/SU-4,60020,1441166195390-splitting 2015-09-02 12:00:45,596 INFO  [MASTER_META_SERVER_OPERATIONS-SU-4:60000-0] master.SplitLogManager: finished splitting (more than or equal to) 83 bytes in 1 log files in [hdfs://SU-1:8020/hbase/WALs/SU-4,60020,1441166195390-splitting] in 4734ms 2015-09-02 12:00:45,597 INFO  [MASTER_META_SERVER_OPERATIONS-SU-4:60000-0] handler.MetaServerShutdownHandler: Server SU-4,60020,1441166195390 was carrying META. Trying to assign. 2015-09-02 12:00:45,597 INFO  [MASTER_META_SERVER_OPERATIONS-SU-4:60000-0] master.RegionStates: Transition {1588230740 state=OPEN, ts=1441166203781, server=SU-4,60020,1441166195390} to {1588230740 state=OFFLINE, ts=1441166445597, server=SU-4,60020,1441166195390} 2015-09-02 12:00:45,597 INFO  [MASTER_META_SERVER_OPERATIONS-SU-4:60000-0] master.RegionStates: Offlined 1588230740 from SU-4,60020,1441166195390 2015-09-02 12:00:45,606 INFO  [MASTER_META_SERVER_OPERATIONS-SU-4:60000-0] zookeeper.MetaTableLocator: Failed verification of hbase:meta,,1 at address=SU-4,60020,1441166195390, exception=Connection refused 2015-09-02 12:00:45,606 INFO  [MASTER_META_SERVER_OPERATIONS-SU-4:60000-0] zookeeper.MetaTableLocator: Deleting hbase:meta region location in ZooKeeper 2015-09-02 12:00:45,613 INFO  [MASTER_META_SERVER_OPERATIONS-SU-4:60000-0] master.AssignmentManager: Assigning hbase:meta,,1.1588230740 to SU-3,60020,1441166207007 2015-09-02 12:00:45,613 INFO  [MASTER_META_SERVER_OPERATIONS-SU-4:60000-0] master.RegionStates: Transition {1588230740 state=OFFLINE, ts=1441166445609, server=SU-4,60020,1441166195390} to {1588230740 state=PENDING_OPEN, ts=1441166445613, server=SU-3,60020,1441166207007} 2015-09-02 12:00:45,729 INFO  [AM.ZK.Worker-pool2-t22] master.RegionStates: Transition {1588230740 state=PENDING_OPEN, ts=1441166445613, server=SU-3,60020,1441166207007} to {1588230740 state=OPENING, ts=1441166445729, server=SU-3,60020,1441166207007} 2015-09-02 12:00:46,768 INFO  [AM.ZK.Worker-pool2-t23] master.RegionStates: Transition {1588230740 state=OPENING, ts=1441166445729, server=SU-3,60020,1441166207007} to {1588230740 state=OPEN, ts=1441166446768, server=SU-3,60020,1441166207007}     2015-09-02 12:00:40,847 INFO  [MASTER_META_SERVER_OPERATIONS-SU-4:60000-0] handler.MetaServerShutdownHandler: Splitting hbase:meta logs for SU-4,60020,1441166195390

suhuadong

赞同来自:

@mopishv0:谢谢了。 hbase的debug级别的日志打开啦,暂时没看出rs挂掉的原因。 改成bulkload方式,现在可以正常从hdfs导入到hbase,没啥问题。 对于bulkload这种方式: job.setSpeculativeExecution(false);  job.setReduceSpeculativeExecution(false); 不知道把推测执行关闭掉,有啥好处呢?  

要回复问题请先登录注册