Job failed with following exception:
java.io.IOException: Error -5 at com.mapr.fs.Inode.throwIfFailed(Inode.java:158) at com.mapr.fs.Inode.getDataIntoCache(Inode.java:568) at com.mapr.fs.MapRFsInStream.dropAndRefillCurrentPage(MapRFsInStream.java:98) at com.mapr.fs.MapRFsInStream.read(MapRFsInStream.java:133) at java.io.DataInputStream.read(DataInputStream.java:83) at org.apache.hadoop.util.LineReader.readDefaultLine(LineReader.java:205) at org.apache.hadoop.util.LineReader.readLine(LineReader.java:169) at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:160) at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:38) at org.apache.hadoop.hive.ql.io.HiveRecordReader.doNext(HiveRecordReader.java:66) at org.apache.hadoop.hive.ql.io.HiveRecordReader.doNext(HiveRecordReader.java:32) at org.apache.hadoop.hive.ql.io.HiveContextAwareRecordReader.next(HiveContextAwareRecordReader.java:67) at org.apache.hadoop.mapred.MapTask$TrackedRecordReader.moveToNext(MapTask.java:210) at org.apache.hadoop.mapred.MapTask$TrackedRecordReader.next(MapTask.java:195) at org.apache.hadoop.mapred.MapRunner.run(MapRunner.java:48) at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:394) at org.apache.hadoop.mapred.MapTask.run(MapTask.java:327) at org.apache.hadoop.mapred.Child$4.run(Child.java:270) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:396) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1109) at org.apache.hadoop.mapred.Child.main(Child.java:264)
While investigation, found that there is a core dump under /opt/cores:
Stack: [0x0000000040abd000,0x0000000040bbe000], sp=0x0000000040bbb970, free space=1018k
Native frames: (J=compiled Java code, j=interpreted, Vv=VM code, C=native code)
C [libMapRClient.so.1+0x224199] mapr::fs::AllocOnePage()+0x19
Java frames: (J=compiled Java code, j=interpreted, Vv=VM code)
j com.mapr.fs.Page.allocateByteBuf(IZ)I+0
j com.mapr.fs.Page.<init>(Ljava/util/concurrent/locks/Lock;Z)V+56
j com.mapr.fs.Inode.<init>(JJLjava/lang/String;Lcom/mapr/fs/MapRFsInStream;)V+102
j com.mapr.fs.MapRFsInStream.<init>(JJLjava/lang/String;Lorg/apache/hadoop/fs/FileSystem$Statistics;)V+35
j com.mapr.fs.MapRClient.open(Ljava/lang/String;)Lcom/mapr/fs/MapRFsInStream;+137
j com.mapr.fs.MapRFileSystem.open(Lorg/apache/hadoop/fs/Path;I)Lorg/apache/hadoop/fs/FSDataInputStream;+18
j org.apache.hadoop.fs.FileSystem.open(Lorg/apache/hadoop/fs/Path;)Lorg/apache/hadoop/fs/FSDataInputStream;+14
j org.apache.hadoop.mapred.Merger$Segment.init(Lorg/apache/hadoop/mapred/Counters$Counter;)V+15
j org.apache.hadoop.mapred.Merger$Segment.access$100(Lorg/apache/hadoop/mapred/Merger$Segment;Lorg/apache/hadoop/mapred/Counters$Counter;)V+2
j org.apache.hadoop.mapred.Merger$MergeQueue.merge(Ljava/lang/Class;Ljava/lang/Class;IILorg/apache/hadoop/fs/Path;Lorg/apache/hadoop/mapred/Counters$Counter;Lorg/apache/hadoop/mapred/Counters$Counter;)Lorg/apache/hadoop/mapred/RawKeyValueIterator;+142
j org.apache.hadoop.mapred.Merger$MergeQueue.merge(Ljava/lang/Class;Ljava/lang/Class;ILorg/apache/hadoop/fs/Path;Lorg/apache/hadoop/mapred/Counters$Counter;Lorg/apache/hadoop/mapred/Counters$Counter;)Lorg/apache/hadoop/mapred/RawKeyValueIterator;+11
j org.apache.hadoop.mapred.Merger.merge(Lorg/apache/hadoop/conf/Configuration;Lorg/apache/hadoop/fs/FileSystem;Ljava/lang/Class;Ljava/lang/Class;Lorg/apache/hadoop/io/compress/CompressionCodec;Ljava/util/List;ILorg/apache/hadoop/fs/Path;Lorg/apache/hadoop/io/RawComparator;Lorg/apache/hadoop/util/Progressable;Lorg/apache/hadoop/mapred/Counters$Counter;Lorg/apache/hadoop/mapred/Counters$Counter;)Lorg/apache/hadoop/mapred/RawKeyValueIterator;+28
j org.apache.hadoop.mapred.MapTask$MapOutputBuffer.mergePerPartition()V+453
j org.apache.hadoop.mapred.MapTask$MapOutputBuffer.flush()V+293
j org.apache.hadoop.mapred.MapTask.runOldMapper(Lorg/apache/hadoop/mapred/JobConf;Lorg/apache/hadoop/mapreduce/split/JobSplit$TaskSplitIndex;Lorg/apache/hadoop/mapred/TaskUmbilicalProtocol;Lorg/apache/hadoop/mapred/Task$TaskReporter;)V+216
j org.apache.hadoop.mapred.MapTask.run(Lorg/apache/hadoop/mapred/JobConf;Lorg/apache/hadoop/mapred/TaskUmbilicalProtocol;)V+114
j org.apache.hadoop.mapred.Child$4.run()Ljava/lang/Object;+29
v ~StubRoutines::call_stub
j java.security.AccessController.doPrivileged(Ljava/security/PrivilegedExceptionAction;Ljava/security/AccessControlContext;)Ljava/lang/Object;+0
j javax.security.auth.Subject.doAs(Ljavax/security/auth/Subject;Ljava/security/PrivilegedExceptionAction;)Ljava/lang/Object;+42
j org.apache.hadoop.security.UserGroupInformation.doAs(Ljava/security/PrivilegedExceptionAction;)Ljava/lang/Object;+5
j org.apache.hadoop.mapred.Child.main([Ljava/lang/String;)V+837
v ~StubRoutines::call_stub
asked
30 Apr '12, 02:53
Ashish
1●2●2●3
accept rate:
0%
I'm also getting the same on a very long job, but no cores. (cores might be disabled)
Exception in thread "main" cascading.flow.FlowException: unhandled exception ...removed...
Caused by: java.io.IOException: Error -5 at com.mapr.fs.Inode.throwIfFailed(Inode.java:158) at com.mapr.fs.Inode.getDataIntoCache(Inode.java:568) at com.mapr.fs.MapRFsInStream.dropAndRefillCurrentPage(MapRFsInStream.java:98) at com.mapr.fs.MapRFsInStream.read(MapRFsInStream.java:133) at java.io.DataInputStream.read(DataInputStream.java:83) at org.apache.hadoop.io.IOUtils.copyBytes(IOUtils.java:49) at com.hadoop.compression.lzo.LzoIndex.readIndex(LzoIndex.java:183) at com.hadoop.mapred.DeprecatedLzoTextInputFormat.listStatus(DeprecatedLzoTextInputFormat.java:91) at org.apache.hadoop.mapred.FileInputFormat.getSplits(FileInputFormat.java:236) at ...