public abstract class ExternalSorter extends Object
Modifier and Type | Class and Description |
---|---|
static class |
ExternalSorter.MapBufferTooSmallException
Exception indicating that the allocated sort buffer is insufficient to hold
the current record.
|
Modifier and Type | Field and Description |
---|---|
protected org.apache.tez.common.counters.TezCounter |
additionalSpillBytesRead |
protected org.apache.tez.common.counters.TezCounter |
additionalSpillBytesWritten |
protected int |
availableMemoryMb |
protected org.apache.hadoop.io.compress.CompressionCodec |
codec |
protected Combiner |
combiner |
protected org.apache.hadoop.io.RawComparator |
comparator |
protected org.apache.hadoop.conf.Configuration |
conf |
protected org.apache.tez.common.counters.TezCounter |
fileOutputByteCounter |
protected int |
ifileBufferSize |
protected boolean |
ifileReadAhead |
protected int |
ifileReadAheadLength |
protected Class |
keyClass |
protected org.apache.hadoop.io.serializer.Serializer |
keySerializer |
protected org.apache.tez.common.counters.TezCounter |
mapOutputByteCounter |
protected TezTaskOutput |
mapOutputFile |
protected org.apache.tez.common.counters.TezCounter |
mapOutputRecordCounter |
protected org.apache.hadoop.util.Progressable |
nullProgressable |
protected org.apache.tez.common.counters.TezCounter |
numAdditionalSpills |
protected org.apache.tez.common.counters.TezCounter |
outputBytesWithOverheadCounter |
protected org.apache.tez.runtime.api.OutputContext |
outputContext |
protected Partitioner |
partitioner |
protected int |
partitions |
protected org.apache.hadoop.fs.FileSystem |
rfs |
protected org.apache.hadoop.io.serializer.SerializationFactory |
serializationFactory |
protected org.apache.hadoop.util.IndexedSorter |
sorter |
protected org.apache.tez.common.counters.TezCounter |
spilledRecordsCounter |
protected Class |
valClass |
protected org.apache.hadoop.io.serializer.Serializer |
valSerializer |
Constructor and Description |
---|
ExternalSorter(org.apache.tez.runtime.api.OutputContext outputContext,
org.apache.hadoop.conf.Configuration conf,
int numOutputs,
long initialMemoryAvailable) |
Modifier and Type | Method and Description |
---|---|
abstract void |
close() |
abstract void |
flush() |
static long |
getInitialMemoryRequirement(org.apache.hadoop.conf.Configuration conf,
long maxAvailableTaskMemory) |
TezTaskOutput |
getMapOutput() |
ShuffleHeader |
getShuffleHeader(int reduce) |
InputStream |
getSortedStream(int partition) |
protected void |
runCombineProcessor(TezRawKeyValueIterator kvIter,
IFile.Writer writer) |
protected void |
sameVolRename(org.apache.hadoop.fs.Path srcPath,
org.apache.hadoop.fs.Path dstPath)
Rename srcPath to dstPath on the same volume.
|
void |
write(Object key,
Iterable<Object> values) |
abstract void |
write(Object key,
Object value) |
protected final org.apache.hadoop.util.Progressable nullProgressable
protected final org.apache.tez.runtime.api.OutputContext outputContext
protected final Combiner combiner
protected final Partitioner partitioner
protected final org.apache.hadoop.conf.Configuration conf
protected final org.apache.hadoop.fs.FileSystem rfs
protected final TezTaskOutput mapOutputFile
protected final int partitions
protected final Class keyClass
protected final Class valClass
protected final org.apache.hadoop.io.RawComparator comparator
protected final org.apache.hadoop.io.serializer.SerializationFactory serializationFactory
protected final org.apache.hadoop.io.serializer.Serializer keySerializer
protected final org.apache.hadoop.io.serializer.Serializer valSerializer
protected final boolean ifileReadAhead
protected final int ifileReadAheadLength
protected final int ifileBufferSize
protected final int availableMemoryMb
protected final org.apache.hadoop.util.IndexedSorter sorter
protected final org.apache.hadoop.io.compress.CompressionCodec codec
protected final org.apache.tez.common.counters.TezCounter mapOutputByteCounter
protected final org.apache.tez.common.counters.TezCounter mapOutputRecordCounter
protected final org.apache.tez.common.counters.TezCounter outputBytesWithOverheadCounter
protected final org.apache.tez.common.counters.TezCounter fileOutputByteCounter
protected final org.apache.tez.common.counters.TezCounter spilledRecordsCounter
protected final org.apache.tez.common.counters.TezCounter additionalSpillBytesWritten
protected final org.apache.tez.common.counters.TezCounter additionalSpillBytesRead
protected final org.apache.tez.common.counters.TezCounter numAdditionalSpills
public ExternalSorter(org.apache.tez.runtime.api.OutputContext outputContext, org.apache.hadoop.conf.Configuration conf, int numOutputs, long initialMemoryAvailable) throws IOException
IOException
public abstract void close() throws IOException
IOException
public abstract void flush() throws IOException
IOException
public abstract void write(Object key, Object value) throws IOException
IOException
public void write(Object key, Iterable<Object> values) throws IOException
IOException
@InterfaceAudience.Private public TezTaskOutput getMapOutput()
protected void runCombineProcessor(TezRawKeyValueIterator kvIter, IFile.Writer writer) throws IOException
IOException
protected void sameVolRename(org.apache.hadoop.fs.Path srcPath, org.apache.hadoop.fs.Path dstPath) throws IOException
IOException
public InputStream getSortedStream(int partition)
public ShuffleHeader getShuffleHeader(int reduce)
public static long getInitialMemoryRequirement(org.apache.hadoop.conf.Configuration conf, long maxAvailableTaskMemory)
Copyright © 2015 Apache Software Foundation. All rights reserved.