public class ShuffleUtils extends Object
Modifier and Type | Field and Description |
---|---|
static String |
SHUFFLE_HANDLER_SERVICE_ID |
Constructor and Description |
---|
ShuffleUtils() |
Modifier and Type | Method and Description |
---|---|
static StringBuilder |
constructBaseURIForShuffleHandler(String host,
int port,
int partition,
String appId,
boolean sslShuffle) |
static StringBuilder |
constructBaseURIForShuffleHandler(String hostIdentifier,
int partition,
String appId,
boolean sslShuffle) |
static HttpConnection.HttpConnectionParams |
constructHttpShuffleConnectionParams(org.apache.hadoop.conf.Configuration conf) |
static URL |
constructInputURL(String baseURI,
List<InputAttemptIdentifier> inputs,
boolean keepAlive) |
static ByteBuffer |
convertJobTokenToBytes(org.apache.hadoop.security.token.Token<org.apache.tez.common.security.JobTokenIdentifier> jobToken) |
static int |
deserializeShuffleProviderMetaData(ByteBuffer meta) |
static void |
generateEventsForNonStartedOutput(List<org.apache.tez.runtime.api.Event> eventList,
int numPhysicalOutputs,
org.apache.tez.runtime.api.OutputContext context,
boolean generateVmEvent,
boolean isCompositeEvent)
Generate events for outputs which have not been started.
|
static SecretKey |
getJobTokenSecretFromTokenBytes(ByteBuffer meta) |
static void |
shuffleToDisk(OutputStream output,
String hostIdentifier,
InputStream input,
long compressedLength,
org.apache.commons.logging.Log LOG,
String identifier) |
static void |
shuffleToMemory(byte[] shuffleData,
InputStream input,
int decompressedLength,
int compressedLength,
org.apache.hadoop.io.compress.CompressionCodec codec,
boolean ifileReadAhead,
int ifileReadAheadLength,
org.apache.commons.logging.Log LOG,
String identifier) |
static String |
stringify(ShuffleUserPayloads.DataMovementEventPayloadProto dmProto) |
public static String SHUFFLE_HANDLER_SERVICE_ID
public static SecretKey getJobTokenSecretFromTokenBytes(ByteBuffer meta) throws IOException
IOException
public static ByteBuffer convertJobTokenToBytes(org.apache.hadoop.security.token.Token<org.apache.tez.common.security.JobTokenIdentifier> jobToken) throws IOException
IOException
public static int deserializeShuffleProviderMetaData(ByteBuffer meta) throws IOException
IOException
public static void shuffleToMemory(byte[] shuffleData, InputStream input, int decompressedLength, int compressedLength, org.apache.hadoop.io.compress.CompressionCodec codec, boolean ifileReadAhead, int ifileReadAheadLength, org.apache.commons.logging.Log LOG, String identifier) throws IOException
IOException
public static void shuffleToDisk(OutputStream output, String hostIdentifier, InputStream input, long compressedLength, org.apache.commons.logging.Log LOG, String identifier) throws IOException
IOException
public static StringBuilder constructBaseURIForShuffleHandler(String host, int port, int partition, String appId, boolean sslShuffle)
public static StringBuilder constructBaseURIForShuffleHandler(String hostIdentifier, int partition, String appId, boolean sslShuffle)
public static URL constructInputURL(String baseURI, List<InputAttemptIdentifier> inputs, boolean keepAlive) throws MalformedURLException
MalformedURLException
public static HttpConnection.HttpConnectionParams constructHttpShuffleConnectionParams(org.apache.hadoop.conf.Configuration conf)
public static void generateEventsForNonStartedOutput(List<org.apache.tez.runtime.api.Event> eventList, int numPhysicalOutputs, org.apache.tez.runtime.api.OutputContext context, boolean generateVmEvent, boolean isCompositeEvent) throws IOException
eventList
- numPhysicalOutputs
- context
- generateVmEvent
- whether to generate a vm event or notisCompositeEvent
- whether to generate a CompositeDataMovementEvent or a DataMovementEventIOException
public static String stringify(ShuffleUserPayloads.DataMovementEventPayloadProto dmProto)
Copyright © 2015 Apache Software Foundation. All rights reserved.