1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.giraph.conf;
19
20 import org.apache.giraph.aggregators.AggregatorWriter;
21 import org.apache.giraph.aggregators.TextAggregatorWriter;
22 import org.apache.giraph.bsp.BspOutputFormat;
23 import org.apache.giraph.bsp.checkpoints.CheckpointSupportedChecker;
24 import org.apache.giraph.bsp.checkpoints.DefaultCheckpointSupportedChecker;
25 import org.apache.giraph.combiner.MessageCombiner;
26 import org.apache.giraph.comm.messages.InMemoryMessageStoreFactory;
27 import org.apache.giraph.comm.messages.MessageEncodeAndStoreType;
28 import org.apache.giraph.comm.messages.MessageStoreFactory;
29 import org.apache.giraph.edge.ByteArrayEdges;
30 import org.apache.giraph.edge.DefaultCreateSourceVertexCallback;
31 import org.apache.giraph.edge.CreateSourceVertexCallback;
32 import org.apache.giraph.edge.EdgeStoreFactory;
33 import org.apache.giraph.edge.InMemoryEdgeStoreFactory;
34 import org.apache.giraph.edge.OutEdges;
35 import org.apache.giraph.factories.ComputationFactory;
36 import org.apache.giraph.factories.DefaultComputationFactory;
37 import org.apache.giraph.factories.DefaultEdgeValueFactory;
38 import org.apache.giraph.factories.DefaultInputOutEdgesFactory;
39 import org.apache.giraph.factories.DefaultMessageValueFactory;
40 import org.apache.giraph.factories.DefaultOutEdgesFactory;
41 import org.apache.giraph.factories.DefaultVertexIdFactory;
42 import org.apache.giraph.factories.DefaultVertexValueFactory;
43 import org.apache.giraph.factories.EdgeValueFactory;
44 import org.apache.giraph.factories.MessageValueFactory;
45 import org.apache.giraph.factories.OutEdgesFactory;
46 import org.apache.giraph.factories.VertexIdFactory;
47 import org.apache.giraph.factories.VertexValueFactory;
48 import org.apache.giraph.graph.Computation;
49 import org.apache.giraph.graph.DefaultVertex;
50 import org.apache.giraph.graph.DefaultVertexResolver;
51 import org.apache.giraph.graph.DefaultVertexValueCombiner;
52 import org.apache.giraph.graph.JobProgressTrackerClient;
53 import org.apache.giraph.graph.Language;
54 import org.apache.giraph.graph.MapperObserver;
55 import org.apache.giraph.graph.RetryableJobProgressTrackerClient;
56 import org.apache.giraph.graph.Vertex;
57 import org.apache.giraph.graph.VertexResolver;
58 import org.apache.giraph.graph.VertexValueCombiner;
59 import org.apache.giraph.io.EdgeInputFormat;
60 import org.apache.giraph.io.EdgeOutputFormat;
61 import org.apache.giraph.io.MappingInputFormat;
62 import org.apache.giraph.io.VertexInputFormat;
63 import org.apache.giraph.io.VertexOutputFormat;
64 import org.apache.giraph.io.filters.DefaultEdgeInputFilter;
65 import org.apache.giraph.io.filters.DefaultVertexInputFilter;
66 import org.apache.giraph.io.filters.EdgeInputFilter;
67 import org.apache.giraph.io.filters.VertexInputFilter;
68 import org.apache.giraph.job.DefaultGiraphJobRetryChecker;
69 import org.apache.giraph.job.DefaultJobObserver;
70 import org.apache.giraph.job.DefaultJobProgressTrackerService;
71 import org.apache.giraph.job.GiraphJobObserver;
72 import org.apache.giraph.job.GiraphJobRetryChecker;
73 import org.apache.giraph.job.HaltApplicationUtils;
74 import org.apache.giraph.job.JobProgressTrackerService;
75 import org.apache.giraph.mapping.MappingStore;
76 import org.apache.giraph.mapping.MappingStoreOps;
77 import org.apache.giraph.mapping.translate.TranslateEdge;
78 import org.apache.giraph.master.DefaultMasterCompute;
79 import org.apache.giraph.master.MasterCompute;
80 import org.apache.giraph.master.MasterObserver;
81 import org.apache.giraph.ooc.persistence.OutOfCoreDataAccessor;
82 import org.apache.giraph.ooc.persistence.LocalDiskDataAccessor;
83 import org.apache.giraph.ooc.policy.MemoryEstimatorOracle;
84 import org.apache.giraph.ooc.policy.OutOfCoreOracle;
85 import org.apache.giraph.partition.GraphPartitionerFactory;
86 import org.apache.giraph.partition.HashPartitionerFactory;
87 import org.apache.giraph.partition.Partition;
88 import org.apache.giraph.partition.SimplePartition;
89 import org.apache.giraph.utils.GcObserver;
90 import org.apache.giraph.worker.DefaultWorkerContext;
91 import org.apache.giraph.worker.WorkerContext;
92 import org.apache.giraph.worker.WorkerObserver;
93 import org.apache.hadoop.io.Writable;
94 import org.apache.hadoop.io.WritableComparable;
95 import org.apache.hadoop.mapreduce.OutputFormat;
96
97 import static java.util.concurrent.TimeUnit.HOURS;
98 import static java.util.concurrent.TimeUnit.MINUTES;
99 import static java.util.concurrent.TimeUnit.SECONDS;
100
101
102
103
104
105 public interface GiraphConstants {
106
107 int ONE_KB = 1024;
108
109 int ONE_MB = 1024 * 1024;
110
111
112 ClassConfOption<MappingStore> MAPPING_STORE_CLASS =
113 ClassConfOption.create("giraph.mappingStoreClass", null,
114 MappingStore.class, "MappingStore Class");
115
116
117 ClassConfOption<MappingStoreOps> MAPPING_STORE_OPS_CLASS =
118 ClassConfOption.create("giraph.mappingStoreOpsClass", null,
119 MappingStoreOps.class, "MappingStoreOps class");
120
121
122 IntConfOption LB_MAPPINGSTORE_UPPER =
123 new IntConfOption("giraph.lbMappingStoreUpper", -1,
124 "'upper' value used by lbmappingstore");
125
126 IntConfOption LB_MAPPINGSTORE_LOWER =
127 new IntConfOption("giraph.lbMappingStoreLower", -1,
128 "'lower' value used by lbMappingstore");
129
130 ClassConfOption EDGE_TRANSLATION_CLASS =
131 ClassConfOption.create("giraph.edgeTranslationClass", null,
132 TranslateEdge.class, "Class used to conduct expensive edge " +
133 "translation during vertex input phase");
134
135
136 ClassConfOption<Computation> COMPUTATION_CLASS =
137 ClassConfOption.create("giraph.computationClass", null,
138 Computation.class, "Computation class - required");
139
140 ClassConfOption<ComputationFactory> COMPUTATION_FACTORY_CLASS =
141 ClassConfOption.create("giraph.computation.factory.class",
142 DefaultComputationFactory.class, ComputationFactory.class,
143 "Computation factory class - optional");
144
145
146 ClassConfOption<TypesHolder> TYPES_HOLDER_CLASS =
147 ClassConfOption.create("giraph.typesHolder", null,
148 TypesHolder.class,
149 "TypesHolder, used if Computation not set - optional");
150
151
152 ClassConfOption<EdgeStoreFactory> EDGE_STORE_FACTORY_CLASS =
153 ClassConfOption.create("giraph.edgeStoreFactoryClass",
154 InMemoryEdgeStoreFactory.class,
155 EdgeStoreFactory.class,
156 "Edge Store Factory class to use for creating edgeStore");
157
158
159 ClassConfOption<MessageStoreFactory> MESSAGE_STORE_FACTORY_CLASS =
160 ClassConfOption.create("giraph.messageStoreFactoryClass",
161 InMemoryMessageStoreFactory.class,
162 MessageStoreFactory.class,
163 "Message Store Factory Class that is to be used");
164
165
166 PerGraphTypeEnumConfOption<Language> GRAPH_TYPE_LANGUAGES =
167 PerGraphTypeEnumConfOption.create("giraph.types.language",
168 Language.class, Language.JAVA,
169 "Language user graph types (IVEMM) are implemented in");
170
171
172 PerGraphTypeBooleanConfOption GRAPH_TYPES_NEEDS_WRAPPERS =
173 new PerGraphTypeBooleanConfOption("giraph.jython.type.wrappers",
174 false, "Whether user graph types (IVEMM) need Jython wrappers");
175
176
177 ClassConfOption<VertexIdFactory> VERTEX_ID_FACTORY_CLASS =
178 ClassConfOption.create("giraph.vertexIdFactoryClass",
179 DefaultVertexIdFactory.class, VertexIdFactory.class,
180 "Vertex ID factory class - optional");
181
182 ClassConfOption<VertexValueFactory> VERTEX_VALUE_FACTORY_CLASS =
183 ClassConfOption.create("giraph.vertexValueFactoryClass",
184 DefaultVertexValueFactory.class, VertexValueFactory.class,
185 "Vertex value factory class - optional");
186
187 ClassConfOption<EdgeValueFactory> EDGE_VALUE_FACTORY_CLASS =
188 ClassConfOption.create("giraph.edgeValueFactoryClass",
189 DefaultEdgeValueFactory.class, EdgeValueFactory.class,
190 "Edge value factory class - optional");
191
192 ClassConfOption<MessageValueFactory>
193 OUTGOING_MESSAGE_VALUE_FACTORY_CLASS =
194 ClassConfOption.create("giraph.outgoingMessageValueFactoryClass",
195 DefaultMessageValueFactory.class, MessageValueFactory.class,
196 "Outgoing message value factory class - optional");
197
198
199 ClassConfOption<OutEdges> VERTEX_EDGES_CLASS =
200 ClassConfOption.create("giraph.outEdgesClass", ByteArrayEdges.class,
201 OutEdges.class, "Vertex edges class - optional");
202
203 ClassConfOption<OutEdges> INPUT_VERTEX_EDGES_CLASS =
204 ClassConfOption.create("giraph.inputOutEdgesClass",
205 ByteArrayEdges.class, OutEdges.class,
206 "Vertex edges class to be used during edge input only - optional");
207
208 ClassConfOption<OutEdgesFactory> VERTEX_EDGES_FACTORY_CLASS =
209 ClassConfOption.create("giraph.outEdgesFactoryClass",
210 DefaultOutEdgesFactory.class, OutEdgesFactory.class,
211 "OutEdges factory class - optional");
212
213 ClassConfOption<OutEdgesFactory> INPUT_VERTEX_EDGES_FACTORY_CLASS =
214 ClassConfOption.create("giraph.inputOutEdgesFactoryClass",
215 DefaultInputOutEdgesFactory.class, OutEdgesFactory.class,
216 "OutEdges for input factory class - optional");
217
218
219 ClassConfOption<MasterCompute> MASTER_COMPUTE_CLASS =
220 ClassConfOption.create("giraph.masterComputeClass",
221 DefaultMasterCompute.class, MasterCompute.class,
222 "Class for Master - optional");
223
224 ClassConfOption<MasterObserver> MASTER_OBSERVER_CLASSES =
225 ClassConfOption.create("giraph.master.observers",
226 null, MasterObserver.class, "Classes for Master Observer - optional");
227
228 ClassConfOption<WorkerObserver> WORKER_OBSERVER_CLASSES =
229 ClassConfOption.create("giraph.worker.observers", null,
230 WorkerObserver.class, "Classes for Worker Observer - optional");
231
232 ClassConfOption<MapperObserver> MAPPER_OBSERVER_CLASSES =
233 ClassConfOption.create("giraph.mapper.observers", null,
234 MapperObserver.class, "Classes for Mapper Observer - optional");
235
236 ClassConfOption<GcObserver> GC_OBSERVER_CLASSES =
237 ClassConfOption.create("giraph.gc.observers", null,
238 GcObserver.class, "Classes for GC oObserver - optional");
239
240 ClassConfOption<MessageCombiner> MESSAGE_COMBINER_CLASS =
241 ClassConfOption.create("giraph.messageCombinerClass", null,
242 MessageCombiner.class, "Message combiner class - optional");
243
244 ClassConfOption<VertexResolver> VERTEX_RESOLVER_CLASS =
245 ClassConfOption.create("giraph.vertexResolverClass",
246 DefaultVertexResolver.class, VertexResolver.class,
247 "Vertex resolver class - optional");
248
249 ClassConfOption<VertexValueCombiner> VERTEX_VALUE_COMBINER_CLASS =
250 ClassConfOption.create("giraph.vertexValueCombinerClass",
251 DefaultVertexValueCombiner.class, VertexValueCombiner.class,
252 "Vertex value combiner class - optional");
253
254
255 EnumConfOption<Language> COMPUTATION_LANGUAGE =
256 EnumConfOption.create("giraph.computation.language",
257 Language.class, Language.JAVA,
258 "Which language computation is implemented in");
259
260
261
262
263
264 BooleanConfOption RESOLVER_CREATE_VERTEX_ON_MSGS =
265 new BooleanConfOption("giraph.vertex.resolver.create.on.msgs", true,
266 "Option of whether to create vertexes that were not existent " +
267 "before but received messages");
268
269 ClassConfOption<GraphPartitionerFactory> GRAPH_PARTITIONER_FACTORY_CLASS =
270 ClassConfOption.create("giraph.graphPartitionerFactoryClass",
271 HashPartitionerFactory.class, GraphPartitionerFactory.class,
272 "Graph partitioner factory class - optional");
273
274
275 ClassConfOption<GiraphJobObserver> JOB_OBSERVER_CLASS =
276 ClassConfOption.create("giraph.jobObserverClass",
277 DefaultJobObserver.class, GiraphJobObserver.class,
278 "Observer class to watch over job status - optional");
279
280
281 ClassConfOption<GiraphJobRetryChecker> JOB_RETRY_CHECKER_CLASS =
282 ClassConfOption.create("giraph.jobRetryCheckerClass",
283 DefaultGiraphJobRetryChecker.class, GiraphJobRetryChecker.class,
284 "Class which decides whether a failed job should be retried - " +
285 "optional");
286
287
288
289
290
291 LongConfOption MAX_ALLOWED_JOB_TIME_MS =
292 new LongConfOption("giraph.maxAllowedJobTimeMilliseconds", -1,
293 "Maximum allowed time for job to run after getting all resources " +
294 "before it will be killed, in milliseconds " +
295 "(-1 if it has no limit)");
296
297
298
299 ClassConfOption<VertexInputFormat> VERTEX_INPUT_FORMAT_CLASS =
300 ClassConfOption.create("giraph.vertexInputFormatClass", null,
301 VertexInputFormat.class, "VertexInputFormat class (at least " +
302 "one of the input format classes is required)");
303
304 ClassConfOption<EdgeInputFormat> EDGE_INPUT_FORMAT_CLASS =
305 ClassConfOption.create("giraph.edgeInputFormatClass", null,
306 EdgeInputFormat.class, "EdgeInputFormat class");
307
308 ClassConfOption<MappingInputFormat> MAPPING_INPUT_FORMAT_CLASS =
309 ClassConfOption.create("giraph.mappingInputFormatClass", null,
310 MappingInputFormat.class, "MappingInputFormat class");
311
312
313 ClassConfOption<EdgeInputFilter> EDGE_INPUT_FILTER_CLASS =
314 ClassConfOption.create("giraph.edgeInputFilterClass",
315 DefaultEdgeInputFilter.class, EdgeInputFilter.class,
316 "EdgeInputFilter class");
317
318 ClassConfOption<VertexInputFilter> VERTEX_INPUT_FILTER_CLASS =
319 ClassConfOption.create("giraph.vertexInputFilterClass",
320 DefaultVertexInputFilter.class, VertexInputFilter.class,
321 "VertexInputFilter class");
322
323 ClassConfOption<Vertex> VERTEX_CLASS =
324 ClassConfOption.create("giraph.vertexClass",
325 DefaultVertex.class, Vertex.class,
326 "Vertex class");
327
328 ClassConfOption<VertexOutputFormat> VERTEX_OUTPUT_FORMAT_CLASS =
329 ClassConfOption.create("giraph.vertexOutputFormatClass", null,
330 VertexOutputFormat.class, "VertexOutputFormat class");
331
332 StrConfOption VERTEX_OUTPUT_FORMAT_SUBDIR =
333 new StrConfOption("giraph.vertex.output.subdir", "",
334 "VertexOutputFormat sub-directory");
335
336 ClassConfOption<EdgeOutputFormat> EDGE_OUTPUT_FORMAT_CLASS =
337 ClassConfOption.create("giraph.edgeOutputFormatClass", null,
338 EdgeOutputFormat.class, "EdgeOutputFormat class");
339
340 StrConfOption EDGE_OUTPUT_FORMAT_SUBDIR =
341 new StrConfOption("giraph.edge.output.subdir", "",
342 "EdgeOutputFormat sub-directory");
343
344
345 StrConfOption GIRAPH_TEXT_OUTPUT_FORMAT_SEPARATOR =
346 new StrConfOption("giraph.textoutputformat.separator", "\t",
347 "GiraphTextOuputFormat Separator");
348
349 BooleanConfOption GIRAPH_TEXT_OUTPUT_FORMAT_REVERSE =
350 new BooleanConfOption("giraph.textoutputformat.reverse", false,
351 "Reverse values in the output");
352
353
354
355
356
357
358
359
360 BooleanConfOption DO_OUTPUT_DURING_COMPUTATION =
361 new BooleanConfOption("giraph.doOutputDuringComputation", false,
362 "If you use this option, instead of having saving vertices in the " +
363 "end of application, saveVertex will be called right after each " +
364 "vertex.compute() is called." +
365 "NOTE: This feature doesn't work well with checkpointing - if you " +
366 "restart from a checkpoint you won't have any ouptut from previous " +
367 "supresteps.");
368
369
370
371
372
373 BooleanConfOption VERTEX_OUTPUT_FORMAT_THREAD_SAFE =
374 new BooleanConfOption("giraph.vertexOutputFormatThreadSafe", false,
375 "Vertex output format thread-safe - if your VertexOutputFormat " +
376 "allows several vertexWriters to be created and written to in " +
377 "parallel, you should set this to true.");
378
379 IntConfOption NUM_OUTPUT_THREADS =
380 new IntConfOption("giraph.numOutputThreads", 1,
381 "Number of threads for writing output in the end of the application");
382
383
384 StrConfOption GIRAPH_YARN_LIBJARS =
385 new StrConfOption("giraph.yarn.libjars", "",
386 "conf key for comma-separated list of jars to export to YARN workers");
387
388 String GIRAPH_YARN_CONF_FILE = "giraph-conf.xml";
389
390 int GIRAPH_YARN_TASK_HEAP_MB_DEFAULT = 1024;
391
392 IntConfOption GIRAPH_YARN_TASK_HEAP_MB = new IntConfOption(
393 "giraph.yarn.task.heap.mb", GIRAPH_YARN_TASK_HEAP_MB_DEFAULT,
394 "Name of Giraph property for user-configurable heap memory per worker");
395
396 int GIRAPH_YARN_PRIORITY = 10;
397
398 BooleanConfOption IS_PURE_YARN_JOB =
399 new BooleanConfOption("giraph.pure.yarn.job", false,
400 "Is this a pure YARN job (i.e. no MapReduce layer managing Giraph " +
401 "tasks)");
402
403
404 ClassConfOption<WritableComparable> VERTEX_ID_CLASS =
405 ClassConfOption.create("giraph.vertexIdClass", null,
406 WritableComparable.class, "Vertex index class");
407
408 ClassConfOption<Writable> VERTEX_VALUE_CLASS =
409 ClassConfOption.create("giraph.vertexValueClass", null, Writable.class,
410 "Vertex value class");
411
412 ClassConfOption<Writable> EDGE_VALUE_CLASS =
413 ClassConfOption.create("giraph.edgeValueClass", null, Writable.class,
414 "Edge value class");
415
416 ClassConfOption<Writable> OUTGOING_MESSAGE_VALUE_CLASS =
417 ClassConfOption.create("giraph.outgoingMessageValueClass", null,
418 Writable.class, "Outgoing message value class");
419
420 ClassConfOption<WorkerContext> WORKER_CONTEXT_CLASS =
421 ClassConfOption.create("giraph.workerContextClass",
422 DefaultWorkerContext.class, WorkerContext.class,
423 "Worker contextclass");
424
425 ClassConfOption<AggregatorWriter> AGGREGATOR_WRITER_CLASS =
426 ClassConfOption.create("giraph.aggregatorWriterClass",
427 TextAggregatorWriter.class, AggregatorWriter.class,
428 "AggregatorWriter class - optional");
429
430
431 ClassConfOption<Partition> PARTITION_CLASS =
432 ClassConfOption.create("giraph.partitionClass", SimplePartition.class,
433 Partition.class, "Partition class - optional");
434
435
436
437
438 String MIN_WORKERS = "giraph.minWorkers";
439
440
441
442 String MAX_WORKERS = "giraph.maxWorkers";
443
444
445
446
447
448 BooleanConfOption SPLIT_MASTER_WORKER =
449 new BooleanConfOption("giraph.SplitMasterWorker", true,
450 "Separate the workers and the master tasks. This is required to " +
451 "support dynamic recovery. (boolean)");
452
453
454 BooleanConfOption LOCAL_TEST_MODE =
455 new BooleanConfOption("giraph.localTestMode", false,
456 "Indicates whether this job is run in an internal unit test");
457
458
459 StrConfOption LOG_LEVEL = new StrConfOption("giraph.logLevel", "info",
460 "Override the Hadoop log level and set the desired log level.");
461
462
463 BooleanConfOption LOG_THREAD_LAYOUT =
464 new BooleanConfOption("giraph.logThreadLayout", false,
465 "Use thread level debugging?");
466
467
468 BooleanConfOption JMAP_ENABLE =
469 new BooleanConfOption("giraph.jmap.histo.enable", false,
470 "Configuration key to enable jmap printing");
471
472
473 IntConfOption JMAP_SLEEP_MILLIS =
474 new IntConfOption("giraph.jmap.histo.msec", SECONDS.toMillis(30),
475 "Configuration key for msec to sleep between calls");
476
477
478 IntConfOption JMAP_PRINT_LINES =
479 new IntConfOption("giraph.jmap.histo.print_lines", 30,
480 "Configuration key for how many lines to print");
481
482
483
484
485
486
487 BooleanConfOption JMAP_LIVE_ONLY =
488 new BooleanConfOption("giraph.jmap.histo.live", false,
489 "Only print live objects in jmap?");
490
491
492
493
494
495 IntConfOption MIN_FREE_MBS_ON_HEAP =
496 new IntConfOption("giraph.heap.minFreeMb", 128, "Option used by " +
497 "worker and master observers to check for imminent OOM exception");
498
499
500
501
502 BooleanConfOption REACTIVE_JMAP_ENABLE =
503 new BooleanConfOption("giraph.heap.enableReactiveJmapDumping", false,
504 "Option to enable dumping jmap histogram reactively based on " +
505 "free memory on heap");
506
507
508
509
510
511 FloatConfOption MIN_PERCENT_RESPONDED =
512 new FloatConfOption("giraph.minPercentResponded", 100.0f,
513 "Minimum percent of the maximum number of workers that have " +
514 "responded in order to continue progressing. (float)");
515
516
517 BooleanConfOption METRICS_ENABLE =
518 new BooleanConfOption("giraph.metrics.enable", false,
519 "Enable the Metrics system");
520
521
522 StrConfOption METRICS_DIRECTORY =
523 new StrConfOption("giraph.metrics.directory", "",
524 "Directory in HDFS to write master metrics to, instead of stderr");
525
526
527
528
529
530
531
532 StrConfOption ZOOKEEPER_LIST =
533 new StrConfOption("giraph.zkList", "",
534 "ZooKeeper comma-separated list (if not set, will start up " +
535 "ZooKeeper locally). Consider that after locally-starting " +
536 "zookeeper, this parameter will updated the configuration with " +
537 "the corrent configuration value.");
538
539
540
541
542
543
544 BooleanConfOption ZOOKEEPER_IS_EXTERNAL =
545 new BooleanConfOption("giraph.zkIsExternal", true,
546 "Zookeeper List will always hold a value during " +
547 "the computation while this option provides " +
548 "information regarding whether the zookeeper was " +
549 "internally started or externally provided.");
550
551
552 IntConfOption ZOOKEEPER_SESSION_TIMEOUT =
553 new IntConfOption("giraph.zkSessionMsecTimeout", MINUTES.toMillis(1),
554 "ZooKeeper session millisecond timeout");
555
556
557 IntConfOption ZOOKEEPER_SERVERLIST_POLL_MSECS =
558 new IntConfOption("giraph.zkServerlistPollMsecs", SECONDS.toMillis(3),
559 "Polling interval to check for the ZooKeeper server data");
560
561
562 IntConfOption ZOOKEEPER_SERVER_PORT =
563 new IntConfOption("giraph.zkServerPort", 22181, "ZooKeeper port to use");
564
565
566 String ZOOKEEPER_DIR = "giraph.zkDir";
567
568
569 IntConfOption ZOOKEEPER_OPS_MAX_ATTEMPTS =
570 new IntConfOption("giraph.zkOpsMaxAttempts", 3,
571 "Max attempts for handling ZooKeeper connection loss");
572
573
574
575
576 IntConfOption ZOOKEEPER_OPS_RETRY_WAIT_MSECS =
577 new IntConfOption("giraph.zkOpsRetryWaitMsecs", SECONDS.toMillis(5),
578 "Msecs to wait before retrying a failed ZooKeeper op due to " +
579 "connection loss.");
580
581
582 IntConfOption TCP_BACKLOG = new IntConfOption("giraph.tcpBacklog", 1,
583 "TCP backlog (defaults to number of workers)");
584
585
586 BooleanConfOption NETTY_USE_POOLED_ALLOCATOR = new BooleanConfOption(
587 "giraph.useNettyPooledAllocator", false, "Should netty use pooled " +
588 "memory allocator?");
589
590
591 BooleanConfOption NETTY_USE_DIRECT_MEMORY = new BooleanConfOption(
592 "giraph.useNettyDirectMemory", false, "Should netty use direct " +
593 "memory buffers");
594
595
596 IntConfOption NETTY_REQUEST_ENCODER_BUFFER_SIZE =
597 new IntConfOption("giraph.nettyRequestEncoderBufferSize", 32 * ONE_KB,
598 "How big to make the encoder buffer?");
599
600
601 IntConfOption NETTY_CLIENT_THREADS =
602 new IntConfOption("giraph.nettyClientThreads", 4, "Netty client threads");
603
604
605 IntConfOption NETTY_SERVER_THREADS =
606 new IntConfOption("giraph.nettyServerThreads", 16,
607 "Netty server threads");
608
609
610 BooleanConfOption NETTY_CLIENT_USE_EXECUTION_HANDLER =
611 new BooleanConfOption("giraph.nettyClientUseExecutionHandler", true,
612 "Use the execution handler in netty on the client?");
613
614
615 IntConfOption NETTY_CLIENT_EXECUTION_THREADS =
616 new IntConfOption("giraph.nettyClientExecutionThreads", 8,
617 "Netty client execution threads (execution handler)");
618
619
620 StrConfOption NETTY_CLIENT_EXECUTION_AFTER_HANDLER =
621 new StrConfOption("giraph.nettyClientExecutionAfterHandler",
622 "request-encoder",
623 "Where to place the netty client execution handle?");
624
625
626 BooleanConfOption NETTY_SERVER_USE_EXECUTION_HANDLER =
627 new BooleanConfOption("giraph.nettyServerUseExecutionHandler", true,
628 "Use the execution handler in netty on the server?");
629
630
631 IntConfOption NETTY_SERVER_EXECUTION_THREADS =
632 new IntConfOption("giraph.nettyServerExecutionThreads", 8,
633 "Netty server execution threads (execution handler)");
634
635
636 StrConfOption NETTY_SERVER_EXECUTION_AFTER_HANDLER =
637 new StrConfOption("giraph.nettyServerExecutionAfterHandler",
638 "requestFrameDecoder",
639 "Where to place the netty server execution handle?");
640
641
642 BooleanConfOption NETTY_SIMULATE_FIRST_REQUEST_CLOSED =
643 new BooleanConfOption("giraph.nettySimulateFirstRequestClosed", false,
644 "Netty simulate a first request closed");
645
646
647 BooleanConfOption NETTY_SIMULATE_FIRST_RESPONSE_FAILED =
648 new BooleanConfOption("giraph.nettySimulateFirstResponseFailed", false,
649 "Netty simulate a first response failed");
650
651
652 StrConfOption NETTY_COMPRESSION_ALGORITHM =
653 new StrConfOption("giraph.nettyCompressionAlgorithm", "",
654 "Which compression algorithm to use in netty");
655
656
657 IntConfOption MAX_RESOLVE_ADDRESS_ATTEMPTS =
658 new IntConfOption("giraph.maxResolveAddressAttempts", 5,
659 "Max resolve address attempts");
660
661
662 IntConfOption WAITING_REQUEST_MSECS =
663 new IntConfOption("giraph.waitingRequestMsecs", SECONDS.toMillis(15),
664 "Msecs to wait between waiting for all requests to finish");
665
666
667 IntConfOption EVENT_WAIT_MSECS =
668 new IntConfOption("giraph.eventWaitMsecs", SECONDS.toMillis(30),
669 "Millseconds to wait for an event before continuing");
670
671
672
673
674
675 IntConfOption MAX_MASTER_SUPERSTEP_WAIT_MSECS =
676 new IntConfOption("giraph.maxMasterSuperstepWaitMsecs",
677 MINUTES.toMillis(10),
678 "Maximum milliseconds to wait before giving up trying to get the " +
679 "minimum number of workers before a superstep (int).");
680
681
682
683
684
685 IntConfOption MAX_COUNTER_WAIT_MSECS = new IntConfOption(
686 "giraph.maxCounterWaitMsecs", MINUTES.toMillis(2),
687 "Maximum milliseconds to wait before giving up waiting for" +
688 "the workers to write their counters to the " +
689 "zookeeper after a superstep");
690
691
692 IntConfOption MAX_REQUEST_MILLISECONDS =
693 new IntConfOption("giraph.maxRequestMilliseconds", MINUTES.toMillis(10),
694 "Milliseconds for a request to complete (or else resend)");
695
696
697
698
699
700 BooleanConfOption RESEND_TIMED_OUT_REQUESTS =
701 new BooleanConfOption("giraph.resendTimedOutRequests", true,
702 "Whether to resend request which timed out or fail the job if " +
703 "timeout happens");
704
705
706 IntConfOption NETTY_MAX_CONNECTION_FAILURES =
707 new IntConfOption("giraph.nettyMaxConnectionFailures", 1000,
708 "Netty max connection failures");
709
710
711 IntConfOption WAIT_TIME_BETWEEN_CONNECTION_RETRIES_MS =
712 new IntConfOption("giraph.waitTimeBetweenConnectionRetriesMs", 500,
713 "");
714
715
716 IntConfOption IPC_INITIAL_PORT =
717 new IntConfOption("giraph.ipcInitialPort", 30000,
718 "Initial port to start using for the IPC communication");
719
720
721 IntConfOption MAX_IPC_PORT_BIND_ATTEMPTS =
722 new IntConfOption("giraph.maxIpcPortBindAttempts", 20,
723 "Maximum bind attempts for different IPC ports");
724
725
726
727
728 BooleanConfOption FAIL_FIRST_IPC_PORT_BIND_ATTEMPT =
729 new BooleanConfOption("giraph.failFirstIpcPortBindAttempt", false,
730 "Fail first IPC port binding attempt, simulate binding failure " +
731 "on real grid testing");
732
733
734 IntConfOption CLIENT_SEND_BUFFER_SIZE =
735 new IntConfOption("giraph.clientSendBufferSize", 512 * ONE_KB,
736 "Client send buffer size");
737
738
739 IntConfOption CLIENT_RECEIVE_BUFFER_SIZE =
740 new IntConfOption("giraph.clientReceiveBufferSize", 32 * ONE_KB,
741 "Client receive buffer size");
742
743
744 IntConfOption SERVER_SEND_BUFFER_SIZE =
745 new IntConfOption("giraph.serverSendBufferSize", 32 * ONE_KB,
746 "Server send buffer size");
747
748
749 IntConfOption SERVER_RECEIVE_BUFFER_SIZE =
750 new IntConfOption("giraph.serverReceiveBufferSize", 512 * ONE_KB,
751 "Server receive buffer size");
752
753
754 IntConfOption MAX_MSG_REQUEST_SIZE =
755 new IntConfOption("giraph.msgRequestSize", 512 * ONE_KB,
756 "Maximum size of messages (in bytes) per peer before flush");
757
758
759
760
761
762
763
764
765 FloatConfOption ADDITIONAL_MSG_REQUEST_SIZE =
766 new FloatConfOption("giraph.additionalMsgRequestSize", 0.2f,
767 "How much bigger than the average per partition size to make " +
768 "initial per partition buffers. If this value is A, message " +
769 "request size is M, and a worker has P partitions, than its " +
770 "initial partition buffer size will be (M / P) * (1 + A).");
771
772
773
774 FloatConfOption REQUEST_SIZE_WARNING_THRESHOLD = new FloatConfOption(
775 "giraph.msgRequestWarningThreshold", 2.0f,
776 "If request sizes are bigger than the buffer size by this factor " +
777 "warnings are printed to the log and to the command line");
778
779
780 IntConfOption MAX_VERTEX_REQUEST_SIZE =
781 new IntConfOption("giraph.vertexRequestSize", 512 * ONE_KB,
782 "Maximum size of vertices (in bytes) per peer before flush");
783
784
785
786
787
788 FloatConfOption ADDITIONAL_VERTEX_REQUEST_SIZE =
789 new FloatConfOption("giraph.additionalVertexRequestSize", 0.2f,
790 "Additional size (expressed as a ratio) of each per-partition " +
791 "buffer on top of the average size.");
792
793
794 IntConfOption MAX_EDGE_REQUEST_SIZE =
795 new IntConfOption("giraph.edgeRequestSize", 512 * ONE_KB,
796 "Maximum size of edges (in bytes) per peer before flush");
797
798
799
800
801
802 FloatConfOption ADDITIONAL_EDGE_REQUEST_SIZE =
803 new FloatConfOption("giraph.additionalEdgeRequestSize", 0.2f,
804 "Additional size (expressed as a ratio) of each per-partition " +
805 "buffer on top of the average size.");
806
807
808 IntConfOption MAX_MUTATIONS_PER_REQUEST =
809 new IntConfOption("giraph.maxMutationsPerRequest", 100,
810 "Maximum number of mutations per partition before flush");
811
812
813
814
815
816 BooleanConfOption USE_MESSAGE_SIZE_ENCODING =
817 new BooleanConfOption("giraph.useMessageSizeEncoding", false,
818 "Use message size encoding (typically better for complex objects, " +
819 "not meant for primitive wrapped messages)");
820
821
822 IntConfOption CHANNELS_PER_SERVER =
823 new IntConfOption("giraph.channelsPerServer", 1,
824 "Number of channels used per server");
825
826
827 String MSG_NUM_FLUSH_THREADS = "giraph.msgNumFlushThreads";
828
829
830 IntConfOption NUM_COMPUTE_THREADS =
831 new IntConfOption("giraph.numComputeThreads", 1,
832 "Number of threads for vertex computation");
833
834
835 IntConfOption NUM_INPUT_THREADS =
836 new IntConfOption("giraph.numInputThreads", 1,
837 "Number of threads for input split loading");
838
839
840 IntConfOption PARTITION_LONG_TAIL_MIN_PRINT =
841 new IntConfOption("giraph.partitionLongTailMinPrint", 1,
842 "Minimum stragglers of the superstep before printing them out");
843
844
845 BooleanConfOption USE_SUPERSTEP_COUNTERS =
846 new BooleanConfOption("giraph.useSuperstepCounters", true,
847 "Use superstep counters? (boolean)");
848
849
850
851
852
853
854
855 FloatConfOption INPUT_SPLIT_SAMPLE_PERCENT =
856 new FloatConfOption("giraph.inputSplitSamplePercent", 100f,
857 "Input split sample percent - Used only for sampling and testing, " +
858 "rather than an actual job. The idea is that to test, you might " +
859 "only want a fraction of the actual input splits from your " +
860 "VertexInputFormat to load (values should be [0, 100]).");
861
862
863
864
865
866
867 LongConfOption INPUT_SPLIT_MAX_VERTICES =
868 new LongConfOption("giraph.InputSplitMaxVertices", -1,
869 "To limit outlier vertex input splits from producing too many " +
870 "vertices or to help with testing, the number of vertices " +
871 "loaded from an input split can be limited. By default, " +
872 "everything is loaded.");
873
874
875
876
877
878
879 LongConfOption INPUT_SPLIT_MAX_EDGES =
880 new LongConfOption("giraph.InputSplitMaxEdges", -1,
881 "To limit outlier vertex input splits from producing too many " +
882 "vertices or to help with testing, the number of edges loaded " +
883 "from an input split can be limited. By default, everything is " +
884 "loaded.");
885
886
887
888
889
890
891
892
893 BooleanConfOption USE_INPUT_SPLIT_LOCALITY =
894 new BooleanConfOption("giraph.useInputSplitLocality", true,
895 "To minimize network usage when reading input splits, each worker " +
896 "can prioritize splits that reside on its host. " +
897 "This, however, comes at the cost of increased load on ZooKeeper. " +
898 "Hence, users with a lot of splits and input threads (or with " +
899 "configurations that can't exploit locality) may want to disable " +
900 "it.");
901
902
903 FloatConfOption PARTITION_COUNT_MULTIPLIER =
904 new FloatConfOption("giraph.masterPartitionCountMultiplier", 1.0f,
905 "Multiplier for the current workers squared");
906
907
908 IntConfOption MIN_PARTITIONS_PER_COMPUTE_THREAD =
909 new IntConfOption("giraph.minPartitionsPerComputeThread", 1,
910 "Minimum number of partitions to have per compute thread");
911
912
913 IntConfOption USER_PARTITION_COUNT =
914 new IntConfOption("giraph.userPartitionCount", -1,
915 "Overrides default partition count calculation if not -1");
916
917
918
919
920 String PARTITION_VERTEX_KEY_SPACE_SIZE = "giraph.vertexKeySpaceSize";
921
922
923
924
925
926 IntConfOption CHECKPOINT_FREQUENCY =
927 new IntConfOption("giraph.checkpointFrequency", 0,
928 "How often to checkpoint (i.e. 0, means no checkpoint, 1 means " +
929 "every superstep, 2 is every two supersteps, etc.).");
930
931
932
933
934 BooleanConfOption CLEANUP_CHECKPOINTS_AFTER_SUCCESS =
935 new BooleanConfOption("giraph.cleanupCheckpointsAfterSuccess", true,
936 "Delete checkpoints after a successful job run?");
937
938
939
940
941
942
943 String RESTART_SUPERSTEP = "giraph.restartSuperstep";
944
945
946
947
948
949 StrConfOption RESTART_JOB_ID = new StrConfOption("giraph.restart.jobId",
950 null, "Which job ID should I try to restart?");
951
952
953
954
955
956 String BASE_ZNODE_KEY = "giraph.zkBaseZNode";
957
958
959
960
961
962 StrConfOption ZOOKEEPER_MANAGER_DIRECTORY =
963 new StrConfOption("giraph.zkManagerDirectory",
964 "_bsp/_defaultZkManagerDir",
965 "If ZOOKEEPER_LIST is not set, then use this directory to manage " +
966 "ZooKeeper");
967
968
969 IntConfOption ZOOKEEPER_CONNECTION_ATTEMPTS =
970 new IntConfOption("giraph.zkConnectionAttempts", 10,
971 "Number of ZooKeeper client connection attempts before giving up.");
972
973
974 StrConfOption CHECKPOINT_DIRECTORY =
975 new StrConfOption("giraph.checkpointDirectory", "_bsp/_checkpoints/",
976 "This directory has/stores the available checkpoint files in HDFS.");
977
978
979
980
981
982 StrConfOption PARTITIONS_DIRECTORY =
983 new StrConfOption("giraph.partitionsDirectory", "_bsp/_partitions",
984 "Comma-separated list of directories in the local filesystem for " +
985 "out-of-core partitions.");
986
987
988
989
990
991
992
993
994 IntConfOption NUM_OUT_OF_CORE_THREADS =
995 new IntConfOption("giraph.numOutOfCoreThreads", 1, "Number of IO " +
996 "threads used in out-of-core mechanism. If using local disk to " +
997 "spill data, this should be equal to the number of available " +
998 "disks. In such case, use giraph.partitionsDirectory to specify " +
999 "mount points on different disks.");
1000
1001
1002 BooleanConfOption USE_OUT_OF_CORE_GRAPH =
1003 new BooleanConfOption("giraph.useOutOfCoreGraph", false,
1004 "Enable out-of-core graph.");
1005
1006
1007 ClassConfOption<OutOfCoreDataAccessor> OUT_OF_CORE_DATA_ACCESSOR =
1008 ClassConfOption.create("giraph.outOfCoreDataAccessor",
1009 LocalDiskDataAccessor.class, OutOfCoreDataAccessor.class,
1010 "Data accessor used in out-of-core computation (local-disk, " +
1011 "in-memory, HDFS, etc.)");
1012
1013
1014
1015
1016
1017
1018 ClassConfOption<OutOfCoreOracle> OUT_OF_CORE_ORACLE =
1019 ClassConfOption.create("giraph.outOfCoreOracle",
1020 MemoryEstimatorOracle.class, OutOfCoreOracle.class,
1021 "Out-of-core oracle that is to be used for adaptive out-of-core " +
1022 "engine");
1023
1024
1025 IntConfOption MAX_PARTITIONS_IN_MEMORY =
1026 new IntConfOption("giraph.maxPartitionsInMemory", 0,
1027 "Maximum number of partitions to hold in memory for each worker. By" +
1028 " default it is set to 0 (for adaptive out-of-core mechanism");
1029
1030
1031 String YOURKIT_OUTPUT_DIR = "giraph.yourkit.outputDir";
1032
1033 String YOURKIT_OUTPUT_DIR_DEFAULT = "/tmp/giraph/%JOB_ID%/%TASK_ID%";
1034
1035
1036 BooleanConfOption KEEP_ZOOKEEPER_DATA =
1037 new BooleanConfOption("giraph.keepZooKeeperData", false,
1038 "Keep the zookeeper output for debugging? Default is to remove it.");
1039
1040 int DEFAULT_ZOOKEEPER_SNAP_COUNT = 50000;
1041
1042 int DEFAULT_ZOOKEEPER_TICK_TIME = 6000;
1043
1044 int DEFAULT_ZOOKEEPER_MAX_CLIENT_CNXNS = 10000;
1045
1046 int ZOOKEEPER_SNAP_RETAIN_COUNT = 3;
1047
1048 int ZOOKEEPER_PURGE_INTERVAL = 1;
1049
1050 IntConfOption ZOOKEEPER_MIN_SESSION_TIMEOUT =
1051 new IntConfOption("giraph.zKMinSessionTimeout", MINUTES.toMillis(10),
1052 "ZooKeeper minimum session timeout");
1053
1054 IntConfOption ZOOKEEPER_MAX_SESSION_TIMEOUT =
1055 new IntConfOption("giraph.zkMaxSessionTimeout", MINUTES.toMillis(15),
1056 "ZooKeeper maximum session timeout");
1057
1058
1059 BooleanConfOption ZOOKEEPER_FORCE_SYNC =
1060 new BooleanConfOption("giraph.zKForceSync", false,
1061 "ZooKeeper force sync");
1062
1063
1064 BooleanConfOption ZOOKEEPER_SKIP_ACL =
1065 new BooleanConfOption("giraph.ZkSkipAcl", true, "ZooKeeper skip ACLs");
1066
1067
1068
1069
1070
1071 BooleanConfOption AUTHENTICATE =
1072 new BooleanConfOption("giraph.authenticate", false,
1073 "Whether to use SASL with DIGEST and Hadoop Job Tokens to " +
1074 "authenticate and authorize Netty BSP Clients to Servers.");
1075
1076
1077 BooleanConfOption USE_UNSAFE_SERIALIZATION =
1078 new BooleanConfOption("giraph.useUnsafeSerialization", true,
1079 "Use unsafe serialization?");
1080
1081
1082
1083
1084
1085
1086
1087
1088 BooleanConfOption USE_BIG_DATA_IO_FOR_MESSAGES =
1089 new BooleanConfOption("giraph.useBigDataIOForMessages", false,
1090 "Use BigDataIO for messages?");
1091
1092
1093
1094
1095
1096
1097 IntConfOption MAX_TASK_ATTEMPTS =
1098 new IntConfOption("mapred.map.max.attempts", -1,
1099 "Maximum number of attempts a master/worker will retry before " +
1100 "killing the job. This directly maps to the number of map task " +
1101 "attempts in Hadoop.");
1102
1103
1104 StrConfOption DNS_INTERFACE =
1105 new StrConfOption("giraph.dns.interface", "default",
1106 "Interface to use for hostname resolution");
1107
1108 StrConfOption DNS_NAMESERVER =
1109 new StrConfOption("giraph.dns.nameserver", "default",
1110 "Server for hostname resolution");
1111
1112
1113
1114
1115
1116
1117 IntConfOption MAX_NUMBER_OF_SUPERSTEPS =
1118 new IntConfOption("giraph.maxNumberOfSupersteps", 1,
1119 "The application will halt after this many supersteps is " +
1120 "completed. For instance, if it is set to 3, the application will " +
1121 "run at most 0, 1, and 2 supersteps and then go into the shutdown " +
1122 "superstep.");
1123
1124
1125
1126
1127
1128 BooleanConfOption STATIC_GRAPH =
1129 new BooleanConfOption("giraph.isStaticGraph", false,
1130 "The application will not mutate the graph topology (the edges). " +
1131 "It is used to optimise out-of-core graph, by not writing back " +
1132 "edges every time.");
1133
1134
1135
1136
1137
1138 EnumConfOption<MessageEncodeAndStoreType> MESSAGE_ENCODE_AND_STORE_TYPE =
1139 EnumConfOption.create("giraph.messageEncodeAndStoreType",
1140 MessageEncodeAndStoreType.class,
1141 MessageEncodeAndStoreType.BYTEARRAY_PER_PARTITION,
1142 "Select the message_encode_and_store_type to use");
1143
1144
1145
1146
1147
1148 BooleanConfOption CREATE_EDGE_SOURCE_VERTICES =
1149 new BooleanConfOption("giraph.createEdgeSourceVertices", true,
1150 "Create a source vertex if present in edge input but not " +
1151 "necessarily in vertex input");
1152
1153
1154
1155
1156
1157 ClassConfOption<CreateSourceVertexCallback>
1158 CREATE_EDGE_SOURCE_VERTICES_CALLBACK =
1159 ClassConfOption.create("giraph.createEdgeSourceVerticesCallback",
1160 DefaultCreateSourceVertexCallback.class,
1161 CreateSourceVertexCallback.class,
1162 "Decide whether we should create a source vertex when id is " +
1163 "present in the edge input but not in vertex input");
1164
1165
1166
1167
1168
1169 String ZOOKEEPER_SERVER_PORT_COUNTER_GROUP = "Zookeeper server:port";
1170
1171
1172
1173
1174
1175 String ZOOKEEPER_HALT_NODE_COUNTER_GROUP = "Zookeeper halt node";
1176
1177
1178
1179
1180
1181 String ZOOKEEPER_BASE_PATH_COUNTER_GROUP = "Zookeeper base path";
1182
1183
1184
1185
1186 ClassConfOption<HaltApplicationUtils.HaltInstructionsWriter>
1187 HALT_INSTRUCTIONS_WRITER_CLASS = ClassConfOption.create(
1188 "giraph.haltInstructionsWriter",
1189 HaltApplicationUtils.DefaultHaltInstructionsWriter.class,
1190 HaltApplicationUtils.HaltInstructionsWriter.class,
1191 "Class used to write instructions on how to halt the application");
1192
1193
1194
1195
1196
1197 IntConfOption WAIT_TASK_DONE_TIMEOUT_MS =
1198 new IntConfOption("giraph.waitTaskDoneTimeoutMs", MINUTES.toMillis(15),
1199 "Maximum timeout (in ms) for waiting for all all tasks to " +
1200 "complete");
1201
1202
1203 BooleanConfOption TRACK_JOB_PROGRESS_ON_CLIENT =
1204 new BooleanConfOption("giraph.trackJobProgressOnClient", false,
1205 "Whether to track job progress on client or not");
1206
1207
1208 ClassConfOption<JobProgressTrackerClient> JOB_PROGRESS_TRACKER_CLIENT_CLASS =
1209 ClassConfOption.create("giraph.jobProgressTrackerClientClass",
1210 RetryableJobProgressTrackerClient.class,
1211 JobProgressTrackerClient.class,
1212 "Class to use to make calls to the job progress tracker service");
1213
1214
1215 ClassConfOption<JobProgressTrackerService>
1216 JOB_PROGRESS_TRACKER_SERVICE_CLASS =
1217 ClassConfOption.create("giraph.jobProgressTrackerServiceClass",
1218 DefaultJobProgressTrackerService.class,
1219 JobProgressTrackerService.class,
1220 "Class to use to track job progress on client");
1221
1222
1223
1224
1225 LongConfOption VERTICES_TO_UPDATE_PROGRESS =
1226 new LongConfOption("giraph.VerticesToUpdateProgress", 100000,
1227 "Minimum number of vertices to compute before " +
1228 "updating worker progress");
1229
1230
1231
1232 IntConfOption HDFS_FILE_CREATION_RETRIES =
1233 new IntConfOption("giraph.hdfs.file.creation.retries", 10,
1234 "Retries to create an HDFS file before failing");
1235
1236
1237 IntConfOption HDFS_FILE_CREATION_RETRY_WAIT_MS =
1238 new IntConfOption("giraph.hdfs.file.creation.retry.wait.ms", 30_000,
1239 "Milliseconds to wait prior to retrying creation of an HDFS file");
1240
1241
1242 IntConfOption NUM_CHECKPOINT_IO_THREADS =
1243 new IntConfOption("giraph.checkpoint.io.threads", 8,
1244 "Number of threads for writing and reading checkpoints");
1245
1246
1247
1248
1249
1250 StrConfOption CHECKPOINT_COMPRESSION_CODEC =
1251 new StrConfOption("giraph.checkpoint.compression.codec",
1252 ".deflate",
1253 "Defines compression algorithm we will be using for " +
1254 "storing checkpoint. Available options include but " +
1255 "not restricted to: .deflate, .gz, .bz2, .lzo");
1256
1257
1258
1259
1260
1261
1262 ClassConfOption<CheckpointSupportedChecker> CHECKPOINT_SUPPORTED_CHECKER =
1263 ClassConfOption.create("giraph.checkpoint.supported.checker",
1264 DefaultCheckpointSupportedChecker.class,
1265 CheckpointSupportedChecker.class,
1266 "This is the way to specify if checkpointing is " +
1267 "supported by the job");
1268
1269
1270
1271
1272 IntConfOption ASYNC_MESSAGE_STORE_THREADS_COUNT =
1273 new IntConfOption("giraph.async.message.store.threads", 0,
1274 "Number of threads to be used in async message store.");
1275
1276
1277 ClassConfOption<OutputFormat> HADOOP_OUTPUT_FORMAT_CLASS =
1278 ClassConfOption.create("giraph.hadoopOutputFormatClass",
1279 BspOutputFormat.class, OutputFormat.class,
1280 "Output format class for hadoop to use (for committing)");
1281
1282
1283
1284
1285
1286 BooleanConfOption PREFER_IP_ADDRESSES =
1287 new BooleanConfOption("giraph.preferIP", false,
1288 "Prefer IP addresses instead of host names");
1289
1290
1291
1292
1293
1294
1295 LongConfOption WAIT_ZOOKEEPER_TIMEOUT_MSEC =
1296 new LongConfOption("giraph.waitZookeeperTimeoutMsec",
1297 MINUTES.toMillis(15),
1298 "How long should we stay in waitForever loops in various " +
1299 "places that require network connection");
1300
1301
1302
1303
1304
1305
1306
1307 LongConfOption WAIT_FOR_OTHER_WORKERS_TIMEOUT_MSEC =
1308 new LongConfOption("giraph.waitForOtherWorkersMsec",
1309 HOURS.toMillis(48),
1310 "How long should workers wait to finish superstep");
1311
1312
1313 IntConfOption SUPERSTEP_COUNT = new IntConfOption("giraph.numSupersteps", -1,
1314 "Number of supersteps job will run for");
1315
1316
1317
1318
1319
1320 BooleanConfOption DISABLE_GIRAPH_CLASS_RESOLVER =
1321 new BooleanConfOption("giraph.disableGiraphClassResolver", false,
1322 "Disables GiraphClassResolver, which is a custom implementation " +
1323 "of kryo class resolver that avoids writing class names to the " +
1324 "underlying stream for faster serialization.");
1325
1326
1327
1328
1329 StrConfOption JMAP_PATH = new StrConfOption("giraph.jmapPath", "jmap",
1330 "Path to use for invoking jmap");
1331
1332
1333
1334
1335 BooleanConfOption FAIL_ON_EMPTY_INPUT = new BooleanConfOption(
1336 "giraph.failOnEmptyInput", true,
1337 "Whether to fail the job or just warn when input is empty");
1338 }
1339