1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.giraph.io.formats.multi;
20
21 import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
22 import org.apache.giraph.conf.StrConfOption;
23 import org.apache.giraph.io.MappingInputFormat;
24 import org.apache.hadoop.conf.Configuration;
25 import org.apache.hadoop.io.Writable;
26 import org.apache.hadoop.io.WritableComparable;
27 import org.json.JSONArray;
28 import org.json.JSONException;
29
30 import com.google.common.collect.Lists;
31
32 import java.util.List;
33
34
35
36
37
38
39
40
41
42
43
44
45 public class MappingInputFormatDescription<I extends WritableComparable,
46 V extends Writable, E extends Writable, B extends Writable>
47 extends InputFormatDescription<MappingInputFormat<I, V, E, B>> {
48
49
50
51
52
53
54
55 public static final StrConfOption MAPPING_INPUT_FORMAT_DESCRIPTIONS =
56 new StrConfOption("giraph.multiMappingInput.descriptions", null,
57 "MappingInputFormats description - JSON array containing a JSON " +
58 "array for each mapping input. Mapping input JSON arrays contain " +
59 "one or two elements - first one is the name of mapping input " +
60 "class, and second one is JSON object with all specific parameters " +
61 "for this mapping input. For example: [[\"VIF1\",{\"p\":\"v1\"}]," +
62 "[\"VIF2\",{\"p\":\"v2\",\"q\":\"v\"}]]\"");
63
64
65
66
67
68
69 public MappingInputFormatDescription(
70 Class<? extends MappingInputFormat<I, V, E, B>> mappingInputFormatClass
71 ) {
72 super(mappingInputFormatClass);
73 }
74
75
76
77
78
79
80 public MappingInputFormatDescription(String description) {
81 super(description);
82 }
83
84
85
86
87
88
89
90
91 private ImmutableClassesGiraphConfiguration<I, V, E>
92 createConfigurationCopy(
93 ImmutableClassesGiraphConfiguration<I, V, E> conf) {
94 ImmutableClassesGiraphConfiguration<I, V, E> confCopy =
95 new ImmutableClassesGiraphConfiguration<I, V, E>(conf);
96 confCopy.setMappingInputFormatClass(getInputFormatClass());
97 putParametersToConfiguration(confCopy);
98 return confCopy;
99 }
100
101
102
103
104
105
106
107
108
109
110
111 public static <I extends WritableComparable, V extends Writable,
112 E extends Writable, B extends Writable>
113 List<MappingInputFormatDescription<I, V, E, B>>
114 getMappingInputFormatDescriptions(Configuration conf) {
115 String mappingInputFormatDescriptions =
116 MAPPING_INPUT_FORMAT_DESCRIPTIONS.get(conf);
117 if (mappingInputFormatDescriptions == null) {
118 return Lists.newArrayList();
119 }
120 try {
121 JSONArray inputFormatsJson = new JSONArray(
122 mappingInputFormatDescriptions);
123 List<MappingInputFormatDescription<I, V, E, B>> descriptions =
124 Lists.newArrayListWithCapacity(inputFormatsJson.length());
125 for (int i = 0; i < inputFormatsJson.length(); i++) {
126 descriptions.add(new MappingInputFormatDescription<I, V, E, B>(
127 inputFormatsJson.getJSONArray(i).toString()));
128 }
129 return descriptions;
130 } catch (JSONException e) {
131 throw new IllegalStateException("getMappingInputFormatDescriptions: " +
132 "JSONException occurred while trying to process " +
133 mappingInputFormatDescriptions, e);
134 }
135 }
136
137
138
139
140
141
142
143
144
145
146
147 public static <I extends WritableComparable, V extends Writable,
148 E extends Writable, B extends Writable>
149 List<MappingInputFormat<I, V, E, B>> createMappingInputFormats(
150 ImmutableClassesGiraphConfiguration<I, V, E> conf) {
151 List<MappingInputFormatDescription<I, V, E, B>> descriptions =
152 getMappingInputFormatDescriptions(conf);
153 List<MappingInputFormat<I, V, E, B>> mappingInputFormats =
154 Lists.newArrayListWithCapacity(descriptions.size());
155 for (MappingInputFormatDescription<I, V, E, B> description : descriptions) {
156 ImmutableClassesGiraphConfiguration<I, V, E> confCopy =
157 description.createConfigurationCopy(conf);
158 mappingInputFormats.add((MappingInputFormat<I, V, E, B>)
159 confCopy.createWrappedMappingInputFormat());
160 }
161 return mappingInputFormats;
162 }
163 }