1 | |
|
2 | |
|
3 | |
|
4 | |
|
5 | |
|
6 | |
|
7 | |
|
8 | |
|
9 | |
|
10 | |
|
11 | |
|
12 | |
|
13 | |
|
14 | |
|
15 | |
|
16 | |
|
17 | |
|
18 | |
package org.apache.giraph.block_app.framework.api.local; |
19 | |
|
20 | |
import java.util.Map; |
21 | |
import java.util.Map.Entry; |
22 | |
|
23 | |
import org.apache.giraph.master.MasterGlobalCommUsage; |
24 | |
import org.apache.giraph.reducers.ReduceOperation; |
25 | |
import org.apache.giraph.reducers.Reducer; |
26 | |
import org.apache.giraph.utils.WritableUtils; |
27 | |
import org.apache.giraph.worker.WorkerGlobalCommUsage; |
28 | |
import org.apache.hadoop.io.Writable; |
29 | |
|
30 | |
import com.google.common.collect.Maps; |
31 | |
|
32 | |
|
33 | |
|
34 | |
|
35 | |
@SuppressWarnings("unchecked") |
36 | |
class InternalAggregators |
37 | |
implements MasterGlobalCommUsage, WorkerGlobalCommUsage { |
38 | |
private final boolean runAllChecks; |
39 | |
|
40 | |
|
41 | 0 | private final Map<String, Reducer<Object, Writable>> reducerMap = |
42 | 0 | Maps.newHashMap(); |
43 | |
|
44 | 0 | private final Map<String, Writable> broadcastMap = |
45 | 0 | Maps.newHashMap(); |
46 | |
|
47 | 0 | private final Map<String, Writable> reducedMap = |
48 | 0 | Maps.newHashMap(); |
49 | |
|
50 | 0 | public InternalAggregators(boolean runAllChecks) { |
51 | 0 | this.runAllChecks = runAllChecks; |
52 | 0 | } |
53 | |
|
54 | |
private static <T> T getOrThrow( |
55 | |
Map<String, T> map, String mapName, String key) { |
56 | 0 | T value = map.get(key); |
57 | 0 | if (value == null) { |
58 | 0 | throw new IllegalArgumentException( |
59 | |
key + " not present in " + mapName); |
60 | |
} |
61 | 0 | return value; |
62 | |
} |
63 | |
|
64 | |
@Override |
65 | |
public void broadcast(String name, Writable value) { |
66 | 0 | broadcastMap.put(name, value); |
67 | 0 | } |
68 | |
|
69 | |
@Override |
70 | |
public <B extends Writable> B getBroadcast(String name) { |
71 | 0 | return (B) getOrThrow(broadcastMap, "broadcastMap", name); |
72 | |
} |
73 | |
|
74 | |
@Override |
75 | |
public <S, R extends Writable> void registerReducer( |
76 | |
String name, ReduceOperation<S, R> reduceOp) { |
77 | 0 | registerReducer(name, reduceOp, reduceOp.createInitialValue()); |
78 | 0 | } |
79 | |
|
80 | |
@Override |
81 | |
public <S, R extends Writable> void registerReducer( |
82 | |
String name, ReduceOperation<S, R> reduceOp, |
83 | |
R globalInitialValue) { |
84 | 0 | if (reducerMap.containsKey(name)) { |
85 | 0 | throw new IllegalArgumentException( |
86 | |
"Reducer with name " + name + " was already registered, " + |
87 | 0 | " and is " + reducerMap.get(name).getReduceOp() + |
88 | |
", and we are trying to " + " register " + reduceOp); |
89 | |
} |
90 | 0 | if (reduceOp == null) { |
91 | 0 | throw new IllegalArgumentException( |
92 | |
"null reducer cannot be registered, with name " + name); |
93 | |
} |
94 | 0 | if (globalInitialValue == null) { |
95 | 0 | throw new IllegalArgumentException( |
96 | |
"global initial value for reducer cannot be null, but is for " + |
97 | |
reduceOp + " with naem" + name); |
98 | |
} |
99 | |
|
100 | 0 | Reducer<S, R> reducer = new Reducer<>(reduceOp, globalInitialValue); |
101 | 0 | reducerMap.put(name, (Reducer<Object, Writable>) reducer); |
102 | 0 | } |
103 | |
|
104 | |
@Override |
105 | |
public void reduce(String name, Object value) { |
106 | 0 | Reducer<Object, Writable> reducer = |
107 | 0 | getOrThrow(reducerMap, "reducerMap", name); |
108 | 0 | synchronized (reducer) { |
109 | 0 | reducer.reduce(value); |
110 | 0 | } |
111 | 0 | } |
112 | |
|
113 | |
@Override |
114 | |
public void reduceMerge(String name, Writable value) { |
115 | 0 | Reducer<Object, Writable> reducer = |
116 | 0 | getOrThrow(reducerMap, "reducerMap", name); |
117 | 0 | synchronized (reducer) { |
118 | 0 | reducer.reduceMerge(value); |
119 | 0 | } |
120 | 0 | } |
121 | |
|
122 | |
@Override |
123 | |
public <R extends Writable> R getReduced(String name) { |
124 | 0 | return (R) getOrThrow(reducedMap, "reducedMap", name); |
125 | |
} |
126 | |
|
127 | |
public synchronized void afterWorkerBeforeMaster() { |
128 | 0 | broadcastMap.clear(); |
129 | 0 | reducedMap.clear(); |
130 | |
for (Entry<String, Reducer<Object, Writable>> entry : |
131 | 0 | reducerMap.entrySet()) { |
132 | 0 | Writable value = entry.getValue().getCurrentValue(); |
133 | 0 | if (runAllChecks) { |
134 | 0 | Writable newValue = entry.getValue().createInitialValue(); |
135 | 0 | WritableUtils.copyInto(value, newValue); |
136 | 0 | value = newValue; |
137 | |
} |
138 | 0 | reducedMap.put(entry.getKey(), value); |
139 | 0 | } |
140 | 0 | reducerMap.clear(); |
141 | 0 | } |
142 | |
} |