1 | |
|
2 | |
|
3 | |
|
4 | |
|
5 | |
|
6 | |
|
7 | |
|
8 | |
|
9 | |
|
10 | |
|
11 | |
|
12 | |
|
13 | |
|
14 | |
|
15 | |
|
16 | |
|
17 | |
|
18 | |
package org.apache.giraph.block_app.reducers.collect; |
19 | |
|
20 | |
import java.util.ArrayList; |
21 | |
import java.util.List; |
22 | |
|
23 | |
import org.apache.giraph.block_app.framework.api.BlockMasterApi; |
24 | |
import org.apache.giraph.block_app.framework.api.CreateReducersApi; |
25 | |
import org.apache.giraph.block_app.framework.piece.global_comm.BroadcastHandle; |
26 | |
import org.apache.giraph.block_app.framework.piece.global_comm.array.BroadcastArrayHandle; |
27 | |
import org.apache.giraph.master.MasterGlobalCommUsage; |
28 | |
import org.apache.giraph.reducers.ReduceOperation; |
29 | |
import org.apache.giraph.types.ops.PrimitiveTypeOps; |
30 | |
import org.apache.giraph.types.ops.TypeOpsUtils; |
31 | |
import org.apache.giraph.types.ops.collections.array.WArrayList; |
32 | |
import org.apache.giraph.worker.WorkerBroadcastUsage; |
33 | |
import org.apache.giraph.writable.kryo.KryoWritableWrapper; |
34 | |
|
35 | |
|
36 | |
|
37 | |
|
38 | |
|
39 | |
|
40 | 0 | @SuppressWarnings("unchecked") |
41 | |
public class CollectShardedTuplesOfPrimitivesReducerHandle |
42 | |
extends ShardedReducerHandle<List<Object>, List<WArrayList>> { |
43 | |
|
44 | |
|
45 | |
|
46 | |
private final List<PrimitiveTypeOps> typeOpsList; |
47 | |
|
48 | |
public CollectShardedTuplesOfPrimitivesReducerHandle( |
49 | 0 | final CreateReducersApi reduceApi, Class<?>... valueClasses) { |
50 | 0 | typeOpsList = new ArrayList<>(); |
51 | 0 | for (Class<?> valueClass : valueClasses) { |
52 | 0 | typeOpsList.add(TypeOpsUtils.getPrimitiveTypeOps(valueClass)); |
53 | |
} |
54 | 0 | register(reduceApi); |
55 | 0 | } |
56 | |
|
57 | |
public List<Object> createSingleValue() { |
58 | 0 | List<Object> ret = new ArrayList<>(); |
59 | 0 | for (PrimitiveTypeOps typeOps : typeOpsList) { |
60 | 0 | ret.add(typeOps.create()); |
61 | 0 | } |
62 | 0 | return ret; |
63 | |
} |
64 | |
|
65 | |
@Override |
66 | |
public ReduceOperation<List<Object>, |
67 | |
KryoWritableWrapper<List<WArrayList>>> createReduceOperation() { |
68 | 0 | return new CollectTuplesOfPrimitivesReduceOperation(typeOpsList); |
69 | |
} |
70 | |
|
71 | |
@Override |
72 | |
public List<WArrayList> createReduceResult( |
73 | |
MasterGlobalCommUsage master) { |
74 | 0 | int size = 0; |
75 | 0 | for (int i = 0; i < REDUCER_COUNT; i++) { |
76 | 0 | size += reducers.get(i).getReducedValue(master).get().get(0).size(); |
77 | |
} |
78 | 0 | return createLists(size); |
79 | |
} |
80 | |
|
81 | |
public List<WArrayList> createLists(int size) { |
82 | 0 | List<WArrayList> ret = new ArrayList<>(); |
83 | 0 | for (PrimitiveTypeOps typeOps : typeOpsList) { |
84 | 0 | ret.add(typeOps.createArrayList(size)); |
85 | 0 | } |
86 | 0 | return ret; |
87 | |
} |
88 | |
|
89 | |
@Override |
90 | |
public BroadcastHandle<List<WArrayList>> createBroadcastHandle( |
91 | |
BroadcastArrayHandle<KryoWritableWrapper<List<WArrayList>>> |
92 | |
broadcasts) { |
93 | 0 | return new CollectShardedTuplesOfPrimitivesBroadcastHandle(broadcasts); |
94 | |
} |
95 | |
|
96 | |
|
97 | |
|
98 | |
|
99 | 0 | public class CollectShardedTuplesOfPrimitivesBroadcastHandle |
100 | |
extends ShardedBroadcastHandle { |
101 | |
public CollectShardedTuplesOfPrimitivesBroadcastHandle( |
102 | |
BroadcastArrayHandle<KryoWritableWrapper<List<WArrayList>>> |
103 | 0 | broadcasts) { |
104 | 0 | super(broadcasts); |
105 | 0 | } |
106 | |
|
107 | |
@Override |
108 | |
public List<WArrayList> createBroadcastResult( |
109 | |
WorkerBroadcastUsage worker) { |
110 | 0 | int size = 0; |
111 | 0 | for (int i = 0; i < REDUCER_COUNT; i++) { |
112 | 0 | size += broadcasts.get(i).getBroadcast(worker).get().size(); |
113 | |
} |
114 | 0 | return createLists(size); |
115 | |
} |
116 | |
} |
117 | |
|
118 | |
|
119 | |
|
120 | |
|
121 | 0 | public static class CollectShardedTuplesOfPrimitivesReduceBroadcast { |
122 | |
private CollectShardedTuplesOfPrimitivesReducerHandle reducerHandle; |
123 | |
private BroadcastHandle<List<WArrayList>> broadcastHandle; |
124 | |
|
125 | |
|
126 | |
public void registeredReducer(CreateReducersApi reduceApi, |
127 | |
Class<?>... valueClasses) { |
128 | 0 | this.reducerHandle = new CollectShardedTuplesOfPrimitivesReducerHandle( |
129 | |
reduceApi, valueClasses); |
130 | 0 | } |
131 | |
|
132 | |
public List<Object> createSingleValue() { |
133 | 0 | return reducerHandle.createSingleValue(); |
134 | |
} |
135 | |
|
136 | |
|
137 | |
public void reduce(List<Object> valueToReduce) { |
138 | 0 | reducerHandle.reduce(valueToReduce); |
139 | 0 | } |
140 | |
|
141 | |
|
142 | |
public List<WArrayList> getReducedValue(MasterGlobalCommUsage master) { |
143 | 0 | return reducerHandle.getReducedValue(master); |
144 | |
} |
145 | |
|
146 | |
|
147 | |
|
148 | |
|
149 | |
public void broadcastValue(BlockMasterApi master) { |
150 | 0 | broadcastHandle = reducerHandle.broadcastValue(master); |
151 | 0 | } |
152 | |
|
153 | |
|
154 | |
public List<WArrayList> getBroadcast(WorkerBroadcastUsage worker) { |
155 | 0 | return broadcastHandle.getBroadcast(worker); |
156 | |
} |
157 | |
} |
158 | |
} |