1 | |
|
2 | |
|
3 | |
|
4 | |
|
5 | |
|
6 | |
|
7 | |
|
8 | |
|
9 | |
|
10 | |
|
11 | |
|
12 | |
|
13 | |
|
14 | |
|
15 | |
|
16 | |
|
17 | |
|
18 | |
package org.apache.giraph.utils; |
19 | |
|
20 | |
import org.apache.commons.io.FilenameUtils; |
21 | |
import org.apache.hadoop.conf.Configuration; |
22 | |
import org.apache.hadoop.filecache.DistributedCache; |
23 | |
import org.apache.hadoop.fs.FileSystem; |
24 | |
import org.apache.hadoop.fs.Path; |
25 | |
import org.apache.log4j.Logger; |
26 | |
|
27 | |
import com.google.common.base.Optional; |
28 | |
|
29 | |
import java.io.IOException; |
30 | |
|
31 | |
import static org.apache.commons.io.FilenameUtils.getBaseName; |
32 | |
|
33 | |
|
34 | |
|
35 | |
|
36 | |
public class DistributedCacheUtils { |
37 | |
|
38 | 0 | private static final Logger LOG = Logger.getLogger( |
39 | |
DistributedCacheUtils.class); |
40 | |
|
41 | |
|
42 | 0 | private DistributedCacheUtils() { } |
43 | |
|
44 | |
|
45 | |
|
46 | |
|
47 | |
|
48 | |
|
49 | |
|
50 | |
|
51 | |
public static Optional<Path> |
52 | |
getLocalCacheFile(Configuration conf, String pathToMatch) { |
53 | 0 | String nameToPath = FilenameUtils.getName(pathToMatch); |
54 | |
Path[] paths; |
55 | |
try { |
56 | 0 | paths = DistributedCache.getLocalCacheFiles(conf); |
57 | 0 | } catch (IOException e) { |
58 | 0 | return Optional.absent(); |
59 | 0 | } |
60 | 0 | for (Path path : paths) { |
61 | 0 | if (FilenameUtils.getName(path.toString()).equals(nameToPath)) { |
62 | 0 | return Optional.of(path); |
63 | |
} |
64 | |
} |
65 | 0 | return Optional.absent(); |
66 | |
} |
67 | |
|
68 | |
|
69 | |
|
70 | |
|
71 | |
|
72 | |
|
73 | |
|
74 | |
|
75 | |
|
76 | |
public static Path copyToHdfs(Path path, Configuration conf) { |
77 | 0 | if (path.toString().startsWith("hdfs://")) { |
78 | |
|
79 | 0 | return path; |
80 | |
} |
81 | |
|
82 | 0 | FileSystem fs = null; |
83 | |
try { |
84 | 0 | fs = FileSystem.get(conf); |
85 | 0 | } catch (IOException e) { |
86 | 0 | throw new IllegalArgumentException("Failed to get HDFS FileSystem", e); |
87 | 0 | } |
88 | 0 | String name = getBaseName(path.toString()) + "-" + System.nanoTime(); |
89 | 0 | Path remotePath = new Path("/tmp/giraph", name); |
90 | 0 | LOG.info("copyToHdfsIfNecessary: Copying " + path + " to " + |
91 | 0 | remotePath + " on hdfs " + fs.getUri()); |
92 | |
try { |
93 | 0 | fs.copyFromLocalFile(false, true, path, remotePath); |
94 | 0 | } catch (IOException e) { |
95 | 0 | throw new IllegalArgumentException( |
96 | |
"Failed to copy jython script from local path " + path + |
97 | 0 | " to hdfs path " + remotePath + " on hdfs " + fs.getUri(), e); |
98 | 0 | } |
99 | 0 | return remotePath; |
100 | |
} |
101 | |
|
102 | |
|
103 | |
|
104 | |
|
105 | |
|
106 | |
|
107 | |
|
108 | |
|
109 | |
public static Path copyAndAdd(Path path, Configuration conf) { |
110 | 0 | Path remotePath = copyToHdfs(path, conf); |
111 | 0 | DistributedCache.addCacheFile(remotePath.toUri(), conf); |
112 | 0 | return remotePath; |
113 | |
} |
114 | |
} |