1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.maven.plugins.shade;
20
21 import javax.inject.Named;
22 import javax.inject.Singleton;
23
24 import java.io.BufferedOutputStream;
25 import java.io.ByteArrayInputStream;
26 import java.io.File;
27 import java.io.FileInputStream;
28 import java.io.IOException;
29 import java.io.InputStream;
30 import java.io.InputStreamReader;
31 import java.io.OutputStreamWriter;
32 import java.io.PushbackInputStream;
33 import java.io.Writer;
34 import java.nio.charset.StandardCharsets;
35 import java.util.ArrayList;
36 import java.util.Arrays;
37 import java.util.Collection;
38 import java.util.Collections;
39 import java.util.Enumeration;
40 import java.util.HashSet;
41 import java.util.Iterator;
42 import java.util.LinkedList;
43 import java.util.List;
44 import java.util.Objects;
45 import java.util.Set;
46 import java.util.concurrent.Callable;
47 import java.util.jar.JarEntry;
48 import java.util.jar.JarFile;
49 import java.util.jar.JarOutputStream;
50 import java.util.regex.Matcher;
51 import java.util.regex.Pattern;
52 import java.util.zip.CRC32;
53 import java.util.zip.ZipEntry;
54 import java.util.zip.ZipException;
55
56 import org.apache.commons.collections4.MultiValuedMap;
57 import org.apache.commons.collections4.multimap.HashSetValuedHashMap;
58 import org.apache.maven.plugin.MojoExecutionException;
59 import org.apache.maven.plugins.shade.filter.Filter;
60 import org.apache.maven.plugins.shade.relocation.Relocator;
61 import org.apache.maven.plugins.shade.resource.ManifestResourceTransformer;
62 import org.apache.maven.plugins.shade.resource.ReproducibleResourceTransformer;
63 import org.apache.maven.plugins.shade.resource.ResourceTransformer;
64 import org.codehaus.plexus.util.IOUtil;
65 import org.codehaus.plexus.util.io.CachingOutputStream;
66 import org.objectweb.asm.ClassReader;
67 import org.objectweb.asm.ClassVisitor;
68 import org.objectweb.asm.ClassWriter;
69 import org.objectweb.asm.commons.ClassRemapper;
70 import org.objectweb.asm.commons.Remapper;
71 import org.slf4j.Logger;
72 import org.slf4j.LoggerFactory;
73
74
75
76
77 @Singleton
78 @Named
79 public class DefaultShader implements Shader {
80 private static final int BUFFER_SIZE = 32 * 1024;
81
82 private final Logger logger;
83
84 public DefaultShader() {
85 this(LoggerFactory.getLogger(DefaultShader.class));
86 }
87
88 public DefaultShader(final Logger logger) {
89 this.logger = Objects.requireNonNull(logger);
90 }
91
92 public void shade(ShadeRequest shadeRequest) throws IOException, MojoExecutionException {
93 Set<String> resources = new HashSet<>();
94
95 ManifestResourceTransformer manifestTransformer = null;
96 List<ResourceTransformer> transformers = new ArrayList<>(shadeRequest.getResourceTransformers());
97 for (Iterator<ResourceTransformer> it = transformers.iterator(); it.hasNext(); ) {
98 ResourceTransformer transformer = it.next();
99 if (transformer instanceof ManifestResourceTransformer) {
100 manifestTransformer = (ManifestResourceTransformer) transformer;
101 it.remove();
102 }
103 }
104
105 final DefaultPackageMapper packageMapper = new DefaultPackageMapper(shadeRequest.getRelocators());
106
107
108 shadeRequest.getUberJar().getParentFile().mkdirs();
109
110 try (JarOutputStream out =
111 new JarOutputStream(new BufferedOutputStream(new CachingOutputStream(shadeRequest.getUberJar())))) {
112 goThroughAllJarEntriesForManifestTransformer(shadeRequest, resources, manifestTransformer, out);
113
114
115 MultiValuedMap<String, File> duplicates = new HashSetValuedHashMap<>(10000, 3);
116
117
118 shadeJars(shadeRequest, resources, transformers, out, duplicates, packageMapper);
119
120
121 MultiValuedMap<Collection<File>, String> overlapping = new HashSetValuedHashMap<>(20, 15);
122
123
124 for (String clazz : duplicates.keySet()) {
125 Collection<File> jarz = duplicates.get(clazz);
126 if (jarz.size() > 1) {
127 overlapping.put(jarz, clazz);
128 }
129 }
130
131
132 logSummaryOfDuplicates(overlapping);
133
134 if (overlapping.keySet().size() > 0) {
135 showOverlappingWarning();
136 }
137
138 for (ResourceTransformer transformer : transformers) {
139 if (transformer.hasTransformedResource()) {
140 transformer.modifyOutputStream(out);
141 }
142 }
143 }
144
145 for (Filter filter : shadeRequest.getFilters()) {
146 filter.finished();
147 }
148 }
149
150
151
152
153 private static class ZipHeaderPeekInputStream extends PushbackInputStream {
154
155 private static final byte[] ZIP_HEADER = new byte[] {0x50, 0x4b, 0x03, 0x04};
156
157 private static final int HEADER_LEN = 4;
158
159 protected ZipHeaderPeekInputStream(InputStream in) {
160 super(in, HEADER_LEN);
161 }
162
163 public boolean hasZipHeader() throws IOException {
164 final byte[] header = new byte[HEADER_LEN];
165 int len = super.read(header, 0, HEADER_LEN);
166 if (len != -1) {
167 super.unread(header, 0, len);
168 }
169 return Arrays.equals(header, ZIP_HEADER);
170 }
171 }
172
173
174
175
176 private static class CrcAndSize {
177
178 private final CRC32 crc = new CRC32();
179
180 private long size;
181
182 CrcAndSize(InputStream inputStream) throws IOException {
183 load(inputStream);
184 }
185
186 private void load(InputStream inputStream) throws IOException {
187 byte[] buffer = new byte[BUFFER_SIZE];
188 int bytesRead;
189 while ((bytesRead = inputStream.read(buffer)) != -1) {
190 this.crc.update(buffer, 0, bytesRead);
191 this.size += bytesRead;
192 }
193 }
194
195 public void setupStoredEntry(JarEntry entry) {
196 entry.setSize(this.size);
197 entry.setCompressedSize(this.size);
198 entry.setCrc(this.crc.getValue());
199 entry.setMethod(ZipEntry.STORED);
200 }
201 }
202
203 private void shadeJars(
204 ShadeRequest shadeRequest,
205 Set<String> resources,
206 List<ResourceTransformer> transformers,
207 JarOutputStream jos,
208 MultiValuedMap<String, File> duplicates,
209 DefaultPackageMapper packageMapper)
210 throws IOException {
211 for (File jar : shadeRequest.getJars()) {
212
213 logger.debug("Processing JAR " + jar);
214
215 List<Filter> jarFilters = getFilters(jar, shadeRequest.getFilters());
216 if (jar.isDirectory()) {
217 shadeDir(
218 shadeRequest,
219 resources,
220 transformers,
221 packageMapper,
222 jos,
223 duplicates,
224 jar,
225 jar,
226 "",
227 jarFilters);
228 } else {
229 shadeJar(shadeRequest, resources, transformers, packageMapper, jos, duplicates, jar, jarFilters);
230 }
231 }
232 }
233
234 private void shadeDir(
235 ShadeRequest shadeRequest,
236 Set<String> resources,
237 List<ResourceTransformer> transformers,
238 DefaultPackageMapper packageMapper,
239 JarOutputStream jos,
240 MultiValuedMap<String, File> duplicates,
241 File jar,
242 File current,
243 String prefix,
244 List<Filter> jarFilters)
245 throws IOException {
246 final File[] children = current.listFiles();
247 if (children == null) {
248 return;
249 }
250 for (final File file : children) {
251 final String name = prefix + file.getName();
252 if (file.isDirectory()) {
253 try {
254 shadeDir(
255 shadeRequest,
256 resources,
257 transformers,
258 packageMapper,
259 jos,
260 duplicates,
261 jar,
262 file,
263 prefix + file.getName() + '/',
264 jarFilters);
265 continue;
266 } catch (Exception e) {
267 throw new IOException(String.format("Problem shading JAR %s entry %s: %s", current, name, e), e);
268 }
269 }
270 if (isFiltered(jarFilters, name) || isExcludedEntry(name)) {
271 continue;
272 }
273
274 try {
275 shadeJarEntry(
276 shadeRequest,
277 resources,
278 transformers,
279 packageMapper,
280 jos,
281 duplicates,
282 jar,
283 new Callable<InputStream>() {
284 @Override
285 public InputStream call() throws Exception {
286 return new FileInputStream(file);
287 }
288 },
289 name,
290 file.lastModified(),
291 -1 );
292 } catch (Exception e) {
293 throw new IOException(String.format("Problem shading JAR %s entry %s: %s", current, name, e), e);
294 }
295 }
296 }
297
298 private void shadeJar(
299 ShadeRequest shadeRequest,
300 Set<String> resources,
301 List<ResourceTransformer> transformers,
302 DefaultPackageMapper packageMapper,
303 JarOutputStream jos,
304 MultiValuedMap<String, File> duplicates,
305 File jar,
306 List<Filter> jarFilters)
307 throws IOException {
308 try (JarFile jarFile = newJarFile(jar)) {
309
310 for (Enumeration<JarEntry> j = jarFile.entries(); j.hasMoreElements(); ) {
311 final JarEntry entry = j.nextElement();
312
313 String name = entry.getName();
314
315 if (entry.isDirectory() || isFiltered(jarFilters, name) || isExcludedEntry(name)) {
316 continue;
317 }
318
319 try {
320 shadeJarEntry(
321 shadeRequest,
322 resources,
323 transformers,
324 packageMapper,
325 jos,
326 duplicates,
327 jar,
328 new Callable<InputStream>() {
329 @Override
330 public InputStream call() throws Exception {
331 return jarFile.getInputStream(entry);
332 }
333 },
334 name,
335 entry.getTime(),
336 entry.getMethod());
337 } catch (Exception e) {
338 throw new IOException(String.format("Problem shading JAR %s entry %s: %s", jar, name, e), e);
339 }
340 }
341 }
342 }
343
344 private boolean isExcludedEntry(final String name) {
345 if ("META-INF/INDEX.LIST".equals(name)) {
346
347
348
349 return true;
350 }
351
352 if ("module-info.class".equals(name)) {
353 logger.warn("Discovered module-info.class. " + "Shading will break its strong encapsulation.");
354 return true;
355 }
356 return false;
357 }
358
359 private void shadeJarEntry(
360 ShadeRequest shadeRequest,
361 Set<String> resources,
362 List<ResourceTransformer> transformers,
363 DefaultPackageMapper packageMapper,
364 JarOutputStream jos,
365 MultiValuedMap<String, File> duplicates,
366 File jar,
367 Callable<InputStream> inputProvider,
368 String name,
369 long time,
370 int method)
371 throws Exception {
372 try (InputStream in = inputProvider.call()) {
373 String mappedName = packageMapper.map(name, true, false);
374
375 int idx = mappedName.lastIndexOf('/');
376 if (idx != -1) {
377
378 String dir = mappedName.substring(0, idx);
379 if (!resources.contains(dir)) {
380 addDirectory(resources, jos, dir, time);
381 }
382 }
383
384 duplicates.put(name, jar);
385 if (name.endsWith(".class")) {
386 addRemappedClass(jos, jar, name, time, in, packageMapper);
387 } else if (shadeRequest.isShadeSourcesContent() && name.endsWith(".java")) {
388
389 if (resources.contains(mappedName)) {
390 return;
391 }
392
393 addJavaSource(resources, jos, mappedName, time, in, shadeRequest.getRelocators());
394 } else {
395 if (!resourceTransformed(transformers, mappedName, in, shadeRequest.getRelocators(), time)) {
396
397 if (resources.contains(mappedName)) {
398 logger.debug("We have a duplicate " + name + " in " + jar);
399 return;
400 }
401
402 addResource(resources, jos, mappedName, inputProvider, time, method);
403 } else {
404 duplicates.removeMapping(name, jar);
405 }
406 }
407 }
408 }
409
410 private void goThroughAllJarEntriesForManifestTransformer(
411 ShadeRequest shadeRequest,
412 Set<String> resources,
413 ManifestResourceTransformer manifestTransformer,
414 JarOutputStream jos)
415 throws IOException {
416 if (manifestTransformer != null) {
417 for (File jar : shadeRequest.getJars()) {
418 try (JarFile jarFile = newJarFile(jar)) {
419 for (Enumeration<JarEntry> en = jarFile.entries(); en.hasMoreElements(); ) {
420 JarEntry entry = en.nextElement();
421 String resource = entry.getName();
422 if (manifestTransformer.canTransformResource(resource)) {
423 resources.add(resource);
424 try (InputStream inputStream = jarFile.getInputStream(entry)) {
425 manifestTransformer.processResource(
426 resource, inputStream, shadeRequest.getRelocators(), entry.getTime());
427 }
428 break;
429 }
430 }
431 }
432 }
433 if (manifestTransformer.hasTransformedResource()) {
434 manifestTransformer.modifyOutputStream(jos);
435 }
436 }
437 }
438
439 private void showOverlappingWarning() {
440 logger.warn("maven-shade-plugin has detected that some files are");
441 logger.warn("present in two or more JARs. When this happens, only one");
442 logger.warn("single version of the file is copied to the uber jar.");
443 logger.warn("Usually this is not harmful and you can skip these warnings,");
444 logger.warn("otherwise try to manually exclude artifacts based on");
445 logger.warn("mvn dependency:tree -Ddetail=true and the above output.");
446 logger.warn("See https://maven.apache.org/plugins/maven-shade-plugin/");
447 }
448
449 private void logSummaryOfDuplicates(MultiValuedMap<Collection<File>, String> overlapping) {
450 for (Collection<File> jarz : overlapping.keySet()) {
451 List<String> jarzS = new ArrayList<>();
452
453 for (File jjar : jarz) {
454 jarzS.add(jjar.getName());
455 }
456
457 Collections.sort(jarzS);
458
459 List<String> classes = new LinkedList<>();
460 List<String> resources = new LinkedList<>();
461
462 for (String name : overlapping.get(jarz)) {
463 if (name.endsWith(".class")) {
464 classes.add(name.replace(".class", "").replace("/", "."));
465 } else {
466 resources.add(name);
467 }
468 }
469
470
471 final Collection<String> overlaps = new ArrayList<>();
472 if (!classes.isEmpty()) {
473 if (resources.size() == 1) {
474 overlaps.add("class");
475 } else {
476 overlaps.add("classes");
477 }
478 }
479 if (!resources.isEmpty()) {
480 if (resources.size() == 1) {
481 overlaps.add("resource");
482 } else {
483 overlaps.add("resources");
484 }
485 }
486
487 final List<String> all = new ArrayList<>(classes.size() + resources.size());
488 all.addAll(classes);
489 all.addAll(resources);
490
491 logger.warn(String.join(", ", jarzS) + " define " + all.size() + " overlapping "
492 + String.join(" and ", overlaps) + ": ");
493
494
495 Collections.sort(all);
496
497 int max = 10;
498
499 for (int i = 0; i < Math.min(max, all.size()); i++) {
500 logger.warn(" - " + all.get(i));
501 }
502
503 if (all.size() > max) {
504 logger.warn(" - " + (all.size() - max) + " more...");
505 }
506 }
507 }
508
509 private JarFile newJarFile(File jar) throws IOException {
510 try {
511 return new JarFile(jar);
512 } catch (ZipException zex) {
513
514
515 throw new ZipException("error in opening zip file " + jar);
516 }
517 }
518
519 private List<Filter> getFilters(File jar, List<Filter> filters) {
520 List<Filter> list = new ArrayList<>();
521
522 for (Filter filter : filters) {
523 if (filter.canFilter(jar)) {
524 list.add(filter);
525 }
526 }
527
528 return list;
529 }
530
531 private void addDirectory(Set<String> resources, JarOutputStream jos, String name, long time) throws IOException {
532 if (name.lastIndexOf('/') > 0) {
533 String parent = name.substring(0, name.lastIndexOf('/'));
534 if (!resources.contains(parent)) {
535 addDirectory(resources, jos, parent, time);
536 }
537 }
538
539
540 JarEntry entry = new JarEntry(name + "/");
541 entry.setTime(time);
542 jos.putNextEntry(entry);
543
544 resources.add(name);
545 }
546
547 private void addRemappedClass(
548 JarOutputStream jos, File jar, String name, long time, InputStream is, DefaultPackageMapper packageMapper)
549 throws IOException, MojoExecutionException {
550 if (packageMapper.relocators.isEmpty()) {
551 try {
552 JarEntry entry = new JarEntry(name);
553 entry.setTime(time);
554 jos.putNextEntry(entry);
555 IOUtil.copy(is, jos);
556 } catch (ZipException e) {
557 logger.debug("We have a duplicate " + name + " in " + jar);
558 }
559
560 return;
561 }
562
563
564
565
566 byte[] originalClass = IOUtil.toByteArray(is);
567
568 ClassReader cr = new ClassReader(new ByteArrayInputStream(originalClass));
569
570
571
572
573
574
575 ClassWriter cw = new ClassWriter(0);
576
577 final String pkg = name.substring(0, name.lastIndexOf('/') + 1);
578 final ShadeClassRemapper cv = new ShadeClassRemapper(cw, pkg, packageMapper);
579
580 try {
581 cr.accept(cv, ClassReader.EXPAND_FRAMES);
582 } catch (Throwable ise) {
583 throw new MojoExecutionException("Error in ASM processing class " + name, ise);
584 }
585
586
587 final byte[] renamedClass;
588 if (cv.remapped) {
589 logger.debug("Rewrote class bytecode: " + name);
590 renamedClass = cw.toByteArray();
591 } else {
592 logger.debug("Keeping original class bytecode: " + name);
593 renamedClass = originalClass;
594 }
595
596
597 String mappedName = packageMapper.map(name.substring(0, name.indexOf('.')), true, false);
598
599 try {
600
601 JarEntry entry = new JarEntry(mappedName + ".class");
602 entry.setTime(time);
603 jos.putNextEntry(entry);
604
605 jos.write(renamedClass);
606 } catch (ZipException e) {
607 logger.debug("We have a duplicate " + mappedName + " in " + jar);
608 }
609 }
610
611 private boolean isFiltered(List<Filter> filters, String name) {
612 for (Filter filter : filters) {
613 if (filter.isFiltered(name)) {
614 return true;
615 }
616 }
617
618 return false;
619 }
620
621 private boolean resourceTransformed(
622 List<ResourceTransformer> resourceTransformers,
623 String name,
624 InputStream is,
625 List<Relocator> relocators,
626 long time)
627 throws IOException {
628 boolean resourceTransformed = false;
629
630 for (ResourceTransformer transformer : resourceTransformers) {
631 if (transformer.canTransformResource(name)) {
632 logger.debug("Transforming " + name + " using "
633 + transformer.getClass().getName());
634
635 if (transformer instanceof ReproducibleResourceTransformer) {
636 ((ReproducibleResourceTransformer) transformer).processResource(name, is, relocators, time);
637 } else {
638 transformer.processResource(name, is, relocators);
639 }
640
641 resourceTransformed = true;
642
643 break;
644 }
645 }
646 return resourceTransformed;
647 }
648
649 private void addJavaSource(
650 Set<String> resources,
651 JarOutputStream jos,
652 String name,
653 long time,
654 InputStream is,
655 List<Relocator> relocators)
656 throws IOException {
657 JarEntry entry = new JarEntry(name);
658 entry.setTime(time);
659 jos.putNextEntry(entry);
660
661 String sourceContent = IOUtil.toString(new InputStreamReader(is, StandardCharsets.UTF_8));
662
663 for (Relocator relocator : relocators) {
664 sourceContent = relocator.applyToSourceContent(sourceContent);
665 }
666
667 final Writer writer = new OutputStreamWriter(jos, StandardCharsets.UTF_8);
668 writer.write(sourceContent);
669 writer.flush();
670
671 resources.add(name);
672 }
673
674 private void addResource(
675 Set<String> resources, JarOutputStream jos, String name, Callable<InputStream> input, long time, int method)
676 throws Exception {
677 ZipHeaderPeekInputStream inputStream = new ZipHeaderPeekInputStream(input.call());
678 try {
679 final JarEntry entry = new JarEntry(name);
680
681
682 if (inputStream.hasZipHeader() && method == ZipEntry.STORED) {
683 new CrcAndSize(inputStream).setupStoredEntry(entry);
684 inputStream.close();
685 inputStream = new ZipHeaderPeekInputStream(input.call());
686 }
687
688 entry.setTime(time);
689
690 jos.putNextEntry(entry);
691
692 IOUtil.copy(inputStream, jos);
693
694 resources.add(name);
695 } finally {
696 inputStream.close();
697 }
698 }
699
700 private interface PackageMapper {
701
702
703
704
705
706
707
708
709 String map(String entityName, boolean mapPaths, boolean mapPackages);
710 }
711
712
713
714
715 private static class DefaultPackageMapper implements PackageMapper {
716 private static final Pattern CLASS_PATTERN = Pattern.compile("(\\[*)?L(.+);");
717
718 private final List<Relocator> relocators;
719
720 private DefaultPackageMapper(final List<Relocator> relocators) {
721 this.relocators = relocators;
722 }
723
724 @Override
725 public String map(String entityName, boolean mapPaths, final boolean mapPackages) {
726 String value = entityName;
727
728 String prefix = "";
729 String suffix = "";
730
731 Matcher m = CLASS_PATTERN.matcher(entityName);
732 if (m.matches()) {
733 prefix = m.group(1) + "L";
734 suffix = ";";
735 entityName = m.group(2);
736 }
737
738 for (Relocator r : relocators) {
739 if (mapPackages && r.canRelocateClass(entityName)) {
740 value = prefix + r.relocateClass(entityName) + suffix;
741 break;
742 } else if (mapPaths && r.canRelocatePath(entityName)) {
743 value = prefix + r.relocatePath(entityName) + suffix;
744 break;
745 }
746 }
747 return value;
748 }
749 }
750
751 private static class LazyInitRemapper extends Remapper {
752 private PackageMapper relocators;
753
754 @Override
755 public Object mapValue(Object object) {
756 return object instanceof String ? relocators.map((String) object, true, true) : super.mapValue(object);
757 }
758
759 @Override
760 public String map(String name) {
761
762
763
764
765
766
767
768
769 return relocators.map(name, true, false);
770 }
771 }
772
773
774
775
776
777
778
779
780
781
782
783 private static class ShadeClassRemapper extends ClassRemapper implements PackageMapper {
784 private final String pkg;
785 private final PackageMapper packageMapper;
786 private boolean remapped;
787
788 ShadeClassRemapper(
789 final ClassVisitor classVisitor, final String pkg, final DefaultPackageMapper packageMapper) {
790 super(classVisitor, new LazyInitRemapper() );
791 this.pkg = pkg;
792 this.packageMapper = packageMapper;
793
794
795 LazyInitRemapper.class.cast(remapper).relocators = this;
796 }
797
798 @Override
799 public void visitSource(final String source, final String debug) {
800 if (source == null) {
801 super.visitSource(null, debug);
802 return;
803 }
804
805 final String fqSource = pkg + source;
806 final String mappedSource = map(fqSource, true, false);
807 final String filename = mappedSource.substring(mappedSource.lastIndexOf('/') + 1);
808 super.visitSource(filename, debug);
809 }
810
811 @Override
812 public String map(final String entityName, boolean mapPaths, final boolean mapPackages) {
813 final String mapped = packageMapper.map(entityName, true, mapPackages);
814 if (!remapped) {
815 remapped = !mapped.equals(entityName);
816 }
817 return mapped;
818 }
819 }
820 }