001package org.apache.maven.wagon.shared.http;
002
003/*
004 * Licensed to the Apache Software Foundation (ASF) under one
005 * or more contributor license agreements.  See the NOTICE file
006 * distributed with this work for additional information
007 * regarding copyright ownership.  The ASF licenses this file
008 * to you under the Apache License, Version 2.0 (the
009 * "License"); you may not use this file except in compliance
010 * with the License.  You may obtain a copy of the License at
011 *
012 *   http://www.apache.org/licenses/LICENSE-2.0
013 *
014 * Unless required by applicable law or agreed to in writing,
015 * software distributed under the License is distributed on an
016 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
017 * KIND, either express or implied.  See the License for the
018 * specific language governing permissions and limitations
019 * under the License.
020 */
021
022import java.io.InputStream;
023import java.util.Collections;
024import java.util.List;
025
026import junit.framework.TestCase;
027
028import org.apache.maven.wagon.TransferFailedException;
029
030import org.slf4j.Logger;
031import org.slf4j.LoggerFactory;
032
033/**
034 * Unit Tests for the HtmlFileListParser
035 */
036public class HtmlFileListParserTest
037    extends TestCase
038{
039    private static Logger logger = LoggerFactory.getLogger( HtmlFileListParserTest.class );
040
041    private void assertContainsExpected( List<String> links, String[] expected )
042    {
043        if ( expected.length != links.size() )
044        {
045            Collections.sort( links );
046            for ( String link : links )
047            {
048                logger.info("   \"" + link + "\", ");
049            }
050            assertEquals( "Links to Expected size", expected.length, links.size() );
051        }
052
053        for ( String exp : expected )
054        {
055            assertTrue( "Should find [" + exp + "] in link list", links.contains( exp ) );
056        }
057    }
058
059    private void assertNotContainingAvoided( List<String> links, String[] avoided )
060    {
061        for ( String a : avoided )
062        {
063            assertFalse( "Should not find [" + a + "] in link list", links.contains( a ) );
064        }
065    }
066
067    private List<String> parseLinks( String url, String filename )
068        throws TransferFailedException
069    {
070        InputStream is = this.getClass().getResourceAsStream( "/filelistings/" + filename );
071        List<String> files = HtmlFileListParser.parseFileList( url, is );
072
073        assertNotNull( "file list should not be null.", files );
074        assertFalse( "file list should not be empty.", files.isEmpty() );
075
076        /* Debug */
077        if ( false )
078        {
079            for ( String file : files )
080            {
081                logger.info( "File: '" + file + "'" );
082            }
083        }
084
085        return files;
086    }
087
088    /**
089     * Example showing jetty directory browsing of commons-lang 
090     * 
091     * @throws TransferFailedException
092     */
093    public void testParseCommonsLang()
094        throws TransferFailedException
095    {
096        List<String> links = parseLinks( "http://localhost/repository/commons-lang/commons-lang/2.3",
097                                             "commons-lang.html" );
098
099        String[] expected = new String[] {
100            "commons-lang-2.3-javadoc.jar",
101            "commons-lang-2.3-javadoc.jar.asc",
102            "commons-lang-2.3-javadoc.jar.md5",
103            "commons-lang-2.3-javadoc.jar.sha1",
104            "commons-lang-2.3-sources.jar",
105            "commons-lang-2.3-sources.jar.asc",
106            "commons-lang-2.3-sources.jar.md5",
107            "commons-lang-2.3-sources.jar.sha1",
108            "commons-lang-2.3.jar",
109            "commons-lang-2.3.jar.asc",
110            "commons-lang-2.3.jar.md5",
111            "commons-lang-2.3.jar.sha1",
112            "commons-lang-2.3.pom",
113            "commons-lang-2.3.pom.md5",
114            "commons-lang-2.3.pom.sha1",
115            "maven-metadata.xml",
116            "maven-metadata.xml.md5",
117            "maven-metadata.xml.sha1" };
118
119        assertContainsExpected( links, expected );
120
121        String[] avoided = new String[] { "../" };
122
123        assertNotContainingAvoided( links, avoided );
124    }
125
126    public void testParseIbiblio()
127        throws Exception
128    {
129        List<String> links = parseLinks( "http://www.ibiblio.org/maven2/org/apache/maven/wagon/",
130                                             "ibiblio-wagon.html" );
131
132        String[] expected = new String[] {
133            "wagon-1.0-alpha-2.pom",
134            "wagon-1.0-alpha-2.pom.asc",
135            "wagon-1.0-alpha-2.pom.md5",
136            "wagon-1.0-alpha-2.pom.sha1",
137            "wagon-1.0-alpha-3-20050413.021234-4.pom",
138            "wagon-1.0-alpha-3-20050413.021234-4.pom.md5",
139            "wagon-1.0-alpha-3-20050413.021234-4.pom.sha1",
140            "wagon-1.0-alpha-3-20050419.043745-5.pom",
141            "wagon-1.0-alpha-3-20050419.043745-5.pom.md5",
142            "wagon-1.0-alpha-3-20050419.043745-5.pom.sha1",
143            "wagon-1.0-alpha-3-20050419.044035-6.pom",
144            "wagon-1.0-alpha-3-20050419.044035-6.pom.md5",
145            "wagon-1.0-alpha-3-20050419.044035-6.pom.sha1",
146            "wagon-1.0-alpha-3-20050421.162738-7.pom",
147            "wagon-1.0-alpha-3-20050421.162738-7.pom.md5",
148            "wagon-1.0-alpha-3-20050421.162738-7.pom.sha1",
149            "wagon-1.0-alpha-3-20050422.075233-8.pom",
150            "wagon-1.0-alpha-3-20050422.075233-8.pom.md5",
151            "wagon-1.0-alpha-3-20050422.075233-8.pom.sha1",
152            "wagon-1.0-alpha-3-20050429.051847-9.pom",
153            "wagon-1.0-alpha-3-20050429.051847-9.pom.md5",
154            "wagon-1.0-alpha-3-20050429.051847-9.pom.sha1",
155            "wagon-file/",
156            "wagon-ftp/",
157            "wagon-http-lightweight/",
158            "wagon-http/",
159            "wagon-lightweight-http/",
160            "wagon-provider-api/",
161            "wagon-provider-test/",
162            "wagon-provider/",
163            "wagon-providers-1.0-alpha-2.pom",
164            "wagon-providers-1.0-alpha-2.pom.asc",
165            "wagon-providers-1.0-alpha-2.pom.md5",
166            "wagon-providers-1.0-alpha-2.pom.sha1",
167            "wagon-providers-1.0-alpha-3-20050407.202848-1.pom",
168            "wagon-providers-1.0-alpha-3-20050407.202848-1.pom.md5",
169            "wagon-providers-1.0-alpha-3-20050407.202848-1.pom.sha1",
170            "wagon-providers-1.0-alpha-3-20050419.044035-2.pom",
171            "wagon-providers-1.0-alpha-3-20050419.044035-2.pom.md5",
172            "wagon-providers-1.0-alpha-3-20050419.044035-2.pom.sha1",
173            "wagon-providers-1.0-alpha-3-20050421.162738-3.pom",
174            "wagon-providers-1.0-alpha-3-20050421.162738-3.pom.md5",
175            "wagon-providers-1.0-alpha-3-20050421.162738-3.pom.sha1",
176            "wagon-providers-1.0-alpha-3-20050422.075233-4.pom",
177            "wagon-providers-1.0-alpha-3-20050422.075233-4.pom.md5",
178            "wagon-providers-1.0-alpha-3-20050422.075233-4.pom.sha1",
179            "wagon-providers-1.0-alpha-3-20050429.051847-5.pom",
180            "wagon-providers-1.0-alpha-3-20050429.051847-5.pom.md5",
181            "wagon-providers-1.0-alpha-3-20050429.051847-5.pom.sha1",
182            "wagon-providers/",
183            "wagon-scm/",
184            "wagon-ssh-external/",
185            "wagon-ssh/",
186            "wagon-webdav/",
187            "wagon/" };
188
189        assertContainsExpected( links, expected );
190
191        String[] avoided = new String[] { "/org/", "?C=S;O=A", "?C=D;O=A", "?C=M;O=A", "?D=A", "?M=A", "?N=D", "?S=A" };
192
193        assertNotContainingAvoided( links, avoided );
194    }
195
196    /**
197     * Test of an html which is improperly formatted, and contains full host-specific paths to the resources.
198     * @throws Exception
199     */
200    public void testParseJetty()
201        throws Exception
202    {
203        List<String> links = parseLinks( "http://www.ibiblio.org/maven2/org/apache/maven/wagon/",
204                                             "jetty-wagon.html" );
205
206        String[] expected = new String[] {
207            "wagon-1.0-alpha-2.pom",
208            "wagon-1.0-alpha-2.pom.asc",
209            "wagon-1.0-alpha-2.pom.md5",
210            "wagon-1.0-alpha-2.pom.sha1",
211            "wagon-1.0-alpha-3-20050413.021234-4.pom",
212            "wagon-1.0-alpha-3-20050413.021234-4.pom.md5",
213            "wagon-1.0-alpha-3-20050413.021234-4.pom.sha1",
214            "wagon-1.0-alpha-3-20050419.043745-5.pom",
215            "wagon-1.0-alpha-3-20050419.043745-5.pom.md5",
216            "wagon-1.0-alpha-3-20050419.043745-5.pom.sha1",
217            "wagon-1.0-alpha-3-20050419.044035-6.pom",
218            "wagon-1.0-alpha-3-20050419.044035-6.pom.md5",
219            "wagon-1.0-alpha-3-20050419.044035-6.pom.sha1",
220            "wagon-1.0-alpha-3-20050421.162738-7.pom",
221            "wagon-1.0-alpha-3-20050421.162738-7.pom.md5",
222            "wagon-1.0-alpha-3-20050421.162738-7.pom.sha1",
223            "wagon-1.0-alpha-3-20050422.075233-8.pom",
224            "wagon-1.0-alpha-3-20050422.075233-8.pom.md5",
225            "wagon-1.0-alpha-3-20050422.075233-8.pom.sha1",
226            "wagon-1.0-alpha-3-20050429.051847-9.pom",
227            "wagon-1.0-alpha-3-20050429.051847-9.pom.md5",
228            "wagon-1.0-alpha-3-20050429.051847-9.pom.sha1",
229            "wagon-file/",
230            "wagon-ftp/",
231            "wagon-http-lightweight/",
232            "wagon-http/",
233            "wagon-lightweight-http/",
234            "wagon-provider-api/",
235            "wagon-provider-test/",
236            "wagon-provider/",
237            "wagon-providers-1.0-alpha-2.pom",
238            "wagon-providers-1.0-alpha-2.pom.asc",
239            "wagon-providers-1.0-alpha-2.pom.md5",
240            "wagon-providers-1.0-alpha-2.pom.sha1",
241            "wagon-providers-1.0-alpha-3-20050407.202848-1.pom",
242            "wagon-providers-1.0-alpha-3-20050407.202848-1.pom.md5",
243            "wagon-providers-1.0-alpha-3-20050407.202848-1.pom.sha1",
244            "wagon-providers-1.0-alpha-3-20050419.044035-2.pom",
245            "wagon-providers-1.0-alpha-3-20050419.044035-2.pom.md5",
246            "wagon-providers-1.0-alpha-3-20050419.044035-2.pom.sha1",
247            "wagon-providers-1.0-alpha-3-20050421.162738-3.pom",
248            "wagon-providers-1.0-alpha-3-20050421.162738-3.pom.md5",
249            "wagon-providers-1.0-alpha-3-20050421.162738-3.pom.sha1",
250            "wagon-providers-1.0-alpha-3-20050422.075233-4.pom",
251            "wagon-providers-1.0-alpha-3-20050422.075233-4.pom.md5",
252            "wagon-providers-1.0-alpha-3-20050422.075233-4.pom.sha1",
253            "wagon-providers-1.0-alpha-3-20050429.051847-5.pom",
254            "wagon-providers-1.0-alpha-3-20050429.051847-5.pom.md5",
255            "wagon-providers-1.0-alpha-3-20050429.051847-5.pom.sha1",
256            "wagon-providers/",
257            "wagon-scm/",
258            "wagon-ssh-external/",
259            "wagon-ssh/",
260            "wagon-webdav/",
261            "wagon/" };
262
263        assertContainsExpected( links, expected );
264
265        String[] avoided = new String[] { "/org/", "?C=S;O=A", "?C=D;O=A", "?C=M;O=A" };
266
267        assertNotContainingAvoided( links, avoided );
268    }
269
270    /**
271     * Test of an html which is improperly formatted, contains full host-specific paths to the resources and uses non-normalized base URI
272     * @throws Exception
273     */
274    public void testParseJettyWithNonNormalizedBaseURI()
275        throws Exception
276    {
277        List<String> links = parseLinks( "http://www.ibiblio.org/maven2/org/apache/maven/wagon//",
278                                             "jetty-wagon.html" );
279
280        String[] expected = new String[] {
281            "wagon-1.0-alpha-2.pom",
282            "wagon-1.0-alpha-2.pom.asc",
283            "wagon-1.0-alpha-2.pom.md5",
284            "wagon-1.0-alpha-2.pom.sha1",
285            "wagon-1.0-alpha-3-20050413.021234-4.pom",
286            "wagon-1.0-alpha-3-20050413.021234-4.pom.md5",
287            "wagon-1.0-alpha-3-20050413.021234-4.pom.sha1",
288            "wagon-1.0-alpha-3-20050419.043745-5.pom",
289            "wagon-1.0-alpha-3-20050419.043745-5.pom.md5",
290            "wagon-1.0-alpha-3-20050419.043745-5.pom.sha1",
291            "wagon-1.0-alpha-3-20050419.044035-6.pom",
292            "wagon-1.0-alpha-3-20050419.044035-6.pom.md5",
293            "wagon-1.0-alpha-3-20050419.044035-6.pom.sha1",
294            "wagon-1.0-alpha-3-20050421.162738-7.pom",
295            "wagon-1.0-alpha-3-20050421.162738-7.pom.md5",
296            "wagon-1.0-alpha-3-20050421.162738-7.pom.sha1",
297            "wagon-1.0-alpha-3-20050422.075233-8.pom",
298            "wagon-1.0-alpha-3-20050422.075233-8.pom.md5",
299            "wagon-1.0-alpha-3-20050422.075233-8.pom.sha1",
300            "wagon-1.0-alpha-3-20050429.051847-9.pom",
301            "wagon-1.0-alpha-3-20050429.051847-9.pom.md5",
302            "wagon-1.0-alpha-3-20050429.051847-9.pom.sha1",
303            "wagon-file/",
304            "wagon-ftp/",
305            "wagon-http-lightweight/",
306            "wagon-http/",
307            "wagon-lightweight-http/",
308            "wagon-provider-api/",
309            "wagon-provider-test/",
310            "wagon-provider/",
311            "wagon-providers-1.0-alpha-2.pom",
312            "wagon-providers-1.0-alpha-2.pom.asc",
313            "wagon-providers-1.0-alpha-2.pom.md5",
314            "wagon-providers-1.0-alpha-2.pom.sha1",
315            "wagon-providers-1.0-alpha-3-20050407.202848-1.pom",
316            "wagon-providers-1.0-alpha-3-20050407.202848-1.pom.md5",
317            "wagon-providers-1.0-alpha-3-20050407.202848-1.pom.sha1",
318            "wagon-providers-1.0-alpha-3-20050419.044035-2.pom",
319            "wagon-providers-1.0-alpha-3-20050419.044035-2.pom.md5",
320            "wagon-providers-1.0-alpha-3-20050419.044035-2.pom.sha1",
321            "wagon-providers-1.0-alpha-3-20050421.162738-3.pom",
322            "wagon-providers-1.0-alpha-3-20050421.162738-3.pom.md5",
323            "wagon-providers-1.0-alpha-3-20050421.162738-3.pom.sha1",
324            "wagon-providers-1.0-alpha-3-20050422.075233-4.pom",
325            "wagon-providers-1.0-alpha-3-20050422.075233-4.pom.md5",
326            "wagon-providers-1.0-alpha-3-20050422.075233-4.pom.sha1",
327            "wagon-providers-1.0-alpha-3-20050429.051847-5.pom",
328            "wagon-providers-1.0-alpha-3-20050429.051847-5.pom.md5",
329            "wagon-providers-1.0-alpha-3-20050429.051847-5.pom.sha1",
330            "wagon-providers/",
331            "wagon-scm/",
332            "wagon-ssh-external/",
333            "wagon-ssh/",
334            "wagon-webdav/",
335            "wagon/" };
336
337        assertContainsExpected( links, expected );
338
339        String[] avoided = new String[] { "/org/", "?C=S;O=A", "?C=D;O=A", "?C=M;O=A" };
340
341        assertNotContainingAvoided( links, avoided );
342    }
343
344    /**
345     * Using repository.codehaus.org output as an example.
346     * This is an example of an older RHEL installation of apache httpd with old fancy indexing output
347     * This example tests how to detect directories properly.
348     * 
349     * @throws TransferFailedException
350     */
351    public void testParseMevenIde()
352        throws TransferFailedException
353    {
354        List<String> links = parseLinks( "http://repository.codehaus.org/org/codehaus/mevenide/", "mevenide.html" );
355
356        String[] expected = new String[] {
357            "apisupport/",
358            "autoupdate/",
359            "continuum-rpc/",
360            "continuum/",
361            "debugger-bridge/",
362            "deployment-bridge/",
363            "feature/",
364            "grammar/",
365            "ide-mojos/",
366            "indexer/",
367            "j2ee/",
368            "junit/",
369            "maven-metadata.xml",
370            "maven-metadata.xml.md5",
371            "maven-metadata.xml.sha1",
372            "mevenide2-parent/",
373            "nb-mvn-embedder/",
374            "nb-project/",
375            "nb-repo-browser/",
376            "netbeans-debugger-plugin/",
377            "netbeans-deploy-plugin/",
378            "netbeans-libs/",
379            "netbeans-nbmreload-plugin/",
380            "netbeans-repository/",
381            "netbeans-run-plugin/",
382            "netbeans/",
383            "persistence/",
384            "plugin-bridges/",
385            "plugins/",
386            "reload-nbm-bridge/",
387            "run-jar-bridge/" };
388
389        assertContainsExpected( links, expected );
390
391        String[] avoided = new String[] { "/org/codehaus/", "?C=S;O=A", "?C=D;O=A", "?C=M;O=A" };
392
393        assertNotContainingAvoided( links, avoided );
394    }
395
396    public void testParseMirror()
397        throws Exception
398    {
399        List<String> links = parseLinks( "http://www.ibiblio.org/maven2/org/apache/maven/wagon/",
400                                             "mirror-wagon.html" );
401
402        String[] expected = new String[] {
403            "wagon-1.0-alpha-2.pom",
404            "wagon-1.0-alpha-2.pom.asc",
405            "wagon-1.0-alpha-2.pom.md5",
406            "wagon-1.0-alpha-2.pom.sha1",
407            "wagon-1.0-alpha-3-20050413.021234-4.pom",
408            "wagon-1.0-alpha-3-20050413.021234-4.pom.md5",
409            "wagon-1.0-alpha-3-20050413.021234-4.pom.sha1",
410            "wagon-1.0-alpha-3-20050419.043745-5.pom",
411            "wagon-1.0-alpha-3-20050419.043745-5.pom.md5",
412            "wagon-1.0-alpha-3-20050419.043745-5.pom.sha1",
413            "wagon-1.0-alpha-3-20050419.044035-6.pom",
414            "wagon-1.0-alpha-3-20050419.044035-6.pom.md5",
415            "wagon-1.0-alpha-3-20050419.044035-6.pom.sha1",
416            "wagon-1.0-alpha-3-20050421.162738-7.pom",
417            "wagon-1.0-alpha-3-20050421.162738-7.pom.md5",
418            "wagon-1.0-alpha-3-20050421.162738-7.pom.sha1",
419            "wagon-1.0-alpha-3-20050422.075233-8.pom",
420            "wagon-1.0-alpha-3-20050422.075233-8.pom.md5",
421            "wagon-1.0-alpha-3-20050422.075233-8.pom.sha1",
422            "wagon-1.0-alpha-3-20050429.051847-9.pom",
423            "wagon-1.0-alpha-3-20050429.051847-9.pom.md5",
424            "wagon-1.0-alpha-3-20050429.051847-9.pom.sha1",
425            "wagon-file/",
426            "wagon-ftp/",
427            "wagon-http-lightweight/",
428            "wagon-http/",
429            "wagon-lightweight-http/",
430            "wagon-provider-api/",
431            "wagon-provider-test/",
432            "wagon-provider/",
433            "wagon-providers-1.0-alpha-2.pom",
434            "wagon-providers-1.0-alpha-2.pom.asc",
435            "wagon-providers-1.0-alpha-2.pom.md5",
436            "wagon-providers-1.0-alpha-2.pom.sha1",
437            "wagon-providers-1.0-alpha-3-20050407.202848-1.pom",
438            "wagon-providers-1.0-alpha-3-20050407.202848-1.pom.md5",
439            "wagon-providers-1.0-alpha-3-20050407.202848-1.pom.sha1",
440            "wagon-providers-1.0-alpha-3-20050419.044035-2.pom",
441            "wagon-providers-1.0-alpha-3-20050419.044035-2.pom.md5",
442            "wagon-providers-1.0-alpha-3-20050419.044035-2.pom.sha1",
443            "wagon-providers-1.0-alpha-3-20050421.162738-3.pom",
444            "wagon-providers-1.0-alpha-3-20050421.162738-3.pom.md5",
445            "wagon-providers-1.0-alpha-3-20050421.162738-3.pom.sha1",
446            "wagon-providers-1.0-alpha-3-20050422.075233-4.pom",
447            "wagon-providers-1.0-alpha-3-20050422.075233-4.pom.md5",
448            "wagon-providers-1.0-alpha-3-20050422.075233-4.pom.sha1",
449            "wagon-providers-1.0-alpha-3-20050429.051847-5.pom",
450            "wagon-providers-1.0-alpha-3-20050429.051847-5.pom.md5",
451            "wagon-providers-1.0-alpha-3-20050429.051847-5.pom.sha1",
452            "wagon-providers/",
453            "wagon-scm/",
454            "wagon-ssh-external/",
455            "wagon-ssh/",
456            "wagon-webdav/",
457            "wagon/" };
458
459        assertContainsExpected( links, expected );
460
461        String[] avoided = new String[] {
462            "/org/codehaus/",
463            "?C=S;O=A",
464            "?C=D;O=A",
465            "?C=M;O=A",
466            "mailto:mirror.admin@mirror.com" };
467
468        assertNotContainingAvoided( links, avoided );
469    }
470
471    /**
472     * Example of output from repo1.maven.org
473     * This example is of nekohtml specifically.
474     * 
475     * @throws TransferFailedException
476     */
477    public void testParseNekoHtml()
478        throws TransferFailedException
479    {
480        List<String> links = parseLinks( "http://repo1.maven.org//maven2/nekohtml/nekohtml/1.9.6/", "nekohtml.html" );
481
482        String[] expected = new String[] {
483            "nekohtml-1.9.6-javadoc.jar",
484            "nekohtml-1.9.6-javadoc.jar.md5",
485            "nekohtml-1.9.6-javadoc.jar.sha1",
486            "nekohtml-1.9.6-sources.jar",
487            "nekohtml-1.9.6-sources.jar.md5",
488            "nekohtml-1.9.6-sources.jar.sha1",
489            "nekohtml-1.9.6.jar",
490            "nekohtml-1.9.6.jar.md5",
491            "nekohtml-1.9.6.jar.sha1",
492            "nekohtml-1.9.6.pom",
493            "nekohtml-1.9.6.pom.md5",
494            "nekohtml-1.9.6.pom.sha1" };
495
496        assertContainsExpected( links, expected );
497
498        String[] avoided = new String[] { "/maven2/nekohtml/nekohtml/", "?C=S;O=A", "?C=D;O=A", "?C=M;O=A" };
499
500        assertNotContainingAvoided( links, avoided );
501    }
502
503    /**
504     * Example of detecting directories on repo1.maven.org
505     * 
506     * @throws TransferFailedException
507     */
508    public void testParseNetSourceforge()
509        throws TransferFailedException
510    {
511        List<String> links = parseLinks( "http://repo1.maven.org/maven2/net/sf/", "net_sf.html" );
512
513        String[] expected = new String[] {
514            "a2j/",
515            "aislib/",
516            "alchim/",
517            "antenna/",
518            "apt-jelly/",
519            "beanlib/",
520            "bluecove/",
521            "buildbox/",
522            "click/",
523            "clirr/",
524            "datavision/",
525            "dozer/",
526            "dtddoc/",
527            "dynpageplus/",
528            "ehcache/",
529            "ezmorph/",
530            "falcon/",
531            "grester/",
532            "gwt-widget/",
533            "hermesftp/",
534            "hibernate/",
535            "jcharts/",
536            "jdatabaseimport/",
537            "jeceira/",
538            "jfcunit/",
539            "jfig/",
540            "jguard/",
541            "jipcam/",
542            "jlynx/",
543            "jour/",
544            "jpf/",
545            "json-lib/",
546            "jsptest/",
547            "jsr107cache/",
548            "jt400/",
549            "jxls/",
550            "kxml/",
551            "ldaptemplate/",
552            "locale4j/",
553            "mapasuta/",
554            "maven-har/",
555            "maven-sar/",
556            "opencsv/",
557            "oval/",
558            "proguard/",
559            "qdwizard/",
560            "resultsetmapper/",
561            "retrotranslator/",
562            "saxon/",
563            "shadesdb/",
564            "smc/",
565            "speculoos/",
566            "springlayout/",
567            "stat-scm/",
568            "statsvn/",
569            "stax/",
570            "struts/",
571            "tacos/",
572            "testextensions/",
573            "webdav-servlet/" };
574
575        assertContainsExpected( links, expected );
576
577        String[] avoided = new String[] { "/maven2/net/", "?C=S;O=A", "?C=D;O=A", "?C=M;O=A" };
578
579        assertNotContainingAvoided( links, avoided );
580    }
581
582    /**
583     * Another larger example of the directory link detection on repository.codehaus.org
584     * 
585     * @throws TransferFailedException
586     */
587    public void testParseOrgCodehaus()
588        throws TransferFailedException
589    {
590        List<String> links = parseLinks( "http://repository.codehaus.org/org/codehaus", "org.codehaus.html" );
591
592        String[] expected = new String[] {
593            "agilifier/",
594            "benji/",
595            "bruce/",
596            "btm/",
597            "cargo/",
598            "castor-spring/",
599            "castor/",
600            "cozmos/",
601            "dataforge/",
602            "dimple/",
603            "droolsdotnet/",
604            "enunciate/",
605            "fabric3/",
606            "gant/",
607            "grails-plugins/",
608            "groovy/",
609            "gsoc/",
610            "guessencoding/",
611            "gumtree/",
612            "gwt-openlayers/",
613            "haus/",
614            "izpack/",
615            "javasim/",
616            "jedi/",
617            "jequel/",
618            "jet/",
619            "jettison/",
620            "jfdi/",
621            "jikesrvm/",
622            "jra/",
623            "jremoting/",
624            "jtestme/",
625            "jtestr/",
626            "labs-ng/",
627            "logicabyss/",
628            "marionette/",
629            "mevenide/",
630            "modello/",
631            "mojo/",
632            "mvel/",
633            "mvflex/",
634            "native-mojo/",
635            "openim/",
636            "plexus/",
637            "polymap/",
638            "prometheus/",
639            "prophit/",
640            "quaere/",
641            "redback/",
642            "rulessandpit/",
643            "rvm/",
644            "savana/",
645            "scala-ide/",
646            "senro/",
647            "sonar/",
648            "staxmate/",
649            "stomp/",
650            "svn4j/",
651            "swiby/",
652            "swizzle/",
653            "sxc/",
654            "testdox/",
655            "wadi/",
656            "waffle/",
657            "woodstox/",
658            "xdoclet/",
659            "xfire/",
660            "xharness/",
661            "xsite/",
662            "xstream/",
663            "xwire/" };
664
665        assertContainsExpected( links, expected );
666
667        String[] avoided = new String[] { "/org/", "?C=S;O=A", "?C=D;O=A", "?C=M;O=A" };
668
669        assertNotContainingAvoided( links, avoided );
670    }
671
672    /**
673     * Test the output found from apache httpd with fancy indexing and dav module.
674     * Using people.apache.org output as source material.
675     * 
676     * @throws TransferFailedException
677     */
678    public void testParsePeopleApacheOrg()
679        throws TransferFailedException
680    {
681        List<String> links = parseLinks(
682                                             "http://people.apache.org/repo/m2-ibiblio-rsync-repository/org/apache/maven/archiva/archiva-plexus-runtime/1.0.1/",
683                                             "org.apache.maven.html" );
684
685        String[] expected = new String[] {
686            "archiva-plexus-runtime-1.0.1-bin.tar.gz",
687            "archiva-plexus-runtime-1.0.1-bin.tar.gz.asc",
688            "archiva-plexus-runtime-1.0.1-bin.tar.gz.asc.md5",
689            "archiva-plexus-runtime-1.0.1-bin.tar.gz.asc.sha1",
690            "archiva-plexus-runtime-1.0.1-bin.tar.gz.md5",
691            "archiva-plexus-runtime-1.0.1-bin.tar.gz.sha1",
692            "archiva-plexus-runtime-1.0.1-bin.zip",
693            "archiva-plexus-runtime-1.0.1-bin.zip.asc",
694            "archiva-plexus-runtime-1.0.1-bin.zip.asc.md5",
695            "archiva-plexus-runtime-1.0.1-bin.zip.asc.sha1",
696            "archiva-plexus-runtime-1.0.1-bin.zip.md5",
697            "archiva-plexus-runtime-1.0.1-bin.zip.sha1",
698            "archiva-plexus-runtime-1.0.1-sources.jar",
699            "archiva-plexus-runtime-1.0.1-sources.jar.asc",
700            "archiva-plexus-runtime-1.0.1-sources.jar.asc.md5",
701            "archiva-plexus-runtime-1.0.1-sources.jar.asc.sha1",
702            "archiva-plexus-runtime-1.0.1-sources.jar.md5",
703            "archiva-plexus-runtime-1.0.1-sources.jar.sha1",
704            "archiva-plexus-runtime-1.0.1.jar",
705            "archiva-plexus-runtime-1.0.1.jar.asc",
706            "archiva-plexus-runtime-1.0.1.jar.asc.md5",
707            "archiva-plexus-runtime-1.0.1.jar.asc.sha1",
708            "archiva-plexus-runtime-1.0.1.jar.md5",
709            "archiva-plexus-runtime-1.0.1.jar.sha1",
710            "archiva-plexus-runtime-1.0.1.pom",
711            "archiva-plexus-runtime-1.0.1.pom.asc",
712            "archiva-plexus-runtime-1.0.1.pom.asc.md5",
713            "archiva-plexus-runtime-1.0.1.pom.asc.sha1",
714            "archiva-plexus-runtime-1.0.1.pom.md5",
715            "archiva-plexus-runtime-1.0.1.pom.sha1",
716            "readme artifacts.txt"};
717
718        assertContainsExpected( links, expected );
719
720        String[] avoided = new String[] {
721            "/repo/m2-ibiblio-rsync-repository/org/apache/maven/archiva/archiva-plexus-runtime/",
722            "?C=S;O=A",
723            "?C=D;O=A",
724            "?C=M;O=A" };
725
726        assertNotContainingAvoided( links, avoided );
727    }
728}