merge r1373478:

IVY-1060 : take into account the provided encoding of the page (Thanks to Robin Fernandes)


git-svn-id: https://svn.apache.org/repos/asf/ant/ivy/core/branches/2.3.x@1373479 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -41,6 +41,7 @@ for detailed view of each issue, please consult http://issues.apache.org/jira/br
 	Stefan De Boey
 	Martin Eigenbrodt
 	Stephen Evanchik
+	Robin Fernandes
 	Gregory Fernandez
 	Danno Ferrin
 	Benjamin Francisoud	
@@ -136,6 +137,7 @@ for detailed view of each issue, please consult http://issues.apache.org/jira/br
 - FIX: buildlist task chokes on absolute path to parent Ivy module (IVY-1364) (thanks to Mitch Gitman and Jean-Louis Boudart)
 - FIX: The ignore circular dependency strategy is clobbering the warn strategy (IVY-1353) (Thanks to Carl Quinn)
 - FIX: Buildnumber and IvyFindRevision Ant tasks should honour defaultBranch setting (IVY-1344) (Thanks to Ales Nosek)
+- FIX: ApacheURLLister.retrieveListing() fails if the encoding of the URL list is different from the default encoding (IVY-1060) (Thanks to Robin Fernandes)
 
    2.3.0-rc1
 =====================================

src/java/org/apache/ivy/util/url/ApacheURLLister.java

@@ -19,6 +19,7 @@ package org.apache.ivy.util.url;
 
 import java.io.BufferedReader;
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.net.URL;
 import java.util.ArrayList;
@@ -106,8 +107,10 @@ public class ApacheURLLister {
             url = new URL(url.getProtocol(), url.getHost(), url.getPort(), url.getPath() + "/");
         }
 
-        BufferedReader r = new BufferedReader(new InputStreamReader(URLHandlerRegistry.getDefault()
-                .openStream(url)));
+        URLHandler urlHandler = URLHandlerRegistry.getDefault();
+        String charset = urlHandler.getURLInfo(url).getBodyCharset();
+        InputStream contentStream = urlHandler.openStream(url);
+        BufferedReader r = new BufferedReader(new InputStreamReader(contentStream, charset));
 
         String htmlText = FileUtil.readEntirely(r);
 

src/java/org/apache/ivy/util/url/BasicURLHandler.java

@@ -71,14 +71,17 @@ public class BasicURLHandler extends AbstractURLHandler {
                     httpCon.setRequestMethod("HEAD");
                 }
                 if (checkStatusCode(url, httpCon)) {
-                    return new URLInfo(true, httpCon.getContentLength(), con.getLastModified());
+                    String bodyCharset = getCharSetFromContentType(con.getContentType());
+                    return new URLInfo(true, httpCon.getContentLength(), con.getLastModified(), bodyCharset);
                 }
             } else {
                 int contentLength = con.getContentLength();
                 if (contentLength <= 0) {
                     return UNAVAILABLE;
                 } else {
-                    return new URLInfo(true, contentLength, con.getLastModified());
+                    // TODO: not HTTP... maybe we *don't* want to default to ISO-8559-1 here?
+                    String bodyCharset = getCharSetFromContentType(con.getContentType());
+                    return new URLInfo(true, contentLength, con.getLastModified(), bodyCharset);
                 }
             }
         } catch (UnknownHostException e) {
@@ -93,6 +96,34 @@ public class BasicURLHandler extends AbstractURLHandler {
         return UNAVAILABLE;
     }
 
+    /**
+     * Extract the charset from the Content-Type header string, or default to ISO-8859-1 as per
+     * rfc2616-sec3.html#sec3.7.1 .
+     * 
+     * @param contentType
+     *            the Content-Type header string
+     * @return the charset as specified in the content type, or ISO-8859-1 if unspecified.
+     */
+    public static String getCharSetFromContentType(String contentType) {
+
+        String charSet = null;
+
+        String[] elements = contentType.split(";");
+        for (int i = 0; i < elements.length; i++) {
+            String element = elements[i].trim();
+            if (element.toLowerCase().startsWith("charset=")) {
+                charSet = element.substring("charset=".length());
+            }
+        }
+
+        if (charSet == null || charSet.length() == 0) {
+            // default to ISO-8859-1 as per rfc2616-sec3.html#sec3.7.1
+            charSet = "ISO-8859-1";
+        }
+
+        return charSet;
+    }
+
     private boolean checkStatusCode(URL url, HttpURLConnection con) throws IOException {
         int status = con.getResponseCode();
         if (status == HttpStatus.SC_OK) {

src/java/org/apache/ivy/util/url/HttpClientHandler.java

@@ -153,7 +153,9 @@ public class HttpClientHandler extends AbstractURLHandler {
                 method = doGet(url, timeout);
             }
             if (checkStatusCode(url, method)) {
-                return new URLInfo(true, getResponseContentLength(method), getLastModified(method));
+                String contentType = method.getResponseHeader("content-type").getValue();
+                String bodyCharset = BasicURLHandler.getCharSetFromContentType(contentType);
+                return new URLInfo(true, getResponseContentLength(method), getLastModified(method), bodyCharset);
             }
         } catch (HttpException e) {
             Message.error("HttpClientHandler: " + e.getMessage() + ":" + e.getReasonCode() + "="

src/java/org/apache/ivy/util/url/URLHandler.java

@@ -48,10 +48,17 @@ public interface URLHandler {
 
         private boolean available;
 
+        private String bodyCharset;
+
         protected URLInfo(boolean available, long contentLength, long lastModified) {
+            this(available, contentLength, lastModified, null);
+        }
+
+        protected URLInfo(boolean available, long contentLength, long lastModified, String bodyCharset) {
             this.available = available;
             this.contentLength = contentLength;
             this.lastModified = lastModified;
+            this.bodyCharset = bodyCharset;
         }
 
         public boolean isReachable() {
@@ -65,6 +72,10 @@ public interface URLHandler {
         public long getLastModified() {
             return lastModified;
         }
+
+        public String getBodyCharset() {
+            return bodyCharset;
+        }
     }
 
     public static final URLInfo UNAVAILABLE = new URLInfo(false, 0, 0);