URLFetcher.java revision 871fe6ed66e9de1369fbc7e4a145f98272b88c0b
16a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen/* 26a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen * Copyright (C) 2015 The Android Open Source Project 36a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen * 46a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen * Licensed under the Apache License, Version 2.0 (the "License"); 56a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen * you may not use this file except in compliance with the License. 66a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen * You may obtain a copy of the License at 76a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen * 86a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen * http://www.apache.org/licenses/LICENSE-2.0 96a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen * 106a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen * Unless required by applicable law or agreed to in writing, software 116a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen * distributed under the License is distributed on an "AS IS" BASIS, 126a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 136a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen * See the License for the specific language governing permissions and 146a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen * limitations under the License. 156a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen */ 166a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen 176a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wenpackage com.android.statementservice.retriever; 186a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen 198c7d99c2b77acbcbbdcbf0dcab61a07767d5dd1bJoseph Wenimport android.util.Log; 208c7d99c2b77acbcbbdcbf0dcab61a07767d5dd1bJoseph Wen 216a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wenimport com.android.volley.Cache; 226a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wenimport com.android.volley.NetworkResponse; 236a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wenimport com.android.volley.toolbox.HttpHeaderParser; 246a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen 256a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wenimport java.io.BufferedInputStream; 266a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wenimport java.io.ByteArrayOutputStream; 276a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wenimport java.io.IOException; 286a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wenimport java.io.InputStream; 296a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wenimport java.net.HttpURLConnection; 306a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wenimport java.net.URL; 316a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wenimport java.util.HashMap; 326a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wenimport java.util.List; 336a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wenimport java.util.Locale; 346a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wenimport java.util.Map; 356a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen 366a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen/** 376a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen * Helper class for fetching HTTP or HTTPS URL. 386a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen * 396a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen * Visible for testing. 406a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen * 416a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen * @hide 426a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen */ 436a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wenpublic class URLFetcher { 448c7d99c2b77acbcbbdcbf0dcab61a07767d5dd1bJoseph Wen private static final String TAG = URLFetcher.class.getSimpleName(); 456a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen 466a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen private static final long DO_NOT_CACHE_RESULT = 0L; 476a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen private static final int INPUT_BUFFER_SIZE_IN_BYTES = 1024; 486a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen 496a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen /** 506a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen * Fetches the specified url and returns the content and ttl. 516a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen * 526a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen * @throws IOException if it can't retrieve the content due to a network problem. 536a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen * @throws AssociationServiceException if the URL scheme is not http or https or the content 546a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen * length exceeds {code fileSizeLimit}. 556a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen */ 566a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen public WebContent getWebContentFromUrl(URL url, long fileSizeLimit, int connectionTimeoutMillis) 576a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen throws AssociationServiceException, IOException { 586a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen final String scheme = url.getProtocol().toLowerCase(Locale.US); 596a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen if (!scheme.equals("http") && !scheme.equals("https")) { 606a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen throw new IllegalArgumentException("The url protocol should be on http or https."); 616a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen } 626a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen 63871fe6ed66e9de1369fbc7e4a145f98272b88c0bJoseph Wen HttpURLConnection connection = null; 64871fe6ed66e9de1369fbc7e4a145f98272b88c0bJoseph Wen try { 65871fe6ed66e9de1369fbc7e4a145f98272b88c0bJoseph Wen connection = (HttpURLConnection) url.openConnection(); 66871fe6ed66e9de1369fbc7e4a145f98272b88c0bJoseph Wen connection.setInstanceFollowRedirects(true); 67871fe6ed66e9de1369fbc7e4a145f98272b88c0bJoseph Wen connection.setConnectTimeout(connectionTimeoutMillis); 68871fe6ed66e9de1369fbc7e4a145f98272b88c0bJoseph Wen connection.setReadTimeout(connectionTimeoutMillis); 69871fe6ed66e9de1369fbc7e4a145f98272b88c0bJoseph Wen connection.setUseCaches(true); 70871fe6ed66e9de1369fbc7e4a145f98272b88c0bJoseph Wen connection.setInstanceFollowRedirects(false); 71871fe6ed66e9de1369fbc7e4a145f98272b88c0bJoseph Wen connection.addRequestProperty("Cache-Control", "max-stale=60"); 726a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen 73871fe6ed66e9de1369fbc7e4a145f98272b88c0bJoseph Wen if (connection.getResponseCode() != HttpURLConnection.HTTP_OK) { 74871fe6ed66e9de1369fbc7e4a145f98272b88c0bJoseph Wen Log.e(TAG, "The responses code is not 200 but " + connection.getResponseCode()); 75871fe6ed66e9de1369fbc7e4a145f98272b88c0bJoseph Wen return new WebContent("", DO_NOT_CACHE_RESULT); 76871fe6ed66e9de1369fbc7e4a145f98272b88c0bJoseph Wen } 778c7d99c2b77acbcbbdcbf0dcab61a07767d5dd1bJoseph Wen 78871fe6ed66e9de1369fbc7e4a145f98272b88c0bJoseph Wen if (connection.getContentLength() > fileSizeLimit) { 79871fe6ed66e9de1369fbc7e4a145f98272b88c0bJoseph Wen Log.e(TAG, "The content size of the url is larger than " + fileSizeLimit); 80871fe6ed66e9de1369fbc7e4a145f98272b88c0bJoseph Wen return new WebContent("", DO_NOT_CACHE_RESULT); 81871fe6ed66e9de1369fbc7e4a145f98272b88c0bJoseph Wen } 826a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen 83871fe6ed66e9de1369fbc7e4a145f98272b88c0bJoseph Wen Long expireTimeMillis = getExpirationTimeMillisFromHTTPHeader( 84871fe6ed66e9de1369fbc7e4a145f98272b88c0bJoseph Wen connection.getHeaderFields()); 856a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen 866a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen return new WebContent(inputStreamToString( 876a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen connection.getInputStream(), connection.getContentLength(), fileSizeLimit), 886a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen expireTimeMillis); 896a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen } finally { 90871fe6ed66e9de1369fbc7e4a145f98272b88c0bJoseph Wen if (connection != null) { 91871fe6ed66e9de1369fbc7e4a145f98272b88c0bJoseph Wen connection.disconnect(); 92871fe6ed66e9de1369fbc7e4a145f98272b88c0bJoseph Wen } 936a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen } 946a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen } 956a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen 966a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen /** 976a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen * Visible for testing. 986a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen * @hide 996a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen */ 1006a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen public static String inputStreamToString(InputStream inputStream, int length, long sizeLimit) 1016a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen throws IOException, AssociationServiceException { 1026a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen if (length < 0) { 1036a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen length = 0; 1046a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen } 1056a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen ByteArrayOutputStream baos = new ByteArrayOutputStream(length); 1066a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen BufferedInputStream bis = new BufferedInputStream(inputStream); 1076a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen byte[] buffer = new byte[INPUT_BUFFER_SIZE_IN_BYTES]; 1086a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen int len = 0; 1096a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen while ((len = bis.read(buffer)) != -1) { 1106a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen baos.write(buffer, 0, len); 1116a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen if (baos.size() > sizeLimit) { 1126a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen throw new AssociationServiceException("The content size of the url is larger than " 1136a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen + sizeLimit); 1146a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen } 1156a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen } 1166a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen return baos.toString("UTF-8"); 1176a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen } 1186a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen 1196a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen /** 1206a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen * Parses the HTTP headers to compute the ttl. 1216a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen * 1226a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen * @param headers a map that map the header key to the header values. Can be null. 1236a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen * @return the ttl in millisecond or null if the ttl is not specified in the header. 1246a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen */ 1256a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen private Long getExpirationTimeMillisFromHTTPHeader(Map<String, List<String>> headers) { 1266a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen if (headers == null) { 1276a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen return null; 1286a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen } 1296a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen Map<String, String> joinedHeaders = joinHttpHeaders(headers); 1306a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen 1316a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen NetworkResponse response = new NetworkResponse(null, joinedHeaders); 1326a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen Cache.Entry cachePolicy = HttpHeaderParser.parseCacheHeaders(response); 1336a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen 1346a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen if (cachePolicy == null) { 1356a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen // Cache is disabled, set the expire time to 0. 1366a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen return DO_NOT_CACHE_RESULT; 1376a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen } else if (cachePolicy.ttl == 0) { 1386a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen // Cache policy is not specified, set the expire time to 0. 1396a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen return DO_NOT_CACHE_RESULT; 1406a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen } else { 1416a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen // cachePolicy.ttl is actually the expire timestamp in millisecond. 1426a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen return cachePolicy.ttl; 1436a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen } 1446a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen } 1456a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen 1466a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen /** 1476a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen * Converts an HTTP header map of the format provided by {@linkHttpUrlConnection} to a map of 1486a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen * the format accepted by {@link HttpHeaderParser}. It does this by joining all the entries for 1496a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen * a given header key with ", ". 1506a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen */ 1516a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen private Map<String, String> joinHttpHeaders(Map<String, List<String>> headers) { 1526a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen Map<String, String> joinedHeaders = new HashMap<String, String>(); 1536a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen for (Map.Entry<String, List<String>> entry : headers.entrySet()) { 1546a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen List<String> values = entry.getValue(); 1556a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen if (values.size() == 1) { 1566a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen joinedHeaders.put(entry.getKey(), values.get(0)); 1576a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen } else { 1586a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen joinedHeaders.put(entry.getKey(), Utils.joinStrings(", ", values)); 1596a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen } 1606a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen } 1616a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen return joinedHeaders; 1626a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen } 1636a34bb2d6a6cbc7a70bdf0c53d238dc28e0b1d58Joseph Wen} 164