HttpEngine.java revision 5d7e0fc1af3141aa41e9c21d74da3c36b933517f
1/*
2 *  Licensed to the Apache Software Foundation (ASF) under one or more
3 *  contributor license agreements.  See the NOTICE file distributed with
4 *  this work for additional information regarding copyright ownership.
5 *  The ASF licenses this file to You under the Apache License, Version 2.0
6 *  (the "License"); you may not use this file except in compliance with
7 *  the License.  You may obtain a copy of the License at
8 *
9 *     http://www.apache.org/licenses/LICENSE-2.0
10 *
11 *  Unless required by applicable law or agreed to in writing, software
12 *  distributed under the License is distributed on an "AS IS" BASIS,
13 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 *  See the License for the specific language governing permissions and
15 *  limitations under the License.
16 */
17
18package libcore.net.http;
19
20import java.io.BufferedOutputStream;
21import java.io.ByteArrayInputStream;
22import java.io.IOException;
23import java.io.InputStream;
24import java.io.OutputStream;
25import java.net.CacheRequest;
26import java.net.CacheResponse;
27import java.net.CookieHandler;
28import java.net.HttpURLConnection;
29import java.net.Proxy;
30import java.net.ResponseCache;
31import java.net.URI;
32import java.net.URISyntaxException;
33import java.net.URL;
34import java.nio.charset.Charsets;
35import java.util.Collections;
36import java.util.Date;
37import java.util.HashMap;
38import java.util.List;
39import java.util.Map;
40import java.util.zip.GZIPInputStream;
41import libcore.io.IoUtils;
42import libcore.io.Streams;
43import libcore.util.EmptyArray;
44
45/**
46 * Handles a single HTTP request/response pair. Each HTTP engine follows this
47 * lifecycle:
48 * <ol>
49 *     <li>It is created.
50 *     <li>The HTTP request message is sent with sendRequest(). Once the request
51 *         is sent it is an error to modify the request headers. After
52 *         sendRequest() has been called the request body can be written to if
53 *         it exists.
54 *     <li>The HTTP response message is read with readResponse(). After the
55 *         response has been read the response headers and body can be read.
56 *         All responses have a response body input stream, though in some
57 *         instances this stream is empty.
58 * </ol>
59 *
60 * <p>The request and response may be served by the HTTP response cache, by the
61 * network, or by both in the event of a conditional GET.
62 *
63 * <p>This class may hold a socket connection that needs to be released or
64 * recycled. By default, this socket connection is held when the last byte of
65 * the response is consumed. To release the connection when it is no longer
66 * required, use {@link #automaticallyReleaseConnectionToPool()}.
67 */
68public class HttpEngine {
69    private static final CacheResponse BAD_GATEWAY_RESPONSE = new CacheResponse() {
70        @Override public Map<String, List<String>> getHeaders() throws IOException {
71            Map<String, List<String>> result = new HashMap<String, List<String>>();
72            result.put(null, Collections.singletonList("HTTP/1.1 502 Bad Gateway"));
73            return result;
74        }
75        @Override public InputStream getBody() throws IOException {
76            return new ByteArrayInputStream(EmptyArray.BYTE);
77        }
78    };
79
80    /**
81     * The maximum number of bytes to buffer when sending headers and a request
82     * body. When the headers and body can be sent in a single write, the
83     * request completes sooner. In one WiFi benchmark, using a large enough
84     * buffer sped up some uploads by half.
85     */
86    private static final int MAX_REQUEST_BUFFER_LENGTH = 32768;
87
88    public static final int DEFAULT_CHUNK_LENGTH = 1024;
89
90    public static final String OPTIONS = "OPTIONS";
91    public static final String GET = "GET";
92    public static final String HEAD = "HEAD";
93    public static final String POST = "POST";
94    public static final String PUT = "PUT";
95    public static final String DELETE = "DELETE";
96    public static final String TRACE = "TRACE";
97    public static final String CONNECT = "CONNECT";
98
99    public static final int HTTP_CONTINUE = 100;
100
101    /**
102     * HTTP 1.1 doesn't specify how many redirects to follow, but HTTP/1.0
103     * recommended 5. http://www.w3.org/Protocols/HTTP/1.0/spec.html#Code3xx
104     */
105    public static final int MAX_REDIRECTS = 5;
106
107    protected final HttpURLConnectionImpl policy;
108
109    protected final String method;
110
111    private ResponseSource responseSource;
112
113    protected HttpConnection connection;
114    private InputStream socketIn;
115    private OutputStream socketOut;
116
117    /**
118     * This stream buffers the request headers and the request body when their
119     * combined size is less than MAX_REQUEST_BUFFER_LENGTH. By combining them
120     * we can save socket writes, which in turn saves a packet transmission.
121     * This is socketOut if the request size is large or unknown.
122     */
123    private OutputStream requestOut;
124    private AbstractHttpOutputStream requestBodyOut;
125
126    private InputStream responseBodyIn;
127
128    private final ResponseCache responseCache = ResponseCache.getDefault();
129    private CacheResponse cacheResponse;
130    private CacheRequest cacheRequest;
131
132    /** The time when the request headers were written, or -1 if they haven't been written yet. */
133    private long sentRequestMillis = -1;
134
135    /**
136     * True if this client added an "Accept-Encoding: gzip" header field and is
137     * therefore responsible for also decompressing the transfer stream.
138     */
139    private boolean transparentGzip;
140
141    boolean sendChunked;
142
143    /**
144     * The version this client will use. Either 0 for HTTP/1.0, or 1 for
145     * HTTP/1.1. Upon receiving a non-HTTP/1.1 response, this client
146     * automatically sets its version to HTTP/1.0.
147     */
148    // TODO: is HTTP minor version tracked across HttpEngines?
149    private int httpMinorVersion = 1; // Assume HTTP/1.1
150
151    private final URI uri;
152
153    private final RequestHeaders requestHeaders;
154
155    /** Null until a response is received from the network or the cache */
156    private ResponseHeaders responseHeaders;
157
158    /*
159     * The cache response currently being validated on a conditional get. Null
160     * if the cached response doesn't exist or doesn't need validation. If the
161     * conditional get succeeds, these will be used for the response headers and
162     * body. If it fails, these be closed and set to null.
163     */
164    private ResponseHeaders cachedResponseHeaders;
165    private InputStream cachedResponseBody;
166
167    /**
168     * True if the socket connection should be released to the connection pool
169     * when the response has been fully read.
170     */
171    private boolean automaticallyReleaseConnectionToPool;
172
173    /** True if the socket connection is no longer needed by this engine. */
174    private boolean connectionReleased;
175
176    /**
177     * @param requestHeaders the client's supplied request headers. This class
178     *     creates a private copy that it can mutate.
179     * @param connection the connection used for an intermediate response
180     *     immediately prior to this request/response pair, such as a same-host
181     *     redirect. This engine assumes ownership of the connection and must
182     *     release it when it is unneeded.
183     */
184    public HttpEngine(HttpURLConnectionImpl policy, String method, RawHeaders requestHeaders,
185            HttpConnection connection, RetryableOutputStream requestBodyOut) throws IOException {
186        this.policy = policy;
187        this.method = method;
188        this.connection = connection;
189        this.requestBodyOut = requestBodyOut;
190
191        try {
192            uri = policy.getURL().toURILenient();
193        } catch (URISyntaxException e) {
194            throw new IOException(e);
195        }
196
197        this.requestHeaders = new RequestHeaders(uri, new RawHeaders(requestHeaders));
198    }
199
200    public URI getUri() {
201        return uri;
202    }
203
204    /**
205     * Figures out what the response source will be, and opens a socket to that
206     * source if necessary. Prepares the request headers and gets ready to start
207     * writing the request body if it exists.
208     */
209    public final void sendRequest() throws IOException {
210        if (responseSource != null) {
211            return;
212        }
213
214        prepareRawRequestHeaders();
215        initResponseSource();
216        if (responseCache instanceof HttpResponseCache) {
217            ((HttpResponseCache) responseCache).trackResponse(responseSource);
218        }
219
220        /*
221         * The raw response source may require the network, but the request
222         * headers may forbid network use. In that case, dispose of the network
223         * response and use a BAD_GATEWAY response instead.
224         */
225        if (requestHeaders.isOnlyIfCached() && responseSource.requiresConnection()) {
226            if (responseSource == ResponseSource.CONDITIONAL_CACHE) {
227                IoUtils.closeQuietly(cachedResponseBody);
228            }
229            this.responseSource = ResponseSource.CACHE;
230            this.cacheResponse = BAD_GATEWAY_RESPONSE;
231            RawHeaders rawResponseHeaders = RawHeaders.fromMultimap(cacheResponse.getHeaders());
232            setResponse(new ResponseHeaders(uri, rawResponseHeaders), cacheResponse.getBody());
233        }
234
235        if (responseSource.requiresConnection()) {
236            sendSocketRequest();
237        } else if (connection != null) {
238            HttpConnectionPool.INSTANCE.recycle(connection);
239            connection = null;
240        }
241    }
242
243    /**
244     * Initialize the source for this response. It may be corrected later if the
245     * request headers forbids network use.
246     */
247    private void initResponseSource() throws IOException {
248        responseSource = ResponseSource.NETWORK;
249        if (!policy.getUseCaches() || responseCache == null) {
250            return;
251        }
252
253        CacheResponse candidate = responseCache.get(uri, method,
254                requestHeaders.getHeaders().toMultimap());
255        if (candidate == null) {
256            return;
257        }
258
259        Map<String, List<String>> responseHeadersMap = candidate.getHeaders();
260        cachedResponseBody = candidate.getBody();
261        if (!acceptCacheResponseType(candidate)
262                || responseHeadersMap == null
263                || cachedResponseBody == null) {
264            IoUtils.closeQuietly(cachedResponseBody);
265            return;
266        }
267
268        RawHeaders rawResponseHeaders = RawHeaders.fromMultimap(responseHeadersMap);
269        cachedResponseHeaders = new ResponseHeaders(uri, rawResponseHeaders);
270        long now = System.currentTimeMillis();
271        this.responseSource = cachedResponseHeaders.chooseResponseSource(now, requestHeaders);
272        if (responseSource == ResponseSource.CACHE) {
273            this.cacheResponse = candidate;
274            setResponse(cachedResponseHeaders, cachedResponseBody);
275        } else if (responseSource == ResponseSource.CONDITIONAL_CACHE) {
276            this.cacheResponse = candidate;
277        } else if (responseSource == ResponseSource.NETWORK) {
278            IoUtils.closeQuietly(cachedResponseBody);
279        } else {
280            throw new AssertionError();
281        }
282    }
283
284    private void sendSocketRequest() throws IOException {
285        if (connection == null) {
286            connect();
287        }
288
289        if (socketOut != null || requestOut != null || socketIn != null) {
290            throw new IllegalStateException();
291        }
292
293        socketOut = connection.getOutputStream();
294        requestOut = socketOut;
295        socketIn = connection.getInputStream();
296
297        if (hasRequestBody()) {
298            initRequestBodyOut();
299        }
300    }
301
302    /**
303     * Connect to the origin server either directly or via a proxy.
304     */
305    protected void connect() throws IOException {
306        if (connection == null) {
307            connection = openSocketConnection();
308        }
309    }
310
311    protected final HttpConnection openSocketConnection() throws IOException {
312        HttpConnection result = HttpConnection.connect(
313                uri, policy.getProxy(), requiresTunnel(), policy.getConnectTimeout());
314        Proxy proxy = result.getAddress().getProxy();
315        if (proxy != null) {
316            policy.setProxy(proxy);
317        }
318        result.setSoTimeout(policy.getReadTimeout());
319        return result;
320    }
321
322    protected void initRequestBodyOut() throws IOException {
323        int chunkLength = policy.getChunkLength();
324        if (chunkLength > 0 || requestHeaders.isChunked()) {
325            sendChunked = true;
326            if (chunkLength == -1) {
327                chunkLength = DEFAULT_CHUNK_LENGTH;
328            }
329        }
330
331        if (socketOut == null) {
332            throw new IllegalStateException("No socket to write to; was a POST cached?");
333        }
334
335        if (httpMinorVersion == 0) {
336            sendChunked = false;
337        }
338
339        int fixedContentLength = policy.getFixedContentLength();
340        if (requestBodyOut != null) {
341            // request body was already initialized by the predecessor HTTP engine
342        } else if (fixedContentLength != -1) {
343            writeRequestHeaders(fixedContentLength);
344            requestBodyOut = new FixedLengthOutputStream(requestOut, fixedContentLength);
345        } else if (sendChunked) {
346            writeRequestHeaders(-1);
347            requestBodyOut = new ChunkedOutputStream(requestOut, chunkLength);
348        } else if (requestHeaders.getContentLength() != -1) {
349            writeRequestHeaders(requestHeaders.getContentLength());
350            requestBodyOut = new RetryableOutputStream(requestHeaders.getContentLength());
351        } else {
352            requestBodyOut = new RetryableOutputStream();
353        }
354    }
355
356    /**
357     * @param body the response body, or null if it doesn't exist or isn't
358     *     available.
359     */
360    private void setResponse(ResponseHeaders headers, InputStream body) throws IOException {
361        if (this.responseBodyIn != null) {
362            throw new IllegalStateException();
363        }
364        this.responseHeaders = headers;
365        this.httpMinorVersion = responseHeaders.getHeaders().getHttpMinorVersion();
366        if (body != null) {
367            initContentStream(body);
368        }
369    }
370
371    private boolean hasRequestBody() {
372        return method == POST || method == PUT;
373    }
374
375    /**
376     * Returns the request body or null if this request doesn't have a body.
377     */
378    public final OutputStream getRequestBody() {
379        if (responseSource == null) {
380            throw new IllegalStateException();
381        }
382        return requestBodyOut;
383    }
384
385    public final boolean hasResponse() {
386        return responseHeaders != null;
387    }
388
389    public final RequestHeaders getRequestHeaders() {
390        return requestHeaders;
391    }
392
393    public final ResponseHeaders getResponseHeaders() {
394        if (responseHeaders == null) {
395            throw new IllegalStateException();
396        }
397        return responseHeaders;
398    }
399
400    public final int getResponseCode() {
401        if (responseHeaders == null) {
402            throw new IllegalStateException();
403        }
404        return responseHeaders.getHeaders().getResponseCode();
405    }
406
407    public final InputStream getResponseBody() {
408        if (responseHeaders == null) {
409            throw new IllegalStateException();
410        }
411        return responseBodyIn;
412    }
413
414    public final CacheResponse getCacheResponse() {
415        if (responseHeaders == null) {
416            throw new IllegalStateException();
417        }
418        return cacheResponse;
419    }
420
421    public final HttpConnection getConnection() {
422        return connection;
423    }
424
425    public final boolean hasRecycledConnection() {
426        return connection != null && connection.isRecycled();
427    }
428
429    /**
430     * Returns true if {@code cacheResponse} is of the right type. This
431     * condition is necessary but not sufficient for the cached response to
432     * be used.
433     */
434    protected boolean acceptCacheResponseType(CacheResponse cacheResponse) {
435        return true;
436    }
437
438    private void maybeCache() throws IOException {
439        // Are we caching at all?
440        if (!policy.getUseCaches() || responseCache == null) {
441            return;
442        }
443
444        // Should we cache this response for this request?
445        if (!responseHeaders.isCacheable(requestHeaders)) {
446            return;
447        }
448
449        // Offer this request to the cache.
450        cacheRequest = responseCache.put(uri, getHttpConnectionToCache());
451    }
452
453    protected HttpURLConnection getHttpConnectionToCache() {
454        return policy;
455    }
456
457    /**
458     * Cause the socket connection to be released to the connection pool when
459     * it is no longer needed. If it is already unneeded, it will be pooled
460     * immediately.
461     */
462    public final void automaticallyReleaseConnectionToPool() {
463        automaticallyReleaseConnectionToPool = true;
464        if (connection != null && connectionReleased) {
465            HttpConnectionPool.INSTANCE.recycle(connection);
466            connection = null;
467        }
468    }
469
470    /**
471     * Releases this engine so that its resources may be either reused or
472     * closed.
473     */
474    public final void release(boolean reusable) {
475        // If the response body comes from the cache, close it.
476        if (responseBodyIn == cachedResponseBody) {
477            IoUtils.closeQuietly(responseBodyIn);
478        }
479
480        if (!connectionReleased && connection != null) {
481            connectionReleased = true;
482
483            // We cannot reuse sockets that have incomplete output.
484            if (requestBodyOut != null && !requestBodyOut.closed) {
485                reusable = false;
486            }
487
488            // If the headers specify that the connection shouldn't be reused, don't reuse it.
489            if (hasConnectionCloseHeader()) {
490                reusable = false;
491            }
492
493            if (responseBodyIn instanceof UnknownLengthHttpInputStream) {
494                reusable = false;
495            }
496
497            if (reusable && responseBodyIn != null) {
498                // We must discard the response body before the connection can be reused.
499                try {
500                    Streams.skipAll(responseBodyIn);
501                } catch (IOException e) {
502                    reusable = false;
503                }
504            }
505
506            if (!reusable) {
507                connection.closeSocketAndStreams();
508                connection = null;
509            } else if (automaticallyReleaseConnectionToPool) {
510                HttpConnectionPool.INSTANCE.recycle(connection);
511                connection = null;
512            }
513        }
514    }
515
516    private void initContentStream(InputStream transferStream) throws IOException {
517        if (transparentGzip && responseHeaders.isContentEncodingGzip()) {
518            /*
519             * If the response was transparently gzipped, remove the gzip header field
520             * so clients don't double decompress. http://b/3009828
521             */
522            responseHeaders.stripContentEncoding();
523            responseBodyIn = new GZIPInputStream(transferStream);
524        } else {
525            responseBodyIn = transferStream;
526        }
527    }
528
529    private InputStream getTransferStream() throws IOException {
530        if (!hasResponseBody()) {
531            return new FixedLengthInputStream(socketIn, cacheRequest, this, 0);
532        }
533
534        if (responseHeaders.isChunked()) {
535            return new ChunkedInputStream(socketIn, cacheRequest, this);
536        }
537
538        if (responseHeaders.getContentLength() != -1) {
539            return new FixedLengthInputStream(socketIn, cacheRequest, this,
540                    responseHeaders.getContentLength());
541        }
542
543        /*
544         * Wrap the input stream from the HttpConnection (rather than
545         * just returning "socketIn" directly here), so that we can control
546         * its use after the reference escapes.
547         */
548        return new UnknownLengthHttpInputStream(socketIn, cacheRequest, this);
549    }
550
551    private void readResponseHeaders() throws IOException {
552        RawHeaders headers;
553        do {
554            headers = new RawHeaders();
555            headers.setStatusLine(Streams.readAsciiLine(socketIn));
556            readHeaders(headers);
557        } while (headers.getResponseCode() == HTTP_CONTINUE);
558        setResponse(new ResponseHeaders(uri, headers), null);
559    }
560
561    /**
562     * Returns true if the response must have a (possibly 0-length) body.
563     * See RFC 2616 section 4.3.
564     */
565    public final boolean hasResponseBody() {
566        int responseCode = responseHeaders.getHeaders().getResponseCode();
567        if (method != HEAD
568                && method != CONNECT
569                && (responseCode < HTTP_CONTINUE || responseCode >= 200)
570                && responseCode != HttpURLConnectionImpl.HTTP_NO_CONTENT
571                && responseCode != HttpURLConnectionImpl.HTTP_NOT_MODIFIED) {
572            return true;
573        }
574
575        /*
576         * If the Content-Length or Transfer-Encoding headers disagree with the
577         * response code, the response is malformed. For best compatibility, we
578         * honor the headers.
579         */
580        if (responseHeaders.getContentLength() != -1 || responseHeaders.isChunked()) {
581            return true;
582        }
583
584        return false;
585    }
586
587    /**
588     * Trailers are headers included after the last chunk of a response encoded
589     * with chunked encoding.
590     */
591    final void readTrailers() throws IOException {
592        readHeaders(responseHeaders.getHeaders());
593    }
594
595    private void readHeaders(RawHeaders headers) throws IOException {
596        // parse the result headers until the first blank line
597        String line;
598        while (!(line = Streams.readAsciiLine(socketIn)).isEmpty()) {
599            headers.addLine(line);
600        }
601
602        CookieHandler cookieHandler = CookieHandler.getDefault();
603        if (cookieHandler != null) {
604            cookieHandler.put(uri, headers.toMultimap());
605        }
606    }
607
608    /**
609     * Prepares the HTTP headers and sends them to the server.
610     *
611     * <p>For streaming requests with a body, headers must be prepared
612     * <strong>before</strong> the output stream has been written to. Otherwise
613     * the body would need to be buffered!
614     *
615     * <p>For non-streaming requests with a body, headers must be prepared
616     * <strong>after</strong> the output stream has been written to and closed.
617     * This ensures that the {@code Content-Length} header field receives the
618     * proper value.
619     *
620     * @param contentLength the number of bytes in the request body, or -1 if
621     *      the request body length is unknown.
622     */
623    private void writeRequestHeaders(int contentLength) throws IOException {
624        if (sentRequestMillis != -1) {
625            throw new IllegalStateException();
626        }
627
628        RawHeaders headersToSend = getNetworkRequestHeaders();
629        byte[] bytes = headersToSend.toHeaderString().getBytes(Charsets.ISO_8859_1);
630
631        if (contentLength != -1 && bytes.length + contentLength <= MAX_REQUEST_BUFFER_LENGTH) {
632            requestOut = new BufferedOutputStream(socketOut, bytes.length + contentLength);
633        }
634
635        sentRequestMillis = System.currentTimeMillis();
636        requestOut.write(bytes);
637    }
638
639    /**
640     * Returns the headers to send on a network request.
641     *
642     * <p>This adds the content length and content-type headers, which are
643     * neither needed nor known when querying the response cache.
644     *
645     * <p>It updates the status line, which may need to be fully qualified if
646     * the connection is using a proxy.
647     */
648    protected RawHeaders getNetworkRequestHeaders() throws IOException {
649        requestHeaders.getHeaders().setStatusLine(getRequestLine());
650
651        int fixedContentLength = policy.getFixedContentLength();
652        if (fixedContentLength != -1) {
653            requestHeaders.setContentLength(fixedContentLength);
654        } else if (sendChunked) {
655            requestHeaders.setChunked();
656        } else if (requestBodyOut instanceof RetryableOutputStream) {
657            int contentLength = ((RetryableOutputStream) requestBodyOut).contentLength();
658            requestHeaders.setContentLength(contentLength);
659        }
660
661        return requestHeaders.getHeaders();
662    }
663
664    /**
665     * Populates requestHeaders with defaults and cookies.
666     *
667     * <p>This client doesn't specify a default {@code Accept} header because it
668     * doesn't know what content types the application is interested in.
669     */
670    private void prepareRawRequestHeaders() throws IOException {
671        requestHeaders.getHeaders().setStatusLine(getRequestLine());
672
673        if (requestHeaders.getUserAgent() == null) {
674            requestHeaders.setUserAgent(getDefaultUserAgent());
675        }
676
677        if (requestHeaders.getHost() == null) {
678            requestHeaders.setHost(getOriginAddress(policy.getURL()));
679        }
680
681        if (httpMinorVersion > 0 && requestHeaders.getConnection() == null) {
682            requestHeaders.setConnection("Keep-Alive");
683        }
684
685        if (requestHeaders.getAcceptEncoding() == null) {
686            transparentGzip = true;
687            requestHeaders.setAcceptEncoding("gzip");
688        }
689
690        if (hasRequestBody() && requestHeaders.getContentType() == null) {
691            requestHeaders.setContentType("application/x-www-form-urlencoded");
692        }
693
694        long ifModifiedSince = policy.getIfModifiedSince();
695        if (ifModifiedSince != 0) {
696            requestHeaders.setIfModifiedSince(new Date(ifModifiedSince));
697        }
698
699        CookieHandler cookieHandler = CookieHandler.getDefault();
700        if (cookieHandler != null) {
701            requestHeaders.addCookies(
702                    cookieHandler.get(uri, requestHeaders.getHeaders().toMultimap()));
703        }
704    }
705
706    private String getRequestLine() {
707        String protocol = (httpMinorVersion == 0) ? "HTTP/1.0" : "HTTP/1.1";
708        return method + " " + requestString() + " " + protocol;
709    }
710
711    private String requestString() {
712        URL url = policy.getURL();
713        if (includeAuthorityInRequestLine()) {
714            return url.toString();
715        } else {
716            String fileOnly = url.getFile();
717            if (fileOnly == null) {
718                fileOnly = "/";
719            } else if (!fileOnly.startsWith("/")) {
720                fileOnly = "/" + fileOnly;
721            }
722            return fileOnly;
723        }
724    }
725
726    /**
727     * Returns true if the request line should contain the full URL with host
728     * and port (like "GET http://android.com/foo HTTP/1.1") or only the path
729     * (like "GET /foo HTTP/1.1").
730     *
731     * <p>This is non-final because for HTTPS it's never necessary to supply the
732     * full URL, even if a proxy is in use.
733     */
734    protected boolean includeAuthorityInRequestLine() {
735        return policy.usingProxy();
736    }
737
738    protected final String getDefaultUserAgent() {
739        String agent = System.getProperty("http.agent");
740        return agent != null ? agent : ("Java" + System.getProperty("java.version"));
741    }
742
743    private boolean hasConnectionCloseHeader() {
744        return (responseHeaders != null && responseHeaders.hasConnectionClose())
745                || requestHeaders.hasConnectionClose();
746    }
747
748    protected final String getOriginAddress(URL url) {
749        int port = url.getPort();
750        String result = url.getHost();
751        if (port > 0 && port != policy.getDefaultPort()) {
752            result = result + ":" + port;
753        }
754        return result;
755    }
756
757    protected boolean requiresTunnel() {
758        return false;
759    }
760
761    /**
762     * Flushes the remaining request header and body, parses the HTTP response
763     * headers and starts reading the HTTP response body if it exists.
764     */
765    public final void readResponse() throws IOException {
766        if (hasResponse()) {
767            return;
768        }
769
770        if (responseSource == null) {
771            throw new IllegalStateException("readResponse() without sendRequest()");
772        }
773
774        if (!responseSource.requiresConnection()) {
775            return;
776        }
777
778        if (sentRequestMillis == -1) {
779            int contentLength = requestBodyOut instanceof RetryableOutputStream
780                    ? ((RetryableOutputStream) requestBodyOut).contentLength()
781                    : -1;
782            writeRequestHeaders(contentLength);
783        }
784
785        if (requestBodyOut != null) {
786            requestBodyOut.close();
787            if (requestBodyOut instanceof RetryableOutputStream) {
788                ((RetryableOutputStream) requestBodyOut).writeToSocket(requestOut);
789            }
790        }
791
792        requestOut.flush();
793        requestOut = socketOut;
794
795        readResponseHeaders();
796        responseHeaders.setLocalTimestamps(sentRequestMillis, System.currentTimeMillis());
797
798        if (responseSource == ResponseSource.CONDITIONAL_CACHE) {
799            if (cachedResponseHeaders.validate(responseHeaders)) {
800                release(true);
801                ResponseHeaders combinedHeaders = cachedResponseHeaders.combine(responseHeaders);
802                setResponse(combinedHeaders, cachedResponseBody);
803                if (responseCache instanceof HttpResponseCache) {
804                    HttpResponseCache httpResponseCache = (HttpResponseCache) responseCache;
805                    httpResponseCache.trackConditionalCacheHit();
806                    httpResponseCache.update(cacheResponse, getHttpConnectionToCache());
807                }
808                return;
809            } else {
810                IoUtils.closeQuietly(cachedResponseBody);
811            }
812        }
813
814        if (hasResponseBody()) {
815            maybeCache(); // reentrant. this calls into user code which may call back into this!
816        }
817
818        initContentStream(getTransferStream());
819    }
820}
821