1/*************************************************************************** 2 * _ _ ____ _ 3 * Project ___| | | | _ \| | 4 * / __| | | | |_) | | 5 * | (__| |_| | _ <| |___ 6 * \___|\___/|_| \_\_____| 7 * 8 * Copyright (C) 1998 - 2016, Daniel Stenberg, <daniel@haxx.se>, et al. 9 * 10 * This software is licensed as described in the file COPYING, which 11 * you should have received as part of this distribution. The terms 12 * are also available at https://curl.haxx.se/docs/copyright.html. 13 * 14 * You may opt to use, copy, modify, merge, publish, distribute and/or sell 15 * copies of the Software, and permit persons to whom the Software is 16 * furnished to do so, under the terms of the COPYING file. 17 * 18 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY 19 * KIND, either express or implied. 20 * 21 ***************************************************************************/ 22 23#include "curl_setup.h" 24 25#include "strtoofft.h" 26#include "strequal.h" 27#include "rawstr.h" 28 29#ifdef HAVE_NETINET_IN_H 30#include <netinet/in.h> 31#endif 32#ifdef HAVE_NETDB_H 33#include <netdb.h> 34#endif 35#ifdef HAVE_ARPA_INET_H 36#include <arpa/inet.h> 37#endif 38#ifdef HAVE_NET_IF_H 39#include <net/if.h> 40#endif 41#ifdef HAVE_SYS_IOCTL_H 42#include <sys/ioctl.h> 43#endif 44#ifdef HAVE_SIGNAL_H 45#include <signal.h> 46#endif 47 48#ifdef HAVE_SYS_PARAM_H 49#include <sys/param.h> 50#endif 51 52#ifdef HAVE_SYS_SELECT_H 53#include <sys/select.h> 54#endif 55 56#ifndef HAVE_SOCKET 57#error "We can't compile without socket() support!" 58#endif 59 60#include "urldata.h" 61#include <curl/curl.h> 62#include "netrc.h" 63 64#include "content_encoding.h" 65#include "hostip.h" 66#include "transfer.h" 67#include "sendf.h" 68#include "speedcheck.h" 69#include "progress.h" 70#include "http.h" 71#include "url.h" 72#include "getinfo.h" 73#include "vtls/vtls.h" 74#include "select.h" 75#include "multiif.h" 76#include "connect.h" 77#include "non-ascii.h" 78 79/* The last 3 #include files should be in this order */ 80#include "curl_printf.h" 81#include "curl_memory.h" 82#include "memdebug.h" 83 84/* 85 * This function will call the read callback to fill our buffer with data 86 * to upload. 87 */ 88CURLcode Curl_fillreadbuffer(struct connectdata *conn, int bytes, int *nreadp) 89{ 90 struct Curl_easy *data = conn->data; 91 size_t buffersize = (size_t)bytes; 92 int nread; 93#ifdef CURL_DOES_CONVERSIONS 94 bool sending_http_headers = FALSE; 95 96 if(conn->handler->protocol&(PROTO_FAMILY_HTTP|CURLPROTO_RTSP)) { 97 const struct HTTP *http = data->req.protop; 98 99 if(http->sending == HTTPSEND_REQUEST) 100 /* We're sending the HTTP request headers, not the data. 101 Remember that so we don't re-translate them into garbage. */ 102 sending_http_headers = TRUE; 103 } 104#endif 105 106 if(data->req.upload_chunky) { 107 /* if chunked Transfer-Encoding */ 108 buffersize -= (8 + 2 + 2); /* 32bit hex + CRLF + CRLF */ 109 data->req.upload_fromhere += (8 + 2); /* 32bit hex + CRLF */ 110 } 111 112 /* this function returns a size_t, so we typecast to int to prevent warnings 113 with picky compilers */ 114 nread = (int)data->state.fread_func(data->req.upload_fromhere, 1, 115 buffersize, data->state.in); 116 117 if(nread == CURL_READFUNC_ABORT) { 118 failf(data, "operation aborted by callback"); 119 *nreadp = 0; 120 return CURLE_ABORTED_BY_CALLBACK; 121 } 122 else if(nread == CURL_READFUNC_PAUSE) { 123 124 if(conn->handler->flags & PROTOPT_NONETWORK) { 125 /* protocols that work without network cannot be paused. This is 126 actually only FILE:// just now, and it can't pause since the transfer 127 isn't done using the "normal" procedure. */ 128 failf(data, "Read callback asked for PAUSE when not supported!"); 129 return CURLE_READ_ERROR; 130 } 131 else { 132 struct SingleRequest *k = &data->req; 133 /* CURL_READFUNC_PAUSE pauses read callbacks that feed socket writes */ 134 k->keepon |= KEEP_SEND_PAUSE; /* mark socket send as paused */ 135 if(data->req.upload_chunky) { 136 /* Back out the preallocation done above */ 137 data->req.upload_fromhere -= (8 + 2); 138 } 139 *nreadp = 0; 140 } 141 return CURLE_OK; /* nothing was read */ 142 } 143 else if((size_t)nread > buffersize) { 144 /* the read function returned a too large value */ 145 *nreadp = 0; 146 failf(data, "read function returned funny value"); 147 return CURLE_READ_ERROR; 148 } 149 150 if(!data->req.forbidchunk && data->req.upload_chunky) { 151 /* if chunked Transfer-Encoding 152 * build chunk: 153 * 154 * <HEX SIZE> CRLF 155 * <DATA> CRLF 156 */ 157 /* On non-ASCII platforms the <DATA> may or may not be 158 translated based on set.prefer_ascii while the protocol 159 portion must always be translated to the network encoding. 160 To further complicate matters, line end conversion might be 161 done later on, so we need to prevent CRLFs from becoming 162 CRCRLFs if that's the case. To do this we use bare LFs 163 here, knowing they'll become CRLFs later on. 164 */ 165 166 char hexbuffer[11]; 167 const char *endofline_native; 168 const char *endofline_network; 169 int hexlen; 170 171 if( 172#ifdef CURL_DO_LINEEND_CONV 173 (data->set.prefer_ascii) || 174#endif 175 (data->set.crlf)) { 176 /* \n will become \r\n later on */ 177 endofline_native = "\n"; 178 endofline_network = "\x0a"; 179 } 180 else { 181 endofline_native = "\r\n"; 182 endofline_network = "\x0d\x0a"; 183 } 184 hexlen = snprintf(hexbuffer, sizeof(hexbuffer), 185 "%x%s", nread, endofline_native); 186 187 /* move buffer pointer */ 188 data->req.upload_fromhere -= hexlen; 189 nread += hexlen; 190 191 /* copy the prefix to the buffer, leaving out the NUL */ 192 memcpy(data->req.upload_fromhere, hexbuffer, hexlen); 193 194 /* always append ASCII CRLF to the data */ 195 memcpy(data->req.upload_fromhere + nread, 196 endofline_network, 197 strlen(endofline_network)); 198 199#ifdef CURL_DOES_CONVERSIONS 200 CURLcode result; 201 int length; 202 if(data->set.prefer_ascii) { 203 /* translate the protocol and data */ 204 length = nread; 205 } 206 else { 207 /* just translate the protocol portion */ 208 length = strlen(hexbuffer); 209 } 210 result = Curl_convert_to_network(data, data->req.upload_fromhere, length); 211 /* Curl_convert_to_network calls failf if unsuccessful */ 212 if(result) 213 return result; 214#endif /* CURL_DOES_CONVERSIONS */ 215 216 if((nread - hexlen) == 0) 217 /* mark this as done once this chunk is transferred */ 218 data->req.upload_done = TRUE; 219 220 nread+=(int)strlen(endofline_native); /* for the added end of line */ 221 } 222#ifdef CURL_DOES_CONVERSIONS 223 else if((data->set.prefer_ascii) && (!sending_http_headers)) { 224 CURLcode result; 225 result = Curl_convert_to_network(data, data->req.upload_fromhere, nread); 226 /* Curl_convert_to_network calls failf if unsuccessful */ 227 if(result) 228 return result; 229 } 230#endif /* CURL_DOES_CONVERSIONS */ 231 232 *nreadp = nread; 233 234 return CURLE_OK; 235} 236 237 238/* 239 * Curl_readrewind() rewinds the read stream. This is typically used for HTTP 240 * POST/PUT with multi-pass authentication when a sending was denied and a 241 * resend is necessary. 242 */ 243CURLcode Curl_readrewind(struct connectdata *conn) 244{ 245 struct Curl_easy *data = conn->data; 246 247 conn->bits.rewindaftersend = FALSE; /* we rewind now */ 248 249 /* explicitly switch off sending data on this connection now since we are 250 about to restart a new transfer and thus we want to avoid inadvertently 251 sending more data on the existing connection until the next transfer 252 starts */ 253 data->req.keepon &= ~KEEP_SEND; 254 255 /* We have sent away data. If not using CURLOPT_POSTFIELDS or 256 CURLOPT_HTTPPOST, call app to rewind 257 */ 258 if(data->set.postfields || 259 (data->set.httpreq == HTTPREQ_POST_FORM)) 260 ; /* do nothing */ 261 else { 262 if(data->set.seek_func) { 263 int err; 264 265 err = (data->set.seek_func)(data->set.seek_client, 0, SEEK_SET); 266 if(err) { 267 failf(data, "seek callback returned error %d", (int)err); 268 return CURLE_SEND_FAIL_REWIND; 269 } 270 } 271 else if(data->set.ioctl_func) { 272 curlioerr err; 273 274 err = (data->set.ioctl_func)(data, CURLIOCMD_RESTARTREAD, 275 data->set.ioctl_client); 276 infof(data, "the ioctl callback returned %d\n", (int)err); 277 278 if(err) { 279 /* FIXME: convert to a human readable error message */ 280 failf(data, "ioctl callback returned error %d", (int)err); 281 return CURLE_SEND_FAIL_REWIND; 282 } 283 } 284 else { 285 /* If no CURLOPT_READFUNCTION is used, we know that we operate on a 286 given FILE * stream and we can actually attempt to rewind that 287 ourselves with fseek() */ 288 if(data->state.fread_func == (curl_read_callback)fread) { 289 if(-1 != fseek(data->state.in, 0, SEEK_SET)) 290 /* successful rewind */ 291 return CURLE_OK; 292 } 293 294 /* no callback set or failure above, makes us fail at once */ 295 failf(data, "necessary data rewind wasn't possible"); 296 return CURLE_SEND_FAIL_REWIND; 297 } 298 } 299 return CURLE_OK; 300} 301 302static int data_pending(const struct connectdata *conn) 303{ 304 /* in the case of libssh2, we can never be really sure that we have emptied 305 its internal buffers so we MUST always try until we get EAGAIN back */ 306 return conn->handler->protocol&(CURLPROTO_SCP|CURLPROTO_SFTP) || 307#if defined(USE_NGHTTP2) 308 Curl_ssl_data_pending(conn, FIRSTSOCKET) || 309 /* For HTTP/2, we may read up everything including responde body 310 with header fields in Curl_http_readwrite_headers. If no 311 content-length is provided, curl waits for the connection 312 close, which we emulate it using conn->proto.httpc.closed = 313 TRUE. The thing is if we read everything, then http2_recv won't 314 be called and we cannot signal the HTTP/2 stream has closed. As 315 a workaround, we return nonzero here to call http2_recv. */ 316 ((conn->handler->protocol&PROTO_FAMILY_HTTP) && conn->httpversion == 20); 317#else 318 Curl_ssl_data_pending(conn, FIRSTSOCKET); 319#endif 320} 321 322static void read_rewind(struct connectdata *conn, 323 size_t thismuch) 324{ 325 DEBUGASSERT(conn->read_pos >= thismuch); 326 327 conn->read_pos -= thismuch; 328 conn->bits.stream_was_rewound = TRUE; 329 330#ifdef DEBUGBUILD 331 { 332 char buf[512 + 1]; 333 size_t show; 334 335 show = CURLMIN(conn->buf_len - conn->read_pos, sizeof(buf)-1); 336 if(conn->master_buffer) { 337 memcpy(buf, conn->master_buffer + conn->read_pos, show); 338 buf[show] = '\0'; 339 } 340 else { 341 buf[0] = '\0'; 342 } 343 344 DEBUGF(infof(conn->data, 345 "Buffer after stream rewind (read_pos = %zu): [%s]\n", 346 conn->read_pos, buf)); 347 } 348#endif 349} 350 351/* 352 * Check to see if CURLOPT_TIMECONDITION was met by comparing the time of the 353 * remote document with the time provided by CURLOPT_TIMEVAL 354 */ 355bool Curl_meets_timecondition(struct Curl_easy *data, time_t timeofdoc) 356{ 357 if((timeofdoc == 0) || (data->set.timevalue == 0)) 358 return TRUE; 359 360 switch(data->set.timecondition) { 361 case CURL_TIMECOND_IFMODSINCE: 362 default: 363 if(timeofdoc <= data->set.timevalue) { 364 infof(data, 365 "The requested document is not new enough\n"); 366 data->info.timecond = TRUE; 367 return FALSE; 368 } 369 break; 370 case CURL_TIMECOND_IFUNMODSINCE: 371 if(timeofdoc >= data->set.timevalue) { 372 infof(data, 373 "The requested document is not old enough\n"); 374 data->info.timecond = TRUE; 375 return FALSE; 376 } 377 break; 378 } 379 380 return TRUE; 381} 382 383/* 384 * Go ahead and do a read if we have a readable socket or if 385 * the stream was rewound (in which case we have data in a 386 * buffer) 387 */ 388static CURLcode readwrite_data(struct Curl_easy *data, 389 struct connectdata *conn, 390 struct SingleRequest *k, 391 int *didwhat, bool *done) 392{ 393 CURLcode result = CURLE_OK; 394 ssize_t nread; /* number of bytes read */ 395 size_t excess = 0; /* excess bytes read */ 396 bool is_empty_data = FALSE; 397 bool readmore = FALSE; /* used by RTP to signal for more data */ 398 int maxloops = 100; 399 400 *done = FALSE; 401 402 /* This is where we loop until we have read everything there is to 403 read or we get a CURLE_AGAIN */ 404 do { 405 size_t buffersize = data->set.buffer_size? 406 data->set.buffer_size : BUFSIZE; 407 size_t bytestoread = buffersize; 408 409 if( 410#if defined(USE_NGHTTP2) 411 /* For HTTP/2, read data without caring about the content 412 length. This is safe because body in HTTP/2 is always 413 segmented thanks to its framing layer. Meanwhile, we have to 414 call Curl_read to ensure that http2_handle_stream_close is 415 called when we read all incoming bytes for a particular 416 stream. */ 417 !((conn->handler->protocol & PROTO_FAMILY_HTTP) && 418 conn->httpversion == 20) && 419#endif 420 k->size != -1 && !k->header) { 421 /* make sure we don't read "too much" if we can help it since we 422 might be pipelining and then someone else might want to read what 423 follows! */ 424 curl_off_t totalleft = k->size - k->bytecount; 425 if(totalleft < (curl_off_t)bytestoread) 426 bytestoread = (size_t)totalleft; 427 } 428 429 if(bytestoread) { 430 /* receive data from the network! */ 431 result = Curl_read(conn, conn->sockfd, k->buf, bytestoread, &nread); 432 433 /* read would've blocked */ 434 if(CURLE_AGAIN == result) 435 break; /* get out of loop */ 436 437 if(result>0) 438 return result; 439 } 440 else { 441 /* read nothing but since we wanted nothing we consider this an OK 442 situation to proceed from */ 443 DEBUGF(infof(data, "readwrite_data: we're done!\n")); 444 nread = 0; 445 } 446 447 if((k->bytecount == 0) && (k->writebytecount == 0)) { 448 Curl_pgrsTime(data, TIMER_STARTTRANSFER); 449 if(k->exp100 > EXP100_SEND_DATA) 450 /* set time stamp to compare with when waiting for the 100 */ 451 k->start100 = Curl_tvnow(); 452 } 453 454 *didwhat |= KEEP_RECV; 455 /* indicates data of zero size, i.e. empty file */ 456 is_empty_data = ((nread == 0) && (k->bodywrites == 0)) ? TRUE : FALSE; 457 458 /* NUL terminate, allowing string ops to be used */ 459 if(0 < nread || is_empty_data) { 460 k->buf[nread] = 0; 461 } 462 else if(0 >= nread) { 463 /* if we receive 0 or less here, the server closed the connection 464 and we bail out from this! */ 465 DEBUGF(infof(data, "nread <= 0, server closed connection, bailing\n")); 466 k->keepon &= ~KEEP_RECV; 467 break; 468 } 469 470 /* Default buffer to use when we write the buffer, it may be changed 471 in the flow below before the actual storing is done. */ 472 k->str = k->buf; 473 474 if(conn->handler->readwrite) { 475 result = conn->handler->readwrite(data, conn, &nread, &readmore); 476 if(result) 477 return result; 478 if(readmore) 479 break; 480 } 481 482#ifndef CURL_DISABLE_HTTP 483 /* Since this is a two-state thing, we check if we are parsing 484 headers at the moment or not. */ 485 if(k->header) { 486 /* we are in parse-the-header-mode */ 487 bool stop_reading = FALSE; 488 result = Curl_http_readwrite_headers(data, conn, &nread, &stop_reading); 489 if(result) 490 return result; 491 492 if(conn->handler->readwrite && 493 (k->maxdownload <= 0 && nread > 0)) { 494 result = conn->handler->readwrite(data, conn, &nread, &readmore); 495 if(result) 496 return result; 497 if(readmore) 498 break; 499 } 500 501 if(stop_reading) { 502 /* We've stopped dealing with input, get out of the do-while loop */ 503 504 if(nread > 0) { 505 if(Curl_pipeline_wanted(conn->data->multi, CURLPIPE_HTTP1)) { 506 infof(data, 507 "Rewinding stream by : %zd" 508 " bytes on url %s (zero-length body)\n", 509 nread, data->state.path); 510 read_rewind(conn, (size_t)nread); 511 } 512 else { 513 infof(data, 514 "Excess found in a non pipelined read:" 515 " excess = %zd" 516 " url = %s (zero-length body)\n", 517 nread, data->state.path); 518 } 519 } 520 521 break; 522 } 523 } 524#endif /* CURL_DISABLE_HTTP */ 525 526 527 /* This is not an 'else if' since it may be a rest from the header 528 parsing, where the beginning of the buffer is headers and the end 529 is non-headers. */ 530 if(k->str && !k->header && (nread > 0 || is_empty_data)) { 531 532#ifndef CURL_DISABLE_HTTP 533 if(0 == k->bodywrites && !is_empty_data) { 534 /* These checks are only made the first time we are about to 535 write a piece of the body */ 536 if(conn->handler->protocol&(PROTO_FAMILY_HTTP|CURLPROTO_RTSP)) { 537 /* HTTP-only checks */ 538 539 if(data->req.newurl) { 540 if(conn->bits.close) { 541 /* Abort after the headers if "follow Location" is set 542 and we're set to close anyway. */ 543 k->keepon &= ~KEEP_RECV; 544 *done = TRUE; 545 return CURLE_OK; 546 } 547 /* We have a new url to load, but since we want to be able 548 to re-use this connection properly, we read the full 549 response in "ignore more" */ 550 k->ignorebody = TRUE; 551 infof(data, "Ignoring the response-body\n"); 552 } 553 if(data->state.resume_from && !k->content_range && 554 (data->set.httpreq==HTTPREQ_GET) && 555 !k->ignorebody) { 556 557 if(k->size == data->state.resume_from) { 558 /* The resume point is at the end of file, consider this fine 559 even if it doesn't allow resume from here. */ 560 infof(data, "The entire document is already downloaded"); 561 connclose(conn, "already downloaded"); 562 /* Abort download */ 563 k->keepon &= ~KEEP_RECV; 564 *done = TRUE; 565 return CURLE_OK; 566 } 567 568 /* we wanted to resume a download, although the server doesn't 569 * seem to support this and we did this with a GET (if it 570 * wasn't a GET we did a POST or PUT resume) */ 571 failf(data, "HTTP server doesn't seem to support " 572 "byte ranges. Cannot resume."); 573 return CURLE_RANGE_ERROR; 574 } 575 576 if(data->set.timecondition && !data->state.range) { 577 /* A time condition has been set AND no ranges have been 578 requested. This seems to be what chapter 13.3.4 of 579 RFC 2616 defines to be the correct action for a 580 HTTP/1.1 client */ 581 582 if(!Curl_meets_timecondition(data, k->timeofdoc)) { 583 *done = TRUE; 584 /* We're simulating a http 304 from server so we return 585 what should have been returned from the server */ 586 data->info.httpcode = 304; 587 infof(data, "Simulate a HTTP 304 response!\n"); 588 /* we abort the transfer before it is completed == we ruin the 589 re-use ability. Close the connection */ 590 connclose(conn, "Simulated 304 handling"); 591 return CURLE_OK; 592 } 593 } /* we have a time condition */ 594 595 } /* this is HTTP or RTSP */ 596 } /* this is the first time we write a body part */ 597#endif /* CURL_DISABLE_HTTP */ 598 599 k->bodywrites++; 600 601 /* pass data to the debug function before it gets "dechunked" */ 602 if(data->set.verbose) { 603 if(k->badheader) { 604 Curl_debug(data, CURLINFO_DATA_IN, data->state.headerbuff, 605 (size_t)k->hbuflen, conn); 606 if(k->badheader == HEADER_PARTHEADER) 607 Curl_debug(data, CURLINFO_DATA_IN, 608 k->str, (size_t)nread, conn); 609 } 610 else 611 Curl_debug(data, CURLINFO_DATA_IN, 612 k->str, (size_t)nread, conn); 613 } 614 615#ifndef CURL_DISABLE_HTTP 616 if(k->chunk) { 617 /* 618 * Here comes a chunked transfer flying and we need to decode this 619 * properly. While the name says read, this function both reads 620 * and writes away the data. The returned 'nread' holds the number 621 * of actual data it wrote to the client. 622 */ 623 624 CHUNKcode res = 625 Curl_httpchunk_read(conn, k->str, nread, &nread); 626 627 if(CHUNKE_OK < res) { 628 if(CHUNKE_WRITE_ERROR == res) { 629 failf(data, "Failed writing data"); 630 return CURLE_WRITE_ERROR; 631 } 632 failf(data, "%s in chunked-encoding", Curl_chunked_strerror(res)); 633 return CURLE_RECV_ERROR; 634 } 635 else if(CHUNKE_STOP == res) { 636 size_t dataleft; 637 /* we're done reading chunks! */ 638 k->keepon &= ~KEEP_RECV; /* read no more */ 639 640 /* There are now possibly N number of bytes at the end of the 641 str buffer that weren't written to the client. 642 643 We DO care about this data if we are pipelining. 644 Push it back to be read on the next pass. */ 645 646 dataleft = conn->chunk.dataleft; 647 if(dataleft != 0) { 648 infof(conn->data, "Leftovers after chunking: %zu bytes\n", 649 dataleft); 650 if(Curl_pipeline_wanted(conn->data->multi, CURLPIPE_HTTP1)) { 651 /* only attempt the rewind if we truly are pipelining */ 652 infof(conn->data, "Rewinding %zu bytes\n",dataleft); 653 read_rewind(conn, dataleft); 654 } 655 } 656 } 657 /* If it returned OK, we just keep going */ 658 } 659#endif /* CURL_DISABLE_HTTP */ 660 661 /* Account for body content stored in the header buffer */ 662 if(k->badheader && !k->ignorebody) { 663 DEBUGF(infof(data, "Increasing bytecount by %zu from hbuflen\n", 664 k->hbuflen)); 665 k->bytecount += k->hbuflen; 666 } 667 668 if((-1 != k->maxdownload) && 669 (k->bytecount + nread >= k->maxdownload)) { 670 671 excess = (size_t)(k->bytecount + nread - k->maxdownload); 672 if(excess > 0 && !k->ignorebody) { 673 if(Curl_pipeline_wanted(conn->data->multi, CURLPIPE_HTTP1)) { 674 /* The 'excess' amount below can't be more than BUFSIZE which 675 always will fit in a size_t */ 676 infof(data, 677 "Rewinding stream by : %zu" 678 " bytes on url %s (size = %" CURL_FORMAT_CURL_OFF_T 679 ", maxdownload = %" CURL_FORMAT_CURL_OFF_T 680 ", bytecount = %" CURL_FORMAT_CURL_OFF_T ", nread = %zd)\n", 681 excess, data->state.path, 682 k->size, k->maxdownload, k->bytecount, nread); 683 read_rewind(conn, excess); 684 } 685 else { 686 infof(data, 687 "Excess found in a non pipelined read:" 688 " excess = %zu" 689 ", size = %" CURL_FORMAT_CURL_OFF_T 690 ", maxdownload = %" CURL_FORMAT_CURL_OFF_T 691 ", bytecount = %" CURL_FORMAT_CURL_OFF_T "\n", 692 excess, k->size, k->maxdownload, k->bytecount); 693 } 694 } 695 696 nread = (ssize_t) (k->maxdownload - k->bytecount); 697 if(nread < 0) /* this should be unusual */ 698 nread = 0; 699 700 k->keepon &= ~KEEP_RECV; /* we're done reading */ 701 } 702 703 k->bytecount += nread; 704 705 Curl_pgrsSetDownloadCounter(data, k->bytecount); 706 707 if(!k->chunk && (nread || k->badheader || is_empty_data)) { 708 /* If this is chunky transfer, it was already written */ 709 710 if(k->badheader && !k->ignorebody) { 711 /* we parsed a piece of data wrongly assuming it was a header 712 and now we output it as body instead */ 713 714 /* Don't let excess data pollute body writes */ 715 if(k->maxdownload == -1 || (curl_off_t)k->hbuflen <= k->maxdownload) 716 result = Curl_client_write(conn, CLIENTWRITE_BODY, 717 data->state.headerbuff, 718 k->hbuflen); 719 else 720 result = Curl_client_write(conn, CLIENTWRITE_BODY, 721 data->state.headerbuff, 722 (size_t)k->maxdownload); 723 724 if(result) 725 return result; 726 } 727 if(k->badheader < HEADER_ALLBAD) { 728 /* This switch handles various content encodings. If there's an 729 error here, be sure to check over the almost identical code 730 in http_chunks.c. 731 Make sure that ALL_CONTENT_ENCODINGS contains all the 732 encodings handled here. */ 733#ifdef HAVE_LIBZ 734 switch (conn->data->set.http_ce_skip ? 735 IDENTITY : k->auto_decoding) { 736 case IDENTITY: 737#endif 738 /* This is the default when the server sends no 739 Content-Encoding header. See Curl_readwrite_init; the 740 memset() call initializes k->auto_decoding to zero. */ 741 if(!k->ignorebody) { 742 743#ifndef CURL_DISABLE_POP3 744 if(conn->handler->protocol&PROTO_FAMILY_POP3) 745 result = Curl_pop3_write(conn, k->str, nread); 746 else 747#endif /* CURL_DISABLE_POP3 */ 748 749 result = Curl_client_write(conn, CLIENTWRITE_BODY, k->str, 750 nread); 751 } 752#ifdef HAVE_LIBZ 753 break; 754 755 case DEFLATE: 756 /* Assume CLIENTWRITE_BODY; headers are not encoded. */ 757 if(!k->ignorebody) 758 result = Curl_unencode_deflate_write(conn, k, nread); 759 break; 760 761 case GZIP: 762 /* Assume CLIENTWRITE_BODY; headers are not encoded. */ 763 if(!k->ignorebody) 764 result = Curl_unencode_gzip_write(conn, k, nread); 765 break; 766 767 default: 768 failf (data, "Unrecognized content encoding type. " 769 "libcurl understands `identity', `deflate' and `gzip' " 770 "content encodings."); 771 result = CURLE_BAD_CONTENT_ENCODING; 772 break; 773 } 774#endif 775 } 776 k->badheader = HEADER_NORMAL; /* taken care of now */ 777 778 if(result) 779 return result; 780 } 781 782 } /* if(!header and data to read) */ 783 784 if(conn->handler->readwrite && 785 (excess > 0 && !conn->bits.stream_was_rewound)) { 786 /* Parse the excess data */ 787 k->str += nread; 788 nread = (ssize_t)excess; 789 790 result = conn->handler->readwrite(data, conn, &nread, &readmore); 791 if(result) 792 return result; 793 794 if(readmore) 795 k->keepon |= KEEP_RECV; /* we're not done reading */ 796 break; 797 } 798 799 if(is_empty_data) { 800 /* if we received nothing, the server closed the connection and we 801 are done */ 802 k->keepon &= ~KEEP_RECV; 803 } 804 805 } while(data_pending(conn) && maxloops--); 806 807 if(((k->keepon & (KEEP_RECV|KEEP_SEND)) == KEEP_SEND) && 808 conn->bits.close) { 809 /* When we've read the entire thing and the close bit is set, the server 810 may now close the connection. If there's now any kind of sending going 811 on from our side, we need to stop that immediately. */ 812 infof(data, "we are done reading and this is set to close, stop send\n"); 813 k->keepon &= ~KEEP_SEND; /* no writing anymore either */ 814 } 815 816 return CURLE_OK; 817} 818 819static CURLcode done_sending(struct connectdata *conn, 820 struct SingleRequest *k) 821{ 822 k->keepon &= ~KEEP_SEND; /* we're done writing */ 823 824 if(conn->bits.rewindaftersend) { 825 CURLcode result = Curl_readrewind(conn); 826 if(result) 827 return result; 828 } 829 return CURLE_OK; 830} 831 832 833/* 834 * Send data to upload to the server, when the socket is writable. 835 */ 836static CURLcode readwrite_upload(struct Curl_easy *data, 837 struct connectdata *conn, 838 struct SingleRequest *k, 839 int *didwhat) 840{ 841 ssize_t i, si; 842 ssize_t bytes_written; 843 CURLcode result; 844 ssize_t nread; /* number of bytes read */ 845 bool sending_http_headers = FALSE; 846 847 if((k->bytecount == 0) && (k->writebytecount == 0)) 848 Curl_pgrsTime(data, TIMER_STARTTRANSFER); 849 850 *didwhat |= KEEP_SEND; 851 852 do { 853 854 /* only read more data if there's no upload data already 855 present in the upload buffer */ 856 if(0 == data->req.upload_present) { 857 /* init the "upload from here" pointer */ 858 data->req.upload_fromhere = k->uploadbuf; 859 860 if(!k->upload_done) { 861 /* HTTP pollution, this should be written nicer to become more 862 protocol agnostic. */ 863 int fillcount; 864 struct HTTP *http = data->req.protop; 865 866 if((k->exp100 == EXP100_SENDING_REQUEST) && 867 (http->sending == HTTPSEND_BODY)) { 868 /* If this call is to send body data, we must take some action: 869 We have sent off the full HTTP 1.1 request, and we shall now 870 go into the Expect: 100 state and await such a header */ 871 k->exp100 = EXP100_AWAITING_CONTINUE; /* wait for the header */ 872 k->keepon &= ~KEEP_SEND; /* disable writing */ 873 k->start100 = Curl_tvnow(); /* timeout count starts now */ 874 *didwhat &= ~KEEP_SEND; /* we didn't write anything actually */ 875 876 /* set a timeout for the multi interface */ 877 Curl_expire(data, data->set.expect_100_timeout); 878 break; 879 } 880 881 if(conn->handler->protocol&(PROTO_FAMILY_HTTP|CURLPROTO_RTSP)) { 882 if(http->sending == HTTPSEND_REQUEST) 883 /* We're sending the HTTP request headers, not the data. 884 Remember that so we don't change the line endings. */ 885 sending_http_headers = TRUE; 886 else 887 sending_http_headers = FALSE; 888 } 889 890 result = Curl_fillreadbuffer(conn, BUFSIZE, &fillcount); 891 if(result) 892 return result; 893 894 nread = (ssize_t)fillcount; 895 } 896 else 897 nread = 0; /* we're done uploading/reading */ 898 899 if(!nread && (k->keepon & KEEP_SEND_PAUSE)) { 900 /* this is a paused transfer */ 901 break; 902 } 903 else if(nread<=0) { 904 result = done_sending(conn, k); 905 if(result) 906 return result; 907 break; 908 } 909 910 /* store number of bytes available for upload */ 911 data->req.upload_present = nread; 912 913 /* convert LF to CRLF if so asked */ 914 if((!sending_http_headers) && ( 915#ifdef CURL_DO_LINEEND_CONV 916 /* always convert if we're FTPing in ASCII mode */ 917 (data->set.prefer_ascii) || 918#endif 919 (data->set.crlf))) { 920 /* Do we need to allocate a scratch buffer? */ 921 if(!data->state.scratch) { 922 data->state.scratch = malloc(2 * BUFSIZE); 923 if(!data->state.scratch) { 924 failf(data, "Failed to alloc scratch buffer!"); 925 926 return CURLE_OUT_OF_MEMORY; 927 } 928 } 929 930 /* 931 * ASCII/EBCDIC Note: This is presumably a text (not binary) 932 * transfer so the data should already be in ASCII. 933 * That means the hex values for ASCII CR (0x0d) & LF (0x0a) 934 * must be used instead of the escape sequences \r & \n. 935 */ 936 for(i = 0, si = 0; i < nread; i++, si++) { 937 if(data->req.upload_fromhere[i] == 0x0a) { 938 data->state.scratch[si++] = 0x0d; 939 data->state.scratch[si] = 0x0a; 940 if(!data->set.crlf) { 941 /* we're here only because FTP is in ASCII mode... 942 bump infilesize for the LF we just added */ 943 if(data->state.infilesize != -1) 944 data->state.infilesize++; 945 } 946 } 947 else 948 data->state.scratch[si] = data->req.upload_fromhere[i]; 949 } 950 951 if(si != nread) { 952 /* only perform the special operation if we really did replace 953 anything */ 954 nread = si; 955 956 /* upload from the new (replaced) buffer instead */ 957 data->req.upload_fromhere = data->state.scratch; 958 959 /* set the new amount too */ 960 data->req.upload_present = nread; 961 } 962 } 963 964#ifndef CURL_DISABLE_SMTP 965 if(conn->handler->protocol & PROTO_FAMILY_SMTP) { 966 result = Curl_smtp_escape_eob(conn, nread); 967 if(result) 968 return result; 969 } 970#endif /* CURL_DISABLE_SMTP */ 971 } /* if 0 == data->req.upload_present */ 972 else { 973 /* We have a partial buffer left from a previous "round". Use 974 that instead of reading more data */ 975 } 976 977 /* write to socket (send away data) */ 978 result = Curl_write(conn, 979 conn->writesockfd, /* socket to send to */ 980 data->req.upload_fromhere, /* buffer pointer */ 981 data->req.upload_present, /* buffer size */ 982 &bytes_written); /* actually sent */ 983 984 if(result) 985 return result; 986 987 if(data->set.verbose) 988 /* show the data before we change the pointer upload_fromhere */ 989 Curl_debug(data, CURLINFO_DATA_OUT, data->req.upload_fromhere, 990 (size_t)bytes_written, conn); 991 992 k->writebytecount += bytes_written; 993 994 if(k->writebytecount == data->state.infilesize) { 995 /* we have sent all data we were supposed to */ 996 k->upload_done = TRUE; 997 infof(data, "We are completely uploaded and fine\n"); 998 } 999 1000 if(data->req.upload_present != bytes_written) { 1001 /* we only wrote a part of the buffer (if anything), deal with it! */ 1002 1003 /* store the amount of bytes left in the buffer to write */ 1004 data->req.upload_present -= bytes_written; 1005 1006 /* advance the pointer where to find the buffer when the next send 1007 is to happen */ 1008 data->req.upload_fromhere += bytes_written; 1009 } 1010 else { 1011 /* we've uploaded that buffer now */ 1012 data->req.upload_fromhere = k->uploadbuf; 1013 data->req.upload_present = 0; /* no more bytes left */ 1014 1015 if(k->upload_done) { 1016 result = done_sending(conn, k); 1017 if(result) 1018 return result; 1019 } 1020 } 1021 1022 Curl_pgrsSetUploadCounter(data, k->writebytecount); 1023 1024 } WHILE_FALSE; /* just to break out from! */ 1025 1026 return CURLE_OK; 1027} 1028 1029/* 1030 * Curl_readwrite() is the low-level function to be called when data is to 1031 * be read and written to/from the connection. 1032 */ 1033CURLcode Curl_readwrite(struct connectdata *conn, 1034 struct Curl_easy *data, 1035 bool *done) 1036{ 1037 struct SingleRequest *k = &data->req; 1038 CURLcode result; 1039 int didwhat=0; 1040 1041 curl_socket_t fd_read; 1042 curl_socket_t fd_write; 1043 int select_res = conn->cselect_bits; 1044 1045 conn->cselect_bits = 0; 1046 1047 /* only use the proper socket if the *_HOLD bit is not set simultaneously as 1048 then we are in rate limiting state in that transfer direction */ 1049 1050 if((k->keepon & KEEP_RECVBITS) == KEEP_RECV) 1051 fd_read = conn->sockfd; 1052 else 1053 fd_read = CURL_SOCKET_BAD; 1054 1055 if((k->keepon & KEEP_SENDBITS) == KEEP_SEND) 1056 fd_write = conn->writesockfd; 1057 else 1058 fd_write = CURL_SOCKET_BAD; 1059 1060 if(conn->data->state.drain) { 1061 select_res |= CURL_CSELECT_IN; 1062 DEBUGF(infof(data, "Curl_readwrite: forcibly told to drain data\n")); 1063 } 1064 1065 if(!select_res) /* Call for select()/poll() only, if read/write/error 1066 status is not known. */ 1067 select_res = Curl_socket_ready(fd_read, fd_write, 0); 1068 1069 if(select_res == CURL_CSELECT_ERR) { 1070 failf(data, "select/poll returned error"); 1071 return CURLE_SEND_ERROR; 1072 } 1073 1074 /* We go ahead and do a read if we have a readable socket or if 1075 the stream was rewound (in which case we have data in a 1076 buffer) */ 1077 if((k->keepon & KEEP_RECV) && 1078 ((select_res & CURL_CSELECT_IN) || conn->bits.stream_was_rewound)) { 1079 1080 result = readwrite_data(data, conn, k, &didwhat, done); 1081 if(result || *done) 1082 return result; 1083 } 1084 1085 /* If we still have writing to do, we check if we have a writable socket. */ 1086 if((k->keepon & KEEP_SEND) && (select_res & CURL_CSELECT_OUT)) { 1087 /* write */ 1088 1089 result = readwrite_upload(data, conn, k, &didwhat); 1090 if(result) 1091 return result; 1092 } 1093 1094 k->now = Curl_tvnow(); 1095 if(didwhat) { 1096 /* Update read/write counters */ 1097 if(k->bytecountp) 1098 *k->bytecountp = k->bytecount; /* read count */ 1099 if(k->writebytecountp) 1100 *k->writebytecountp = k->writebytecount; /* write count */ 1101 } 1102 else { 1103 /* no read no write, this is a timeout? */ 1104 if(k->exp100 == EXP100_AWAITING_CONTINUE) { 1105 /* This should allow some time for the header to arrive, but only a 1106 very short time as otherwise it'll be too much wasted time too 1107 often. */ 1108 1109 /* Quoting RFC2616, section "8.2.3 Use of the 100 (Continue) Status": 1110 1111 Therefore, when a client sends this header field to an origin server 1112 (possibly via a proxy) from which it has never seen a 100 (Continue) 1113 status, the client SHOULD NOT wait for an indefinite period before 1114 sending the request body. 1115 1116 */ 1117 1118 long ms = Curl_tvdiff(k->now, k->start100); 1119 if(ms >= data->set.expect_100_timeout) { 1120 /* we've waited long enough, continue anyway */ 1121 k->exp100 = EXP100_SEND_DATA; 1122 k->keepon |= KEEP_SEND; 1123 infof(data, "Done waiting for 100-continue\n"); 1124 } 1125 } 1126 } 1127 1128 if(Curl_pgrsUpdate(conn)) 1129 result = CURLE_ABORTED_BY_CALLBACK; 1130 else 1131 result = Curl_speedcheck(data, k->now); 1132 if(result) 1133 return result; 1134 1135 if(k->keepon) { 1136 if(0 > Curl_timeleft(data, &k->now, FALSE)) { 1137 if(k->size != -1) { 1138 failf(data, "Operation timed out after %ld milliseconds with %" 1139 CURL_FORMAT_CURL_OFF_T " out of %" 1140 CURL_FORMAT_CURL_OFF_T " bytes received", 1141 Curl_tvdiff(k->now, data->progress.t_startsingle), k->bytecount, 1142 k->size); 1143 } 1144 else { 1145 failf(data, "Operation timed out after %ld milliseconds with %" 1146 CURL_FORMAT_CURL_OFF_T " bytes received", 1147 Curl_tvdiff(k->now, data->progress.t_startsingle), k->bytecount); 1148 } 1149 return CURLE_OPERATION_TIMEDOUT; 1150 } 1151 } 1152 else { 1153 /* 1154 * The transfer has been performed. Just make some general checks before 1155 * returning. 1156 */ 1157 1158 if(!(data->set.opt_no_body) && (k->size != -1) && 1159 (k->bytecount != k->size) && 1160#ifdef CURL_DO_LINEEND_CONV 1161 /* Most FTP servers don't adjust their file SIZE response for CRLFs, 1162 so we'll check to see if the discrepancy can be explained 1163 by the number of CRLFs we've changed to LFs. 1164 */ 1165 (k->bytecount != (k->size + data->state.crlf_conversions)) && 1166#endif /* CURL_DO_LINEEND_CONV */ 1167 !data->req.newurl) { 1168 failf(data, "transfer closed with %" CURL_FORMAT_CURL_OFF_T 1169 " bytes remaining to read", 1170 k->size - k->bytecount); 1171 return CURLE_PARTIAL_FILE; 1172 } 1173 else if(!(data->set.opt_no_body) && 1174 k->chunk && 1175 (conn->chunk.state != CHUNK_STOP)) { 1176 /* 1177 * In chunked mode, return an error if the connection is closed prior to 1178 * the empty (terminating) chunk is read. 1179 * 1180 * The condition above used to check for 1181 * conn->proto.http->chunk.datasize != 0 which is true after reading 1182 * *any* chunk, not just the empty chunk. 1183 * 1184 */ 1185 failf(data, "transfer closed with outstanding read data remaining"); 1186 return CURLE_PARTIAL_FILE; 1187 } 1188 if(Curl_pgrsUpdate(conn)) 1189 return CURLE_ABORTED_BY_CALLBACK; 1190 } 1191 1192 /* Now update the "done" boolean we return */ 1193 *done = (0 == (k->keepon&(KEEP_RECV|KEEP_SEND| 1194 KEEP_RECV_PAUSE|KEEP_SEND_PAUSE))) ? TRUE : FALSE; 1195 1196 return CURLE_OK; 1197} 1198 1199/* 1200 * Curl_single_getsock() gets called by the multi interface code when the app 1201 * has requested to get the sockets for the current connection. This function 1202 * will then be called once for every connection that the multi interface 1203 * keeps track of. This function will only be called for connections that are 1204 * in the proper state to have this information available. 1205 */ 1206int Curl_single_getsock(const struct connectdata *conn, 1207 curl_socket_t *sock, /* points to numsocks number 1208 of sockets */ 1209 int numsocks) 1210{ 1211 const struct Curl_easy *data = conn->data; 1212 int bitmap = GETSOCK_BLANK; 1213 unsigned sockindex = 0; 1214 1215 if(conn->handler->perform_getsock) 1216 return conn->handler->perform_getsock(conn, sock, numsocks); 1217 1218 if(numsocks < 2) 1219 /* simple check but we might need two slots */ 1220 return GETSOCK_BLANK; 1221 1222 /* don't include HOLD and PAUSE connections */ 1223 if((data->req.keepon & KEEP_RECVBITS) == KEEP_RECV) { 1224 1225 DEBUGASSERT(conn->sockfd != CURL_SOCKET_BAD); 1226 1227 bitmap |= GETSOCK_READSOCK(sockindex); 1228 sock[sockindex] = conn->sockfd; 1229 } 1230 1231 /* don't include HOLD and PAUSE connections */ 1232 if((data->req.keepon & KEEP_SENDBITS) == KEEP_SEND) { 1233 1234 if((conn->sockfd != conn->writesockfd) || 1235 bitmap == GETSOCK_BLANK) { 1236 /* only if they are not the same socket and we have a readable 1237 one, we increase index */ 1238 if(bitmap != GETSOCK_BLANK) 1239 sockindex++; /* increase index if we need two entries */ 1240 1241 DEBUGASSERT(conn->writesockfd != CURL_SOCKET_BAD); 1242 1243 sock[sockindex] = conn->writesockfd; 1244 } 1245 1246 bitmap |= GETSOCK_WRITESOCK(sockindex); 1247 } 1248 1249 return bitmap; 1250} 1251 1252/* 1253 * Determine optimum sleep time based on configured rate, current rate, 1254 * and packet size. 1255 * Returns value in milliseconds. 1256 * 1257 * The basic idea is to adjust the desired rate up/down in this method 1258 * based on whether we are running too slow or too fast. Then, calculate 1259 * how many milliseconds to wait for the next packet to achieve this new 1260 * rate. 1261 */ 1262long Curl_sleep_time(curl_off_t rate_bps, curl_off_t cur_rate_bps, 1263 int pkt_size) 1264{ 1265 curl_off_t min_sleep = 0; 1266 curl_off_t rv = 0; 1267 1268 if(rate_bps == 0) 1269 return 0; 1270 1271 /* If running faster than about .1% of the desired speed, slow 1272 * us down a bit. Use shift instead of division as the 0.1% 1273 * cutoff is arbitrary anyway. 1274 */ 1275 if(cur_rate_bps > (rate_bps + (rate_bps >> 10))) { 1276 /* running too fast, decrease target rate by 1/64th of rate */ 1277 rate_bps -= rate_bps >> 6; 1278 min_sleep = 1; 1279 } 1280 else if(cur_rate_bps < (rate_bps - (rate_bps >> 10))) { 1281 /* running too slow, increase target rate by 1/64th of rate */ 1282 rate_bps += rate_bps >> 6; 1283 } 1284 1285 /* Determine number of milliseconds to wait until we do 1286 * the next packet at the adjusted rate. We should wait 1287 * longer when using larger packets, for instance. 1288 */ 1289 rv = ((curl_off_t)(pkt_size * 1000) / rate_bps); 1290 1291 /* Catch rounding errors and always slow down at least 1ms if 1292 * we are running too fast. 1293 */ 1294 if(rv < min_sleep) 1295 rv = min_sleep; 1296 1297 /* Bound value to fit in 'long' on 32-bit platform. That's 1298 * plenty long enough anyway! 1299 */ 1300 if(rv > 0x7fffffff) 1301 rv = 0x7fffffff; 1302 1303 return (long)rv; 1304} 1305 1306/* Curl_init_CONNECT() gets called each time the handle switches to CONNECT 1307 which means this gets called once for each subsequent redirect etc */ 1308void Curl_init_CONNECT(struct Curl_easy *data) 1309{ 1310 data->state.fread_func = data->set.fread_func_set; 1311 data->state.in = data->set.in_set; 1312} 1313 1314/* 1315 * Curl_pretransfer() is called immediately before a transfer starts, and only 1316 * once for one transfer no matter if it has redirects or do multi-pass 1317 * authentication etc. 1318 */ 1319CURLcode Curl_pretransfer(struct Curl_easy *data) 1320{ 1321 CURLcode result; 1322 if(!data->change.url) { 1323 /* we can't do anything without URL */ 1324 failf(data, "No URL set!"); 1325 return CURLE_URL_MALFORMAT; 1326 } 1327 1328 /* Init the SSL session ID cache here. We do it here since we want to do it 1329 after the *_setopt() calls (that could specify the size of the cache) but 1330 before any transfer takes place. */ 1331 result = Curl_ssl_initsessions(data, data->set.ssl.max_ssl_sessions); 1332 if(result) 1333 return result; 1334 1335 data->set.followlocation=0; /* reset the location-follow counter */ 1336 data->state.this_is_a_follow = FALSE; /* reset this */ 1337 data->state.errorbuf = FALSE; /* no error has occurred */ 1338 data->state.httpversion = 0; /* don't assume any particular server version */ 1339 1340 data->state.authproblem = FALSE; 1341 data->state.authhost.want = data->set.httpauth; 1342 data->state.authproxy.want = data->set.proxyauth; 1343 Curl_safefree(data->info.wouldredirect); 1344 data->info.wouldredirect = NULL; 1345 1346 if(data->set.httpreq == HTTPREQ_PUT) 1347 data->state.infilesize = data->set.filesize; 1348 else 1349 data->state.infilesize = data->set.postfieldsize; 1350 1351 /* If there is a list of cookie files to read, do it now! */ 1352 if(data->change.cookielist) 1353 Curl_cookie_loadfiles(data); 1354 1355 /* If there is a list of host pairs to deal with */ 1356 if(data->change.resolve) 1357 result = Curl_loadhostpairs(data); 1358 1359 if(!result) { 1360 /* Allow data->set.use_port to set which port to use. This needs to be 1361 * disabled for example when we follow Location: headers to URLs using 1362 * different ports! */ 1363 data->state.allow_port = TRUE; 1364 1365#if defined(HAVE_SIGNAL) && defined(SIGPIPE) && !defined(HAVE_MSG_NOSIGNAL) 1366 /************************************************************* 1367 * Tell signal handler to ignore SIGPIPE 1368 *************************************************************/ 1369 if(!data->set.no_signal) 1370 data->state.prev_signal = signal(SIGPIPE, SIG_IGN); 1371#endif 1372 1373 Curl_initinfo(data); /* reset session-specific information "variables" */ 1374 Curl_pgrsResetTimesSizes(data); 1375 Curl_pgrsStartNow(data); 1376 1377 if(data->set.timeout) 1378 Curl_expire(data, data->set.timeout); 1379 1380 if(data->set.connecttimeout) 1381 Curl_expire(data, data->set.connecttimeout); 1382 1383 /* In case the handle is re-used and an authentication method was picked 1384 in the session we need to make sure we only use the one(s) we now 1385 consider to be fine */ 1386 data->state.authhost.picked &= data->state.authhost.want; 1387 data->state.authproxy.picked &= data->state.authproxy.want; 1388 1389 if(data->set.wildcardmatch) { 1390 struct WildcardData *wc = &data->wildcard; 1391 if(!wc->filelist) { 1392 result = Curl_wildcard_init(wc); /* init wildcard structures */ 1393 if(result) 1394 return CURLE_OUT_OF_MEMORY; 1395 } 1396 } 1397 1398 } 1399 1400 return result; 1401} 1402 1403/* 1404 * Curl_posttransfer() is called immediately after a transfer ends 1405 */ 1406CURLcode Curl_posttransfer(struct Curl_easy *data) 1407{ 1408#if defined(HAVE_SIGNAL) && defined(SIGPIPE) && !defined(HAVE_MSG_NOSIGNAL) 1409 /* restore the signal handler for SIGPIPE before we get back */ 1410 if(!data->set.no_signal) 1411 signal(SIGPIPE, data->state.prev_signal); 1412#else 1413 (void)data; /* unused parameter */ 1414#endif 1415 1416 return CURLE_OK; 1417} 1418 1419#ifndef CURL_DISABLE_HTTP 1420/* 1421 * strlen_url() returns the length of the given URL if the spaces within the 1422 * URL were properly URL encoded. 1423 */ 1424static size_t strlen_url(const char *url) 1425{ 1426 const unsigned char *ptr; 1427 size_t newlen=0; 1428 bool left=TRUE; /* left side of the ? */ 1429 1430 for(ptr=(unsigned char *)url; *ptr; ptr++) { 1431 switch(*ptr) { 1432 case '?': 1433 left=FALSE; 1434 /* fall through */ 1435 default: 1436 if(*ptr >= 0x80) 1437 newlen += 2; 1438 newlen++; 1439 break; 1440 case ' ': 1441 if(left) 1442 newlen+=3; 1443 else 1444 newlen++; 1445 break; 1446 } 1447 } 1448 return newlen; 1449} 1450 1451/* strcpy_url() copies a url to a output buffer and URL-encodes the spaces in 1452 * the source URL accordingly. 1453 */ 1454static void strcpy_url(char *output, const char *url) 1455{ 1456 /* we must add this with whitespace-replacing */ 1457 bool left=TRUE; 1458 const unsigned char *iptr; 1459 char *optr = output; 1460 for(iptr = (unsigned char *)url; /* read from here */ 1461 *iptr; /* until zero byte */ 1462 iptr++) { 1463 switch(*iptr) { 1464 case '?': 1465 left=FALSE; 1466 /* fall through */ 1467 default: 1468 if(*iptr >= 0x80) { 1469 snprintf(optr, 4, "%%%02x", *iptr); 1470 optr += 3; 1471 } 1472 else 1473 *optr++=*iptr; 1474 break; 1475 case ' ': 1476 if(left) { 1477 *optr++='%'; /* add a '%' */ 1478 *optr++='2'; /* add a '2' */ 1479 *optr++='0'; /* add a '0' */ 1480 } 1481 else 1482 *optr++='+'; /* add a '+' here */ 1483 break; 1484 } 1485 } 1486 *optr=0; /* zero terminate output buffer */ 1487 1488} 1489 1490/* 1491 * Returns true if the given URL is absolute (as opposed to relative) 1492 */ 1493static bool is_absolute_url(const char *url) 1494{ 1495 char prot[16]; /* URL protocol string storage */ 1496 char letter; /* used for a silly sscanf */ 1497 1498 return (2 == sscanf(url, "%15[^?&/:]://%c", prot, &letter)) ? TRUE : FALSE; 1499} 1500 1501/* 1502 * Concatenate a relative URL to a base URL making it absolute. 1503 * URL-encodes any spaces. 1504 * The returned pointer must be freed by the caller unless NULL 1505 * (returns NULL on out of memory). 1506 */ 1507static char *concat_url(const char *base, const char *relurl) 1508{ 1509 /*** 1510 TRY to append this new path to the old URL 1511 to the right of the host part. Oh crap, this is doomed to cause 1512 problems in the future... 1513 */ 1514 char *newest; 1515 char *protsep; 1516 char *pathsep; 1517 size_t newlen; 1518 1519 const char *useurl = relurl; 1520 size_t urllen; 1521 1522 /* we must make our own copy of the URL to play with, as it may 1523 point to read-only data */ 1524 char *url_clone=strdup(base); 1525 1526 if(!url_clone) 1527 return NULL; /* skip out of this NOW */ 1528 1529 /* protsep points to the start of the host name */ 1530 protsep=strstr(url_clone, "//"); 1531 if(!protsep) 1532 protsep=url_clone; 1533 else 1534 protsep+=2; /* pass the slashes */ 1535 1536 if('/' != relurl[0]) { 1537 int level=0; 1538 1539 /* First we need to find out if there's a ?-letter in the URL, 1540 and cut it and the right-side of that off */ 1541 pathsep = strchr(protsep, '?'); 1542 if(pathsep) 1543 *pathsep=0; 1544 1545 /* we have a relative path to append to the last slash if there's one 1546 available, or if the new URL is just a query string (starts with a 1547 '?') we append the new one at the end of the entire currently worked 1548 out URL */ 1549 if(useurl[0] != '?') { 1550 pathsep = strrchr(protsep, '/'); 1551 if(pathsep) 1552 *pathsep=0; 1553 } 1554 1555 /* Check if there's any slash after the host name, and if so, remember 1556 that position instead */ 1557 pathsep = strchr(protsep, '/'); 1558 if(pathsep) 1559 protsep = pathsep+1; 1560 else 1561 protsep = NULL; 1562 1563 /* now deal with one "./" or any amount of "../" in the newurl 1564 and act accordingly */ 1565 1566 if((useurl[0] == '.') && (useurl[1] == '/')) 1567 useurl+=2; /* just skip the "./" */ 1568 1569 while((useurl[0] == '.') && 1570 (useurl[1] == '.') && 1571 (useurl[2] == '/')) { 1572 level++; 1573 useurl+=3; /* pass the "../" */ 1574 } 1575 1576 if(protsep) { 1577 while(level--) { 1578 /* cut off one more level from the right of the original URL */ 1579 pathsep = strrchr(protsep, '/'); 1580 if(pathsep) 1581 *pathsep=0; 1582 else { 1583 *protsep=0; 1584 break; 1585 } 1586 } 1587 } 1588 } 1589 else { 1590 /* We got a new absolute path for this server */ 1591 1592 if((relurl[0] == '/') && (relurl[1] == '/')) { 1593 /* the new URL starts with //, just keep the protocol part from the 1594 original one */ 1595 *protsep=0; 1596 useurl = &relurl[2]; /* we keep the slashes from the original, so we 1597 skip the new ones */ 1598 } 1599 else { 1600 /* cut off the original URL from the first slash, or deal with URLs 1601 without slash */ 1602 pathsep = strchr(protsep, '/'); 1603 if(pathsep) { 1604 /* When people use badly formatted URLs, such as 1605 "http://www.url.com?dir=/home/daniel" we must not use the first 1606 slash, if there's a ?-letter before it! */ 1607 char *sep = strchr(protsep, '?'); 1608 if(sep && (sep < pathsep)) 1609 pathsep = sep; 1610 *pathsep=0; 1611 } 1612 else { 1613 /* There was no slash. Now, since we might be operating on a badly 1614 formatted URL, such as "http://www.url.com?id=2380" which doesn't 1615 use a slash separator as it is supposed to, we need to check for a 1616 ?-letter as well! */ 1617 pathsep = strchr(protsep, '?'); 1618 if(pathsep) 1619 *pathsep=0; 1620 } 1621 } 1622 } 1623 1624 /* If the new part contains a space, this is a mighty stupid redirect 1625 but we still make an effort to do "right". To the left of a '?' 1626 letter we replace each space with %20 while it is replaced with '+' 1627 on the right side of the '?' letter. 1628 */ 1629 newlen = strlen_url(useurl); 1630 1631 urllen = strlen(url_clone); 1632 1633 newest = malloc(urllen + 1 + /* possible slash */ 1634 newlen + 1 /* zero byte */); 1635 1636 if(!newest) { 1637 free(url_clone); /* don't leak this */ 1638 return NULL; 1639 } 1640 1641 /* copy over the root url part */ 1642 memcpy(newest, url_clone, urllen); 1643 1644 /* check if we need to append a slash */ 1645 if(('/' == useurl[0]) || (protsep && !*protsep) || ('?' == useurl[0])) 1646 ; 1647 else 1648 newest[urllen++]='/'; 1649 1650 /* then append the new piece on the right side */ 1651 strcpy_url(&newest[urllen], useurl); 1652 1653 free(url_clone); 1654 1655 return newest; 1656} 1657#endif /* CURL_DISABLE_HTTP */ 1658 1659/* 1660 * Curl_follow() handles the URL redirect magic. Pass in the 'newurl' string 1661 * as given by the remote server and set up the new URL to request. 1662 */ 1663CURLcode Curl_follow(struct Curl_easy *data, 1664 char *newurl, /* this 'newurl' is the Location: string, 1665 and it must be malloc()ed before passed 1666 here */ 1667 followtype type) /* see transfer.h */ 1668{ 1669#ifdef CURL_DISABLE_HTTP 1670 (void)data; 1671 (void)newurl; 1672 (void)type; 1673 /* Location: following will not happen when HTTP is disabled */ 1674 return CURLE_TOO_MANY_REDIRECTS; 1675#else 1676 1677 /* Location: redirect */ 1678 bool disallowport = FALSE; 1679 1680 if(type == FOLLOW_REDIR) { 1681 if((data->set.maxredirs != -1) && 1682 (data->set.followlocation >= data->set.maxredirs)) { 1683 failf(data, "Maximum (%ld) redirects followed", data->set.maxredirs); 1684 return CURLE_TOO_MANY_REDIRECTS; 1685 } 1686 1687 /* mark the next request as a followed location: */ 1688 data->state.this_is_a_follow = TRUE; 1689 1690 data->set.followlocation++; /* count location-followers */ 1691 1692 if(data->set.http_auto_referer) { 1693 /* We are asked to automatically set the previous URL as the referer 1694 when we get the next URL. We pick the ->url field, which may or may 1695 not be 100% correct */ 1696 1697 if(data->change.referer_alloc) { 1698 Curl_safefree(data->change.referer); 1699 data->change.referer_alloc = FALSE; 1700 } 1701 1702 data->change.referer = strdup(data->change.url); 1703 if(!data->change.referer) 1704 return CURLE_OUT_OF_MEMORY; 1705 data->change.referer_alloc = TRUE; /* yes, free this later */ 1706 } 1707 } 1708 1709 if(!is_absolute_url(newurl)) { 1710 /*** 1711 *DANG* this is an RFC 2068 violation. The URL is supposed 1712 to be absolute and this doesn't seem to be that! 1713 */ 1714 char *absolute = concat_url(data->change.url, newurl); 1715 if(!absolute) 1716 return CURLE_OUT_OF_MEMORY; 1717 free(newurl); 1718 newurl = absolute; 1719 } 1720 else { 1721 /* The new URL MAY contain space or high byte values, that means a mighty 1722 stupid redirect URL but we still make an effort to do "right". */ 1723 char *newest; 1724 size_t newlen = strlen_url(newurl); 1725 1726 /* This is an absolute URL, don't allow the custom port number */ 1727 disallowport = TRUE; 1728 1729 newest = malloc(newlen+1); /* get memory for this */ 1730 if(!newest) 1731 return CURLE_OUT_OF_MEMORY; 1732 strcpy_url(newest, newurl); /* create a space-free URL */ 1733 1734 free(newurl); /* that was no good */ 1735 newurl = newest; /* use this instead now */ 1736 1737 } 1738 1739 if(type == FOLLOW_FAKE) { 1740 /* we're only figuring out the new url if we would've followed locations 1741 but now we're done so we can get out! */ 1742 data->info.wouldredirect = newurl; 1743 return CURLE_OK; 1744 } 1745 1746 if(disallowport) 1747 data->state.allow_port = FALSE; 1748 1749 if(data->change.url_alloc) { 1750 Curl_safefree(data->change.url); 1751 data->change.url_alloc = FALSE; 1752 } 1753 1754 data->change.url = newurl; 1755 data->change.url_alloc = TRUE; 1756 newurl = NULL; /* don't free! */ 1757 1758 infof(data, "Issue another request to this URL: '%s'\n", data->change.url); 1759 1760 /* 1761 * We get here when the HTTP code is 300-399 (and 401). We need to perform 1762 * differently based on exactly what return code there was. 1763 * 1764 * News from 7.10.6: we can also get here on a 401 or 407, in case we act on 1765 * a HTTP (proxy-) authentication scheme other than Basic. 1766 */ 1767 switch(data->info.httpcode) { 1768 /* 401 - Act on a WWW-Authenticate, we keep on moving and do the 1769 Authorization: XXXX header in the HTTP request code snippet */ 1770 /* 407 - Act on a Proxy-Authenticate, we keep on moving and do the 1771 Proxy-Authorization: XXXX header in the HTTP request code snippet */ 1772 /* 300 - Multiple Choices */ 1773 /* 306 - Not used */ 1774 /* 307 - Temporary Redirect */ 1775 default: /* for all above (and the unknown ones) */ 1776 /* Some codes are explicitly mentioned since I've checked RFC2616 and they 1777 * seem to be OK to POST to. 1778 */ 1779 break; 1780 case 301: /* Moved Permanently */ 1781 /* (quote from RFC7231, section 6.4.2) 1782 * 1783 * Note: For historical reasons, a user agent MAY change the request 1784 * method from POST to GET for the subsequent request. If this 1785 * behavior is undesired, the 307 (Temporary Redirect) status code 1786 * can be used instead. 1787 * 1788 * ---- 1789 * 1790 * Many webservers expect this, so these servers often answers to a POST 1791 * request with an error page. To be sure that libcurl gets the page that 1792 * most user agents would get, libcurl has to force GET. 1793 * 1794 * This behaviour is forbidden by RFC1945 and the obsolete RFC2616, and 1795 * can be overridden with CURLOPT_POSTREDIR. 1796 */ 1797 if((data->set.httpreq == HTTPREQ_POST 1798 || data->set.httpreq == HTTPREQ_POST_FORM) 1799 && !(data->set.keep_post & CURL_REDIR_POST_301)) { 1800 infof(data, "Switch from POST to GET\n"); 1801 data->set.httpreq = HTTPREQ_GET; 1802 } 1803 break; 1804 case 302: /* Found */ 1805 /* (quote from RFC7231, section 6.4.3) 1806 * 1807 * Note: For historical reasons, a user agent MAY change the request 1808 * method from POST to GET for the subsequent request. If this 1809 * behavior is undesired, the 307 (Temporary Redirect) status code 1810 * can be used instead. 1811 * 1812 * ---- 1813 * 1814 * Many webservers expect this, so these servers often answers to a POST 1815 * request with an error page. To be sure that libcurl gets the page that 1816 * most user agents would get, libcurl has to force GET. 1817 * 1818 * This behaviour is forbidden by RFC1945 and the obsolete RFC2616, and 1819 * can be overridden with CURLOPT_POSTREDIR. 1820 */ 1821 if((data->set.httpreq == HTTPREQ_POST 1822 || data->set.httpreq == HTTPREQ_POST_FORM) 1823 && !(data->set.keep_post & CURL_REDIR_POST_302)) { 1824 infof(data, "Switch from POST to GET\n"); 1825 data->set.httpreq = HTTPREQ_GET; 1826 } 1827 break; 1828 1829 case 303: /* See Other */ 1830 /* Disable both types of POSTs, unless the user explicitely 1831 asks for POST after POST */ 1832 if(data->set.httpreq != HTTPREQ_GET 1833 && !(data->set.keep_post & CURL_REDIR_POST_303)) { 1834 data->set.httpreq = HTTPREQ_GET; /* enforce GET request */ 1835 infof(data, "Disables POST, goes with %s\n", 1836 data->set.opt_no_body?"HEAD":"GET"); 1837 } 1838 break; 1839 case 304: /* Not Modified */ 1840 /* 304 means we did a conditional request and it was "Not modified". 1841 * We shouldn't get any Location: header in this response! 1842 */ 1843 break; 1844 case 305: /* Use Proxy */ 1845 /* (quote from RFC2616, section 10.3.6): 1846 * "The requested resource MUST be accessed through the proxy given 1847 * by the Location field. The Location field gives the URI of the 1848 * proxy. The recipient is expected to repeat this single request 1849 * via the proxy. 305 responses MUST only be generated by origin 1850 * servers." 1851 */ 1852 break; 1853 } 1854 Curl_pgrsTime(data, TIMER_REDIRECT); 1855 Curl_pgrsResetTimesSizes(data); 1856 1857 return CURLE_OK; 1858#endif /* CURL_DISABLE_HTTP */ 1859} 1860 1861/* Returns CURLE_OK *and* sets '*url' if a request retry is wanted. 1862 1863 NOTE: that the *url is malloc()ed. */ 1864CURLcode Curl_retry_request(struct connectdata *conn, 1865 char **url) 1866{ 1867 struct Curl_easy *data = conn->data; 1868 1869 *url = NULL; 1870 1871 /* if we're talking upload, we can't do the checks below, unless the protocol 1872 is HTTP as when uploading over HTTP we will still get a response */ 1873 if(data->set.upload && 1874 !(conn->handler->protocol&(PROTO_FAMILY_HTTP|CURLPROTO_RTSP))) 1875 return CURLE_OK; 1876 1877 if((data->req.bytecount + data->req.headerbytecount == 0) && 1878 conn->bits.reuse && 1879 !data->set.opt_no_body && 1880 (data->set.rtspreq != RTSPREQ_RECEIVE)) { 1881 /* We got no data, we attempted to re-use a connection and yet we want a 1882 "body". This might happen if the connection was left alive when we were 1883 done using it before, but that was closed when we wanted to read from 1884 it again. Bad luck. Retry the same request on a fresh connect! */ 1885 infof(conn->data, "Connection died, retrying a fresh connect\n"); 1886 *url = strdup(conn->data->change.url); 1887 if(!*url) 1888 return CURLE_OUT_OF_MEMORY; 1889 1890 connclose(conn, "retry"); /* close this connection */ 1891 conn->bits.retry = TRUE; /* mark this as a connection we're about 1892 to retry. Marking it this way should 1893 prevent i.e HTTP transfers to return 1894 error just because nothing has been 1895 transferred! */ 1896 1897 1898 if(conn->handler->protocol&PROTO_FAMILY_HTTP) { 1899 struct HTTP *http = data->req.protop; 1900 if(http->writebytecount) 1901 return Curl_readrewind(conn); 1902 } 1903 } 1904 return CURLE_OK; 1905} 1906 1907/* 1908 * Curl_setup_transfer() is called to setup some basic properties for the 1909 * upcoming transfer. 1910 */ 1911void 1912Curl_setup_transfer( 1913 struct connectdata *conn, /* connection data */ 1914 int sockindex, /* socket index to read from or -1 */ 1915 curl_off_t size, /* -1 if unknown at this point */ 1916 bool getheader, /* TRUE if header parsing is wanted */ 1917 curl_off_t *bytecountp, /* return number of bytes read or NULL */ 1918 int writesockindex, /* socket index to write to, it may very well be 1919 the same we read from. -1 disables */ 1920 curl_off_t *writecountp /* return number of bytes written or NULL */ 1921 ) 1922{ 1923 struct Curl_easy *data; 1924 struct SingleRequest *k; 1925 1926 DEBUGASSERT(conn != NULL); 1927 1928 data = conn->data; 1929 k = &data->req; 1930 1931 DEBUGASSERT((sockindex <= 1) && (sockindex >= -1)); 1932 1933 /* now copy all input parameters */ 1934 conn->sockfd = sockindex == -1 ? 1935 CURL_SOCKET_BAD : conn->sock[sockindex]; 1936 conn->writesockfd = writesockindex == -1 ? 1937 CURL_SOCKET_BAD:conn->sock[writesockindex]; 1938 k->getheader = getheader; 1939 1940 k->size = size; 1941 k->bytecountp = bytecountp; 1942 k->writebytecountp = writecountp; 1943 1944 /* The code sequence below is placed in this function just because all 1945 necessary input is not always known in do_complete() as this function may 1946 be called after that */ 1947 1948 if(!k->getheader) { 1949 k->header = FALSE; 1950 if(size > 0) 1951 Curl_pgrsSetDownloadSize(data, size); 1952 } 1953 /* we want header and/or body, if neither then don't do this! */ 1954 if(k->getheader || !data->set.opt_no_body) { 1955 1956 if(conn->sockfd != CURL_SOCKET_BAD) 1957 k->keepon |= KEEP_RECV; 1958 1959 if(conn->writesockfd != CURL_SOCKET_BAD) { 1960 struct HTTP *http = data->req.protop; 1961 /* HTTP 1.1 magic: 1962 1963 Even if we require a 100-return code before uploading data, we might 1964 need to write data before that since the REQUEST may not have been 1965 finished sent off just yet. 1966 1967 Thus, we must check if the request has been sent before we set the 1968 state info where we wait for the 100-return code 1969 */ 1970 if((data->state.expect100header) && 1971 (conn->handler->protocol&PROTO_FAMILY_HTTP) && 1972 (http->sending == HTTPSEND_BODY)) { 1973 /* wait with write until we either got 100-continue or a timeout */ 1974 k->exp100 = EXP100_AWAITING_CONTINUE; 1975 k->start100 = Curl_tvnow(); 1976 1977 /* Set a timeout for the multi interface. Add the inaccuracy margin so 1978 that we don't fire slightly too early and get denied to run. */ 1979 Curl_expire(data, data->set.expect_100_timeout); 1980 } 1981 else { 1982 if(data->state.expect100header) 1983 /* when we've sent off the rest of the headers, we must await a 1984 100-continue but first finish sending the request */ 1985 k->exp100 = EXP100_SENDING_REQUEST; 1986 1987 /* enable the write bit when we're not waiting for continue */ 1988 k->keepon |= KEEP_SEND; 1989 } 1990 } /* if(conn->writesockfd != CURL_SOCKET_BAD) */ 1991 } /* if(k->getheader || !data->set.opt_no_body) */ 1992 1993} 1994