1// Copyright (c) 2012 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "content/browser/appcache/appcache_update_job.h"
6
7#include "base/bind.h"
8#include "base/bind_helpers.h"
9#include "base/compiler_specific.h"
10#include "base/message_loop/message_loop.h"
11#include "base/strings/string_util.h"
12#include "base/strings/stringprintf.h"
13#include "content/browser/appcache/appcache_group.h"
14#include "content/browser/appcache/appcache_histograms.h"
15#include "net/base/host_port_pair.h"
16#include "net/base/io_buffer.h"
17#include "net/base/load_flags.h"
18#include "net/base/net_errors.h"
19#include "net/base/request_priority.h"
20#include "net/http/http_request_headers.h"
21#include "net/http/http_response_headers.h"
22#include "net/url_request/url_request_context.h"
23
24namespace {
25bool IsDataReductionProxy(const net::HostPortPair& proxy_server) {
26  return (
27      proxy_server.Equals(net::HostPortPair("proxy.googlezip.net", 443)) ||
28      proxy_server.Equals(net::HostPortPair("compress.googlezip.net", 80)) ||
29      proxy_server.Equals(net::HostPortPair("proxy-dev.googlezip.net", 80)));
30}
31}  // namspace
32
33namespace content {
34
35static const int kBufferSize = 32768;
36static const size_t kMaxConcurrentUrlFetches = 2;
37static const int kMax503Retries = 3;
38
39static std::string FormatUrlErrorMessage(
40      const char* format, const GURL& url,
41      AppCacheUpdateJob::ResultType error,
42      int response_code) {
43    // Show the net response code if we have one.
44    int code = response_code;
45    if (error != AppCacheUpdateJob::SERVER_ERROR)
46      code = static_cast<int>(error);
47    return base::StringPrintf(format, code, url.spec().c_str());
48}
49
50// Helper class for collecting hosts per frontend when sending notifications
51// so that only one notification is sent for all hosts using the same frontend.
52class HostNotifier {
53 public:
54  typedef std::vector<int> HostIds;
55  typedef std::map<AppCacheFrontend*, HostIds> NotifyHostMap;
56
57  // Caller is responsible for ensuring there will be no duplicate hosts.
58  void AddHost(AppCacheHost* host) {
59    std::pair<NotifyHostMap::iterator , bool> ret = hosts_to_notify.insert(
60        NotifyHostMap::value_type(host->frontend(), HostIds()));
61    ret.first->second.push_back(host->host_id());
62  }
63
64  void AddHosts(const std::set<AppCacheHost*>& hosts) {
65    for (std::set<AppCacheHost*>::const_iterator it = hosts.begin();
66         it != hosts.end(); ++it) {
67      AddHost(*it);
68    }
69  }
70
71  void SendNotifications(AppCacheEventID event_id) {
72    for (NotifyHostMap::iterator it = hosts_to_notify.begin();
73         it != hosts_to_notify.end(); ++it) {
74      AppCacheFrontend* frontend = it->first;
75      frontend->OnEventRaised(it->second, event_id);
76    }
77  }
78
79  void SendProgressNotifications(
80      const GURL& url, int num_total, int num_complete) {
81    for (NotifyHostMap::iterator it = hosts_to_notify.begin();
82         it != hosts_to_notify.end(); ++it) {
83      AppCacheFrontend* frontend = it->first;
84      frontend->OnProgressEventRaised(it->second, url,
85                                      num_total, num_complete);
86    }
87  }
88
89  void SendErrorNotifications(const AppCacheErrorDetails& details) {
90    DCHECK(!details.message.empty());
91    for (NotifyHostMap::iterator it = hosts_to_notify.begin();
92         it != hosts_to_notify.end(); ++it) {
93      AppCacheFrontend* frontend = it->first;
94      frontend->OnErrorEventRaised(it->second, details);
95    }
96  }
97
98  void SendLogMessage(const std::string& message) {
99    for (NotifyHostMap::iterator it = hosts_to_notify.begin();
100         it != hosts_to_notify.end(); ++it) {
101      AppCacheFrontend* frontend = it->first;
102      for (HostIds::iterator id = it->second.begin();
103           id != it->second.end(); ++id) {
104        frontend->OnLogMessage(*id, APPCACHE_LOG_WARNING, message);
105      }
106    }
107  }
108
109 private:
110  NotifyHostMap hosts_to_notify;
111};
112
113AppCacheUpdateJob::UrlToFetch::UrlToFetch(const GURL& url,
114                                          bool checked,
115                                          AppCacheResponseInfo* info)
116    : url(url),
117      storage_checked(checked),
118      existing_response_info(info) {
119}
120
121AppCacheUpdateJob::UrlToFetch::~UrlToFetch() {
122}
123
124// Helper class to fetch resources. Depending on the fetch type,
125// can either fetch to an in-memory string or write the response
126// data out to the disk cache.
127AppCacheUpdateJob::URLFetcher::URLFetcher(const GURL& url,
128                                          FetchType fetch_type,
129                                          AppCacheUpdateJob* job)
130    : url_(url),
131      job_(job),
132      fetch_type_(fetch_type),
133      retry_503_attempts_(0),
134      buffer_(new net::IOBuffer(kBufferSize)),
135      request_(job->service_->request_context()
136                   ->CreateRequest(url, net::DEFAULT_PRIORITY, this, NULL)),
137      result_(UPDATE_OK),
138      redirect_response_code_(-1) {}
139
140AppCacheUpdateJob::URLFetcher::~URLFetcher() {
141}
142
143void AppCacheUpdateJob::URLFetcher::Start() {
144  request_->set_first_party_for_cookies(job_->manifest_url_);
145  request_->SetLoadFlags(request_->load_flags() |
146                         net::LOAD_DISABLE_INTERCEPT);
147  if (existing_response_headers_.get())
148    AddConditionalHeaders(existing_response_headers_.get());
149  request_->Start();
150}
151
152void AppCacheUpdateJob::URLFetcher::OnReceivedRedirect(
153    net::URLRequest* request,
154    const net::RedirectInfo& redirect_info,
155    bool* defer_redirect) {
156  DCHECK(request_ == request);
157  // TODO(bengr): Remove this special case logic when crbug.com/429505 is
158  // resolved. Until then, the data reduction proxy client logic uses the
159  // redirect mechanism to resend requests over a direct connection when
160  // the proxy instructs it to do so. The redirect is to the same location
161  // as the original URL.
162  if ((request->load_flags() & net::LOAD_BYPASS_PROXY) &&
163      IsDataReductionProxy(request->proxy_server())) {
164    DCHECK_EQ(request->original_url(), request->url());
165    return;
166  }
167  // Redirect is not allowed by the update process.
168  job_->MadeProgress();
169  redirect_response_code_ = request->GetResponseCode();
170  request->Cancel();
171  result_ = REDIRECT_ERROR;
172  OnResponseCompleted();
173}
174
175void AppCacheUpdateJob::URLFetcher::OnResponseStarted(
176    net::URLRequest *request) {
177  DCHECK(request == request_);
178  int response_code = -1;
179  if (request->status().is_success()) {
180    response_code = request->GetResponseCode();
181    job_->MadeProgress();
182  }
183  if ((response_code / 100) == 2) {
184
185    // See http://code.google.com/p/chromium/issues/detail?id=69594
186    // We willfully violate the HTML5 spec at this point in order
187    // to support the appcaching of cross-origin HTTPS resources.
188    // We've opted for a milder constraint and allow caching unless
189    // the resource has a "no-store" header. A spec change has been
190    // requested on the whatwg list.
191    // TODO(michaeln): Consider doing this for cross-origin HTTP resources too.
192    if (url_.SchemeIsSecure() &&
193        url_.GetOrigin() != job_->manifest_url_.GetOrigin()) {
194      if (request->response_headers()->
195              HasHeaderValue("cache-control", "no-store")) {
196        DCHECK_EQ(-1, redirect_response_code_);
197        request->Cancel();
198        result_ = SERVER_ERROR;  // Not the best match?
199        OnResponseCompleted();
200        return;
201      }
202    }
203
204    // Write response info to storage for URL fetches. Wait for async write
205    // completion before reading any response data.
206    if (fetch_type_ == URL_FETCH || fetch_type_ == MASTER_ENTRY_FETCH) {
207      response_writer_.reset(job_->CreateResponseWriter());
208      scoped_refptr<HttpResponseInfoIOBuffer> io_buffer(
209          new HttpResponseInfoIOBuffer(
210              new net::HttpResponseInfo(request->response_info())));
211      response_writer_->WriteInfo(
212          io_buffer.get(),
213          base::Bind(&URLFetcher::OnWriteComplete, base::Unretained(this)));
214    } else {
215      ReadResponseData();
216    }
217  } else {
218    if (response_code > 0)
219      result_ = SERVER_ERROR;
220    else
221      result_ = NETWORK_ERROR;
222    OnResponseCompleted();
223  }
224}
225
226void AppCacheUpdateJob::URLFetcher::OnReadCompleted(
227    net::URLRequest* request, int bytes_read) {
228  DCHECK(request_ == request);
229  bool data_consumed = true;
230  if (request->status().is_success() && bytes_read > 0) {
231    job_->MadeProgress();
232    data_consumed = ConsumeResponseData(bytes_read);
233    if (data_consumed) {
234      bytes_read = 0;
235      while (request->Read(buffer_.get(), kBufferSize, &bytes_read)) {
236        if (bytes_read > 0) {
237          data_consumed = ConsumeResponseData(bytes_read);
238          if (!data_consumed)
239            break;  // wait for async data processing, then read more
240        } else {
241          break;
242        }
243      }
244    }
245  }
246  if (data_consumed && !request->status().is_io_pending()) {
247    DCHECK_EQ(UPDATE_OK, result_);
248    OnResponseCompleted();
249  }
250}
251
252void AppCacheUpdateJob::URLFetcher::AddConditionalHeaders(
253    const net::HttpResponseHeaders* headers) {
254  DCHECK(request_.get() && headers);
255  net::HttpRequestHeaders extra_headers;
256
257  // Add If-Modified-Since header if response info has Last-Modified header.
258  const std::string last_modified = "Last-Modified";
259  std::string last_modified_value;
260  headers->EnumerateHeader(NULL, last_modified, &last_modified_value);
261  if (!last_modified_value.empty()) {
262    extra_headers.SetHeader(net::HttpRequestHeaders::kIfModifiedSince,
263                            last_modified_value);
264  }
265
266  // Add If-None-Match header if response info has ETag header.
267  const std::string etag = "ETag";
268  std::string etag_value;
269  headers->EnumerateHeader(NULL, etag, &etag_value);
270  if (!etag_value.empty()) {
271    extra_headers.SetHeader(net::HttpRequestHeaders::kIfNoneMatch,
272                            etag_value);
273  }
274  if (!extra_headers.IsEmpty())
275    request_->SetExtraRequestHeaders(extra_headers);
276}
277
278void  AppCacheUpdateJob::URLFetcher::OnWriteComplete(int result) {
279  if (result < 0) {
280    request_->Cancel();
281    result_ = DISKCACHE_ERROR;
282    OnResponseCompleted();
283    return;
284  }
285  ReadResponseData();
286}
287
288void AppCacheUpdateJob::URLFetcher::ReadResponseData() {
289  InternalUpdateState state = job_->internal_state_;
290  if (state == CACHE_FAILURE || state == CANCELLED || state == COMPLETED)
291    return;
292  int bytes_read = 0;
293  request_->Read(buffer_.get(), kBufferSize, &bytes_read);
294  OnReadCompleted(request_.get(), bytes_read);
295}
296
297// Returns false if response data is processed asynchronously, in which
298// case ReadResponseData will be invoked when it is safe to continue
299// reading more response data from the request.
300bool AppCacheUpdateJob::URLFetcher::ConsumeResponseData(int bytes_read) {
301  DCHECK_GT(bytes_read, 0);
302  switch (fetch_type_) {
303    case MANIFEST_FETCH:
304    case MANIFEST_REFETCH:
305      manifest_data_.append(buffer_->data(), bytes_read);
306      break;
307    case URL_FETCH:
308    case MASTER_ENTRY_FETCH:
309      DCHECK(response_writer_.get());
310      response_writer_->WriteData(
311          buffer_.get(),
312          bytes_read,
313          base::Bind(&URLFetcher::OnWriteComplete, base::Unretained(this)));
314      return false;  // wait for async write completion to continue reading
315    default:
316      NOTREACHED();
317  }
318  return true;
319}
320
321void AppCacheUpdateJob::URLFetcher::OnResponseCompleted() {
322  if (request_->status().is_success())
323    job_->MadeProgress();
324
325  // Retry for 503s where retry-after is 0.
326  if (request_->status().is_success() &&
327      request_->GetResponseCode() == 503 &&
328      MaybeRetryRequest()) {
329    return;
330  }
331
332  switch (fetch_type_) {
333    case MANIFEST_FETCH:
334      job_->HandleManifestFetchCompleted(this);
335      break;
336    case URL_FETCH:
337      job_->HandleUrlFetchCompleted(this);
338      break;
339    case MASTER_ENTRY_FETCH:
340      job_->HandleMasterEntryFetchCompleted(this);
341      break;
342    case MANIFEST_REFETCH:
343      job_->HandleManifestRefetchCompleted(this);
344      break;
345    default:
346      NOTREACHED();
347  }
348
349  delete this;
350}
351
352bool AppCacheUpdateJob::URLFetcher::MaybeRetryRequest() {
353  if (retry_503_attempts_ >= kMax503Retries ||
354      !request_->response_headers()->HasHeaderValue("retry-after", "0")) {
355    return false;
356  }
357  ++retry_503_attempts_;
358  result_ = UPDATE_OK;
359  request_ = job_->service_->request_context()->CreateRequest(
360      url_, net::DEFAULT_PRIORITY, this, NULL);
361  Start();
362  return true;
363}
364
365AppCacheUpdateJob::AppCacheUpdateJob(AppCacheServiceImpl* service,
366                                     AppCacheGroup* group)
367    : service_(service),
368      manifest_url_(group->manifest_url()),
369      group_(group),
370      update_type_(UNKNOWN_TYPE),
371      internal_state_(FETCH_MANIFEST),
372      master_entries_completed_(0),
373      url_fetches_completed_(0),
374      manifest_fetcher_(NULL),
375      manifest_has_valid_mime_type_(false),
376      stored_state_(UNSTORED),
377      storage_(service->storage()) {
378    service_->AddObserver(this);
379}
380
381AppCacheUpdateJob::~AppCacheUpdateJob() {
382  if (service_)
383    service_->RemoveObserver(this);
384  if (internal_state_ != COMPLETED)
385    Cancel();
386
387  DCHECK(!manifest_fetcher_);
388  DCHECK(pending_url_fetches_.empty());
389  DCHECK(!inprogress_cache_.get());
390  DCHECK(pending_master_entries_.empty());
391  DCHECK(master_entry_fetches_.empty());
392
393  if (group_)
394    group_->SetUpdateAppCacheStatus(AppCacheGroup::IDLE);
395}
396
397void AppCacheUpdateJob::StartUpdate(AppCacheHost* host,
398                                    const GURL& new_master_resource) {
399  DCHECK(group_->update_job() == this);
400  DCHECK(!group_->is_obsolete());
401
402  bool is_new_pending_master_entry = false;
403  if (!new_master_resource.is_empty()) {
404    DCHECK(new_master_resource == host->pending_master_entry_url());
405    DCHECK(!new_master_resource.has_ref());
406    DCHECK(new_master_resource.GetOrigin() == manifest_url_.GetOrigin());
407
408    // Cannot add more to this update if already terminating.
409    if (IsTerminating()) {
410      group_->QueueUpdate(host, new_master_resource);
411      return;
412    }
413
414    std::pair<PendingMasters::iterator, bool> ret =
415        pending_master_entries_.insert(
416            PendingMasters::value_type(new_master_resource, PendingHosts()));
417    is_new_pending_master_entry = ret.second;
418    ret.first->second.push_back(host);
419    host->AddObserver(this);
420  }
421
422  // Notify host (if any) if already checking or downloading.
423  AppCacheGroup::UpdateAppCacheStatus update_status = group_->update_status();
424  if (update_status == AppCacheGroup::CHECKING ||
425      update_status == AppCacheGroup::DOWNLOADING) {
426    if (host) {
427      NotifySingleHost(host, APPCACHE_CHECKING_EVENT);
428      if (update_status == AppCacheGroup::DOWNLOADING)
429        NotifySingleHost(host, APPCACHE_DOWNLOADING_EVENT);
430
431      // Add to fetch list or an existing entry if already fetched.
432      if (!new_master_resource.is_empty()) {
433        AddMasterEntryToFetchList(host, new_master_resource,
434                                  is_new_pending_master_entry);
435      }
436    }
437    return;
438  }
439
440  // Begin update process for the group.
441  MadeProgress();
442  group_->SetUpdateAppCacheStatus(AppCacheGroup::CHECKING);
443  if (group_->HasCache()) {
444    update_type_ = UPGRADE_ATTEMPT;
445    NotifyAllAssociatedHosts(APPCACHE_CHECKING_EVENT);
446  } else {
447    update_type_ = CACHE_ATTEMPT;
448    DCHECK(host);
449    NotifySingleHost(host, APPCACHE_CHECKING_EVENT);
450  }
451
452  if (!new_master_resource.is_empty()) {
453    AddMasterEntryToFetchList(host, new_master_resource,
454                              is_new_pending_master_entry);
455  }
456
457  FetchManifest(true);
458}
459
460AppCacheResponseWriter* AppCacheUpdateJob::CreateResponseWriter() {
461  AppCacheResponseWriter* writer =
462      storage_->CreateResponseWriter(manifest_url_,
463                                                group_->group_id());
464  stored_response_ids_.push_back(writer->response_id());
465  return writer;
466}
467
468void AppCacheUpdateJob::HandleCacheFailure(
469    const AppCacheErrorDetails& error_details,
470    ResultType result,
471    const GURL& failed_resource_url) {
472  // 6.9.4 cache failure steps 2-8.
473  DCHECK(internal_state_ != CACHE_FAILURE);
474  DCHECK(!error_details.message.empty());
475  DCHECK(result != UPDATE_OK);
476  internal_state_ = CACHE_FAILURE;
477  LogHistogramStats(result, failed_resource_url);
478  CancelAllUrlFetches();
479  CancelAllMasterEntryFetches(error_details);
480  NotifyAllError(error_details);
481  DiscardInprogressCache();
482  internal_state_ = COMPLETED;
483  DeleteSoon();  // To unwind the stack prior to deletion.
484}
485
486void AppCacheUpdateJob::FetchManifest(bool is_first_fetch) {
487  DCHECK(!manifest_fetcher_);
488  manifest_fetcher_ = new URLFetcher(
489     manifest_url_,
490     is_first_fetch ? URLFetcher::MANIFEST_FETCH :
491                      URLFetcher::MANIFEST_REFETCH,
492     this);
493
494  // Add any necessary Http headers before sending fetch request.
495  if (is_first_fetch) {
496    AppCacheEntry* entry = (update_type_ == UPGRADE_ATTEMPT) ?
497        group_->newest_complete_cache()->GetEntry(manifest_url_) : NULL;
498    if (entry) {
499      // Asynchronously load response info for manifest from newest cache.
500      storage_->LoadResponseInfo(manifest_url_, group_->group_id(),
501                                 entry->response_id(), this);
502    } else {
503      manifest_fetcher_->Start();
504    }
505  } else {
506    DCHECK(internal_state_ == REFETCH_MANIFEST);
507    DCHECK(manifest_response_info_.get());
508    manifest_fetcher_->set_existing_response_headers(
509        manifest_response_info_->headers.get());
510    manifest_fetcher_->Start();
511  }
512}
513
514
515void AppCacheUpdateJob::HandleManifestFetchCompleted(
516    URLFetcher* fetcher) {
517  DCHECK_EQ(internal_state_, FETCH_MANIFEST);
518  DCHECK_EQ(manifest_fetcher_, fetcher);
519  manifest_fetcher_ = NULL;
520
521  net::URLRequest* request = fetcher->request();
522  int response_code = -1;
523  bool is_valid_response_code = false;
524  if (request->status().is_success()) {
525    response_code = request->GetResponseCode();
526    is_valid_response_code = (response_code / 100 == 2);
527
528    std::string mime_type;
529    request->GetMimeType(&mime_type);
530    manifest_has_valid_mime_type_ = (mime_type == "text/cache-manifest");
531  }
532
533  if (is_valid_response_code) {
534    manifest_data_ = fetcher->manifest_data();
535    manifest_response_info_.reset(
536        new net::HttpResponseInfo(request->response_info()));
537    if (update_type_ == UPGRADE_ATTEMPT)
538      CheckIfManifestChanged();  // continues asynchronously
539    else
540      ContinueHandleManifestFetchCompleted(true);
541  } else if (response_code == 304 && update_type_ == UPGRADE_ATTEMPT) {
542    ContinueHandleManifestFetchCompleted(false);
543  } else if ((response_code == 404 || response_code == 410) &&
544             update_type_ == UPGRADE_ATTEMPT) {
545    storage_->MakeGroupObsolete(group_, this, response_code);  // async
546  } else {
547    const char* kFormatString = "Manifest fetch failed (%d) %s";
548    std::string message = FormatUrlErrorMessage(
549        kFormatString, manifest_url_, fetcher->result(), response_code);
550    HandleCacheFailure(AppCacheErrorDetails(message,
551                                    APPCACHE_MANIFEST_ERROR,
552                                    manifest_url_,
553                                    response_code,
554                                    false /*is_cross_origin*/),
555                       fetcher->result(),
556                       GURL());
557  }
558}
559
560void AppCacheUpdateJob::OnGroupMadeObsolete(AppCacheGroup* group,
561                                            bool success,
562                                            int response_code) {
563  DCHECK(master_entry_fetches_.empty());
564  CancelAllMasterEntryFetches(AppCacheErrorDetails(
565      "The cache has been made obsolete, "
566      "the manifest file returned 404 or 410",
567      APPCACHE_MANIFEST_ERROR,
568      GURL(),
569      response_code,
570      false /*is_cross_origin*/));
571  if (success) {
572    DCHECK(group->is_obsolete());
573    NotifyAllAssociatedHosts(APPCACHE_OBSOLETE_EVENT);
574    internal_state_ = COMPLETED;
575    MaybeCompleteUpdate();
576  } else {
577    // Treat failure to mark group obsolete as a cache failure.
578    HandleCacheFailure(AppCacheErrorDetails(
579        "Failed to mark the cache as obsolete",
580        APPCACHE_UNKNOWN_ERROR,
581        GURL(),
582        0,
583        false /*is_cross_origin*/),
584                       DB_ERROR,
585                       GURL());
586  }
587}
588
589void AppCacheUpdateJob::ContinueHandleManifestFetchCompleted(bool changed) {
590  DCHECK(internal_state_ == FETCH_MANIFEST);
591
592  if (!changed) {
593    DCHECK(update_type_ == UPGRADE_ATTEMPT);
594    internal_state_ = NO_UPDATE;
595
596    // Wait for pending master entries to download.
597    FetchMasterEntries();
598    MaybeCompleteUpdate();  // if not done, run async 6.9.4 step 7 substeps
599    return;
600  }
601
602  AppCacheManifest manifest;
603  if (!ParseManifest(manifest_url_, manifest_data_.data(),
604                     manifest_data_.length(),
605                     manifest_has_valid_mime_type_ ?
606                        PARSE_MANIFEST_ALLOWING_INTERCEPTS :
607                        PARSE_MANIFEST_PER_STANDARD,
608                     manifest)) {
609    const char* kFormatString = "Failed to parse manifest %s";
610    const std::string message = base::StringPrintf(kFormatString,
611        manifest_url_.spec().c_str());
612    HandleCacheFailure(
613        AppCacheErrorDetails(
614            message, APPCACHE_SIGNATURE_ERROR, GURL(), 0,
615            false /*is_cross_origin*/),
616        MANIFEST_ERROR,
617        GURL());
618    VLOG(1) << message;
619    return;
620  }
621
622  // Proceed with update process. Section 6.9.4 steps 8-20.
623  internal_state_ = DOWNLOADING;
624  inprogress_cache_ = new AppCache(storage_, storage_->NewCacheId());
625  BuildUrlFileList(manifest);
626  inprogress_cache_->InitializeWithManifest(&manifest);
627
628  // Associate all pending master hosts with the newly created cache.
629  for (PendingMasters::iterator it = pending_master_entries_.begin();
630       it != pending_master_entries_.end(); ++it) {
631    PendingHosts& hosts = it->second;
632    for (PendingHosts::iterator host_it = hosts.begin();
633         host_it != hosts.end(); ++host_it) {
634      (*host_it)
635          ->AssociateIncompleteCache(inprogress_cache_.get(), manifest_url_);
636    }
637  }
638
639  if (manifest.did_ignore_intercept_namespaces) {
640    // Must be done after associating all pending master hosts.
641    std::string message(
642        "Ignoring the INTERCEPT section of the application cache manifest "
643        "because the content type is not text/cache-manifest");
644    LogConsoleMessageToAll(message);
645  }
646
647  group_->SetUpdateAppCacheStatus(AppCacheGroup::DOWNLOADING);
648  NotifyAllAssociatedHosts(APPCACHE_DOWNLOADING_EVENT);
649  FetchUrls();
650  FetchMasterEntries();
651  MaybeCompleteUpdate();  // if not done, continues when async fetches complete
652}
653
654void AppCacheUpdateJob::HandleUrlFetchCompleted(URLFetcher* fetcher) {
655  DCHECK(internal_state_ == DOWNLOADING);
656
657  net::URLRequest* request = fetcher->request();
658  const GURL& url = request->original_url();
659  pending_url_fetches_.erase(url);
660  NotifyAllProgress(url);
661  ++url_fetches_completed_;
662
663  int response_code = request->status().is_success()
664                          ? request->GetResponseCode()
665                          : fetcher->redirect_response_code();
666
667  AppCacheEntry& entry = url_file_list_.find(url)->second;
668
669  if (response_code / 100 == 2) {
670    // Associate storage with the new entry.
671    DCHECK(fetcher->response_writer());
672    entry.set_response_id(fetcher->response_writer()->response_id());
673    entry.set_response_size(fetcher->response_writer()->amount_written());
674    if (!inprogress_cache_->AddOrModifyEntry(url, entry))
675      duplicate_response_ids_.push_back(entry.response_id());
676
677    // TODO(michaeln): Check for <html manifest=xxx>
678    // See http://code.google.com/p/chromium/issues/detail?id=97930
679    // if (entry.IsMaster() && !(entry.IsExplicit() || fallback || intercept))
680    //   if (!manifestAttribute) skip it
681
682    // Foreign entries will be detected during cache selection.
683    // Note: 6.9.4, step 17.9 possible optimization: if resource is HTML or XML
684    // file whose root element is an html element with a manifest attribute
685    // whose value doesn't match the manifest url of the application cache
686    // being processed, mark the entry as being foreign.
687  } else {
688    VLOG(1) << "Request status: " << request->status().status()
689            << " error: " << request->status().error()
690            << " response code: " << response_code;
691    if (entry.IsExplicit() || entry.IsFallback() || entry.IsIntercept()) {
692      if (response_code == 304 && fetcher->existing_entry().has_response_id()) {
693        // Keep the existing response.
694        entry.set_response_id(fetcher->existing_entry().response_id());
695        entry.set_response_size(fetcher->existing_entry().response_size());
696        inprogress_cache_->AddOrModifyEntry(url, entry);
697      } else {
698        const char* kFormatString = "Resource fetch failed (%d) %s";
699        std::string message = FormatUrlErrorMessage(
700            kFormatString, url, fetcher->result(), response_code);
701        ResultType result = fetcher->result();
702        bool is_cross_origin = url.GetOrigin() != manifest_url_.GetOrigin();
703        switch (result) {
704          case DISKCACHE_ERROR:
705            HandleCacheFailure(
706                AppCacheErrorDetails(
707                    message, APPCACHE_UNKNOWN_ERROR, GURL(), 0,
708                    is_cross_origin),
709                result,
710                url);
711            break;
712          case NETWORK_ERROR:
713            HandleCacheFailure(
714                AppCacheErrorDetails(message, APPCACHE_RESOURCE_ERROR, url, 0,
715                    is_cross_origin),
716                result,
717                url);
718            break;
719          default:
720            HandleCacheFailure(AppCacheErrorDetails(message,
721                                            APPCACHE_RESOURCE_ERROR,
722                                            url,
723                                            response_code,
724                                            is_cross_origin),
725                               result,
726                               url);
727            break;
728        }
729        return;
730      }
731    } else if (response_code == 404 || response_code == 410) {
732      // Entry is skipped.  They are dropped from the cache.
733    } else if (update_type_ == UPGRADE_ATTEMPT &&
734               fetcher->existing_entry().has_response_id()) {
735      // Keep the existing response.
736      // TODO(michaeln): Not sure this is a good idea. This is spec compliant
737      // but the old resource may or may not be compatible with the new contents
738      // of the cache. Impossible to know one way or the other.
739      entry.set_response_id(fetcher->existing_entry().response_id());
740      entry.set_response_size(fetcher->existing_entry().response_size());
741      inprogress_cache_->AddOrModifyEntry(url, entry);
742    }
743  }
744
745  // Fetch another URL now that one request has completed.
746  DCHECK(internal_state_ != CACHE_FAILURE);
747  FetchUrls();
748  MaybeCompleteUpdate();
749}
750
751void AppCacheUpdateJob::HandleMasterEntryFetchCompleted(
752    URLFetcher* fetcher) {
753  DCHECK(internal_state_ == NO_UPDATE || internal_state_ == DOWNLOADING);
754
755  // TODO(jennb): Handle downloads completing during cache failure when update
756  // no longer fetches master entries directly. For now, we cancel all pending
757  // master entry fetches when entering cache failure state so this will never
758  // be called in CACHE_FAILURE state.
759
760  net::URLRequest* request = fetcher->request();
761  const GURL& url = request->original_url();
762  master_entry_fetches_.erase(url);
763  ++master_entries_completed_;
764
765  int response_code = request->status().is_success()
766      ? request->GetResponseCode() : -1;
767
768  PendingMasters::iterator found = pending_master_entries_.find(url);
769  DCHECK(found != pending_master_entries_.end());
770  PendingHosts& hosts = found->second;
771
772  // Section 6.9.4. No update case: step 7.3, else step 22.
773  if (response_code / 100 == 2) {
774    // Add fetched master entry to the appropriate cache.
775    AppCache* cache = inprogress_cache_.get() ? inprogress_cache_.get()
776                                              : group_->newest_complete_cache();
777    DCHECK(fetcher->response_writer());
778    AppCacheEntry master_entry(AppCacheEntry::MASTER,
779                               fetcher->response_writer()->response_id(),
780                               fetcher->response_writer()->amount_written());
781    if (cache->AddOrModifyEntry(url, master_entry))
782      added_master_entries_.push_back(url);
783    else
784      duplicate_response_ids_.push_back(master_entry.response_id());
785
786    // In no-update case, associate host with the newest cache.
787    if (!inprogress_cache_.get()) {
788      // TODO(michaeln): defer until the updated cache has been stored
789      DCHECK(cache == group_->newest_complete_cache());
790      for (PendingHosts::iterator host_it = hosts.begin();
791           host_it != hosts.end(); ++host_it) {
792        (*host_it)->AssociateCompleteCache(cache);
793      }
794    }
795  } else {
796    HostNotifier host_notifier;
797    for (PendingHosts::iterator host_it = hosts.begin();
798         host_it != hosts.end(); ++host_it) {
799      AppCacheHost* host = *host_it;
800      host_notifier.AddHost(host);
801
802      // In downloading case, disassociate host from inprogress cache.
803      if (inprogress_cache_.get())
804        host->AssociateNoCache(GURL());
805
806      host->RemoveObserver(this);
807    }
808    hosts.clear();
809
810    const char* kFormatString = "Manifest fetch failed (%d) %s";
811    std::string message = FormatUrlErrorMessage(
812        kFormatString, request->url(), fetcher->result(), response_code);
813    host_notifier.SendErrorNotifications(
814        AppCacheErrorDetails(message,
815                     APPCACHE_MANIFEST_ERROR,
816                     request->url(),
817                     response_code,
818                     false /*is_cross_origin*/));
819
820    // In downloading case, update result is different if all master entries
821    // failed vs. only some failing.
822    if (inprogress_cache_.get()) {
823      // Only count successful downloads to know if all master entries failed.
824      pending_master_entries_.erase(found);
825      --master_entries_completed_;
826
827      // Section 6.9.4, step 22.3.
828      if (update_type_ == CACHE_ATTEMPT && pending_master_entries_.empty()) {
829        HandleCacheFailure(AppCacheErrorDetails(message,
830                                        APPCACHE_MANIFEST_ERROR,
831                                        request->url(),
832                                        response_code,
833                                        false /*is_cross_origin*/),
834                           fetcher->result(),
835                           GURL());
836        return;
837      }
838    }
839  }
840
841  DCHECK(internal_state_ != CACHE_FAILURE);
842  FetchMasterEntries();
843  MaybeCompleteUpdate();
844}
845
846void AppCacheUpdateJob::HandleManifestRefetchCompleted(
847    URLFetcher* fetcher) {
848  DCHECK(internal_state_ == REFETCH_MANIFEST);
849  DCHECK(manifest_fetcher_ == fetcher);
850  manifest_fetcher_ = NULL;
851
852  net::URLRequest* request = fetcher->request();
853  int response_code = request->status().is_success()
854      ? request->GetResponseCode() : -1;
855  if (response_code == 304 || manifest_data_ == fetcher->manifest_data()) {
856    // Only need to store response in storage if manifest is not already
857    // an entry in the cache.
858    AppCacheEntry* entry = inprogress_cache_->GetEntry(manifest_url_);
859    if (entry) {
860      entry->add_types(AppCacheEntry::MANIFEST);
861      StoreGroupAndCache();
862    } else {
863      manifest_response_writer_.reset(CreateResponseWriter());
864      scoped_refptr<HttpResponseInfoIOBuffer> io_buffer(
865          new HttpResponseInfoIOBuffer(manifest_response_info_.release()));
866      manifest_response_writer_->WriteInfo(
867          io_buffer.get(),
868          base::Bind(&AppCacheUpdateJob::OnManifestInfoWriteComplete,
869                     base::Unretained(this)));
870    }
871  } else {
872    VLOG(1) << "Request status: " << request->status().status()
873            << " error: " << request->status().error()
874            << " response code: " << response_code;
875    ScheduleUpdateRetry(kRerunDelayMs);
876    if (response_code == 200) {
877      HandleCacheFailure(AppCacheErrorDetails("Manifest changed during update",
878                                      APPCACHE_CHANGED_ERROR,
879                                      GURL(),
880                                      0,
881                                      false /*is_cross_origin*/),
882                         MANIFEST_ERROR,
883                         GURL());
884    } else {
885      const char* kFormatString = "Manifest re-fetch failed (%d) %s";
886      std::string message = FormatUrlErrorMessage(
887          kFormatString, manifest_url_, fetcher->result(), response_code);
888      HandleCacheFailure(AppCacheErrorDetails(message,
889                                      APPCACHE_MANIFEST_ERROR,
890                                      GURL(),
891                                      response_code,
892                                      false /*is_cross_origin*/),
893                         fetcher->result(),
894                         GURL());
895    }
896  }
897}
898
899void AppCacheUpdateJob::OnManifestInfoWriteComplete(int result) {
900  if (result > 0) {
901    scoped_refptr<net::StringIOBuffer> io_buffer(
902        new net::StringIOBuffer(manifest_data_));
903    manifest_response_writer_->WriteData(
904        io_buffer.get(),
905        manifest_data_.length(),
906        base::Bind(&AppCacheUpdateJob::OnManifestDataWriteComplete,
907                   base::Unretained(this)));
908  } else {
909    HandleCacheFailure(
910        AppCacheErrorDetails("Failed to write the manifest headers to storage",
911                     APPCACHE_UNKNOWN_ERROR,
912                     GURL(),
913                     0,
914                     false /*is_cross_origin*/),
915        DISKCACHE_ERROR,
916        GURL());
917  }
918}
919
920void AppCacheUpdateJob::OnManifestDataWriteComplete(int result) {
921  if (result > 0) {
922    AppCacheEntry entry(AppCacheEntry::MANIFEST,
923        manifest_response_writer_->response_id(),
924        manifest_response_writer_->amount_written());
925    if (!inprogress_cache_->AddOrModifyEntry(manifest_url_, entry))
926      duplicate_response_ids_.push_back(entry.response_id());
927    StoreGroupAndCache();
928  } else {
929    HandleCacheFailure(
930        AppCacheErrorDetails("Failed to write the manifest data to storage",
931                     APPCACHE_UNKNOWN_ERROR,
932                     GURL(),
933                     0,
934                     false /*is_cross_origin*/),
935        DISKCACHE_ERROR,
936        GURL());
937  }
938}
939
940void AppCacheUpdateJob::StoreGroupAndCache() {
941  DCHECK(stored_state_ == UNSTORED);
942  stored_state_ = STORING;
943  scoped_refptr<AppCache> newest_cache;
944  if (inprogress_cache_.get())
945    newest_cache.swap(inprogress_cache_);
946  else
947    newest_cache = group_->newest_complete_cache();
948  newest_cache->set_update_time(base::Time::Now());
949
950  // TODO(michaeln): dcheck is fishing for clues to crbug/95101
951  DCHECK_EQ(manifest_url_, group_->manifest_url());
952  storage_->StoreGroupAndNewestCache(group_, newest_cache.get(), this);
953}
954
955void AppCacheUpdateJob::OnGroupAndNewestCacheStored(AppCacheGroup* group,
956                                                    AppCache* newest_cache,
957                                                    bool success,
958                                                    bool would_exceed_quota) {
959  DCHECK(stored_state_ == STORING);
960  if (success) {
961    stored_state_ = STORED;
962    MaybeCompleteUpdate();  // will definitely complete
963  } else {
964    stored_state_ = UNSTORED;
965
966    // Restore inprogress_cache_ to get the proper events delivered
967    // and the proper cleanup to occur.
968    if (newest_cache != group->newest_complete_cache())
969      inprogress_cache_ = newest_cache;
970
971    ResultType result = DB_ERROR;
972    AppCacheErrorReason reason = APPCACHE_UNKNOWN_ERROR;
973    std::string message("Failed to commit new cache to storage");
974    if (would_exceed_quota) {
975      message.append(", would exceed quota");
976      result = QUOTA_ERROR;
977      reason = APPCACHE_QUOTA_ERROR;
978    }
979    HandleCacheFailure(
980        AppCacheErrorDetails(message, reason, GURL(), 0,
981            false /*is_cross_origin*/),
982        result,
983        GURL());
984  }
985}
986
987void AppCacheUpdateJob::NotifySingleHost(AppCacheHost* host,
988                                         AppCacheEventID event_id) {
989  std::vector<int> ids(1, host->host_id());
990  host->frontend()->OnEventRaised(ids, event_id);
991}
992
993void AppCacheUpdateJob::NotifyAllAssociatedHosts(AppCacheEventID event_id) {
994  HostNotifier host_notifier;
995  AddAllAssociatedHostsToNotifier(&host_notifier);
996  host_notifier.SendNotifications(event_id);
997}
998
999void AppCacheUpdateJob::NotifyAllProgress(const GURL& url) {
1000  HostNotifier host_notifier;
1001  AddAllAssociatedHostsToNotifier(&host_notifier);
1002  host_notifier.SendProgressNotifications(
1003      url, url_file_list_.size(), url_fetches_completed_);
1004}
1005
1006void AppCacheUpdateJob::NotifyAllFinalProgress() {
1007  DCHECK(url_file_list_.size() == url_fetches_completed_);
1008  NotifyAllProgress(GURL());
1009}
1010
1011void AppCacheUpdateJob::NotifyAllError(const AppCacheErrorDetails& details) {
1012  HostNotifier host_notifier;
1013  AddAllAssociatedHostsToNotifier(&host_notifier);
1014  host_notifier.SendErrorNotifications(details);
1015}
1016
1017void AppCacheUpdateJob::LogConsoleMessageToAll(const std::string& message) {
1018  HostNotifier host_notifier;
1019  AddAllAssociatedHostsToNotifier(&host_notifier);
1020  host_notifier.SendLogMessage(message);
1021}
1022
1023void AppCacheUpdateJob::AddAllAssociatedHostsToNotifier(
1024    HostNotifier* host_notifier) {
1025  // Collect hosts so we only send one notification per frontend.
1026  // A host can only be associated with a single cache so no need to worry
1027  // about duplicate hosts being added to the notifier.
1028  if (inprogress_cache_.get()) {
1029    DCHECK(internal_state_ == DOWNLOADING || internal_state_ == CACHE_FAILURE);
1030    host_notifier->AddHosts(inprogress_cache_->associated_hosts());
1031  }
1032
1033  AppCacheGroup::Caches old_caches = group_->old_caches();
1034  for (AppCacheGroup::Caches::const_iterator it = old_caches.begin();
1035       it != old_caches.end(); ++it) {
1036    host_notifier->AddHosts((*it)->associated_hosts());
1037  }
1038
1039  AppCache* newest_cache = group_->newest_complete_cache();
1040  if (newest_cache)
1041    host_notifier->AddHosts(newest_cache->associated_hosts());
1042}
1043
1044void AppCacheUpdateJob::OnDestructionImminent(AppCacheHost* host) {
1045  // The host is about to be deleted; remove from our collection.
1046  PendingMasters::iterator found =
1047      pending_master_entries_.find(host->pending_master_entry_url());
1048  DCHECK(found != pending_master_entries_.end());
1049  PendingHosts& hosts = found->second;
1050  PendingHosts::iterator it = std::find(hosts.begin(), hosts.end(), host);
1051  DCHECK(it != hosts.end());
1052  hosts.erase(it);
1053}
1054
1055void AppCacheUpdateJob::OnServiceReinitialized(
1056    AppCacheStorageReference* old_storage_ref) {
1057  // We continue to use the disabled instance, but arrange for its
1058  // deletion when its no longer needed.
1059  if (old_storage_ref->storage() == storage_)
1060    disabled_storage_reference_ = old_storage_ref;
1061}
1062
1063void AppCacheUpdateJob::CheckIfManifestChanged() {
1064  DCHECK(update_type_ == UPGRADE_ATTEMPT);
1065  AppCacheEntry* entry = NULL;
1066  if (group_->newest_complete_cache())
1067    entry = group_->newest_complete_cache()->GetEntry(manifest_url_);
1068  if (!entry) {
1069    // TODO(michaeln): This is just a bandaid to avoid a crash.
1070    // http://code.google.com/p/chromium/issues/detail?id=95101
1071    if (service_->storage() == storage_) {
1072      // Use a local variable because service_ is reset in HandleCacheFailure.
1073      AppCacheServiceImpl* service = service_;
1074      HandleCacheFailure(
1075          AppCacheErrorDetails("Manifest entry not found in existing cache",
1076                       APPCACHE_UNKNOWN_ERROR,
1077                       GURL(),
1078                       0,
1079                       false /*is_cross_origin*/),
1080          DB_ERROR,
1081          GURL());
1082      AppCacheHistograms::AddMissingManifestEntrySample();
1083      service->DeleteAppCacheGroup(manifest_url_, net::CompletionCallback());
1084    }
1085    return;
1086  }
1087
1088  // Load manifest data from storage to compare against fetched manifest.
1089  manifest_response_reader_.reset(
1090      storage_->CreateResponseReader(manifest_url_,
1091                                     group_->group_id(),
1092                                     entry->response_id()));
1093  read_manifest_buffer_ = new net::IOBuffer(kBufferSize);
1094  manifest_response_reader_->ReadData(
1095      read_manifest_buffer_.get(),
1096      kBufferSize,
1097      base::Bind(&AppCacheUpdateJob::OnManifestDataReadComplete,
1098                 base::Unretained(this)));  // async read
1099}
1100
1101void AppCacheUpdateJob::OnManifestDataReadComplete(int result) {
1102  if (result > 0) {
1103    loaded_manifest_data_.append(read_manifest_buffer_->data(), result);
1104    manifest_response_reader_->ReadData(
1105        read_manifest_buffer_.get(),
1106        kBufferSize,
1107        base::Bind(&AppCacheUpdateJob::OnManifestDataReadComplete,
1108                   base::Unretained(this)));  // read more
1109  } else {
1110    read_manifest_buffer_ = NULL;
1111    manifest_response_reader_.reset();
1112    ContinueHandleManifestFetchCompleted(
1113        result < 0 || manifest_data_ != loaded_manifest_data_);
1114  }
1115}
1116
1117void AppCacheUpdateJob::BuildUrlFileList(const AppCacheManifest& manifest) {
1118  for (base::hash_set<std::string>::const_iterator it =
1119           manifest.explicit_urls.begin();
1120       it != manifest.explicit_urls.end(); ++it) {
1121    AddUrlToFileList(GURL(*it), AppCacheEntry::EXPLICIT);
1122  }
1123
1124  const std::vector<AppCacheNamespace>& intercepts =
1125      manifest.intercept_namespaces;
1126  for (std::vector<AppCacheNamespace>::const_iterator it = intercepts.begin();
1127       it != intercepts.end(); ++it) {
1128    int flags = AppCacheEntry::INTERCEPT;
1129    if (it->is_executable)
1130      flags |= AppCacheEntry::EXECUTABLE;
1131    AddUrlToFileList(it->target_url, flags);
1132  }
1133
1134  const std::vector<AppCacheNamespace>& fallbacks =
1135      manifest.fallback_namespaces;
1136  for (std::vector<AppCacheNamespace>::const_iterator it = fallbacks.begin();
1137       it != fallbacks.end(); ++it) {
1138     AddUrlToFileList(it->target_url, AppCacheEntry::FALLBACK);
1139  }
1140
1141  // Add all master entries from newest complete cache.
1142  if (update_type_ == UPGRADE_ATTEMPT) {
1143    const AppCache::EntryMap& entries =
1144        group_->newest_complete_cache()->entries();
1145    for (AppCache::EntryMap::const_iterator it = entries.begin();
1146         it != entries.end(); ++it) {
1147      const AppCacheEntry& entry = it->second;
1148      if (entry.IsMaster())
1149        AddUrlToFileList(it->first, AppCacheEntry::MASTER);
1150    }
1151  }
1152}
1153
1154void AppCacheUpdateJob::AddUrlToFileList(const GURL& url, int type) {
1155  std::pair<AppCache::EntryMap::iterator, bool> ret = url_file_list_.insert(
1156      AppCache::EntryMap::value_type(url, AppCacheEntry(type)));
1157
1158  if (ret.second)
1159    urls_to_fetch_.push_back(UrlToFetch(url, false, NULL));
1160  else
1161    ret.first->second.add_types(type);  // URL already exists. Merge types.
1162}
1163
1164void AppCacheUpdateJob::FetchUrls() {
1165  DCHECK(internal_state_ == DOWNLOADING);
1166
1167  // Fetch each URL in the list according to section 6.9.4 step 17.1-17.3.
1168  // Fetch up to the concurrent limit. Other fetches will be triggered as each
1169  // each fetch completes.
1170  while (pending_url_fetches_.size() < kMaxConcurrentUrlFetches &&
1171         !urls_to_fetch_.empty()) {
1172    UrlToFetch url_to_fetch = urls_to_fetch_.front();
1173    urls_to_fetch_.pop_front();
1174
1175    AppCache::EntryMap::iterator it = url_file_list_.find(url_to_fetch.url);
1176    DCHECK(it != url_file_list_.end());
1177    AppCacheEntry& entry = it->second;
1178    if (ShouldSkipUrlFetch(entry)) {
1179      NotifyAllProgress(url_to_fetch.url);
1180      ++url_fetches_completed_;
1181    } else if (AlreadyFetchedEntry(url_to_fetch.url, entry.types())) {
1182      NotifyAllProgress(url_to_fetch.url);
1183      ++url_fetches_completed_;  // saved a URL request
1184    } else if (!url_to_fetch.storage_checked &&
1185               MaybeLoadFromNewestCache(url_to_fetch.url, entry)) {
1186      // Continues asynchronously after data is loaded from newest cache.
1187    } else {
1188      URLFetcher* fetcher = new URLFetcher(
1189          url_to_fetch.url, URLFetcher::URL_FETCH, this);
1190      if (url_to_fetch.existing_response_info.get()) {
1191        DCHECK(group_->newest_complete_cache());
1192        AppCacheEntry* existing_entry =
1193            group_->newest_complete_cache()->GetEntry(url_to_fetch.url);
1194        DCHECK(existing_entry);
1195        DCHECK(existing_entry->response_id() ==
1196               url_to_fetch.existing_response_info->response_id());
1197        fetcher->set_existing_response_headers(
1198            url_to_fetch.existing_response_info->http_response_info()->headers
1199                .get());
1200        fetcher->set_existing_entry(*existing_entry);
1201      }
1202      fetcher->Start();
1203      pending_url_fetches_.insert(
1204          PendingUrlFetches::value_type(url_to_fetch.url, fetcher));
1205    }
1206  }
1207}
1208
1209void AppCacheUpdateJob::CancelAllUrlFetches() {
1210  // Cancel any pending URL requests.
1211  for (PendingUrlFetches::iterator it = pending_url_fetches_.begin();
1212       it != pending_url_fetches_.end(); ++it) {
1213    delete it->second;
1214  }
1215
1216  url_fetches_completed_ +=
1217      pending_url_fetches_.size() + urls_to_fetch_.size();
1218  pending_url_fetches_.clear();
1219  urls_to_fetch_.clear();
1220}
1221
1222bool AppCacheUpdateJob::ShouldSkipUrlFetch(const AppCacheEntry& entry) {
1223  // 6.6.4 Step 17
1224  // If the resource URL being processed was flagged as neither an
1225  // "explicit entry" nor or a "fallback entry", then the user agent
1226  // may skip this URL.
1227  if (entry.IsExplicit() || entry.IsFallback() || entry.IsIntercept())
1228    return false;
1229
1230  // TODO(jennb): decide if entry should be skipped to expire it from cache
1231  return false;
1232}
1233
1234bool AppCacheUpdateJob::AlreadyFetchedEntry(const GURL& url,
1235                                            int entry_type) {
1236  DCHECK(internal_state_ == DOWNLOADING || internal_state_ == NO_UPDATE);
1237  AppCacheEntry* existing =
1238      inprogress_cache_.get() ? inprogress_cache_->GetEntry(url)
1239                              : group_->newest_complete_cache()->GetEntry(url);
1240  if (existing) {
1241    existing->add_types(entry_type);
1242    return true;
1243  }
1244  return false;
1245}
1246
1247void AppCacheUpdateJob::AddMasterEntryToFetchList(AppCacheHost* host,
1248                                                  const GURL& url,
1249                                                  bool is_new) {
1250  DCHECK(!IsTerminating());
1251
1252  if (internal_state_ == DOWNLOADING || internal_state_ == NO_UPDATE) {
1253    AppCache* cache;
1254    if (inprogress_cache_.get()) {
1255      // always associate
1256      host->AssociateIncompleteCache(inprogress_cache_.get(), manifest_url_);
1257      cache = inprogress_cache_.get();
1258    } else {
1259      cache = group_->newest_complete_cache();
1260    }
1261
1262    // Update existing entry if it has already been fetched.
1263    AppCacheEntry* entry = cache->GetEntry(url);
1264    if (entry) {
1265      entry->add_types(AppCacheEntry::MASTER);
1266      if (internal_state_ == NO_UPDATE && !inprogress_cache_.get()) {
1267        // only associate if have entry
1268        host->AssociateCompleteCache(cache);
1269      }
1270      if (is_new)
1271        ++master_entries_completed_;  // pretend fetching completed
1272      return;
1273    }
1274  }
1275
1276  // Add to fetch list if not already fetching.
1277  if (master_entry_fetches_.find(url) == master_entry_fetches_.end()) {
1278    master_entries_to_fetch_.insert(url);
1279    if (internal_state_ == DOWNLOADING || internal_state_ == NO_UPDATE)
1280      FetchMasterEntries();
1281  }
1282}
1283
1284void AppCacheUpdateJob::FetchMasterEntries() {
1285  DCHECK(internal_state_ == NO_UPDATE || internal_state_ == DOWNLOADING);
1286
1287  // Fetch each master entry in the list, up to the concurrent limit.
1288  // Additional fetches will be triggered as each fetch completes.
1289  while (master_entry_fetches_.size() < kMaxConcurrentUrlFetches &&
1290         !master_entries_to_fetch_.empty()) {
1291    const GURL& url = *master_entries_to_fetch_.begin();
1292
1293    if (AlreadyFetchedEntry(url, AppCacheEntry::MASTER)) {
1294      ++master_entries_completed_;  // saved a URL request
1295
1296      // In no update case, associate hosts to newest cache in group
1297      // now that master entry has been "successfully downloaded".
1298      if (internal_state_ == NO_UPDATE) {
1299        // TODO(michaeln): defer until the updated cache has been stored.
1300        DCHECK(!inprogress_cache_.get());
1301        AppCache* cache = group_->newest_complete_cache();
1302        PendingMasters::iterator found = pending_master_entries_.find(url);
1303        DCHECK(found != pending_master_entries_.end());
1304        PendingHosts& hosts = found->second;
1305        for (PendingHosts::iterator host_it = hosts.begin();
1306             host_it != hosts.end(); ++host_it) {
1307          (*host_it)->AssociateCompleteCache(cache);
1308        }
1309      }
1310    } else {
1311      URLFetcher* fetcher = new URLFetcher(
1312          url, URLFetcher::MASTER_ENTRY_FETCH, this);
1313      fetcher->Start();
1314      master_entry_fetches_.insert(PendingUrlFetches::value_type(url, fetcher));
1315    }
1316
1317    master_entries_to_fetch_.erase(master_entries_to_fetch_.begin());
1318  }
1319}
1320
1321void AppCacheUpdateJob::CancelAllMasterEntryFetches(
1322    const AppCacheErrorDetails& error_details) {
1323  // For now, cancel all in-progress fetches for master entries and pretend
1324  // all master entries fetches have completed.
1325  // TODO(jennb): Delete this when update no longer fetches master entries
1326  // directly.
1327
1328  // Cancel all in-progress fetches.
1329  for (PendingUrlFetches::iterator it = master_entry_fetches_.begin();
1330       it != master_entry_fetches_.end(); ++it) {
1331    delete it->second;
1332    master_entries_to_fetch_.insert(it->first);  // back in unfetched list
1333  }
1334  master_entry_fetches_.clear();
1335
1336  master_entries_completed_ += master_entries_to_fetch_.size();
1337
1338  // Cache failure steps, step 2.
1339  // Pretend all master entries that have not yet been fetched have completed
1340  // downloading. Unassociate hosts from any appcache and send ERROR event.
1341  HostNotifier host_notifier;
1342  while (!master_entries_to_fetch_.empty()) {
1343    const GURL& url = *master_entries_to_fetch_.begin();
1344    PendingMasters::iterator found = pending_master_entries_.find(url);
1345    DCHECK(found != pending_master_entries_.end());
1346    PendingHosts& hosts = found->second;
1347    for (PendingHosts::iterator host_it = hosts.begin();
1348         host_it != hosts.end(); ++host_it) {
1349      AppCacheHost* host = *host_it;
1350      host->AssociateNoCache(GURL());
1351      host_notifier.AddHost(host);
1352      host->RemoveObserver(this);
1353    }
1354    hosts.clear();
1355
1356    master_entries_to_fetch_.erase(master_entries_to_fetch_.begin());
1357  }
1358  host_notifier.SendErrorNotifications(error_details);
1359}
1360
1361bool AppCacheUpdateJob::MaybeLoadFromNewestCache(const GURL& url,
1362                                                 AppCacheEntry& entry) {
1363  if (update_type_ != UPGRADE_ATTEMPT)
1364    return false;
1365
1366  AppCache* newest = group_->newest_complete_cache();
1367  AppCacheEntry* copy_me = newest->GetEntry(url);
1368  if (!copy_me || !copy_me->has_response_id())
1369    return false;
1370
1371  // Load HTTP headers for entry from newest cache.
1372  loading_responses_.insert(
1373      LoadingResponses::value_type(copy_me->response_id(), url));
1374  storage_->LoadResponseInfo(manifest_url_, group_->group_id(),
1375                             copy_me->response_id(),
1376                             this);
1377  // Async: wait for OnResponseInfoLoaded to complete.
1378  return true;
1379}
1380
1381void AppCacheUpdateJob::OnResponseInfoLoaded(
1382    AppCacheResponseInfo* response_info, int64 response_id) {
1383  const net::HttpResponseInfo* http_info = response_info ?
1384      response_info->http_response_info() : NULL;
1385
1386  // Needed response info for a manifest fetch request.
1387  if (internal_state_ == FETCH_MANIFEST) {
1388    if (http_info)
1389      manifest_fetcher_->set_existing_response_headers(
1390          http_info->headers.get());
1391    manifest_fetcher_->Start();
1392    return;
1393  }
1394
1395  LoadingResponses::iterator found = loading_responses_.find(response_id);
1396  DCHECK(found != loading_responses_.end());
1397  const GURL& url = found->second;
1398
1399  if (!http_info) {
1400    LoadFromNewestCacheFailed(url, NULL);  // no response found
1401  } else {
1402    // Check if response can be re-used according to HTTP caching semantics.
1403    // Responses with a "vary" header get treated as expired.
1404    const std::string name = "vary";
1405    std::string value;
1406    void* iter = NULL;
1407    if (!http_info->headers.get() ||
1408        http_info->headers->RequiresValidation(http_info->request_time,
1409                                               http_info->response_time,
1410                                               base::Time::Now()) ||
1411        http_info->headers->EnumerateHeader(&iter, name, &value)) {
1412      LoadFromNewestCacheFailed(url, response_info);
1413    } else {
1414      DCHECK(group_->newest_complete_cache());
1415      AppCacheEntry* copy_me = group_->newest_complete_cache()->GetEntry(url);
1416      DCHECK(copy_me);
1417      DCHECK(copy_me->response_id() == response_id);
1418
1419      AppCache::EntryMap::iterator it = url_file_list_.find(url);
1420      DCHECK(it != url_file_list_.end());
1421      AppCacheEntry& entry = it->second;
1422      entry.set_response_id(response_id);
1423      entry.set_response_size(copy_me->response_size());
1424      inprogress_cache_->AddOrModifyEntry(url, entry);
1425      NotifyAllProgress(url);
1426      ++url_fetches_completed_;
1427    }
1428  }
1429  loading_responses_.erase(found);
1430
1431  MaybeCompleteUpdate();
1432}
1433
1434void AppCacheUpdateJob::LoadFromNewestCacheFailed(
1435    const GURL& url, AppCacheResponseInfo* response_info) {
1436  if (internal_state_ == CACHE_FAILURE)
1437    return;
1438
1439  // Re-insert url at front of fetch list. Indicate storage has been checked.
1440  urls_to_fetch_.push_front(UrlToFetch(url, true, response_info));
1441  FetchUrls();
1442}
1443
1444void AppCacheUpdateJob::MaybeCompleteUpdate() {
1445  DCHECK(internal_state_ != CACHE_FAILURE);
1446
1447  // Must wait for any pending master entries or url fetches to complete.
1448  if (master_entries_completed_ != pending_master_entries_.size() ||
1449      url_fetches_completed_ != url_file_list_.size()) {
1450    DCHECK(internal_state_ != COMPLETED);
1451    return;
1452  }
1453
1454  switch (internal_state_) {
1455    case NO_UPDATE:
1456      if (master_entries_completed_ > 0) {
1457        switch (stored_state_) {
1458          case UNSTORED:
1459            StoreGroupAndCache();
1460            return;
1461          case STORING:
1462            return;
1463          case STORED:
1464            break;
1465        }
1466      }
1467      // 6.9.4 steps 7.3-7.7.
1468      NotifyAllAssociatedHosts(APPCACHE_NO_UPDATE_EVENT);
1469      DiscardDuplicateResponses();
1470      internal_state_ = COMPLETED;
1471      break;
1472    case DOWNLOADING:
1473      internal_state_ = REFETCH_MANIFEST;
1474      FetchManifest(false);
1475      break;
1476    case REFETCH_MANIFEST:
1477      DCHECK(stored_state_ == STORED);
1478      NotifyAllFinalProgress();
1479      if (update_type_ == CACHE_ATTEMPT)
1480        NotifyAllAssociatedHosts(APPCACHE_CACHED_EVENT);
1481      else
1482        NotifyAllAssociatedHosts(APPCACHE_UPDATE_READY_EVENT);
1483      DiscardDuplicateResponses();
1484      internal_state_ = COMPLETED;
1485      LogHistogramStats(UPDATE_OK, GURL());
1486      break;
1487    case CACHE_FAILURE:
1488      NOTREACHED();  // See HandleCacheFailure
1489      break;
1490    default:
1491      break;
1492  }
1493
1494  // Let the stack unwind before deletion to make it less risky as this
1495  // method is called from multiple places in this file.
1496  if (internal_state_ == COMPLETED)
1497    DeleteSoon();
1498}
1499
1500void AppCacheUpdateJob::ScheduleUpdateRetry(int delay_ms) {
1501  // TODO(jennb): post a delayed task with the "same parameters" as this job
1502  // to retry the update at a later time. Need group, URLs of pending master
1503  // entries and their hosts.
1504}
1505
1506void AppCacheUpdateJob::Cancel() {
1507  internal_state_ = CANCELLED;
1508
1509  LogHistogramStats(CANCELLED_ERROR, GURL());
1510
1511  if (manifest_fetcher_) {
1512    delete manifest_fetcher_;
1513    manifest_fetcher_ = NULL;
1514  }
1515
1516  for (PendingUrlFetches::iterator it = pending_url_fetches_.begin();
1517       it != pending_url_fetches_.end(); ++it) {
1518    delete it->second;
1519  }
1520  pending_url_fetches_.clear();
1521
1522  for (PendingUrlFetches::iterator it = master_entry_fetches_.begin();
1523       it != master_entry_fetches_.end(); ++it) {
1524    delete it->second;
1525  }
1526  master_entry_fetches_.clear();
1527
1528  ClearPendingMasterEntries();
1529  DiscardInprogressCache();
1530
1531  // Delete response writer to avoid any callbacks.
1532  if (manifest_response_writer_)
1533    manifest_response_writer_.reset();
1534
1535  storage_->CancelDelegateCallbacks(this);
1536}
1537
1538void AppCacheUpdateJob::ClearPendingMasterEntries() {
1539  for (PendingMasters::iterator it = pending_master_entries_.begin();
1540       it != pending_master_entries_.end(); ++it) {
1541    PendingHosts& hosts = it->second;
1542    for (PendingHosts::iterator host_it = hosts.begin();
1543         host_it != hosts.end(); ++host_it) {
1544      (*host_it)->RemoveObserver(this);
1545    }
1546  }
1547
1548  pending_master_entries_.clear();
1549}
1550
1551void AppCacheUpdateJob::DiscardInprogressCache() {
1552  if (stored_state_ == STORING) {
1553    // We can make no assumptions about whether the StoreGroupAndCacheTask
1554    // actually completed or not. This condition should only be reachable
1555    // during shutdown. Free things up and return to do no harm.
1556    inprogress_cache_ = NULL;
1557    added_master_entries_.clear();
1558    return;
1559  }
1560
1561  storage_->DoomResponses(manifest_url_, stored_response_ids_);
1562
1563  if (!inprogress_cache_.get()) {
1564    // We have to undo the changes we made, if any, to the existing cache.
1565    if (group_ && group_->newest_complete_cache()) {
1566      for (std::vector<GURL>::iterator iter = added_master_entries_.begin();
1567           iter != added_master_entries_.end(); ++iter) {
1568        group_->newest_complete_cache()->RemoveEntry(*iter);
1569      }
1570    }
1571    added_master_entries_.clear();
1572    return;
1573  }
1574
1575  AppCache::AppCacheHosts& hosts = inprogress_cache_->associated_hosts();
1576  while (!hosts.empty())
1577    (*hosts.begin())->AssociateNoCache(GURL());
1578
1579  inprogress_cache_ = NULL;
1580  added_master_entries_.clear();
1581}
1582
1583void AppCacheUpdateJob::DiscardDuplicateResponses() {
1584  storage_->DoomResponses(manifest_url_, duplicate_response_ids_);
1585}
1586
1587void AppCacheUpdateJob::LogHistogramStats(
1588      ResultType result, const GURL& failed_resource_url) {
1589  AppCacheHistograms::CountUpdateJobResult(result, manifest_url_.GetOrigin());
1590  if (result == UPDATE_OK)
1591    return;
1592
1593  int percent_complete = 0;
1594  if (url_file_list_.size() > 0) {
1595    size_t actual_fetches_completed = url_fetches_completed_;
1596    if (!failed_resource_url.is_empty() && actual_fetches_completed)
1597      --actual_fetches_completed;
1598    percent_complete = (static_cast<double>(actual_fetches_completed) /
1599                            static_cast<double>(url_file_list_.size())) * 100.0;
1600    percent_complete = std::min(percent_complete, 99);
1601  }
1602
1603  bool was_making_progress =
1604      base::Time::Now() - last_progress_time_ <
1605          base::TimeDelta::FromMinutes(5);
1606
1607  bool off_origin_resource_failure =
1608      !failed_resource_url.is_empty() &&
1609          (failed_resource_url.GetOrigin() != manifest_url_.GetOrigin());
1610
1611  AppCacheHistograms::LogUpdateFailureStats(
1612      manifest_url_.GetOrigin(),
1613      percent_complete,
1614      was_making_progress,
1615      off_origin_resource_failure);
1616}
1617
1618void AppCacheUpdateJob::DeleteSoon() {
1619  ClearPendingMasterEntries();
1620  manifest_response_writer_.reset();
1621  storage_->CancelDelegateCallbacks(this);
1622  service_->RemoveObserver(this);
1623  service_ = NULL;
1624
1625  // Break the connection with the group so the group cannot call delete
1626  // on this object after we've posted a task to delete ourselves.
1627  group_->SetUpdateAppCacheStatus(AppCacheGroup::IDLE);
1628  group_ = NULL;
1629
1630  base::MessageLoop::current()->DeleteSoon(FROM_HERE, this);
1631}
1632
1633}  // namespace content
1634