1// Copyright (c) 2013 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "tools/gn/input_file_manager.h"
6
7#include "base/bind.h"
8#include "base/stl_util.h"
9#include "tools/gn/filesystem_utils.h"
10#include "tools/gn/parser.h"
11#include "tools/gn/scheduler.h"
12#include "tools/gn/scope_per_file_provider.h"
13#include "tools/gn/tokenizer.h"
14#include "tools/gn/trace.h"
15
16namespace {
17
18void InvokeFileLoadCallback(const InputFileManager::FileLoadCallback& cb,
19                            const ParseNode* node) {
20  cb.Run(node);
21}
22
23}  // namespace
24
25InputFileManager::InputFileData::InputFileData(const SourceFile& file_name)
26    : file(file_name),
27      loaded(false),
28      sync_invocation(false) {
29}
30
31InputFileManager::InputFileData::~InputFileData() {
32}
33
34InputFileManager::InputFileManager() {
35}
36
37InputFileManager::~InputFileManager() {
38  // Should be single-threaded by now.
39  STLDeleteContainerPairSecondPointers(input_files_.begin(),
40                                       input_files_.end());
41}
42
43bool InputFileManager::AsyncLoadFile(const LocationRange& origin,
44                                     const BuildSettings* build_settings,
45                                     const SourceFile& file_name,
46                                     const FileLoadCallback& callback,
47                                     Err* err) {
48  // Try not to schedule callbacks while holding the lock. All cases that don't
49  // want to schedule should return early. Otherwise, this will be scheduled
50  // after we leave the lock.
51  base::Closure schedule_this;
52  {
53    base::AutoLock lock(lock_);
54
55    InputFileMap::const_iterator found = input_files_.find(file_name);
56    if (found == input_files_.end()) {
57      // New file, schedule load.
58      InputFileData* data = new InputFileData(file_name);
59      data->scheduled_callbacks.push_back(callback);
60      input_files_[file_name] = data;
61
62      schedule_this = base::Bind(&InputFileManager::BackgroundLoadFile,
63                                 this,
64                                 origin,
65                                 build_settings,
66                                 file_name,
67                                 &data->file);
68    } else {
69      InputFileData* data = found->second;
70
71      // Prevent mixing async and sync loads. See SyncLoadFile for discussion.
72      if (data->sync_invocation) {
73        g_scheduler->FailWithError(Err(
74            origin, "Load type mismatch.",
75            "The file \"" + file_name.value() + "\" was previously loaded\n"
76            "synchronously (via an import) and now you're trying to load it "
77            "asynchronously\n(via a deps rule). This is a class 2 misdemeanor: "
78            "a single input file must\nbe loaded the same way each time to "
79            "avoid blowing my tiny, tiny mind."));
80        return false;
81      }
82
83      if (data->loaded) {
84        // Can just directly issue the callback on the background thread.
85        schedule_this = base::Bind(&InvokeFileLoadCallback, callback,
86                                   data->parsed_root.get());
87      } else {
88        // Load is pending on this file, schedule the invoke.
89        data->scheduled_callbacks.push_back(callback);
90        return true;
91      }
92    }
93  }
94  g_scheduler->pool()->PostWorkerTaskWithShutdownBehavior(
95      FROM_HERE, schedule_this,
96      base::SequencedWorkerPool::BLOCK_SHUTDOWN);
97  return true;
98}
99
100const ParseNode* InputFileManager::SyncLoadFile(
101    const LocationRange& origin,
102    const BuildSettings* build_settings,
103    const SourceFile& file_name,
104    Err* err) {
105  base::AutoLock lock(lock_);
106
107  InputFileData* data = NULL;
108  InputFileMap::iterator found = input_files_.find(file_name);
109  if (found == input_files_.end()) {
110    // Haven't seen this file yet, start loading right now.
111    data = new InputFileData(file_name);
112    data->sync_invocation = true;
113    input_files_[file_name] = data;
114
115    base::AutoUnlock unlock(lock_);
116    if (!LoadFile(origin, build_settings, file_name, &data->file, err))
117      return NULL;
118  } else {
119    // This file has either been loaded or is pending loading.
120    data = found->second;
121
122    if (!data->sync_invocation) {
123      // Don't allow mixing of sync and async loads. If an async load is
124      // scheduled and then a bunch of threads need to load it synchronously
125      // and block on it loading, it could deadlock or at least cause a lot
126      // of wasted CPU while those threads wait for the load to complete (which
127      // may be far back in the input queue).
128      //
129      // We could work around this by promoting the load to a sync load. This
130      // requires a bunch of extra code to either check flags and likely do
131      // extra locking (bad) or to just do both types of load on the file and
132      // deal with the race condition.
133      //
134      // I have no practical way to test this, and generally we should have
135      // all include files processed synchronously and all build files
136      // processed asynchronously, so it doesn't happen in practice.
137      *err = Err(
138          origin, "Load type mismatch.",
139          "The file \"" + file_name.value() + "\" was previously loaded\n"
140          "asynchronously (via a deps rule) and now you're trying to load it "
141          "synchronously.\nThis is a class 2 misdemeanor: a single input file "
142          "must be loaded the same way\neach time to avoid blowing my tiny, "
143          "tiny mind.");
144      return NULL;
145    }
146
147    if (!data->loaded) {
148      // Wait for the already-pending sync load to complete.
149      if (!data->completion_event)
150        data->completion_event.reset(new base::WaitableEvent(false, false));
151      {
152        base::AutoUnlock unlock(lock_);
153        data->completion_event->Wait();
154      }
155      // If there were multiple waiters on the same event, we now need to wake
156      // up the next one.
157      data->completion_event->Signal();
158    }
159  }
160
161  // The other load could have failed. In this case that error will be printed
162  // to the console, but we need to return something here, so make up a
163  // dummy error.
164  if (!data->parsed_root)
165    *err = Err(origin, "File parse failed");
166  return data->parsed_root.get();
167}
168
169int InputFileManager::GetInputFileCount() const {
170  base::AutoLock lock(lock_);
171  return static_cast<int>(input_files_.size());
172}
173
174void InputFileManager::GetAllPhysicalInputFileNames(
175    std::vector<base::FilePath>* result) const {
176  base::AutoLock lock(lock_);
177  result->reserve(input_files_.size());
178  for (InputFileMap::const_iterator i = input_files_.begin();
179       i != input_files_.end(); ++i) {
180    if (!i->second->file.physical_name().empty())
181      result->push_back(i->second->file.physical_name());
182  }
183}
184
185void InputFileManager::BackgroundLoadFile(const LocationRange& origin,
186                                          const BuildSettings* build_settings,
187                                          const SourceFile& name,
188                                          InputFile* file) {
189  Err err;
190  if (!LoadFile(origin, build_settings, name, file, &err))
191    g_scheduler->FailWithError(err);
192}
193
194bool InputFileManager::LoadFile(const LocationRange& origin,
195                                const BuildSettings* build_settings,
196                                const SourceFile& name,
197                                InputFile* file,
198                                Err* err) {
199  // Do all of this stuff outside the lock. We should not give out file
200  // pointers until the read is complete.
201  if (g_scheduler->verbose_logging()) {
202    std::string logmsg = name.value();
203    if (origin.begin().file())
204      logmsg += " (referenced from " + origin.begin().Describe(false) + ")";
205    g_scheduler->Log("Loading", logmsg);
206  }
207
208  // Read.
209  base::FilePath primary_path = build_settings->GetFullPath(name);
210  ScopedTrace load_trace(TraceItem::TRACE_FILE_LOAD, name.value());
211  if (!file->Load(primary_path)) {
212    if (!build_settings->secondary_source_path().empty()) {
213      // Fall back to secondary source tree.
214      base::FilePath secondary_path =
215          build_settings->GetFullPathSecondary(name);
216      if (!file->Load(secondary_path)) {
217        *err = Err(origin, "Can't load input file.",
218                   "Unable to load either \n" +
219                   FilePathToUTF8(primary_path) + " or \n" +
220                   FilePathToUTF8(secondary_path));
221        return false;
222      }
223    } else {
224      *err = Err(origin,
225                 "Unable to load \"" + FilePathToUTF8(primary_path) + "\".");
226      return false;
227    }
228  }
229  load_trace.Done();
230
231  ScopedTrace exec_trace(TraceItem::TRACE_FILE_PARSE, name.value());
232
233  // Tokenize.
234  std::vector<Token> tokens = Tokenizer::Tokenize(file, err);
235  if (err->has_error())
236    return false;
237
238  // Parse.
239  scoped_ptr<ParseNode> root = Parser::Parse(tokens, err);
240  if (err->has_error())
241    return false;
242  ParseNode* unowned_root = root.get();
243
244  exec_trace.Done();
245
246  std::vector<FileLoadCallback> callbacks;
247  {
248    base::AutoLock lock(lock_);
249    DCHECK(input_files_.find(name) != input_files_.end());
250
251    InputFileData* data = input_files_[name];
252    data->loaded = true;
253    data->tokens.swap(tokens);
254    data->parsed_root = root.Pass();
255
256    // Unblock waiters on this event.
257    //
258    // It's somewhat bad to signal this inside the lock. When it's used, it's
259    // lazily created inside the lock. So we need to do the check and signal
260    // inside the lock to avoid race conditions on the lazy creation of the
261    // lock.
262    //
263    // We could avoid this by creating the lock every time, but the lock is
264    // very seldom used and will generally be NULL, so my current theory is that
265    // several signals of a completion event inside a lock is better than
266    // creating about 1000 extra locks (one for each file).
267    if (data->completion_event)
268      data->completion_event->Signal();
269
270    callbacks.swap(data->scheduled_callbacks);
271  }
272
273  // Run pending invocations. Theoretically we could schedule each of these
274  // separately to get some parallelism. But normally there will only be one
275  // item in the list, so that's extra overhead and complexity for no gain.
276  for (size_t i = 0; i < callbacks.size(); i++)
277    callbacks[i].Run(unowned_root);
278  return true;
279}
280