1// Copyright (c) 2013 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "base/command_line.h"
6#include "base/file_util.h"
7#include "base/logging.h"
8#include "base/process/kill.h"
9#include "base/process/launch.h"
10#include "base/strings/string_number_conversions.h"
11#include "base/strings/utf_string_conversions.h"
12#include "base/time/time.h"
13#include "build/build_config.h"
14#include "tools/gn/err.h"
15#include "tools/gn/filesystem_utils.h"
16#include "tools/gn/functions.h"
17#include "tools/gn/input_conversion.h"
18#include "tools/gn/input_file.h"
19#include "tools/gn/parse_tree.h"
20#include "tools/gn/scheduler.h"
21#include "tools/gn/trace.h"
22#include "tools/gn/value.h"
23
24#if defined(OS_WIN)
25#include <windows.h>
26
27#include "base/win/scoped_handle.h"
28#include "base/win/scoped_process_information.h"
29#endif
30
31#if defined(OS_POSIX)
32#include <fcntl.h>
33#include <unistd.h>
34
35#include "base/posix/file_descriptor_shuffle.h"
36#endif
37
38namespace functions {
39
40namespace {
41
42const char kNoExecSwitch[] = "no-exec";
43
44#if defined(OS_WIN)
45bool ExecProcess(const CommandLine& cmdline,
46                 const base::FilePath& startup_dir,
47                 std::string* std_out,
48                 std::string* std_err,
49                 int* exit_code) {
50  SECURITY_ATTRIBUTES sa_attr;
51  // Set the bInheritHandle flag so pipe handles are inherited.
52  sa_attr.nLength = sizeof(SECURITY_ATTRIBUTES);
53  sa_attr.bInheritHandle = TRUE;
54  sa_attr.lpSecurityDescriptor = NULL;
55
56  // Create the pipe for the child process's STDOUT.
57  HANDLE out_read = NULL;
58  HANDLE out_write = NULL;
59  if (!CreatePipe(&out_read, &out_write, &sa_attr, 0)) {
60    NOTREACHED() << "Failed to create pipe";
61    return false;
62  }
63  base::win::ScopedHandle scoped_out_read(out_read);
64  base::win::ScopedHandle scoped_out_write(out_write);
65
66  // Create the pipe for the child process's STDERR.
67  HANDLE err_read = NULL;
68  HANDLE err_write = NULL;
69  if (!CreatePipe(&err_read, &err_write, &sa_attr, 0)) {
70    NOTREACHED() << "Failed to create pipe";
71    return false;
72  }
73  base::win::ScopedHandle scoped_err_read(err_read);
74  base::win::ScopedHandle scoped_err_write(err_write);
75
76  // Ensure the read handle to the pipe for STDOUT/STDERR is not inherited.
77  if (!SetHandleInformation(out_read, HANDLE_FLAG_INHERIT, 0)) {
78    NOTREACHED() << "Failed to disabled pipe inheritance";
79    return false;
80  }
81  if (!SetHandleInformation(err_read, HANDLE_FLAG_INHERIT, 0)) {
82    NOTREACHED() << "Failed to disabled pipe inheritance";
83    return false;
84  }
85
86  base::FilePath::StringType cmdline_str(cmdline.GetCommandLineString());
87
88  STARTUPINFO start_info = {};
89
90  start_info.cb = sizeof(STARTUPINFO);
91  start_info.hStdOutput = out_write;
92  // Keep the normal stdin.
93  start_info.hStdInput = GetStdHandle(STD_INPUT_HANDLE);
94  // FIXME(brettw) set stderr here when we actually read it below.
95  //start_info.hStdError = err_write;
96  start_info.hStdError = GetStdHandle(STD_ERROR_HANDLE);
97  start_info.dwFlags |= STARTF_USESTDHANDLES;
98
99  // Create the child process.
100  PROCESS_INFORMATION temp_process_info = {};
101  if (!CreateProcess(NULL,
102                     &cmdline_str[0],
103                     NULL, NULL,
104                     TRUE,  // Handles are inherited.
105                     0, NULL,
106                     startup_dir.value().c_str(),
107                     &start_info, &temp_process_info)) {
108    return false;
109  }
110  base::win::ScopedProcessInformation proc_info(temp_process_info);
111
112  // Close our writing end of pipes now. Otherwise later read would not be able
113  // to detect end of child's output.
114  scoped_out_write.Close();
115  scoped_err_write.Close();
116
117  // Read output from the child process's pipe for STDOUT
118  const int kBufferSize = 1024;
119  char buffer[kBufferSize];
120
121  // FIXME(brettw) read from stderr here! This is complicated because we want
122  // to read both of them at the same time, probably need overlapped I/O.
123  // Also uncomment start_info code above.
124  for (;;) {
125    DWORD bytes_read = 0;
126    BOOL success = ReadFile(out_read, buffer, kBufferSize, &bytes_read, NULL);
127    if (!success || bytes_read == 0)
128      break;
129    std_out->append(buffer, bytes_read);
130  }
131
132  // Let's wait for the process to finish.
133  WaitForSingleObject(proc_info.process_handle(), INFINITE);
134
135  DWORD dw_exit_code;
136  GetExitCodeProcess(proc_info.process_handle(), &dw_exit_code);
137  *exit_code = static_cast<int>(dw_exit_code);
138
139  return true;
140}
141#else
142bool ExecProcess(const CommandLine& cmdline,
143                 const base::FilePath& startup_dir,
144                 std::string* std_out,
145                 std::string* std_err,
146                 int* exit_code) {
147  *exit_code = EXIT_FAILURE;
148
149  std::vector<std::string> argv = cmdline.argv();
150
151  int pipe_fd[2];
152  pid_t pid;
153  base::InjectiveMultimap fd_shuffle1, fd_shuffle2;
154  scoped_ptr<char*[]> argv_cstr(new char*[argv.size() + 1]);
155
156  fd_shuffle1.reserve(3);
157  fd_shuffle2.reserve(3);
158
159  if (pipe(pipe_fd) < 0)
160    return false;
161
162  switch (pid = fork()) {
163    case -1:  // error
164      close(pipe_fd[0]);
165      close(pipe_fd[1]);
166      return false;
167    case 0:  // child
168      {
169        // DANGER: no calls to malloc are allowed from now on:
170        // http://crbug.com/36678
171
172        // Obscure fork() rule: in the child, if you don't end up doing exec*(),
173        // you call _exit() instead of exit(). This is because _exit() does not
174        // call any previously-registered (in the parent) exit handlers, which
175        // might do things like block waiting for threads that don't even exist
176        // in the child.
177        int dev_null = open("/dev/null", O_WRONLY);
178        if (dev_null < 0)
179          _exit(127);
180
181        fd_shuffle1.push_back(
182            base::InjectionArc(pipe_fd[1], STDOUT_FILENO, true));
183        fd_shuffle1.push_back(
184            base::InjectionArc(dev_null, STDERR_FILENO, true));
185        fd_shuffle1.push_back(
186            base::InjectionArc(dev_null, STDIN_FILENO, true));
187        // Adding another element here? Remeber to increase the argument to
188        // reserve(), above.
189
190        std::copy(fd_shuffle1.begin(), fd_shuffle1.end(),
191                  std::back_inserter(fd_shuffle2));
192
193        if (!ShuffleFileDescriptors(&fd_shuffle1))
194          _exit(127);
195
196        file_util::SetCurrentDirectory(startup_dir);
197
198        // TODO(brettw) the base version GetAppOutput does a
199        // CloseSuperfluousFds call here. Do we need this?
200
201        for (size_t i = 0; i < argv.size(); i++)
202          argv_cstr[i] = const_cast<char*>(argv[i].c_str());
203        argv_cstr[argv.size()] = NULL;
204        execvp(argv_cstr[0], argv_cstr.get());
205        _exit(127);
206      }
207    default:  // parent
208      {
209        // Close our writing end of pipe now. Otherwise later read would not
210        // be able to detect end of child's output (in theory we could still
211        // write to the pipe).
212        close(pipe_fd[1]);
213
214        char buffer[256];
215        ssize_t bytes_read = 0;
216
217        while (true) {
218          bytes_read = HANDLE_EINTR(read(pipe_fd[0], buffer, sizeof(buffer)));
219          if (bytes_read <= 0)
220            break;
221          std_out->append(buffer, bytes_read);
222        }
223        close(pipe_fd[0]);
224
225        return base::WaitForExitCode(pid, exit_code);
226      }
227  }
228
229  return false;
230}
231#endif
232
233}  // namespace
234
235const char kExecScript[] = "exec_script";
236const char kExecScript_Help[] =
237    "exec_script: Synchronously run a script and return the output.\n"
238    "\n"
239    "  exec_script(filename, arguments, input_conversion,\n"
240    "              [file_dependencies])\n"
241    "\n"
242    "  Runs the given script, returning the stdout of the script. The build\n"
243    "  generation will fail if the script does not exist or returns a nonzero\n"
244    "  exit code.\n"
245    "\n"
246    "  The current directory when executing the script will be the root\n"
247    "  build directory. If you are passing file names, you will want to use\n"
248    "  the to_build_dir() function to make file names relative to this\n"
249    "  path (see \"gn help to_build_dir\").\n"
250    "\n"
251    "Arguments:\n"
252    "\n"
253    "  filename:\n"
254    "      File name of python script to execute. Non-absolute names will\n"
255    "      be treated as relative to the current build file.\n"
256    "\n"
257    "  arguments:\n"
258    "      A list of strings to be passed to the script as arguments.\n"
259    "\n"
260    "  input_conversion:\n"
261    "      Controls how the file is read and parsed.\n"
262    "      See \"gn help input_conversion\".\n"
263    "\n"
264    "  dependencies:\n"
265    "      (Optional) A list of files that this script reads or otherwise\n"
266    "      depends on. These dependencies will be added to the build result\n"
267    "      such that if any of them change, the build will be regenerated and\n"
268    "      the script will be re-run.\n"
269    "\n"
270    "      The script itself will be an implicit dependency so you do not\n"
271    "      need to list it.\n"
272    "\n"
273    "Example:\n"
274    "\n"
275    "  all_lines = exec_script(\"myscript.py\", [some_input], \"list lines\",\n"
276    "                          [ to_build_dir(\"data_file.txt\") ])\n";
277
278Value RunExecScript(Scope* scope,
279                    const FunctionCallNode* function,
280                    const std::vector<Value>& args,
281                    Err* err) {
282  if (args.size() != 3 && args.size() != 4) {
283    *err = Err(function->function(), "Wrong number of args to write_file",
284               "I expected three or four arguments.");
285    return Value();
286  }
287
288  const Settings* settings = scope->settings();
289  const BuildSettings* build_settings = settings->build_settings();
290  const SourceDir& cur_dir = scope->GetSourceDir();
291
292  // Find the python script to run.
293  if (!args[0].VerifyTypeIs(Value::STRING, err))
294    return Value();
295  SourceFile script_source =
296      cur_dir.ResolveRelativeFile(args[0].string_value());
297  base::FilePath script_path = build_settings->GetFullPath(script_source);
298  if (!build_settings->secondary_source_path().empty() &&
299      !base::PathExists(script_path)) {
300    // Fall back to secondary source root when the file doesn't exist.
301    script_path = build_settings->GetFullPathSecondary(script_source);
302  }
303
304  ScopedTrace trace(TraceItem::TRACE_SCRIPT_EXECUTE, script_source.value());
305  trace.SetToolchain(settings->toolchain_label());
306
307  // Add all dependencies of this script, including the script itself, to the
308  // build deps.
309  g_scheduler->AddGenDependency(script_path);
310  if (args.size() == 4) {
311    const Value& deps_value = args[3];
312    if (!deps_value.VerifyTypeIs(Value::LIST, err))
313      return Value();
314
315    for (size_t i = 0; i < deps_value.list_value().size(); i++) {
316      if (!deps_value.list_value()[0].VerifyTypeIs(Value::STRING, err))
317        return Value();
318      g_scheduler->AddGenDependency(
319          build_settings->GetFullPath(cur_dir.ResolveRelativeFile(
320              deps_value.list_value()[0].string_value())));
321    }
322  }
323
324  // Make the command line.
325  const base::FilePath& python_path = build_settings->python_path();
326  CommandLine cmdline(python_path);
327  cmdline.AppendArgPath(script_path);
328
329  const Value& script_args = args[1];
330  if (!script_args.VerifyTypeIs(Value::LIST, err))
331    return Value();
332  for (size_t i = 0; i < script_args.list_value().size(); i++) {
333    if (!script_args.list_value()[i].VerifyTypeIs(Value::STRING, err))
334      return Value();
335    cmdline.AppendArg(script_args.list_value()[i].string_value());
336  }
337
338  // Log command line for debugging help.
339  trace.SetCommandLine(cmdline);
340  base::TimeTicks begin_exec;
341  if (g_scheduler->verbose_logging()) {
342#if defined(OS_WIN)
343    g_scheduler->Log("Pythoning", UTF16ToUTF8(cmdline.GetCommandLineString()));
344#else
345    g_scheduler->Log("Pythoning", cmdline.GetCommandLineString());
346#endif
347    begin_exec = base::TimeTicks::Now();
348  }
349
350  base::FilePath startup_dir =
351      build_settings->GetFullPath(build_settings->build_dir());
352  // The first time a build is run, no targets will have been written so the
353  // build output directory won't exist. We need to make sure it does before
354  // running any scripts with this as its startup directory, although it will
355  // be relatively rare that the directory won't exist by the time we get here.
356  //
357  // If this shows up on benchmarks, we can cache whether we've done this
358  // or not and skip creating the directory.
359  base::CreateDirectory(startup_dir);
360
361  // Execute the process.
362  // TODO(brettw) set the environment block.
363  std::string output;
364  std::string stderr_output;  // TODO(brettw) not hooked up, see above.
365  int exit_code = 0;
366  if (!CommandLine::ForCurrentProcess()->HasSwitch(kNoExecSwitch)) {
367    if (!ExecProcess(cmdline, startup_dir,
368                     &output, &stderr_output, &exit_code)) {
369      *err = Err(function->function(), "Could not execute python.",
370          "I was trying to execute \"" + FilePathToUTF8(python_path) + "\".");
371      return Value();
372    }
373  }
374  if (g_scheduler->verbose_logging()) {
375    g_scheduler->Log("Pythoning", script_source.value() + " took " +
376        base::Int64ToString(
377            (base::TimeTicks::Now() - begin_exec).InMilliseconds()) +
378        "ms");
379  }
380
381  // TODO(brettw) maybe we need stderr also for reasonable stack dumps.
382  if (exit_code != 0) {
383    std::string msg = "Current dir: " + FilePathToUTF8(startup_dir) +
384        "\nCommand: " + FilePathToUTF8(cmdline.GetCommandLineString()) +
385        "\nReturned " + base::IntToString(exit_code);
386    if (!output.empty())
387      msg += " and printed out:\n\n" + output;
388    else
389      msg += ".";
390    *err = Err(function->function(), "Script returned non-zero exit code.",
391               msg);
392    return Value();
393  }
394
395  return ConvertInputToValue(output, function, args[2], err);
396}
397
398}  // namespace functions
399