func.cc revision 4e950e6f4e4d0204412550b2a67be7c231018eb8 
1// Copyright 2015 Google Inc. All rights reserved
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//      http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15// +build ignore
16
17#include "func.h"
18
19#include <errno.h>
20#include <glob.h>
21#include <limits.h>
22#include <stdio.h>
23#include <stdlib.h>
24#include <sys/types.h>
25#include <sys/wait.h>
26#include <unistd.h>
27
28#include <algorithm>
29#include <iterator>
30#include <memory>
31#include <unordered_map>
32
33#include "ast.h"
34#include "eval.h"
35#include "find.h"
36#include "log.h"
37#include "parser.h"
38#include "stats.h"
39#include "strutil.h"
40#include "symtab.h"
41#include "var.h"
42
43namespace {
44
45void PatsubstFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
46  shared_ptr<string> pat_str = args[0]->Eval(ev);
47  shared_ptr<string> repl = args[1]->Eval(ev);
48  shared_ptr<string> str = args[2]->Eval(ev);
49  WordWriter ww(s);
50  Pattern pat(*pat_str);
51  for (StringPiece tok : WordScanner(*str)) {
52    ww.MaybeAddWhitespace();
53    pat.AppendSubst(tok, *repl, s);
54  }
55}
56
57void StripFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
58  shared_ptr<string> str = args[0]->Eval(ev);
59  WordWriter ww(s);
60  for (StringPiece tok : WordScanner(*str)) {
61    ww.Write(tok);
62  }
63}
64
65void SubstFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
66  shared_ptr<string> pat = args[0]->Eval(ev);
67  shared_ptr<string> repl = args[1]->Eval(ev);
68  shared_ptr<string> str = args[2]->Eval(ev);
69  size_t index = 0;
70  while (index < str->size()) {
71    size_t found = str->find(*pat, index);
72    if (found == string::npos)
73      break;
74    AppendString(StringPiece(*str).substr(index, found - index), s);
75    AppendString(*repl, s);
76    index = found + pat->size();
77  }
78  AppendString(StringPiece(*str).substr(index), s);
79}
80
81void FindstringFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
82  shared_ptr<string> find = args[0]->Eval(ev);
83  shared_ptr<string> in = args[1]->Eval(ev);
84  if (in->find(*find) != string::npos)
85    AppendString(*find, s);
86}
87
88void FilterFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
89  shared_ptr<string> pat_buf = args[0]->Eval(ev);
90  shared_ptr<string> text = args[1]->Eval(ev);
91  vector<Pattern> pats;
92  for (StringPiece pat : WordScanner(*pat_buf)) {
93    pats.push_back(Pattern(pat));
94  }
95  WordWriter ww(s);
96  for (StringPiece tok : WordScanner(*text)) {
97    for (const Pattern& pat : pats) {
98      if (pat.Match(tok)) {
99        ww.Write(tok);
100        break;
101      }
102    }
103  }
104}
105
106void FilterOutFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
107  shared_ptr<string> pat_buf = args[0]->Eval(ev);
108  shared_ptr<string> text = args[1]->Eval(ev);
109  vector<Pattern> pats;
110  for (StringPiece pat : WordScanner(*pat_buf)) {
111    pats.push_back(Pattern(pat));
112  }
113  WordWriter ww(s);
114  for (StringPiece tok : WordScanner(*text)) {
115    bool matched = false;
116    for (const Pattern& pat : pats) {
117      if (pat.Match(tok)) {
118        matched = true;
119        break;
120      }
121    }
122    if (!matched)
123      ww.Write(tok);
124  }
125}
126
127void SortFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
128  shared_ptr<string> list = args[0]->Eval(ev);
129  vector<StringPiece> toks;
130  WordScanner(*list).Split(&toks);
131  sort(toks.begin(), toks.end());
132  WordWriter ww(s);
133  StringPiece prev;
134  for (StringPiece tok : toks) {
135    if (prev != tok) {
136      ww.Write(tok);
137      prev = tok;
138    }
139  }
140}
141
142static int GetNumericValueForFunc(const string& buf) {
143  StringPiece s = TrimLeftSpace(buf);
144  char* end;
145  long n = strtol(s.data(), &end, 10);
146  if (n < 0 || n == LONG_MAX || s.data() + s.size() != end) {
147    return -1;
148  }
149  return n;
150}
151
152void WordFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
153  shared_ptr<string> n_str = args[0]->Eval(ev);
154  int n = GetNumericValueForFunc(*n_str);
155  if (n < 0) {
156    ev->Error(StringPrintf(
157        "*** non-numeric first argument to `word' function: '%s'.",
158        n_str->c_str()));
159  }
160  if (n == 0) {
161    ev->Error("*** first argument to `word' function must be greater than 0.");
162  }
163
164  shared_ptr<string> text = args[1]->Eval(ev);
165  for (StringPiece tok : WordScanner(*text)) {
166    n--;
167    if (n == 0) {
168      AppendString(tok, s);
169      break;
170    }
171  }
172}
173
174void WordlistFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
175  shared_ptr<string> s_str = args[0]->Eval(ev);
176  int si = GetNumericValueForFunc(*s_str);
177  if (si < 0) {
178    ev->Error(StringPrintf(
179        "*** non-numeric first argument to `wordlist' function: '%s'.",
180        s_str->c_str()));
181  }
182  if (si == 0) {
183    ev->Error(StringPrintf(
184        "*** invalid first argument to `wordlist' function: %s`",
185        s_str->c_str()));
186  }
187
188  shared_ptr<string> e_str = args[1]->Eval(ev);
189  int ei = GetNumericValueForFunc(*e_str);
190  if (ei < 0) {
191    ev->Error(StringPrintf(
192        "*** non-numeric second argument to `wordlist' function: '%s'.",
193        e_str->c_str()));
194  }
195
196  shared_ptr<string> text = args[2]->Eval(ev);
197  int i = 0;
198  WordWriter ww(s);
199  for (StringPiece tok : WordScanner(*text)) {
200    i++;
201    if (si <= i && i <= ei) {
202      ww.Write(tok);
203    }
204  }
205}
206
207void WordsFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
208  shared_ptr<string> text = args[0]->Eval(ev);
209  WordScanner ws(*text);
210  int n = 0;
211  for (auto iter = ws.begin(); iter != ws.end(); ++iter)
212    n++;
213  char buf[32];
214  sprintf(buf, "%d", n);
215  *s += buf;
216}
217
218void FirstwordFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
219  shared_ptr<string> text = args[0]->Eval(ev);
220  for (StringPiece tok : WordScanner(*text)) {
221    AppendString(tok, s);
222    return;
223  }
224}
225
226void LastwordFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
227  shared_ptr<string> text = args[0]->Eval(ev);
228  StringPiece last;
229  for (StringPiece tok : WordScanner(*text)) {
230    last = tok;
231  }
232  AppendString(last, s);
233}
234
235void JoinFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
236  shared_ptr<string> list1 = args[0]->Eval(ev);
237  shared_ptr<string> list2 = args[1]->Eval(ev);
238  WordScanner ws1(*list1);
239  WordScanner ws2(*list2);
240  WordWriter ww(s);
241  for (WordScanner::Iterator iter1 = ws1.begin(), iter2 = ws2.begin();
242       iter1 != ws1.end() && iter2 != ws2.end();
243       ++iter1, ++iter2) {
244    ww.Write(*iter1);
245    // Use |AppendString| not to append extra ' '.
246    AppendString(*iter2, s);
247  }
248}
249
250void WildcardFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
251  COLLECT_STATS("func wildcard time");
252  shared_ptr<string> pat = args[0]->Eval(ev);
253  if (ev->avoid_io()) {
254    *s += "$(/bin/ls -d ";
255    *s += *pat;
256    *s += " 2> /dev/null)";
257    return;
258  }
259
260  WordWriter ww(s);
261  vector<const char*> files;
262  for (StringPiece tok : WordScanner(*pat)) {
263    ScopedTerminator st(tok);
264    // TODO: Make this faster by not always using glob.
265    files.clear();
266    glob_t gl;
267    glob(tok.data(), GLOB_NOSORT, NULL, &gl);
268    for (size_t i = 0; i < gl.gl_pathc; i++) {
269      files.push_back(gl.gl_pathv[i]);
270    }
271    sort(files.begin(), files.end(),
272         [](const char* a, const char* b) {
273           return strcmp(a, b) < 0;
274         });
275    for (const char* file : files) {
276      ww.Write(file);
277    }
278
279    globfree(&gl);
280  }
281}
282
283void DirFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
284  shared_ptr<string> text = args[0]->Eval(ev);
285  WordWriter ww(s);
286  for (StringPiece tok : WordScanner(*text)) {
287    ww.Write(Dirname(tok));
288    s->push_back('/');
289  }
290}
291
292void NotdirFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
293  shared_ptr<string> text = args[0]->Eval(ev);
294  WordWriter ww(s);
295  for (StringPiece tok : WordScanner(*text)) {
296    if (tok == "/") {
297      ww.Write(STRING_PIECE(""));
298    } else {
299      ww.Write(Basename(tok));
300    }
301  }
302}
303
304void SuffixFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
305  shared_ptr<string> text = args[0]->Eval(ev);
306  WordWriter ww(s);
307  for (StringPiece tok : WordScanner(*text)) {
308    StringPiece suf = GetExt(tok);
309    if (!suf.empty())
310      ww.Write(suf);
311  }
312}
313
314void BasenameFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
315  shared_ptr<string> text = args[0]->Eval(ev);
316  WordWriter ww(s);
317  for (StringPiece tok : WordScanner(*text)) {
318    ww.Write(StripExt(tok));
319  }
320}
321
322void AddsuffixFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
323  shared_ptr<string> suf = args[0]->Eval(ev);
324  shared_ptr<string> text = args[1]->Eval(ev);
325  WordWriter ww(s);
326  for (StringPiece tok : WordScanner(*text)) {
327    ww.Write(tok);
328    *s += *suf;
329  }
330}
331
332void AddprefixFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
333  shared_ptr<string> pre = args[0]->Eval(ev);
334  shared_ptr<string> text = args[1]->Eval(ev);
335  WordWriter ww(s);
336  for (StringPiece tok : WordScanner(*text)) {
337    ww.Write(*pre);
338    AppendString(tok, s);
339  }
340}
341
342void RealpathFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
343  shared_ptr<string> text = args[0]->Eval(ev);
344  if (ev->avoid_io()) {
345    *s += "KATI_TODO(realpath)";
346    return;
347  }
348
349  WordWriter ww(s);
350  for (StringPiece tok : WordScanner(*text)) {
351    ScopedTerminator st(tok);
352    char buf[PATH_MAX];
353    if (realpath(tok.data(), buf))
354      *s += buf;
355  }
356}
357
358void AbspathFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
359  shared_ptr<string> text = args[0]->Eval(ev);
360  WordWriter ww(s);
361  string buf;
362  for (StringPiece tok : WordScanner(*text)) {
363    AbsPath(tok, &buf);
364    ww.Write(buf);
365  }
366}
367
368void IfFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
369  shared_ptr<string> cond = args[0]->Eval(ev);
370  if (cond->empty()) {
371    if (args.size() > 2)
372      args[2]->Eval(ev, s);
373  } else {
374    args[1]->Eval(ev, s);
375  }
376}
377
378void AndFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
379  shared_ptr<string> cond;
380  for (Value* a : args) {
381    cond = a->Eval(ev);
382    if (cond->empty())
383      return;
384  }
385  if (cond.get()) {
386    *s += *cond;
387  }
388}
389
390void OrFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
391  for (Value* a : args) {
392    shared_ptr<string> cond = a->Eval(ev);
393    if (!cond->empty()) {
394      *s += *cond;
395      return;
396    }
397  }
398}
399
400void ValueFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
401  shared_ptr<string> var_name = args[0]->Eval(ev);
402  Var* var = ev->LookupVar(Intern(*var_name));
403  AppendString(var->String().as_string(), s);
404}
405
406void EvalFunc(const vector<Value*>& args, Evaluator* ev, string*) {
407  // TODO: eval leaks everything... for now.
408  //shared_ptr<string> text = args[0]->Eval(ev);
409  string* text = new string;
410  args[0]->Eval(ev, text);
411  vector<AST*> asts;
412  Parse(*text, ev->loc(), &asts);
413  for (AST* ast : asts) {
414    LOG("%s", ast->DebugString().c_str());
415    ast->Eval(ev);
416    //delete ast;
417  }
418}
419
420//#define TEST_FIND_EMULATOR
421
422#ifdef TEST_FIND_EMULATOR
423static string SortWordsInString(StringPiece s) {
424  vector<string> toks;
425  for (StringPiece tok : WordScanner(s)) {
426    toks.push_back(tok.as_string());
427  }
428  sort(toks.begin(), toks.end());
429  return JoinStrings(toks, " ");
430}
431#endif
432
433static void RunCommand(const string& cmd, string* s) {
434  int pipefd[2];
435  if (pipe(pipefd) != 0)
436    PERROR("pipe failed");
437  int pid;
438  if ((pid = vfork())) {
439    close(pipefd[1]);
440    while (true) {
441      int status;
442      int result = waitpid(pid, &status, WNOHANG);
443      if (result < 0)
444        PERROR("waitpid failed");
445
446      while (true) {
447        char buf[4096];
448        ssize_t r = read(pipefd[0], buf, 4096);
449        if (r < 0)
450          PERROR("read failed");
451        if (r == 0)
452          break;
453        s->append(buf, buf+r);
454      }
455
456      if (result != 0) {
457        break;
458      }
459    }
460    close(pipefd[0]);
461  } else {
462    close(pipefd[0]);
463    if (dup2(pipefd[1], 1) < 0)
464      PERROR("dup2 failed");
465    close(pipefd[1]);
466
467    const char* argv[] = {
468      // TODO: Handle $(SHELL).
469      "/bin/sh", "-c", cmd.c_str(), NULL
470    };
471    execvp(argv[0], const_cast<char**>(argv));
472  }
473}
474
475void ShellFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
476  shared_ptr<string> cmd = args[0]->Eval(ev);
477  if (ev->avoid_io()) {
478    *s += "$(";
479    *s += *cmd;
480    *s += ")";
481    return;
482  }
483
484  LOG("ShellFunc: %s", cmd->c_str());
485
486#ifdef TEST_FIND_EMULATOR
487  bool need_check = false;
488  string out2;
489  if (FindEmulator::Get() && FindEmulator::Get()->HandleFind(*cmd, &out2)) {
490    need_check = true;
491  }
492#else
493  if (FindEmulator::Get() && FindEmulator::Get()->HandleFind(*cmd, s))
494    return;
495#endif
496
497  COLLECT_STATS_WITH_SLOW_REPORT("func shell time", cmd->c_str());
498  string out;
499#if 0
500  // TODO: Handle $(SHELL).
501  FILE* fp = popen(cmd->c_str(), "r");
502  while (true) {
503    char buf[4096];
504    size_t r = fread(buf, 1, 4096, fp);
505    out.append(buf, buf+r);
506    if (r == 0) {
507      fclose(fp);
508      break;
509    }
510  }
511#else
512  RunCommand(*cmd, &out);
513#endif
514
515  while (out[out.size()-1] == '\n')
516    out.pop_back();
517  for (size_t i = 0; i < out.size(); i++) {
518    if (out[i] == '\n')
519      out[i] = ' ';
520  }
521
522#ifdef TEST_FIND_EMULATOR
523  if (need_check) {
524    string sorted = SortWordsInString(out);
525    out2 = SortWordsInString(out2);
526    if (sorted != out2) {
527      ERROR("FindEmulator is broken: %s\n%s\nvs\n%s",
528            cmd->c_str(), sorted.c_str(), out2.c_str());
529    }
530  }
531#endif
532
533  *s += out;
534}
535
536void CallFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
537  static const char* tmpvar_names[] = {
538    "0", "1",  "2",  "3",  "4",  "5",  "6",  "7",  "8",  "9"
539  };
540
541  shared_ptr<string> func_name = args[0]->Eval(ev);
542  Var* func = ev->LookupVar(Intern(*func_name));
543  vector<unique_ptr<SimpleVar>> av;
544  for (size_t i = 1; i < args.size(); i++) {
545    unique_ptr<SimpleVar> s(
546        new SimpleVar(args[i]->Eval(ev), VarOrigin::AUTOMATIC));
547    av.push_back(move(s));
548  }
549  vector<unique_ptr<ScopedVar>> sv;
550  for (size_t i = 1; i < args.size(); i++) {
551    sv.push_back(move(unique_ptr<ScopedVar>(
552        new ScopedVar(ev->mutable_vars(),
553                      Intern(tmpvar_names[i]), av[i-1].get()))));
554  }
555  func->Eval(ev, s);
556}
557
558void ForeachFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
559  shared_ptr<string> varname = args[0]->Eval(ev);
560  shared_ptr<string> list = args[1]->Eval(ev);
561  WordWriter ww(s);
562  for (StringPiece tok : WordScanner(*list)) {
563    unique_ptr<SimpleVar> v(new SimpleVar(
564        make_shared<string>(tok.data(), tok.size()), VarOrigin::AUTOMATIC));
565    ScopedVar sv(ev->mutable_vars(), Intern(*varname), v.get());
566    ww.MaybeAddWhitespace();
567    args[2]->Eval(ev, s);
568  }
569}
570
571void OriginFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
572  shared_ptr<string> var_name = args[0]->Eval(ev);
573  Var* var = ev->LookupVar(Intern(*var_name));
574  *s += GetOriginStr(var->Origin());
575}
576
577void FlavorFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
578  shared_ptr<string> var_name = args[0]->Eval(ev);
579  Var* var = ev->LookupVar(Intern(*var_name));
580  *s += var->Flavor();
581}
582
583void InfoFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
584  shared_ptr<string> a = args[0]->Eval(ev);
585  if (ev->avoid_io()) {
586    *s += "KATI_TODO(info)";
587    return;
588  }
589  printf("%s\n", a->c_str());
590  fflush(stdout);
591}
592
593void WarningFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
594  shared_ptr<string> a = args[0]->Eval(ev);
595  if (ev->avoid_io()) {
596    *s += "KATI_TODO(warning)";
597    return;
598  }
599  printf("%s:%d: %s\n", LOCF(ev->loc()), a->c_str());
600  fflush(stdout);
601}
602
603void ErrorFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
604  shared_ptr<string> a = args[0]->Eval(ev);
605  if (ev->avoid_io()) {
606    *s += "KATI_TODO(error)";
607    return;
608  }
609  ev->Error(StringPrintf("*** %s.", a->c_str()));
610}
611
612FuncInfo g_func_infos[] = {
613  { "patsubst", &PatsubstFunc, 3, 3, false, false },
614  { "strip", &StripFunc, 1, 1, false, false },
615  { "subst", &SubstFunc, 3, 3, false, false },
616  { "findstring", &FindstringFunc, 2, 2, false, false },
617  { "filter", &FilterFunc, 2, 2, false, false },
618  { "filter-out", &FilterOutFunc, 2, 2, false, false },
619  { "sort", &SortFunc, 1, 1, false, false },
620  { "word", &WordFunc, 2, 2, false, false },
621  { "wordlist", &WordlistFunc, 3, 3, false, false },
622  { "words", &WordsFunc, 1, 1, false, false },
623  { "firstword", &FirstwordFunc, 1, 1, false, false },
624  { "lastword", &LastwordFunc, 1, 1, false, false },
625
626  { "join", &JoinFunc, 2, 2, false, false },
627  { "wildcard", &WildcardFunc, 1, 1, false, false },
628  { "dir", &DirFunc, 1, 1, false, false },
629  { "notdir", &NotdirFunc, 1, 1, false, false },
630  { "suffix", &SuffixFunc, 1, 1, false, false },
631  { "basename", &BasenameFunc, 1, 1, false, false },
632  { "addsuffix", &AddsuffixFunc, 2, 2, false, false },
633  { "addprefix", &AddprefixFunc, 2, 2, false, false },
634  { "realpath", &RealpathFunc, 1, 1, false, false },
635  { "abspath", &AbspathFunc, 1, 1, false, false },
636
637  { "if", &IfFunc, 3, 2, false, true },
638  { "and", &AndFunc, 0, 0, true, false },
639  { "or", &OrFunc, 0, 0, true, false },
640
641  { "value", &ValueFunc, 1, 1, false, false },
642  { "eval", &EvalFunc, 1, 1, false, false },
643  { "shell", &ShellFunc, 1, 1, false, false },
644  { "call", &CallFunc, 0, 0, false, false },
645  { "foreach", &ForeachFunc, 3, 3, false, false },
646
647  { "origin", &OriginFunc, 1, 1, false, false },
648  { "flavor", &FlavorFunc, 1, 1, false, false },
649
650  { "info", &InfoFunc, 1, 1, false, false },
651  { "warning", &WarningFunc, 1, 1, false, false },
652  { "error", &ErrorFunc, 1, 1, false, false },
653};
654
655unordered_map<StringPiece, FuncInfo*>* g_func_info_map;
656
657}  // namespace
658
659void InitFuncTable() {
660  g_func_info_map = new unordered_map<StringPiece, FuncInfo*>;
661  for (size_t i = 0; i < sizeof(g_func_infos) / sizeof(g_func_infos[0]); i++) {
662    FuncInfo* fi = &g_func_infos[i];
663    bool ok = g_func_info_map->emplace(fi->name, fi).second;
664    CHECK(ok);
665  }
666}
667
668void QuitFuncTable() {
669  delete g_func_info_map;
670}
671
672FuncInfo* GetFuncInfo(StringPiece name) {
673  auto found = g_func_info_map->find(name);
674  if (found == g_func_info_map->end())
675    return NULL;
676  return found->second;
677}
678