func.cc revision 62b16e7b9424f88e4907a6879d15ee832ef49477
1// Copyright 2015 Google Inc. All rights reserved 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15// +build ignore 16 17#include "func.h" 18 19#include <errno.h> 20#include <glob.h> 21#include <limits.h> 22#include <stdio.h> 23#include <stdlib.h> 24#include <sys/types.h> 25#include <sys/wait.h> 26#include <unistd.h> 27 28#include <algorithm> 29#include <iterator> 30#include <memory> 31#include <unordered_map> 32 33#include "ast.h" 34#include "eval.h" 35#include "find.h" 36#include "log.h" 37#include "parser.h" 38#include "stats.h" 39#include "strutil.h" 40#include "symtab.h" 41#include "var.h" 42 43namespace { 44 45void PatsubstFunc(const vector<Value*>& args, Evaluator* ev, string* s) { 46 shared_ptr<string> pat_str = args[0]->Eval(ev); 47 shared_ptr<string> repl = args[1]->Eval(ev); 48 shared_ptr<string> str = args[2]->Eval(ev); 49 WordWriter ww(s); 50 Pattern pat(*pat_str); 51 for (StringPiece tok : WordScanner(*str)) { 52 ww.MaybeAddWhitespace(); 53 pat.AppendSubst(tok, *repl, s); 54 } 55} 56 57void StripFunc(const vector<Value*>& args, Evaluator* ev, string* s) { 58 shared_ptr<string> str = args[0]->Eval(ev); 59 WordWriter ww(s); 60 for (StringPiece tok : WordScanner(*str)) { 61 ww.Write(tok); 62 } 63} 64 65void SubstFunc(const vector<Value*>& args, Evaluator* ev, string* s) { 66 shared_ptr<string> pat = args[0]->Eval(ev); 67 shared_ptr<string> repl = args[1]->Eval(ev); 68 shared_ptr<string> str = args[2]->Eval(ev); 69 size_t index = 0; 70 while (index < str->size()) { 71 size_t found = str->find(*pat, index); 72 if (found == string::npos) 73 break; 74 AppendString(StringPiece(*str).substr(index, found - index), s); 75 AppendString(*repl, s); 76 index = found + pat->size(); 77 } 78 AppendString(StringPiece(*str).substr(index), s); 79} 80 81void FindstringFunc(const vector<Value*>& args, Evaluator* ev, string* s) { 82 shared_ptr<string> find = args[0]->Eval(ev); 83 shared_ptr<string> in = args[1]->Eval(ev); 84 if (in->find(*find) != string::npos) 85 AppendString(*find, s); 86} 87 88void FilterFunc(const vector<Value*>& args, Evaluator* ev, string* s) { 89 shared_ptr<string> pat_buf = args[0]->Eval(ev); 90 shared_ptr<string> text = args[1]->Eval(ev); 91 vector<Pattern> pats; 92 for (StringPiece pat : WordScanner(*pat_buf)) { 93 pats.push_back(Pattern(pat)); 94 } 95 WordWriter ww(s); 96 for (StringPiece tok : WordScanner(*text)) { 97 for (const Pattern& pat : pats) { 98 if (pat.Match(tok)) { 99 ww.Write(tok); 100 break; 101 } 102 } 103 } 104} 105 106void FilterOutFunc(const vector<Value*>& args, Evaluator* ev, string* s) { 107 shared_ptr<string> pat_buf = args[0]->Eval(ev); 108 shared_ptr<string> text = args[1]->Eval(ev); 109 vector<Pattern> pats; 110 for (StringPiece pat : WordScanner(*pat_buf)) { 111 pats.push_back(Pattern(pat)); 112 } 113 WordWriter ww(s); 114 for (StringPiece tok : WordScanner(*text)) { 115 bool matched = false; 116 for (const Pattern& pat : pats) { 117 if (pat.Match(tok)) { 118 matched = true; 119 break; 120 } 121 } 122 if (!matched) 123 ww.Write(tok); 124 } 125} 126 127void SortFunc(const vector<Value*>& args, Evaluator* ev, string* s) { 128 shared_ptr<string> list = args[0]->Eval(ev); 129 vector<StringPiece> toks; 130 WordScanner(*list).Split(&toks); 131 sort(toks.begin(), toks.end()); 132 WordWriter ww(s); 133 StringPiece prev; 134 for (StringPiece tok : toks) { 135 if (prev != tok) { 136 ww.Write(tok); 137 prev = tok; 138 } 139 } 140} 141 142static int GetNumericValueForFunc(const string& buf) { 143 StringPiece s = TrimLeftSpace(buf); 144 char* end; 145 long n = strtol(s.data(), &end, 10); 146 if (n < 0 || n == LONG_MAX || s.data() + s.size() != end) { 147 return -1; 148 } 149 return n; 150} 151 152void WordFunc(const vector<Value*>& args, Evaluator* ev, string* s) { 153 shared_ptr<string> n_str = args[0]->Eval(ev); 154 int n = GetNumericValueForFunc(*n_str); 155 if (n < 0) { 156 ev->Error(StringPrintf( 157 "*** non-numeric first argument to `word' function: '%s'.", 158 n_str->c_str())); 159 } 160 if (n == 0) { 161 ev->Error("*** first argument to `word' function must be greater than 0."); 162 } 163 164 shared_ptr<string> text = args[1]->Eval(ev); 165 for (StringPiece tok : WordScanner(*text)) { 166 n--; 167 if (n == 0) { 168 AppendString(tok, s); 169 break; 170 } 171 } 172} 173 174void WordlistFunc(const vector<Value*>& args, Evaluator* ev, string* s) { 175 shared_ptr<string> s_str = args[0]->Eval(ev); 176 int si = GetNumericValueForFunc(*s_str); 177 if (si < 0) { 178 ev->Error(StringPrintf( 179 "*** non-numeric first argument to `wordlist' function: '%s'.", 180 s_str->c_str())); 181 } 182 if (si == 0) { 183 ev->Error(StringPrintf( 184 "*** invalid first argument to `wordlist' function: %s`", 185 s_str->c_str())); 186 } 187 188 shared_ptr<string> e_str = args[1]->Eval(ev); 189 int ei = GetNumericValueForFunc(*e_str); 190 if (ei < 0) { 191 ev->Error(StringPrintf( 192 "*** non-numeric second argument to `wordlist' function: '%s'.", 193 e_str->c_str())); 194 } 195 196 shared_ptr<string> text = args[2]->Eval(ev); 197 int i = 0; 198 WordWriter ww(s); 199 for (StringPiece tok : WordScanner(*text)) { 200 i++; 201 if (si <= i && i <= ei) { 202 ww.Write(tok); 203 } 204 } 205} 206 207void WordsFunc(const vector<Value*>& args, Evaluator* ev, string* s) { 208 shared_ptr<string> text = args[0]->Eval(ev); 209 WordScanner ws(*text); 210 int n = 0; 211 for (auto iter = ws.begin(); iter != ws.end(); ++iter) 212 n++; 213 char buf[32]; 214 sprintf(buf, "%d", n); 215 *s += buf; 216} 217 218void FirstwordFunc(const vector<Value*>& args, Evaluator* ev, string* s) { 219 shared_ptr<string> text = args[0]->Eval(ev); 220 for (StringPiece tok : WordScanner(*text)) { 221 AppendString(tok, s); 222 return; 223 } 224} 225 226void LastwordFunc(const vector<Value*>& args, Evaluator* ev, string* s) { 227 shared_ptr<string> text = args[0]->Eval(ev); 228 StringPiece last; 229 for (StringPiece tok : WordScanner(*text)) { 230 last = tok; 231 } 232 AppendString(last, s); 233} 234 235void JoinFunc(const vector<Value*>& args, Evaluator* ev, string* s) { 236 shared_ptr<string> list1 = args[0]->Eval(ev); 237 shared_ptr<string> list2 = args[1]->Eval(ev); 238 WordScanner ws1(*list1); 239 WordScanner ws2(*list2); 240 WordWriter ww(s); 241 for (WordScanner::Iterator iter1 = ws1.begin(), iter2 = ws2.begin(); 242 iter1 != ws1.end() && iter2 != ws2.end(); 243 ++iter1, ++iter2) { 244 ww.Write(*iter1); 245 // Use |AppendString| not to append extra ' '. 246 AppendString(*iter2, s); 247 } 248} 249 250void WildcardFunc(const vector<Value*>& args, Evaluator* ev, string* s) { 251 COLLECT_STATS("func wildcard time"); 252 shared_ptr<string> pat = args[0]->Eval(ev); 253 if (ev->avoid_io()) { 254 *s += "$(/bin/ls -d "; 255 *s += *pat; 256 *s += " 2> /dev/null)"; 257 return; 258 } 259 260 WordWriter ww(s); 261 vector<const char*> files; 262 for (StringPiece tok : WordScanner(*pat)) { 263 ScopedTerminator st(tok); 264 // TODO: Make this faster by not always using glob. 265 files.clear(); 266 glob_t gl; 267 glob(tok.data(), GLOB_NOSORT, NULL, &gl); 268 for (size_t i = 0; i < gl.gl_pathc; i++) { 269 files.push_back(gl.gl_pathv[i]); 270 } 271 sort(files.begin(), files.end(), 272 [](const char* a, const char* b) { 273 return strcmp(a, b) < 0; 274 }); 275 for (const char* file : files) { 276 ww.Write(file); 277 } 278 279 globfree(&gl); 280 } 281} 282 283void DirFunc(const vector<Value*>& args, Evaluator* ev, string* s) { 284 shared_ptr<string> text = args[0]->Eval(ev); 285 WordWriter ww(s); 286 for (StringPiece tok : WordScanner(*text)) { 287 ww.Write(Dirname(tok)); 288 s->push_back('/'); 289 } 290} 291 292void NotdirFunc(const vector<Value*>& args, Evaluator* ev, string* s) { 293 shared_ptr<string> text = args[0]->Eval(ev); 294 WordWriter ww(s); 295 for (StringPiece tok : WordScanner(*text)) { 296 if (tok == "/") { 297 ww.Write(STRING_PIECE("")); 298 } else { 299 ww.Write(Basename(tok)); 300 } 301 } 302} 303 304void SuffixFunc(const vector<Value*>& args, Evaluator* ev, string* s) { 305 shared_ptr<string> text = args[0]->Eval(ev); 306 WordWriter ww(s); 307 for (StringPiece tok : WordScanner(*text)) { 308 StringPiece suf = GetExt(tok); 309 if (!suf.empty()) 310 ww.Write(suf); 311 } 312} 313 314void BasenameFunc(const vector<Value*>& args, Evaluator* ev, string* s) { 315 shared_ptr<string> text = args[0]->Eval(ev); 316 WordWriter ww(s); 317 for (StringPiece tok : WordScanner(*text)) { 318 ww.Write(StripExt(tok)); 319 } 320} 321 322void AddsuffixFunc(const vector<Value*>& args, Evaluator* ev, string* s) { 323 shared_ptr<string> suf = args[0]->Eval(ev); 324 shared_ptr<string> text = args[1]->Eval(ev); 325 WordWriter ww(s); 326 for (StringPiece tok : WordScanner(*text)) { 327 ww.Write(tok); 328 *s += *suf; 329 } 330} 331 332void AddprefixFunc(const vector<Value*>& args, Evaluator* ev, string* s) { 333 shared_ptr<string> pre = args[0]->Eval(ev); 334 shared_ptr<string> text = args[1]->Eval(ev); 335 WordWriter ww(s); 336 for (StringPiece tok : WordScanner(*text)) { 337 ww.Write(*pre); 338 AppendString(tok, s); 339 } 340} 341 342void RealpathFunc(const vector<Value*>& args, Evaluator* ev, string* s) { 343 shared_ptr<string> text = args[0]->Eval(ev); 344 if (ev->avoid_io()) { 345 *s += "KATI_TODO(realpath)"; 346 return; 347 } 348 349 WordWriter ww(s); 350 for (StringPiece tok : WordScanner(*text)) { 351 ScopedTerminator st(tok); 352 char buf[PATH_MAX]; 353 if (realpath(tok.data(), buf)) 354 *s += buf; 355 } 356} 357 358void AbspathFunc(const vector<Value*>& args, Evaluator* ev, string* s) { 359 shared_ptr<string> text = args[0]->Eval(ev); 360 WordWriter ww(s); 361 string buf; 362 for (StringPiece tok : WordScanner(*text)) { 363 AbsPath(tok, &buf); 364 ww.Write(buf); 365 } 366} 367 368void IfFunc(const vector<Value*>& args, Evaluator* ev, string* s) { 369 shared_ptr<string> cond = args[0]->Eval(ev); 370 if (cond->empty()) { 371 if (args.size() > 2) 372 args[2]->Eval(ev, s); 373 } else { 374 args[1]->Eval(ev, s); 375 } 376} 377 378void AndFunc(const vector<Value*>& args, Evaluator* ev, string* s) { 379 shared_ptr<string> cond; 380 for (Value* a : args) { 381 cond = a->Eval(ev); 382 if (cond->empty()) 383 return; 384 } 385 if (cond.get()) { 386 *s += *cond; 387 } 388} 389 390void OrFunc(const vector<Value*>& args, Evaluator* ev, string* s) { 391 for (Value* a : args) { 392 shared_ptr<string> cond = a->Eval(ev); 393 if (!cond->empty()) { 394 *s += *cond; 395 return; 396 } 397 } 398} 399 400void ValueFunc(const vector<Value*>& args, Evaluator* ev, string* s) { 401 shared_ptr<string> var_name = args[0]->Eval(ev); 402 Var* var = ev->LookupVar(Intern(*var_name)); 403 AppendString(var->String().as_string(), s); 404} 405 406void EvalFunc(const vector<Value*>& args, Evaluator* ev, string*) { 407 // TODO: eval leaks everything... for now. 408 //shared_ptr<string> text = args[0]->Eval(ev); 409 string* text = new string; 410 args[0]->Eval(ev, text); 411 vector<AST*> asts; 412 Parse(*text, ev->loc(), &asts); 413 for (AST* ast : asts) { 414 LOG("%s", ast->DebugString().c_str()); 415 ast->Eval(ev); 416 //delete ast; 417 } 418} 419 420//#define TEST_FIND_EMULATOR 421 422#ifdef TEST_FIND_EMULATOR 423static string SortWordsInString(StringPiece s) { 424 vector<string> toks; 425 for (StringPiece tok : WordScanner(s)) { 426 toks.push_back(tok.as_string()); 427 } 428 sort(toks.begin(), toks.end()); 429 return JoinStrings(toks, " "); 430} 431#endif 432 433static void RunCommand(const string& cmd, string* s) { 434 int pipefd[2]; 435 if (pipe(pipefd) != 0) 436 PERROR("pipe failed"); 437 int pid; 438 if ((pid = vfork())) { 439 close(pipefd[1]); 440 while (true) { 441 int status; 442 int result = waitpid(pid, &status, WNOHANG); 443 if (result < 0) 444 PERROR("waitpid failed"); 445 446 while (true) { 447 char buf[4096]; 448 ssize_t r = read(pipefd[0], buf, 4096); 449 if (r < 0) 450 PERROR("read failed"); 451 if (r == 0) 452 break; 453 s->append(buf, buf+r); 454 } 455 456 if (result != 0) { 457 break; 458 } 459 } 460 close(pipefd[0]); 461 } else { 462 close(pipefd[0]); 463 if (dup2(pipefd[1], 1) < 0) 464 PERROR("dup2 failed"); 465 close(pipefd[1]); 466 467 const char* argv[] = { 468 // TODO: Handle $(SHELL). 469 "/bin/sh", "-c", cmd.c_str(), NULL 470 }; 471 execvp(argv[0], const_cast<char**>(argv)); 472 } 473} 474 475void ShellFunc(const vector<Value*>& args, Evaluator* ev, string* s) { 476 shared_ptr<string> cmd = args[0]->Eval(ev); 477 if (ev->avoid_io()) { 478 *s += "$("; 479 *s += *cmd; 480 *s += ")"; 481 return; 482 } 483 484 LOG("ShellFunc: %s", cmd->c_str()); 485 486#ifdef TEST_FIND_EMULATOR 487 bool need_check = false; 488 string out2; 489 if (FindEmulator::Get()->HandleFind(*cmd, &out2)) { 490 need_check = true; 491 } 492#else 493 if (FindEmulator::Get()->HandleFind(*cmd, s)) 494 return; 495#endif 496 497 COLLECT_STATS_WITH_SLOW_REPORT("func shell time", cmd->c_str()); 498 string out; 499#if 0 500 // TODO: Handle $(SHELL). 501 FILE* fp = popen(cmd->c_str(), "r"); 502 while (true) { 503 char buf[4096]; 504 size_t r = fread(buf, 1, 4096, fp); 505 out.append(buf, buf+r); 506 if (r == 0) { 507 fclose(fp); 508 break; 509 } 510 } 511#else 512 RunCommand(*cmd, &out); 513#endif 514 515 while (out[out.size()-1] == '\n') 516 out.pop_back(); 517 for (size_t i = 0; i < out.size(); i++) { 518 if (out[i] == '\n') 519 out[i] = ' '; 520 } 521 522#ifdef TEST_FIND_EMULATOR 523 if (need_check) { 524 string sorted = SortWordsInString(out); 525 out2 = SortWordsInString(out2); 526 if (sorted != out2) { 527 ERROR("FindEmulator is broken: %s\n%s\nvs\n%s", 528 cmd->c_str(), sorted.c_str(), out2.c_str()); 529 } 530 } 531#endif 532 533 *s += out; 534} 535 536void CallFunc(const vector<Value*>& args, Evaluator* ev, string* s) { 537 static const char* tmpvar_names[] = { 538 "0", "1", "2", "3", "4", "5", "6", "7", "8", "9" 539 }; 540 541 shared_ptr<string> func_name = args[0]->Eval(ev); 542 Var* func = ev->LookupVar(Intern(*func_name)); 543 vector<unique_ptr<SimpleVar>> av; 544 for (size_t i = 1; i < args.size(); i++) { 545 unique_ptr<SimpleVar> s( 546 new SimpleVar(args[i]->Eval(ev), VarOrigin::AUTOMATIC)); 547 av.push_back(move(s)); 548 } 549 vector<unique_ptr<ScopedVar>> sv; 550 for (size_t i = 1; i < args.size(); i++) { 551 sv.push_back(move(unique_ptr<ScopedVar>( 552 new ScopedVar(ev->mutable_vars(), 553 Intern(tmpvar_names[i]), av[i-1].get())))); 554 } 555 func->Eval(ev, s); 556} 557 558void ForeachFunc(const vector<Value*>& args, Evaluator* ev, string* s) { 559 shared_ptr<string> varname = args[0]->Eval(ev); 560 shared_ptr<string> list = args[1]->Eval(ev); 561 WordWriter ww(s); 562 for (StringPiece tok : WordScanner(*list)) { 563 unique_ptr<SimpleVar> v(new SimpleVar( 564 make_shared<string>(tok.data(), tok.size()), VarOrigin::AUTOMATIC)); 565 ScopedVar sv(ev->mutable_vars(), Intern(*varname), v.get()); 566 ww.MaybeAddWhitespace(); 567 args[2]->Eval(ev, s); 568 } 569} 570 571void OriginFunc(const vector<Value*>& args, Evaluator* ev, string* s) { 572 shared_ptr<string> var_name = args[0]->Eval(ev); 573 Var* var = ev->LookupVar(Intern(*var_name)); 574 *s += GetOriginStr(var->Origin()); 575} 576 577void FlavorFunc(const vector<Value*>& args, Evaluator* ev, string* s) { 578 shared_ptr<string> var_name = args[0]->Eval(ev); 579 Var* var = ev->LookupVar(Intern(*var_name)); 580 *s += var->Flavor(); 581} 582 583void InfoFunc(const vector<Value*>& args, Evaluator* ev, string* s) { 584 shared_ptr<string> a = args[0]->Eval(ev); 585 if (ev->avoid_io()) { 586 *s += "KATI_TODO(info)"; 587 return; 588 } 589 printf("%s\n", a->c_str()); 590 fflush(stdout); 591} 592 593void WarningFunc(const vector<Value*>& args, Evaluator* ev, string* s) { 594 shared_ptr<string> a = args[0]->Eval(ev); 595 if (ev->avoid_io()) { 596 *s += "KATI_TODO(warning)"; 597 return; 598 } 599 printf("%s:%d: %s\n", LOCF(ev->loc()), a->c_str()); 600 fflush(stdout); 601} 602 603void ErrorFunc(const vector<Value*>& args, Evaluator* ev, string* s) { 604 shared_ptr<string> a = args[0]->Eval(ev); 605 if (ev->avoid_io()) { 606 *s += "KATI_TODO(error)"; 607 return; 608 } 609 ev->Error(StringPrintf("*** %s.", a->c_str())); 610} 611 612FuncInfo g_func_infos[] = { 613 { "patsubst", &PatsubstFunc, 3, 3, false, false }, 614 { "strip", &StripFunc, 1, 1, false, false }, 615 { "subst", &SubstFunc, 3, 3, false, false }, 616 { "findstring", &FindstringFunc, 2, 2, false, false }, 617 { "filter", &FilterFunc, 2, 2, false, false }, 618 { "filter-out", &FilterOutFunc, 2, 2, false, false }, 619 { "sort", &SortFunc, 1, 1, false, false }, 620 { "word", &WordFunc, 2, 2, false, false }, 621 { "wordlist", &WordlistFunc, 3, 3, false, false }, 622 { "words", &WordsFunc, 1, 1, false, false }, 623 { "firstword", &FirstwordFunc, 1, 1, false, false }, 624 { "lastword", &LastwordFunc, 1, 1, false, false }, 625 626 { "join", &JoinFunc, 2, 2, false, false }, 627 { "wildcard", &WildcardFunc, 1, 1, false, false }, 628 { "dir", &DirFunc, 1, 1, false, false }, 629 { "notdir", &NotdirFunc, 1, 1, false, false }, 630 { "suffix", &SuffixFunc, 1, 1, false, false }, 631 { "basename", &BasenameFunc, 1, 1, false, false }, 632 { "addsuffix", &AddsuffixFunc, 2, 2, false, false }, 633 { "addprefix", &AddprefixFunc, 2, 2, false, false }, 634 { "realpath", &RealpathFunc, 1, 1, false, false }, 635 { "abspath", &AbspathFunc, 1, 1, false, false }, 636 637 { "if", &IfFunc, 3, 2, false, true }, 638 { "and", &AndFunc, 0, 0, true, false }, 639 { "or", &OrFunc, 0, 0, true, false }, 640 641 { "value", &ValueFunc, 1, 1, false, false }, 642 { "eval", &EvalFunc, 1, 1, false, false }, 643 { "shell", &ShellFunc, 1, 1, false, false }, 644 { "call", &CallFunc, 0, 0, false, false }, 645 { "foreach", &ForeachFunc, 3, 3, false, false }, 646 647 { "origin", &OriginFunc, 1, 1, false, false }, 648 { "flavor", &FlavorFunc, 1, 1, false, false }, 649 650 { "info", &InfoFunc, 1, 1, false, false }, 651 { "warning", &WarningFunc, 1, 1, false, false }, 652 { "error", &ErrorFunc, 1, 1, false, false }, 653}; 654 655unordered_map<StringPiece, FuncInfo*>* g_func_info_map; 656 657} // namespace 658 659void InitFuncTable() { 660 g_func_info_map = new unordered_map<StringPiece, FuncInfo*>; 661 for (size_t i = 0; i < sizeof(g_func_infos) / sizeof(g_func_infos[0]); i++) { 662 FuncInfo* fi = &g_func_infos[i]; 663 bool ok = g_func_info_map->emplace(fi->name, fi).second; 664 CHECK(ok); 665 } 666} 667 668void QuitFuncTable() { 669 delete g_func_info_map; 670} 671 672FuncInfo* GetFuncInfo(StringPiece name) { 673 auto found = g_func_info_map->find(name); 674 if (found == g_func_info_map->end()) 675 return NULL; 676 return found->second; 677} 678