1b23bb61316537e0be83940005c37f731ad090d9dmrdmnd#!/bin/bash 2b23bb61316537e0be83940005c37f731ad090d9dmrdmnd 3b23bb61316537e0be83940005c37f731ad090d9dmrdmnd# These should be on the local filesystem. We'll be hitting it hard. 4b23bb61316537e0be83940005c37f731ad090d9dmrdmndDATA_DIR=/usr/local/google/home/${USER}/ 5b23bb61316537e0be83940005c37f731ad090d9dmrdmndSYMBOL_CACHE=${DATA_DIR}cache/ 6b23bb61316537e0be83940005c37f731ad090d9dmrdmndREPORT_DIR=${DATA_DIR}reports/ 7b23bb61316537e0be83940005c37f731ad090d9dmrdmndSAMPLE_DIR=${DATA_DIR}samples/ 8b23bb61316537e0be83940005c37f731ad090d9dmrdmndRECORD_FILE=/tmp/profiles.rio 9b23bb61316537e0be83940005c37f731ad090d9dmrdmndCOLUMN_FILE=/tmp/profiles.cio 10b23bb61316537e0be83940005c37f731ad090d9dmrdmndmkdir -p ${SYMBOL_CACHE} 11b23bb61316537e0be83940005c37f731ad090d9dmrdmndmkdir -p ${REPORT_DIR} 12b23bb61316537e0be83940005c37f731ad090d9dmrdmndmkdir -p ${SAMPLE_DIR} 13b23bb61316537e0be83940005c37f731ad090d9dmrdmnd 14b23bb61316537e0be83940005c37f731ad090d9dmrdmnd# Directory that has the scripts app_engine_pull.py and symbolizer.py 15b23bb61316537e0be83940005c37f731ad090d9dmrdmndINTERPRETER_DIR=/google/src/files/p2/head/depot2/gcctools/chromeos/v14/cwp/interpreter/ 16b23bb61316537e0be83940005c37f731ad090d9dmrdmndV14_DIR=$(dirname $(dirname ${INTERPRETER_DIR})) 17b23bb61316537e0be83940005c37f731ad090d9dmrdmnd 18b23bb61316537e0be83940005c37f731ad090d9dmrdmndPYTHONPATH=$PYTHONPATH:$V14_DIR 19b23bb61316537e0be83940005c37f731ad090d9dmrdmnd 20b23bb61316537e0be83940005c37f731ad090d9dmrdmnd# Profile util binary 21b23bb61316537e0be83940005c37f731ad090d9dmrdmndPROFILE_UTIL_BINARY=/home/mrdmnd/${USER}-profiledb/google3/blaze-bin/perftools/gwp/chromeos/profile_util 22b23bb61316537e0be83940005c37f731ad090d9dmrdmnd 23b23bb61316537e0be83940005c37f731ad090d9dmrdmnd# mr-convert binary 24b23bb61316537e0be83940005c37f731ad090d9dmrdmndMR_CONVERT_BINARY=/home/build/static/projects/dremel/mr-convert 25b23bb61316537e0be83940005c37f731ad090d9dmrdmnd 26b23bb61316537e0be83940005c37f731ad090d9dmrdmndCNS_LOC=/cns/ag-d/home/${USER}/profiledb/ 27b23bb61316537e0be83940005c37f731ad090d9dmrdmnd 28b23bb61316537e0be83940005c37f731ad090d9dmrdmnd# Protofile location 29b23bb61316537e0be83940005c37f731ad090d9dmrdmndPROTO_LOC=${CNS_LOC}cwp_profile_db_entry.proto 30b23bb61316537e0be83940005c37f731ad090d9dmrdmnd 31b23bb61316537e0be83940005c37f731ad090d9dmrdmndecho "0. Cleaning up old data." 32b23bb61316537e0be83940005c37f731ad090d9dmrdmndrm /tmp/profiles.* 33b23bb61316537e0be83940005c37f731ad090d9dmrdmndrm ${REPORT_DIR}* 34b23bb61316537e0be83940005c37f731ad090d9dmrdmndrm ${SAMPLE_DIR}* 35b23bb61316537e0be83940005c37f731ad090d9dmrdmnd 36b23bb61316537e0be83940005c37f731ad090d9dmrdmnd 37b23bb61316537e0be83940005c37f731ad090d9dmrdmndecho "Starting CWP Pipeline..." 38b23bb61316537e0be83940005c37f731ad090d9dmrdmndecho "1. Pulling samples to local filesystem from server." 39b23bb61316537e0be83940005c37f731ad090d9dmrdmndecho " For demo purposes, UN=${USER}@google.com, PW=xjpbmshkzefutlrm" 40b23bb61316537e0be83940005c37f731ad090d9dmrdmndpython ${INTERPRETER_DIR}app_engine_pull.py --output_dir=${SAMPLE_DIR} 41b23bb61316537e0be83940005c37f731ad090d9dmrdmndecho "2. Symbolizing samples to perf reports. Hold on..." 42b23bb61316537e0be83940005c37f731ad090d9dmrdmnd 43b23bb61316537e0be83940005c37f731ad090d9dmrdmndpython ${INTERPRETER_DIR}symbolizer.py --in=${SAMPLE_DIR} --out=${REPORT_DIR} --cache=${SYMBOL_CACHE} 44b23bb61316537e0be83940005c37f731ad090d9dmrdmndecho "3. Loading reports into RecordIO format..." 45b23bb61316537e0be83940005c37f731ad090d9dmrdmnd# Will need to make append_dir more clever / incremental 46b23bb61316537e0be83940005c37f731ad090d9dmrdmnd${PROFILE_UTIL_BINARY} --record=${RECORD_FILE} --append_dir=${REPORT_DIR} 47b23bb61316537e0be83940005c37f731ad090d9dmrdmndecho "Done." 48b23bb61316537e0be83940005c37f731ad090d9dmrdmndecho "4. Converting records to columnio." 49b23bb61316537e0be83940005c37f731ad090d9dmrdmnd${MR_CONVERT_BINARY} --mapreduce_input_map=recordio:${RECORD_FILE} --mapreduce_output_map=${COLUMN_FILE}@1 --columnio_mroutput_message_type=CwpProfileDbEntry --columnio_mroutput_protofiles=${PROTO_LOC} 50b23bb61316537e0be83940005c37f731ad090d9dmrdmndecho "5. Uploading columnio to colossus." 51b23bb61316537e0be83940005c37f731ad090d9dmrdmndfileutil cp -f /tmp/profiles.cio-* ${CNS_LOC} 52b23bb61316537e0be83940005c37f731ad090d9dmrdmndecho "6. Let's try some dremel queries..." 53b23bb61316537e0be83940005c37f731ad090d9dmrdmndecho " dremel> define table t /cns/ag-d/home/${USER}/profiledb/profiles.cio-*" 54b23bb61316537e0be83940005c37f731ad090d9dmrdmndecho " Like, say, dremel> select sum(frames.count) as count, left(frames.function_name, 80) as name from t group by name order by count desc limit 25;" 55b23bb61316537e0be83940005c37f731ad090d9dmrdmnd 56