1#!/bin/bash
2
3# These should be on the local filesystem. We'll be hitting it hard.
4DATA_DIR=/usr/local/google/home/${USER}/
5SYMBOL_CACHE=${DATA_DIR}cache/
6REPORT_DIR=${DATA_DIR}reports/
7SAMPLE_DIR=${DATA_DIR}samples/
8RECORD_FILE=/tmp/profiles.rio
9COLUMN_FILE=/tmp/profiles.cio
10mkdir -p ${SYMBOL_CACHE}
11mkdir -p ${REPORT_DIR}
12mkdir -p ${SAMPLE_DIR}
13
14# Directory that has the scripts app_engine_pull.py and symbolizer.py
15INTERPRETER_DIR=/google/src/files/p2/head/depot2/gcctools/chromeos/v14/cwp/interpreter/
16V14_DIR=$(dirname $(dirname ${INTERPRETER_DIR}))
17
18PYTHONPATH=$PYTHONPATH:$V14_DIR
19
20# Profile util binary
21PROFILE_UTIL_BINARY=/home/mrdmnd/${USER}-profiledb/google3/blaze-bin/perftools/gwp/chromeos/profile_util
22
23# mr-convert binary
24MR_CONVERT_BINARY=/home/build/static/projects/dremel/mr-convert
25
26CNS_LOC=/cns/ag-d/home/${USER}/profiledb/
27
28# Protofile location
29PROTO_LOC=${CNS_LOC}cwp_profile_db_entry.proto
30
31echo "0. Cleaning up old data."
32rm /tmp/profiles.*
33rm ${REPORT_DIR}*
34rm ${SAMPLE_DIR}*
35
36
37echo "Starting CWP Pipeline..."
38echo "1. Pulling samples to local filesystem from server."
39echo "   For demo purposes, UN=${USER}@google.com, PW=xjpbmshkzefutlrm"
40python ${INTERPRETER_DIR}app_engine_pull.py --output_dir=${SAMPLE_DIR}
41echo "2. Symbolizing samples to perf reports. Hold on..."
42
43python ${INTERPRETER_DIR}symbolizer.py --in=${SAMPLE_DIR} --out=${REPORT_DIR} --cache=${SYMBOL_CACHE}
44echo "3. Loading reports into RecordIO format..."
45# Will need to make append_dir more clever / incremental
46${PROFILE_UTIL_BINARY} --record=${RECORD_FILE} --append_dir=${REPORT_DIR}
47echo "Done."
48echo "4. Converting records to columnio."
49${MR_CONVERT_BINARY} --mapreduce_input_map=recordio:${RECORD_FILE} --mapreduce_output_map=${COLUMN_FILE}@1 --columnio_mroutput_message_type=CwpProfileDbEntry --columnio_mroutput_protofiles=${PROTO_LOC}
50echo "5. Uploading columnio to colossus."
51fileutil cp -f /tmp/profiles.cio-* ${CNS_LOC}
52echo "6. Let's try some dremel queries..."
53echo "   dremel> define table t /cns/ag-d/home/${USER}/profiledb/profiles.cio-*"
54echo "   Like, say, dremel> select sum(frames.count) as count, left(frames.function_name, 80) as name from t group by name order by count desc limit 25;"
55
56