19bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker#!/usr/bin/env python 29bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker# 39bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker# Copyright (C) 2009 The Android Open Source Project 49bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker# 59bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker# Licensed under the Apache License, Version 2.0 (the "License"); 69bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker# you may not use this file except in compliance with the License. 79bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker# You may obtain a copy of the License at 89bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker# 99bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker# http://www.apache.org/licenses/LICENSE-2.0 109bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker# 119bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker# Unless required by applicable law or agreed to in writing, software 129bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker# distributed under the License is distributed on an "AS IS" BASIS, 139bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 149bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker# See the License for the specific language governing permissions and 159bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker# limitations under the License. 169bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker 179bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker""" 189bd4962af87257c6a97e9026af7e4764394412c2Doug ZongkerUsage: merge-event-log-tags.py [-o output_file] [input_files...] 199bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker 209bd4962af87257c6a97e9026af7e4764394412c2Doug ZongkerMerge together zero or more event-logs-tags files to produce a single 219bd4962af87257c6a97e9026af7e4764394412c2Doug Zongkeroutput file, stripped of comments. Checks that no tag numbers conflict 229bd4962af87257c6a97e9026af7e4764394412c2Doug Zongkerand fails if they do. 239bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker 249bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker-h to display this usage message and exit. 259bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker""" 269bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker 279bd4962af87257c6a97e9026af7e4764394412c2Doug Zongkerimport cStringIO 289bd4962af87257c6a97e9026af7e4764394412c2Doug Zongkerimport getopt 296ce87a1fc87873510e571c08d5995773a16cddf1Doug Zongkertry: 306ce87a1fc87873510e571c08d5995773a16cddf1Doug Zongker import hashlib 316ce87a1fc87873510e571c08d5995773a16cddf1Doug Zongkerexcept ImportError: 326ce87a1fc87873510e571c08d5995773a16cddf1Doug Zongker import md5 as hashlib 33abfbbe2e1dc0d8dc01b87492427c670dab70f81fDoug Zongkerimport struct 349bd4962af87257c6a97e9026af7e4764394412c2Doug Zongkerimport sys 359bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker 369bd4962af87257c6a97e9026af7e4764394412c2Doug Zongkerimport event_log_tags 379bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker 389bd4962af87257c6a97e9026af7e4764394412c2Doug Zongkererrors = [] 399bd4962af87257c6a97e9026af7e4764394412c2Doug Zongkerwarnings = [] 409bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker 419bd4962af87257c6a97e9026af7e4764394412c2Doug Zongkeroutput_file = None 42b75105315187ec2719bdc8adfdea6fc4f6b296efJoe Onoratopre_merged_file = None 439bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker 44abfbbe2e1dc0d8dc01b87492427c670dab70f81fDoug Zongker# Tags with a tag number of ? are assigned a tag in the range 45abfbbe2e1dc0d8dc01b87492427c670dab70f81fDoug Zongker# [ASSIGN_START, ASSIGN_LIMIT). 46abfbbe2e1dc0d8dc01b87492427c670dab70f81fDoug ZongkerASSIGN_START = 900000 47abfbbe2e1dc0d8dc01b87492427c670dab70f81fDoug ZongkerASSIGN_LIMIT = 1000000 48abfbbe2e1dc0d8dc01b87492427c670dab70f81fDoug Zongker 499bd4962af87257c6a97e9026af7e4764394412c2Doug Zongkertry: 50b75105315187ec2719bdc8adfdea6fc4f6b296efJoe Onorato opts, args = getopt.getopt(sys.argv[1:], "ho:m:") 519bd4962af87257c6a97e9026af7e4764394412c2Doug Zongkerexcept getopt.GetoptError, err: 529bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker print str(err) 539bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker print __doc__ 549bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker sys.exit(2) 559bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker 569bd4962af87257c6a97e9026af7e4764394412c2Doug Zongkerfor o, a in opts: 579bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker if o == "-h": 589bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker print __doc__ 599bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker sys.exit(2) 609bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker elif o == "-o": 619bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker output_file = a 62b75105315187ec2719bdc8adfdea6fc4f6b296efJoe Onorato elif o == "-m": 63b75105315187ec2719bdc8adfdea6fc4f6b296efJoe Onorato pre_merged_file = a 649bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker else: 659bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker print >> sys.stderr, "unhandled option %s" % (o,) 669bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker sys.exit(1) 679bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker 68abfbbe2e1dc0d8dc01b87492427c670dab70f81fDoug Zongker# Restrictions on tags: 69abfbbe2e1dc0d8dc01b87492427c670dab70f81fDoug Zongker# 70abfbbe2e1dc0d8dc01b87492427c670dab70f81fDoug Zongker# Tag names must be unique. (If the tag number and description are 71abfbbe2e1dc0d8dc01b87492427c670dab70f81fDoug Zongker# also the same, a warning is issued instead of an error.) 72abfbbe2e1dc0d8dc01b87492427c670dab70f81fDoug Zongker# 73abfbbe2e1dc0d8dc01b87492427c670dab70f81fDoug Zongker# Explicit tag numbers must be unique. (If the tag name is also the 74abfbbe2e1dc0d8dc01b87492427c670dab70f81fDoug Zongker# same, no error is issued because the above rule will issue a 75abfbbe2e1dc0d8dc01b87492427c670dab70f81fDoug Zongker# warning or error.) 76abfbbe2e1dc0d8dc01b87492427c670dab70f81fDoug Zongker 77abfbbe2e1dc0d8dc01b87492427c670dab70f81fDoug Zongkerby_tagname = {} 78abfbbe2e1dc0d8dc01b87492427c670dab70f81fDoug Zongkerby_tagnum = {} 79abfbbe2e1dc0d8dc01b87492427c670dab70f81fDoug Zongker 80b75105315187ec2719bdc8adfdea6fc4f6b296efJoe Onoratopre_merged_tags = {} 81b75105315187ec2719bdc8adfdea6fc4f6b296efJoe Onoratoif pre_merged_file: 82b75105315187ec2719bdc8adfdea6fc4f6b296efJoe Onorato for t in event_log_tags.TagFile(pre_merged_file).tags: 83b75105315187ec2719bdc8adfdea6fc4f6b296efJoe Onorato pre_merged_tags[t.tagname] = t 84b75105315187ec2719bdc8adfdea6fc4f6b296efJoe Onorato 859bd4962af87257c6a97e9026af7e4764394412c2Doug Zongkerfor fn in args: 869bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker tagfile = event_log_tags.TagFile(fn) 879bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker 889bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker for t in tagfile.tags: 899bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker tagnum = t.tagnum 909bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker tagname = t.tagname 919bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker description = t.description 929bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker 93abfbbe2e1dc0d8dc01b87492427c670dab70f81fDoug Zongker if t.tagname in by_tagname: 94abfbbe2e1dc0d8dc01b87492427c670dab70f81fDoug Zongker orig = by_tagname[t.tagname] 959bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker 965fe3b35846bd1a0abfee68091daf58d7d92e51baDan Egnor # Allow an explicit tag number to define an implicit tag number 975fe3b35846bd1a0abfee68091daf58d7d92e51baDan Egnor if orig.tagnum is None: 985fe3b35846bd1a0abfee68091daf58d7d92e51baDan Egnor orig.tagnum = t.tagnum 995fe3b35846bd1a0abfee68091daf58d7d92e51baDan Egnor elif t.tagnum is None: 1005fe3b35846bd1a0abfee68091daf58d7d92e51baDan Egnor t.tagnum = orig.tagnum 1015fe3b35846bd1a0abfee68091daf58d7d92e51baDan Egnor 102abfbbe2e1dc0d8dc01b87492427c670dab70f81fDoug Zongker if (t.tagnum == orig.tagnum and 1039bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker t.description == orig.description): 1049bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker # if the name and description are identical, issue a warning 1059bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker # instead of failing (to make it easier to move tags between 1069bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker # projects without breaking the build). 107abfbbe2e1dc0d8dc01b87492427c670dab70f81fDoug Zongker tagfile.AddWarning("tag \"%s\" (%s) duplicated in %s:%d" % 108abfbbe2e1dc0d8dc01b87492427c670dab70f81fDoug Zongker (t.tagname, t.tagnum, orig.filename, orig.linenum), 1099bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker linenum=t.linenum) 1109bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker else: 111abfbbe2e1dc0d8dc01b87492427c670dab70f81fDoug Zongker tagfile.AddError( 112abfbbe2e1dc0d8dc01b87492427c670dab70f81fDoug Zongker "tag name \"%s\" used by conflicting tag %s from %s:%d" % 113abfbbe2e1dc0d8dc01b87492427c670dab70f81fDoug Zongker (t.tagname, orig.tagnum, orig.filename, orig.linenum), 114abfbbe2e1dc0d8dc01b87492427c670dab70f81fDoug Zongker linenum=t.linenum) 1159bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker continue 1169bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker 117abfbbe2e1dc0d8dc01b87492427c670dab70f81fDoug Zongker if t.tagnum is not None and t.tagnum in by_tagnum: 118abfbbe2e1dc0d8dc01b87492427c670dab70f81fDoug Zongker orig = by_tagnum[t.tagnum] 119abfbbe2e1dc0d8dc01b87492427c670dab70f81fDoug Zongker 120abfbbe2e1dc0d8dc01b87492427c670dab70f81fDoug Zongker if t.tagname != orig.tagname: 121abfbbe2e1dc0d8dc01b87492427c670dab70f81fDoug Zongker tagfile.AddError( 122abfbbe2e1dc0d8dc01b87492427c670dab70f81fDoug Zongker "tag number %d used by conflicting tag \"%s\" from %s:%d" % 123abfbbe2e1dc0d8dc01b87492427c670dab70f81fDoug Zongker (t.tagnum, orig.tagname, orig.filename, orig.linenum), 124abfbbe2e1dc0d8dc01b87492427c670dab70f81fDoug Zongker linenum=t.linenum) 125abfbbe2e1dc0d8dc01b87492427c670dab70f81fDoug Zongker continue 126abfbbe2e1dc0d8dc01b87492427c670dab70f81fDoug Zongker 127abfbbe2e1dc0d8dc01b87492427c670dab70f81fDoug Zongker by_tagname[t.tagname] = t 128abfbbe2e1dc0d8dc01b87492427c670dab70f81fDoug Zongker if t.tagnum is not None: 129abfbbe2e1dc0d8dc01b87492427c670dab70f81fDoug Zongker by_tagnum[t.tagnum] = t 1309bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker 1319bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker errors.extend(tagfile.errors) 1329bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker warnings.extend(tagfile.warnings) 1339bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker 1349bd4962af87257c6a97e9026af7e4764394412c2Doug Zongkerif errors: 1359bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker for fn, ln, msg in errors: 1369bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker print >> sys.stderr, "%s:%d: error: %s" % (fn, ln, msg) 1379bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker sys.exit(1) 1389bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker 1399bd4962af87257c6a97e9026af7e4764394412c2Doug Zongkerif warnings: 1409bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker for fn, ln, msg in warnings: 1419bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker print >> sys.stderr, "%s:%d: warning: %s" % (fn, ln, msg) 1429bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker 143abfbbe2e1dc0d8dc01b87492427c670dab70f81fDoug Zongker# Python's hash function (a) isn't great and (b) varies between 144abfbbe2e1dc0d8dc01b87492427c670dab70f81fDoug Zongker# versions of python. Using md5 is overkill here but is the same from 145abfbbe2e1dc0d8dc01b87492427c670dab70f81fDoug Zongker# platform to platform and speed shouldn't matter in practice. 146abfbbe2e1dc0d8dc01b87492427c670dab70f81fDoug Zongkerdef hashname(str): 1476ce87a1fc87873510e571c08d5995773a16cddf1Doug Zongker d = hashlib.md5(str).digest()[:4] 148abfbbe2e1dc0d8dc01b87492427c670dab70f81fDoug Zongker return struct.unpack("!I", d)[0] 149abfbbe2e1dc0d8dc01b87492427c670dab70f81fDoug Zongker 150abfbbe2e1dc0d8dc01b87492427c670dab70f81fDoug Zongker# Assign a tag number to all the entries that say they want one 151abfbbe2e1dc0d8dc01b87492427c670dab70f81fDoug Zongker# assigned. We do this based on a hash of the tag name so that the 152abfbbe2e1dc0d8dc01b87492427c670dab70f81fDoug Zongker# numbers should stay relatively stable as tags are added. 153abfbbe2e1dc0d8dc01b87492427c670dab70f81fDoug Zongker 154b75105315187ec2719bdc8adfdea6fc4f6b296efJoe Onorato# If we were provided pre-merged tags (w/ the -m option), then don't 155b75105315187ec2719bdc8adfdea6fc4f6b296efJoe Onorato# ever try to allocate one, just fail if we don't have a number 156b75105315187ec2719bdc8adfdea6fc4f6b296efJoe Onorato 157abfbbe2e1dc0d8dc01b87492427c670dab70f81fDoug Zongkerfor name, t in sorted(by_tagname.iteritems()): 158abfbbe2e1dc0d8dc01b87492427c670dab70f81fDoug Zongker if t.tagnum is None: 159b75105315187ec2719bdc8adfdea6fc4f6b296efJoe Onorato if pre_merged_tags: 160b75105315187ec2719bdc8adfdea6fc4f6b296efJoe Onorato try: 161b75105315187ec2719bdc8adfdea6fc4f6b296efJoe Onorato t.tagnum = pre_merged_tags[t.tagname] 162b75105315187ec2719bdc8adfdea6fc4f6b296efJoe Onorato except KeyError: 163b75105315187ec2719bdc8adfdea6fc4f6b296efJoe Onorato print >> sys.stderr, ("Error: Tag number not defined for tag `%s'." 164b75105315187ec2719bdc8adfdea6fc4f6b296efJoe Onorato +" Have you done a full build?") % t.tagname 165b75105315187ec2719bdc8adfdea6fc4f6b296efJoe Onorato sys.exit(1) 166b75105315187ec2719bdc8adfdea6fc4f6b296efJoe Onorato else: 167b75105315187ec2719bdc8adfdea6fc4f6b296efJoe Onorato while True: 168b75105315187ec2719bdc8adfdea6fc4f6b296efJoe Onorato x = (hashname(name) % (ASSIGN_LIMIT - ASSIGN_START - 1)) + ASSIGN_START 169b75105315187ec2719bdc8adfdea6fc4f6b296efJoe Onorato if x not in by_tagnum: 170b75105315187ec2719bdc8adfdea6fc4f6b296efJoe Onorato t.tagnum = x 171b75105315187ec2719bdc8adfdea6fc4f6b296efJoe Onorato by_tagnum[x] = t 172b75105315187ec2719bdc8adfdea6fc4f6b296efJoe Onorato break 173b75105315187ec2719bdc8adfdea6fc4f6b296efJoe Onorato name = "_" + name 174abfbbe2e1dc0d8dc01b87492427c670dab70f81fDoug Zongker 175abfbbe2e1dc0d8dc01b87492427c670dab70f81fDoug Zongker# by_tagnum should be complete now; we've assigned numbers to all tags. 176abfbbe2e1dc0d8dc01b87492427c670dab70f81fDoug Zongker 1779bd4962af87257c6a97e9026af7e4764394412c2Doug Zongkerbuffer = cStringIO.StringIO() 178abfbbe2e1dc0d8dc01b87492427c670dab70f81fDoug Zongkerfor n, t in sorted(by_tagnum.iteritems()): 1799bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker if t.description: 1809bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker buffer.write("%d %s %s\n" % (t.tagnum, t.tagname, t.description)) 1819bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker else: 1829bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker buffer.write("%d %s\n" % (t.tagnum, t.tagname)) 1839bd4962af87257c6a97e9026af7e4764394412c2Doug Zongker 1849bd4962af87257c6a97e9026af7e4764394412c2Doug Zongkerevent_log_tags.WriteOutput(output_file, buffer) 185