HprofConv.c revision 4e1205528c6047cd000b6c5617f8ea8ae3ede82a 
1/*
2 * Copyright (C) 2009 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17/*
18 * Strip Android-specific records out of hprof data, back-converting from
19 * 1.0.3 to 1.0.2.  This removes some useful information, but allows
20 * Android hprof data to be handled by widely-available tools (like "jhat").
21 */
22#include <stdio.h>
23#include <string.h>
24#include <stdlib.h>
25#include <stdint.h>
26#include <errno.h>
27#include <assert.h>
28
29//#define VERBOSE_DEBUG
30#ifdef VERBOSE_DEBUG
31# define DBUG(...) fprintf(stderr, __VA_ARGS__)
32#else
33# define DBUG(...)
34#endif
35
36#ifndef FALSE
37# define FALSE 0
38# define TRUE (!FALSE)
39#endif
40
41typedef enum HprofBasicType {
42    HPROF_BASIC_OBJECT = 2,
43    HPROF_BASIC_BOOLEAN = 4,
44    HPROF_BASIC_CHAR = 5,
45    HPROF_BASIC_FLOAT = 6,
46    HPROF_BASIC_DOUBLE = 7,
47    HPROF_BASIC_BYTE = 8,
48    HPROF_BASIC_SHORT = 9,
49    HPROF_BASIC_INT = 10,
50    HPROF_BASIC_LONG = 11,
51} HprofBasicType;
52
53typedef enum HprofTag {
54    /* tags we must handle specially */
55    HPROF_TAG_HEAP_DUMP                 = 0x0c,
56    HPROF_TAG_HEAP_DUMP_SEGMENT         = 0x1c,
57} HprofTag;
58
59typedef enum HprofHeapTag {
60    /* 1.0.2 tags */
61    HPROF_ROOT_UNKNOWN                  = 0xff,
62    HPROF_ROOT_JNI_GLOBAL               = 0x01,
63    HPROF_ROOT_JNI_LOCAL                = 0x02,
64    HPROF_ROOT_JAVA_FRAME               = 0x03,
65    HPROF_ROOT_NATIVE_STACK             = 0x04,
66    HPROF_ROOT_STICKY_CLASS             = 0x05,
67    HPROF_ROOT_THREAD_BLOCK             = 0x06,
68    HPROF_ROOT_MONITOR_USED             = 0x07,
69    HPROF_ROOT_THREAD_OBJECT            = 0x08,
70    HPROF_CLASS_DUMP                    = 0x20,
71    HPROF_INSTANCE_DUMP                 = 0x21,
72    HPROF_OBJECT_ARRAY_DUMP             = 0x22,
73    HPROF_PRIMITIVE_ARRAY_DUMP          = 0x23,
74
75    /* Android 1.0.3 tags */
76    HPROF_HEAP_DUMP_INFO                = 0xfe,
77    HPROF_ROOT_INTERNED_STRING          = 0x89,
78    HPROF_ROOT_FINALIZING               = 0x8a,
79    HPROF_ROOT_DEBUGGER                 = 0x8b,
80    HPROF_ROOT_REFERENCE_CLEANUP        = 0x8c,
81    HPROF_ROOT_VM_INTERNAL              = 0x8d,
82    HPROF_ROOT_JNI_MONITOR              = 0x8e,
83    HPROF_UNREACHABLE                   = 0x90,  /* deprecated */
84    HPROF_PRIMITIVE_ARRAY_NODATA_DUMP   = 0xc3,
85} HprofHeapTag;
86
87#define kIdentSize  4
88#define kRecHdrLen  9
89
90
91/*
92 * ===========================================================================
93 *      Expanding buffer
94 * ===========================================================================
95 */
96
97/* simple struct */
98typedef struct {
99    unsigned char* storage;
100    size_t curLen;
101    size_t maxLen;
102} ExpandBuf;
103
104/*
105 * Create an ExpandBuf.
106 */
107static ExpandBuf* ebAlloc(void)
108{
109    static const int kInitialSize = 64;
110
111    ExpandBuf* newBuf = (ExpandBuf*) malloc(sizeof(ExpandBuf));
112    if (newBuf == NULL)
113        return NULL;
114    newBuf->storage = (unsigned char*) malloc(kInitialSize);
115    newBuf->curLen = 0;
116    newBuf->maxLen = kInitialSize;
117
118    return newBuf;
119}
120
121/*
122 * Release the storage associated with an ExpandBuf.
123 */
124static void ebFree(ExpandBuf* pBuf)
125{
126    if (pBuf != NULL) {
127        free(pBuf->storage);
128        free(pBuf);
129    }
130}
131
132/*
133 * Return a pointer to the data buffer.
134 *
135 * The pointer may change as data is added to the buffer, so this value
136 * should not be cached.
137 */
138static inline unsigned char* ebGetBuffer(ExpandBuf* pBuf)
139{
140    return pBuf->storage;
141}
142
143/*
144 * Get the amount of data currently in the buffer.
145 */
146static inline size_t ebGetLength(ExpandBuf* pBuf)
147{
148    return pBuf->curLen;
149}
150
151/*
152 * Empty the buffer.
153 */
154static void ebClear(ExpandBuf* pBuf)
155{
156    pBuf->curLen = 0;
157}
158
159/*
160 * Ensure that the buffer can hold at least "size" additional bytes.
161 */
162static int ebEnsureCapacity(ExpandBuf* pBuf, int size)
163{
164    assert(size > 0);
165
166    if (pBuf->curLen + size > pBuf->maxLen) {
167        int newSize = pBuf->curLen + size + 128;    /* oversize slightly */
168        unsigned char* newStorage = realloc(pBuf->storage, newSize);
169        if (newStorage == NULL) {
170            fprintf(stderr, "ERROR: realloc failed on size=%d\n", newSize);
171            return -1;
172        }
173
174        pBuf->storage = newStorage;
175        pBuf->maxLen = newSize;
176    }
177
178    assert(pBuf->curLen + size <= pBuf->maxLen);
179    return 0;
180}
181
182/*
183 * Add data to the buffer after ensuring it can hold it.
184 */
185static int ebAddData(ExpandBuf* pBuf, const void* data, size_t count)
186{
187    ebEnsureCapacity(pBuf, count);
188    memcpy(pBuf->storage + pBuf->curLen, data, count);
189    pBuf->curLen += count;
190    return 0;
191}
192
193/*
194 * Read a NULL-terminated string from the input.
195 */
196static int ebReadString(ExpandBuf* pBuf, FILE* in)
197{
198    int ic;
199
200    do {
201        ebEnsureCapacity(pBuf, 1);
202
203        ic = getc(in);
204        if (feof(in) || ferror(in)) {
205            fprintf(stderr, "ERROR: failed reading input\n");
206            return -1;
207        }
208
209        pBuf->storage[pBuf->curLen++] = (unsigned char) ic;
210    } while (ic != 0);
211
212    return 0;
213}
214
215/*
216 * Read some data, adding it to the expanding buffer.
217 *
218 * This will ensure that the buffer has enough space to hold the new data
219 * (plus the previous contents).
220 */
221static int ebReadData(ExpandBuf* pBuf, FILE* in, size_t count, int eofExpected)
222{
223    size_t actual;
224
225    assert(count > 0);
226
227    ebEnsureCapacity(pBuf, count);
228    actual = fread(pBuf->storage + pBuf->curLen, 1, count, in);
229    if (actual != count) {
230        if (eofExpected && feof(in) && !ferror(in)) {
231            /* return without reporting an error */
232        } else {
233            fprintf(stderr, "ERROR: read %d of %d bytes\n", actual, count);
234            return -1;
235        }
236    }
237
238    pBuf->curLen += count;
239    assert(pBuf->curLen <= pBuf->maxLen);
240
241    return 0;
242}
243
244/*
245 * Write the data from the buffer.  Resets the data count to zero.
246 */
247static int ebWriteData(ExpandBuf* pBuf, FILE* out)
248{
249    size_t actual;
250
251    assert(pBuf->curLen > 0);
252    assert(pBuf->curLen <= pBuf->maxLen);
253
254    actual = fwrite(pBuf->storage, 1, pBuf->curLen, out);
255    if (actual != pBuf->curLen) {
256        fprintf(stderr, "ERROR: write %d of %d bytes\n", actual, pBuf->curLen);
257        return -1;
258    }
259
260    pBuf->curLen = 0;
261
262    return 0;
263}
264
265
266/*
267 * ===========================================================================
268 *      Hprof stuff
269 * ===========================================================================
270 */
271
272/*
273 * Get a 2-byte value, in big-endian order, from memory.
274 */
275static uint16_t get2BE(const unsigned char* buf)
276{
277    uint16_t val;
278
279    val = (buf[0] << 8) | buf[1];
280    return val;
281}
282
283/*
284 * Get a 4-byte value, in big-endian order, from memory.
285 */
286static uint32_t get4BE(const unsigned char* buf)
287{
288    uint32_t val;
289
290    val = (buf[0] << 24) | (buf[1] << 16) | (buf[2] << 8) | buf[3];
291    return val;
292}
293
294/*
295 * Set a 4-byte value, in big-endian order.
296 */
297static void set4BE(unsigned char* buf, uint32_t val)
298{
299    buf[0] = val >> 24;
300    buf[1] = val >> 16;
301    buf[2] = val >> 8;
302    buf[3] = val;
303}
304
305/*
306 * Get the size, in bytes, of one of the "basic types".
307 */
308static int computeBasicLen(HprofBasicType basicType)
309{
310    static const int sizes[] = { -1, -1, 4, -1, 1, 2, 4, 8, 1, 2, 4, 8  };
311    static const size_t maxSize = sizeof(sizes) / sizeof(sizes[0]);
312
313    assert(basicType >= 0);
314    if (basicType >= maxSize)
315        return -1;
316    return sizes[basicType];
317}
318
319/*
320 * Compute the length of a HPROF_CLASS_DUMP block.
321 */
322static int computeClassDumpLen(const unsigned char* origBuf, int len)
323{
324    const unsigned char* buf = origBuf;
325    int blockLen = 0;
326    int i, count;
327
328    blockLen += kIdentSize * 7 + 8;
329    buf += blockLen;
330    len -= blockLen;
331
332    if (len < 0)
333        return -1;
334
335    count = get2BE(buf);
336    buf += 2;
337    len -= 2;
338    DBUG("CDL: 1st count is %d\n", count);
339    for (i = 0; i < count; i++) {
340        HprofBasicType basicType;
341        int basicLen;
342
343        basicType = buf[2];
344        basicLen = computeBasicLen(basicType);
345        if (basicLen < 0) {
346            DBUG("ERROR: invalid basicType %d\n", basicType);
347            return -1;
348        }
349
350        buf += 2 + 1 + basicLen;
351        len -= 2 + 1 + basicLen;
352        if (len < 0)
353            return -1;
354    }
355
356    count = get2BE(buf);
357    buf += 2;
358    len -= 2;
359    DBUG("CDL: 2nd count is %d\n", count);
360    for (i = 0; i < count; i++) {
361        HprofBasicType basicType;
362        int basicLen;
363
364        basicType = buf[kIdentSize];
365        basicLen = computeBasicLen(basicType);
366        if (basicLen < 0) {
367            fprintf(stderr, "ERROR: invalid basicType %d\n", basicType);
368            return -1;
369        }
370
371        buf += kIdentSize + 1 + basicLen;
372        len -= kIdentSize + 1 + basicLen;
373        if (len < 0)
374            return -1;
375    }
376
377    count = get2BE(buf);
378    buf += 2;
379    len -= 2;
380    DBUG("CDL: 3rd count is %d\n", count);
381    for (i = 0; i < count; i++) {
382        buf += kIdentSize + 1;
383        len -= kIdentSize + 1;
384        if (len < 0)
385            return -1;
386    }
387
388    DBUG("Total class dump len: %d\n", buf - origBuf);
389    return buf - origBuf;
390}
391
392/*
393 * Compute the length of a HPROF_INSTANCE_DUMP block.
394 */
395static int computeInstanceDumpLen(const unsigned char* origBuf, int len)
396{
397    int extraCount = get4BE(origBuf + kIdentSize * 2 + 4);
398    return kIdentSize * 2 + 8 + extraCount;
399}
400
401/*
402 * Compute the length of a HPROF_OBJECT_ARRAY_DUMP block.
403 */
404static int computeObjectArrayDumpLen(const unsigned char* origBuf, int len)
405{
406    int arrayCount = get4BE(origBuf + kIdentSize + 4);
407    return kIdentSize * 2 + 8 + arrayCount * kIdentSize;
408}
409
410/*
411 * Compute the length of a HPROF_PRIMITIVE_ARRAY_DUMP block.
412 */
413static int computePrimitiveArrayDumpLen(const unsigned char* origBuf, int len)
414{
415    int arrayCount = get4BE(origBuf + kIdentSize + 4);
416    HprofBasicType basicType = origBuf[kIdentSize + 8];
417    int basicLen = computeBasicLen(basicType);
418
419    return kIdentSize + 9 + arrayCount * basicLen;
420}
421
422/*
423 * Crunch through a heap dump record, writing the original or converted
424 * data to "out".
425 */
426static int processHeapDump(ExpandBuf* pBuf, FILE* out)
427{
428    ExpandBuf* pOutBuf = ebAlloc();
429    unsigned char* origBuf = ebGetBuffer(pBuf);
430    unsigned char* buf = origBuf;
431    int len = ebGetLength(pBuf);
432    int result = -1;
433
434    pBuf = NULL;        /* we just use the raw pointer from here forward */
435
436    /* copy the original header to the output buffer */
437    if (ebAddData(pOutBuf, buf, kRecHdrLen) != 0)
438        goto bail;
439
440    buf += kRecHdrLen;      /* skip past record header */
441    len -= kRecHdrLen;
442
443    while (len > 0) {
444        unsigned char subType = buf[0];
445        int justCopy = TRUE;
446        int subLen;
447
448        DBUG("--- 0x%02x  ", subType);
449        switch (subType) {
450        /* 1.0.2 types */
451        case HPROF_ROOT_UNKNOWN:
452            subLen = kIdentSize;
453            break;
454        case HPROF_ROOT_JNI_GLOBAL:
455            subLen = kIdentSize * 2;
456            break;
457        case HPROF_ROOT_JNI_LOCAL:
458            subLen = kIdentSize + 8;
459            break;
460        case HPROF_ROOT_JAVA_FRAME:
461            subLen = kIdentSize + 8;
462            break;
463        case HPROF_ROOT_NATIVE_STACK:
464            subLen = kIdentSize + 4;
465            break;
466        case HPROF_ROOT_STICKY_CLASS:
467            subLen = kIdentSize;
468            break;
469        case HPROF_ROOT_THREAD_BLOCK:
470            subLen = kIdentSize + 4;
471            break;
472        case HPROF_ROOT_MONITOR_USED:
473            subLen = kIdentSize;
474            break;
475        case HPROF_ROOT_THREAD_OBJECT:
476            subLen = kIdentSize + 8;
477            break;
478        case HPROF_CLASS_DUMP:
479            subLen = computeClassDumpLen(buf+1, len-1);
480            break;
481        case HPROF_INSTANCE_DUMP:
482            subLen = computeInstanceDumpLen(buf+1, len-1);
483            break;
484        case HPROF_OBJECT_ARRAY_DUMP:
485            subLen = computeObjectArrayDumpLen(buf+1, len-1);
486            break;
487        case HPROF_PRIMITIVE_ARRAY_DUMP:
488            subLen = computePrimitiveArrayDumpLen(buf+1, len-1);
489            break;
490
491        /* these were added for Android in 1.0.3 */
492        case HPROF_HEAP_DUMP_INFO:
493            justCopy = FALSE;
494            subLen = kIdentSize + 4;
495            // no 1.0.2 equivalent for this
496            break;
497        case HPROF_ROOT_INTERNED_STRING:
498            buf[0] = HPROF_ROOT_UNKNOWN;
499            subLen = kIdentSize;
500            break;
501        case HPROF_ROOT_FINALIZING:
502            buf[0] = HPROF_ROOT_UNKNOWN;
503            subLen = kIdentSize;
504            break;
505        case HPROF_ROOT_DEBUGGER:
506            buf[0] = HPROF_ROOT_UNKNOWN;
507            subLen = kIdentSize;
508            break;
509        case HPROF_ROOT_REFERENCE_CLEANUP:
510            buf[0] = HPROF_ROOT_UNKNOWN;
511            subLen = kIdentSize;
512            break;
513        case HPROF_ROOT_VM_INTERNAL:
514            buf[0] = HPROF_ROOT_UNKNOWN;
515            subLen = kIdentSize;
516            break;
517        case HPROF_ROOT_JNI_MONITOR:
518            /* keep the ident, drop the next 8 bytes */
519            buf[0] = HPROF_ROOT_UNKNOWN;
520            justCopy = FALSE;
521            ebAddData(pOutBuf, buf, 1 + kIdentSize);
522            subLen = kIdentSize + 8;
523            break;
524        case HPROF_UNREACHABLE:
525            buf[0] = HPROF_ROOT_UNKNOWN;
526            subLen = kIdentSize;
527            break;
528        case HPROF_PRIMITIVE_ARRAY_NODATA_DUMP:
529            buf[0] = HPROF_PRIMITIVE_ARRAY_DUMP;
530            buf[5] = buf[6] = buf[7] = buf[8] = 0;  /* set array len to 0 */
531            subLen = kIdentSize + 9;
532            break;
533
534        /* shouldn't get here */
535        default:
536            fprintf(stderr, "ERROR: unexpected subtype 0x%02x at offset %d\n",
537                subType, buf - origBuf);
538            goto bail;
539        }
540
541        if (justCopy) {
542            /* copy source data */
543            DBUG("(%d)\n", 1 + subLen);
544            ebAddData(pOutBuf, buf, 1 + subLen);
545        } else {
546            /* other data has been written, or the sub-record omitted */
547            DBUG("(adv %d)\n", 1 + subLen);
548        }
549
550        /* advance to next entry */
551        buf += 1 + subLen;
552        len -= 1 + subLen;
553    }
554
555    /*
556     * Update the record length.
557     */
558    set4BE(ebGetBuffer(pOutBuf) + 5, ebGetLength(pOutBuf) - kRecHdrLen);
559
560    if (ebWriteData(pOutBuf, out) != 0)
561        goto bail;
562
563    result = 0;
564
565bail:
566    ebFree(pOutBuf);
567    return result;
568}
569
570/*
571 * Filter an hprof data file.
572 */
573static int filterData(FILE* in, FILE* out)
574{
575    const char *magicString;
576    ExpandBuf* pBuf;
577    int result = -1;
578
579    pBuf = ebAlloc();
580    if (pBuf == NULL)
581        goto bail;
582
583    /*
584     * Start with the header.
585     */
586    if (ebReadString(pBuf, in) != 0)
587        goto bail;
588
589    magicString = (const char*)ebGetBuffer(pBuf);
590    if (strcmp(magicString, "JAVA PROFILE 1.0.3") != 0) {
591        if (strcmp(magicString, "JAVA PROFILE 1.0.2") == 0) {
592            fprintf(stderr, "ERROR: HPROF file already in 1.0.2 format.\n");
593        } else {
594            fprintf(stderr, "ERROR: expecting HPROF file format 1.0.3\n");
595        }
596        goto bail;
597    }
598
599    /* downgrade to 1.0.2 */
600    (ebGetBuffer(pBuf))[17] = '2';
601    if (ebWriteData(pBuf, out) != 0)
602        goto bail;
603
604    /*
605     * Copy:
606     * (4b) identifier size, always 4
607     * (8b) file creation date
608     */
609    if (ebReadData(pBuf, in, 12, FALSE) != 0)
610        goto bail;
611    if (ebWriteData(pBuf, out) != 0)
612        goto bail;
613
614    /*
615     * Read records until we hit EOF.  Each record begins with:
616     * (1b) type
617     * (4b) timestamp
618     * (4b) length of data that follows
619     */
620    while (1) {
621        assert(ebGetLength(pBuf) == 0);
622
623        /* read type char */
624        if (ebReadData(pBuf, in, 1, TRUE) != 0)
625            goto bail;
626        if (feof(in))
627            break;
628
629        /* read the rest of the header */
630        if (ebReadData(pBuf, in, kRecHdrLen-1, FALSE) != 0)
631            goto bail;
632
633        unsigned char* buf = ebGetBuffer(pBuf);
634        unsigned char type;
635        unsigned int timestamp, length;
636
637        type = buf[0];
638        timestamp = get4BE(buf + 1);
639        length = get4BE(buf + 5);
640        buf = NULL;     /* ptr invalid after next read op */
641
642        /* read the record data */
643        if (length != 0) {
644            if (ebReadData(pBuf, in, length, FALSE) != 0)
645                goto bail;
646        }
647
648        if (type == HPROF_TAG_HEAP_DUMP ||
649            type == HPROF_TAG_HEAP_DUMP_SEGMENT)
650        {
651            DBUG("Processing heap dump 0x%02x (%d bytes)\n",
652                type, length);
653            if (processHeapDump(pBuf, out) != 0)
654                goto bail;
655            ebClear(pBuf);
656        } else {
657            /* keep */
658            DBUG("Keeping 0x%02x (%d bytes)\n", type, length);
659            if (ebWriteData(pBuf, out) != 0)
660                goto bail;
661        }
662    }
663
664    result = 0;
665
666bail:
667    ebFree(pBuf);
668    return result;
669}
670
671/*
672 * Get args.
673 */
674int main(int argc, char** argv)
675{
676    FILE* in = stdin;
677    FILE* out = stdout;
678    int cc;
679
680    if (argc != 3) {
681        fprintf(stderr, "Usage: hprof-conf infile outfile\n\n");
682        fprintf(stderr,
683            "Specify '-' for either or both to use stdin/stdout.\n\n");
684
685        fprintf(stderr,
686            "Copyright (C) 2009 The Android Open Source Project\n\n"
687            "This software is built from source code licensed under the "
688            "Apache License,\n"
689            "Version 2.0 (the \"License\"). You may obtain a copy of the "
690            "License at\n\n"
691            "     http://www.apache.org/licenses/LICENSE-2.0\n\n"
692            "See the associated NOTICE file for this software for further "
693            "details.\n");
694
695        return 2;
696    }
697
698    if (strcmp(argv[1], "-") != 0) {
699        in = fopen(argv[1], "rb");
700        if (in == NULL) {
701            fprintf(stderr, "ERROR: failed to open input '%s': %s\n",
702                argv[1], strerror(errno));
703            return 1;
704        }
705    }
706    if (strcmp(argv[2], "-") != 0) {
707        out = fopen(argv[2], "wb");
708        if (out == NULL) {
709            fprintf(stderr, "ERROR: failed to open output '%s': %s\n",
710                argv[2], strerror(errno));
711            if (in != stdin)
712                fclose(in);
713            return 1;
714        }
715    }
716
717    cc = filterData(in, out);
718
719    if (in != stdin)
720        fclose(in);
721    if (out != stdout)
722        fclose(out);
723    return (cc != 0);
724}
725