xref: /dalvik/docs/prettify.js

// Copyright (C) 2006 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.


/**
 * @fileoverview
 * some functions for browser-side pretty printing of code contained in html.
 *
 * The lexer should work on a number of languages including C and friends,
 * Java, Python, Bash, SQL, HTML, XML, CSS, Javascript, and Makefiles.
 * It works passably on Ruby, PHP and Awk and a decent subset of Perl, but,
 * because of commenting conventions, doesn't work on Smalltalk, Lisp-like, or
 * CAML-like languages.
 *
 * If there's a language not mentioned here, then I don't know it, and don't
 * know whether it works.  If it has a C-like, Bash-like, or XML-like syntax
 * then it should work passably.
 *
 * Usage:
 * 1) include this source file in an html page via
 * <script type="text/javascript" src="/path/to/prettify.js"></script>
 * 2) define style rules.  See the example page for examples.
 * 3) mark the <pre> and <code> tags in your source with class=prettyprint.
 *    You can also use the (html deprecated) <xmp> tag, but the pretty printer
 *    needs to do more substantial DOM manipulations to support that, so some
 *    css styles may not be preserved.
 * That's it.  I wanted to keep the API as simple as possible, so there's no
 * need to specify which language the code is in.
 *
 * Change log:
 * cbeust, 2006/08/22
 *   Java annotations (start with "@") are now captured as literals ("lit")
 */

// JSLint declarations
/*global console, document, navigator, setTimeout, window */

/**
 * Split {@code prettyPrint} into multiple timeouts so as not to interfere with
 * UI events.
 * If set to {@code false}, {@code prettyPrint()} is synchronous.
 */
var PR_SHOULD_USE_CONTINUATION = true;

/** the number of characters between tab columns */
var PR_TAB_WIDTH = 8;

/** Walks the DOM returning a properly escaped version of innerHTML.
  * @param {Node} node
  * @param {Array.<string>} out output buffer that receives chunks of HTML.
  */
var PR_normalizedHtml;

/** Contains functions for creating and registering new language handlers.
  * @type {Object}
  */
var PR;

/** Pretty print a chunk of code.
  *
  * @param {string} sourceCodeHtml code as html
  * @return {string} code as html, but prettier
  */
var prettyPrintOne;
/** find all the < pre > and < code > tags in the DOM with class=prettyprint
  * and prettify them.
  * @param {Function} opt_whenDone if specified, called when the last entry
  *     has been finished.
  */
var prettyPrint;

/** browser detection. @extern */
function _pr_isIE6() {
  var isIE6 = navigator && navigator.userAgent &&
      /\bMSIE 6\./.test(navigator.userAgent);
  _pr_isIE6 = function () { return isIE6; };
  return isIE6;
}


(function () {
  /** Splits input on space and returns an Object mapping each non-empty part to
    * true.
    */
  function wordSet(words) {
    words = words.split(/ /g);
    var set = {};
    for (var i = words.length; --i >= 0;) {
      var w = words[i];
      if (w) { set[w] = null; }
    }
    return set;
  }

  // Keyword lists for various languages.
  var FLOW_CONTROL_KEYWORDS =
      "break continue do else for if return while ";
  var C_KEYWORDS = FLOW_CONTROL_KEYWORDS + "auto case char const default " +
      "double enum extern float goto int long register short signed sizeof " +
      "static struct switch typedef union unsigned void volatile ";
  var COMMON_KEYWORDS = C_KEYWORDS + "catch class delete false import " +
      "new operator private protected public this throw true try ";
  var CPP_KEYWORDS = COMMON_KEYWORDS + "alignof align_union asm axiom bool " +
      "concept concept_map const_cast constexpr decltype " +
      "dynamic_cast explicit export friend inline late_check " +
      "mutable namespace nullptr reinterpret_cast static_assert static_cast " +
      "template typeid typename typeof using virtual wchar_t where ";
  var JAVA_KEYWORDS = COMMON_KEYWORDS +
      "boolean byte extends final finally implements import instanceof null " +
      "native package strictfp super synchronized throws transient ";
  var CSHARP_KEYWORDS = JAVA_KEYWORDS +
      "as base by checked decimal delegate descending event " +
      "fixed foreach from group implicit in interface internal into is lock " +
      "object out override orderby params readonly ref sbyte sealed " +
      "stackalloc string select uint ulong unchecked unsafe ushort var ";
  var JSCRIPT_KEYWORDS = COMMON_KEYWORDS +
      "debugger eval export function get null set undefined var with " +
      "Infinity NaN ";
  var PERL_KEYWORDS = "caller delete die do dump elsif eval exit foreach for " +
      "goto if import last local my next no our print package redo require " +
      "sub undef unless until use wantarray while BEGIN END ";
  var PYTHON_KEYWORDS = FLOW_CONTROL_KEYWORDS + "and as assert class def del " +
      "elif except exec finally from global import in is lambda " +
      "nonlocal not or pass print raise try with yield " +
      "False True None ";
  var RUBY_KEYWORDS = FLOW_CONTROL_KEYWORDS + "alias and begin case class def" +
      " defined elsif end ensure false in module next nil not or redo rescue " +
      "retry self super then true undef unless until when yield BEGIN END ";
  var SH_KEYWORDS = FLOW_CONTROL_KEYWORDS + "case done elif esac eval fi " +
      "function in local set then until ";
  var ALL_KEYWORDS = (
      CPP_KEYWORDS + CSHARP_KEYWORDS + JSCRIPT_KEYWORDS + PERL_KEYWORDS +
      PYTHON_KEYWORDS + RUBY_KEYWORDS + SH_KEYWORDS);

  // token style names.  correspond to css classes
  /** token style for a string literal */
  var PR_STRING = 'str';
  /** token style for a keyword */
  var PR_KEYWORD = 'kwd';
  /** token style for a comment */
  var PR_COMMENT = 'com';
  /** token style for a type */
  var PR_TYPE = 'typ';
  /** token style for a literal value.  e.g. 1, null, true. */
  var PR_LITERAL = 'lit';
  /** token style for a punctuation string. */
  var PR_PUNCTUATION = 'pun';
  /** token style for a punctuation string. */
  var PR_PLAIN = 'pln';

  /** token style for an sgml tag. */
  var PR_TAG = 'tag';
  /** token style for a markup declaration such as a DOCTYPE. */
  var PR_DECLARATION = 'dec';
  /** token style for embedded source. */
  var PR_SOURCE = 'src';
  /** token style for an sgml attribute name. */
  var PR_ATTRIB_NAME = 'atn';
  /** token style for an sgml attribute value. */
  var PR_ATTRIB_VALUE = 'atv';

  /**
   * A class that indicates a section of markup that is not code, e.g. to allow
   * embedding of line numbers within code listings.
   */
  var PR_NOCODE = 'nocode';

  function isWordChar(ch) {
    return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z');
  }

  /** Splice one array into another.
    * Like the python <code>
    * container[containerPosition:containerPosition + countReplaced] = inserted
    * </code>
    * @param {Array} inserted
    * @param {Array} container modified in place
    * @param {Number} containerPosition
    * @param {Number} countReplaced
    */
  function spliceArrayInto(
      inserted, container, containerPosition, countReplaced) {
    inserted.unshift(containerPosition, countReplaced || 0);
    try {
      container.splice.apply(container, inserted);
    } finally {
      inserted.splice(0, 2);
    }
  }

  /** A set of tokens that can precede a regular expression literal in
    * javascript.
    * http://www.mozilla.org/js/language/js20/rationale/syntax.html has the full
    * list, but I've removed ones that might be problematic when seen in
    * languages that don't support regular expression literals.
    *
    * <p>Specifically, I've removed any keywords that can't precede a regexp
    * literal in a syntactically legal javascript program, and I've removed the
    * "in" keyword since it's not a keyword in many languages, and might be used
    * as a count of inches.
    * @private
    */
  var REGEXP_PRECEDER_PATTERN = function () {
      var preceders = [
          "!", "!=", "!==", "#", "%", "%=", "&", "&&", "&&=",
          "&=", "(", "*", "*=", /* "+", */ "+=", ",", /* "-", */ "-=",
          "->", /*".", "..", "...", handled below */ "/", "/=", ":", "::", ";",
          "<", "<<", "<<=", "<=", "=", "==", "===", ">",
          ">=", ">>", ">>=", ">>>", ">>>=", "?", "@", "[",
          "^", "^=", "^^", "^^=", "{", "|", "|=", "||",
          "||=", "~" /* handles =~ and !~ */,
          "break", "case", "continue", "delete",
          "do", "else", "finally", "instanceof",
          "return", "throw", "try", "typeof"
          ];
      var pattern = '(?:' +
          '(?:(?:^|[^0-9.])\\.{1,3})|' +  // a dot that's not part of a number
          '(?:(?:^|[^\\+])\\+)|' +  // allow + but not ++
          '(?:(?:^|[^\\-])-)';  // allow - but not --
      for (var i = 0; i < preceders.length; ++i) {
        var preceder = preceders[i];
        if (isWordChar(preceder.charAt(0))) {
          pattern += '|\\b' + preceder;
        } else {
          pattern += '|' + preceder.replace(/([^=<>:&])/g, '\\$1');
        }
      }
      pattern += '|^)\\s*$';  // matches at end, and matches empty string
      return new RegExp(pattern);
      // CAVEAT: this does not properly handle the case where a regular
      // expression immediately follows another since a regular expression may
      // have flags for case-sensitivity and the like.  Having regexp tokens
      // adjacent is not
      // valid in any language I'm aware of, so I'm punting.
      // TODO: maybe style special characters inside a regexp as punctuation.
    }();

  // Define regexps here so that the interpreter doesn't have to create an
  // object each time the function containing them is called.
  // The language spec requires a new object created even if you don't access
  // the $1 members.
  var pr_amp = /&/g;
  var pr_lt = /</g;
  var pr_gt = />/g;
  var pr_quot = /\"/g;
  /** like textToHtml but escapes double quotes to be attribute safe. */
  function attribToHtml(str) {
    return str.replace(pr_amp, '&amp;')
        .replace(pr_lt, '&lt;')
        .replace(pr_gt, '&gt;')
        .replace(pr_quot, '&quot;');
  }

  /** escapest html special characters to html. */
  function textToHtml(str) {
    return str.replace(pr_amp, '&amp;')
        .replace(pr_lt, '&lt;')
        .replace(pr_gt, '&gt;');
  }


  var pr_ltEnt = /&lt;/g;
  var pr_gtEnt = /&gt;/g;
  var pr_aposEnt = /&apos;/g;
  var pr_quotEnt = /&quot;/g;
  var pr_ampEnt = /&amp;/g;
  var pr_nbspEnt = /&nbsp;/g;
  /** unescapes html to plain text. */
  function htmlToText(html) {
    var pos = html.indexOf('&');
    if (pos < 0) { return html; }
    // Handle numeric entities specially.  We can't use functional substitution
    // since that doesn't work in older versions of Safari.
    // These should be rare since most browsers convert them to normal chars.
    for (--pos; (pos = html.indexOf('&#', pos + 1)) >= 0;) {
      var end = html.indexOf(';', pos);
      if (end >= 0) {
        var num = html.substring(pos + 3, end);
        var radix = 10;
        if (num && num.charAt(0) === 'x') {
          num = num.substring(1);
          radix = 16;
        }
        var codePoint = parseInt(num, radix);
        if (!isNaN(codePoint)) {
          html = (html.substring(0, pos) + String.fromCharCode(codePoint) +
                  html.substring(end + 1));
        }
      }
    }

    return html.replace(pr_ltEnt, '<')
        .replace(pr_gtEnt, '>')
        .replace(pr_aposEnt, "'")
        .replace(pr_quotEnt, '"')
        .replace(pr_ampEnt, '&')
        .replace(pr_nbspEnt, ' ');
  }

  /** is the given node's innerHTML normally unescaped? */
  function isRawContent(node) {
    return 'XMP' === node.tagName;
  }

  function normalizedHtml(node, out) {
    switch (node.nodeType) {
      case 1:  // an element
        var name = node.tagName.toLowerCase();
        out.push('<', name);
        for (var i = 0; i < node.attributes.length; ++i) {
          var attr = node.attributes[i];
          if (!attr.specified) { continue; }
          out.push(' ');
          normalizedHtml(attr, out);
        }
        out.push('>');
        for (var child = node.firstChild; child; child = child.nextSibling) {
          normalizedHtml(child, out);
        }
        if (node.firstChild || !/^(?:br|link|img)$/.test(name)) {
          out.push('<\/', name, '>');
        }
        break;
      case 2: // an attribute
        out.push(node.name.toLowerCase(), '="', attribToHtml(node.value), '"');
        break;
      case 3: case 4: // text
        out.push(textToHtml(node.nodeValue));
        break;
    }
  }

  var PR_innerHtmlWorks = null;
  function getInnerHtml(node) {
    // inner html is hopelessly broken in Safari 2.0.4 when the content is
    // an html description of well formed XML and the containing tag is a PRE
    // tag, so we detect that case and emulate innerHTML.
    if (null === PR_innerHtmlWorks) {
      var testNode = document.createElement('PRE');
      testNode.appendChild(
          document.createTextNode('<!DOCTYPE foo PUBLIC "foo bar">\n<foo />'));
      PR_innerHtmlWorks = !/</.test(testNode.innerHTML);
    }

    if (PR_innerHtmlWorks) {
      var content = node.innerHTML;
      // XMP tags contain unescaped entities so require special handling.
      if (isRawContent(node)) {
        content = textToHtml(content);
      }
      return content;
    }

    var out = [];
    for (var child = node.firstChild; child; child = child.nextSibling) {
      normalizedHtml(child, out);
    }
    return out.join('');
  }

  /** returns a function that expand tabs to spaces.  This function can be fed
    * successive chunks of text, and will maintain its own internal state to
    * keep track of how tabs are expanded.
    * @return {function (string) : string} a function that takes
    *   plain text and return the text with tabs expanded.
    * @private
    */
  function makeTabExpander(tabWidth) {
    var SPACES = '                ';
    var charInLine = 0;

    return function (plainText) {
      // walk over each character looking for tabs and newlines.
      // On tabs, expand them.  On newlines, reset charInLine.
      // Otherwise increment charInLine
      var out = null;
      var pos = 0;
      for (var i = 0, n = plainText.length; i < n; ++i) {
        var ch = plainText.charAt(i);

        switch (ch) {
          case '\t':
            if (!out) { out = []; }
            out.push(plainText.substring(pos, i));
            // calculate how much space we need in front of this part
            // nSpaces is the amount of padding -- the number of spaces needed
            // to move us to the next column, where columns occur at factors of
            // tabWidth.
            var nSpaces = tabWidth - (charInLine % tabWidth);
            charInLine += nSpaces;
            for (; nSpaces >= 0; nSpaces -= SPACES.length) {
              out.push(SPACES.substring(0, nSpaces));
            }
            pos = i + 1;
            break;
          case '\n':
            charInLine = 0;
            break;
          default:
            ++charInLine;
        }
      }
      if (!out) { return plainText; }
      out.push(plainText.substring(pos));
      return out.join('');
    };
  }

  // The below pattern matches one of the following
  // (1) /[^<]+/ : A run of characters other than '<'
  // (2) /<!--.*?-->/: an HTML comment
  // (3) /<!\[CDATA\[.*?\]\]>/: a cdata section
  // (3) /<\/?[a-zA-Z][^>]*>/ : A probably tag that should not be highlighted
  // (4) /</ : A '<' that does not begin a larger chunk.  Treated as 1
  var pr_chunkPattern =
  /(?:[^<]+|<!--[\s\S]*?-->|<!\[CDATA\[([\s\S]*?)\]\]>|<\/?[a-zA-Z][^>]*>|<)/g;
  var pr_commentPrefix = /^<!--/;
  var pr_cdataPrefix = /^<\[CDATA\[/;
  var pr_brPrefix = /^<br\b/i;
  var pr_tagNameRe = /^<(\/?)([a-zA-Z]+)/;

  /** split markup into chunks of html tags (style null) and
    * plain text (style {@link #PR_PLAIN}), converting tags which are
    * significant for tokenization (<br>) into their textual equivalent.
    *
    * @param {string} s html where whitespace is considered significant.
    * @return {Object} source code and extracted tags.
    * @private
    */
  function extractTags(s) {
    // since the pattern has the 'g' modifier and defines no capturing groups,
    // this will return a list of all chunks which we then classify and wrap as
    // PR_Tokens
    var matches = s.match(pr_chunkPattern);
    var sourceBuf = [];
    var sourceBufLen = 0;
    var extractedTags = [];
    if (matches) {
      for (var i = 0, n = matches.length; i < n; ++i) {
        var match = matches[i];
        if (match.length > 1 && match.charAt(0) === '<') {
          if (pr_commentPrefix.test(match)) { continue; }
          if (pr_cdataPrefix.test(match)) {
            // strip CDATA prefix and suffix.  Don't unescape since it's CDATA
            sourceBuf.push(match.substring(9, match.length - 3));
            sourceBufLen += match.length - 12;
          } else if (pr_brPrefix.test(match)) {
            // <br> tags are lexically significant so convert them to text.
            // This is undone later.
            sourceBuf.push('\n');
            ++sourceBufLen;
          } else {
            if (match.indexOf(PR_NOCODE) >= 0 && isNoCodeTag(match)) {
              // A <span class="nocode"> will start a section that should be
              // ignored.  Continue walking the list until we see a matching end
              // tag.
              var name = match.match(pr_tagNameRe)[2];
              var depth = 1;
              end_tag_loop:
              for (var j = i + 1; j < n; ++j) {
                var name2 = matches[j].match(pr_tagNameRe);
                if (name2 && name2[2] === name) {
                  if (name2[1] === '/') {
                    if (--depth === 0) { break end_tag_loop; }
                  } else {
                    ++depth;
                  }
                }
              }
              if (j < n) {
                extractedTags.push(
                    sourceBufLen, matches.slice(i, j + 1).join(''));
                i = j;
              } else {  // Ignore unclosed sections.
                extractedTags.push(sourceBufLen, match);
              }
            } else {
              extractedTags.push(sourceBufLen, match);
            }
          }
        } else {
          var literalText = htmlToText(match);
          sourceBuf.push(literalText);
          sourceBufLen += literalText.length;
        }
      }
    }
    return { source: sourceBuf.join(''), tags: extractedTags };
  }

  /** True if the given tag contains a class attribute with the nocode class. */
  function isNoCodeTag(tag) {
    return !!tag
        // First canonicalize the representation of attributes
        .replace(/\s(\w+)\s*=\s*(?:\"([^\"]*)\"|'([^\']*)'|(\S+))/g,
                 ' $1="$2$3$4"')
        // Then look for the attribute we want.
        .match(/[cC][lL][aA][sS][sS]=\"[^\"]*\bnocode\b/);
  }

  /** Given triples of [style, pattern, context] returns a lexing function,
    * The lexing function interprets the patterns to find token boundaries and
    * returns a decoration list of the form
    * [index_0, style_0, index_1, style_1, ..., index_n, style_n]
    * where index_n is an index into the sourceCode, and style_n is a style
    * constant like PR_PLAIN.  index_n-1 <= index_n, and style_n-1 applies to
    * all characters in sourceCode[index_n-1:index_n].
    *
    * The stylePatterns is a list whose elements have the form
    * [style : string, pattern : RegExp, context : RegExp, shortcut : string].
    &
    * Style is a style constant like PR_PLAIN.
    *
    * Pattern must only match prefixes, and if it matches a prefix and context
    * is null or matches the last non-comment token parsed, then that match is
    * considered a token with the same style.
    *
    * Context is applied to the last non-whitespace, non-comment token
    * recognized.
    *
    * Shortcut is an optional string of characters, any of which, if the first
    * character, gurantee that this pattern and only this pattern matches.
    *
    * @param {Array} shortcutStylePatterns patterns that always start with
    *   a known character.  Must have a shortcut string.
    * @param {Array} fallthroughStylePatterns patterns that will be tried in
    *   order if the shortcut ones fail.  May have shortcuts.
    *
    * @return {function (string, number?) : Array.<number|string>} a
    *   function that takes source code and returns a list of decorations.
    */
  function createSimpleLexer(shortcutStylePatterns,
                             fallthroughStylePatterns) {
    var shortcuts = {};
    (function () {
      var allPatterns = shortcutStylePatterns.concat(fallthroughStylePatterns);
      for (var i = allPatterns.length; --i >= 0;) {
        var patternParts = allPatterns[i];
        var shortcutChars = patternParts[3];
        if (shortcutChars) {
          for (var c = shortcutChars.length; --c >= 0;) {
            shortcuts[shortcutChars.charAt(c)] = patternParts;
          }
        }
      }
    })();

    var nPatterns = fallthroughStylePatterns.length;
    var notWs = /\S/;

    return function (sourceCode, opt_basePos) {
      opt_basePos = opt_basePos || 0;
      var decorations = [opt_basePos, PR_PLAIN];
      var lastToken = '';
      var pos = 0;  // index into sourceCode
      var tail = sourceCode;

      while (tail.length) {
        var style;
        var token = null;
        var match;

        var patternParts = shortcuts[tail.charAt(0)];
        if (patternParts) {
          match = tail.match(patternParts[1]);
          token = match[0];
          style = patternParts[0];
        } else {
          for (var i = 0; i < nPatterns; ++i) {
            patternParts = fallthroughStylePatterns[i];
            var contextPattern = patternParts[2];
            if (contextPattern && !contextPattern.test(lastToken)) {
              // rule can't be used
              continue;
            }
            match = tail.match(patternParts[1]);
            if (match) {
              token = match[0];
              style = patternParts[0];
              break;
            }
          }

          if (!token) {  // make sure that we make progress
            style = PR_PLAIN;
            token = tail.substring(0, 1);
          }
        }

        decorations.push(opt_basePos + pos, style);
        pos += token.length;
        tail = tail.substring(token.length);
        if (style !== PR_COMMENT && notWs.test(token)) { lastToken = token; }
      }
      return decorations;
    };
  }

  var PR_MARKUP_LEXER = createSimpleLexer([], [
      [PR_PLAIN,       /^[^<]+/, null],
      [PR_DECLARATION, /^<!\w[^>]*(?:>|$)/, null],
      [PR_COMMENT,     /^<!--[\s\S]*?(?:-->|$)/, null],
      [PR_SOURCE,      /^<\?[\s\S]*?(?:\?>|$)/, null],
      [PR_SOURCE,      /^<%[\s\S]*?(?:%>|$)/, null],
      [PR_SOURCE,
       // Tags whose content is not escaped, and which contain source code.
       /^<(script|style|xmp)\b[^>]*>[\s\S]*?<\/\1\b[^>]*>/i, null],
      [PR_TAG,         /^<\/?\w[^<>]*>/, null]
      ]);
  // Splits any of the source|style|xmp entries above into a start tag,
  // source content, and end tag.
  var PR_SOURCE_CHUNK_PARTS = /^(<[^>]*>)([\s\S]*)(<\/[^>]*>)$/;
  /** split markup on tags, comments, application directives, and other top
    * level constructs.  Tags are returned as a single token - attributes are
    * not yet broken out.
    * @private
    */
  function tokenizeMarkup(source) {
    var decorations = PR_MARKUP_LEXER(source);
    for (var i = 0; i < decorations.length; i += 2) {
      if (decorations[i + 1] === PR_SOURCE) {
        var start, end;
        start = decorations[i];
        end = i + 2 < decorations.length ? decorations[i + 2] : source.length;
        // Split out start and end script tags as actual tags, and leave the
        // body with style SCRIPT.
        var sourceChunk = source.substring(start, end);
        var match = sourceChunk.match(PR_SOURCE_CHUNK_PARTS);
        if (match) {
          decorations.splice(
              i, 2,
              start, PR_TAG,  // the open chunk
              start + match[1].length, PR_SOURCE,
              start + match[1].length + (match[2] || '').length, PR_TAG);
        }
      }
    }
    return decorations;
  }

  var PR_TAG_LEXER = createSimpleLexer([
      [PR_ATTRIB_VALUE, /^\'[^\']*(?:\'|$)/, null, "'"],
      [PR_ATTRIB_VALUE, /^\"[^\"]*(?:\"|$)/, null, '"'],
      [PR_PUNCTUATION,  /^[<>\/=]+/, null, '<>/=']
      ], [
      [PR_TAG,          /^[\w:\-]+/, /^</],
      [PR_ATTRIB_VALUE, /^[\w\-]+/, /^=/],
      [PR_ATTRIB_NAME,  /^[\w:\-]+/, null],
      [PR_PLAIN,        /^\s+/, null, ' \t\r\n']
      ]);
  /** split tags attributes and their values out from the tag name, and
    * recursively lex source chunks.
    * @private
    */
  function splitTagAttributes(source, decorations) {
    for (var i = 0; i < decorations.length; i += 2) {
      var style = decorations[i + 1];
      if (style === PR_TAG) {
        var start, end;
        start = decorations[i];
        end = i + 2 < decorations.length ? decorations[i + 2] : source.length;
        var chunk = source.substring(start, end);
        var subDecorations = PR_TAG_LEXER(chunk, start);
        spliceArrayInto(subDecorations, decorations, i, 2);
        i += subDecorations.length - 2;
      }
    }
    return decorations;
  }

  /** returns a function that produces a list of decorations from source text.
    *
    * This code treats ", ', and ` as string delimiters, and \ as a string
    * escape.  It does not recognize perl's qq() style strings.
    * It has no special handling for double delimiter escapes as in basic, or
    * the tripled delimiters used in python, but should work on those regardless
    * although in those cases a single string literal may be broken up into
    * multiple adjacent string literals.
    *
    * It recognizes C, C++, and shell style comments.
    *
    * @param {Object} options a set of optional parameters.
    * @return {function (string) : Array.<string|number>} a
    *     decorator that takes sourceCode as plain text and that returns a
    *     decoration list
    */
  function sourceDecorator(options) {
    var shortcutStylePatterns = [], fallthroughStylePatterns = [];
    if (options.tripleQuotedStrings) {
      // '''multi-line-string''', 'single-line-string', and double-quoted
      shortcutStylePatterns.push(
          [PR_STRING,  /^(?:\'\'\'(?:[^\'\\]|\\[\s\S]|\'{1,2}(?=[^\']))*(?:\'\'\'|$)|\"\"\"(?:[^\"\\]|\\[\s\S]|\"{1,2}(?=[^\"]))*(?:\"\"\"|$)|\'(?:[^\\\']|\\[\s\S])*(?:\'|$)|\"(?:[^\\\"]|\\[\s\S])*(?:\"|$))/,
           null, '\'"']);
    } else if (options.multiLineStrings) {
      // 'multi-line-string', "multi-line-string"
      shortcutStylePatterns.push(
          [PR_STRING,  /^(?:\'(?:[^\\\']|\\[\s\S])*(?:\'|$)|\"(?:[^\\\"]|\\[\s\S])*(?:\"|$)|\`(?:[^\\\`]|\\[\s\S])*(?:\`|$))/,
           null, '\'"`']);
    } else {
      // 'single-line-string', "single-line-string"
      shortcutStylePatterns.push(
          [PR_STRING,
           /^(?:\'(?:[^\\\'\r\n]|\\.)*(?:\'|$)|\"(?:[^\\\"\r\n]|\\.)*(?:\"|$))/,
           null, '"\'']);
    }
    fallthroughStylePatterns.push(
        [PR_PLAIN,   /^(?:[^\'\"\`\/\#]+)/, null, ' \r\n']);
    if (options.hashComments) {
      shortcutStylePatterns.push([PR_COMMENT, /^#[^\r\n]*/, null, '#']);
    }
    if (options.cStyleComments) {
      fallthroughStylePatterns.push([PR_COMMENT, /^\/\/[^\r\n]*/, null]);
      fallthroughStylePatterns.push(
          [PR_COMMENT, /^\/\*[\s\S]*?(?:\*\/|$)/, null]);
    }
    if (options.regexLiterals) {
      var REGEX_LITERAL = (
          // A regular expression literal starts with a slash that is
          // not followed by * or / so that it is not confused with
          // comments.
          '^/(?=[^/*])'
          // and then contains any number of raw characters,
          + '(?:[^/\\x5B\\x5C]'
          // escape sequences (\x5C),
          +    '|\\x5C[\\s\\S]'
          // or non-nesting character sets (\x5B\x5D);
          +    '|\\x5B(?:[^\\x5C\\x5D]|\\x5C[\\s\\S])*(?:\\x5D|$))+'
          // finally closed by a /.
          + '(?:/|$)');
      fallthroughStylePatterns.push(
          [PR_STRING, new RegExp(REGEX_LITERAL), REGEXP_PRECEDER_PATTERN]);
    }

    var keywords = wordSet(options.keywords);

    options = null;

    /** splits the given string into comment, string, and "other" tokens.
      * @param {string} sourceCode as plain text
      * @return {Array.<number|string>} a decoration list.
      * @private
      */
    var splitStringAndCommentTokens = createSimpleLexer(
        shortcutStylePatterns, fallthroughStylePatterns);

    var styleLiteralIdentifierPuncRecognizer = createSimpleLexer([], [
        [PR_PLAIN,       /^\s+/, null, ' \r\n'],
        // TODO(mikesamuel): recognize non-latin letters and numerals in idents
        [PR_PLAIN,       /^[a-z_$@][a-z_$@0-9]*/i, null],
        // A hex number
        [PR_LITERAL,     /^0x[a-f0-9]+[a-z]/i, null],
        // An octal or decimal number, possibly in scientific notation
        [PR_LITERAL,
         /^(?:\d(?:_\d+)*\d*(?:\.\d*)?|\.\d+)(?:e[+\-]?\d+)?[a-z]*/i,
         null, '123456789'],
        [PR_PUNCTUATION, /^[^\s\w\.$@]+/, null]
        // Fallback will handle decimal points not adjacent to a digit
      ]);

    /** splits plain text tokens into more specific tokens, and then tries to
      * recognize keywords, and types.
      * @private
      */
    function splitNonStringNonCommentTokens(source, decorations) {
      for (var i = 0; i < decorations.length; i += 2) {
        var style = decorations[i + 1];
        if (style === PR_PLAIN) {
          var start, end, chunk, subDecs;
          start = decorations[i];
          end = i + 2 < decorations.length ? decorations[i + 2] : source.length;
          chunk = source.substring(start, end);
          subDecs = styleLiteralIdentifierPuncRecognizer(chunk, start);
          for (var j = 0, m = subDecs.length; j < m; j += 2) {
            var subStyle = subDecs[j + 1];
            if (subStyle === PR_PLAIN) {
              var subStart = subDecs[j];
              var subEnd = j + 2 < m ? subDecs[j + 2] : chunk.length;
              var token = source.substring(subStart, subEnd);
              if (token === '.') {
                subDecs[j + 1] = PR_PUNCTUATION;
              } else if (token in keywords) {
                subDecs[j + 1] = PR_KEYWORD;
              } else if (/^@?[A-Z][A-Z$]*[a-z][A-Za-z$]*$/.test(token)) {
                // classify types and annotations using Java's style conventions
                subDecs[j + 1] = token.charAt(0) === '@' ? PR_LITERAL : PR_TYPE;
              }
            }
          }
          spliceArrayInto(subDecs, decorations, i, 2);
          i += subDecs.length - 2;
        }
      }
      return decorations;
    }

    return function (sourceCode) {
      // Split into strings, comments, and other.
      // We do this because strings and comments are easily recognizable and can
      // contain stuff that looks like other tokens, so we want to mark those
      // early so we don't recurse into them.
      var decorations = splitStringAndCommentTokens(sourceCode);

      // Split non comment|string tokens on whitespace and word boundaries
      decorations = splitNonStringNonCommentTokens(sourceCode, decorations);

      return decorations;
    };
  }

  var decorateSource = sourceDecorator({
        keywords: ALL_KEYWORDS,
        hashComments: true,
        cStyleComments: true,
        multiLineStrings: true,
        regexLiterals: true
      });

  /** identify regions of markup that are really source code, and recursivley
    * lex them.
    * @private
    */
  function splitSourceNodes(source, decorations) {
    for (var i = 0; i < decorations.length; i += 2) {
      var style = decorations[i + 1];
      if (style === PR_SOURCE) {
        // Recurse using the non-markup lexer
        var start, end;
        start = decorations[i];
        end = i + 2 < decorations.length ? decorations[i + 2] : source.length;
        var subDecorations = decorateSource(source.substring(start, end));
        for (var j = 0, m = subDecorations.length; j < m; j += 2) {
          subDecorations[j] += start;
        }
        spliceArrayInto(subDecorations, decorations, i, 2);
        i += subDecorations.length - 2;
      }
    }
    return decorations;
  }

  /** identify attribute values that really contain source code and recursively
    * lex them.
    * @private
    */
  function splitSourceAttributes(source, decorations) {
    var nextValueIsSource = false;
    for (var i = 0; i < decorations.length; i += 2) {
      var style = decorations[i + 1];
      var start, end;
      if (style === PR_ATTRIB_NAME) {
        start = decorations[i];
        end = i + 2 < decorations.length ? decorations[i + 2] : source.length;
        nextValueIsSource = /^on|^style$/i.test(source.substring(start, end));
      } else if (style === PR_ATTRIB_VALUE) {
        if (nextValueIsSource) {
          start = decorations[i];
          end = i + 2 < decorations.length ? decorations[i + 2] : source.length;
          var attribValue = source.substring(start, end);
          var attribLen = attribValue.length;
          var quoted =
              (attribLen >= 2 && /^[\"\']/.test(attribValue) &&
               attribValue.charAt(0) === attribValue.charAt(attribLen - 1));

          var attribSource;
          var attribSourceStart;
          var attribSourceEnd;
          if (quoted) {
            attribSourceStart = start + 1;
            attribSourceEnd = end - 1;
            attribSource = attribValue;
          } else {
            attribSourceStart = start + 1;
            attribSourceEnd = end - 1;
            attribSource = attribValue.substring(1, attribValue.length - 1);
          }

          var attribSourceDecorations = decorateSource(attribSource);
          for (var j = 0, m = attribSourceDecorations.length; j < m; j += 2) {
            attribSourceDecorations[j] += attribSourceStart;
          }

          if (quoted) {
            attribSourceDecorations.push(attribSourceEnd, PR_ATTRIB_VALUE);
            spliceArrayInto(attribSourceDecorations, decorations, i + 2, 0);
          } else {
            spliceArrayInto(attribSourceDecorations, decorations, i, 2);
          }
        }
        nextValueIsSource = false;
      }
    }
    return decorations;
  }

  /** returns a decoration list given a string of markup.
    *
    * This code recognizes a number of constructs.
    * <!-- ... --> comment
    * <!\w ... >   declaration
    * <\w ... >    tag
    * </\w ... >   tag
    * <?...?>      embedded source
    * <%...%>      embedded source
    * &[#\w]...;   entity
    *
    * It does not recognizes %foo; doctype entities from  .
    *
    * It will recurse into any <style>, <script>, and on* attributes using
    * PR_lexSource.
    */
  function decorateMarkup(sourceCode) {
    // This function works as follows:
    // 1) Start by splitting the markup into text and tag chunks
    //    Input:  string s
    //    Output: List<PR_Token> where style in (PR_PLAIN, null)
    // 2) Then split the text chunks further into comments, declarations,
    //    tags, etc.
    //    After each split, consider whether the token is the start of an
    //    embedded source section, i.e. is an open <script> tag.  If it is, find
    //    the corresponding close token, and don't bother to lex in between.
    //    Input:  List<string>
    //    Output: List<PR_Token> with style in
    //            (PR_TAG, PR_PLAIN, PR_SOURCE, null)
    // 3) Finally go over each tag token and split out attribute names and
    //    values.
    //    Input:  List<PR_Token>
    //    Output: List<PR_Token> where style in
    //            (PR_TAG, PR_PLAIN, PR_SOURCE, NAME, VALUE, null)
    var decorations = tokenizeMarkup(sourceCode);
    decorations = splitTagAttributes(sourceCode, decorations);
    decorations = splitSourceNodes(sourceCode, decorations);
    decorations = splitSourceAttributes(sourceCode, decorations);
    return decorations;
  }

  /**
    * @param {string} sourceText plain text
    * @param {Array.<number|string>} extractedTags chunks of raw html preceded
    *   by their position in sourceText in order.
    * @param {Array.<number|string>} decorations style classes preceded by their
    *   position in sourceText in order.
    * @return {string} html
    * @private
    */
  function recombineTagsAndDecorations(sourceText, extractedTags, decorations) {
    var html = [];
    // index past the last char in sourceText written to html
    var outputIdx = 0;

    var openDecoration = null;
    var currentDecoration = null;
    var tagPos = 0;  // index into extractedTags
    var decPos = 0;  // index into decorations
    var tabExpander = makeTabExpander(PR_TAB_WIDTH);

    var adjacentSpaceRe = /([\r\n ]) /g;
    var startOrSpaceRe = /(^| ) /gm;
    var newlineRe = /\r\n?|\n/g;
    var trailingSpaceRe = /[ \r\n]$/;
    var lastWasSpace = true;  // the last text chunk emitted ended with a space.

    // A helper function that is responsible for opening sections of decoration
    // and outputing properly escaped chunks of source
    function emitTextUpTo(sourceIdx) {
      if (sourceIdx > outputIdx) {
        if (openDecoration && openDecoration !== currentDecoration) {
          // Close the current decoration
          html.push('</span>');
          openDecoration = null;
        }
        if (!openDecoration && currentDecoration) {
          openDecoration = currentDecoration;
          html.push('<span class="', openDecoration, '">');
        }
        // This interacts badly with some wikis which introduces paragraph tags
        // into pre blocks for some strange reason.
        // It's necessary for IE though which seems to lose the preformattedness
        // of <pre> tags when their innerHTML is assigned.
        // http://stud3.tuwien.ac.at/~e0226430/innerHtmlQuirk.html
        // and it serves to undo the conversion of <br>s to newlines done in
        // chunkify.
        var htmlChunk = textToHtml(
            tabExpander(sourceText.substring(outputIdx, sourceIdx)))
            .replace(lastWasSpace
                     ? startOrSpaceRe
                     : adjacentSpaceRe, '$1&nbsp;');
        // Keep track of whether we need to escape space at the beginning of the
        // next chunk.
        lastWasSpace = trailingSpaceRe.test(htmlChunk);
        html.push(htmlChunk.replace(newlineRe, '<br />'));
        outputIdx = sourceIdx;
      }
    }

    while (true) {
      // Determine if we're going to consume a tag this time around.  Otherwise
      // we consume a decoration or exit.
      var outputTag;
      if (tagPos < extractedTags.length) {
        if (decPos < decorations.length) {
          // Pick one giving preference to extractedTags since we shouldn't open
          // a new style that we're going to have to immediately close in order
          // to output a tag.
          outputTag = extractedTags[tagPos] <= decorations[decPos];
        } else {
          outputTag = true;
        }
      } else {
        outputTag = false;
      }
      // Consume either a decoration or a tag or exit.
      if (outputTag) {
        emitTextUpTo(extractedTags[tagPos]);
        if (openDecoration) {
          // Close the current decoration
          html.push('</span>');
          openDecoration = null;
        }
        html.push(extractedTags[tagPos + 1]);
        tagPos += 2;
      } else if (decPos < decorations.length) {
        emitTextUpTo(decorations[decPos]);
        currentDecoration = decorations[decPos + 1];
        decPos += 2;
      } else {
        break;
      }
    }
    emitTextUpTo(sourceText.length);
    if (openDecoration) {
      html.push('</span>');
    }

    return html.join('');
  }

  /** Maps language-specific file extensions to handlers. */
  var langHandlerRegistry = {};
  /** Register a language handler for the given file extensions.
    * @param {function (string) : Array.<number|string>} handler
    *     a function from source code to a list of decorations.
    * @param {Array.<string>} fileExtensions
    */
  function registerLangHandler(handler, fileExtensions) {
    for (var i = fileExtensions.length; --i >= 0;) {
      var ext = fileExtensions[i];
      if (!langHandlerRegistry.hasOwnProperty(ext)) {
        langHandlerRegistry[ext] = handler;
      } else if ('console' in window) {
        console.log('cannot override language handler %s', ext);
      }
    }
  }
  registerLangHandler(decorateSource, ['default-code']);
  registerLangHandler(decorateMarkup,
                      ['default-markup', 'html', 'htm', 'xhtml', 'xml', 'xsl']);
  registerLangHandler(sourceDecorator({
          keywords: CPP_KEYWORDS,
          hashComments: true,
          cStyleComments: true
        }), ['c', 'cc', 'cpp', 'cxx', 'cyc']);
  registerLangHandler(sourceDecorator({
          keywords: CSHARP_KEYWORDS,
          hashComments: true,
          cStyleComments: true
        }), ['cs']);
  registerLangHandler(sourceDecorator({
          keywords: JAVA_KEYWORDS,
          cStyleComments: true
        }), ['java']);
  registerLangHandler(sourceDecorator({
          keywords: SH_KEYWORDS,
          hashComments: true,
          multiLineStrings: true
        }), ['bsh', 'csh', 'sh']);
  registerLangHandler(sourceDecorator({
          keywords: PYTHON_KEYWORDS,
          hashComments: true,
          multiLineStrings: true,
          tripleQuotedStrings: true
        }), ['cv', 'py']);
  registerLangHandler(sourceDecorator({
          keywords: PERL_KEYWORDS,
          hashComments: true,
          multiLineStrings: true,
          regexLiterals: true
        }), ['perl', 'pl', 'pm']);
  registerLangHandler(sourceDecorator({
          keywords: RUBY_KEYWORDS,
          hashComments: true,
          multiLineStrings: true,
          regexLiterals: true
        }), ['rb']);
  registerLangHandler(sourceDecorator({
          keywords: JSCRIPT_KEYWORDS,
          cStyleComments: true,
          regexLiterals: true
        }), ['js']);

  function prettyPrintOne(sourceCodeHtml, opt_langExtension) {
    try {
      // Extract tags, and convert the source code to plain text.
      var sourceAndExtractedTags = extractTags(sourceCodeHtml);
      /** Plain text. @type {string} */
      var source = sourceAndExtractedTags.source;

      /** Even entries are positions in source in ascending order.  Odd entries
        * are tags that were extracted at that position.
        * @type {Array.<number|string>}
        */
      var extractedTags = sourceAndExtractedTags.tags;

      // Pick a lexer and apply it.
      if (!langHandlerRegistry.hasOwnProperty(opt_langExtension)) {
        // Treat it as markup if the first non whitespace character is a < and
        // the last non-whitespace character is a >.
        opt_langExtension =
            /^\s*</.test(source) ? 'default-markup' : 'default-code';
      }

      /** Even entries are positions in source in ascending order.  Odd enties
        * are style markers (e.g., PR_COMMENT) that run from that position until
        * the end.
        * @type {Array.<number|string>}
        */
      var decorations = langHandlerRegistry[opt_langExtension].call({}, source);

      // Integrate the decorations and tags back into the source code to produce
      // a decorated html string.
      return recombineTagsAndDecorations(source, extractedTags, decorations);
    } catch (e) {
      if ('console' in window) {
        console.log(e);
        console.trace();
      }
      return sourceCodeHtml;
    }
  }

  function prettyPrint(opt_whenDone) {
    var isIE6 = _pr_isIE6();

    // fetch a list of nodes to rewrite
    var codeSegments = [
        document.getElementsByTagName('pre'),
        document.getElementsByTagName('code'),
        document.getElementsByTagName('xmp') ];
    var elements = [];
    for (var i = 0; i < codeSegments.length; ++i) {
      for (var j = 0; j < codeSegments[i].length; ++j) {
        elements.push(codeSegments[i][j]);
      }
    }
    codeSegments = null;

    // the loop is broken into a series of continuations to make sure that we
    // don't make the browser unresponsive when rewriting a large page.
    var k = 0;

    function doWork() {
      var endTime = (PR_SHOULD_USE_CONTINUATION ?
                     new Date().getTime() + 250 /* ms */ :
                     Infinity);
      for (; k < elements.length && new Date().getTime() < endTime; k++) {
        var cs = elements[k];
        if (cs.className && cs.className.indexOf('prettyprint') >= 0) {
          // If the classes includes a language extensions, use it.
          // Language extensions can be specified like
          //     <pre class="prettyprint lang-cpp">
          // the language extension "cpp" is used to find a language handler as
          // passed to PR_registerLangHandler.
          var langExtension = cs.className.match(/\blang-(\w+)\b/);
          if (langExtension) { langExtension = langExtension[1]; }

          // make sure this is not nested in an already prettified element
          var nested = false;
          for (var p = cs.parentNode; p; p = p.parentNode) {
            if ((p.tagName === 'pre' || p.tagName === 'code' ||
                 p.tagName === 'xmp') &&
                p.className && p.className.indexOf('prettyprint') >= 0) {
              nested = true;
              break;
            }
          }
          if (!nested) {
            // fetch the content as a snippet of properly escaped HTML.
            // Firefox adds newlines at the end.
            var content = getInnerHtml(cs);
            content = content.replace(/(?:\r\n?|\n)$/, '');

            // do the pretty printing
            var newContent = prettyPrintOne(content, langExtension);

            // push the prettified html back into the tag.
            if (!isRawContent(cs)) {
              // just replace the old html with the new
              cs.innerHTML = newContent;
            } else {
              // we need to change the tag to a <pre> since <xmp>s do not allow
              // embedded tags such as the span tags used to attach styles to
              // sections of source code.
              var pre = document.createElement('PRE');
              for (var i = 0; i < cs.attributes.length; ++i) {
                var a = cs.attributes[i];
                if (a.specified) {
                  var aname = a.name.toLowerCase();
                  if (aname === 'class') {
                    pre.className = a.value;  // For IE 6
                  } else {
                    pre.setAttribute(a.name, a.value);
                  }
                }
              }
              pre.innerHTML = newContent;

              // remove the old
              cs.parentNode.replaceChild(pre, cs);
              cs = pre;
            }

            // Replace <br>s with line-feeds so that copying and pasting works
            // on IE 6.
            // Doing this on other browsers breaks lots of stuff since \r\n is
            // treated as two newlines on Firefox, and doing this also slows
            // down rendering.
            if (isIE6 && cs.tagName === 'PRE') {
              var lineBreaks = cs.getElementsByTagName('br');
              for (var j = lineBreaks.length; --j >= 0;) {
                var lineBreak = lineBreaks[j];
                lineBreak.parentNode.replaceChild(
                    document.createTextNode('\r\n'), lineBreak);
              }
            }
          }
        }
      }
      if (k < elements.length) {
        // finish up in a continuation
        setTimeout(doWork, 250);
      } else if (opt_whenDone) {
        opt_whenDone();
      }
    }

    doWork();
  }

  window['PR_normalizedHtml'] = normalizedHtml;
  window['prettyPrintOne'] = prettyPrintOne;
  window['prettyPrint'] = prettyPrint;
  window['PR'] = {
        'createSimpleLexer': createSimpleLexer,
        'registerLangHandler': registerLangHandler,
        'sourceDecorator': sourceDecorator,
        'PR_ATTRIB_NAME': PR_ATTRIB_NAME,
        'PR_ATTRIB_VALUE': PR_ATTRIB_VALUE,
        'PR_COMMENT': PR_COMMENT,
        'PR_DECLARATION': PR_DECLARATION,
        'PR_KEYWORD': PR_KEYWORD,
        'PR_LITERAL': PR_LITERAL,
        'PR_NOCODE': PR_NOCODE,
        'PR_PLAIN': PR_PLAIN,
        'PR_PUNCTUATION': PR_PUNCTUATION,
        'PR_SOURCE': PR_SOURCE,
        'PR_STRING': PR_STRING,
        'PR_TAG': PR_TAG,
        'PR_TYPE': PR_TYPE
      };
})();