1# Protocol Buffers - Google's data interchange format
2# Copyright 2008 Google Inc.  All rights reserved.
3# https://developers.google.com/protocol-buffers/
4#
5# Redistribution and use in source and binary forms, with or without
6# modification, are permitted provided that the following conditions are
7# met:
8#
9#     * Redistributions of source code must retain the above copyright
10# notice, this list of conditions and the following disclaimer.
11#     * Redistributions in binary form must reproduce the above
12# copyright notice, this list of conditions and the following disclaimer
13# in the documentation and/or other materials provided with the
14# distribution.
15#     * Neither the name of Google Inc. nor the names of its
16# contributors may be used to endorse or promote products derived from
17# this software without specific prior written permission.
18#
19# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31"""Contains routines for printing protocol messages in JSON format.
32
33Simple usage example:
34
35  # Create a proto object and serialize it to a json format string.
36  message = my_proto_pb2.MyMessage(foo='bar')
37  json_string = json_format.MessageToJson(message)
38
39  # Parse a json format string to proto object.
40  message = json_format.Parse(json_string, my_proto_pb2.MyMessage())
41"""
42
43__author__ = 'jieluo@google.com (Jie Luo)'
44
45import base64
46import json
47import math
48import six
49import sys
50
51from google.protobuf import descriptor
52from google.protobuf import symbol_database
53
54_TIMESTAMPFOMAT = '%Y-%m-%dT%H:%M:%S'
55_INT_TYPES = frozenset([descriptor.FieldDescriptor.CPPTYPE_INT32,
56                        descriptor.FieldDescriptor.CPPTYPE_UINT32,
57                        descriptor.FieldDescriptor.CPPTYPE_INT64,
58                        descriptor.FieldDescriptor.CPPTYPE_UINT64])
59_INT64_TYPES = frozenset([descriptor.FieldDescriptor.CPPTYPE_INT64,
60                          descriptor.FieldDescriptor.CPPTYPE_UINT64])
61_FLOAT_TYPES = frozenset([descriptor.FieldDescriptor.CPPTYPE_FLOAT,
62                          descriptor.FieldDescriptor.CPPTYPE_DOUBLE])
63_INFINITY = 'Infinity'
64_NEG_INFINITY = '-Infinity'
65_NAN = 'NaN'
66
67
68class Error(Exception):
69  """Top-level module error for json_format."""
70
71
72class SerializeToJsonError(Error):
73  """Thrown if serialization to JSON fails."""
74
75
76class ParseError(Error):
77  """Thrown in case of parsing error."""
78
79
80def MessageToJson(message, including_default_value_fields=False):
81  """Converts protobuf message to JSON format.
82
83  Args:
84    message: The protocol buffers message instance to serialize.
85    including_default_value_fields: If True, singular primitive fields,
86        repeated fields, and map fields will always be serialized.  If
87        False, only serialize non-empty fields.  Singular message fields
88        and oneof fields are not affected by this option.
89
90  Returns:
91    A string containing the JSON formatted protocol buffer message.
92  """
93  js = _MessageToJsonObject(message, including_default_value_fields)
94  return json.dumps(js, indent=2)
95
96
97def _MessageToJsonObject(message, including_default_value_fields):
98  """Converts message to an object according to Proto3 JSON Specification."""
99  message_descriptor = message.DESCRIPTOR
100  full_name = message_descriptor.full_name
101  if _IsWrapperMessage(message_descriptor):
102    return _WrapperMessageToJsonObject(message)
103  if full_name in _WKTJSONMETHODS:
104    return _WKTJSONMETHODS[full_name][0](
105        message, including_default_value_fields)
106  js = {}
107  return _RegularMessageToJsonObject(
108      message, js, including_default_value_fields)
109
110
111def _IsMapEntry(field):
112  return (field.type == descriptor.FieldDescriptor.TYPE_MESSAGE and
113          field.message_type.has_options and
114          field.message_type.GetOptions().map_entry)
115
116
117def _RegularMessageToJsonObject(message, js, including_default_value_fields):
118  """Converts normal message according to Proto3 JSON Specification."""
119  fields = message.ListFields()
120  include_default = including_default_value_fields
121
122  try:
123    for field, value in fields:
124      name = field.camelcase_name
125      if _IsMapEntry(field):
126        # Convert a map field.
127        v_field = field.message_type.fields_by_name['value']
128        js_map = {}
129        for key in value:
130          if isinstance(key, bool):
131            if key:
132              recorded_key = 'true'
133            else:
134              recorded_key = 'false'
135          else:
136            recorded_key = key
137          js_map[recorded_key] = _FieldToJsonObject(
138              v_field, value[key], including_default_value_fields)
139        js[name] = js_map
140      elif field.label == descriptor.FieldDescriptor.LABEL_REPEATED:
141        # Convert a repeated field.
142        js[name] = [_FieldToJsonObject(field, k, include_default)
143                    for k in value]
144      else:
145        js[name] = _FieldToJsonObject(field, value, include_default)
146
147    # Serialize default value if including_default_value_fields is True.
148    if including_default_value_fields:
149      message_descriptor = message.DESCRIPTOR
150      for field in message_descriptor.fields:
151        # Singular message fields and oneof fields will not be affected.
152        if ((field.label != descriptor.FieldDescriptor.LABEL_REPEATED and
153             field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE) or
154            field.containing_oneof):
155          continue
156        name = field.camelcase_name
157        if name in js:
158          # Skip the field which has been serailized already.
159          continue
160        if _IsMapEntry(field):
161          js[name] = {}
162        elif field.label == descriptor.FieldDescriptor.LABEL_REPEATED:
163          js[name] = []
164        else:
165          js[name] = _FieldToJsonObject(field, field.default_value)
166
167  except ValueError as e:
168    raise SerializeToJsonError(
169        'Failed to serialize {0} field: {1}.'.format(field.name, e))
170
171  return js
172
173
174def _FieldToJsonObject(
175    field, value, including_default_value_fields=False):
176  """Converts field value according to Proto3 JSON Specification."""
177  if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
178    return _MessageToJsonObject(value, including_default_value_fields)
179  elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_ENUM:
180    enum_value = field.enum_type.values_by_number.get(value, None)
181    if enum_value is not None:
182      return enum_value.name
183    else:
184      raise SerializeToJsonError('Enum field contains an integer value '
185                                 'which can not mapped to an enum value.')
186  elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_STRING:
187    if field.type == descriptor.FieldDescriptor.TYPE_BYTES:
188      # Use base64 Data encoding for bytes
189      return base64.b64encode(value).decode('utf-8')
190    else:
191      return value
192  elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_BOOL:
193    return bool(value)
194  elif field.cpp_type in _INT64_TYPES:
195    return str(value)
196  elif field.cpp_type in _FLOAT_TYPES:
197    if math.isinf(value):
198      if value < 0.0:
199        return _NEG_INFINITY
200      else:
201        return _INFINITY
202    if math.isnan(value):
203      return _NAN
204  return value
205
206
207def _AnyMessageToJsonObject(message, including_default):
208  """Converts Any message according to Proto3 JSON Specification."""
209  if not message.ListFields():
210    return {}
211  js = {}
212  type_url = message.type_url
213  js['@type'] = type_url
214  sub_message = _CreateMessageFromTypeUrl(type_url)
215  sub_message.ParseFromString(message.value)
216  message_descriptor = sub_message.DESCRIPTOR
217  full_name = message_descriptor.full_name
218  if _IsWrapperMessage(message_descriptor):
219    js['value'] = _WrapperMessageToJsonObject(sub_message)
220    return js
221  if full_name in _WKTJSONMETHODS:
222    js['value'] = _WKTJSONMETHODS[full_name][0](sub_message, including_default)
223    return js
224  return _RegularMessageToJsonObject(sub_message, js, including_default)
225
226
227def _CreateMessageFromTypeUrl(type_url):
228  # TODO(jieluo): Should add a way that users can register the type resolver
229  # instead of the default one.
230  db = symbol_database.Default()
231  type_name = type_url.split('/')[-1]
232  try:
233    message_descriptor = db.pool.FindMessageTypeByName(type_name)
234  except KeyError:
235    raise TypeError(
236        'Can not find message descriptor by type_url: {0}.'.format(type_url))
237  message_class = db.GetPrototype(message_descriptor)
238  return message_class()
239
240
241def _GenericMessageToJsonObject(message, unused_including_default):
242  """Converts message by ToJsonString according to Proto3 JSON Specification."""
243  # Duration, Timestamp and FieldMask have ToJsonString method to do the
244  # convert. Users can also call the method directly.
245  return message.ToJsonString()
246
247
248def _ValueMessageToJsonObject(message, unused_including_default=False):
249  """Converts Value message according to Proto3 JSON Specification."""
250  which = message.WhichOneof('kind')
251  # If the Value message is not set treat as null_value when serialize
252  # to JSON. The parse back result will be different from original message.
253  if which is None or which == 'null_value':
254    return None
255  if which == 'list_value':
256    return _ListValueMessageToJsonObject(message.list_value)
257  if which == 'struct_value':
258    value = message.struct_value
259  else:
260    value = getattr(message, which)
261  oneof_descriptor = message.DESCRIPTOR.fields_by_name[which]
262  return _FieldToJsonObject(oneof_descriptor, value)
263
264
265def _ListValueMessageToJsonObject(message, unused_including_default=False):
266  """Converts ListValue message according to Proto3 JSON Specification."""
267  return [_ValueMessageToJsonObject(value)
268          for value in message.values]
269
270
271def _StructMessageToJsonObject(message, unused_including_default=False):
272  """Converts Struct message according to Proto3 JSON Specification."""
273  fields = message.fields
274  ret = {}
275  for key in fields:
276    ret[key] = _ValueMessageToJsonObject(fields[key])
277  return ret
278
279
280def _IsWrapperMessage(message_descriptor):
281  return message_descriptor.file.name == 'google/protobuf/wrappers.proto'
282
283
284def _WrapperMessageToJsonObject(message):
285  return _FieldToJsonObject(
286      message.DESCRIPTOR.fields_by_name['value'], message.value)
287
288
289def _DuplicateChecker(js):
290  result = {}
291  for name, value in js:
292    if name in result:
293      raise ParseError('Failed to load JSON: duplicate key {0}.'.format(name))
294    result[name] = value
295  return result
296
297
298def Parse(text, message):
299  """Parses a JSON representation of a protocol message into a message.
300
301  Args:
302    text: Message JSON representation.
303    message: A protocol beffer message to merge into.
304
305  Returns:
306    The same message passed as argument.
307
308  Raises::
309    ParseError: On JSON parsing problems.
310  """
311  if not isinstance(text, six.text_type): text = text.decode('utf-8')
312  try:
313    if sys.version_info < (2, 7):
314      # object_pair_hook is not supported before python2.7
315      js = json.loads(text)
316    else:
317      js = json.loads(text, object_pairs_hook=_DuplicateChecker)
318  except ValueError as e:
319    raise ParseError('Failed to load JSON: {0}.'.format(str(e)))
320  _ConvertMessage(js, message)
321  return message
322
323
324def _ConvertFieldValuePair(js, message):
325  """Convert field value pairs into regular message.
326
327  Args:
328    js: A JSON object to convert the field value pairs.
329    message: A regular protocol message to record the data.
330
331  Raises:
332    ParseError: In case of problems converting.
333  """
334  names = []
335  message_descriptor = message.DESCRIPTOR
336  for name in js:
337    try:
338      field = message_descriptor.fields_by_camelcase_name.get(name, None)
339      if not field:
340        raise ParseError(
341            'Message type "{0}" has no field named "{1}".'.format(
342                message_descriptor.full_name, name))
343      if name in names:
344        raise ParseError(
345            'Message type "{0}" should not have multiple "{1}" fields.'.format(
346                message.DESCRIPTOR.full_name, name))
347      names.append(name)
348      # Check no other oneof field is parsed.
349      if field.containing_oneof is not None:
350        oneof_name = field.containing_oneof.name
351        if oneof_name in names:
352          raise ParseError('Message type "{0}" should not have multiple "{1}" '
353                           'oneof fields.'.format(
354                               message.DESCRIPTOR.full_name, oneof_name))
355        names.append(oneof_name)
356
357      value = js[name]
358      if value is None:
359        message.ClearField(field.name)
360        continue
361
362      # Parse field value.
363      if _IsMapEntry(field):
364        message.ClearField(field.name)
365        _ConvertMapFieldValue(value, message, field)
366      elif field.label == descriptor.FieldDescriptor.LABEL_REPEATED:
367        message.ClearField(field.name)
368        if not isinstance(value, list):
369          raise ParseError('repeated field {0} must be in [] which is '
370                           '{1}.'.format(name, value))
371        if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
372          # Repeated message field.
373          for item in value:
374            sub_message = getattr(message, field.name).add()
375            # None is a null_value in Value.
376            if (item is None and
377                sub_message.DESCRIPTOR.full_name != 'google.protobuf.Value'):
378              raise ParseError('null is not allowed to be used as an element'
379                               ' in a repeated field.')
380            _ConvertMessage(item, sub_message)
381        else:
382          # Repeated scalar field.
383          for item in value:
384            if item is None:
385              raise ParseError('null is not allowed to be used as an element'
386                               ' in a repeated field.')
387            getattr(message, field.name).append(
388                _ConvertScalarFieldValue(item, field))
389      elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
390        sub_message = getattr(message, field.name)
391        _ConvertMessage(value, sub_message)
392      else:
393        setattr(message, field.name, _ConvertScalarFieldValue(value, field))
394    except ParseError as e:
395      if field and field.containing_oneof is None:
396        raise ParseError('Failed to parse {0} field: {1}'.format(name, e))
397      else:
398        raise ParseError(str(e))
399    except ValueError as e:
400      raise ParseError('Failed to parse {0} field: {1}.'.format(name, e))
401    except TypeError as e:
402      raise ParseError('Failed to parse {0} field: {1}.'.format(name, e))
403
404
405def _ConvertMessage(value, message):
406  """Convert a JSON object into a message.
407
408  Args:
409    value: A JSON object.
410    message: A WKT or regular protocol message to record the data.
411
412  Raises:
413    ParseError: In case of convert problems.
414  """
415  message_descriptor = message.DESCRIPTOR
416  full_name = message_descriptor.full_name
417  if _IsWrapperMessage(message_descriptor):
418    _ConvertWrapperMessage(value, message)
419  elif full_name in _WKTJSONMETHODS:
420    _WKTJSONMETHODS[full_name][1](value, message)
421  else:
422    _ConvertFieldValuePair(value, message)
423
424
425def _ConvertAnyMessage(value, message):
426  """Convert a JSON representation into Any message."""
427  if isinstance(value, dict) and not value:
428    return
429  try:
430    type_url = value['@type']
431  except KeyError:
432    raise ParseError('@type is missing when parsing any message.')
433
434  sub_message = _CreateMessageFromTypeUrl(type_url)
435  message_descriptor = sub_message.DESCRIPTOR
436  full_name = message_descriptor.full_name
437  if _IsWrapperMessage(message_descriptor):
438    _ConvertWrapperMessage(value['value'], sub_message)
439  elif full_name in _WKTJSONMETHODS:
440    _WKTJSONMETHODS[full_name][1](value['value'], sub_message)
441  else:
442    del value['@type']
443    _ConvertFieldValuePair(value, sub_message)
444  # Sets Any message
445  message.value = sub_message.SerializeToString()
446  message.type_url = type_url
447
448
449def _ConvertGenericMessage(value, message):
450  """Convert a JSON representation into message with FromJsonString."""
451  # Durantion, Timestamp, FieldMask have FromJsonString method to do the
452  # convert. Users can also call the method directly.
453  message.FromJsonString(value)
454
455
456_INT_OR_FLOAT = six.integer_types + (float,)
457
458
459def _ConvertValueMessage(value, message):
460  """Convert a JSON representation into Value message."""
461  if isinstance(value, dict):
462    _ConvertStructMessage(value, message.struct_value)
463  elif isinstance(value, list):
464    _ConvertListValueMessage(value, message.list_value)
465  elif value is None:
466    message.null_value = 0
467  elif isinstance(value, bool):
468    message.bool_value = value
469  elif isinstance(value, six.string_types):
470    message.string_value = value
471  elif isinstance(value, _INT_OR_FLOAT):
472    message.number_value = value
473  else:
474    raise ParseError('Unexpected type for Value message.')
475
476
477def _ConvertListValueMessage(value, message):
478  """Convert a JSON representation into ListValue message."""
479  if not isinstance(value, list):
480    raise ParseError(
481        'ListValue must be in [] which is {0}.'.format(value))
482  message.ClearField('values')
483  for item in value:
484    _ConvertValueMessage(item, message.values.add())
485
486
487def _ConvertStructMessage(value, message):
488  """Convert a JSON representation into Struct message."""
489  if not isinstance(value, dict):
490    raise ParseError(
491        'Struct must be in a dict which is {0}.'.format(value))
492  for key in value:
493    _ConvertValueMessage(value[key], message.fields[key])
494  return
495
496
497def _ConvertWrapperMessage(value, message):
498  """Convert a JSON representation into Wrapper message."""
499  field = message.DESCRIPTOR.fields_by_name['value']
500  setattr(message, 'value', _ConvertScalarFieldValue(value, field))
501
502
503def _ConvertMapFieldValue(value, message, field):
504  """Convert map field value for a message map field.
505
506  Args:
507    value: A JSON object to convert the map field value.
508    message: A protocol message to record the converted data.
509    field: The descriptor of the map field to be converted.
510
511  Raises:
512    ParseError: In case of convert problems.
513  """
514  if not isinstance(value, dict):
515    raise ParseError(
516        'Map field {0} must be in a dict which is {1}.'.format(
517            field.name, value))
518  key_field = field.message_type.fields_by_name['key']
519  value_field = field.message_type.fields_by_name['value']
520  for key in value:
521    key_value = _ConvertScalarFieldValue(key, key_field, True)
522    if value_field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
523      _ConvertMessage(value[key], getattr(message, field.name)[key_value])
524    else:
525      getattr(message, field.name)[key_value] = _ConvertScalarFieldValue(
526          value[key], value_field)
527
528
529def _ConvertScalarFieldValue(value, field, require_str=False):
530  """Convert a single scalar field value.
531
532  Args:
533    value: A scalar value to convert the scalar field value.
534    field: The descriptor of the field to convert.
535    require_str: If True, the field value must be a str.
536
537  Returns:
538    The converted scalar field value
539
540  Raises:
541    ParseError: In case of convert problems.
542  """
543  if field.cpp_type in _INT_TYPES:
544    return _ConvertInteger(value)
545  elif field.cpp_type in _FLOAT_TYPES:
546    return _ConvertFloat(value)
547  elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_BOOL:
548    return _ConvertBool(value, require_str)
549  elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_STRING:
550    if field.type == descriptor.FieldDescriptor.TYPE_BYTES:
551      return base64.b64decode(value)
552    else:
553      return value
554  elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_ENUM:
555    # Convert an enum value.
556    enum_value = field.enum_type.values_by_name.get(value, None)
557    if enum_value is None:
558      raise ParseError(
559          'Enum value must be a string literal with double quotes. '
560          'Type "{0}" has no value named {1}.'.format(
561              field.enum_type.full_name, value))
562    return enum_value.number
563
564
565def _ConvertInteger(value):
566  """Convert an integer.
567
568  Args:
569    value: A scalar value to convert.
570
571  Returns:
572    The integer value.
573
574  Raises:
575    ParseError: If an integer couldn't be consumed.
576  """
577  if isinstance(value, float):
578    raise ParseError('Couldn\'t parse integer: {0}.'.format(value))
579
580  if isinstance(value, six.text_type) and value.find(' ') != -1:
581    raise ParseError('Couldn\'t parse integer: "{0}".'.format(value))
582
583  return int(value)
584
585
586def _ConvertFloat(value):
587  """Convert an floating point number."""
588  if value == 'nan':
589    raise ParseError('Couldn\'t parse float "nan", use "NaN" instead.')
590  try:
591    # Assume Python compatible syntax.
592    return float(value)
593  except ValueError:
594    # Check alternative spellings.
595    if value == _NEG_INFINITY:
596      return float('-inf')
597    elif value == _INFINITY:
598      return float('inf')
599    elif value == _NAN:
600      return float('nan')
601    else:
602      raise ParseError('Couldn\'t parse float: {0}.'.format(value))
603
604
605def _ConvertBool(value, require_str):
606  """Convert a boolean value.
607
608  Args:
609    value: A scalar value to convert.
610    require_str: If True, value must be a str.
611
612  Returns:
613    The bool parsed.
614
615  Raises:
616    ParseError: If a boolean value couldn't be consumed.
617  """
618  if require_str:
619    if value == 'true':
620      return True
621    elif value == 'false':
622      return False
623    else:
624      raise ParseError('Expected "true" or "false", not {0}.'.format(value))
625
626  if not isinstance(value, bool):
627    raise ParseError('Expected true or false without quotes.')
628  return value
629
630_WKTJSONMETHODS = {
631    'google.protobuf.Any': [_AnyMessageToJsonObject,
632                            _ConvertAnyMessage],
633    'google.protobuf.Duration': [_GenericMessageToJsonObject,
634                                 _ConvertGenericMessage],
635    'google.protobuf.FieldMask': [_GenericMessageToJsonObject,
636                                  _ConvertGenericMessage],
637    'google.protobuf.ListValue': [_ListValueMessageToJsonObject,
638                                  _ConvertListValueMessage],
639    'google.protobuf.Struct': [_StructMessageToJsonObject,
640                               _ConvertStructMessage],
641    'google.protobuf.Timestamp': [_GenericMessageToJsonObject,
642                                  _ConvertGenericMessage],
643    'google.protobuf.Value': [_ValueMessageToJsonObject,
644                              _ConvertValueMessage]
645}
646