protobuf/pyext/descriptor_pool.cc

// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc.  All rights reserved.
// https://developers.google.com/protocol-buffers/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
//     * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
//     * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

// Implements the DescriptorPool, which collects all descriptors.

#include <Python.h>

#include <google/protobuf/descriptor.pb.h>
#include <google/protobuf/dynamic_message.h>
#include <google/protobuf/pyext/descriptor.h>
#include <google/protobuf/pyext/descriptor_database.h>
#include <google/protobuf/pyext/descriptor_pool.h>
#include <google/protobuf/pyext/message.h>
#include <google/protobuf/pyext/scoped_pyobject_ptr.h>

#if PY_MAJOR_VERSION >= 3
  #define PyString_FromStringAndSize PyUnicode_FromStringAndSize
  #if PY_VERSION_HEX < 0x03030000
    #error "Python 3.0 - 3.2 are not supported."
  #endif
  #define PyString_AsStringAndSize(ob, charpp, sizep) \
    (PyUnicode_Check(ob)? \
       ((*(charpp) = PyUnicode_AsUTF8AndSize(ob, (sizep))) == NULL? -1: 0): \
       PyBytes_AsStringAndSize(ob, (charpp), (sizep)))
#endif

namespace google {
namespace protobuf {
namespace python {

// A map to cache Python Pools per C++ pointer.
// Pointers are not owned here, and belong to the PyDescriptorPool.
static hash_map<const DescriptorPool*, PyDescriptorPool*> descriptor_pool_map;

namespace cdescriptor_pool {

// Create a Python DescriptorPool object, but does not fill the "pool"
// attribute.
static PyDescriptorPool* _CreateDescriptorPool() {
  PyDescriptorPool* cpool = PyObject_New(
      PyDescriptorPool, &PyDescriptorPool_Type);
  if (cpool == NULL) {
    return NULL;
  }

  cpool->underlay = NULL;
  cpool->database = NULL;

  DynamicMessageFactory* message_factory = new DynamicMessageFactory();
  // This option might be the default some day.
  message_factory->SetDelegateToGeneratedFactory(true);
  cpool->message_factory = message_factory;

  // TODO(amauryfa): Rewrite the SymbolDatabase in C so that it uses the same
  // storage.
  cpool->classes_by_descriptor =
      new PyDescriptorPool::ClassesByMessageMap();
  cpool->descriptor_options =
      new hash_map<const void*, PyObject *>();

  return cpool;
}

// Create a Python DescriptorPool, using the given pool as an underlay:
// new messages will be added to a custom pool, not to the underlay.
//
// Ownership of the underlay is not transferred, its pointer should
// stay alive.
static PyDescriptorPool* PyDescriptorPool_NewWithUnderlay(
    const DescriptorPool* underlay) {
  PyDescriptorPool* cpool = _CreateDescriptorPool();
  if (cpool == NULL) {
    return NULL;
  }
  cpool->pool = new DescriptorPool(underlay);
  cpool->underlay = underlay;

  if (!descriptor_pool_map.insert(
      std::make_pair(cpool->pool, cpool)).second) {
    // Should never happen -- would indicate an internal error / bug.
    PyErr_SetString(PyExc_ValueError, "DescriptorPool already registered");
    return NULL;
  }

  return cpool;
}

static PyDescriptorPool* PyDescriptorPool_NewWithDatabase(
    DescriptorDatabase* database) {
  PyDescriptorPool* cpool = _CreateDescriptorPool();
  if (cpool == NULL) {
    return NULL;
  }
  if (database != NULL) {
    cpool->pool = new DescriptorPool(database);
    cpool->database = database;
  } else {
    cpool->pool = new DescriptorPool();
  }

  if (!descriptor_pool_map.insert(std::make_pair(cpool->pool, cpool)).second) {
    // Should never happen -- would indicate an internal error / bug.
    PyErr_SetString(PyExc_ValueError, "DescriptorPool already registered");
    return NULL;
  }

  return cpool;
}

// The public DescriptorPool constructor.
static PyObject* New(PyTypeObject* type,
                     PyObject* args, PyObject* kwargs) {
  static char* kwlist[] = {"descriptor_db", 0};
  PyObject* py_database = NULL;
  if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O", kwlist, &py_database)) {
    return NULL;
  }
  DescriptorDatabase* database = NULL;
  if (py_database && py_database != Py_None) {
    database = new PyDescriptorDatabase(py_database);
  }
  return reinterpret_cast<PyObject*>(
      PyDescriptorPool_NewWithDatabase(database));
}

static void Dealloc(PyDescriptorPool* self) {
  typedef PyDescriptorPool::ClassesByMessageMap::iterator iterator;
  descriptor_pool_map.erase(self->pool);
  for (iterator it = self->classes_by_descriptor->begin();
       it != self->classes_by_descriptor->end(); ++it) {
    Py_DECREF(it->second);
  }
  delete self->classes_by_descriptor;
  for (hash_map<const void*, PyObject*>::iterator it =
           self->descriptor_options->begin();
       it != self->descriptor_options->end(); ++it) {
    Py_DECREF(it->second);
  }
  delete self->descriptor_options;
  delete self->message_factory;
  delete self->database;
  delete self->pool;
  Py_TYPE(self)->tp_free(reinterpret_cast<PyObject*>(self));
}

PyObject* FindMessageByName(PyDescriptorPool* self, PyObject* arg) {
  Py_ssize_t name_size;
  char* name;
  if (PyString_AsStringAndSize(arg, &name, &name_size) < 0) {
    return NULL;
  }

  const Descriptor* message_descriptor =
      self->pool->FindMessageTypeByName(string(name, name_size));

  if (message_descriptor == NULL) {
    PyErr_Format(PyExc_KeyError, "Couldn't find message %.200s", name);
    return NULL;
  }

  return PyMessageDescriptor_FromDescriptor(message_descriptor);
}

// Add a message class to our database.
int RegisterMessageClass(PyDescriptorPool* self,
                         const Descriptor* message_descriptor,
                         CMessageClass* message_class) {
  Py_INCREF(message_class);
  typedef PyDescriptorPool::ClassesByMessageMap::iterator iterator;
  std::pair<iterator, bool> ret = self->classes_by_descriptor->insert(
      std::make_pair(message_descriptor, message_class));
  if (!ret.second) {
    // Update case: DECREF the previous value.
    Py_DECREF(ret.first->second);
    ret.first->second = message_class;
  }
  return 0;
}

// Retrieve the message class added to our database.
CMessageClass* GetMessageClass(PyDescriptorPool* self,
                               const Descriptor* message_descriptor) {
  typedef PyDescriptorPool::ClassesByMessageMap::iterator iterator;
  iterator ret = self->classes_by_descriptor->find(message_descriptor);
  if (ret == self->classes_by_descriptor->end()) {
    PyErr_Format(PyExc_TypeError, "No message class registered for '%s'",
                 message_descriptor->full_name().c_str());
    return NULL;
  } else {
    return ret->second;
  }
}

PyObject* FindFileByName(PyDescriptorPool* self, PyObject* arg) {
  Py_ssize_t name_size;
  char* name;
  if (PyString_AsStringAndSize(arg, &name, &name_size) < 0) {
    return NULL;
  }

  const FileDescriptor* file_descriptor =
      self->pool->FindFileByName(string(name, name_size));
  if (file_descriptor == NULL) {
    PyErr_Format(PyExc_KeyError, "Couldn't find file %.200s",
                 name);
    return NULL;
  }

  return PyFileDescriptor_FromDescriptor(file_descriptor);
}

PyObject* FindFieldByName(PyDescriptorPool* self, PyObject* arg) {
  Py_ssize_t name_size;
  char* name;
  if (PyString_AsStringAndSize(arg, &name, &name_size) < 0) {
    return NULL;
  }

  const FieldDescriptor* field_descriptor =
      self->pool->FindFieldByName(string(name, name_size));
  if (field_descriptor == NULL) {
    PyErr_Format(PyExc_KeyError, "Couldn't find field %.200s",
                 name);
    return NULL;
  }

  return PyFieldDescriptor_FromDescriptor(field_descriptor);
}

PyObject* FindExtensionByName(PyDescriptorPool* self, PyObject* arg) {
  Py_ssize_t name_size;
  char* name;
  if (PyString_AsStringAndSize(arg, &name, &name_size) < 0) {
    return NULL;
  }

  const FieldDescriptor* field_descriptor =
      self->pool->FindExtensionByName(string(name, name_size));
  if (field_descriptor == NULL) {
    PyErr_Format(PyExc_KeyError, "Couldn't find extension field %.200s", name);
    return NULL;
  }

  return PyFieldDescriptor_FromDescriptor(field_descriptor);
}

PyObject* FindEnumTypeByName(PyDescriptorPool* self, PyObject* arg) {
  Py_ssize_t name_size;
  char* name;
  if (PyString_AsStringAndSize(arg, &name, &name_size) < 0) {
    return NULL;
  }

  const EnumDescriptor* enum_descriptor =
      self->pool->FindEnumTypeByName(string(name, name_size));
  if (enum_descriptor == NULL) {
    PyErr_Format(PyExc_KeyError, "Couldn't find enum %.200s", name);
    return NULL;
  }

  return PyEnumDescriptor_FromDescriptor(enum_descriptor);
}

PyObject* FindOneofByName(PyDescriptorPool* self, PyObject* arg) {
  Py_ssize_t name_size;
  char* name;
  if (PyString_AsStringAndSize(arg, &name, &name_size) < 0) {
    return NULL;
  }

  const OneofDescriptor* oneof_descriptor =
      self->pool->FindOneofByName(string(name, name_size));
  if (oneof_descriptor == NULL) {
    PyErr_Format(PyExc_KeyError, "Couldn't find oneof %.200s", name);
    return NULL;
  }

  return PyOneofDescriptor_FromDescriptor(oneof_descriptor);
}

PyObject* FindFileContainingSymbol(PyDescriptorPool* self, PyObject* arg) {
  Py_ssize_t name_size;
  char* name;
  if (PyString_AsStringAndSize(arg, &name, &name_size) < 0) {
    return NULL;
  }

  const FileDescriptor* file_descriptor =
      self->pool->FindFileContainingSymbol(string(name, name_size));
  if (file_descriptor == NULL) {
    PyErr_Format(PyExc_KeyError, "Couldn't find symbol %.200s", name);
    return NULL;
  }

  return PyFileDescriptor_FromDescriptor(file_descriptor);
}

// These functions should not exist -- the only valid way to create
// descriptors is to call Add() or AddSerializedFile().
// But these AddDescriptor() functions were created in Python and some people
// call them, so we support them for now for compatibility.
// However we do check that the existing descriptor already exists in the pool,
// which appears to always be true for existing calls -- but then why do people
// call a function that will just be a no-op?
// TODO(amauryfa): Need to investigate further.

PyObject* AddFileDescriptor(PyDescriptorPool* self, PyObject* descriptor) {
  const FileDescriptor* file_descriptor =
      PyFileDescriptor_AsDescriptor(descriptor);
  if (!file_descriptor) {
    return NULL;
  }
  if (file_descriptor !=
      self->pool->FindFileByName(file_descriptor->name())) {
    PyErr_Format(PyExc_ValueError,
                 "The file descriptor %s does not belong to this pool",
                 file_descriptor->name().c_str());
    return NULL;
  }
  Py_RETURN_NONE;
}

PyObject* AddDescriptor(PyDescriptorPool* self, PyObject* descriptor) {
  const Descriptor* message_descriptor =
      PyMessageDescriptor_AsDescriptor(descriptor);
  if (!message_descriptor) {
    return NULL;
  }
  if (message_descriptor !=
      self->pool->FindMessageTypeByName(message_descriptor->full_name())) {
    PyErr_Format(PyExc_ValueError,
                 "The message descriptor %s does not belong to this pool",
                 message_descriptor->full_name().c_str());
    return NULL;
  }
  Py_RETURN_NONE;
}

PyObject* AddEnumDescriptor(PyDescriptorPool* self, PyObject* descriptor) {
  const EnumDescriptor* enum_descriptor =
      PyEnumDescriptor_AsDescriptor(descriptor);
  if (!enum_descriptor) {
    return NULL;
  }
  if (enum_descriptor !=
      self->pool->FindEnumTypeByName(enum_descriptor->full_name())) {
    PyErr_Format(PyExc_ValueError,
                 "The enum descriptor %s does not belong to this pool",
                 enum_descriptor->full_name().c_str());
    return NULL;
  }
  Py_RETURN_NONE;
}

// The code below loads new Descriptors from a serialized FileDescriptorProto.


// Collects errors that occur during proto file building to allow them to be
// propagated in the python exception instead of only living in ERROR logs.
class BuildFileErrorCollector : public DescriptorPool::ErrorCollector {
 public:
  BuildFileErrorCollector() : error_message(""), had_errors(false) {}

  void AddError(const string& filename, const string& element_name,
                const Message* descriptor, ErrorLocation location,
                const string& message) {
    // Replicates the logging behavior that happens in the C++ implementation
    // when an error collector is not passed in.
    if (!had_errors) {
      error_message +=
          ("Invalid proto descriptor for file \"" + filename + "\":\n");
      had_errors = true;
    }
    // As this only happens on failure and will result in the program not
    // running at all, no effort is made to optimize this string manipulation.
    error_message += ("  " + element_name + ": " + message + "\n");
  }

  string error_message;
  bool had_errors;
};

PyObject* AddSerializedFile(PyDescriptorPool* self, PyObject* serialized_pb) {
  char* message_type;
  Py_ssize_t message_len;

  if (self->database != NULL) {
    PyErr_SetString(
        PyExc_ValueError,
        "Cannot call Add on a DescriptorPool that uses a DescriptorDatabase. "
        "Add your file to the underlying database.");
    return NULL;
  }

  if (PyBytes_AsStringAndSize(serialized_pb, &message_type, &message_len) < 0) {
    return NULL;
  }

  FileDescriptorProto file_proto;
  if (!file_proto.ParseFromArray(message_type, message_len)) {
    PyErr_SetString(PyExc_TypeError, "Couldn't parse file content!");
    return NULL;
  }

  // If the file was already part of a C++ library, all its descriptors are in
  // the underlying pool.  No need to do anything else.
  const FileDescriptor* generated_file = NULL;
  if (self->underlay) {
    generated_file = self->underlay->FindFileByName(file_proto.name());
  }
  if (generated_file != NULL) {
    return PyFileDescriptor_FromDescriptorWithSerializedPb(
        generated_file, serialized_pb);
  }

  BuildFileErrorCollector error_collector;
  const FileDescriptor* descriptor =
      self->pool->BuildFileCollectingErrors(file_proto,
                                            &error_collector);
  if (descriptor == NULL) {
    PyErr_Format(PyExc_TypeError,
                 "Couldn't build proto file into descriptor pool!\n%s",
                 error_collector.error_message.c_str());
    return NULL;
  }

  return PyFileDescriptor_FromDescriptorWithSerializedPb(
      descriptor, serialized_pb);
}

PyObject* Add(PyDescriptorPool* self, PyObject* file_descriptor_proto) {
  ScopedPyObjectPtr serialized_pb(
      PyObject_CallMethod(file_descriptor_proto, "SerializeToString", NULL));
  if (serialized_pb == NULL) {
    return NULL;
  }
  return AddSerializedFile(self, serialized_pb.get());
}

static PyMethodDef Methods[] = {
  { "Add", (PyCFunction)Add, METH_O,
    "Adds the FileDescriptorProto and its types to this pool." },
  { "AddSerializedFile", (PyCFunction)AddSerializedFile, METH_O,
    "Adds a serialized FileDescriptorProto to this pool." },

  // TODO(amauryfa): Understand why the Python implementation differs from
  // this one, ask users to use another API and deprecate these functions.
  { "AddFileDescriptor", (PyCFunction)AddFileDescriptor, METH_O,
    "No-op. Add() must have been called before." },
  { "AddDescriptor", (PyCFunction)AddDescriptor, METH_O,
    "No-op. Add() must have been called before." },
  { "AddEnumDescriptor", (PyCFunction)AddEnumDescriptor, METH_O,
    "No-op. Add() must have been called before." },

  { "FindFileByName", (PyCFunction)FindFileByName, METH_O,
    "Searches for a file descriptor by its .proto name." },
  { "FindMessageTypeByName", (PyCFunction)FindMessageByName, METH_O,
    "Searches for a message descriptor by full name." },
  { "FindFieldByName", (PyCFunction)FindFieldByName, METH_O,
    "Searches for a field descriptor by full name." },
  { "FindExtensionByName", (PyCFunction)FindExtensionByName, METH_O,
    "Searches for extension descriptor by full name." },
  { "FindEnumTypeByName", (PyCFunction)FindEnumTypeByName, METH_O,
    "Searches for enum type descriptor by full name." },
  { "FindOneofByName", (PyCFunction)FindOneofByName, METH_O,
    "Searches for oneof descriptor by full name." },

  { "FindFileContainingSymbol", (PyCFunction)FindFileContainingSymbol, METH_O,
    "Gets the FileDescriptor containing the specified symbol." },
  {NULL}
};

}  // namespace cdescriptor_pool

PyTypeObject PyDescriptorPool_Type = {
  PyVarObject_HEAD_INIT(&PyType_Type, 0)
  FULL_MODULE_NAME ".DescriptorPool",  // tp_name
  sizeof(PyDescriptorPool),            // tp_basicsize
  0,                                   // tp_itemsize
  (destructor)cdescriptor_pool::Dealloc,  // tp_dealloc
  0,                                   // tp_print
  0,                                   // tp_getattr
  0,                                   // tp_setattr
  0,                                   // tp_compare
  0,                                   // tp_repr
  0,                                   // tp_as_number
  0,                                   // tp_as_sequence
  0,                                   // tp_as_mapping
  0,                                   // tp_hash
  0,                                   // tp_call
  0,                                   // tp_str
  0,                                   // tp_getattro
  0,                                   // tp_setattro
  0,                                   // tp_as_buffer
  Py_TPFLAGS_DEFAULT,                  // tp_flags
  "A Descriptor Pool",                 // tp_doc
  0,                                   // tp_traverse
  0,                                   // tp_clear
  0,                                   // tp_richcompare
  0,                                   // tp_weaklistoffset
  0,                                   // tp_iter
  0,                                   // tp_iternext
  cdescriptor_pool::Methods,           // tp_methods
  0,                                   // tp_members
  0,                                   // tp_getset
  0,                                   // tp_base
  0,                                   // tp_dict
  0,                                   // tp_descr_get
  0,                                   // tp_descr_set
  0,                                   // tp_dictoffset
  0,                                   // tp_init
  0,                                   // tp_alloc
  cdescriptor_pool::New,               // tp_new
  PyObject_Del,                        // tp_free
};

// This is the DescriptorPool which contains all the definitions from the
// generated _pb2.py modules.
static PyDescriptorPool* python_generated_pool = NULL;

bool InitDescriptorPool() {
  if (PyType_Ready(&PyDescriptorPool_Type) < 0)
    return false;

  // The Pool of messages declared in Python libraries.
  // generated_pool() contains all messages already linked in C++ libraries, and
  // is used as underlay.
  python_generated_pool = cdescriptor_pool::PyDescriptorPool_NewWithUnderlay(
      DescriptorPool::generated_pool());
  if (python_generated_pool == NULL) {
    return false;
  }
  // Register this pool to be found for C++-generated descriptors.
  descriptor_pool_map.insert(
      std::make_pair(DescriptorPool::generated_pool(),
                     python_generated_pool));

  return true;
}

// The default DescriptorPool used everywhere in this module.
// Today it's the python_generated_pool.
// TODO(amauryfa): Remove all usages of this function: the pool should be
// derived from the context.
PyDescriptorPool* GetDefaultDescriptorPool() {
  return python_generated_pool;
}

PyDescriptorPool* GetDescriptorPool_FromPool(const DescriptorPool* pool) {
  // Fast path for standard descriptors.
  if (pool == python_generated_pool->pool ||
      pool == DescriptorPool::generated_pool()) {
    return python_generated_pool;
  }
  hash_map<const DescriptorPool*, PyDescriptorPool*>::iterator it =
      descriptor_pool_map.find(pool);
  if (it == descriptor_pool_map.end()) {
    PyErr_SetString(PyExc_KeyError, "Unknown descriptor pool");
    return NULL;
  }
  return it->second;
}

}  // namespace python
}  // namespace protobuf
}  // namespace google