17eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel/* 27eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel * multibytecodec.c: Common Multibyte Codec Implementation 37eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel * 47eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel * Written by Hye-Shik Chang <perky@FreeBSD.org> 57eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel */ 67eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 77eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel#define PY_SSIZE_T_CLEAN 87eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel#include "Python.h" 97eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel#include "structmember.h" 107eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel#include "multibytecodec.h" 117eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 127eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanieltypedef struct { 137eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel const Py_UNICODE *inbuf, *inbuf_top, *inbuf_end; 147eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel unsigned char *outbuf, *outbuf_end; 157eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyObject *excobj, *outobj; 167eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel} MultibyteEncodeBuffer; 177eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 187eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanieltypedef struct { 197eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel const unsigned char *inbuf, *inbuf_top, *inbuf_end; 207eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_UNICODE *outbuf, *outbuf_end; 217eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyObject *excobj, *outobj; 227eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel} MultibyteDecodeBuffer; 237eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 247eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielPyDoc_STRVAR(MultibyteCodec_Encode__doc__, 257eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel"I.encode(unicode[, errors]) -> (string, length consumed)\n\ 267eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel\n\ 277eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielReturn an encoded string version of `unicode'. errors may be given to\n\ 287eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielset a different error handling scheme. Default is 'strict' meaning that\n\ 297eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielencoding errors raise a UnicodeEncodeError. Other possible values are\n\ 307eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel'ignore', 'replace' and 'xmlcharrefreplace' as well as any other name\n\ 317eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielregistered with codecs.register_error that can handle UnicodeEncodeErrors."); 327eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 337eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielPyDoc_STRVAR(MultibyteCodec_Decode__doc__, 347eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel"I.decode(string[, errors]) -> (unicodeobject, length consumed)\n\ 357eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel\n\ 367eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielDecodes `string' using I, an MultibyteCodec instance. errors may be given\n\ 377eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielto set a different error handling scheme. Default is 'strict' meaning\n\ 387eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielthat encoding errors raise a UnicodeDecodeError. Other possible values\n\ 397eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielare 'ignore' and 'replace' as well as any other name registered with\n\ 407eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielcodecs.register_error that is able to handle UnicodeDecodeErrors."); 417eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 427eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic char *codeckwarglist[] = {"input", "errors", NULL}; 437eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic char *incnewkwarglist[] = {"errors", NULL}; 447eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic char *incrementalkwarglist[] = {"input", "final", NULL}; 457eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic char *streamkwarglist[] = {"stream", "errors", NULL}; 467eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 477eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic PyObject *multibytecodec_encode(MultibyteCodec *, 487eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel MultibyteCodec_State *, const Py_UNICODE **, Py_ssize_t, 497eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyObject *, int); 507eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 517eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel#define MBENC_RESET MBENC_MAX<<1 /* reset after an encoding session */ 527eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 537eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic PyObject * 547eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielmake_tuple(PyObject *object, Py_ssize_t len) 557eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel{ 567eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyObject *v, *w; 577eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 587eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (object == NULL) 597eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return NULL; 607eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 617eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel v = PyTuple_New(2); 627eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (v == NULL) { 637eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_DECREF(object); 647eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return NULL; 657eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 667eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyTuple_SET_ITEM(v, 0, object); 677eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 687eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel w = PyInt_FromSsize_t(len); 697eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (w == NULL) { 707eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_DECREF(v); 717eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return NULL; 727eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 737eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyTuple_SET_ITEM(v, 1, w); 747eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 757eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return v; 767eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel} 777eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 787eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic PyObject * 797eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielinternal_error_callback(const char *errors) 807eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel{ 817eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (errors == NULL || strcmp(errors, "strict") == 0) 827eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return ERROR_STRICT; 837eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel else if (strcmp(errors, "ignore") == 0) 847eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return ERROR_IGNORE; 857eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel else if (strcmp(errors, "replace") == 0) 867eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return ERROR_REPLACE; 877eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel else 887eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return PyString_FromString(errors); 897eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel} 907eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 917eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic PyObject * 927eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielcall_error_callback(PyObject *errors, PyObject *exc) 937eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel{ 947eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyObject *args, *cb, *r; 957eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 967eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel assert(PyString_Check(errors)); 977eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel cb = PyCodec_LookupError(PyString_AS_STRING(errors)); 987eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (cb == NULL) 997eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return NULL; 1007eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 1017eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel args = PyTuple_New(1); 1027eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (args == NULL) { 1037eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_DECREF(cb); 1047eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return NULL; 1057eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 1067eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 1077eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyTuple_SET_ITEM(args, 0, exc); 1087eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_INCREF(exc); 1097eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 1107eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel r = PyObject_CallObject(cb, args); 1117eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_DECREF(args); 1127eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_DECREF(cb); 1137eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return r; 1147eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel} 1157eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 1167eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic PyObject * 1177eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielcodecctx_errors_get(MultibyteStatefulCodecContext *self) 1187eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel{ 1197eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel const char *errors; 1207eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 1217eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (self->errors == ERROR_STRICT) 1227eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel errors = "strict"; 1237eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel else if (self->errors == ERROR_IGNORE) 1247eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel errors = "ignore"; 1257eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel else if (self->errors == ERROR_REPLACE) 1267eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel errors = "replace"; 1277eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel else { 1287eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_INCREF(self->errors); 1297eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return self->errors; 1307eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 1317eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 1327eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return PyString_FromString(errors); 1337eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel} 1347eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 1357eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic int 1367eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielcodecctx_errors_set(MultibyteStatefulCodecContext *self, PyObject *value, 1377eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel void *closure) 1387eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel{ 1397eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyObject *cb; 1407eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 1417eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (!PyString_Check(value)) { 1427eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyErr_SetString(PyExc_TypeError, "errors must be a string"); 1437eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return -1; 1447eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 1457eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 1467eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel cb = internal_error_callback(PyString_AS_STRING(value)); 1477eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (cb == NULL) 1487eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return -1; 1497eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 1507eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel ERROR_DECREF(self->errors); 1517eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel self->errors = cb; 1527eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return 0; 1537eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel} 1547eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 1557eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel/* This getset handlers list is used by all the stateful codec objects */ 1567eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic PyGetSetDef codecctx_getsets[] = { 1577eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel {"errors", (getter)codecctx_errors_get, 1587eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel (setter)codecctx_errors_set, 1597eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyDoc_STR("how to treat errors")}, 1607eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel {NULL,} 1617eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel}; 1627eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 1637eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic int 1647eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielexpand_encodebuffer(MultibyteEncodeBuffer *buf, Py_ssize_t esize) 1657eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel{ 1667eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_ssize_t orgpos, orgsize, incsize; 1677eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 1687eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel orgpos = (Py_ssize_t)((char *)buf->outbuf - 1697eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyString_AS_STRING(buf->outobj)); 1707eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel orgsize = PyString_GET_SIZE(buf->outobj); 1717eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel incsize = (esize < (orgsize >> 1) ? (orgsize >> 1) | 1 : esize); 1727eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 1737eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (orgsize > PY_SSIZE_T_MAX - incsize) { 1747eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyErr_NoMemory(); 1757eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return -1; 1767eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 1777eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 1787eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (_PyString_Resize(&buf->outobj, orgsize + incsize) == -1) 1797eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return -1; 1807eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 1817eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel buf->outbuf = (unsigned char *)PyString_AS_STRING(buf->outobj) +orgpos; 1827eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel buf->outbuf_end = (unsigned char *)PyString_AS_STRING(buf->outobj) 1837eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel + PyString_GET_SIZE(buf->outobj); 1847eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 1857eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return 0; 1867eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel} 1877eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel#define REQUIRE_ENCODEBUFFER(buf, s) do { \ 1887eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if ((s) < 0 || (s) > (buf)->outbuf_end - (buf)->outbuf) \ 1897eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (expand_encodebuffer(buf, s) == -1) \ 1907eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; \ 1917eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel} while(0) 1927eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 1937eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic int 1947eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielexpand_decodebuffer(MultibyteDecodeBuffer *buf, Py_ssize_t esize) 1957eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel{ 1967eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_ssize_t orgpos, orgsize; 1977eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 1987eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel orgpos = (Py_ssize_t)(buf->outbuf - PyUnicode_AS_UNICODE(buf->outobj)); 1997eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel orgsize = PyUnicode_GET_SIZE(buf->outobj); 2007eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (PyUnicode_Resize(&buf->outobj, orgsize + ( 2017eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel esize < (orgsize >> 1) ? (orgsize >> 1) | 1 : esize)) == -1) 2027eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return -1; 2037eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 2047eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel buf->outbuf = PyUnicode_AS_UNICODE(buf->outobj) + orgpos; 2057eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel buf->outbuf_end = PyUnicode_AS_UNICODE(buf->outobj) 2067eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel + PyUnicode_GET_SIZE(buf->outobj); 2077eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 2087eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return 0; 2097eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel} 2107eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel#define REQUIRE_DECODEBUFFER(buf, s) do { \ 2117eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if ((s) < 0 || (s) > (buf)->outbuf_end - (buf)->outbuf) \ 2127eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (expand_decodebuffer(buf, s) == -1) \ 2137eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; \ 2147eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel} while(0) 2157eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 2167eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 2177eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel/** 2187eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel * MultibyteCodec object 2197eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel */ 2207eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 2217eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic int 2227eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielmultibytecodec_encerror(MultibyteCodec *codec, 2237eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel MultibyteCodec_State *state, 2247eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel MultibyteEncodeBuffer *buf, 2257eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyObject *errors, Py_ssize_t e) 2267eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel{ 2277eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyObject *retobj = NULL, *retstr = NULL, *tobj; 2287eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_ssize_t retstrsize, newpos; 2297eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_ssize_t esize, start, end; 2307eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel const char *reason; 2317eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 2327eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (e > 0) { 2337eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel reason = "illegal multibyte sequence"; 2347eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel esize = e; 2357eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 2367eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel else { 2377eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel switch (e) { 2387eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel case MBERR_TOOSMALL: 2397eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel REQUIRE_ENCODEBUFFER(buf, -1); 2407eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return 0; /* retry it */ 2417eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel case MBERR_TOOFEW: 2427eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel reason = "incomplete multibyte sequence"; 2437eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel esize = (Py_ssize_t)(buf->inbuf_end - buf->inbuf); 2447eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel break; 2457eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel case MBERR_INTERNAL: 2467eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyErr_SetString(PyExc_RuntimeError, 2477eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel "internal codec error"); 2487eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return -1; 2497eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel default: 2507eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyErr_SetString(PyExc_RuntimeError, 2517eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel "unknown runtime error"); 2527eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return -1; 2537eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 2547eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 2557eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 2567eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (errors == ERROR_REPLACE) { 2577eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel const Py_UNICODE replchar = '?', *inbuf = &replchar; 2587eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_ssize_t r; 2597eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 2607eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel for (;;) { 2617eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_ssize_t outleft; 2627eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 2637eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel outleft = (Py_ssize_t)(buf->outbuf_end - buf->outbuf); 2647eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel r = codec->encode(state, codec->config, &inbuf, 1, 2657eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel &buf->outbuf, outleft, 0); 2667eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (r == MBERR_TOOSMALL) { 2677eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel REQUIRE_ENCODEBUFFER(buf, -1); 2687eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel continue; 2697eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 2707eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel else 2717eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel break; 2727eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 2737eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 2747eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (r != 0) { 2757eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel REQUIRE_ENCODEBUFFER(buf, 1); 2767eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel *buf->outbuf++ = '?'; 2777eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 2787eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 2797eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (errors == ERROR_IGNORE || errors == ERROR_REPLACE) { 2807eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel buf->inbuf += esize; 2817eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return 0; 2827eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 2837eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 2847eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel start = (Py_ssize_t)(buf->inbuf - buf->inbuf_top); 2857eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel end = start + esize; 2867eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 2877eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel /* use cached exception object if available */ 2887eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (buf->excobj == NULL) { 2897eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel buf->excobj = PyUnicodeEncodeError_Create(codec->encoding, 2907eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel buf->inbuf_top, 2917eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel buf->inbuf_end - buf->inbuf_top, 2927eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel start, end, reason); 2937eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (buf->excobj == NULL) 2947eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 2957eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 2967eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel else 2977eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (PyUnicodeEncodeError_SetStart(buf->excobj, start) != 0 || 2987eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyUnicodeEncodeError_SetEnd(buf->excobj, end) != 0 || 2997eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyUnicodeEncodeError_SetReason(buf->excobj, reason) != 0) 3007eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 3017eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 3027eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (errors == ERROR_STRICT) { 3037eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyCodec_StrictErrors(buf->excobj); 3047eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 3057eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 3067eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 3077eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel retobj = call_error_callback(errors, buf->excobj); 3087eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (retobj == NULL) 3097eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 3107eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 3117eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (!PyTuple_Check(retobj) || PyTuple_GET_SIZE(retobj) != 2 || 3127eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel !PyUnicode_Check((tobj = PyTuple_GET_ITEM(retobj, 0))) || 3137eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel !(PyInt_Check(PyTuple_GET_ITEM(retobj, 1)) || 3147eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyLong_Check(PyTuple_GET_ITEM(retobj, 1)))) { 3157eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyErr_SetString(PyExc_TypeError, 3167eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel "encoding error handler must return " 3177eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel "(unicode, int) tuple"); 3187eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 3197eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 3207eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 3217eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel { 3227eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel const Py_UNICODE *uraw = PyUnicode_AS_UNICODE(tobj); 3237eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 3247eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel retstr = multibytecodec_encode(codec, state, &uraw, 3257eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyUnicode_GET_SIZE(tobj), ERROR_STRICT, 3267eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel MBENC_FLUSH); 3277eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (retstr == NULL) 3287eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 3297eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 3307eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 3317eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel retstrsize = PyString_GET_SIZE(retstr); 3327eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (retstrsize > 0) { 3337eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel REQUIRE_ENCODEBUFFER(buf, retstrsize); 3347eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel memcpy(buf->outbuf, PyString_AS_STRING(retstr), retstrsize); 3357eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel buf->outbuf += retstrsize; 3367eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 3377eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 3387eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel newpos = PyInt_AsSsize_t(PyTuple_GET_ITEM(retobj, 1)); 3397eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (newpos < 0 && !PyErr_Occurred()) 3407eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel newpos += (Py_ssize_t)(buf->inbuf_end - buf->inbuf_top); 3417eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (newpos < 0 || buf->inbuf_top + newpos > buf->inbuf_end) { 3427eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyErr_Clear(); 3437eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyErr_Format(PyExc_IndexError, 3447eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel "position %zd from error handler out of bounds", 3457eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel newpos); 3467eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 3477eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 3487eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel buf->inbuf = buf->inbuf_top + newpos; 3497eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 3507eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_DECREF(retobj); 3517eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_DECREF(retstr); 3527eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return 0; 3537eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 3547eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielerrorexit: 3557eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_XDECREF(retobj); 3567eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_XDECREF(retstr); 3577eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return -1; 3587eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel} 3597eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 3607eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic int 3617eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielmultibytecodec_decerror(MultibyteCodec *codec, 3627eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel MultibyteCodec_State *state, 3637eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel MultibyteDecodeBuffer *buf, 3647eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyObject *errors, Py_ssize_t e) 3657eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel{ 3667eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyObject *retobj = NULL, *retuni = NULL; 3677eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_ssize_t retunisize, newpos; 3687eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel const char *reason; 3697eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_ssize_t esize, start, end; 3707eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 3717eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (e > 0) { 3727eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel reason = "illegal multibyte sequence"; 3737eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel esize = e; 3747eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 3757eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel else { 3767eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel switch (e) { 3777eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel case MBERR_TOOSMALL: 3787eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel REQUIRE_DECODEBUFFER(buf, -1); 3797eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return 0; /* retry it */ 3807eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel case MBERR_TOOFEW: 3817eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel reason = "incomplete multibyte sequence"; 3827eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel esize = (Py_ssize_t)(buf->inbuf_end - buf->inbuf); 3837eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel break; 3847eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel case MBERR_INTERNAL: 3857eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyErr_SetString(PyExc_RuntimeError, 3867eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel "internal codec error"); 3877eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return -1; 3887eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel default: 3897eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyErr_SetString(PyExc_RuntimeError, 3907eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel "unknown runtime error"); 3917eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return -1; 3927eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 3937eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 3947eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 3957eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (errors == ERROR_REPLACE) { 3967eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel REQUIRE_DECODEBUFFER(buf, 1); 3977eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel *buf->outbuf++ = Py_UNICODE_REPLACEMENT_CHARACTER; 3987eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 3997eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (errors == ERROR_IGNORE || errors == ERROR_REPLACE) { 4007eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel buf->inbuf += esize; 4017eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return 0; 4027eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 4037eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 4047eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel start = (Py_ssize_t)(buf->inbuf - buf->inbuf_top); 4057eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel end = start + esize; 4067eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 4077eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel /* use cached exception object if available */ 4087eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (buf->excobj == NULL) { 4097eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel buf->excobj = PyUnicodeDecodeError_Create(codec->encoding, 4107eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel (const char *)buf->inbuf_top, 4117eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel (Py_ssize_t)(buf->inbuf_end - buf->inbuf_top), 4127eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel start, end, reason); 4137eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (buf->excobj == NULL) 4147eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 4157eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 4167eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel else 4177eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (PyUnicodeDecodeError_SetStart(buf->excobj, start) || 4187eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyUnicodeDecodeError_SetEnd(buf->excobj, end) || 4197eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyUnicodeDecodeError_SetReason(buf->excobj, reason)) 4207eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 4217eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 4227eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (errors == ERROR_STRICT) { 4237eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyCodec_StrictErrors(buf->excobj); 4247eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 4257eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 4267eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 4277eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel retobj = call_error_callback(errors, buf->excobj); 4287eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (retobj == NULL) 4297eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 4307eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 4317eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (!PyTuple_Check(retobj) || PyTuple_GET_SIZE(retobj) != 2 || 4327eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel !PyUnicode_Check((retuni = PyTuple_GET_ITEM(retobj, 0))) || 4337eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel !(PyInt_Check(PyTuple_GET_ITEM(retobj, 1)) || 4347eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyLong_Check(PyTuple_GET_ITEM(retobj, 1)))) { 4357eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyErr_SetString(PyExc_TypeError, 4367eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel "decoding error handler must return " 4377eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel "(unicode, int) tuple"); 4387eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 4397eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 4407eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 4417eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel retunisize = PyUnicode_GET_SIZE(retuni); 4427eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (retunisize > 0) { 4437eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel REQUIRE_DECODEBUFFER(buf, retunisize); 4447eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel memcpy((char *)buf->outbuf, PyUnicode_AS_DATA(retuni), 4457eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel retunisize * Py_UNICODE_SIZE); 4467eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel buf->outbuf += retunisize; 4477eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 4487eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 4497eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel newpos = PyInt_AsSsize_t(PyTuple_GET_ITEM(retobj, 1)); 4507eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (newpos < 0 && !PyErr_Occurred()) 4517eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel newpos += (Py_ssize_t)(buf->inbuf_end - buf->inbuf_top); 4527eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (newpos < 0 || buf->inbuf_top + newpos > buf->inbuf_end) { 4537eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyErr_Clear(); 4547eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyErr_Format(PyExc_IndexError, 4557eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel "position %zd from error handler out of bounds", 4567eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel newpos); 4577eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 4587eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 4597eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel buf->inbuf = buf->inbuf_top + newpos; 4607eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_DECREF(retobj); 4617eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return 0; 4627eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 4637eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielerrorexit: 4647eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_XDECREF(retobj); 4657eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return -1; 4667eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel} 4677eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 4687eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic PyObject * 4697eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielmultibytecodec_encode(MultibyteCodec *codec, 4707eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel MultibyteCodec_State *state, 4717eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel const Py_UNICODE **data, Py_ssize_t datalen, 4727eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyObject *errors, int flags) 4737eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel{ 4747eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel MultibyteEncodeBuffer buf; 4757eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_ssize_t finalsize, r = 0; 4767eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 4777eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (datalen == 0 && !(flags & MBENC_RESET)) 4787eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return PyString_FromString(""); 4797eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 4807eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel buf.excobj = NULL; 4817eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel buf.inbuf = buf.inbuf_top = *data; 4827eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel buf.inbuf_end = buf.inbuf_top + datalen; 4837eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 4847eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (datalen > (PY_SSIZE_T_MAX - 16) / 2) { 4857eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyErr_NoMemory(); 4867eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 4877eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 4887eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 4897eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel buf.outobj = PyString_FromStringAndSize(NULL, datalen * 2 + 16); 4907eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (buf.outobj == NULL) 4917eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 4927eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel buf.outbuf = (unsigned char *)PyString_AS_STRING(buf.outobj); 4937eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel buf.outbuf_end = buf.outbuf + PyString_GET_SIZE(buf.outobj); 4947eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 4957eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel while (buf.inbuf < buf.inbuf_end) { 4967eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_ssize_t inleft, outleft; 4977eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 4987eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel /* we don't reuse inleft and outleft here. 4997eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel * error callbacks can relocate the cursor anywhere on buffer*/ 5007eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel inleft = (Py_ssize_t)(buf.inbuf_end - buf.inbuf); 5017eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel outleft = (Py_ssize_t)(buf.outbuf_end - buf.outbuf); 5027eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel r = codec->encode(state, codec->config, &buf.inbuf, inleft, 5037eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel &buf.outbuf, outleft, flags); 5047eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if ((r == 0) || (r == MBERR_TOOFEW && !(flags & MBENC_FLUSH))) 5057eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel break; 5067eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel else if (multibytecodec_encerror(codec, state, &buf, errors,r)) 5077eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 5087eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel else if (r == MBERR_TOOFEW) 5097eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel break; 5107eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 5117eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 5127eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (codec->encreset != NULL && (flags & MBENC_RESET)) 5137eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel for (;;) { 5147eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_ssize_t outleft; 5157eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 5167eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel outleft = (Py_ssize_t)(buf.outbuf_end - buf.outbuf); 5177eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel r = codec->encreset(state, codec->config, &buf.outbuf, 5187eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel outleft); 5197eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (r == 0) 5207eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel break; 5217eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel else if (multibytecodec_encerror(codec, state, 5227eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel &buf, errors, r)) 5237eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 5247eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 5257eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 5267eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel finalsize = (Py_ssize_t)((char *)buf.outbuf - 5277eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyString_AS_STRING(buf.outobj)); 5287eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 5297eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (finalsize != PyString_GET_SIZE(buf.outobj)) 5307eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (_PyString_Resize(&buf.outobj, finalsize) == -1) 5317eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 5327eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 5337eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel *data = buf.inbuf; 5347eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_XDECREF(buf.excobj); 5357eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return buf.outobj; 5367eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 5377eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielerrorexit: 5387eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_XDECREF(buf.excobj); 5397eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_XDECREF(buf.outobj); 5407eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return NULL; 5417eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel} 5427eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 5437eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic PyObject * 5447eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielMultibyteCodec_Encode(MultibyteCodecObject *self, 5457eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyObject *args, PyObject *kwargs) 5467eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel{ 5477eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel MultibyteCodec_State state; 5487eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_UNICODE *data; 5497eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyObject *errorcb, *r, *arg, *ucvt; 5507eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel const char *errors = NULL; 5517eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_ssize_t datalen; 5527eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 5537eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|z:encode", 5547eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel codeckwarglist, &arg, &errors)) 5557eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return NULL; 5567eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 5577eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (PyUnicode_Check(arg)) 5587eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel ucvt = NULL; 5597eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel else { 5607eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel arg = ucvt = PyObject_Unicode(arg); 5617eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (arg == NULL) 5627eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return NULL; 5637eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel else if (!PyUnicode_Check(arg)) { 5647eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyErr_SetString(PyExc_TypeError, 5657eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel "couldn't convert the object to unicode."); 5667eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_DECREF(ucvt); 5677eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return NULL; 5687eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 5697eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 5707eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 5717eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel data = PyUnicode_AS_UNICODE(arg); 5727eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel datalen = PyUnicode_GET_SIZE(arg); 5737eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 5747eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel errorcb = internal_error_callback(errors); 5757eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (errorcb == NULL) { 5767eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_XDECREF(ucvt); 5777eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return NULL; 5787eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 5797eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 5807eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (self->codec->encinit != NULL && 5817eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel self->codec->encinit(&state, self->codec->config) != 0) 5827eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 5837eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel r = multibytecodec_encode(self->codec, &state, 5847eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel (const Py_UNICODE **)&data, datalen, errorcb, 5857eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel MBENC_FLUSH | MBENC_RESET); 5867eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (r == NULL) 5877eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 5887eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 5897eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel ERROR_DECREF(errorcb); 5907eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_XDECREF(ucvt); 5917eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return make_tuple(r, datalen); 5927eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 5937eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielerrorexit: 5947eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel ERROR_DECREF(errorcb); 5957eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_XDECREF(ucvt); 5967eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return NULL; 5977eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel} 5987eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 5997eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic PyObject * 6007eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielMultibyteCodec_Decode(MultibyteCodecObject *self, 6017eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyObject *args, PyObject *kwargs) 6027eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel{ 6037eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel MultibyteCodec_State state; 6047eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel MultibyteDecodeBuffer buf; 6057eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyObject *errorcb; 6067eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_buffer pdata; 6077eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel const char *data, *errors = NULL; 6087eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_ssize_t datalen, finalsize; 6097eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 6107eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s*|z:decode", 6117eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel codeckwarglist, &pdata, &errors)) 6127eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return NULL; 6137eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel data = pdata.buf; 6147eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel datalen = pdata.len; 6157eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 6167eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel errorcb = internal_error_callback(errors); 6177eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (errorcb == NULL) { 6187eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyBuffer_Release(&pdata); 6197eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return NULL; 6207eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 6217eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 6227eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (datalen == 0) { 6237eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyBuffer_Release(&pdata); 6247eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel ERROR_DECREF(errorcb); 6257eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return make_tuple(PyUnicode_FromUnicode(NULL, 0), 0); 6267eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 6277eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 6287eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel buf.excobj = NULL; 6297eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel buf.inbuf = buf.inbuf_top = (unsigned char *)data; 6307eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel buf.inbuf_end = buf.inbuf_top + datalen; 6317eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel buf.outobj = PyUnicode_FromUnicode(NULL, datalen); 6327eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (buf.outobj == NULL) 6337eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 6347eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel buf.outbuf = PyUnicode_AS_UNICODE(buf.outobj); 6357eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel buf.outbuf_end = buf.outbuf + PyUnicode_GET_SIZE(buf.outobj); 6367eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 6377eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (self->codec->decinit != NULL && 6387eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel self->codec->decinit(&state, self->codec->config) != 0) 6397eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 6407eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 6417eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel while (buf.inbuf < buf.inbuf_end) { 6427eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_ssize_t inleft, outleft, r; 6437eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 6447eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel inleft = (Py_ssize_t)(buf.inbuf_end - buf.inbuf); 6457eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel outleft = (Py_ssize_t)(buf.outbuf_end - buf.outbuf); 6467eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 6477eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel r = self->codec->decode(&state, self->codec->config, 6487eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel &buf.inbuf, inleft, &buf.outbuf, outleft); 6497eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (r == 0) 6507eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel break; 6517eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel else if (multibytecodec_decerror(self->codec, &state, 6527eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel &buf, errorcb, r)) 6537eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 6547eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 6557eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 6567eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel finalsize = (Py_ssize_t)(buf.outbuf - 6577eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyUnicode_AS_UNICODE(buf.outobj)); 6587eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 6597eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (finalsize != PyUnicode_GET_SIZE(buf.outobj)) 6607eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (PyUnicode_Resize(&buf.outobj, finalsize) == -1) 6617eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 6627eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 6637eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyBuffer_Release(&pdata); 6647eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_XDECREF(buf.excobj); 6657eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel ERROR_DECREF(errorcb); 6667eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return make_tuple(buf.outobj, datalen); 6677eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 6687eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielerrorexit: 6697eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyBuffer_Release(&pdata); 6707eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel ERROR_DECREF(errorcb); 6717eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_XDECREF(buf.excobj); 6727eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_XDECREF(buf.outobj); 6737eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 6747eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return NULL; 6757eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel} 6767eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 6777eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic struct PyMethodDef multibytecodec_methods[] = { 6787eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel {"encode", (PyCFunction)MultibyteCodec_Encode, 6797eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel METH_VARARGS | METH_KEYWORDS, 6807eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel MultibyteCodec_Encode__doc__}, 6817eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel {"decode", (PyCFunction)MultibyteCodec_Decode, 6827eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel METH_VARARGS | METH_KEYWORDS, 6837eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel MultibyteCodec_Decode__doc__}, 6847eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel {NULL, NULL}, 6857eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel}; 6867eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 6877eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic void 6887eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielmultibytecodec_dealloc(MultibyteCodecObject *self) 6897eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel{ 6907eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyObject_Del(self); 6917eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel} 6927eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 6937eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic PyTypeObject MultibyteCodec_Type = { 6947eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyVarObject_HEAD_INIT(NULL, 0) 6957eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel "MultibyteCodec", /* tp_name */ 6967eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel sizeof(MultibyteCodecObject), /* tp_basicsize */ 6977eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_itemsize */ 6987eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel /* methods */ 6997eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel (destructor)multibytecodec_dealloc, /* tp_dealloc */ 7007eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_print */ 7017eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_getattr */ 7027eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_setattr */ 7037eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_compare */ 7047eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_repr */ 7057eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_as_number */ 7067eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_as_sequence */ 7077eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_as_mapping */ 7087eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_hash */ 7097eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_call */ 7107eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_str */ 7117eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyObject_GenericGetAttr, /* tp_getattro */ 7127eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_setattro */ 7137eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_as_buffer */ 7147eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_TPFLAGS_DEFAULT, /* tp_flags */ 7157eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_doc */ 7167eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_traverse */ 7177eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_clear */ 7187eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_richcompare */ 7197eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_weaklistoffset */ 7207eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_iter */ 7217eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_iterext */ 7227eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel multibytecodec_methods, /* tp_methods */ 7237eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel}; 7247eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 7257eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 7267eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel/** 7277eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel * Utility functions for stateful codec mechanism 7287eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel */ 7297eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 7307eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel#define STATEFUL_DCTX(o) ((MultibyteStatefulDecoderContext *)(o)) 7317eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel#define STATEFUL_ECTX(o) ((MultibyteStatefulEncoderContext *)(o)) 7327eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 7337eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic PyObject * 7347eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielencoder_encode_stateful(MultibyteStatefulEncoderContext *ctx, 7357eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyObject *unistr, int final) 7367eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel{ 7377eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyObject *ucvt, *r = NULL; 7387eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_UNICODE *inbuf, *inbuf_end, *inbuf_tmp = NULL; 7397eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_ssize_t datalen, origpending; 7407eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 7417eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (PyUnicode_Check(unistr)) 7427eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel ucvt = NULL; 7437eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel else { 7447eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel unistr = ucvt = PyObject_Unicode(unistr); 7457eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (unistr == NULL) 7467eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return NULL; 7477eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel else if (!PyUnicode_Check(unistr)) { 7487eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyErr_SetString(PyExc_TypeError, 7497eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel "couldn't convert the object to unicode."); 7507eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_DECREF(ucvt); 7517eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return NULL; 7527eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 7537eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 7547eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 7557eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel datalen = PyUnicode_GET_SIZE(unistr); 7567eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel origpending = ctx->pendingsize; 7577eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 7587eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (origpending > 0) { 7597eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (datalen > PY_SSIZE_T_MAX - ctx->pendingsize) { 7607eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyErr_NoMemory(); 7617eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel /* inbuf_tmp == NULL */ 7627eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 7637eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 7647eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel inbuf_tmp = PyMem_New(Py_UNICODE, datalen + ctx->pendingsize); 7657eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (inbuf_tmp == NULL) 7667eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 7677eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel memcpy(inbuf_tmp, ctx->pending, 7687eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_UNICODE_SIZE * ctx->pendingsize); 7697eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel memcpy(inbuf_tmp + ctx->pendingsize, 7707eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyUnicode_AS_UNICODE(unistr), 7717eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_UNICODE_SIZE * datalen); 7727eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel datalen += ctx->pendingsize; 7737eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel ctx->pendingsize = 0; 7747eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel inbuf = inbuf_tmp; 7757eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 7767eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel else 7777eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel inbuf = (Py_UNICODE *)PyUnicode_AS_UNICODE(unistr); 7787eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 7797eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel inbuf_end = inbuf + datalen; 7807eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 7817eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel r = multibytecodec_encode(ctx->codec, &ctx->state, 7827eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel (const Py_UNICODE **)&inbuf, datalen, 7837eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel ctx->errors, final ? MBENC_FLUSH | MBENC_RESET : 0); 7847eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (r == NULL) { 7857eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel /* recover the original pending buffer */ 7867eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (origpending > 0) 7877eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel memcpy(ctx->pending, inbuf_tmp, 7887eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_UNICODE_SIZE * origpending); 7897eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel ctx->pendingsize = origpending; 7907eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 7917eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 7927eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 7937eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (inbuf < inbuf_end) { 7947eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel ctx->pendingsize = (Py_ssize_t)(inbuf_end - inbuf); 7957eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (ctx->pendingsize > MAXENCPENDING) { 7967eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel /* normal codecs can't reach here */ 7977eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel ctx->pendingsize = 0; 7987eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyErr_SetString(PyExc_UnicodeError, 7997eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel "pending buffer overflow"); 8007eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 8017eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 8027eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel memcpy(ctx->pending, inbuf, 8037eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel ctx->pendingsize * Py_UNICODE_SIZE); 8047eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 8057eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 8067eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (inbuf_tmp != NULL) 8077eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyMem_Del(inbuf_tmp); 8087eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_XDECREF(ucvt); 8097eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return r; 8107eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 8117eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielerrorexit: 8127eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (inbuf_tmp != NULL) 8137eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyMem_Del(inbuf_tmp); 8147eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_XDECREF(r); 8157eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_XDECREF(ucvt); 8167eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return NULL; 8177eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel} 8187eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 8197eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic int 8207eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanieldecoder_append_pending(MultibyteStatefulDecoderContext *ctx, 8217eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel MultibyteDecodeBuffer *buf) 8227eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel{ 8237eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_ssize_t npendings; 8247eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 8257eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel npendings = (Py_ssize_t)(buf->inbuf_end - buf->inbuf); 8267eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (npendings + ctx->pendingsize > MAXDECPENDING || 8277eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel npendings > PY_SSIZE_T_MAX - ctx->pendingsize) { 8287eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyErr_SetString(PyExc_UnicodeError, "pending buffer overflow"); 8297eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return -1; 8307eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 8317eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel memcpy(ctx->pending + ctx->pendingsize, buf->inbuf, npendings); 8327eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel ctx->pendingsize += npendings; 8337eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return 0; 8347eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel} 8357eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 8367eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic int 8377eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanieldecoder_prepare_buffer(MultibyteDecodeBuffer *buf, const char *data, 8387eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_ssize_t size) 8397eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel{ 8407eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel buf->inbuf = buf->inbuf_top = (const unsigned char *)data; 8417eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel buf->inbuf_end = buf->inbuf_top + size; 8427eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (buf->outobj == NULL) { /* only if outobj is not allocated yet */ 8437eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel buf->outobj = PyUnicode_FromUnicode(NULL, size); 8447eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (buf->outobj == NULL) 8457eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return -1; 8467eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel buf->outbuf = PyUnicode_AS_UNICODE(buf->outobj); 8477eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel buf->outbuf_end = buf->outbuf + 8487eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyUnicode_GET_SIZE(buf->outobj); 8497eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 8507eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 8517eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return 0; 8527eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel} 8537eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 8547eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic int 8557eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanieldecoder_feed_buffer(MultibyteStatefulDecoderContext *ctx, 8567eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel MultibyteDecodeBuffer *buf) 8577eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel{ 8587eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel while (buf->inbuf < buf->inbuf_end) { 8597eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_ssize_t inleft, outleft; 8607eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_ssize_t r; 8617eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 8627eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel inleft = (Py_ssize_t)(buf->inbuf_end - buf->inbuf); 8637eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel outleft = (Py_ssize_t)(buf->outbuf_end - buf->outbuf); 8647eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 8657eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel r = ctx->codec->decode(&ctx->state, ctx->codec->config, 8667eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel &buf->inbuf, inleft, &buf->outbuf, outleft); 8677eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (r == 0 || r == MBERR_TOOFEW) 8687eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel break; 8697eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel else if (multibytecodec_decerror(ctx->codec, &ctx->state, 8707eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel buf, ctx->errors, r)) 8717eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return -1; 8727eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 8737eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return 0; 8747eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel} 8757eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 8767eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 8777eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel/** 8787eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel * MultibyteIncrementalEncoder object 8797eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel */ 8807eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 8817eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic PyObject * 8827eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielmbiencoder_encode(MultibyteIncrementalEncoderObject *self, 8837eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyObject *args, PyObject *kwargs) 8847eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel{ 8857eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyObject *data; 8867eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel int final = 0; 8877eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 8887eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|i:encode", 8897eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel incrementalkwarglist, &data, &final)) 8907eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return NULL; 8917eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 8927eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return encoder_encode_stateful(STATEFUL_ECTX(self), data, final); 8937eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel} 8947eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 8957eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic PyObject * 8967eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielmbiencoder_reset(MultibyteIncrementalEncoderObject *self) 8977eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel{ 8987eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (self->codec->decreset != NULL && 8997eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel self->codec->decreset(&self->state, self->codec->config) != 0) 9007eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return NULL; 9017eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel self->pendingsize = 0; 9027eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 9037eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_RETURN_NONE; 9047eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel} 9057eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 9067eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic struct PyMethodDef mbiencoder_methods[] = { 9077eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel {"encode", (PyCFunction)mbiencoder_encode, 9087eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel METH_VARARGS | METH_KEYWORDS, NULL}, 9097eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel {"reset", (PyCFunction)mbiencoder_reset, 9107eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel METH_NOARGS, NULL}, 9117eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel {NULL, NULL}, 9127eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel}; 9137eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 9147eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic PyObject * 9157eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielmbiencoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds) 9167eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel{ 9177eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel MultibyteIncrementalEncoderObject *self; 9187eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyObject *codec = NULL; 9197eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel char *errors = NULL; 9207eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 9217eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (!PyArg_ParseTupleAndKeywords(args, kwds, "|s:IncrementalEncoder", 9227eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel incnewkwarglist, &errors)) 9237eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return NULL; 9247eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 9257eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel self = (MultibyteIncrementalEncoderObject *)type->tp_alloc(type, 0); 9267eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (self == NULL) 9277eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return NULL; 9287eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 9297eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel codec = PyObject_GetAttrString((PyObject *)type, "codec"); 9307eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (codec == NULL) 9317eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 9327eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (!MultibyteCodec_Check(codec)) { 9337eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyErr_SetString(PyExc_TypeError, "codec is unexpected type"); 9347eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 9357eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 9367eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 9377eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel self->codec = ((MultibyteCodecObject *)codec)->codec; 9387eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel self->pendingsize = 0; 9397eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel self->errors = internal_error_callback(errors); 9407eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (self->errors == NULL) 9417eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 9427eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (self->codec->encinit != NULL && 9437eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel self->codec->encinit(&self->state, self->codec->config) != 0) 9447eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 9457eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 9467eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_DECREF(codec); 9477eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return (PyObject *)self; 9487eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 9497eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielerrorexit: 9507eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_XDECREF(self); 9517eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_XDECREF(codec); 9527eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return NULL; 9537eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel} 9547eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 9557eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic int 9567eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielmbiencoder_init(PyObject *self, PyObject *args, PyObject *kwds) 9577eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel{ 9587eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return 0; 9597eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel} 9607eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 9617eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic int 9627eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielmbiencoder_traverse(MultibyteIncrementalEncoderObject *self, 9637eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel visitproc visit, void *arg) 9647eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel{ 9657eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (ERROR_ISCUSTOM(self->errors)) 9667eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_VISIT(self->errors); 9677eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return 0; 9687eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel} 9697eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 9707eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic void 9717eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielmbiencoder_dealloc(MultibyteIncrementalEncoderObject *self) 9727eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel{ 9737eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyObject_GC_UnTrack(self); 9747eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel ERROR_DECREF(self->errors); 9757eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_TYPE(self)->tp_free(self); 9767eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel} 9777eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 9787eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic PyTypeObject MultibyteIncrementalEncoder_Type = { 9797eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyVarObject_HEAD_INIT(NULL, 0) 9807eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel "MultibyteIncrementalEncoder", /* tp_name */ 9817eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel sizeof(MultibyteIncrementalEncoderObject), /* tp_basicsize */ 9827eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_itemsize */ 9837eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel /* methods */ 9847eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel (destructor)mbiencoder_dealloc, /* tp_dealloc */ 9857eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_print */ 9867eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_getattr */ 9877eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_setattr */ 9887eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_compare */ 9897eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_repr */ 9907eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_as_number */ 9917eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_as_sequence */ 9927eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_as_mapping */ 9937eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_hash */ 9947eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_call */ 9957eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_str */ 9967eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyObject_GenericGetAttr, /* tp_getattro */ 9977eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_setattro */ 9987eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_as_buffer */ 9997eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC 10007eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel | Py_TPFLAGS_BASETYPE, /* tp_flags */ 10017eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_doc */ 10027eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel (traverseproc)mbiencoder_traverse, /* tp_traverse */ 10037eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_clear */ 10047eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_richcompare */ 10057eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_weaklistoffset */ 10067eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_iter */ 10077eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_iterext */ 10087eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel mbiencoder_methods, /* tp_methods */ 10097eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_members */ 10107eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel codecctx_getsets, /* tp_getset */ 10117eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_base */ 10127eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_dict */ 10137eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_descr_get */ 10147eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_descr_set */ 10157eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_dictoffset */ 10167eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel mbiencoder_init, /* tp_init */ 10177eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_alloc */ 10187eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel mbiencoder_new, /* tp_new */ 10197eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel}; 10207eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 10217eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 10227eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel/** 10237eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel * MultibyteIncrementalDecoder object 10247eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel */ 10257eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 10267eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic PyObject * 10277eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielmbidecoder_decode(MultibyteIncrementalDecoderObject *self, 10287eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyObject *args, PyObject *kwargs) 10297eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel{ 10307eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel MultibyteDecodeBuffer buf; 10317eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel char *data, *wdata = NULL; 10327eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_buffer pdata; 10337eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_ssize_t wsize, finalsize = 0, size, origpending; 10347eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel int final = 0; 10357eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 10367eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s*|i:decode", 10377eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel incrementalkwarglist, &pdata, &final)) 10387eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return NULL; 10397eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel data = pdata.buf; 10407eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel size = pdata.len; 10417eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 10427eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel buf.outobj = buf.excobj = NULL; 10437eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel origpending = self->pendingsize; 10447eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 10457eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (self->pendingsize == 0) { 10467eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel wsize = size; 10477eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel wdata = data; 10487eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 10497eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel else { 10507eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (size > PY_SSIZE_T_MAX - self->pendingsize) { 10517eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyErr_NoMemory(); 10527eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 10537eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 10547eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel wsize = size + self->pendingsize; 10557eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel wdata = PyMem_Malloc(wsize); 10567eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (wdata == NULL) 10577eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 10587eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel memcpy(wdata, self->pending, self->pendingsize); 10597eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel memcpy(wdata + self->pendingsize, data, size); 10607eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel self->pendingsize = 0; 10617eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 10627eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 10637eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (decoder_prepare_buffer(&buf, wdata, wsize) != 0) 10647eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 10657eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 10667eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (decoder_feed_buffer(STATEFUL_DCTX(self), &buf)) 10677eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 10687eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 10697eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (final && buf.inbuf < buf.inbuf_end) { 10707eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (multibytecodec_decerror(self->codec, &self->state, 10717eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel &buf, self->errors, MBERR_TOOFEW)) { 10727eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel /* recover the original pending buffer */ 10737eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel memcpy(self->pending, wdata, origpending); 10747eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel self->pendingsize = origpending; 10757eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 10767eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 10777eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 10787eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 10797eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (buf.inbuf < buf.inbuf_end) { /* pending sequence still exists */ 10807eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (decoder_append_pending(STATEFUL_DCTX(self), &buf) != 0) 10817eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 10827eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 10837eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 10847eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel finalsize = (Py_ssize_t)(buf.outbuf - PyUnicode_AS_UNICODE(buf.outobj)); 10857eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (finalsize != PyUnicode_GET_SIZE(buf.outobj)) 10867eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (PyUnicode_Resize(&buf.outobj, finalsize) == -1) 10877eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 10887eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 10897eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyBuffer_Release(&pdata); 10907eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (wdata != data) 10917eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyMem_Del(wdata); 10927eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_XDECREF(buf.excobj); 10937eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return buf.outobj; 10947eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 10957eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielerrorexit: 10967eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyBuffer_Release(&pdata); 10977eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (wdata != NULL && wdata != data) 10987eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyMem_Del(wdata); 10997eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_XDECREF(buf.excobj); 11007eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_XDECREF(buf.outobj); 11017eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return NULL; 11027eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel} 11037eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 11047eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic PyObject * 11057eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielmbidecoder_reset(MultibyteIncrementalDecoderObject *self) 11067eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel{ 11077eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (self->codec->decreset != NULL && 11087eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel self->codec->decreset(&self->state, self->codec->config) != 0) 11097eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return NULL; 11107eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel self->pendingsize = 0; 11117eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 11127eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_RETURN_NONE; 11137eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel} 11147eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 11157eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic struct PyMethodDef mbidecoder_methods[] = { 11167eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel {"decode", (PyCFunction)mbidecoder_decode, 11177eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel METH_VARARGS | METH_KEYWORDS, NULL}, 11187eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel {"reset", (PyCFunction)mbidecoder_reset, 11197eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel METH_NOARGS, NULL}, 11207eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel {NULL, NULL}, 11217eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel}; 11227eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 11237eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic PyObject * 11247eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielmbidecoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds) 11257eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel{ 11267eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel MultibyteIncrementalDecoderObject *self; 11277eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyObject *codec = NULL; 11287eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel char *errors = NULL; 11297eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 11307eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (!PyArg_ParseTupleAndKeywords(args, kwds, "|s:IncrementalDecoder", 11317eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel incnewkwarglist, &errors)) 11327eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return NULL; 11337eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 11347eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel self = (MultibyteIncrementalDecoderObject *)type->tp_alloc(type, 0); 11357eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (self == NULL) 11367eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return NULL; 11377eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 11387eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel codec = PyObject_GetAttrString((PyObject *)type, "codec"); 11397eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (codec == NULL) 11407eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 11417eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (!MultibyteCodec_Check(codec)) { 11427eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyErr_SetString(PyExc_TypeError, "codec is unexpected type"); 11437eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 11447eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 11457eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 11467eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel self->codec = ((MultibyteCodecObject *)codec)->codec; 11477eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel self->pendingsize = 0; 11487eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel self->errors = internal_error_callback(errors); 11497eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (self->errors == NULL) 11507eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 11517eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (self->codec->decinit != NULL && 11527eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel self->codec->decinit(&self->state, self->codec->config) != 0) 11537eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 11547eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 11557eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_DECREF(codec); 11567eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return (PyObject *)self; 11577eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 11587eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielerrorexit: 11597eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_XDECREF(self); 11607eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_XDECREF(codec); 11617eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return NULL; 11627eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel} 11637eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 11647eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic int 11657eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielmbidecoder_init(PyObject *self, PyObject *args, PyObject *kwds) 11667eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel{ 11677eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return 0; 11687eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel} 11697eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 11707eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic int 11717eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielmbidecoder_traverse(MultibyteIncrementalDecoderObject *self, 11727eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel visitproc visit, void *arg) 11737eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel{ 11747eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (ERROR_ISCUSTOM(self->errors)) 11757eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_VISIT(self->errors); 11767eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return 0; 11777eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel} 11787eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 11797eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic void 11807eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielmbidecoder_dealloc(MultibyteIncrementalDecoderObject *self) 11817eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel{ 11827eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyObject_GC_UnTrack(self); 11837eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel ERROR_DECREF(self->errors); 11847eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_TYPE(self)->tp_free(self); 11857eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel} 11867eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 11877eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic PyTypeObject MultibyteIncrementalDecoder_Type = { 11887eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyVarObject_HEAD_INIT(NULL, 0) 11897eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel "MultibyteIncrementalDecoder", /* tp_name */ 11907eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel sizeof(MultibyteIncrementalDecoderObject), /* tp_basicsize */ 11917eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_itemsize */ 11927eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel /* methods */ 11937eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel (destructor)mbidecoder_dealloc, /* tp_dealloc */ 11947eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_print */ 11957eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_getattr */ 11967eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_setattr */ 11977eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_compare */ 11987eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_repr */ 11997eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_as_number */ 12007eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_as_sequence */ 12017eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_as_mapping */ 12027eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_hash */ 12037eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_call */ 12047eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_str */ 12057eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyObject_GenericGetAttr, /* tp_getattro */ 12067eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_setattro */ 12077eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_as_buffer */ 12087eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC 12097eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel | Py_TPFLAGS_BASETYPE, /* tp_flags */ 12107eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_doc */ 12117eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel (traverseproc)mbidecoder_traverse, /* tp_traverse */ 12127eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_clear */ 12137eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_richcompare */ 12147eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_weaklistoffset */ 12157eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_iter */ 12167eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_iterext */ 12177eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel mbidecoder_methods, /* tp_methods */ 12187eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_members */ 12197eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel codecctx_getsets, /* tp_getset */ 12207eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_base */ 12217eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_dict */ 12227eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_descr_get */ 12237eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_descr_set */ 12247eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_dictoffset */ 12257eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel mbidecoder_init, /* tp_init */ 12267eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_alloc */ 12277eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel mbidecoder_new, /* tp_new */ 12287eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel}; 12297eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 12307eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 12317eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel/** 12327eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel * MultibyteStreamReader object 12337eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel */ 12347eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 12357eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic PyObject * 12367eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielmbstreamreader_iread(MultibyteStreamReaderObject *self, 12377eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel const char *method, Py_ssize_t sizehint) 12387eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel{ 12397eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel MultibyteDecodeBuffer buf; 12407eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyObject *cres; 12417eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_ssize_t rsize, finalsize = 0; 12427eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 12437eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (sizehint == 0) 12447eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return PyUnicode_FromUnicode(NULL, 0); 12457eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 12467eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel buf.outobj = buf.excobj = NULL; 12477eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel cres = NULL; 12487eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 12497eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel for (;;) { 12507eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel int endoffile; 12517eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 12527eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (sizehint < 0) 12537eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel cres = PyObject_CallMethod(self->stream, 12547eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel (char *)method, NULL); 12557eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel else 12567eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel cres = PyObject_CallMethod(self->stream, 12577eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel (char *)method, "i", sizehint); 12587eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (cres == NULL) 12597eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 12607eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 12617eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (!PyString_Check(cres)) { 12627eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyErr_SetString(PyExc_TypeError, 12637eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel "stream function returned a " 12647eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel "non-string object"); 12657eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 12667eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 12677eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 12687eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel endoffile = (PyString_GET_SIZE(cres) == 0); 12697eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 12707eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (self->pendingsize > 0) { 12717eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyObject *ctr; 12727eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel char *ctrdata; 12737eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 12747eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (PyString_GET_SIZE(cres) > PY_SSIZE_T_MAX - self->pendingsize) { 12757eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyErr_NoMemory(); 12767eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 12777eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 12787eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel rsize = PyString_GET_SIZE(cres) + self->pendingsize; 12797eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel ctr = PyString_FromStringAndSize(NULL, rsize); 12807eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (ctr == NULL) 12817eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 12827eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel ctrdata = PyString_AS_STRING(ctr); 12837eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel memcpy(ctrdata, self->pending, self->pendingsize); 12847eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel memcpy(ctrdata + self->pendingsize, 12857eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyString_AS_STRING(cres), 12867eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyString_GET_SIZE(cres)); 12877eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_DECREF(cres); 12887eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel cres = ctr; 12897eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel self->pendingsize = 0; 12907eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 12917eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 12927eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel rsize = PyString_GET_SIZE(cres); 12937eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (decoder_prepare_buffer(&buf, PyString_AS_STRING(cres), 12947eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel rsize) != 0) 12957eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 12967eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 12977eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (rsize > 0 && decoder_feed_buffer( 12987eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel (MultibyteStatefulDecoderContext *)self, &buf)) 12997eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 13007eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 13017eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (endoffile || sizehint < 0) { 13027eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (buf.inbuf < buf.inbuf_end && 13037eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel multibytecodec_decerror(self->codec, &self->state, 13047eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel &buf, self->errors, MBERR_TOOFEW)) 13057eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 13067eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 13077eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 13087eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (buf.inbuf < buf.inbuf_end) { /* pending sequence exists */ 13097eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (decoder_append_pending(STATEFUL_DCTX(self), 13107eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel &buf) != 0) 13117eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 13127eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 13137eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 13147eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel finalsize = (Py_ssize_t)(buf.outbuf - 13157eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyUnicode_AS_UNICODE(buf.outobj)); 13167eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_DECREF(cres); 13177eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel cres = NULL; 13187eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 13197eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (sizehint < 0 || finalsize != 0 || rsize == 0) 13207eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel break; 13217eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 13227eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel sizehint = 1; /* read 1 more byte and retry */ 13237eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 13247eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 13257eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (finalsize != PyUnicode_GET_SIZE(buf.outobj)) 13267eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (PyUnicode_Resize(&buf.outobj, finalsize) == -1) 13277eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 13287eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 13297eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_XDECREF(cres); 13307eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_XDECREF(buf.excobj); 13317eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return buf.outobj; 13327eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 13337eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielerrorexit: 13347eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_XDECREF(cres); 13357eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_XDECREF(buf.excobj); 13367eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_XDECREF(buf.outobj); 13377eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return NULL; 13387eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel} 13397eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 13407eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic PyObject * 13417eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielmbstreamreader_read(MultibyteStreamReaderObject *self, PyObject *args) 13427eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel{ 13437eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyObject *sizeobj = NULL; 13447eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_ssize_t size; 13457eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 13467eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (!PyArg_UnpackTuple(args, "read", 0, 1, &sizeobj)) 13477eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return NULL; 13487eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 13497eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (sizeobj == Py_None || sizeobj == NULL) 13507eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel size = -1; 13517eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel else if (PyInt_Check(sizeobj)) 13527eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel size = PyInt_AsSsize_t(sizeobj); 13537eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel else { 13547eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyErr_SetString(PyExc_TypeError, "arg 1 must be an integer"); 13557eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return NULL; 13567eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 13577eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 13587eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return mbstreamreader_iread(self, "read", size); 13597eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel} 13607eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 13617eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic PyObject * 13627eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielmbstreamreader_readline(MultibyteStreamReaderObject *self, PyObject *args) 13637eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel{ 13647eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyObject *sizeobj = NULL; 13657eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_ssize_t size; 13667eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 13677eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (!PyArg_UnpackTuple(args, "readline", 0, 1, &sizeobj)) 13687eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return NULL; 13697eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 13707eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (sizeobj == Py_None || sizeobj == NULL) 13717eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel size = -1; 13727eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel else if (PyInt_Check(sizeobj)) 13737eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel size = PyInt_AsSsize_t(sizeobj); 13747eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel else { 13757eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyErr_SetString(PyExc_TypeError, "arg 1 must be an integer"); 13767eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return NULL; 13777eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 13787eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 13797eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return mbstreamreader_iread(self, "readline", size); 13807eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel} 13817eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 13827eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic PyObject * 13837eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielmbstreamreader_readlines(MultibyteStreamReaderObject *self, PyObject *args) 13847eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel{ 13857eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyObject *sizehintobj = NULL, *r, *sr; 13867eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_ssize_t sizehint; 13877eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 13887eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (!PyArg_UnpackTuple(args, "readlines", 0, 1, &sizehintobj)) 13897eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return NULL; 13907eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 13917eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (sizehintobj == Py_None || sizehintobj == NULL) 13927eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel sizehint = -1; 13937eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel else if (PyInt_Check(sizehintobj)) 13947eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel sizehint = PyInt_AsSsize_t(sizehintobj); 13957eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel else { 13967eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyErr_SetString(PyExc_TypeError, "arg 1 must be an integer"); 13977eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return NULL; 13987eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 13997eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 14007eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel r = mbstreamreader_iread(self, "read", sizehint); 14017eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (r == NULL) 14027eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return NULL; 14037eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 14047eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel sr = PyUnicode_Splitlines(r, 1); 14057eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_DECREF(r); 14067eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return sr; 14077eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel} 14087eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 14097eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic PyObject * 14107eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielmbstreamreader_reset(MultibyteStreamReaderObject *self) 14117eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel{ 14127eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (self->codec->decreset != NULL && 14137eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel self->codec->decreset(&self->state, self->codec->config) != 0) 14147eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return NULL; 14157eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel self->pendingsize = 0; 14167eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 14177eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_RETURN_NONE; 14187eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel} 14197eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 14207eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic struct PyMethodDef mbstreamreader_methods[] = { 14217eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel {"read", (PyCFunction)mbstreamreader_read, 14227eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel METH_VARARGS, NULL}, 14237eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel {"readline", (PyCFunction)mbstreamreader_readline, 14247eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel METH_VARARGS, NULL}, 14257eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel {"readlines", (PyCFunction)mbstreamreader_readlines, 14267eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel METH_VARARGS, NULL}, 14277eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel {"reset", (PyCFunction)mbstreamreader_reset, 14287eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel METH_NOARGS, NULL}, 14297eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel {NULL, NULL}, 14307eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel}; 14317eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 14327eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic PyMemberDef mbstreamreader_members[] = { 14337eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel {"stream", T_OBJECT, 14347eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel offsetof(MultibyteStreamReaderObject, stream), 14357eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel READONLY, NULL}, 14367eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel {NULL,} 14377eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel}; 14387eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 14397eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic PyObject * 14407eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielmbstreamreader_new(PyTypeObject *type, PyObject *args, PyObject *kwds) 14417eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel{ 14427eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel MultibyteStreamReaderObject *self; 14437eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyObject *stream, *codec = NULL; 14447eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel char *errors = NULL; 14457eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 14467eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|s:StreamReader", 14477eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel streamkwarglist, &stream, &errors)) 14487eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return NULL; 14497eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 14507eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel self = (MultibyteStreamReaderObject *)type->tp_alloc(type, 0); 14517eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (self == NULL) 14527eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return NULL; 14537eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 14547eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel codec = PyObject_GetAttrString((PyObject *)type, "codec"); 14557eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (codec == NULL) 14567eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 14577eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (!MultibyteCodec_Check(codec)) { 14587eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyErr_SetString(PyExc_TypeError, "codec is unexpected type"); 14597eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 14607eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 14617eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 14627eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel self->codec = ((MultibyteCodecObject *)codec)->codec; 14637eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel self->stream = stream; 14647eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_INCREF(stream); 14657eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel self->pendingsize = 0; 14667eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel self->errors = internal_error_callback(errors); 14677eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (self->errors == NULL) 14687eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 14697eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (self->codec->decinit != NULL && 14707eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel self->codec->decinit(&self->state, self->codec->config) != 0) 14717eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 14727eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 14737eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_DECREF(codec); 14747eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return (PyObject *)self; 14757eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 14767eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielerrorexit: 14777eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_XDECREF(self); 14787eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_XDECREF(codec); 14797eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return NULL; 14807eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel} 14817eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 14827eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic int 14837eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielmbstreamreader_init(PyObject *self, PyObject *args, PyObject *kwds) 14847eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel{ 14857eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return 0; 14867eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel} 14877eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 14887eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic int 14897eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielmbstreamreader_traverse(MultibyteStreamReaderObject *self, 14907eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel visitproc visit, void *arg) 14917eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel{ 14927eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (ERROR_ISCUSTOM(self->errors)) 14937eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_VISIT(self->errors); 14947eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_VISIT(self->stream); 14957eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return 0; 14967eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel} 14977eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 14987eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic void 14997eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielmbstreamreader_dealloc(MultibyteStreamReaderObject *self) 15007eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel{ 15017eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyObject_GC_UnTrack(self); 15027eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel ERROR_DECREF(self->errors); 15037eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_XDECREF(self->stream); 15047eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_TYPE(self)->tp_free(self); 15057eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel} 15067eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 15077eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic PyTypeObject MultibyteStreamReader_Type = { 15087eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyVarObject_HEAD_INIT(NULL, 0) 15097eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel "MultibyteStreamReader", /* tp_name */ 15107eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel sizeof(MultibyteStreamReaderObject), /* tp_basicsize */ 15117eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_itemsize */ 15127eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel /* methods */ 15137eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel (destructor)mbstreamreader_dealloc, /* tp_dealloc */ 15147eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_print */ 15157eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_getattr */ 15167eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_setattr */ 15177eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_compare */ 15187eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_repr */ 15197eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_as_number */ 15207eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_as_sequence */ 15217eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_as_mapping */ 15227eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_hash */ 15237eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_call */ 15247eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_str */ 15257eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyObject_GenericGetAttr, /* tp_getattro */ 15267eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_setattro */ 15277eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_as_buffer */ 15287eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC 15297eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel | Py_TPFLAGS_BASETYPE, /* tp_flags */ 15307eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_doc */ 15317eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel (traverseproc)mbstreamreader_traverse, /* tp_traverse */ 15327eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_clear */ 15337eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_richcompare */ 15347eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_weaklistoffset */ 15357eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_iter */ 15367eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_iterext */ 15377eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel mbstreamreader_methods, /* tp_methods */ 15387eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel mbstreamreader_members, /* tp_members */ 15397eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel codecctx_getsets, /* tp_getset */ 15407eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_base */ 15417eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_dict */ 15427eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_descr_get */ 15437eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_descr_set */ 15447eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_dictoffset */ 15457eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel mbstreamreader_init, /* tp_init */ 15467eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_alloc */ 15477eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel mbstreamreader_new, /* tp_new */ 15487eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel}; 15497eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 15507eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 15517eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel/** 15527eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel * MultibyteStreamWriter object 15537eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel */ 15547eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 15557eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic int 15567eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielmbstreamwriter_iwrite(MultibyteStreamWriterObject *self, 15577eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyObject *unistr) 15587eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel{ 15597eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyObject *str, *wr; 15607eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 15617eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel str = encoder_encode_stateful(STATEFUL_ECTX(self), unistr, 0); 15627eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (str == NULL) 15637eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return -1; 15647eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 15657eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel wr = PyObject_CallMethod(self->stream, "write", "O", str); 15667eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_DECREF(str); 15677eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (wr == NULL) 15687eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return -1; 15697eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 15707eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_DECREF(wr); 15717eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return 0; 15727eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel} 15737eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 15747eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic PyObject * 15757eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielmbstreamwriter_write(MultibyteStreamWriterObject *self, PyObject *strobj) 15767eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel{ 15777eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (mbstreamwriter_iwrite(self, strobj)) 15787eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return NULL; 15797eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel else 15807eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_RETURN_NONE; 15817eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel} 15827eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 15837eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic PyObject * 15847eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielmbstreamwriter_writelines(MultibyteStreamWriterObject *self, PyObject *lines) 15857eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel{ 15867eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyObject *strobj; 15877eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel int i, r; 15887eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 15897eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (!PySequence_Check(lines)) { 15907eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyErr_SetString(PyExc_TypeError, 15917eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel "arg must be a sequence object"); 15927eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return NULL; 15937eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 15947eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 15957eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel for (i = 0; i < PySequence_Length(lines); i++) { 15967eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel /* length can be changed even within this loop */ 15977eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel strobj = PySequence_GetItem(lines, i); 15987eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (strobj == NULL) 15997eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return NULL; 16007eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 16017eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel r = mbstreamwriter_iwrite(self, strobj); 16027eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_DECREF(strobj); 16037eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (r == -1) 16047eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return NULL; 16057eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 16067eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 16077eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_RETURN_NONE; 16087eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel} 16097eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 16107eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic PyObject * 16117eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielmbstreamwriter_reset(MultibyteStreamWriterObject *self) 16127eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel{ 16137eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel const Py_UNICODE *pending; 16147eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyObject *pwrt; 16157eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 16167eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel pending = self->pending; 16177eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel pwrt = multibytecodec_encode(self->codec, &self->state, 16187eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel &pending, self->pendingsize, self->errors, 16197eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel MBENC_FLUSH | MBENC_RESET); 16207eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel /* some pending buffer can be truncated when UnicodeEncodeError is 16217eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel * raised on 'strict' mode. but, 'reset' method is designed to 16227eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel * reset the pending buffer or states so failed string sequence 16237eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel * ought to be missed */ 16247eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel self->pendingsize = 0; 16257eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (pwrt == NULL) 16267eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return NULL; 16277eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 16287eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (PyString_Size(pwrt) > 0) { 16297eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyObject *wr; 16307eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel wr = PyObject_CallMethod(self->stream, "write", "O", pwrt); 16317eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (wr == NULL) { 16327eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_DECREF(pwrt); 16337eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return NULL; 16347eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 16357eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 16367eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_DECREF(pwrt); 16377eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 16387eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_RETURN_NONE; 16397eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel} 16407eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 16417eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic PyObject * 16427eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielmbstreamwriter_new(PyTypeObject *type, PyObject *args, PyObject *kwds) 16437eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel{ 16447eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel MultibyteStreamWriterObject *self; 16457eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyObject *stream, *codec = NULL; 16467eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel char *errors = NULL; 16477eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 16487eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|s:StreamWriter", 16497eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel streamkwarglist, &stream, &errors)) 16507eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return NULL; 16517eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 16527eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel self = (MultibyteStreamWriterObject *)type->tp_alloc(type, 0); 16537eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (self == NULL) 16547eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return NULL; 16557eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 16567eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel codec = PyObject_GetAttrString((PyObject *)type, "codec"); 16577eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (codec == NULL) 16587eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 16597eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (!MultibyteCodec_Check(codec)) { 16607eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyErr_SetString(PyExc_TypeError, "codec is unexpected type"); 16617eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 16627eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 16637eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 16647eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel self->codec = ((MultibyteCodecObject *)codec)->codec; 16657eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel self->stream = stream; 16667eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_INCREF(stream); 16677eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel self->pendingsize = 0; 16687eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel self->errors = internal_error_callback(errors); 16697eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (self->errors == NULL) 16707eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 16717eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (self->codec->encinit != NULL && 16727eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel self->codec->encinit(&self->state, self->codec->config) != 0) 16737eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel goto errorexit; 16747eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 16757eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_DECREF(codec); 16767eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return (PyObject *)self; 16777eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 16787eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielerrorexit: 16797eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_XDECREF(self); 16807eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_XDECREF(codec); 16817eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return NULL; 16827eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel} 16837eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 16847eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic int 16857eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielmbstreamwriter_init(PyObject *self, PyObject *args, PyObject *kwds) 16867eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel{ 16877eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return 0; 16887eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel} 16897eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 16907eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic int 16917eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielmbstreamwriter_traverse(MultibyteStreamWriterObject *self, 16927eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel visitproc visit, void *arg) 16937eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel{ 16947eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (ERROR_ISCUSTOM(self->errors)) 16957eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_VISIT(self->errors); 16967eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_VISIT(self->stream); 16977eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return 0; 16987eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel} 16997eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 17007eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic void 17017eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielmbstreamwriter_dealloc(MultibyteStreamWriterObject *self) 17027eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel{ 17037eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyObject_GC_UnTrack(self); 17047eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel ERROR_DECREF(self->errors); 17057eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_XDECREF(self->stream); 17067eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_TYPE(self)->tp_free(self); 17077eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel} 17087eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 17097eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic struct PyMethodDef mbstreamwriter_methods[] = { 17107eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel {"write", (PyCFunction)mbstreamwriter_write, 17117eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel METH_O, NULL}, 17127eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel {"writelines", (PyCFunction)mbstreamwriter_writelines, 17137eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel METH_O, NULL}, 17147eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel {"reset", (PyCFunction)mbstreamwriter_reset, 17157eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel METH_NOARGS, NULL}, 17167eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel {NULL, NULL}, 17177eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel}; 17187eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 17197eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic PyMemberDef mbstreamwriter_members[] = { 17207eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel {"stream", T_OBJECT, 17217eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel offsetof(MultibyteStreamWriterObject, stream), 17227eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel READONLY, NULL}, 17237eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel {NULL,} 17247eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel}; 17257eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 17267eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic PyTypeObject MultibyteStreamWriter_Type = { 17277eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyVarObject_HEAD_INIT(NULL, 0) 17287eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel "MultibyteStreamWriter", /* tp_name */ 17297eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel sizeof(MultibyteStreamWriterObject), /* tp_basicsize */ 17307eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_itemsize */ 17317eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel /* methods */ 17327eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel (destructor)mbstreamwriter_dealloc, /* tp_dealloc */ 17337eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_print */ 17347eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_getattr */ 17357eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_setattr */ 17367eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_compare */ 17377eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_repr */ 17387eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_as_number */ 17397eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_as_sequence */ 17407eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_as_mapping */ 17417eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_hash */ 17427eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_call */ 17437eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_str */ 17447eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyObject_GenericGetAttr, /* tp_getattro */ 17457eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_setattro */ 17467eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_as_buffer */ 17477eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC 17487eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel | Py_TPFLAGS_BASETYPE, /* tp_flags */ 17497eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_doc */ 17507eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel (traverseproc)mbstreamwriter_traverse, /* tp_traverse */ 17517eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_clear */ 17527eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_richcompare */ 17537eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_weaklistoffset */ 17547eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_iter */ 17557eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_iterext */ 17567eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel mbstreamwriter_methods, /* tp_methods */ 17577eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel mbstreamwriter_members, /* tp_members */ 17587eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel codecctx_getsets, /* tp_getset */ 17597eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_base */ 17607eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_dict */ 17617eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_descr_get */ 17627eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_descr_set */ 17637eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_dictoffset */ 17647eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel mbstreamwriter_init, /* tp_init */ 17657eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 0, /* tp_alloc */ 17667eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel mbstreamwriter_new, /* tp_new */ 17677eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel}; 17687eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 17697eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 17707eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel/** 17717eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel * Exposed factory function 17727eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel */ 17737eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 17747eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic PyObject * 17757eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel__create_codec(PyObject *ignore, PyObject *arg) 17767eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel{ 17777eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel MultibyteCodecObject *self; 17787eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel MultibyteCodec *codec; 17797eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 17807eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (!PyCapsule_IsValid(arg, PyMultibyteCodec_CAPSULE_NAME)) { 17817eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyErr_SetString(PyExc_ValueError, "argument type invalid"); 17827eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return NULL; 17837eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 17847eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 17857eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel codec = PyCapsule_GetPointer(arg, PyMultibyteCodec_CAPSULE_NAME); 17867eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (codec->codecinit != NULL && codec->codecinit(codec->config) != 0) 17877eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return NULL; 17887eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 17897eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel self = PyObject_New(MultibyteCodecObject, &MultibyteCodec_Type); 17907eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (self == NULL) 17917eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return NULL; 17927eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel self->codec = codec; 17937eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 17947eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return (PyObject *)self; 17957eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel} 17967eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 17977eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielstatic struct PyMethodDef __methods[] = { 17987eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel {"__create_codec", (PyCFunction)__create_codec, METH_O}, 17997eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel {NULL, NULL}, 18007eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel}; 18017eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 18027eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielPyMODINIT_FUNC 18037eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDanielinit_multibytecodec(void) 18047eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel{ 18057eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel int i; 18067eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyObject *m; 18077eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyTypeObject *typelist[] = { 18087eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel &MultibyteIncrementalEncoder_Type, 18097eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel &MultibyteIncrementalDecoder_Type, 18107eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel &MultibyteStreamReader_Type, 18117eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel &MultibyteStreamWriter_Type, 18127eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel NULL 18137eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel }; 18147eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 18157eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (PyType_Ready(&MultibyteCodec_Type) < 0) 18167eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return; 18177eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 18187eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel m = Py_InitModule("_multibytecodec", __methods); 18197eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (m == NULL) 18207eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return; 18217eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 18227eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel for (i = 0; typelist[i] != NULL; i++) { 18237eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (PyType_Ready(typelist[i]) < 0) 18247eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel return; 18257eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_INCREF(typelist[i]); 18267eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel PyModule_AddObject(m, typelist[i]->tp_name, 18277eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel (PyObject *)typelist[i]); 18287eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel } 18297eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel 18307eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel if (PyErr_Occurred()) 18317eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel Py_FatalError("can't initialize the _multibytecodec module"); 18327eb75bccb5dacb658c63db1a9a980950c3d54d42Daryl McDaniel} 1833