1/* 2 * multibytecodec.c: Common Multibyte Codec Implementation 3 * 4 * Written by Hye-Shik Chang <perky@FreeBSD.org> 5 */ 6 7#define PY_SSIZE_T_CLEAN 8#include "Python.h" 9#include "structmember.h" 10#include "multibytecodec.h" 11 12typedef struct { 13 const Py_UNICODE *inbuf, *inbuf_top, *inbuf_end; 14 unsigned char *outbuf, *outbuf_end; 15 PyObject *excobj, *outobj; 16} MultibyteEncodeBuffer; 17 18typedef struct { 19 const unsigned char *inbuf, *inbuf_top, *inbuf_end; 20 Py_UNICODE *outbuf, *outbuf_end; 21 PyObject *excobj, *outobj; 22} MultibyteDecodeBuffer; 23 24PyDoc_STRVAR(MultibyteCodec_Encode__doc__, 25"I.encode(unicode[, errors]) -> (string, length consumed)\n\ 26\n\ 27Return an encoded string version of `unicode'. errors may be given to\n\ 28set a different error handling scheme. Default is 'strict' meaning that\n\ 29encoding errors raise a UnicodeEncodeError. Other possible values are\n\ 30'ignore', 'replace' and 'xmlcharrefreplace' as well as any other name\n\ 31registered with codecs.register_error that can handle UnicodeEncodeErrors."); 32 33PyDoc_STRVAR(MultibyteCodec_Decode__doc__, 34"I.decode(string[, errors]) -> (unicodeobject, length consumed)\n\ 35\n\ 36Decodes `string' using I, an MultibyteCodec instance. errors may be given\n\ 37to set a different error handling scheme. Default is 'strict' meaning\n\ 38that encoding errors raise a UnicodeDecodeError. Other possible values\n\ 39are 'ignore' and 'replace' as well as any other name registered with\n\ 40codecs.register_error that is able to handle UnicodeDecodeErrors."); 41 42static char *codeckwarglist[] = {"input", "errors", NULL}; 43static char *incnewkwarglist[] = {"errors", NULL}; 44static char *incrementalkwarglist[] = {"input", "final", NULL}; 45static char *streamkwarglist[] = {"stream", "errors", NULL}; 46 47static PyObject *multibytecodec_encode(MultibyteCodec *, 48 MultibyteCodec_State *, const Py_UNICODE **, Py_ssize_t, 49 PyObject *, int); 50 51#define MBENC_RESET MBENC_MAX<<1 /* reset after an encoding session */ 52 53static PyObject * 54make_tuple(PyObject *object, Py_ssize_t len) 55{ 56 PyObject *v, *w; 57 58 if (object == NULL) 59 return NULL; 60 61 v = PyTuple_New(2); 62 if (v == NULL) { 63 Py_DECREF(object); 64 return NULL; 65 } 66 PyTuple_SET_ITEM(v, 0, object); 67 68 w = PyInt_FromSsize_t(len); 69 if (w == NULL) { 70 Py_DECREF(v); 71 return NULL; 72 } 73 PyTuple_SET_ITEM(v, 1, w); 74 75 return v; 76} 77 78static PyObject * 79internal_error_callback(const char *errors) 80{ 81 if (errors == NULL || strcmp(errors, "strict") == 0) 82 return ERROR_STRICT; 83 else if (strcmp(errors, "ignore") == 0) 84 return ERROR_IGNORE; 85 else if (strcmp(errors, "replace") == 0) 86 return ERROR_REPLACE; 87 else 88 return PyString_FromString(errors); 89} 90 91static PyObject * 92call_error_callback(PyObject *errors, PyObject *exc) 93{ 94 PyObject *args, *cb, *r; 95 96 assert(PyString_Check(errors)); 97 cb = PyCodec_LookupError(PyString_AS_STRING(errors)); 98 if (cb == NULL) 99 return NULL; 100 101 args = PyTuple_New(1); 102 if (args == NULL) { 103 Py_DECREF(cb); 104 return NULL; 105 } 106 107 PyTuple_SET_ITEM(args, 0, exc); 108 Py_INCREF(exc); 109 110 r = PyObject_CallObject(cb, args); 111 Py_DECREF(args); 112 Py_DECREF(cb); 113 return r; 114} 115 116static PyObject * 117codecctx_errors_get(MultibyteStatefulCodecContext *self) 118{ 119 const char *errors; 120 121 if (self->errors == ERROR_STRICT) 122 errors = "strict"; 123 else if (self->errors == ERROR_IGNORE) 124 errors = "ignore"; 125 else if (self->errors == ERROR_REPLACE) 126 errors = "replace"; 127 else { 128 Py_INCREF(self->errors); 129 return self->errors; 130 } 131 132 return PyString_FromString(errors); 133} 134 135static int 136codecctx_errors_set(MultibyteStatefulCodecContext *self, PyObject *value, 137 void *closure) 138{ 139 PyObject *cb; 140 141 if (!PyString_Check(value)) { 142 PyErr_SetString(PyExc_TypeError, "errors must be a string"); 143 return -1; 144 } 145 146 cb = internal_error_callback(PyString_AS_STRING(value)); 147 if (cb == NULL) 148 return -1; 149 150 ERROR_DECREF(self->errors); 151 self->errors = cb; 152 return 0; 153} 154 155/* This getset handlers list is used by all the stateful codec objects */ 156static PyGetSetDef codecctx_getsets[] = { 157 {"errors", (getter)codecctx_errors_get, 158 (setter)codecctx_errors_set, 159 PyDoc_STR("how to treat errors")}, 160 {NULL,} 161}; 162 163static int 164expand_encodebuffer(MultibyteEncodeBuffer *buf, Py_ssize_t esize) 165{ 166 Py_ssize_t orgpos, orgsize, incsize; 167 168 orgpos = (Py_ssize_t)((char *)buf->outbuf - 169 PyString_AS_STRING(buf->outobj)); 170 orgsize = PyString_GET_SIZE(buf->outobj); 171 incsize = (esize < (orgsize >> 1) ? (orgsize >> 1) | 1 : esize); 172 173 if (orgsize > PY_SSIZE_T_MAX - incsize) 174 return -1; 175 176 if (_PyString_Resize(&buf->outobj, orgsize + incsize) == -1) 177 return -1; 178 179 buf->outbuf = (unsigned char *)PyString_AS_STRING(buf->outobj) +orgpos; 180 buf->outbuf_end = (unsigned char *)PyString_AS_STRING(buf->outobj) 181 + PyString_GET_SIZE(buf->outobj); 182 183 return 0; 184} 185#define REQUIRE_ENCODEBUFFER(buf, s) { \ 186 if ((s) < 1 || (buf)->outbuf + (s) > (buf)->outbuf_end) \ 187 if (expand_encodebuffer(buf, s) == -1) \ 188 goto errorexit; \ 189} 190 191static int 192expand_decodebuffer(MultibyteDecodeBuffer *buf, Py_ssize_t esize) 193{ 194 Py_ssize_t orgpos, orgsize; 195 196 orgpos = (Py_ssize_t)(buf->outbuf - PyUnicode_AS_UNICODE(buf->outobj)); 197 orgsize = PyUnicode_GET_SIZE(buf->outobj); 198 if (PyUnicode_Resize(&buf->outobj, orgsize + ( 199 esize < (orgsize >> 1) ? (orgsize >> 1) | 1 : esize)) == -1) 200 return -1; 201 202 buf->outbuf = PyUnicode_AS_UNICODE(buf->outobj) + orgpos; 203 buf->outbuf_end = PyUnicode_AS_UNICODE(buf->outobj) 204 + PyUnicode_GET_SIZE(buf->outobj); 205 206 return 0; 207} 208#define REQUIRE_DECODEBUFFER(buf, s) { \ 209 if ((s) < 1 || (buf)->outbuf + (s) > (buf)->outbuf_end) \ 210 if (expand_decodebuffer(buf, s) == -1) \ 211 goto errorexit; \ 212} 213 214 215/** 216 * MultibyteCodec object 217 */ 218 219static int 220multibytecodec_encerror(MultibyteCodec *codec, 221 MultibyteCodec_State *state, 222 MultibyteEncodeBuffer *buf, 223 PyObject *errors, Py_ssize_t e) 224{ 225 PyObject *retobj = NULL, *retstr = NULL, *tobj; 226 Py_ssize_t retstrsize, newpos; 227 Py_ssize_t esize, start, end; 228 const char *reason; 229 230 if (e > 0) { 231 reason = "illegal multibyte sequence"; 232 esize = e; 233 } 234 else { 235 switch (e) { 236 case MBERR_TOOSMALL: 237 REQUIRE_ENCODEBUFFER(buf, -1); 238 return 0; /* retry it */ 239 case MBERR_TOOFEW: 240 reason = "incomplete multibyte sequence"; 241 esize = (Py_ssize_t)(buf->inbuf_end - buf->inbuf); 242 break; 243 case MBERR_INTERNAL: 244 PyErr_SetString(PyExc_RuntimeError, 245 "internal codec error"); 246 return -1; 247 default: 248 PyErr_SetString(PyExc_RuntimeError, 249 "unknown runtime error"); 250 return -1; 251 } 252 } 253 254 if (errors == ERROR_REPLACE) { 255 const Py_UNICODE replchar = '?', *inbuf = &replchar; 256 Py_ssize_t r; 257 258 for (;;) { 259 Py_ssize_t outleft; 260 261 outleft = (Py_ssize_t)(buf->outbuf_end - buf->outbuf); 262 r = codec->encode(state, codec->config, &inbuf, 1, 263 &buf->outbuf, outleft, 0); 264 if (r == MBERR_TOOSMALL) { 265 REQUIRE_ENCODEBUFFER(buf, -1); 266 continue; 267 } 268 else 269 break; 270 } 271 272 if (r != 0) { 273 REQUIRE_ENCODEBUFFER(buf, 1); 274 *buf->outbuf++ = '?'; 275 } 276 } 277 if (errors == ERROR_IGNORE || errors == ERROR_REPLACE) { 278 buf->inbuf += esize; 279 return 0; 280 } 281 282 start = (Py_ssize_t)(buf->inbuf - buf->inbuf_top); 283 end = start + esize; 284 285 /* use cached exception object if available */ 286 if (buf->excobj == NULL) { 287 buf->excobj = PyUnicodeEncodeError_Create(codec->encoding, 288 buf->inbuf_top, 289 buf->inbuf_end - buf->inbuf_top, 290 start, end, reason); 291 if (buf->excobj == NULL) 292 goto errorexit; 293 } 294 else 295 if (PyUnicodeEncodeError_SetStart(buf->excobj, start) != 0 || 296 PyUnicodeEncodeError_SetEnd(buf->excobj, end) != 0 || 297 PyUnicodeEncodeError_SetReason(buf->excobj, reason) != 0) 298 goto errorexit; 299 300 if (errors == ERROR_STRICT) { 301 PyCodec_StrictErrors(buf->excobj); 302 goto errorexit; 303 } 304 305 retobj = call_error_callback(errors, buf->excobj); 306 if (retobj == NULL) 307 goto errorexit; 308 309 if (!PyTuple_Check(retobj) || PyTuple_GET_SIZE(retobj) != 2 || 310 !PyUnicode_Check((tobj = PyTuple_GET_ITEM(retobj, 0))) || 311 !(PyInt_Check(PyTuple_GET_ITEM(retobj, 1)) || 312 PyLong_Check(PyTuple_GET_ITEM(retobj, 1)))) { 313 PyErr_SetString(PyExc_TypeError, 314 "encoding error handler must return " 315 "(unicode, int) tuple"); 316 goto errorexit; 317 } 318 319 { 320 const Py_UNICODE *uraw = PyUnicode_AS_UNICODE(tobj); 321 322 retstr = multibytecodec_encode(codec, state, &uraw, 323 PyUnicode_GET_SIZE(tobj), ERROR_STRICT, 324 MBENC_FLUSH); 325 if (retstr == NULL) 326 goto errorexit; 327 } 328 329 retstrsize = PyString_GET_SIZE(retstr); 330 REQUIRE_ENCODEBUFFER(buf, retstrsize); 331 332 memcpy(buf->outbuf, PyString_AS_STRING(retstr), retstrsize); 333 buf->outbuf += retstrsize; 334 335 newpos = PyInt_AsSsize_t(PyTuple_GET_ITEM(retobj, 1)); 336 if (newpos < 0 && !PyErr_Occurred()) 337 newpos += (Py_ssize_t)(buf->inbuf_end - buf->inbuf_top); 338 if (newpos < 0 || buf->inbuf_top + newpos > buf->inbuf_end) { 339 PyErr_Clear(); 340 PyErr_Format(PyExc_IndexError, 341 "position %zd from error handler out of bounds", 342 newpos); 343 goto errorexit; 344 } 345 buf->inbuf = buf->inbuf_top + newpos; 346 347 Py_DECREF(retobj); 348 Py_DECREF(retstr); 349 return 0; 350 351errorexit: 352 Py_XDECREF(retobj); 353 Py_XDECREF(retstr); 354 return -1; 355} 356 357static int 358multibytecodec_decerror(MultibyteCodec *codec, 359 MultibyteCodec_State *state, 360 MultibyteDecodeBuffer *buf, 361 PyObject *errors, Py_ssize_t e) 362{ 363 PyObject *retobj = NULL, *retuni = NULL; 364 Py_ssize_t retunisize, newpos; 365 const char *reason; 366 Py_ssize_t esize, start, end; 367 368 if (e > 0) { 369 reason = "illegal multibyte sequence"; 370 esize = e; 371 } 372 else { 373 switch (e) { 374 case MBERR_TOOSMALL: 375 REQUIRE_DECODEBUFFER(buf, -1); 376 return 0; /* retry it */ 377 case MBERR_TOOFEW: 378 reason = "incomplete multibyte sequence"; 379 esize = (Py_ssize_t)(buf->inbuf_end - buf->inbuf); 380 break; 381 case MBERR_INTERNAL: 382 PyErr_SetString(PyExc_RuntimeError, 383 "internal codec error"); 384 return -1; 385 default: 386 PyErr_SetString(PyExc_RuntimeError, 387 "unknown runtime error"); 388 return -1; 389 } 390 } 391 392 if (errors == ERROR_REPLACE) { 393 REQUIRE_DECODEBUFFER(buf, 1); 394 *buf->outbuf++ = Py_UNICODE_REPLACEMENT_CHARACTER; 395 } 396 if (errors == ERROR_IGNORE || errors == ERROR_REPLACE) { 397 buf->inbuf += esize; 398 return 0; 399 } 400 401 start = (Py_ssize_t)(buf->inbuf - buf->inbuf_top); 402 end = start + esize; 403 404 /* use cached exception object if available */ 405 if (buf->excobj == NULL) { 406 buf->excobj = PyUnicodeDecodeError_Create(codec->encoding, 407 (const char *)buf->inbuf_top, 408 (Py_ssize_t)(buf->inbuf_end - buf->inbuf_top), 409 start, end, reason); 410 if (buf->excobj == NULL) 411 goto errorexit; 412 } 413 else 414 if (PyUnicodeDecodeError_SetStart(buf->excobj, start) || 415 PyUnicodeDecodeError_SetEnd(buf->excobj, end) || 416 PyUnicodeDecodeError_SetReason(buf->excobj, reason)) 417 goto errorexit; 418 419 if (errors == ERROR_STRICT) { 420 PyCodec_StrictErrors(buf->excobj); 421 goto errorexit; 422 } 423 424 retobj = call_error_callback(errors, buf->excobj); 425 if (retobj == NULL) 426 goto errorexit; 427 428 if (!PyTuple_Check(retobj) || PyTuple_GET_SIZE(retobj) != 2 || 429 !PyUnicode_Check((retuni = PyTuple_GET_ITEM(retobj, 0))) || 430 !(PyInt_Check(PyTuple_GET_ITEM(retobj, 1)) || 431 PyLong_Check(PyTuple_GET_ITEM(retobj, 1)))) { 432 PyErr_SetString(PyExc_TypeError, 433 "decoding error handler must return " 434 "(unicode, int) tuple"); 435 goto errorexit; 436 } 437 438 retunisize = PyUnicode_GET_SIZE(retuni); 439 if (retunisize > 0) { 440 REQUIRE_DECODEBUFFER(buf, retunisize); 441 memcpy((char *)buf->outbuf, PyUnicode_AS_DATA(retuni), 442 retunisize * Py_UNICODE_SIZE); 443 buf->outbuf += retunisize; 444 } 445 446 newpos = PyInt_AsSsize_t(PyTuple_GET_ITEM(retobj, 1)); 447 if (newpos < 0 && !PyErr_Occurred()) 448 newpos += (Py_ssize_t)(buf->inbuf_end - buf->inbuf_top); 449 if (newpos < 0 || buf->inbuf_top + newpos > buf->inbuf_end) { 450 PyErr_Clear(); 451 PyErr_Format(PyExc_IndexError, 452 "position %zd from error handler out of bounds", 453 newpos); 454 goto errorexit; 455 } 456 buf->inbuf = buf->inbuf_top + newpos; 457 Py_DECREF(retobj); 458 return 0; 459 460errorexit: 461 Py_XDECREF(retobj); 462 return -1; 463} 464 465static PyObject * 466multibytecodec_encode(MultibyteCodec *codec, 467 MultibyteCodec_State *state, 468 const Py_UNICODE **data, Py_ssize_t datalen, 469 PyObject *errors, int flags) 470{ 471 MultibyteEncodeBuffer buf; 472 Py_ssize_t finalsize, r = 0; 473 474 if (datalen == 0 && !(flags & MBENC_RESET)) 475 return PyString_FromString(""); 476 477 buf.excobj = NULL; 478 buf.inbuf = buf.inbuf_top = *data; 479 buf.inbuf_end = buf.inbuf_top + datalen; 480 481 if (datalen > (PY_SSIZE_T_MAX - 16) / 2) { 482 PyErr_NoMemory(); 483 goto errorexit; 484 } 485 486 buf.outobj = PyString_FromStringAndSize(NULL, datalen * 2 + 16); 487 if (buf.outobj == NULL) 488 goto errorexit; 489 buf.outbuf = (unsigned char *)PyString_AS_STRING(buf.outobj); 490 buf.outbuf_end = buf.outbuf + PyString_GET_SIZE(buf.outobj); 491 492 while (buf.inbuf < buf.inbuf_end) { 493 Py_ssize_t inleft, outleft; 494 495 /* we don't reuse inleft and outleft here. 496 * error callbacks can relocate the cursor anywhere on buffer*/ 497 inleft = (Py_ssize_t)(buf.inbuf_end - buf.inbuf); 498 outleft = (Py_ssize_t)(buf.outbuf_end - buf.outbuf); 499 r = codec->encode(state, codec->config, &buf.inbuf, inleft, 500 &buf.outbuf, outleft, flags); 501 if ((r == 0) || (r == MBERR_TOOFEW && !(flags & MBENC_FLUSH))) 502 break; 503 else if (multibytecodec_encerror(codec, state, &buf, errors,r)) 504 goto errorexit; 505 else if (r == MBERR_TOOFEW) 506 break; 507 } 508 509 if (codec->encreset != NULL && (flags & MBENC_RESET)) 510 for (;;) { 511 Py_ssize_t outleft; 512 513 outleft = (Py_ssize_t)(buf.outbuf_end - buf.outbuf); 514 r = codec->encreset(state, codec->config, &buf.outbuf, 515 outleft); 516 if (r == 0) 517 break; 518 else if (multibytecodec_encerror(codec, state, 519 &buf, errors, r)) 520 goto errorexit; 521 } 522 523 finalsize = (Py_ssize_t)((char *)buf.outbuf - 524 PyString_AS_STRING(buf.outobj)); 525 526 if (finalsize != PyString_GET_SIZE(buf.outobj)) 527 if (_PyString_Resize(&buf.outobj, finalsize) == -1) 528 goto errorexit; 529 530 *data = buf.inbuf; 531 Py_XDECREF(buf.excobj); 532 return buf.outobj; 533 534errorexit: 535 Py_XDECREF(buf.excobj); 536 Py_XDECREF(buf.outobj); 537 return NULL; 538} 539 540static PyObject * 541MultibyteCodec_Encode(MultibyteCodecObject *self, 542 PyObject *args, PyObject *kwargs) 543{ 544 MultibyteCodec_State state; 545 Py_UNICODE *data; 546 PyObject *errorcb, *r, *arg, *ucvt; 547 const char *errors = NULL; 548 Py_ssize_t datalen; 549 550 if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|z:encode", 551 codeckwarglist, &arg, &errors)) 552 return NULL; 553 554 if (PyUnicode_Check(arg)) 555 ucvt = NULL; 556 else { 557 arg = ucvt = PyObject_Unicode(arg); 558 if (arg == NULL) 559 return NULL; 560 else if (!PyUnicode_Check(arg)) { 561 PyErr_SetString(PyExc_TypeError, 562 "couldn't convert the object to unicode."); 563 Py_DECREF(ucvt); 564 return NULL; 565 } 566 } 567 568 data = PyUnicode_AS_UNICODE(arg); 569 datalen = PyUnicode_GET_SIZE(arg); 570 571 errorcb = internal_error_callback(errors); 572 if (errorcb == NULL) { 573 Py_XDECREF(ucvt); 574 return NULL; 575 } 576 577 if (self->codec->encinit != NULL && 578 self->codec->encinit(&state, self->codec->config) != 0) 579 goto errorexit; 580 r = multibytecodec_encode(self->codec, &state, 581 (const Py_UNICODE **)&data, datalen, errorcb, 582 MBENC_FLUSH | MBENC_RESET); 583 if (r == NULL) 584 goto errorexit; 585 586 ERROR_DECREF(errorcb); 587 Py_XDECREF(ucvt); 588 return make_tuple(r, datalen); 589 590errorexit: 591 ERROR_DECREF(errorcb); 592 Py_XDECREF(ucvt); 593 return NULL; 594} 595 596static PyObject * 597MultibyteCodec_Decode(MultibyteCodecObject *self, 598 PyObject *args, PyObject *kwargs) 599{ 600 MultibyteCodec_State state; 601 MultibyteDecodeBuffer buf; 602 PyObject *errorcb; 603 Py_buffer pdata; 604 const char *data, *errors = NULL; 605 Py_ssize_t datalen, finalsize; 606 607 if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s*|z:decode", 608 codeckwarglist, &pdata, &errors)) 609 return NULL; 610 data = pdata.buf; 611 datalen = pdata.len; 612 613 errorcb = internal_error_callback(errors); 614 if (errorcb == NULL) { 615 PyBuffer_Release(&pdata); 616 return NULL; 617 } 618 619 if (datalen == 0) { 620 PyBuffer_Release(&pdata); 621 ERROR_DECREF(errorcb); 622 return make_tuple(PyUnicode_FromUnicode(NULL, 0), 0); 623 } 624 625 buf.excobj = NULL; 626 buf.inbuf = buf.inbuf_top = (unsigned char *)data; 627 buf.inbuf_end = buf.inbuf_top + datalen; 628 buf.outobj = PyUnicode_FromUnicode(NULL, datalen); 629 if (buf.outobj == NULL) 630 goto errorexit; 631 buf.outbuf = PyUnicode_AS_UNICODE(buf.outobj); 632 buf.outbuf_end = buf.outbuf + PyUnicode_GET_SIZE(buf.outobj); 633 634 if (self->codec->decinit != NULL && 635 self->codec->decinit(&state, self->codec->config) != 0) 636 goto errorexit; 637 638 while (buf.inbuf < buf.inbuf_end) { 639 Py_ssize_t inleft, outleft, r; 640 641 inleft = (Py_ssize_t)(buf.inbuf_end - buf.inbuf); 642 outleft = (Py_ssize_t)(buf.outbuf_end - buf.outbuf); 643 644 r = self->codec->decode(&state, self->codec->config, 645 &buf.inbuf, inleft, &buf.outbuf, outleft); 646 if (r == 0) 647 break; 648 else if (multibytecodec_decerror(self->codec, &state, 649 &buf, errorcb, r)) 650 goto errorexit; 651 } 652 653 finalsize = (Py_ssize_t)(buf.outbuf - 654 PyUnicode_AS_UNICODE(buf.outobj)); 655 656 if (finalsize != PyUnicode_GET_SIZE(buf.outobj)) 657 if (PyUnicode_Resize(&buf.outobj, finalsize) == -1) 658 goto errorexit; 659 660 PyBuffer_Release(&pdata); 661 Py_XDECREF(buf.excobj); 662 ERROR_DECREF(errorcb); 663 return make_tuple(buf.outobj, datalen); 664 665errorexit: 666 PyBuffer_Release(&pdata); 667 ERROR_DECREF(errorcb); 668 Py_XDECREF(buf.excobj); 669 Py_XDECREF(buf.outobj); 670 671 return NULL; 672} 673 674static struct PyMethodDef multibytecodec_methods[] = { 675 {"encode", (PyCFunction)MultibyteCodec_Encode, 676 METH_VARARGS | METH_KEYWORDS, 677 MultibyteCodec_Encode__doc__}, 678 {"decode", (PyCFunction)MultibyteCodec_Decode, 679 METH_VARARGS | METH_KEYWORDS, 680 MultibyteCodec_Decode__doc__}, 681 {NULL, NULL}, 682}; 683 684static void 685multibytecodec_dealloc(MultibyteCodecObject *self) 686{ 687 PyObject_Del(self); 688} 689 690static PyTypeObject MultibyteCodec_Type = { 691 PyVarObject_HEAD_INIT(NULL, 0) 692 "MultibyteCodec", /* tp_name */ 693 sizeof(MultibyteCodecObject), /* tp_basicsize */ 694 0, /* tp_itemsize */ 695 /* methods */ 696 (destructor)multibytecodec_dealloc, /* tp_dealloc */ 697 0, /* tp_print */ 698 0, /* tp_getattr */ 699 0, /* tp_setattr */ 700 0, /* tp_compare */ 701 0, /* tp_repr */ 702 0, /* tp_as_number */ 703 0, /* tp_as_sequence */ 704 0, /* tp_as_mapping */ 705 0, /* tp_hash */ 706 0, /* tp_call */ 707 0, /* tp_str */ 708 PyObject_GenericGetAttr, /* tp_getattro */ 709 0, /* tp_setattro */ 710 0, /* tp_as_buffer */ 711 Py_TPFLAGS_DEFAULT, /* tp_flags */ 712 0, /* tp_doc */ 713 0, /* tp_traverse */ 714 0, /* tp_clear */ 715 0, /* tp_richcompare */ 716 0, /* tp_weaklistoffset */ 717 0, /* tp_iter */ 718 0, /* tp_iterext */ 719 multibytecodec_methods, /* tp_methods */ 720}; 721 722 723/** 724 * Utility functions for stateful codec mechanism 725 */ 726 727#define STATEFUL_DCTX(o) ((MultibyteStatefulDecoderContext *)(o)) 728#define STATEFUL_ECTX(o) ((MultibyteStatefulEncoderContext *)(o)) 729 730static PyObject * 731encoder_encode_stateful(MultibyteStatefulEncoderContext *ctx, 732 PyObject *unistr, int final) 733{ 734 PyObject *ucvt, *r = NULL; 735 Py_UNICODE *inbuf, *inbuf_end, *inbuf_tmp = NULL; 736 Py_ssize_t datalen, origpending; 737 738 if (PyUnicode_Check(unistr)) 739 ucvt = NULL; 740 else { 741 unistr = ucvt = PyObject_Unicode(unistr); 742 if (unistr == NULL) 743 return NULL; 744 else if (!PyUnicode_Check(unistr)) { 745 PyErr_SetString(PyExc_TypeError, 746 "couldn't convert the object to unicode."); 747 Py_DECREF(ucvt); 748 return NULL; 749 } 750 } 751 752 datalen = PyUnicode_GET_SIZE(unistr); 753 origpending = ctx->pendingsize; 754 755 if (origpending > 0) { 756 if (datalen > PY_SSIZE_T_MAX - ctx->pendingsize) { 757 PyErr_NoMemory(); 758 /* inbuf_tmp == NULL */ 759 goto errorexit; 760 } 761 inbuf_tmp = PyMem_New(Py_UNICODE, datalen + ctx->pendingsize); 762 if (inbuf_tmp == NULL) 763 goto errorexit; 764 memcpy(inbuf_tmp, ctx->pending, 765 Py_UNICODE_SIZE * ctx->pendingsize); 766 memcpy(inbuf_tmp + ctx->pendingsize, 767 PyUnicode_AS_UNICODE(unistr), 768 Py_UNICODE_SIZE * datalen); 769 datalen += ctx->pendingsize; 770 ctx->pendingsize = 0; 771 inbuf = inbuf_tmp; 772 } 773 else 774 inbuf = (Py_UNICODE *)PyUnicode_AS_UNICODE(unistr); 775 776 inbuf_end = inbuf + datalen; 777 778 r = multibytecodec_encode(ctx->codec, &ctx->state, 779 (const Py_UNICODE **)&inbuf, datalen, 780 ctx->errors, final ? MBENC_FLUSH | MBENC_RESET : 0); 781 if (r == NULL) { 782 /* recover the original pending buffer */ 783 if (origpending > 0) 784 memcpy(ctx->pending, inbuf_tmp, 785 Py_UNICODE_SIZE * origpending); 786 ctx->pendingsize = origpending; 787 goto errorexit; 788 } 789 790 if (inbuf < inbuf_end) { 791 ctx->pendingsize = (Py_ssize_t)(inbuf_end - inbuf); 792 if (ctx->pendingsize > MAXENCPENDING) { 793 /* normal codecs can't reach here */ 794 ctx->pendingsize = 0; 795 PyErr_SetString(PyExc_UnicodeError, 796 "pending buffer overflow"); 797 goto errorexit; 798 } 799 memcpy(ctx->pending, inbuf, 800 ctx->pendingsize * Py_UNICODE_SIZE); 801 } 802 803 if (inbuf_tmp != NULL) 804 PyMem_Del(inbuf_tmp); 805 Py_XDECREF(ucvt); 806 return r; 807 808errorexit: 809 if (inbuf_tmp != NULL) 810 PyMem_Del(inbuf_tmp); 811 Py_XDECREF(r); 812 Py_XDECREF(ucvt); 813 return NULL; 814} 815 816static int 817decoder_append_pending(MultibyteStatefulDecoderContext *ctx, 818 MultibyteDecodeBuffer *buf) 819{ 820 Py_ssize_t npendings; 821 822 npendings = (Py_ssize_t)(buf->inbuf_end - buf->inbuf); 823 if (npendings + ctx->pendingsize > MAXDECPENDING || 824 npendings > PY_SSIZE_T_MAX - ctx->pendingsize) { 825 PyErr_SetString(PyExc_UnicodeError, "pending buffer overflow"); 826 return -1; 827 } 828 memcpy(ctx->pending + ctx->pendingsize, buf->inbuf, npendings); 829 ctx->pendingsize += npendings; 830 return 0; 831} 832 833static int 834decoder_prepare_buffer(MultibyteDecodeBuffer *buf, const char *data, 835 Py_ssize_t size) 836{ 837 buf->inbuf = buf->inbuf_top = (const unsigned char *)data; 838 buf->inbuf_end = buf->inbuf_top + size; 839 if (buf->outobj == NULL) { /* only if outobj is not allocated yet */ 840 buf->outobj = PyUnicode_FromUnicode(NULL, size); 841 if (buf->outobj == NULL) 842 return -1; 843 buf->outbuf = PyUnicode_AS_UNICODE(buf->outobj); 844 buf->outbuf_end = buf->outbuf + 845 PyUnicode_GET_SIZE(buf->outobj); 846 } 847 848 return 0; 849} 850 851static int 852decoder_feed_buffer(MultibyteStatefulDecoderContext *ctx, 853 MultibyteDecodeBuffer *buf) 854{ 855 while (buf->inbuf < buf->inbuf_end) { 856 Py_ssize_t inleft, outleft; 857 Py_ssize_t r; 858 859 inleft = (Py_ssize_t)(buf->inbuf_end - buf->inbuf); 860 outleft = (Py_ssize_t)(buf->outbuf_end - buf->outbuf); 861 862 r = ctx->codec->decode(&ctx->state, ctx->codec->config, 863 &buf->inbuf, inleft, &buf->outbuf, outleft); 864 if (r == 0 || r == MBERR_TOOFEW) 865 break; 866 else if (multibytecodec_decerror(ctx->codec, &ctx->state, 867 buf, ctx->errors, r)) 868 return -1; 869 } 870 return 0; 871} 872 873 874/** 875 * MultibyteIncrementalEncoder object 876 */ 877 878static PyObject * 879mbiencoder_encode(MultibyteIncrementalEncoderObject *self, 880 PyObject *args, PyObject *kwargs) 881{ 882 PyObject *data; 883 int final = 0; 884 885 if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|i:encode", 886 incrementalkwarglist, &data, &final)) 887 return NULL; 888 889 return encoder_encode_stateful(STATEFUL_ECTX(self), data, final); 890} 891 892static PyObject * 893mbiencoder_reset(MultibyteIncrementalEncoderObject *self) 894{ 895 if (self->codec->decreset != NULL && 896 self->codec->decreset(&self->state, self->codec->config) != 0) 897 return NULL; 898 self->pendingsize = 0; 899 900 Py_RETURN_NONE; 901} 902 903static struct PyMethodDef mbiencoder_methods[] = { 904 {"encode", (PyCFunction)mbiencoder_encode, 905 METH_VARARGS | METH_KEYWORDS, NULL}, 906 {"reset", (PyCFunction)mbiencoder_reset, 907 METH_NOARGS, NULL}, 908 {NULL, NULL}, 909}; 910 911static PyObject * 912mbiencoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds) 913{ 914 MultibyteIncrementalEncoderObject *self; 915 PyObject *codec = NULL; 916 char *errors = NULL; 917 918 if (!PyArg_ParseTupleAndKeywords(args, kwds, "|s:IncrementalEncoder", 919 incnewkwarglist, &errors)) 920 return NULL; 921 922 self = (MultibyteIncrementalEncoderObject *)type->tp_alloc(type, 0); 923 if (self == NULL) 924 return NULL; 925 926 codec = PyObject_GetAttrString((PyObject *)type, "codec"); 927 if (codec == NULL) 928 goto errorexit; 929 if (!MultibyteCodec_Check(codec)) { 930 PyErr_SetString(PyExc_TypeError, "codec is unexpected type"); 931 goto errorexit; 932 } 933 934 self->codec = ((MultibyteCodecObject *)codec)->codec; 935 self->pendingsize = 0; 936 self->errors = internal_error_callback(errors); 937 if (self->errors == NULL) 938 goto errorexit; 939 if (self->codec->encinit != NULL && 940 self->codec->encinit(&self->state, self->codec->config) != 0) 941 goto errorexit; 942 943 Py_DECREF(codec); 944 return (PyObject *)self; 945 946errorexit: 947 Py_XDECREF(self); 948 Py_XDECREF(codec); 949 return NULL; 950} 951 952static int 953mbiencoder_init(PyObject *self, PyObject *args, PyObject *kwds) 954{ 955 return 0; 956} 957 958static int 959mbiencoder_traverse(MultibyteIncrementalEncoderObject *self, 960 visitproc visit, void *arg) 961{ 962 if (ERROR_ISCUSTOM(self->errors)) 963 Py_VISIT(self->errors); 964 return 0; 965} 966 967static void 968mbiencoder_dealloc(MultibyteIncrementalEncoderObject *self) 969{ 970 PyObject_GC_UnTrack(self); 971 ERROR_DECREF(self->errors); 972 Py_TYPE(self)->tp_free(self); 973} 974 975static PyTypeObject MultibyteIncrementalEncoder_Type = { 976 PyVarObject_HEAD_INIT(NULL, 0) 977 "MultibyteIncrementalEncoder", /* tp_name */ 978 sizeof(MultibyteIncrementalEncoderObject), /* tp_basicsize */ 979 0, /* tp_itemsize */ 980 /* methods */ 981 (destructor)mbiencoder_dealloc, /* tp_dealloc */ 982 0, /* tp_print */ 983 0, /* tp_getattr */ 984 0, /* tp_setattr */ 985 0, /* tp_compare */ 986 0, /* tp_repr */ 987 0, /* tp_as_number */ 988 0, /* tp_as_sequence */ 989 0, /* tp_as_mapping */ 990 0, /* tp_hash */ 991 0, /* tp_call */ 992 0, /* tp_str */ 993 PyObject_GenericGetAttr, /* tp_getattro */ 994 0, /* tp_setattro */ 995 0, /* tp_as_buffer */ 996 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC 997 | Py_TPFLAGS_BASETYPE, /* tp_flags */ 998 0, /* tp_doc */ 999 (traverseproc)mbiencoder_traverse, /* tp_traverse */ 1000 0, /* tp_clear */ 1001 0, /* tp_richcompare */ 1002 0, /* tp_weaklistoffset */ 1003 0, /* tp_iter */ 1004 0, /* tp_iterext */ 1005 mbiencoder_methods, /* tp_methods */ 1006 0, /* tp_members */ 1007 codecctx_getsets, /* tp_getset */ 1008 0, /* tp_base */ 1009 0, /* tp_dict */ 1010 0, /* tp_descr_get */ 1011 0, /* tp_descr_set */ 1012 0, /* tp_dictoffset */ 1013 mbiencoder_init, /* tp_init */ 1014 0, /* tp_alloc */ 1015 mbiencoder_new, /* tp_new */ 1016}; 1017 1018 1019/** 1020 * MultibyteIncrementalDecoder object 1021 */ 1022 1023static PyObject * 1024mbidecoder_decode(MultibyteIncrementalDecoderObject *self, 1025 PyObject *args, PyObject *kwargs) 1026{ 1027 MultibyteDecodeBuffer buf; 1028 char *data, *wdata = NULL; 1029 Py_buffer pdata; 1030 Py_ssize_t wsize, finalsize = 0, size, origpending; 1031 int final = 0; 1032 1033 if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s*|i:decode", 1034 incrementalkwarglist, &pdata, &final)) 1035 return NULL; 1036 data = pdata.buf; 1037 size = pdata.len; 1038 1039 buf.outobj = buf.excobj = NULL; 1040 origpending = self->pendingsize; 1041 1042 if (self->pendingsize == 0) { 1043 wsize = size; 1044 wdata = data; 1045 } 1046 else { 1047 if (size > PY_SSIZE_T_MAX - self->pendingsize) { 1048 PyErr_NoMemory(); 1049 goto errorexit; 1050 } 1051 wsize = size + self->pendingsize; 1052 wdata = PyMem_Malloc(wsize); 1053 if (wdata == NULL) 1054 goto errorexit; 1055 memcpy(wdata, self->pending, self->pendingsize); 1056 memcpy(wdata + self->pendingsize, data, size); 1057 self->pendingsize = 0; 1058 } 1059 1060 if (decoder_prepare_buffer(&buf, wdata, wsize) != 0) 1061 goto errorexit; 1062 1063 if (decoder_feed_buffer(STATEFUL_DCTX(self), &buf)) 1064 goto errorexit; 1065 1066 if (final && buf.inbuf < buf.inbuf_end) { 1067 if (multibytecodec_decerror(self->codec, &self->state, 1068 &buf, self->errors, MBERR_TOOFEW)) { 1069 /* recover the original pending buffer */ 1070 memcpy(self->pending, wdata, origpending); 1071 self->pendingsize = origpending; 1072 goto errorexit; 1073 } 1074 } 1075 1076 if (buf.inbuf < buf.inbuf_end) { /* pending sequence still exists */ 1077 if (decoder_append_pending(STATEFUL_DCTX(self), &buf) != 0) 1078 goto errorexit; 1079 } 1080 1081 finalsize = (Py_ssize_t)(buf.outbuf - PyUnicode_AS_UNICODE(buf.outobj)); 1082 if (finalsize != PyUnicode_GET_SIZE(buf.outobj)) 1083 if (PyUnicode_Resize(&buf.outobj, finalsize) == -1) 1084 goto errorexit; 1085 1086 PyBuffer_Release(&pdata); 1087 if (wdata != data) 1088 PyMem_Del(wdata); 1089 Py_XDECREF(buf.excobj); 1090 return buf.outobj; 1091 1092errorexit: 1093 PyBuffer_Release(&pdata); 1094 if (wdata != NULL && wdata != data) 1095 PyMem_Del(wdata); 1096 Py_XDECREF(buf.excobj); 1097 Py_XDECREF(buf.outobj); 1098 return NULL; 1099} 1100 1101static PyObject * 1102mbidecoder_reset(MultibyteIncrementalDecoderObject *self) 1103{ 1104 if (self->codec->decreset != NULL && 1105 self->codec->decreset(&self->state, self->codec->config) != 0) 1106 return NULL; 1107 self->pendingsize = 0; 1108 1109 Py_RETURN_NONE; 1110} 1111 1112static struct PyMethodDef mbidecoder_methods[] = { 1113 {"decode", (PyCFunction)mbidecoder_decode, 1114 METH_VARARGS | METH_KEYWORDS, NULL}, 1115 {"reset", (PyCFunction)mbidecoder_reset, 1116 METH_NOARGS, NULL}, 1117 {NULL, NULL}, 1118}; 1119 1120static PyObject * 1121mbidecoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds) 1122{ 1123 MultibyteIncrementalDecoderObject *self; 1124 PyObject *codec = NULL; 1125 char *errors = NULL; 1126 1127 if (!PyArg_ParseTupleAndKeywords(args, kwds, "|s:IncrementalDecoder", 1128 incnewkwarglist, &errors)) 1129 return NULL; 1130 1131 self = (MultibyteIncrementalDecoderObject *)type->tp_alloc(type, 0); 1132 if (self == NULL) 1133 return NULL; 1134 1135 codec = PyObject_GetAttrString((PyObject *)type, "codec"); 1136 if (codec == NULL) 1137 goto errorexit; 1138 if (!MultibyteCodec_Check(codec)) { 1139 PyErr_SetString(PyExc_TypeError, "codec is unexpected type"); 1140 goto errorexit; 1141 } 1142 1143 self->codec = ((MultibyteCodecObject *)codec)->codec; 1144 self->pendingsize = 0; 1145 self->errors = internal_error_callback(errors); 1146 if (self->errors == NULL) 1147 goto errorexit; 1148 if (self->codec->decinit != NULL && 1149 self->codec->decinit(&self->state, self->codec->config) != 0) 1150 goto errorexit; 1151 1152 Py_DECREF(codec); 1153 return (PyObject *)self; 1154 1155errorexit: 1156 Py_XDECREF(self); 1157 Py_XDECREF(codec); 1158 return NULL; 1159} 1160 1161static int 1162mbidecoder_init(PyObject *self, PyObject *args, PyObject *kwds) 1163{ 1164 return 0; 1165} 1166 1167static int 1168mbidecoder_traverse(MultibyteIncrementalDecoderObject *self, 1169 visitproc visit, void *arg) 1170{ 1171 if (ERROR_ISCUSTOM(self->errors)) 1172 Py_VISIT(self->errors); 1173 return 0; 1174} 1175 1176static void 1177mbidecoder_dealloc(MultibyteIncrementalDecoderObject *self) 1178{ 1179 PyObject_GC_UnTrack(self); 1180 ERROR_DECREF(self->errors); 1181 Py_TYPE(self)->tp_free(self); 1182} 1183 1184static PyTypeObject MultibyteIncrementalDecoder_Type = { 1185 PyVarObject_HEAD_INIT(NULL, 0) 1186 "MultibyteIncrementalDecoder", /* tp_name */ 1187 sizeof(MultibyteIncrementalDecoderObject), /* tp_basicsize */ 1188 0, /* tp_itemsize */ 1189 /* methods */ 1190 (destructor)mbidecoder_dealloc, /* tp_dealloc */ 1191 0, /* tp_print */ 1192 0, /* tp_getattr */ 1193 0, /* tp_setattr */ 1194 0, /* tp_compare */ 1195 0, /* tp_repr */ 1196 0, /* tp_as_number */ 1197 0, /* tp_as_sequence */ 1198 0, /* tp_as_mapping */ 1199 0, /* tp_hash */ 1200 0, /* tp_call */ 1201 0, /* tp_str */ 1202 PyObject_GenericGetAttr, /* tp_getattro */ 1203 0, /* tp_setattro */ 1204 0, /* tp_as_buffer */ 1205 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC 1206 | Py_TPFLAGS_BASETYPE, /* tp_flags */ 1207 0, /* tp_doc */ 1208 (traverseproc)mbidecoder_traverse, /* tp_traverse */ 1209 0, /* tp_clear */ 1210 0, /* tp_richcompare */ 1211 0, /* tp_weaklistoffset */ 1212 0, /* tp_iter */ 1213 0, /* tp_iterext */ 1214 mbidecoder_methods, /* tp_methods */ 1215 0, /* tp_members */ 1216 codecctx_getsets, /* tp_getset */ 1217 0, /* tp_base */ 1218 0, /* tp_dict */ 1219 0, /* tp_descr_get */ 1220 0, /* tp_descr_set */ 1221 0, /* tp_dictoffset */ 1222 mbidecoder_init, /* tp_init */ 1223 0, /* tp_alloc */ 1224 mbidecoder_new, /* tp_new */ 1225}; 1226 1227 1228/** 1229 * MultibyteStreamReader object 1230 */ 1231 1232static PyObject * 1233mbstreamreader_iread(MultibyteStreamReaderObject *self, 1234 const char *method, Py_ssize_t sizehint) 1235{ 1236 MultibyteDecodeBuffer buf; 1237 PyObject *cres; 1238 Py_ssize_t rsize, finalsize = 0; 1239 1240 if (sizehint == 0) 1241 return PyUnicode_FromUnicode(NULL, 0); 1242 1243 buf.outobj = buf.excobj = NULL; 1244 cres = NULL; 1245 1246 for (;;) { 1247 int endoffile; 1248 1249 if (sizehint < 0) 1250 cres = PyObject_CallMethod(self->stream, 1251 (char *)method, NULL); 1252 else 1253 cres = PyObject_CallMethod(self->stream, 1254 (char *)method, "i", sizehint); 1255 if (cres == NULL) 1256 goto errorexit; 1257 1258 if (!PyString_Check(cres)) { 1259 PyErr_SetString(PyExc_TypeError, 1260 "stream function returned a " 1261 "non-string object"); 1262 goto errorexit; 1263 } 1264 1265 endoffile = (PyString_GET_SIZE(cres) == 0); 1266 1267 if (self->pendingsize > 0) { 1268 PyObject *ctr; 1269 char *ctrdata; 1270 1271 if (PyString_GET_SIZE(cres) > PY_SSIZE_T_MAX - self->pendingsize) { 1272 PyErr_NoMemory(); 1273 goto errorexit; 1274 } 1275 rsize = PyString_GET_SIZE(cres) + self->pendingsize; 1276 ctr = PyString_FromStringAndSize(NULL, rsize); 1277 if (ctr == NULL) 1278 goto errorexit; 1279 ctrdata = PyString_AS_STRING(ctr); 1280 memcpy(ctrdata, self->pending, self->pendingsize); 1281 memcpy(ctrdata + self->pendingsize, 1282 PyString_AS_STRING(cres), 1283 PyString_GET_SIZE(cres)); 1284 Py_DECREF(cres); 1285 cres = ctr; 1286 self->pendingsize = 0; 1287 } 1288 1289 rsize = PyString_GET_SIZE(cres); 1290 if (decoder_prepare_buffer(&buf, PyString_AS_STRING(cres), 1291 rsize) != 0) 1292 goto errorexit; 1293 1294 if (rsize > 0 && decoder_feed_buffer( 1295 (MultibyteStatefulDecoderContext *)self, &buf)) 1296 goto errorexit; 1297 1298 if (endoffile || sizehint < 0) { 1299 if (buf.inbuf < buf.inbuf_end && 1300 multibytecodec_decerror(self->codec, &self->state, 1301 &buf, self->errors, MBERR_TOOFEW)) 1302 goto errorexit; 1303 } 1304 1305 if (buf.inbuf < buf.inbuf_end) { /* pending sequence exists */ 1306 if (decoder_append_pending(STATEFUL_DCTX(self), 1307 &buf) != 0) 1308 goto errorexit; 1309 } 1310 1311 finalsize = (Py_ssize_t)(buf.outbuf - 1312 PyUnicode_AS_UNICODE(buf.outobj)); 1313 Py_DECREF(cres); 1314 cres = NULL; 1315 1316 if (sizehint < 0 || finalsize != 0 || rsize == 0) 1317 break; 1318 1319 sizehint = 1; /* read 1 more byte and retry */ 1320 } 1321 1322 if (finalsize != PyUnicode_GET_SIZE(buf.outobj)) 1323 if (PyUnicode_Resize(&buf.outobj, finalsize) == -1) 1324 goto errorexit; 1325 1326 Py_XDECREF(cres); 1327 Py_XDECREF(buf.excobj); 1328 return buf.outobj; 1329 1330errorexit: 1331 Py_XDECREF(cres); 1332 Py_XDECREF(buf.excobj); 1333 Py_XDECREF(buf.outobj); 1334 return NULL; 1335} 1336 1337static PyObject * 1338mbstreamreader_read(MultibyteStreamReaderObject *self, PyObject *args) 1339{ 1340 PyObject *sizeobj = NULL; 1341 Py_ssize_t size; 1342 1343 if (!PyArg_UnpackTuple(args, "read", 0, 1, &sizeobj)) 1344 return NULL; 1345 1346 if (sizeobj == Py_None || sizeobj == NULL) 1347 size = -1; 1348 else if (PyInt_Check(sizeobj)) 1349 size = PyInt_AsSsize_t(sizeobj); 1350 else { 1351 PyErr_SetString(PyExc_TypeError, "arg 1 must be an integer"); 1352 return NULL; 1353 } 1354 1355 return mbstreamreader_iread(self, "read", size); 1356} 1357 1358static PyObject * 1359mbstreamreader_readline(MultibyteStreamReaderObject *self, PyObject *args) 1360{ 1361 PyObject *sizeobj = NULL; 1362 Py_ssize_t size; 1363 1364 if (!PyArg_UnpackTuple(args, "readline", 0, 1, &sizeobj)) 1365 return NULL; 1366 1367 if (sizeobj == Py_None || sizeobj == NULL) 1368 size = -1; 1369 else if (PyInt_Check(sizeobj)) 1370 size = PyInt_AsSsize_t(sizeobj); 1371 else { 1372 PyErr_SetString(PyExc_TypeError, "arg 1 must be an integer"); 1373 return NULL; 1374 } 1375 1376 return mbstreamreader_iread(self, "readline", size); 1377} 1378 1379static PyObject * 1380mbstreamreader_readlines(MultibyteStreamReaderObject *self, PyObject *args) 1381{ 1382 PyObject *sizehintobj = NULL, *r, *sr; 1383 Py_ssize_t sizehint; 1384 1385 if (!PyArg_UnpackTuple(args, "readlines", 0, 1, &sizehintobj)) 1386 return NULL; 1387 1388 if (sizehintobj == Py_None || sizehintobj == NULL) 1389 sizehint = -1; 1390 else if (PyInt_Check(sizehintobj)) 1391 sizehint = PyInt_AsSsize_t(sizehintobj); 1392 else { 1393 PyErr_SetString(PyExc_TypeError, "arg 1 must be an integer"); 1394 return NULL; 1395 } 1396 1397 r = mbstreamreader_iread(self, "read", sizehint); 1398 if (r == NULL) 1399 return NULL; 1400 1401 sr = PyUnicode_Splitlines(r, 1); 1402 Py_DECREF(r); 1403 return sr; 1404} 1405 1406static PyObject * 1407mbstreamreader_reset(MultibyteStreamReaderObject *self) 1408{ 1409 if (self->codec->decreset != NULL && 1410 self->codec->decreset(&self->state, self->codec->config) != 0) 1411 return NULL; 1412 self->pendingsize = 0; 1413 1414 Py_RETURN_NONE; 1415} 1416 1417static struct PyMethodDef mbstreamreader_methods[] = { 1418 {"read", (PyCFunction)mbstreamreader_read, 1419 METH_VARARGS, NULL}, 1420 {"readline", (PyCFunction)mbstreamreader_readline, 1421 METH_VARARGS, NULL}, 1422 {"readlines", (PyCFunction)mbstreamreader_readlines, 1423 METH_VARARGS, NULL}, 1424 {"reset", (PyCFunction)mbstreamreader_reset, 1425 METH_NOARGS, NULL}, 1426 {NULL, NULL}, 1427}; 1428 1429static PyMemberDef mbstreamreader_members[] = { 1430 {"stream", T_OBJECT, 1431 offsetof(MultibyteStreamReaderObject, stream), 1432 READONLY, NULL}, 1433 {NULL,} 1434}; 1435 1436static PyObject * 1437mbstreamreader_new(PyTypeObject *type, PyObject *args, PyObject *kwds) 1438{ 1439 MultibyteStreamReaderObject *self; 1440 PyObject *stream, *codec = NULL; 1441 char *errors = NULL; 1442 1443 if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|s:StreamReader", 1444 streamkwarglist, &stream, &errors)) 1445 return NULL; 1446 1447 self = (MultibyteStreamReaderObject *)type->tp_alloc(type, 0); 1448 if (self == NULL) 1449 return NULL; 1450 1451 codec = PyObject_GetAttrString((PyObject *)type, "codec"); 1452 if (codec == NULL) 1453 goto errorexit; 1454 if (!MultibyteCodec_Check(codec)) { 1455 PyErr_SetString(PyExc_TypeError, "codec is unexpected type"); 1456 goto errorexit; 1457 } 1458 1459 self->codec = ((MultibyteCodecObject *)codec)->codec; 1460 self->stream = stream; 1461 Py_INCREF(stream); 1462 self->pendingsize = 0; 1463 self->errors = internal_error_callback(errors); 1464 if (self->errors == NULL) 1465 goto errorexit; 1466 if (self->codec->decinit != NULL && 1467 self->codec->decinit(&self->state, self->codec->config) != 0) 1468 goto errorexit; 1469 1470 Py_DECREF(codec); 1471 return (PyObject *)self; 1472 1473errorexit: 1474 Py_XDECREF(self); 1475 Py_XDECREF(codec); 1476 return NULL; 1477} 1478 1479static int 1480mbstreamreader_init(PyObject *self, PyObject *args, PyObject *kwds) 1481{ 1482 return 0; 1483} 1484 1485static int 1486mbstreamreader_traverse(MultibyteStreamReaderObject *self, 1487 visitproc visit, void *arg) 1488{ 1489 if (ERROR_ISCUSTOM(self->errors)) 1490 Py_VISIT(self->errors); 1491 Py_VISIT(self->stream); 1492 return 0; 1493} 1494 1495static void 1496mbstreamreader_dealloc(MultibyteStreamReaderObject *self) 1497{ 1498 PyObject_GC_UnTrack(self); 1499 ERROR_DECREF(self->errors); 1500 Py_XDECREF(self->stream); 1501 Py_TYPE(self)->tp_free(self); 1502} 1503 1504static PyTypeObject MultibyteStreamReader_Type = { 1505 PyVarObject_HEAD_INIT(NULL, 0) 1506 "MultibyteStreamReader", /* tp_name */ 1507 sizeof(MultibyteStreamReaderObject), /* tp_basicsize */ 1508 0, /* tp_itemsize */ 1509 /* methods */ 1510 (destructor)mbstreamreader_dealloc, /* tp_dealloc */ 1511 0, /* tp_print */ 1512 0, /* tp_getattr */ 1513 0, /* tp_setattr */ 1514 0, /* tp_compare */ 1515 0, /* tp_repr */ 1516 0, /* tp_as_number */ 1517 0, /* tp_as_sequence */ 1518 0, /* tp_as_mapping */ 1519 0, /* tp_hash */ 1520 0, /* tp_call */ 1521 0, /* tp_str */ 1522 PyObject_GenericGetAttr, /* tp_getattro */ 1523 0, /* tp_setattro */ 1524 0, /* tp_as_buffer */ 1525 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC 1526 | Py_TPFLAGS_BASETYPE, /* tp_flags */ 1527 0, /* tp_doc */ 1528 (traverseproc)mbstreamreader_traverse, /* tp_traverse */ 1529 0, /* tp_clear */ 1530 0, /* tp_richcompare */ 1531 0, /* tp_weaklistoffset */ 1532 0, /* tp_iter */ 1533 0, /* tp_iterext */ 1534 mbstreamreader_methods, /* tp_methods */ 1535 mbstreamreader_members, /* tp_members */ 1536 codecctx_getsets, /* tp_getset */ 1537 0, /* tp_base */ 1538 0, /* tp_dict */ 1539 0, /* tp_descr_get */ 1540 0, /* tp_descr_set */ 1541 0, /* tp_dictoffset */ 1542 mbstreamreader_init, /* tp_init */ 1543 0, /* tp_alloc */ 1544 mbstreamreader_new, /* tp_new */ 1545}; 1546 1547 1548/** 1549 * MultibyteStreamWriter object 1550 */ 1551 1552static int 1553mbstreamwriter_iwrite(MultibyteStreamWriterObject *self, 1554 PyObject *unistr) 1555{ 1556 PyObject *str, *wr; 1557 1558 str = encoder_encode_stateful(STATEFUL_ECTX(self), unistr, 0); 1559 if (str == NULL) 1560 return -1; 1561 1562 wr = PyObject_CallMethod(self->stream, "write", "O", str); 1563 Py_DECREF(str); 1564 if (wr == NULL) 1565 return -1; 1566 1567 Py_DECREF(wr); 1568 return 0; 1569} 1570 1571static PyObject * 1572mbstreamwriter_write(MultibyteStreamWriterObject *self, PyObject *strobj) 1573{ 1574 if (mbstreamwriter_iwrite(self, strobj)) 1575 return NULL; 1576 else 1577 Py_RETURN_NONE; 1578} 1579 1580static PyObject * 1581mbstreamwriter_writelines(MultibyteStreamWriterObject *self, PyObject *lines) 1582{ 1583 PyObject *strobj; 1584 int i, r; 1585 1586 if (!PySequence_Check(lines)) { 1587 PyErr_SetString(PyExc_TypeError, 1588 "arg must be a sequence object"); 1589 return NULL; 1590 } 1591 1592 for (i = 0; i < PySequence_Length(lines); i++) { 1593 /* length can be changed even within this loop */ 1594 strobj = PySequence_GetItem(lines, i); 1595 if (strobj == NULL) 1596 return NULL; 1597 1598 r = mbstreamwriter_iwrite(self, strobj); 1599 Py_DECREF(strobj); 1600 if (r == -1) 1601 return NULL; 1602 } 1603 1604 Py_RETURN_NONE; 1605} 1606 1607static PyObject * 1608mbstreamwriter_reset(MultibyteStreamWriterObject *self) 1609{ 1610 const Py_UNICODE *pending; 1611 PyObject *pwrt; 1612 1613 pending = self->pending; 1614 pwrt = multibytecodec_encode(self->codec, &self->state, 1615 &pending, self->pendingsize, self->errors, 1616 MBENC_FLUSH | MBENC_RESET); 1617 /* some pending buffer can be truncated when UnicodeEncodeError is 1618 * raised on 'strict' mode. but, 'reset' method is designed to 1619 * reset the pending buffer or states so failed string sequence 1620 * ought to be missed */ 1621 self->pendingsize = 0; 1622 if (pwrt == NULL) 1623 return NULL; 1624 1625 if (PyString_Size(pwrt) > 0) { 1626 PyObject *wr; 1627 wr = PyObject_CallMethod(self->stream, "write", "O", pwrt); 1628 if (wr == NULL) { 1629 Py_DECREF(pwrt); 1630 return NULL; 1631 } 1632 } 1633 Py_DECREF(pwrt); 1634 1635 Py_RETURN_NONE; 1636} 1637 1638static PyObject * 1639mbstreamwriter_new(PyTypeObject *type, PyObject *args, PyObject *kwds) 1640{ 1641 MultibyteStreamWriterObject *self; 1642 PyObject *stream, *codec = NULL; 1643 char *errors = NULL; 1644 1645 if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|s:StreamWriter", 1646 streamkwarglist, &stream, &errors)) 1647 return NULL; 1648 1649 self = (MultibyteStreamWriterObject *)type->tp_alloc(type, 0); 1650 if (self == NULL) 1651 return NULL; 1652 1653 codec = PyObject_GetAttrString((PyObject *)type, "codec"); 1654 if (codec == NULL) 1655 goto errorexit; 1656 if (!MultibyteCodec_Check(codec)) { 1657 PyErr_SetString(PyExc_TypeError, "codec is unexpected type"); 1658 goto errorexit; 1659 } 1660 1661 self->codec = ((MultibyteCodecObject *)codec)->codec; 1662 self->stream = stream; 1663 Py_INCREF(stream); 1664 self->pendingsize = 0; 1665 self->errors = internal_error_callback(errors); 1666 if (self->errors == NULL) 1667 goto errorexit; 1668 if (self->codec->encinit != NULL && 1669 self->codec->encinit(&self->state, self->codec->config) != 0) 1670 goto errorexit; 1671 1672 Py_DECREF(codec); 1673 return (PyObject *)self; 1674 1675errorexit: 1676 Py_XDECREF(self); 1677 Py_XDECREF(codec); 1678 return NULL; 1679} 1680 1681static int 1682mbstreamwriter_init(PyObject *self, PyObject *args, PyObject *kwds) 1683{ 1684 return 0; 1685} 1686 1687static int 1688mbstreamwriter_traverse(MultibyteStreamWriterObject *self, 1689 visitproc visit, void *arg) 1690{ 1691 if (ERROR_ISCUSTOM(self->errors)) 1692 Py_VISIT(self->errors); 1693 Py_VISIT(self->stream); 1694 return 0; 1695} 1696 1697static void 1698mbstreamwriter_dealloc(MultibyteStreamWriterObject *self) 1699{ 1700 PyObject_GC_UnTrack(self); 1701 ERROR_DECREF(self->errors); 1702 Py_XDECREF(self->stream); 1703 Py_TYPE(self)->tp_free(self); 1704} 1705 1706static struct PyMethodDef mbstreamwriter_methods[] = { 1707 {"write", (PyCFunction)mbstreamwriter_write, 1708 METH_O, NULL}, 1709 {"writelines", (PyCFunction)mbstreamwriter_writelines, 1710 METH_O, NULL}, 1711 {"reset", (PyCFunction)mbstreamwriter_reset, 1712 METH_NOARGS, NULL}, 1713 {NULL, NULL}, 1714}; 1715 1716static PyMemberDef mbstreamwriter_members[] = { 1717 {"stream", T_OBJECT, 1718 offsetof(MultibyteStreamWriterObject, stream), 1719 READONLY, NULL}, 1720 {NULL,} 1721}; 1722 1723static PyTypeObject MultibyteStreamWriter_Type = { 1724 PyVarObject_HEAD_INIT(NULL, 0) 1725 "MultibyteStreamWriter", /* tp_name */ 1726 sizeof(MultibyteStreamWriterObject), /* tp_basicsize */ 1727 0, /* tp_itemsize */ 1728 /* methods */ 1729 (destructor)mbstreamwriter_dealloc, /* tp_dealloc */ 1730 0, /* tp_print */ 1731 0, /* tp_getattr */ 1732 0, /* tp_setattr */ 1733 0, /* tp_compare */ 1734 0, /* tp_repr */ 1735 0, /* tp_as_number */ 1736 0, /* tp_as_sequence */ 1737 0, /* tp_as_mapping */ 1738 0, /* tp_hash */ 1739 0, /* tp_call */ 1740 0, /* tp_str */ 1741 PyObject_GenericGetAttr, /* tp_getattro */ 1742 0, /* tp_setattro */ 1743 0, /* tp_as_buffer */ 1744 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC 1745 | Py_TPFLAGS_BASETYPE, /* tp_flags */ 1746 0, /* tp_doc */ 1747 (traverseproc)mbstreamwriter_traverse, /* tp_traverse */ 1748 0, /* tp_clear */ 1749 0, /* tp_richcompare */ 1750 0, /* tp_weaklistoffset */ 1751 0, /* tp_iter */ 1752 0, /* tp_iterext */ 1753 mbstreamwriter_methods, /* tp_methods */ 1754 mbstreamwriter_members, /* tp_members */ 1755 codecctx_getsets, /* tp_getset */ 1756 0, /* tp_base */ 1757 0, /* tp_dict */ 1758 0, /* tp_descr_get */ 1759 0, /* tp_descr_set */ 1760 0, /* tp_dictoffset */ 1761 mbstreamwriter_init, /* tp_init */ 1762 0, /* tp_alloc */ 1763 mbstreamwriter_new, /* tp_new */ 1764}; 1765 1766 1767/** 1768 * Exposed factory function 1769 */ 1770 1771static PyObject * 1772__create_codec(PyObject *ignore, PyObject *arg) 1773{ 1774 MultibyteCodecObject *self; 1775 MultibyteCodec *codec; 1776 1777 if (!PyCapsule_IsValid(arg, PyMultibyteCodec_CAPSULE_NAME)) { 1778 PyErr_SetString(PyExc_ValueError, "argument type invalid"); 1779 return NULL; 1780 } 1781 1782 codec = PyCapsule_GetPointer(arg, PyMultibyteCodec_CAPSULE_NAME); 1783 if (codec->codecinit != NULL && codec->codecinit(codec->config) != 0) 1784 return NULL; 1785 1786 self = PyObject_New(MultibyteCodecObject, &MultibyteCodec_Type); 1787 if (self == NULL) 1788 return NULL; 1789 self->codec = codec; 1790 1791 return (PyObject *)self; 1792} 1793 1794static struct PyMethodDef __methods[] = { 1795 {"__create_codec", (PyCFunction)__create_codec, METH_O}, 1796 {NULL, NULL}, 1797}; 1798 1799PyMODINIT_FUNC 1800init_multibytecodec(void) 1801{ 1802 int i; 1803 PyObject *m; 1804 PyTypeObject *typelist[] = { 1805 &MultibyteIncrementalEncoder_Type, 1806 &MultibyteIncrementalDecoder_Type, 1807 &MultibyteStreamReader_Type, 1808 &MultibyteStreamWriter_Type, 1809 NULL 1810 }; 1811 1812 if (PyType_Ready(&MultibyteCodec_Type) < 0) 1813 return; 1814 1815 m = Py_InitModule("_multibytecodec", __methods); 1816 if (m == NULL) 1817 return; 1818 1819 for (i = 0; typelist[i] != NULL; i++) { 1820 if (PyType_Ready(typelist[i]) < 0) 1821 return; 1822 Py_INCREF(typelist[i]); 1823 PyModule_AddObject(m, typelist[i]->tp_name, 1824 (PyObject *)typelist[i]); 1825 } 1826 1827 if (PyErr_Occurred()) 1828 Py_FatalError("can't initialize the _multibytecodec module"); 1829} 1830