1 /*
2 * multibytecodec.c: Common Multibyte Codec Implementation
3 *
4 * Written by Hye-Shik Chang <perky@FreeBSD.org>
5 */
6
7 #define PY_SSIZE_T_CLEAN
8 #include "Python.h"
9 #include "structmember.h"
10 #include "multibytecodec.h"
11
12 typedef struct {
13 const Py_UNICODE *inbuf, *inbuf_top, *inbuf_end;
14 unsigned char *outbuf, *outbuf_end;
15 PyObject *excobj, *outobj;
16 } MultibyteEncodeBuffer;
17
18 typedef struct {
19 const unsigned char *inbuf, *inbuf_top, *inbuf_end;
20 Py_UNICODE *outbuf, *outbuf_end;
21 PyObject *excobj, *outobj;
22 } MultibyteDecodeBuffer;
23
24 PyDoc_STRVAR(MultibyteCodec_Encode__doc__,
25 "I.encode(unicode[, errors]) -> (string, length consumed)\n\
26 \n\
27 Return an encoded string version of `unicode'. errors may be given to\n\
28 set a different error handling scheme. Default is 'strict' meaning that\n\
29 encoding errors raise a UnicodeEncodeError. Other possible values are\n\
30 'ignore', 'replace' and 'xmlcharrefreplace' as well as any other name\n\
31 registered with codecs.register_error that can handle UnicodeEncodeErrors.");
32
33 PyDoc_STRVAR(MultibyteCodec_Decode__doc__,
34 "I.decode(string[, errors]) -> (unicodeobject, length consumed)\n\
35 \n\
36 Decodes `string' using I, an MultibyteCodec instance. errors may be given\n\
37 to set a different error handling scheme. Default is 'strict' meaning\n\
38 that encoding errors raise a UnicodeDecodeError. Other possible values\n\
39 are 'ignore' and 'replace' as well as any other name registered with\n\
40 codecs.register_error that is able to handle UnicodeDecodeErrors.");
41
42 static char *codeckwarglist[] = {"input", "errors", NULL};
43 static char *incnewkwarglist[] = {"errors", NULL};
44 static char *incrementalkwarglist[] = {"input", "final", NULL};
45 static char *streamkwarglist[] = {"stream", "errors", NULL};
46
47 static PyObject *multibytecodec_encode(MultibyteCodec *,
48 MultibyteCodec_State *, const Py_UNICODE **, Py_ssize_t,
49 PyObject *, int);
50
51 #define MBENC_RESET MBENC_MAX<<1 /* reset after an encoding session */
52
53 static PyObject *
make_tuple(PyObject * object,Py_ssize_t len)54 make_tuple(PyObject *object, Py_ssize_t len)
55 {
56 PyObject *v, *w;
57
58 if (object == NULL)
59 return NULL;
60
61 v = PyTuple_New(2);
62 if (v == NULL) {
63 Py_DECREF(object);
64 return NULL;
65 }
66 PyTuple_SET_ITEM(v, 0, object);
67
68 w = PyInt_FromSsize_t(len);
69 if (w == NULL) {
70 Py_DECREF(v);
71 return NULL;
72 }
73 PyTuple_SET_ITEM(v, 1, w);
74
75 return v;
76 }
77
78 static PyObject *
internal_error_callback(const char * errors)79 internal_error_callback(const char *errors)
80 {
81 if (errors == NULL || strcmp(errors, "strict") == 0)
82 return ERROR_STRICT;
83 else if (strcmp(errors, "ignore") == 0)
84 return ERROR_IGNORE;
85 else if (strcmp(errors, "replace") == 0)
86 return ERROR_REPLACE;
87 else
88 return PyString_FromString(errors);
89 }
90
91 static PyObject *
call_error_callback(PyObject * errors,PyObject * exc)92 call_error_callback(PyObject *errors, PyObject *exc)
93 {
94 PyObject *args, *cb, *r;
95
96 assert(PyString_Check(errors));
97 cb = PyCodec_LookupError(PyString_AS_STRING(errors));
98 if (cb == NULL)
99 return NULL;
100
101 args = PyTuple_New(1);
102 if (args == NULL) {
103 Py_DECREF(cb);
104 return NULL;
105 }
106
107 PyTuple_SET_ITEM(args, 0, exc);
108 Py_INCREF(exc);
109
110 r = PyObject_CallObject(cb, args);
111 Py_DECREF(args);
112 Py_DECREF(cb);
113 return r;
114 }
115
116 static PyObject *
codecctx_errors_get(MultibyteStatefulCodecContext * self)117 codecctx_errors_get(MultibyteStatefulCodecContext *self)
118 {
119 const char *errors;
120
121 if (self->errors == ERROR_STRICT)
122 errors = "strict";
123 else if (self->errors == ERROR_IGNORE)
124 errors = "ignore";
125 else if (self->errors == ERROR_REPLACE)
126 errors = "replace";
127 else {
128 Py_INCREF(self->errors);
129 return self->errors;
130 }
131
132 return PyString_FromString(errors);
133 }
134
135 static int
codecctx_errors_set(MultibyteStatefulCodecContext * self,PyObject * value,void * closure)136 codecctx_errors_set(MultibyteStatefulCodecContext *self, PyObject *value,
137 void *closure)
138 {
139 PyObject *cb;
140
141 if (!PyString_Check(value)) {
142 PyErr_SetString(PyExc_TypeError, "errors must be a string");
143 return -1;
144 }
145
146 cb = internal_error_callback(PyString_AS_STRING(value));
147 if (cb == NULL)
148 return -1;
149
150 ERROR_DECREF(self->errors);
151 self->errors = cb;
152 return 0;
153 }
154
155 /* This getset handlers list is used by all the stateful codec objects */
156 static PyGetSetDef codecctx_getsets[] = {
157 {"errors", (getter)codecctx_errors_get,
158 (setter)codecctx_errors_set,
159 PyDoc_STR("how to treat errors")},
160 {NULL,}
161 };
162
163 static int
expand_encodebuffer(MultibyteEncodeBuffer * buf,Py_ssize_t esize)164 expand_encodebuffer(MultibyteEncodeBuffer *buf, Py_ssize_t esize)
165 {
166 Py_ssize_t orgpos, orgsize, incsize;
167
168 orgpos = (Py_ssize_t)((char *)buf->outbuf -
169 PyString_AS_STRING(buf->outobj));
170 orgsize = PyString_GET_SIZE(buf->outobj);
171 incsize = (esize < (orgsize >> 1) ? (orgsize >> 1) | 1 : esize);
172
173 if (orgsize > PY_SSIZE_T_MAX - incsize)
174 return -1;
175
176 if (_PyString_Resize(&buf->outobj, orgsize + incsize) == -1)
177 return -1;
178
179 buf->outbuf = (unsigned char *)PyString_AS_STRING(buf->outobj) +orgpos;
180 buf->outbuf_end = (unsigned char *)PyString_AS_STRING(buf->outobj)
181 + PyString_GET_SIZE(buf->outobj);
182
183 return 0;
184 }
185 #define REQUIRE_ENCODEBUFFER(buf, s) { \
186 if ((s) < 1 || (buf)->outbuf + (s) > (buf)->outbuf_end) \
187 if (expand_encodebuffer(buf, s) == -1) \
188 goto errorexit; \
189 }
190
191 static int
expand_decodebuffer(MultibyteDecodeBuffer * buf,Py_ssize_t esize)192 expand_decodebuffer(MultibyteDecodeBuffer *buf, Py_ssize_t esize)
193 {
194 Py_ssize_t orgpos, orgsize;
195
196 orgpos = (Py_ssize_t)(buf->outbuf - PyUnicode_AS_UNICODE(buf->outobj));
197 orgsize = PyUnicode_GET_SIZE(buf->outobj);
198 if (PyUnicode_Resize(&buf->outobj, orgsize + (
199 esize < (orgsize >> 1) ? (orgsize >> 1) | 1 : esize)) == -1)
200 return -1;
201
202 buf->outbuf = PyUnicode_AS_UNICODE(buf->outobj) + orgpos;
203 buf->outbuf_end = PyUnicode_AS_UNICODE(buf->outobj)
204 + PyUnicode_GET_SIZE(buf->outobj);
205
206 return 0;
207 }
208 #define REQUIRE_DECODEBUFFER(buf, s) { \
209 if ((s) < 1 || (buf)->outbuf + (s) > (buf)->outbuf_end) \
210 if (expand_decodebuffer(buf, s) == -1) \
211 goto errorexit; \
212 }
213
214
215 /**
216 * MultibyteCodec object
217 */
218
219 static int
multibytecodec_encerror(MultibyteCodec * codec,MultibyteCodec_State * state,MultibyteEncodeBuffer * buf,PyObject * errors,Py_ssize_t e)220 multibytecodec_encerror(MultibyteCodec *codec,
221 MultibyteCodec_State *state,
222 MultibyteEncodeBuffer *buf,
223 PyObject *errors, Py_ssize_t e)
224 {
225 PyObject *retobj = NULL, *retstr = NULL, *tobj;
226 Py_ssize_t retstrsize, newpos;
227 Py_ssize_t esize, start, end;
228 const char *reason;
229
230 if (e > 0) {
231 reason = "illegal multibyte sequence";
232 esize = e;
233 }
234 else {
235 switch (e) {
236 case MBERR_TOOSMALL:
237 REQUIRE_ENCODEBUFFER(buf, -1);
238 return 0; /* retry it */
239 case MBERR_TOOFEW:
240 reason = "incomplete multibyte sequence";
241 esize = (Py_ssize_t)(buf->inbuf_end - buf->inbuf);
242 break;
243 case MBERR_INTERNAL:
244 PyErr_SetString(PyExc_RuntimeError,
245 "internal codec error");
246 return -1;
247 default:
248 PyErr_SetString(PyExc_RuntimeError,
249 "unknown runtime error");
250 return -1;
251 }
252 }
253
254 if (errors == ERROR_REPLACE) {
255 const Py_UNICODE replchar = '?', *inbuf = &replchar;
256 Py_ssize_t r;
257
258 for (;;) {
259 Py_ssize_t outleft;
260
261 outleft = (Py_ssize_t)(buf->outbuf_end - buf->outbuf);
262 r = codec->encode(state, codec->config, &inbuf, 1,
263 &buf->outbuf, outleft, 0);
264 if (r == MBERR_TOOSMALL) {
265 REQUIRE_ENCODEBUFFER(buf, -1);
266 continue;
267 }
268 else
269 break;
270 }
271
272 if (r != 0) {
273 REQUIRE_ENCODEBUFFER(buf, 1);
274 *buf->outbuf++ = '?';
275 }
276 }
277 if (errors == ERROR_IGNORE || errors == ERROR_REPLACE) {
278 buf->inbuf += esize;
279 return 0;
280 }
281
282 start = (Py_ssize_t)(buf->inbuf - buf->inbuf_top);
283 end = start + esize;
284
285 /* use cached exception object if available */
286 if (buf->excobj == NULL) {
287 buf->excobj = PyUnicodeEncodeError_Create(codec->encoding,
288 buf->inbuf_top,
289 buf->inbuf_end - buf->inbuf_top,
290 start, end, reason);
291 if (buf->excobj == NULL)
292 goto errorexit;
293 }
294 else
295 if (PyUnicodeEncodeError_SetStart(buf->excobj, start) != 0 ||
296 PyUnicodeEncodeError_SetEnd(buf->excobj, end) != 0 ||
297 PyUnicodeEncodeError_SetReason(buf->excobj, reason) != 0)
298 goto errorexit;
299
300 if (errors == ERROR_STRICT) {
301 PyCodec_StrictErrors(buf->excobj);
302 goto errorexit;
303 }
304
305 retobj = call_error_callback(errors, buf->excobj);
306 if (retobj == NULL)
307 goto errorexit;
308
309 if (!PyTuple_Check(retobj) || PyTuple_GET_SIZE(retobj) != 2 ||
310 !PyUnicode_Check((tobj = PyTuple_GET_ITEM(retobj, 0))) ||
311 !(PyInt_Check(PyTuple_GET_ITEM(retobj, 1)) ||
312 PyLong_Check(PyTuple_GET_ITEM(retobj, 1)))) {
313 PyErr_SetString(PyExc_TypeError,
314 "encoding error handler must return "
315 "(unicode, int) tuple");
316 goto errorexit;
317 }
318
319 {
320 const Py_UNICODE *uraw = PyUnicode_AS_UNICODE(tobj);
321
322 retstr = multibytecodec_encode(codec, state, &uraw,
323 PyUnicode_GET_SIZE(tobj), ERROR_STRICT,
324 MBENC_FLUSH);
325 if (retstr == NULL)
326 goto errorexit;
327 }
328
329 retstrsize = PyString_GET_SIZE(retstr);
330 REQUIRE_ENCODEBUFFER(buf, retstrsize);
331
332 memcpy(buf->outbuf, PyString_AS_STRING(retstr), retstrsize);
333 buf->outbuf += retstrsize;
334
335 newpos = PyInt_AsSsize_t(PyTuple_GET_ITEM(retobj, 1));
336 if (newpos < 0 && !PyErr_Occurred())
337 newpos += (Py_ssize_t)(buf->inbuf_end - buf->inbuf_top);
338 if (newpos < 0 || buf->inbuf_top + newpos > buf->inbuf_end) {
339 PyErr_Clear();
340 PyErr_Format(PyExc_IndexError,
341 "position %zd from error handler out of bounds",
342 newpos);
343 goto errorexit;
344 }
345 buf->inbuf = buf->inbuf_top + newpos;
346
347 Py_DECREF(retobj);
348 Py_DECREF(retstr);
349 return 0;
350
351 errorexit:
352 Py_XDECREF(retobj);
353 Py_XDECREF(retstr);
354 return -1;
355 }
356
357 static int
multibytecodec_decerror(MultibyteCodec * codec,MultibyteCodec_State * state,MultibyteDecodeBuffer * buf,PyObject * errors,Py_ssize_t e)358 multibytecodec_decerror(MultibyteCodec *codec,
359 MultibyteCodec_State *state,
360 MultibyteDecodeBuffer *buf,
361 PyObject *errors, Py_ssize_t e)
362 {
363 PyObject *retobj = NULL, *retuni = NULL;
364 Py_ssize_t retunisize, newpos;
365 const char *reason;
366 Py_ssize_t esize, start, end;
367
368 if (e > 0) {
369 reason = "illegal multibyte sequence";
370 esize = e;
371 }
372 else {
373 switch (e) {
374 case MBERR_TOOSMALL:
375 REQUIRE_DECODEBUFFER(buf, -1);
376 return 0; /* retry it */
377 case MBERR_TOOFEW:
378 reason = "incomplete multibyte sequence";
379 esize = (Py_ssize_t)(buf->inbuf_end - buf->inbuf);
380 break;
381 case MBERR_INTERNAL:
382 PyErr_SetString(PyExc_RuntimeError,
383 "internal codec error");
384 return -1;
385 default:
386 PyErr_SetString(PyExc_RuntimeError,
387 "unknown runtime error");
388 return -1;
389 }
390 }
391
392 if (errors == ERROR_REPLACE) {
393 REQUIRE_DECODEBUFFER(buf, 1);
394 *buf->outbuf++ = Py_UNICODE_REPLACEMENT_CHARACTER;
395 }
396 if (errors == ERROR_IGNORE || errors == ERROR_REPLACE) {
397 buf->inbuf += esize;
398 return 0;
399 }
400
401 start = (Py_ssize_t)(buf->inbuf - buf->inbuf_top);
402 end = start + esize;
403
404 /* use cached exception object if available */
405 if (buf->excobj == NULL) {
406 buf->excobj = PyUnicodeDecodeError_Create(codec->encoding,
407 (const char *)buf->inbuf_top,
408 (Py_ssize_t)(buf->inbuf_end - buf->inbuf_top),
409 start, end, reason);
410 if (buf->excobj == NULL)
411 goto errorexit;
412 }
413 else
414 if (PyUnicodeDecodeError_SetStart(buf->excobj, start) ||
415 PyUnicodeDecodeError_SetEnd(buf->excobj, end) ||
416 PyUnicodeDecodeError_SetReason(buf->excobj, reason))
417 goto errorexit;
418
419 if (errors == ERROR_STRICT) {
420 PyCodec_StrictErrors(buf->excobj);
421 goto errorexit;
422 }
423
424 retobj = call_error_callback(errors, buf->excobj);
425 if (retobj == NULL)
426 goto errorexit;
427
428 if (!PyTuple_Check(retobj) || PyTuple_GET_SIZE(retobj) != 2 ||
429 !PyUnicode_Check((retuni = PyTuple_GET_ITEM(retobj, 0))) ||
430 !(PyInt_Check(PyTuple_GET_ITEM(retobj, 1)) ||
431 PyLong_Check(PyTuple_GET_ITEM(retobj, 1)))) {
432 PyErr_SetString(PyExc_TypeError,
433 "decoding error handler must return "
434 "(unicode, int) tuple");
435 goto errorexit;
436 }
437
438 retunisize = PyUnicode_GET_SIZE(retuni);
439 if (retunisize > 0) {
440 REQUIRE_DECODEBUFFER(buf, retunisize);
441 memcpy((char *)buf->outbuf, PyUnicode_AS_DATA(retuni),
442 retunisize * Py_UNICODE_SIZE);
443 buf->outbuf += retunisize;
444 }
445
446 newpos = PyInt_AsSsize_t(PyTuple_GET_ITEM(retobj, 1));
447 if (newpos < 0 && !PyErr_Occurred())
448 newpos += (Py_ssize_t)(buf->inbuf_end - buf->inbuf_top);
449 if (newpos < 0 || buf->inbuf_top + newpos > buf->inbuf_end) {
450 PyErr_Clear();
451 PyErr_Format(PyExc_IndexError,
452 "position %zd from error handler out of bounds",
453 newpos);
454 goto errorexit;
455 }
456 buf->inbuf = buf->inbuf_top + newpos;
457 Py_DECREF(retobj);
458 return 0;
459
460 errorexit:
461 Py_XDECREF(retobj);
462 return -1;
463 }
464
465 static PyObject *
multibytecodec_encode(MultibyteCodec * codec,MultibyteCodec_State * state,const Py_UNICODE ** data,Py_ssize_t datalen,PyObject * errors,int flags)466 multibytecodec_encode(MultibyteCodec *codec,
467 MultibyteCodec_State *state,
468 const Py_UNICODE **data, Py_ssize_t datalen,
469 PyObject *errors, int flags)
470 {
471 MultibyteEncodeBuffer buf;
472 Py_ssize_t finalsize, r = 0;
473
474 if (datalen == 0 && !(flags & MBENC_RESET))
475 return PyString_FromString("");
476
477 buf.excobj = NULL;
478 buf.inbuf = buf.inbuf_top = *data;
479 buf.inbuf_end = buf.inbuf_top + datalen;
480
481 if (datalen > (PY_SSIZE_T_MAX - 16) / 2) {
482 PyErr_NoMemory();
483 goto errorexit;
484 }
485
486 buf.outobj = PyString_FromStringAndSize(NULL, datalen * 2 + 16);
487 if (buf.outobj == NULL)
488 goto errorexit;
489 buf.outbuf = (unsigned char *)PyString_AS_STRING(buf.outobj);
490 buf.outbuf_end = buf.outbuf + PyString_GET_SIZE(buf.outobj);
491
492 while (buf.inbuf < buf.inbuf_end) {
493 Py_ssize_t inleft, outleft;
494
495 /* we don't reuse inleft and outleft here.
496 * error callbacks can relocate the cursor anywhere on buffer*/
497 inleft = (Py_ssize_t)(buf.inbuf_end - buf.inbuf);
498 outleft = (Py_ssize_t)(buf.outbuf_end - buf.outbuf);
499 r = codec->encode(state, codec->config, &buf.inbuf, inleft,
500 &buf.outbuf, outleft, flags);
501 if ((r == 0) || (r == MBERR_TOOFEW && !(flags & MBENC_FLUSH)))
502 break;
503 else if (multibytecodec_encerror(codec, state, &buf, errors,r))
504 goto errorexit;
505 else if (r == MBERR_TOOFEW)
506 break;
507 }
508
509 if (codec->encreset != NULL && (flags & MBENC_RESET))
510 for (;;) {
511 Py_ssize_t outleft;
512
513 outleft = (Py_ssize_t)(buf.outbuf_end - buf.outbuf);
514 r = codec->encreset(state, codec->config, &buf.outbuf,
515 outleft);
516 if (r == 0)
517 break;
518 else if (multibytecodec_encerror(codec, state,
519 &buf, errors, r))
520 goto errorexit;
521 }
522
523 finalsize = (Py_ssize_t)((char *)buf.outbuf -
524 PyString_AS_STRING(buf.outobj));
525
526 if (finalsize != PyString_GET_SIZE(buf.outobj))
527 if (_PyString_Resize(&buf.outobj, finalsize) == -1)
528 goto errorexit;
529
530 *data = buf.inbuf;
531 Py_XDECREF(buf.excobj);
532 return buf.outobj;
533
534 errorexit:
535 Py_XDECREF(buf.excobj);
536 Py_XDECREF(buf.outobj);
537 return NULL;
538 }
539
540 static PyObject *
MultibyteCodec_Encode(MultibyteCodecObject * self,PyObject * args,PyObject * kwargs)541 MultibyteCodec_Encode(MultibyteCodecObject *self,
542 PyObject *args, PyObject *kwargs)
543 {
544 MultibyteCodec_State state;
545 Py_UNICODE *data;
546 PyObject *errorcb, *r, *arg, *ucvt;
547 const char *errors = NULL;
548 Py_ssize_t datalen;
549
550 if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|z:encode",
551 codeckwarglist, &arg, &errors))
552 return NULL;
553
554 if (PyUnicode_Check(arg))
555 ucvt = NULL;
556 else {
557 arg = ucvt = PyObject_Unicode(arg);
558 if (arg == NULL)
559 return NULL;
560 else if (!PyUnicode_Check(arg)) {
561 PyErr_SetString(PyExc_TypeError,
562 "couldn't convert the object to unicode.");
563 Py_DECREF(ucvt);
564 return NULL;
565 }
566 }
567
568 data = PyUnicode_AS_UNICODE(arg);
569 datalen = PyUnicode_GET_SIZE(arg);
570
571 errorcb = internal_error_callback(errors);
572 if (errorcb == NULL) {
573 Py_XDECREF(ucvt);
574 return NULL;
575 }
576
577 if (self->codec->encinit != NULL &&
578 self->codec->encinit(&state, self->codec->config) != 0)
579 goto errorexit;
580 r = multibytecodec_encode(self->codec, &state,
581 (const Py_UNICODE **)&data, datalen, errorcb,
582 MBENC_FLUSH | MBENC_RESET);
583 if (r == NULL)
584 goto errorexit;
585
586 ERROR_DECREF(errorcb);
587 Py_XDECREF(ucvt);
588 return make_tuple(r, datalen);
589
590 errorexit:
591 ERROR_DECREF(errorcb);
592 Py_XDECREF(ucvt);
593 return NULL;
594 }
595
596 static PyObject *
MultibyteCodec_Decode(MultibyteCodecObject * self,PyObject * args,PyObject * kwargs)597 MultibyteCodec_Decode(MultibyteCodecObject *self,
598 PyObject *args, PyObject *kwargs)
599 {
600 MultibyteCodec_State state;
601 MultibyteDecodeBuffer buf;
602 PyObject *errorcb;
603 Py_buffer pdata;
604 const char *data, *errors = NULL;
605 Py_ssize_t datalen, finalsize;
606
607 if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s*|z:decode",
608 codeckwarglist, &pdata, &errors))
609 return NULL;
610 data = pdata.buf;
611 datalen = pdata.len;
612
613 errorcb = internal_error_callback(errors);
614 if (errorcb == NULL) {
615 PyBuffer_Release(&pdata);
616 return NULL;
617 }
618
619 if (datalen == 0) {
620 PyBuffer_Release(&pdata);
621 ERROR_DECREF(errorcb);
622 return make_tuple(PyUnicode_FromUnicode(NULL, 0), 0);
623 }
624
625 buf.excobj = NULL;
626 buf.inbuf = buf.inbuf_top = (unsigned char *)data;
627 buf.inbuf_end = buf.inbuf_top + datalen;
628 buf.outobj = PyUnicode_FromUnicode(NULL, datalen);
629 if (buf.outobj == NULL)
630 goto errorexit;
631 buf.outbuf = PyUnicode_AS_UNICODE(buf.outobj);
632 buf.outbuf_end = buf.outbuf + PyUnicode_GET_SIZE(buf.outobj);
633
634 if (self->codec->decinit != NULL &&
635 self->codec->decinit(&state, self->codec->config) != 0)
636 goto errorexit;
637
638 while (buf.inbuf < buf.inbuf_end) {
639 Py_ssize_t inleft, outleft, r;
640
641 inleft = (Py_ssize_t)(buf.inbuf_end - buf.inbuf);
642 outleft = (Py_ssize_t)(buf.outbuf_end - buf.outbuf);
643
644 r = self->codec->decode(&state, self->codec->config,
645 &buf.inbuf, inleft, &buf.outbuf, outleft);
646 if (r == 0)
647 break;
648 else if (multibytecodec_decerror(self->codec, &state,
649 &buf, errorcb, r))
650 goto errorexit;
651 }
652
653 finalsize = (Py_ssize_t)(buf.outbuf -
654 PyUnicode_AS_UNICODE(buf.outobj));
655
656 if (finalsize != PyUnicode_GET_SIZE(buf.outobj))
657 if (PyUnicode_Resize(&buf.outobj, finalsize) == -1)
658 goto errorexit;
659
660 PyBuffer_Release(&pdata);
661 Py_XDECREF(buf.excobj);
662 ERROR_DECREF(errorcb);
663 return make_tuple(buf.outobj, datalen);
664
665 errorexit:
666 PyBuffer_Release(&pdata);
667 ERROR_DECREF(errorcb);
668 Py_XDECREF(buf.excobj);
669 Py_XDECREF(buf.outobj);
670
671 return NULL;
672 }
673
674 static struct PyMethodDef multibytecodec_methods[] = {
675 {"encode", (PyCFunction)MultibyteCodec_Encode,
676 METH_VARARGS | METH_KEYWORDS,
677 MultibyteCodec_Encode__doc__},
678 {"decode", (PyCFunction)MultibyteCodec_Decode,
679 METH_VARARGS | METH_KEYWORDS,
680 MultibyteCodec_Decode__doc__},
681 {NULL, NULL},
682 };
683
684 static void
multibytecodec_dealloc(MultibyteCodecObject * self)685 multibytecodec_dealloc(MultibyteCodecObject *self)
686 {
687 PyObject_Del(self);
688 }
689
690 static PyTypeObject MultibyteCodec_Type = {
691 PyVarObject_HEAD_INIT(NULL, 0)
692 "MultibyteCodec", /* tp_name */
693 sizeof(MultibyteCodecObject), /* tp_basicsize */
694 0, /* tp_itemsize */
695 /* methods */
696 (destructor)multibytecodec_dealloc, /* tp_dealloc */
697 0, /* tp_print */
698 0, /* tp_getattr */
699 0, /* tp_setattr */
700 0, /* tp_compare */
701 0, /* tp_repr */
702 0, /* tp_as_number */
703 0, /* tp_as_sequence */
704 0, /* tp_as_mapping */
705 0, /* tp_hash */
706 0, /* tp_call */
707 0, /* tp_str */
708 PyObject_GenericGetAttr, /* tp_getattro */
709 0, /* tp_setattro */
710 0, /* tp_as_buffer */
711 Py_TPFLAGS_DEFAULT, /* tp_flags */
712 0, /* tp_doc */
713 0, /* tp_traverse */
714 0, /* tp_clear */
715 0, /* tp_richcompare */
716 0, /* tp_weaklistoffset */
717 0, /* tp_iter */
718 0, /* tp_iterext */
719 multibytecodec_methods, /* tp_methods */
720 };
721
722
723 /**
724 * Utility functions for stateful codec mechanism
725 */
726
727 #define STATEFUL_DCTX(o) ((MultibyteStatefulDecoderContext *)(o))
728 #define STATEFUL_ECTX(o) ((MultibyteStatefulEncoderContext *)(o))
729
730 static PyObject *
encoder_encode_stateful(MultibyteStatefulEncoderContext * ctx,PyObject * unistr,int final)731 encoder_encode_stateful(MultibyteStatefulEncoderContext *ctx,
732 PyObject *unistr, int final)
733 {
734 PyObject *ucvt, *r = NULL;
735 Py_UNICODE *inbuf, *inbuf_end, *inbuf_tmp = NULL;
736 Py_ssize_t datalen, origpending;
737
738 if (PyUnicode_Check(unistr))
739 ucvt = NULL;
740 else {
741 unistr = ucvt = PyObject_Unicode(unistr);
742 if (unistr == NULL)
743 return NULL;
744 else if (!PyUnicode_Check(unistr)) {
745 PyErr_SetString(PyExc_TypeError,
746 "couldn't convert the object to unicode.");
747 Py_DECREF(ucvt);
748 return NULL;
749 }
750 }
751
752 datalen = PyUnicode_GET_SIZE(unistr);
753 origpending = ctx->pendingsize;
754
755 if (origpending > 0) {
756 if (datalen > PY_SSIZE_T_MAX - ctx->pendingsize) {
757 PyErr_NoMemory();
758 /* inbuf_tmp == NULL */
759 goto errorexit;
760 }
761 inbuf_tmp = PyMem_New(Py_UNICODE, datalen + ctx->pendingsize);
762 if (inbuf_tmp == NULL)
763 goto errorexit;
764 memcpy(inbuf_tmp, ctx->pending,
765 Py_UNICODE_SIZE * ctx->pendingsize);
766 memcpy(inbuf_tmp + ctx->pendingsize,
767 PyUnicode_AS_UNICODE(unistr),
768 Py_UNICODE_SIZE * datalen);
769 datalen += ctx->pendingsize;
770 ctx->pendingsize = 0;
771 inbuf = inbuf_tmp;
772 }
773 else
774 inbuf = (Py_UNICODE *)PyUnicode_AS_UNICODE(unistr);
775
776 inbuf_end = inbuf + datalen;
777
778 r = multibytecodec_encode(ctx->codec, &ctx->state,
779 (const Py_UNICODE **)&inbuf, datalen,
780 ctx->errors, final ? MBENC_FLUSH | MBENC_RESET : 0);
781 if (r == NULL) {
782 /* recover the original pending buffer */
783 if (origpending > 0)
784 memcpy(ctx->pending, inbuf_tmp,
785 Py_UNICODE_SIZE * origpending);
786 ctx->pendingsize = origpending;
787 goto errorexit;
788 }
789
790 if (inbuf < inbuf_end) {
791 ctx->pendingsize = (Py_ssize_t)(inbuf_end - inbuf);
792 if (ctx->pendingsize > MAXENCPENDING) {
793 /* normal codecs can't reach here */
794 ctx->pendingsize = 0;
795 PyErr_SetString(PyExc_UnicodeError,
796 "pending buffer overflow");
797 goto errorexit;
798 }
799 memcpy(ctx->pending, inbuf,
800 ctx->pendingsize * Py_UNICODE_SIZE);
801 }
802
803 if (inbuf_tmp != NULL)
804 PyMem_Del(inbuf_tmp);
805 Py_XDECREF(ucvt);
806 return r;
807
808 errorexit:
809 if (inbuf_tmp != NULL)
810 PyMem_Del(inbuf_tmp);
811 Py_XDECREF(r);
812 Py_XDECREF(ucvt);
813 return NULL;
814 }
815
816 static int
decoder_append_pending(MultibyteStatefulDecoderContext * ctx,MultibyteDecodeBuffer * buf)817 decoder_append_pending(MultibyteStatefulDecoderContext *ctx,
818 MultibyteDecodeBuffer *buf)
819 {
820 Py_ssize_t npendings;
821
822 npendings = (Py_ssize_t)(buf->inbuf_end - buf->inbuf);
823 if (npendings + ctx->pendingsize > MAXDECPENDING ||
824 npendings > PY_SSIZE_T_MAX - ctx->pendingsize) {
825 PyErr_SetString(PyExc_UnicodeError, "pending buffer overflow");
826 return -1;
827 }
828 memcpy(ctx->pending + ctx->pendingsize, buf->inbuf, npendings);
829 ctx->pendingsize += npendings;
830 return 0;
831 }
832
833 static int
decoder_prepare_buffer(MultibyteDecodeBuffer * buf,const char * data,Py_ssize_t size)834 decoder_prepare_buffer(MultibyteDecodeBuffer *buf, const char *data,
835 Py_ssize_t size)
836 {
837 buf->inbuf = buf->inbuf_top = (const unsigned char *)data;
838 buf->inbuf_end = buf->inbuf_top + size;
839 if (buf->outobj == NULL) { /* only if outobj is not allocated yet */
840 buf->outobj = PyUnicode_FromUnicode(NULL, size);
841 if (buf->outobj == NULL)
842 return -1;
843 buf->outbuf = PyUnicode_AS_UNICODE(buf->outobj);
844 buf->outbuf_end = buf->outbuf +
845 PyUnicode_GET_SIZE(buf->outobj);
846 }
847
848 return 0;
849 }
850
851 static int
decoder_feed_buffer(MultibyteStatefulDecoderContext * ctx,MultibyteDecodeBuffer * buf)852 decoder_feed_buffer(MultibyteStatefulDecoderContext *ctx,
853 MultibyteDecodeBuffer *buf)
854 {
855 while (buf->inbuf < buf->inbuf_end) {
856 Py_ssize_t inleft, outleft;
857 Py_ssize_t r;
858
859 inleft = (Py_ssize_t)(buf->inbuf_end - buf->inbuf);
860 outleft = (Py_ssize_t)(buf->outbuf_end - buf->outbuf);
861
862 r = ctx->codec->decode(&ctx->state, ctx->codec->config,
863 &buf->inbuf, inleft, &buf->outbuf, outleft);
864 if (r == 0 || r == MBERR_TOOFEW)
865 break;
866 else if (multibytecodec_decerror(ctx->codec, &ctx->state,
867 buf, ctx->errors, r))
868 return -1;
869 }
870 return 0;
871 }
872
873
874 /**
875 * MultibyteIncrementalEncoder object
876 */
877
878 static PyObject *
mbiencoder_encode(MultibyteIncrementalEncoderObject * self,PyObject * args,PyObject * kwargs)879 mbiencoder_encode(MultibyteIncrementalEncoderObject *self,
880 PyObject *args, PyObject *kwargs)
881 {
882 PyObject *data;
883 int final = 0;
884
885 if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|i:encode",
886 incrementalkwarglist, &data, &final))
887 return NULL;
888
889 return encoder_encode_stateful(STATEFUL_ECTX(self), data, final);
890 }
891
892 static PyObject *
mbiencoder_reset(MultibyteIncrementalEncoderObject * self)893 mbiencoder_reset(MultibyteIncrementalEncoderObject *self)
894 {
895 if (self->codec->decreset != NULL &&
896 self->codec->decreset(&self->state, self->codec->config) != 0)
897 return NULL;
898 self->pendingsize = 0;
899
900 Py_RETURN_NONE;
901 }
902
903 static struct PyMethodDef mbiencoder_methods[] = {
904 {"encode", (PyCFunction)mbiencoder_encode,
905 METH_VARARGS | METH_KEYWORDS, NULL},
906 {"reset", (PyCFunction)mbiencoder_reset,
907 METH_NOARGS, NULL},
908 {NULL, NULL},
909 };
910
911 static PyObject *
mbiencoder_new(PyTypeObject * type,PyObject * args,PyObject * kwds)912 mbiencoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
913 {
914 MultibyteIncrementalEncoderObject *self;
915 PyObject *codec = NULL;
916 char *errors = NULL;
917
918 if (!PyArg_ParseTupleAndKeywords(args, kwds, "|s:IncrementalEncoder",
919 incnewkwarglist, &errors))
920 return NULL;
921
922 self = (MultibyteIncrementalEncoderObject *)type->tp_alloc(type, 0);
923 if (self == NULL)
924 return NULL;
925
926 codec = PyObject_GetAttrString((PyObject *)type, "codec");
927 if (codec == NULL)
928 goto errorexit;
929 if (!MultibyteCodec_Check(codec)) {
930 PyErr_SetString(PyExc_TypeError, "codec is unexpected type");
931 goto errorexit;
932 }
933
934 self->codec = ((MultibyteCodecObject *)codec)->codec;
935 self->pendingsize = 0;
936 self->errors = internal_error_callback(errors);
937 if (self->errors == NULL)
938 goto errorexit;
939 if (self->codec->encinit != NULL &&
940 self->codec->encinit(&self->state, self->codec->config) != 0)
941 goto errorexit;
942
943 Py_DECREF(codec);
944 return (PyObject *)self;
945
946 errorexit:
947 Py_XDECREF(self);
948 Py_XDECREF(codec);
949 return NULL;
950 }
951
952 static int
mbiencoder_init(PyObject * self,PyObject * args,PyObject * kwds)953 mbiencoder_init(PyObject *self, PyObject *args, PyObject *kwds)
954 {
955 return 0;
956 }
957
958 static int
mbiencoder_traverse(MultibyteIncrementalEncoderObject * self,visitproc visit,void * arg)959 mbiencoder_traverse(MultibyteIncrementalEncoderObject *self,
960 visitproc visit, void *arg)
961 {
962 if (ERROR_ISCUSTOM(self->errors))
963 Py_VISIT(self->errors);
964 return 0;
965 }
966
967 static void
mbiencoder_dealloc(MultibyteIncrementalEncoderObject * self)968 mbiencoder_dealloc(MultibyteIncrementalEncoderObject *self)
969 {
970 PyObject_GC_UnTrack(self);
971 ERROR_DECREF(self->errors);
972 Py_TYPE(self)->tp_free(self);
973 }
974
975 static PyTypeObject MultibyteIncrementalEncoder_Type = {
976 PyVarObject_HEAD_INIT(NULL, 0)
977 "MultibyteIncrementalEncoder", /* tp_name */
978 sizeof(MultibyteIncrementalEncoderObject), /* tp_basicsize */
979 0, /* tp_itemsize */
980 /* methods */
981 (destructor)mbiencoder_dealloc, /* tp_dealloc */
982 0, /* tp_print */
983 0, /* tp_getattr */
984 0, /* tp_setattr */
985 0, /* tp_compare */
986 0, /* tp_repr */
987 0, /* tp_as_number */
988 0, /* tp_as_sequence */
989 0, /* tp_as_mapping */
990 0, /* tp_hash */
991 0, /* tp_call */
992 0, /* tp_str */
993 PyObject_GenericGetAttr, /* tp_getattro */
994 0, /* tp_setattro */
995 0, /* tp_as_buffer */
996 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC
997 | Py_TPFLAGS_BASETYPE, /* tp_flags */
998 0, /* tp_doc */
999 (traverseproc)mbiencoder_traverse, /* tp_traverse */
1000 0, /* tp_clear */
1001 0, /* tp_richcompare */
1002 0, /* tp_weaklistoffset */
1003 0, /* tp_iter */
1004 0, /* tp_iterext */
1005 mbiencoder_methods, /* tp_methods */
1006 0, /* tp_members */
1007 codecctx_getsets, /* tp_getset */
1008 0, /* tp_base */
1009 0, /* tp_dict */
1010 0, /* tp_descr_get */
1011 0, /* tp_descr_set */
1012 0, /* tp_dictoffset */
1013 mbiencoder_init, /* tp_init */
1014 0, /* tp_alloc */
1015 mbiencoder_new, /* tp_new */
1016 };
1017
1018
1019 /**
1020 * MultibyteIncrementalDecoder object
1021 */
1022
1023 static PyObject *
mbidecoder_decode(MultibyteIncrementalDecoderObject * self,PyObject * args,PyObject * kwargs)1024 mbidecoder_decode(MultibyteIncrementalDecoderObject *self,
1025 PyObject *args, PyObject *kwargs)
1026 {
1027 MultibyteDecodeBuffer buf;
1028 char *data, *wdata = NULL;
1029 Py_buffer pdata;
1030 Py_ssize_t wsize, finalsize = 0, size, origpending;
1031 int final = 0;
1032
1033 if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s*|i:decode",
1034 incrementalkwarglist, &pdata, &final))
1035 return NULL;
1036 data = pdata.buf;
1037 size = pdata.len;
1038
1039 buf.outobj = buf.excobj = NULL;
1040 origpending = self->pendingsize;
1041
1042 if (self->pendingsize == 0) {
1043 wsize = size;
1044 wdata = data;
1045 }
1046 else {
1047 if (size > PY_SSIZE_T_MAX - self->pendingsize) {
1048 PyErr_NoMemory();
1049 goto errorexit;
1050 }
1051 wsize = size + self->pendingsize;
1052 wdata = PyMem_Malloc(wsize);
1053 if (wdata == NULL)
1054 goto errorexit;
1055 memcpy(wdata, self->pending, self->pendingsize);
1056 memcpy(wdata + self->pendingsize, data, size);
1057 self->pendingsize = 0;
1058 }
1059
1060 if (decoder_prepare_buffer(&buf, wdata, wsize) != 0)
1061 goto errorexit;
1062
1063 if (decoder_feed_buffer(STATEFUL_DCTX(self), &buf))
1064 goto errorexit;
1065
1066 if (final && buf.inbuf < buf.inbuf_end) {
1067 if (multibytecodec_decerror(self->codec, &self->state,
1068 &buf, self->errors, MBERR_TOOFEW)) {
1069 /* recover the original pending buffer */
1070 memcpy(self->pending, wdata, origpending);
1071 self->pendingsize = origpending;
1072 goto errorexit;
1073 }
1074 }
1075
1076 if (buf.inbuf < buf.inbuf_end) { /* pending sequence still exists */
1077 if (decoder_append_pending(STATEFUL_DCTX(self), &buf) != 0)
1078 goto errorexit;
1079 }
1080
1081 finalsize = (Py_ssize_t)(buf.outbuf - PyUnicode_AS_UNICODE(buf.outobj));
1082 if (finalsize != PyUnicode_GET_SIZE(buf.outobj))
1083 if (PyUnicode_Resize(&buf.outobj, finalsize) == -1)
1084 goto errorexit;
1085
1086 PyBuffer_Release(&pdata);
1087 if (wdata != data)
1088 PyMem_Del(wdata);
1089 Py_XDECREF(buf.excobj);
1090 return buf.outobj;
1091
1092 errorexit:
1093 PyBuffer_Release(&pdata);
1094 if (wdata != NULL && wdata != data)
1095 PyMem_Del(wdata);
1096 Py_XDECREF(buf.excobj);
1097 Py_XDECREF(buf.outobj);
1098 return NULL;
1099 }
1100
1101 static PyObject *
mbidecoder_reset(MultibyteIncrementalDecoderObject * self)1102 mbidecoder_reset(MultibyteIncrementalDecoderObject *self)
1103 {
1104 if (self->codec->decreset != NULL &&
1105 self->codec->decreset(&self->state, self->codec->config) != 0)
1106 return NULL;
1107 self->pendingsize = 0;
1108
1109 Py_RETURN_NONE;
1110 }
1111
1112 static struct PyMethodDef mbidecoder_methods[] = {
1113 {"decode", (PyCFunction)mbidecoder_decode,
1114 METH_VARARGS | METH_KEYWORDS, NULL},
1115 {"reset", (PyCFunction)mbidecoder_reset,
1116 METH_NOARGS, NULL},
1117 {NULL, NULL},
1118 };
1119
1120 static PyObject *
mbidecoder_new(PyTypeObject * type,PyObject * args,PyObject * kwds)1121 mbidecoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
1122 {
1123 MultibyteIncrementalDecoderObject *self;
1124 PyObject *codec = NULL;
1125 char *errors = NULL;
1126
1127 if (!PyArg_ParseTupleAndKeywords(args, kwds, "|s:IncrementalDecoder",
1128 incnewkwarglist, &errors))
1129 return NULL;
1130
1131 self = (MultibyteIncrementalDecoderObject *)type->tp_alloc(type, 0);
1132 if (self == NULL)
1133 return NULL;
1134
1135 codec = PyObject_GetAttrString((PyObject *)type, "codec");
1136 if (codec == NULL)
1137 goto errorexit;
1138 if (!MultibyteCodec_Check(codec)) {
1139 PyErr_SetString(PyExc_TypeError, "codec is unexpected type");
1140 goto errorexit;
1141 }
1142
1143 self->codec = ((MultibyteCodecObject *)codec)->codec;
1144 self->pendingsize = 0;
1145 self->errors = internal_error_callback(errors);
1146 if (self->errors == NULL)
1147 goto errorexit;
1148 if (self->codec->decinit != NULL &&
1149 self->codec->decinit(&self->state, self->codec->config) != 0)
1150 goto errorexit;
1151
1152 Py_DECREF(codec);
1153 return (PyObject *)self;
1154
1155 errorexit:
1156 Py_XDECREF(self);
1157 Py_XDECREF(codec);
1158 return NULL;
1159 }
1160
1161 static int
mbidecoder_init(PyObject * self,PyObject * args,PyObject * kwds)1162 mbidecoder_init(PyObject *self, PyObject *args, PyObject *kwds)
1163 {
1164 return 0;
1165 }
1166
1167 static int
mbidecoder_traverse(MultibyteIncrementalDecoderObject * self,visitproc visit,void * arg)1168 mbidecoder_traverse(MultibyteIncrementalDecoderObject *self,
1169 visitproc visit, void *arg)
1170 {
1171 if (ERROR_ISCUSTOM(self->errors))
1172 Py_VISIT(self->errors);
1173 return 0;
1174 }
1175
1176 static void
mbidecoder_dealloc(MultibyteIncrementalDecoderObject * self)1177 mbidecoder_dealloc(MultibyteIncrementalDecoderObject *self)
1178 {
1179 PyObject_GC_UnTrack(self);
1180 ERROR_DECREF(self->errors);
1181 Py_TYPE(self)->tp_free(self);
1182 }
1183
1184 static PyTypeObject MultibyteIncrementalDecoder_Type = {
1185 PyVarObject_HEAD_INIT(NULL, 0)
1186 "MultibyteIncrementalDecoder", /* tp_name */
1187 sizeof(MultibyteIncrementalDecoderObject), /* tp_basicsize */
1188 0, /* tp_itemsize */
1189 /* methods */
1190 (destructor)mbidecoder_dealloc, /* tp_dealloc */
1191 0, /* tp_print */
1192 0, /* tp_getattr */
1193 0, /* tp_setattr */
1194 0, /* tp_compare */
1195 0, /* tp_repr */
1196 0, /* tp_as_number */
1197 0, /* tp_as_sequence */
1198 0, /* tp_as_mapping */
1199 0, /* tp_hash */
1200 0, /* tp_call */
1201 0, /* tp_str */
1202 PyObject_GenericGetAttr, /* tp_getattro */
1203 0, /* tp_setattro */
1204 0, /* tp_as_buffer */
1205 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC
1206 | Py_TPFLAGS_BASETYPE, /* tp_flags */
1207 0, /* tp_doc */
1208 (traverseproc)mbidecoder_traverse, /* tp_traverse */
1209 0, /* tp_clear */
1210 0, /* tp_richcompare */
1211 0, /* tp_weaklistoffset */
1212 0, /* tp_iter */
1213 0, /* tp_iterext */
1214 mbidecoder_methods, /* tp_methods */
1215 0, /* tp_members */
1216 codecctx_getsets, /* tp_getset */
1217 0, /* tp_base */
1218 0, /* tp_dict */
1219 0, /* tp_descr_get */
1220 0, /* tp_descr_set */
1221 0, /* tp_dictoffset */
1222 mbidecoder_init, /* tp_init */
1223 0, /* tp_alloc */
1224 mbidecoder_new, /* tp_new */
1225 };
1226
1227
1228 /**
1229 * MultibyteStreamReader object
1230 */
1231
1232 static PyObject *
mbstreamreader_iread(MultibyteStreamReaderObject * self,const char * method,Py_ssize_t sizehint)1233 mbstreamreader_iread(MultibyteStreamReaderObject *self,
1234 const char *method, Py_ssize_t sizehint)
1235 {
1236 MultibyteDecodeBuffer buf;
1237 PyObject *cres;
1238 Py_ssize_t rsize, finalsize = 0;
1239
1240 if (sizehint == 0)
1241 return PyUnicode_FromUnicode(NULL, 0);
1242
1243 buf.outobj = buf.excobj = NULL;
1244 cres = NULL;
1245
1246 for (;;) {
1247 int endoffile;
1248
1249 if (sizehint < 0)
1250 cres = PyObject_CallMethod(self->stream,
1251 (char *)method, NULL);
1252 else
1253 cres = PyObject_CallMethod(self->stream,
1254 (char *)method, "i", sizehint);
1255 if (cres == NULL)
1256 goto errorexit;
1257
1258 if (!PyString_Check(cres)) {
1259 PyErr_SetString(PyExc_TypeError,
1260 "stream function returned a "
1261 "non-string object");
1262 goto errorexit;
1263 }
1264
1265 endoffile = (PyString_GET_SIZE(cres) == 0);
1266
1267 if (self->pendingsize > 0) {
1268 PyObject *ctr;
1269 char *ctrdata;
1270
1271 if (PyString_GET_SIZE(cres) > PY_SSIZE_T_MAX - self->pendingsize) {
1272 PyErr_NoMemory();
1273 goto errorexit;
1274 }
1275 rsize = PyString_GET_SIZE(cres) + self->pendingsize;
1276 ctr = PyString_FromStringAndSize(NULL, rsize);
1277 if (ctr == NULL)
1278 goto errorexit;
1279 ctrdata = PyString_AS_STRING(ctr);
1280 memcpy(ctrdata, self->pending, self->pendingsize);
1281 memcpy(ctrdata + self->pendingsize,
1282 PyString_AS_STRING(cres),
1283 PyString_GET_SIZE(cres));
1284 Py_DECREF(cres);
1285 cres = ctr;
1286 self->pendingsize = 0;
1287 }
1288
1289 rsize = PyString_GET_SIZE(cres);
1290 if (decoder_prepare_buffer(&buf, PyString_AS_STRING(cres),
1291 rsize) != 0)
1292 goto errorexit;
1293
1294 if (rsize > 0 && decoder_feed_buffer(
1295 (MultibyteStatefulDecoderContext *)self, &buf))
1296 goto errorexit;
1297
1298 if (endoffile || sizehint < 0) {
1299 if (buf.inbuf < buf.inbuf_end &&
1300 multibytecodec_decerror(self->codec, &self->state,
1301 &buf, self->errors, MBERR_TOOFEW))
1302 goto errorexit;
1303 }
1304
1305 if (buf.inbuf < buf.inbuf_end) { /* pending sequence exists */
1306 if (decoder_append_pending(STATEFUL_DCTX(self),
1307 &buf) != 0)
1308 goto errorexit;
1309 }
1310
1311 finalsize = (Py_ssize_t)(buf.outbuf -
1312 PyUnicode_AS_UNICODE(buf.outobj));
1313 Py_DECREF(cres);
1314 cres = NULL;
1315
1316 if (sizehint < 0 || finalsize != 0 || rsize == 0)
1317 break;
1318
1319 sizehint = 1; /* read 1 more byte and retry */
1320 }
1321
1322 if (finalsize != PyUnicode_GET_SIZE(buf.outobj))
1323 if (PyUnicode_Resize(&buf.outobj, finalsize) == -1)
1324 goto errorexit;
1325
1326 Py_XDECREF(cres);
1327 Py_XDECREF(buf.excobj);
1328 return buf.outobj;
1329
1330 errorexit:
1331 Py_XDECREF(cres);
1332 Py_XDECREF(buf.excobj);
1333 Py_XDECREF(buf.outobj);
1334 return NULL;
1335 }
1336
1337 static PyObject *
mbstreamreader_read(MultibyteStreamReaderObject * self,PyObject * args)1338 mbstreamreader_read(MultibyteStreamReaderObject *self, PyObject *args)
1339 {
1340 PyObject *sizeobj = NULL;
1341 Py_ssize_t size;
1342
1343 if (!PyArg_UnpackTuple(args, "read", 0, 1, &sizeobj))
1344 return NULL;
1345
1346 if (sizeobj == Py_None || sizeobj == NULL)
1347 size = -1;
1348 else if (PyInt_Check(sizeobj))
1349 size = PyInt_AsSsize_t(sizeobj);
1350 else {
1351 PyErr_SetString(PyExc_TypeError, "arg 1 must be an integer");
1352 return NULL;
1353 }
1354
1355 return mbstreamreader_iread(self, "read", size);
1356 }
1357
1358 static PyObject *
mbstreamreader_readline(MultibyteStreamReaderObject * self,PyObject * args)1359 mbstreamreader_readline(MultibyteStreamReaderObject *self, PyObject *args)
1360 {
1361 PyObject *sizeobj = NULL;
1362 Py_ssize_t size;
1363
1364 if (!PyArg_UnpackTuple(args, "readline", 0, 1, &sizeobj))
1365 return NULL;
1366
1367 if (sizeobj == Py_None || sizeobj == NULL)
1368 size = -1;
1369 else if (PyInt_Check(sizeobj))
1370 size = PyInt_AsSsize_t(sizeobj);
1371 else {
1372 PyErr_SetString(PyExc_TypeError, "arg 1 must be an integer");
1373 return NULL;
1374 }
1375
1376 return mbstreamreader_iread(self, "readline", size);
1377 }
1378
1379 static PyObject *
mbstreamreader_readlines(MultibyteStreamReaderObject * self,PyObject * args)1380 mbstreamreader_readlines(MultibyteStreamReaderObject *self, PyObject *args)
1381 {
1382 PyObject *sizehintobj = NULL, *r, *sr;
1383 Py_ssize_t sizehint;
1384
1385 if (!PyArg_UnpackTuple(args, "readlines", 0, 1, &sizehintobj))
1386 return NULL;
1387
1388 if (sizehintobj == Py_None || sizehintobj == NULL)
1389 sizehint = -1;
1390 else if (PyInt_Check(sizehintobj))
1391 sizehint = PyInt_AsSsize_t(sizehintobj);
1392 else {
1393 PyErr_SetString(PyExc_TypeError, "arg 1 must be an integer");
1394 return NULL;
1395 }
1396
1397 r = mbstreamreader_iread(self, "read", sizehint);
1398 if (r == NULL)
1399 return NULL;
1400
1401 sr = PyUnicode_Splitlines(r, 1);
1402 Py_DECREF(r);
1403 return sr;
1404 }
1405
1406 static PyObject *
mbstreamreader_reset(MultibyteStreamReaderObject * self)1407 mbstreamreader_reset(MultibyteStreamReaderObject *self)
1408 {
1409 if (self->codec->decreset != NULL &&
1410 self->codec->decreset(&self->state, self->codec->config) != 0)
1411 return NULL;
1412 self->pendingsize = 0;
1413
1414 Py_RETURN_NONE;
1415 }
1416
1417 static struct PyMethodDef mbstreamreader_methods[] = {
1418 {"read", (PyCFunction)mbstreamreader_read,
1419 METH_VARARGS, NULL},
1420 {"readline", (PyCFunction)mbstreamreader_readline,
1421 METH_VARARGS, NULL},
1422 {"readlines", (PyCFunction)mbstreamreader_readlines,
1423 METH_VARARGS, NULL},
1424 {"reset", (PyCFunction)mbstreamreader_reset,
1425 METH_NOARGS, NULL},
1426 {NULL, NULL},
1427 };
1428
1429 static PyMemberDef mbstreamreader_members[] = {
1430 {"stream", T_OBJECT,
1431 offsetof(MultibyteStreamReaderObject, stream),
1432 READONLY, NULL},
1433 {NULL,}
1434 };
1435
1436 static PyObject *
mbstreamreader_new(PyTypeObject * type,PyObject * args,PyObject * kwds)1437 mbstreamreader_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
1438 {
1439 MultibyteStreamReaderObject *self;
1440 PyObject *stream, *codec = NULL;
1441 char *errors = NULL;
1442
1443 if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|s:StreamReader",
1444 streamkwarglist, &stream, &errors))
1445 return NULL;
1446
1447 self = (MultibyteStreamReaderObject *)type->tp_alloc(type, 0);
1448 if (self == NULL)
1449 return NULL;
1450
1451 codec = PyObject_GetAttrString((PyObject *)type, "codec");
1452 if (codec == NULL)
1453 goto errorexit;
1454 if (!MultibyteCodec_Check(codec)) {
1455 PyErr_SetString(PyExc_TypeError, "codec is unexpected type");
1456 goto errorexit;
1457 }
1458
1459 self->codec = ((MultibyteCodecObject *)codec)->codec;
1460 self->stream = stream;
1461 Py_INCREF(stream);
1462 self->pendingsize = 0;
1463 self->errors = internal_error_callback(errors);
1464 if (self->errors == NULL)
1465 goto errorexit;
1466 if (self->codec->decinit != NULL &&
1467 self->codec->decinit(&self->state, self->codec->config) != 0)
1468 goto errorexit;
1469
1470 Py_DECREF(codec);
1471 return (PyObject *)self;
1472
1473 errorexit:
1474 Py_XDECREF(self);
1475 Py_XDECREF(codec);
1476 return NULL;
1477 }
1478
1479 static int
mbstreamreader_init(PyObject * self,PyObject * args,PyObject * kwds)1480 mbstreamreader_init(PyObject *self, PyObject *args, PyObject *kwds)
1481 {
1482 return 0;
1483 }
1484
1485 static int
mbstreamreader_traverse(MultibyteStreamReaderObject * self,visitproc visit,void * arg)1486 mbstreamreader_traverse(MultibyteStreamReaderObject *self,
1487 visitproc visit, void *arg)
1488 {
1489 if (ERROR_ISCUSTOM(self->errors))
1490 Py_VISIT(self->errors);
1491 Py_VISIT(self->stream);
1492 return 0;
1493 }
1494
1495 static void
mbstreamreader_dealloc(MultibyteStreamReaderObject * self)1496 mbstreamreader_dealloc(MultibyteStreamReaderObject *self)
1497 {
1498 PyObject_GC_UnTrack(self);
1499 ERROR_DECREF(self->errors);
1500 Py_XDECREF(self->stream);
1501 Py_TYPE(self)->tp_free(self);
1502 }
1503
1504 static PyTypeObject MultibyteStreamReader_Type = {
1505 PyVarObject_HEAD_INIT(NULL, 0)
1506 "MultibyteStreamReader", /* tp_name */
1507 sizeof(MultibyteStreamReaderObject), /* tp_basicsize */
1508 0, /* tp_itemsize */
1509 /* methods */
1510 (destructor)mbstreamreader_dealloc, /* tp_dealloc */
1511 0, /* tp_print */
1512 0, /* tp_getattr */
1513 0, /* tp_setattr */
1514 0, /* tp_compare */
1515 0, /* tp_repr */
1516 0, /* tp_as_number */
1517 0, /* tp_as_sequence */
1518 0, /* tp_as_mapping */
1519 0, /* tp_hash */
1520 0, /* tp_call */
1521 0, /* tp_str */
1522 PyObject_GenericGetAttr, /* tp_getattro */
1523 0, /* tp_setattro */
1524 0, /* tp_as_buffer */
1525 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC
1526 | Py_TPFLAGS_BASETYPE, /* tp_flags */
1527 0, /* tp_doc */
1528 (traverseproc)mbstreamreader_traverse, /* tp_traverse */
1529 0, /* tp_clear */
1530 0, /* tp_richcompare */
1531 0, /* tp_weaklistoffset */
1532 0, /* tp_iter */
1533 0, /* tp_iterext */
1534 mbstreamreader_methods, /* tp_methods */
1535 mbstreamreader_members, /* tp_members */
1536 codecctx_getsets, /* tp_getset */
1537 0, /* tp_base */
1538 0, /* tp_dict */
1539 0, /* tp_descr_get */
1540 0, /* tp_descr_set */
1541 0, /* tp_dictoffset */
1542 mbstreamreader_init, /* tp_init */
1543 0, /* tp_alloc */
1544 mbstreamreader_new, /* tp_new */
1545 };
1546
1547
1548 /**
1549 * MultibyteStreamWriter object
1550 */
1551
1552 static int
mbstreamwriter_iwrite(MultibyteStreamWriterObject * self,PyObject * unistr)1553 mbstreamwriter_iwrite(MultibyteStreamWriterObject *self,
1554 PyObject *unistr)
1555 {
1556 PyObject *str, *wr;
1557
1558 str = encoder_encode_stateful(STATEFUL_ECTX(self), unistr, 0);
1559 if (str == NULL)
1560 return -1;
1561
1562 wr = PyObject_CallMethod(self->stream, "write", "O", str);
1563 Py_DECREF(str);
1564 if (wr == NULL)
1565 return -1;
1566
1567 Py_DECREF(wr);
1568 return 0;
1569 }
1570
1571 static PyObject *
mbstreamwriter_write(MultibyteStreamWriterObject * self,PyObject * strobj)1572 mbstreamwriter_write(MultibyteStreamWriterObject *self, PyObject *strobj)
1573 {
1574 if (mbstreamwriter_iwrite(self, strobj))
1575 return NULL;
1576 else
1577 Py_RETURN_NONE;
1578 }
1579
1580 static PyObject *
mbstreamwriter_writelines(MultibyteStreamWriterObject * self,PyObject * lines)1581 mbstreamwriter_writelines(MultibyteStreamWriterObject *self, PyObject *lines)
1582 {
1583 PyObject *strobj;
1584 int i, r;
1585
1586 if (!PySequence_Check(lines)) {
1587 PyErr_SetString(PyExc_TypeError,
1588 "arg must be a sequence object");
1589 return NULL;
1590 }
1591
1592 for (i = 0; i < PySequence_Length(lines); i++) {
1593 /* length can be changed even within this loop */
1594 strobj = PySequence_GetItem(lines, i);
1595 if (strobj == NULL)
1596 return NULL;
1597
1598 r = mbstreamwriter_iwrite(self, strobj);
1599 Py_DECREF(strobj);
1600 if (r == -1)
1601 return NULL;
1602 }
1603
1604 Py_RETURN_NONE;
1605 }
1606
1607 static PyObject *
mbstreamwriter_reset(MultibyteStreamWriterObject * self)1608 mbstreamwriter_reset(MultibyteStreamWriterObject *self)
1609 {
1610 const Py_UNICODE *pending;
1611 PyObject *pwrt;
1612
1613 pending = self->pending;
1614 pwrt = multibytecodec_encode(self->codec, &self->state,
1615 &pending, self->pendingsize, self->errors,
1616 MBENC_FLUSH | MBENC_RESET);
1617 /* some pending buffer can be truncated when UnicodeEncodeError is
1618 * raised on 'strict' mode. but, 'reset' method is designed to
1619 * reset the pending buffer or states so failed string sequence
1620 * ought to be missed */
1621 self->pendingsize = 0;
1622 if (pwrt == NULL)
1623 return NULL;
1624
1625 if (PyString_Size(pwrt) > 0) {
1626 PyObject *wr;
1627 wr = PyObject_CallMethod(self->stream, "write", "O", pwrt);
1628 if (wr == NULL) {
1629 Py_DECREF(pwrt);
1630 return NULL;
1631 }
1632 }
1633 Py_DECREF(pwrt);
1634
1635 Py_RETURN_NONE;
1636 }
1637
1638 static PyObject *
mbstreamwriter_new(PyTypeObject * type,PyObject * args,PyObject * kwds)1639 mbstreamwriter_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
1640 {
1641 MultibyteStreamWriterObject *self;
1642 PyObject *stream, *codec = NULL;
1643 char *errors = NULL;
1644
1645 if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|s:StreamWriter",
1646 streamkwarglist, &stream, &errors))
1647 return NULL;
1648
1649 self = (MultibyteStreamWriterObject *)type->tp_alloc(type, 0);
1650 if (self == NULL)
1651 return NULL;
1652
1653 codec = PyObject_GetAttrString((PyObject *)type, "codec");
1654 if (codec == NULL)
1655 goto errorexit;
1656 if (!MultibyteCodec_Check(codec)) {
1657 PyErr_SetString(PyExc_TypeError, "codec is unexpected type");
1658 goto errorexit;
1659 }
1660
1661 self->codec = ((MultibyteCodecObject *)codec)->codec;
1662 self->stream = stream;
1663 Py_INCREF(stream);
1664 self->pendingsize = 0;
1665 self->errors = internal_error_callback(errors);
1666 if (self->errors == NULL)
1667 goto errorexit;
1668 if (self->codec->encinit != NULL &&
1669 self->codec->encinit(&self->state, self->codec->config) != 0)
1670 goto errorexit;
1671
1672 Py_DECREF(codec);
1673 return (PyObject *)self;
1674
1675 errorexit:
1676 Py_XDECREF(self);
1677 Py_XDECREF(codec);
1678 return NULL;
1679 }
1680
1681 static int
mbstreamwriter_init(PyObject * self,PyObject * args,PyObject * kwds)1682 mbstreamwriter_init(PyObject *self, PyObject *args, PyObject *kwds)
1683 {
1684 return 0;
1685 }
1686
1687 static int
mbstreamwriter_traverse(MultibyteStreamWriterObject * self,visitproc visit,void * arg)1688 mbstreamwriter_traverse(MultibyteStreamWriterObject *self,
1689 visitproc visit, void *arg)
1690 {
1691 if (ERROR_ISCUSTOM(self->errors))
1692 Py_VISIT(self->errors);
1693 Py_VISIT(self->stream);
1694 return 0;
1695 }
1696
1697 static void
mbstreamwriter_dealloc(MultibyteStreamWriterObject * self)1698 mbstreamwriter_dealloc(MultibyteStreamWriterObject *self)
1699 {
1700 PyObject_GC_UnTrack(self);
1701 ERROR_DECREF(self->errors);
1702 Py_XDECREF(self->stream);
1703 Py_TYPE(self)->tp_free(self);
1704 }
1705
1706 static struct PyMethodDef mbstreamwriter_methods[] = {
1707 {"write", (PyCFunction)mbstreamwriter_write,
1708 METH_O, NULL},
1709 {"writelines", (PyCFunction)mbstreamwriter_writelines,
1710 METH_O, NULL},
1711 {"reset", (PyCFunction)mbstreamwriter_reset,
1712 METH_NOARGS, NULL},
1713 {NULL, NULL},
1714 };
1715
1716 static PyMemberDef mbstreamwriter_members[] = {
1717 {"stream", T_OBJECT,
1718 offsetof(MultibyteStreamWriterObject, stream),
1719 READONLY, NULL},
1720 {NULL,}
1721 };
1722
1723 static PyTypeObject MultibyteStreamWriter_Type = {
1724 PyVarObject_HEAD_INIT(NULL, 0)
1725 "MultibyteStreamWriter", /* tp_name */
1726 sizeof(MultibyteStreamWriterObject), /* tp_basicsize */
1727 0, /* tp_itemsize */
1728 /* methods */
1729 (destructor)mbstreamwriter_dealloc, /* tp_dealloc */
1730 0, /* tp_print */
1731 0, /* tp_getattr */
1732 0, /* tp_setattr */
1733 0, /* tp_compare */
1734 0, /* tp_repr */
1735 0, /* tp_as_number */
1736 0, /* tp_as_sequence */
1737 0, /* tp_as_mapping */
1738 0, /* tp_hash */
1739 0, /* tp_call */
1740 0, /* tp_str */
1741 PyObject_GenericGetAttr, /* tp_getattro */
1742 0, /* tp_setattro */
1743 0, /* tp_as_buffer */
1744 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC
1745 | Py_TPFLAGS_BASETYPE, /* tp_flags */
1746 0, /* tp_doc */
1747 (traverseproc)mbstreamwriter_traverse, /* tp_traverse */
1748 0, /* tp_clear */
1749 0, /* tp_richcompare */
1750 0, /* tp_weaklistoffset */
1751 0, /* tp_iter */
1752 0, /* tp_iterext */
1753 mbstreamwriter_methods, /* tp_methods */
1754 mbstreamwriter_members, /* tp_members */
1755 codecctx_getsets, /* tp_getset */
1756 0, /* tp_base */
1757 0, /* tp_dict */
1758 0, /* tp_descr_get */
1759 0, /* tp_descr_set */
1760 0, /* tp_dictoffset */
1761 mbstreamwriter_init, /* tp_init */
1762 0, /* tp_alloc */
1763 mbstreamwriter_new, /* tp_new */
1764 };
1765
1766
1767 /**
1768 * Exposed factory function
1769 */
1770
1771 static PyObject *
__create_codec(PyObject * ignore,PyObject * arg)1772 __create_codec(PyObject *ignore, PyObject *arg)
1773 {
1774 MultibyteCodecObject *self;
1775 MultibyteCodec *codec;
1776
1777 if (!PyCapsule_IsValid(arg, PyMultibyteCodec_CAPSULE_NAME)) {
1778 PyErr_SetString(PyExc_ValueError, "argument type invalid");
1779 return NULL;
1780 }
1781
1782 codec = PyCapsule_GetPointer(arg, PyMultibyteCodec_CAPSULE_NAME);
1783 if (codec->codecinit != NULL && codec->codecinit(codec->config) != 0)
1784 return NULL;
1785
1786 self = PyObject_New(MultibyteCodecObject, &MultibyteCodec_Type);
1787 if (self == NULL)
1788 return NULL;
1789 self->codec = codec;
1790
1791 return (PyObject *)self;
1792 }
1793
1794 static struct PyMethodDef __methods[] = {
1795 {"__create_codec", (PyCFunction)__create_codec, METH_O},
1796 {NULL, NULL},
1797 };
1798
1799 PyMODINIT_FUNC
init_multibytecodec(void)1800 init_multibytecodec(void)
1801 {
1802 int i;
1803 PyObject *m;
1804 PyTypeObject *typelist[] = {
1805 &MultibyteIncrementalEncoder_Type,
1806 &MultibyteIncrementalDecoder_Type,
1807 &MultibyteStreamReader_Type,
1808 &MultibyteStreamWriter_Type,
1809 NULL
1810 };
1811
1812 if (PyType_Ready(&MultibyteCodec_Type) < 0)
1813 return;
1814
1815 m = Py_InitModule("_multibytecodec", __methods);
1816 if (m == NULL)
1817 return;
1818
1819 for (i = 0; typelist[i] != NULL; i++) {
1820 if (PyType_Ready(typelist[i]) < 0)
1821 return;
1822 Py_INCREF(typelist[i]);
1823 PyModule_AddObject(m, typelist[i]->tp_name,
1824 (PyObject *)typelist[i]);
1825 }
1826
1827 if (PyErr_Occurred())
1828 Py_FatalError("can't initialize the _multibytecodec module");
1829 }
1830