| /* | |
| * multibytecodec.c: Common Multibyte Codec Implementation | |
| * | |
| * Written by Hye-Shik Chang <perky@FreeBSD.org> | |
| */ | |
| #ifndef Py_BUILD_CORE_BUILTIN | |
| # define Py_BUILD_CORE_MODULE 1 | |
| #endif | |
| #include "Python.h" | |
| #include "multibytecodec.h" | |
| #include "clinic/multibytecodec.c.h" | |
| #include "pycore_tuple.h" // _PyTuple_FromPairSteal | |
| #include <stddef.h> // offsetof() | |
| #define MODULE_NAME "_multibytecodec" | |
| typedef struct { | |
| PyTypeObject *encoder_type; | |
| PyTypeObject *decoder_type; | |
| PyTypeObject *reader_type; | |
| PyTypeObject *writer_type; | |
| PyTypeObject *multibytecodec_type; | |
| PyObject *str_write; | |
| } module_state; | |
| static module_state * | |
| get_module_state(PyObject *module) | |
| { | |
| module_state *state = PyModule_GetState(module); | |
| assert(state != NULL); | |
| return state; | |
| } | |
| static struct PyModuleDef _multibytecodecmodule; | |
| static module_state * | |
| find_state_by_def(PyTypeObject *type) | |
| { | |
| PyObject *module = PyType_GetModuleByDef(type, &_multibytecodecmodule); | |
| assert(module != NULL); | |
| return get_module_state(module); | |
| } | |
| #define clinic_get_state() find_state_by_def(type) | |
| /*[clinic input] | |
| module _multibytecodec | |
| class _multibytecodec.MultibyteCodec "MultibyteCodecObject *" "clinic_get_state()->multibytecodec_type" | |
| class _multibytecodec.MultibyteIncrementalEncoder "MultibyteIncrementalEncoderObject *" "clinic_get_state()->encoder_type" | |
| class _multibytecodec.MultibyteIncrementalDecoder "MultibyteIncrementalDecoderObject *" "clinic_get_state()->decoder_type" | |
| class _multibytecodec.MultibyteStreamReader "MultibyteStreamReaderObject *" "clinic_get_state()->reader_type" | |
| class _multibytecodec.MultibyteStreamWriter "MultibyteStreamWriterObject *" "clinic_get_state()->writer_type" | |
| [clinic start generated code]*/ | |
| /*[clinic end generated code: output=da39a3ee5e6b4b0d input=305a76dfdd24b99c]*/ | |
| #undef clinic_get_state | |
| #define _MultibyteCodec_CAST(op) ((MultibyteCodec *)(op)) | |
| #define _MultibyteCodecObject_CAST(op) ((MultibyteCodecObject *)(op)) | |
| #define _MultibyteStatefulCodecContext_CAST(op) \ | |
| ((MultibyteStatefulCodecContext *)(op)) | |
| #define _MultibyteStatefulEncoderContext_CAST(op) \ | |
| ((MultibyteStatefulEncoderContext *)(op)) | |
| #define _MultibyteStatefulDecoderContext_CAST(op) \ | |
| ((MultibyteStatefulDecoderContext *)(op)) | |
| #define _MultibyteIncrementalEncoderObject_CAST(op) \ | |
| ((MultibyteIncrementalEncoderObject *)(op)) | |
| #define _MultibyteIncrementalDecoderObject_CAST(op) \ | |
| ((MultibyteIncrementalDecoderObject *)(op)) | |
| #define _MultibyteStreamReaderObject_CAST(op) \ | |
| ((MultibyteStreamReaderObject *)(op)) | |
| #define _MultibyteStreamWriterObject_CAST(op) \ | |
| ((MultibyteStreamWriterObject *)(op)) | |
| typedef struct { | |
| PyObject *inobj; | |
| Py_ssize_t inpos, inlen; | |
| unsigned char *outbuf, *outbuf_end; | |
| PyObject *excobj; | |
| PyBytesWriter *writer; | |
| } MultibyteEncodeBuffer; | |
| typedef struct { | |
| const unsigned char *inbuf, *inbuf_top, *inbuf_end; | |
| PyObject *excobj; | |
| _PyUnicodeWriter writer; | |
| } MultibyteDecodeBuffer; | |
| static char *incnewkwarglist[] = {"errors", NULL}; | |
| static char *streamkwarglist[] = {"stream", "errors", NULL}; | |
| static PyObject *multibytecodec_encode(const MultibyteCodec *, | |
| MultibyteCodec_State *, PyObject *, Py_ssize_t *, | |
| PyObject *, int); | |
| #define MBENC_RESET MBENC_MAX<<1 /* reset after an encoding session */ | |
| static PyObject * | |
| make_tuple(PyObject *object, Py_ssize_t len) | |
| { | |
| if (object == NULL) { | |
| return NULL; | |
| } | |
| PyObject* len_obj = PyLong_FromSsize_t(len); | |
| if (len_obj == NULL) { | |
| Py_DECREF(object); | |
| return NULL; | |
| } | |
| return _PyTuple_FromPairSteal(object, len_obj); | |
| } | |
| static PyObject * | |
| internal_error_callback(const char *errors) | |
| { | |
| if (errors == NULL || strcmp(errors, "strict") == 0) | |
| return ERROR_STRICT; | |
| else if (strcmp(errors, "ignore") == 0) | |
| return ERROR_IGNORE; | |
| else if (strcmp(errors, "replace") == 0) | |
| return ERROR_REPLACE; | |
| else | |
| return PyUnicode_FromString(errors); | |
| } | |
| static PyObject * | |
| call_error_callback(PyObject *errors, PyObject *exc) | |
| { | |
| PyObject *cb, *r; | |
| const char *str; | |
| assert(PyUnicode_Check(errors)); | |
| str = PyUnicode_AsUTF8(errors); | |
| if (str == NULL) | |
| return NULL; | |
| cb = PyCodec_LookupError(str); | |
| if (cb == NULL) | |
| return NULL; | |
| r = PyObject_CallOneArg(cb, exc); | |
| Py_DECREF(cb); | |
| return r; | |
| } | |
| static PyObject * | |
| codecctx_errors_get(PyObject *op, void *Py_UNUSED(closure)) | |
| { | |
| const char *errors; | |
| MultibyteStatefulCodecContext *self = _MultibyteStatefulCodecContext_CAST(op); | |
| if (self->errors == ERROR_STRICT) | |
| errors = "strict"; | |
| else if (self->errors == ERROR_IGNORE) | |
| errors = "ignore"; | |
| else if (self->errors == ERROR_REPLACE) | |
| errors = "replace"; | |
| else { | |
| return Py_NewRef(self->errors); | |
| } | |
| return PyUnicode_FromString(errors); | |
| } | |
| static int | |
| codecctx_errors_set(PyObject *op, PyObject *value, void *Py_UNUSED(closure)) | |
| { | |
| PyObject *cb; | |
| const char *str; | |
| MultibyteStatefulCodecContext *self = _MultibyteStatefulCodecContext_CAST(op); | |
| if (value == NULL) { | |
| PyErr_SetString(PyExc_AttributeError, "cannot delete attribute"); | |
| return -1; | |
| } | |
| if (!PyUnicode_Check(value)) { | |
| PyErr_SetString(PyExc_TypeError, "errors must be a string"); | |
| return -1; | |
| } | |
| str = PyUnicode_AsUTF8(value); | |
| if (str == NULL) | |
| return -1; | |
| cb = internal_error_callback(str); | |
| if (cb == NULL) | |
| return -1; | |
| ERROR_DECREF(self->errors); | |
| self->errors = cb; | |
| return 0; | |
| } | |
| /* This getset handlers list is used by all the stateful codec objects */ | |
| static PyGetSetDef codecctx_getsets[] = { | |
| {"errors", codecctx_errors_get, codecctx_errors_set, | |
| PyDoc_STR("how to treat errors")}, | |
| {NULL,} | |
| }; | |
| static int | |
| expand_encodebuffer(MultibyteEncodeBuffer *buf, Py_ssize_t esize) | |
| { | |
| Py_ssize_t orgpos, orgsize, incsize; | |
| orgpos = (Py_ssize_t)((char *)buf->outbuf - | |
| (char *)PyBytesWriter_GetData(buf->writer)); | |
| orgsize = PyBytesWriter_GetSize(buf->writer); | |
| incsize = (esize < (orgsize >> 1) ? (orgsize >> 1) | 1 : esize); | |
| if (orgsize > PY_SSIZE_T_MAX - incsize) { | |
| PyErr_NoMemory(); | |
| return -1; | |
| } | |
| if (PyBytesWriter_Resize(buf->writer, orgsize + incsize) == -1) | |
| return -1; | |
| unsigned char *data = PyBytesWriter_GetData(buf->writer); | |
| buf->outbuf = data + orgpos; | |
| buf->outbuf_end = data + PyBytesWriter_GetSize(buf->writer); | |
| return 0; | |
| } | |
| #define REQUIRE_ENCODEBUFFER(buf, s) do { \ | |
| if ((s) < 0 || (s) > (buf)->outbuf_end - (buf)->outbuf) \ | |
| if (expand_encodebuffer(buf, s) == -1) \ | |
| goto errorexit; \ | |
| } while(0) | |
| /** | |
| * MultibyteCodec object | |
| */ | |
| static int | |
| multibytecodec_encerror(const MultibyteCodec *codec, | |
| MultibyteCodec_State *state, | |
| MultibyteEncodeBuffer *buf, | |
| PyObject *errors, Py_ssize_t e) | |
| { | |
| PyObject *retobj = NULL, *retstr = NULL, *tobj; | |
| Py_ssize_t retstrsize, newpos; | |
| Py_ssize_t esize, start, end; | |
| const char *reason; | |
| if (e > 0) { | |
| reason = "illegal multibyte sequence"; | |
| esize = e; | |
| } | |
| else { | |
| switch (e) { | |
| case MBERR_TOOSMALL: | |
| REQUIRE_ENCODEBUFFER(buf, -1); | |
| return 0; /* retry it */ | |
| case MBERR_TOOFEW: | |
| reason = "incomplete multibyte sequence"; | |
| esize = (Py_ssize_t)buf->inpos; | |
| break; | |
| case MBERR_INTERNAL: | |
| PyErr_SetString(PyExc_RuntimeError, | |
| "internal codec error"); | |
| return -1; | |
| default: | |
| PyErr_SetString(PyExc_RuntimeError, | |
| "unknown runtime error"); | |
| return -1; | |
| } | |
| } | |
| if (errors == ERROR_REPLACE) { | |
| PyObject *replchar; | |
| Py_ssize_t r; | |
| Py_ssize_t inpos; | |
| int kind; | |
| const void *data; | |
| replchar = PyUnicode_FromOrdinal('?'); | |
| if (replchar == NULL) | |
| goto errorexit; | |
| kind = PyUnicode_KIND(replchar); | |
| data = PyUnicode_DATA(replchar); | |
| inpos = 0; | |
| for (;;) { | |
| Py_ssize_t outleft = (Py_ssize_t)(buf->outbuf_end - buf->outbuf); | |
| r = codec->encode(state, codec, | |
| kind, data, &inpos, 1, | |
| &buf->outbuf, outleft, 0); | |
| if (r == MBERR_TOOSMALL) { | |
| REQUIRE_ENCODEBUFFER(buf, -1); | |
| continue; | |
| } | |
| else | |
| break; | |
| } | |
| Py_DECREF(replchar); | |
| if (r != 0) { | |
| REQUIRE_ENCODEBUFFER(buf, 1); | |
| *buf->outbuf++ = '?'; | |
| } | |
| } | |
| if (errors == ERROR_IGNORE || errors == ERROR_REPLACE) { | |
| buf->inpos += esize; | |
| return 0; | |
| } | |
| start = (Py_ssize_t)buf->inpos; | |
| end = start + esize; | |
| /* use cached exception object if available */ | |
| if (buf->excobj == NULL) { | |
| buf->excobj = PyObject_CallFunction(PyExc_UnicodeEncodeError, | |
| "sOnns", | |
| codec->encoding, buf->inobj, | |
| start, end, reason); | |
| if (buf->excobj == NULL) | |
| goto errorexit; | |
| } | |
| else | |
| if (PyUnicodeEncodeError_SetStart(buf->excobj, start) != 0 || | |
| PyUnicodeEncodeError_SetEnd(buf->excobj, end) != 0 || | |
| PyUnicodeEncodeError_SetReason(buf->excobj, reason) != 0) | |
| goto errorexit; | |
| if (errors == ERROR_STRICT) { | |
| PyCodec_StrictErrors(buf->excobj); | |
| goto errorexit; | |
| } | |
| retobj = call_error_callback(errors, buf->excobj); | |
| if (retobj == NULL) | |
| goto errorexit; | |
| if (!PyTuple_Check(retobj) || PyTuple_GET_SIZE(retobj) != 2 || | |
| (!PyUnicode_Check((tobj = PyTuple_GET_ITEM(retobj, 0))) && !PyBytes_Check(tobj)) || | |
| !PyLong_Check(PyTuple_GET_ITEM(retobj, 1))) { | |
| PyErr_SetString(PyExc_TypeError, | |
| "encoding error handler must return " | |
| "(str, int) tuple"); | |
| goto errorexit; | |
| } | |
| if (PyUnicode_Check(tobj)) { | |
| Py_ssize_t inpos; | |
| retstr = multibytecodec_encode(codec, state, tobj, | |
| &inpos, ERROR_STRICT, | |
| MBENC_FLUSH); | |
| if (retstr == NULL) | |
| goto errorexit; | |
| } | |
| else { | |
| retstr = Py_NewRef(tobj); | |
| } | |
| assert(PyBytes_Check(retstr)); | |
| retstrsize = PyBytes_GET_SIZE(retstr); | |
| if (retstrsize > 0) { | |
| REQUIRE_ENCODEBUFFER(buf, retstrsize); | |
| memcpy(buf->outbuf, PyBytes_AS_STRING(retstr), retstrsize); | |
| buf->outbuf += retstrsize; | |
| } | |
| newpos = PyLong_AsSsize_t(PyTuple_GET_ITEM(retobj, 1)); | |
| if (newpos < 0 && !PyErr_Occurred()) | |
| newpos += (Py_ssize_t)buf->inlen; | |
| if (newpos < 0 || newpos > buf->inlen) { | |
| PyErr_Clear(); | |
| PyErr_Format(PyExc_IndexError, | |
| "position %zd from error handler out of bounds", | |
| newpos); | |
| goto errorexit; | |
| } | |
| buf->inpos = newpos; | |
| Py_DECREF(retobj); | |
| Py_DECREF(retstr); | |
| return 0; | |
| errorexit: | |
| Py_XDECREF(retobj); | |
| Py_XDECREF(retstr); | |
| return -1; | |
| } | |
| static int | |
| multibytecodec_decerror(const MultibyteCodec *codec, | |
| MultibyteCodec_State *state, | |
| MultibyteDecodeBuffer *buf, | |
| PyObject *errors, Py_ssize_t e) | |
| { | |
| PyObject *retobj = NULL, *retuni = NULL; | |
| Py_ssize_t newpos; | |
| const char *reason; | |
| Py_ssize_t esize, start, end; | |
| if (e > 0) { | |
| reason = "illegal multibyte sequence"; | |
| esize = e; | |
| } | |
| else { | |
| switch (e) { | |
| case MBERR_TOOSMALL: | |
| return 0; /* retry it */ | |
| case MBERR_TOOFEW: | |
| reason = "incomplete multibyte sequence"; | |
| esize = (Py_ssize_t)(buf->inbuf_end - buf->inbuf); | |
| break; | |
| case MBERR_INTERNAL: | |
| PyErr_SetString(PyExc_RuntimeError, | |
| "internal codec error"); | |
| return -1; | |
| case MBERR_EXCEPTION: | |
| return -1; | |
| default: | |
| PyErr_SetString(PyExc_RuntimeError, | |
| "unknown runtime error"); | |
| return -1; | |
| } | |
| } | |
| if (errors == ERROR_REPLACE) { | |
| if (_PyUnicodeWriter_WriteCharInline( | |
| &buf->writer, Py_UNICODE_REPLACEMENT_CHARACTER) < 0) | |
| goto errorexit; | |
| } | |
| if (errors == ERROR_IGNORE || errors == ERROR_REPLACE) { | |
| buf->inbuf += esize; | |
| return 0; | |
| } | |
| start = (Py_ssize_t)(buf->inbuf - buf->inbuf_top); | |
| end = start + esize; | |
| /* use cached exception object if available */ | |
| if (buf->excobj == NULL) { | |
| buf->excobj = PyUnicodeDecodeError_Create(codec->encoding, | |
| (const char *)buf->inbuf_top, | |
| (Py_ssize_t)(buf->inbuf_end - buf->inbuf_top), | |
| start, end, reason); | |
| if (buf->excobj == NULL) | |
| goto errorexit; | |
| } | |
| else | |
| if (PyUnicodeDecodeError_SetStart(buf->excobj, start) || | |
| PyUnicodeDecodeError_SetEnd(buf->excobj, end) || | |
| PyUnicodeDecodeError_SetReason(buf->excobj, reason)) | |
| goto errorexit; | |
| if (errors == ERROR_STRICT) { | |
| PyCodec_StrictErrors(buf->excobj); | |
| goto errorexit; | |
| } | |
| retobj = call_error_callback(errors, buf->excobj); | |
| if (retobj == NULL) | |
| goto errorexit; | |
| if (!PyTuple_Check(retobj) || PyTuple_GET_SIZE(retobj) != 2 || | |
| !PyUnicode_Check((retuni = PyTuple_GET_ITEM(retobj, 0))) || | |
| !PyLong_Check(PyTuple_GET_ITEM(retobj, 1))) { | |
| PyErr_SetString(PyExc_TypeError, | |
| "decoding error handler must return " | |
| "(str, int) tuple"); | |
| goto errorexit; | |
| } | |
| if (_PyUnicodeWriter_WriteStr(&buf->writer, retuni) < 0) | |
| goto errorexit; | |
| newpos = PyLong_AsSsize_t(PyTuple_GET_ITEM(retobj, 1)); | |
| if (newpos < 0 && !PyErr_Occurred()) | |
| newpos += (Py_ssize_t)(buf->inbuf_end - buf->inbuf_top); | |
| if (newpos < 0 || buf->inbuf_top + newpos > buf->inbuf_end) { | |
| PyErr_Clear(); | |
| PyErr_Format(PyExc_IndexError, | |
| "position %zd from error handler out of bounds", | |
| newpos); | |
| goto errorexit; | |
| } | |
| buf->inbuf = buf->inbuf_top + newpos; | |
| Py_DECREF(retobj); | |
| return 0; | |
| errorexit: | |
| Py_XDECREF(retobj); | |
| return -1; | |
| } | |
| static PyObject * | |
| multibytecodec_encode(const MultibyteCodec *codec, | |
| MultibyteCodec_State *state, | |
| PyObject *text, Py_ssize_t *inpos_t, | |
| PyObject *errors, int flags) | |
| { | |
| MultibyteEncodeBuffer buf; | |
| Py_ssize_t r = 0; | |
| Py_ssize_t datalen; | |
| int kind; | |
| const void *data; | |
| datalen = PyUnicode_GET_LENGTH(text); | |
| if (datalen == 0 && !(flags & MBENC_RESET)) | |
| return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); | |
| buf.excobj = NULL; | |
| buf.writer = NULL; | |
| buf.inobj = text; /* borrowed reference */ | |
| buf.inpos = 0; | |
| buf.inlen = datalen; | |
| kind = PyUnicode_KIND(buf.inobj); | |
| data = PyUnicode_DATA(buf.inobj); | |
| if (datalen > (PY_SSIZE_T_MAX - 16) / 2) { | |
| PyErr_NoMemory(); | |
| goto errorexit; | |
| } | |
| buf.writer = PyBytesWriter_Create(datalen * 2 + 16); | |
| if (buf.writer == NULL) | |
| goto errorexit; | |
| buf.outbuf = (unsigned char *)PyBytesWriter_GetData(buf.writer); | |
| buf.outbuf_end = buf.outbuf + PyBytesWriter_GetSize(buf.writer); | |
| while (buf.inpos < buf.inlen) { | |
| /* we don't reuse inleft and outleft here. | |
| * error callbacks can relocate the cursor anywhere on buffer*/ | |
| Py_ssize_t outleft = (Py_ssize_t)(buf.outbuf_end - buf.outbuf); | |
| r = codec->encode(state, codec, | |
| kind, data, | |
| &buf.inpos, buf.inlen, | |
| &buf.outbuf, outleft, flags); | |
| if ((r == 0) || (r == MBERR_TOOFEW && !(flags & MBENC_FLUSH))) | |
| break; | |
| else if (multibytecodec_encerror(codec, state, &buf, errors,r)) | |
| goto errorexit; | |
| else if (r == MBERR_TOOFEW) | |
| break; | |
| } | |
| if (codec->encreset != NULL && (flags & MBENC_RESET)) | |
| for (;;) { | |
| Py_ssize_t outleft; | |
| outleft = (Py_ssize_t)(buf.outbuf_end - buf.outbuf); | |
| r = codec->encreset(state, codec, &buf.outbuf, | |
| outleft); | |
| if (r == 0) | |
| break; | |
| else if (multibytecodec_encerror(codec, state, | |
| &buf, errors, r)) | |
| goto errorexit; | |
| } | |
| if (inpos_t) | |
| *inpos_t = buf.inpos; | |
| Py_XDECREF(buf.excobj); | |
| return PyBytesWriter_FinishWithPointer(buf.writer, buf.outbuf); | |
| errorexit: | |
| Py_XDECREF(buf.excobj); | |
| PyBytesWriter_Discard(buf.writer); | |
| return NULL; | |
| } | |
| /*[clinic input] | |
| _multibytecodec.MultibyteCodec.encode | |
| input: object | |
| errors: str(accept={str, NoneType}) = None | |
| Return an encoded string version of 'input'. | |
| 'errors' may be given to set a different error handling scheme. | |
| Default is 'strict' meaning that encoding errors raise | |
| a UnicodeEncodeError. Other possible values are 'ignore', 'replace' | |
| and 'xmlcharrefreplace' as well as any other name registered with | |
| codecs.register_error that can handle UnicodeEncodeErrors. | |
| [clinic start generated code]*/ | |
| static PyObject * | |
| _multibytecodec_MultibyteCodec_encode_impl(MultibyteCodecObject *self, | |
| PyObject *input, | |
| const char *errors) | |
| /*[clinic end generated code: output=7b26652045ba56a9 input=980002ed1447697b]*/ | |
| { | |
| MultibyteCodec_State state; | |
| PyObject *errorcb, *r, *ucvt; | |
| Py_ssize_t datalen; | |
| if (PyUnicode_Check(input)) | |
| ucvt = NULL; | |
| else { | |
| input = ucvt = PyObject_Str(input); | |
| if (input == NULL) | |
| return NULL; | |
| else if (!PyUnicode_Check(input)) { | |
| PyErr_SetString(PyExc_TypeError, | |
| "couldn't convert the object to unicode."); | |
| Py_DECREF(ucvt); | |
| return NULL; | |
| } | |
| } | |
| datalen = PyUnicode_GET_LENGTH(input); | |
| errorcb = internal_error_callback(errors); | |
| if (errorcb == NULL) { | |
| Py_XDECREF(ucvt); | |
| return NULL; | |
| } | |
| if (self->codec->encinit != NULL && | |
| self->codec->encinit(&state, self->codec) != 0) | |
| goto errorexit; | |
| r = multibytecodec_encode(self->codec, &state, | |
| input, NULL, errorcb, | |
| MBENC_FLUSH | MBENC_RESET); | |
| if (r == NULL) | |
| goto errorexit; | |
| ERROR_DECREF(errorcb); | |
| Py_XDECREF(ucvt); | |
| return make_tuple(r, datalen); | |
| errorexit: | |
| ERROR_DECREF(errorcb); | |
| Py_XDECREF(ucvt); | |
| return NULL; | |
| } | |
| /*[clinic input] | |
| _multibytecodec.MultibyteCodec.decode | |
| input: Py_buffer | |
| errors: str(accept={str, NoneType}) = None | |
| Decodes 'input'. | |
| 'errors' may be given to set a different error handling scheme. | |
| Default is 'strict' meaning that encoding errors raise | |
| a UnicodeDecodeError. Other possible values are 'ignore' and | |
| 'replace' as well as any other name registered with | |
| codecs.register_error that is able to handle UnicodeDecodeErrors." | |
| [clinic start generated code]*/ | |
| static PyObject * | |
| _multibytecodec_MultibyteCodec_decode_impl(MultibyteCodecObject *self, | |
| Py_buffer *input, | |
| const char *errors) | |
| /*[clinic end generated code: output=ff419f65bad6cc77 input=dbf93d8bb98ca440]*/ | |
| { | |
| MultibyteCodec_State state; | |
| MultibyteDecodeBuffer buf; | |
| PyObject *errorcb, *res; | |
| const char *data; | |
| Py_ssize_t datalen; | |
| data = input->buf; | |
| datalen = input->len; | |
| errorcb = internal_error_callback(errors); | |
| if (errorcb == NULL) { | |
| return NULL; | |
| } | |
| if (datalen == 0) { | |
| ERROR_DECREF(errorcb); | |
| return make_tuple(Py_GetConstant(Py_CONSTANT_EMPTY_STR), 0); | |
| } | |
| _PyUnicodeWriter_Init(&buf.writer); | |
| buf.writer.min_length = datalen; | |
| buf.excobj = NULL; | |
| buf.inbuf = buf.inbuf_top = (unsigned char *)data; | |
| buf.inbuf_end = buf.inbuf_top + datalen; | |
| if (self->codec->decinit != NULL && | |
| self->codec->decinit(&state, self->codec) != 0) | |
| goto errorexit; | |
| while (buf.inbuf < buf.inbuf_end) { | |
| Py_ssize_t inleft, r; | |
| inleft = (Py_ssize_t)(buf.inbuf_end - buf.inbuf); | |
| r = self->codec->decode(&state, self->codec, | |
| &buf.inbuf, inleft, &buf.writer); | |
| if (r == 0) | |
| break; | |
| else if (multibytecodec_decerror(self->codec, &state, | |
| &buf, errorcb, r)) | |
| goto errorexit; | |
| } | |
| res = _PyUnicodeWriter_Finish(&buf.writer); | |
| if (res == NULL) | |
| goto errorexit; | |
| Py_XDECREF(buf.excobj); | |
| ERROR_DECREF(errorcb); | |
| return make_tuple(res, datalen); | |
| errorexit: | |
| ERROR_DECREF(errorcb); | |
| Py_XDECREF(buf.excobj); | |
| _PyUnicodeWriter_Dealloc(&buf.writer); | |
| return NULL; | |
| } | |
| static struct PyMethodDef multibytecodec_methods[] = { | |
| _MULTIBYTECODEC_MULTIBYTECODEC_ENCODE_METHODDEF | |
| _MULTIBYTECODEC_MULTIBYTECODEC_DECODE_METHODDEF | |
| {NULL, NULL}, | |
| }; | |
| static int | |
| multibytecodec_clear(PyObject *op) | |
| { | |
| MultibyteCodecObject *self = _MultibyteCodecObject_CAST(op); | |
| Py_CLEAR(self->cjk_module); | |
| return 0; | |
| } | |
| static int | |
| multibytecodec_traverse(PyObject *op, visitproc visit, void *arg) | |
| { | |
| MultibyteCodecObject *self = _MultibyteCodecObject_CAST(op); | |
| Py_VISIT(Py_TYPE(self)); | |
| Py_VISIT(self->cjk_module); | |
| return 0; | |
| } | |
| static void | |
| multibytecodec_dealloc(PyObject *self) | |
| { | |
| PyObject_GC_UnTrack(self); | |
| PyTypeObject *tp = Py_TYPE(self); | |
| (void)multibytecodec_clear(self); | |
| tp->tp_free(self); | |
| Py_DECREF(tp); | |
| } | |
| static PyType_Slot multibytecodec_slots[] = { | |
| {Py_tp_dealloc, multibytecodec_dealloc}, | |
| {Py_tp_getattro, PyObject_GenericGetAttr}, | |
| {Py_tp_methods, multibytecodec_methods}, | |
| {Py_tp_traverse, multibytecodec_traverse}, | |
| {Py_tp_clear, multibytecodec_clear}, | |
| {0, NULL}, | |
| }; | |
| static PyType_Spec multibytecodec_spec = { | |
| .name = MODULE_NAME ".MultibyteCodec", | |
| .basicsize = sizeof(MultibyteCodecObject), | |
| .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | | |
| Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_IMMUTABLETYPE), | |
| .slots = multibytecodec_slots, | |
| }; | |
| /** | |
| * Utility functions for stateful codec mechanism | |
| */ | |
| #define STATEFUL_DCTX(o) ((MultibyteStatefulDecoderContext *)(o)) | |
| #define STATEFUL_ECTX(o) ((MultibyteStatefulEncoderContext *)(o)) | |
| static PyObject * | |
| encoder_encode_stateful(MultibyteStatefulEncoderContext *ctx, | |
| PyObject *unistr, int final) | |
| { | |
| PyObject *ucvt, *r = NULL; | |
| PyObject *inbuf = NULL; | |
| Py_ssize_t inpos, datalen; | |
| PyObject *origpending = NULL; | |
| if (PyUnicode_Check(unistr)) | |
| ucvt = NULL; | |
| else { | |
| unistr = ucvt = PyObject_Str(unistr); | |
| if (unistr == NULL) | |
| return NULL; | |
| else if (!PyUnicode_Check(unistr)) { | |
| PyErr_SetString(PyExc_TypeError, | |
| "couldn't convert the object to str."); | |
| Py_DECREF(ucvt); | |
| return NULL; | |
| } | |
| } | |
| if (ctx->pending) { | |
| PyObject *inbuf_tmp; | |
| origpending = Py_NewRef(ctx->pending); | |
| inbuf_tmp = Py_NewRef(ctx->pending); | |
| PyUnicode_Append(&inbuf_tmp, unistr); | |
| if (inbuf_tmp == NULL) | |
| goto errorexit; | |
| Py_CLEAR(ctx->pending); | |
| inbuf = inbuf_tmp; | |
| } | |
| else { | |
| origpending = NULL; | |
| inbuf = Py_NewRef(unistr); | |
| } | |
| inpos = 0; | |
| datalen = PyUnicode_GET_LENGTH(inbuf); | |
| r = multibytecodec_encode(ctx->codec, &ctx->state, | |
| inbuf, &inpos, | |
| ctx->errors, final ? MBENC_FLUSH | MBENC_RESET : 0); | |
| if (r == NULL) { | |
| /* recover the original pending buffer */ | |
| Py_XSETREF(ctx->pending, origpending); | |
| origpending = NULL; | |
| goto errorexit; | |
| } | |
| Py_XDECREF(origpending); | |
| if (inpos < datalen) { | |
| if (datalen - inpos > MAXENCPENDING) { | |
| /* normal codecs can't reach here */ | |
| PyObject *excobj = PyObject_CallFunction(PyExc_UnicodeEncodeError, | |
| "sOnns", | |
| ctx->codec->encoding, | |
| inbuf, | |
| inpos, datalen, | |
| "pending buffer overflow"); | |
| if (excobj == NULL) goto errorexit; | |
| PyErr_SetObject(PyExc_UnicodeEncodeError, excobj); | |
| Py_DECREF(excobj); | |
| goto errorexit; | |
| } | |
| ctx->pending = PyUnicode_Substring(inbuf, inpos, datalen); | |
| if (ctx->pending == NULL) { | |
| /* normal codecs can't reach here */ | |
| goto errorexit; | |
| } | |
| } | |
| Py_DECREF(inbuf); | |
| Py_XDECREF(ucvt); | |
| return r; | |
| errorexit: | |
| Py_XDECREF(r); | |
| Py_XDECREF(ucvt); | |
| Py_XDECREF(origpending); | |
| Py_XDECREF(inbuf); | |
| return NULL; | |
| } | |
| static int | |
| decoder_append_pending(MultibyteStatefulDecoderContext *ctx, | |
| MultibyteDecodeBuffer *buf) | |
| { | |
| Py_ssize_t npendings; | |
| npendings = (Py_ssize_t)(buf->inbuf_end - buf->inbuf); | |
| if (npendings + ctx->pendingsize > MAXDECPENDING || | |
| npendings > PY_SSIZE_T_MAX - ctx->pendingsize) { | |
| Py_ssize_t bufsize = (Py_ssize_t)(buf->inbuf_end - buf->inbuf_top); | |
| PyObject *excobj = PyUnicodeDecodeError_Create(ctx->codec->encoding, | |
| (const char *)buf->inbuf_top, | |
| bufsize, | |
| 0, | |
| bufsize, | |
| "pending buffer overflow"); | |
| if (excobj == NULL) return -1; | |
| PyErr_SetObject(PyExc_UnicodeDecodeError, excobj); | |
| Py_DECREF(excobj); | |
| return -1; | |
| } | |
| memcpy(ctx->pending + ctx->pendingsize, buf->inbuf, npendings); | |
| ctx->pendingsize += npendings; | |
| return 0; | |
| } | |
| static int | |
| decoder_prepare_buffer(MultibyteDecodeBuffer *buf, const char *data, | |
| Py_ssize_t size) | |
| { | |
| buf->inbuf = buf->inbuf_top = (const unsigned char *)data; | |
| buf->inbuf_end = buf->inbuf_top + size; | |
| buf->writer.min_length += size; | |
| return 0; | |
| } | |
| static int | |
| decoder_feed_buffer(MultibyteStatefulDecoderContext *ctx, | |
| MultibyteDecodeBuffer *buf) | |
| { | |
| while (buf->inbuf < buf->inbuf_end) { | |
| Py_ssize_t inleft; | |
| Py_ssize_t r; | |
| inleft = (Py_ssize_t)(buf->inbuf_end - buf->inbuf); | |
| r = ctx->codec->decode(&ctx->state, ctx->codec, | |
| &buf->inbuf, inleft, &buf->writer); | |
| if (r == 0 || r == MBERR_TOOFEW) | |
| break; | |
| else if (multibytecodec_decerror(ctx->codec, &ctx->state, | |
| buf, ctx->errors, r)) | |
| return -1; | |
| } | |
| return 0; | |
| } | |
| /*[clinic input] | |
| _multibytecodec.MultibyteIncrementalEncoder.encode | |
| input: object | |
| final: bool = False | |
| [clinic start generated code]*/ | |
| static PyObject * | |
| _multibytecodec_MultibyteIncrementalEncoder_encode_impl(MultibyteIncrementalEncoderObject *self, | |
| PyObject *input, | |
| int final) | |
| /*[clinic end generated code: output=123361b6c505e2c1 input=bd5f7d40d43e99b0]*/ | |
| { | |
| return encoder_encode_stateful(STATEFUL_ECTX(self), input, final); | |
| } | |
| /*[clinic input] | |
| _multibytecodec.MultibyteIncrementalEncoder.getstate | |
| [clinic start generated code]*/ | |
| static PyObject * | |
| _multibytecodec_MultibyteIncrementalEncoder_getstate_impl(MultibyteIncrementalEncoderObject *self) | |
| /*[clinic end generated code: output=9794a5ace70d7048 input=4a2a82874ffa40bb]*/ | |
| { | |
| /* state made up of 1 byte for buffer size, up to MAXENCPENDING*4 bytes | |
| for UTF-8 encoded buffer (each character can use up to 4 | |
| bytes), and required bytes for MultibyteCodec_State.c. A byte | |
| array is used to avoid different compilers generating different | |
| values for the same state, e.g. as a result of struct padding. | |
| */ | |
| unsigned char statebytes[1 + MAXENCPENDING*4 + sizeof(self->state.c)]; | |
| Py_ssize_t statesize; | |
| const char *pendingbuffer = NULL; | |
| Py_ssize_t pendingsize; | |
| if (self->pending != NULL) { | |
| pendingbuffer = PyUnicode_AsUTF8AndSize(self->pending, &pendingsize); | |
| if (pendingbuffer == NULL) { | |
| return NULL; | |
| } | |
| if (pendingsize > MAXENCPENDING*4) { | |
| PyObject *excobj = PyObject_CallFunction(PyExc_UnicodeEncodeError, | |
| "sOnns", | |
| self->codec->encoding, | |
| self->pending, | |
| 0, PyUnicode_GET_LENGTH(self->pending), | |
| "pending buffer too large"); | |
| if (excobj == NULL) { | |
| return NULL; | |
| } | |
| PyErr_SetObject(PyExc_UnicodeEncodeError, excobj); | |
| Py_DECREF(excobj); | |
| return NULL; | |
| } | |
| statebytes[0] = (unsigned char)pendingsize; | |
| memcpy(statebytes + 1, pendingbuffer, pendingsize); | |
| statesize = 1 + pendingsize; | |
| } else { | |
| statebytes[0] = 0; | |
| statesize = 1; | |
| } | |
| memcpy(statebytes+statesize, self->state.c, | |
| sizeof(self->state.c)); | |
| statesize += sizeof(self->state.c); | |
| return (PyObject *)_PyLong_FromByteArray(statebytes, statesize, | |
| 1 /* little-endian */ , | |
| 0 /* unsigned */ ); | |
| } | |
| /*[clinic input] | |
| _multibytecodec.MultibyteIncrementalEncoder.setstate | |
| state as statelong: object(type='PyLongObject *', subclass_of='&PyLong_Type') | |
| / | |
| [clinic start generated code]*/ | |
| static PyObject * | |
| _multibytecodec_MultibyteIncrementalEncoder_setstate_impl(MultibyteIncrementalEncoderObject *self, | |
| PyLongObject *statelong) | |
| /*[clinic end generated code: output=4e5e98ac1f4039ca input=c80fb5830d4d2f76]*/ | |
| { | |
| PyObject *pending = NULL; | |
| unsigned char statebytes[1 + MAXENCPENDING*4 + sizeof(self->state.c)]; | |
| if (_PyLong_AsByteArray(statelong, statebytes, sizeof(statebytes), | |
| 1 /* little-endian */ , | |
| 0 /* unsigned */ , | |
| 1 /* with_exceptions */) < 0) { | |
| goto errorexit; | |
| } | |
| if (statebytes[0] > MAXENCPENDING*4) { | |
| PyErr_SetString(PyExc_UnicodeError, "pending buffer too large"); | |
| return NULL; | |
| } | |
| pending = PyUnicode_DecodeUTF8((const char *)statebytes+1, | |
| statebytes[0], "strict"); | |
| if (pending == NULL) { | |
| goto errorexit; | |
| } | |
| Py_XSETREF(self->pending, pending); | |
| memcpy(self->state.c, statebytes+1+statebytes[0], | |
| sizeof(self->state.c)); | |
| Py_RETURN_NONE; | |
| errorexit: | |
| Py_XDECREF(pending); | |
| return NULL; | |
| } | |
| /*[clinic input] | |
| _multibytecodec.MultibyteIncrementalEncoder.reset | |
| [clinic start generated code]*/ | |
| static PyObject * | |
| _multibytecodec_MultibyteIncrementalEncoder_reset_impl(MultibyteIncrementalEncoderObject *self) | |
| /*[clinic end generated code: output=b4125d8f537a253f input=930f06760707b6ea]*/ | |
| { | |
| /* Longest output: 4 bytes (b'\x0F\x1F(B') with ISO 2022 */ | |
| unsigned char buffer[4], *outbuf; | |
| Py_ssize_t r; | |
| if (self->codec->encreset != NULL) { | |
| outbuf = buffer; | |
| r = self->codec->encreset(&self->state, self->codec, | |
| &outbuf, sizeof(buffer)); | |
| if (r != 0) | |
| return NULL; | |
| } | |
| Py_CLEAR(self->pending); | |
| Py_RETURN_NONE; | |
| } | |
| static struct PyMethodDef mbiencoder_methods[] = { | |
| _MULTIBYTECODEC_MULTIBYTEINCREMENTALENCODER_ENCODE_METHODDEF | |
| _MULTIBYTECODEC_MULTIBYTEINCREMENTALENCODER_GETSTATE_METHODDEF | |
| _MULTIBYTECODEC_MULTIBYTEINCREMENTALENCODER_SETSTATE_METHODDEF | |
| _MULTIBYTECODEC_MULTIBYTEINCREMENTALENCODER_RESET_METHODDEF | |
| {NULL, NULL}, | |
| }; | |
| static PyObject * | |
| mbiencoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds) | |
| { | |
| MultibyteIncrementalEncoderObject *self; | |
| PyObject *codec = NULL; | |
| char *errors = NULL; | |
| if (!PyArg_ParseTupleAndKeywords(args, kwds, "|s:IncrementalEncoder", | |
| incnewkwarglist, &errors)) | |
| return NULL; | |
| self = (MultibyteIncrementalEncoderObject *)type->tp_alloc(type, 0); | |
| if (self == NULL) | |
| return NULL; | |
| codec = PyObject_GetAttrString((PyObject *)type, "codec"); | |
| if (codec == NULL) | |
| goto errorexit; | |
| module_state *state = find_state_by_def(type); | |
| if (!MultibyteCodec_Check(state, codec)) { | |
| PyErr_SetString(PyExc_TypeError, "codec is unexpected type"); | |
| goto errorexit; | |
| } | |
| self->codec = ((MultibyteCodecObject *)codec)->codec; | |
| self->pending = NULL; | |
| self->errors = internal_error_callback(errors); | |
| if (self->errors == NULL) | |
| goto errorexit; | |
| if (self->codec->encinit != NULL && | |
| self->codec->encinit(&self->state, self->codec) != 0) | |
| goto errorexit; | |
| Py_DECREF(codec); | |
| return (PyObject *)self; | |
| errorexit: | |
| Py_XDECREF(self); | |
| Py_XDECREF(codec); | |
| return NULL; | |
| } | |
| static int | |
| mbiencoder_init(PyObject *self, PyObject *args, PyObject *kwds) | |
| { | |
| return 0; | |
| } | |
| static int | |
| mbiencoder_traverse(PyObject *op, visitproc visit, void *arg) | |
| { | |
| MultibyteIncrementalEncoderObject *self = _MultibyteIncrementalEncoderObject_CAST(op); | |
| if (ERROR_ISCUSTOM(self->errors)) | |
| Py_VISIT(self->errors); | |
| return 0; | |
| } | |
| static void | |
| mbiencoder_dealloc(PyObject *op) | |
| { | |
| MultibyteIncrementalEncoderObject *self = _MultibyteIncrementalEncoderObject_CAST(op); | |
| PyTypeObject *tp = Py_TYPE(self); | |
| PyObject_GC_UnTrack(self); | |
| ERROR_DECREF(self->errors); | |
| Py_CLEAR(self->pending); | |
| tp->tp_free(self); | |
| Py_DECREF(tp); | |
| } | |
| static PyType_Slot encoder_slots[] = { | |
| {Py_tp_dealloc, mbiencoder_dealloc}, | |
| {Py_tp_getattro, PyObject_GenericGetAttr}, | |
| {Py_tp_traverse, mbiencoder_traverse}, | |
| {Py_tp_methods, mbiencoder_methods}, | |
| {Py_tp_getset, codecctx_getsets}, | |
| {Py_tp_init, mbiencoder_init}, | |
| {Py_tp_new, mbiencoder_new}, | |
| {0, NULL}, | |
| }; | |
| static PyType_Spec encoder_spec = { | |
| .name = MODULE_NAME ".MultibyteIncrementalEncoder", | |
| .basicsize = sizeof(MultibyteIncrementalEncoderObject), | |
| .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE | | |
| Py_TPFLAGS_IMMUTABLETYPE), | |
| .slots = encoder_slots, | |
| }; | |
| /*[clinic input] | |
| _multibytecodec.MultibyteIncrementalDecoder.decode | |
| input: Py_buffer | |
| final: bool = False | |
| [clinic start generated code]*/ | |
| static PyObject * | |
| _multibytecodec_MultibyteIncrementalDecoder_decode_impl(MultibyteIncrementalDecoderObject *self, | |
| Py_buffer *input, | |
| int final) | |
| /*[clinic end generated code: output=b9b9090e8a9ce2ba input=8795fbb20860027a]*/ | |
| { | |
| MultibyteDecodeBuffer buf; | |
| char *data, *wdata = NULL; | |
| Py_ssize_t wsize, size, origpending; | |
| PyObject *res; | |
| data = input->buf; | |
| size = input->len; | |
| _PyUnicodeWriter_Init(&buf.writer); | |
| buf.excobj = NULL; | |
| origpending = self->pendingsize; | |
| if (self->pendingsize == 0) { | |
| wsize = size; | |
| wdata = data; | |
| } | |
| else { | |
| if (size > PY_SSIZE_T_MAX - self->pendingsize) { | |
| PyErr_NoMemory(); | |
| goto errorexit; | |
| } | |
| wsize = size + self->pendingsize; | |
| wdata = PyMem_Malloc(wsize); | |
| if (wdata == NULL) { | |
| PyErr_NoMemory(); | |
| goto errorexit; | |
| } | |
| memcpy(wdata, self->pending, self->pendingsize); | |
| memcpy(wdata + self->pendingsize, data, size); | |
| self->pendingsize = 0; | |
| } | |
| if (decoder_prepare_buffer(&buf, wdata, wsize) != 0) | |
| goto errorexit; | |
| if (decoder_feed_buffer(STATEFUL_DCTX(self), &buf)) | |
| goto errorexit; | |
| if (final && buf.inbuf < buf.inbuf_end) { | |
| if (multibytecodec_decerror(self->codec, &self->state, | |
| &buf, self->errors, MBERR_TOOFEW)) { | |
| /* recover the original pending buffer */ | |
| memcpy(self->pending, wdata, origpending); | |
| self->pendingsize = origpending; | |
| goto errorexit; | |
| } | |
| } | |
| if (buf.inbuf < buf.inbuf_end) { /* pending sequence still exists */ | |
| if (decoder_append_pending(STATEFUL_DCTX(self), &buf) != 0) | |
| goto errorexit; | |
| } | |
| res = _PyUnicodeWriter_Finish(&buf.writer); | |
| if (res == NULL) | |
| goto errorexit; | |
| if (wdata != data) | |
| PyMem_Free(wdata); | |
| Py_XDECREF(buf.excobj); | |
| return res; | |
| errorexit: | |
| if (wdata != NULL && wdata != data) | |
| PyMem_Free(wdata); | |
| Py_XDECREF(buf.excobj); | |
| _PyUnicodeWriter_Dealloc(&buf.writer); | |
| return NULL; | |
| } | |
| /*[clinic input] | |
| _multibytecodec.MultibyteIncrementalDecoder.getstate | |
| [clinic start generated code]*/ | |
| static PyObject * | |
| _multibytecodec_MultibyteIncrementalDecoder_getstate_impl(MultibyteIncrementalDecoderObject *self) | |
| /*[clinic end generated code: output=255009c4713b7f82 input=4006aa49bddbaa75]*/ | |
| { | |
| PyObject *buffer; | |
| PyObject *statelong; | |
| buffer = PyBytes_FromStringAndSize((const char *)self->pending, | |
| self->pendingsize); | |
| if (buffer == NULL) { | |
| return NULL; | |
| } | |
| statelong = (PyObject *)_PyLong_FromByteArray(self->state.c, | |
| sizeof(self->state.c), | |
| 1 /* little-endian */ , | |
| 0 /* unsigned */ ); | |
| if (statelong == NULL) { | |
| Py_DECREF(buffer); | |
| return NULL; | |
| } | |
| return Py_BuildValue("NN", buffer, statelong); | |
| } | |
| /*[clinic input] | |
| _multibytecodec.MultibyteIncrementalDecoder.setstate | |
| state: object(subclass_of='&PyTuple_Type') | |
| / | |
| [clinic start generated code]*/ | |
| static PyObject * | |
| _multibytecodec_MultibyteIncrementalDecoder_setstate_impl(MultibyteIncrementalDecoderObject *self, | |
| PyObject *state) | |
| /*[clinic end generated code: output=106b2fbca3e2dcc2 input=e5d794e8baba1a47]*/ | |
| { | |
| PyObject *buffer; | |
| PyLongObject *statelong; | |
| Py_ssize_t buffersize; | |
| const char *bufferstr; | |
| unsigned char statebytes[8]; | |
| if (!PyArg_ParseTuple(state, "SO!;setstate(): illegal state argument", | |
| &buffer, &PyLong_Type, &statelong)) | |
| { | |
| return NULL; | |
| } | |
| if (_PyLong_AsByteArray(statelong, statebytes, sizeof(statebytes), | |
| 1 /* little-endian */ , | |
| 0 /* unsigned */ , | |
| 1 /* with_exceptions */) < 0) { | |
| return NULL; | |
| } | |
| buffersize = PyBytes_Size(buffer); | |
| if (buffersize == -1) { | |
| return NULL; | |
| } | |
| if (buffersize > MAXDECPENDING) { | |
| PyObject *excobj = PyUnicodeDecodeError_Create(self->codec->encoding, | |
| PyBytes_AS_STRING(buffer), buffersize, | |
| 0, buffersize, | |
| "pending buffer too large"); | |
| if (excobj == NULL) return NULL; | |
| PyErr_SetObject(PyExc_UnicodeDecodeError, excobj); | |
| Py_DECREF(excobj); | |
| return NULL; | |
| } | |
| bufferstr = PyBytes_AsString(buffer); | |
| if (bufferstr == NULL) { | |
| return NULL; | |
| } | |
| self->pendingsize = buffersize; | |
| memcpy(self->pending, bufferstr, self->pendingsize); | |
| memcpy(self->state.c, statebytes, sizeof(statebytes)); | |
| Py_RETURN_NONE; | |
| } | |
| /*[clinic input] | |
| _multibytecodec.MultibyteIncrementalDecoder.reset | |
| [clinic start generated code]*/ | |
| static PyObject * | |
| _multibytecodec_MultibyteIncrementalDecoder_reset_impl(MultibyteIncrementalDecoderObject *self) | |
| /*[clinic end generated code: output=da423b1782c23ed1 input=3b63b3be85b2fb45]*/ | |
| { | |
| if (self->codec->decreset != NULL && | |
| self->codec->decreset(&self->state, self->codec) != 0) | |
| return NULL; | |
| self->pendingsize = 0; | |
| Py_RETURN_NONE; | |
| } | |
| static struct PyMethodDef mbidecoder_methods[] = { | |
| _MULTIBYTECODEC_MULTIBYTEINCREMENTALDECODER_DECODE_METHODDEF | |
| _MULTIBYTECODEC_MULTIBYTEINCREMENTALDECODER_GETSTATE_METHODDEF | |
| _MULTIBYTECODEC_MULTIBYTEINCREMENTALDECODER_SETSTATE_METHODDEF | |
| _MULTIBYTECODEC_MULTIBYTEINCREMENTALDECODER_RESET_METHODDEF | |
| {NULL, NULL}, | |
| }; | |
| static PyObject * | |
| mbidecoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds) | |
| { | |
| MultibyteIncrementalDecoderObject *self; | |
| PyObject *codec = NULL; | |
| char *errors = NULL; | |
| if (!PyArg_ParseTupleAndKeywords(args, kwds, "|s:IncrementalDecoder", | |
| incnewkwarglist, &errors)) | |
| return NULL; | |
| self = (MultibyteIncrementalDecoderObject *)type->tp_alloc(type, 0); | |
| if (self == NULL) | |
| return NULL; | |
| codec = PyObject_GetAttrString((PyObject *)type, "codec"); | |
| if (codec == NULL) | |
| goto errorexit; | |
| module_state *state = find_state_by_def(type); | |
| if (!MultibyteCodec_Check(state, codec)) { | |
| PyErr_SetString(PyExc_TypeError, "codec is unexpected type"); | |
| goto errorexit; | |
| } | |
| self->codec = ((MultibyteCodecObject *)codec)->codec; | |
| self->pendingsize = 0; | |
| self->errors = internal_error_callback(errors); | |
| if (self->errors == NULL) | |
| goto errorexit; | |
| if (self->codec->decinit != NULL && | |
| self->codec->decinit(&self->state, self->codec) != 0) | |
| goto errorexit; | |
| Py_DECREF(codec); | |
| return (PyObject *)self; | |
| errorexit: | |
| Py_XDECREF(self); | |
| Py_XDECREF(codec); | |
| return NULL; | |
| } | |
| static int | |
| mbidecoder_init(PyObject *self, PyObject *args, PyObject *kwds) | |
| { | |
| return 0; | |
| } | |
| static int | |
| mbidecoder_traverse(PyObject *op, visitproc visit, void *arg) | |
| { | |
| MultibyteIncrementalDecoderObject *self = _MultibyteIncrementalDecoderObject_CAST(op); | |
| if (ERROR_ISCUSTOM(self->errors)) | |
| Py_VISIT(self->errors); | |
| return 0; | |
| } | |
| static void | |
| mbidecoder_dealloc(PyObject *op) | |
| { | |
| MultibyteIncrementalDecoderObject *self = _MultibyteIncrementalDecoderObject_CAST(op); | |
| PyTypeObject *tp = Py_TYPE(self); | |
| PyObject_GC_UnTrack(self); | |
| ERROR_DECREF(self->errors); | |
| tp->tp_free(self); | |
| Py_DECREF(tp); | |
| } | |
| static PyType_Slot decoder_slots[] = { | |
| {Py_tp_dealloc, mbidecoder_dealloc}, | |
| {Py_tp_getattro, PyObject_GenericGetAttr}, | |
| {Py_tp_traverse, mbidecoder_traverse}, | |
| {Py_tp_methods, mbidecoder_methods}, | |
| {Py_tp_getset, codecctx_getsets}, | |
| {Py_tp_init, mbidecoder_init}, | |
| {Py_tp_new, mbidecoder_new}, | |
| {0, NULL}, | |
| }; | |
| static PyType_Spec decoder_spec = { | |
| .name = MODULE_NAME ".MultibyteIncrementalDecoder", | |
| .basicsize = sizeof(MultibyteIncrementalDecoderObject), | |
| .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE | | |
| Py_TPFLAGS_IMMUTABLETYPE), | |
| .slots = decoder_slots, | |
| }; | |
| static PyObject * | |
| mbstreamreader_iread(MultibyteStreamReaderObject *self, | |
| const char *method, Py_ssize_t sizehint) | |
| { | |
| MultibyteDecodeBuffer buf; | |
| PyObject *cres, *res; | |
| Py_ssize_t rsize; | |
| if (sizehint == 0) | |
| return Py_GetConstant(Py_CONSTANT_EMPTY_STR); | |
| _PyUnicodeWriter_Init(&buf.writer); | |
| buf.excobj = NULL; | |
| cres = NULL; | |
| for (;;) { | |
| int endoffile; | |
| if (sizehint < 0) | |
| cres = PyObject_CallMethod(self->stream, | |
| method, NULL); | |
| else | |
| cres = PyObject_CallMethod(self->stream, | |
| method, "i", sizehint); | |
| if (cres == NULL) | |
| goto errorexit; | |
| if (!PyBytes_Check(cres)) { | |
| PyErr_Format(PyExc_TypeError, | |
| "stream function returned a " | |
| "non-bytes object (%.100s)", | |
| Py_TYPE(cres)->tp_name); | |
| goto errorexit; | |
| } | |
| endoffile = (PyBytes_GET_SIZE(cres) == 0); | |
| if (self->pendingsize > 0) { | |
| if (PyBytes_GET_SIZE(cres) > PY_SSIZE_T_MAX - self->pendingsize) { | |
| PyErr_NoMemory(); | |
| goto errorexit; | |
| } | |
| rsize = PyBytes_GET_SIZE(cres) + self->pendingsize; | |
| PyBytesWriter *writer = PyBytesWriter_Create(rsize); | |
| if (writer == NULL) { | |
| goto errorexit; | |
| } | |
| char *ctrdata = PyBytesWriter_GetData(writer); | |
| memcpy(ctrdata, self->pending, self->pendingsize); | |
| memcpy(ctrdata + self->pendingsize, | |
| PyBytes_AS_STRING(cres), | |
| PyBytes_GET_SIZE(cres)); | |
| Py_SETREF(cres, PyBytesWriter_Finish(writer)); | |
| if (cres == NULL) { | |
| goto errorexit; | |
| } | |
| self->pendingsize = 0; | |
| } | |
| rsize = PyBytes_GET_SIZE(cres); | |
| if (decoder_prepare_buffer(&buf, PyBytes_AS_STRING(cres), | |
| rsize) != 0) | |
| goto errorexit; | |
| if (rsize > 0 && decoder_feed_buffer( | |
| (MultibyteStatefulDecoderContext *)self, &buf)) | |
| goto errorexit; | |
| if (endoffile || sizehint < 0) { | |
| if (buf.inbuf < buf.inbuf_end && | |
| multibytecodec_decerror(self->codec, &self->state, | |
| &buf, self->errors, MBERR_TOOFEW)) | |
| goto errorexit; | |
| } | |
| if (buf.inbuf < buf.inbuf_end) { /* pending sequence exists */ | |
| if (decoder_append_pending(STATEFUL_DCTX(self), | |
| &buf) != 0) | |
| goto errorexit; | |
| } | |
| Py_SETREF(cres, NULL); | |
| if (sizehint < 0 || buf.writer.pos != 0 || rsize == 0) | |
| break; | |
| sizehint = 1; /* read 1 more byte and retry */ | |
| } | |
| res = _PyUnicodeWriter_Finish(&buf.writer); | |
| if (res == NULL) | |
| goto errorexit; | |
| Py_XDECREF(cres); | |
| Py_XDECREF(buf.excobj); | |
| return res; | |
| errorexit: | |
| Py_XDECREF(cres); | |
| Py_XDECREF(buf.excobj); | |
| _PyUnicodeWriter_Dealloc(&buf.writer); | |
| return NULL; | |
| } | |
| /*[clinic input] | |
| _multibytecodec.MultibyteStreamReader.read | |
| sizeobj: object = None | |
| / | |
| [clinic start generated code]*/ | |
| static PyObject * | |
| _multibytecodec_MultibyteStreamReader_read_impl(MultibyteStreamReaderObject *self, | |
| PyObject *sizeobj) | |
| /*[clinic end generated code: output=35621eb75355d5b8 input=015b0d3ff2fca485]*/ | |
| { | |
| Py_ssize_t size; | |
| if (sizeobj == Py_None) | |
| size = -1; | |
| else if (PyLong_Check(sizeobj)) | |
| size = PyLong_AsSsize_t(sizeobj); | |
| else { | |
| PyErr_SetString(PyExc_TypeError, "arg 1 must be an integer"); | |
| return NULL; | |
| } | |
| if (size == -1 && PyErr_Occurred()) | |
| return NULL; | |
| return mbstreamreader_iread(self, "read", size); | |
| } | |
| /*[clinic input] | |
| _multibytecodec.MultibyteStreamReader.readline | |
| sizeobj: object = None | |
| / | |
| [clinic start generated code]*/ | |
| static PyObject * | |
| _multibytecodec_MultibyteStreamReader_readline_impl(MultibyteStreamReaderObject *self, | |
| PyObject *sizeobj) | |
| /*[clinic end generated code: output=4fbfaae1ed457a11 input=41ccc64f9bb0cec3]*/ | |
| { | |
| Py_ssize_t size; | |
| if (sizeobj == Py_None) | |
| size = -1; | |
| else if (PyLong_Check(sizeobj)) | |
| size = PyLong_AsSsize_t(sizeobj); | |
| else { | |
| PyErr_SetString(PyExc_TypeError, "arg 1 must be an integer"); | |
| return NULL; | |
| } | |
| if (size == -1 && PyErr_Occurred()) | |
| return NULL; | |
| return mbstreamreader_iread(self, "readline", size); | |
| } | |
| /*[clinic input] | |
| _multibytecodec.MultibyteStreamReader.readlines | |
| sizehintobj: object = None | |
| / | |
| [clinic start generated code]*/ | |
| static PyObject * | |
| _multibytecodec_MultibyteStreamReader_readlines_impl(MultibyteStreamReaderObject *self, | |
| PyObject *sizehintobj) | |
| /*[clinic end generated code: output=e7c4310768ed2ad4 input=54932f5d4d88e880]*/ | |
| { | |
| PyObject *r, *sr; | |
| Py_ssize_t sizehint; | |
| if (sizehintobj == Py_None) | |
| sizehint = -1; | |
| else if (PyLong_Check(sizehintobj)) | |
| sizehint = PyLong_AsSsize_t(sizehintobj); | |
| else { | |
| PyErr_SetString(PyExc_TypeError, "arg 1 must be an integer"); | |
| return NULL; | |
| } | |
| if (sizehint == -1 && PyErr_Occurred()) | |
| return NULL; | |
| r = mbstreamreader_iread(self, "read", sizehint); | |
| if (r == NULL) | |
| return NULL; | |
| sr = PyUnicode_Splitlines(r, 1); | |
| Py_DECREF(r); | |
| return sr; | |
| } | |
| /*[clinic input] | |
| _multibytecodec.MultibyteStreamReader.reset | |
| [clinic start generated code]*/ | |
| static PyObject * | |
| _multibytecodec_MultibyteStreamReader_reset_impl(MultibyteStreamReaderObject *self) | |
| /*[clinic end generated code: output=138490370a680abc input=5d4140db84b5e1e2]*/ | |
| { | |
| if (self->codec->decreset != NULL && | |
| self->codec->decreset(&self->state, self->codec) != 0) | |
| return NULL; | |
| self->pendingsize = 0; | |
| Py_RETURN_NONE; | |
| } | |
| static struct PyMethodDef mbstreamreader_methods[] = { | |
| _MULTIBYTECODEC_MULTIBYTESTREAMREADER_READ_METHODDEF | |
| _MULTIBYTECODEC_MULTIBYTESTREAMREADER_READLINE_METHODDEF | |
| _MULTIBYTECODEC_MULTIBYTESTREAMREADER_READLINES_METHODDEF | |
| _MULTIBYTECODEC_MULTIBYTESTREAMREADER_RESET_METHODDEF | |
| {NULL, NULL}, | |
| }; | |
| static PyMemberDef mbstreamreader_members[] = { | |
| {"stream", _Py_T_OBJECT, | |
| offsetof(MultibyteStreamReaderObject, stream), | |
| Py_READONLY, NULL}, | |
| {NULL,} | |
| }; | |
| static PyObject * | |
| mbstreamreader_new(PyTypeObject *type, PyObject *args, PyObject *kwds) | |
| { | |
| MultibyteStreamReaderObject *self; | |
| PyObject *stream, *codec = NULL; | |
| char *errors = NULL; | |
| if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|s:StreamReader", | |
| streamkwarglist, &stream, &errors)) | |
| return NULL; | |
| self = (MultibyteStreamReaderObject *)type->tp_alloc(type, 0); | |
| if (self == NULL) | |
| return NULL; | |
| codec = PyObject_GetAttrString((PyObject *)type, "codec"); | |
| if (codec == NULL) | |
| goto errorexit; | |
| module_state *state = find_state_by_def(type); | |
| if (!MultibyteCodec_Check(state, codec)) { | |
| PyErr_SetString(PyExc_TypeError, "codec is unexpected type"); | |
| goto errorexit; | |
| } | |
| self->codec = ((MultibyteCodecObject *)codec)->codec; | |
| self->stream = Py_NewRef(stream); | |
| self->pendingsize = 0; | |
| self->errors = internal_error_callback(errors); | |
| if (self->errors == NULL) | |
| goto errorexit; | |
| if (self->codec->decinit != NULL && | |
| self->codec->decinit(&self->state, self->codec) != 0) | |
| goto errorexit; | |
| Py_DECREF(codec); | |
| return (PyObject *)self; | |
| errorexit: | |
| Py_XDECREF(self); | |
| Py_XDECREF(codec); | |
| return NULL; | |
| } | |
| static int | |
| mbstreamreader_init(PyObject *self, PyObject *args, PyObject *kwds) | |
| { | |
| return 0; | |
| } | |
| static int | |
| mbstreamreader_traverse(PyObject *op, visitproc visit, void *arg) | |
| { | |
| MultibyteStreamReaderObject *self = _MultibyteStreamReaderObject_CAST(op); | |
| if (ERROR_ISCUSTOM(self->errors)) | |
| Py_VISIT(self->errors); | |
| Py_VISIT(self->stream); | |
| return 0; | |
| } | |
| static void | |
| mbstreamreader_dealloc(PyObject *op) | |
| { | |
| MultibyteStreamReaderObject *self = _MultibyteStreamReaderObject_CAST(op); | |
| PyTypeObject *tp = Py_TYPE(self); | |
| PyObject_GC_UnTrack(self); | |
| ERROR_DECREF(self->errors); | |
| Py_XDECREF(self->stream); | |
| tp->tp_free(self); | |
| Py_DECREF(tp); | |
| } | |
| static PyType_Slot reader_slots[] = { | |
| {Py_tp_dealloc, mbstreamreader_dealloc}, | |
| {Py_tp_getattro, PyObject_GenericGetAttr}, | |
| {Py_tp_traverse, mbstreamreader_traverse}, | |
| {Py_tp_methods, mbstreamreader_methods}, | |
| {Py_tp_members, mbstreamreader_members}, | |
| {Py_tp_getset, codecctx_getsets}, | |
| {Py_tp_init, mbstreamreader_init}, | |
| {Py_tp_new, mbstreamreader_new}, | |
| {0, NULL}, | |
| }; | |
| static PyType_Spec reader_spec = { | |
| .name = MODULE_NAME ".MultibyteStreamReader", | |
| .basicsize = sizeof(MultibyteStreamReaderObject), | |
| .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE | | |
| Py_TPFLAGS_IMMUTABLETYPE), | |
| .slots = reader_slots, | |
| }; | |
| static int | |
| mbstreamwriter_iwrite(MultibyteStreamWriterObject *self, | |
| PyObject *unistr, PyObject *str_write) | |
| { | |
| PyObject *str, *wr; | |
| str = encoder_encode_stateful(STATEFUL_ECTX(self), unistr, 0); | |
| if (str == NULL) | |
| return -1; | |
| wr = PyObject_CallMethodOneArg(self->stream, str_write, str); | |
| Py_DECREF(str); | |
| if (wr == NULL) | |
| return -1; | |
| Py_DECREF(wr); | |
| return 0; | |
| } | |
| /*[clinic input] | |
| _multibytecodec.MultibyteStreamWriter.write | |
| cls: defining_class | |
| strobj: object | |
| / | |
| [clinic start generated code]*/ | |
| static PyObject * | |
| _multibytecodec_MultibyteStreamWriter_write_impl(MultibyteStreamWriterObject *self, | |
| PyTypeObject *cls, | |
| PyObject *strobj) | |
| /*[clinic end generated code: output=68ade3aea26410ac input=199f26f68bd8425a]*/ | |
| { | |
| module_state *state = PyType_GetModuleState(cls); | |
| assert(state != NULL); | |
| if (mbstreamwriter_iwrite(self, strobj, state->str_write)) { | |
| return NULL; | |
| } | |
| Py_RETURN_NONE; | |
| } | |
| /*[clinic input] | |
| _multibytecodec.MultibyteStreamWriter.writelines | |
| cls: defining_class | |
| lines: object | |
| / | |
| [clinic start generated code]*/ | |
| static PyObject * | |
| _multibytecodec_MultibyteStreamWriter_writelines_impl(MultibyteStreamWriterObject *self, | |
| PyTypeObject *cls, | |
| PyObject *lines) | |
| /*[clinic end generated code: output=b4c99d2cf23ffb88 input=a6d5fe7c74972a34]*/ | |
| { | |
| PyObject *strobj; | |
| int i, r; | |
| if (!PySequence_Check(lines)) { | |
| PyErr_SetString(PyExc_TypeError, | |
| "arg must be a sequence object"); | |
| return NULL; | |
| } | |
| module_state *state = PyType_GetModuleState(cls); | |
| assert(state != NULL); | |
| for (i = 0; i < PySequence_Length(lines); i++) { | |
| /* length can be changed even within this loop */ | |
| strobj = PySequence_GetItem(lines, i); | |
| if (strobj == NULL) | |
| return NULL; | |
| r = mbstreamwriter_iwrite(self, strobj, state->str_write); | |
| Py_DECREF(strobj); | |
| if (r == -1) | |
| return NULL; | |
| } | |
| /* PySequence_Length() can fail */ | |
| if (PyErr_Occurred()) | |
| return NULL; | |
| Py_RETURN_NONE; | |
| } | |
| /*[clinic input] | |
| _multibytecodec.MultibyteStreamWriter.reset | |
| cls: defining_class | |
| / | |
| [clinic start generated code]*/ | |
| static PyObject * | |
| _multibytecodec_MultibyteStreamWriter_reset_impl(MultibyteStreamWriterObject *self, | |
| PyTypeObject *cls) | |
| /*[clinic end generated code: output=32ef224c2a38aa3d input=28af6a9cd38d1979]*/ | |
| { | |
| PyObject *pwrt; | |
| if (!self->pending) | |
| Py_RETURN_NONE; | |
| pwrt = multibytecodec_encode(self->codec, &self->state, | |
| self->pending, NULL, self->errors, | |
| MBENC_FLUSH | MBENC_RESET); | |
| /* some pending buffer can be truncated when UnicodeEncodeError is | |
| * raised on 'strict' mode. but, 'reset' method is designed to | |
| * reset the pending buffer or states so failed string sequence | |
| * ought to be missed */ | |
| Py_CLEAR(self->pending); | |
| if (pwrt == NULL) | |
| return NULL; | |
| assert(PyBytes_Check(pwrt)); | |
| module_state *state = PyType_GetModuleState(cls); | |
| assert(state != NULL); | |
| if (PyBytes_Size(pwrt) > 0) { | |
| PyObject *wr; | |
| wr = PyObject_CallMethodOneArg(self->stream, state->str_write, pwrt); | |
| if (wr == NULL) { | |
| Py_DECREF(pwrt); | |
| return NULL; | |
| } | |
| } | |
| Py_DECREF(pwrt); | |
| Py_RETURN_NONE; | |
| } | |
| static PyObject * | |
| mbstreamwriter_new(PyTypeObject *type, PyObject *args, PyObject *kwds) | |
| { | |
| MultibyteStreamWriterObject *self; | |
| PyObject *stream, *codec = NULL; | |
| char *errors = NULL; | |
| if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|s:StreamWriter", | |
| streamkwarglist, &stream, &errors)) | |
| return NULL; | |
| self = (MultibyteStreamWriterObject *)type->tp_alloc(type, 0); | |
| if (self == NULL) | |
| return NULL; | |
| codec = PyObject_GetAttrString((PyObject *)type, "codec"); | |
| if (codec == NULL) | |
| goto errorexit; | |
| module_state *state = find_state_by_def(type); | |
| if (!MultibyteCodec_Check(state, codec)) { | |
| PyErr_SetString(PyExc_TypeError, "codec is unexpected type"); | |
| goto errorexit; | |
| } | |
| self->codec = ((MultibyteCodecObject *)codec)->codec; | |
| self->stream = Py_NewRef(stream); | |
| self->pending = NULL; | |
| self->errors = internal_error_callback(errors); | |
| if (self->errors == NULL) | |
| goto errorexit; | |
| if (self->codec->encinit != NULL && | |
| self->codec->encinit(&self->state, self->codec) != 0) | |
| goto errorexit; | |
| Py_DECREF(codec); | |
| return (PyObject *)self; | |
| errorexit: | |
| Py_XDECREF(self); | |
| Py_XDECREF(codec); | |
| return NULL; | |
| } | |
| static int | |
| mbstreamwriter_init(PyObject *self, PyObject *args, PyObject *kwds) | |
| { | |
| return 0; | |
| } | |
| static int | |
| mbstreamwriter_traverse(PyObject *op, visitproc visit, void *arg) | |
| { | |
| MultibyteStreamWriterObject *self = _MultibyteStreamWriterObject_CAST(op); | |
| if (ERROR_ISCUSTOM(self->errors)) | |
| Py_VISIT(self->errors); | |
| Py_VISIT(self->stream); | |
| return 0; | |
| } | |
| static void | |
| mbstreamwriter_dealloc(PyObject *op) | |
| { | |
| MultibyteStreamWriterObject *self = _MultibyteStreamWriterObject_CAST(op); | |
| PyTypeObject *tp = Py_TYPE(self); | |
| PyObject_GC_UnTrack(self); | |
| ERROR_DECREF(self->errors); | |
| Py_XDECREF(self->stream); | |
| tp->tp_free(self); | |
| Py_DECREF(tp); | |
| } | |
| static struct PyMethodDef mbstreamwriter_methods[] = { | |
| _MULTIBYTECODEC_MULTIBYTESTREAMWRITER_WRITE_METHODDEF | |
| _MULTIBYTECODEC_MULTIBYTESTREAMWRITER_WRITELINES_METHODDEF | |
| _MULTIBYTECODEC_MULTIBYTESTREAMWRITER_RESET_METHODDEF | |
| {NULL, NULL}, | |
| }; | |
| static PyMemberDef mbstreamwriter_members[] = { | |
| {"stream", _Py_T_OBJECT, | |
| offsetof(MultibyteStreamWriterObject, stream), | |
| Py_READONLY, NULL}, | |
| {NULL,} | |
| }; | |
| static PyType_Slot writer_slots[] = { | |
| {Py_tp_dealloc, mbstreamwriter_dealloc}, | |
| {Py_tp_getattro, PyObject_GenericGetAttr}, | |
| {Py_tp_traverse, mbstreamwriter_traverse}, | |
| {Py_tp_methods, mbstreamwriter_methods}, | |
| {Py_tp_members, mbstreamwriter_members}, | |
| {Py_tp_getset, codecctx_getsets}, | |
| {Py_tp_init, mbstreamwriter_init}, | |
| {Py_tp_new, mbstreamwriter_new}, | |
| {0, NULL}, | |
| }; | |
| static PyType_Spec writer_spec = { | |
| .name = MODULE_NAME ".MultibyteStreamWriter", | |
| .basicsize = sizeof(MultibyteStreamWriterObject), | |
| .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE | | |
| Py_TPFLAGS_IMMUTABLETYPE), | |
| .slots = writer_slots, | |
| }; | |
| /*[clinic input] | |
| _multibytecodec.__create_codec | |
| arg: object | |
| / | |
| [clinic start generated code]*/ | |
| static PyObject * | |
| _multibytecodec___create_codec(PyObject *module, PyObject *arg) | |
| /*[clinic end generated code: output=cfa3dce8260e809d input=6840b2a6b183fcfa]*/ | |
| { | |
| MultibyteCodecObject *self; | |
| if (!PyCapsule_IsValid(arg, CODEC_CAPSULE)) { | |
| PyErr_SetString(PyExc_ValueError, "argument type invalid"); | |
| return NULL; | |
| } | |
| codec_capsule *data = PyCapsule_GetPointer(arg, CODEC_CAPSULE); | |
| const MultibyteCodec *codec = data->codec; | |
| if (codec->codecinit != NULL && codec->codecinit(codec) != 0) | |
| return NULL; | |
| module_state *state = get_module_state(module); | |
| self = PyObject_GC_New(MultibyteCodecObject, state->multibytecodec_type); | |
| if (self == NULL) | |
| return NULL; | |
| self->codec = codec; | |
| self->cjk_module = Py_NewRef(data->cjk_module); | |
| PyObject_GC_Track(self); | |
| return (PyObject *)self; | |
| } | |
| static int | |
| _multibytecodec_traverse(PyObject *mod, visitproc visit, void *arg) | |
| { | |
| module_state *state = get_module_state(mod); | |
| Py_VISIT(state->multibytecodec_type); | |
| Py_VISIT(state->encoder_type); | |
| Py_VISIT(state->decoder_type); | |
| Py_VISIT(state->reader_type); | |
| Py_VISIT(state->writer_type); | |
| return 0; | |
| } | |
| static int | |
| _multibytecodec_clear(PyObject *mod) | |
| { | |
| module_state *state = get_module_state(mod); | |
| Py_CLEAR(state->multibytecodec_type); | |
| Py_CLEAR(state->encoder_type); | |
| Py_CLEAR(state->decoder_type); | |
| Py_CLEAR(state->reader_type); | |
| Py_CLEAR(state->writer_type); | |
| Py_CLEAR(state->str_write); | |
| return 0; | |
| } | |
| static void | |
| _multibytecodec_free(void *mod) | |
| { | |
| (void)_multibytecodec_clear((PyObject *)mod); | |
| } | |
| #define CREATE_TYPE(module, type, spec) \ | |
| do { \ | |
| type = (PyTypeObject *)PyType_FromModuleAndSpec(module, spec, NULL); \ | |
| if (!type) { \ | |
| return -1; \ | |
| } \ | |
| } while (0) | |
| #define ADD_TYPE(module, type) \ | |
| do { \ | |
| if (PyModule_AddType(module, type) < 0) { \ | |
| return -1; \ | |
| } \ | |
| } while (0) | |
| static int | |
| _multibytecodec_exec(PyObject *mod) | |
| { | |
| module_state *state = get_module_state(mod); | |
| state->str_write = PyUnicode_InternFromString("write"); | |
| if (state->str_write == NULL) { | |
| return -1; | |
| } | |
| CREATE_TYPE(mod, state->multibytecodec_type, &multibytecodec_spec); | |
| CREATE_TYPE(mod, state->encoder_type, &encoder_spec); | |
| CREATE_TYPE(mod, state->decoder_type, &decoder_spec); | |
| CREATE_TYPE(mod, state->reader_type, &reader_spec); | |
| CREATE_TYPE(mod, state->writer_type, &writer_spec); | |
| ADD_TYPE(mod, state->encoder_type); | |
| ADD_TYPE(mod, state->decoder_type); | |
| ADD_TYPE(mod, state->reader_type); | |
| ADD_TYPE(mod, state->writer_type); | |
| return 0; | |
| } | |
| #undef CREATE_TYPE | |
| #undef ADD_TYPE | |
| static struct PyMethodDef _multibytecodec_methods[] = { | |
| _MULTIBYTECODEC___CREATE_CODEC_METHODDEF | |
| {NULL, NULL}, | |
| }; | |
| static PyModuleDef_Slot _multibytecodec_slots[] = { | |
| _Py_ABI_SLOT, | |
| {Py_mod_exec, _multibytecodec_exec}, | |
| {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, | |
| {Py_mod_gil, Py_MOD_GIL_NOT_USED}, | |
| {0, NULL} | |
| }; | |
| static struct PyModuleDef _multibytecodecmodule = { | |
| .m_base = PyModuleDef_HEAD_INIT, | |
| .m_name = "_multibytecodec", | |
| .m_size = sizeof(module_state), | |
| .m_methods = _multibytecodec_methods, | |
| .m_slots = _multibytecodec_slots, | |
| .m_traverse = _multibytecodec_traverse, | |
| .m_clear = _multibytecodec_clear, | |
| .m_free = _multibytecodec_free, | |
| }; | |
| PyMODINIT_FUNC | |
| PyInit__multibytecodec(void) | |
| { | |
| return PyModuleDef_Init(&_multibytecodecmodule); | |
| } |