1 /*
2 An implementation of Text I/O as defined by PEP 3116 - "New I/O"
3
4 Classes defined here: TextIOBase, IncrementalNewlineDecoder, TextIOWrapper.
5
6 Written by Amaury Forgeot d'Arc and Antoine Pitrou
7 */
8
9 #define PY_SSIZE_T_CLEAN
10 #include "Python.h"
11 #include "structmember.h"
12 #include "_iomodule.h"
13
14 /* TextIOBase */
15
16 PyDoc_STRVAR(textiobase_doc,
17 "Base class for text I/O.\n"
18 "\n"
19 "This class provides a character and line based interface to stream\n"
20 "I/O. There is no readinto method because Python's character strings\n"
21 "are immutable. There is no public constructor.\n"
22 );
23
24 static PyObject *
_unsupported(const char * message)25 _unsupported(const char *message)
26 {
27 PyErr_SetString(_PyIO_unsupported_operation, message);
28 return NULL;
29 }
30
31 PyDoc_STRVAR(textiobase_detach_doc,
32 "Separate the underlying buffer from the TextIOBase and return it.\n"
33 "\n"
34 "After the underlying buffer has been detached, the TextIO is in an\n"
35 "unusable state.\n"
36 );
37
38 static PyObject *
textiobase_detach(PyObject * self)39 textiobase_detach(PyObject *self)
40 {
41 return _unsupported("detach");
42 }
43
44 PyDoc_STRVAR(textiobase_read_doc,
45 "Read at most n characters from stream.\n"
46 "\n"
47 "Read from underlying buffer until we have n characters or we hit EOF.\n"
48 "If n is negative or omitted, read until EOF.\n"
49 );
50
51 static PyObject *
textiobase_read(PyObject * self,PyObject * args)52 textiobase_read(PyObject *self, PyObject *args)
53 {
54 return _unsupported("read");
55 }
56
57 PyDoc_STRVAR(textiobase_readline_doc,
58 "Read until newline or EOF.\n"
59 "\n"
60 "Returns an empty string if EOF is hit immediately.\n"
61 );
62
63 static PyObject *
textiobase_readline(PyObject * self,PyObject * args)64 textiobase_readline(PyObject *self, PyObject *args)
65 {
66 return _unsupported("readline");
67 }
68
69 PyDoc_STRVAR(textiobase_write_doc,
70 "Write string to stream.\n"
71 "Returns the number of characters written (which is always equal to\n"
72 "the length of the string).\n"
73 );
74
75 static PyObject *
textiobase_write(PyObject * self,PyObject * args)76 textiobase_write(PyObject *self, PyObject *args)
77 {
78 return _unsupported("write");
79 }
80
81 PyDoc_STRVAR(textiobase_encoding_doc,
82 "Encoding of the text stream.\n"
83 "\n"
84 "Subclasses should override.\n"
85 );
86
87 static PyObject *
textiobase_encoding_get(PyObject * self,void * context)88 textiobase_encoding_get(PyObject *self, void *context)
89 {
90 Py_RETURN_NONE;
91 }
92
93 PyDoc_STRVAR(textiobase_newlines_doc,
94 "Line endings translated so far.\n"
95 "\n"
96 "Only line endings translated during reading are considered.\n"
97 "\n"
98 "Subclasses should override.\n"
99 );
100
101 static PyObject *
textiobase_newlines_get(PyObject * self,void * context)102 textiobase_newlines_get(PyObject *self, void *context)
103 {
104 Py_RETURN_NONE;
105 }
106
107 PyDoc_STRVAR(textiobase_errors_doc,
108 "The error setting of the decoder or encoder.\n"
109 "\n"
110 "Subclasses should override.\n"
111 );
112
113 static PyObject *
textiobase_errors_get(PyObject * self,void * context)114 textiobase_errors_get(PyObject *self, void *context)
115 {
116 Py_RETURN_NONE;
117 }
118
119
120 static PyMethodDef textiobase_methods[] = {
121 {"detach", (PyCFunction)textiobase_detach, METH_NOARGS, textiobase_detach_doc},
122 {"read", textiobase_read, METH_VARARGS, textiobase_read_doc},
123 {"readline", textiobase_readline, METH_VARARGS, textiobase_readline_doc},
124 {"write", textiobase_write, METH_VARARGS, textiobase_write_doc},
125 {NULL, NULL}
126 };
127
128 static PyGetSetDef textiobase_getset[] = {
129 {"encoding", (getter)textiobase_encoding_get, NULL, textiobase_encoding_doc},
130 {"newlines", (getter)textiobase_newlines_get, NULL, textiobase_newlines_doc},
131 {"errors", (getter)textiobase_errors_get, NULL, textiobase_errors_doc},
132 {NULL}
133 };
134
135 PyTypeObject PyTextIOBase_Type = {
136 PyVarObject_HEAD_INIT(NULL, 0)
137 "_io._TextIOBase", /*tp_name*/
138 0, /*tp_basicsize*/
139 0, /*tp_itemsize*/
140 0, /*tp_dealloc*/
141 0, /*tp_print*/
142 0, /*tp_getattr*/
143 0, /*tp_setattr*/
144 0, /*tp_compare */
145 0, /*tp_repr*/
146 0, /*tp_as_number*/
147 0, /*tp_as_sequence*/
148 0, /*tp_as_mapping*/
149 0, /*tp_hash */
150 0, /*tp_call*/
151 0, /*tp_str*/
152 0, /*tp_getattro*/
153 0, /*tp_setattro*/
154 0, /*tp_as_buffer*/
155 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/
156 textiobase_doc, /* tp_doc */
157 0, /* tp_traverse */
158 0, /* tp_clear */
159 0, /* tp_richcompare */
160 0, /* tp_weaklistoffset */
161 0, /* tp_iter */
162 0, /* tp_iternext */
163 textiobase_methods, /* tp_methods */
164 0, /* tp_members */
165 textiobase_getset, /* tp_getset */
166 &PyIOBase_Type, /* tp_base */
167 0, /* tp_dict */
168 0, /* tp_descr_get */
169 0, /* tp_descr_set */
170 0, /* tp_dictoffset */
171 0, /* tp_init */
172 0, /* tp_alloc */
173 0, /* tp_new */
174 };
175
176
177 /* IncrementalNewlineDecoder */
178
179 PyDoc_STRVAR(incrementalnewlinedecoder_doc,
180 "Codec used when reading a file in universal newlines mode. It wraps\n"
181 "another incremental decoder, translating \\r\\n and \\r into \\n. It also\n"
182 "records the types of newlines encountered. When used with\n"
183 "translate=False, it ensures that the newline sequence is returned in\n"
184 "one piece. When used with decoder=None, it expects unicode strings as\n"
185 "decode input and translates newlines without first invoking an external\n"
186 "decoder.\n"
187 );
188
189 typedef struct {
190 PyObject_HEAD
191 PyObject *decoder;
192 PyObject *errors;
193 signed int pendingcr: 1;
194 signed int translate: 1;
195 unsigned int seennl: 3;
196 } nldecoder_object;
197
198 static int
incrementalnewlinedecoder_init(nldecoder_object * self,PyObject * args,PyObject * kwds)199 incrementalnewlinedecoder_init(nldecoder_object *self,
200 PyObject *args, PyObject *kwds)
201 {
202 PyObject *decoder;
203 int translate;
204 PyObject *errors = NULL;
205 char *kwlist[] = {"decoder", "translate", "errors", NULL};
206
207 if (!PyArg_ParseTupleAndKeywords(args, kwds, "Oi|O:IncrementalNewlineDecoder",
208 kwlist, &decoder, &translate, &errors))
209 return -1;
210
211 self->decoder = decoder;
212 Py_INCREF(decoder);
213
214 if (errors == NULL) {
215 self->errors = PyUnicode_FromString("strict");
216 if (self->errors == NULL)
217 return -1;
218 }
219 else {
220 Py_INCREF(errors);
221 self->errors = errors;
222 }
223
224 self->translate = translate;
225 self->seennl = 0;
226 self->pendingcr = 0;
227
228 return 0;
229 }
230
231 static void
incrementalnewlinedecoder_dealloc(nldecoder_object * self)232 incrementalnewlinedecoder_dealloc(nldecoder_object *self)
233 {
234 Py_CLEAR(self->decoder);
235 Py_CLEAR(self->errors);
236 Py_TYPE(self)->tp_free((PyObject *)self);
237 }
238
239 static int
check_decoded(PyObject * decoded)240 check_decoded(PyObject *decoded)
241 {
242 if (decoded == NULL)
243 return -1;
244 if (!PyUnicode_Check(decoded)) {
245 PyErr_Format(PyExc_TypeError,
246 "decoder should return a string result, not '%.200s'",
247 Py_TYPE(decoded)->tp_name);
248 Py_DECREF(decoded);
249 return -1;
250 }
251 return 0;
252 }
253
254 #define SEEN_CR 1
255 #define SEEN_LF 2
256 #define SEEN_CRLF 4
257 #define SEEN_ALL (SEEN_CR | SEEN_LF | SEEN_CRLF)
258
259 PyObject *
_PyIncrementalNewlineDecoder_decode(PyObject * _self,PyObject * input,int final)260 _PyIncrementalNewlineDecoder_decode(PyObject *_self,
261 PyObject *input, int final)
262 {
263 PyObject *output;
264 Py_ssize_t output_len;
265 nldecoder_object *self = (nldecoder_object *) _self;
266
267 if (self->decoder == NULL) {
268 PyErr_SetString(PyExc_ValueError,
269 "IncrementalNewlineDecoder.__init__ not called");
270 return NULL;
271 }
272
273 /* decode input (with the eventual \r from a previous pass) */
274 if (self->decoder != Py_None) {
275 output = PyObject_CallMethodObjArgs(self->decoder,
276 _PyIO_str_decode, input, final ? Py_True : Py_False, NULL);
277 }
278 else {
279 output = input;
280 Py_INCREF(output);
281 }
282
283 if (check_decoded(output) < 0)
284 return NULL;
285
286 output_len = PyUnicode_GET_SIZE(output);
287 if (self->pendingcr && (final || output_len > 0)) {
288 Py_UNICODE *out;
289 PyObject *modified = PyUnicode_FromUnicode(NULL, output_len + 1);
290 if (modified == NULL)
291 goto error;
292 out = PyUnicode_AS_UNICODE(modified);
293 out[0] = '\r';
294 memcpy(out + 1, PyUnicode_AS_UNICODE(output),
295 output_len * sizeof(Py_UNICODE));
296 Py_DECREF(output);
297 output = modified;
298 self->pendingcr = 0;
299 output_len++;
300 }
301
302 /* retain last \r even when not translating data:
303 * then readline() is sure to get \r\n in one pass
304 */
305 if (!final) {
306 if (output_len > 0
307 && PyUnicode_AS_UNICODE(output)[output_len - 1] == '\r') {
308
309 if (Py_REFCNT(output) == 1) {
310 if (PyUnicode_Resize(&output, output_len - 1) < 0)
311 goto error;
312 }
313 else {
314 PyObject *modified = PyUnicode_FromUnicode(
315 PyUnicode_AS_UNICODE(output),
316 output_len - 1);
317 if (modified == NULL)
318 goto error;
319 Py_DECREF(output);
320 output = modified;
321 }
322 self->pendingcr = 1;
323 }
324 }
325
326 /* Record which newlines are read and do newline translation if desired,
327 all in one pass. */
328 {
329 Py_UNICODE *in_str;
330 Py_ssize_t len;
331 int seennl = self->seennl;
332 int only_lf = 0;
333
334 in_str = PyUnicode_AS_UNICODE(output);
335 len = PyUnicode_GET_SIZE(output);
336
337 if (len == 0)
338 return output;
339
340 /* If, up to now, newlines are consistently \n, do a quick check
341 for the \r *byte* with the libc's optimized memchr.
342 */
343 if (seennl == SEEN_LF || seennl == 0) {
344 only_lf = (memchr(in_str, '\r', len * sizeof(Py_UNICODE)) == NULL);
345 }
346
347 if (only_lf) {
348 /* If not already seen, quick scan for a possible "\n" character.
349 (there's nothing else to be done, even when in translation mode)
350 */
351 if (seennl == 0 &&
352 memchr(in_str, '\n', len * sizeof(Py_UNICODE)) != NULL) {
353 Py_UNICODE *s, *end;
354 s = in_str;
355 end = in_str + len;
356 for (;;) {
357 Py_UNICODE c;
358 /* Fast loop for non-control characters */
359 while (*s > '\n')
360 s++;
361 c = *s++;
362 if (c == '\n') {
363 seennl |= SEEN_LF;
364 break;
365 }
366 if (s > end)
367 break;
368 }
369 }
370 /* Finished: we have scanned for newlines, and none of them
371 need translating */
372 }
373 else if (!self->translate) {
374 Py_UNICODE *s, *end;
375 /* We have already seen all newline types, no need to scan again */
376 if (seennl == SEEN_ALL)
377 goto endscan;
378 s = in_str;
379 end = in_str + len;
380 for (;;) {
381 Py_UNICODE c;
382 /* Fast loop for non-control characters */
383 while (*s > '\r')
384 s++;
385 c = *s++;
386 if (c == '\n')
387 seennl |= SEEN_LF;
388 else if (c == '\r') {
389 if (*s == '\n') {
390 seennl |= SEEN_CRLF;
391 s++;
392 }
393 else
394 seennl |= SEEN_CR;
395 }
396 if (s > end)
397 break;
398 if (seennl == SEEN_ALL)
399 break;
400 }
401 endscan:
402 ;
403 }
404 else {
405 PyObject *translated = NULL;
406 Py_UNICODE *out_str;
407 Py_UNICODE *in, *out, *end;
408 if (Py_REFCNT(output) != 1) {
409 /* We could try to optimize this so that we only do a copy
410 when there is something to translate. On the other hand,
411 most decoders should only output non-shared strings, i.e.
412 translation is done in place. */
413 translated = PyUnicode_FromUnicode(NULL, len);
414 if (translated == NULL)
415 goto error;
416 assert(Py_REFCNT(translated) == 1);
417 memcpy(PyUnicode_AS_UNICODE(translated),
418 PyUnicode_AS_UNICODE(output),
419 len * sizeof(Py_UNICODE));
420 }
421 else {
422 translated = output;
423 }
424 out_str = PyUnicode_AS_UNICODE(translated);
425 in = in_str;
426 out = out_str;
427 end = in_str + len;
428 for (;;) {
429 Py_UNICODE c;
430 /* Fast loop for non-control characters */
431 while ((c = *in++) > '\r')
432 *out++ = c;
433 if (c == '\n') {
434 *out++ = c;
435 seennl |= SEEN_LF;
436 continue;
437 }
438 if (c == '\r') {
439 if (*in == '\n') {
440 in++;
441 seennl |= SEEN_CRLF;
442 }
443 else
444 seennl |= SEEN_CR;
445 *out++ = '\n';
446 continue;
447 }
448 if (in > end)
449 break;
450 *out++ = c;
451 }
452 if (translated != output) {
453 Py_DECREF(output);
454 output = translated;
455 }
456 if (out - out_str != len) {
457 if (PyUnicode_Resize(&output, out - out_str) < 0)
458 goto error;
459 }
460 }
461 self->seennl |= seennl;
462 }
463
464 return output;
465
466 error:
467 Py_DECREF(output);
468 return NULL;
469 }
470
471 static PyObject *
incrementalnewlinedecoder_decode(nldecoder_object * self,PyObject * args,PyObject * kwds)472 incrementalnewlinedecoder_decode(nldecoder_object *self,
473 PyObject *args, PyObject *kwds)
474 {
475 char *kwlist[] = {"input", "final", NULL};
476 PyObject *input;
477 int final = 0;
478
479 if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|i:IncrementalNewlineDecoder",
480 kwlist, &input, &final))
481 return NULL;
482 return _PyIncrementalNewlineDecoder_decode((PyObject *) self, input, final);
483 }
484
485 static PyObject *
incrementalnewlinedecoder_getstate(nldecoder_object * self,PyObject * args)486 incrementalnewlinedecoder_getstate(nldecoder_object *self, PyObject *args)
487 {
488 PyObject *buffer;
489 unsigned PY_LONG_LONG flag;
490
491 if (self->decoder != Py_None) {
492 PyObject *state = PyObject_CallMethodObjArgs(self->decoder,
493 _PyIO_str_getstate, NULL);
494 if (state == NULL)
495 return NULL;
496 if (!PyArg_Parse(state, "(OK)", &buffer, &flag)) {
497 Py_DECREF(state);
498 return NULL;
499 }
500 Py_INCREF(buffer);
501 Py_DECREF(state);
502 }
503 else {
504 buffer = PyBytes_FromString("");
505 flag = 0;
506 }
507 flag <<= 1;
508 if (self->pendingcr)
509 flag |= 1;
510 return Py_BuildValue("NK", buffer, flag);
511 }
512
513 static PyObject *
incrementalnewlinedecoder_setstate(nldecoder_object * self,PyObject * state)514 incrementalnewlinedecoder_setstate(nldecoder_object *self, PyObject *state)
515 {
516 PyObject *buffer;
517 unsigned PY_LONG_LONG flag;
518
519 if (!PyArg_Parse(state, "(OK)", &buffer, &flag))
520 return NULL;
521
522 self->pendingcr = (int) flag & 1;
523 flag >>= 1;
524
525 if (self->decoder != Py_None)
526 return PyObject_CallMethod(self->decoder,
527 "setstate", "((OK))", buffer, flag);
528 else
529 Py_RETURN_NONE;
530 }
531
532 static PyObject *
incrementalnewlinedecoder_reset(nldecoder_object * self,PyObject * args)533 incrementalnewlinedecoder_reset(nldecoder_object *self, PyObject *args)
534 {
535 self->seennl = 0;
536 self->pendingcr = 0;
537 if (self->decoder != Py_None)
538 return PyObject_CallMethodObjArgs(self->decoder, _PyIO_str_reset, NULL);
539 else
540 Py_RETURN_NONE;
541 }
542
543 static PyObject *
incrementalnewlinedecoder_newlines_get(nldecoder_object * self,void * context)544 incrementalnewlinedecoder_newlines_get(nldecoder_object *self, void *context)
545 {
546 switch (self->seennl) {
547 case SEEN_CR:
548 return PyUnicode_FromString("\r");
549 case SEEN_LF:
550 return PyUnicode_FromString("\n");
551 case SEEN_CRLF:
552 return PyUnicode_FromString("\r\n");
553 case SEEN_CR | SEEN_LF:
554 return Py_BuildValue("ss", "\r", "\n");
555 case SEEN_CR | SEEN_CRLF:
556 return Py_BuildValue("ss", "\r", "\r\n");
557 case SEEN_LF | SEEN_CRLF:
558 return Py_BuildValue("ss", "\n", "\r\n");
559 case SEEN_CR | SEEN_LF | SEEN_CRLF:
560 return Py_BuildValue("sss", "\r", "\n", "\r\n");
561 default:
562 Py_RETURN_NONE;
563 }
564
565 }
566
567
568 static PyMethodDef incrementalnewlinedecoder_methods[] = {
569 {"decode", (PyCFunction)incrementalnewlinedecoder_decode, METH_VARARGS|METH_KEYWORDS},
570 {"getstate", (PyCFunction)incrementalnewlinedecoder_getstate, METH_NOARGS},
571 {"setstate", (PyCFunction)incrementalnewlinedecoder_setstate, METH_O},
572 {"reset", (PyCFunction)incrementalnewlinedecoder_reset, METH_NOARGS},
573 {NULL}
574 };
575
576 static PyGetSetDef incrementalnewlinedecoder_getset[] = {
577 {"newlines", (getter)incrementalnewlinedecoder_newlines_get, NULL, NULL},
578 {NULL}
579 };
580
581 PyTypeObject PyIncrementalNewlineDecoder_Type = {
582 PyVarObject_HEAD_INIT(NULL, 0)
583 "_io.IncrementalNewlineDecoder", /*tp_name*/
584 sizeof(nldecoder_object), /*tp_basicsize*/
585 0, /*tp_itemsize*/
586 (destructor)incrementalnewlinedecoder_dealloc, /*tp_dealloc*/
587 0, /*tp_print*/
588 0, /*tp_getattr*/
589 0, /*tp_setattr*/
590 0, /*tp_compare */
591 0, /*tp_repr*/
592 0, /*tp_as_number*/
593 0, /*tp_as_sequence*/
594 0, /*tp_as_mapping*/
595 0, /*tp_hash */
596 0, /*tp_call*/
597 0, /*tp_str*/
598 0, /*tp_getattro*/
599 0, /*tp_setattro*/
600 0, /*tp_as_buffer*/
601 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/
602 incrementalnewlinedecoder_doc, /* tp_doc */
603 0, /* tp_traverse */
604 0, /* tp_clear */
605 0, /* tp_richcompare */
606 0, /*tp_weaklistoffset*/
607 0, /* tp_iter */
608 0, /* tp_iternext */
609 incrementalnewlinedecoder_methods, /* tp_methods */
610 0, /* tp_members */
611 incrementalnewlinedecoder_getset, /* tp_getset */
612 0, /* tp_base */
613 0, /* tp_dict */
614 0, /* tp_descr_get */
615 0, /* tp_descr_set */
616 0, /* tp_dictoffset */
617 (initproc)incrementalnewlinedecoder_init, /* tp_init */
618 0, /* tp_alloc */
619 PyType_GenericNew, /* tp_new */
620 };
621
622
623 /* TextIOWrapper */
624
625 PyDoc_STRVAR(textiowrapper_doc,
626 "Character and line based layer over a BufferedIOBase object, buffer.\n"
627 "\n"
628 "encoding gives the name of the encoding that the stream will be\n"
629 "decoded or encoded with. It defaults to locale.getpreferredencoding.\n"
630 "\n"
631 "errors determines the strictness of encoding and decoding (see the\n"
632 "codecs.register) and defaults to \"strict\".\n"
633 "\n"
634 "newline controls how line endings are handled. It can be None, '',\n"
635 "'\\n', '\\r', and '\\r\\n'. It works as follows:\n"
636 "\n"
637 "* On input, if newline is None, universal newlines mode is\n"
638 " enabled. Lines in the input can end in '\\n', '\\r', or '\\r\\n', and\n"
639 " these are translated into '\\n' before being returned to the\n"
640 " caller. If it is '', universal newline mode is enabled, but line\n"
641 " endings are returned to the caller untranslated. If it has any of\n"
642 " the other legal values, input lines are only terminated by the given\n"
643 " string, and the line ending is returned to the caller untranslated.\n"
644 "\n"
645 "* On output, if newline is None, any '\\n' characters written are\n"
646 " translated to the system default line separator, os.linesep. If\n"
647 " newline is '', no translation takes place. If newline is any of the\n"
648 " other legal values, any '\\n' characters written are translated to\n"
649 " the given string.\n"
650 "\n"
651 "If line_buffering is True, a call to flush is implied when a call to\n"
652 "write contains a newline character."
653 );
654
655 typedef PyObject *
656 (*encodefunc_t)(PyObject *, PyObject *);
657
658 typedef struct
659 {
660 PyObject_HEAD
661 int ok; /* initialized? */
662 int detached;
663 Py_ssize_t chunk_size;
664 PyObject *buffer;
665 PyObject *encoding;
666 PyObject *encoder;
667 PyObject *decoder;
668 PyObject *readnl;
669 PyObject *errors;
670 const char *writenl; /* utf-8 encoded, NULL stands for \n */
671 char line_buffering;
672 char readuniversal;
673 char readtranslate;
674 char writetranslate;
675 char seekable;
676 char telling;
677 /* Specialized encoding func (see below) */
678 encodefunc_t encodefunc;
679 /* Whether or not it's the start of the stream */
680 char encoding_start_of_stream;
681
682 /* Reads and writes are internally buffered in order to speed things up.
683 However, any read will first flush the write buffer if itsn't empty.
684
685 Please also note that text to be written is first encoded before being
686 buffered. This is necessary so that encoding errors are immediately
687 reported to the caller, but it unfortunately means that the
688 IncrementalEncoder (whose encode() method is always written in Python)
689 becomes a bottleneck for small writes.
690 */
691 PyObject *decoded_chars; /* buffer for text returned from decoder */
692 Py_ssize_t decoded_chars_used; /* offset into _decoded_chars for read() */
693 PyObject *pending_bytes; /* list of bytes objects waiting to be
694 written, or NULL */
695 Py_ssize_t pending_bytes_count;
696 PyObject *snapshot;
697 /* snapshot is either None, or a tuple (dec_flags, next_input) where
698 * dec_flags is the second (integer) item of the decoder state and
699 * next_input is the chunk of input bytes that comes next after the
700 * snapshot point. We use this to reconstruct decoder states in tell().
701 */
702
703 /* Cache raw object if it's a FileIO object */
704 PyObject *raw;
705
706 PyObject *weakreflist;
707 PyObject *dict;
708 } textio;
709
710 static void
711 textiowrapper_set_decoded_chars(textio *self, PyObject *chars);
712
713 /* A couple of specialized cases in order to bypass the slow incremental
714 encoding methods for the most popular encodings. */
715
716 static PyObject *
ascii_encode(textio * self,PyObject * text)717 ascii_encode(textio *self, PyObject *text)
718 {
719 return PyUnicode_EncodeASCII(PyUnicode_AS_UNICODE(text),
720 PyUnicode_GET_SIZE(text),
721 PyBytes_AS_STRING(self->errors));
722 }
723
724 static PyObject *
utf16be_encode(textio * self,PyObject * text)725 utf16be_encode(textio *self, PyObject *text)
726 {
727 return PyUnicode_EncodeUTF16(PyUnicode_AS_UNICODE(text),
728 PyUnicode_GET_SIZE(text),
729 PyBytes_AS_STRING(self->errors), 1);
730 }
731
732 static PyObject *
utf16le_encode(textio * self,PyObject * text)733 utf16le_encode(textio *self, PyObject *text)
734 {
735 return PyUnicode_EncodeUTF16(PyUnicode_AS_UNICODE(text),
736 PyUnicode_GET_SIZE(text),
737 PyBytes_AS_STRING(self->errors), -1);
738 }
739
740 static PyObject *
utf16_encode(textio * self,PyObject * text)741 utf16_encode(textio *self, PyObject *text)
742 {
743 if (!self->encoding_start_of_stream) {
744 /* Skip the BOM and use native byte ordering */
745 #if defined(WORDS_BIGENDIAN)
746 return utf16be_encode(self, text);
747 #else
748 return utf16le_encode(self, text);
749 #endif
750 }
751 return PyUnicode_EncodeUTF16(PyUnicode_AS_UNICODE(text),
752 PyUnicode_GET_SIZE(text),
753 PyBytes_AS_STRING(self->errors), 0);
754 }
755
756 static PyObject *
utf32be_encode(textio * self,PyObject * text)757 utf32be_encode(textio *self, PyObject *text)
758 {
759 return PyUnicode_EncodeUTF32(PyUnicode_AS_UNICODE(text),
760 PyUnicode_GET_SIZE(text),
761 PyBytes_AS_STRING(self->errors), 1);
762 }
763
764 static PyObject *
utf32le_encode(textio * self,PyObject * text)765 utf32le_encode(textio *self, PyObject *text)
766 {
767 return PyUnicode_EncodeUTF32(PyUnicode_AS_UNICODE(text),
768 PyUnicode_GET_SIZE(text),
769 PyBytes_AS_STRING(self->errors), -1);
770 }
771
772 static PyObject *
utf32_encode(textio * self,PyObject * text)773 utf32_encode(textio *self, PyObject *text)
774 {
775 if (!self->encoding_start_of_stream) {
776 /* Skip the BOM and use native byte ordering */
777 #if defined(WORDS_BIGENDIAN)
778 return utf32be_encode(self, text);
779 #else
780 return utf32le_encode(self, text);
781 #endif
782 }
783 return PyUnicode_EncodeUTF32(PyUnicode_AS_UNICODE(text),
784 PyUnicode_GET_SIZE(text),
785 PyBytes_AS_STRING(self->errors), 0);
786 }
787
788 static PyObject *
utf8_encode(textio * self,PyObject * text)789 utf8_encode(textio *self, PyObject *text)
790 {
791 return PyUnicode_EncodeUTF8(PyUnicode_AS_UNICODE(text),
792 PyUnicode_GET_SIZE(text),
793 PyBytes_AS_STRING(self->errors));
794 }
795
796 static PyObject *
latin1_encode(textio * self,PyObject * text)797 latin1_encode(textio *self, PyObject *text)
798 {
799 return PyUnicode_EncodeLatin1(PyUnicode_AS_UNICODE(text),
800 PyUnicode_GET_SIZE(text),
801 PyBytes_AS_STRING(self->errors));
802 }
803
804 /* Map normalized encoding names onto the specialized encoding funcs */
805
806 typedef struct {
807 const char *name;
808 encodefunc_t encodefunc;
809 } encodefuncentry;
810
811 static encodefuncentry encodefuncs[] = {
812 {"ascii", (encodefunc_t) ascii_encode},
813 {"iso8859-1", (encodefunc_t) latin1_encode},
814 {"utf-8", (encodefunc_t) utf8_encode},
815 {"utf-16-be", (encodefunc_t) utf16be_encode},
816 {"utf-16-le", (encodefunc_t) utf16le_encode},
817 {"utf-16", (encodefunc_t) utf16_encode},
818 {"utf-32-be", (encodefunc_t) utf32be_encode},
819 {"utf-32-le", (encodefunc_t) utf32le_encode},
820 {"utf-32", (encodefunc_t) utf32_encode},
821 {NULL, NULL}
822 };
823
824
825 static int
textiowrapper_init(textio * self,PyObject * args,PyObject * kwds)826 textiowrapper_init(textio *self, PyObject *args, PyObject *kwds)
827 {
828 char *kwlist[] = {"buffer", "encoding", "errors",
829 "newline", "line_buffering",
830 NULL};
831 PyObject *buffer, *raw, *codec_info = NULL;
832 char *encoding = NULL;
833 char *errors = NULL;
834 char *newline = NULL;
835 int line_buffering = 0;
836
837 PyObject *res;
838 int r;
839
840 self->ok = 0;
841 self->detached = 0;
842 if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|zzzi:fileio",
843 kwlist, &buffer, &encoding, &errors,
844 &newline, &line_buffering))
845 return -1;
846
847 if (newline && newline[0] != '\0'
848 && !(newline[0] == '\n' && newline[1] == '\0')
849 && !(newline[0] == '\r' && newline[1] == '\0')
850 && !(newline[0] == '\r' && newline[1] == '\n' && newline[2] == '\0')) {
851 PyErr_Format(PyExc_ValueError,
852 "illegal newline value: %s", newline);
853 return -1;
854 }
855
856 Py_CLEAR(self->buffer);
857 Py_CLEAR(self->encoding);
858 Py_CLEAR(self->encoder);
859 Py_CLEAR(self->decoder);
860 Py_CLEAR(self->readnl);
861 Py_CLEAR(self->decoded_chars);
862 Py_CLEAR(self->pending_bytes);
863 Py_CLEAR(self->snapshot);
864 Py_CLEAR(self->errors);
865 Py_CLEAR(self->raw);
866 self->decoded_chars_used = 0;
867 self->pending_bytes_count = 0;
868 self->encodefunc = NULL;
869 self->writenl = NULL;
870
871 if (encoding == NULL && self->encoding == NULL) {
872 if (_PyIO_locale_module == NULL) {
873 _PyIO_locale_module = PyImport_ImportModule("locale");
874 if (_PyIO_locale_module == NULL)
875 goto catch_ImportError;
876 else
877 goto use_locale;
878 }
879 else {
880 use_locale:
881 self->encoding = PyObject_CallMethod(
882 _PyIO_locale_module, "getpreferredencoding", NULL);
883 if (self->encoding == NULL) {
884 catch_ImportError:
885 /*
886 Importing locale can raise an ImportError because of
887 _functools, and locale.getpreferredencoding can raise an
888 ImportError if _locale is not available. These will happen
889 during module building.
890 */
891 if (PyErr_ExceptionMatches(PyExc_ImportError)) {
892 PyErr_Clear();
893 self->encoding = PyString_FromString("ascii");
894 }
895 else
896 goto error;
897 }
898 else if (!PyString_Check(self->encoding))
899 Py_CLEAR(self->encoding);
900 }
901 }
902 if (self->encoding != NULL)
903 encoding = PyString_AsString(self->encoding);
904 else if (encoding != NULL) {
905 self->encoding = PyString_FromString(encoding);
906 if (self->encoding == NULL)
907 goto error;
908 }
909 else {
910 PyErr_SetString(PyExc_IOError,
911 "could not determine default encoding");
912 goto error;
913 }
914
915 /* Check we have been asked for a real text encoding */
916 codec_info = _PyCodec_LookupTextEncoding(encoding, "codecs.open()");
917 if (codec_info == NULL) {
918 Py_CLEAR(self->encoding);
919 goto error;
920 }
921
922 /* XXX: Failures beyond this point have the potential to leak elements
923 * of the partially constructed object (like self->encoding)
924 */
925
926 if (errors == NULL)
927 errors = "strict";
928 self->errors = PyBytes_FromString(errors);
929 if (self->errors == NULL)
930 goto error;
931
932 self->chunk_size = 8192;
933 self->readuniversal = (newline == NULL || newline[0] == '\0');
934 self->line_buffering = line_buffering;
935 self->readtranslate = (newline == NULL);
936 if (newline) {
937 self->readnl = PyString_FromString(newline);
938 if (self->readnl == NULL)
939 goto error;
940 }
941 self->writetranslate = (newline == NULL || newline[0] != '\0');
942 if (!self->readuniversal && self->writetranslate) {
943 self->writenl = PyString_AsString(self->readnl);
944 if (!strcmp(self->writenl, "\n"))
945 self->writenl = NULL;
946 }
947 #ifdef MS_WINDOWS
948 else
949 self->writenl = "\r\n";
950 #endif
951
952 /* Build the decoder object */
953 res = PyObject_CallMethod(buffer, "readable", NULL);
954 if (res == NULL)
955 goto error;
956 r = PyObject_IsTrue(res);
957 Py_DECREF(res);
958 if (r == -1)
959 goto error;
960 if (r == 1) {
961 self->decoder = _PyCodecInfo_GetIncrementalDecoder(codec_info,
962 errors);
963 if (self->decoder == NULL)
964 goto error;
965
966 if (self->readuniversal) {
967 PyObject *incrementalDecoder = PyObject_CallFunction(
968 (PyObject *)&PyIncrementalNewlineDecoder_Type,
969 "Oi", self->decoder, (int)self->readtranslate);
970 if (incrementalDecoder == NULL)
971 goto error;
972 Py_XSETREF(self->decoder, incrementalDecoder);
973 }
974 }
975
976 /* Build the encoder object */
977 res = PyObject_CallMethod(buffer, "writable", NULL);
978 if (res == NULL)
979 goto error;
980 r = PyObject_IsTrue(res);
981 Py_DECREF(res);
982 if (r == -1)
983 goto error;
984 if (r == 1) {
985 self->encoder = _PyCodecInfo_GetIncrementalEncoder(codec_info,
986 errors);
987 if (self->encoder == NULL)
988 goto error;
989 /* Get the normalized name of the codec */
990 res = PyObject_GetAttrString(codec_info, "name");
991 if (res == NULL) {
992 if (PyErr_ExceptionMatches(PyExc_AttributeError))
993 PyErr_Clear();
994 else
995 goto error;
996 }
997 else if (PyString_Check(res)) {
998 encodefuncentry *e = encodefuncs;
999 while (e->name != NULL) {
1000 if (!strcmp(PyString_AS_STRING(res), e->name)) {
1001 self->encodefunc = e->encodefunc;
1002 break;
1003 }
1004 e++;
1005 }
1006 }
1007 Py_XDECREF(res);
1008 }
1009
1010 /* Finished sorting out the codec details */
1011 Py_DECREF(codec_info);
1012
1013 self->buffer = buffer;
1014 Py_INCREF(buffer);
1015
1016 if (Py_TYPE(buffer) == &PyBufferedReader_Type ||
1017 Py_TYPE(buffer) == &PyBufferedWriter_Type ||
1018 Py_TYPE(buffer) == &PyBufferedRandom_Type) {
1019 raw = PyObject_GetAttrString(buffer, "raw");
1020 /* Cache the raw FileIO object to speed up 'closed' checks */
1021 if (raw == NULL) {
1022 if (PyErr_ExceptionMatches(PyExc_AttributeError))
1023 PyErr_Clear();
1024 else
1025 goto error;
1026 }
1027 else if (Py_TYPE(raw) == &PyFileIO_Type)
1028 self->raw = raw;
1029 else
1030 Py_DECREF(raw);
1031 }
1032
1033 res = PyObject_CallMethod(buffer, "seekable", NULL);
1034 if (res == NULL)
1035 goto error;
1036 r = PyObject_IsTrue(res);
1037 Py_DECREF(res);
1038 if (r < 0)
1039 goto error;
1040 self->seekable = self->telling = r;
1041
1042 self->encoding_start_of_stream = 0;
1043 if (self->seekable && self->encoder) {
1044 PyObject *cookieObj;
1045 int cmp;
1046
1047 self->encoding_start_of_stream = 1;
1048
1049 cookieObj = PyObject_CallMethodObjArgs(buffer, _PyIO_str_tell, NULL);
1050 if (cookieObj == NULL)
1051 goto error;
1052
1053 cmp = PyObject_RichCompareBool(cookieObj, _PyIO_zero, Py_EQ);
1054 Py_DECREF(cookieObj);
1055 if (cmp < 0) {
1056 goto error;
1057 }
1058
1059 if (cmp == 0) {
1060 self->encoding_start_of_stream = 0;
1061 res = PyObject_CallMethodObjArgs(self->encoder, _PyIO_str_setstate,
1062 _PyIO_zero, NULL);
1063 if (res == NULL)
1064 goto error;
1065 Py_DECREF(res);
1066 }
1067 }
1068
1069 self->ok = 1;
1070 return 0;
1071
1072 error:
1073 Py_XDECREF(codec_info);
1074 return -1;
1075 }
1076
1077 static void
_textiowrapper_clear(textio * self)1078 _textiowrapper_clear(textio *self)
1079 {
1080 self->ok = 0;
1081 Py_CLEAR(self->buffer);
1082 Py_CLEAR(self->encoding);
1083 Py_CLEAR(self->encoder);
1084 Py_CLEAR(self->decoder);
1085 Py_CLEAR(self->readnl);
1086 Py_CLEAR(self->decoded_chars);
1087 Py_CLEAR(self->pending_bytes);
1088 Py_CLEAR(self->snapshot);
1089 Py_CLEAR(self->errors);
1090 Py_CLEAR(self->raw);
1091
1092 Py_CLEAR(self->dict);
1093 }
1094
1095 static void
textiowrapper_dealloc(textio * self)1096 textiowrapper_dealloc(textio *self)
1097 {
1098 if (self->ok && _PyIOBase_finalize((PyObject *) self) < 0)
1099 return;
1100 _PyObject_GC_UNTRACK(self);
1101 if (self->weakreflist != NULL)
1102 PyObject_ClearWeakRefs((PyObject *)self);
1103 _textiowrapper_clear(self);
1104 Py_TYPE(self)->tp_free((PyObject *)self);
1105 }
1106
1107 static int
textiowrapper_traverse(textio * self,visitproc visit,void * arg)1108 textiowrapper_traverse(textio *self, visitproc visit, void *arg)
1109 {
1110 Py_VISIT(self->buffer);
1111 Py_VISIT(self->encoding);
1112 Py_VISIT(self->encoder);
1113 Py_VISIT(self->decoder);
1114 Py_VISIT(self->readnl);
1115 Py_VISIT(self->decoded_chars);
1116 Py_VISIT(self->pending_bytes);
1117 Py_VISIT(self->snapshot);
1118 Py_VISIT(self->errors);
1119 Py_VISIT(self->raw);
1120
1121 Py_VISIT(self->dict);
1122 return 0;
1123 }
1124
1125 static int
textiowrapper_clear(textio * self)1126 textiowrapper_clear(textio *self)
1127 {
1128 if (self->ok && _PyIOBase_finalize((PyObject *) self) < 0)
1129 return -1;
1130 _textiowrapper_clear(self);
1131 return 0;
1132 }
1133
1134 static PyObject *
1135 textiowrapper_closed_get(textio *self, void *context);
1136
1137 /* This macro takes some shortcuts to make the common case faster. */
1138 #define CHECK_CLOSED(self) \
1139 do { \
1140 int r; \
1141 PyObject *_res; \
1142 if (Py_TYPE(self) == &PyTextIOWrapper_Type) { \
1143 if (self->raw != NULL) \
1144 r = _PyFileIO_closed(self->raw); \
1145 else { \
1146 _res = textiowrapper_closed_get(self, NULL); \
1147 if (_res == NULL) \
1148 return NULL; \
1149 r = PyObject_IsTrue(_res); \
1150 Py_DECREF(_res); \
1151 if (r < 0) \
1152 return NULL; \
1153 } \
1154 if (r > 0) { \
1155 PyErr_SetString(PyExc_ValueError, \
1156 "I/O operation on closed file."); \
1157 return NULL; \
1158 } \
1159 } \
1160 else if (_PyIOBase_check_closed((PyObject *)self, Py_True) == NULL) \
1161 return NULL; \
1162 } while (0)
1163
1164 #define CHECK_INITIALIZED(self) \
1165 if (self->ok <= 0) { \
1166 PyErr_SetString(PyExc_ValueError, \
1167 "I/O operation on uninitialized object"); \
1168 return NULL; \
1169 }
1170
1171 #define CHECK_ATTACHED(self) \
1172 CHECK_INITIALIZED(self); \
1173 if (self->detached) { \
1174 PyErr_SetString(PyExc_ValueError, \
1175 "underlying buffer has been detached"); \
1176 return NULL; \
1177 }
1178
1179 #define CHECK_ATTACHED_INT(self) \
1180 if (self->ok <= 0) { \
1181 PyErr_SetString(PyExc_ValueError, \
1182 "I/O operation on uninitialized object"); \
1183 return -1; \
1184 } else if (self->detached) { \
1185 PyErr_SetString(PyExc_ValueError, \
1186 "underlying buffer has been detached"); \
1187 return -1; \
1188 }
1189
1190
1191 static PyObject *
textiowrapper_detach(textio * self)1192 textiowrapper_detach(textio *self)
1193 {
1194 PyObject *buffer, *res;
1195 CHECK_ATTACHED(self);
1196 res = PyObject_CallMethodObjArgs((PyObject *)self, _PyIO_str_flush, NULL);
1197 if (res == NULL)
1198 return NULL;
1199 Py_DECREF(res);
1200 buffer = self->buffer;
1201 self->buffer = NULL;
1202 self->detached = 1;
1203 return buffer;
1204 }
1205
1206 Py_LOCAL_INLINE(const Py_UNICODE *)
findchar(const Py_UNICODE * s,Py_ssize_t size,Py_UNICODE ch)1207 findchar(const Py_UNICODE *s, Py_ssize_t size, Py_UNICODE ch)
1208 {
1209 /* like wcschr, but doesn't stop at NULL characters */
1210 while (size-- > 0) {
1211 if (*s == ch)
1212 return s;
1213 s++;
1214 }
1215 return NULL;
1216 }
1217
1218 /* Flush the internal write buffer. This doesn't explicitly flush the
1219 underlying buffered object, though. */
1220 static int
_textiowrapper_writeflush(textio * self)1221 _textiowrapper_writeflush(textio *self)
1222 {
1223 PyObject *pending, *b, *ret;
1224
1225 if (self->pending_bytes == NULL)
1226 return 0;
1227
1228 pending = self->pending_bytes;
1229 Py_INCREF(pending);
1230 self->pending_bytes_count = 0;
1231 Py_CLEAR(self->pending_bytes);
1232
1233 b = _PyBytes_Join(_PyIO_empty_bytes, pending);
1234 Py_DECREF(pending);
1235 if (b == NULL)
1236 return -1;
1237 ret = NULL;
1238 do {
1239 ret = PyObject_CallMethodObjArgs(self->buffer,
1240 _PyIO_str_write, b, NULL);
1241 } while (ret == NULL && _PyIO_trap_eintr());
1242 Py_DECREF(b);
1243 if (ret == NULL)
1244 return -1;
1245 Py_DECREF(ret);
1246 return 0;
1247 }
1248
1249 static PyObject *
textiowrapper_write(textio * self,PyObject * args)1250 textiowrapper_write(textio *self, PyObject *args)
1251 {
1252 PyObject *ret;
1253 PyObject *text; /* owned reference */
1254 PyObject *b;
1255 Py_ssize_t textlen;
1256 int haslf = 0;
1257 int needflush = 0;
1258
1259 CHECK_ATTACHED(self);
1260
1261 if (!PyArg_ParseTuple(args, "U:write", &text)) {
1262 return NULL;
1263 }
1264
1265 CHECK_CLOSED(self);
1266
1267 if (self->encoder == NULL) {
1268 PyErr_SetString(PyExc_IOError, "not writable");
1269 return NULL;
1270 }
1271
1272 Py_INCREF(text);
1273
1274 textlen = PyUnicode_GetSize(text);
1275
1276 if ((self->writetranslate && self->writenl != NULL) || self->line_buffering)
1277 if (findchar(PyUnicode_AS_UNICODE(text),
1278 PyUnicode_GET_SIZE(text), '\n'))
1279 haslf = 1;
1280
1281 if (haslf && self->writetranslate && self->writenl != NULL) {
1282 PyObject *newtext = PyObject_CallMethod(
1283 text, "replace", "ss", "\n", self->writenl);
1284 Py_DECREF(text);
1285 if (newtext == NULL)
1286 return NULL;
1287 text = newtext;
1288 }
1289
1290 if (self->line_buffering &&
1291 (haslf ||
1292 findchar(PyUnicode_AS_UNICODE(text),
1293 PyUnicode_GET_SIZE(text), '\r')))
1294 needflush = 1;
1295
1296 /* XXX What if we were just reading? */
1297 if (self->encodefunc != NULL) {
1298 b = (*self->encodefunc)((PyObject *) self, text);
1299 self->encoding_start_of_stream = 0;
1300 }
1301 else
1302 b = PyObject_CallMethodObjArgs(self->encoder,
1303 _PyIO_str_encode, text, NULL);
1304 Py_DECREF(text);
1305 if (b == NULL)
1306 return NULL;
1307
1308 if (self->pending_bytes == NULL) {
1309 self->pending_bytes = PyList_New(0);
1310 if (self->pending_bytes == NULL) {
1311 Py_DECREF(b);
1312 return NULL;
1313 }
1314 self->pending_bytes_count = 0;
1315 }
1316 if (PyList_Append(self->pending_bytes, b) < 0) {
1317 Py_DECREF(b);
1318 return NULL;
1319 }
1320 self->pending_bytes_count += PyBytes_GET_SIZE(b);
1321 Py_DECREF(b);
1322 if (self->pending_bytes_count > self->chunk_size || needflush) {
1323 if (_textiowrapper_writeflush(self) < 0)
1324 return NULL;
1325 }
1326
1327 if (needflush) {
1328 ret = PyObject_CallMethodObjArgs(self->buffer, _PyIO_str_flush, NULL);
1329 if (ret == NULL)
1330 return NULL;
1331 Py_DECREF(ret);
1332 }
1333
1334 textiowrapper_set_decoded_chars(self, NULL);
1335 Py_CLEAR(self->snapshot);
1336
1337 if (self->decoder) {
1338 ret = PyObject_CallMethod(self->decoder, "reset", NULL);
1339 if (ret == NULL)
1340 return NULL;
1341 Py_DECREF(ret);
1342 }
1343
1344 return PyLong_FromSsize_t(textlen);
1345 }
1346
1347 /* Steal a reference to chars and store it in the decoded_char buffer;
1348 */
1349 static void
textiowrapper_set_decoded_chars(textio * self,PyObject * chars)1350 textiowrapper_set_decoded_chars(textio *self, PyObject *chars)
1351 {
1352 Py_XSETREF(self->decoded_chars, chars);
1353 self->decoded_chars_used = 0;
1354 }
1355
1356 static PyObject *
textiowrapper_get_decoded_chars(textio * self,Py_ssize_t n)1357 textiowrapper_get_decoded_chars(textio *self, Py_ssize_t n)
1358 {
1359 PyObject *chars;
1360 Py_ssize_t avail;
1361
1362 if (self->decoded_chars == NULL)
1363 return PyUnicode_FromStringAndSize(NULL, 0);
1364
1365 avail = (PyUnicode_GET_SIZE(self->decoded_chars)
1366 - self->decoded_chars_used);
1367
1368 assert(avail >= 0);
1369
1370 if (n < 0 || n > avail)
1371 n = avail;
1372
1373 if (self->decoded_chars_used > 0 || n < avail) {
1374 chars = PyUnicode_FromUnicode(
1375 PyUnicode_AS_UNICODE(self->decoded_chars)
1376 + self->decoded_chars_used, n);
1377 if (chars == NULL)
1378 return NULL;
1379 }
1380 else {
1381 chars = self->decoded_chars;
1382 Py_INCREF(chars);
1383 }
1384
1385 self->decoded_chars_used += n;
1386 return chars;
1387 }
1388
1389 /* Read and decode the next chunk of data from the BufferedReader.
1390 */
1391 static int
textiowrapper_read_chunk(textio * self)1392 textiowrapper_read_chunk(textio *self)
1393 {
1394 PyObject *dec_buffer = NULL;
1395 PyObject *dec_flags = NULL;
1396 PyObject *input_chunk = NULL;
1397 PyObject *decoded_chars, *chunk_size;
1398 int eof;
1399
1400 /* The return value is True unless EOF was reached. The decoded string is
1401 * placed in self._decoded_chars (replacing its previous value). The
1402 * entire input chunk is sent to the decoder, though some of it may remain
1403 * buffered in the decoder, yet to be converted.
1404 */
1405
1406 if (self->decoder == NULL) {
1407 PyErr_SetString(PyExc_IOError, "not readable");
1408 return -1;
1409 }
1410
1411 if (self->telling) {
1412 /* To prepare for tell(), we need to snapshot a point in the file
1413 * where the decoder's input buffer is empty.
1414 */
1415
1416 PyObject *state = PyObject_CallMethodObjArgs(self->decoder,
1417 _PyIO_str_getstate, NULL);
1418 if (state == NULL)
1419 return -1;
1420 /* Given this, we know there was a valid snapshot point
1421 * len(dec_buffer) bytes ago with decoder state (b'', dec_flags).
1422 */
1423 if (!PyArg_Parse(state, "(OO)", &dec_buffer, &dec_flags)) {
1424 Py_DECREF(state);
1425 return -1;
1426 }
1427 Py_INCREF(dec_buffer);
1428 Py_INCREF(dec_flags);
1429 Py_DECREF(state);
1430 }
1431
1432 /* Read a chunk, decode it, and put the result in self._decoded_chars. */
1433 chunk_size = PyLong_FromSsize_t(self->chunk_size);
1434 if (chunk_size == NULL)
1435 goto fail;
1436 input_chunk = PyObject_CallMethodObjArgs(self->buffer,
1437 _PyIO_str_read1, chunk_size, NULL);
1438 Py_DECREF(chunk_size);
1439 if (input_chunk == NULL)
1440 goto fail;
1441 if (!PyBytes_Check(input_chunk)) {
1442 PyErr_Format(PyExc_TypeError,
1443 "underlying read1() should have returned a bytes object, "
1444 "not '%.200s'", Py_TYPE(input_chunk)->tp_name);
1445 goto fail;
1446 }
1447
1448 eof = (PyBytes_Size(input_chunk) == 0);
1449
1450 if (Py_TYPE(self->decoder) == &PyIncrementalNewlineDecoder_Type) {
1451 decoded_chars = _PyIncrementalNewlineDecoder_decode(
1452 self->decoder, input_chunk, eof);
1453 }
1454 else {
1455 decoded_chars = PyObject_CallMethodObjArgs(self->decoder,
1456 _PyIO_str_decode, input_chunk, eof ? Py_True : Py_False, NULL);
1457 }
1458
1459 if (check_decoded(decoded_chars) < 0)
1460 goto fail;
1461 textiowrapper_set_decoded_chars(self, decoded_chars);
1462 if (PyUnicode_GET_SIZE(decoded_chars) > 0)
1463 eof = 0;
1464
1465 if (self->telling) {
1466 /* At the snapshot point, len(dec_buffer) bytes before the read, the
1467 * next input to be decoded is dec_buffer + input_chunk.
1468 */
1469 PyObject *next_input = PyNumber_Add(dec_buffer, input_chunk);
1470 if (next_input == NULL)
1471 goto fail;
1472 if (!PyBytes_Check(next_input)) {
1473 PyErr_Format(PyExc_TypeError,
1474 "decoder getstate() should have returned a bytes "
1475 "object, not '%.200s'",
1476 Py_TYPE(next_input)->tp_name);
1477 Py_DECREF(next_input);
1478 goto fail;
1479 }
1480 Py_DECREF(dec_buffer);
1481 Py_XSETREF(self->snapshot, Py_BuildValue("NN", dec_flags, next_input));
1482 }
1483 Py_DECREF(input_chunk);
1484
1485 return (eof == 0);
1486
1487 fail:
1488 Py_XDECREF(dec_buffer);
1489 Py_XDECREF(dec_flags);
1490 Py_XDECREF(input_chunk);
1491 return -1;
1492 }
1493
1494 static PyObject *
textiowrapper_read(textio * self,PyObject * args)1495 textiowrapper_read(textio *self, PyObject *args)
1496 {
1497 Py_ssize_t n = -1;
1498 PyObject *result = NULL, *chunks = NULL;
1499
1500 CHECK_ATTACHED(self);
1501
1502 if (!PyArg_ParseTuple(args, "|O&:read", &_PyIO_ConvertSsize_t, &n))
1503 return NULL;
1504
1505 CHECK_CLOSED(self);
1506
1507 if (self->decoder == NULL) {
1508 PyErr_SetString(PyExc_IOError, "not readable");
1509 return NULL;
1510 }
1511
1512 if (_textiowrapper_writeflush(self) < 0)
1513 return NULL;
1514
1515 if (n < 0) {
1516 /* Read everything */
1517 PyObject *bytes = PyObject_CallMethod(self->buffer, "read", NULL);
1518 PyObject *decoded, *final;
1519 if (bytes == NULL)
1520 goto fail;
1521 decoded = PyObject_CallMethodObjArgs(self->decoder, _PyIO_str_decode,
1522 bytes, Py_True, NULL);
1523 Py_DECREF(bytes);
1524 if (check_decoded(decoded) < 0)
1525 goto fail;
1526
1527 result = textiowrapper_get_decoded_chars(self, -1);
1528
1529 if (result == NULL) {
1530 Py_DECREF(decoded);
1531 return NULL;
1532 }
1533
1534 final = PyUnicode_Concat(result, decoded);
1535 Py_DECREF(result);
1536 Py_DECREF(decoded);
1537 if (final == NULL)
1538 goto fail;
1539
1540 textiowrapper_set_decoded_chars(self, NULL);
1541 Py_CLEAR(self->snapshot);
1542 return final;
1543 }
1544 else {
1545 int res = 1;
1546 Py_ssize_t remaining = n;
1547
1548 result = textiowrapper_get_decoded_chars(self, n);
1549 if (result == NULL)
1550 goto fail;
1551 remaining -= PyUnicode_GET_SIZE(result);
1552
1553 /* Keep reading chunks until we have n characters to return */
1554 while (remaining > 0) {
1555 res = textiowrapper_read_chunk(self);
1556 if (res < 0) {
1557 /* NOTE: PyErr_SetFromErrno() calls PyErr_CheckSignals()
1558 when EINTR occurs so we needn't do it ourselves. */
1559 if (_PyIO_trap_eintr()) {
1560 continue;
1561 }
1562 goto fail;
1563 }
1564 if (res == 0) /* EOF */
1565 break;
1566 if (chunks == NULL) {
1567 chunks = PyList_New(0);
1568 if (chunks == NULL)
1569 goto fail;
1570 }
1571 if (PyList_Append(chunks, result) < 0)
1572 goto fail;
1573 Py_DECREF(result);
1574 result = textiowrapper_get_decoded_chars(self, remaining);
1575 if (result == NULL)
1576 goto fail;
1577 remaining -= PyUnicode_GET_SIZE(result);
1578 }
1579 if (chunks != NULL) {
1580 if (result != NULL && PyList_Append(chunks, result) < 0)
1581 goto fail;
1582 Py_XSETREF(result, PyUnicode_Join(_PyIO_empty_str, chunks));
1583 if (result == NULL)
1584 goto fail;
1585 Py_CLEAR(chunks);
1586 }
1587 return result;
1588 }
1589 fail:
1590 Py_XDECREF(result);
1591 Py_XDECREF(chunks);
1592 return NULL;
1593 }
1594
1595
1596 /* NOTE: `end` must point to the real end of the Py_UNICODE storage,
1597 that is to the NUL character. Otherwise the function will produce
1598 incorrect results. */
1599 static Py_UNICODE *
find_control_char(Py_UNICODE * start,Py_UNICODE * end,Py_UNICODE ch)1600 find_control_char(Py_UNICODE *start, Py_UNICODE *end, Py_UNICODE ch)
1601 {
1602 Py_UNICODE *s = start;
1603 for (;;) {
1604 while (*s > ch)
1605 s++;
1606 if (*s == ch)
1607 return s;
1608 if (s == end)
1609 return NULL;
1610 s++;
1611 }
1612 }
1613
1614 Py_ssize_t
_PyIO_find_line_ending(int translated,int universal,PyObject * readnl,Py_UNICODE * start,Py_UNICODE * end,Py_ssize_t * consumed)1615 _PyIO_find_line_ending(
1616 int translated, int universal, PyObject *readnl,
1617 Py_UNICODE *start, Py_UNICODE *end, Py_ssize_t *consumed)
1618 {
1619 Py_ssize_t len = end - start;
1620
1621 if (translated) {
1622 /* Newlines are already translated, only search for \n */
1623 Py_UNICODE *pos = find_control_char(start, end, '\n');
1624 if (pos != NULL)
1625 return pos - start + 1;
1626 else {
1627 *consumed = len;
1628 return -1;
1629 }
1630 }
1631 else if (universal) {
1632 /* Universal newline search. Find any of \r, \r\n, \n
1633 * The decoder ensures that \r\n are not split in two pieces
1634 */
1635 Py_UNICODE *s = start;
1636 for (;;) {
1637 Py_UNICODE ch;
1638 /* Fast path for non-control chars. The loop always ends
1639 since the Py_UNICODE storage is NUL-terminated. */
1640 while (*s > '\r')
1641 s++;
1642 if (s >= end) {
1643 *consumed = len;
1644 return -1;
1645 }
1646 ch = *s++;
1647 if (ch == '\n')
1648 return s - start;
1649 if (ch == '\r') {
1650 if (*s == '\n')
1651 return s - start + 1;
1652 else
1653 return s - start;
1654 }
1655 }
1656 }
1657 else {
1658 /* Non-universal mode. */
1659 Py_ssize_t readnl_len = PyString_GET_SIZE(readnl);
1660 unsigned char *nl = (unsigned char *) PyString_AS_STRING(readnl);
1661 if (readnl_len == 1) {
1662 Py_UNICODE *pos = find_control_char(start, end, nl[0]);
1663 if (pos != NULL)
1664 return pos - start + 1;
1665 *consumed = len;
1666 return -1;
1667 }
1668 else {
1669 Py_UNICODE *s = start;
1670 Py_UNICODE *e = end - readnl_len + 1;
1671 Py_UNICODE *pos;
1672 if (e < s)
1673 e = s;
1674 while (s < e) {
1675 Py_ssize_t i;
1676 Py_UNICODE *pos = find_control_char(s, end, nl[0]);
1677 if (pos == NULL || pos >= e)
1678 break;
1679 for (i = 1; i < readnl_len; i++) {
1680 if (pos[i] != nl[i])
1681 break;
1682 }
1683 if (i == readnl_len)
1684 return pos - start + readnl_len;
1685 s = pos + 1;
1686 }
1687 pos = find_control_char(e, end, nl[0]);
1688 if (pos == NULL)
1689 *consumed = len;
1690 else
1691 *consumed = pos - start;
1692 return -1;
1693 }
1694 }
1695 }
1696
1697 static PyObject *
_textiowrapper_readline(textio * self,Py_ssize_t limit)1698 _textiowrapper_readline(textio *self, Py_ssize_t limit)
1699 {
1700 PyObject *line = NULL, *chunks = NULL, *remaining = NULL;
1701 Py_ssize_t start, endpos, chunked, offset_to_buffer;
1702 int res;
1703
1704 CHECK_CLOSED(self);
1705
1706 if (_textiowrapper_writeflush(self) < 0)
1707 return NULL;
1708
1709 chunked = 0;
1710
1711 while (1) {
1712 Py_UNICODE *ptr;
1713 Py_ssize_t line_len;
1714 Py_ssize_t consumed = 0;
1715
1716 /* First, get some data if necessary */
1717 res = 1;
1718 while (!self->decoded_chars ||
1719 !PyUnicode_GET_SIZE(self->decoded_chars)) {
1720 res = textiowrapper_read_chunk(self);
1721 if (res < 0) {
1722 /* NOTE: PyErr_SetFromErrno() calls PyErr_CheckSignals()
1723 when EINTR occurs so we needn't do it ourselves. */
1724 if (_PyIO_trap_eintr()) {
1725 continue;
1726 }
1727 goto error;
1728 }
1729 if (res == 0)
1730 break;
1731 }
1732 if (res == 0) {
1733 /* end of file */
1734 textiowrapper_set_decoded_chars(self, NULL);
1735 Py_CLEAR(self->snapshot);
1736 start = endpos = offset_to_buffer = 0;
1737 break;
1738 }
1739
1740 if (remaining == NULL) {
1741 line = self->decoded_chars;
1742 start = self->decoded_chars_used;
1743 offset_to_buffer = 0;
1744 Py_INCREF(line);
1745 }
1746 else {
1747 assert(self->decoded_chars_used == 0);
1748 line = PyUnicode_Concat(remaining, self->decoded_chars);
1749 start = 0;
1750 offset_to_buffer = PyUnicode_GET_SIZE(remaining);
1751 Py_CLEAR(remaining);
1752 if (line == NULL)
1753 goto error;
1754 }
1755
1756 ptr = PyUnicode_AS_UNICODE(line);
1757 line_len = PyUnicode_GET_SIZE(line);
1758
1759 endpos = _PyIO_find_line_ending(
1760 self->readtranslate, self->readuniversal, self->readnl,
1761 ptr + start, ptr + line_len, &consumed);
1762 if (endpos >= 0) {
1763 endpos += start;
1764 if (limit >= 0 && (endpos - start) + chunked >= limit)
1765 endpos = start + limit - chunked;
1766 break;
1767 }
1768
1769 /* We can put aside up to `endpos` */
1770 endpos = consumed + start;
1771 if (limit >= 0 && (endpos - start) + chunked >= limit) {
1772 /* Didn't find line ending, but reached length limit */
1773 endpos = start + limit - chunked;
1774 break;
1775 }
1776
1777 if (endpos > start) {
1778 /* No line ending seen yet - put aside current data */
1779 PyObject *s;
1780 if (chunks == NULL) {
1781 chunks = PyList_New(0);
1782 if (chunks == NULL)
1783 goto error;
1784 }
1785 s = PyUnicode_FromUnicode(ptr + start, endpos - start);
1786 if (s == NULL)
1787 goto error;
1788 if (PyList_Append(chunks, s) < 0) {
1789 Py_DECREF(s);
1790 goto error;
1791 }
1792 chunked += PyUnicode_GET_SIZE(s);
1793 Py_DECREF(s);
1794 }
1795 /* There may be some remaining bytes we'll have to prepend to the
1796 next chunk of data */
1797 if (endpos < line_len) {
1798 remaining = PyUnicode_FromUnicode(
1799 ptr + endpos, line_len - endpos);
1800 if (remaining == NULL)
1801 goto error;
1802 }
1803 Py_CLEAR(line);
1804 /* We have consumed the buffer */
1805 textiowrapper_set_decoded_chars(self, NULL);
1806 }
1807
1808 if (line != NULL) {
1809 /* Our line ends in the current buffer */
1810 self->decoded_chars_used = endpos - offset_to_buffer;
1811 if (start > 0 || endpos < PyUnicode_GET_SIZE(line)) {
1812 if (start == 0 && Py_REFCNT(line) == 1) {
1813 if (PyUnicode_Resize(&line, endpos) < 0)
1814 goto error;
1815 }
1816 else {
1817 PyObject *s = PyUnicode_FromUnicode(
1818 PyUnicode_AS_UNICODE(line) + start, endpos - start);
1819 Py_CLEAR(line);
1820 if (s == NULL)
1821 goto error;
1822 line = s;
1823 }
1824 }
1825 }
1826 if (remaining != NULL) {
1827 if (chunks == NULL) {
1828 chunks = PyList_New(0);
1829 if (chunks == NULL)
1830 goto error;
1831 }
1832 if (PyList_Append(chunks, remaining) < 0)
1833 goto error;
1834 Py_CLEAR(remaining);
1835 }
1836 if (chunks != NULL) {
1837 if (line != NULL && PyList_Append(chunks, line) < 0)
1838 goto error;
1839 Py_XSETREF(line, PyUnicode_Join(_PyIO_empty_str, chunks));
1840 if (line == NULL)
1841 goto error;
1842 Py_DECREF(chunks);
1843 }
1844 if (line == NULL)
1845 line = PyUnicode_FromStringAndSize(NULL, 0);
1846
1847 return line;
1848
1849 error:
1850 Py_XDECREF(chunks);
1851 Py_XDECREF(remaining);
1852 Py_XDECREF(line);
1853 return NULL;
1854 }
1855
1856 static PyObject *
textiowrapper_readline(textio * self,PyObject * args)1857 textiowrapper_readline(textio *self, PyObject *args)
1858 {
1859 PyObject *limitobj = NULL;
1860 Py_ssize_t limit = -1;
1861
1862 CHECK_ATTACHED(self);
1863 if (!PyArg_ParseTuple(args, "|O:readline", &limitobj)) {
1864 return NULL;
1865 }
1866 if (limitobj) {
1867 if (!PyNumber_Check(limitobj)) {
1868 PyErr_Format(PyExc_TypeError,
1869 "integer argument expected, got '%.200s'",
1870 Py_TYPE(limitobj)->tp_name);
1871 return NULL;
1872 }
1873 limit = PyNumber_AsSsize_t(limitobj, PyExc_OverflowError);
1874 if (limit == -1 && PyErr_Occurred())
1875 return NULL;
1876 }
1877 return _textiowrapper_readline(self, limit);
1878 }
1879
1880 /* Seek and Tell */
1881
1882 typedef struct {
1883 Py_off_t start_pos;
1884 int dec_flags;
1885 int bytes_to_feed;
1886 int chars_to_skip;
1887 char need_eof;
1888 } cookie_type;
1889
1890 /*
1891 To speed up cookie packing/unpacking, we store the fields in a temporary
1892 string and call _PyLong_FromByteArray() or _PyLong_AsByteArray (resp.).
1893 The following macros define at which offsets in the intermediary byte
1894 string the various CookieStruct fields will be stored.
1895 */
1896
1897 #define COOKIE_BUF_LEN (sizeof(Py_off_t) + 3 * sizeof(int) + sizeof(char))
1898
1899 #if defined(WORDS_BIGENDIAN)
1900
1901 # define IS_LITTLE_ENDIAN 0
1902
1903 /* We want the least significant byte of start_pos to also be the least
1904 significant byte of the cookie, which means that in big-endian mode we
1905 must copy the fields in reverse order. */
1906
1907 # define OFF_START_POS (sizeof(char) + 3 * sizeof(int))
1908 # define OFF_DEC_FLAGS (sizeof(char) + 2 * sizeof(int))
1909 # define OFF_BYTES_TO_FEED (sizeof(char) + sizeof(int))
1910 # define OFF_CHARS_TO_SKIP (sizeof(char))
1911 # define OFF_NEED_EOF 0
1912
1913 #else
1914
1915 # define IS_LITTLE_ENDIAN 1
1916
1917 /* Little-endian mode: the least significant byte of start_pos will
1918 naturally end up the least significant byte of the cookie. */
1919
1920 # define OFF_START_POS 0
1921 # define OFF_DEC_FLAGS (sizeof(Py_off_t))
1922 # define OFF_BYTES_TO_FEED (sizeof(Py_off_t) + sizeof(int))
1923 # define OFF_CHARS_TO_SKIP (sizeof(Py_off_t) + 2 * sizeof(int))
1924 # define OFF_NEED_EOF (sizeof(Py_off_t) + 3 * sizeof(int))
1925
1926 #endif
1927
1928 static int
textiowrapper_parse_cookie(cookie_type * cookie,PyObject * cookieObj)1929 textiowrapper_parse_cookie(cookie_type *cookie, PyObject *cookieObj)
1930 {
1931 unsigned char buffer[COOKIE_BUF_LEN];
1932 PyLongObject *cookieLong = (PyLongObject *)PyNumber_Long(cookieObj);
1933 if (cookieLong == NULL)
1934 return -1;
1935
1936 if (_PyLong_AsByteArray(cookieLong, buffer, sizeof(buffer),
1937 IS_LITTLE_ENDIAN, 0) < 0) {
1938 Py_DECREF(cookieLong);
1939 return -1;
1940 }
1941 Py_DECREF(cookieLong);
1942
1943 memcpy(&cookie->start_pos, buffer + OFF_START_POS, sizeof(cookie->start_pos));
1944 memcpy(&cookie->dec_flags, buffer + OFF_DEC_FLAGS, sizeof(cookie->dec_flags));
1945 memcpy(&cookie->bytes_to_feed, buffer + OFF_BYTES_TO_FEED, sizeof(cookie->bytes_to_feed));
1946 memcpy(&cookie->chars_to_skip, buffer + OFF_CHARS_TO_SKIP, sizeof(cookie->chars_to_skip));
1947 memcpy(&cookie->need_eof, buffer + OFF_NEED_EOF, sizeof(cookie->need_eof));
1948
1949 return 0;
1950 }
1951
1952 static PyObject *
textiowrapper_build_cookie(cookie_type * cookie)1953 textiowrapper_build_cookie(cookie_type *cookie)
1954 {
1955 unsigned char buffer[COOKIE_BUF_LEN];
1956
1957 memcpy(buffer + OFF_START_POS, &cookie->start_pos, sizeof(cookie->start_pos));
1958 memcpy(buffer + OFF_DEC_FLAGS, &cookie->dec_flags, sizeof(cookie->dec_flags));
1959 memcpy(buffer + OFF_BYTES_TO_FEED, &cookie->bytes_to_feed, sizeof(cookie->bytes_to_feed));
1960 memcpy(buffer + OFF_CHARS_TO_SKIP, &cookie->chars_to_skip, sizeof(cookie->chars_to_skip));
1961 memcpy(buffer + OFF_NEED_EOF, &cookie->need_eof, sizeof(cookie->need_eof));
1962
1963 return _PyLong_FromByteArray(buffer, sizeof(buffer), IS_LITTLE_ENDIAN, 0);
1964 }
1965 #undef IS_LITTLE_ENDIAN
1966
1967 static int
_textiowrapper_decoder_setstate(textio * self,cookie_type * cookie)1968 _textiowrapper_decoder_setstate(textio *self, cookie_type *cookie)
1969 {
1970 PyObject *res;
1971 /* When seeking to the start of the stream, we call decoder.reset()
1972 rather than decoder.getstate().
1973 This is for a few decoders such as utf-16 for which the state value
1974 at start is not (b"", 0) but e.g. (b"", 2) (meaning, in the case of
1975 utf-16, that we are expecting a BOM).
1976 */
1977 if (cookie->start_pos == 0 && cookie->dec_flags == 0)
1978 res = PyObject_CallMethodObjArgs(self->decoder, _PyIO_str_reset, NULL);
1979 else
1980 res = PyObject_CallMethod(self->decoder, "setstate",
1981 "((si))", "", cookie->dec_flags);
1982 if (res == NULL)
1983 return -1;
1984 Py_DECREF(res);
1985 return 0;
1986 }
1987
1988 static int
_textiowrapper_encoder_setstate(textio * self,cookie_type * cookie)1989 _textiowrapper_encoder_setstate(textio *self, cookie_type *cookie)
1990 {
1991 PyObject *res;
1992 /* Same as _textiowrapper_decoder_setstate() above. */
1993 if (cookie->start_pos == 0 && cookie->dec_flags == 0) {
1994 res = PyObject_CallMethodObjArgs(self->encoder, _PyIO_str_reset, NULL);
1995 self->encoding_start_of_stream = 1;
1996 }
1997 else {
1998 res = PyObject_CallMethodObjArgs(self->encoder, _PyIO_str_setstate,
1999 _PyIO_zero, NULL);
2000 self->encoding_start_of_stream = 0;
2001 }
2002 if (res == NULL)
2003 return -1;
2004 Py_DECREF(res);
2005 return 0;
2006 }
2007
2008 static PyObject *
textiowrapper_seek(textio * self,PyObject * args)2009 textiowrapper_seek(textio *self, PyObject *args)
2010 {
2011 PyObject *cookieObj, *posobj;
2012 cookie_type cookie;
2013 int whence = 0;
2014 PyObject *res;
2015 int cmp;
2016
2017 CHECK_ATTACHED(self);
2018
2019 if (!PyArg_ParseTuple(args, "O|i:seek", &cookieObj, &whence))
2020 return NULL;
2021 CHECK_CLOSED(self);
2022
2023 Py_INCREF(cookieObj);
2024
2025 if (!self->seekable) {
2026 PyErr_SetString(PyExc_IOError,
2027 "underlying stream is not seekable");
2028 goto fail;
2029 }
2030
2031 if (whence == 1) {
2032 /* seek relative to current position */
2033 cmp = PyObject_RichCompareBool(cookieObj, _PyIO_zero, Py_EQ);
2034 if (cmp < 0)
2035 goto fail;
2036
2037 if (cmp == 0) {
2038 PyErr_SetString(PyExc_IOError,
2039 "can't do nonzero cur-relative seeks");
2040 goto fail;
2041 }
2042
2043 /* Seeking to the current position should attempt to
2044 * sync the underlying buffer with the current position.
2045 */
2046 Py_DECREF(cookieObj);
2047 cookieObj = PyObject_CallMethod((PyObject *)self, "tell", NULL);
2048 if (cookieObj == NULL)
2049 goto fail;
2050 }
2051 else if (whence == 2) {
2052 /* seek relative to end of file */
2053
2054 cmp = PyObject_RichCompareBool(cookieObj, _PyIO_zero, Py_EQ);
2055 if (cmp < 0)
2056 goto fail;
2057
2058 if (cmp == 0) {
2059 PyErr_SetString(PyExc_IOError,
2060 "can't do nonzero end-relative seeks");
2061 goto fail;
2062 }
2063
2064 res = PyObject_CallMethod((PyObject *)self, "flush", NULL);
2065 if (res == NULL)
2066 goto fail;
2067 Py_DECREF(res);
2068
2069 textiowrapper_set_decoded_chars(self, NULL);
2070 Py_CLEAR(self->snapshot);
2071 if (self->decoder) {
2072 res = PyObject_CallMethod(self->decoder, "reset", NULL);
2073 if (res == NULL)
2074 goto fail;
2075 Py_DECREF(res);
2076 }
2077
2078 res = PyObject_CallMethod(self->buffer, "seek", "ii", 0, 2);
2079 Py_XDECREF(cookieObj);
2080 return res;
2081 }
2082 else if (whence != 0) {
2083 PyErr_Format(PyExc_ValueError,
2084 "invalid whence (%d, should be 0, 1 or 2)", whence);
2085 goto fail;
2086 }
2087
2088 cmp = PyObject_RichCompareBool(cookieObj, _PyIO_zero, Py_LT);
2089 if (cmp < 0)
2090 goto fail;
2091
2092 if (cmp == 1) {
2093 PyObject *repr = PyObject_Repr(cookieObj);
2094 if (repr != NULL) {
2095 PyErr_Format(PyExc_ValueError,
2096 "negative seek position %s",
2097 PyString_AS_STRING(repr));
2098 Py_DECREF(repr);
2099 }
2100 goto fail;
2101 }
2102
2103 res = PyObject_CallMethodObjArgs((PyObject *)self, _PyIO_str_flush, NULL);
2104 if (res == NULL)
2105 goto fail;
2106 Py_DECREF(res);
2107
2108 /* The strategy of seek() is to go back to the safe start point
2109 * and replay the effect of read(chars_to_skip) from there.
2110 */
2111 if (textiowrapper_parse_cookie(&cookie, cookieObj) < 0)
2112 goto fail;
2113
2114 /* Seek back to the safe start point. */
2115 posobj = PyLong_FromOff_t(cookie.start_pos);
2116 if (posobj == NULL)
2117 goto fail;
2118 res = PyObject_CallMethodObjArgs(self->buffer,
2119 _PyIO_str_seek, posobj, NULL);
2120 Py_DECREF(posobj);
2121 if (res == NULL)
2122 goto fail;
2123 Py_DECREF(res);
2124
2125 textiowrapper_set_decoded_chars(self, NULL);
2126 Py_CLEAR(self->snapshot);
2127
2128 /* Restore the decoder to its state from the safe start point. */
2129 if (self->decoder) {
2130 if (_textiowrapper_decoder_setstate(self, &cookie) < 0)
2131 goto fail;
2132 }
2133
2134 if (cookie.chars_to_skip) {
2135 /* Just like _read_chunk, feed the decoder and save a snapshot. */
2136 PyObject *input_chunk = PyObject_CallMethod(
2137 self->buffer, "read", "i", cookie.bytes_to_feed);
2138 PyObject *decoded;
2139
2140 if (input_chunk == NULL)
2141 goto fail;
2142
2143 if (!PyBytes_Check(input_chunk)) {
2144 PyErr_Format(PyExc_TypeError,
2145 "underlying read() should have returned a bytes "
2146 "object, not '%.200s'",
2147 Py_TYPE(input_chunk)->tp_name);
2148 Py_DECREF(input_chunk);
2149 goto fail;
2150 }
2151
2152 self->snapshot = Py_BuildValue("iN", cookie.dec_flags, input_chunk);
2153 if (self->snapshot == NULL) {
2154 Py_DECREF(input_chunk);
2155 goto fail;
2156 }
2157
2158 decoded = PyObject_CallMethod(self->decoder, "decode",
2159 "Oi", input_chunk, (int)cookie.need_eof);
2160
2161 if (check_decoded(decoded) < 0)
2162 goto fail;
2163
2164 textiowrapper_set_decoded_chars(self, decoded);
2165
2166 /* Skip chars_to_skip of the decoded characters. */
2167 if (PyUnicode_GetSize(self->decoded_chars) < cookie.chars_to_skip) {
2168 PyErr_SetString(PyExc_IOError, "can't restore logical file position");
2169 goto fail;
2170 }
2171 self->decoded_chars_used = cookie.chars_to_skip;
2172 }
2173 else {
2174 self->snapshot = Py_BuildValue("is", cookie.dec_flags, "");
2175 if (self->snapshot == NULL)
2176 goto fail;
2177 }
2178
2179 /* Finally, reset the encoder (merely useful for proper BOM handling) */
2180 if (self->encoder) {
2181 if (_textiowrapper_encoder_setstate(self, &cookie) < 0)
2182 goto fail;
2183 }
2184 return cookieObj;
2185 fail:
2186 Py_XDECREF(cookieObj);
2187 return NULL;
2188
2189 }
2190
2191 static PyObject *
textiowrapper_tell(textio * self,PyObject * args)2192 textiowrapper_tell(textio *self, PyObject *args)
2193 {
2194 PyObject *res;
2195 PyObject *posobj = NULL;
2196 cookie_type cookie = {0,0,0,0,0};
2197 PyObject *next_input;
2198 Py_ssize_t chars_to_skip, chars_decoded;
2199 PyObject *saved_state = NULL;
2200 char *input, *input_end;
2201
2202 CHECK_ATTACHED(self);
2203 CHECK_CLOSED(self);
2204
2205 if (!self->seekable) {
2206 PyErr_SetString(PyExc_IOError,
2207 "underlying stream is not seekable");
2208 goto fail;
2209 }
2210 if (!self->telling) {
2211 PyErr_SetString(PyExc_IOError,
2212 "telling position disabled by next() call");
2213 goto fail;
2214 }
2215
2216 if (_textiowrapper_writeflush(self) < 0)
2217 return NULL;
2218 res = PyObject_CallMethod((PyObject *)self, "flush", NULL);
2219 if (res == NULL)
2220 goto fail;
2221 Py_DECREF(res);
2222
2223 posobj = PyObject_CallMethod(self->buffer, "tell", NULL);
2224 if (posobj == NULL)
2225 goto fail;
2226
2227 if (self->decoder == NULL || self->snapshot == NULL) {
2228 assert (self->decoded_chars == NULL || PyUnicode_GetSize(self->decoded_chars) == 0);
2229 return posobj;
2230 }
2231
2232 #if defined(HAVE_LARGEFILE_SUPPORT)
2233 cookie.start_pos = PyLong_AsLongLong(posobj);
2234 #else
2235 cookie.start_pos = PyLong_AsLong(posobj);
2236 #endif
2237 if (PyErr_Occurred())
2238 goto fail;
2239
2240 /* Skip backward to the snapshot point (see _read_chunk). */
2241 if (!PyArg_Parse(self->snapshot, "(iO)", &cookie.dec_flags, &next_input))
2242 goto fail;
2243
2244 assert (PyBytes_Check(next_input));
2245
2246 cookie.start_pos -= PyBytes_GET_SIZE(next_input);
2247
2248 /* How many decoded characters have been used up since the snapshot? */
2249 if (self->decoded_chars_used == 0) {
2250 /* We haven't moved from the snapshot point. */
2251 Py_DECREF(posobj);
2252 return textiowrapper_build_cookie(&cookie);
2253 }
2254
2255 chars_to_skip = self->decoded_chars_used;
2256
2257 /* Starting from the snapshot position, we will walk the decoder
2258 * forward until it gives us enough decoded characters.
2259 */
2260 saved_state = PyObject_CallMethodObjArgs(self->decoder,
2261 _PyIO_str_getstate, NULL);
2262 if (saved_state == NULL)
2263 goto fail;
2264
2265 /* Note our initial start point. */
2266 if (_textiowrapper_decoder_setstate(self, &cookie) < 0)
2267 goto fail;
2268
2269 /* Feed the decoder one byte at a time. As we go, note the
2270 * nearest "safe start point" before the current location
2271 * (a point where the decoder has nothing buffered, so seek()
2272 * can safely start from there and advance to this location).
2273 */
2274 chars_decoded = 0;
2275 input = PyBytes_AS_STRING(next_input);
2276 input_end = input + PyBytes_GET_SIZE(next_input);
2277 while (input < input_end) {
2278 PyObject *state;
2279 char *dec_buffer;
2280 Py_ssize_t dec_buffer_len;
2281 int dec_flags;
2282
2283 PyObject *decoded = PyObject_CallMethod(
2284 self->decoder, "decode", "s#", input, (Py_ssize_t)1);
2285 if (check_decoded(decoded) < 0)
2286 goto fail;
2287 chars_decoded += PyUnicode_GET_SIZE(decoded);
2288 Py_DECREF(decoded);
2289
2290 cookie.bytes_to_feed += 1;
2291
2292 state = PyObject_CallMethodObjArgs(self->decoder,
2293 _PyIO_str_getstate, NULL);
2294 if (state == NULL)
2295 goto fail;
2296 if (!PyArg_Parse(state, "(s#i)", &dec_buffer, &dec_buffer_len, &dec_flags)) {
2297 Py_DECREF(state);
2298 goto fail;
2299 }
2300 Py_DECREF(state);
2301
2302 if (dec_buffer_len == 0 && chars_decoded <= chars_to_skip) {
2303 /* Decoder buffer is empty, so this is a safe start point. */
2304 cookie.start_pos += cookie.bytes_to_feed;
2305 chars_to_skip -= chars_decoded;
2306 cookie.dec_flags = dec_flags;
2307 cookie.bytes_to_feed = 0;
2308 chars_decoded = 0;
2309 }
2310 if (chars_decoded >= chars_to_skip)
2311 break;
2312 input++;
2313 }
2314 if (input == input_end) {
2315 /* We didn't get enough decoded data; signal EOF to get more. */
2316 PyObject *decoded = PyObject_CallMethod(
2317 self->decoder, "decode", "si", "", /* final = */ 1);
2318 if (check_decoded(decoded) < 0)
2319 goto fail;
2320 chars_decoded += PyUnicode_GET_SIZE(decoded);
2321 Py_DECREF(decoded);
2322 cookie.need_eof = 1;
2323
2324 if (chars_decoded < chars_to_skip) {
2325 PyErr_SetString(PyExc_IOError,
2326 "can't reconstruct logical file position");
2327 goto fail;
2328 }
2329 }
2330
2331 /* finally */
2332 Py_XDECREF(posobj);
2333 res = PyObject_CallMethod(self->decoder, "setstate", "(O)", saved_state);
2334 Py_DECREF(saved_state);
2335 if (res == NULL)
2336 return NULL;
2337 Py_DECREF(res);
2338
2339 /* The returned cookie corresponds to the last safe start point. */
2340 cookie.chars_to_skip = Py_SAFE_DOWNCAST(chars_to_skip, Py_ssize_t, int);
2341 return textiowrapper_build_cookie(&cookie);
2342
2343 fail:
2344 Py_XDECREF(posobj);
2345 if (saved_state) {
2346 PyObject *type, *value, *traceback;
2347 PyErr_Fetch(&type, &value, &traceback);
2348
2349 res = PyObject_CallMethod(self->decoder, "setstate", "(O)", saved_state);
2350 _PyErr_ReplaceException(type, value, traceback);
2351 Py_DECREF(saved_state);
2352 Py_XDECREF(res);
2353 }
2354 return NULL;
2355 }
2356
2357 static PyObject *
textiowrapper_truncate(textio * self,PyObject * args)2358 textiowrapper_truncate(textio *self, PyObject *args)
2359 {
2360 PyObject *pos = Py_None;
2361 PyObject *res;
2362
2363 CHECK_ATTACHED(self)
2364 if (!PyArg_ParseTuple(args, "|O:truncate", &pos)) {
2365 return NULL;
2366 }
2367
2368 res = PyObject_CallMethodObjArgs((PyObject *) self, _PyIO_str_flush, NULL);
2369 if (res == NULL)
2370 return NULL;
2371 Py_DECREF(res);
2372
2373 return PyObject_CallMethodObjArgs(self->buffer, _PyIO_str_truncate, pos, NULL);
2374 }
2375
2376 static PyObject *
textiowrapper_repr(textio * self)2377 textiowrapper_repr(textio *self)
2378 {
2379 PyObject *nameobj, *res;
2380 PyObject *namerepr = NULL, *encrepr = NULL;
2381
2382 CHECK_INITIALIZED(self);
2383
2384 nameobj = PyObject_GetAttrString((PyObject *) self, "name");
2385 if (nameobj == NULL) {
2386 if (PyErr_ExceptionMatches(PyExc_Exception))
2387 PyErr_Clear();
2388 else
2389 goto error;
2390 encrepr = PyObject_Repr(self->encoding);
2391 res = PyString_FromFormat("<_io.TextIOWrapper encoding=%s>",
2392 PyString_AS_STRING(encrepr));
2393 }
2394 else {
2395 encrepr = PyObject_Repr(self->encoding);
2396 namerepr = PyObject_Repr(nameobj);
2397 res = PyString_FromFormat("<_io.TextIOWrapper name=%s encoding=%s>",
2398 PyString_AS_STRING(namerepr),
2399 PyString_AS_STRING(encrepr));
2400 Py_DECREF(nameobj);
2401 }
2402 Py_XDECREF(namerepr);
2403 Py_XDECREF(encrepr);
2404 return res;
2405
2406 error:
2407 Py_XDECREF(namerepr);
2408 Py_XDECREF(encrepr);
2409 return NULL;
2410 }
2411
2412
2413 /* Inquiries */
2414
2415 static PyObject *
textiowrapper_fileno(textio * self,PyObject * args)2416 textiowrapper_fileno(textio *self, PyObject *args)
2417 {
2418 CHECK_ATTACHED(self);
2419 return PyObject_CallMethod(self->buffer, "fileno", NULL);
2420 }
2421
2422 static PyObject *
textiowrapper_seekable(textio * self,PyObject * args)2423 textiowrapper_seekable(textio *self, PyObject *args)
2424 {
2425 CHECK_ATTACHED(self);
2426 return PyObject_CallMethod(self->buffer, "seekable", NULL);
2427 }
2428
2429 static PyObject *
textiowrapper_readable(textio * self,PyObject * args)2430 textiowrapper_readable(textio *self, PyObject *args)
2431 {
2432 CHECK_ATTACHED(self);
2433 return PyObject_CallMethod(self->buffer, "readable", NULL);
2434 }
2435
2436 static PyObject *
textiowrapper_writable(textio * self,PyObject * args)2437 textiowrapper_writable(textio *self, PyObject *args)
2438 {
2439 CHECK_ATTACHED(self);
2440 return PyObject_CallMethod(self->buffer, "writable", NULL);
2441 }
2442
2443 static PyObject *
textiowrapper_isatty(textio * self,PyObject * args)2444 textiowrapper_isatty(textio *self, PyObject *args)
2445 {
2446 CHECK_ATTACHED(self);
2447 return PyObject_CallMethod(self->buffer, "isatty", NULL);
2448 }
2449
2450 static PyObject *
textiowrapper_flush(textio * self,PyObject * args)2451 textiowrapper_flush(textio *self, PyObject *args)
2452 {
2453 CHECK_ATTACHED(self);
2454 CHECK_CLOSED(self);
2455 self->telling = self->seekable;
2456 if (_textiowrapper_writeflush(self) < 0)
2457 return NULL;
2458 return PyObject_CallMethod(self->buffer, "flush", NULL);
2459 }
2460
2461 static PyObject *
textiowrapper_close(textio * self,PyObject * args)2462 textiowrapper_close(textio *self, PyObject *args)
2463 {
2464 PyObject *res;
2465 int r;
2466 CHECK_ATTACHED(self);
2467
2468 res = textiowrapper_closed_get(self, NULL);
2469 if (res == NULL)
2470 return NULL;
2471 r = PyObject_IsTrue(res);
2472 Py_DECREF(res);
2473 if (r < 0)
2474 return NULL;
2475
2476 if (r > 0) {
2477 Py_RETURN_NONE; /* stream already closed */
2478 }
2479 else {
2480 PyObject *exc = NULL, *val, *tb;
2481 res = PyObject_CallMethod((PyObject *)self, "flush", NULL);
2482 if (res == NULL)
2483 PyErr_Fetch(&exc, &val, &tb);
2484 else
2485 Py_DECREF(res);
2486
2487 res = PyObject_CallMethod(self->buffer, "close", NULL);
2488 if (exc != NULL) {
2489 _PyErr_ReplaceException(exc, val, tb);
2490 Py_CLEAR(res);
2491 }
2492 return res;
2493 }
2494 }
2495
2496 static PyObject *
textiowrapper_iternext(textio * self)2497 textiowrapper_iternext(textio *self)
2498 {
2499 PyObject *line;
2500
2501 CHECK_ATTACHED(self);
2502
2503 self->telling = 0;
2504 if (Py_TYPE(self) == &PyTextIOWrapper_Type) {
2505 /* Skip method call overhead for speed */
2506 line = _textiowrapper_readline(self, -1);
2507 }
2508 else {
2509 line = PyObject_CallMethodObjArgs((PyObject *)self,
2510 _PyIO_str_readline, NULL);
2511 if (line && !PyUnicode_Check(line)) {
2512 PyErr_Format(PyExc_IOError,
2513 "readline() should have returned an str object, "
2514 "not '%.200s'", Py_TYPE(line)->tp_name);
2515 Py_DECREF(line);
2516 return NULL;
2517 }
2518 }
2519
2520 if (line == NULL)
2521 return NULL;
2522
2523 if (PyUnicode_GET_SIZE(line) == 0) {
2524 /* Reached EOF or would have blocked */
2525 Py_DECREF(line);
2526 Py_CLEAR(self->snapshot);
2527 self->telling = self->seekable;
2528 return NULL;
2529 }
2530
2531 return line;
2532 }
2533
2534 static PyObject *
textiowrapper_name_get(textio * self,void * context)2535 textiowrapper_name_get(textio *self, void *context)
2536 {
2537 CHECK_ATTACHED(self);
2538 return PyObject_GetAttrString(self->buffer, "name");
2539 }
2540
2541 static PyObject *
textiowrapper_closed_get(textio * self,void * context)2542 textiowrapper_closed_get(textio *self, void *context)
2543 {
2544 CHECK_ATTACHED(self);
2545 return PyObject_GetAttr(self->buffer, _PyIO_str_closed);
2546 }
2547
2548 static PyObject *
textiowrapper_newlines_get(textio * self,void * context)2549 textiowrapper_newlines_get(textio *self, void *context)
2550 {
2551 PyObject *res;
2552 CHECK_ATTACHED(self);
2553 if (self->decoder == NULL)
2554 Py_RETURN_NONE;
2555 res = PyObject_GetAttr(self->decoder, _PyIO_str_newlines);
2556 if (res == NULL) {
2557 if (PyErr_ExceptionMatches(PyExc_AttributeError)) {
2558 PyErr_Clear();
2559 Py_RETURN_NONE;
2560 }
2561 else {
2562 return NULL;
2563 }
2564 }
2565 return res;
2566 }
2567
2568 static PyObject *
textiowrapper_errors_get(textio * self,void * context)2569 textiowrapper_errors_get(textio *self, void *context)
2570 {
2571 CHECK_INITIALIZED(self);
2572 Py_INCREF(self->errors);
2573 return self->errors;
2574 }
2575
2576 static PyObject *
textiowrapper_chunk_size_get(textio * self,void * context)2577 textiowrapper_chunk_size_get(textio *self, void *context)
2578 {
2579 CHECK_ATTACHED(self);
2580 return PyLong_FromSsize_t(self->chunk_size);
2581 }
2582
2583 static int
textiowrapper_chunk_size_set(textio * self,PyObject * arg,void * context)2584 textiowrapper_chunk_size_set(textio *self, PyObject *arg, void *context)
2585 {
2586 Py_ssize_t n;
2587 CHECK_ATTACHED_INT(self);
2588 n = PyNumber_AsSsize_t(arg, PyExc_TypeError);
2589 if (n == -1 && PyErr_Occurred())
2590 return -1;
2591 if (n <= 0) {
2592 PyErr_SetString(PyExc_ValueError,
2593 "a strictly positive integer is required");
2594 return -1;
2595 }
2596 self->chunk_size = n;
2597 return 0;
2598 }
2599
2600 static PyMethodDef textiowrapper_methods[] = {
2601 {"detach", (PyCFunction)textiowrapper_detach, METH_NOARGS},
2602 {"write", (PyCFunction)textiowrapper_write, METH_VARARGS},
2603 {"read", (PyCFunction)textiowrapper_read, METH_VARARGS},
2604 {"readline", (PyCFunction)textiowrapper_readline, METH_VARARGS},
2605 {"flush", (PyCFunction)textiowrapper_flush, METH_NOARGS},
2606 {"close", (PyCFunction)textiowrapper_close, METH_NOARGS},
2607
2608 {"fileno", (PyCFunction)textiowrapper_fileno, METH_NOARGS},
2609 {"seekable", (PyCFunction)textiowrapper_seekable, METH_NOARGS},
2610 {"readable", (PyCFunction)textiowrapper_readable, METH_NOARGS},
2611 {"writable", (PyCFunction)textiowrapper_writable, METH_NOARGS},
2612 {"isatty", (PyCFunction)textiowrapper_isatty, METH_NOARGS},
2613
2614 {"seek", (PyCFunction)textiowrapper_seek, METH_VARARGS},
2615 {"tell", (PyCFunction)textiowrapper_tell, METH_NOARGS},
2616 {"truncate", (PyCFunction)textiowrapper_truncate, METH_VARARGS},
2617 {NULL, NULL}
2618 };
2619
2620 static PyMemberDef textiowrapper_members[] = {
2621 {"encoding", T_OBJECT, offsetof(textio, encoding), READONLY},
2622 {"buffer", T_OBJECT, offsetof(textio, buffer), READONLY},
2623 {"line_buffering", T_BOOL, offsetof(textio, line_buffering), READONLY},
2624 {NULL}
2625 };
2626
2627 static PyGetSetDef textiowrapper_getset[] = {
2628 {"name", (getter)textiowrapper_name_get, NULL, NULL},
2629 {"closed", (getter)textiowrapper_closed_get, NULL, NULL},
2630 /* {"mode", (getter)TextIOWrapper_mode_get, NULL, NULL},
2631 */
2632 {"newlines", (getter)textiowrapper_newlines_get, NULL, NULL},
2633 {"errors", (getter)textiowrapper_errors_get, NULL, NULL},
2634 {"_CHUNK_SIZE", (getter)textiowrapper_chunk_size_get,
2635 (setter)textiowrapper_chunk_size_set, NULL},
2636 {NULL}
2637 };
2638
2639 PyTypeObject PyTextIOWrapper_Type = {
2640 PyVarObject_HEAD_INIT(NULL, 0)
2641 "_io.TextIOWrapper", /*tp_name*/
2642 sizeof(textio), /*tp_basicsize*/
2643 0, /*tp_itemsize*/
2644 (destructor)textiowrapper_dealloc, /*tp_dealloc*/
2645 0, /*tp_print*/
2646 0, /*tp_getattr*/
2647 0, /*tps_etattr*/
2648 0, /*tp_compare */
2649 (reprfunc)textiowrapper_repr,/*tp_repr*/
2650 0, /*tp_as_number*/
2651 0, /*tp_as_sequence*/
2652 0, /*tp_as_mapping*/
2653 0, /*tp_hash */
2654 0, /*tp_call*/
2655 0, /*tp_str*/
2656 0, /*tp_getattro*/
2657 0, /*tp_setattro*/
2658 0, /*tp_as_buffer*/
2659 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE
2660 | Py_TPFLAGS_HAVE_GC, /*tp_flags*/
2661 textiowrapper_doc, /* tp_doc */
2662 (traverseproc)textiowrapper_traverse, /* tp_traverse */
2663 (inquiry)textiowrapper_clear, /* tp_clear */
2664 0, /* tp_richcompare */
2665 offsetof(textio, weakreflist), /*tp_weaklistoffset*/
2666 0, /* tp_iter */
2667 (iternextfunc)textiowrapper_iternext, /* tp_iternext */
2668 textiowrapper_methods, /* tp_methods */
2669 textiowrapper_members, /* tp_members */
2670 textiowrapper_getset, /* tp_getset */
2671 0, /* tp_base */
2672 0, /* tp_dict */
2673 0, /* tp_descr_get */
2674 0, /* tp_descr_set */
2675 offsetof(textio, dict), /*tp_dictoffset*/
2676 (initproc)textiowrapper_init, /* tp_init */
2677 0, /* tp_alloc */
2678 PyType_GenericNew, /* tp_new */
2679 };
2680