1  /* struct module -- pack values into and (out of) bytes objects */
2  
3  /* New version supporting byte order, alignment and size options,
4     character strings, and unsigned numbers */
5  
6  #ifndef Py_BUILD_CORE_BUILTIN
7  #  define Py_BUILD_CORE_MODULE 1
8  #endif
9  
10  #define PY_SSIZE_T_CLEAN
11  
12  #include "Python.h"
13  #include "pycore_moduleobject.h"  // _PyModule_GetState()
14  #include "structmember.h"         // PyMemberDef
15  #include <ctype.h>
16  
17  /*[clinic input]
18  class Struct "PyStructObject *" "&PyStructType"
19  [clinic start generated code]*/
20  /*[clinic end generated code: output=da39a3ee5e6b4b0d input=9b032058a83ed7c3]*/
21  
22  typedef struct {
23      PyObject *cache;
24      PyObject *PyStructType;
25      PyObject *unpackiter_type;
26      PyObject *StructError;
27  } _structmodulestate;
28  
29  static inline _structmodulestate*
get_struct_state(PyObject * module)30  get_struct_state(PyObject *module)
31  {
32      void *state = _PyModule_GetState(module);
33      assert(state != NULL);
34      return (_structmodulestate *)state;
35  }
36  
37  static struct PyModuleDef _structmodule;
38  
39  #define get_struct_state_structinst(self) \
40      (get_struct_state(PyType_GetModuleByDef(Py_TYPE(self), &_structmodule)))
41  #define get_struct_state_iterinst(self) \
42      (get_struct_state(PyType_GetModule(Py_TYPE(self))))
43  
44  /* The translation function for each format character is table driven */
45  typedef struct _formatdef {
46      char format;
47      Py_ssize_t size;
48      Py_ssize_t alignment;
49      PyObject* (*unpack)(_structmodulestate *, const char *,
50                          const struct _formatdef *);
51      int (*pack)(_structmodulestate *, char *, PyObject *,
52                  const struct _formatdef *);
53  } formatdef;
54  
55  typedef struct _formatcode {
56      const struct _formatdef *fmtdef;
57      Py_ssize_t offset;
58      Py_ssize_t size;
59      Py_ssize_t repeat;
60  } formatcode;
61  
62  /* Struct object interface */
63  
64  typedef struct {
65      PyObject_HEAD
66      Py_ssize_t s_size;
67      Py_ssize_t s_len;
68      formatcode *s_codes;
69      PyObject *s_format;
70      PyObject *weakreflist; /* List of weak references */
71  } PyStructObject;
72  
73  #define PyStruct_Check(op, state) PyObject_TypeCheck(op, (PyTypeObject *)(state)->PyStructType)
74  
75  /* Define various structs to figure out the alignments of types */
76  
77  
78  typedef struct { char c; short x; } st_short;
79  typedef struct { char c; int x; } st_int;
80  typedef struct { char c; long x; } st_long;
81  typedef struct { char c; float x; } st_float;
82  typedef struct { char c; double x; } st_double;
83  typedef struct { char c; void *x; } st_void_p;
84  typedef struct { char c; size_t x; } st_size_t;
85  typedef struct { char c; _Bool x; } st_bool;
86  
87  #define SHORT_ALIGN (sizeof(st_short) - sizeof(short))
88  #define INT_ALIGN (sizeof(st_int) - sizeof(int))
89  #define LONG_ALIGN (sizeof(st_long) - sizeof(long))
90  #define FLOAT_ALIGN (sizeof(st_float) - sizeof(float))
91  #define DOUBLE_ALIGN (sizeof(st_double) - sizeof(double))
92  #define VOID_P_ALIGN (sizeof(st_void_p) - sizeof(void *))
93  #define SIZE_T_ALIGN (sizeof(st_size_t) - sizeof(size_t))
94  #define BOOL_ALIGN (sizeof(st_bool) - sizeof(_Bool))
95  
96  /* We can't support q and Q in native mode unless the compiler does;
97     in std mode, they're 8 bytes on all platforms. */
98  typedef struct { char c; long long x; } s_long_long;
99  #define LONG_LONG_ALIGN (sizeof(s_long_long) - sizeof(long long))
100  
101  #ifdef __powerc
102  #pragma options align=reset
103  #endif
104  
105  /*[python input]
106  class cache_struct_converter(CConverter):
107      type = 'PyStructObject *'
108      converter = 'cache_struct_converter'
109      c_default = "NULL"
110  
111      def parse_arg(self, argname, displayname):
112          return """
113              if (!{converter}(module, {argname}, &{paramname})) {{{{
114                  goto exit;
115              }}}}
116              """.format(argname=argname, paramname=self.name,
117                         converter=self.converter)
118  
119      def cleanup(self):
120          return "Py_XDECREF(%s);\n" % self.name
121  [python start generated code]*/
122  /*[python end generated code: output=da39a3ee5e6b4b0d input=d6746621c2fb1a7d]*/
123  
124  static int cache_struct_converter(PyObject *, PyObject *, PyStructObject **);
125  
126  #include "clinic/_struct.c.h"
127  
128  /* Helper for integer format codes: converts an arbitrary Python object to a
129     PyLongObject if possible, otherwise fails.  Caller should decref. */
130  
131  static PyObject *
get_pylong(_structmodulestate * state,PyObject * v)132  get_pylong(_structmodulestate *state, PyObject *v)
133  {
134      assert(v != NULL);
135      if (!PyLong_Check(v)) {
136          /* Not an integer;  try to use __index__ to convert. */
137          if (PyIndex_Check(v)) {
138              v = _PyNumber_Index(v);
139              if (v == NULL)
140                  return NULL;
141          }
142          else {
143              PyErr_SetString(state->StructError,
144                              "required argument is not an integer");
145              return NULL;
146          }
147      }
148      else
149          Py_INCREF(v);
150  
151      assert(PyLong_Check(v));
152      return v;
153  }
154  
155  /* Helper routine to get a C long and raise the appropriate error if it isn't
156     one */
157  
158  static int
get_long(_structmodulestate * state,PyObject * v,long * p)159  get_long(_structmodulestate *state, PyObject *v, long *p)
160  {
161      long x;
162  
163      v = get_pylong(state, v);
164      if (v == NULL)
165          return -1;
166      assert(PyLong_Check(v));
167      x = PyLong_AsLong(v);
168      Py_DECREF(v);
169      if (x == (long)-1 && PyErr_Occurred()) {
170          if (PyErr_ExceptionMatches(PyExc_OverflowError))
171              PyErr_SetString(state->StructError,
172                              "argument out of range");
173          return -1;
174      }
175      *p = x;
176      return 0;
177  }
178  
179  
180  /* Same, but handling unsigned long */
181  
182  static int
get_ulong(_structmodulestate * state,PyObject * v,unsigned long * p)183  get_ulong(_structmodulestate *state, PyObject *v, unsigned long *p)
184  {
185      unsigned long x;
186  
187      v = get_pylong(state, v);
188      if (v == NULL)
189          return -1;
190      assert(PyLong_Check(v));
191      x = PyLong_AsUnsignedLong(v);
192      Py_DECREF(v);
193      if (x == (unsigned long)-1 && PyErr_Occurred()) {
194          if (PyErr_ExceptionMatches(PyExc_OverflowError))
195              PyErr_SetString(state->StructError,
196                              "argument out of range");
197          return -1;
198      }
199      *p = x;
200      return 0;
201  }
202  
203  /* Same, but handling native long long. */
204  
205  static int
get_longlong(_structmodulestate * state,PyObject * v,long long * p)206  get_longlong(_structmodulestate *state, PyObject *v, long long *p)
207  {
208      long long x;
209  
210      v = get_pylong(state, v);
211      if (v == NULL)
212          return -1;
213      assert(PyLong_Check(v));
214      x = PyLong_AsLongLong(v);
215      Py_DECREF(v);
216      if (x == (long long)-1 && PyErr_Occurred()) {
217          if (PyErr_ExceptionMatches(PyExc_OverflowError))
218              PyErr_SetString(state->StructError,
219                              "argument out of range");
220          return -1;
221      }
222      *p = x;
223      return 0;
224  }
225  
226  /* Same, but handling native unsigned long long. */
227  
228  static int
get_ulonglong(_structmodulestate * state,PyObject * v,unsigned long long * p)229  get_ulonglong(_structmodulestate *state, PyObject *v, unsigned long long *p)
230  {
231      unsigned long long x;
232  
233      v = get_pylong(state, v);
234      if (v == NULL)
235          return -1;
236      assert(PyLong_Check(v));
237      x = PyLong_AsUnsignedLongLong(v);
238      Py_DECREF(v);
239      if (x == (unsigned long long)-1 && PyErr_Occurred()) {
240          if (PyErr_ExceptionMatches(PyExc_OverflowError))
241              PyErr_SetString(state->StructError,
242                              "argument out of range");
243          return -1;
244      }
245      *p = x;
246      return 0;
247  }
248  
249  /* Same, but handling Py_ssize_t */
250  
251  static int
get_ssize_t(_structmodulestate * state,PyObject * v,Py_ssize_t * p)252  get_ssize_t(_structmodulestate *state, PyObject *v, Py_ssize_t *p)
253  {
254      Py_ssize_t x;
255  
256      v = get_pylong(state, v);
257      if (v == NULL)
258          return -1;
259      assert(PyLong_Check(v));
260      x = PyLong_AsSsize_t(v);
261      Py_DECREF(v);
262      if (x == (Py_ssize_t)-1 && PyErr_Occurred()) {
263          if (PyErr_ExceptionMatches(PyExc_OverflowError))
264              PyErr_SetString(state->StructError,
265                              "argument out of range");
266          return -1;
267      }
268      *p = x;
269      return 0;
270  }
271  
272  /* Same, but handling size_t */
273  
274  static int
get_size_t(_structmodulestate * state,PyObject * v,size_t * p)275  get_size_t(_structmodulestate *state, PyObject *v, size_t *p)
276  {
277      size_t x;
278  
279      v = get_pylong(state, v);
280      if (v == NULL)
281          return -1;
282      assert(PyLong_Check(v));
283      x = PyLong_AsSize_t(v);
284      Py_DECREF(v);
285      if (x == (size_t)-1 && PyErr_Occurred()) {
286          if (PyErr_ExceptionMatches(PyExc_OverflowError))
287              PyErr_SetString(state->StructError,
288                              "argument out of range");
289          return -1;
290      }
291      *p = x;
292      return 0;
293  }
294  
295  
296  #define RANGE_ERROR(state, x, f, flag, mask) return _range_error(state, f, flag)
297  
298  
299  /* Floating point helpers */
300  
301  static PyObject *
unpack_halffloat(const char * p,int le)302  unpack_halffloat(const char *p,  /* start of 2-byte string */
303                   int le)         /* true for little-endian, false for big-endian */
304  {
305      double x = PyFloat_Unpack2(p, le);
306      if (x == -1.0 && PyErr_Occurred()) {
307          return NULL;
308      }
309      return PyFloat_FromDouble(x);
310  }
311  
312  static int
pack_halffloat(_structmodulestate * state,char * p,PyObject * v,int le)313  pack_halffloat(_structmodulestate *state,
314                 char *p,      /* start of 2-byte string */
315                 PyObject *v,  /* value to pack */
316                 int le)       /* true for little-endian, false for big-endian */
317  {
318      double x = PyFloat_AsDouble(v);
319      if (x == -1.0 && PyErr_Occurred()) {
320          PyErr_SetString(state->StructError,
321                          "required argument is not a float");
322          return -1;
323      }
324      return PyFloat_Pack2(x, p, le);
325  }
326  
327  static PyObject *
unpack_float(const char * p,int le)328  unpack_float(const char *p,  /* start of 4-byte string */
329           int le)             /* true for little-endian, false for big-endian */
330  {
331      double x;
332  
333      x = PyFloat_Unpack4(p, le);
334      if (x == -1.0 && PyErr_Occurred())
335          return NULL;
336      return PyFloat_FromDouble(x);
337  }
338  
339  static PyObject *
unpack_double(const char * p,int le)340  unpack_double(const char *p,  /* start of 8-byte string */
341            int le)         /* true for little-endian, false for big-endian */
342  {
343      double x;
344  
345      x = PyFloat_Unpack8(p, le);
346      if (x == -1.0 && PyErr_Occurred())
347          return NULL;
348      return PyFloat_FromDouble(x);
349  }
350  
351  /* Helper to format the range error exceptions */
352  static int
_range_error(_structmodulestate * state,const formatdef * f,int is_unsigned)353  _range_error(_structmodulestate *state, const formatdef *f, int is_unsigned)
354  {
355      /* ulargest is the largest unsigned value with f->size bytes.
356       * Note that the simpler:
357       *     ((size_t)1 << (f->size * 8)) - 1
358       * doesn't work when f->size == sizeof(size_t) because C doesn't
359       * define what happens when a left shift count is >= the number of
360       * bits in the integer being shifted; e.g., on some boxes it doesn't
361       * shift at all when they're equal.
362       */
363      const size_t ulargest = (size_t)-1 >> ((SIZEOF_SIZE_T - f->size)*8);
364      assert(f->size >= 1 && f->size <= SIZEOF_SIZE_T);
365      if (is_unsigned)
366          PyErr_Format(state->StructError,
367              "'%c' format requires 0 <= number <= %zu",
368              f->format,
369              ulargest);
370      else {
371          const Py_ssize_t largest = (Py_ssize_t)(ulargest >> 1);
372          PyErr_Format(state->StructError,
373              "'%c' format requires %zd <= number <= %zd",
374              f->format,
375              ~ largest,
376              largest);
377      }
378  
379      return -1;
380  }
381  
382  
383  
384  /* A large number of small routines follow, with names of the form
385  
386     [bln][up]_TYPE
387  
388     [bln] distinguishes among big-endian, little-endian and native.
389     [pu] distinguishes between pack (to struct) and unpack (from struct).
390     TYPE is one of char, byte, ubyte, etc.
391  */
392  
393  /* Native mode routines. ****************************************************/
394  /* NOTE:
395     In all n[up]_<type> routines handling types larger than 1 byte, there is
396     *no* guarantee that the p pointer is properly aligned for each type,
397     therefore memcpy is called.  An intermediate variable is used to
398     compensate for big-endian architectures.
399     Normally both the intermediate variable and the memcpy call will be
400     skipped by C optimisation in little-endian architectures (gcc >= 2.91
401     does this). */
402  
403  static PyObject *
nu_char(_structmodulestate * state,const char * p,const formatdef * f)404  nu_char(_structmodulestate *state, const char *p, const formatdef *f)
405  {
406      return PyBytes_FromStringAndSize(p, 1);
407  }
408  
409  static PyObject *
nu_byte(_structmodulestate * state,const char * p,const formatdef * f)410  nu_byte(_structmodulestate *state, const char *p, const formatdef *f)
411  {
412      return PyLong_FromLong((long) *(signed char *)p);
413  }
414  
415  static PyObject *
nu_ubyte(_structmodulestate * state,const char * p,const formatdef * f)416  nu_ubyte(_structmodulestate *state, const char *p, const formatdef *f)
417  {
418      return PyLong_FromLong((long) *(unsigned char *)p);
419  }
420  
421  static PyObject *
nu_short(_structmodulestate * state,const char * p,const formatdef * f)422  nu_short(_structmodulestate *state, const char *p, const formatdef *f)
423  {
424      short x;
425      memcpy((char *)&x, p, sizeof x);
426      return PyLong_FromLong((long)x);
427  }
428  
429  static PyObject *
nu_ushort(_structmodulestate * state,const char * p,const formatdef * f)430  nu_ushort(_structmodulestate *state, const char *p, const formatdef *f)
431  {
432      unsigned short x;
433      memcpy((char *)&x, p, sizeof x);
434      return PyLong_FromLong((long)x);
435  }
436  
437  static PyObject *
nu_int(_structmodulestate * state,const char * p,const formatdef * f)438  nu_int(_structmodulestate *state, const char *p, const formatdef *f)
439  {
440      int x;
441      memcpy((char *)&x, p, sizeof x);
442      return PyLong_FromLong((long)x);
443  }
444  
445  static PyObject *
nu_uint(_structmodulestate * state,const char * p,const formatdef * f)446  nu_uint(_structmodulestate *state, const char *p, const formatdef *f)
447  {
448      unsigned int x;
449      memcpy((char *)&x, p, sizeof x);
450      return PyLong_FromUnsignedLong((unsigned long)x);
451  }
452  
453  static PyObject *
nu_long(_structmodulestate * state,const char * p,const formatdef * f)454  nu_long(_structmodulestate *state, const char *p, const formatdef *f)
455  {
456      long x;
457      memcpy((char *)&x, p, sizeof x);
458      return PyLong_FromLong(x);
459  }
460  
461  static PyObject *
nu_ulong(_structmodulestate * state,const char * p,const formatdef * f)462  nu_ulong(_structmodulestate *state, const char *p, const formatdef *f)
463  {
464      unsigned long x;
465      memcpy((char *)&x, p, sizeof x);
466      return PyLong_FromUnsignedLong(x);
467  }
468  
469  static PyObject *
nu_ssize_t(_structmodulestate * state,const char * p,const formatdef * f)470  nu_ssize_t(_structmodulestate *state, const char *p, const formatdef *f)
471  {
472      Py_ssize_t x;
473      memcpy((char *)&x, p, sizeof x);
474      return PyLong_FromSsize_t(x);
475  }
476  
477  static PyObject *
nu_size_t(_structmodulestate * state,const char * p,const formatdef * f)478  nu_size_t(_structmodulestate *state, const char *p, const formatdef *f)
479  {
480      size_t x;
481      memcpy((char *)&x, p, sizeof x);
482      return PyLong_FromSize_t(x);
483  }
484  
485  static PyObject *
nu_longlong(_structmodulestate * state,const char * p,const formatdef * f)486  nu_longlong(_structmodulestate *state, const char *p, const formatdef *f)
487  {
488      long long x;
489      memcpy((char *)&x, p, sizeof x);
490      return PyLong_FromLongLong(x);
491  }
492  
493  static PyObject *
nu_ulonglong(_structmodulestate * state,const char * p,const formatdef * f)494  nu_ulonglong(_structmodulestate *state, const char *p, const formatdef *f)
495  {
496      unsigned long long x;
497      memcpy((char *)&x, p, sizeof x);
498      return PyLong_FromUnsignedLongLong(x);
499  }
500  
501  static PyObject *
nu_bool(_structmodulestate * state,const char * p,const formatdef * f)502  nu_bool(_structmodulestate *state, const char *p, const formatdef *f)
503  {
504      _Bool x;
505      memcpy((char *)&x, p, sizeof x);
506      return PyBool_FromLong(x != 0);
507  }
508  
509  
510  static PyObject *
nu_halffloat(_structmodulestate * state,const char * p,const formatdef * f)511  nu_halffloat(_structmodulestate *state, const char *p, const formatdef *f)
512  {
513  #if PY_LITTLE_ENDIAN
514      return unpack_halffloat(p, 1);
515  #else
516      return unpack_halffloat(p, 0);
517  #endif
518  }
519  
520  static PyObject *
nu_float(_structmodulestate * state,const char * p,const formatdef * f)521  nu_float(_structmodulestate *state, const char *p, const formatdef *f)
522  {
523      float x;
524      memcpy((char *)&x, p, sizeof x);
525      return PyFloat_FromDouble((double)x);
526  }
527  
528  static PyObject *
nu_double(_structmodulestate * state,const char * p,const formatdef * f)529  nu_double(_structmodulestate *state, const char *p, const formatdef *f)
530  {
531      double x;
532      memcpy((char *)&x, p, sizeof x);
533      return PyFloat_FromDouble(x);
534  }
535  
536  static PyObject *
nu_void_p(_structmodulestate * state,const char * p,const formatdef * f)537  nu_void_p(_structmodulestate *state, const char *p, const formatdef *f)
538  {
539      void *x;
540      memcpy((char *)&x, p, sizeof x);
541      return PyLong_FromVoidPtr(x);
542  }
543  
544  static int
np_byte(_structmodulestate * state,char * p,PyObject * v,const formatdef * f)545  np_byte(_structmodulestate *state, char *p, PyObject *v, const formatdef *f)
546  {
547      long x;
548      if (get_long(state, v, &x) < 0)
549          return -1;
550      if (x < -128 || x > 127) {
551          PyErr_SetString(state->StructError,
552                          "byte format requires -128 <= number <= 127");
553          return -1;
554      }
555      *p = (char)x;
556      return 0;
557  }
558  
559  static int
np_ubyte(_structmodulestate * state,char * p,PyObject * v,const formatdef * f)560  np_ubyte(_structmodulestate *state, char *p, PyObject *v, const formatdef *f)
561  {
562      long x;
563      if (get_long(state, v, &x) < 0)
564          return -1;
565      if (x < 0 || x > 255) {
566          PyErr_SetString(state->StructError,
567                          "ubyte format requires 0 <= number <= 255");
568          return -1;
569      }
570      *(unsigned char *)p = (unsigned char)x;
571      return 0;
572  }
573  
574  static int
np_char(_structmodulestate * state,char * p,PyObject * v,const formatdef * f)575  np_char(_structmodulestate *state, char *p, PyObject *v, const formatdef *f)
576  {
577      if (!PyBytes_Check(v) || PyBytes_Size(v) != 1) {
578          PyErr_SetString(state->StructError,
579                          "char format requires a bytes object of length 1");
580          return -1;
581      }
582      *p = *PyBytes_AS_STRING(v);
583      return 0;
584  }
585  
586  static int
np_short(_structmodulestate * state,char * p,PyObject * v,const formatdef * f)587  np_short(_structmodulestate *state, char *p, PyObject *v, const formatdef *f)
588  {
589      long x;
590      short y;
591      if (get_long(state, v, &x) < 0)
592          return -1;
593      if (x < SHRT_MIN || x > SHRT_MAX) {
594          PyErr_Format(state->StructError,
595                       "short format requires %d <= number <= %d",
596                       (int)SHRT_MIN, (int)SHRT_MAX);
597          return -1;
598      }
599      y = (short)x;
600      memcpy(p, (char *)&y, sizeof y);
601      return 0;
602  }
603  
604  static int
np_ushort(_structmodulestate * state,char * p,PyObject * v,const formatdef * f)605  np_ushort(_structmodulestate *state, char *p, PyObject *v, const formatdef *f)
606  {
607      long x;
608      unsigned short y;
609      if (get_long(state, v, &x) < 0)
610          return -1;
611      if (x < 0 || x > USHRT_MAX) {
612          PyErr_Format(state->StructError,
613                       "ushort format requires 0 <= number <= %u",
614                       (unsigned int)USHRT_MAX);
615          return -1;
616      }
617      y = (unsigned short)x;
618      memcpy(p, (char *)&y, sizeof y);
619      return 0;
620  }
621  
622  static int
np_int(_structmodulestate * state,char * p,PyObject * v,const formatdef * f)623  np_int(_structmodulestate *state, char *p, PyObject *v, const formatdef *f)
624  {
625      long x;
626      int y;
627      if (get_long(state, v, &x) < 0)
628          return -1;
629  #if (SIZEOF_LONG > SIZEOF_INT)
630      if ((x < ((long)INT_MIN)) || (x > ((long)INT_MAX)))
631          RANGE_ERROR(state, x, f, 0, -1);
632  #endif
633      y = (int)x;
634      memcpy(p, (char *)&y, sizeof y);
635      return 0;
636  }
637  
638  static int
np_uint(_structmodulestate * state,char * p,PyObject * v,const formatdef * f)639  np_uint(_structmodulestate *state, char *p, PyObject *v, const formatdef *f)
640  {
641      unsigned long x;
642      unsigned int y;
643      if (get_ulong(state, v, &x) < 0)
644          return -1;
645      y = (unsigned int)x;
646  #if (SIZEOF_LONG > SIZEOF_INT)
647      if (x > ((unsigned long)UINT_MAX))
648          RANGE_ERROR(state, y, f, 1, -1);
649  #endif
650      memcpy(p, (char *)&y, sizeof y);
651      return 0;
652  }
653  
654  static int
np_long(_structmodulestate * state,char * p,PyObject * v,const formatdef * f)655  np_long(_structmodulestate *state, char *p, PyObject *v, const formatdef *f)
656  {
657      long x;
658      if (get_long(state, v, &x) < 0)
659          return -1;
660      memcpy(p, (char *)&x, sizeof x);
661      return 0;
662  }
663  
664  static int
np_ulong(_structmodulestate * state,char * p,PyObject * v,const formatdef * f)665  np_ulong(_structmodulestate *state, char *p, PyObject *v, const formatdef *f)
666  {
667      unsigned long x;
668      if (get_ulong(state, v, &x) < 0)
669          return -1;
670      memcpy(p, (char *)&x, sizeof x);
671      return 0;
672  }
673  
674  static int
np_ssize_t(_structmodulestate * state,char * p,PyObject * v,const formatdef * f)675  np_ssize_t(_structmodulestate *state, char *p, PyObject *v, const formatdef *f)
676  {
677      Py_ssize_t x;
678      if (get_ssize_t(state, v, &x) < 0)
679          return -1;
680      memcpy(p, (char *)&x, sizeof x);
681      return 0;
682  }
683  
684  static int
np_size_t(_structmodulestate * state,char * p,PyObject * v,const formatdef * f)685  np_size_t(_structmodulestate *state, char *p, PyObject *v, const formatdef *f)
686  {
687      size_t x;
688      if (get_size_t(state, v, &x) < 0)
689          return -1;
690      memcpy(p, (char *)&x, sizeof x);
691      return 0;
692  }
693  
694  static int
np_longlong(_structmodulestate * state,char * p,PyObject * v,const formatdef * f)695  np_longlong(_structmodulestate *state, char *p, PyObject *v, const formatdef *f)
696  {
697      long long x;
698      if (get_longlong(state, v, &x) < 0)
699          return -1;
700      memcpy(p, (char *)&x, sizeof x);
701      return 0;
702  }
703  
704  static int
np_ulonglong(_structmodulestate * state,char * p,PyObject * v,const formatdef * f)705  np_ulonglong(_structmodulestate *state, char *p, PyObject *v, const formatdef *f)
706  {
707      unsigned long long x;
708      if (get_ulonglong(state, v, &x) < 0)
709          return -1;
710      memcpy(p, (char *)&x, sizeof x);
711      return 0;
712  }
713  
714  
715  static int
np_bool(_structmodulestate * state,char * p,PyObject * v,const formatdef * f)716  np_bool(_structmodulestate *state, char *p, PyObject *v, const formatdef *f)
717  {
718      int y;
719      _Bool x;
720      y = PyObject_IsTrue(v);
721      if (y < 0)
722          return -1;
723      x = y;
724      memcpy(p, (char *)&x, sizeof x);
725      return 0;
726  }
727  
728  static int
np_halffloat(_structmodulestate * state,char * p,PyObject * v,const formatdef * f)729  np_halffloat(_structmodulestate *state, char *p, PyObject *v, const formatdef *f)
730  {
731  #if PY_LITTLE_ENDIAN
732      return pack_halffloat(state, p, v, 1);
733  #else
734      return pack_halffloat(state, p, v, 0);
735  #endif
736  }
737  
738  static int
np_float(_structmodulestate * state,char * p,PyObject * v,const formatdef * f)739  np_float(_structmodulestate *state, char *p, PyObject *v, const formatdef *f)
740  {
741      float x = (float)PyFloat_AsDouble(v);
742      if (x == -1 && PyErr_Occurred()) {
743          PyErr_SetString(state->StructError,
744                          "required argument is not a float");
745          return -1;
746      }
747      memcpy(p, (char *)&x, sizeof x);
748      return 0;
749  }
750  
751  static int
np_double(_structmodulestate * state,char * p,PyObject * v,const formatdef * f)752  np_double(_structmodulestate *state, char *p, PyObject *v, const formatdef *f)
753  {
754      double x = PyFloat_AsDouble(v);
755      if (x == -1 && PyErr_Occurred()) {
756          PyErr_SetString(state->StructError,
757                          "required argument is not a float");
758          return -1;
759      }
760      memcpy(p, (char *)&x, sizeof(double));
761      return 0;
762  }
763  
764  static int
np_void_p(_structmodulestate * state,char * p,PyObject * v,const formatdef * f)765  np_void_p(_structmodulestate *state, char *p, PyObject *v, const formatdef *f)
766  {
767      void *x;
768  
769      v = get_pylong(state, v);
770      if (v == NULL)
771          return -1;
772      assert(PyLong_Check(v));
773      x = PyLong_AsVoidPtr(v);
774      Py_DECREF(v);
775      if (x == NULL && PyErr_Occurred())
776          return -1;
777      memcpy(p, (char *)&x, sizeof x);
778      return 0;
779  }
780  
781  static const formatdef native_table[] = {
782      {'x',       sizeof(char),   0,              NULL},
783      {'b',       sizeof(char),   0,              nu_byte,        np_byte},
784      {'B',       sizeof(char),   0,              nu_ubyte,       np_ubyte},
785      {'c',       sizeof(char),   0,              nu_char,        np_char},
786      {'s',       sizeof(char),   0,              NULL},
787      {'p',       sizeof(char),   0,              NULL},
788      {'h',       sizeof(short),  SHORT_ALIGN,    nu_short,       np_short},
789      {'H',       sizeof(short),  SHORT_ALIGN,    nu_ushort,      np_ushort},
790      {'i',       sizeof(int),    INT_ALIGN,      nu_int,         np_int},
791      {'I',       sizeof(int),    INT_ALIGN,      nu_uint,        np_uint},
792      {'l',       sizeof(long),   LONG_ALIGN,     nu_long,        np_long},
793      {'L',       sizeof(long),   LONG_ALIGN,     nu_ulong,       np_ulong},
794      {'n',       sizeof(size_t), SIZE_T_ALIGN,   nu_ssize_t,     np_ssize_t},
795      {'N',       sizeof(size_t), SIZE_T_ALIGN,   nu_size_t,      np_size_t},
796      {'q',       sizeof(long long), LONG_LONG_ALIGN, nu_longlong, np_longlong},
797      {'Q',       sizeof(long long), LONG_LONG_ALIGN, nu_ulonglong,np_ulonglong},
798      {'?',       sizeof(_Bool),      BOOL_ALIGN,     nu_bool,        np_bool},
799      {'e',       sizeof(short),  SHORT_ALIGN,    nu_halffloat,   np_halffloat},
800      {'f',       sizeof(float),  FLOAT_ALIGN,    nu_float,       np_float},
801      {'d',       sizeof(double), DOUBLE_ALIGN,   nu_double,      np_double},
802      {'P',       sizeof(void *), VOID_P_ALIGN,   nu_void_p,      np_void_p},
803      {0}
804  };
805  
806  /* Big-endian routines. *****************************************************/
807  
808  static PyObject *
bu_int(_structmodulestate * state,const char * p,const formatdef * f)809  bu_int(_structmodulestate *state, const char *p, const formatdef *f)
810  {
811      long x = 0;
812      Py_ssize_t i = f->size;
813      const unsigned char *bytes = (const unsigned char *)p;
814      do {
815          x = (x<<8) | *bytes++;
816      } while (--i > 0);
817      /* Extend the sign bit. */
818      if (SIZEOF_LONG > f->size)
819          x |= -(x & (1L << ((8 * f->size) - 1)));
820      return PyLong_FromLong(x);
821  }
822  
823  static PyObject *
bu_uint(_structmodulestate * state,const char * p,const formatdef * f)824  bu_uint(_structmodulestate *state, const char *p, const formatdef *f)
825  {
826      unsigned long x = 0;
827      Py_ssize_t i = f->size;
828      const unsigned char *bytes = (const unsigned char *)p;
829      do {
830          x = (x<<8) | *bytes++;
831      } while (--i > 0);
832      return PyLong_FromUnsignedLong(x);
833  }
834  
835  static PyObject *
bu_longlong(_structmodulestate * state,const char * p,const formatdef * f)836  bu_longlong(_structmodulestate *state, const char *p, const formatdef *f)
837  {
838      long long x = 0;
839      Py_ssize_t i = f->size;
840      const unsigned char *bytes = (const unsigned char *)p;
841      do {
842          x = (x<<8) | *bytes++;
843      } while (--i > 0);
844      /* Extend the sign bit. */
845      if (SIZEOF_LONG_LONG > f->size)
846          x |= -(x & ((long long)1 << ((8 * f->size) - 1)));
847      return PyLong_FromLongLong(x);
848  }
849  
850  static PyObject *
bu_ulonglong(_structmodulestate * state,const char * p,const formatdef * f)851  bu_ulonglong(_structmodulestate *state, const char *p, const formatdef *f)
852  {
853      unsigned long long x = 0;
854      Py_ssize_t i = f->size;
855      const unsigned char *bytes = (const unsigned char *)p;
856      do {
857          x = (x<<8) | *bytes++;
858      } while (--i > 0);
859      return PyLong_FromUnsignedLongLong(x);
860  }
861  
862  static PyObject *
bu_halffloat(_structmodulestate * state,const char * p,const formatdef * f)863  bu_halffloat(_structmodulestate *state, const char *p, const formatdef *f)
864  {
865      return unpack_halffloat(p, 0);
866  }
867  
868  static PyObject *
bu_float(_structmodulestate * state,const char * p,const formatdef * f)869  bu_float(_structmodulestate *state, const char *p, const formatdef *f)
870  {
871      return unpack_float(p, 0);
872  }
873  
874  static PyObject *
bu_double(_structmodulestate * state,const char * p,const formatdef * f)875  bu_double(_structmodulestate *state, const char *p, const formatdef *f)
876  {
877      return unpack_double(p, 0);
878  }
879  
880  static PyObject *
bu_bool(_structmodulestate * state,const char * p,const formatdef * f)881  bu_bool(_structmodulestate *state, const char *p, const formatdef *f)
882  {
883      return PyBool_FromLong(*p != 0);
884  }
885  
886  static int
bp_int(_structmodulestate * state,char * p,PyObject * v,const formatdef * f)887  bp_int(_structmodulestate *state, char *p, PyObject *v, const formatdef *f)
888  {
889      long x;
890      Py_ssize_t i;
891      unsigned char *q = (unsigned char *)p;
892      if (get_long(state, v, &x) < 0)
893          return -1;
894      i = f->size;
895      if (i != SIZEOF_LONG) {
896          if ((i == 2) && (x < -32768 || x > 32767))
897              RANGE_ERROR(state, x, f, 0, 0xffffL);
898  #if (SIZEOF_LONG != 4)
899          else if ((i == 4) && (x < -2147483648L || x > 2147483647L))
900              RANGE_ERROR(state, x, f, 0, 0xffffffffL);
901  #endif
902      }
903      do {
904          q[--i] = (unsigned char)(x & 0xffL);
905          x >>= 8;
906      } while (i > 0);
907      return 0;
908  }
909  
910  static int
bp_uint(_structmodulestate * state,char * p,PyObject * v,const formatdef * f)911  bp_uint(_structmodulestate *state, char *p, PyObject *v, const formatdef *f)
912  {
913      unsigned long x;
914      Py_ssize_t i;
915      unsigned char *q = (unsigned char *)p;
916      if (get_ulong(state, v, &x) < 0)
917          return -1;
918      i = f->size;
919      if (i != SIZEOF_LONG) {
920          unsigned long maxint = 1;
921          maxint <<= (unsigned long)(i * 8);
922          if (x >= maxint)
923              RANGE_ERROR(state, x, f, 1, maxint - 1);
924      }
925      do {
926          q[--i] = (unsigned char)(x & 0xffUL);
927          x >>= 8;
928      } while (i > 0);
929      return 0;
930  }
931  
932  static int
bp_longlong(_structmodulestate * state,char * p,PyObject * v,const formatdef * f)933  bp_longlong(_structmodulestate *state, char *p, PyObject *v, const formatdef *f)
934  {
935      int res;
936      v = get_pylong(state, v);
937      if (v == NULL)
938          return -1;
939      res = _PyLong_AsByteArray((PyLongObject *)v,
940                                (unsigned char *)p,
941                                8,
942                                0, /* little_endian */
943                                1  /* signed */);
944      Py_DECREF(v);
945      return res;
946  }
947  
948  static int
bp_ulonglong(_structmodulestate * state,char * p,PyObject * v,const formatdef * f)949  bp_ulonglong(_structmodulestate *state, char *p, PyObject *v, const formatdef *f)
950  {
951      int res;
952      v = get_pylong(state, v);
953      if (v == NULL)
954          return -1;
955      res = _PyLong_AsByteArray((PyLongObject *)v,
956                                (unsigned char *)p,
957                                8,
958                                0, /* little_endian */
959                                0  /* signed */);
960      Py_DECREF(v);
961      return res;
962  }
963  
964  static int
bp_halffloat(_structmodulestate * state,char * p,PyObject * v,const formatdef * f)965  bp_halffloat(_structmodulestate *state, char *p, PyObject *v, const formatdef *f)
966  {
967      return pack_halffloat(state, p, v, 0);
968  }
969  
970  static int
bp_float(_structmodulestate * state,char * p,PyObject * v,const formatdef * f)971  bp_float(_structmodulestate *state, char *p, PyObject *v, const formatdef *f)
972  {
973      double x = PyFloat_AsDouble(v);
974      if (x == -1 && PyErr_Occurred()) {
975          PyErr_SetString(state->StructError,
976                          "required argument is not a float");
977          return -1;
978      }
979      return PyFloat_Pack4(x, p, 0);
980  }
981  
982  static int
bp_double(_structmodulestate * state,char * p,PyObject * v,const formatdef * f)983  bp_double(_structmodulestate *state, char *p, PyObject *v, const formatdef *f)
984  {
985      double x = PyFloat_AsDouble(v);
986      if (x == -1 && PyErr_Occurred()) {
987          PyErr_SetString(state->StructError,
988                          "required argument is not a float");
989          return -1;
990      }
991      return PyFloat_Pack8(x, p, 0);
992  }
993  
994  static int
bp_bool(_structmodulestate * state,char * p,PyObject * v,const formatdef * f)995  bp_bool(_structmodulestate *state, char *p, PyObject *v, const formatdef *f)
996  {
997      int y;
998      y = PyObject_IsTrue(v);
999      if (y < 0)
1000          return -1;
1001      *p = (char)y;
1002      return 0;
1003  }
1004  
1005  static formatdef bigendian_table[] = {
1006      {'x',       1,              0,              NULL},
1007      {'b',       1,              0,              nu_byte,        np_byte},
1008      {'B',       1,              0,              nu_ubyte,       np_ubyte},
1009      {'c',       1,              0,              nu_char,        np_char},
1010      {'s',       1,              0,              NULL},
1011      {'p',       1,              0,              NULL},
1012      {'h',       2,              0,              bu_int,         bp_int},
1013      {'H',       2,              0,              bu_uint,        bp_uint},
1014      {'i',       4,              0,              bu_int,         bp_int},
1015      {'I',       4,              0,              bu_uint,        bp_uint},
1016      {'l',       4,              0,              bu_int,         bp_int},
1017      {'L',       4,              0,              bu_uint,        bp_uint},
1018      {'q',       8,              0,              bu_longlong,    bp_longlong},
1019      {'Q',       8,              0,              bu_ulonglong,   bp_ulonglong},
1020      {'?',       1,              0,              bu_bool,        bp_bool},
1021      {'e',       2,              0,              bu_halffloat,   bp_halffloat},
1022      {'f',       4,              0,              bu_float,       bp_float},
1023      {'d',       8,              0,              bu_double,      bp_double},
1024      {0}
1025  };
1026  
1027  /* Little-endian routines. *****************************************************/
1028  
1029  static PyObject *
lu_int(_structmodulestate * state,const char * p,const formatdef * f)1030  lu_int(_structmodulestate *state, const char *p, const formatdef *f)
1031  {
1032      long x = 0;
1033      Py_ssize_t i = f->size;
1034      const unsigned char *bytes = (const unsigned char *)p;
1035      do {
1036          x = (x<<8) | bytes[--i];
1037      } while (i > 0);
1038      /* Extend the sign bit. */
1039      if (SIZEOF_LONG > f->size)
1040          x |= -(x & (1L << ((8 * f->size) - 1)));
1041      return PyLong_FromLong(x);
1042  }
1043  
1044  static PyObject *
lu_uint(_structmodulestate * state,const char * p,const formatdef * f)1045  lu_uint(_structmodulestate *state, const char *p, const formatdef *f)
1046  {
1047      unsigned long x = 0;
1048      Py_ssize_t i = f->size;
1049      const unsigned char *bytes = (const unsigned char *)p;
1050      do {
1051          x = (x<<8) | bytes[--i];
1052      } while (i > 0);
1053      return PyLong_FromUnsignedLong(x);
1054  }
1055  
1056  static PyObject *
lu_longlong(_structmodulestate * state,const char * p,const formatdef * f)1057  lu_longlong(_structmodulestate *state, const char *p, const formatdef *f)
1058  {
1059      long long x = 0;
1060      Py_ssize_t i = f->size;
1061      const unsigned char *bytes = (const unsigned char *)p;
1062      do {
1063          x = (x<<8) | bytes[--i];
1064      } while (i > 0);
1065      /* Extend the sign bit. */
1066      if (SIZEOF_LONG_LONG > f->size)
1067          x |= -(x & ((long long)1 << ((8 * f->size) - 1)));
1068      return PyLong_FromLongLong(x);
1069  }
1070  
1071  static PyObject *
lu_ulonglong(_structmodulestate * state,const char * p,const formatdef * f)1072  lu_ulonglong(_structmodulestate *state, const char *p, const formatdef *f)
1073  {
1074      unsigned long long x = 0;
1075      Py_ssize_t i = f->size;
1076      const unsigned char *bytes = (const unsigned char *)p;
1077      do {
1078          x = (x<<8) | bytes[--i];
1079      } while (i > 0);
1080      return PyLong_FromUnsignedLongLong(x);
1081  }
1082  
1083  static PyObject *
lu_halffloat(_structmodulestate * state,const char * p,const formatdef * f)1084  lu_halffloat(_structmodulestate *state, const char *p, const formatdef *f)
1085  {
1086      return unpack_halffloat(p, 1);
1087  }
1088  
1089  static PyObject *
lu_float(_structmodulestate * state,const char * p,const formatdef * f)1090  lu_float(_structmodulestate *state, const char *p, const formatdef *f)
1091  {
1092      return unpack_float(p, 1);
1093  }
1094  
1095  static PyObject *
lu_double(_structmodulestate * state,const char * p,const formatdef * f)1096  lu_double(_structmodulestate *state, const char *p, const formatdef *f)
1097  {
1098      return unpack_double(p, 1);
1099  }
1100  
1101  static int
lp_int(_structmodulestate * state,char * p,PyObject * v,const formatdef * f)1102  lp_int(_structmodulestate *state, char *p, PyObject *v, const formatdef *f)
1103  {
1104      long x;
1105      Py_ssize_t i;
1106      unsigned char *q = (unsigned char *)p;
1107      if (get_long(state, v, &x) < 0)
1108          return -1;
1109      i = f->size;
1110      if (i != SIZEOF_LONG) {
1111          if ((i == 2) && (x < -32768 || x > 32767))
1112              RANGE_ERROR(state, x, f, 0, 0xffffL);
1113  #if (SIZEOF_LONG != 4)
1114          else if ((i == 4) && (x < -2147483648L || x > 2147483647L))
1115              RANGE_ERROR(state, x, f, 0, 0xffffffffL);
1116  #endif
1117      }
1118      do {
1119          *q++ = (unsigned char)(x & 0xffL);
1120          x >>= 8;
1121      } while (--i > 0);
1122      return 0;
1123  }
1124  
1125  static int
lp_uint(_structmodulestate * state,char * p,PyObject * v,const formatdef * f)1126  lp_uint(_structmodulestate *state, char *p, PyObject *v, const formatdef *f)
1127  {
1128      unsigned long x;
1129      Py_ssize_t i;
1130      unsigned char *q = (unsigned char *)p;
1131      if (get_ulong(state, v, &x) < 0)
1132          return -1;
1133      i = f->size;
1134      if (i != SIZEOF_LONG) {
1135          unsigned long maxint = 1;
1136          maxint <<= (unsigned long)(i * 8);
1137          if (x >= maxint)
1138              RANGE_ERROR(state, x, f, 1, maxint - 1);
1139      }
1140      do {
1141          *q++ = (unsigned char)(x & 0xffUL);
1142          x >>= 8;
1143      } while (--i > 0);
1144      return 0;
1145  }
1146  
1147  static int
lp_longlong(_structmodulestate * state,char * p,PyObject * v,const formatdef * f)1148  lp_longlong(_structmodulestate *state, char *p, PyObject *v, const formatdef *f)
1149  {
1150      int res;
1151      v = get_pylong(state, v);
1152      if (v == NULL)
1153          return -1;
1154      res = _PyLong_AsByteArray((PyLongObject*)v,
1155                                (unsigned char *)p,
1156                                8,
1157                                1, /* little_endian */
1158                                1  /* signed */);
1159      Py_DECREF(v);
1160      return res;
1161  }
1162  
1163  static int
lp_ulonglong(_structmodulestate * state,char * p,PyObject * v,const formatdef * f)1164  lp_ulonglong(_structmodulestate *state, char *p, PyObject *v, const formatdef *f)
1165  {
1166      int res;
1167      v = get_pylong(state, v);
1168      if (v == NULL)
1169          return -1;
1170      res = _PyLong_AsByteArray((PyLongObject*)v,
1171                                (unsigned char *)p,
1172                                8,
1173                                1, /* little_endian */
1174                                0  /* signed */);
1175      Py_DECREF(v);
1176      return res;
1177  }
1178  
1179  static int
lp_halffloat(_structmodulestate * state,char * p,PyObject * v,const formatdef * f)1180  lp_halffloat(_structmodulestate *state, char *p, PyObject *v, const formatdef *f)
1181  {
1182      return pack_halffloat(state, p, v, 1);
1183  }
1184  
1185  static int
lp_float(_structmodulestate * state,char * p,PyObject * v,const formatdef * f)1186  lp_float(_structmodulestate *state, char *p, PyObject *v, const formatdef *f)
1187  {
1188      double x = PyFloat_AsDouble(v);
1189      if (x == -1 && PyErr_Occurred()) {
1190          PyErr_SetString(state->StructError,
1191                          "required argument is not a float");
1192          return -1;
1193      }
1194      return PyFloat_Pack4(x, p, 1);
1195  }
1196  
1197  static int
lp_double(_structmodulestate * state,char * p,PyObject * v,const formatdef * f)1198  lp_double(_structmodulestate *state, char *p, PyObject *v, const formatdef *f)
1199  {
1200      double x = PyFloat_AsDouble(v);
1201      if (x == -1 && PyErr_Occurred()) {
1202          PyErr_SetString(state->StructError,
1203                          "required argument is not a float");
1204          return -1;
1205      }
1206      return PyFloat_Pack8(x, p, 1);
1207  }
1208  
1209  static formatdef lilendian_table[] = {
1210      {'x',       1,              0,              NULL},
1211      {'b',       1,              0,              nu_byte,        np_byte},
1212      {'B',       1,              0,              nu_ubyte,       np_ubyte},
1213      {'c',       1,              0,              nu_char,        np_char},
1214      {'s',       1,              0,              NULL},
1215      {'p',       1,              0,              NULL},
1216      {'h',       2,              0,              lu_int,         lp_int},
1217      {'H',       2,              0,              lu_uint,        lp_uint},
1218      {'i',       4,              0,              lu_int,         lp_int},
1219      {'I',       4,              0,              lu_uint,        lp_uint},
1220      {'l',       4,              0,              lu_int,         lp_int},
1221      {'L',       4,              0,              lu_uint,        lp_uint},
1222      {'q',       8,              0,              lu_longlong,    lp_longlong},
1223      {'Q',       8,              0,              lu_ulonglong,   lp_ulonglong},
1224      {'?',       1,              0,              bu_bool,        bp_bool}, /* Std rep not endian dep,
1225          but potentially different from native rep -- reuse bx_bool funcs. */
1226      {'e',       2,              0,              lu_halffloat,   lp_halffloat},
1227      {'f',       4,              0,              lu_float,       lp_float},
1228      {'d',       8,              0,              lu_double,      lp_double},
1229      {0}
1230  };
1231  
1232  
1233  static const formatdef *
whichtable(const char ** pfmt)1234  whichtable(const char **pfmt)
1235  {
1236      const char *fmt = (*pfmt)++; /* May be backed out of later */
1237      switch (*fmt) {
1238      case '<':
1239          return lilendian_table;
1240      case '>':
1241      case '!': /* Network byte order is big-endian */
1242          return bigendian_table;
1243      case '=': { /* Host byte order -- different from native in alignment! */
1244  #if PY_LITTLE_ENDIAN
1245          return lilendian_table;
1246  #else
1247          return bigendian_table;
1248  #endif
1249      }
1250      default:
1251          --*pfmt; /* Back out of pointer increment */
1252          /* Fall through */
1253      case '@':
1254          return native_table;
1255      }
1256  }
1257  
1258  
1259  /* Get the table entry for a format code */
1260  
1261  static const formatdef *
getentry(_structmodulestate * state,int c,const formatdef * f)1262  getentry(_structmodulestate *state, int c, const formatdef *f)
1263  {
1264      for (; f->format != '\0'; f++) {
1265          if (f->format == c) {
1266              return f;
1267          }
1268      }
1269      PyErr_SetString(state->StructError, "bad char in struct format");
1270      return NULL;
1271  }
1272  
1273  
1274  /* Align a size according to a format code.  Return -1 on overflow. */
1275  
1276  static Py_ssize_t
align(Py_ssize_t size,char c,const formatdef * e)1277  align(Py_ssize_t size, char c, const formatdef *e)
1278  {
1279      Py_ssize_t extra;
1280  
1281      if (e->format == c) {
1282          if (e->alignment && size > 0) {
1283              extra = (e->alignment - 1) - (size - 1) % (e->alignment);
1284              if (extra > PY_SSIZE_T_MAX - size)
1285                  return -1;
1286              size += extra;
1287          }
1288      }
1289      return size;
1290  }
1291  
1292  /*
1293   * Struct object implementation.
1294   */
1295  
1296  /* calculate the size of a format string */
1297  
1298  static int
prepare_s(PyStructObject * self)1299  prepare_s(PyStructObject *self)
1300  {
1301      const formatdef *f;
1302      const formatdef *e;
1303      formatcode *codes;
1304  
1305      const char *s;
1306      const char *fmt;
1307      char c;
1308      Py_ssize_t size, len, num, itemsize;
1309      size_t ncodes;
1310  
1311      _structmodulestate *state = get_struct_state_structinst(self);
1312  
1313      fmt = PyBytes_AS_STRING(self->s_format);
1314      if (strlen(fmt) != (size_t)PyBytes_GET_SIZE(self->s_format)) {
1315          PyErr_SetString(state->StructError,
1316                          "embedded null character");
1317          return -1;
1318      }
1319  
1320      f = whichtable(&fmt);
1321  
1322      s = fmt;
1323      size = 0;
1324      len = 0;
1325      ncodes = 0;
1326      while ((c = *s++) != '\0') {
1327          if (Py_ISSPACE(c))
1328              continue;
1329          if ('0' <= c && c <= '9') {
1330              num = c - '0';
1331              while ('0' <= (c = *s++) && c <= '9') {
1332                  /* overflow-safe version of
1333                     if (num*10 + (c - '0') > PY_SSIZE_T_MAX) { ... } */
1334                  if (num >= PY_SSIZE_T_MAX / 10 && (
1335                          num > PY_SSIZE_T_MAX / 10 ||
1336                          (c - '0') > PY_SSIZE_T_MAX % 10))
1337                      goto overflow;
1338                  num = num*10 + (c - '0');
1339              }
1340              if (c == '\0') {
1341                  PyErr_SetString(state->StructError,
1342                                  "repeat count given without format specifier");
1343                  return -1;
1344              }
1345          }
1346          else
1347              num = 1;
1348  
1349          e = getentry(state, c, f);
1350          if (e == NULL)
1351              return -1;
1352  
1353          switch (c) {
1354              case 's': /* fall through */
1355              case 'p': len++; ncodes++; break;
1356              case 'x': break;
1357              default: len += num; if (num) ncodes++; break;
1358          }
1359  
1360          itemsize = e->size;
1361          size = align(size, c, e);
1362          if (size == -1)
1363              goto overflow;
1364  
1365          /* if (size + num * itemsize > PY_SSIZE_T_MAX) { ... } */
1366          if (num > (PY_SSIZE_T_MAX - size) / itemsize)
1367              goto overflow;
1368          size += num * itemsize;
1369      }
1370  
1371      /* check for overflow */
1372      if ((ncodes + 1) > ((size_t)PY_SSIZE_T_MAX / sizeof(formatcode))) {
1373          PyErr_NoMemory();
1374          return -1;
1375      }
1376  
1377      self->s_size = size;
1378      self->s_len = len;
1379      codes = PyMem_Malloc((ncodes + 1) * sizeof(formatcode));
1380      if (codes == NULL) {
1381          PyErr_NoMemory();
1382          return -1;
1383      }
1384      /* Free any s_codes value left over from a previous initialization. */
1385      if (self->s_codes != NULL)
1386          PyMem_Free(self->s_codes);
1387      self->s_codes = codes;
1388  
1389      s = fmt;
1390      size = 0;
1391      while ((c = *s++) != '\0') {
1392          if (Py_ISSPACE(c))
1393              continue;
1394          if ('0' <= c && c <= '9') {
1395              num = c - '0';
1396              while ('0' <= (c = *s++) && c <= '9')
1397                  num = num*10 + (c - '0');
1398          }
1399          else
1400              num = 1;
1401  
1402          e = getentry(state, c, f);
1403  
1404          size = align(size, c, e);
1405          if (c == 's' || c == 'p') {
1406              codes->offset = size;
1407              codes->size = num;
1408              codes->fmtdef = e;
1409              codes->repeat = 1;
1410              codes++;
1411              size += num;
1412          } else if (c == 'x') {
1413              size += num;
1414          } else if (num) {
1415              codes->offset = size;
1416              codes->size = e->size;
1417              codes->fmtdef = e;
1418              codes->repeat = num;
1419              codes++;
1420              size += e->size * num;
1421          }
1422      }
1423      codes->fmtdef = NULL;
1424      codes->offset = size;
1425      codes->size = 0;
1426      codes->repeat = 0;
1427  
1428      return 0;
1429  
1430    overflow:
1431      PyErr_SetString(state->StructError,
1432                      "total struct size too long");
1433      return -1;
1434  }
1435  
1436  static PyObject *
s_new(PyTypeObject * type,PyObject * args,PyObject * kwds)1437  s_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
1438  {
1439      PyObject *self;
1440  
1441      assert(type != NULL);
1442      allocfunc alloc_func = PyType_GetSlot(type, Py_tp_alloc);
1443      assert(alloc_func != NULL);
1444  
1445      self = alloc_func(type, 0);
1446      if (self != NULL) {
1447          PyStructObject *s = (PyStructObject*)self;
1448          s->s_format = Py_NewRef(Py_None);
1449          s->s_codes = NULL;
1450          s->s_size = -1;
1451          s->s_len = -1;
1452      }
1453      return self;
1454  }
1455  
1456  /*[clinic input]
1457  Struct.__init__
1458  
1459      format: object
1460  
1461  Create a compiled struct object.
1462  
1463  Return a new Struct object which writes and reads binary data according to
1464  the format string.
1465  
1466  See help(struct) for more on format strings.
1467  [clinic start generated code]*/
1468  
1469  static int
Struct___init___impl(PyStructObject * self,PyObject * format)1470  Struct___init___impl(PyStructObject *self, PyObject *format)
1471  /*[clinic end generated code: output=b8e80862444e92d0 input=192a4575a3dde802]*/
1472  {
1473      int ret = 0;
1474  
1475      if (PyUnicode_Check(format)) {
1476          format = PyUnicode_AsASCIIString(format);
1477          if (format == NULL)
1478              return -1;
1479      }
1480      else {
1481          Py_INCREF(format);
1482      }
1483  
1484      if (!PyBytes_Check(format)) {
1485          Py_DECREF(format);
1486          PyErr_Format(PyExc_TypeError,
1487                       "Struct() argument 1 must be a str or bytes object, "
1488                       "not %.200s",
1489                       _PyType_Name(Py_TYPE(format)));
1490          return -1;
1491      }
1492  
1493      Py_SETREF(self->s_format, format);
1494  
1495      ret = prepare_s(self);
1496      return ret;
1497  }
1498  
1499  static int
s_clear(PyStructObject * s)1500  s_clear(PyStructObject *s)
1501  {
1502      Py_CLEAR(s->s_format);
1503      return 0;
1504  }
1505  
1506  static int
s_traverse(PyStructObject * s,visitproc visit,void * arg)1507  s_traverse(PyStructObject *s, visitproc visit, void *arg)
1508  {
1509      Py_VISIT(Py_TYPE(s));
1510      Py_VISIT(s->s_format);
1511      return 0;
1512  }
1513  
1514  static void
s_dealloc(PyStructObject * s)1515  s_dealloc(PyStructObject *s)
1516  {
1517      PyTypeObject *tp = Py_TYPE(s);
1518      PyObject_GC_UnTrack(s);
1519      if (s->weakreflist != NULL)
1520          PyObject_ClearWeakRefs((PyObject *)s);
1521      if (s->s_codes != NULL) {
1522          PyMem_Free(s->s_codes);
1523      }
1524      Py_XDECREF(s->s_format);
1525      freefunc free_func = PyType_GetSlot(Py_TYPE(s), Py_tp_free);
1526      free_func(s);
1527      Py_DECREF(tp);
1528  }
1529  
1530  static PyObject *
s_unpack_internal(PyStructObject * soself,const char * startfrom,_structmodulestate * state)1531  s_unpack_internal(PyStructObject *soself, const char *startfrom,
1532                    _structmodulestate *state) {
1533      formatcode *code;
1534      Py_ssize_t i = 0;
1535      PyObject *result = PyTuple_New(soself->s_len);
1536      if (result == NULL)
1537          return NULL;
1538  
1539      for (code = soself->s_codes; code->fmtdef != NULL; code++) {
1540          const formatdef *e = code->fmtdef;
1541          const char *res = startfrom + code->offset;
1542          Py_ssize_t j = code->repeat;
1543          while (j--) {
1544              PyObject *v;
1545              if (e->format == 's') {
1546                  v = PyBytes_FromStringAndSize(res, code->size);
1547              } else if (e->format == 'p') {
1548                  Py_ssize_t n = *(unsigned char*)res;
1549                  if (n >= code->size)
1550                      n = code->size - 1;
1551                  v = PyBytes_FromStringAndSize(res + 1, n);
1552              } else {
1553                  v = e->unpack(state, res, e);
1554              }
1555              if (v == NULL)
1556                  goto fail;
1557              PyTuple_SET_ITEM(result, i++, v);
1558              res += code->size;
1559          }
1560      }
1561  
1562      return result;
1563  fail:
1564      Py_DECREF(result);
1565      return NULL;
1566  }
1567  
1568  
1569  /*[clinic input]
1570  Struct.unpack
1571  
1572      buffer: Py_buffer
1573      /
1574  
1575  Return a tuple containing unpacked values.
1576  
1577  Unpack according to the format string Struct.format. The buffer's size
1578  in bytes must be Struct.size.
1579  
1580  See help(struct) for more on format strings.
1581  [clinic start generated code]*/
1582  
1583  static PyObject *
Struct_unpack_impl(PyStructObject * self,Py_buffer * buffer)1584  Struct_unpack_impl(PyStructObject *self, Py_buffer *buffer)
1585  /*[clinic end generated code: output=873a24faf02e848a input=3113f8e7038b2f6c]*/
1586  {
1587      _structmodulestate *state = get_struct_state_structinst(self);
1588      assert(self->s_codes != NULL);
1589      if (buffer->len != self->s_size) {
1590          PyErr_Format(state->StructError,
1591                       "unpack requires a buffer of %zd bytes",
1592                       self->s_size);
1593          return NULL;
1594      }
1595      return s_unpack_internal(self, buffer->buf, state);
1596  }
1597  
1598  /*[clinic input]
1599  Struct.unpack_from
1600  
1601      buffer: Py_buffer
1602      offset: Py_ssize_t = 0
1603  
1604  Return a tuple containing unpacked values.
1605  
1606  Values are unpacked according to the format string Struct.format.
1607  
1608  The buffer's size in bytes, starting at position offset, must be
1609  at least Struct.size.
1610  
1611  See help(struct) for more on format strings.
1612  [clinic start generated code]*/
1613  
1614  static PyObject *
Struct_unpack_from_impl(PyStructObject * self,Py_buffer * buffer,Py_ssize_t offset)1615  Struct_unpack_from_impl(PyStructObject *self, Py_buffer *buffer,
1616                          Py_ssize_t offset)
1617  /*[clinic end generated code: output=57fac875e0977316 input=cafd4851d473c894]*/
1618  {
1619      _structmodulestate *state = get_struct_state_structinst(self);
1620      assert(self->s_codes != NULL);
1621  
1622      if (offset < 0) {
1623          if (offset + self->s_size > 0) {
1624              PyErr_Format(state->StructError,
1625                           "not enough data to unpack %zd bytes at offset %zd",
1626                           self->s_size,
1627                           offset);
1628              return NULL;
1629          }
1630  
1631          if (offset + buffer->len < 0) {
1632              PyErr_Format(state->StructError,
1633                           "offset %zd out of range for %zd-byte buffer",
1634                           offset,
1635                           buffer->len);
1636              return NULL;
1637          }
1638          offset += buffer->len;
1639      }
1640  
1641      if ((buffer->len - offset) < self->s_size) {
1642          PyErr_Format(state->StructError,
1643                       "unpack_from requires a buffer of at least %zu bytes for "
1644                       "unpacking %zd bytes at offset %zd "
1645                       "(actual buffer size is %zd)",
1646                       (size_t)self->s_size + (size_t)offset,
1647                       self->s_size,
1648                       offset,
1649                       buffer->len);
1650          return NULL;
1651      }
1652      return s_unpack_internal(self, (char*)buffer->buf + offset, state);
1653  }
1654  
1655  
1656  
1657  /* Unpack iterator type */
1658  
1659  typedef struct {
1660      PyObject_HEAD
1661      PyStructObject *so;
1662      Py_buffer buf;
1663      Py_ssize_t index;
1664  } unpackiterobject;
1665  
1666  static void
unpackiter_dealloc(unpackiterobject * self)1667  unpackiter_dealloc(unpackiterobject *self)
1668  {
1669      /* bpo-31095: UnTrack is needed before calling any callbacks */
1670      PyTypeObject *tp = Py_TYPE(self);
1671      PyObject_GC_UnTrack(self);
1672      Py_XDECREF(self->so);
1673      PyBuffer_Release(&self->buf);
1674      PyObject_GC_Del(self);
1675      Py_DECREF(tp);
1676  }
1677  
1678  static int
unpackiter_traverse(unpackiterobject * self,visitproc visit,void * arg)1679  unpackiter_traverse(unpackiterobject *self, visitproc visit, void *arg)
1680  {
1681      Py_VISIT(Py_TYPE(self));
1682      Py_VISIT(self->so);
1683      Py_VISIT(self->buf.obj);
1684      return 0;
1685  }
1686  
1687  static PyObject *
unpackiter_len(unpackiterobject * self,PyObject * Py_UNUSED (ignored))1688  unpackiter_len(unpackiterobject *self, PyObject *Py_UNUSED(ignored))
1689  {
1690      Py_ssize_t len;
1691      if (self->so == NULL)
1692          len = 0;
1693      else
1694          len = (self->buf.len - self->index) / self->so->s_size;
1695      return PyLong_FromSsize_t(len);
1696  }
1697  
1698  static PyMethodDef unpackiter_methods[] = {
1699      {"__length_hint__", (PyCFunction) unpackiter_len, METH_NOARGS, NULL},
1700      {NULL,              NULL}           /* sentinel */
1701  };
1702  
1703  static PyObject *
unpackiter_iternext(unpackiterobject * self)1704  unpackiter_iternext(unpackiterobject *self)
1705  {
1706      _structmodulestate *state = get_struct_state_iterinst(self);
1707      PyObject *result;
1708      if (self->so == NULL)
1709          return NULL;
1710      if (self->index >= self->buf.len) {
1711          /* Iterator exhausted */
1712          Py_CLEAR(self->so);
1713          PyBuffer_Release(&self->buf);
1714          return NULL;
1715      }
1716      assert(self->index + self->so->s_size <= self->buf.len);
1717      result = s_unpack_internal(self->so,
1718                                 (char*) self->buf.buf + self->index,
1719                                 state);
1720      self->index += self->so->s_size;
1721      return result;
1722  }
1723  
unpackiter_new(PyTypeObject * type,PyObject * args,PyObject * kwds)1724  PyObject *unpackiter_new(PyTypeObject *type, PyObject *args, PyObject *kwds) {
1725      PyErr_Format(PyExc_TypeError, "Cannot create '%.200s objects", _PyType_Name(type));
1726      return NULL;
1727  }
1728  
1729  static PyType_Slot unpackiter_type_slots[] = {
1730      {Py_tp_dealloc, unpackiter_dealloc},
1731      {Py_tp_getattro, PyObject_GenericGetAttr},
1732      {Py_tp_traverse, unpackiter_traverse},
1733      {Py_tp_iter, PyObject_SelfIter},
1734      {Py_tp_iternext, unpackiter_iternext},
1735      {Py_tp_methods, unpackiter_methods},
1736      {Py_tp_new, unpackiter_new},
1737      {0, 0},
1738  };
1739  
1740  static PyType_Spec unpackiter_type_spec = {
1741      "_struct.unpack_iterator",
1742      sizeof(unpackiterobject),
1743      0,
1744      (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC |
1745       Py_TPFLAGS_IMMUTABLETYPE),
1746      unpackiter_type_slots
1747  };
1748  
1749  /*[clinic input]
1750  Struct.iter_unpack
1751  
1752      buffer: object
1753      /
1754  
1755  Return an iterator yielding tuples.
1756  
1757  Tuples are unpacked from the given bytes source, like a repeated
1758  invocation of unpack_from().
1759  
1760  Requires that the bytes length be a multiple of the struct size.
1761  [clinic start generated code]*/
1762  
1763  static PyObject *
Struct_iter_unpack(PyStructObject * self,PyObject * buffer)1764  Struct_iter_unpack(PyStructObject *self, PyObject *buffer)
1765  /*[clinic end generated code: output=172d83d0cd15dbab input=6d65b3f3107dbc99]*/
1766  {
1767      _structmodulestate *state = get_struct_state_structinst(self);
1768      unpackiterobject *iter;
1769  
1770      assert(self->s_codes != NULL);
1771  
1772      if (self->s_size == 0) {
1773          PyErr_Format(state->StructError,
1774                       "cannot iteratively unpack with a struct of length 0");
1775          return NULL;
1776      }
1777  
1778      iter = (unpackiterobject *) PyType_GenericAlloc((PyTypeObject *)state->unpackiter_type, 0);
1779      if (iter == NULL)
1780          return NULL;
1781  
1782      if (PyObject_GetBuffer(buffer, &iter->buf, PyBUF_SIMPLE) < 0) {
1783          Py_DECREF(iter);
1784          return NULL;
1785      }
1786      if (iter->buf.len % self->s_size != 0) {
1787          PyErr_Format(state->StructError,
1788                       "iterative unpacking requires a buffer of "
1789                       "a multiple of %zd bytes",
1790                       self->s_size);
1791          Py_DECREF(iter);
1792          return NULL;
1793      }
1794      Py_INCREF(self);
1795      iter->so = self;
1796      iter->index = 0;
1797      return (PyObject *)iter;
1798  }
1799  
1800  
1801  /*
1802   * Guts of the pack function.
1803   *
1804   * Takes a struct object, a tuple of arguments, and offset in that tuple of
1805   * argument for where to start processing the arguments for packing, and a
1806   * character buffer for writing the packed string.  The caller must insure
1807   * that the buffer may contain the required length for packing the arguments.
1808   * 0 is returned on success, 1 is returned if there is an error.
1809   *
1810   */
1811  static int
s_pack_internal(PyStructObject * soself,PyObject * const * args,int offset,char * buf,_structmodulestate * state)1812  s_pack_internal(PyStructObject *soself, PyObject *const *args, int offset,
1813                  char* buf, _structmodulestate *state)
1814  {
1815      formatcode *code;
1816      /* XXX(nnorwitz): why does i need to be a local?  can we use
1817         the offset parameter or do we need the wider width? */
1818      Py_ssize_t i;
1819  
1820      memset(buf, '\0', soself->s_size);
1821      i = offset;
1822      for (code = soself->s_codes; code->fmtdef != NULL; code++) {
1823          const formatdef *e = code->fmtdef;
1824          char *res = buf + code->offset;
1825          Py_ssize_t j = code->repeat;
1826          while (j--) {
1827              PyObject *v = args[i++];
1828              if (e->format == 's') {
1829                  Py_ssize_t n;
1830                  int isstring;
1831                  const void *p;
1832                  isstring = PyBytes_Check(v);
1833                  if (!isstring && !PyByteArray_Check(v)) {
1834                      PyErr_SetString(state->StructError,
1835                                      "argument for 's' must be a bytes object");
1836                      return -1;
1837                  }
1838                  if (isstring) {
1839                      n = PyBytes_GET_SIZE(v);
1840                      p = PyBytes_AS_STRING(v);
1841                  }
1842                  else {
1843                      n = PyByteArray_GET_SIZE(v);
1844                      p = PyByteArray_AS_STRING(v);
1845                  }
1846                  if (n > code->size)
1847                      n = code->size;
1848                  if (n > 0)
1849                      memcpy(res, p, n);
1850              } else if (e->format == 'p') {
1851                  Py_ssize_t n;
1852                  int isstring;
1853                  const void *p;
1854                  isstring = PyBytes_Check(v);
1855                  if (!isstring && !PyByteArray_Check(v)) {
1856                      PyErr_SetString(state->StructError,
1857                                      "argument for 'p' must be a bytes object");
1858                      return -1;
1859                  }
1860                  if (isstring) {
1861                      n = PyBytes_GET_SIZE(v);
1862                      p = PyBytes_AS_STRING(v);
1863                  }
1864                  else {
1865                      n = PyByteArray_GET_SIZE(v);
1866                      p = PyByteArray_AS_STRING(v);
1867                  }
1868                  if (n > (code->size - 1))
1869                      n = code->size - 1;
1870                  if (n > 0)
1871                      memcpy(res + 1, p, n);
1872                  if (n > 255)
1873                      n = 255;
1874                  *res = Py_SAFE_DOWNCAST(n, Py_ssize_t, unsigned char);
1875              } else {
1876                  if (e->pack(state, res, v, e) < 0) {
1877                      if (PyLong_Check(v) && PyErr_ExceptionMatches(PyExc_OverflowError))
1878                          PyErr_SetString(state->StructError,
1879                                          "int too large to convert");
1880                      return -1;
1881                  }
1882              }
1883              res += code->size;
1884          }
1885      }
1886  
1887      /* Success */
1888      return 0;
1889  }
1890  
1891  
1892  PyDoc_STRVAR(s_pack__doc__,
1893  "S.pack(v1, v2, ...) -> bytes\n\
1894  \n\
1895  Return a bytes object containing values v1, v2, ... packed according\n\
1896  to the format string S.format.  See help(struct) for more on format\n\
1897  strings.");
1898  
1899  static PyObject *
s_pack(PyObject * self,PyObject * const * args,Py_ssize_t nargs)1900  s_pack(PyObject *self, PyObject *const *args, Py_ssize_t nargs)
1901  {
1902      char *buf;
1903      PyStructObject *soself;
1904      _structmodulestate *state = get_struct_state_structinst(self);
1905  
1906      /* Validate arguments. */
1907      soself = (PyStructObject *)self;
1908      assert(PyStruct_Check(self, state));
1909      assert(soself->s_codes != NULL);
1910      if (nargs != soself->s_len)
1911      {
1912          PyErr_Format(state->StructError,
1913              "pack expected %zd items for packing (got %zd)", soself->s_len, nargs);
1914          return NULL;
1915      }
1916  
1917      /* Allocate a new string */
1918      _PyBytesWriter writer;
1919      _PyBytesWriter_Init(&writer);
1920      buf = _PyBytesWriter_Alloc(&writer, soself->s_size);
1921      if (buf == NULL) {
1922          _PyBytesWriter_Dealloc(&writer);
1923          return NULL;
1924      }
1925  
1926      /* Call the guts */
1927      if ( s_pack_internal(soself, args, 0, buf, state) != 0 ) {
1928          _PyBytesWriter_Dealloc(&writer);
1929          return NULL;
1930      }
1931  
1932      return _PyBytesWriter_Finish(&writer, buf + soself->s_size);
1933  }
1934  
1935  PyDoc_STRVAR(s_pack_into__doc__,
1936  "S.pack_into(buffer, offset, v1, v2, ...)\n\
1937  \n\
1938  Pack the values v1, v2, ... according to the format string S.format\n\
1939  and write the packed bytes into the writable buffer buf starting at\n\
1940  offset.  Note that the offset is a required argument.  See\n\
1941  help(struct) for more on format strings.");
1942  
1943  static PyObject *
s_pack_into(PyObject * self,PyObject * const * args,Py_ssize_t nargs)1944  s_pack_into(PyObject *self, PyObject *const *args, Py_ssize_t nargs)
1945  {
1946      PyStructObject *soself;
1947      Py_buffer buffer;
1948      Py_ssize_t offset;
1949      _structmodulestate *state = get_struct_state_structinst(self);
1950  
1951      /* Validate arguments.  +1 is for the first arg as buffer. */
1952      soself = (PyStructObject *)self;
1953      assert(PyStruct_Check(self, state));
1954      assert(soself->s_codes != NULL);
1955      if (nargs != (soself->s_len + 2))
1956      {
1957          if (nargs == 0) {
1958              PyErr_Format(state->StructError,
1959                          "pack_into expected buffer argument");
1960          }
1961          else if (nargs == 1) {
1962              PyErr_Format(state->StructError,
1963                          "pack_into expected offset argument");
1964          }
1965          else {
1966              PyErr_Format(state->StructError,
1967                          "pack_into expected %zd items for packing (got %zd)",
1968                          soself->s_len, (nargs - 2));
1969          }
1970          return NULL;
1971      }
1972  
1973      /* Extract a writable memory buffer from the first argument */
1974      if (!PyArg_Parse(args[0], "w*", &buffer))
1975          return NULL;
1976      assert(buffer.len >= 0);
1977  
1978      /* Extract the offset from the first argument */
1979      offset = PyNumber_AsSsize_t(args[1], PyExc_IndexError);
1980      if (offset == -1 && PyErr_Occurred()) {
1981          PyBuffer_Release(&buffer);
1982          return NULL;
1983      }
1984  
1985      /* Support negative offsets. */
1986      if (offset < 0) {
1987           /* Check that negative offset is low enough to fit data */
1988          if (offset + soself->s_size > 0) {
1989              PyErr_Format(state->StructError,
1990                           "no space to pack %zd bytes at offset %zd",
1991                           soself->s_size,
1992                           offset);
1993              PyBuffer_Release(&buffer);
1994              return NULL;
1995          }
1996  
1997          /* Check that negative offset is not crossing buffer boundary */
1998          if (offset + buffer.len < 0) {
1999              PyErr_Format(state->StructError,
2000                           "offset %zd out of range for %zd-byte buffer",
2001                           offset,
2002                           buffer.len);
2003              PyBuffer_Release(&buffer);
2004              return NULL;
2005          }
2006  
2007          offset += buffer.len;
2008      }
2009  
2010      /* Check boundaries */
2011      if ((buffer.len - offset) < soself->s_size) {
2012          assert(offset >= 0);
2013          assert(soself->s_size >= 0);
2014  
2015          PyErr_Format(state->StructError,
2016                       "pack_into requires a buffer of at least %zu bytes for "
2017                       "packing %zd bytes at offset %zd "
2018                       "(actual buffer size is %zd)",
2019                       (size_t)soself->s_size + (size_t)offset,
2020                       soself->s_size,
2021                       offset,
2022                       buffer.len);
2023          PyBuffer_Release(&buffer);
2024          return NULL;
2025      }
2026  
2027      /* Call the guts */
2028      if (s_pack_internal(soself, args, 2, (char*)buffer.buf + offset, state) != 0) {
2029          PyBuffer_Release(&buffer);
2030          return NULL;
2031      }
2032  
2033      PyBuffer_Release(&buffer);
2034      Py_RETURN_NONE;
2035  }
2036  
2037  static PyObject *
s_get_format(PyStructObject * self,void * unused)2038  s_get_format(PyStructObject *self, void *unused)
2039  {
2040      return PyUnicode_FromStringAndSize(PyBytes_AS_STRING(self->s_format),
2041                                         PyBytes_GET_SIZE(self->s_format));
2042  }
2043  
2044  static PyObject *
s_get_size(PyStructObject * self,void * unused)2045  s_get_size(PyStructObject *self, void *unused)
2046  {
2047      return PyLong_FromSsize_t(self->s_size);
2048  }
2049  
2050  PyDoc_STRVAR(s_sizeof__doc__,
2051  "S.__sizeof__() -> size of S in memory, in bytes");
2052  
2053  static PyObject *
s_sizeof(PyStructObject * self,void * unused)2054  s_sizeof(PyStructObject *self, void *unused)
2055  {
2056      Py_ssize_t size;
2057      formatcode *code;
2058  
2059      size = _PyObject_SIZE(Py_TYPE(self)) + sizeof(formatcode);
2060      for (code = self->s_codes; code->fmtdef != NULL; code++)
2061          size += sizeof(formatcode);
2062      return PyLong_FromSsize_t(size);
2063  }
2064  
2065  /* List of functions */
2066  
2067  static struct PyMethodDef s_methods[] = {
2068      STRUCT_ITER_UNPACK_METHODDEF
2069      {"pack",            _PyCFunction_CAST(s_pack), METH_FASTCALL, s_pack__doc__},
2070      {"pack_into",       _PyCFunction_CAST(s_pack_into), METH_FASTCALL, s_pack_into__doc__},
2071      STRUCT_UNPACK_METHODDEF
2072      STRUCT_UNPACK_FROM_METHODDEF
2073      {"__sizeof__",      (PyCFunction)s_sizeof, METH_NOARGS, s_sizeof__doc__},
2074      {NULL,       NULL}          /* sentinel */
2075  };
2076  
2077  static PyMemberDef s_members[] = {
2078      {"__weaklistoffset__", T_PYSSIZET, offsetof(PyStructObject, weakreflist), READONLY},
2079      {NULL}  /* sentinel */
2080  };
2081  
2082  static PyGetSetDef s_getsetlist[] = {
2083      {"format", (getter)s_get_format, (setter)NULL, "struct format string", NULL},
2084      {"size", (getter)s_get_size, (setter)NULL, "struct size in bytes", NULL},
2085      {NULL} /* sentinel */
2086  };
2087  
2088  PyDoc_STRVAR(s__doc__,
2089  "Struct(fmt) --> compiled struct object\n"
2090  "\n"
2091  );
2092  
2093  static PyType_Slot PyStructType_slots[] = {
2094      {Py_tp_dealloc, s_dealloc},
2095      {Py_tp_getattro, PyObject_GenericGetAttr},
2096      {Py_tp_setattro, PyObject_GenericSetAttr},
2097      {Py_tp_doc, (void*)s__doc__},
2098      {Py_tp_traverse, s_traverse},
2099      {Py_tp_clear, s_clear},
2100      {Py_tp_methods, s_methods},
2101      {Py_tp_members, s_members},
2102      {Py_tp_getset, s_getsetlist},
2103      {Py_tp_init, Struct___init__},
2104      {Py_tp_alloc, PyType_GenericAlloc},
2105      {Py_tp_new, s_new},
2106      {Py_tp_free, PyObject_GC_Del},
2107      {0, 0},
2108  };
2109  
2110  static PyType_Spec PyStructType_spec = {
2111      "_struct.Struct",
2112      sizeof(PyStructObject),
2113      0,
2114      (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC |
2115       Py_TPFLAGS_BASETYPE | Py_TPFLAGS_IMMUTABLETYPE),
2116      PyStructType_slots
2117  };
2118  
2119  
2120  /* ---- Standalone functions  ---- */
2121  
2122  #define MAXCACHE 100
2123  
2124  static int
cache_struct_converter(PyObject * module,PyObject * fmt,PyStructObject ** ptr)2125  cache_struct_converter(PyObject *module, PyObject *fmt, PyStructObject **ptr)
2126  {
2127      PyObject * s_object;
2128      _structmodulestate *state = get_struct_state(module);
2129  
2130      if (fmt == NULL) {
2131          Py_DECREF(*ptr);
2132          *ptr = NULL;
2133          return 1;
2134      }
2135  
2136      if (state->cache == NULL) {
2137          state->cache = PyDict_New();
2138          if (state->cache == NULL)
2139              return 0;
2140      }
2141  
2142      s_object = PyDict_GetItemWithError(state->cache, fmt);
2143      if (s_object != NULL) {
2144          Py_INCREF(s_object);
2145          *ptr = (PyStructObject *)s_object;
2146          return Py_CLEANUP_SUPPORTED;
2147      }
2148      else if (PyErr_Occurred()) {
2149          return 0;
2150      }
2151  
2152      s_object = PyObject_CallOneArg(state->PyStructType, fmt);
2153      if (s_object != NULL) {
2154          if (PyDict_GET_SIZE(state->cache) >= MAXCACHE)
2155              PyDict_Clear(state->cache);
2156          /* Attempt to cache the result */
2157          if (PyDict_SetItem(state->cache, fmt, s_object) == -1)
2158              PyErr_Clear();
2159          *ptr = (PyStructObject *)s_object;
2160          return Py_CLEANUP_SUPPORTED;
2161      }
2162      return 0;
2163  }
2164  
2165  /*[clinic input]
2166  _clearcache
2167  
2168  Clear the internal cache.
2169  [clinic start generated code]*/
2170  
2171  static PyObject *
_clearcache_impl(PyObject * module)2172  _clearcache_impl(PyObject *module)
2173  /*[clinic end generated code: output=ce4fb8a7bf7cb523 input=463eaae04bab3211]*/
2174  {
2175      Py_CLEAR(get_struct_state(module)->cache);
2176      Py_RETURN_NONE;
2177  }
2178  
2179  
2180  /*[clinic input]
2181  calcsize -> Py_ssize_t
2182  
2183      format as s_object: cache_struct
2184      /
2185  
2186  Return size in bytes of the struct described by the format string.
2187  [clinic start generated code]*/
2188  
2189  static Py_ssize_t
calcsize_impl(PyObject * module,PyStructObject * s_object)2190  calcsize_impl(PyObject *module, PyStructObject *s_object)
2191  /*[clinic end generated code: output=db7d23d09c6932c4 input=96a6a590c7717ecd]*/
2192  {
2193      return s_object->s_size;
2194  }
2195  
2196  PyDoc_STRVAR(pack_doc,
2197  "pack(format, v1, v2, ...) -> bytes\n\
2198  \n\
2199  Return a bytes object containing the values v1, v2, ... packed according\n\
2200  to the format string.  See help(struct) for more on format strings.");
2201  
2202  static PyObject *
pack(PyObject * module,PyObject * const * args,Py_ssize_t nargs)2203  pack(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
2204  {
2205      PyObject *s_object = NULL;
2206      PyObject *format, *result;
2207  
2208      if (nargs == 0) {
2209          PyErr_SetString(PyExc_TypeError, "missing format argument");
2210          return NULL;
2211      }
2212      format = args[0];
2213  
2214      if (!cache_struct_converter(module, format, (PyStructObject **)&s_object)) {
2215          return NULL;
2216      }
2217      result = s_pack(s_object, args + 1, nargs - 1);
2218      Py_DECREF(s_object);
2219      return result;
2220  }
2221  
2222  PyDoc_STRVAR(pack_into_doc,
2223  "pack_into(format, buffer, offset, v1, v2, ...)\n\
2224  \n\
2225  Pack the values v1, v2, ... according to the format string and write\n\
2226  the packed bytes into the writable buffer buf starting at offset.  Note\n\
2227  that the offset is a required argument.  See help(struct) for more\n\
2228  on format strings.");
2229  
2230  static PyObject *
pack_into(PyObject * module,PyObject * const * args,Py_ssize_t nargs)2231  pack_into(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
2232  {
2233      PyObject *s_object = NULL;
2234      PyObject *format, *result;
2235  
2236      if (nargs == 0) {
2237          PyErr_SetString(PyExc_TypeError, "missing format argument");
2238          return NULL;
2239      }
2240      format = args[0];
2241  
2242      if (!cache_struct_converter(module, format, (PyStructObject **)&s_object)) {
2243          return NULL;
2244      }
2245      result = s_pack_into(s_object, args + 1, nargs - 1);
2246      Py_DECREF(s_object);
2247      return result;
2248  }
2249  
2250  /*[clinic input]
2251  unpack
2252  
2253      format as s_object: cache_struct
2254      buffer: Py_buffer
2255      /
2256  
2257  Return a tuple containing values unpacked according to the format string.
2258  
2259  The buffer's size in bytes must be calcsize(format).
2260  
2261  See help(struct) for more on format strings.
2262  [clinic start generated code]*/
2263  
2264  static PyObject *
unpack_impl(PyObject * module,PyStructObject * s_object,Py_buffer * buffer)2265  unpack_impl(PyObject *module, PyStructObject *s_object, Py_buffer *buffer)
2266  /*[clinic end generated code: output=48ddd4d88eca8551 input=05fa3b91678da727]*/
2267  {
2268      return Struct_unpack_impl(s_object, buffer);
2269  }
2270  
2271  /*[clinic input]
2272  unpack_from
2273  
2274      format as s_object: cache_struct
2275      /
2276      buffer: Py_buffer
2277      offset: Py_ssize_t = 0
2278  
2279  Return a tuple containing values unpacked according to the format string.
2280  
2281  The buffer's size, minus offset, must be at least calcsize(format).
2282  
2283  See help(struct) for more on format strings.
2284  [clinic start generated code]*/
2285  
2286  static PyObject *
unpack_from_impl(PyObject * module,PyStructObject * s_object,Py_buffer * buffer,Py_ssize_t offset)2287  unpack_from_impl(PyObject *module, PyStructObject *s_object,
2288                   Py_buffer *buffer, Py_ssize_t offset)
2289  /*[clinic end generated code: output=1042631674c6e0d3 input=6e80a5398e985025]*/
2290  {
2291      return Struct_unpack_from_impl(s_object, buffer, offset);
2292  }
2293  
2294  /*[clinic input]
2295  iter_unpack
2296  
2297      format as s_object: cache_struct
2298      buffer: object
2299      /
2300  
2301  Return an iterator yielding tuples unpacked from the given bytes.
2302  
2303  The bytes are unpacked according to the format string, like
2304  a repeated invocation of unpack_from().
2305  
2306  Requires that the bytes length be a multiple of the format struct size.
2307  [clinic start generated code]*/
2308  
2309  static PyObject *
iter_unpack_impl(PyObject * module,PyStructObject * s_object,PyObject * buffer)2310  iter_unpack_impl(PyObject *module, PyStructObject *s_object,
2311                   PyObject *buffer)
2312  /*[clinic end generated code: output=0ae50e250d20e74d input=b214a58869a3c98d]*/
2313  {
2314      return Struct_iter_unpack(s_object, buffer);
2315  }
2316  
2317  static struct PyMethodDef module_functions[] = {
2318      _CLEARCACHE_METHODDEF
2319      CALCSIZE_METHODDEF
2320      ITER_UNPACK_METHODDEF
2321      {"pack",            _PyCFunction_CAST(pack), METH_FASTCALL,   pack_doc},
2322      {"pack_into",       _PyCFunction_CAST(pack_into), METH_FASTCALL,   pack_into_doc},
2323      UNPACK_METHODDEF
2324      UNPACK_FROM_METHODDEF
2325      {NULL,       NULL}          /* sentinel */
2326  };
2327  
2328  
2329  /* Module initialization */
2330  
2331  PyDoc_STRVAR(module_doc,
2332  "Functions to convert between Python values and C structs.\n\
2333  Python bytes objects are used to hold the data representing the C struct\n\
2334  and also as format strings (explained below) to describe the layout of data\n\
2335  in the C struct.\n\
2336  \n\
2337  The optional first format char indicates byte order, size and alignment:\n\
2338    @: native order, size & alignment (default)\n\
2339    =: native order, std. size & alignment\n\
2340    <: little-endian, std. size & alignment\n\
2341    >: big-endian, std. size & alignment\n\
2342    !: same as >\n\
2343  \n\
2344  The remaining chars indicate types of args and must match exactly;\n\
2345  these can be preceded by a decimal repeat count:\n\
2346    x: pad byte (no data); c:char; b:signed byte; B:unsigned byte;\n\
2347    ?: _Bool (requires C99; if not available, char is used instead)\n\
2348    h:short; H:unsigned short; i:int; I:unsigned int;\n\
2349    l:long; L:unsigned long; f:float; d:double; e:half-float.\n\
2350  Special cases (preceding decimal count indicates length):\n\
2351    s:string (array of char); p: pascal string (with count byte).\n\
2352  Special cases (only available in native format):\n\
2353    n:ssize_t; N:size_t;\n\
2354    P:an integer type that is wide enough to hold a pointer.\n\
2355  Special case (not in native mode unless 'long long' in platform C):\n\
2356    q:long long; Q:unsigned long long\n\
2357  Whitespace between formats is ignored.\n\
2358  \n\
2359  The variable struct.error is an exception raised on errors.\n");
2360  
2361  
2362  static int
_structmodule_traverse(PyObject * module,visitproc visit,void * arg)2363  _structmodule_traverse(PyObject *module, visitproc visit, void *arg)
2364  {
2365      _structmodulestate *state = get_struct_state(module);
2366      if (state) {
2367          Py_VISIT(state->cache);
2368          Py_VISIT(state->PyStructType);
2369          Py_VISIT(state->unpackiter_type);
2370          Py_VISIT(state->StructError);
2371      }
2372      return 0;
2373  }
2374  
2375  static int
_structmodule_clear(PyObject * module)2376  _structmodule_clear(PyObject *module)
2377  {
2378      _structmodulestate *state = get_struct_state(module);
2379      if (state) {
2380          Py_CLEAR(state->cache);
2381          Py_CLEAR(state->PyStructType);
2382          Py_CLEAR(state->unpackiter_type);
2383          Py_CLEAR(state->StructError);
2384      }
2385      return 0;
2386  }
2387  
2388  static void
_structmodule_free(void * module)2389  _structmodule_free(void *module)
2390  {
2391      _structmodule_clear((PyObject *)module);
2392  }
2393  
2394  static int
_structmodule_exec(PyObject * m)2395  _structmodule_exec(PyObject *m)
2396  {
2397      _structmodulestate *state = get_struct_state(m);
2398  
2399      state->PyStructType = PyType_FromModuleAndSpec(
2400          m, &PyStructType_spec, NULL);
2401      if (state->PyStructType == NULL) {
2402          return -1;
2403      }
2404      if (PyModule_AddType(m, (PyTypeObject *)state->PyStructType) < 0) {
2405          return -1;
2406      }
2407  
2408      state->unpackiter_type = PyType_FromModuleAndSpec(
2409          m, &unpackiter_type_spec, NULL);
2410      if (state->unpackiter_type == NULL) {
2411          return -1;
2412      }
2413  
2414      /* Check endian and swap in faster functions */
2415      {
2416          const formatdef *native = native_table;
2417          formatdef *other, *ptr;
2418  #if PY_LITTLE_ENDIAN
2419          other = lilendian_table;
2420  #else
2421          other = bigendian_table;
2422  #endif
2423          /* Scan through the native table, find a matching
2424             entry in the endian table and swap in the
2425             native implementations whenever possible
2426             (64-bit platforms may not have "standard" sizes) */
2427          while (native->format != '\0' && other->format != '\0') {
2428              ptr = other;
2429              while (ptr->format != '\0') {
2430                  if (ptr->format == native->format) {
2431                      /* Match faster when formats are
2432                         listed in the same order */
2433                      if (ptr == other)
2434                          other++;
2435                      /* Only use the trick if the
2436                         size matches */
2437                      if (ptr->size != native->size)
2438                          break;
2439                      /* Skip float and double, could be
2440                         "unknown" float format */
2441                      if (ptr->format == 'd' || ptr->format == 'f')
2442                          break;
2443                      /* Skip _Bool, semantics are different for standard size */
2444                      if (ptr->format == '?')
2445                          break;
2446                      ptr->pack = native->pack;
2447                      ptr->unpack = native->unpack;
2448                      break;
2449                  }
2450                  ptr++;
2451              }
2452              native++;
2453          }
2454      }
2455  
2456      /* Add some symbolic constants to the module */
2457      state->StructError = PyErr_NewException("struct.error", NULL, NULL);
2458      if (state->StructError == NULL) {
2459          return -1;
2460      }
2461      if (PyModule_AddObjectRef(m, "error", state->StructError) < 0) {
2462          return -1;
2463      }
2464  
2465      return 0;
2466  }
2467  
2468  static PyModuleDef_Slot _structmodule_slots[] = {
2469      {Py_mod_exec, _structmodule_exec},
2470      {0, NULL}
2471  };
2472  
2473  static struct PyModuleDef _structmodule = {
2474      PyModuleDef_HEAD_INIT,
2475      .m_name = "_struct",
2476      .m_doc = module_doc,
2477      .m_size = sizeof(_structmodulestate),
2478      .m_methods = module_functions,
2479      .m_slots = _structmodule_slots,
2480      .m_traverse = _structmodule_traverse,
2481      .m_clear = _structmodule_clear,
2482      .m_free = _structmodule_free,
2483  };
2484  
2485  PyMODINIT_FUNC
PyInit__struct(void)2486  PyInit__struct(void)
2487  {
2488      return PyModuleDef_Init(&_structmodule);
2489  }
2490