Codebase list python-cbor / 0314c94
Imported Upstream version 0.1.21 Agustin Henze 8 years ago
13 changed file(s) with 2514 addition(s) and 0 deletion(s). Raw diff Collapse all Expand all
0 Metadata-Version: 1.1
1 Name: cbor
2 Version: 0.1.21
3 Summary: RFC 7049 - Concise Binary Object Representation
4 Home-page: https://bitbucket.org/bodhisnarkva/cbor
5 Author: Brian Olson
6 Author-email: bolson@bolson.org
7 License: Apache
8 Description:
9 An implementation of RFC 7049 - Concise Binary Object Representation (CBOR).
10
11 CBOR is comparable to JSON, has a superset of JSON's ability, but serializes to a binary format which is smaller and faster to generate and parse.
12
13 The two primary functions are cbor.loads() and cbor.dumps().
14
15 This library includes a C implementation which runs 3-5 times faster than the Python standard library's C-accelerated implementanion of JSON. This is also includes a 100% Python implementation.
16
17 Platform: UNKNOWN
18 Classifier: Development Status :: 4 - Beta
19 Classifier: Intended Audience :: Developers
20 Classifier: License :: OSI Approved :: Apache Software License
21 Classifier: Operating System :: OS Independent
22 Classifier: Programming Language :: Python :: 2.7
23 Classifier: Programming Language :: Python :: 3.3
24 Classifier: Programming Language :: Python :: 3.4
25 Classifier: Programming Language :: C
26 Classifier: Topic :: Software Development :: Libraries :: Python Modules
0 #ifndef CBOR_H
1 #define CBOR_H
2
3 #define CBOR_TYPE_MASK 0xE0 /* top 3 bits */
4 #define CBOR_INFO_BITS 0x1F /* low 5 bits */
5
6 #define CBOR_UINT 0x00
7 #define CBOR_NEGINT 0x20
8 #define CBOR_BYTES 0x40
9 #define CBOR_TEXT 0x60
10 #define CBOR_ARRAY 0x80
11 #define CBOR_MAP 0xA0
12 #define CBOR_TAG 0xC0
13 #define CBOR_7 0xE0 /* float and other types */
14
15 #define CBOR_ADDITIONAL_INFORMATION 0x1F
16
17 /* read the "additional information" of a tag byte which is often a
18 * small literal integer describing the length in bytes of the data
19 * item */
20 #define IS_SMALL_LITERAL(n) (((n) & 0x1f) < 24)
21 #define SMALL_LITERAL(n) ((n) & 0x1f)
22
23
24 #define CBOR_UINT8_FOLLOWS 24 // 0x18
25 #define CBOR_UINT16_FOLLOWS 25 // 0x19
26 #define CBOR_UINT32_FOLLOWS 26 // 0x1A
27 #define CBOR_UINT64_FOLLOWS 27 // 0x1B
28 #define CBOR_VAR_FOLLOWS 31 // 0x1F
29
30 #define CBOR_UINT8 (CBOR_UINT | CBOR_UINT8_FOLLOWS)
31 #define CBOR_UINT16 (CBOR_UINT | CBOR_UINT16_FOLLOWS)
32 #define CBOR_UINT32 (CBOR_UINT | CBOR_UINT32_FOLLOWS)
33 #define CBOR_UINT64 (CBOR_UINT | CBOR_UINT64_FOLLOWS)
34
35 #define CBOR_NEGINT8 (CBOR_NEGINT | CBOR_UINT8_FOLLOWS)
36 #define CBOR_NEGINT16 (CBOR_NEGINT | CBOR_UINT16_FOLLOWS)
37 #define CBOR_NEGINT32 (CBOR_NEGINT | CBOR_UINT32_FOLLOWS)
38 #define CBOR_NEGINT64 (CBOR_NEGINT | CBOR_UINT64_FOLLOWS)
39
40
41 #define CBOR_BREAK 0xFF
42
43 #define CBOR_FALSE (CBOR_7 | 20)
44 #define CBOR_TRUE (CBOR_7 | 21)
45 #define CBOR_NULL (CBOR_7 | 22)
46 #define CBOR_UNDEFINED (CBOR_7 | 23)
47
48 #define CBOR_FLOAT16 (CBOR_7 | 25)
49 #define CBOR_FLOAT32 (CBOR_7 | 26)
50 #define CBOR_FLOAT64 (CBOR_7 | 27)
51
52
53 #define CBOR_TAG_DATE_STRING (0) /* RFC3339 */
54 #define CBOR_TAG_DATE_ARRAY (1) /* any number type follows, seconds since 1970-01-01T00:00:00 UTC */
55 #define CBOR_TAG_BIGNUM (2) /* big endian byte string follows */
56 #define CBOR_TAG_NEGBIGNUM (3) /* big endian byte string follows */
57 #define CBOR_TAG_DECIMAL (4) /* [ 10^x exponent, number ] */
58 #define CBOR_TAG_BIGFLOAT (5) /* [ 2^x exponent, number ] */
59 //#define CBOR_TAG_BASE64URL (21)
60 //#define CBOR_TAG_BASE64 (22)
61 #define CBOR_TAG_BASE16 (23)
62 #define CBOR_TAG_CBOR (24) /* following byte string is embedded CBOR data */
63
64 #define CBOR_TAG_URI 32
65 //#define CBOR_TAG_BASE64URL 33
66 //#define CBOR_TAG_BASE64 34
67 #define CBOR_TAG_REGEX 35
68 #define CBOR_TAG_MIME 36 /* following text is MIME message, headers, separators and all */
69 #define CBOR_TAG_CBOR_FILEHEADER 55799 /* can open a file with 0xd9d9f7 */
70
71
72 /* Content-Type: application/cbor */
73
74
75 #endif /* CBOR_H */
0 #include "Python.h"
1
2 #include "cbor.h"
3
4 #include <math.h>
5 #include <stdint.h>
6
7 //#include <stdio.h>
8 #include <arpa/inet.h>
9
10
11 #ifndef DEBUG_LOGGING
12 // causes things to be written to stderr
13 #define DEBUG_LOGGING 0
14 //#define DEBUG_LOGGING 1
15 #endif
16
17
18 #ifdef Py_InitModule
19 // Python 2.7
20
21 #define HAS_FILE_READER 1
22 #define IS_PY3 0
23
24 #else
25
26 #define HAS_FILE_READER 0
27 #define IS_PY3 1
28
29 #endif
30
31 // Hey Look! It's a polymorphic object structure in C!
32
33 // read(, len): read len bytes and return in buffer, or NULL on error
34 // read1(, uint8_t*): read one byte and return 0 on success
35 // return_buffer(, *): release result of read(, len)
36 // delete(): destructor. free thiz and contents.
37 #define READER_FUNCTIONS \
38 void* (*read)(void* self, Py_ssize_t len); \
39 int (*read1)(void* self, uint8_t* oneByte); \
40 void (*return_buffer)(void* self, void* buffer); \
41 void (*delete)(void* self);
42
43 #define SET_READER_FUNCTIONS(thiz, clazz) (thiz)->read = clazz##_read;\
44 (thiz)->read1 = clazz##_read1;\
45 (thiz)->return_buffer = clazz##_return_buffer;\
46 (thiz)->delete = clazz##_delete;
47
48 typedef struct _Reader {
49 READER_FUNCTIONS;
50 } Reader;
51
52 static Reader* NewBufferReader(PyObject* ob);
53 static Reader* NewObjectReader(PyObject* ob);
54 #if HAS_FILE_READER
55 static Reader* NewFileReader(PyObject* ob);
56 #endif
57
58
59 static PyObject* loads_tag(Reader* rin, uint64_t aux);
60 static int loads_kv(PyObject* out, Reader* rin);
61
62 typedef struct VarBufferPart {
63 void* start;
64 uint64_t len;
65 struct VarBufferPart* next;
66 } VarBufferPart;
67
68
69 static int logprintf(const char* fmt, ...) {
70 va_list ap;
71 int ret;
72 va_start(ap, fmt);
73 #if DEBUG_LOGGING
74 ret = vfprintf(stderr, fmt, ap);
75 #else
76 ret = 0;
77 #endif
78 va_end(ap);
79 return ret;
80 }
81
82 // TODO: portably work this out at compile time
83 static int _is_big_endian = 0;
84
85 static int is_big_endian(void) {
86 uint32_t val = 1234;
87 _is_big_endian = val == htonl(val);
88 //logprintf("is_big_endian=%d\n", _is_big_endian);
89 return _is_big_endian;
90 }
91
92
93 PyObject* decodeFloat16(Reader* rin) {
94 // float16 parsing adapted from example code in spec
95 uint8_t hibyte, lobyte;// = raw[pos];
96 int err;
97 int exp;
98 int mant;
99 double val;
100
101 err = rin->read1(rin, &hibyte);
102 if (err) { logprintf("fail in float16[0]\n"); return NULL; }
103 err = rin->read1(rin, &lobyte);
104 if (err) { logprintf("fail in float16[1]\n"); return NULL; }
105
106 exp = (hibyte >> 2) & 0x1f;
107 mant = ((hibyte & 0x3) << 8) | lobyte;
108 if (exp == 0) {
109 val = ldexp(mant, -24);
110 } else if (exp != 31) {
111 val = ldexp(mant + 1024, exp - 25);
112 } else {
113 val = mant == 0 ? INFINITY : NAN;
114 }
115 if (hibyte & 0x80) {
116 val = -val;
117 }
118 return PyFloat_FromDouble(val);
119 }
120 PyObject* decodeFloat32(Reader* rin) {
121 float val;
122 uint8_t* raw = rin->read(rin, 4);
123 if (!raw) { logprintf("fail in float32\n"); return NULL; }
124 if (_is_big_endian) {
125 // easy!
126 val = *((float*)raw);
127 } else {
128 uint8_t* dest = (uint8_t*)(&val);
129 dest[3] = raw[0];
130 dest[2] = raw[1];
131 dest[1] = raw[2];
132 dest[0] = raw[3];
133 }
134 rin->return_buffer(rin, raw);
135 return PyFloat_FromDouble(val);
136 }
137 PyObject* decodeFloat64(Reader* rin) {
138 int si;
139 uint64_t aux = 0;
140 uint8_t* raw = rin->read(rin, 8);
141 if (!raw) { logprintf("fail in float64\n"); return NULL; }
142 for (si = 0; si < 8; si++) {
143 aux = aux << 8;
144 aux |= raw[si];
145 }
146 rin->return_buffer(rin, raw);
147 return PyFloat_FromDouble(*((double*)(&aux)));
148 }
149
150 // parse following int value into *auxP
151 // return 0 on success, -1 on fail
152 static int handle_info_bits(Reader* rin, uint8_t cbor_info, uint64_t* auxP) {
153 uint64_t aux;
154
155 if (cbor_info <= 23) {
156 // literal value <=23
157 aux = cbor_info;
158 } else if (cbor_info == CBOR_UINT8_FOLLOWS) {
159 uint8_t taux;
160 if (rin->read1(rin, &taux)) { logprintf("fail in uint8\n"); return -1; }
161 aux = taux;
162 } else if (cbor_info == CBOR_UINT16_FOLLOWS) {
163 uint8_t hibyte, lobyte;
164 if (rin->read1(rin, &hibyte)) { logprintf("fail in uint16[0]\n"); return -1; }
165 if (rin->read1(rin, &lobyte)) { logprintf("fail in uint16[1]\n"); return -1; }
166 aux = (hibyte << 8) | lobyte;
167 } else if (cbor_info == CBOR_UINT32_FOLLOWS) {
168 uint8_t* raw = (uint8_t*)rin->read(rin, 4);
169 if (!raw) { logprintf("fail in uint32[1]\n"); return -1; }
170 aux =
171 (((uint64_t)raw[0]) << 24) |
172 (((uint64_t)raw[1]) << 16) |
173 (((uint64_t)raw[2]) << 8) |
174 ((uint64_t)raw[3]);
175 rin->return_buffer(rin, raw);
176 } else if (cbor_info == CBOR_UINT64_FOLLOWS) {
177 int si;
178 uint8_t* raw = (uint8_t*)rin->read(rin, 8);
179 if (!raw) { logprintf("fail in uint64[1]\n"); return -1; }
180 aux = 0;
181 for (si = 0; si < 8; si++) {
182 aux = aux << 8;
183 aux |= raw[si];
184 }
185 rin->return_buffer(rin, raw);
186 } else {
187 aux = 0;
188 }
189 *auxP = aux;
190 return 0;
191 }
192
193 static PyObject* inner_loads_c(Reader* rin, uint8_t c);
194
195 static PyObject* inner_loads(Reader* rin) {
196 uint8_t c;
197 int err;
198
199 err = rin->read1(rin, &c);
200 if (err) { logprintf("fail in loads tag\n"); return NULL; }
201 return inner_loads_c(rin, c);
202 }
203
204 PyObject* inner_loads_c(Reader* rin, uint8_t c) {
205 uint8_t cbor_type;
206 uint8_t cbor_info;
207 uint64_t aux;
208
209 cbor_type = c & CBOR_TYPE_MASK;
210 cbor_info = c & CBOR_INFO_BITS;
211
212 #if 0
213 if (pos > len) {
214 PyErr_SetString(PyExc_ValueError, "misparse, token went longer than buffer");
215 return NULL;
216 }
217
218 pos += 1;
219 #endif
220
221 if (cbor_type == CBOR_7) {
222 if (cbor_info == CBOR_UINT16_FOLLOWS) { // float16
223 return decodeFloat16(rin);
224 } else if (cbor_info == CBOR_UINT32_FOLLOWS) { // float32
225 return decodeFloat32(rin);
226 } else if (cbor_info == CBOR_UINT64_FOLLOWS) { // float64
227 return decodeFloat64(rin);
228 }
229 // not a float, fall through to other CBOR_7 interpretations
230 }
231 if (handle_info_bits(rin, cbor_info, &aux)) { logprintf("info bits failed\n"); return NULL; }
232
233 PyObject* out = NULL;
234 switch (cbor_type) {
235 case CBOR_UINT:
236 out = PyLong_FromUnsignedLongLong(aux);
237 if (out == NULL) {
238 PyErr_SetString(PyExc_RuntimeError, "unknown error decoding UINT");
239 }
240 return out;
241 case CBOR_NEGINT:
242 if (aux > 0x7fffffffffffffff) {
243 PyObject* bignum = PyLong_FromUnsignedLongLong(aux);
244 PyObject* minusOne = PyLong_FromLong(-1);
245 out = PyNumber_Subtract(minusOne, bignum);
246 Py_DECREF(minusOne);
247 Py_DECREF(bignum);
248 } else {
249 out = PyLong_FromLongLong((long long)(((long long)-1) - aux));
250 }
251 if (out == NULL) {
252 PyErr_SetString(PyExc_RuntimeError, "unknown error decoding NEGINT");
253 }
254 return out;
255 case CBOR_BYTES:
256 if (cbor_info == CBOR_VAR_FOLLOWS) {
257 size_t total = 0;
258 VarBufferPart* parts = NULL;
259 VarBufferPart* parts_tail = NULL;
260 uint8_t sc;
261 if (rin->read1(rin, &sc)) { logprintf("r1 fail in var bytes tag\n"); return NULL; }
262 while (sc != CBOR_BREAK) {
263 uint8_t scbor_type = sc & CBOR_TYPE_MASK;
264 uint8_t scbor_info = sc & CBOR_INFO_BITS;
265 uint64_t saux;
266 void* blob;
267
268 if (scbor_type != CBOR_BYTES) {
269 PyErr_Format(PyExc_ValueError, "expected subordinate BYTES block under VAR BYTES, but got %x", scbor_type);
270 return NULL;
271 }
272 if(handle_info_bits(rin, scbor_info, &saux)) { logprintf("var bytes sub infobits failed\n"); return NULL; }
273 blob = rin->read(rin, saux);
274 if (!blob) { logprintf("var bytes sub bytes read failed\n"); return NULL; }
275 if (parts_tail == NULL) {
276 parts = parts_tail = (VarBufferPart*)PyMem_Malloc(sizeof(VarBufferPart) + saux);
277 } else {
278 parts_tail->next = (VarBufferPart*)PyMem_Malloc(sizeof(VarBufferPart) + saux);
279 parts_tail = parts_tail->next;
280 }
281 parts_tail->start = (void*)(parts_tail + 1);
282 memcpy(parts_tail->start, blob, saux);
283 rin->return_buffer(rin, blob);
284 parts_tail->len = saux;
285 parts_tail->next = NULL;
286 total += saux;
287 if (rin->read1(rin, &sc)) { logprintf("r1 fail in var bytes tag\n"); return NULL; }
288 }
289 // Done
290 {
291 uint8_t* allbytes = (uint8_t*)PyMem_Malloc(total);
292 uintptr_t op = 0;
293 while (parts != NULL) {
294 VarBufferPart* next;
295 memcpy(allbytes + op, parts->start, parts->len);
296 op += parts->len;
297 next = parts->next;
298 PyMem_Free(parts);
299 parts = next;
300 }
301 out = PyBytes_FromStringAndSize((char*)allbytes, total);
302 PyMem_Free(allbytes);
303 }
304 if (out == NULL) {
305 PyErr_SetString(PyExc_RuntimeError, "unknown error decoding VAR BYTES");
306 }
307 } else {
308 void* raw;
309 if (aux == 0) {
310 static void* empty_string = "";
311 raw = empty_string;
312 } else {
313 raw = rin->read(rin, aux);
314 if (!raw) { logprintf("bytes read failed\n"); return NULL; }
315 }
316 out = PyBytes_FromStringAndSize(raw, (Py_ssize_t)aux);
317 if (out == NULL) {
318 PyErr_SetString(PyExc_RuntimeError, "unknown error decoding BYTES");
319 }
320 if (aux != 0) {
321 rin->return_buffer(rin, raw);
322 }
323 }
324 return out;
325 case CBOR_TEXT:
326 if (cbor_info == CBOR_VAR_FOLLOWS) {
327 PyObject* parts = PyList_New(0);
328 PyObject* joiner = PyUnicode_FromString("");
329 uint8_t sc;
330 if (rin->read1(rin, &sc)) { logprintf("r1 fail in var text tag\n"); return NULL; }
331 while (sc != CBOR_BREAK) {
332 PyObject* subitem = inner_loads_c(rin, sc);
333 if (subitem == NULL) { logprintf("fail in var text subitem\n"); return NULL; }
334 PyList_Append(parts, subitem);
335 Py_DECREF(subitem);
336 if (rin->read1(rin, &sc)) { logprintf("r1 fail in var text tag\n"); return NULL; }
337 }
338 // Done
339 out = PyUnicode_Join(joiner, parts);
340 Py_DECREF(joiner);
341 Py_DECREF(parts);
342 if (out == NULL) {
343 PyErr_SetString(PyExc_RuntimeError, "unknown error decoding VAR TEXT");
344 }
345 } else {
346 void* raw;
347 if (aux == 0) {
348 static void* empty_string = "";
349 raw = empty_string;
350 } else {
351 raw = rin->read(rin, aux);
352 if (!raw) { logprintf("read text failed\n"); return NULL; }
353 }
354 out = PyUnicode_FromStringAndSize((char*)raw, (Py_ssize_t)aux);
355 if (out == NULL) {
356 PyErr_SetString(PyExc_RuntimeError, "unknown error decoding TEXT");
357 }
358 if (aux != 0) {
359 rin->return_buffer(rin, raw);
360 }
361 }
362 return out;
363 case CBOR_ARRAY:
364 if (cbor_info == CBOR_VAR_FOLLOWS) {
365 uint8_t sc;
366 out = PyList_New(0);
367 if (rin->read1(rin, &sc)) { logprintf("r1 fail in var array tag\n"); return NULL; }
368 while (sc != CBOR_BREAK) {
369 PyObject* subitem = inner_loads_c(rin, sc);
370 if (subitem == NULL) { logprintf("fail in var array subitem\n"); return NULL; }
371 PyList_Append(out, subitem);
372 Py_DECREF(subitem);
373 if (rin->read1(rin, &sc)) { logprintf("r1 fail in var array tag\n"); return NULL; }
374 }
375 // Done
376 if (out == NULL) {
377 PyErr_SetString(PyExc_RuntimeError, "unknown error decoding VAR ARRAY");
378 }
379 } else {
380 unsigned int i;
381 out = PyList_New((Py_ssize_t)aux);
382 for (i = 0; i < aux; i++) {
383 PyObject* subitem = inner_loads(rin);
384 if (subitem == NULL) { logprintf("array subitem[%d] (of %d) failed\n", i, aux); return NULL; }
385 PyList_SetItem(out, (Py_ssize_t)i, subitem);
386 // PyList_SetItem became the owner of the reference count of subitem, we don't need to DECREF it
387 }
388 if (out == NULL) {
389 PyErr_SetString(PyExc_RuntimeError, "unknown error decoding ARRAY");
390 }
391 }
392 return out;
393 case CBOR_MAP:
394 out = PyDict_New();
395 if (cbor_info == CBOR_VAR_FOLLOWS) {
396 uint8_t sc;
397 if (rin->read1(rin, &sc)) { logprintf("r1 fail in var map tag\n"); return NULL; }
398 while (sc != CBOR_BREAK) {
399 PyObject* key = inner_loads_c(rin, sc);
400 PyObject* value;
401 if (key == NULL) { logprintf("var map key fail\n"); return NULL; }
402 value = inner_loads(rin);
403 if (value == NULL) { logprintf("var map val vail\n"); return NULL; }
404 PyDict_SetItem(out, key, value);
405 Py_DECREF(key);
406 Py_DECREF(value);
407
408 if (rin->read1(rin, &sc)) { logprintf("r1 fail in var map tag\n"); return NULL; }
409 }
410 if (out == NULL) {
411 PyErr_SetString(PyExc_RuntimeError, "unknown error decoding VAR MAP");
412 }
413 } else {
414 unsigned int i;
415 for (i = 0; i < aux; i++) {
416 if (loads_kv(out, rin) != 0) {
417 logprintf("map kv[%d] failed\n", i);
418 return NULL;
419 }
420 }
421 if (out == NULL) {
422 PyErr_SetString(PyExc_RuntimeError, "unknown error decoding MAP");
423 }
424 }
425 return out;
426 case CBOR_TAG:
427 return loads_tag(rin, aux);
428 case CBOR_7:
429 if (aux == 20) {
430 out = Py_False;
431 Py_INCREF(out);
432 } else if (aux == 21) {
433 out = Py_True;
434 Py_INCREF(out);
435 } else if (aux == 22) {
436 out = Py_None;
437 Py_INCREF(out);
438 } else if (aux == 23) {
439 // js `undefined`, closest is py None
440 out = Py_None;
441 Py_INCREF(out);
442 }
443 if (out == NULL) {
444 PyErr_Format(PyExc_ValueError, "unknown section 7 marker %02x, aux=%llu", c, aux);
445 }
446 return out;
447 default:
448 PyErr_Format(PyExc_RuntimeError, "unknown cbor marker %02x", c);
449 return NULL;
450 }
451 PyErr_SetString(PyExc_RuntimeError, "cbor library internal error moof!");
452 return NULL;
453 }
454
455 static int loads_kv(PyObject* out, Reader* rin) {
456 PyObject* key = inner_loads(rin);
457 PyObject* value;
458 if (key == NULL) { logprintf("map key fail\n"); return -1; }
459 value = inner_loads(rin);
460 if (value == NULL) { logprintf("map val fail\n"); return -1; }
461 PyDict_SetItem(out, key, value);
462 Py_DECREF(key);
463 Py_DECREF(value);
464 return 0;
465 }
466
467 static PyObject* loads_bignum(Reader* rin, uint8_t c) {
468 PyObject* out = NULL;
469
470 uint8_t bytes_info = c & CBOR_INFO_BITS;
471 if (bytes_info < 24) {
472 int i;
473 PyObject* eight = PyLong_FromLong(8);
474 out = PyLong_FromLong(0);
475 for (i = 0; i < bytes_info; i++) {
476 // TODO: is this leaking like crazy?
477 PyObject* curbyte;
478 PyObject* tout = PyNumber_Lshift(out, eight);
479 Py_DECREF(out);
480 out = tout;
481 uint8_t cb;
482 if (rin->read1(rin, &cb)) {
483 logprintf("r1 fail in bignum %d/%d\n", i, bytes_info);
484 Py_DECREF(eight);
485 Py_DECREF(out);
486 return NULL;
487 }
488 curbyte = PyLong_FromLong(cb);
489 tout = PyNumber_Or(out, curbyte);
490 Py_DECREF(curbyte);
491 Py_DECREF(out);
492 out = tout;
493 }
494 Py_DECREF(eight);
495 return out;
496 } else {
497 PyErr_Format(PyExc_NotImplementedError, "TODO: TAG BIGNUM for bigger bignum bytes_info=%d, len(ull)=%lu\n", bytes_info, sizeof(unsigned long long));
498 return NULL;
499 }
500 }
501
502
503 // returns a PyObject for cbor.cbor.Tag
504 // Returned PyObject* is a BORROWED reference from the module dict
505 static PyObject* getCborTagClass(void) {
506 PyObject* cbor_module = PyImport_ImportModule("cbor.cbor");
507 PyObject* moddict = PyModule_GetDict(cbor_module);
508 PyObject* tag_class = PyDict_GetItemString(moddict, "Tag");
509 // moddict and tag_class are 'borrowed reference'
510 Py_DECREF(cbor_module);
511
512 return tag_class;
513 }
514
515
516 static PyObject* loads_tag(Reader* rin, uint64_t aux) {
517 PyObject* out = NULL;
518 // return an object CBORTag(tagnum, nextob)
519 if (aux == CBOR_TAG_BIGNUM) {
520 // If the next object is bytes, interpret it here without making a PyObject for it.
521 uint8_t sc;
522 if (rin->read1(rin, &sc)) { logprintf("r1 fail in bignum tag\n"); return NULL; }
523 if ((sc & CBOR_TYPE_MASK) == CBOR_BYTES) {
524 return loads_bignum(rin, sc);
525 } else {
526 PyErr_Format(PyExc_ValueError, "TAG BIGNUM not followed by bytes but %02x", sc);
527 return NULL;
528 }
529 PyErr_Format(PyExc_ValueError, "TODO: WRITEME CBOR TAG BIGNUM %02x ...\n", sc);
530 return NULL;
531 } else if (aux == CBOR_TAG_NEGBIGNUM) {
532 // If the next object is bytes, interpret it here without making a PyObject for it.
533 uint8_t sc;
534 if (rin->read1(rin, &sc)) { logprintf("r1 fail in negbignum tag\n"); return NULL; }
535 if ((sc & CBOR_TYPE_MASK) == CBOR_BYTES) {
536 out = loads_bignum(rin, sc);
537 if (out == NULL) { logprintf("loads_bignum fail inside TAG_NEGBIGNUM\n"); return NULL; }
538 PyObject* minusOne = PyLong_FromLong(-1);
539 PyObject* tout = PyNumber_Subtract(minusOne, out);
540 Py_DECREF(minusOne);
541 Py_DECREF(out);
542 out = tout;
543 return out;
544 } else {
545 PyErr_Format(PyExc_ValueError, "TAG NEGBIGNUM not followed by bytes but %02x", sc);
546 return NULL;
547 }
548 PyErr_Format(PyExc_ValueError, "TODO: WRITEME CBOR TAG NEGBIGNUM %02x ...\n", sc);
549 return NULL;
550 }
551 out = inner_loads(rin);
552 if (out == NULL) { return NULL; }
553 {
554 PyObject* tag_class = getCborTagClass();
555 PyObject* args = Py_BuildValue("(K,O)", aux, out);
556 PyObject* tout = PyObject_CallObject(tag_class, args);
557 Py_DECREF(args);
558 Py_DECREF(out);
559 // tag_class was just a borrowed reference
560 out = tout;
561 }
562 return out;
563 }
564
565
566 static PyObject*
567 cbor_loads(PyObject* noself, PyObject* args) {
568 PyObject* ob;
569 is_big_endian();
570 if (PyType_IsSubtype(Py_TYPE(args), &PyList_Type)) {
571 ob = PyList_GetItem(args, 0);
572 } else if (PyType_IsSubtype(Py_TYPE(args), &PyTuple_Type)) {
573 ob = PyTuple_GetItem(args, 0);
574 } else {
575 PyErr_Format(PyExc_ValueError, "args not list or tuple: %R\n", args);
576 return NULL;
577 }
578
579 if (ob == Py_None) {
580 PyErr_SetString(PyExc_ValueError, "got None for buffer to decode in loads");
581 return NULL;
582 }
583
584 {
585 PyObject* out = NULL;
586 Reader* r = NewBufferReader(ob);
587 if (!r) {
588 return NULL;
589 }
590 out = inner_loads(r);
591 r->delete(r);
592 return out;
593 }
594 }
595
596
597 #if HAS_FILE_READER
598
599 typedef struct _FileReader {
600 READER_FUNCTIONS;
601 FILE* fin;
602 void* dst;
603 Py_ssize_t dst_size;
604 Py_ssize_t read_count;
605 } FileReader;
606
607 // read from a python builtin file which contains a C FILE*
608 static void* FileReader_read(void* self, Py_ssize_t len) {
609 FileReader* thiz = (FileReader*)self;
610 Py_ssize_t rtotal = 0;
611 uintptr_t opos;
612 //logprintf("file read %d\n", len);
613 if (len > thiz->dst_size) {
614 thiz->dst = PyMem_Realloc(thiz->dst, len);
615 thiz->dst_size = len;
616 } else if ((thiz->dst_size > (128 * 1024)) && (len < 4096)) {
617 PyMem_Free(thiz->dst);
618 thiz->dst = PyMem_Malloc(len);
619 thiz->dst_size = len;
620 }
621 opos = (uintptr_t)(thiz->dst);
622 while (1) {
623 size_t rlen = fread((void*)opos, 1, len, thiz->fin);
624 if (rlen == 0) {
625 // file isn't going to give any more
626 PyErr_Format(PyExc_ValueError, "only got %zd bytes with %zd stil to read from file", rtotal, len);
627 PyMem_Free(thiz->dst);
628 thiz->dst = NULL;
629 thiz->dst_size = 0;
630 return NULL;
631 }
632 thiz->read_count += rlen;
633 rtotal += rlen;
634 opos += rlen;
635 len -= rlen;
636 if (rtotal >= len) {
637 if (thiz->dst == NULL) {
638 PyErr_SetString(PyExc_RuntimeError, "known error in file reader, NULL dst");
639 return NULL;
640 }
641 return thiz->dst;
642 }
643 }
644 }
645 static int FileReader_read1(void* self, uint8_t* oneByte) {
646 FileReader* thiz = (FileReader*)self;
647 size_t didread = fread((void*)oneByte, 1, 1, thiz->fin);
648 if (didread == 0) {
649 logprintf("failed to read 1 from file\n");
650 PyErr_SetString(PyExc_ValueError, "got nothing reading 1 from file");
651 return -1;
652 }
653 thiz->read_count++;
654 return 0;
655 }
656 static void FileReader_return_buffer(void* self, void* buffer) {
657 // Nothing to do, we hold onto the buffer and maybe reuse it for next read
658 }
659 static void FileReader_delete(void* self) {
660 FileReader* thiz = (FileReader*)self;
661 if (thiz->dst) {
662 PyMem_Free(thiz->dst);
663 }
664 PyMem_Free(thiz);
665 }
666 static Reader* NewFileReader(PyObject* ob) {
667 FileReader* fr = (FileReader*)PyMem_Malloc(sizeof(FileReader));
668 if (fr == NULL) {
669 PyErr_SetString(PyExc_MemoryError, "failed to allocate FileReader");
670 return NULL;
671 }
672 fr->fin = PyFile_AsFile(ob);
673 if (fr->fin == NULL) {
674 PyErr_SetString(PyExc_RuntimeError, "PyFile_AsFile NULL");
675 PyMem_Free(fr);
676 return NULL;
677 }
678 fr->dst = NULL;
679 fr->dst_size = 0;
680 fr->read_count = 0;
681 SET_READER_FUNCTIONS(fr, FileReader);
682 return (Reader*)fr;
683 }
684
685 #endif /* Python 2.7 FileReader */
686
687
688 typedef struct _ObjectReader {
689 READER_FUNCTIONS;
690 PyObject* ob;
691
692 // We got one object with all the bytes neccessary, and need to
693 // DECREF it later.
694 PyObject* retval;
695 void* bytes;
696
697 // OR, we got several objects, we DECREFed them as we went, and
698 // need to Free() this buffer at the end.
699 void* dst;
700
701 Py_ssize_t read_count;
702 int exception_is_external;
703 } ObjectReader;
704
705 // read from a python file-like object which has a .read(n) method
706 static void* ObjectReader_read(void* context, Py_ssize_t len) {
707 ObjectReader* thiz = (ObjectReader*)context;
708 Py_ssize_t rtotal = 0;
709 uintptr_t opos = 0;
710 //logprintf("ob read %d\n", len);
711 assert(!thiz->dst);
712 assert(!thiz->bytes);
713 while (rtotal < len) {
714 PyObject* retval = PyObject_CallMethod(thiz->ob, "read", "n", len - rtotal, NULL);
715 Py_ssize_t rlen;
716 if (retval == NULL) {
717 thiz->exception_is_external = 1;
718 logprintf("exception in object.read()\n");
719 return NULL;
720 }
721 if (!PyBytes_Check(retval)) {
722 logprintf("object.read() is not bytes\n");
723 PyErr_SetString(PyExc_ValueError, "expected ob.read() to return a bytes object\n");
724 Py_DECREF(retval);
725 return NULL;
726 }
727 rlen = PyBytes_Size(retval);
728 thiz->read_count += rlen;
729 if (rlen > len - rtotal) {
730 logprintf("object.read() is too much!\n");
731 PyErr_Format(PyExc_ValueError, "ob.read() returned %ld bytes but only wanted %lu\n", rlen, len - rtotal);
732 Py_DECREF(retval);
733 return NULL;
734 }
735 if (rlen == len) {
736 // best case! All in one call to read()
737 // We _keep_ a reference to retval until later.
738 thiz->retval = retval;
739 thiz->bytes = PyBytes_AsString(retval);
740 assert(thiz->bytes);
741 thiz->dst = NULL;
742 opos = 0;
743 return thiz->bytes;
744 }
745 if (thiz->dst == NULL) {
746 thiz->dst = PyMem_Malloc(len);
747 opos = (uintptr_t)thiz->dst;
748 }
749 // else, not enough all in one go
750 memcpy((void*)opos, PyBytes_AsString(retval), rlen);
751 Py_DECREF(retval);
752 opos += rlen;
753 rtotal += rlen;
754 }
755 assert(thiz->dst);
756 return thiz->dst;
757 }
758 static int ObjectReader_read1(void* self, uint8_t* oneByte) {
759 ObjectReader* thiz = (ObjectReader*)self;
760 PyObject* retval = PyObject_CallMethod(thiz->ob, "read", "i", 1, NULL);
761 Py_ssize_t rlen;
762 if (retval == NULL) {
763 thiz->exception_is_external = 1;
764 //logprintf("call ob read(1) failed\n");
765 return -1;
766 }
767 if (!PyBytes_Check(retval)) {
768 PyErr_SetString(PyExc_ValueError, "expected ob.read() to return a bytes object\n");
769 return -1;
770 }
771 rlen = PyBytes_Size(retval);
772 thiz->read_count += rlen;
773 if (rlen > 1) {
774 PyErr_Format(PyExc_ValueError, "TODO: raise exception: WAT ob.read() returned %ld bytes but only wanted 1\n", rlen);
775 return -1;
776 }
777 if (rlen == 1) {
778 *oneByte = PyBytes_AsString(retval)[0];
779 Py_DECREF(retval);
780 return 0;
781 }
782 PyErr_SetString(PyExc_ValueError, "got nothing reading 1");
783 return -1;
784 }
785 static void ObjectReader_return_buffer(void* context, void* buffer) {
786 ObjectReader* thiz = (ObjectReader*)context;
787 if (buffer == thiz->bytes) {
788 Py_DECREF(thiz->retval);
789 thiz->retval = NULL;
790 thiz->bytes = NULL;
791 } else if (buffer == thiz->dst) {
792 PyMem_Free(thiz->dst);
793 thiz->dst = NULL;
794 } else {
795 logprintf("TODO: raise exception, could not release buffer %p, wanted dst=%p or bytes=%p\n", buffer, thiz->dst, thiz->bytes);
796 }
797 }
798 static void ObjectReader_delete(void* context) {
799 ObjectReader* thiz = (ObjectReader*)context;
800 if (thiz->retval != NULL) {
801 Py_DECREF(thiz->retval);
802 }
803 if (thiz->dst != NULL) {
804 PyMem_Free(thiz->dst);
805 }
806 PyMem_Free(thiz);
807 }
808 static Reader* NewObjectReader(PyObject* ob) {
809 ObjectReader* r = (ObjectReader*)PyMem_Malloc(sizeof(ObjectReader));
810 r->ob = ob;
811 r->retval = NULL;
812 r->bytes = NULL;
813 r->dst = NULL;
814 r->read_count = 0;
815 r->exception_is_external = 0;
816 SET_READER_FUNCTIONS(r, ObjectReader);
817 return (Reader*)r;
818 }
819
820 typedef struct _BufferReader {
821 READER_FUNCTIONS;
822 uint8_t* raw;
823 Py_ssize_t len;
824 uintptr_t pos;
825 } BufferReader;
826
827 // read from a buffer, aka loads()
828 static void* BufferReader_read(void* context, Py_ssize_t len) {
829 BufferReader* thiz = (BufferReader*)context;
830 //logprintf("br %p %d (%d)\n", thiz, len, thiz->len);
831 if (len <= thiz->len) {
832 void* out = (void*)thiz->pos;
833 thiz->pos += len;
834 thiz->len -= len;
835 assert(out);
836 return out;
837 }
838 PyErr_Format(PyExc_ValueError, "buffer read for %zd but only have %zd\n", len, thiz->len);
839 return NULL;
840 }
841 static int BufferReader_read1(void* self, uint8_t* oneByte) {
842 BufferReader* thiz = (BufferReader*)self;
843 //logprintf("br %p _1_ (%d)\n", thiz, thiz->len);
844 if (thiz->len <= 0) {
845 PyErr_SetString(PyExc_LookupError, "buffer exhausted");
846 return -1;
847 }
848 *oneByte = *((uint8_t*)thiz->pos);
849 thiz->pos += 1;
850 thiz->len -= 1;
851 return 0;
852 }
853 static void BufferReader_return_buffer(void* context, void* buffer) {
854 // nothing to do
855 }
856 static void BufferReader_delete(void* context) {
857 BufferReader* thiz = (BufferReader*)context;
858 PyMem_Free(thiz);
859 }
860 static Reader* NewBufferReader(PyObject* ob) {
861 BufferReader* r = (BufferReader*)PyMem_Malloc(sizeof(BufferReader));
862 SET_READER_FUNCTIONS(r, BufferReader);
863 if (PyByteArray_Check(ob)) {
864 r->raw = (uint8_t*)PyByteArray_AsString(ob);
865 r->len = PyByteArray_Size(ob);
866 } else if (PyBytes_Check(ob)) {
867 r->raw = (uint8_t*)PyBytes_AsString(ob);
868 r->len = PyBytes_Size(ob);
869 } else {
870 PyErr_SetString(PyExc_ValueError, "input of unknown type not bytes or bytearray");
871 return NULL;
872 }
873 r->pos = (uintptr_t)r->raw;
874 if (r->len == 0) {
875 PyErr_SetString(PyExc_ValueError, "got zero length string in loads");
876 return NULL;
877 }
878 if (r->raw == NULL) {
879 PyErr_SetString(PyExc_ValueError, "got NULL buffer for string");
880 return NULL;
881 }
882 //logprintf("NBR(%llu, %ld)\n", r->pos, r->len);
883 return (Reader*)r;
884 }
885
886
887 static PyObject*
888 cbor_load(PyObject* noself, PyObject* args) {
889 PyObject* ob;
890 Reader* reader;
891 is_big_endian();
892 if (PyType_IsSubtype(Py_TYPE(args), &PyList_Type)) {
893 ob = PyList_GetItem(args, 0);
894 } else if (PyType_IsSubtype(Py_TYPE(args), &PyTuple_Type)) {
895 ob = PyTuple_GetItem(args, 0);
896 } else {
897 PyErr_Format(PyExc_ValueError, "args not list or tuple: %R\n", args);
898 return NULL;
899 }
900
901 if (ob == Py_None) {
902 PyErr_SetString(PyExc_ValueError, "got None for buffer to decode in loads");
903 return NULL;
904 }
905 PyObject* retval;
906 #if HAS_FILE_READER
907 if (PyFile_Check(ob)) {
908 reader = NewFileReader(ob);
909 if (reader == NULL) { return NULL; }
910 retval = inner_loads(reader);
911 if ((retval == NULL) &&
912 (((FileReader*)reader)->read_count == 0) &&
913 (feof(((FileReader*)reader)->fin) != 0)) {
914 // never got anything, started at EOF
915 PyErr_Clear();
916 PyErr_SetString(PyExc_EOFError, "read nothing, apparent EOF");
917 }
918 reader->delete(reader);
919 } else
920 #endif
921 {
922 reader = NewObjectReader(ob);
923 retval = inner_loads(reader);
924 if ((retval == NULL) &&
925 (!((ObjectReader*)reader)->exception_is_external) &&
926 ((ObjectReader*)reader)->read_count == 0) {
927 // never got anything, assume EOF
928 PyErr_Clear();
929 PyErr_SetString(PyExc_EOFError, "read nothing, apparent EOF");
930 }
931 reader->delete(reader);
932 }
933 return retval;
934 }
935
936
937 static void tag_u64_out(uint8_t cbor_type, uint64_t aux, uint8_t* out, uintptr_t* posp) {
938 uintptr_t pos = *posp;
939 if (out != NULL) {
940 out[pos] = cbor_type | CBOR_UINT64_FOLLOWS;
941 out[pos+1] = (aux >> 56) & 0x0ff;
942 out[pos+2] = (aux >> 48) & 0x0ff;
943 out[pos+3] = (aux >> 40) & 0x0ff;
944 out[pos+4] = (aux >> 32) & 0x0ff;
945 out[pos+5] = (aux >> 24) & 0x0ff;
946 out[pos+6] = (aux >> 16) & 0x0ff;
947 out[pos+7] = (aux >> 8) & 0x0ff;
948 out[pos+8] = aux & 0x0ff;
949 }
950 pos += 9;
951 *posp = pos;
952 }
953
954
955 static void tag_aux_out(uint8_t cbor_type, uint64_t aux, uint8_t* out, uintptr_t* posp) {
956 uintptr_t pos = *posp;
957 if (aux <= 23) {
958 // tiny literal
959 if (out != NULL) {
960 out[pos] = cbor_type | aux;
961 }
962 pos += 1;
963 } else if (aux <= 0x0ff) {
964 // one byte value
965 if (out != NULL) {
966 out[pos] = cbor_type | CBOR_UINT8_FOLLOWS;
967 out[pos+1] = aux;
968 }
969 pos += 2;
970 } else if (aux <= 0x0ffff) {
971 // two byte value
972 if (out != NULL) {
973 out[pos] = cbor_type | CBOR_UINT16_FOLLOWS;
974 out[pos+1] = (aux >> 8) & 0x0ff;
975 out[pos+2] = aux & 0x0ff;
976 }
977 pos += 3;
978 } else if (aux <= 0x0ffffffffL) {
979 // four byte value
980 if (out != NULL) {
981 out[pos] = cbor_type | CBOR_UINT32_FOLLOWS;
982 out[pos+1] = (aux >> 24) & 0x0ff;
983 out[pos+2] = (aux >> 16) & 0x0ff;
984 out[pos+3] = (aux >> 8) & 0x0ff;
985 out[pos+4] = aux & 0x0ff;
986 }
987 pos += 5;
988 } else {
989 // eight byte value
990 tag_u64_out(cbor_type, aux, out, posp);
991 return;
992 }
993 *posp = pos;
994 return;
995 }
996
997 static int inner_dumps(PyObject* ob, uint8_t* out, uintptr_t* posp);
998
999 static int dumps_dict(PyObject* ob, uint8_t* out, uintptr_t* posp) {
1000 uintptr_t pos = *posp;
1001 Py_ssize_t dictiter = 0;
1002 PyObject* key;
1003 PyObject* val;
1004 Py_ssize_t dictlen = PyDict_Size(ob);
1005 int err;
1006 tag_aux_out(CBOR_MAP, dictlen, out, &pos);
1007 while (PyDict_Next(ob, &dictiter, &key, &val)) {
1008 err = inner_dumps(key, out, &pos);
1009 if (err != 0) { return err; }
1010 err = inner_dumps(val, out, &pos);
1011 if (err != 0) { return err; }
1012 }
1013 *posp = pos;
1014 return 0;
1015 }
1016
1017
1018 static void dumps_bignum(uint8_t tag, PyObject* val, uint8_t* out, uintptr_t* posp) {
1019 uintptr_t pos = (posp != NULL) ? *posp : 0;
1020 PyObject* eight = PyLong_FromLong(8);
1021 PyObject* bytemask = NULL;
1022 PyObject* nval = NULL;
1023 uint8_t* revbytes = NULL;
1024 int revbytepos = 0;
1025 int val_is_orig = 1;
1026 if (out != NULL) {
1027 bytemask = PyLong_FromLongLong(0x0ff);
1028 revbytes = PyMem_Malloc(23);
1029 }
1030 while (PyObject_IsTrue(val) && (revbytepos < 23)) {
1031 if (revbytes != NULL) {
1032 PyObject* tbyte = PyNumber_And(val, bytemask);
1033 revbytes[revbytepos] = PyLong_AsLong(tbyte);
1034 Py_DECREF(tbyte);
1035 }
1036 revbytepos++;
1037 nval = PyNumber_InPlaceRshift(val, eight);
1038 if (val_is_orig) {
1039 val_is_orig = 0;
1040 } else {
1041 Py_DECREF(val);
1042 }
1043 val = nval;
1044 }
1045 if (revbytes != NULL) {
1046 out[pos] = CBOR_TAG | tag;
1047 pos++;
1048 out[pos] = CBOR_BYTES | revbytepos;
1049 pos++;
1050 revbytepos--;
1051 while (revbytepos >= 0) {
1052 out[pos] = revbytes[revbytepos];
1053 pos++;
1054 revbytepos--;
1055 }
1056 PyMem_Free(revbytes);
1057 Py_DECREF(bytemask);
1058 } else {
1059 pos += 2 + revbytepos;
1060 }
1061 if (!val_is_orig) {
1062 Py_DECREF(val);
1063 }
1064 Py_DECREF(eight);
1065 *posp = pos;
1066 }
1067
1068 static int dumps_tag(PyObject* ob, uint8_t* out, uintptr_t* posp) {
1069 uintptr_t pos = (posp != NULL) ? *posp : 0;
1070 int err = 0;
1071
1072
1073 PyObject* tag_num;
1074 PyObject* tag_value;
1075 err = 0;
1076
1077 tag_num = PyObject_GetAttrString(ob, "tag");
1078 if (tag_num != NULL) {
1079 tag_value = PyObject_GetAttrString(ob, "value");
1080 if (tag_value != NULL) {
1081 #ifdef Py_INTOBJECT_H
1082 if (PyInt_Check(tag_num)) {
1083 long val = PyInt_AsLong(tag_num);
1084 if (val > 0) {
1085 tag_aux_out(CBOR_TAG, val, out, &pos);
1086 err = inner_dumps(tag_value, out, &pos);
1087 } else {
1088 PyErr_Format(PyExc_ValueError, "tag cannot be a negative int: %ld", val);
1089 err = -1;
1090 }
1091 } else
1092 #endif
1093 if (PyLong_Check(tag_num)) {
1094 int overflow = -1;
1095 long long val = PyLong_AsLongLongAndOverflow(tag_num, &overflow);
1096 if (overflow == 0) {
1097 if (val >= 0) {
1098 tag_aux_out(CBOR_TAG, val, out, &pos);
1099 err = inner_dumps(tag_value, out, &pos);
1100 } else {
1101 PyErr_Format(PyExc_ValueError, "tag cannot be a negative long: %lld", val);
1102 err = -1;
1103 }
1104 } else {
1105 PyErr_SetString(PyExc_ValueError, "tag number too large");
1106 err = -1;
1107 }
1108 }
1109 Py_DECREF(tag_value);
1110 } else {
1111 PyErr_SetString(PyExc_ValueError, "broken Tag object has .tag but not .value");
1112 err = -1;
1113 }
1114 Py_DECREF(tag_num);
1115 } else {
1116 PyErr_SetString(PyExc_ValueError, "broken Tag object with no .tag");
1117 err = -1;
1118 }
1119 if (err != 0) { return err; }
1120
1121 *posp = pos;
1122 return err;
1123 }
1124
1125
1126 // With out=NULL it just counts the length.
1127 // return err, 0=OK
1128 static int inner_dumps(PyObject* ob, uint8_t* out, uintptr_t* posp) {
1129 uintptr_t pos = (posp != NULL) ? *posp : 0;
1130
1131 if (PyBool_Check(ob)) {
1132 if (out != NULL) {
1133 if (PyObject_IsTrue(ob)) {
1134 out[pos] = CBOR_TRUE;
1135 } else {
1136 out[pos] = CBOR_FALSE;
1137 }
1138 }
1139 pos += 1;
1140 } else if (ob == Py_None) {
1141 if (out != NULL) {
1142 out[pos] = CBOR_NULL;
1143 }
1144 pos += 1;
1145 } else if (PyDict_Check(ob)) {
1146 int err = dumps_dict(ob, out, &pos);
1147 if (err != 0) { return err; }
1148 } else if (PyList_Check(ob)) {
1149 Py_ssize_t i;
1150 Py_ssize_t listlen = PyList_Size(ob);
1151 tag_aux_out(CBOR_ARRAY, listlen, out, &pos);
1152 for (i = 0; i < listlen; i++) {
1153 int err = inner_dumps(PyList_GetItem(ob, i), out, &pos);
1154 if (err != 0) { return err; }
1155 }
1156 } else if (PyTuple_Check(ob)) {
1157 Py_ssize_t i;
1158 Py_ssize_t listlen = PyTuple_Size(ob);
1159 tag_aux_out(CBOR_ARRAY, listlen, out, &pos);
1160 for (i = 0; i < listlen; i++) {
1161 int err = inner_dumps(PyTuple_GetItem(ob, i), out, &pos);
1162 if (err != 0) { return err; }
1163 }
1164 // TODO: accept other enumerables and emit a variable length array
1165 #ifdef Py_INTOBJECT_H
1166 // PyInt exists in Python 2 but not 3
1167 } else if (PyInt_Check(ob)) {
1168 long val = PyInt_AsLong(ob);
1169 if (val >= 0) {
1170 tag_aux_out(CBOR_UINT, val, out, &pos);
1171 } else {
1172 tag_aux_out(CBOR_NEGINT, -1 - val, out, &pos);
1173 }
1174 #endif
1175 } else if (PyLong_Check(ob)) {
1176 int overflow = 0;
1177 long long val = PyLong_AsLongLongAndOverflow(ob, &overflow);
1178 if (overflow == 0) {
1179 if (val >= 0) {
1180 tag_aux_out(CBOR_UINT, val, out, &pos);
1181 } else {
1182 tag_aux_out(CBOR_NEGINT, -1L - val, out, &pos);
1183 }
1184 } else {
1185 if (overflow < 0) {
1186 // BIG NEGINT
1187 PyObject* minusone = PyLong_FromLongLong(-1L);
1188 PyObject* val = PyNumber_Subtract(minusone, ob);
1189 Py_DECREF(minusone);
1190 dumps_bignum(CBOR_TAG_NEGBIGNUM, val, out, &pos);
1191 Py_DECREF(val);
1192 } else {
1193 // BIG INT
1194 dumps_bignum(CBOR_TAG_BIGNUM, ob, out, &pos);
1195 }
1196 }
1197 } else if (PyFloat_Check(ob)) {
1198 double val = PyFloat_AsDouble(ob);
1199 tag_u64_out(CBOR_7, *((uint64_t*)(&val)), out, &pos);
1200 } else if (PyBytes_Check(ob)) {
1201 Py_ssize_t len = PyBytes_Size(ob);
1202 tag_aux_out(CBOR_BYTES, len, out, &pos);
1203 if (out != NULL) {
1204 memcpy(out + pos, PyBytes_AsString(ob), len);
1205 }
1206 pos += len;
1207 } else if (PyUnicode_Check(ob)) {
1208 PyObject* utf8 = PyUnicode_AsUTF8String(ob);
1209 Py_ssize_t len = PyBytes_Size(utf8);
1210 tag_aux_out(CBOR_TEXT, len, out, &pos);
1211 if (out != NULL) {
1212 memcpy(out + pos, PyBytes_AsString(utf8), len);
1213 }
1214 pos += len;
1215 Py_DECREF(utf8);
1216 } else {
1217 int handled = 0;
1218 {
1219 PyObject* tag_class = getCborTagClass();
1220 if (PyObject_IsInstance(ob, tag_class)) {
1221 int err = dumps_tag(ob, out, &pos);
1222 if (err != 0) { return err; }
1223 handled = 1;
1224 }
1225 // tag_class was just a borrowed reference
1226 }
1227
1228 // TODO: other special object serializations here
1229
1230 if (!handled) {
1231 #if IS_PY3
1232 PyErr_Format(PyExc_ValueError, "cannot serialize unknown object: %R", ob);
1233 #else
1234 PyObject* badtype = PyObject_Type(ob);
1235 PyObject* badtypename = PyObject_Str(badtype);
1236 PyErr_Format(PyExc_ValueError, "cannot serialize unknown object of type %s", PyString_AsString(badtypename));
1237 Py_DECREF(badtypename);
1238 Py_DECREF(badtype);
1239 #endif
1240 return -1;
1241 }
1242 }
1243 if (posp != NULL) {
1244 *posp = pos;
1245 }
1246 return 0;
1247 }
1248
1249 static PyObject*
1250 cbor_dumps(PyObject* noself, PyObject* args) {
1251 PyObject* ob;
1252 is_big_endian();
1253 if (PyType_IsSubtype(Py_TYPE(args), &PyList_Type)) {
1254 ob = PyList_GetItem(args, 0);
1255 } else if (PyType_IsSubtype(Py_TYPE(args), &PyTuple_Type)) {
1256 ob = PyTuple_GetItem(args, 0);
1257 } else {
1258 PyErr_Format(PyExc_ValueError, "args not list or tuple: %R\n", args);
1259 return NULL;
1260 }
1261
1262 {
1263 Py_ssize_t outlen = 0;
1264 uintptr_t pos = 0;
1265 void* out = NULL;
1266 PyObject* obout = NULL;
1267 int err;
1268
1269 // first pass just to count length
1270 err = inner_dumps(ob, NULL, &pos);
1271 if (err != 0) {
1272 return NULL;
1273 }
1274
1275 outlen = pos;
1276
1277 out = PyMem_Malloc(outlen);
1278 if (out == NULL) {
1279 PyErr_NoMemory();
1280 return NULL;
1281 }
1282
1283 err = inner_dumps(ob, out, NULL);
1284 if (err != 0) {
1285 PyMem_Free(out);
1286 return NULL;
1287 }
1288
1289 // TODO: I wish there was a way to do this without this copy.
1290 obout = PyBytes_FromStringAndSize(out, outlen);
1291 PyMem_Free(out);
1292 return obout;
1293 }
1294 }
1295
1296 static PyObject*
1297 cbor_dump(PyObject* noself, PyObject* args) {
1298 // args should be (obj, fp)
1299 PyObject* ob;
1300 PyObject* fp;
1301
1302 is_big_endian();
1303 if (PyType_IsSubtype(Py_TYPE(args), &PyList_Type)) {
1304 ob = PyList_GetItem(args, 0);
1305 fp = PyList_GetItem(args, 1);
1306 } else if (PyType_IsSubtype(Py_TYPE(args), &PyTuple_Type)) {
1307 ob = PyTuple_GetItem(args, 0);
1308 fp = PyTuple_GetItem(args, 1);
1309 } else {
1310 PyErr_Format(PyExc_ValueError, "args not list or tuple: %R\n", args);
1311 return NULL;
1312 }
1313
1314 {
1315 // TODO: make this smarter, right now it is justt fp.write(dumps(ob))
1316 Py_ssize_t outlen = 0;
1317 uintptr_t pos = 0;
1318 void* out = NULL;
1319 int err;
1320
1321 // first pass just to count length
1322 err = inner_dumps(ob, NULL, &pos);
1323 if (err != 0) {
1324 return NULL;
1325 }
1326
1327 outlen = pos;
1328
1329 out = PyMem_Malloc(outlen);
1330 if (out == NULL) {
1331 PyErr_NoMemory();
1332 return NULL;
1333 }
1334
1335 err = inner_dumps(ob, out, NULL);
1336 if (err != 0) {
1337 PyMem_Free(out);
1338 return NULL;
1339 }
1340
1341 #if HAS_FILE_READER
1342 if (PyFile_Check(fp)) {
1343 FILE* fout = PyFile_AsFile(fp);
1344 fwrite(out, 1, outlen, fout);
1345 } else
1346 #endif
1347 {
1348 PyObject* ret;
1349 PyObject* obout = NULL;
1350 #if IS_PY3
1351 PyObject* writeStr = PyUnicode_FromString("write");
1352 #else
1353 PyObject* writeStr = PyString_FromString("write");
1354 #endif
1355 obout = PyBytes_FromStringAndSize(out, outlen);
1356 //logprintf("write %zd bytes to %p.write() as %p\n", outlen, fp, obout);
1357 ret = PyObject_CallMethodObjArgs(fp, writeStr, obout, NULL);
1358 Py_DECREF(writeStr);
1359 Py_DECREF(obout);
1360 if (ret != NULL) {
1361 Py_DECREF(ret);
1362 } else {
1363 // exception in fp.write()
1364 PyMem_Free(out);
1365 return NULL;
1366 }
1367 //logprintf("wrote %zd bytes to %p.write() as %p\n", outlen, fp, obout);
1368 }
1369 PyMem_Free(out);
1370 }
1371
1372 Py_RETURN_NONE;
1373 }
1374
1375
1376 static PyMethodDef CborMethods[] = {
1377 {"loads", cbor_loads, METH_VARARGS,
1378 "parse cbor from data buffer to objects"},
1379 {"dumps", cbor_dumps, METH_VARARGS,
1380 "serialize python object to bytes"},
1381 {"load", cbor_load, METH_VARARGS,
1382 "Parse cbor from data buffer to objects.\n"
1383 "Takes a file-like object capable of .read(N)\n"},
1384 {"dump", cbor_dump, METH_VARARGS,
1385 "Serialize python object to bytes.\n"
1386 "dump(obj, fp)\n"
1387 "obj: object to output; fp: file-like object to .write() to\n"},
1388 {NULL, NULL, 0, NULL} /* Sentinel */
1389 };
1390
1391 #ifdef Py_InitModule
1392 // Python 2.7
1393 PyMODINIT_FUNC
1394 init_cbor(void)
1395 {
1396 (void) Py_InitModule("cbor._cbor", CborMethods);
1397 }
1398 #else
1399 // Python 3
1400 PyMODINIT_FUNC
1401 PyInit__cbor(void)
1402 {
1403 static PyModuleDef modef = {
1404 PyModuleDef_HEAD_INIT,
1405 };
1406 //modef.m_base = PyModuleDef_HEAD_INIT;
1407 modef.m_name = "cbor._cbor";
1408 modef.m_doc = NULL;
1409 modef.m_size = 0;
1410 modef.m_methods = CborMethods;
1411 modef.m_reload = NULL;
1412 modef.m_traverse = NULL;
1413 modef.m_clear = NULL;
1414 modef.m_free = NULL;
1415 return PyModule_Create(&modef);
1416 }
1417 #endif
1418
0 #!python
1
2 try:
3 # try C library _cbor.so
4 from ._cbor import loads, dumps, load, dump
5 except:
6 # fall back to 100% python implementation
7 from .cbor import loads, dumps, load, dump
8
9 from .cbor import Tag
10 from .tagmap import TagMapper, ClassTag, UnknownTagException
11
12 __all__ = [
13 'loads', 'dumps', 'load', 'dump',
14 'Tag',
15 'TagMapper', 'ClassTag', 'UnknownTagException',
16 ]
0 #!python
1 # -*- Python -*-
2
3 import datetime
4 import re
5 import struct
6 import sys
7
8 _IS_PY3 = sys.version_info[0] >= 3
9
10 if _IS_PY3:
11 from io import BytesIO as StringIO
12 else:
13 try:
14 from cStringIO import StringIO
15 except:
16 from StringIO import StringIO
17
18
19 CBOR_TYPE_MASK = 0xE0 # top 3 bits
20 CBOR_INFO_BITS = 0x1F # low 5 bits
21
22
23 CBOR_UINT = 0x00
24 CBOR_NEGINT = 0x20
25 CBOR_BYTES = 0x40
26 CBOR_TEXT = 0x60
27 CBOR_ARRAY = 0x80
28 CBOR_MAP = 0xA0
29 CBOR_TAG = 0xC0
30 CBOR_7 = 0xE0 # float and other types
31
32 CBOR_UINT8_FOLLOWS = 24 # 0x18
33 CBOR_UINT16_FOLLOWS = 25 # 0x19
34 CBOR_UINT32_FOLLOWS = 26 # 0x1a
35 CBOR_UINT64_FOLLOWS = 27 # 0x1b
36 CBOR_VAR_FOLLOWS = 31 # 0x1f
37
38 CBOR_BREAK = 0xFF
39
40 CBOR_FALSE = (CBOR_7 | 20)
41 CBOR_TRUE = (CBOR_7 | 21)
42 CBOR_NULL = (CBOR_7 | 22)
43 CBOR_UNDEFINED = (CBOR_7 | 23) # js 'undefined' value
44
45 CBOR_FLOAT16 = (CBOR_7 | 25)
46 CBOR_FLOAT32 = (CBOR_7 | 26)
47 CBOR_FLOAT64 = (CBOR_7 | 27)
48
49 CBOR_TAG_DATE_STRING = 0 # RFC3339
50 CBOR_TAG_DATE_ARRAY = 1 # any number type follows, seconds since 1970-01-01T00:00:00 UTC
51 CBOR_TAG_BIGNUM = 2 # big endian byte string follows
52 CBOR_TAG_NEGBIGNUM = 3 # big endian byte string follows
53 CBOR_TAG_DECIMAL = 4 # [ 10^x exponent, number ]
54 CBOR_TAG_BIGFLOAT = 5 # [ 2^x exponent, number ]
55 CBOR_TAG_BASE64URL = 21
56 CBOR_TAG_BASE64 = 22
57 CBOR_TAG_BASE16 = 23
58 CBOR_TAG_CBOR = 24 # following byte string is embedded CBOR data
59
60 CBOR_TAG_URI = 32
61 CBOR_TAG_BASE64URL = 33
62 CBOR_TAG_BASE64 = 34
63 CBOR_TAG_REGEX = 35
64 CBOR_TAG_MIME = 36 # following text is MIME message, headers, separators and all
65 CBOR_TAG_CBOR_FILEHEADER = 55799 # can open a file with 0xd9d9f7
66
67 _CBOR_TAG_BIGNUM_BYTES = struct.pack('B', CBOR_TAG | CBOR_TAG_BIGNUM)
68
69
70 def dumps_int(val):
71 "return bytes representing int val in CBOR"
72 if val >= 0:
73 # CBOR_UINT is 0, so I'm lazy/efficient about not OR-ing it in.
74 if val <= 23:
75 return struct.pack('B', val)
76 if val <= 0x0ff:
77 return struct.pack('BB', CBOR_UINT8_FOLLOWS, val)
78 if val <= 0x0ffff:
79 return struct.pack('!BH', CBOR_UINT16_FOLLOWS, val)
80 if val <= 0x0ffffffff:
81 return struct.pack('!BI', CBOR_UINT32_FOLLOWS, val)
82 if val <= 0x0ffffffffffffffff:
83 return struct.pack('!BQ', CBOR_UINT64_FOLLOWS, val)
84 outb = _dumps_bignum_to_bytearray(val)
85 return _CBOR_TAG_BIGNUM_BYTES + _encode_type_num(CBOR_BYTES, len(outb)) + outb
86 val = -1 - val
87 return _encode_type_num(CBOR_NEGINT, val)
88
89
90 if _IS_PY3:
91 def _dumps_bignum_to_bytearray(val):
92 out = []
93 while val > 0:
94 out.insert(0, val & 0x0ff)
95 val = val >> 8
96 return bytes(out)
97 else:
98 def _dumps_bignum_to_bytearray(val):
99 out = []
100 while val > 0:
101 out.insert(0, chr(val & 0x0ff))
102 val = val >> 8
103 return b''.join(out)
104
105
106 def dumps_float(val):
107 return struct.pack("!Bd", CBOR_FLOAT64, val)
108
109
110 _CBOR_TAG_NEGBIGNUM_BYTES = struct.pack('B', CBOR_TAG | CBOR_TAG_NEGBIGNUM)
111
112
113 def _encode_type_num(cbor_type, val):
114 """For some CBOR primary type [0..7] and an auxiliary unsigned number, return CBOR encoded bytes"""
115 assert val >= 0
116 if val <= 23:
117 return struct.pack('B', cbor_type | val)
118 if val <= 0x0ff:
119 return struct.pack('BB', cbor_type | CBOR_UINT8_FOLLOWS, val)
120 if val <= 0x0ffff:
121 return struct.pack('!BH', cbor_type | CBOR_UINT16_FOLLOWS, val)
122 if val <= 0x0ffffffff:
123 return struct.pack('!BI', cbor_type | CBOR_UINT32_FOLLOWS, val)
124 if (((cbor_type == CBOR_NEGINT) and (val <= 0x07fffffffffffffff)) or
125 ((cbor_type != CBOR_NEGINT) and (val <= 0x0ffffffffffffffff))):
126 return struct.pack('!BQ', cbor_type | CBOR_UINT64_FOLLOWS, val)
127 if cbor_type != CBOR_NEGINT:
128 raise Exception("value too big for CBOR unsigned number: {0!r}".format(val))
129 outb = _dumps_bignum_to_bytearray(val)
130 return _CBOR_TAG_NEGBIGNUM_BYTES + _encode_type_num(CBOR_BYTES, len(outb)) + outb
131
132
133 if _IS_PY3:
134 def _is_unicode(val):
135 return isinstance(val, str)
136 else:
137 def _is_unicode(val):
138 return isinstance(val, unicode)
139
140
141 def dumps_string(val, is_text=None, is_bytes=None):
142 if _is_unicode(val):
143 val = val.encode('utf8')
144 is_text = True
145 is_bytes = False
146 if (is_bytes) or not (is_text == True):
147 return _encode_type_num(CBOR_BYTES, len(val)) + val
148 return _encode_type_num(CBOR_TEXT, len(val)) + val
149
150
151 def dumps_array(arr):
152 head = _encode_type_num(CBOR_ARRAY, len(arr))
153 parts = [dumps(x) for x in arr]
154 return head + b''.join(parts)
155
156
157 if _IS_PY3:
158 def dumps_dict(d):
159 head = _encode_type_num(CBOR_MAP, len(d))
160 parts = [head]
161 for k,v in d.items():
162 parts.append(dumps(k))
163 parts.append(dumps(v))
164 return b''.join(parts)
165 else:
166 def dumps_dict(d):
167 head = _encode_type_num(CBOR_MAP, len(d))
168 parts = [head]
169 for k,v in d.iteritems():
170 parts.append(dumps(k))
171 parts.append(dumps(v))
172 return b''.join(parts)
173
174
175 def dumps_bool(b):
176 if b:
177 return struct.pack('B', CBOR_TRUE)
178 return struct.pack('B', CBOR_FALSE)
179
180
181 def dumps_tag(t):
182 return _encode_type_num(CBOR_TAG, t.tag) + dumps(t.value)
183
184
185 if _IS_PY3:
186 def _is_stringish(x):
187 return isinstance(x, (str, bytes))
188 def _is_intish(x):
189 return isinstance(x, int)
190 else:
191 def _is_stringish(x):
192 return isinstance(x, (str, basestring, bytes, unicode))
193 def _is_intish(x):
194 return isinstance(x, (int, long))
195
196
197 def dumps(ob):
198 if ob is None:
199 return struct.pack('B', CBOR_NULL)
200 if isinstance(ob, bool):
201 return dumps_bool(ob)
202 if _is_stringish(ob):
203 return dumps_string(ob)
204 if isinstance(ob, (list, tuple)):
205 return dumps_array(ob)
206 # TODO: accept other enumerables and emit a variable length array
207 if isinstance(ob, dict):
208 return dumps_dict(ob)
209 if isinstance(ob, float):
210 return dumps_float(ob)
211 if _is_intish(ob):
212 return dumps_int(ob)
213 if isinstance(ob, Tag):
214 return dumps_tag(ob)
215 raise Exception("don't know how to cbor serialize object of type %s", type(ob))
216
217
218 # same basic signature as json.dump, but with no options (yet)
219 def dump(obj, fp):
220 """
221 obj: Python object to serialize
222 fp: file-like object capable of .write(bytes)
223 """
224 # this is kinda lame, but probably not inefficient for non-huge objects
225 # TODO: .write() to fp as we go as each inner object is serialized
226 blob = dumps(obj)
227 fp.write(blob)
228
229
230 class Tag(object):
231 def __init__(self, tag=None, value=None):
232 self.tag = tag
233 self.value = value
234
235 def __repr__(self):
236 return "Tag({0!r}, {1!r})".format(self.tag, self.value)
237
238 def __eq__(self, other):
239 if not isinstance(other, Tag):
240 return False
241 return (self.tag == other.tag) and (self.value == other.value)
242
243
244 def loads(data):
245 """
246 Parse CBOR bytes and return Python objects.
247 """
248 if data is None:
249 raise ValueError("got None for buffer to decode in loads")
250 fp = StringIO(data)
251 return _loads(fp)[0]
252
253
254 def load(fp):
255 """
256 Parse and return object from fp, a file-like object supporting .read(n)
257 """
258 return _loads(fp)[0]
259
260
261 _MAX_DEPTH = 100
262
263
264 def _tag_aux(fp, tb):
265 bytes_read = 1
266 tag = tb & CBOR_TYPE_MASK
267 tag_aux = tb & CBOR_INFO_BITS
268 if tag_aux <= 23:
269 aux = tag_aux
270 elif tag_aux == CBOR_UINT8_FOLLOWS:
271 data = fp.read(1)
272 aux = struct.unpack_from("!B", data, 0)[0]
273 bytes_read += 1
274 elif tag_aux == CBOR_UINT16_FOLLOWS:
275 data = fp.read(2)
276 aux = struct.unpack_from("!H", data, 0)[0]
277 bytes_read += 2
278 elif tag_aux == CBOR_UINT32_FOLLOWS:
279 data = fp.read(4)
280 aux = struct.unpack_from("!I", data, 0)[0]
281 bytes_read += 4
282 elif tag_aux == CBOR_UINT64_FOLLOWS:
283 data = fp.read(8)
284 aux = struct.unpack_from("!Q", data, 0)[0]
285 bytes_read += 8
286 else:
287 assert tag_aux == CBOR_VAR_FOLLOWS, "bogus tag {0:02x}".format(tb)
288 aux = None
289
290 return tag, tag_aux, aux, bytes_read
291
292
293 def _read_byte(fp):
294 tb = fp.read(1)
295 if len(tb) == 0:
296 # I guess not all file-like objects do this
297 raise EOFError()
298 return ord(tb)
299
300
301 def _loads_var_array(fp, limit, depth, returntags, bytes_read):
302 ob = []
303 tb = _read_byte(fp)
304 while tb != CBOR_BREAK:
305 (subob, sub_len) = _loads_tb(fp, tb, limit, depth, returntags)
306 bytes_read += 1 + sub_len
307 ob.append(subob)
308 tb = _read_byte(fp)
309 return (ob, bytes_read + 1)
310
311
312 def _loads_var_map(fp, limit, depth, returntags, bytes_read):
313 ob = {}
314 tb = _read_byte(fp)
315 while tb != CBOR_BREAK:
316 (subk, sub_len) = _loads_tb(fp, tb, limit, depth, returntags)
317 bytes_read += 1 + sub_len
318 (subv, sub_len) = _loads(fp, limit, depth, returntags)
319 bytes_read += sub_len
320 ob[subk] = subv
321 tb = _read_byte(fp)
322 return (ob, bytes_read + 1)
323
324
325 if _IS_PY3:
326 def _loads_array(fp, limit, depth, returntags, aux, bytes_read):
327 ob = []
328 for i in range(aux):
329 subob, subpos = _loads(fp)
330 bytes_read += subpos
331 ob.append(subob)
332 return ob, bytes_read
333 def _loads_map(fp, limit, depth, returntags, aux, bytes_read):
334 ob = {}
335 for i in range(aux):
336 subk, subpos = _loads(fp)
337 bytes_read += subpos
338 subv, subpos = _loads(fp)
339 bytes_read += subpos
340 ob[subk] = subv
341 return ob, bytes_read
342 else:
343 def _loads_array(fp, limit, depth, returntags, aux, bytes_read):
344 ob = []
345 for i in xrange(aux):
346 subob, subpos = _loads(fp)
347 bytes_read += subpos
348 ob.append(subob)
349 return ob, bytes_read
350 def _loads_map(fp, limit, depth, returntags, aux, bytes_read):
351 ob = {}
352 for i in xrange(aux):
353 subk, subpos = _loads(fp)
354 bytes_read += subpos
355 subv, subpos = _loads(fp)
356 bytes_read += subpos
357 ob[subk] = subv
358 return ob, bytes_read
359
360
361 def _loads(fp, limit=None, depth=0, returntags=False):
362 "return (object, bytes read)"
363 if depth > _MAX_DEPTH:
364 raise Exception("hit CBOR loads recursion depth limit")
365
366 tb = _read_byte(fp)
367
368 return _loads_tb(fp, tb, limit, depth, returntags)
369
370 def _loads_tb(fp, tb, limit=None, depth=0, returntags=False):
371 # Some special cases of CBOR_7 best handled by special struct.unpack logic here
372 if tb == CBOR_FLOAT16:
373 data = fp.read(2)
374 hibyte, lowbyte = struct.unpack_from("BB", data, 0)
375 exp = (hibyte >> 2) & 0x1F
376 mant = ((hibyte & 0x03) << 8) | lowbyte
377 if exp == 0:
378 val = mant * (2.0 ** -24)
379 elif exp == 31:
380 if mant == 0:
381 val = float('Inf')
382 else:
383 val = float('NaN')
384 else:
385 val = (mant + 1024.0) * (2 ** (exp - 25))
386 if hibyte & 0x80:
387 val = -1.0 * val
388 return (val, 3)
389 elif tb == CBOR_FLOAT32:
390 data = fp.read(4)
391 pf = struct.unpack_from("!f", data, 0)
392 return (pf[0], 5)
393 elif tb == CBOR_FLOAT64:
394 data = fp.read(8)
395 pf = struct.unpack_from("!d", data, 0)
396 return (pf[0], 9)
397
398 tag, tag_aux, aux, bytes_read = _tag_aux(fp, tb)
399
400 if tag == CBOR_UINT:
401 return (aux, bytes_read)
402 elif tag == CBOR_NEGINT:
403 return (-1 - aux, bytes_read)
404 elif tag == CBOR_BYTES:
405 ob, subpos = loads_bytes(fp, aux)
406 return (ob, bytes_read + subpos)
407 elif tag == CBOR_TEXT:
408 raw, subpos = loads_bytes(fp, aux, btag=CBOR_TEXT)
409 ob = raw.decode('utf8')
410 return (ob, bytes_read + subpos)
411 elif tag == CBOR_ARRAY:
412 if aux is None:
413 return _loads_var_array(fp, limit, depth, returntags, bytes_read)
414 return _loads_array(fp, limit, depth, returntags, aux, bytes_read)
415 elif tag == CBOR_MAP:
416 if aux is None:
417 return _loads_var_map(fp, limit, depth, returntags, bytes_read)
418 return _loads_map(fp, limit, depth, returntags, aux, bytes_read)
419 elif tag == CBOR_TAG:
420 ob, subpos = _loads(fp)
421 bytes_read += subpos
422 if returntags:
423 # Don't interpret the tag, return it and the tagged object.
424 ob = Tag(aux, ob)
425 else:
426 # attempt to interpet the tag and the value into a Python object.
427 ob = tagify(ob, aux)
428 return ob, bytes_read
429 elif tag == CBOR_7:
430 if tb == CBOR_TRUE:
431 return (True, bytes_read)
432 if tb == CBOR_FALSE:
433 return (False, bytes_read)
434 if tb == CBOR_NULL:
435 return (None, bytes_read)
436 if tb == CBOR_UNDEFINED:
437 return (None, bytes_read)
438 raise ValueError("unknown cbor tag 7 byte: {:02x}".format(tb))
439
440
441 def loads_bytes(fp, aux, btag=CBOR_BYTES):
442 # TODO: limit to some maximum number of chunks and some maximum total bytes
443 if aux is not None:
444 # simple case
445 ob = fp.read(aux)
446 return (ob, aux)
447 # read chunks of bytes
448 chunklist = []
449 total_bytes_read = 0
450 while True:
451 tb = fp.read(1)[0]
452 if not _IS_PY3:
453 tb = ord(tb)
454 if tb == CBOR_BREAK:
455 total_bytes_read += 1
456 break
457 tag, tag_aux, aux, bytes_read = _tag_aux(fp, tb)
458 assert tag == btag, 'variable length value contains unexpected component'
459 ob = fp.read(aux)
460 chunklist.append(ob)
461 total_bytes_read += bytes_read + aux
462 return (b''.join(chunklist), total_bytes_read)
463
464
465 if _IS_PY3:
466 def _bytes_to_biguint(bs):
467 out = 0
468 for ch in bs:
469 out = out << 8
470 out = out | ch
471 return out
472 else:
473 def _bytes_to_biguint(bs):
474 out = 0
475 for ch in bs:
476 out = out << 8
477 out = out | ord(ch)
478 return out
479
480
481 def tagify(ob, aux):
482 # TODO: make this extensible?
483 # cbor.register_tag_handler(tagnumber, tag_handler)
484 # where tag_handler takes (tagnumber, tagged_object)
485 if aux == CBOR_TAG_DATE_STRING:
486 # TODO: parse RFC3339 date string
487 pass
488 if aux == CBOR_TAG_DATE_ARRAY:
489 return datetime.datetime.utcfromtimestamp(ob)
490 if aux == CBOR_TAG_BIGNUM:
491 return _bytes_to_biguint(ob)
492 if aux == CBOR_TAG_NEGBIGNUM:
493 return -1 - _bytes_to_biguint(ob)
494 if aux == CBOR_TAG_REGEX:
495 # Is this actually a good idea? Should we just return the tag and the raw value to the user somehow?
496 return re.compile(ob)
497 return Tag(aux, ob)
0 from __future__ import absolute_import
1 import logging
2 import random
3 import socket
4 import time
5
6 import cbor
7
8
9 logger = logging.getLogger(__name__)
10
11
12 class SocketReader(object):
13 '''
14 Simple adapter from socket.recv to file-like-read
15 '''
16 def __init__(self, sock):
17 self.socket = sock
18 self.timeout_seconds = 10.0
19
20 def read(self, num):
21 start = time.time()
22 data = self.socket.recv(num)
23 while len(data) < num:
24 now = time.time()
25 if now > (start + self.timeout_seconds):
26 break
27 ndat = self.socket.recv(num - len(data))
28 if ndat:
29 data += ndat
30 return data
31
32
33 class CborRpcClient(object):
34 '''Base class for all client objects.
35
36 This provides common `addr_family`, `address`, and `registry_addresses`
37 configuration parameters, and manages the connection back to the server.
38
39 Automatic retry and time based fallback is managed from
40 configuration parameters `retries` (default 5), and
41 `base_retry_seconds` (default 0.5). Retry time doubles on each
42 retry. E.g. try 0; wait 0.5s; try 1; wait 1s; try 2; wait 2s; try
43 3; wait 4s; try 4; wait 8s; try 5; FAIL. Total time waited just
44 under base_retry_seconds * (2 ** retries).
45
46 .. automethod:: __init__
47 .. automethod:: _rpc
48 .. automethod:: close
49
50 '''
51
52 def __init__(self, config=None):
53 self._socket_family = config.get('addr_family', socket.AF_INET)
54 # may need to be ('host', port)
55 self._socket_addr = config.get('address')
56 if self._socket_family == socket.AF_INET:
57 if not isinstance(self._socket_addr, tuple):
58 # python socket standard library insists this be tuple!
59 tsocket_addr = tuple(self._socket_addr)
60 assert len(tsocket_addr) == 2, 'address must be length-2 tuple ("hostname", port number), got {!r} tuplified to {!r}'.format(self._socket_addr, tsocket_addr)
61 self._socket_addr = tsocket_addr
62 self._socket = None
63 self._rfile = None
64 self._local_addr = None
65 self._message_count = 0
66 self._retries = config.get('retries', 5)
67 self._base_retry_seconds = float(config.get('base_retry_seconds', 0.5))
68
69 def _conn(self):
70 # lazy socket opener
71 if self._socket is None:
72 try:
73 self._socket = socket.create_connection(self._socket_addr)
74 self._local_addr = self._socket.getsockname()
75 except:
76 logger.error('error connecting to %r:%r', self._socket_addr[0],
77 self._socket_addr[1], exc_info=True)
78 raise
79 return self._socket
80
81 def close(self):
82 '''Close the connection to the server.
83
84 The next RPC call will reopen the connection.
85
86 '''
87 if self._socket is not None:
88 self._rfile = None
89 try:
90 self._socket.shutdown(socket.SHUT_RDWR)
91 self._socket.close()
92 except socket.error:
93 logger.warn('error closing lockd client socket',
94 exc_info=True)
95 self._socket = None
96
97 @property
98 def rfile(self):
99 if self._rfile is None:
100 conn = self._conn()
101 self._rfile = SocketReader(conn)
102 return self._rfile
103
104 def _rpc(self, method_name, params):
105 '''Call a method on the server.
106
107 Calls ``method_name(*params)`` remotely, and returns the results
108 of that function call. Expected return types are primitives, lists,
109 and dictionaries.
110
111 :raise Exception: if the server response was a failure
112
113 '''
114 mlog = logging.getLogger('cborrpc')
115 tryn = 0
116 delay = self._base_retry_seconds
117 self._message_count += 1
118 message = {
119 'id': self._message_count,
120 'method': method_name,
121 'params': params
122 }
123 mlog.debug('request %r', message)
124 buf = cbor.dumps(message)
125
126 errormessage = None
127 while True:
128 try:
129 conn = self._conn()
130 conn.send(buf)
131 response = cbor.load(self.rfile)
132 mlog.debug('response %r', response)
133 assert response['id'] == message['id']
134 if 'result' in response:
135 return response['result']
136 # From here on out we got a response, the server
137 # didn't have some weird intermittent error or
138 # non-connectivity, it gave us an error message. We
139 # don't retry that, we raise it to the user.
140 errormessage = response.get('error')
141 if errormessage and hasattr(errormessage,'get'):
142 errormessage = errormessage.get('message')
143 if not errormessage:
144 errormessage = repr(response)
145 break
146 except Exception as ex:
147 if tryn < self._retries:
148 tryn += 1
149 logger.debug('ex in %r (%s), retrying %s in %s sec...',
150 method_name, ex, tryn, delay, exc_info=True)
151 self.close()
152 time.sleep(delay)
153 delay *= 2
154 continue
155 logger.error('failed in rpc %r %r', method_name, params,
156 exc_info=True)
157 raise
158 raise Exception(errormessage)
159
160
161 if __name__ == '__main__':
162 import sys
163 logging.basicConfig(level=logging.DEBUG)
164 host,port = sys.argv[1].split(':')
165 if not host:
166 host = 'localhost'
167 port = int(port)
168 client = CborRpcClient({'address':(host,port)})
169 print(client._rpc(u'connect', [u'127.0.0.1:5432', u'root', u'aoeu']))
170 print(client._rpc(u'put', [[('k1','v1'), ('k2','v2')]]))
171 #print(client._rpc(u'ping', []))
172 #print(client._rpc(u'gnip', []))
173 client.close()
174
0 try:
1 # try C library _cbor.so
2 from ._cbor import loads, dumps, load, dump
3 except:
4 # fall back to 100% python implementation
5 from .cbor import loads, dumps, load, dump
6
7 from .cbor import Tag, CBOR_TAG_CBOR
8
9
10 class ClassTag(object):
11 '''
12 For some CBOR tag_number, encode/decode Python class_type.
13 class_type manily used for isintance(foo, class_type)
14 Call encode_function() taking a Python instance and returning CBOR primitive types.
15 Call decode_function() on CBOR primitive types and return an instance of the Python class_type (a factory function).
16 '''
17 def __init__(self, tag_number, class_type, encode_function, decode_function):
18 self.tag_number = tag_number
19 self.class_type = class_type
20 self.encode_function = encode_function
21 self.decode_function = decode_function
22
23
24 # TODO: This would be more efficient if it moved into cbor.py and
25 # cbormodule.c, happening inline so that there is only one traversal
26 # of the objects. But that would require two implementations. When
27 # this API has been used more and can be considered settled I should
28 # do that. -- Brian Olson 20140917_172229
29 class TagMapper(object):
30 '''
31 Translate Python objects and CBOR tagged data.
32 Use the CBOR TAG system to note that some data is of a certain class.
33 Dump while translating Python objects into a CBOR compatible representation.
34 Load and translate CBOR primitives back into Python objects.
35 '''
36 def __init__(self, class_tags=None, raise_on_unknown_tag=False):
37 '''
38 class_tags: list of ClassTag objects
39 '''
40 self.class_tags = class_tags
41 self.raise_on_unknown_tag = raise_on_unknown_tag
42
43 def encode(self, obj):
44 for ct in self.class_tags:
45 if (ct.class_type is None) or (ct.encode_function is None):
46 continue
47 if isinstance(obj, ct.class_type):
48 return Tag(ct.tag_number, ct.encode_function(obj))
49 if isinstance(obj, (list, tuple)):
50 return [self.encode(x) for x in obj]
51 if isinstance(obj, dict):
52 # assume key is a primitive
53 # can't do this in Python 2.6:
54 #return {k:self.encode(v) for k,v in obj.iteritems()}
55 out = {}
56 for k,v in obj.iteritems():
57 out[k] = self.encode(v)
58 return out
59 # fall through, let underlying cbor.dump decide if it can encode object
60 return obj
61
62 def decode(self, obj):
63 if isinstance(obj, Tag):
64 for ct in self.class_tags:
65 if ct.tag_number == obj.tag:
66 return ct.decode_function(obj.value)
67 # unknown Tag
68 if self.raise_on_unknown_tag:
69 raise UnknownTagException(str(obj.tag))
70 # otherwise, pass it through
71 return obj
72 if isinstance(obj, list):
73 # update in place. cbor only decodes to list, not tuple
74 for i,v in enumerate(obj):
75 obj[i] = self.decode(v)
76 return obj
77 if isinstance(obj, dict):
78 # update in place
79 for k,v in obj.iteritems():
80 # assume key is a primitive
81 obj[k] = self.decode(v)
82 return obj
83 # non-recursive object (num,bool,blob,string)
84 return obj
85
86 def dump(self, obj, fp):
87 dump(self.encode(obj), fp)
88
89 def dumps(self, obj):
90 return dumps(self.encode(obj))
91
92 def load(self, fp):
93 return self.decode(load(fp))
94
95 def loads(self, blob):
96 return self.decode(loads(blob))
97
98
99 class WrappedCBOR(ClassTag):
100 """Handles Tag 24, where a byte array is sub encoded CBOR.
101 Unpacks sub encoded object on finding such a tag.
102 Does not convert anyting into such a tag.
103
104 Usage:
105 >>> import cbor
106 >>> import cbor.tagmap
107 >>> tm=cbor.TagMapper([cbor.tagmap.WrappedCBOR()])
108 >>> x = cbor.dumps(cbor.Tag(24, cbor.dumps({"a":[1,2,3]})))
109 >>> x
110 '\xd8\x18G\xa1Aa\x83\x01\x02\x03'
111 >>> tm.loads(x)
112 {'a': [1L, 2L, 3L]}
113 >>> cbor.loads(x)
114 Tag(24L, '\xa1Aa\x83\x01\x02\x03')
115 """
116 def __init__(self):
117 super(WrappedCBOR, self).__init__(CBOR_TAG_CBOR, None, None, loads)
118
119 @staticmethod
120 def wrap(ob):
121 return Tag(CBOR_TAG_CBOR, dumps(ob))
122
123 @staticmethod
124 def dump(ob, fp):
125 return dump(Tag(CBOR_TAG_CBOR, dumps(ob)), fp)
126
127 @staticmethod
128 def dumps(ob):
129 return dumps(Tag(CBOR_TAG_CBOR, dumps(ob)))
130
131
132 class UnknownTagException(BaseException):
133 pass
0 Metadata-Version: 1.1
1 Name: cbor
2 Version: 0.1.21
3 Summary: RFC 7049 - Concise Binary Object Representation
4 Home-page: https://bitbucket.org/bodhisnarkva/cbor
5 Author: Brian Olson
6 Author-email: bolson@bolson.org
7 License: Apache
8 Description:
9 An implementation of RFC 7049 - Concise Binary Object Representation (CBOR).
10
11 CBOR is comparable to JSON, has a superset of JSON's ability, but serializes to a binary format which is smaller and faster to generate and parse.
12
13 The two primary functions are cbor.loads() and cbor.dumps().
14
15 This library includes a C implementation which runs 3-5 times faster than the Python standard library's C-accelerated implementanion of JSON. This is also includes a 100% Python implementation.
16
17 Platform: UNKNOWN
18 Classifier: Development Status :: 4 - Beta
19 Classifier: Intended Audience :: Developers
20 Classifier: License :: OSI Approved :: Apache Software License
21 Classifier: Operating System :: OS Independent
22 Classifier: Programming Language :: Python :: 2.7
23 Classifier: Programming Language :: Python :: 3.3
24 Classifier: Programming Language :: Python :: 3.4
25 Classifier: Programming Language :: C
26 Classifier: Topic :: Software Development :: Libraries :: Python Modules
0 setup.py
1 c/cbor.h
2 c/cbormodule.c
3 cbor/__init__.py
4 cbor/cbor.py
5 cbor/cbor_rpc_client.py
6 cbor/tagmap.py
7 cbor.egg-info/PKG-INFO
8 cbor.egg-info/SOURCES.txt
9 cbor.egg-info/dependency_links.txt
10 cbor.egg-info/top_level.txt
0 [egg_info]
1 tag_build =
2 tag_date = 0
3 tag_svn_revision = 0
4
0 #! /usr/bin/env python
1 # Copyright 2014 Brian Olson
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 # http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14
15 from distutils.command.build_ext import build_ext
16 from distutils.errors import (CCompilerError, DistutilsExecError,
17 DistutilsPlatformError)
18 import sys
19
20 from setuptools import setup, Extension
21
22
23 build_errors = (CCompilerError, DistutilsExecError, DistutilsPlatformError)
24 if sys.platform == 'win32' and sys.version_info > (2, 6):
25 # 2.6's distutils.msvc9compiler can raise an IOError when failing to
26 # find the compiler
27 build_errors += (IOError,)
28
29
30 class BuildError(Exception):
31 """Raised if compiling extensions failed."""
32
33
34 class optional_build_ext(build_ext):
35 """build_ext implementation with optional C speedups."""
36
37 def run(self):
38 try:
39 build_ext.run(self)
40 except DistutilsPlatformError:
41 raise BuildError()
42
43 def build_extension(self, ext):
44 try:
45 build_ext.build_extension(self, ext)
46 except build_errors as be:
47 raise BuildError(be)
48 except ValueError as ve:
49 # this can happen on Windows 64 bit, see Python issue 7511
50 if "'path'" in str(sys.exc_info()[1]): # works with Python 2 and 3
51 raise BuildError(ve)
52 raise
53
54
55 setup_options = dict(
56 name='cbor',
57 version='0.1.21',
58 description='RFC 7049 - Concise Binary Object Representation',
59 long_description="""
60 An implementation of RFC 7049 - Concise Binary Object Representation (CBOR).
61
62 CBOR is comparable to JSON, has a superset of JSON's ability, but serializes to a binary format which is smaller and faster to generate and parse.
63
64 The two primary functions are cbor.loads() and cbor.dumps().
65
66 This library includes a C implementation which runs 3-5 times faster than the Python standard library's C-accelerated implementanion of JSON. This is also includes a 100% Python implementation.
67 """,
68 author='Brian Olson',
69 author_email='bolson@bolson.org',
70 url='https://bitbucket.org/bodhisnarkva/cbor',
71 packages=['cbor'],
72 package_dir={'cbor':'cbor'},
73 ext_modules=[
74 Extension(
75 'cbor._cbor',
76 include_dirs=['c/'],
77 sources=['c/cbormodule.c'],
78 headers=['c/cbor.h'],
79 )
80 ],
81 license='Apache',
82 classifiers=[
83 'Development Status :: 4 - Beta',
84 'Intended Audience :: Developers',
85 'License :: OSI Approved :: Apache Software License',
86 'Operating System :: OS Independent',
87 'Programming Language :: Python :: 2.7',
88 'Programming Language :: Python :: 3.3',
89 'Programming Language :: Python :: 3.4',
90 'Programming Language :: C',
91 'Topic :: Software Development :: Libraries :: Python Modules',
92 ],
93 cmdclass={'build_ext': optional_build_ext},
94 )
95
96
97 def main():
98 """ Perform setup with optional C speedups.
99
100 Optional extension compilation stolen from markupsafe, which again stole
101 it from simplejson. Creds to Bob Ippolito for the original code.
102 """
103 is_jython = 'java' in sys.platform
104 is_pypy = hasattr(sys, 'pypy_translation_info')
105
106 if is_jython or is_pypy:
107 del setup_options['ext_modules']
108
109 try:
110 setup(**setup_options)
111 except BuildError as be:
112 sys.stderr.write('''
113 BUILD ERROR:
114 %s
115 RETRYING WITHOUT C EXTENSIONS
116 ''' % (be,))
117 del setup_options['ext_modules']
118 setup(**setup_options)
119
120
121 if __name__ == '__main__':
122 main()