Codebase list python-pyeclib / 3c8d65c
Merge tag '1.5.0' into debian/pike Release 1.5.0 Ondřej Nový 6 years ago
12 changed file(s) with 276 addition(s) and 171 deletion(s). Raw diff Collapse all Expand all
+1
-0
.mailmap less more
44 Tushar Gohad <tushar.gohad@intel.com> <Tushar Gohad>
55 Eric Lambert <eric_lambert@xyratex.com> <eric.lambert@seagate.com>
66 Kota Tsuyuzaki <bloodeagle40234@gmail.com> <tsuyuzaki.kota@lab.ntt.co.jp>
7 Kota Tsuyuzaki <bloodeagle40234@gmail.com> <bloodeagle40123@gmail.com>
+12
-8
AUTHORS less more
00 Original Authors
11 ----------------
2 Kevin Greenan (kmgreen2@gmail.com)
23 Tushar Gohad (tushar.gohad@intel.com)
3 Kevin Greenan (kmgreen2@gmail.com)
44
55 Contributors
66 ------------
7 Clay Gerrard (clay.gerrard@gmail.com)
8 Davanum Srinivas (davanum@gmail.com)
9 Eric Lambert (eric_lambert@xyratex.com)
10 Jim Cheung (jim.cheung@phazr.io)
11 John Dickinson (me@not.mn)
12 Kota Tsuyuzaki (bloodeagle40234@gmail.com)
13 Mark Storer (Mark.Storer@evault.com)
14 Ondřej Nový (ondrej.novy@firma.seznam.cz)
15 Pete Zaitcev (zaitcev@kotori.zaitcev.us)
16 Thiago da Silva (thiago@redhat.com)
717 Timur Alperovich (timuralp@swiftstack.com)
818 Tim Burke (tim.burke@gmail.com)
9 Thiago da Silva (thiago@redhat.com)
10 Eric Lambert (eric_lambert@xyratex.com)
11 Ondřej Nový (ondrej.novy@firma.seznam.cz)
12 Davanum Srinivas (davanum@gmail.com)
1319 Victor Stinner (vstinner@redhat.com)
14 Mark Storer (Mark.Storer@evault.com)
15 Kota Tsuyuzaki (bloodeagle40123@gmail.com)
16 Pete Zaitcev (zaitcev@kotori.zaitcev.us)
1720 Yuan Zhou (yuan.zhou@intel.com)
21
+10
-1
ChangeLog less more
0 New in 1.5.0
1 ------------
2
3 * Added support for Phazr.IO libphazr library.
4 * Fixed error handling that caused segfaults.
5 * Use version number exposed by liberasurecode.
6 * various other minor bugs fixes and improvements.
7
8
09 New in 1.4.0
110 ------------
211
3 * Add support for ISA-L Cauchy
12 * Added support for ISA-L Cauchy
413 * Fixed memory leak in get_metadata
514 * Added soft warning log line when using liberasurecode <1.3.1
615
+4
-1
README.rst less more
5050 * ``isa_l_rs_vand`` => Intel Storage Acceleration Library (ISA-L) - SIMD accelerated Erasure Coding backends [4]
5151 * ``isa_l_rs_cauchy`` => Cauchy Reed-Solomon encoding (ISA-L variant) [4]
5252 * ``shss`` => NTT Lab Japan's Erasure Coding Library [5]
53 * ``libphazr`` => Phazr.IO's erasure code library with built-in privacy [6]
5354
5455
5556 The Python API supports the following functions:
239240 An example for ubuntu to install dependency packages::
240241
241242 $ sudo apt-get install build-essential python-dev python-pip liberasurecode-dev
242 $ sudo pip install -U bindep -r test-requirement.txt
243 $ sudo pip install -U bindep -r test-requirements.txt
243244
244245 If you want to confirm all dependency packages installed successfully, try::
245246
279280 [4] Intel(R) Storage Acceleration Library (Open Source Version), https://01.org/intel%C2%AE-storage-acceleration-library-open-source-version
280281
281282 [5] Kota Tsuyuzaki <tsuyuzaki.kota@lab.ntt.co.jp>, "NTT SHSS Erasure Coding backend"
283
284 [6] Jim Cheung <support@phazr.io>, "Phazr.IO libphazr erasure code backend with built-in privacy"
+2
-2
doc/source/conf.py less more
4747 # built documents.
4848 #
4949 # The short X.Y version.
50 version = '1.4.0'
50 version = '1.5.0'
5151 # The full version, including alpha/beta/rc tags.
52 release = '1.4.0'
52 release = '1.5.0'
5353
5454 # The language for content autogenerated by Sphinx. Refer to documentation
5555 # for a list of supported languages.
+22
-23
pyeclib/ec_iface.py less more
123123 shss = 5
124124 liberasurecode_rs_vand = 6
125125 isa_l_rs_cauchy = 7
126 libphazr = 8
126127
127128
128129 # Output of Erasure (en)Coding process are data "fragments". Fragment data
166167 self.m = positive_int_value(value)
167168 except ValueError:
168169 raise ECDriverError(
169 "Invalid number of data fragments (m)")
170 "Invalid number of parity fragments (m)")
170171 elif key == "ec_type":
171172 if value in ["flat_xor_hd", "flat_xor_hd_3", "flat_xor_hd_4"]:
172173 if value == "flat_xor_hd" or value == "flat_xor_hd_3":
174175 elif value == "flat_xor_hd_4":
175176 self.hd = 4
176177 value = "flat_xor_hd"
178 elif value == "libphazr":
179 self.hd = 1
177180 if PyECLib_EC_Types.has_enum(value):
178181 self.ec_type = PyECLib_EC_Types.get_by_name(value)
179182 else:
211214 #
212215 # Verify that the imported library implements the required functions
213216 #
214 required_methods = {
215 'decode': 0,
216 'encode': 0,
217 'reconstruct': 0,
218 'fragments_needed': 0,
219 'min_parity_fragments_needed': 0,
220 'get_metadata': 0,
221 'verify_stripe_metadata': 0,
222 'get_segment_info': 0
223 }
224
225 for attr in dir(self.ec_lib_reference):
226 if hasattr(getattr(self.ec_lib_reference, attr), "__call__"):
227 required_methods[attr] = 1
228
229 not_implemented_str = ""
230 for (method, is_implemented) in required_methods.items():
231 if is_implemented == 0:
232 not_implemented_str += method + " "
233
234 if len(not_implemented_str) > 0:
217 required_methods = [
218 'decode',
219 'encode',
220 'reconstruct',
221 'fragments_needed',
222 'min_parity_fragments_needed',
223 'get_metadata',
224 'verify_stripe_metadata',
225 'get_segment_info',
226 ]
227
228 missing_methods = ' '.join(
229 method for method in required_methods
230 if not callable(getattr(self.ec_lib_reference, method, None)))
231
232 if missing_methods:
235233 raise ECDriverError(
236234 "The following required methods are not implemented "
237 "in %s: %s" % (self.library_import_str, not_implemented_str))
235 "in %s: %s" % (self.library_import_str, missing_methods))
238236
239237 def __repr__(self):
240238 return '%s(ec_type=%r, k=%r, m=%r)' % (
520518 'shss',
521519 'liberasurecode_rs_vand',
522520 'isa_l_rs_cauchy',
521 'libphazr',
523522 ]
524523
525524
+2
-2
setup.py less more
158158
159159 module = Extension('pyeclib_c',
160160 define_macros=[('MAJOR VERSION', '1'),
161 ('MINOR VERSION', '4')],
161 ('MINOR VERSION', '5')],
162162 include_dirs=[default_python_incdir,
163163 'src/c/pyeclib_c',
164164 '/usr/include',
171171 sources=['src/c/pyeclib_c/pyeclib_c.c'])
172172
173173 setup(name='pyeclib',
174 version='1.4.0',
174 version='1.5.0',
175175 author='Kevin Greenan',
176176 author_email='kmgreen2@gmail.com',
177177 maintainer='Kevin Greenan and Tushar Gohad',
+120
-126
src/c/pyeclib_c/pyeclib_c.c less more
8989 static PyObject *import_class(const char *module, const char *cls)
9090 {
9191 PyObject *s = PyImport_ImportModule(module);
92 if (s == NULL) {
93 return NULL;
94 }
9295 return (PyObject *) PyObject_GetAttrString(s, cls);
9396 }
9497
137140 return NULL;
138141 }
139142
140 void
141 pyeclib_c_seterr(int ret, const char * prefix)
142 {
143 void pyeclib_c_seterr(int ret, const char * prefix) {
144 char *err_class;
145 char *err_msg;
143146 char err[255];
144 PyObject *eo = NULL;
145
146 strcpy(err, prefix);
147
148 // If any error was previously set, we're explicitly ignoring it
149 // to raise something new
150 PyErr_Clear();
151
147152 switch (ret) {
148153 case -EBACKENDNOTAVAIL:
149 eo = import_class("pyeclib.ec_iface",
150 "ECBackendInstanceNotAvailable");
151 strcat(err, "Backend instance not found");
154 err_class = "ECBackendInstanceNotAvailable";
155 err_msg = "Backend instance not found";
152156 break;
153157 case -EINSUFFFRAGS:
154 eo = import_class("pyeclib.ec_iface",
155 "ECInsufficientFragments");
156 strcat(err, "Insufficient number of fragments");
158 err_class = "ECInsufficientFragments";
159 err_msg = "Insufficient number of fragments";
157160 break;
158161 case -EBACKENDNOTSUPP:
159 eo = import_class("pyeclib.ec_iface",
160 "ECBackendNotSupported");
161 strcat(err, "Backend not supported");
162 err_class = "ECBackendNotSupported";
163 err_msg = "Backend not supported";
162164 break;
163165 case -EINVALIDPARAMS:
164 eo = import_class("pyeclib.ec_iface",
165 "ECInvalidParameter");
166 strcat(err, "Invalid arguments");
166 err_class = "ECInvalidParameter";
167 err_msg = "Invalid arguments";
167168 break;
168169 case -EBADCHKSUM:
169 eo = import_class("pyeclib.ec_iface",
170 "ECBadFragmentChecksum");
171 strcat(err, "Fragment integrity check failed");
170 err_class = "ECBadFragmentChecksum";
171 err_msg = "Fragment integrity check failed";
172172 break;
173173 case -EBADHEADER:
174 eo = import_class("pyeclib.ec_iface",
175 "ECInvalidFragmentMetadata");
176 strcat(err, "Fragment integrity check failed");
174 err_class = "ECInvalidFragmentMetadata";
175 err_msg = "Fragment integrity check failed";
177176 break;
178177 case -ENOMEM:
179 eo = import_class("pyeclib.ec_iface",
180 "ECOutOfMemory");
181 strcat(err, "Out of memory");
178 err_class = "ECOutOfMemory";
179 err_msg = "Out of memory";
182180 break;
183181 default:
184 eo = import_class("pyeclib.ec_iface",
185 "ECDriverError");
186 strcat(err, "Unknown error");
182 err_class = "ECDriverError";
183 err_msg = "Unknown error";
187184 break;
188185 }
189 strcat(err, ". Please inspect syslog for liberasurecode error report.");
190 PyErr_SetString(eo, err);
186 PyObject *eo = import_class("pyeclib.ec_iface", err_class);
187 if (eo != NULL) {
188 snprintf(err, 255,
189 "%s ERROR: %s. Please inspect syslog for liberasurecode error report.",
190 prefix, err_msg);
191 PyErr_SetString(eo, err);
192 }
191193 }
192194
193195 static int stderr_fd;
238240 if (!PyArg_ParseTuple(args, "iii|iiiii",
239241 &k, &m, &backend_id, &hd, &use_inline_chksum,
240242 &use_algsig_chksum, &validate)) {
241 pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_init ERROR: ");
243 pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_init");
242244 return NULL;
243245 }
244246
245247 /* Allocate and initialize the pyeclib object */
246248 pyeclib_handle = (pyeclib_t *) alloc_zeroed_buffer(sizeof(pyeclib_t));
247249 if (NULL == pyeclib_handle) {
248 pyeclib_c_seterr(-ENOMEM, "pyeclib_c_init ERROR: ");
250 pyeclib_c_seterr(-ENOMEM, "pyeclib_c_init");
249251 goto cleanup;
250252 }
251253
261263 if (pyeclib_handle->ec_desc <= 0) {
262264 /* liberasurecode returns status in ec_desc as one of the error codes
263265 * (LIBERASURECODE_ERROR_CODES) defined in erasurecode.h */
264 pyeclib_c_seterr(pyeclib_handle->ec_desc, "pyeclib_c_init ERROR: ");
266 pyeclib_c_seterr(pyeclib_handle->ec_desc, "pyeclib_c_init");
265267 goto cleanup;
266268 }
267269
277279
278280 /* Clean up the allocated memory on error, or update the ref count */
279281 if (pyeclib_obj_handle == NULL) {
280 pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_init ERROR: ");
282 pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_init");
281283 goto cleanup;
282284 } else {
283285 Py_INCREF(pyeclib_obj_handle);
304306 pyeclib_t *pyeclib_handle = NULL; /* pyeclib object to destroy */
305307
306308 if (!PyCapsule_CheckExact(obj)) {
307 pyeclib_c_seterr(-1, "pyeclib_c_destructor ERROR: ");
309 pyeclib_c_seterr(-1, "pyeclib_c_destructor");
308310 return;
309311 }
310312
311313 pyeclib_handle = (pyeclib_t*)PyCapsule_GetPointer(obj, PYECC_HANDLE_NAME);
312314 if (pyeclib_handle == NULL) {
313 pyeclib_c_seterr(-1, "pyeclib_c_destructor ERROR: ");
315 pyeclib_c_seterr(-1, "pyeclib_c_destructor");
314316 } else {
315317 check_and_free_buffer(pyeclib_handle);
316318 }
354356 pyeclib_t *pyeclib_handle = NULL;
355357 PyObject *ret_dict = NULL; /* python dictionary to return */
356358
357 // Prepare variables for return dict to cleanup on exit
358 PyObject *segment_size_key = NULL;
359 PyObject *segment_size_value = NULL;
360 PyObject *last_segment_size_key = NULL;
361 PyObject *last_segment_size_value = NULL;
362 PyObject *fragment_size_key = NULL;
363 PyObject *fragment_size_value = NULL;
364 PyObject *last_fragment_size_key = NULL;
365 PyObject *last_fragment_size_value = NULL;
366 PyObject *num_segments_key = NULL;
367 PyObject *num_segments_value = NULL;
368
369359 int data_len; /* data length from user in bytes */
370360 int segment_size, last_segment_size; /* segment sizes in bytes */
371361 int num_segments; /* total number of segments */
374364
375365 /* Obtain and validate the method parameters */
376366 if (!PyArg_ParseTuple(args, "Oii", &pyeclib_obj_handle, &data_len, &segment_size)) {
377 pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_get_segment_info ERROR: ");
367 pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_get_segment_info");
378368 return NULL;
379369 }
380370 pyeclib_handle = (pyeclib_t*)PyCapsule_GetPointer(pyeclib_obj_handle, PYECC_HANDLE_NAME);
381371 if (pyeclib_handle == NULL) {
382 pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_get_segment_info ERROR: ");
372 pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_get_segment_info");
383373 return NULL;
384374 }
385375
386376 /* The minimum segment size depends on the EC algorithm */
387377 min_segment_size = liberasurecode_get_minimum_encode_size(pyeclib_handle->ec_desc);
388378 if (min_segment_size < 0) {
389 pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_get_segment_info ERROR: ");
379 pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_get_segment_info");
390380 return NULL;
391381 }
392382
415405
416406 fragment_size = liberasurecode_get_fragment_size(pyeclib_handle->ec_desc, data_len);
417407 if (fragment_size < 0) {
418 pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_get_segment_info ERROR: ");
408 pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_get_segment_info");
419409 return NULL;
420410 }
421411
431421
432422 fragment_size = liberasurecode_get_fragment_size(pyeclib_handle->ec_desc, segment_size);
433423 if (fragment_size < 0) {
434 pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_get_segment_info ERROR: ");
424 pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_get_segment_info");
435425 return NULL;
436426 }
437427
451441
452442 last_fragment_size = liberasurecode_get_fragment_size(pyeclib_handle->ec_desc, last_segment_size);
453443 if (fragment_size < 0) {
454 pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_get_segment_info ERROR: ");
444 pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_get_segment_info");
455445 return NULL;
456446 }
457447 }
461451 fragment_size += sizeof(fragment_header_t);
462452
463453 /* Create and return the python dictionary of segment info */
464 ret_dict = PyDict_New();
454 ret_dict = Py_BuildValue(
455 "{s:i, s:i, s:i, s:i, s:i}",
456 "segment_size", segment_size,
457 "last_segment_size", last_segment_size,
458 "fragment_size", fragment_size,
459 "last_fragment_size", last_fragment_size,
460 "num_segments", num_segments);
465461 if (NULL == ret_dict) {
466462 goto error;
467 } else {
468 if((segment_size_key = PyString_FromString("segment_size\0")) == NULL ||
469 (segment_size_value = PyInt_FromLong(segment_size)) == NULL ||
470 PyDict_SetItem(ret_dict, segment_size_key, segment_size_value)) goto error;
471
472 if((last_segment_size_key = PyString_FromString("last_segment_size\0")) == NULL ||
473 (last_segment_size_value = PyInt_FromLong(last_segment_size)) == NULL ||
474 PyDict_SetItem(ret_dict, last_segment_size_key, last_segment_size_value)) goto error;
475
476 if((fragment_size_key = PyString_FromString("fragment_size\0")) == NULL ||
477 (fragment_size_value = PyInt_FromLong(fragment_size)) == NULL ||
478 PyDict_SetItem(ret_dict, fragment_size_key, fragment_size_value)) goto error;
479
480 if((last_fragment_size_key = PyString_FromString("last_fragment_size\0")) == NULL ||
481 (last_fragment_size_value = PyInt_FromLong(last_fragment_size)) == NULL ||
482 PyDict_SetItem(ret_dict, last_fragment_size_key, last_fragment_size_value)) goto error;
483
484 if((num_segments_key = PyString_FromString("num_segments\0")) == NULL ||
485 (num_segments_value = PyInt_FromLong(num_segments)) == NULL ||
486 PyDict_SetItem(ret_dict, num_segments_key, num_segments_value)) goto error;
487463 }
488464
489465 exit:
490 Py_XDECREF(segment_size_key);
491 Py_XDECREF(segment_size_value);
492 Py_XDECREF(last_segment_size_key);
493 Py_XDECREF(last_segment_size_value);
494 Py_XDECREF(fragment_size_key);
495 Py_XDECREF(fragment_size_value);
496 Py_XDECREF(last_fragment_size_key);
497 Py_XDECREF(last_fragment_size_value);
498 Py_XDECREF(num_segments_key);
499 Py_XDECREF(num_segments_value);
500466 return ret_dict;
501467
502468 error:
503469 // To prevent unexpected call, this is placed after return call
504 pyeclib_c_seterr(-ENOMEM, "pyeclib_c_get_segment_info ERROR: ");
470 pyeclib_c_seterr(-ENOMEM, "pyeclib_c_get_segment_info");
505471 Py_XDECREF(ret_dict);
506472 ret_dict = NULL;
507473 goto exit;
532498
533499 /* Assume binary data (force "byte array" input) */
534500 if (!PyArg_ParseTuple(args, ENCODE_ARGS, &pyeclib_obj_handle, &data, &data_len)) {
535 pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_encode ERROR: ");
501 pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_encode");
536502 return NULL;
537503 }
538504 pyeclib_handle = (pyeclib_t*)PyCapsule_GetPointer(pyeclib_obj_handle, PYECC_HANDLE_NAME);
539505 if (pyeclib_handle == NULL) {
540 pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_encode ERROR: ");
506 pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_encode");
541507 return NULL;
542508 }
543509
544510 ret = liberasurecode_encode(pyeclib_handle->ec_desc, data, data_len, &encoded_data, &encoded_parity, &fragment_len);
545511 if (ret < 0) {
546 pyeclib_c_seterr(ret, "pyeclib_c_encode ERROR: ");
512 pyeclib_c_seterr(ret, "pyeclib_c_encode");
547513 return NULL;
548514 }
549515
550516 /* Create the python list of fragments to return */
551517 list_of_strips = PyList_New(pyeclib_handle->ec_args.k + pyeclib_handle->ec_args.m);
552518 if (NULL == list_of_strips) {
553 pyeclib_c_seterr(-ENOMEM, "pyeclib_c_encode ERROR: ");
519 pyeclib_c_seterr(-ENOMEM, "pyeclib_c_encode");
554520 return NULL;
555521 }
556522
600566
601567 /* Obtain and validate the method parameters */
602568 if (!PyArg_ParseTuple(args, "OOO", &pyeclib_obj_handle, &reconstruct_list, &exclude_list)) {
603 pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_get_required_fragments ERROR: ");
569 pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_get_required_fragments");
604570 return NULL;
605571 }
606572 pyeclib_handle = (pyeclib_t*)PyCapsule_GetPointer(pyeclib_obj_handle, PYECC_HANDLE_NAME);
607573 if (pyeclib_handle == NULL) {
608 pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_get_required_fragments ERROR: ");
574 pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_get_required_fragments");
609575 return NULL;
610576 }
611577 k = pyeclib_handle->ec_args.k;
615581 num_missing = (int) PyList_Size(reconstruct_list);
616582 c_reconstruct_list = (int *) alloc_zeroed_buffer((num_missing + 1) * sizeof(int));
617583 if (NULL == c_reconstruct_list) {
618 pyeclib_c_seterr(-ENOMEM, "pyeclib_c_get_required_fragments ERROR: ");
584 pyeclib_c_seterr(-ENOMEM, "pyeclib_c_get_required_fragments");
619585 return NULL;
620586 }
621587 c_reconstruct_list[num_missing] = -1;
628594 num_exclude = (int) PyList_Size(exclude_list);
629595 c_exclude_list = (int *) alloc_zeroed_buffer((num_exclude + 1) * sizeof(int));
630596 if (NULL == c_exclude_list) {
631 pyeclib_c_seterr(-ENOMEM, "pyeclib_c_get_required_fragments ERROR: ");
597 pyeclib_c_seterr(-ENOMEM, "pyeclib_c_get_required_fragments");
632598 goto exit;
633599 }
634600 c_exclude_list[num_exclude] = -1;
640606
641607 fragments_needed = alloc_zeroed_buffer(sizeof(int) * (k + m));
642608 if (NULL == fragments_needed) {
643 pyeclib_c_seterr(-ENOMEM, "pyeclib_c_get_required_fragments ERROR: ");
609 pyeclib_c_seterr(-ENOMEM, "pyeclib_c_get_required_fragments");
644610 goto exit;
645611 }
646612
647613 ret = liberasurecode_fragments_needed(pyeclib_handle->ec_desc, c_reconstruct_list,
648614 c_exclude_list, fragments_needed);
649615 if (ret < 0) {
650 pyeclib_c_seterr(ret, "pyeclib_c_get_required_fragments ERROR: ");
616 pyeclib_c_seterr(ret, "pyeclib_c_get_required_fragments");
651617 goto exit;
652618 }
653619
654620 /* Post-process into a Python list */
655621 fragment_idx_list = PyList_New(0);
656622 if (NULL == fragment_idx_list) {
657 pyeclib_c_seterr(-ENOMEM, "pyeclib_c_get_required_fragments ERROR: ");
623 pyeclib_c_seterr(-ENOMEM, "pyeclib_c_get_required_fragments");
658624 goto exit;
659625 }
660626
703669 /* Obtain and validate the method parameters */
704670 if (!PyArg_ParseTuple(args, "OOii", &pyeclib_obj_handle, &fragments,
705671 &fragment_len, &destination_idx)) {
706 pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_reconstruct ERROR: ");
672 pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_reconstruct");
707673 return NULL;
708674 }
709675 pyeclib_handle = (pyeclib_t*)PyCapsule_GetPointer(pyeclib_obj_handle, PYECC_HANDLE_NAME);
710676 if (pyeclib_handle == NULL) {
711 pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_reconstruct ERROR: ");
677 pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_reconstruct");
712678 return NULL;
713679 }
714680
715681 /* Pre-processing Python data structures */
716682 if (!PyList_Check(fragments)) {
717 pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_reconstruct ERROR: ");
683 pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_reconstruct");
718684 return NULL;
719685 }
720686
722688
723689 c_fragments = (char **) alloc_zeroed_buffer(sizeof(char *) * num_fragments);
724690 if (NULL == c_fragments) {
725 pyeclib_c_seterr(-ENOMEM, "pyeclib_c_reconstruct ERROR: ");
691 pyeclib_c_seterr(-ENOMEM, "pyeclib_c_reconstruct");
726692 goto error;
727693 }
728694
729695 c_reconstructed = (char*) alloc_zeroed_buffer(sizeof(char) * fragment_len);
730696 if (NULL == c_fragments) {
731 pyeclib_c_seterr(-ENOMEM, "pyeclib_c_reconstruct ERROR: ");
697 pyeclib_c_seterr(-ENOMEM, "pyeclib_c_reconstruct");
732698 goto error;
733699 }
734700
746712 destination_idx,
747713 c_reconstructed);
748714 if (ret < 0) {
749 pyeclib_c_seterr(ret, "pyeclib_c_reconstruct ERROR: ");
715 pyeclib_c_seterr(ret, "pyeclib_c_reconstruct");
750716 reconstructed = NULL;
751717 } else {
752718 reconstructed = PY_BUILDVALUE_OBJ_LEN(c_reconstructed, fragment_len);
798764 /* Obtain and validate the method parameters */
799765 if (!PyArg_ParseTuple(args, "OOi|OO",&pyeclib_obj_handle, &fragments,
800766 &fragment_len, &ranges, &metadata_checks_obj)) {
801 pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_decode ERROR: ");
767 pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_decode");
802768 return NULL;
803769 }
804770
814780
815781 pyeclib_handle = (pyeclib_t*)PyCapsule_GetPointer(pyeclib_obj_handle, PYECC_HANDLE_NAME);
816782 if (pyeclib_handle == NULL) {
817 pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_decode ERROR: ");
783 pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_decode");
818784 return NULL;
819785 }
820786 if (!PyList_Check(fragments)) {
821 pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_decode ERROR: ");
787 pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_decode");
822788 return NULL;
823789 }
824790
829795 }
830796
831797 if (pyeclib_handle->ec_args.k > num_fragments) {
832 pyeclib_c_seterr(-EINSUFFFRAGS, "pyeclib_c_decode ERROR: ");
798 pyeclib_c_seterr(-EINSUFFFRAGS, "pyeclib_c_decode");
833799 return NULL;
834800 }
835801
836802 if (num_ranges > 0) {
837803 c_ranges = (pyeclib_byte_range_t*)malloc(sizeof(pyeclib_byte_range_t) * num_ranges);
838804 if (NULL == c_ranges) {
839 pyeclib_c_seterr(-ENOMEM, "pyeclib_c_decode ERROR: ");
805 pyeclib_c_seterr(-ENOMEM, "pyeclib_c_decode");
840806 goto error;
841807 }
842808 for (i = 0; i < num_ranges; i++) {
853819 begin = PyInt_AsLong(py_begin);
854820 #endif
855821 else {
856 pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_decode invalid range ERROR: ");
822 pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_decode invalid range");
857823 goto error;
858824 }
859825 if (PyLong_Check(py_end))
863829 end = PyInt_AsLong(py_end);
864830 #endif
865831 else {
866 pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_decode invalid range ERROR: ");
832 pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_decode invalid range");
867833 goto error;
868834 }
869835
872838
873839 range_payload_size += c_ranges[i].length;
874840 } else {
875 pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_decode invalid range ERROR: ");
841 pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_decode invalid range");
876842 goto error;
877843 }
878844 }
899865 &orig_data_size);
900866
901867 if (ret < 0) {
902 pyeclib_c_seterr(ret, "pyeclib_c_decode ERROR: ");
868 pyeclib_c_seterr(ret, "pyeclib_c_decode");
903869 goto error;
904870 }
905871
908874 } else {
909875 ret_payload = PyList_New(num_ranges);
910876 if (NULL == ret_payload) {
911 pyeclib_c_seterr(-ENOMEM, "pyeclib_c_decode ERROR: ");
877 pyeclib_c_seterr(-ENOMEM, "pyeclib_c_decode");
912878 goto error;
913879 }
914880 range_payload_size = 0;
915881 for (i = 0; i < num_ranges; i++) {
916882 /* Check that range is within the original buffer */
917883 if (c_ranges[i].offset + c_ranges[i].length > orig_data_size) {
918 pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_decode invalid range ERROR: ");
884 pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_decode invalid range");
919885 goto error;
920886 }
921887 PyList_SET_ITEM(ret_payload, i,
1005971 break;
1006972 case 7:
1007973 backend_id_str = "isa_l_rs_cauchy\0";
974 break;
975 case 8:
976 backend_id_str = "libphazr\0";
1008977 break;
1009978 default:
1010979 backend_id_str = "unknown\0";
10481017 "backend_version", fragment_metadata->backend_version);
10491018 encoded_chksum = check_and_free_buffer(encoded_chksum);
10501019 if (metadata_dict == NULL) {
1051 pyeclib_c_seterr(-ENOMEM, "fragment_metadata_to_dict ERROR: ");
1020 pyeclib_c_seterr(-ENOMEM, "fragment_metadata_to_dict");
10521021 return NULL;
10531022 }
10541023 return metadata_dict;
10761045
10771046 /* Obtain and validate the method parameters */
10781047 if (!PyArg_ParseTuple(args, GET_METADATA_ARGS, &pyeclib_obj_handle, &fragment, &fragment_len, &formatted)) {
1079 pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_get_metadata ERROR: ");
1048 pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_get_metadata");
10801049 return NULL;
10811050 }
10821051 pyeclib_handle = (pyeclib_t*)PyCapsule_GetPointer(pyeclib_obj_handle, PYECC_HANDLE_NAME);
10831052 if (pyeclib_handle == NULL) {
1084 pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_get_metadata ERROR: ");
1053 pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_get_metadata");
10851054 return NULL;
10861055 }
10871056
10881057 ret = liberasurecode_get_fragment_metadata(fragment, &c_fragment_metadata);
10891058
10901059 if (ret < 0) {
1091 pyeclib_c_seterr(ret, "pyeclib_c_get_metadata ERROR: ");
1060 pyeclib_c_seterr(ret, "pyeclib_c_get_metadata");
10921061 fragment_metadata = NULL;
10931062 } else {
10941063 if (formatted) {
11321101
11331102 /* Obtain and validate the method parameters */
11341103 if (!PyArg_ParseTuple(args, "OO", &pyeclib_obj_handle, &fragment_metadata_list)) {
1135 pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_encode ERROR: ");
1104 pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_encode");
11361105 return NULL;
11371106 }
11381107 pyeclib_handle = (pyeclib_t*)PyCapsule_GetPointer(pyeclib_obj_handle, PYECC_HANDLE_NAME);
11391108 if (pyeclib_handle == NULL) {
1140 pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_encode ERROR: ");
1109 pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_encode");
11411110 return NULL;
11421111 }
11431112 k = pyeclib_handle->ec_args.k;
11441113 m = pyeclib_handle->ec_args.m;
11451114 num_fragments = k + m;
11461115 if (num_fragments != PyList_Size(fragment_metadata_list)) {
1147 pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_encode ERROR: ");
1116 pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_encode");
11481117 return NULL;
11491118 }
11501119
11521121 size = sizeof(char * ) * num_fragments;
11531122 c_fragment_metadata_list = (char **) alloc_zeroed_buffer(size);
11541123 if (NULL == c_fragment_metadata_list) {
1155 pyeclib_c_seterr(-ENOMEM, "pyeclib_c_encode ERROR: ");
1124 pyeclib_c_seterr(-ENOMEM, "pyeclib_c_encode");
11561125 goto error;
11571126 }
11581127
12031172 const ec_backend_id_t backend_id;
12041173
12051174 if (!PyArg_ParseTuple(args, "i", &backend_id)) {
1206 pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_check_backend_available ERROR: ");
1175 pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_check_backend_available");
12071176 return NULL;
12081177 }
12091178
12161185
12171186 static PyObject*
12181187 pyeclib_c_liberasurecode_version(PyObject *self, PyObject *args) {
1219 return PyInt_FromLong(LIBERASURECODE_VERSION);
1188 void *hLib;
1189 char *err;
1190 uint32_t (*hGetVersion)(void);
1191
1192 dlerror();
1193 hLib = dlopen("liberasurecode.so", RTLD_LAZY);
1194 /* It's important that we clear the last error before calling dysym */
1195 err = dlerror();
1196 if (err) {
1197 /* This should never actually get hit; since we're using various
1198 symbols already, liberasurecode.so should *already* be loaded. */
1199 return PyInt_FromLong(LIBERASURECODE_VERSION);
1200 }
1201
1202 hGetVersion = dlsym(hLib, "liberasurecode_get_version");
1203 err = dlerror();
1204 if (err) {
1205 /* This is the important bit. Old version, doesn't have get_version
1206 support; fall back to old behavior. */
1207 dlclose(hLib);
1208 return PyInt_FromLong(LIBERASURECODE_VERSION);
1209 }
1210
1211 uint32_t version = (*hGetVersion)();
1212 dlclose(hLib);
1213 return Py_BuildValue("k", version);
12201214 }
12211215
12221216 static PyMethodDef PyECLibMethods[] = {
+66
-5
test/test_pyeclib_api.py less more
3030 from distutils.version import StrictVersion
3131 from itertools import combinations
3232
33 import pyeclib.ec_iface
3334 from pyeclib.ec_iface import ECBackendNotSupported
3435 from pyeclib.ec_iface import ECDriver
3536 from pyeclib.ec_iface import ECDriverError
3637 from pyeclib.ec_iface import ECInsufficientFragments
3738 from pyeclib.ec_iface import ECInvalidFragmentMetadata
39 from pyeclib.ec_iface import ECInvalidParameter
3840 from pyeclib.ec_iface import PyECLib_EC_Types
3941 from pyeclib.ec_iface import ALL_EC_TYPES
4042 from pyeclib.ec_iface import VALID_EC_TYPES
133135 "Invalid Argument: m is required")
134136
135137 with self.assertRaises(ECDriverError) as err_context:
136 # m is smaller than 1
138 # k is smaller than 1
137139 ECDriver(ec_type=ec_type, k=-100, m=1)
138140 self.assertEqual(str(err_context.exception),
139141 "Invalid number of data fragments (k)")
142144 # m is smaller than 1
143145 ECDriver(ec_type=ec_type, k=1, m=-100)
144146 self.assertEqual(str(err_context.exception),
145 "Invalid number of data fragments (m)")
147 "Invalid number of parity fragments (m)")
146148
147149 def test_valid_ec_types(self):
148150 # Build list of available types and compare to VALID_EC_TYPES
150152 for _type in ALL_EC_TYPES:
151153 try:
152154 if _type is 'shss':
155 _k = 10
156 _m = 4
157 elif _type is 'libphazr':
158 _k = 4
153159 _m = 4
154160 else:
161 _k = 10
155162 _m = 5
156 ECDriver(k=10, m=_m, ec_type=_type, validate=True)
163 ECDriver(k=_k, m=_m, ec_type=_type, validate=True)
157164 available_ec_types.append(_type)
158165 except Exception:
159166 # ignore any errors, assume backend not available
170177 try:
171178 if _type is 'shss':
172179 _instance = ECDriver(k=10, m=4, ec_type=_type)
180 elif _type is 'libphazr':
181 _instance = ECDriver(k=4, m=4, ec_type=_type)
173182 else:
174183 _instance = ECDriver(k=10, m=5, ec_type=_type)
175184 except ECDriverError:
176 self.fail("%p: %s algorithm not supported" % _instance, _type)
185 self.fail("%s algorithm not supported" % _type)
177186
178187 self.assertRaises(ECBackendNotSupported, ECDriver, k=10, m=5,
179188 ec_type="invalid_algo")
243252 chksum_type=csum))
244253 pyeclib_drivers.append(ECDriver(k=11, m=7, ec_type=_type6,
245254 chksum_type=csum))
255
256 _type7 = 'libphazr'
257 if _type7 in VALID_EC_TYPES:
258 pyeclib_drivers.append(ECDriver(k=4, m=4, ec_type=_type7,
259 chksum_type=csum))
246260 return pyeclib_drivers
247261
248262 def test_small_encode(self):
255269 decoded_str = pyeclib_driver.decode(encoded_fragments)
256270
257271 self.assertTrue(decoded_str == encode_str)
272
273 def test_encode_invalid_params(self):
274 pyeclib_drivers = self.get_pyeclib_testspec()
275 encode_args = [u'\U0001F0A1', 3, object(), None, {}, []]
276
277 for pyeclib_driver in pyeclib_drivers:
278 for encode_str in encode_args:
279 with self.assertRaises(ECInvalidParameter):
280 pyeclib_driver.encode(encode_str)
281
282 def test_attribute_error_in_the_error_handling(self):
283 pyeclib_drivers = self.get_pyeclib_testspec()
284 self.assertGreater(len(pyeclib_drivers), 0)
285 pyeclib_driver = pyeclib_drivers[0]
286
287 del pyeclib.ec_iface.ECInvalidParameter
288 try:
289 with self.assertRaises(AttributeError): # !!
290 pyeclib_driver.encode(3)
291 finally:
292 pyeclib.ec_iface.ECInvalidParameter = ECInvalidParameter
293
294 def test_import_error_in_the_error_handling(self):
295 pyeclib_drivers = self.get_pyeclib_testspec()
296 self.assertGreater(len(pyeclib_drivers), 0)
297 pyeclib_driver = pyeclib_drivers[0]
298
299 from six.moves import builtins
300 real_import = builtins.__import__
301
302 def fake_import(*a, **kw):
303 raise ImportError
304
305 builtins.__import__ = fake_import
306 try:
307 with self.assertRaises(ImportError): # !!
308 pyeclib_driver.encode(3)
309 finally:
310 builtins.__import__ = real_import
258311
259312 def test_decode_reconstruct_with_fragment_iterator(self):
260313 pyeclib_drivers = self.get_pyeclib_testspec()
739792 def dummy(self, ec_type=ec_type):
740793 if ec_type not in VALID_EC_TYPES:
741794 raise unittest.SkipTest
742 k, m = 10, 4 if ec_type == 'shss' else 5
795 if ec_type == 'shss':
796 k = 10
797 m = 4
798 elif ec_type == 'libphazr':
799 k = 4
800 m = 4
801 else:
802 k = 10
803 m = 5
743804 ECDriver(k=k, m=m, ec_type=ec_type)
744805 dummy.__name__ = 'test_%s_available' % ec_type
745806 cls_dict[dummy.__name__] = dummy
+34
-1
test/test_pyeclib_c.py less more
7777 (PyECLib_EC_Types.shss, 10, 4),
7878 (PyECLib_EC_Types.shss, 20, 4),
7979 (PyECLib_EC_Types.shss, 11, 7)]
80 self.libphazr = [(PyECLib_EC_Types.libphazr, 4, 4)]
8081
8182 # Input temp files for testing
8283 self.sizes = ["101-K", "202-K", "303-K"]
146147 :return average encode time
147148 """
148149 timer = Timer()
149 tsum = 0
150150 handle = pyeclib_c.init(num_data, num_parity, ec_type, hd)
151151 whole_file_bytes = self.get_tmp_file(file_size).read()
152152
288288 size_desc = size_str.split("-")
289289 size = float(size_desc[0])
290290
291 if avg_time == 0:
292 return '? (test finished too fast to calculate throughput)'
293
291294 if size_desc[1] == 'M':
292295 throughput = size / avg_time
293296 elif size_desc[1] == 'K':
326329 @require_backend("shss")
327330 def test_shss(self):
328331 for (ec_type, k, m) in self.shss:
332 print(("\nRunning tests for %s k=%d, m=%d" % (ec_type, k, m)))
333
334 success = self._test_get_required_fragments(k, m, ec_type)
335 self.assertTrue(success)
336
337 for size_str in self.sizes:
338 avg_time = self.time_encode(k, m, ec_type.value, 0,
339 size_str,
340 self.iterations)
341 print("Encode (%s): %s" %
342 (size_str, self.get_throughput(avg_time, size_str)))
343
344 for size_str in self.sizes:
345 success, avg_time = self.time_decode(k, m, ec_type.value, 0,
346 size_str,
347 self.iterations)
348 self.assertTrue(success)
349 print("Decode (%s): %s" %
350 (size_str, self.get_throughput(avg_time, size_str)))
351
352 for size_str in self.sizes:
353 success, avg_time = self.time_reconstruct(
354 k, m, ec_type.value, 0, size_str, self.iterations)
355 self.assertTrue(success)
356 print("Reconstruct (%s): %s" %
357 (size_str, self.get_throughput(avg_time, size_str)))
358
359 @require_backend("libphazr")
360 def test_libphazr(self):
361 for (ec_type, k, m) in self.libphazr:
329362 print(("\nRunning tests for %s k=%d, m=%d" % (ec_type, k, m)))
330363
331364 success = self._test_get_required_fragments(k, m, ec_type)
+1
-0
test-requirements.txt less more
33
44 coverage
55 nose
6 six>=1.9.0
67 sphinx>=1.1.2,!=1.2.0,!=1.3b1,<1.3 # BSD
+2
-2
tox.ini less more
00 [tox]
1 envlist = py27,py34,pep8
1 envlist = py27,py35,pep8
22
33 [testenv]
44 deps =
55 -r{toxinidir}/test-requirements.txt
66 commands=
7 nosetests -v test/
7 nosetests --no-path-adjustment -v test/
88
99 [testenv:pep8]
1010 deps=