Codebase list python-pyeclib / 5026e16
Make library_import_str arg to ECDriver conditional Signed-off-by: Tushar Gohad <tushar.gohad@intel.com> Tushar Gohad 9 years ago
8 changed file(s) with 69 addition(s) and 122 deletion(s). Raw diff Collapse all Expand all
+2
-2
README less more
3131
3232 PyEClib initialization::
3333
34 ec_driver = ECDriver("pyeclib.core.ECPyECLibDriver",
35 k=<num_encoded_data_fragments>,
34 ec_driver = ECDriver(k=<num_encoded_data_fragments>,
3635 m=<num_encoded_parity_fragments>,
3736 ec_type=<ec_scheme>))
3837
3938 Supported ``ec_type`` values:
4039
4140 * ``jerasure_rs_vand`` => Vandermonde Reed-Solomon encoding
41 * ``jerasure_rs_cauchy`` => Cauchy Reed-Solomon encoding (Jerasure variant)
4242 * ``flat_xor_hd_3``, ``flat_xor_hd_4`` => Flat-XOR based HD combination codes
4343
4444 A configuration utility is provided to help compare available EC schemes in
+5
-10
src/python/pyeclib/ec_iface.py less more
114114 # Main ECDriver class
115115 class ECDriver(object):
116116
117 def __init__(self, library_import_str, *args, **kwargs):
117 def __init__(self, *args, **kwargs):
118118 self.k = -1
119119 self.m = -1
120120 self.ec_type = None
121121 self.chksum_type = None
122 self.library_import_str = None
123122 for (key, value) in kwargs.items():
124123 if key == "k":
125124 try:
148147 raise ECDriverError(
149148 "%s is not a valid checksum type for PyECLib!" % value)
150149
151 if library_import_str is not None:
152 self.library_import_str = library_import_str
153 else:
154 raise ECDriverError(
155 "Library import string (library_import_str) was not specified "
156 "and is a required argument!")
150 self.library_import_str = kwargs.pop('library_import_str',
151 'pyeclib.core.ECPyECLibDriver')
157152 #
158153 # Instantiate EC backend driver
159154 #
160155 self.ec_lib_reference = create_instance(
161 library_import_str,
156 self.library_import_str,
162157 k=self.k,
163158 m=self.m,
164159 ec_type=self.ec_type,
188183 if len(not_implemented_str) > 0:
189184 raise ECDriverError(
190185 "The following required methods are not implemented "
191 "in %s: %s" % (library_import_str, not_implemented_str))
186 "in %s: %s" % (self.library_import_str, not_implemented_str))
192187
193188 def encode(self, data_bytes):
194189 """
+56
-85
test/test_pyeclib_api.py less more
4040 class TestNullDriver(unittest.TestCase):
4141
4242 def setUp(self):
43 self.null_driver = ECDriver("pyeclib.core.ECNullDriver", k=8, m=2)
43 self.null_driver = ECDriver(
44 library_import_str="pyeclib.core.ECNullDriver", k=8, m=2)
4445
4546 def tearDown(self):
4647 pass
5354 class TestStripeDriver(unittest.TestCase):
5455
5556 def setUp(self):
56 self.stripe_driver = ECDriver("pyeclib.core.ECStripingDriver", k=8, m=0)
57 self.stripe_driver = ECDriver(
58 library_import_str="pyeclib.core.ECStripingDriver", k=8, m=0)
5759
5860 def tearDown(self):
5961 pass
101103
102104 def test_small_encode(self):
103105 pyeclib_drivers = []
104 pyeclib_drivers.append(ECDriver("pyeclib.core.ECPyECLibDriver",
105 k=12, m=2, ec_type="jerasure_rs_vand"))
106 pyeclib_drivers.append(ECDriver("pyeclib.core.ECPyECLibDriver",
107 k=11, m=2, ec_type="jerasure_rs_vand"))
108 pyeclib_drivers.append(ECDriver("pyeclib.core.ECPyECLibDriver",
109 k=10, m=2, ec_type="jerasure_rs_vand"))
106 pyeclib_drivers.append(ECDriver(k=12, m=2, ec_type="jerasure_rs_vand"))
107 pyeclib_drivers.append(ECDriver(k=11, m=2, ec_type="jerasure_rs_vand"))
108 pyeclib_drivers.append(ECDriver(k=10, m=2, ec_type="jerasure_rs_vand"))
110109
111110 encode_strs = [b"a", b"hello", b"hellohyhi", b"yo"]
112111
120119 def disabled_test_verify_fragment_algsig_chksum_fail(self):
121120 pyeclib_drivers = []
122121 pyeclib_drivers.append(
123 ECDriver("pyeclib.core.ECPyECLibDriver",
124 k=12, m=2, ec_type="jerasure_rs_vand", chksum_type="algsig"))
125 pyeclib_drivers.append(
126 ECDriver("pyeclib.core.ECPyECLibDriver",
127 k=12, m=3, ec_type="jerasure_rs_vand", chksum_type="algsig"))
128 pyeclib_drivers.append(
129 ECDriver("pyeclib.core.ECPyECLibDriver",
130 k=12, m=6, ec_type="flat_xor_hd", chksum_type="algsig"))
131 pyeclib_drivers.append(
132 ECDriver("pyeclib.core.ECPyECLibDriver",
133 k=10, m=5, ec_type="flat_xor_hd", chksum_type="algsig"))
122 ECDriver(k=12, m=2, ec_type="jerasure_rs_vand", chksum_type="algsig"))
123 pyeclib_drivers.append(
124 ECDriver(k=12, m=3, ec_type="jerasure_rs_vand", chksum_type="algsig"))
125 pyeclib_drivers.append(
126 ECDriver(k=12, m=6, ec_type="flat_xor_hd", chksum_type="algsig"))
127 pyeclib_drivers.append(
128 ECDriver(k=10, m=5, ec_type="flat_xor_hd", chksum_type="algsig"))
134129
135130 filesize = 1024 * 1024 * 3
136131 file_str = ''.join(random.choice(ascii_letters) for i in range(filesize))
158153 def disabled_test_verify_fragment_inline_succeed(self):
159154 pyeclib_drivers = []
160155 pyeclib_drivers.append(
161 ECDriver("pyeclib.core.ECPyECLibDriver",
162 k=12, m=2, ec_type="jerasure_rs_vand", chksum_type="algsig"))
163 pyeclib_drivers.append(
164 ECDriver("pyeclib.core.ECPyECLibDriver",
165 k=12, m=3, ec_type="jerasure_rs_vand", chksum_type="algsig"))
166 pyeclib_drivers.append(
167 ECDriver("pyeclib.core.ECPyECLibDriver",
168 k=12, m=6, ec_type="flat_xor_hd", chksum_type="algsig"))
169 pyeclib_drivers.append(
170 ECDriver("pyeclib.core.ECPyECLibDriver",
171 k=10, m=5, ec_type="flat_xor_hd", chksum_type="algsig"))
156 ECDriver(k=12, m=2, ec_type="jerasure_rs_vand", chksum_type="algsig"))
157 pyeclib_drivers.append(
158 ECDriver(k=12, m=3, ec_type="jerasure_rs_vand", chksum_type="algsig"))
159 pyeclib_drivers.append(
160 ECDriver(k=12, m=6, ec_type="flat_xor_hd", chksum_type="algsig"))
161 pyeclib_drivers.append(
162 ECDriver(k=10, m=5, ec_type="flat_xor_hd", chksum_type="algsig"))
172163
173164 filesize = 1024 * 1024 * 3
174165 file_str = ''.join(random.choice(ascii_letters) for i in range(filesize))
189180 def disabled_test_verify_fragment_inline_chksum_fail(self):
190181 pyeclib_drivers = []
191182 pyeclib_drivers.append(
192 ECDriver("pyeclib.core.ECPyECLibDriver",
193 k=12, m=2, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
194 pyeclib_drivers.append(
195 ECDriver("pyeclib.core.ECPyECLibDriver",
196 k=12, m=3, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
197 pyeclib_drivers.append(
198 ECDriver("pyeclib.core.ECPyECLibDriver",
199 k=12, m=4, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
200 pyeclib_drivers.append(
201 ECDriver("pyeclib.core.ECPyECLibDriver",
202 k=12, m=2, ec_type="jerasure_rs_cauchy", chksum_type="inline_crc32"))
183 ECDriver(k=12, m=2, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
184 pyeclib_drivers.append(
185 ECDriver(k=12, m=3, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
186 pyeclib_drivers.append(
187 ECDriver(k=12, m=4, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
188 pyeclib_drivers.append(
189 ECDriver(k=12, m=2, ec_type="jerasure_rs_cauchy", chksum_type="inline_crc32"))
203190
204191 filesize = 1024 * 1024 * 3
205192 file_str = ''.join(random.choice(ascii_letters) for i in range(filesize))
232219 def disabled_test_verify_fragment_inline_chksum_succeed(self):
233220 pyeclib_drivers = []
234221 pyeclib_drivers.append(
235 ECDriver("pyeclib.core.ECPyECLibDriver",
236 k=12, m=2, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
237 pyeclib_drivers.append(
238 ECDriver("pyeclib.core.ECPyECLibDriver",
239 k=12, m=3, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
240 pyeclib_drivers.append(
241 ECDriver("pyeclib.core.ECPyECLibDriver",
242 k=12, m=4, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
243 pyeclib_drivers.append(
244 ECDriver("pyeclib.core.ECPyECLibDriver",
245 k=12, m=2, ec_type="jerasure_rs_cauchy", chksum_type="inline_crc32"))
222 ECDriver(k=12, m=2, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
223 pyeclib_drivers.append(
224 ECDriver(k=12, m=3, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
225 pyeclib_drivers.append(
226 ECDriver(k=12, m=4, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
227 pyeclib_drivers.append(
228 ECDriver(k=12, m=2, ec_type="jerasure_rs_cauchy", chksum_type="inline_crc32"))
246229
247230 filesize = 1024 * 1024 * 3
248231 file_str = ''.join(random.choice(ascii_letters) for i in range(filesize))
263246 def test_get_segment_byterange_info(self):
264247 pyeclib_drivers = []
265248 pyeclib_drivers.append(
266 ECDriver("pyeclib.core.ECPyECLibDriver",
267 k=12, m=2, ec_type="jerasure_rs_vand"))
268 pyeclib_drivers.append(
269 ECDriver("pyeclib.core.ECPyECLibDriver",
270 k=11, m=2, ec_type="jerasure_rs_vand"))
271 pyeclib_drivers.append(
272 ECDriver("pyeclib.core.ECPyECLibDriver",
273 k=10, m=2, ec_type="jerasure_rs_vand"))
249 ECDriver(k=12, m=2, ec_type="jerasure_rs_vand"))
250 pyeclib_drivers.append(
251 ECDriver(k=11, m=2, ec_type="jerasure_rs_vand"))
252 pyeclib_drivers.append(
253 ECDriver(k=10, m=2, ec_type="jerasure_rs_vand"))
274254
275255 file_size = 1024 * 1024
276256 segment_size = 3 * 1024
299279 def test_get_segment_info(self):
300280 pyeclib_drivers = []
301281 pyeclib_drivers.append(
302 ECDriver("pyeclib.core.ECPyECLibDriver",
303 k=12, m=2, ec_type="jerasure_rs_vand"))
304 pyeclib_drivers.append(
305 ECDriver("pyeclib.core.ECPyECLibDriver",
306 k=11, m=2, ec_type="jerasure_rs_vand"))
307 pyeclib_drivers.append(
308 ECDriver("pyeclib.core.ECPyECLibDriver",
309 k=10, m=2, ec_type="jerasure_rs_vand"))
282 ECDriver(k=12, m=2, ec_type="jerasure_rs_vand"))
283 pyeclib_drivers.append(
284 ECDriver(k=11, m=2, ec_type="jerasure_rs_vand"))
285 pyeclib_drivers.append(
286 ECDriver(k=10, m=2, ec_type="jerasure_rs_vand"))
310287
311288 file_sizes = [
312289 1024 * 1024,
374351 def test_rs(self):
375352 pyeclib_drivers = []
376353 pyeclib_drivers.append(
377 ECDriver("pyeclib.core.ECPyECLibDriver",
378 k=12, m=2, ec_type="jerasure_rs_vand"))
379 pyeclib_drivers.append(
380 ECDriver("pyeclib.core.ECPyECLibDriver",
381 k=12, m=2, ec_type="jerasure_rs_cauchy"))
382 pyeclib_drivers.append(
383 ECDriver("pyeclib.core.ECPyECLibDriver",
384 k=12, m=3, ec_type="jerasure_rs_vand"))
385 pyeclib_drivers.append(
386 ECDriver("pyeclib.core.ECPyECLibDriver",
387 k=12, m=3, ec_type="jerasure_rs_cauchy"))
388 pyeclib_drivers.append(
389 ECDriver("pyeclib.core.ECPyECLibDriver",
390 k=12, m=6, ec_type="flat_xor_hd"))
391 pyeclib_drivers.append(
392 ECDriver("pyeclib.core.ECPyECLibDriver",
393 k=10, m=5, ec_type="flat_xor_hd"))
354 ECDriver(k=12, m=2, ec_type="jerasure_rs_vand"))
355 pyeclib_drivers.append(
356 ECDriver(k=12, m=2, ec_type="jerasure_rs_cauchy"))
357 pyeclib_drivers.append(
358 ECDriver(k=12, m=3, ec_type="jerasure_rs_vand"))
359 pyeclib_drivers.append(
360 ECDriver(k=12, m=3, ec_type="jerasure_rs_cauchy"))
361 pyeclib_drivers.append(
362 ECDriver(k=12, m=6, ec_type="flat_xor_hd"))
363 pyeclib_drivers.append(
364 ECDriver(k=10, m=5, ec_type="flat_xor_hd"))
394365
395366 for pyeclib_driver in pyeclib_drivers:
396367 for file_size in self.file_sizes:
+2
-2
test/test_pyeclib_c.py less more
183183
184184 fragments = pyeclib_c.encode(handle, whole_file_bytes)
185185 orig_fragments = fragments[:]
186
186
187187 for i in range(iterations):
188188 missing_idxs = []
189189 num_missing = hd - 1
366366 self.num_parities[i],
367367 ec_type.value, self.num_parities[i] + 1,
368368 size_str, self.iterations)
369
369
370370 self.assertTrue(success)
371371 print(("Range Decode (%s): %s" %
372372 (size_str, self.get_throughput(avg_time, size_str))))
+1
-5
tools/pyeclib_conf_tool.py less more
5757 # (limit 10)
5858 #
5959
60 import pyeclib
6160 from pyeclib.ec_iface import ECDriver
6261 import random
6362 import string
6463 import sys
65 import os
6664 import argparse
6765 import time
6866 import math
237235 string.ascii_uppercase + string.digits) for x in range(args.s))
238236
239237 try:
240 ec_driver = ECDriver(
241 "pyeclib.core.ECPyECLibDriver",
242 k=scheme.k, m=scheme.m, ec_type=scheme.ec_type)
238 ec_driver = ECDriver(k=scheme.k, m=scheme.m, ec_type=scheme.ec_type)
243239 except Exception as e:
244240 print("Scheme %s is not defined (%s)." % (scheme, e))
245241 continue
+1
-7
tools/pyeclib_decode.py less more
2020 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
2121 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
2222
23 import pyeclib
2423 from pyeclib.ec_iface import ECDriver
25 import random
26 import string
27 import sys
28 import os
2924 import argparse
3025
3126 parser = argparse.ArgumentParser(description='Decoder for PyECLib.')
4338 print("fragments = %s" % args.fragments)
4439 print("filename = %s" % args.filename)
4540
46 ec_driver = ECDriver(
47 "pyeclib.core.ECPyECLibDriver", k=args.k, m=args.m, ec_type=args.ec_type)
41 ec_driver = ECDriver(k=args.k, m=args.m, ec_type=args.ec_type)
4842
4943 fragment_list = []
5044
+1
-8
tools/pyeclib_encode.py less more
2020 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
2121 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
2222
23 import pyeclib
2423 from pyeclib.ec_iface import ECDriver
25 import random
26 import string
27 import sys
28 import os
2924 import argparse
3025
3126 parser = argparse.ArgumentParser(description='Encoder for PyECLib.')
4237 print("ec_type = %s" % args.ec_type)
4338 print("filename = %s" % args.filename)
4439
45 ec_driver = ECDriver(
46 "pyeclib.core.ECPyECLibDriver",
47 k=args.k, m=args.m, ec_type=args.ec_type)
40 ec_driver = ECDriver(k=args.k, m=args.m, ec_type=args.ec_type)
4841
4942 # read
5043 with open(("%s/%s" % (args.file_dir, args.filename)), "rb") as fp:
+1
-3
tools/pyeclib_fragments_needed.py less more
3939
4040 args = parser.parse_args()
4141
42 ec_driver = ECDriver(
43 "pyeclib.core.ECPyECLibDriver",
44 k=args.k, m=args.m, ec_type=args.ec_type)
42 ec_driver = ECDriver(k=args.k, m=args.m, ec_type=args.ec_type)
4543
4644 fragments_needed = ec_driver.fragments_needed(args.missing_fragments)
4745