Commit f7498ab93c74f101be709086f186a3c7263a7211 - python-pyeclib

+3

-1

README less more

15	15	and "Screaming fast Galois Field arithmetic using Intel SIMD instructions" in
16	16	USENIX FAST 2013).
17	17
18		Examples of using this library are provided in ./test:
	18	Examples of using this library are provided in ./tools:
19	19
20	20	Command-line encoder: ec_pyeclib_encode.py
21	21
22	22	Command-line decoder: ec_pyeclib_decode.py
	23
	24	Utility to determine what is needed to reconstruct missing fragments: pyeclib_fragments_needed.py
23	25
24	26	The main Python interface only contains 6 functions:
25	27

+0

-1

src/python/pyeclib/core.py less more

31	31	self.ec_flat_xor_3 = "flat_xor_3"
32	32	self.ec_flat_xor_4 = "flat_xor_4"
33	33	self.ec_types = [self.ec_rs_vand, self.ec_rs_cauchy_orig, self.ec_flat_xor_3, self.ec_flat_xor_4]
34		self.ec_valid_xor_params = ["12_6_4", "10_5_3"]
35	34	self.ec_rs_vand_best_w = 16
36	35	self.ec_default_w = 32
37	36	self.ec_rs_cauchy_best_w = 4

+0

-67

~~test/ec_pyeclib_decode.py~~ less more

0
1		# Copyright (c) 2013, Kevin Greenan (kmgreen2@gmail.com)
2		# All rights reserved.
3		#
4		# Redistribution and use in source and binary forms, with or without
5		# modification, are permitted provided that the following conditions are met:
6		#
7		# Redistributions of source code must retain the above copyright notice, this
8		# list of conditions and the following disclaimer.
9		#
10		# Redistributions in binary form must reproduce the above copyright notice, this
11		# list of conditions and the following disclaimer in the documentation and/or
12		# other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY
13		# THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED
14		# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
15		# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
16		# EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
17		# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
18		# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
19		# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
20		# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
21		# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
22		# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
23
24		import pyeclib
25		from pyeclib.ec_iface import ECDriver
26		import random
27		import string
28		import sys
29		import os
30		import argparse
31
32		parser = argparse.ArgumentParser(description='Decoder for PyECLib.')
33		parser.add_argument('k', type=int, help='number of data elements')
34		parser.add_argument('m', type=int, help='number of parity elements')
35		parser.add_argument('type', help='EC algorithm used')
36		parser.add_argument('fragments', metavar='fragment', nargs='+',
37		help='fragments to decode')
38		parser.add_argument('filename', help='output file')
39
40		args = parser.parse_args()
41
42		print args.k
43		print args.m
44		print args.type
45		print args.fragments
46		print args.filename
47
48		ec_driver = ECDriver("pyeclib.core.ECPyECLibDriver", k=args.k, m=args.m, type=args.type)
49
50		fragment_list = []
51
52		for fragment in args.fragments:
53		fp = open("%s" % fragment, "r")
54
55		fragment_list.append(fp.read())
56
57		fp.close()
58
59
60		fp = open("%s.decoded" % args.filename, "w")
61
62		decoded_file=ec_driver.decode(fragment_list)
63
64		fp.write(decoded_file)
65
66		fp.close()

+0

-64

~~test/ec_pyeclib_encode.py~~ less more

0
1		# Copyright (c) 2013, Kevin Greenan (kmgreen2@gmail.com)
2		# All rights reserved.
3		#
4		# Redistribution and use in source and binary forms, with or without
5		# modification, are permitted provided that the following conditions are met:
6		#
7		# Redistributions of source code must retain the above copyright notice, this
8		# list of conditions and the following disclaimer.
9		#
10		# Redistributions in binary form must reproduce the above copyright notice, this
11		# list of conditions and the following disclaimer in the documentation and/or
12		# other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY
13		# THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED
14		# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
15		# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
16		# EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
17		# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
18		# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
19		# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
20		# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
21		# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
22		# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
23
24		import pyeclib
25		from pyeclib.ec_iface import ECDriver
26		import random
27		import string
28		import sys
29		import os
30		import argparse
31
32		parser = argparse.ArgumentParser(description='Encoder for PyECLib.')
33		parser.add_argument('k', type=int, help='number of data elements')
34		parser.add_argument('m', type=int, help='number of parity elements')
35		parser.add_argument('type', help='EC algorithm used')
36		parser.add_argument('file_dir', help='directory with the file')
37		parser.add_argument('filename', help='file to encode')
38		parser.add_argument('fragment_dir', help='directory to drop encoded fragments')
39
40		args = parser.parse_args()
41
42		print args.k
43		print args.m
44		print args.type
45		print args.filename
46
47		ec_driver = ECDriver("pyeclib.core.ECPyECLibDriver", k=args.k, m=args.m, type=args.type)
48
49		fp = open("%s/%s" % (args.file_dir, args.filename), "r")
50
51		whole_file_str = fp.read()
52
53		fragments=ec_driver.encode(whole_file_str)
54
55		fp.close()
56
57		i = 0
58		for fragment in fragments:
59		fp = open("%s/%s.%d" % (args.fragment_dir, args.filename, i), "w")
60		fp.write(fragment)
61		fp.close()
62		i += 1
63

+4

-3

test/ec_pyeclib_file_test.sh less more

25	25	DECODED_DIR=./decoded_files
26	26	FRAGMENT_DIR=./test_fragments
27	27	FILES=`echo ${FILE_DIR}`
	28	TOOLS_DIR=../tools
28	29
29	30	if [ ! -d ${DECODED_DIR} ]; then
30	31	mkdir ${DECODED_DIR}

61	62	FAULT_TOL="2"
62	63	fi
63	64	for file in `cd ${FILES}; echo *; cd ..`; do
64		python ec_pyeclib_encode.py ${NUM_DATA} ${NUM_PARITY} ${TYPE} ${FILE_DIR} ${file} ${FRAGMENT_DIR}
	65	python ${TOOLS_DIR}/pyeclib_encode.py ${NUM_DATA} ${NUM_PARITY} ${TYPE} ${FILE_DIR} ${file} ${FRAGMENT_DIR}
65	66	done
66	67
67	68	for file in `cd ${FILES}; echo *; cd ..`; do

72	73	fragments[${index}]=""
73	74	let i=$i+1
74	75	done
75		python ec_pyeclib_decode.py ${NUM_DATA} ${NUM_PARITY} ${TYPE} ${fragments[*]} ${DECODED_DIR}/${file}
	76	python ${TOOLS_DIR}/pyeclib_decode.py ${NUM_DATA} ${NUM_PARITY} ${TYPE} ${fragments[*]} ${DECODED_DIR}/${file}
76	77	diff ${FILE_DIR}/${file} ${DECODED_DIR}/${file}.decoded
77	78	if [[ $? != 0 ]]; then
78	79	echo "${FILE_DIR}/${file} != ${DECODED_DIR}/${file}.decoded"

84	85	done
85	86
86	87	rm ${DECODED_DIR}/*
87		rm ${FRAGMENT_DIR}/*⏎
	88	rm ${FRAGMENT_DIR}/*

+0

-47

~~test/ec_pyeclib_fragments_needed.py~~ less more

0
1		# Copyright (c) 2013, Kevin Greenan (kmgreen2@gmail.com)
2		# All rights reserved.
3		#
4		# Redistribution and use in source and binary forms, with or without
5		# modification, are permitted provided that the following conditions are met:
6		#
7		# Redistributions of source code must retain the above copyright notice, this
8		# list of conditions and the following disclaimer.
9		#
10		# Redistributions in binary form must reproduce the above copyright notice, this
11		# list of conditions and the following disclaimer in the documentation and/or
12		# other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY
13		# THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED
14		# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
15		# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
16		# EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
17		# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
18		# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
19		# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
20		# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
21		# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
22		# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
23
24		import pyeclib
25		from pyeclib.ec_iface import ECDriver
26		import random
27		import string
28		import sys
29		import os
30		import argparse
31
32		parser = argparse.ArgumentParser(description='Encoder for PyECLib.')
33		parser.add_argument('k', type=int, help='number of data elements')
34		parser.add_argument('m', type=int, help='number of parity elements')
35		parser.add_argument('type', help='EC algorithm used')
36		parser.add_argument('missing_fragments', type=int, metavar='missing_fragment', nargs='+',
37		help='missing_fragments')
38
39		args = parser.parse_args()
40
41		ec_driver = ECDriver("pyeclib.core.ECPyECLibDriver", k=args.k, m=args.m, type=args.type)
42
43		fragments_needed = ec_driver.fragments_needed(args.missing_fragments)
44
45		print fragments_needed
46

+1

-1

test/run_tests.py less more

77	77
78	78	test_c_stuff()
79	79	pyeclib_core_test()
80		pyeclib_core_valgrind()
	80	#pyeclib_core_valgrind()

+67

-0

tools/pyeclib_decode.py less more

	0
	1	# Copyright (c) 2013, Kevin Greenan (kmgreen2@gmail.com)
	2	# All rights reserved.
	3	#
	4	# Redistribution and use in source and binary forms, with or without
	5	# modification, are permitted provided that the following conditions are met:
	6	#
	7	# Redistributions of source code must retain the above copyright notice, this
	8	# list of conditions and the following disclaimer.
	9	#
	10	# Redistributions in binary form must reproduce the above copyright notice, this
	11	# list of conditions and the following disclaimer in the documentation and/or
	12	# other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY
	13	# THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED
	14	# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
	15	# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
	16	# EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
	17	# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
	18	# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	19	# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
	20	# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
	21	# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
	22	# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	23
	24	import pyeclib
	25	from pyeclib.ec_iface import ECDriver
	26	import random
	27	import string
	28	import sys
	29	import os
	30	import argparse
	31
	32	parser = argparse.ArgumentParser(description='Decoder for PyECLib.')
	33	parser.add_argument('k', type=int, help='number of data elements')
	34	parser.add_argument('m', type=int, help='number of parity elements')
	35	parser.add_argument('type', help='EC algorithm used')
	36	parser.add_argument('fragments', metavar='fragment', nargs='+',
	37	help='fragments to decode')
	38	parser.add_argument('filename', help='output file')
	39
	40	args = parser.parse_args()
	41
	42	print args.k
	43	print args.m
	44	print args.type
	45	print args.fragments
	46	print args.filename
	47
	48	ec_driver = ECDriver("pyeclib.core.ECPyECLibDriver", k=args.k, m=args.m, type=args.type)
	49
	50	fragment_list = []
	51
	52	for fragment in args.fragments:
	53	fp = open("%s" % fragment, "r")
	54
	55	fragment_list.append(fp.read())
	56
	57	fp.close()
	58
	59
	60	fp = open("%s.decoded" % args.filename, "w")
	61
	62	decoded_file=ec_driver.decode(fragment_list)
	63
	64	fp.write(decoded_file)
	65
	66	fp.close()

+64

-0

tools/pyeclib_encode.py less more

	0
	1	# Copyright (c) 2013, Kevin Greenan (kmgreen2@gmail.com)
	2	# All rights reserved.
	3	#
	4	# Redistribution and use in source and binary forms, with or without
	5	# modification, are permitted provided that the following conditions are met:
	6	#
	7	# Redistributions of source code must retain the above copyright notice, this
	8	# list of conditions and the following disclaimer.
	9	#
	10	# Redistributions in binary form must reproduce the above copyright notice, this
	11	# list of conditions and the following disclaimer in the documentation and/or
	12	# other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY
	13	# THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED
	14	# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
	15	# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
	16	# EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
	17	# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
	18	# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	19	# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
	20	# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
	21	# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
	22	# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	23
	24	import pyeclib
	25	from pyeclib.ec_iface import ECDriver
	26	import random
	27	import string
	28	import sys
	29	import os
	30	import argparse
	31
	32	parser = argparse.ArgumentParser(description='Encoder for PyECLib.')
	33	parser.add_argument('k', type=int, help='number of data elements')
	34	parser.add_argument('m', type=int, help='number of parity elements')
	35	parser.add_argument('type', help='EC algorithm used')
	36	parser.add_argument('file_dir', help='directory with the file')
	37	parser.add_argument('filename', help='file to encode')
	38	parser.add_argument('fragment_dir', help='directory to drop encoded fragments')
	39
	40	args = parser.parse_args()
	41
	42	print args.k
	43	print args.m
	44	print args.type
	45	print args.filename
	46
	47	ec_driver = ECDriver("pyeclib.core.ECPyECLibDriver", k=args.k, m=args.m, type=args.type)
	48
	49	fp = open("%s/%s" % (args.file_dir, args.filename), "r")
	50
	51	whole_file_str = fp.read()
	52
	53	fragments=ec_driver.encode(whole_file_str)
	54
	55	fp.close()
	56
	57	i = 0
	58	for fragment in fragments:
	59	fp = open("%s/%s.%d" % (args.fragment_dir, args.filename, i), "w")
	60	fp.write(fragment)
	61	fp.close()
	62	i += 1
	63

+47

-0

tools/pyeclib_fragments_needed.py less more

	0
	1	# Copyright (c) 2013, Kevin Greenan (kmgreen2@gmail.com)
	2	# All rights reserved.
	3	#
	4	# Redistribution and use in source and binary forms, with or without
	5	# modification, are permitted provided that the following conditions are met:
	6	#
	7	# Redistributions of source code must retain the above copyright notice, this
	8	# list of conditions and the following disclaimer.
	9	#
	10	# Redistributions in binary form must reproduce the above copyright notice, this
	11	# list of conditions and the following disclaimer in the documentation and/or
	12	# other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY
	13	# THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED
	14	# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
	15	# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
	16	# EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
	17	# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
	18	# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	19	# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
	20	# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
	21	# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
	22	# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	23
	24	import pyeclib
	25	from pyeclib.ec_iface import ECDriver
	26	import random
	27	import string
	28	import sys
	29	import os
	30	import argparse
	31
	32	parser = argparse.ArgumentParser(description='PyECLib tool to determine fragment indexes needed to recover missing fragments.')
	33	parser.add_argument('k', type=int, help='number of data elements')
	34	parser.add_argument('m', type=int, help='number of parity elements')
	35	parser.add_argument('type', help='EC algorithm used')
	36	parser.add_argument('missing_fragments', type=int, metavar='missing_fragment', nargs='+',
	37	help='missing_fragments')
	38
	39	args = parser.parse_args()
	40
	41	ec_driver = ECDriver("pyeclib.core.ECPyECLibDriver", k=args.k, m=args.m, type=args.type)
	42
	43	fragments_needed = ec_driver.fragments_needed(args.missing_fragments)
	44
	45	print fragments_needed
	46