Codebase list liberasurecode / 0d1e413
Fix nasty rebuild bug where partiy would be reconstructed incorrectly when both data and parity was missing. The fix is to just call decode when reconstructing parity, since it will have to do extra work anyway when data is missing. We did a little extra work in ISA-L to do better, but can save that for later, since 99% of the time decode will perform just fine. Kevin Greenan 9 years ago
3 changed file(s) with 100 addition(s) and 52 deletion(s). Raw diff Collapse all Expand all
+38
-21
src/backends/jerasure/jerasure_rs_cauchy.c less more
136136 int *missing_idxs, int destination_idx, int blocksize)
137137 {
138138 int k, m, w; /* erasure code paramters */
139 int ret = 1; /* return code */
139 int ret = 0; /* return code */
140140 int *decoding_row = NULL; /* decoding matrix row for decode */
141141 int *erased = NULL; /* k+m length list of erased frag ids */
142142 int *dm_ids = NULL; /* k length list of fragment ids */
148148 m = jerasure_desc->m;
149149 w = jerasure_desc->w;
150150
151 dm_ids = (int *) alloc_zeroed_buffer(sizeof(int) * k);
152 decoding_matrix = (int *) alloc_zeroed_buffer(sizeof(int *) * k * k * w * w);
153 erased = jerasure_desc->jerasure_erasures_to_erased(k, m, missing_idxs);
154 if (NULL == decoding_matrix || NULL == dm_ids || NULL == erased) {
155 goto out;
156 }
157
158 ret = jerasure_desc->jerasure_make_decoding_bitmatrix(k, m, w,
151 if (destination_idx < k) {
152 dm_ids = (int *) alloc_zeroed_buffer(sizeof(int) * k);
153 decoding_matrix = (int *) alloc_zeroed_buffer(sizeof(int *) * k * k * w * w);
154 erased = jerasure_desc->jerasure_erasures_to_erased(k, m, missing_idxs);
155 if (NULL == decoding_matrix || NULL == dm_ids || NULL == erased) {
156 goto out;
157 }
158
159 ret = jerasure_desc->jerasure_make_decoding_bitmatrix(k, m, w,
159160 jerasure_desc->bitmatrix,
160161 erased, decoding_matrix, dm_ids);
161 if (destination_idx < k) {
162 decoding_row = decoding_matrix + (destination_idx * k * w * w);
162 if (ret == 0) {
163 decoding_row = decoding_matrix + (destination_idx * k * w * w);
164
165 jerasure_desc->jerasure_bitmatrix_dotprod(jerasure_desc->k, jerasure_desc->w,
166 decoding_row, dm_ids, destination_idx,
167 data, parity, blocksize, PYECC_CAUCHY_PACKETSIZE);
168 } else {
169 /*
170 * ToDo (KMG) I know this is not needed, but keeping to prevent future
171 * memory leaks, as this function will be better optimized for decoding
172 * missing parity
173 */
174 goto out;
175 }
163176 } else {
164 decoding_row = jerasure_desc->bitmatrix + ((destination_idx - k) * k * w * w);
165 }
166
167 if (ret == 0) {
168 jerasure_desc->jerasure_bitmatrix_dotprod(jerasure_desc->k, jerasure_desc->w,
169 decoding_row, dm_ids, destination_idx,
170 data, parity, blocksize,
171 PYECC_CAUCHY_PACKETSIZE);
172 } else {
173 goto out;
177 /*
178 * If it is parity we are reconstructing, then just call decode.
179 * ToDo (KMG): We can do better than this, but this should perform just
180 * fine for most cases. We can adjust the decoding matrix like we
181 * did with ISA-L.
182 */
183 jerasure_desc->jerasure_bitmatrix_decode(k, m, w,
184 jerasure_desc->bitmatrix,
185 0,
186 missing_idxs,
187 data,
188 parity,
189 blocksize,
190 PYECC_CAUCHY_PACKETSIZE);
174191 }
175192
176193 out:
+38
-27
src/backends/jerasure/jerasure_rs_vand.c less more
107107 static int jerasure_rs_vand_reconstruct(void *desc, char **data, char **parity,
108108 int *missing_idxs, int destination_idx, int blocksize)
109109 {
110 int ret = 1; /* return code */
110 int ret = 0; /* return code */
111111 int *decoding_row; /* decoding matrix row for decode */
112112 int *erased = NULL; /* k+m length list of erased frag ids */
113113 int *dm_ids = NULL; /* k length list of frag ids */
116116 struct jerasure_rs_vand_descriptor *jerasure_desc =
117117 (struct jerasure_rs_vand_descriptor*) desc;
118118
119 dm_ids = (int *) alloc_zeroed_buffer(sizeof(int) * jerasure_desc->k);
120 decoding_matrix = (int *)
121 alloc_zeroed_buffer(sizeof(int*) * jerasure_desc->k * jerasure_desc->k);
122 if (NULL == decoding_matrix || NULL == dm_ids) {
123 goto out;
124 }
125
126 erased = jerasure_desc->jerasure_erasures_to_erased(jerasure_desc->k,
127 jerasure_desc->m, missing_idxs);
128 if (NULL == erased) {
129 goto out;
130 }
131
132 ret = jerasure_desc->jerasure_make_decoding_matrix(jerasure_desc->k,
133 jerasure_desc->m, jerasure_desc->w, jerasure_desc->matrix,
134 erased, decoding_matrix, dm_ids);
135119 if (destination_idx < jerasure_desc->k) {
120 dm_ids = (int *) alloc_zeroed_buffer(sizeof(int) * jerasure_desc->k);
121 decoding_matrix = (int *)
122 alloc_zeroed_buffer(sizeof(int*) * jerasure_desc->k * jerasure_desc->k);
123 erased = jerasure_desc->jerasure_erasures_to_erased(jerasure_desc->k,
124 jerasure_desc->m, missing_idxs);
125 if (NULL == decoding_matrix || NULL == dm_ids || NULL == erased) {
126 goto out;
127 }
128
129 ret = jerasure_desc->jerasure_make_decoding_matrix(jerasure_desc->k,
130 jerasure_desc->m, jerasure_desc->w, jerasure_desc->matrix,
131 erased, decoding_matrix, dm_ids);
132
136133 decoding_row = decoding_matrix + (destination_idx * jerasure_desc->k);
134
135 if (ret == 0) {
136 jerasure_desc->jerasure_matrix_dotprod(jerasure_desc->k,
137 jerasure_desc->w, decoding_row, dm_ids, destination_idx,
138 data, parity, blocksize);
139 } else {
140 /*
141 * ToDo (KMG) I know this is not needed, but keeping to prevent future
142 * memory leaks, as this function will be better optimized for decoding
143 * missing parity
144 */
145 goto out;
146 }
137147 } else {
138 decoding_row = jerasure_desc->matrix +
139 ((destination_idx - jerasure_desc->k) * jerasure_desc->k);
140 }
141
142 if (ret == 0) {
143 jerasure_desc->jerasure_matrix_dotprod(jerasure_desc->k,
144 jerasure_desc->w, decoding_row, dm_ids, destination_idx,
145 data, parity, blocksize);
146 } else {
147 goto out;
148 /*
149 * If it is parity we are reconstructing, then just call decode.
150 * ToDo (KMG): We can do better than this, but this should perform just
151 * fine for most cases. We can adjust the decoding matrix like we
152 * did with ISA-L.
153 */
154 jerasure_desc->jerasure_matrix_decode(jerasure_desc->k,
155 jerasure_desc->m, jerasure_desc->w,
156 jerasure_desc->matrix, 1, missing_idxs, data, parity, blocksize);
157 goto parity_reconstr_out;
148158 }
149159
150160 out:
152162 free(decoding_matrix);
153163 free(dm_ids);
154164
165 parity_reconstr_out:
155166 return ret;
156167 }
157168
+24
-4
test/liberasurecode_test.c less more
767767 free(avail_frags);
768768 }
769769
770 /**
771 * Note: this test will attempt to reconstruct a single fragment when
772 * one or more other fragments are missing (specified by skip).
773 *
774 * For example, if skip is [0, 0, 0, 1, 0, 0] and we are reconstructing
775 * fragment 5, then it will test the reconstruction of fragment 5 when 3
776 * and 5 are assumed unavailable.
777 *
778 * We only mark at most 2 as unavailable, as we cannot guarantee every situation
779 * will be able to habndle 3 failures.
780 */
770781 static void reconstruct_test_impl(const ec_backend_id_t be_id,
771782 struct ec_args *args,
772783 int *skip)
795806 rc = liberasurecode_encode(desc, orig_data, orig_data_size,
796807 &encoded_data, &encoded_parity, &encoded_fragment_len);
797808 assert(rc == 0);
798 num_avail_frags = create_frags_array(&avail_frags, encoded_data,
799 encoded_parity, args, skip);
800809 out = malloc(encoded_fragment_len);
801810 assert(out != NULL);
802811 for (i = 0; i < num_fragments; i++) {
812 char *cmp = NULL;
813 // If the current fragment was not chosen as fragments to skip,
814 // remove it and the chosen fragments to skip from the available list
815 // and reset its state
803816 if (skip[i] == 0) {
804 continue;
817 skip[i] = 1;
818 num_avail_frags = create_frags_array(&avail_frags, encoded_data,
819 encoded_parity, args, skip);
820 skip[i] = 0;
821 // Do not reset the skip state if the fragment was chosen as a fragment
822 // to skip for this invocation of the test
823 } else {
824 num_avail_frags = create_frags_array(&avail_frags, encoded_data,
825 encoded_parity, args, skip);
805826 }
806 char *cmp = NULL;
807827 if (i < args->k) {
808828 cmp = encoded_data[i];
809829 }