liberasurecode  1.5.0
Erasure Code API library
 All Data Structures Files Functions Variables Typedefs Macros
erasurecode.c
Go to the documentation of this file.
1 /*
2  * Copyright 2014 Tushar Gohad, Kevin M Greenan, Eric Lambert, Mark Storer
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions are met:
6  *
7  * Redistributions of source code must retain the above copyright notice, this
8  * list of conditions and the following disclaimer.
9  *
10  * Redistributions in binary form must reproduce the above copyright notice, this
11  * list of conditions and the following disclaimer in the documentation and/or
12  * other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY
13  * THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED
14  * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
15  * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
16  * EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
17  * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
18  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
19  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
20  * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
21  * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
22  * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
23  *
24  * liberasurecode API implementation
25  *
26  * vi: set noai tw=79 ts=4 sw=4:
27  */
28 
29 #include "assert.h"
30 #include "list.h"
31 #include "erasurecode.h"
32 #include "erasurecode_backend.h"
33 #include "erasurecode_helpers.h"
34 #include "erasurecode_helpers_ext.h"
35 #include "erasurecode_preprocessing.h"
36 #include "erasurecode_postprocessing.h"
37 #include "erasurecode_stdinc.h"
38 
39 #include "alg_sig.h"
40 #include "erasurecode_log.h"
41 
42 /* =~=*=~==~=*=~==~=*=~= Supported EC backends =~=*=~==~=*=~==~=*=~==~=*=~== */
43 
44 /* EC backend references */
45 extern struct ec_backend_common backend_null;
46 extern struct ec_backend_common backend_flat_xor_hd;
47 extern struct ec_backend_common backend_jerasure_rs_vand;
48 extern struct ec_backend_common backend_jerasure_rs_cauchy;
49 extern struct ec_backend_common backend_isa_l_rs_vand;
50 extern struct ec_backend_common backend_shss;
51 extern struct ec_backend_common backend_liberasurecode_rs_vand;
52 extern struct ec_backend_common backend_isa_l_rs_cauchy;
53 extern struct ec_backend_common backend_libphazr;
54 
55 ec_backend_t ec_backends_supported[] = {
56  (ec_backend_t) &backend_null,
57  (ec_backend_t) &backend_jerasure_rs_vand,
58  (ec_backend_t) &backend_jerasure_rs_cauchy,
59  (ec_backend_t) &backend_flat_xor_hd,
60  (ec_backend_t) &backend_isa_l_rs_vand,
61  (ec_backend_t) &backend_shss,
62  (ec_backend_t) &backend_liberasurecode_rs_vand,
63  (ec_backend_t) &backend_isa_l_rs_cauchy,
64  (ec_backend_t) &backend_libphazr,
65  NULL,
66 };
67 
68 /* backend list to return to the caller */
70 char *ec_backends_supported_str[EC_BACKENDS_MAX];
71 
72 /* =~=*=~==~=*=~==~=*=~= EC backend instance management =~=*=~==~=*=~==~=*= */
73 
74 /* Registered erasure code backend instances */
75 SLIST_HEAD(backend_list, ec_backend) active_instances =
76  SLIST_HEAD_INITIALIZER(active_instances);
77 rwlock_t active_instances_rwlock = RWLOCK_INITIALIZER;
78 
79 /* Backend instance id */
80 int next_backend_desc = 0;
81 
88 ec_backend_t liberasurecode_backend_instance_get_by_desc(int desc)
89 {
90  struct ec_backend *b = NULL;
91  SLIST_FOREACH(b, &active_instances, link) {
92  if (b->idesc == desc)
93  break;
94  }
95  return b;
96 }
97 
105 {
106  for (;;) {
107  if (++next_backend_desc <= 0)
108  next_backend_desc = 1;
109  if (!liberasurecode_backend_instance_get_by_desc(next_backend_desc))
110  return next_backend_desc;
111  }
112 }
113 
122 {
123  int desc = -1; /* descriptor to return */
124  int rc = 0; /* return call value */
125 
126  rc = rwlock_wrlock(&active_instances_rwlock);
127  if (rc == 0) {
128  SLIST_INSERT_HEAD(&active_instances, instance, link);
130  if (desc <= 0)
131  goto register_out;
132  instance->idesc = desc;
133  } else {
134  goto exit;
135  }
136 
137 register_out:
138  rwlock_unlock(&active_instances_rwlock);
139 exit:
140  return desc;
141 }
142 
149 {
150  int rc = 0; /* return call value */
151 
152  rc = rwlock_wrlock(&active_instances_rwlock);
153  if (rc == 0) {
154  SLIST_REMOVE(&active_instances, instance, ec_backend, link);
155  } else {
156  goto exit;
157  }
158  rwlock_unlock(&active_instances_rwlock);
159 
160 exit:
161  return rc;
162 }
163 
164 /* =~=*=~==~=*=~== liberasurecode backend API helpers =~=*=~==~=*=~== */
165 
166 static void print_dlerror(const char *caller)
167 {
168  char *msg = dlerror();
169  if (NULL == msg)
170  log_error("%s: unknown dynamic linking error\n", caller);
171  else
172  log_error("%s: dynamic linking error %s\n", caller, msg);
173 }
174 
175 /* Generic dlopen/dlclose routines */
176 void* liberasurecode_backend_open(ec_backend_t instance)
177 {
178  if (NULL == instance)
179  return NULL;
180  /* Use RTLD_LOCAL to avoid symbol collisions */
181  return dlopen(instance->common.soname, RTLD_LAZY | RTLD_LOCAL);
182 }
183 
184 int liberasurecode_backend_close(ec_backend_t instance)
185 {
186  if (NULL == instance || NULL == instance->desc.backend_sohandle)
187  return 0;
188 
189  dlclose(instance->desc.backend_sohandle);
190  dlerror(); /* Clear any existing errors */
191 
192  instance->desc.backend_sohandle = NULL;
193  return 0;
194 }
195 
196 /* =*=~==~=*=~==~=*=~= liberasurecode init/exit routines =~=*=~==~=*=~==~=*= */
197 
198 void __attribute__ ((constructor))
199 liberasurecode_init(void) {
200  /* init logging */
201  openlog("liberasurecode", LOG_PID | LOG_CONS, LOG_USER);
202 
203  /* populate supported backends list as a string */
204  {
205  int i;
206  for (i = 0; ec_backends_supported[i]; ++i) {
207  ec_backends_supported_str[i] = strdup(
208  ec_backends_supported[i]->common.name);
209  }
210  num_supported_backends = i;
211  }
212 }
213 
214 void __attribute__ ((destructor))
215 liberasurecode_exit(void) {
216  int i;
217  for (i = 0; i < num_supported_backends; ++i)
218  free(ec_backends_supported_str[i]);
219  closelog();
220 }
221 
222 /* =~=*=~==~=*=~= liberasurecode frontend API implementation =~=*=~==~=*=~== */
223 
231 int liberasurecode_backend_available(const ec_backend_id_t backend_id) {
232  struct ec_backend backend;
233  if (backend_id >= EC_BACKENDS_MAX)
234  return 0;
235 
236  backend.desc.backend_sohandle = liberasurecode_backend_open(
237  ec_backends_supported[backend_id]);
238  if (!backend.desc.backend_sohandle) {
239  return 0;
240  }
241 
243  return 1;
244 }
245 
265 int liberasurecode_instance_create(const ec_backend_id_t id,
266  struct ec_args *args)
267 {
268  ec_backend_t instance = NULL;
269  struct ec_backend_args bargs;
270  if (!args)
271  return -EINVALIDPARAMS;
272 
273  if (id >= EC_BACKENDS_MAX)
274  return -EBACKENDNOTSUPP;
275 
276  if ((args->k + args->m) > EC_MAX_FRAGMENTS) {
277  log_error("Total number of fragments (k + m) must be less than %d\n",
278  EC_MAX_FRAGMENTS);
279  return -EINVALIDPARAMS;
280  }
281 
282  /* Allocate memory for ec_backend instance */
283  instance = calloc(1, sizeof(*instance));
284  if (NULL == instance)
285  return -ENOMEM;
286 
287  /* Copy common backend, args struct */
288  instance->common = ec_backends_supported[id]->common;
289  memcpy(&(bargs.uargs), args, sizeof (struct ec_args));
290  instance->args = bargs;
291 
292  /* Open backend .so if not already open */
293  /* .so handle is returned in instance->desc.backend_sohandle */
294  if (!instance->desc.backend_sohandle) {
295  instance->desc.backend_sohandle = liberasurecode_backend_open(instance);
296  if (!instance->desc.backend_sohandle) {
297  /* ignore during init, return the same handle */
298  print_dlerror(__func__);
299  free(instance);
300  return -EBACKENDNOTAVAIL;
301  }
302  }
303 
304  /* Call private init() for the backend */
305  instance->desc.backend_desc = instance->common.ops->init(
306  &instance->args, instance->desc.backend_sohandle);
307  if (NULL == instance->desc.backend_desc) {
308  free (instance);
309  return -EBACKENDINITERR;
310  }
311 
312  /* Register instance and return a descriptor/instance id */
313  instance->idesc = liberasurecode_backend_instance_register(instance);
314 
315  return instance->idesc;
316 }
317 
324 {
325  ec_backend_t instance = NULL; /* instance to destroy */
326  int rc = 0; /* return code */
327 
328  instance = liberasurecode_backend_instance_get_by_desc(desc);
329  if (NULL == instance)
330  return -EBACKENDNOTAVAIL;
331 
332  /* Call private exit() for the backend */
333  instance->common.ops->exit(instance->desc.backend_desc);
334 
335  /* dlclose() backend library */
337 
338  /* Remove instance from registry */
340  if (rc == 0) {
341  free(instance);
342  }
343 
344  return rc;
345 }
346 
363  char **encoded_data,
364  char **encoded_parity)
365 {
366  int i, k, m;
367 
368  ec_backend_t instance = liberasurecode_backend_instance_get_by_desc(desc);
369  if (NULL == instance) {
370  return -EBACKENDNOTAVAIL;
371  }
372 
373  k = instance->args.uargs.k;
374  m = instance->args.uargs.m;
375 
376  if (encoded_data) {
377  for (i = 0; i < k; i++) {
378  free(encoded_data[i]);
379  }
380 
381  free(encoded_data);
382  }
383 
384  if (encoded_parity) {
385  for (i = 0; i < m; i++) {
386  free(encoded_parity[i]);
387  }
388  free(encoded_parity);
389  }
390 
391  return 0;
392 }
393 
411  const char *orig_data, uint64_t orig_data_size, /* input */
412  char ***encoded_data, char ***encoded_parity, /* output */
413  uint64_t *fragment_len) /* output */
414 {
415  int k, m;
416  int ret = 0; /* return code */
417 
418  int blocksize = 0; /* length of each of k data elements */
419 
420  if (orig_data == NULL) {
421  log_error("Pointer to data buffer is null!");
422  ret = -EINVALIDPARAMS;
423  goto out;
424  }
425 
426  if (encoded_data == NULL) {
427  log_error("Pointer to encoded data buffers is null!");
428  return -EINVALIDPARAMS;
429  }
430 
431  if (encoded_parity == NULL) {
432  log_error("Pointer to encoded parity buffers is null!");
433  return -EINVALIDPARAMS;
434  }
435 
436  if (fragment_len == NULL) {
437  log_error("Pointer to fragment length is null!");
438  ret = -EINVALIDPARAMS;
439  goto out;
440  }
441 
442  ec_backend_t instance = liberasurecode_backend_instance_get_by_desc(desc);
443  if (NULL == instance) {
444  ret = -EBACKENDNOTAVAIL;
445  goto out;
446  }
447 
448  k = instance->args.uargs.k;
449  m = instance->args.uargs.m;
450 
451  /*
452  * Allocate arrays for data, parity and missing_idxs
453  */
454  *encoded_data = (char **) alloc_zeroed_buffer(sizeof(char *) * k);
455  if (NULL == *encoded_data) {
456  log_error("Could not allocate data buffer!");
457  goto out;
458  }
459 
460  *encoded_parity = (char **) alloc_zeroed_buffer(sizeof(char *) * m);
461  if (NULL == *encoded_parity) {
462  log_error("Could not allocate parity buffer!");
463  goto out;
464  }
465 
466  ret = prepare_fragments_for_encode(instance, k, m, orig_data, orig_data_size,
467  *encoded_data, *encoded_parity, &blocksize);
468  if (ret < 0) {
469  // ensure encoded_data/parity point the head of fragment_ptr
470  get_fragment_ptr_array_from_data(*encoded_data, *encoded_data, k);
471  get_fragment_ptr_array_from_data(*encoded_parity, *encoded_parity, m);
472  goto out;
473  }
474 
475  /* call the backend encode function passing it desc instance */
476  ret = instance->common.ops->encode(instance->desc.backend_desc,
477  *encoded_data, *encoded_parity, blocksize);
478  if (ret < 0) {
479  // ensure encoded_data/parity point the head of fragment_ptr
480  get_fragment_ptr_array_from_data(*encoded_data, *encoded_data, k);
481  get_fragment_ptr_array_from_data(*encoded_parity, *encoded_parity, m);
482  goto out;
483  }
484 
485  ret = finalize_fragments_after_encode(instance, k, m, blocksize, orig_data_size,
486  *encoded_data, *encoded_parity);
487 
488  *fragment_len = get_fragment_size((*encoded_data)[0]);
489 
490 out:
491  if (ret) {
492  /* Cleanup the allocations we have done */
493  liberasurecode_encode_cleanup(desc, *encoded_data, *encoded_parity);
494  log_error("Error in liberasurecode_encode %d", ret);
495  }
496  return ret;
497 }
498 
512 int liberasurecode_decode_cleanup(int desc, char *data)
513 {
514  ec_backend_t instance = liberasurecode_backend_instance_get_by_desc(desc);
515  if (NULL == instance) {
516  return -EBACKENDNOTAVAIL;
517  }
518 
519  free(data);
520 
521  return 0;
522 }
523 
538  char **available_fragments, /* input */
539  int num_fragments, uint64_t fragment_len, /* input */
540  int force_metadata_checks, /* input */
541  char **out_data, uint64_t *out_data_len) /* output */
542 {
543  int i, j;
544  int ret = 0;
545 
546  int k = -1, m = -1;
547  int orig_data_size = 0;
548 
549  int blocksize = 0;
550  char **data = NULL;
551  char **parity = NULL;
552  char **data_segments = NULL;
553  char **parity_segments = NULL;
554  int *missing_idxs = NULL;
555 
556  uint64_t realloc_bm = 0;
557 
558  ec_backend_t instance = liberasurecode_backend_instance_get_by_desc(desc);
559  if (NULL == instance) {
560  ret = -EBACKENDNOTAVAIL;
561  goto out;
562  }
563 
564  if (NULL == available_fragments) {
565  log_error("Pointer to encoded fragments buffer is null!");
566  ret = -EINVALIDPARAMS;
567  goto out;
568  }
569 
570  if (NULL == out_data) {
571  log_error("Pointer to decoded data buffer is null!");
572  ret = -EINVALIDPARAMS;
573  goto out;
574  }
575 
576  if (NULL == out_data_len) {
577  log_error("Pointer to decoded data length variable is null!");
578  ret = -EINVALIDPARAMS;
579  goto out;
580  }
581 
582  k = instance->args.uargs.k;
583  m = instance->args.uargs.m;
584 
585  if (num_fragments < k) {
586  log_error("Not enough fragments to decode, got %d, need %d!",
587  num_fragments, k);
588  ret = -EINSUFFFRAGS;
589  goto out;
590  }
591 
592  if (fragment_len < sizeof(fragment_header_t)) {
593  log_error("Fragments not long enough to include headers! "
594  "Need %zu, but got %lu.", sizeof(fragment_header_t),
595  (unsigned long)fragment_len);
596  ret = -EBADHEADER;
597  goto out;
598  }
599  for (i = 0; i < num_fragments; ++i) {
600  /* Verify metadata checksum */
602  (fragment_header_t *) available_fragments[i])) {
603  log_error("Invalid fragment header information!");
604  ret = -EBADHEADER;
605  goto out;
606  }
607  }
608 
609  if (instance->common.id != EC_BACKEND_SHSS && instance->common.id != EC_BACKEND_LIBPHAZR) {
610  /* shss (ntt_backend) & libphazr backend must force to decode */
611  // TODO: Add a frag and function to handle whether the backend want to decode or not.
612  /*
613  * Try to re-assebmle the original data before attempting a decode
614  */
615  ret = fragments_to_string(k, m,
616  available_fragments, num_fragments,
617  out_data, out_data_len);
618 
619  if (ret == 0) {
620  /* We were able to get the original data without decoding! */
621  goto out;
622  }
623  }
624 
625  /*
626  * Allocate arrays for data, parity and missing_idxs
627  */
628  data = alloc_zeroed_buffer(sizeof(char*) * k);
629  if (NULL == data) {
630  log_error("Could not allocate data buffer!");
631  goto out;
632  }
633 
634  parity = alloc_zeroed_buffer(sizeof(char*) * m);
635  if (NULL == parity) {
636  log_error("Could not allocate parity buffer!");
637  goto out;
638  }
639 
640  missing_idxs = alloc_and_set_buffer(sizeof(char*) * (k + m), -1);
641  if (NULL == missing_idxs) {
642  log_error("Could not allocate missing_idxs buffer!");
643  goto out;
644  }
645 
646  /* If metadata checks requested, check fragment integrity upfront */
647  if (force_metadata_checks) {
648  int num_invalid_fragments = 0;
649  for (i = 0; i < num_fragments; ++i) {
650  if (is_invalid_fragment(desc, available_fragments[i])) {
651  ++num_invalid_fragments;
652  }
653  }
654  if ((num_fragments - num_invalid_fragments) < k) {
655  ret = -EINSUFFFRAGS;
656  log_error("Not enough valid fragments available for decode!");
657  goto out;
658  }
659  }
660 
661  /*
662  * Separate the fragments into data and parity. Also determine which
663  * pieces are missing.
664  */
665  ret = get_fragment_partition(k, m, available_fragments, num_fragments,
666  data, parity, missing_idxs);
667 
668  if (ret < 0) {
669  log_error("Could not properly partition the fragments!");
670  goto out;
671  }
672 
673  /*
674  * Preparing the fragments for decode. This will alloc aligned buffers
675  * when unaligned buffers were passed in available_fragments. It passes
676  * back a bitmap telling us which buffers need to be freed by us
677  * (realloc_bm).
678  *
679  */
680  ret = prepare_fragments_for_decode(k, m,
681  data, parity, missing_idxs,
682  &orig_data_size, &blocksize,
683  fragment_len, &realloc_bm);
684  if (ret < 0) {
685  log_error("Could not prepare fragments for decode!");
686  goto out;
687  }
688 
689  data_segments = alloc_zeroed_buffer(k * sizeof(char *));
690  parity_segments = alloc_zeroed_buffer(m * sizeof(char *));
691  get_data_ptr_array_from_fragments(data_segments, data, k);
692  get_data_ptr_array_from_fragments(parity_segments, parity, m);
693 
694  /* call the backend decode function passing it desc instance */
695  ret = instance->common.ops->decode(instance->desc.backend_desc,
696  data_segments, parity_segments,
697  missing_idxs, blocksize);
698 
699  if (ret < 0) {
700  log_error("Encountered error in backend decode function!");
701  goto out;
702  }
703 
704  /*
705  * Need to fill in the missing data headers so we can generate
706  * the original string.
707  */
708  j = 0;
709  while (missing_idxs[j] >= 0) {
710  int set_chksum = 1;
711  int missing_idx = missing_idxs[j];
712  if (missing_idx < k) {
713  /* Generate headers */
714  char *fragment_ptr = data[missing_idx];
715  init_fragment_header(fragment_ptr);
716  add_fragment_metadata(instance, fragment_ptr, missing_idx,
717  orig_data_size, blocksize, instance->args.uargs.ct,
718  !set_chksum);
719  }
720  j++;
721  }
722 
723  /* Try to generate the original string */
724  ret = fragments_to_string(k, m, data, k, out_data, out_data_len);
725 
726  if (ret < 0) {
727  log_error("Could not convert decoded fragments to a string!");
728  }
729 
730 out:
731  /* Free the buffers allocated in prepare_fragments_for_decode */
732  if (realloc_bm != 0) {
733  for (i = 0; i < k; i++) {
734  if (realloc_bm & (1 << i)) {
735  free(data[i]);
736  }
737  }
738 
739  for (i = 0; i < m; i++) {
740  if (realloc_bm & (1 << (i + k))) {
741  free(parity[i]);
742  }
743  }
744  }
745 
746  free(data);
747  free(parity);
748  free(missing_idxs);
749  free(data_segments);
750  free(parity_segments);
751 
752  return ret;
753 }
754 
768  char **available_fragments, /* input */
769  int num_fragments, uint64_t fragment_len, /* input */
770  int destination_idx, /* input */
771  char* out_fragment) /* output */
772 {
773  int ret = 0;
774  int blocksize = 0;
775  int orig_data_size = 0;
776  char **data = NULL;
777  char **parity = NULL;
778  int *missing_idxs = NULL;
779  char *fragment_ptr = NULL;
780  int is_destination_missing = 0;
781  int k = -1;
782  int m = -1;
783  int i;
784  uint64_t realloc_bm = 0;
785  char **data_segments = NULL;
786  char **parity_segments = NULL;
787  int set_chksum = 1;
788 
789  ec_backend_t instance = liberasurecode_backend_instance_get_by_desc(desc);
790  if (NULL == instance) {
791  ret = -EBACKENDNOTAVAIL;
792  goto out;
793  }
794 
795  if (NULL == available_fragments) {
796  log_error("Can not reconstruct fragment, available fragments pointer is NULL");
797  ret = -EINVALIDPARAMS;
798  goto out;
799  }
800 
801  if (NULL == out_fragment) {
802  log_error("Can not reconstruct fragment, output fragment pointer is NULL");
803  ret = -EINVALIDPARAMS;
804  goto out;
805  }
806 
807  k = instance->args.uargs.k;
808  m = instance->args.uargs.m;
809 
810  for (i = 0; i < num_fragments; i++) {
811  /* Verify metadata checksum */
813  (fragment_header_t *) available_fragments[i])) {
814  log_error("Invalid fragment header information!");
815  ret = -EBADHEADER;
816  goto out;
817  }
818  }
819 
820  /*
821  * Allocate arrays for data, parity and missing_idxs
822  */
823  data = alloc_zeroed_buffer(sizeof(char*) * k);
824  if (NULL == data) {
825  log_error("Could not allocate data buffer!");
826  ret = -ENOMEM;
827  goto out;
828  }
829 
830  parity = alloc_zeroed_buffer(sizeof(char*) * m);
831  if (NULL == parity) {
832  log_error("Could not allocate parity buffer!");
833  ret = -ENOMEM;
834  goto out;
835  }
836 
837  missing_idxs = alloc_and_set_buffer(sizeof(int*) * (k + m), -1);
838  if (NULL == missing_idxs) {
839  log_error("Could not allocate missing_idxs buffer!");
840  ret = -ENOMEM;
841  goto out;
842  }
843 
844  /*
845  * Separate the fragments into data and parity. Also determine which
846  * pieces are missing.
847  */
848  ret = get_fragment_partition(k, m, available_fragments, num_fragments,
849  data, parity, missing_idxs);
850 
851  if (ret < 0) {
852  log_error("Could not properly partition the fragments!");
853  goto out;
854  }
855 
856  /*
857  * Odd corner-case: If the caller passes in a destination_idx that
858  * is also included in the available fragments list, we should *not*
859  * try to reconstruct.
860  *
861  * For now, we will log a warning and do nothing. In the future, we
862  * should probably log and return an error.
863  *
864  */
865  i = 0;
866  while (missing_idxs[i] > -1) {
867  if (missing_idxs[i] == destination_idx) {
868  is_destination_missing = 1;
869  }
870  i++;
871  }
872 
873  if (!is_destination_missing) {
874  if (destination_idx < k) {
875  fragment_ptr = data[destination_idx];
876  } else {
877  fragment_ptr = parity[destination_idx - k];
878  }
879  log_warn("Dest idx for reconstruction was supplied as available buffer!");
880  goto destination_available;
881  }
882 
883  /*
884  * Preparing the fragments for reconstruction. This will alloc aligned
885  * buffers when unaligned buffers were passed in available_fragments.
886  * It passes back a bitmap telling us which buffers need to be freed by
887  * us (realloc_bm).
888  */
889  ret = prepare_fragments_for_decode(k, m, data, parity, missing_idxs,
890  &orig_data_size, &blocksize,
891  fragment_len, &realloc_bm);
892  if (ret < 0) {
893  log_error("Could not prepare fragments for reconstruction!");
894  goto out;
895  }
896  data_segments = alloc_zeroed_buffer(k * sizeof(char *));
897  parity_segments = alloc_zeroed_buffer(m * sizeof(char *));
898  get_data_ptr_array_from_fragments(data_segments, data, k);
899  get_data_ptr_array_from_fragments(parity_segments, parity, m);
900 
901 
902  /* call the backend reconstruct function passing it desc instance */
903  ret = instance->common.ops->reconstruct(instance->desc.backend_desc,
904  data_segments, parity_segments,
905  missing_idxs, destination_idx,
906  blocksize);
907  if (ret < 0) {
908  log_error("Could not reconstruct fragment!");
909  goto out;
910  }
911 
912  /*
913  * Update the header to reflect the newly constructed fragment
914  */
915  if (destination_idx < k) {
916  fragment_ptr = data[destination_idx];
917  } else {
918  fragment_ptr = parity[destination_idx - k];
919  }
920  init_fragment_header(fragment_ptr);
921  add_fragment_metadata(instance, fragment_ptr, destination_idx,
922  orig_data_size, blocksize, instance->args.uargs.ct,
923  set_chksum);
924 
925 destination_available:
926  /*
927  * Copy the reconstructed fragment to the output buffer
928  *
929  * Note: the address stored in fragment_ptr will be freed below
930  */
931  memcpy(out_fragment, fragment_ptr, fragment_len);
932 
933 out:
934  /* Free the buffers allocated in prepare_fragments_for_decode */
935  if (realloc_bm != 0) {
936  for (i = 0; i < k; i++) {
937  if (realloc_bm & (1 << i)) {
938  free(data[i]);
939  }
940  }
941 
942  for (i = 0; i < m; i++) {
943  if (realloc_bm & (1 << (i + k))) {
944  free(parity[i]);
945  }
946  }
947  }
948 
949  free(data);
950  free(parity);
951  free(missing_idxs);
952  free(data_segments);
953  free(parity_segments);
954 
955  return ret;
956 }
957 
974  int *fragments_to_reconstruct,
975  int *fragments_to_exclude,
976  int *fragments_needed)
977 {
978  int ret = 0;
979 
980  ec_backend_t instance = liberasurecode_backend_instance_get_by_desc(desc);
981  if (NULL == instance) {
982  ret = -EBACKENDNOTAVAIL;
983  goto out_error;
984  }
985  if (NULL == fragments_to_reconstruct) {
986  log_error("Unable to determine list of fragments needed, pointer to list of indexes to reconstruct is NULL.");
987  ret = -EINVALIDPARAMS;
988  goto out_error;
989  }
990 
991  if (NULL == fragments_to_exclude) {
992  log_error("Unable to determine list of fragments needed, pointer to list of fragments to exclude is NULL.");
993  ret = -EINVALIDPARAMS;
994  goto out_error;
995  }
996 
997  if (NULL == fragments_needed) {
998  log_error("Unable to determine list of fragments needed, pointer to list of fragments to reconstruct is NULL.");
999  ret = -EINVALIDPARAMS;
1000  goto out_error;
1001  }
1002 
1003  /* FIXME preprocessing */
1004 
1005  /* call the backend fragments_needed function passing it desc instance */
1006  ret = instance->common.ops->fragments_needed(
1007  instance->desc.backend_desc,
1008  fragments_to_reconstruct, fragments_to_exclude, fragments_needed);
1009 
1010 out_error:
1011  return ret;
1012 }
1013 
1014 /* =~=*=~==~=*=~==~=*=~==~=*=~===~=*=~==~=*=~===~=*=~==~=*=~===~=*=~==~=*=~= */
1015 
1027  fragment_metadata_t *fragment_metadata)
1028 {
1029  int ret = 0;
1030  fragment_header_t *fragment_hdr = NULL;
1031 
1032  if (NULL == fragment) {
1033  log_error("Need valid fragment object to get metadata for");
1034  ret = -EINVALIDPARAMS;
1035  goto out;
1036  }
1037 
1038  if (NULL == fragment_metadata) {
1039  log_error("Need valid fragment_metadata object for return value");
1040  ret = -EINVALIDPARAMS;
1041  goto out;
1042  }
1043 
1044  /* Verify metadata checksum */
1046  (fragment_header_t *) fragment)) {
1047  log_error("Invalid fragment header information!");
1048  ret = -EBADHEADER;
1049  goto out;
1050  }
1051 
1052  memcpy(fragment_metadata, fragment, sizeof(struct fragment_metadata));
1053  fragment_hdr = (fragment_header_t *) fragment;
1054  if (LIBERASURECODE_FRAG_HEADER_MAGIC != fragment_hdr->magic) {
1055  log_error("Invalid fragment, illegal magic value");
1056  ret = -EINVALIDPARAMS;
1057  goto out;
1058  }
1059 
1060  switch(fragment_hdr->meta.chksum_type) {
1061  case CHKSUM_CRC32: {
1062  uint32_t computed_chksum = 0;
1063  uint32_t stored_chksum = fragment_hdr->meta.chksum[0];
1064  char *fragment_data = get_data_ptr_from_fragment(fragment);
1065  uint64_t fragment_size = fragment_hdr->meta.size;
1066  computed_chksum = crc32(0, fragment_data, fragment_size);
1067  if (stored_chksum != computed_chksum) {
1068  fragment_metadata->chksum_mismatch = 1;
1069  } else {
1070  fragment_metadata->chksum_mismatch = 0;
1071  }
1072  break;
1073  }
1074  case CHKSUM_MD5:
1075  break;
1076  case CHKSUM_NONE:
1077  default:
1078  break;
1079  }
1080 
1081 out:
1082  return ret;
1083 }
1084 
1085 int is_invalid_fragment_header(fragment_header_t *header)
1086 {
1087  uint32_t *stored_csum = NULL, csum = 0;
1088  assert (NULL != header);
1089  if (header->libec_version == 0)
1090  /* libec_version must be bigger than 0 */
1091  return 1;
1092  if (header->libec_version < _VERSION(1,2,0))
1093  /* no metadata checksum support */
1094  return 0;
1095  stored_csum = get_metadata_chksum((char *) header);
1096  if (NULL == stored_csum)
1097  return 1; /* can't verify, possibly crc32 call error */
1098  csum = crc32(0, &header->meta, sizeof(fragment_metadata_t));
1099  return (*stored_csum != csum);
1100 }
1101 
1103  fragment_metadata_t *md)
1104 {
1105  int k = be->args.uargs.k;
1106  int m = be->args.uargs.m;
1107  if (md->idx > (k + m)) {
1108  return 1;
1109  }
1110  if (md->backend_id != be->common.id) {
1111  return 1;
1112  }
1113  if (!be->common.ops->is_compatible_with(md->backend_version)) {
1114  return 1;
1115  }
1116  return 0;
1117 }
1118 
1119 int is_invalid_fragment_metadata(int desc, fragment_metadata_t *fragment_metadata)
1120 {
1121  ec_backend_t be = liberasurecode_backend_instance_get_by_desc(desc);
1122  if (!be) {
1123  log_error("Unable to verify fragment metadata: invalid backend id %d.",
1124  desc);
1125  return -EINVALIDPARAMS;
1126  }
1128  fragment_metadata) != 0) {
1129  return -EBADHEADER;
1130  }
1131  if (!be->common.ops->is_compatible_with(fragment_metadata->backend_version)) {
1132  return -EBADHEADER;
1133  }
1134  if (fragment_metadata->chksum_mismatch == 1) {
1135  return -EBADCHKSUM;
1136  }
1137  return 0;
1138 }
1139 
1140 int is_invalid_fragment(int desc, char *fragment)
1141 {
1142  uint32_t ver = 0;
1143  fragment_metadata_t fragment_metadata;
1144  ec_backend_t be = liberasurecode_backend_instance_get_by_desc(desc);
1145  if (!be) {
1146  log_error("Unable to verify fragment metadata: invalid backend id %d.",
1147  desc);
1148  return 1;
1149  }
1150  if (!fragment) {
1151  log_error("Unable to verify fragment validity: fragments missing.");
1152  return 1;
1153  }
1154  if (get_libec_version(fragment, &ver) != 0 ||
1155  ver > LIBERASURECODE_VERSION) {
1156  return 1;
1157  }
1158  if (liberasurecode_get_fragment_metadata(fragment, &fragment_metadata) != 0) {
1159  return 1;
1160  }
1161  if (is_invalid_fragment_metadata(desc, &fragment_metadata) != 0) {
1162  return 1;
1163  }
1164  return 0;
1165 }
1166 
1168  char **fragments, int num_fragments)
1169 {
1170  int i = 0;
1171  if (!fragments) {
1172  log_error("Unable to verify stripe metadata: fragments missing.");
1173  return -EINVALIDPARAMS;
1174  }
1175  if (num_fragments <= 0) {
1176  log_error("Unable to verify stripe metadata: "
1177  "number of fragments must be greater than 0.");
1178  return -EINVALIDPARAMS;
1179  }
1180 
1181  for (i = 0; i < num_fragments; i++) {
1182  fragment_metadata_t *fragment_metadata = (fragment_metadata_t*)fragments[i];
1183  int ret = is_invalid_fragment_metadata(desc, fragment_metadata);
1184  if (ret < 0) {
1185  return ret;
1186  }
1187  }
1188 
1189  return 0;
1190 }
1191 
1192 /* =~=*=~==~=*=~==~=*=~==~=*=~===~=*=~==~=*=~===~=*=~==~=*=~===~=*=~==~=*=~= */
1193 
1201 int liberasurecode_get_aligned_data_size(int desc, uint64_t data_len)
1202 {
1203  int k;
1204  int ret = 0;
1205  int word_size;
1206  int alignment_multiple;
1207 
1208  ec_backend_t instance = liberasurecode_backend_instance_get_by_desc(desc);
1209  if (NULL == instance) {
1210  ret = -EBACKENDNOTAVAIL;
1211  goto out;
1212  }
1213 
1214  k = instance->args.uargs.k;
1215 
1216  word_size = instance->common.ops->element_size(
1217  instance->desc.backend_desc) / 8;
1218 
1219  alignment_multiple = k * word_size;
1220 
1221  ret = ((data_len + alignment_multiple - 1) / alignment_multiple)
1222  * alignment_multiple;
1223 
1224 out:
1225  return ret;
1226 }
1227 
1233 {
1234  return liberasurecode_get_aligned_data_size(desc, 1);
1235 }
1236 
1237 int liberasurecode_get_fragment_size(int desc, int data_len)
1238 {
1239  ec_backend_t instance = liberasurecode_backend_instance_get_by_desc(desc);
1240  // TODO: Create a common function to calculate fragment size also for preprocessing
1241  if (NULL == instance)
1242  return -EBACKENDNOTAVAIL;
1243  int aligned_data_len = get_aligned_data_size(instance, data_len);
1244  int blocksize = aligned_data_len / instance->args.uargs.k;
1245  int metadata_size = instance->common.ops->get_backend_metadata_size(
1246  instance->desc.backend_desc,
1247  blocksize);
1248  int size = blocksize + metadata_size;
1249 
1250  return size;
1251 }
1252 
1253 
1259 {
1260  return LIBERASURECODE_VERSION;
1261 }
1262 
1263 /* ==~=*=~==~=*=~==~=*=~==~=*=~==~=* misc *=~==~=*=~==~=*=~==~=*=~==~=*=~== */
1264 
1265 #if 0
1266 /* Validate backend before calling init */
1267 int liberasurecode_backend_validate(ec_backend_t backend)
1268 {
1269  /* Verify that the backend implements all required methods */
1270 }
1271 
1272 /* FIXME - do we need to use reference counts if we are creating
1273 * a new instance per user */
1274 
1275 /* Get a reference to an EC backend */
1276 ec_backend_t liberasurecode_backend_get(const char *name)
1277 {
1278  ec_backend_t b = liberasurecode_backend_lookup_by_name(name);
1279  if (NULL != b)
1280  ++b->users;
1281  return b;
1282 }
1283 
1284 /* Drop an EC backend reference held */
1285 void liberasurecode_backend_put(ec_backend_t backend)
1286 {
1287  if (backend->users > 0)
1288  --backend->users;
1289 }
1290 
1291 /* Query interface for active instances */
1292 ec_backend_t liberasurecode_backend_instance_active(ec_backend_t instance)
1293 {
1294  ec_backend_t b;
1295 
1296  SLIST_FOREACH(b, &active_instances, link) {
1297  if (strcmp(b->name, name) == 0)
1298  return b;
1299  }
1300 
1301  return NULL;
1302 }
1303 
1304 ec_backend_t liberasurecode_backend_lookup_by_soname(const char *soname)
1305 {
1306  ec_backend_t b;
1307 
1308  SLIST_FOREACH(b, &active_instances, link) {
1309  if (strcmp(b->soname, soname) == 0)
1310  return b;
1311  }
1312 
1313  return NULL;
1314 }
1315 #endif
1316 
1317 /* ==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~== */
void * alloc_and_set_buffer(int size, int value)
Allocate a buffer of a specific size and set its' contents to the specified value.
int get_fragment_ptr_array_from_data(char **frag_array, char **data, int num_data)
struct ec_backend_common backend_jerasure_rs_cauchy
SLIST_HEAD(backend_list, ec_backend)
Look up a backend instance by descriptor.
Definition: erasurecode.c:75
int get_fragment_partition(int k, int m, char **fragments, int num_fragments, char **data, char **parity, int *missing)
struct ec_backend_common backend_jerasure_rs_vand
int get_data_ptr_array_from_fragments(char **data_array, char **fragments, int num_fragments)
int liberasurecode_reconstruct_fragment(int desc, char **available_fragments, int num_fragments, uint64_t fragment_len, int destination_idx, char *out_fragment)
Reconstruct a missing fragment from a subset of available fragments.
Definition: erasurecode.c:767
int liberasurecode_get_minimum_encode_size(int desc)
This will return the minumum encode size, which is the minimum buffer size that can be encoded...
Definition: erasurecode.c:1232
void * liberasurecode_backend_open(ec_backend_t instance)
Definition: erasurecode.c:176
uint32_t * get_metadata_chksum(char *buf)
int liberasurecode_verify_fragment_metadata(ec_backend_t be, fragment_metadata_t *md)
Definition: erasurecode.c:1102
int liberasurecode_get_aligned_data_size(int desc, uint64_t data_len)
This computes the aligned size of a buffer passed into the encode function.
Definition: erasurecode.c:1201
int get_aligned_data_size(ec_backend_t instance, int data_len)
Compute a size aligned to the number of data and the underlying wordsize of the EC algorithm...
int liberasurecode_backend_close(ec_backend_t instance)
Definition: erasurecode.c:184
struct ec_backend_common backend_isa_l_rs_vand
Definition: isa_l_rs_vand.c:48
int liberasurecode_backend_available(const ec_backend_id_t backend_id)
Checks if a given backend is available.
Definition: erasurecode.c:231
struct ec_backend_common backend_null
Definition: null.c:231
char * get_data_ptr_from_fragment(char *buf)
int liberasurecode_verify_stripe_metadata(int desc, char **fragments, int num_fragments)
Definition: erasurecode.c:1167
void * alloc_zeroed_buffer(int size)
Allocate a zero-ed buffer of a specific size.
int prepare_fragments_for_decode(int k, int m, char **data, char **parity, int *missing_idxs, int *orig_size, int *fragment_payload_size, int fragment_size, uint64_t *realloc_bm)
int liberasurecode_instance_destroy(int desc)
Close a liberasurecode instance.
Definition: erasurecode.c:323
int liberasurecode_get_fragment_metadata(char *fragment, fragment_metadata_t *fragment_metadata)
Get opaque metadata for a fragment.
Definition: erasurecode.c:1026
int is_invalid_fragment(int desc, char *fragment)
Definition: erasurecode.c:1140
int finalize_fragments_after_encode(ec_backend_t instance, int k, int m, int blocksize, uint64_t orig_data_size, char **encoded_data, char **encoded_parity)
void add_fragment_metadata(ec_backend_t be, char *fragment, int idx, uint64_t orig_data_size, int blocksize, ec_checksum_type_t ct, int add_chksum)
struct ec_backend_common backend_liberasurecode_rs_vand
struct ec_backend_common backend_isa_l_rs_cauchy
int liberasurecode_encode_cleanup(int desc, char **encoded_data, char **encoded_parity)
Cleanup structures allocated by librasurecode_encode.
Definition: erasurecode.c:362
int is_invalid_fragment_header(fragment_header_t *header)
Definition: erasurecode.c:1085
int liberasurecode_backend_alloc_desc(void)
Allocated backend instance descriptor.
Definition: erasurecode.c:104
uint64_t get_fragment_size(char *buf)
Return total fragment length (on-disk, on-wire)
char * ec_backends_supported_str[EC_BACKENDS_MAX]
Definition: erasurecode.c:70
struct ec_backend_common backend_flat_xor_hd
Definition: flat_xor_hd.c:51
int crc32(int crc, const void *buf, size_t size)
Definition: crc32.c:119
int liberasurecode_backend_instance_unregister(ec_backend_t instance)
Unregister a backend instance.
Definition: erasurecode.c:148
int liberasurecode_fragments_needed(int desc, int *fragments_to_reconstruct, int *fragments_to_exclude, int *fragments_needed)
Return a list of lists with valid rebuild indexes given a list of missing indexes.
Definition: erasurecode.c:973
int liberasurecode_get_fragment_size(int desc, int data_len)
Definition: erasurecode.c:1237
ec_backend_t ec_backends_supported[]
Definition: erasurecode.c:55
static void print_dlerror(const char *caller)
Definition: erasurecode.c:166
int num_supported_backends
Definition: erasurecode.c:69
uint32_t liberasurecode_get_version()
This will return the liberasurecode version for the descriptor.
Definition: erasurecode.c:1258
int liberasurecode_instance_create(const ec_backend_id_t id, struct ec_args *args)
Create a liberasurecode instance and return a descriptor for use with EC operations (encode...
Definition: erasurecode.c:265
int fragments_to_string(int k, int m, char **fragments, int num_fragments, char **orig_payload, uint64_t *payload_len)
struct ec_backend_common backend_shss
Definition: shss.c:43
void __attribute__((constructor))
Definition: erasurecode.c:198
int get_libec_version(char *buf, uint32_t *ver)
struct ec_backend_common backend_libphazr
Definition: libphazr.c:50
int is_invalid_fragment_metadata(int desc, fragment_metadata_t *fragment_metadata)
Definition: erasurecode.c:1119
int prepare_fragments_for_encode(ec_backend_t instance, int k, int m, const char *orig_data, uint64_t orig_data_size, char **encoded_data, char **encoded_parity, int *blocksize)
int liberasurecode_decode(int desc, char **available_fragments, int num_fragments, uint64_t fragment_len, int force_metadata_checks, char **out_data, uint64_t *out_data_len)
Reconstruct original data from a set of k encoded fragments.
Definition: erasurecode.c:537
int liberasurecode_backend_instance_register(ec_backend_t instance)
Register a backend instance with liberasurecode.
Definition: erasurecode.c:121
int liberasurecode_encode(int desc, const char *orig_data, uint64_t orig_data_size, char ***encoded_data, char ***encoded_parity, uint64_t *fragment_len)
Erasure encode a data buffer.
Definition: erasurecode.c:410
int liberasurecode_decode_cleanup(int desc, char *data)
Cleanup structures allocated by librasurecode_decode.
Definition: erasurecode.c:512