liberasurecode  1.2.0
Erasure Code API library
 All Data Structures Files Functions Variables Typedefs Macros
erasurecode.c
Go to the documentation of this file.
1 /*
2  * Copyright 2014 Tushar Gohad, Kevin M Greenan, Eric Lambert, Mark Storer
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions are met:
6  *
7  * Redistributions of source code must retain the above copyright notice, this
8  * list of conditions and the following disclaimer.
9  *
10  * Redistributions in binary form must reproduce the above copyright notice, this
11  * list of conditions and the following disclaimer in the documentation and/or
12  * other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY
13  * THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED
14  * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
15  * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
16  * EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
17  * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
18  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
19  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
20  * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
21  * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
22  * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
23  *
24  * liberasurecode API implementation
25  *
26  * vi: set noai tw=79 ts=4 sw=4:
27  */
28 
29 #include "assert.h"
30 #include "list.h"
31 #include "erasurecode.h"
32 #include "erasurecode_backend.h"
33 #include "erasurecode_helpers.h"
34 #include "erasurecode_helpers_ext.h"
35 #include "erasurecode_preprocessing.h"
36 #include "erasurecode_postprocessing.h"
37 #include "erasurecode_stdinc.h"
38 
39 #include "alg_sig.h"
40 #include "erasurecode_log.h"
41 
42 /* =~=*=~==~=*=~==~=*=~= Supported EC backends =~=*=~==~=*=~==~=*=~==~=*=~== */
43 
44 /* EC backend references */
45 extern struct ec_backend_common backend_null;
46 extern struct ec_backend_common backend_flat_xor_hd;
47 extern struct ec_backend_common backend_jerasure_rs_vand;
48 extern struct ec_backend_common backend_jerasure_rs_cauchy;
49 extern struct ec_backend_common backend_isa_l_rs_vand;
50 extern struct ec_backend_common backend_shss;
51 extern struct ec_backend_common backend_liberasurecode_rs_vand;
52 
53 ec_backend_t ec_backends_supported[] = {
54  (ec_backend_t) &backend_null,
55  (ec_backend_t) &backend_jerasure_rs_vand,
56  (ec_backend_t) &backend_jerasure_rs_cauchy,
57  (ec_backend_t) &backend_flat_xor_hd,
58  (ec_backend_t) &backend_isa_l_rs_vand,
59  (ec_backend_t) &backend_shss,
60  (ec_backend_t) &backend_liberasurecode_rs_vand,
61  NULL,
62 };
63 
64 /* backend list to return to the caller */
66 char *ec_backends_supported_str[EC_BACKENDS_MAX];
67 
68 /* =~=*=~==~=*=~==~=*=~= EC backend instance management =~=*=~==~=*=~==~=*= */
69 
70 /* Registered erasure code backend instances */
71 SLIST_HEAD(backend_list, ec_backend) active_instances =
72  SLIST_HEAD_INITIALIZER(active_instances);
73 rwlock_t active_instances_rwlock = RWLOCK_INITIALIZER;
74 
75 /* Backend instance id */
76 int next_backend_desc = 0;
77 
84 ec_backend_t liberasurecode_backend_instance_get_by_desc(int desc)
85 {
86  struct ec_backend *b = NULL;
87  SLIST_FOREACH(b, &active_instances, link) {
88  if (b->idesc == desc)
89  break;
90  }
91  return b;
92 }
93 
101 {
102  for (;;) {
103  if (++next_backend_desc <= 0)
104  next_backend_desc = 1;
105  if (!liberasurecode_backend_instance_get_by_desc(next_backend_desc))
106  return next_backend_desc;
107  }
108 }
109 
118 {
119  int desc = -1; /* descriptor to return */
120  int rc = 0; /* return call value */
121 
122  rc = rwlock_wrlock(&active_instances_rwlock);
123  if (rc == 0) {
124  SLIST_INSERT_HEAD(&active_instances, instance, link);
126  if (desc <= 0)
127  goto register_out;
128  instance->idesc = desc;
129  } else {
130  goto exit;
131  }
132 
133 register_out:
134  rwlock_unlock(&active_instances_rwlock);
135 exit:
136  return desc;
137 }
138 
145 {
146  int rc = 0; /* return call value */
147 
148  rc = rwlock_wrlock(&active_instances_rwlock);
149  if (rc == 0) {
150  SLIST_REMOVE(&active_instances, instance, ec_backend, link);
151  } else {
152  goto exit;
153  }
154  rwlock_unlock(&active_instances_rwlock);
155 
156 exit:
157  return rc;
158 }
159 
160 /* =~=*=~==~=*=~== liberasurecode backend API helpers =~=*=~==~=*=~== */
161 
162 static void print_dlerror(const char *caller)
163 {
164  char *msg = dlerror();
165  if (NULL == msg)
166  log_error("%s: unknown dynamic linking error\n", caller);
167  else
168  log_error("%s: dynamic linking error %s\n", caller, msg);
169 }
170 
171 /* Generic dlopen/dlclose routines */
172 void* liberasurecode_backend_open(ec_backend_t instance)
173 {
174  if (NULL == instance)
175  return NULL;
176  /* Use RTLD_LOCAL to avoid symbol collisions */
177  return dlopen(instance->common.soname, RTLD_LAZY | RTLD_LOCAL);
178 }
179 
180 int liberasurecode_backend_close(ec_backend_t instance)
181 {
182  if (NULL == instance || NULL == instance->desc.backend_sohandle)
183  return 0;
184 
185  dlclose(instance->desc.backend_sohandle);
186  dlerror(); /* Clear any existing errors */
187 
188  instance->desc.backend_sohandle = NULL;
189  return 0;
190 }
191 
192 /* =*=~==~=*=~==~=*=~= liberasurecode init/exit routines =~=*=~==~=*=~==~=*= */
193 
194 void __attribute__ ((constructor))
195 liberasurecode_init(void) {
196  /* init logging */
197  openlog("liberasurecode", LOG_PID | LOG_CONS, LOG_USER);
198 
199  /* populate supported backends list as a string */
200  {
201  int i;
202  for (i = 0; ec_backends_supported[i]; ++i) {
203  ec_backends_supported_str[i] = strdup(
204  ec_backends_supported[i]->common.name);
205  }
206  num_supported_backends = i;
207  }
208 }
209 
210 void __attribute__ ((destructor))
211 liberasurecode_exit(void) {
212  int i;
213  for (i = 0; i < num_supported_backends; ++i)
214  free(ec_backends_supported_str[i]);
215  closelog();
216 }
217 
218 /* =~=*=~==~=*=~= liberasurecode frontend API implementation =~=*=~==~=*=~== */
219 
227 int liberasurecode_backend_available(const ec_backend_id_t backend_id) {
228  struct ec_backend backend;
229  if (backend_id >= EC_BACKENDS_MAX)
230  return 0;
231 
232  backend.desc.backend_sohandle = liberasurecode_backend_open(
233  ec_backends_supported[backend_id]);
234  if (!backend.desc.backend_sohandle) {
235  return 0;
236  }
237 
239  return 1;
240 }
241 
261 int liberasurecode_instance_create(const ec_backend_id_t id,
262  struct ec_args *args)
263 {
264  ec_backend_t instance = NULL;
265  struct ec_backend_args bargs;
266  if (!args)
267  return -EINVALIDPARAMS;
268 
269  if (id >= EC_BACKENDS_MAX)
270  return -EBACKENDNOTSUPP;
271 
272  if ((args->k + args->m) > EC_MAX_FRAGMENTS) {
273  log_error("Total number of fragments (k + m) must be less than %d\n",
274  EC_MAX_FRAGMENTS);
275  return -EINVALIDPARAMS;
276  }
277 
278  /* Allocate memory for ec_backend instance */
279  instance = calloc(1, sizeof(*instance));
280  if (NULL == instance)
281  return -ENOMEM;
282 
283  /* Copy common backend, args struct */
284  instance->common = ec_backends_supported[id]->common;
285  memcpy(&(bargs.uargs), args, sizeof (struct ec_args));
286  instance->args = bargs;
287 
288  /* Open backend .so if not already open */
289  /* .so handle is returned in instance->desc.backend_sohandle */
290  if (!instance->desc.backend_sohandle) {
291  instance->desc.backend_sohandle = liberasurecode_backend_open(instance);
292  if (!instance->desc.backend_sohandle) {
293  /* ignore during init, return the same handle */
294  print_dlerror(__func__);
295  free(instance);
296  return -EBACKENDNOTAVAIL;
297  }
298  }
299 
300  /* Call private init() for the backend */
301  instance->desc.backend_desc = instance->common.ops->init(
302  &instance->args, instance->desc.backend_sohandle);
303  if (NULL == instance->desc.backend_desc) {
304  free (instance);
305  return -EBACKENDINITERR;
306  }
307 
308  /* Register instance and return a descriptor/instance id */
309  instance->idesc = liberasurecode_backend_instance_register(instance);
310 
311  return instance->idesc;
312 }
313 
320 {
321  ec_backend_t instance = NULL; /* instance to destroy */
322  int rc = 0; /* return code */
323 
324  instance = liberasurecode_backend_instance_get_by_desc(desc);
325  if (NULL == instance)
326  return -EBACKENDNOTAVAIL;
327 
328  /* Call private exit() for the backend */
329  instance->common.ops->exit(instance->desc.backend_desc);
330 
331  /* dlclose() backend library */
333 
334  /* Remove instace from registry */
336  if (rc == 0) {
337  free(instance);
338  }
339 
340  return rc;
341 }
342 
359  char **encoded_data,
360  char **encoded_parity)
361 {
362  int i, k, m;
363 
364  ec_backend_t instance = liberasurecode_backend_instance_get_by_desc(desc);
365  if (NULL == instance) {
366  return -EBACKENDNOTAVAIL;
367  }
368 
369  k = instance->args.uargs.k;
370  m = instance->args.uargs.m;
371 
372  if (encoded_data) {
373  for (i = 0; i < k; i++) {
374  free(encoded_data[i]);
375  }
376 
377  free(encoded_data);
378  }
379 
380  if (encoded_parity) {
381  for (i = 0; i < m; i++) {
382  free(encoded_parity[i]);
383  }
384  free(encoded_parity);
385  }
386 
387  return 0;
388 }
389 
407  const char *orig_data, uint64_t orig_data_size, /* input */
408  char ***encoded_data, char ***encoded_parity, /* output */
409  uint64_t *fragment_len) /* output */
410 {
411  int k, m;
412  int ret = 0; /* return code */
413 
414  int blocksize = 0; /* length of each of k data elements */
415 
416  if (orig_data == NULL) {
417  log_error("Pointer to data buffer is null!");
418  ret = -EINVALIDPARAMS;
419  goto out;
420  }
421 
422  if (encoded_data == NULL) {
423  log_error("Pointer to encoded data buffers is null!");
424  return -EINVALIDPARAMS;
425  }
426 
427  if (encoded_parity == NULL) {
428  log_error("Pointer to encoded parity buffers is null!");
429  return -EINVALIDPARAMS;
430  }
431 
432  if (fragment_len == NULL) {
433  log_error("Pointer to fragment length is null!");
434  ret = -EINVALIDPARAMS;
435  goto out;
436  }
437 
438  ec_backend_t instance = liberasurecode_backend_instance_get_by_desc(desc);
439  if (NULL == instance) {
440  ret = -EBACKENDNOTAVAIL;
441  goto out;
442  }
443 
444  k = instance->args.uargs.k;
445  m = instance->args.uargs.m;
446 
447  /*
448  * Allocate arrays for data, parity and missing_idxs
449  */
450  *encoded_data = (char **) alloc_zeroed_buffer(sizeof(char *) * k);
451  if (NULL == *encoded_data) {
452  log_error("Could not allocate data buffer!");
453  goto out;
454  }
455 
456  *encoded_parity = (char **) alloc_zeroed_buffer(sizeof(char *) * m);
457  if (NULL == *encoded_parity) {
458  log_error("Could not allocate parity buffer!");
459  goto out;
460  }
461 
462  ret = prepare_fragments_for_encode(instance, k, m, orig_data, orig_data_size,
463  *encoded_data, *encoded_parity, &blocksize);
464  if (ret < 0) {
465  // ensure encoded_data/parity point the head of fragment_ptr
466  get_fragment_ptr_array_from_data(*encoded_data, *encoded_data, k);
467  get_fragment_ptr_array_from_data(*encoded_parity, *encoded_parity, m);
468  goto out;
469  }
470 
471  /* call the backend encode function passing it desc instance */
472  ret = instance->common.ops->encode(instance->desc.backend_desc,
473  *encoded_data, *encoded_parity, blocksize);
474  if (ret < 0) {
475  // ensure encoded_data/parity point the head of fragment_ptr
476  get_fragment_ptr_array_from_data(*encoded_data, *encoded_data, k);
477  get_fragment_ptr_array_from_data(*encoded_parity, *encoded_parity, m);
478  goto out;
479  }
480 
481  ret = finalize_fragments_after_encode(instance, k, m, blocksize, orig_data_size,
482  *encoded_data, *encoded_parity);
483 
484  *fragment_len = get_fragment_size((*encoded_data)[0]);
485 
486 out:
487  if (ret) {
488  /* Cleanup the allocations we have done */
489  liberasurecode_encode_cleanup(desc, *encoded_data, *encoded_parity);
490  log_error("Error in liberasurecode_encode %d", ret);
491  }
492  return ret;
493 }
494 
508 int liberasurecode_decode_cleanup(int desc, char *data)
509 {
510  ec_backend_t instance = liberasurecode_backend_instance_get_by_desc(desc);
511  if (NULL == instance) {
512  return -EBACKENDNOTAVAIL;
513  }
514 
515  free(data);
516 
517  return 0;
518 }
519 
534  char **available_fragments, /* input */
535  int num_fragments, uint64_t fragment_len, /* input */
536  int force_metadata_checks, /* input */
537  char **out_data, uint64_t *out_data_len) /* output */
538 {
539  int i, j;
540  int ret = 0;
541 
542  int k = -1, m = -1;
543  int orig_data_size = 0;
544 
545  int blocksize = 0;
546  char **data = NULL;
547  char **parity = NULL;
548  char **data_segments = NULL;
549  char **parity_segments = NULL;
550  int *missing_idxs = NULL;
551 
552  uint64_t realloc_bm = 0;
553 
554  ec_backend_t instance = liberasurecode_backend_instance_get_by_desc(desc);
555  if (NULL == instance) {
556  ret = -EBACKENDNOTAVAIL;
557  goto out;
558  }
559 
560  if (NULL == available_fragments) {
561  log_error("Pointer to encoded fragments buffer is null!");
562  ret = -EINVALIDPARAMS;
563  goto out;
564  }
565 
566  if (NULL == out_data) {
567  log_error("Pointer to decoded data buffer is null!");
568  ret = -EINVALIDPARAMS;
569  goto out;
570  }
571 
572  if (NULL == out_data_len) {
573  log_error("Pointer to decoded data length variable is null!");
574  ret = -EINVALIDPARAMS;
575  goto out;
576  }
577 
578  k = instance->args.uargs.k;
579  m = instance->args.uargs.m;
580 
581  if (num_fragments < k) {
582  log_error("Not enough fragments to decode, got %d, need %d!",
583  num_fragments, k);
584  ret = -EINSUFFFRAGS;
585  goto out;
586  }
587 
588  for (i = 0; i < num_fragments; ++i) {
589  /* Verify metadata checksum */
591  (fragment_header_t *) available_fragments[i])) {
592  log_error("Invalid fragment header information!");
593  ret = -EBADHEADER;
594  goto out;
595  }
596  }
597 
598  if (instance->common.id != EC_BACKEND_SHSS) {
599  /* shss (ntt_backend) must force to decode */
600  // TODO: Add a frag and function to handle whether the backend want to decode or not.
601  /*
602  * Try to re-assebmle the original data before attempting a decode
603  */
604  ret = fragments_to_string(k, m,
605  available_fragments, num_fragments,
606  out_data, out_data_len);
607 
608  if (ret == 0) {
609  /* We were able to get the original data without decoding! */
610  goto out;
611  }
612  }
613 
614  /*
615  * Allocate arrays for data, parity and missing_idxs
616  */
617  data = alloc_zeroed_buffer(sizeof(char*) * k);
618  if (NULL == data) {
619  log_error("Could not allocate data buffer!");
620  goto out;
621  }
622 
623  parity = alloc_zeroed_buffer(sizeof(char*) * m);
624  if (NULL == parity) {
625  log_error("Could not allocate parity buffer!");
626  goto out;
627  }
628 
629  missing_idxs = alloc_and_set_buffer(sizeof(char*) * (k + m), -1);
630  if (NULL == missing_idxs) {
631  log_error("Could not allocate missing_idxs buffer!");
632  goto out;
633  }
634 
635  /* If metadata checks requested, check fragment integrity upfront */
636  if (force_metadata_checks) {
637  int num_invalid_fragments = 0;
638  for (i = 0; i < num_fragments; ++i) {
639  if (is_invalid_fragment(desc, available_fragments[i])) {
640  ++num_invalid_fragments;
641  }
642  }
643  if ((num_fragments - num_invalid_fragments) < k) {
644  ret = -EINSUFFFRAGS;
645  log_error("Not enough valid fragments available for decode!");
646  goto out;
647  }
648  }
649 
650  /*
651  * Separate the fragments into data and parity. Also determine which
652  * pieces are missing.
653  */
654  ret = get_fragment_partition(k, m, available_fragments, num_fragments,
655  data, parity, missing_idxs);
656 
657  if (ret < 0) {
658  log_error("Could not properly partition the fragments!");
659  goto out;
660  }
661 
662  /*
663  * Preparing the fragments for decode. This will alloc aligned buffers
664  * when unaligned buffers were passed in available_fragments. It passes
665  * back a bitmap telling us which buffers need to be freed by us
666  * (realloc_bm).
667  *
668  */
669  ret = prepare_fragments_for_decode(k, m,
670  data, parity, missing_idxs,
671  &orig_data_size, &blocksize,
672  fragment_len, &realloc_bm);
673  if (ret < 0) {
674  log_error("Could not prepare fragments for decode!");
675  goto out;
676  }
677 
678  data_segments = alloc_zeroed_buffer(k * sizeof(char *));
679  parity_segments = alloc_zeroed_buffer(m * sizeof(char *));
680  get_data_ptr_array_from_fragments(data_segments, data, k);
681  get_data_ptr_array_from_fragments(parity_segments, parity, m);
682 
683  /* call the backend decode function passing it desc instance */
684  ret = instance->common.ops->decode(instance->desc.backend_desc,
685  data_segments, parity_segments,
686  missing_idxs, blocksize);
687 
688  if (ret < 0) {
689  log_error("Encountered error in backend decode function!");
690  goto out;
691  }
692 
693  /*
694  * Need to fill in the missing data headers so we can generate
695  * the original string.
696  */
697  j = 0;
698  while (missing_idxs[j] >= 0) {
699  int set_chksum = 1;
700  int missing_idx = missing_idxs[j];
701  if (missing_idx < k) {
702  /* Generate headers */
703  char *fragment_ptr = data[missing_idx];
704  init_fragment_header(fragment_ptr);
705  add_fragment_metadata(instance, fragment_ptr, missing_idx,
706  orig_data_size, blocksize, instance->args.uargs.ct,
707  !set_chksum);
708  }
709  j++;
710  }
711 
712  /* Try to generate the original string */
713  ret = fragments_to_string(k, m, data, k, out_data, out_data_len);
714 
715  if (ret < 0) {
716  log_error("Could not convert decoded fragments to a string!");
717  }
718 
719 out:
720  /* Free the buffers allocated in prepare_fragments_for_decode */
721  if (realloc_bm != 0) {
722  for (i = 0; i < k; i++) {
723  if (realloc_bm & (1 << i)) {
724  free(data[i]);
725  }
726  }
727 
728  for (i = 0; i < m; i++) {
729  if (realloc_bm & (1 << (i + k))) {
730  free(parity[i]);
731  }
732  }
733  }
734 
735  free(data);
736  free(parity);
737  free(missing_idxs);
738  free(data_segments);
739  free(parity_segments);
740 
741  return ret;
742 }
743 
757  char **available_fragments, /* input */
758  int num_fragments, uint64_t fragment_len, /* input */
759  int destination_idx, /* input */
760  char* out_fragment) /* output */
761 {
762  int ret = 0;
763  int blocksize = 0;
764  int orig_data_size = 0;
765  char **data = NULL;
766  char **parity = NULL;
767  int *missing_idxs = NULL;
768  char *fragment_ptr = NULL;
769  int is_destination_missing = 0;
770  int k = -1;
771  int m = -1;
772  int i;
773  uint64_t realloc_bm = 0;
774  char **data_segments = NULL;
775  char **parity_segments = NULL;
776  int set_chksum = 1;
777 
778  ec_backend_t instance = liberasurecode_backend_instance_get_by_desc(desc);
779  if (NULL == instance) {
780  ret = -EBACKENDNOTAVAIL;
781  goto out;
782  }
783 
784  if (NULL == available_fragments) {
785  log_error("Can not reconstruct fragment, available fragments pointer is NULL");
786  ret = -EINVALIDPARAMS;
787  goto out;
788  }
789 
790  if (NULL == out_fragment) {
791  log_error("Can not reconstruct fragment, output fragment pointer is NULL");
792  ret = -EINVALIDPARAMS;
793  goto out;
794  }
795 
796  k = instance->args.uargs.k;
797  m = instance->args.uargs.m;
798 
799  for (i = 0; i < num_fragments; i++) {
800  /* Verify metadata checksum */
802  (fragment_header_t *) available_fragments[i])) {
803  log_error("Invalid fragment header information!");
804  ret = -EBADHEADER;
805  goto out;
806  }
807  }
808 
809  /*
810  * Allocate arrays for data, parity and missing_idxs
811  */
812  data = alloc_zeroed_buffer(sizeof(char*) * k);
813  if (NULL == data) {
814  log_error("Could not allocate data buffer!");
815  goto out;
816  }
817 
818  parity = alloc_zeroed_buffer(sizeof(char*) * m);
819  if (NULL == parity) {
820  log_error("Could not allocate parity buffer!");
821  goto out;
822  }
823 
824  missing_idxs = alloc_and_set_buffer(sizeof(int*) * (k + m), -1);
825  if (NULL == missing_idxs) {
826  log_error("Could not allocate missing_idxs buffer!");
827  goto out;
828  }
829 
830  /*
831  * Separate the fragments into data and parity. Also determine which
832  * pieces are missing.
833  */
834  ret = get_fragment_partition(k, m, available_fragments, num_fragments,
835  data, parity, missing_idxs);
836 
837  if (ret < 0) {
838  log_error("Could not properly partition the fragments!");
839  goto out;
840  }
841 
842  /*
843  * Odd corner-case: If the caller passes in a destination_idx that
844  * is also included in the available fragments list, we should *not*
845  * try to reconstruct.
846  *
847  * For now, we will log a warning and do nothing. In the future, we
848  * should probably log and return an error.
849  *
850  */
851  i = 0;
852  while (missing_idxs[i] > -1) {
853  if (missing_idxs[i] == destination_idx) {
854  is_destination_missing = 1;
855  }
856  i++;
857  }
858 
859  if (!is_destination_missing) {
860  if (destination_idx < k) {
861  fragment_ptr = data[destination_idx];
862  } else {
863  fragment_ptr = parity[destination_idx - k];
864  }
865  log_warn("Dest idx for reconstruction was supplied as available buffer!");
866  goto destination_available;
867  }
868 
869  /*
870  * Preparing the fragments for reconstruction. This will alloc aligned
871  * buffers when unaligned buffers were passed in available_fragments.
872  * It passes back a bitmap telling us which buffers need to be freed by
873  * us (realloc_bm).
874  */
875  ret = prepare_fragments_for_decode(k, m, data, parity, missing_idxs,
876  &orig_data_size, &blocksize,
877  fragment_len, &realloc_bm);
878  if (ret < 0) {
879  log_error("Could not prepare fragments for reconstruction!");
880  goto out;
881  }
882  data_segments = alloc_zeroed_buffer(k * sizeof(char *));
883  parity_segments = alloc_zeroed_buffer(m * sizeof(char *));
884  get_data_ptr_array_from_fragments(data_segments, data, k);
885  get_data_ptr_array_from_fragments(parity_segments, parity, m);
886 
887 
888  /* call the backend reconstruct function passing it desc instance */
889  ret = instance->common.ops->reconstruct(instance->desc.backend_desc,
890  data_segments, parity_segments,
891  missing_idxs, destination_idx,
892  blocksize);
893  if (ret < 0) {
894  log_error("Could not reconstruct fragment!");
895  goto out;
896  }
897 
898  /*
899  * Update the header to reflect the newly constructed fragment
900  */
901  if (destination_idx < k) {
902  fragment_ptr = data[destination_idx];
903  } else {
904  fragment_ptr = parity[destination_idx - k];
905  }
906  init_fragment_header(fragment_ptr);
907  add_fragment_metadata(instance, fragment_ptr, destination_idx,
908  orig_data_size, blocksize, instance->args.uargs.ct,
909  set_chksum);
910 
911 destination_available:
912  /*
913  * Copy the reconstructed fragment to the output buffer
914  *
915  * Note: the address stored in fragment_ptr will be freed below
916  */
917  memcpy(out_fragment, fragment_ptr, fragment_len);
918 
919 out:
920  /* Free the buffers allocated in prepare_fragments_for_decode */
921  if (realloc_bm != 0) {
922  for (i = 0; i < k; i++) {
923  if (realloc_bm & (1 << i)) {
924  free(data[i]);
925  }
926  }
927 
928  for (i = 0; i < m; i++) {
929  if (realloc_bm & (1 << (i + k))) {
930  free(parity[i]);
931  }
932  }
933  }
934 
935  free(data);
936  free(parity);
937  free(missing_idxs);
938  free(data_segments);
939  free(parity_segments);
940 
941  return ret;
942 }
943 
960  int *fragments_to_reconstruct,
961  int *fragments_to_exclude,
962  int *fragments_needed)
963 {
964  int ret = 0;
965 
966  ec_backend_t instance = liberasurecode_backend_instance_get_by_desc(desc);
967  if (NULL == instance) {
968  ret = -EBACKENDNOTAVAIL;
969  goto out_error;
970  }
971  if (NULL == fragments_to_reconstruct) {
972  log_error("Unable to determine list of fragments needed, pointer to list of indexes to reconstruct is NULL.");
973  ret = -EINVALIDPARAMS;
974  goto out_error;
975  }
976 
977  if (NULL == fragments_to_exclude) {
978  log_error("Unable to determine list of fragments needed, pointer to list of fragments to exclude is NULL.");
979  ret = -EINVALIDPARAMS;
980  goto out_error;
981  }
982 
983  if (NULL == fragments_needed) {
984  log_error("Unable to determine list of fragments needed, pointer to list of fragments to reconstruct is NULL.");
985  ret = -EINVALIDPARAMS;
986  goto out_error;
987  }
988 
989  /* FIXME preprocessing */
990 
991  /* call the backend fragments_needed function passing it desc instance */
992  ret = instance->common.ops->fragments_needed(
993  instance->desc.backend_desc,
994  fragments_to_reconstruct, fragments_to_exclude, fragments_needed);
995 
996 out_error:
997  return ret;
998 }
999 
1000 /* =~=*=~==~=*=~==~=*=~==~=*=~===~=*=~==~=*=~===~=*=~==~=*=~===~=*=~==~=*=~= */
1001 
1013  fragment_metadata_t *fragment_metadata)
1014 {
1015  int ret = 0;
1016  fragment_header_t *fragment_hdr = NULL;
1017 
1018  if (NULL == fragment) {
1019  log_error("Need valid fragment object to get metadata for");
1020  ret = -EINVALIDPARAMS;
1021  goto out;
1022  }
1023 
1024  if (NULL == fragment_metadata) {
1025  log_error("Need valid fragment_metadata object for return value");
1026  ret = -EINVALIDPARAMS;
1027  goto out;
1028  }
1029 
1030  /* Verify metadata checksum */
1032  (fragment_header_t *) fragment)) {
1033  log_error("Invalid fragment header information!");
1034  ret = -EBADHEADER;
1035  goto out;
1036  }
1037 
1038  memcpy(fragment_metadata, fragment, sizeof(struct fragment_metadata));
1039  fragment_hdr = (fragment_header_t *) fragment;
1040  if (LIBERASURECODE_FRAG_HEADER_MAGIC != fragment_hdr->magic) {
1041  log_error("Invalid fragment, illegal magic value");
1042  ret = -EINVALIDPARAMS;
1043  goto out;
1044  }
1045 
1046  switch(fragment_hdr->meta.chksum_type) {
1047  case CHKSUM_CRC32: {
1048  uint32_t computed_chksum = 0;
1049  uint32_t stored_chksum = fragment_hdr->meta.chksum[0];
1050  char *fragment_data = get_data_ptr_from_fragment(fragment);
1051  uint64_t fragment_size = fragment_hdr->meta.size;
1052  computed_chksum = crc32(0, fragment_data, fragment_size);
1053  if (stored_chksum != computed_chksum) {
1054  fragment_metadata->chksum_mismatch = 1;
1055  } else {
1056  fragment_metadata->chksum_mismatch = 0;
1057  }
1058  break;
1059  }
1060  case CHKSUM_MD5:
1061  break;
1062  case CHKSUM_NONE:
1063  default:
1064  break;
1065  }
1066 
1067 out:
1068  return ret;
1069 }
1070 
1071 int is_invalid_fragment_header(fragment_header_t *header)
1072 {
1073  uint32_t *stored_csum = NULL, csum = 0;
1074  assert (NULL != header);
1075  if (header->libec_version < _VERSION(1,2,0))
1076  /* no metadata checksum support */
1077  return 0;
1078  stored_csum = get_metadata_chksum((char *) header);
1079  if (NULL == stored_csum)
1080  return 1; /* can't verify, possibly crc32 call error */
1081  csum = crc32(0, &header->meta, sizeof(fragment_metadata_t));
1082  return (*stored_csum != csum);
1083 }
1084 
1086  fragment_metadata_t *md)
1087 {
1088  int k = be->args.uargs.k;
1089  int m = be->args.uargs.m;
1090  if (md->idx < 0 || (md->idx > (k + m))) {
1091  return 1;
1092  }
1093  if (md->backend_id != be->common.id) {
1094  return 1;
1095  }
1096  if (!be->common.ops->is_compatible_with(md->backend_version)) {
1097  return 1;
1098  }
1099  return 0;
1100 }
1101 
1102 int is_invalid_fragment_metadata(int desc, fragment_metadata_t *fragment_metadata)
1103 {
1104  ec_backend_t be = liberasurecode_backend_instance_get_by_desc(desc);
1105  if (!be) {
1106  log_error("Unable to verify fragment metadata: invalid backend id %d.",
1107  desc);
1108  return -EINVALIDPARAMS;
1109  }
1111  fragment_metadata) != 0) {
1112  return -EBADHEADER;
1113  }
1114  if (!be->common.ops->is_compatible_with(fragment_metadata->backend_version)) {
1115  return -EBADHEADER;
1116  }
1117  if (fragment_metadata->chksum_mismatch == 1) {
1118  return -EBADCHKSUM;
1119  }
1120  return 0;
1121 }
1122 
1123 int is_invalid_fragment(int desc, char *fragment)
1124 {
1125  uint32_t ver = 0;
1126  fragment_metadata_t fragment_metadata;
1127  ec_backend_t be = liberasurecode_backend_instance_get_by_desc(desc);
1128  if (!be) {
1129  log_error("Unable to verify fragment metadata: invalid backend id %d.",
1130  desc);
1131  return 1;
1132  }
1133  if (!fragment) {
1134  log_error("Unable to verify fragment validity: fragments missing.");
1135  return 1;
1136  }
1137  if (get_libec_version(fragment, &ver) != 0 ||
1138  ver > LIBERASURECODE_VERSION) {
1139  return 1;
1140  }
1141  if (liberasurecode_get_fragment_metadata(fragment, &fragment_metadata) != 0) {
1142  return 1;
1143  }
1144  if (is_invalid_fragment_metadata(desc, &fragment_metadata) != 0) {
1145  return 1;
1146  }
1147  return 0;
1148 }
1149 
1151  char **fragments, int num_fragments)
1152 {
1153  int i = 0;
1154  if (!fragments) {
1155  log_error("Unable to verify stripe metadata: fragments missing.");
1156  return -EINVALIDPARAMS;
1157  }
1158  if (num_fragments <= 0) {
1159  log_error("Unable to verify stripe metadata: "
1160  "number of fragments must be greater than 0.");
1161  return -EINVALIDPARAMS;
1162  }
1163 
1164  for (i = 0; i < num_fragments; i++) {
1165  fragment_metadata_t *fragment_metadata = (fragment_metadata_t*)fragments[i];
1166  int ret = is_invalid_fragment_metadata(desc, fragment_metadata);
1167  if (ret < 0) {
1168  return ret;
1169  }
1170  }
1171 
1172  return 0;
1173 }
1174 
1175 /* =~=*=~==~=*=~==~=*=~==~=*=~===~=*=~==~=*=~===~=*=~==~=*=~===~=*=~==~=*=~= */
1176 
1184 int liberasurecode_get_aligned_data_size(int desc, uint64_t data_len)
1185 {
1186  int k;
1187  int ret = 0;
1188  int word_size;
1189  int alignment_multiple;
1190 
1191  ec_backend_t instance = liberasurecode_backend_instance_get_by_desc(desc);
1192  if (NULL == instance) {
1193  ret = -EBACKENDNOTAVAIL;
1194  goto out;
1195  }
1196 
1197  k = instance->args.uargs.k;
1198 
1199  word_size = instance->common.ops->element_size(
1200  instance->desc.backend_desc) / 8;
1201 
1202  alignment_multiple = k * word_size;
1203 
1204  ret = (int) ceill( (double)
1205  data_len / alignment_multiple) * alignment_multiple;
1206 
1207 out:
1208  return ret;
1209 }
1210 
1216 {
1217  return liberasurecode_get_aligned_data_size(desc, 1);
1218 }
1219 
1220 int liberasurecode_get_fragment_size(int desc, int data_len)
1221 {
1222  ec_backend_t instance = liberasurecode_backend_instance_get_by_desc(desc);
1223  // TODO: Create a common function to calculate fragment size also for preprocessing
1224  int aligned_data_len = get_aligned_data_size(instance, data_len);
1225  int size = (aligned_data_len / instance->args.uargs.k) + instance->common.backend_metadata_size;
1226 
1227  return size;
1228 }
1229 
1230 /* ==~=*=~==~=*=~==~=*=~==~=*=~==~=* misc *=~==~=*=~==~=*=~==~=*=~==~=*=~== */
1231 
1232 #if 0
1233 /* Validate backend before calling init */
1234 int liberasurecode_backend_validate(ec_backend_t backend)
1235 {
1236  /* Verify that the backend implements all required methods */
1237 }
1238 
1239 /* FIXME - do we need to use reference counts if we are creating
1240 * a new instance per user */
1241 
1242 /* Get a reference to an EC backend */
1243 ec_backend_t liberasurecode_backend_get(const char *name)
1244 {
1245  ec_backend_t b = liberasurecode_backend_lookup_by_name(name);
1246  if (NULL != b)
1247  ++b->users;
1248  return b;
1249 }
1250 
1251 /* Drop an EC backend reference held */
1252 void liberasurecode_backend_put(ec_backend_t backend)
1253 {
1254  if (backend->users > 0)
1255  --backend->users;
1256 }
1257 
1258 /* Query interface for active instances */
1259 ec_backend_t liberasurecode_backend_instance_active(ec_backend_t instance)
1260 {
1261  ec_backend_t b;
1262 
1263  SLIST_FOREACH(b, &active_instances, link) {
1264  if (strcmp(b->name, name) == 0)
1265  return b;
1266  }
1267 
1268  return NULL;
1269 }
1270 
1271 ec_backend_t liberasurecode_backend_lookup_by_soname(const char *soname)
1272 {
1273  ec_backend_t b;
1274 
1275  SLIST_FOREACH(b, &active_instances, link) {
1276  if (strcmp(b->soname, soname) == 0)
1277  return b;
1278  }
1279 
1280  return NULL;
1281 }
1282 #endif
1283 
1284 /* ==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~== */
void * alloc_and_set_buffer(int size, int value)
Allocate a buffer of a specific size and set its&#39; contents to the specified value.
int get_fragment_ptr_array_from_data(char **frag_array, char **data, int num_data)
SLIST_HEAD(backend_list, ec_backend)
Look up a backend instance by descriptor.
Definition: erasurecode.c:71
int get_fragment_partition(int k, int m, char **fragments, int num_fragments, char **data, char **parity, int *missing)
int get_data_ptr_array_from_fragments(char **data_array, char **fragments, int num_fragments)
int liberasurecode_reconstruct_fragment(int desc, char **available_fragments, int num_fragments, uint64_t fragment_len, int destination_idx, char *out_fragment)
Reconstruct a missing fragment from a subset of available fragments.
Definition: erasurecode.c:756
struct ec_backend_common backend_jerasure_rs_vand
int liberasurecode_get_minimum_encode_size(int desc)
This will return the minumum encode size, which is the minimum buffer size that can be encoded...
Definition: erasurecode.c:1215
void * liberasurecode_backend_open(ec_backend_t instance)
Definition: erasurecode.c:172
uint32_t * get_metadata_chksum(char *buf)
int liberasurecode_verify_fragment_metadata(ec_backend_t be, fragment_metadata_t *md)
Definition: erasurecode.c:1085
int liberasurecode_get_aligned_data_size(int desc, uint64_t data_len)
This computes the aligned size of a buffer passed into the encode function.
Definition: erasurecode.c:1184
int get_aligned_data_size(ec_backend_t instance, int data_len)
Compute a size aligned to the number of data and the underlying wordsize of the EC algorithm...
int liberasurecode_backend_close(ec_backend_t instance)
Definition: erasurecode.c:180
int liberasurecode_backend_available(const ec_backend_id_t backend_id)
Checks if a given backend is available.
Definition: erasurecode.c:227
char * get_data_ptr_from_fragment(char *buf)
int liberasurecode_verify_stripe_metadata(int desc, char **fragments, int num_fragments)
Definition: erasurecode.c:1150
void * alloc_zeroed_buffer(int size)
Allocate a zero-ed buffer of a specific size.
int prepare_fragments_for_decode(int k, int m, char **data, char **parity, int *missing_idxs, int *orig_size, int *fragment_payload_size, int fragment_size, uint64_t *realloc_bm)
struct ec_backend_common backend_isa_l_rs_vand
Definition: isa_l_rs_vand.c:52
int liberasurecode_instance_destroy(int desc)
Close a liberasurecode instance.
Definition: erasurecode.c:319
struct ec_backend_common backend_null
Definition: null.c:229
int liberasurecode_get_fragment_metadata(char *fragment, fragment_metadata_t *fragment_metadata)
Get opaque metadata for a fragment.
Definition: erasurecode.c:1012
int is_invalid_fragment(int desc, char *fragment)
Definition: erasurecode.c:1123
int finalize_fragments_after_encode(ec_backend_t instance, int k, int m, int blocksize, uint64_t orig_data_size, char **encoded_data, char **encoded_parity)
void add_fragment_metadata(ec_backend_t be, char *fragment, int idx, uint64_t orig_data_size, int blocksize, ec_checksum_type_t ct, int add_chksum)
int liberasurecode_encode_cleanup(int desc, char **encoded_data, char **encoded_parity)
Cleanup structures allocated by librasurecode_encode.
Definition: erasurecode.c:358
struct ec_backend_common backend_jerasure_rs_cauchy
int is_invalid_fragment_header(fragment_header_t *header)
Definition: erasurecode.c:1071
int liberasurecode_backend_alloc_desc(void)
Allocated backend instance descriptor.
Definition: erasurecode.c:100
struct ec_backend_common backend_flat_xor_hd
Definition: flat_xor_hd.c:51
uint64_t get_fragment_size(char *buf)
Return total fragment length (on-disk, on-wire)
struct ec_backend_common backend_liberasurecode_rs_vand
char * ec_backends_supported_str[EC_BACKENDS_MAX]
Definition: erasurecode.c:66
struct ec_backend_common backend_shss
Definition: shss.c:43
int crc32(int crc, const void *buf, size_t size)
Definition: crc32.c:119
int liberasurecode_backend_instance_unregister(ec_backend_t instance)
Unregister a backend instance.
Definition: erasurecode.c:144
int liberasurecode_fragments_needed(int desc, int *fragments_to_reconstruct, int *fragments_to_exclude, int *fragments_needed)
Return a list of lists with valid rebuild indexes given a list of missing indexes.
Definition: erasurecode.c:959
int liberasurecode_get_fragment_size(int desc, int data_len)
Definition: erasurecode.c:1220
ec_backend_t ec_backends_supported[]
Definition: erasurecode.c:53
static void print_dlerror(const char *caller)
Definition: erasurecode.c:162
int num_supported_backends
Definition: erasurecode.c:65
int liberasurecode_instance_create(const ec_backend_id_t id, struct ec_args *args)
Create a liberasurecode instance and return a descriptor for use with EC operations (encode...
Definition: erasurecode.c:261
int fragments_to_string(int k, int m, char **fragments, int num_fragments, char **orig_payload, uint64_t *payload_len)
void __attribute__((constructor))
Definition: erasurecode.c:194
int get_libec_version(char *buf, uint32_t *ver)
int is_invalid_fragment_metadata(int desc, fragment_metadata_t *fragment_metadata)
Definition: erasurecode.c:1102
int prepare_fragments_for_encode(ec_backend_t instance, int k, int m, const char *orig_data, uint64_t orig_data_size, char **encoded_data, char **encoded_parity, int *blocksize)
int liberasurecode_decode(int desc, char **available_fragments, int num_fragments, uint64_t fragment_len, int force_metadata_checks, char **out_data, uint64_t *out_data_len)
Reconstruct original data from a set of k encoded fragments.
Definition: erasurecode.c:533
int liberasurecode_backend_instance_register(ec_backend_t instance)
Register a backend instance with liberasurecode.
Definition: erasurecode.c:117
int liberasurecode_encode(int desc, const char *orig_data, uint64_t orig_data_size, char ***encoded_data, char ***encoded_parity, uint64_t *fragment_len)
Erasure encode a data buffer.
Definition: erasurecode.c:406
int liberasurecode_decode_cleanup(int desc, char *data)
Cleanup structures allocated by librasurecode_decode.
Definition: erasurecode.c:508