Commit d8c84dcbb9d36e787ecd88d43f76558575ca4e88

Authored by KMG
1 parent e5170f70
Exists in master and in 1 other branch v2

Added tests for the GF init helper functions in galois.c

Updated the README to explain the GF change procedure.
Examples/makefile
... ... @@ -63,6 +63,7 @@ ALL = jerasure_01 \
63 63 reed_sol_04 \
64 64 reed_sol_test_gf \
65 65 reed_sol_time_gf \
  66 + reed_sol_hard_time_gf \
66 67 cauchy_01 \
67 68 cauchy_02 \
68 69 cauchy_03 \
... ... @@ -190,9 +191,13 @@ decoder: decoder.o galois.o jerasure.o liberation.o reed_sol.o cauchy.o
190 191 $(CC) $(CFLAGS) -o decoder decoder.o liberation.o jerasure.o galois.o reed_sol.o cauchy.o -lgf_complete
191 192  
192 193 reed_sol_test_gf.o: galois.h reed_sol.h jerasure.h
193   -reed_sol_test_gf: reed_sol_test_gf.o galois.o ${LIBDIR}/gf_complete.a jerasure.o reed_sol.o
194   - $(CC) $(CFLAGS) -o reed_sol_test_gf reed_sol_test_gf.o reed_sol.o jerasure.o galois.o ${LIBDIR}/gf_complete.a
  194 +reed_sol_test_gf: reed_sol_test_gf.o galois.o jerasure.o reed_sol.o
  195 + $(CC) $(CFLAGS) -o reed_sol_test_gf reed_sol_test_gf.o reed_sol.o jerasure.o galois.o -lgf_complete
195 196  
196 197 reed_sol_time_gf.o: galois.h reed_sol.h jerasure.h
197   -reed_sol_time_gf: reed_sol_time_gf.o galois.o ${LIBDIR}/gf_complete.a jerasure.o reed_sol.o
198   - $(CC) $(CFLAGS) -o reed_sol_time_gf reed_sol_time_gf.o reed_sol.o jerasure.o galois.o ${LIBDIR}/gf_complete.a
  198 +reed_sol_time_gf: reed_sol_time_gf.o galois.o jerasure.o reed_sol.o
  199 + $(CC) $(CFLAGS) -o reed_sol_time_gf reed_sol_time_gf.o reed_sol.o jerasure.o galois.o -lgf_complete
  200 +
  201 +reed_sol_hard_time_gf.o: galois.h reed_sol.h jerasure.h
  202 +reed_sol_hard_time_gf: reed_sol_hard_time_gf.o galois.o jerasure.o reed_sol.o
  203 + $(CC) $(CFLAGS) -o reed_sol_hard_time_gf reed_sol_hard_time_gf.o reed_sol.o jerasure.o galois.o -lgf_complete
... ...
Examples/reed_sol_hard_time_gf.c 0 โ†’ 100644
... ... @@ -0,0 +1,361 @@
  1 +/* *
  2 + * Copyright (c) 2013, James S. Plank and Kevin Greenan
  3 + * All rights reserved.
  4 + *
  5 + * Jerasure - A C/C++ Library for a Variety of Reed-Solomon and RAID-6 Erasure
  6 + * Coding Techniques
  7 + *
  8 + * Revision 2.0: Galois Field backend now links to GF-Complete
  9 + *
  10 + * Redistribution and use in source and binary forms, with or without
  11 + * modification, are permitted provided that the following conditions
  12 + * are met:
  13 + *
  14 + * - Redistributions of source code must retain the above copyright
  15 + * notice, this list of conditions and the following disclaimer.
  16 + *
  17 + * - Redistributions in binary form must reproduce the above copyright
  18 + * notice, this list of conditions and the following disclaimer in
  19 + * the documentation and/or other materials provided with the
  20 + * distribution.
  21 + *
  22 + * - Neither the name of the University of Tennessee nor the names of its
  23 + * contributors may be used to endorse or promote products derived
  24 + * from this software without specific prior written permission.
  25 + *
  26 + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  27 + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  28 + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  29 + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  30 + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  31 + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
  32 + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
  33 + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  34 + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  35 + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY
  36 + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  37 + * POSSIBILITY OF SUCH DAMAGE.
  38 + */
  39 +
  40 +
  41 +
  42 +#include <stdio.h>
  43 +#include <stdlib.h>
  44 +#include <string.h>
  45 +#include <gf_complete.h>
  46 +#include "jerasure.h"
  47 +#include "reed_sol.h"
  48 +
  49 +#define BUFSIZE 4096
  50 +
  51 +int get_gfp_from_argv(gf_t **gfp, int w, int argc, char **argv, int starting)
  52 +{
  53 + int mult_type, divide_type, region_type;
  54 + int arg1, arg2, subrg_size, degree;
  55 + uint64_t prim_poly;
  56 + gf_t *base;
  57 + char *crt, *x, *y;
  58 +
  59 + mult_type = GF_MULT_DEFAULT;
  60 + region_type = GF_REGION_DEFAULT;
  61 + divide_type = GF_DIVIDE_DEFAULT;
  62 + prim_poly = 0;
  63 + base = NULL;
  64 + arg1 = 0;
  65 + arg2 = 0;
  66 + degree = 0;
  67 + while (1) {
  68 + if (argc > starting) {
  69 + if (strcmp(argv[starting], "-m") == 0) {
  70 + starting++;
  71 + if (mult_type != GF_MULT_DEFAULT) {
  72 + if (base != NULL) gf_free(base, 1);
  73 + return 0;
  74 + }
  75 + if (strcmp(argv[starting], "SHIFT") == 0) {
  76 + mult_type = GF_MULT_SHIFT;
  77 + starting++;
  78 + } else if (strcmp(argv[starting], "CARRY_FREE") == 0) {
  79 + mult_type = GF_MULT_CARRY_FREE;
  80 + starting++;
  81 + } else if (strcmp(argv[starting], "GROUP") == 0) {
  82 + mult_type = GF_MULT_GROUP;
  83 + if (argc < starting + 3) {
  84 + return 0;
  85 + }
  86 + if (sscanf(argv[starting+1], "%d", &arg1) == 0 ||
  87 + sscanf(argv[starting+2], "%d", &arg2) == 0) {
  88 + return 0;
  89 + }
  90 + starting += 3;
  91 + } else if (strcmp(argv[starting], "BYTWO_p") == 0) {
  92 + mult_type = GF_MULT_BYTWO_p;
  93 + starting++;
  94 + } else if (strcmp(argv[starting], "BYTWO_b") == 0) {
  95 + mult_type = GF_MULT_BYTWO_b;
  96 + starting++;
  97 + } else if (strcmp(argv[starting], "TABLE") == 0) {
  98 + mult_type = GF_MULT_TABLE;
  99 + starting++;
  100 + } else if (strcmp(argv[starting], "LOG") == 0) {
  101 + mult_type = GF_MULT_LOG_TABLE;
  102 + starting++;
  103 + } else if (strcmp(argv[starting], "LOG_ZERO") == 0) {
  104 + mult_type = GF_MULT_LOG_ZERO;
  105 + starting++;
  106 + } else if (strcmp(argv[starting], "LOG_ZERO_EXT") == 0) {
  107 + mult_type = GF_MULT_LOG_ZERO_EXT;
  108 + starting++;
  109 + } else if (strcmp(argv[starting], "SPLIT") == 0) {
  110 + mult_type = GF_MULT_SPLIT_TABLE;
  111 + if (argc < starting + 3) {
  112 + return 0;
  113 + }
  114 + if (sscanf(argv[starting+1], "%d", &arg1) == 0 ||
  115 + sscanf(argv[starting+2], "%d", &arg2) == 0) {
  116 + return 0;
  117 + }
  118 + starting += 3;
  119 + } else if (strcmp(argv[starting], "COMPOSITE") == 0) {
  120 + mult_type = GF_MULT_COMPOSITE;
  121 + if (argc < starting + 2) { return 0; }
  122 + if (sscanf(argv[starting+1], "%d", &arg1) == 0) {
  123 + return 0;
  124 + }
  125 + starting += 2;
  126 + degree = arg1;
  127 + starting = get_gfp_from_argv(&base, w/degree, argc, argv, starting);
  128 + if (starting == 0) {
  129 + free(base);
  130 + return 0;
  131 + }
  132 + } else {
  133 + if (base != NULL) gf_free(base, 1);
  134 + return 0;
  135 + }
  136 + } else if (strcmp(argv[starting], "-r") == 0) {
  137 + starting++;
  138 + if (strcmp(argv[starting], "DOUBLE") == 0) {
  139 + region_type |= GF_REGION_DOUBLE_TABLE;
  140 + starting++;
  141 + } else if (strcmp(argv[starting], "QUAD") == 0) {
  142 + region_type |= GF_REGION_QUAD_TABLE;
  143 + starting++;
  144 + } else if (strcmp(argv[starting], "LAZY") == 0) {
  145 + region_type |= GF_REGION_LAZY;
  146 + starting++;
  147 + } else if (strcmp(argv[starting], "SSE") == 0) {
  148 + region_type |= GF_REGION_SSE;
  149 + starting++;
  150 + } else if (strcmp(argv[starting], "NOSSE") == 0) {
  151 + region_type |= GF_REGION_NOSSE;
  152 + starting++;
  153 + } else if (strcmp(argv[starting], "CAUCHY") == 0) {
  154 + region_type |= GF_REGION_CAUCHY;
  155 + starting++;
  156 + } else if (strcmp(argv[starting], "ALTMAP") == 0) {
  157 + region_type |= GF_REGION_ALTMAP;
  158 + starting++;
  159 + } else {
  160 + if (base != NULL) gf_free(base, 1);
  161 + return 0;
  162 + }
  163 + } else if (strcmp(argv[starting], "-p") == 0) {
  164 + starting++;
  165 + if (sscanf(argv[starting], "%llx", (long long unsigned int *)(&prim_poly)) == 0) {
  166 + if (base != NULL) gf_free(base, 1);
  167 + return 0;
  168 + }
  169 + starting++;
  170 + } else if (strcmp(argv[starting], "-d") == 0) {
  171 + starting++;
  172 + if (divide_type != GF_DIVIDE_DEFAULT) {
  173 + if (base != NULL) gf_free(base, 1);
  174 + return 0;
  175 + } else if (strcmp(argv[starting], "EUCLID") == 0) {
  176 + divide_type = GF_DIVIDE_EUCLID;
  177 + starting++;
  178 + } else if (strcmp(argv[starting], "MATRIX") == 0) {
  179 + divide_type = GF_DIVIDE_MATRIX;
  180 + starting++;
  181 + } else {
  182 + return 0;
  183 + }
  184 + } else if (strcmp(argv[starting], "-") == 0) {
  185 + if (mult_type == GF_MULT_COMPOSITE) {
  186 + *gfp = galois_init_composite_field(w, region_type, divide_type, degree, base);
  187 + } else {
  188 + *gfp = galois_init_field(w, mult_type, region_type, divide_type, prim_poly, arg1, arg2);
  189 + }
  190 + starting++;
  191 + break;
  192 + } else {
  193 + if (base != NULL) gf_free(base, 1);
  194 + return 0;
  195 + }
  196 + }
  197 + }
  198 +
  199 + return starting;
  200 +}
  201 +
  202 +static void *malloc16(int size) {
  203 + void *mem = malloc(size+16+sizeof(void*));
  204 + void **ptr = (void**)((long)(mem+16+sizeof(void*)) & ~(15));
  205 + ptr[-1] = mem;
  206 + return ptr;
  207 +}
  208 +
  209 +static void free16(void *ptr) {
  210 + free(((void**)ptr)[-1]);
  211 +}
  212 +
  213 +#define talloc(type, num) (type *) malloc16(sizeof(type)*(num))
  214 +
  215 +void
  216 +timer_start (double *t)
  217 +{
  218 + struct timeval tv;
  219 +
  220 + gettimeofday (&tv, NULL);
  221 + *t = (double)tv.tv_sec + (double)tv.tv_usec * 1e-6;
  222 +}
  223 +
  224 +double
  225 +timer_split (const double *t)
  226 +{
  227 + struct timeval tv;
  228 + double cur_t;
  229 +
  230 + gettimeofday (&tv, NULL);
  231 + cur_t = (double)tv.tv_sec + (double)tv.tv_usec * 1e-6;
  232 + return (cur_t - *t);
  233 +}
  234 +
  235 +usage(char *s)
  236 +{
  237 + fprintf(stderr, "usage: reed_sol_hard_test_gf k m w [additional GF args]- Test and time Reed-Solomon in a particular GF(2^w).\n");
  238 + fprintf(stderr, " \n");
  239 + fprintf(stderr, " w must be 8, 16 or 32. k+m must be <= 2^w.\n");
  240 + fprintf(stderr, " See the README for information on the additional GF args.\n");
  241 + fprintf(stderr, " Set up a Vandermonde-based distribution matrix and encodes k devices of\n");
  242 + fprintf(stderr, " %d bytes each with it. Then it decodes.\n", BUFSIZE);
  243 + fprintf(stderr, " \n");
  244 + fprintf(stderr, "This tests: jerasure_matrix_encode()\n");
  245 + fprintf(stderr, " jerasure_matrix_decode()\n");
  246 + fprintf(stderr, " jerasure_print_matrix()\n");
  247 + fprintf(stderr, " galois_init_composite_field()\n");
  248 + fprintf(stderr, " galois_init_field()\n");
  249 + fprintf(stderr, " galois_change_technique()\n");
  250 + fprintf(stderr, " reed_sol_vandermonde_coding_matrix()\n");
  251 + if (s != NULL) fprintf(stderr, "%s\n", s);
  252 + exit(1);
  253 +}
  254 +
  255 +static void fill_buffer(unsigned char *buf, int size)
  256 +{
  257 + int i;
  258 +
  259 + buf[0] = (char)(lrand48() % 256);
  260 +
  261 + for (i=1; i < size; i++) {
  262 + buf[i] = ((buf[i-1] + i) % 256);
  263 + }
  264 +}
  265 +
  266 +int main(int argc, char **argv)
  267 +{
  268 + long l;
  269 + int k, w, i, j, m, iterations, bufsize;
  270 + int *matrix;
  271 + char **data, **coding, **old_values;
  272 + int *erasures, *erased;
  273 + int *decoding_matrix, *dm_ids;
  274 + double t = 0, total_time = 0;
  275 + gf_t *gf = NULL;
  276 + int ret = 0;
  277 +
  278 + if (argc < 6) usage(NULL);
  279 + if (sscanf(argv[1], "%d", &k) == 0 || k <= 0) usage("Bad k");
  280 + if (sscanf(argv[2], "%d", &m) == 0 || m <= 0) usage("Bad m");
  281 + if (sscanf(argv[3], "%d", &w) == 0 || (w != 8 && w != 16 && w != 32)) usage("Bad w");
  282 + if (sscanf(argv[4], "%d", &iterations) == 0) usage("Bad iterations");
  283 + if (sscanf(argv[5], "%d", &bufsize) == 0) usage("Bad bufsize");
  284 + if (w <= 16 && k + m > (1 << w)) usage("k + m is too big");
  285 +
  286 + srand48(time(0));
  287 +
  288 + ret = get_gfp_from_argv(&gf, w, argc, argv, 6);
  289 +
  290 + if (ret == 0 || gf == NULL) {
  291 + usage("Invalid arguments given for GF!\n");
  292 + }
  293 +
  294 + galois_change_technique(gf, w);
  295 +
  296 + matrix = reed_sol_vandermonde_coding_matrix(k, m, w);
  297 +
  298 + printf("Last m rows of the Distribution Matrix:\n\n");
  299 + jerasure_print_matrix(matrix, m, k, w);
  300 + printf("\n");
  301 +
  302 + data = talloc(char *, k);
  303 + for (i = 0; i < k; i++) {
  304 + data[i] = talloc(char, bufsize);
  305 + fill_buffer(data[i], bufsize);
  306 + }
  307 +
  308 + coding = talloc(char *, m);
  309 + old_values = talloc(char *, m);
  310 + for (i = 0; i < m; i++) {
  311 + coding[i] = talloc(char, bufsize);
  312 + old_values[i] = talloc(char, bufsize);
  313 + }
  314 +
  315 + for (i = 0; i < iterations; i++) {
  316 + timer_start(&t);
  317 + jerasure_matrix_encode(k, m, w, matrix, data, coding, bufsize);
  318 + total_time += timer_split(&t);
  319 + }
  320 +
  321 + fprintf(stderr, "Encode thput for %d iterations: %.2f MB/s (%.2f sec)\n", iterations, (double)(k*iterations*bufsize/1024/1024) / total_time, total_time);
  322 +
  323 + erasures = talloc(int, (m+1));
  324 + erased = talloc(int, (k+m));
  325 + for (i = 0; i < m+k; i++) erased[i] = 0;
  326 + l = 0;
  327 + for (i = 0; i < m; ) {
  328 + erasures[i] = ((unsigned int)lrand48())%(k+m);
  329 + if (erased[erasures[i]] == 0) {
  330 + erased[erasures[i]] = 1;
  331 + memcpy(old_values[i], (erasures[i] < k) ? data[erasures[i]] : coding[erasures[i]-k], bufsize);
  332 + bzero((erasures[i] < k) ? data[erasures[i]] : coding[erasures[i]-k], bufsize);
  333 + i++;
  334 + }
  335 + }
  336 + erasures[i] = -1;
  337 +
  338 + for (i = 0; i < iterations; i++) {
  339 + timer_start(&t);
  340 + jerasure_matrix_decode(k, m, w, matrix, 1, erasures, data, coding, bufsize);
  341 + total_time += timer_split(&t);
  342 + }
  343 +
  344 + fprintf(stderr, "Decode thput for %d iterations: %.2f MB/s (%.2f sec)\n", iterations, (double)(m*iterations*bufsize/1024/1024) / total_time, total_time);
  345 +
  346 + for (i = 0; i < m; i++) {
  347 + if (erasures[i] < k) {
  348 + if (memcmp(data[erasures[i]], old_values[i], bufsize)) {
  349 + fprintf(stderr, "Decoding failed for %d!\n", erasures[i]);
  350 + exit(1);
  351 + }
  352 + } else {
  353 + if (memcmp(coding[erasures[i]-k], old_values[i], bufsize)) {
  354 + fprintf(stderr, "Decoding failed for %d!\n", erasures[i]);
  355 + exit(1);
  356 + }
  357 + }
  358 + }
  359 +
  360 + return 0;
  361 +}
... ...
Examples/time_all_gfs.sh
... ... @@ -1,92 +0,0 @@
1   -#
2   -#
3   -# Copyright (c) 2013, James S. Plank and Kevin Greenan
4   -# All rights reserved.
5   -#
6   -# Jerasure - A C/C++ Library for a Variety of Reed-Solomon and RAID-6 Erasure
7   -# Coding Techniques
8   -#
9   -# Revision 2.0: Galois Field backend now links to GF-Complete
10   -#
11   -# Redistribution and use in source and binary forms, with or without
12   -# modification, are permitted provided that the following conditions
13   -# are met:
14   -#
15   -# - Redistributions of source code must retain the above copyright
16   -# notice, this list of conditions and the following disclaimer.
17   -#
18   -# - Redistributions in binary form must reproduce the above copyright
19   -# notice, this list of conditions and the following disclaimer in
20   -# the documentation and/or other materials provided with the
21   -# distribution.
22   -#
23   -# - Neither the name of the University of Tennessee nor the names of its
24   -# contributors may be used to endorse or promote products derived
25   -# from this software without specific prior written permission.
26   -#
27   -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
28   -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
29   -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
30   -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
31   -# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
32   -# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
33   -# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
34   -# OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
35   -# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
36   -# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY
37   -# WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
38   -# POSSIBILITY OF SUCH DAMAGE.
39   -#
40   -GF_COMPLETE_DIR=/usr/local/bin
41   -GF_METHODS=${GF_COMPLETE_DIR}/gf_methods
42   -ITERATIONS=128
43   -BUFSIZE=65536
44   -k=12
45   -m=3
46   -
47   -# Test all w=8
48   -${GF_METHODS} | awk -F: '{ if ($1 == "w=8") print $2; }' |
49   -while read method; do
50   - echo "Testing ${k} ${m} 8 ${ITERATIONS} ${BUFSIZE} ${method}"
51   - ./reed_sol_time_gf ${k} ${m} 8 ${ITERATIONS} ${BUFSIZE} ${method}
52   - if [[ $? != "0" ]]; then
53   - echo "Failed test for ${k} ${m} 8 ${ITERATIONS} ${BUFSIZE} ${method}"
54   - exit 1
55   - fi
56   -done
57   -
58   -if [[ $? == "1" ]]; then
59   - exit 1
60   -fi
61   -
62   -# Test all w=16
63   -${GF_METHODS} | grep -v 'TABLE' | awk -F: '{ if ($1 == "w=16") print $2; }' |
64   -while read method; do
65   - echo "Testing ${k} ${m} 16 ${ITERATIONS} ${BUFSIZE} ${method}"
66   - ./reed_sol_time_gf ${k} ${m} 16 ${ITERATIONS} ${BUFSIZE} ${method}
67   - if [[ $? != "0" ]]; then
68   - echo "Failed test for ${k} ${m} 16 ${ITERATIONS} ${BUFSIZE} ${method}"
69   - exit 1
70   - fi
71   -done
72   -
73   -if [[ $? == "1" ]]; then
74   - exit 1
75   -fi
76   -
77   -# Test all w=32
78   -${GF_METHODS} | awk -F: '{ if ($1 == "w=32") print $2; }' |
79   -while read method; do
80   - echo "Testing ${k} ${m} 32 ${ITERATIONS} ${BUFSIZE} ${method}"
81   - ./reed_sol_time_gf ${k} ${m} 32 ${ITERATIONS} ${BUFSIZE} ${method}
82   - if [[ $? != "0" ]]; then
83   - echo "Failed test for ${k} ${m} 32 ${ITERATIONS} ${BUFSIZE} ${method}"
84   - exit 1
85   - fi
86   -done
87   -
88   -if [[ $? == "1" ]]; then
89   - exit 1
90   -fi
91   -
92   -echo "Passed all tests!"
Examples/time_all_gfs_argv_init.sh 0 โ†’ 100755
... ... @@ -0,0 +1,92 @@
  1 +#
  2 +#
  3 +# Copyright (c) 2013, James S. Plank and Kevin Greenan
  4 +# All rights reserved.
  5 +#
  6 +# Jerasure - A C/C++ Library for a Variety of Reed-Solomon and RAID-6 Erasure
  7 +# Coding Techniques
  8 +#
  9 +# Revision 2.0: Galois Field backend now links to GF-Complete
  10 +#
  11 +# Redistribution and use in source and binary forms, with or without
  12 +# modification, are permitted provided that the following conditions
  13 +# are met:
  14 +#
  15 +# - Redistributions of source code must retain the above copyright
  16 +# notice, this list of conditions and the following disclaimer.
  17 +#
  18 +# - Redistributions in binary form must reproduce the above copyright
  19 +# notice, this list of conditions and the following disclaimer in
  20 +# the documentation and/or other materials provided with the
  21 +# distribution.
  22 +#
  23 +# - Neither the name of the University of Tennessee nor the names of its
  24 +# contributors may be used to endorse or promote products derived
  25 +# from this software without specific prior written permission.
  26 +#
  27 +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  28 +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  29 +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  30 +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  31 +# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  32 +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
  33 +# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
  34 +# OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  35 +# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  36 +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY
  37 +# WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  38 +# POSSIBILITY OF SUCH DAMAGE.
  39 +#
  40 +GF_COMPLETE_DIR=/usr/local/bin
  41 +GF_METHODS=${GF_COMPLETE_DIR}/gf_methods
  42 +ITERATIONS=128
  43 +BUFSIZE=65536
  44 +k=12
  45 +m=3
  46 +
  47 +# Test all w=8
  48 +${GF_METHODS} | awk -F: '{ if ($1 == "w=8") print $2; }' |
  49 +while read method; do
  50 + echo "Testing ${k} ${m} 8 ${ITERATIONS} ${BUFSIZE} ${method}"
  51 + ./reed_sol_time_gf ${k} ${m} 8 ${ITERATIONS} ${BUFSIZE} ${method}
  52 + if [[ $? != "0" ]]; then
  53 + echo "Failed test for ${k} ${m} 8 ${ITERATIONS} ${BUFSIZE} ${method}"
  54 + exit 1
  55 + fi
  56 +done
  57 +
  58 +if [[ $? == "1" ]]; then
  59 + exit 1
  60 +fi
  61 +
  62 +# Test all w=16
  63 +${GF_METHODS} | grep -v 'TABLE' | awk -F: '{ if ($1 == "w=16") print $2; }' |
  64 +while read method; do
  65 + echo "Testing ${k} ${m} 16 ${ITERATIONS} ${BUFSIZE} ${method}"
  66 + ./reed_sol_time_gf ${k} ${m} 16 ${ITERATIONS} ${BUFSIZE} ${method}
  67 + if [[ $? != "0" ]]; then
  68 + echo "Failed test for ${k} ${m} 16 ${ITERATIONS} ${BUFSIZE} ${method}"
  69 + exit 1
  70 + fi
  71 +done
  72 +
  73 +if [[ $? == "1" ]]; then
  74 + exit 1
  75 +fi
  76 +
  77 +# Test all w=32
  78 +${GF_METHODS} | awk -F: '{ if ($1 == "w=32") print $2; }' |
  79 +while read method; do
  80 + echo "Testing ${k} ${m} 32 ${ITERATIONS} ${BUFSIZE} ${method}"
  81 + ./reed_sol_time_gf ${k} ${m} 32 ${ITERATIONS} ${BUFSIZE} ${method}
  82 + if [[ $? != "0" ]]; then
  83 + echo "Failed test for ${k} ${m} 32 ${ITERATIONS} ${BUFSIZE} ${method}"
  84 + exit 1
  85 + fi
  86 +done
  87 +
  88 +if [[ $? == "1" ]]; then
  89 + exit 1
  90 +fi
  91 +
  92 +echo "Passed all tests!"
... ...
Examples/time_all_gfs_hard_init.sh 0 โ†’ 100755
... ... @@ -0,0 +1,92 @@
  1 +#
  2 +#
  3 +# Copyright (c) 2013, James S. Plank and Kevin Greenan
  4 +# All rights reserved.
  5 +#
  6 +# Jerasure - A C/C++ Library for a Variety of Reed-Solomon and RAID-6 Erasure
  7 +# Coding Techniques
  8 +#
  9 +# Revision 2.0: Galois Field backend now links to GF-Complete
  10 +#
  11 +# Redistribution and use in source and binary forms, with or without
  12 +# modification, are permitted provided that the following conditions
  13 +# are met:
  14 +#
  15 +# - Redistributions of source code must retain the above copyright
  16 +# notice, this list of conditions and the following disclaimer.
  17 +#
  18 +# - Redistributions in binary form must reproduce the above copyright
  19 +# notice, this list of conditions and the following disclaimer in
  20 +# the documentation and/or other materials provided with the
  21 +# distribution.
  22 +#
  23 +# - Neither the name of the University of Tennessee nor the names of its
  24 +# contributors may be used to endorse or promote products derived
  25 +# from this software without specific prior written permission.
  26 +#
  27 +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  28 +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  29 +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  30 +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  31 +# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  32 +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
  33 +# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
  34 +# OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  35 +# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  36 +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY
  37 +# WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  38 +# POSSIBILITY OF SUCH DAMAGE.
  39 +#
  40 +GF_COMPLETE_DIR=/usr/local/bin
  41 +GF_METHODS=${GF_COMPLETE_DIR}/gf_methods
  42 +ITERATIONS=128
  43 +BUFSIZE=65536
  44 +k=12
  45 +m=3
  46 +
  47 +# Test all w=8
  48 +${GF_METHODS} | awk -F: '{ if ($1 == "w=8") print $2; }' |
  49 +while read method; do
  50 + echo "Testing ${k} ${m} 8 ${ITERATIONS} ${BUFSIZE} ${method}"
  51 + ./reed_sol_hard_time_gf ${k} ${m} 8 ${ITERATIONS} ${BUFSIZE} ${method}
  52 + if [[ $? != "0" ]]; then
  53 + echo "Failed test for ${k} ${m} 8 ${ITERATIONS} ${BUFSIZE} ${method}"
  54 + exit 1
  55 + fi
  56 +done
  57 +
  58 +if [[ $? == "1" ]]; then
  59 + exit 1
  60 +fi
  61 +
  62 +# Test all w=16
  63 +${GF_METHODS} | grep -v 'TABLE' | awk -F: '{ if ($1 == "w=16") print $2; }' |
  64 +while read method; do
  65 + echo "Testing ${k} ${m} 16 ${ITERATIONS} ${BUFSIZE} ${method}"
  66 + ./reed_sol_hard_time_gf ${k} ${m} 16 ${ITERATIONS} ${BUFSIZE} ${method}
  67 + if [[ $? != "0" ]]; then
  68 + echo "Failed test for ${k} ${m} 16 ${ITERATIONS} ${BUFSIZE} ${method}"
  69 + exit 1
  70 + fi
  71 +done
  72 +
  73 +if [[ $? == "1" ]]; then
  74 + exit 1
  75 +fi
  76 +
  77 +# Test all w=32
  78 +${GF_METHODS} | awk -F: '{ if ($1 == "w=32") print $2; }' |
  79 +while read method; do
  80 + echo "Testing ${k} ${m} 32 ${ITERATIONS} ${BUFSIZE} ${method}"
  81 + ./reed_sol_hard_time_gf ${k} ${m} 32 ${ITERATIONS} ${BUFSIZE} ${method}
  82 + if [[ $? != "0" ]]; then
  83 + echo "Failed test for ${k} ${m} 32 ${ITERATIONS} ${BUFSIZE} ${method}"
  84 + exit 1
  85 + fi
  86 +done
  87 +
  88 +if [[ $? == "1" ]]; then
  89 + exit 1
  90 +fi
  91 +
  92 +echo "Passed all tests!"
... ...
README.nd
... ... @@ -29,4 +29,112 @@ determine the default field to use, if one is not specified.
29 29 If you would like to explore a using a different Galois Field implementation,
30 30 you can dynamically set the backend GF for a given word-size (w).
31 31  
32   -<TBD - Add instructions>
  32 +The new galois.[ch] exports the following functions to be used by applications
  33 +for dynamically setting the backend GF:
  34 +
  35 +1.) galois_change_technique
  36 +
  37 + Function signature:
  38 +
  39 + void galois_change_technique(gf_t *gf, int w);
  40 +
  41 + This function will take a pointer to a Galois field structure and set it as the
  42 + current backend for all operations in GF(2^w). Note that you must specify 'w'
  43 + here, since the internal GF structure is mostly opaque to Jerasure. Be sure to
  44 + change the technique with the correct structure and word-size.
  45 +
  46 + There are a few ways to get a pointer to a gf_t structure: via GF-Complete or
  47 + using the helper functions provided by Jerasure: galois_init_field and
  48 + galois_init_composite_field.
  49 +
  50 + GF-Complete exposes create_gf_from_argv, gf_init_easy and gf_init_hard. See
  51 + the GF-Complete documentation for more detail on how to use those functions.
  52 + You can definitely create more complicated fields with the GF-Complete
  53 + initialization functions, but the two helper functions provided by Jerasure
  54 + (galois_init_field and galois_init_composite_field) can be used to create most
  55 + of the supported Galois Fields.
  56 +
  57 +2.) galois_init_field
  58 +
  59 + Function signature:
  60 +
  61 + gf_t* galois_init_field(int w,
  62 + int mult_type,
  63 + int region_type,
  64 + int divide_type,
  65 + uint64_t prim_poly,
  66 + int arg1,
  67 + int arg2);
  68 +
  69 + This is a helper function that will initialize a Galois field. See the GF-Complete
  70 + documentation for more info on what the arguments mean. Here is a brief description
  71 + of the arguments:
  72 +
  73 + mult_type can be any *one* of the following:
  74 +
  75 + GF_MULT_DEFAULT
  76 + GF_MULT_SHIFT
  77 + GF_MULT_CARRY_FREE
  78 + GF_MULT_GROUP
  79 + GF_MULT_BYTWO_p
  80 + GF_MULT_BYTWO_b
  81 + GF_MULT_TABLE
  82 + GF_MULT_LOG_TABLE
  83 + GF_MULT_LOG_ZERO
  84 + GF_MULT_LOG_ZERO_EXT
  85 + GF_MULT_SPLIT_TABLE
  86 +
  87 + region_type can be a combination of the following (some combinations will not
  88 + be valid):
  89 +
  90 + GF_REGION_DEFAULT
  91 + GF_REGION_DOUBLE_TABLE
  92 + GF_REGION_QUAD_TABLE
  93 + GF_REGION_LAZY
  94 + GF_REGION_SSE
  95 + GF_REGION_NOSSE
  96 + GF_REGION_ALTMAP
  97 + GF_REGION_CAUCHY
  98 +
  99 + divide_type can be one of the following:
  100 +
  101 + GF_DIVIDE_DEFAULT
  102 + GF_DIVIDE_MATRIX
  103 + GF_DIVIDE_EUCLID
  104 +
  105 + prim_poly is the field-defining primitive polynomial
  106 +
  107 + arg1 and arg2 are special arguments usually used for defining SPLIT and GROUP
  108 + operations
  109 +
  110 +3.) galois_init_composite_field
  111 +
  112 + Function signature:
  113 +
  114 + gf_t* galois_init_composite_field(int w,
  115 + int region_type,
  116 + int divide_type,
  117 + int degree,
  118 + gf_t* base_gf);
  119 +
  120 + This is a helper function designed to make creating Composite fields easier. All you
  121 + need to do is hand it w, region mult type, divide type, degree and a pointer to a base
  122 + field. Note that the base_gf must have degree w/degree in order for this to work.
  123 + For example, if we create a GF using:
  124 +
  125 + galois_init_composite_field(32, GF_REGION_DEFAULT, GF_DIVIDE_DEFAULT, 2, base_gf);
  126 +
  127 + Then base_gf must have w=16.
  128 +
  129 +For more information on how to change the backing fields for Jerasure, please refer to
  130 +
  131 + 1.) Examples/reed_sol_test_gf.c: Runs basic tests for Reed-Solomon given args
  132 + for a backing GF (uses create_gf_from_argv to get gf_t pointer)
  133 +
  134 + 2.) Examples/reed_sol_time_gf.c: Runs more thorough timing and validation tests
  135 + for a backing GF (uses create_gf_from_argv to get gf_t pointer)
  136 +
  137 + 3.) Examples/reed_sol_hard_time_gf.c: Runs more thorough timing and validation tests
  138 + for a backing GF (uses galois_init_field and galois_init_composite_field to get gf_t
  139 + pointer)
  140 +
... ...
README.txt
... ... @@ -29,4 +29,112 @@ determine the default field to use, if one is not specified.
29 29 If you would like to explore a using a different Galois Field implementation,
30 30 you can dynamically set the backend GF for a given word-size (w).
31 31  
32   -<TBD - Add instructions>
  32 +The new galois.[ch] exports the following functions to be used by applications
  33 +for dynamically setting the backend GF:
  34 +
  35 +1.) galois_change_technique
  36 +
  37 + Function signature:
  38 +
  39 + void galois_change_technique(gf_t *gf, int w);
  40 +
  41 + This function will take a pointer to a Galois field structure and set it as the
  42 + current backend for all operations in GF(2^w). Note that you must specify 'w'
  43 + here, since the internal GF structure is mostly opaque to Jerasure. Be sure to
  44 + change the technique with the correct structure and word-size.
  45 +
  46 + There are a few ways to get a pointer to a gf_t structure: via GF-Complete or
  47 + using the helper functions provided by Jerasure: galois_init_field and
  48 + galois_init_composite_field.
  49 +
  50 + GF-Complete exposes create_gf_from_argv, gf_init_easy and gf_init_hard. See
  51 + the GF-Complete documentation for more detail on how to use those functions.
  52 + You can definitely create more complicated fields with the GF-Complete
  53 + initialization functions, but the two helper functions provided by Jerasure
  54 + (galois_init_field and galois_init_composite_field) can be used to create most
  55 + of the supported Galois Fields.
  56 +
  57 +2.) galois_init_field
  58 +
  59 + Function signature:
  60 +
  61 + gf_t* galois_init_field(int w,
  62 + int mult_type,
  63 + int region_type,
  64 + int divide_type,
  65 + uint64_t prim_poly,
  66 + int arg1,
  67 + int arg2);
  68 +
  69 + This is a helper function that will initialize a Galois field. See the GF-Complete
  70 + documentation for more info on what the arguments mean. Here is a brief description
  71 + of the arguments:
  72 +
  73 + mult_type can be any *one* of the following:
  74 +
  75 + GF_MULT_DEFAULT
  76 + GF_MULT_SHIFT
  77 + GF_MULT_CARRY_FREE
  78 + GF_MULT_GROUP
  79 + GF_MULT_BYTWO_p
  80 + GF_MULT_BYTWO_b
  81 + GF_MULT_TABLE
  82 + GF_MULT_LOG_TABLE
  83 + GF_MULT_LOG_ZERO
  84 + GF_MULT_LOG_ZERO_EXT
  85 + GF_MULT_SPLIT_TABLE
  86 +
  87 + region_type can be a combination of the following (some combinations will not
  88 + be valid):
  89 +
  90 + GF_REGION_DEFAULT
  91 + GF_REGION_DOUBLE_TABLE
  92 + GF_REGION_QUAD_TABLE
  93 + GF_REGION_LAZY
  94 + GF_REGION_SSE
  95 + GF_REGION_NOSSE
  96 + GF_REGION_ALTMAP
  97 + GF_REGION_CAUCHY
  98 +
  99 + divide_type can be one of the following:
  100 +
  101 + GF_DIVIDE_DEFAULT
  102 + GF_DIVIDE_MATRIX
  103 + GF_DIVIDE_EUCLID
  104 +
  105 + prim_poly is the field-defining primitive polynomial
  106 +
  107 + arg1 and arg2 are special arguments usually used for defining SPLIT and GROUP
  108 + operations
  109 +
  110 +3.) galois_init_composite_field
  111 +
  112 + Function signature:
  113 +
  114 + gf_t* galois_init_composite_field(int w,
  115 + int region_type,
  116 + int divide_type,
  117 + int degree,
  118 + gf_t* base_gf);
  119 +
  120 + This is a helper function designed to make creating Composite fields easier. All you
  121 + need to do is hand it w, region mult type, divide type, degree and a pointer to a base
  122 + field. Note that the base_gf must have degree w/degree in order for this to work.
  123 + For example, if we create a GF using:
  124 +
  125 + galois_init_composite_field(32, GF_REGION_DEFAULT, GF_DIVIDE_DEFAULT, 2, base_gf);
  126 +
  127 + Then base_gf must have w=16.
  128 +
  129 +For more information on how to change the backing fields for Jerasure, please refer to
  130 +
  131 + 1.) Examples/reed_sol_test_gf.c: Runs basic tests for Reed-Solomon given args
  132 + for a backing GF (uses create_gf_from_argv to get gf_t pointer)
  133 +
  134 + 2.) Examples/reed_sol_time_gf.c: Runs more thorough timing and validation tests
  135 + for a backing GF (uses create_gf_from_argv to get gf_t pointer)
  136 +
  137 + 3.) Examples/reed_sol_hard_time_gf.c: Runs more thorough timing and validation tests
  138 + for a backing GF (uses galois_init_field and galois_init_composite_field to get gf_t
  139 + pointer)
  140 +
... ...
galois.c
... ... @@ -44,8 +44,9 @@
44 44  
45 45 #include "galois.h"
46 46  
47   -#define MAX_GF_INSTANCES 128
  47 +#define MAX_GF_INSTANCES 64
48 48 gf_t *gfp_array[MAX_GF_INSTANCES] = { 0 };
  49 +int gfp_is_composite[MAX_GF_INSTANCES] = { 0 };
49 50  
50 51 gf_t *galois_get_field_ptr(int w)
51 52 {
... ... @@ -56,23 +57,29 @@ gf_t *galois_get_field_ptr(int w)
56 57 return NULL;
57 58 }
58 59  
59   -int galois_init_base_field(int w,
60   - int mult_type,
61   - int region_type,
62   - int divide_type,
63   - uint64_t prim_poly,
64   - int arg1,
65   - int arg2)
  60 +gf_t* galois_init_field(int w,
  61 + int mult_type,
  62 + int region_type,
  63 + int divide_type,
  64 + uint64_t prim_poly,
  65 + int arg1,
  66 + int arg2)
66 67 {
67 68 int scratch_size;
68 69 void *scratch_memory;
69   - int ret;
  70 + gf_t *gfp;
70 71  
71 72 if (w <= 0 || w > 32) {
72 73 fprintf(stderr, "ERROR -- cannot init default Galois field for w=%d\n", w);
73 74 exit(1);
74 75 }
75 76  
  77 + gfp = (gf_t *) malloc(sizeof(gf_t));
  78 + if (!gfp) {
  79 + fprintf(stderr, "ERROR -- cannot allocate memory for Galois field w=%d\n", w);
  80 + exit(1);
  81 + }
  82 +
76 83 scratch_size = gf_scratch_size(w, mult_type, region_type, divide_type, arg1, arg2);
77 84 if (!scratch_size) {
78 85 fprintf(stderr, "ERROR -- cannot get scratch size for base field w=%d\n", w);
... ... @@ -85,20 +92,7 @@ int galois_init_base_field(int w,
85 92 exit(1);
86 93 }
87 94  
88   - /*
89   - * Properly free up the old field
90   - */
91   - if (gfp_array[w] != NULL) {
92   - gf_free(gfp_array[w], 0);
93   - }
94   -
95   - gfp_array[w] = (gf_t*)malloc(sizeof(gf_t));
96   - if (gfp_array[w] == NULL) {
97   - fprintf(stderr, "ERROR -- cannot allocate memory for Galois field w=%d\n", w);
98   - exit(1);
99   - }
100   -
101   - if(!gf_init_hard(gfp_array[w],
  95 + if(!gf_init_hard(gfp,
102 96 w,
103 97 mult_type,
104 98 region_type,
... ... @@ -112,27 +106,33 @@ int galois_init_base_field(int w,
112 106 fprintf(stderr, "ERROR -- cannot init default Galois field for w=%d\n", w);
113 107 exit(1);
114 108 }
115   - return 0;
  109 +
  110 + gfp_is_composite[w] = 0;
  111 + return gfp;
116 112 }
117 113  
118   -int galois_init_composite_field(int w,
  114 +gf_t* galois_init_composite_field(int w,
119 115 int region_type,
120 116 int divide_type,
121   - uint64_t prim_poly,
122   - int arg1,
123   - int arg2,
  117 + int degree,
124 118 gf_t* base_gf)
125 119 {
126 120 int scratch_size;
127 121 void *scratch_memory;
128   - int ret;
  122 + gf_t *gfp;
129 123  
130 124 if (w <= 0 || w > 32) {
131 125 fprintf(stderr, "ERROR -- cannot init composite field for w=%d\n", w);
132 126 exit(1);
133 127 }
  128 +
  129 + gfp = (gf_t *) malloc(sizeof(gf_t));
  130 + if (!gfp) {
  131 + fprintf(stderr, "ERROR -- cannot allocate memory for Galois field w=%d\n", w);
  132 + exit(1);
  133 + }
134 134  
135   - scratch_size = gf_scratch_size(w, GF_MULT_COMPOSITE, region_type, divide_type, arg1, arg2);
  135 + scratch_size = gf_scratch_size(w, GF_MULT_COMPOSITE, region_type, divide_type, degree, 0);
136 136 if (!scratch_size) {
137 137 fprintf(stderr, "ERROR -- cannot get scratch size for composite field w=%d\n", w);
138 138 exit(1);
... ... @@ -144,34 +144,22 @@ int galois_init_composite_field(int w,
144 144 exit(1);
145 145 }
146 146  
147   - /*
148   - * Properly free up the old field
149   - */
150   - if (gfp_array[w] != NULL) {
151   - gf_free(gfp_array[w], 1);
152   - }
153   -
154   - gfp_array[w] = (gf_t*)malloc(sizeof(gf_t));
155