Commit a6a9a0eba33eda7f6e38dd7fb195e1f7cce16e79

Authored by Kevin Greenan
1 parent d23dfbc8
Exists in master and in 1 other branch v2

Added some performance tests, set the default GF for w=16 to SPLIT 16,4

and created function for autoconf to call during configure (spot check).
Test/makefile
... ... @@ -47,10 +47,9 @@ BINDIR=${PREFIX}/bin
47 47 LIBDIR=${PREFIX}/lib
48 48 INCDIR=${PREFIX}/include
49 49 CC = gcc
50   -#CFLAGS = -O3 -I$(HOME)/include
51   -CFLAGS = -g -I$(HOME)/include
  50 +CFLAGS = -O3 -I$(HOME)/include
52 51  
53   -ALL = reed_sol_test_01
  52 +ALL = reed_sol_test_01 reed_sol_test_02
54 53  
55 54 all: $(ALL)
56 55  
... ... @@ -99,3 +98,6 @@ reed_sol_test_01.o: galois.h reed_sol.h jerasure.h
99 98 reed_sol_test_01: reed_sol_test_01.o galois.o ${LIBDIR}/gf_complete.a jerasure.o reed_sol.o
100 99 $(CC) $(CFLAGS) -o reed_sol_test_01 reed_sol_test_01.o reed_sol.o jerasure.o galois.o ${LIBDIR}/gf_complete.a
101 100  
  101 +reed_sol_test_02.o: galois.h reed_sol.h jerasure.h
  102 +reed_sol_test_02: reed_sol_test_02.o galois.o ${LIBDIR}/gf_complete.a jerasure.o reed_sol.o
  103 + $(CC) $(CFLAGS) -o reed_sol_test_02 reed_sol_test_02.o reed_sol.o jerasure.o galois.o ${LIBDIR}/gf_complete.a
... ...
Test/reed_sol_test_02.c 0 → 100755
... ... @@ -0,0 +1,213 @@
  1 +/* Examples/reed_sol_01.c
  2 +Jerasure - A C/C++ Library for a Variety of Reed-Solomon and RAID-6 Erasure Coding Techniques
  3 +
  4 +Revision 1.2A
  5 +May 24, 2011
  6 +
  7 +James S. Plank
  8 +Department of Electrical Engineering and Computer Science
  9 +University of Tennessee
  10 +Knoxville, TN 37996
  11 +plank@cs.utk.edu
  12 +
  13 +Copyright (c) 2011, James S. Plank
  14 +All rights reserved.
  15 +
  16 +Redistribution and use in source and binary forms, with or without
  17 +modification, are permitted provided that the following conditions
  18 +are met:
  19 +
  20 + - Redistributions of source code must retain the above copyright
  21 + notice, this list of conditions and the following disclaimer.
  22 +
  23 + - Redistributions in binary form must reproduce the above copyright
  24 + notice, this list of conditions and the following disclaimer in
  25 + the documentation and/or other materials provided with the
  26 + distribution.
  27 +
  28 + - Neither the name of the University of Tennessee nor the names of its
  29 + contributors may be used to endorse or promote products derived
  30 + from this software without specific prior written permission.
  31 +
  32 +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  33 +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  34 +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  35 +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  36 +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  37 +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
  38 +BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
  39 +OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  40 +AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  41 +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY
  42 +WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  43 +POSSIBILITY OF SUCH DAMAGE.
  44 +
  45 +
  46 + */
  47 +
  48 +
  49 +/*
  50 + revised by S. Simmerman
  51 + 2/25/08
  52 +*/
  53 +#include <stdio.h>
  54 +#include <stdlib.h>
  55 +#include <string.h>
  56 +#include <gf_complete.h>
  57 +#include "jerasure.h"
  58 +#include "reed_sol.h"
  59 +
  60 +#define talloc(type, num) (type *) malloc(sizeof(type)*(num))
  61 +
  62 +void
  63 +timer_start (double *t)
  64 +{
  65 + struct timeval tv;
  66 +
  67 + gettimeofday (&tv, NULL);
  68 + *t = (double)tv.tv_sec + (double)tv.tv_usec * 1e-6;
  69 +}
  70 +
  71 +double
  72 +timer_split (const double *t)
  73 +{
  74 + struct timeval tv;
  75 + double cur_t;
  76 +
  77 + gettimeofday (&tv, NULL);
  78 + cur_t = (double)tv.tv_sec + (double)tv.tv_usec * 1e-6;
  79 + return (cur_t - *t);
  80 +}
  81 +
  82 +usage(char *s)
  83 +{
  84 + fprintf(stderr, "usage: reed_sol_test_01 k m w iterations bufsize [additional GF args]- Tests Reed-Solomon in GF(2^w).\n");
  85 + fprintf(stderr, " \n");
  86 + fprintf(stderr, " w must be 8, 16 or 32. k+m must be <= 2^w. It sets up a classic\n");
  87 + fprintf(stderr, " Vandermonde-based distribution matrix and encodes k devices of\n");
  88 + fprintf(stderr, " <bufsize> bytes each with it. Then it decodes.\n");
  89 + fprintf(stderr, " \n");
  90 + fprintf(stderr, "This tests: jerasure_matrix_encode()\n");
  91 + fprintf(stderr, " jerasure_matrix_decode()\n");
  92 + fprintf(stderr, " jerasure_print_matrix()\n");
  93 + fprintf(stderr, " reed_sol_vandermonde_coding_matrix()\n");
  94 + if (s != NULL) fprintf(stderr, "%s\n", s);
  95 + exit(1);
  96 +}
  97 +
  98 +gf_t* get_gf(int w, int argc, char **argv, int starting)
  99 +{
  100 + gf_t *gf = (gf_t*)malloc(sizeof(gf_t));
  101 + if (create_gf_from_argv(gf, w, argc, argv, starting) == 0) {
  102 + free(gf);
  103 + gf = NULL;
  104 + }
  105 + return gf;
  106 +}
  107 +
  108 +static void fill_buffer(unsigned char *buf, int size)
  109 +{
  110 + int i;
  111 +
  112 + buf[0] = (char)(lrand48() % 256);
  113 +
  114 + for (i=1; i < size; i++) {
  115 + buf[i] = ((buf[i-1] + i) % 256);
  116 + }
  117 +}
  118 +
  119 +int main(int argc, char **argv)
  120 +{
  121 + long l;
  122 + int k, w, i, j, m, iterations, bufsize;
  123 + int *matrix;
  124 + char **data, **coding, **old_values;
  125 + int *erasures, *erased;
  126 + int *decoding_matrix, *dm_ids;
  127 + double t = 0, total_time = 0;
  128 + gf_t *gf = NULL;
  129 +
  130 + if (argc < 6) usage(NULL);
  131 + if (sscanf(argv[1], "%d", &k) == 0 || k <= 0) usage("Bad k");
  132 + if (sscanf(argv[2], "%d", &m) == 0 || m <= 0) usage("Bad m");
  133 + if (sscanf(argv[3], "%d", &w) == 0 || (w != 8 && w != 16 && w != 32)) usage("Bad w");
  134 + if (sscanf(argv[4], "%d", &iterations) == 0) usage("Bad iterations");
  135 + if (sscanf(argv[5], "%d", &bufsize) == 0) usage("Bad bufsize");
  136 + if (w <= 16 && k + m > (1 << w)) usage("k + m is too big");
  137 +
  138 + srand48(time(0));
  139 +
  140 + gf = get_gf(w, argc, argv, 6);
  141 +
  142 + if (gf == NULL) {
  143 + usage("Invalid arguments given for GF!\n");
  144 + }
  145 +
  146 + galois_change_technique(gf, w);
  147 +
  148 + matrix = reed_sol_vandermonde_coding_matrix(k, m, w);
  149 +
  150 + printf("Last m rows of the Distribution Matrix:\n\n");
  151 + jerasure_print_matrix(matrix, m, k, w);
  152 + printf("\n");
  153 +
  154 + data = talloc(char *, k);
  155 + for (i = 0; i < k; i++) {
  156 + data[i] = talloc(char, bufsize);
  157 + fill_buffer(data[i], bufsize);
  158 + }
  159 +
  160 + coding = talloc(char *, m);
  161 + old_values = talloc(char *, m);
  162 + for (i = 0; i < m; i++) {
  163 + coding[i] = talloc(char, bufsize);
  164 + old_values[i] = talloc(char, bufsize);
  165 + }
  166 +
  167 + for (i = 0; i < iterations; i++) {
  168 + timer_start(&t);
  169 + jerasure_matrix_encode(k, m, w, matrix, data, coding, bufsize);
  170 + total_time += timer_split(&t);
  171 + }
  172 +
  173 + fprintf(stderr, "Encode thput for %d iterations: %.2f MB/s (%.2f sec)\n", iterations, (double)(k*iterations*bufsize/1024/1024) / total_time, total_time);
  174 +
  175 + erasures = talloc(int, (m+1));
  176 + erased = talloc(int, (k+m));
  177 + for (i = 0; i < m+k; i++) erased[i] = 0;
  178 + l = 0;
  179 + for (i = 0; i < m; ) {
  180 + erasures[i] = ((unsigned int)lrand48())%(k+m);
  181 + if (erased[erasures[i]] == 0) {
  182 + erased[erasures[i]] = 1;
  183 + memcpy(old_values[i], (erasures[i] < k) ? data[erasures[i]] : coding[erasures[i]-k], bufsize);
  184 + bzero((erasures[i] < k) ? data[erasures[i]] : coding[erasures[i]-k], bufsize);
  185 + i++;
  186 + }
  187 + }
  188 + erasures[i] = -1;
  189 +
  190 + for (i = 0; i < iterations; i++) {
  191 + timer_start(&t);
  192 + jerasure_matrix_decode(k, m, w, matrix, 1, erasures, data, coding, bufsize);
  193 + total_time += timer_split(&t);
  194 + }
  195 +
  196 + fprintf(stderr, "Decode thput for %d iterations: %.2f MB/s (%.2f sec)\n", iterations, (double)(m*iterations*bufsize/1024/1024) / total_time, total_time);
  197 +
  198 + for (i = 0; i < m; i++) {
  199 + if (erasures[i] < k) {
  200 + if (memcmp(data[erasures[i]], old_values[i], bufsize)) {
  201 + fprintf(stderr, "Decoding failed for %d!\n", erasures[i]);
  202 + exit(1);
  203 + }
  204 + } else {
  205 + if (memcmp(coding[erasures[i]-k], old_values[i], bufsize)) {
  206 + fprintf(stderr, "Decoding failed for %d!\n", erasures[i]);
  207 + exit(1);
  208 + }
  209 + }
  210 + }
  211 +
  212 + return 0;
  213 +}
... ...
Test/time_all_gfs.sh 0 → 100755
... ... @@ -0,0 +1,41 @@
  1 +GF_COMPLETE_DIR=/usr/local/bin
  2 +GF_METHODS=${GF_COMPLETE_DIR}/gf_methods
  3 +ITERATIONS=128
  4 +BUFSIZE=65536
  5 +k=12
  6 +m=2
  7 +
  8 +# Test all w=8
  9 +${GF_METHODS} | awk -F: '{ if ($1 == "w=8") print $2; }' |
  10 +while read method; do
  11 + echo "Testing ${k} ${m} 8 ${ITERATIONS} ${BUFSIZE} ${method}"
  12 + ./reed_sol_test_02 ${k} ${m} 8 ${ITERATIONS} ${BUFSIZE} ${method}
  13 + if [[ $? != "0" ]]; then
  14 + echo "Failed test for ${k} ${m} 8 ${ITERATIONS} ${BUFSIZE} ${method}"
  15 + exit 1
  16 + fi
  17 +done
  18 +
  19 +# Test all w=16
  20 +${GF_METHODS} | grep -v 'TABLE' | awk -F: '{ if ($1 == "w=16") print $2; }' |
  21 +while read method; do
  22 + echo "Testing ${k} ${m} 16 ${ITERATIONS} ${BUFSIZE} ${method}"
  23 + ./reed_sol_test_02 ${k} ${m} 16 ${ITERATIONS} ${BUFSIZE} ${method}
  24 + if [[ $? != "0" ]]; then
  25 + echo "Failed test for ${k} ${m} 16 ${ITERATIONS} ${BUFSIZE} ${method}"
  26 + exit 1
  27 + fi
  28 +done
  29 +
  30 +# Test all w=32
  31 +${GF_METHODS} | awk -F: '{ if ($1 == "w=32") print $2; }' |
  32 +while read method; do
  33 + echo "Testing ${k} ${m} 32 ${ITERATIONS} ${BUFSIZE} ${method}"
  34 + ./reed_sol_test_02 ${k} ${m} 32 ${ITERATIONS} ${BUFSIZE} ${method}
  35 + if [[ $? != "0" ]]; then
  36 + echo "Failed test for ${k} ${m} 32 ${ITERATIONS} ${BUFSIZE} ${method}"
  37 + exit 1
  38 + fi
  39 +done
  40 +
  41 +echo "Passed all tests!"
... ...
galois.c
... ... @@ -196,6 +196,14 @@ static void galois_init_default_field(int w)
196 196 exit(1);
197 197 }
198 198 }
  199 +
  200 + if (w == 16) {
  201 + if (!gf_init_hard(gfp_array[w], w, GF_MULT_SPLIT_TABLE, GF_REGION_ALTMAP, GF_DIVIDE_DEFAULT, 0, 16, 4, NULL, NULL)) {
  202 + fprintf(stderr, "ERROR -- cannot init default Galois field for w=%d\n", w);
  203 + exit(1);
  204 + }
  205 + }
  206 +
199 207 if (!gf_init_easy(gfp_array[w], w)) {
200 208 fprintf(stderr, "ERROR -- cannot init default Galois field for w=%d\n", w);
201 209 exit(1);
... ...
jerasure.c
... ... @@ -1380,3 +1380,16 @@ void jerasure_bitmatrix_encode(int k, int m, int w, int *bitmatrix,
1380 1380 }
1381 1381 }
1382 1382  
  1383 +/*
  1384 + * Exported function for use by autoconf to perform quick
  1385 + * spot-check.
  1386 + */
  1387 +int jerasure_autoconf_test()
  1388 +{
  1389 + int x = galois_single_multiply(1, 2, 8);
  1390 + if (x != 2) {
  1391 + return -1;
  1392 + }
  1393 + return 0;
  1394 +}
  1395 +
... ...
jerasure.h
... ... @@ -297,4 +297,6 @@ int *jerasure_matrix_multiply(int *m1, int *m2, int r1, int c1, int r2, int c2,
297 297  
298 298 void jerasure_get_stats(double *fill_in);
299 299  
  300 +int jerasure_autoconf_test();
  301 +
300 302 #endif
... ...