From 26cfeb68c1da44bc454806933dde5459455e113b Mon Sep 17 00:00:00 2001 From: couturie <you@example.com> Date: Sun, 27 May 2018 20:50:59 +0200 Subject: [PATCH] ida --- IDA/Makefile | 3 + IDA/ida_gf65.cpp | 565 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 568 insertions(+) create mode 100644 IDA/ida_gf65.cpp diff --git a/IDA/Makefile b/IDA/Makefile index 8d7d811..71745c6 100644 --- a/IDA/Makefile +++ b/IDA/Makefile @@ -15,5 +15,8 @@ ida: ida.cpp ida_gf64: ida_gf64.cpp g++ -o $@ $< -std=c++11 -O3 -lm -mmmx -msse -msse2 -msse3 -mssse3 -msse4.1 -msse4.2 -mavx -g -O3 -Wall -I/home/couturie/ajeter/jerasure/include /home/couturie/ajeter/jerasure/src/.libs/jerasure.o /home/couturie/ajeter/jerasure/src/.libs/galois.o -lgf_complete -fpermissive +ida_gf65: ida_gf65.cpp + g++ -o $@ $< -std=c++11 -O3 -lm -mmmx -msse -msse2 -msse3 -mssse3 -msse4.1 -msse4.2 -mavx -g -O3 -Wall -I/home/couturie/ajeter/jerasure/include /home/couturie/ajeter/jerasure/src/.libs/jerasure.o /home/couturie/ajeter/jerasure/src/.libs/galois.o -lgf_complete -fpermissive + clean: rm test_mat2 ida ida_gf64 diff --git a/IDA/ida_gf65.cpp b/IDA/ida_gf65.cpp new file mode 100644 index 0000000..09c4162 --- /dev/null +++ b/IDA/ida_gf65.cpp @@ -0,0 +1,565 @@ + + +#include <iostream> +#include <fstream> +#include <chrono> +#include <sstream> +#include <ctime> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <algorithm> // std::random_shuffle +#include <vector> // std::vector +extern "C" { + #include "jerasure.h" +} + +typedef unsigned long mylong; +#define LLUI (long long unsigned int) + + +using namespace std; + +string cloud[5]={"dropboxida1","googleida1","megaida1","onedriveida1","pcloudida1"}; + + +void display(mylong *mat, int r, int c) { + for(int i=0;i<r;i++) { + for(int j=0;j<c;j++) { + printf("%016llu ",LLUI mat[i*c+j]); + } + printf("\n"); + } + printf("\n"); +} + +mylong *matrix_multiply(gf_t *gf, mylong *m1, mylong *m2, int r1, int c1, int r2, int c2, int w) +{ + mylong *product; + int i, j, k; + + product = (mylong *) malloc(sizeof(mylong)*r1*c2); + for (i = 0; i < r1*c2; i++) product[i] = 0; + + for (i = 0; i < r1; i++) { + for (j = 0; j < c2; j++) { + for (k = 0; k < r2; k++) { + product[i*c2+j] ^= gf->multiply.w64(gf,m1[i*c1+k], m2[k*c2+j]); + } + } + } + return product; +} + + + +int invert_matrix(gf_t *gf, mylong *mat, mylong *inv, int rows) +{ + int cols, i, j, k, x, rs2; + int row_start; + mylong tmp, inverse; + + cols = rows; + + k = 0; + for (i = 0; i < rows; i++) { + for (j = 0; j < cols; j++) { + inv[k] = (i == j) ? 1 : 0; + k++; + } + } +// display(inv, rows, rows); +// printf("\n"); + + /* First -- convert into upper triangular */ + for (i = 0; i < cols; i++) { + row_start = cols*i; + + /* Swap rows if we ave a zero i,i element. If we can't swap, then the + matrix was not invertible */ + + if (mat[row_start+i] == 0) { + for (j = i+1; j < rows && mat[cols*j+i] == 0; j++) ; + if (j == rows) return -1; + rs2 = j*cols; + for (k = 0; k < cols; k++) { + tmp = mat[row_start+k]; + mat[row_start+k] = mat[rs2+k]; + mat[rs2+k] = tmp; + tmp = inv[row_start+k]; + inv[row_start+k] = inv[rs2+k]; + inv[rs2+k] = tmp; + } + } + + /* Multiply the row by 1/element i,i */ + tmp = mat[row_start+i]; + if (tmp != 1) { + inverse = gf->divide.w64(gf,1, tmp); + for (j = 0; j < cols; j++) { + mat[row_start+j] = gf->multiply.w64(gf,mat[row_start+j], inverse); + inv[row_start+j] = gf->multiply.w64(gf,inv[row_start+j], inverse); + } + } + + /* Now for each j>i, add A_ji*Ai to Aj */ + k = row_start+i; + for (j = i+1; j != cols; j++) { + k += cols; + if (mat[k] != 0) { + if (mat[k] == 1) { + rs2 = cols*j; + for (x = 0; x < cols; x++) { + mat[rs2+x] ^= mat[row_start+x]; + inv[rs2+x] ^= inv[row_start+x]; + } + } else { + tmp = mat[k]; + rs2 = cols*j; + for (x = 0; x < cols; x++) { + mat[rs2+x] ^= gf->multiply.w64(gf,tmp, mat[row_start+x]); + inv[rs2+x] ^= gf->multiply.w64(gf,tmp, inv[row_start+x]); + } + } + } + } + } + + /* Now the matrix is upper triangular. Start at the top and multiply down */ + + for (i = rows-1; i >= 0; i--) { + row_start = i*cols; + for (j = 0; j < i; j++) { + rs2 = j*cols; + if (mat[rs2+i] != 0) { + tmp = mat[rs2+i]; + mat[rs2+i] = 0; + for (k = 0; k < cols; k++) { + inv[rs2+k] ^= gf->multiply.w64(gf,tmp, inv[row_start+k]); + } + } + } + } + +/* printf("mat\n"); + display(mat, rows, rows); + printf("\n"); + printf("inv\n"); + display(inv, rows, rows); + printf("\n"); +*/ + return 0; +} + + + + +int invertible_matrix(gf_t *gf, int *mat, int rows, int w) +{ + int cols, i, j, k, x, rs2; + int row_start; + mylong tmp, inverse; + + cols = rows; + + /* First -- convert into upper triangular */ + for (i = 0; i < cols; i++) { + row_start = cols*i; + + /* Swap rows if we ave a zero i,i element. If we can't swap, then the + matrix was not invertible */ + + if (mat[row_start+i] == 0) { + for (j = i+1; j < rows && mat[cols*j+i] == 0; j++) ; + if (j == rows) return 0; + rs2 = j*cols; + for (k = 0; k < cols; k++) { + tmp = mat[row_start+k]; + mat[row_start+k] = mat[rs2+k]; + mat[rs2+k] = tmp; + } + } + + /* Multiply the row by 1/element i,i */ + tmp = mat[row_start+i]; + if (tmp != 1) { + inverse = gf->divide.w64(gf,1, tmp); + for (j = 0; j < cols; j++) { + mat[row_start+j] = gf->multiply.w64(gf,mat[row_start+j], inverse); + } + } + + /* Now for each j>i, add A_ji*Ai to Aj */ + k = row_start+i; + for (j = i+1; j != cols; j++) { + k += cols; + if (mat[k] != 0) { + if (mat[k] == 1) { + rs2 = cols*j; + for (x = 0; x < cols; x++) { + mat[rs2+x] ^= mat[row_start+x]; + } + } else { + tmp = mat[k]; + rs2 = cols*j; + for (x = 0; x < cols; x++) { + mat[rs2+x] ^= gf->multiply.w64(gf,tmp,mat[row_start+x]); + } + } + } + } + } + return 1; +} + + + + + +mylong* readFullFile(int n, int t, mylong& sizeFile, mylong & padded_size) { + + ifstream stream("lena.png", ios::in | ios::binary | ios::ate); +// ifstream stream("lena_small.png", ios::in | ios::binary | ios::ate); +// ifstream stream("/home/couturie/Downloads/CARCARIASS.zip", ios::in | ios::binary | ios::ate); + + sizeFile=stream.tellg(); + std::cout << sizeFile << std::endl; + stream.seekg(0, ios::beg); + + + + + + + vector<uint8_t> contents((istreambuf_iterator<char>(stream)), istreambuf_iterator<char>()); + + + + + + + + //make padding, we need to pad to be divisible by 8*t, we + if((sizeFile+8)%(8*t)!=0) { + cout<<(int)(sizeFile/(8*t))<<endl; + + int remainder=(8*t)*(1+(int)((sizeFile+8)/(8*t)))-sizeFile-8; + cout << "remainder " << remainder << endl; + uint8_t vec[remainder]; + for(int i=0;i<remainder;i++) + vec[i]=0; + //add remainder elements at the end + contents.insert(contents.end(),vec,vec+remainder); + //add 8 elements at the beginning for the size + contents.insert(contents.begin(),vec,vec+8); + } + + + + cout << "res contents " << contents.size() << endl; + +// for(int i=0;i<contents.size();i++) +// cout << (int)contents[i] << " "; + cout << endl; + + uint8_t *p_contents=&contents[0]; +// mylong *p_contents2=reinterpret_cast<mylong*>(p_contents); + + padded_size=contents.size()/8; + + mylong *p_contents2=new mylong[padded_size]; + memcpy(p_contents2,p_contents,sizeof(mylong)*padded_size); + //mylong *p_contents2=(mylong*)p_contents; + + p_contents2[0]=sizeFile; + + + + +/* for(int i=0;i<padded_size;i++) + cout << p_contents2[i] << " "; + cout << endl; +*/ + /* long res=0; + for(int i=8-1;i>=0;i--) { + res<<=8; + res+=p_contents[i]; + } + + cout << "convert val " << (long)res << endl; + + res=0; + for(int i=16-1;i>=8;i--) { + res<<=8; + res+=p_contents[i]; + } + + cout << "convert val " << (long)res << endl; + */ + + return p_contents2; +} + +void sendChunk(string name,int cloud_id) { + stringstream ss; + ss <<"rclone copy "<<name<<" "<<cloud[cloud_id]<<":"; + string str = ss.str(); + cout<<str<<endl; + system(str.c_str()); + ss.str(""); + ss<<"rm -f "<<name; + str = ss.str(); + cout<<str<<endl; + system(str.c_str()); +} + + +void retrieveChunk(string name,int cloud_id) { + stringstream ss; + ss <<"rclone copy "<<cloud[cloud_id]<<":"<<name<<" ."; + string str = ss.str(); + cout<<str<<endl; + system(str.c_str()); +} + + +void saveFile(uint8_t *data, const char *fileName,long size_file) { + cout<<"size file "<<size_file<<endl; + FILE* pFile = fopen (fileName, "wb"); + fwrite (data , sizeof(uint8_t), size_file, pFile); + + fclose (pFile); +} + + +void readFile(uint8_t *data, const char *fileName,long size_file) { + cout<<"size file "<<size_file<<endl; + FILE* pFile = fopen (fileName, "rb"); + + fseek(pFile, 0L, SEEK_END); + int sz = ftell(pFile); + rewind(pFile); + if(sz!=size_file) { + cout << "error : size of this chunk is not correct !! " << endl; + exit(0); + } + fread (data , sizeof(uint8_t), size_file, pFile); + + fclose (pFile); +} + + +void saveFile8(mylong *data, const char *fileName,long size_file) { + + cout<<"size file 8 "<<size_file<<endl; + display(data,1,1); + FILE* pFile = fopen (fileName, "wb"); + + + + fwrite (data , sizeof(mylong), size_file, pFile); + fclose (pFile); + + + +} + + +int main(int argc, char **argv) +{ + unsigned int w; + int invert; + mylong *matS; + mylong *matG; + mylong *matC; + mylong *inverse; + mylong *identity; + + + int size=5000000; + int t=3; + int n=5; + + + w=64; + gf_t gf; + gf_init_easy(&gf, w); + + mylong padded_size; + mylong sizeFile; + matS=readFullFile(n,t,sizeFile,padded_size); + + cout<<padded_size*8<<endl; + +/* for(int i=0;i<padded_size;i++) + cout << matS[i] << " "; + cout << endl; +*/ + + int len=padded_size/t; +// int len=size/t; + cout<<" len "<<len<<" "<<padded_size<<endl; + + // matS = malloc(sizeof(mylong)*t*len); + matG = malloc(sizeof(mylong)*t*n); + + + + + srand48(time(0)); + std::srand ( unsigned ( std::time(0) ) ); + +/* + for(int i=0;i<t;i++) { + for(int j=0;j<len;j++) { + matS[i*len+j]=lrand48()<<32|lrand48(); + } + } + */ + + for(int i=0;i<n;i++) { + for(int j=0;j<t;j++) { + matG[i*t+j]=lrand48()<<32|lrand48(); + } + } + + + + printf("Matrix S:\n"); +// display(matS, len, t); + display(matS, t, t); + + printf("Matrix G:\n"); + display(matG, n, t); + + + auto start = std::chrono::system_clock::now(); + matC=matrix_multiply(&gf, matG, matS, n, t, t, len, w); + auto end = std::chrono::system_clock::now(); + std::chrono::duration<double> elapsed_seconds = end-start; + std::cout << "elapsed time: " << elapsed_seconds.count() << "s\n"; + +// display(matC,t,t); + + + //Save trunks + for(int i=0;i<n;i++) { + stringstream ss; + ss <<"lena_"<<i<<".png"; + string str = ss.str(); + saveFile((uint8_t*)&matC[i*len], str.c_str(),len*sizeof(mylong)); + sendChunk( str,i); + } + + + + mylong *matCs = malloc(sizeof(mylong)*t*len); + mylong *matGs = malloc(sizeof(mylong)*t*t); + + + std::vector<int> myvector; + + // set some values: + for (int i=0; i<n; ++i) myvector.push_back(i); // 1 2 3 4 5 6 7 8 9 + + // using built-in random generator: + random_shuffle ( myvector.begin(), myvector.end() ); + + + + std::cout << "random chunk" << std::endl; + int ind=0; + for(int ii=0;ii<t;ii++) { +// for(int i=n-1;i>=t;i--) { +// for(int i=0;i<n;i+=2) { + + auto i=myvector[ii]; + std::cout << myvector[i] << " "; + + + stringstream ss; + ss <<"lena_"<<i<<".png"; + string str = ss.str(); + + retrieveChunk(str,i); + readFile((uint8_t*)&matCs[ind*len], str.c_str(),len*sizeof(mylong)); + +// display(&matCs[ind*len],1,1); +// display(&matC[i*len],1,1); + + /*for(int j=0;j<len;j++) { + matCs[ind*len+j]=matC[i*len+j]; + }*/ + + + + for(int j=0;j<t;j++) { + matGs[ind*t+j]=matG[i*t+j]; + } + ind++; + } + std::cout << std::endl; + + + printf("Matrix Gs:\n"); + display(matGs, t, t); + +// printf("Matrix Cs:\n"); +// display(matCs, t, len); + + mylong* matGs_copy = malloc(sizeof(mylong)* t*t); + + //WARNING invert changes the matrix +// invert = invertible_matrix(&gf,matGs, t, w); +// printf("\nInvertible Gs: %s\n", (invert == 1) ? "Yes" : "No"); + + memcpy(matGs_copy, matGs, sizeof(mylong)*t*t); + mylong *matInvGs = malloc(sizeof(mylong)*t*t); + invert_matrix(&gf, matGs_copy, matInvGs, t); + + /* + WARNING this changes matGs + identity=matrix_multiply(&gf, matInvGs, matGs, t, t, t, t, w); + printf("Identity:\n"); + display(identity, t, t); + */ + + + //printf("Matrix Gs:\n"); + //display(matGs, t, t); + + mylong *matS2 = malloc(sizeof(mylong)*t*len); + + + start = std::chrono::system_clock::now(); + matS2=matrix_multiply(&gf, matInvGs, matCs, t, t, t, len, w); + end = std::chrono::system_clock::now(); + elapsed_seconds = end-start; + std::cout << "elapsed time: " << elapsed_seconds.count() << "s\n"; + + + printf("Matrix S2:\n"); +// display(matS2, len, t); + display(matS2, t, t); + + int equal=1; + for(int i=0;i<padded_size && equal;i++) { + equal=matS[i]==matS2[i]; +// if(!equal) +// printf("ARGH\n"); + } + if(equal) + printf("EQUAL !!!\n"); + + + mylong new_size=matS2[0]; + cout << "size of data " << new_size << endl; + + + //first elements that contains the size is removed + uint8_t *reconstucted_data=reinterpret_cast<uint8_t*>(&matS2[1]); + saveFile(reconstucted_data, "lena2.png",new_size); + return 0; +} + + -- 2.39.5