From 4e348a4bb3b30e44371ca9aa293ce2573de92b3d Mon Sep 17 00:00:00 2001 From: Jean-Marie Verdun Date: Tue, 15 Oct 2019 21:03:44 +0200 Subject: [PATCH] Refactor code to prepare Amazon s3 v4 API support We do support currently only v2 which will be EOL in june 2020 according to Amazon --- src/Mod/Cloud/App/AppCloud.cpp | 337 ++++++++++++++------------------- src/Mod/Cloud/App/AppCloud.h | 9 + 2 files changed, 154 insertions(+), 192 deletions(-) diff --git a/src/Mod/Cloud/App/AppCloud.cpp b/src/Mod/Cloud/App/AppCloud.cpp index 3d18fa8e88..f6490e01b9 100644 --- a/src/Mod/Cloud/App/AppCloud.cpp +++ b/src/Mod/Cloud/App/AppCloud.cpp @@ -193,31 +193,15 @@ void Cloud::CloudWriter::checkText(DOMText* text) { void Cloud::CloudWriter::createBucket() { - printf("CREATING BUCKET %s\n", this->Bucket); - struct timeval tv; - struct tm *tm; - char date_formatted[256]; - char StringToSign[1024]; - unsigned char *digest; - + struct Cloud::AmzData *RequestData; CURL *curl; CURLcode res; struct data_buffer curl_buffer; -#if defined(FC_OS_WIN32) -#else - gettimeofday(&tv,NULL); - tm = localtime(&tv.tv_sec); - strftime(date_formatted,256,"%a, %d %b %Y %T %z", tm); -#endif - - // CHANGEME - sprintf(StringToSign,"PUT\n\napplication/xml\n%s\n/%s/", date_formatted, this->Bucket); - - // We have to use HMAC encoding and SHA1 - digest=HMAC(EVP_sha1(),this->SecretKey,strlen(this->SecretKey), - (const unsigned char *)&StringToSign,strlen(StringToSign),NULL,NULL); + char path[1024]; + sprintf(path, "/%s/", this->Bucket); + RequestData = Cloud::ComputeDigestAmzS3v2("PUT", "application/xml", path, this->SecretKey); // Let's build the Header and call to curl curl_global_init(CURL_GLOBAL_ALL); @@ -226,26 +210,26 @@ void Cloud::CloudWriter::createBucket() { struct curl_slist *chunk = NULL; char header_data[1024]; + char Url[256]; // Let's build our own header std::string strUrl(this->Url); eraseSubStr(strUrl,"http://"); eraseSubStr(strUrl,"https://"); - sprintf(header_data,"Host: %s", strUrl.c_str()); - chunk = curl_slist_append(chunk, header_data); - sprintf(header_data,"Date: %s", date_formatted); - chunk = curl_slist_append(chunk, header_data); - chunk = curl_slist_append(chunk, "Content-Type: application/xml"); - std::string digest_str; - digest_str=Base::base64_encode(digest,strlen((const char *)digest)); - sprintf(header_data,"Authorization: AWS %s:%s", this->AccessKey, - digest_str.c_str()); - chunk = curl_slist_append(chunk, header_data); - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, chunk); -// curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L); sprintf(header_data,"%s:%s/%s/", this->Url,this->TcpPort, this->Bucket); curl_easy_setopt(curl, CURLOPT_URL, header_data); + + chunk = Cloud::BuildHeaderAmzS3v2( strUrl.c_str(), this->TcpPort, this->AccessKey, RequestData); + + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, chunk); + + // Lets build the Url for our Curl call + + sprintf(Url,"%s:%s/%s/", this->Url,this->TcpPort, + this->Bucket); + curl_easy_setopt(curl, CURLOPT_URL, Url); + curl_easy_setopt(curl, CURLOPT_UPLOAD, 1L); curl_easy_setopt(curl, CURLOPT_PUT, 1L); // curl read a file not a memory buffer (it shall be able to do it) @@ -266,13 +250,78 @@ void Cloud::CloudWriter::createBucket() } } -Cloud::CloudWriter::CloudWriter(const char* Url, const char* AccessKey, const char* SecretKey, const char* TcpPort, const char* Bucket) +struct Cloud::AmzData *Cloud::ComputeDigestAmzS3v2(char *operation, char *data_type, const char *target, const char *Secret) { + struct AmzData *returnData; struct timeval tv; struct tm *tm; char date_formatted[256]; - char StringToSign[1024]; - unsigned char *digest; + char StringToSign[1024]; + unsigned char *digest; + // Amazon S3 and Swift require the timezone to be define to GMT. + // As to simplify the conversion this is performed through the TZ + // environment variable and a call to localtime as to convert output of gettimeofday + returnData = ( struct Cloud::AmzData *) malloc(sizeof(struct Cloud::AmzData)); + + strcpy(returnData->ContentType, data_type); + +#if defined(FC_OS_WIN32) + _putenv("TZ=GMT"); +#else + setenv("TZ","GMT",1); +#endif + // We must check that the bucket exists or not. If not, we have to create it. + // We must request the bucketlist if we receive a 404 error. This means that it + // doesn't exist. The other option is that the content list is empty. + // This piece of code is the same as the Reader except we do not interpret it ! +#if defined(FC_OS_WIN32) +#else + gettimeofday(&tv,NULL); + tm = localtime(&tv.tv_sec); + strftime(date_formatted,256,"%a, %d %b %Y %T %z", tm); +#endif + sprintf(StringToSign,"%s\n\n%s\n%s\n%s", operation, data_type, date_formatted, target); + // We have to use HMAC encoding and SHA1 + digest=HMAC(EVP_sha1(),Secret,strlen(Secret), + (const unsigned char *)&StringToSign,strlen(StringToSign),NULL,NULL); + returnData->digest = Base::base64_encode(digest,strlen((const char *)digest)); + strcpy(returnData->dateFormatted,date_formatted); + return returnData; + +} + +struct curl_slist *Cloud::BuildHeaderAmzS3v2(const char *Url, const char *TcpPort, const char *PublicKey, struct Cloud::AmzData *Data) +{ + char header_data[1024]; + struct curl_slist *chunk = NULL; + + // Build the Host: entry + + sprintf(header_data,"Host: %s:%s", Url, TcpPort); + chunk = curl_slist_append(chunk, header_data); + + // Build the Date entry + + sprintf(header_data,"Date: %s", Data->dateFormatted); + chunk = curl_slist_append(chunk, header_data); + + // Build the Content-Type entry + + sprintf(header_data,"Content-Type:%s", Data->ContentType); + chunk = curl_slist_append(chunk, header_data); + + // build the Auth entry + + sprintf(header_data,"Authorization: AWS %s:%s", PublicKey, + Data->digest.c_str()); + chunk = curl_slist_append(chunk, header_data); + + return chunk; +} + +Cloud::CloudWriter::CloudWriter(const char* Url, const char* AccessKey, const char* SecretKey, const char* TcpPort, const char* Bucket) +{ + struct Cloud::AmzData *RequestData; CURL *curl; CURLcode res; @@ -284,58 +333,31 @@ Cloud::CloudWriter::CloudWriter(const char* Url, const char* AccessKey, const ch this->TcpPort=TcpPort; this->Bucket=Bucket; this->FileName=""; - // Amazon S3 and Swift require the timezone to be define to GMT. - // As to simplify the conversion this is performed through the TZ - // environment variable and a call to localtime as to convert output of gettimeofday -#if defined(FC_OS_WIN32) - _putenv("TZ=GMT"); -#else - setenv("TZ","GMT",1); -#endif - // We must check that the bucket exists or not. If not, we have to create it. - // We must request the bucketlist if we receive a 404 error. This means that it - // doesn't exist. The other option is that the content list is empty. - // This piece of code is the same as the Reader except we do not interpret it ! -#if defined(FC_OS_WIN32) -#else - gettimeofday(&tv,NULL); - tm = localtime(&tv.tv_sec); - strftime(date_formatted,256,"%a, %d %b %Y %T %z", tm); -#endif - - sprintf(StringToSign,"GET\n\napplication/xml\n%s\n/%s/", date_formatted, this->Bucket); - - // We have to use HMAC encoding and SHA1 - digest=HMAC(EVP_sha1(),this->SecretKey,strlen(this->SecretKey), - (const unsigned char *)&StringToSign,strlen(StringToSign),NULL,NULL); - + char path[1024]; + sprintf(path,"/%s/", this->Bucket); + RequestData = Cloud::ComputeDigestAmzS3v2("GET", "application/xml", path, this->SecretKey); // Let's build the Header and call to curl curl_global_init(CURL_GLOBAL_ALL); curl = curl_easy_init(); if ( curl ) { - struct curl_slist *chunk = NULL; - char header_data[1024]; // Let's build our own header + struct curl_slist *chunk = NULL; + char Url[256]; std::string strUrl(this->Url); eraseSubStr(strUrl,"http://"); eraseSubStr(strUrl,"https://"); - sprintf(header_data,"Host: %s:%s", strUrl.c_str(), - this->TcpPort); - chunk = curl_slist_append(chunk, header_data); - sprintf(header_data,"Date: %s", date_formatted); - chunk = curl_slist_append(chunk, header_data); - chunk = curl_slist_append(chunk, "Content-Type: application/xml"); - std::string digest_str; - digest_str=Base::base64_encode(digest,strlen((const char *)digest)); - sprintf(header_data,"Authorization: AWS %s:%s", this->AccessKey, - digest_str.c_str()); - chunk = curl_slist_append(chunk, header_data); + chunk = Cloud::BuildHeaderAmzS3v2( strUrl.c_str(), this->TcpPort, this->AccessKey, RequestData); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, chunk); - sprintf(header_data,"%s:%s/%s/", this->Url,this->TcpPort, + + // Lets build the Url for our Curl call + + sprintf(Url,"%s:%s/%s/", this->Url,this->TcpPort, this->Bucket); - curl_easy_setopt(curl, CURLOPT_URL, header_data); + curl_easy_setopt(curl, CURLOPT_URL, Url); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, CurlWrite_CallbackFunc_StdString); curl_easy_setopt(curl, CURLOPT_WRITEDATA, &s); // curl read a file not a memory buffer (it shall be able to do it) @@ -344,7 +366,8 @@ Cloud::CloudWriter::CloudWriter(const char* Url, const char* AccessKey, const ch fprintf(stderr, "curl_easy_perform() failed: %s\n", curl_easy_strerror(res)); curl_easy_cleanup(curl); - + free( RequestData); + // Let's dump temporarly for debug purpose of s3v4 implementation std::stringstream input(s); @@ -458,11 +481,7 @@ Cloud::CloudReader::~CloudReader() Cloud::CloudReader::CloudReader(const char* Url, const char* AccessKey, const char* SecretKey, const char* TcpPort, const char* Bucket) { - struct timeval tv; - struct tm *tm; - char date_formatted[256]; - char StringToSign[1024]; - unsigned char *digest; + struct Cloud::AmzData *RequestData; CURL *curl; CURLcode res; @@ -475,58 +494,32 @@ Cloud::CloudReader::CloudReader(const char* Url, const char* AccessKey, const ch this->TcpPort=TcpPort; this->Bucket=Bucket; - // Amazon S3 and Swift require the timezone to be define to - // GMT. As to simplify the conversion this is performed through the TZ - // environment variable and a call to localtime as to convert output of gettimeofday -#if defined(FC_OS_WIN32) - _putenv("TZ=GMT"); -#else - setenv("TZ","GMT",1); -#endif + char path[1024]; + sprintf(path,"/%s/", this->Bucket); + RequestData = Cloud::ComputeDigestAmzS3v2("GET", "application/xml", path, this->SecretKey); - // We must get the directory content - - -#if defined(FC_OS_WIN32) -#else - gettimeofday(&tv,NULL); - tm = localtime(&tv.tv_sec); - strftime(date_formatted,256,"%a, %d %b %Y %T %z", tm); -#endif - - sprintf(StringToSign,"GET\n\napplication/xml\n%s\n/%s/", date_formatted, this->Bucket); - - // We have to use HMAC encoding and SHA1 - digest=HMAC(EVP_sha1(),this->SecretKey,strlen(this->SecretKey), - (const unsigned char *)&StringToSign,strlen(StringToSign),NULL,NULL); // Let's build the Header and call to curl curl_global_init(CURL_GLOBAL_ALL); curl = curl_easy_init(); if ( curl ) { + // Let's build our own header struct curl_slist *chunk = NULL; - char header_data[1024]; - // Let's build our own header - std::string strUrl(this->Url); + char Url[256]; + std::string strUrl(this->Url); eraseSubStr(strUrl,"http://"); eraseSubStr(strUrl,"https://"); - sprintf(header_data,"Host: %s:%s", strUrl.c_str(), - this->TcpPort); - chunk = curl_slist_append(chunk, header_data); - sprintf(header_data,"Date: %s", date_formatted); - chunk = curl_slist_append(chunk, header_data); - chunk = curl_slist_append(chunk, "Content-Type: application/xml"); - std::string digest_str; - digest_str=Base::base64_encode(digest,strlen((const char *)digest)); - sprintf(header_data,"Authorization: AWS %s:%s", this->AccessKey, - digest_str.c_str()); - chunk = curl_slist_append(chunk, header_data); + chunk = Cloud::BuildHeaderAmzS3v2( strUrl.c_str(), this->TcpPort, this->AccessKey, RequestData); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, chunk); - sprintf(header_data,"%s:%s/%s/", this->Url,this->TcpPort, + + sprintf(Url,"%s:%s/%s/", this->Url,this->TcpPort, this->Bucket); - curl_easy_setopt(curl, CURLOPT_URL, header_data); + curl_easy_setopt(curl, CURLOPT_URL, Url); + + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, CurlWrite_CallbackFunc_StdString); curl_easy_setopt(curl, CURLOPT_WRITEDATA, &s); // curl read a file not a memory buffer (it shall be able to do it) @@ -564,31 +557,16 @@ Cloud::CloudReader::CloudReader(const char* Url, const char* AccessKey, const ch void Cloud::CloudReader::DownloadFile(Cloud::CloudReader::FileEntry *entry) { - struct timeval tv; - struct tm *tm; - char date_formatted[256]; - char StringToSign[1024]; - unsigned char *digest; + struct Cloud::AmzData *RequestData; CURL *curl; CURLcode res; std::string s; // We must get the directory content - - -#if defined(FC_OS_WIN32) -#else - gettimeofday(&tv,NULL); - tm = localtime(&tv.tv_sec); - strftime(date_formatted,256,"%a, %d %b %Y %T %z", tm); -#endif - - // CHANGEME - sprintf(StringToSign,"GET\n\napplication/octet-stream\n%s\n/%s/%s", date_formatted, this->Bucket, entry->FileName); - // We have to use HMAC encoding and SHA1 - digest=HMAC(EVP_sha1(),this->SecretKey,strlen(this->SecretKey), - (const unsigned char *)&StringToSign,strlen(StringToSign),NULL,NULL); + char path[1024]; + sprintf(path, "/%s/%s", this->Bucket, entry->FileName); + RequestData = Cloud::ComputeDigestAmzS3v2("GET", "application/octet-stream", path, this->SecretKey); // Let's build the Header and call to curl curl_global_init(CURL_GLOBAL_ALL); @@ -596,26 +574,20 @@ void Cloud::CloudReader::DownloadFile(Cloud::CloudReader::FileEntry *entry) if ( curl ) { struct curl_slist *chunk = NULL; - char header_data[1024]; + char Url[256]; // Let's build our own header std::string strUrl(this->Url); eraseSubStr(strUrl,"http://"); eraseSubStr(strUrl,"https://"); - sprintf(header_data,"Host: %s:%s", strUrl.c_str(), - this->TcpPort); - chunk = curl_slist_append(chunk, header_data); - sprintf(header_data,"Date: %s", date_formatted); - chunk = curl_slist_append(chunk, header_data); - chunk = curl_slist_append(chunk, "Content-Type: application/octet-stream"); - std::string digest_str; - digest_str=Base::base64_encode(digest,strlen((const char *)digest)); - sprintf(header_data,"Authorization: AWS %s:%s", this->AccessKey, - digest_str.c_str()); - chunk = curl_slist_append(chunk, header_data); + + chunk = Cloud::BuildHeaderAmzS3v2( strUrl.c_str(), this->TcpPort, this->AccessKey, RequestData); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, chunk); - sprintf(header_data,"%s:%s/%s/%s", this->Url,this->TcpPort, - this->Bucket,entry->FileName); - curl_easy_setopt(curl, CURLOPT_URL, header_data); + + sprintf(Url,"%s:%s/%s/%s", this->Url,this->TcpPort, + this->Bucket, entry->FileName); + curl_easy_setopt(curl, CURLOPT_URL, Url); + // curl_easy_setopt(curl, CURLOPT_VERBOSE, 1); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, CurlWrite_CallbackFunc_StdString); curl_easy_setopt(curl, CURLOPT_WRITEDATA, &s); @@ -696,31 +668,14 @@ bool Cloud::CloudWriter::shouldWrite(const std::string& , const Base::Persistenc void Cloud::CloudWriter::pushCloud(const char *FileName, const char *data, long size) { - struct timeval tv; - struct tm *tm; - char date_formatted[256]; - char StringToSign[1024]; - unsigned char *digest; - // char my_data[1024]="a que coucou"; - + struct Cloud::AmzData *RequestData; CURL *curl; CURLcode res; + struct data_buffer curl_buffer; - struct data_buffer curl_buffer; - -#if defined(FC_OS_WIN32) -#else - gettimeofday(&tv,NULL); - tm = localtime(&tv.tv_sec); - strftime(date_formatted,256,"%a, %d %b %Y %T %z", tm); -#endif - - // CHANGEME - sprintf(StringToSign,"PUT\n\napplication/octet-stream\n%s\n/%s/%s", date_formatted, this->Bucket, FileName); - - // We have to use HMAC encoding and SHA1 - digest=HMAC(EVP_sha1(),this->SecretKey,strlen(this->SecretKey), - (const unsigned char *)&StringToSign,strlen(StringToSign),NULL,NULL); + char path[1024]; + sprintf(path, "/%s/%s", this->Bucket, FileName); + RequestData = Cloud::ComputeDigestAmzS3v2("PUT", "application/octet-stream", path, this->SecretKey); // Let's build the Header and call to curl curl_global_init(CURL_GLOBAL_ALL); @@ -728,28 +683,26 @@ void Cloud::CloudWriter::pushCloud(const char *FileName, const char *data, long if ( curl ) { struct curl_slist *chunk = NULL; - char header_data[1024]; + char Url[256]; // Let's build our own header std::string strUrl(this->Url); eraseSubStr(strUrl,"http://"); eraseSubStr(strUrl,"https://"); - - sprintf(header_data,"Host: %s:%s", strUrl.c_str(), - this->TcpPort); - chunk = curl_slist_append(chunk, header_data); - sprintf(header_data,"Date: %s", date_formatted); - chunk = curl_slist_append(chunk, header_data); - chunk = curl_slist_append(chunk, "Content-Type: application/octet-stream"); - std::string digest_str; - digest_str=Base::base64_encode(digest,strlen((const char *)digest)); - sprintf(header_data,"Authorization: AWS %s:%s", this->AccessKey, - digest_str.c_str()); - chunk = curl_slist_append(chunk, header_data); + + + chunk = Cloud::BuildHeaderAmzS3v2( strUrl.c_str(), this->TcpPort, this->AccessKey, RequestData); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, chunk); + + // Lets build the Url for our Curl call + + sprintf(Url,"%s:%s/%s/", this->Url,this->TcpPort, + this->Bucket); + curl_easy_setopt(curl, CURLOPT_URL, Url); + + // curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L); - sprintf(header_data,"%s:%s/%s/%s", this->Url,this->TcpPort, - this->Bucket,FileName); - curl_easy_setopt(curl, CURLOPT_URL, header_data); + curl_easy_setopt(curl, CURLOPT_UPLOAD, 1L); curl_easy_setopt(curl, CURLOPT_PUT, 1L); // curl read a file not a memory buffer (it shall be able to do it) diff --git a/src/Mod/Cloud/App/AppCloud.h b/src/Mod/Cloud/App/AppCloud.h index 1650d73425..9ba2a55e57 100644 --- a/src/Mod/Cloud/App/AppCloud.h +++ b/src/Mod/Cloud/App/AppCloud.h @@ -47,8 +47,17 @@ XERCES_CPP_NAMESPACE_END namespace Cloud { +struct AmzData { + std::string digest; + char dateFormatted[256]; + char ContentType[256]; + char Host[256]; +}; + void eraseSubStr(std::string & Str, const std::string & toErase); size_t CurlWrite_CallbackFunc_StdString(void *contents, size_t size, size_t nmemb, std::string *s); +struct AmzData *ComputeDigestAmzS3v2(char *operation, char *data_type, const char *target, const char *Secret); +struct curl_slist *BuildHeaderAmzS3v2(const char *Url, const char *TcpPort, const char *PublicKey, struct AmzData *Data); class CloudAppExport CloudReader {