C++ 下载 Minio 文件
下载 aws-sdk-cpp
- git clone --recurse-submodules GitHub - aws/aws-sdk-cpp: AWS SDK for C++
麒麟上编译AWS
cmake ../aws-sdk-cpp -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH=/home/ht706/aws/tcs/install -DCMAKE_INSTALL_PREFIX=/home/ht706/aws/tcs/install -DLibCrypto_INCLUDE_DIR=/home/ht706/aws/tcs/install/include -DBUILD_SHARED_LIBS=on -DNO_WGNU=OFF -DBUILD_ONLY="s3" -DENABLE_TESTING=OFF
多线程下载
// 只要设置HTTP请求头的Range属性, 就可以实现从指定位置开始下载
/*
表示头500个字节:Range: bytes=0-499
表示第二个500字节:Range: bytes=500-999
表示最后500个字节:Range: bytes=-500
表示500字节以后的范围:Range: bytes=500-
*/
std::vector<MultiDownloadCtx> downloadParts;
for (size_t i = 0; i < partNum; i++)
{
std::stringstream rangeStream;
rangeStream << "bytes=" << i * DOWNLOAD_DATA_SIZE_THRESHOLD << '-' <<
((i == partNum - 1) ? filesize - 1
: ((i + 1) * DOWNLOAD_DATA_SIZE_THRESHOLD - 1));
downloadParts.emplace_back();
downloadParts.back().m_partNumber = i;
downloadParts.back().getObjectRequest
.WithBucket(fromBucket)
.WithKey(objectFile)
.WithRange(rangeStream.str());
}
for (int i = 0; i < downloadParts.size(); i++)
{
MultiDownloadCtx& ctx = downloadParts[i];
m_threadPool->AddJob([&]()-> void {
objectFile;
int partSize = ctx.m_partNumber == partNum - 1 ?
(int)(filesize -(size_t)ctx.m_partNumber * DOWNLOAD_DATA_SIZE_THRESHOLD)
: DOWNLOAD_DATA_SIZE_THRESHOLD;
ByteArrayStreamBuf* buf = appender.GetBuffer(m_threadPool->GetThreadIdx());
LogDEBUG("Start downloading part %d, range: %s, part size: %d\n",
ctx.m_partNumber, ctx.getObjectRequest.GetRange().c_str(),
partSize);
// Clear the stream as aws sdk expects a clean stream returned from
// response stream factory each time when retry on network errors
ctx.getObjectRequest.SetResponseStreamFactory([&buf]()-> Aws::IOStream*
{ buf->reset(); return Aws::New<Aws::IOStream>("SR_MULTI_PART_DOWNLOAD", buf); });
Aws::S3::Model::GetObjectOutcome outcome = s3_client->GetObject(
ctx.getObjectRequest);
buf->updateSize(partSize);
if (outcome.IsSuccess())
{
LogDEBUG("Download part %d succeed, download size: %d",
ctx.m_partNumber, partSize);
ctx.m_outcome = RemoteStorageRequestOutcome::SUCCESS;
appender.WritePartToOutputStream(m_threadPool->GetThreadIdx(),ctx.m_partNumber);
doDownloadStatus(objectFile, filesize, partSize);
}
else
{
ctx.m_outcome = handleError(outcome.GetError());
}
});
}
m_threadPool->WaitAll();
运行现象
视频
交流qq :690759587 qq群:691104042
本文内容由网友自发贡献,版权归原作者所有,本站不承担相应法律责任。如您发现有涉嫌抄袭侵权的内容,请联系:hwhale#tublm.com(使用前将#替换为@)