33#include <zypp-curl/parser/MetaLinkParser>
36#include <zypp-curl/auth/CurlAuthData>
43#undef CURLVERSION_AT_LEAST
44#define CURLVERSION_AT_LEAST(M,N,O) LIBCURL_VERSION_NUM >= ((((M)<<8)+(N))<<8)+(O)
187 void adddnsfd( std::vector<GPollFD> &waitFds );
188 void dnsevent(
const std::vector<GPollFD> &waitFds );
219 size_t writefunction (
char *ptr, std::optional<off_t> offset,
size_t bytes )
override;
221 bool beginRange ( off_t range, std::string &cancelReason )
override;
222 bool finishedRange ( off_t range,
bool validated, std::string &cancelReason )
override;
235 Url baseurl, CURLM *multi, FILE *fp,
244 void run(std::vector<Url> &urllist);
266 std::list< std::unique_ptr<multifetchworker> >
_workers;
319#if _POSIX_C_SOURCE >= 199309L
321 if ( clock_gettime( CLOCK_MONOTONIC, &ts) )
323 return ts.tv_sec + ts.tv_nsec / 1000000000.;
326 if (gettimeofday(&tv, NULL))
328 return tv.tv_sec + tv.tv_usec / 1000000.;
336 return bytes ? 0 : 1;
358 const auto &blk =
_blocks[*currRange];
359 off_t seekTo = blk.start + blk.bytesWritten;
364 return bytes ? 0 : 1;
376 const auto &currRangeState = stripeDesc.blockStates[stripeRangeOff];
379 cancelReason =
"Cancelled because stripe block is already finalized";
381 WAR <<
"#" <<
_workerno <<
": trying to start a range ("<<stripeRangeOff<<
"["<<
_blocks[workerRangeOff].start <<
" : "<<
_blocks[workerRangeOff].len<<
"]) that was already finalized, cancelling. Stealing was: " <<
_request->
_stealing << endl;
392 const auto &currRangeState = stripeDesc.blockStates[stripeRangeOff];
396 cancelReason =
"Block failed to validate";
423 if (l > 9 && !strncasecmp(p,
"Location:", 9)) {
424 std::string line(p + 9, l - 9);
425 if (line[l - 10] ==
'\r')
426 line.erase(l - 10, 1);
427 XXX <<
"#" <<
_workerno <<
": redirecting to" << line << endl;
433 XXX <<
"#" <<
_workerno <<
": filesize mismatch" << endl;
445, _maxspeed( request._maxspeed )
446, _request ( &request )
452 XXX <<
"reused worker from pool" << endl;
472 curl_easy_cleanup(
_curl);
478 curl_easy_setopt(
_curl, CURLOPT_PRIVATE,
this);
491 if (use_auth.empty())
492 use_auth =
"digest,basic";
494 if( auth != CURLAUTH_NONE)
496 XXX <<
"#" <<
_workerno <<
": Enabling HTTP authentication methods: " << use_auth
497 <<
" (CURLOPT_HTTPAUTH=" << auth <<
")" << std::endl;
498 curl_easy_setopt(
_curl, CURLOPT_HTTPAUTH, auth);
513#if CURLVERSION_AT_LEAST(7,15,5)
514 curl_easy_setopt(
_curl, CURLOPT_MAX_RECV_SPEED_LARGE, (curl_off_t)0);
516 curl_easy_setopt(
_curl, CURLOPT_PRIVATE, (
void *)0);
517 curl_easy_setopt(
_curl, CURLOPT_WRITEFUNCTION, (
void *)0);
518 curl_easy_setopt(
_curl, CURLOPT_WRITEDATA, (
void *)0);
519 curl_easy_setopt(
_curl, CURLOPT_HEADERFUNCTION, (
void *)0);
520 curl_easy_setopt(
_curl, CURLOPT_HEADERDATA, (
void *)0);
524 curl_easy_cleanup(
_curl);
531 while (waitpid(
_pid, &status, 0) == -1)
548 const char *s = getenv(name.c_str());
549 return s && *s ? true :
false;
565 if (inet_pton(AF_INET, host.c_str(), addrbuf) == 1)
567 if (inet_pton(AF_INET6, host.c_str(), addrbuf) == 1)
578 if (schemeproxy !=
"http_proxy")
580 std::transform(schemeproxy.begin(), schemeproxy.end(), schemeproxy.begin(), ::toupper);
585 XXX <<
"checking DNS lookup of " << host << endl;
594 if (
_pid == pid_t(-1))
607 struct addrinfo *ai =
nullptr, aihints;
608 memset(&aihints, 0,
sizeof(aihints));
609 aihints.ai_family = PF_UNSPEC;
610 int tstsock = socket(PF_INET6, SOCK_DGRAM | SOCK_CLOEXEC, 0);
612 aihints.ai_family = PF_INET;
615 aihints.ai_socktype = SOCK_STREAM;
616 aihints.ai_flags = AI_CANONNAME;
619 alarm(connecttimeout);
620 signal(SIGALRM, SIG_DFL);
621 if (getaddrinfo(host.c_str(), NULL, &aihints, &ai))
639 .events = G_IO_IN | G_IO_HUP | G_IO_ERR,
647 bool hasEvent = std::any_of( waitFds.begin (), waitFds.end(),[
this](
const GPollFD &waitfd ){
648 return ( waitfd.fd == _dnspipe && waitfd.revents != 0 );
654 while (waitpid(
_pid, &status, 0) == -1)
665 if (!WIFEXITED(status))
672 int exitcode = WEXITSTATUS(status);
673 XXX <<
"#" <<
_workerno <<
": DNS lookup returned " << exitcode << endl;
691 auto &blk =
_blocks[workerRangeIdx];
707 size_t cnt = l >
sizeof(buf) ?
sizeof(buf) : l;
717 blk._digest = std::move(newDig);
732 std::optional<zypp::Digest> digest;
733 std::optional<size_t> relDigLen;
734 std::optional<size_t> blkSumPad;
739 relDigLen = sum.size( );
745 return MultiByteHandler::Range::make(
751 std::move(relDigLen),
752 std::move(blkSumPad) );
759 XXX <<
"start stealing!" << endl;
773 if (worker->
_pass == -1)
840 XXX <<
"#" <<
_workerno <<
": going to sleep for " << sl * 1000 <<
" ms" << endl;
897 for ( uint i = 0; i < stripeDesc.blocks.size(); i++ ) {
917 DBG <<
"#" <<
_workerno <<
"Done adding blocks to download, going to download: " <<
_blocks.size() <<
" nr of block with " <<
_datasize <<
" nr of bytes" << std::endl;
928 bool hadRangeFail =
_multiByteHandler->lastError() == MultiByteHandler::Code::RangeFail;
934 if ( hadRangeFail ) {
941 curl_easy_reset(
_curl );
979 :
internal::CurlPollHelper::CurlPoll{ multi }
981 , _filename(
std::move(filename))
982 , _baseurl(
std::move(baseurl))
985 , _blklist(
std::move(blklist))
986 , _filesize(filesize)
988 , _timeout(context->_settings.timeout())
989 , _connect_timeout(context->_settings.connectTimeout())
990 , _maxspeed(context->_settings.maxDownloadSpeed())
991 , _maxworkers(context->_settings.maxConcurrentConnections())
1018 currStripeSize += blk.
size;
1033 std::vector<Url>::iterator urliter = urllist.begin();
1039 if (mcode != CURLM_OK)
1045 std::vector<GPollFD> waitFds;
1050 XXX <<
"finished!" << endl;
1057 _workers.push_back(std::make_unique<multifetchworker>(workerno++, *
this, *urliter));
1074 WAR <<
"No more active workers!" << endl;
1076 for (
auto workeriter =
_workers.begin(); workeriter !=
_workers.end(); ++workeriter)
1086 for (
auto workeriter =
_workers.begin(); workeriter !=
_workers.end(); ++workeriter)
1087 (*workeriter)->adddnsfd( waitFds );
1094 for (
auto workeriter =
_workers.begin(); workeriter !=
_workers.end(); ++workeriter) {
1108 timeoutMs = sl * 1000;
1112 timeoutMs = std::min<long>( timeoutMs, _curlHelper.
timeout_ms.value() );
1114 dnsFdCount = waitFds.size();
1115 waitFds.insert( waitFds.end(), _curlHelper.
socks.begin(), _curlHelper.
socks.end() );
1121 for (
auto workeriter =
_workers.begin(); workeriter !=
_workers.end(); ++workeriter)
1126 (*workeriter)->
dnsevent( waitFds );
1136 if (mcode != CURLM_OK)
1140 if (mcode != CURLM_OK)
1160 for (
auto workeriter =
_workers.begin(); workeriter !=
_workers.end(); ++workeriter)
1169 XXX <<
"#" << worker->
_workerno <<
": sleep done, wake up" << endl;
1177 CURLMsg *msg =
nullptr;
1179 while ((msg = curl_multi_info_read(
_multi, &nqueue)) != 0)
1181 if (msg->msg != CURLMSG_DONE)
1183 CURL *easy = msg->easy_handle;
1184 CURLcode cc = msg->data.result;
1187 if (curl_easy_getinfo(easy, CURLINFO_PRIVATE, &worker) != CURLE_OK)
1198 curl_multi_remove_handle(
_multi, easy);
1200 const auto &setWorkerBroken = [&](
const std::string &
str = {} ){
1202 if ( !
str.empty () )
1213 WAR <<
"#" << worker->
_workerno <<
": has no multibyte handler, this is a bug" << endl;
1214 setWorkerBroken(
"Multibyte handler error");
1223 WAR <<
"#" << worker->
_workerno <<
": still has work to do or can recover from a error, continuing the job!" << endl;
1234 if ( cc != CURLE_OK ) {
1240 WAR <<
"#" << worker->
_workerno <<
": failed, but was set to discard, reusing for new requests" << endl;
1259 bool done = std::all_of( wrkerStripe.blockStates.begin(), wrkerStripe.blockStates.begin(), [](
const Stripe::RState s ) { return s == Stripe::FINALIZED; } );
1262 std::for_each( wrkerStripe.blockStates.begin(), wrkerStripe.blockStates.begin(), [](
Stripe::RState &s ) {
1263 if ( s != Stripe::FINALIZED)
1264 s = Stripe::PENDING;
1275 int maxworkerno = 0;
1277 for (
auto workeriter =
_workers.begin(); workeriter !=
_workers.end(); ++workeriter)
1292 double ratio = worker->
_avgspeed / maxavg;
1295 ratio = ratio * ratio;
1298 XXX <<
"#" << worker->
_workerno <<
": too slow ("<< ratio <<
", " << worker->
_avgspeed <<
", #" << maxworkerno <<
": " << maxavg <<
"), going to sleep for " << ratio * 1000 <<
" ms" << endl;
1319#if CURLVERSION_AT_LEAST(7,15,5)
1320 curl_easy_setopt(worker->
_curl, CURLOPT_MAX_RECV_SPEED_LARGE, (curl_off_t)(avg));
1352 WAR <<
"overall result" << endl;
1353 for (
auto workeriter =
_workers.begin(); workeriter !=
_workers.end(); ++workeriter)
1373 MIL <<
"MediaMultiCurl::MediaMultiCurl(" << url_r <<
", " << attach_point_hint_r <<
")" << endl;
1387 curl_multi_cleanup(
_multi);
1390 std::map<std::string, CURL *>::iterator it;
1393 CURL *easy = it->second;
1396 curl_easy_cleanup(easy);
1412 for (; sl; sl = sl->next)
1430 if ( curl_easy_getinfo(
_curl, CURLINFO_PRIVATE, &fp ) != CURLE_OK || !fp )
1432 if ( ftell( fp ) == 0 )
1437 long httpReturnCode = 0;
1438 if (curl_easy_getinfo(
_curl, CURLINFO_RESPONSE_CODE, &httpReturnCode ) != CURLE_OK || httpReturnCode == 0)
1442 bool ismetalink =
false;
1443 if (curl_easy_getinfo(
_curl, CURLINFO_CONTENT_TYPE, &ptr) == CURLE_OK && ptr)
1445 std::string ct = std::string(ptr);
1446 if (ct.find(
"application/x-zsync") == 0 || ct.find(
"application/metalink+xml") == 0 || ct.find(
"application/metalink4+xml") == 0)
1449 if (!ismetalink && dlnow < 256)
1458 DBG <<
"looks_like_meta_file: " << ismetalink << endl;
1475 if( assert_dir( dest.
dirname() ) )
1477 DBG <<
"assert_dir " << dest.
dirname() <<
" failed" << endl;
1487 ERR <<
"out of memory for temp file name" << endl;
1491 AutoFD tmp_fd { ::mkostemp( buf, O_CLOEXEC ) };
1494 ERR <<
"mkstemp failed for file '" << destNew <<
"'" << endl;
1499 file = ::fdopen( tmp_fd,
"we" );
1502 ERR <<
"fopen failed for file '" << destNew <<
"'" << endl;
1505 tmp_fd.resetDispose();
1508 DBG <<
"dest: " << dest << endl;
1509 DBG <<
"temp: " << destNew << endl;
1514 curl_easy_setopt(
_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE);
1515 curl_easy_setopt(
_curl, CURLOPT_TIMEVALUE, (
long)
PathInfo(target).mtime());
1519 curl_easy_setopt(
_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1520 curl_easy_setopt(
_curl, CURLOPT_TIMEVALUE, 0L);
1526 curl_easy_setopt(
_curl, CURLOPT_PRIVATE, (*file) );
1533 curl_easy_setopt(
_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1534 curl_easy_setopt(
_curl, CURLOPT_TIMEVALUE, 0L);
1536 curl_easy_setopt(
_curl, CURLOPT_PRIVATE, (
void *)0);
1539 curl_easy_setopt(
_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1540 curl_easy_setopt(
_curl, CURLOPT_TIMEVALUE, 0L);
1542 curl_easy_setopt(
_curl, CURLOPT_PRIVATE, (
void *)0);
1543 long httpReturnCode = 0;
1544 CURLcode infoRet = curl_easy_getinfo(
_curl, CURLINFO_RESPONSE_CODE, &httpReturnCode);
1545 if (infoRet == CURLE_OK)
1548 if ( httpReturnCode == 304
1551 DBG <<
"not modified: " <<
PathInfo(dest) << endl;
1557 WAR <<
"Could not get the response code." << endl;
1563 if (curl_easy_getinfo(
_curl, CURLINFO_CONTENT_TYPE, &ptr) == CURLE_OK && ptr)
1565 std::string ct = std::string(ptr);
1566 if (ct.find(
"application/x-zsync") == 0 )
1568 else if (ct.find(
"application/metalink+xml") == 0 || ct.find(
"application/metalink4+xml") == 0)
1582 bool userabort =
false;
1588 std::vector<Url> urls;
1591 parser.
parse( destNew );
1608 XXX <<
"With no blocks" << std::endl;
1615 XXX <<
"No filesize in metalink file and no expected filesize, aborting multicurl." << std::endl;
1620 Disabling
this workaround
for now, since we now
do zip ranges into bigger requests
1631 file = fopen((*destNew).c_str(),
"w+e");
1636 XXX <<
"reusing blocks from file " << target << endl;
1642 XXX <<
"reusing blocks from file " << failedFile << endl;
1650 XXX <<
"reusing blocks from file " << df << endl;
1660 userabort = ex.
errstr() ==
"User abort";
1671 WAR<<
"Failed to multifetch file " << ex <<
" falling back to single Curl download!" << std::endl;
1672 if (
PathInfo(destNew).size() >= 63336)
1674 ::unlink(failedFile.
asString().c_str());
1681 file = fopen((*destNew).c_str(),
"w+e");
1693 ERR <<
"Failed to chmod file " << destNew << endl;
1700 ERR <<
"Fclose failed for file '" << destNew <<
"'" << endl;
1704 if ( rename( destNew, dest ) != 0 )
1706 ERR <<
"Rename failed" << endl;
1709 destNew.resetDispose();
1717 if (filesize == off_t(-1) && blklist.haveFilesize())
1718 filesize = blklist.getFilesize();
1719 if (!blklist.haveBlocks() && filesize != 0) {
1720 if ( filesize == -1 ) {
1725 MIL <<
"Generate blocklist, since there was none in the metalink file." << std::endl;
1730 while ( currOff < filesize ) {
1732 auto blksize = filesize - currOff ;
1733 if ( blksize > prefSize )
1736 blklist.addBlock( currOff, blksize );
1740 XXX <<
"Generated blocklist: " << std::endl << blklist << std::endl <<
" End blocklist " << std::endl;
1743 if (filesize == 0 || !blklist.numBlocks()) {
1752 _multi = curl_multi_init();
1758 std::vector<Url> myurllist;
1759 for (std::vector<Url>::iterator urliter = urllist->begin(); urliter != urllist->end(); ++urliter)
1763 std::string scheme = urliter->getScheme();
1764 if (scheme ==
"http" || scheme ==
"https" || scheme ==
"ftp" || scheme ==
"tftp")
1774 if (!myurllist.size())
1775 myurllist.push_back(baseurl);
1784 if (fseeko(fp, off_t(0), SEEK_SET))
1790 while ((l = fread(buf, 1,
sizeof(buf), fp)) > 0)
1798 return _dnsok.find(host) ==
_dnsok.end() ? false :
true;
1820 curl_easy_cleanup(oldeasy);
void resetDispose()
Set no dispose function.
Store and operate with byte count.
static const Unit MB
1000^2 Byte
SizeType blocks(ByteCount blocksize_r=K) const
Return number of blocks of size blocksize_r (default 1K).
static const Unit K
1024 Byte
std::string asString(unsigned field_width_r=0, unsigned unit_width_r=1) const
Auto selected Unit and precision.
Compute Message Digests (MD5, SHA1 etc)
bool update(const char *bytes, size_t len)
feed data into digest computation algorithm
Digest clone() const
Returns a clone of the current Digest and returns it.
Base class for Exception.
std::string getScheme() const
Returns the scheme name of the URL.
std::string asString() const
Returns a default string representation of the Url object.
std::string getHost(EEncoding eflag=zypp::url::E_DECODED) const
Returns the hostname or IP from the URL authority.
Pathname repoCachePath() const
Path where the caches are kept (/var/cache/zypp)
static ZConfig & instance()
Singleton ctor.
Wrapper class for stat/lstat.
Pathname extend(const std::string &r) const
Append string r to the last component of the path.
Pathname dirname() const
Return all but the last component od this path.
const std::string & asString() const
String representation.
Pathname absolutename() const
Return this path, adding a leading '/' if relative.
The CurlMultiPartHandler class.
zypp::callback::SendReport< zypp::KeyRingReport > _report
zypp::Url propagateQueryParams(zypp::Url url_r, const zypp::Url &template_r)
String related utilities and Regular expression matching.
mode_t applyUmaskTo(mode_t mode_r)
Modify mode_r according to the current umask ( mode_r & ~getUmask() ).
int hardlinkCopy(const Pathname &oldpath, const Pathname &newpath)
Create newpath as hardlink or copy of oldpath.
int unlink(const Pathname &path)
Like 'unlink'.
std::string numstring(char n, int w=0)
int zypp_poll(std::vector< GPollFD > &fds, int timeout)
Small wrapper around g_poll that additionally listens to the shutdown FD returned by ZYpp::shutdownSi...
Easy-to use interface to the ZYPP dependency resolver.
AutoDispose< const Pathname > ManagedFile
A Pathname plus associated cleanup code to be executed when path is no longer needed.
CURLMcode handleSocketActions(const std::vector< GPollFD > &actionsFds, int first=0)
std::vector< GPollFD > socks
std::optional< long > timeout_ms
AutoDispose<int> calling close
AutoDispose<FILE*> calling fclose
#define ZYPP_RETHROW(EXCPT)
Drops a logline and rethrows, updating the CodeLocation.
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.