13#include <zypp-core/zyppng/base/EventDispatcher>
15#include <zypp-core/zyppng/core/String>
18#include <zypp-curl/CurlConfig>
19#include <zypp-curl/auth/CurlAuthData>
20#include <zypp-media/MediaConfig>
31#include <boost/variant.hpp>
32#include <boost/variant/polymorphic_get.hpp>
38 static size_t nwr_headerCallback (
char *ptr,
size_t size,
size_t nmemb,
void *userdata ) {
42 NetworkRequestPrivate *that =
reinterpret_cast<NetworkRequestPrivate *
>( userdata );
43 return that->headerfunction( ptr, size * nmemb );
45 static size_t nwr_writeCallback (
char *ptr,
size_t size,
size_t nmemb,
void *userdata ) {
49 NetworkRequestPrivate *that =
reinterpret_cast<NetworkRequestPrivate *
>( userdata );
50 return that->writefunction( ptr, {}, size * nmemb );
54 template<
class T>
struct always_false : std::false_type {};
152 for (
const auto &[key, value] :
i->second ) {
154 "%s: %s", key.c_str(), value.c_str() )
160 locSet.addHeader(
"Pragma:");
191 if(
locSet.verifyPeerEnabled() ||
192 locSet.verifyHostEnabled() )
197 if( !
locSet.clientCertificatePath().empty() )
201 if( !
locSet.clientKeyPath().empty() )
206#ifdef CURLSSLOPT_ALLOW_BEAST
231 if (
locSet.userPassword().size() )
241 <<
" (CURLOPT_HTTPAUTH=" <<
auth <<
")" << std::endl;
246 if (
locSet.proxyEnabled() && !
locSet.proxy().empty() )
259 std::string proxyuserpwd =
locSet.proxyUserPassword();
261 if ( proxyuserpwd.empty() )
265 if (
curlconf.proxyuserpwd.empty() )
266 DBG <<
_easyHandle <<
" " <<
"Proxy: ~/.curlrc does not contain the proxy-user option" << std::endl;
269 proxyuserpwd =
curlconf.proxyuserpwd;
270 DBG <<
_easyHandle <<
" " <<
"Proxy: using proxy-user from ~/.curlrc" << std::endl;
278 if ( ! proxyuserpwd.empty() )
283#if CURLVERSION_AT_LEAST(7,19,4)
288 DBG <<
_easyHandle <<
" " <<
"Proxy: explicitly NOPROXY" << std::endl;
296 if (
locSet.minDownloadSpeed() != 0 )
303#if CURLVERSION_AT_LEAST(7,15,5)
304 if (
locSet.maxDownloadSpeed() != 0 )
315#if CURLVERSION_AT_LEAST(7,18,0)
322 for (
const auto &header :
locSet.headers() ) {
323 if ( !
z_func()->addRequestHeader( header.c_str() ) )
335 return ( elem1.start < elem2.start );
342 initState->_partialHelper = std::make_unique<CurlMultiPartHandler>(
353 errBuf =
"Request is in invalid state to call setupHandle";
357 if ( !
helper->prepare () ) {
377 DBG <<
_easyHandle <<
"Can only create output file in running mode" << std::endl;
381 if ( !
rmode->_outFile ) {
393 if ( !
rmode->_outFile ) {
407 if (
rmode->_partialHelper )
return rmode->_partialHelper->canRecover();
415 errBuf =
"NetworkRequestPrivate::prepareToContinue called in invalid state";
419 if (
rmode->_partialHelper &&
rmode->_partialHelper->hasMoreWork() ) {
426 if ( !
prepMode._partialHelper->prepareToContinue() ) {
445 errBuf =
"Request has no more work";
453 return std::any_of(
_requestedRanges.begin(),
_requestedRanges.end(), [](
const auto &range ){ return range._rangeState == CurlMultiPartHandler::Pending; });
460 MIL <<
_easyHandle <<
" " <<
"Continuing a previously started range batch." << std::endl;
468 if (
m._activityTimer ) {
480 if ( std::holds_alternative<running_t>(
_runningMode) ) {
482 if (
rmode._partialHelper )
483 rmode._partialHelper->finalize();
492 if ( std::holds_alternative<running_t>(
_runningMode) ) {
501 if ( !
rmode._partialHelper->verifyData() ){
511 constexpr size_t bufSize = 4096;
558 MIL_MEDIA <<
_easyHandle <<
" Request timeout interval: " <<
t.interval()<<
" remaining: " <<
t.remaining() << std::endl;
559 std::map<std::string, boost::any> extraInfo;
560 extraInfo.insert( {
"requestUrl",
_url } );
572 if ( std::holds_alternative<running_t>(
_runningMode ) ){
574 if (
rmode._activityTimer &&
rmode._activityTimer->isRunning() )
575 rmode._activityTimer->start();
585 if ( !std::holds_alternative<running_t>(
that->_runningMode) ){
586 DBG <<
that->_easyHandle <<
" " <<
"Curl progress callback was called in invalid state "<<
that->z_func()->state() << std::endl;
590 auto &
rmode = std::get<running_t>(
that->_runningMode );
593 that->resetActivityTimer();
595 rmode._isInCallback =
true;
600 rmode._isInCallback =
false;
617 hdr.remove_prefix( std::min(
hdr.find_first_not_of(
" \t\r\n"),
hdr.size() ) );
622 hdr = std::string_view();
629 const auto &
repSize =
rmode._partialHelper->reportedFileSize ();
635 if ( zypp::strv::hasPrefixCI(
hdr,
"HTTP/" ) ) {
644 }
else if ( zypp::strv::hasPrefixCI(
hdr,
"Location:" ) ) {
648 }
else if ( zypp::strv::hasPrefixCI(
hdr,
"Content-Length:") ) {
651 auto len = zypp::str::strtonum<typename zypp::ByteCount::SizeType>(
str.data() );
653 DBG <<
_easyHandle <<
" " <<
"Got Content-Length Header: " << len << std::endl;
685 "Unable to set output file pointer." );
688 rmode._currentFileOffset = *offset;
692 const auto &
repSize =
rmode._partialHelper->reportedFileSize ();
726 if ( !
rmode || !
rmode->_partialHelper || !
rmode->_partialHelper->hasError() )
730 if (
rmode->_cachedResult )
749 if (
d->_dispatcher )
750 d->_dispatcher->cancel( *
this,
"Request destroyed while still running" );
755 d_func()->_expectedFileSize = std::move( expectedFileSize );
763 d->_dispatcher->reschedule();
768 return d_func()->_priority;
778 return d_func()->_options;
796 d->_requestedRanges.push_back( std::move(range) );
797 auto &
rng =
d->_requestedRanges.back();
799 rng.bytesWritten = 0;
801 rng._digest->reset();
826 d->_requestedRanges.clear();
837 std::vector<Range> failed;
838 for (
auto &
r :
d->_requestedRanges ) {
840 failed.push_back(
r.clone() );
848 return d_func()->_requestedRanges;
853 return d_func()->_lastRedirect;
858 return d_func()->_easyHandle;
871 using FPSeconds = std::chrono::duration<double, std::chrono::seconds::period>;
875 target = std::chrono::duration_cast<std::chrono::microseconds>(
FPSeconds(val) );
893 if ( !std::holds_alternative<NetworkRequestPrivate::running_t>(
d->_runningMode) )
896 const auto &
rmode = std::get<NetworkRequestPrivate::running_t>(
d->_runningMode );
916 return d_func()->_targetFile;
924 d->_targetFile = path;
937 d->_fMode = std::move( mode );
944 return std::string(
ptr);
945 return std::string();
952 if constexpr (std::is_same_v<T, NetworkRequestPrivate::pending_t> || std::is_same_v<T, NetworkRequestPrivate::prepareNextRangeBatch_t> )
954 else if constexpr (std::is_same_v<T, NetworkRequestPrivate::running_t>
955 || std::is_same_v<T, NetworkRequestPrivate::finished_t>)
956 return arg._contentLenght;
958 static_assert(always_false<T>::value,
"Unhandled state type");
959 },
d_func()->_runningMode);
966 if constexpr (std::is_same_v<T, NetworkRequestPrivate::pending_t>)
968 else if constexpr (std::is_same_v<T, NetworkRequestPrivate::running_t>
969 || std::is_same_v<T, NetworkRequestPrivate::prepareNextRangeBatch_t>
970 || std::is_same_v<T, NetworkRequestPrivate::finished_t>)
971 return arg._downloaded;
973 static_assert(always_false<T>::value,
"Unhandled state type");
974 },
d_func()->_runningMode);
979 return d_func()->_settings;
984 return std::visit([
this](
auto&
arg) {
986 if constexpr (std::is_same_v<T, NetworkRequestPrivate::pending_t>)
988 else if constexpr (std::is_same_v<T, NetworkRequestPrivate::running_t> || std::is_same_v<T, NetworkRequestPrivate::prepareNextRangeBatch_t> )
990 else if constexpr (std::is_same_v<T, NetworkRequestPrivate::finished_t>) {
991 if ( std::get<NetworkRequestPrivate::finished_t>(
d_func()->_runningMode ).
_result.isError() )
997 static_assert(always_false<T>::value,
"Unhandled state type");
998 },
d_func()->_runningMode);
1006 return std::get<NetworkRequestPrivate::finished_t>(
d_func()->_runningMode)._result;
1012 return std::string();
1038 return d_func()->_sigStarted;
1043 return d_func()->_sigBytesDownloaded;
1048 return d_func()->_sigProgress;
1053 return d_func()->_sigFinished;
ZYppCommitResult & _result
Reference counted access to a Tp object calling a custom Dispose function when the last AutoDispose h...
void reset()
Reset to default Ctor values.
Store and operate with byte count.
static const Unit B
1 Byte
Compute Message Digests (MD5, SHA1 etc)
Base class for Exception.
std::string getScheme() const
Returns the scheme name of the URL.
std::string asString() const
Returns a default string representation of the Url object.
std::string getQueryParam(const std::string ¶m, EEncoding eflag=zypp::url::E_DECODED) const
Return the value for the specified query parameter.
std::string getHost(EEncoding eflag=zypp::url::E_DECODED) const
Returns the hostname or IP from the URL authority.
const std::string & asString() const
String representation.
The CurlMultiPartHandler class.
static zyppng::NetworkRequestError customError(NetworkRequestError::Type t, std::string &&errorMsg="", std::map< std::string, boost::any > &&extraInfo={})
The NetworkRequestError class Represents a error that occured in.
std::string nativeErrorString() const
bool isError() const
isError Will return true if this is a actual error
size_t headerfunction(char *ptr, size_t bytes) override
std::optional< FileVerifyInfo > _fileVerification
The digest for the full file.
enum zyppng::NetworkRequestPrivate::ProtocolMode _protocolMode
const std::string _currentCookieFile
void notifyErrorCodeChanged() override
zypp::Pathname _targetFile
void resetActivityTimer()
NetworkRequestDispatcher * _dispatcher
std::vector< NetworkRequest::Range > _requestedRanges
the requested ranges that need to be downloaded
size_t writefunction(char *ptr, std::optional< off_t > offset, size_t bytes) override
static int curlProgressCallback(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow)
Signal< void(NetworkRequest &req) _sigStarted)
std::string errorMessage() const
NetworkRequest::FileMode _fMode
std::variant< pending_t, running_t, prepareNextRangeBatch_t, finished_t > _runningMode
bool initialize(std::string &errBuf)
void onActivityTimeout(Timer &)
Signal< void(NetworkRequest &req, const NetworkRequestError &err) _sigFinished)
std::string _lastRedirect
to log/report redirections
NetworkRequest::Options _options
bool prepareToContinue(std::string &errBuf)
void setResult(NetworkRequestError &&err)
~NetworkRequestPrivate() override
std::array< char, CURL_ERROR_SIZE+1 > _errorBuf
std::unique_ptr< curl_slist, decltype(&curl_slist_free_all) _headers)
bool setupHandle(std::string &errBuf)
NetworkRequestPrivate(Url &&url, zypp::Pathname &&targetFile, NetworkRequest::FileMode fMode, NetworkRequest &p)
TransferSettings _settings
void setCurlOption(CURLoption opt, T data)
zypp::ByteCount _expectedFileSize
Signal< void(NetworkRequest &req, zypp::ByteCount count) _sigBytesDownloaded)
bool setExpectedFileChecksum(const zypp::CheckSum &expected)
zypp::ByteCount reportedByteCount() const
Returns the number of bytes that are reported from the backend as the full download size,...
const zypp::Pathname & targetFilePath() const
Returns the target filename path.
zypp::ByteCount downloadedByteCount() const
Returns the number of already downloaded bytes as reported by the backend.
void resetRequestRanges()
void setUrl(const Url &url)
This will change the URL of the request.
void setExpectedFileSize(zypp::ByteCount expectedFileSize)
void setPriority(Priority prio, bool triggerReschedule=true)
std::vector< char > peekData(off_t offset, size_t count) const
std::string contentType() const
Returns the content type as reported from the server.
void setFileOpenMode(FileMode mode)
Sets the file open mode to mode.
bool addRequestHeader(const std::string &header)
~NetworkRequest() override
void setOptions(Options opt)
FileMode fileOpenMode() const
Returns the currently configured file open mode.
bool hasError() const
Checks if there was a error with the request.
State state() const
Returns the current state the HttpDownloadRequest is in.
std::optional< Timings > timings() const
After the request is finished query the timings that were collected during download.
std::string extendedErrorString() const
In some cases, curl can provide extended error information collected at runtime.
Priority priority() const
NetworkRequestError error() const
Returns the last set Error.
void setTargetFilePath(const zypp::Pathname &path)
Changes the target file path of the download.
SignalProxy< void(NetworkRequest &req, const NetworkRequestError &err) sigFinished)()
Signals that the download finished.
void * nativeHandle() const
void addRequestRange(size_t start, size_t len=0, std::optional< zypp::Digest > &&digest={}, CheckSumBytes expectedChkSum=CheckSumBytes(), std::any userData=std::any(), std::optional< size_t > digestCompareLen={}, std::optional< size_t > chksumpad={})
SignalProxy< void(NetworkRequest &req, zypp::ByteCount count) sigBytesDownloaded)()
Signals that new data has been downloaded, this is only the payload and does not include control data...
std::vector< Range > failedRanges() const
const std::vector< Range > & requestedRanges() const
SignalProxy< void(NetworkRequest &req) sigStarted)()
Signals that the dispatcher dequeued the request and actually starts downloading data.
TransferSettings & transferSettings()
const std::string & lastRedirectInfo() const
SignalProxy< void(NetworkRequest &req, off_t dltotal, off_t dlnow, off_t ultotal, off_t ulnow) sigProgress)()
Signals if there was data read from the download.
The Timer class provides repetitive and single-shot timers.
SignalProxy< void(Timer &t) sigExpired)()
This signal is always emitted when the timer expires.
#define EXPLICITLY_NO_PROXY
std::string curlUnEscape(const std::string &text_r)
void setupZYPP_MEDIA_CURL_DEBUG(CURL *curl)
Setup CURLOPT_VERBOSE and CURLOPT_DEBUGFUNCTION according to env::ZYPP_MEDIA_CURL_DEBUG.
CURLcode setCurlRedirProtocols(CURL *curl)
typename decay< T >::type decay_t
String related utilities and Regular expression matching.
int ZYPP_MEDIA_CURL_IPRESOLVE()
4/6 to force IPv4/v6
int assert_file_mode(const Pathname &path, unsigned mode)
Like assert_file but enforce mode even if the file already exists.
constexpr bool always_false
std::vector< char > peek_data_fd(FILE *fd, off_t offset, size_t count)
std::string form(const char *format,...) __attribute__((format(printf
Printf style construction of std::string.
bool strToBool(const C_Str &str, bool default_r)
Parse str into a bool depending on the default value.
std::string trim(const std::string &s, const Trim trim_r)
Easy-to use interface to the ZYPP dependency resolver.
T trim(StrType &&s, const Trim trim_r)
std::string strerr_cxx(const int err=-1)
static Range make(size_t start, size_t len=0, std::optional< zypp::Digest > &&digest={}, CheckSumBytes &&expectedChkSum=CheckSumBytes(), std::any &&userData=std::any(), std::optional< size_t > digestCompareLen={}, std::optional< size_t > _dataBlockPadding={})
running_t(pending_t &&prevState)
#define ZYPP_CAUGHT(EXCPT)
Drops a logline telling the Exception was caught (in order to handle it).
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
#define ZYPP_IMPL_PRIVATE(Class)