ref: refs/heads/master commit 21b807b024cbe0d1bcc41a9ceb5c27c1eadf7eaf Author: Duncan Mac-Vicar P Date: Sat Feb 21 18:28:12 2009 +0100 finally I saw the light, MediaAria2 now inherits MediaCurl, therefore has access to curl methods, or any functionality which aria does not provides. Implemented the following options with aria: download.max_concurrent_connections (default 2) download.min_download_speed (default no limit) download.max_download_speed (default no limit) download.max_silent_tries (default 5) Lot of duplicated code removed, and no longer require command line curl. Now lets see if we can remove duplicated code in curl iteself and also implement range support. --- zypp.conf | 25 ++++ zypp/ZConfig.cc | 38 ++++++- zypp/ZConfig.h | 23 ++++ zypp/media/MediaAccess.cc | 11 +-- zypp/media/MediaAccess.h | 2 +- zypp/media/MediaAria2c.cc | 269 +++++---------------------------------- zypp/media/MediaAria2c.h | 3 +- zypp/media/TransferSettings.cc | 50 ++++++++ zypp/media/TransferSettings.h | 50 +++++++- 9 files changed, 219 insertions(+), 252 deletions(-) diff --git a/zypp.conf b/zypp.conf index 0e089e0..8812e8f 100644 --- a/zypp.conf +++ b/zypp.conf @@ -120,6 +120,31 @@ ## # repo.refresh.delay = 10 +## +## Maximum number of concurrent connections to use per transfer +## This setting is only used if more than one is possible +## Setting it to a reasonable number avoids flooding servers +## +# download.max_concurrent_connections = 2 + +## +## Sets the minimum download speed (bytes per second) +## until the connection is dropped +## This can be useful to prevent security attacks on hosts by +## providing updates at very low speeds. +## +## 0 means no limit +## +# download.min_download_speed = 0 + +## Maximum download speed (bytes per second) +## 0 means no limit +# download.max_download_speed = 0 + +## Number of tries per download which will be +## done without user interaction +## 0 means no limit (use with caution) +# download.max_silent_tries = 5 ## ## Whether to consider using a .delta.rpm when downloading a package diff --git a/zypp/ZConfig.cc b/zypp/ZConfig.cc index 816b5fc..33b13f5 100644 --- a/zypp/ZConfig.cc +++ b/zypp/ZConfig.cc @@ -154,6 +154,10 @@ namespace zypp , repo_refresh_delay ( 10 ) , download_use_deltarpm ( true ) , download_use_deltarpm_always ( false ) + , download_max_concurrent_connections(2) + , download_min_download_speed(0) + , download_max_download_speed(0) + , download_max_silent_tries(5) , solver_onlyRequires ( false ) , apply_locks_file ( true ) @@ -240,6 +244,22 @@ namespace zypp { download_use_deltarpm_always = str::strToBool( value, download_use_deltarpm_always ); } + else if ( entry == "download.max_concurrent_connections" ) + { + str::strtonum(value, download_max_concurrent_connections); + } + else if ( entry == "download.min_download_speed" ) + { + str::strtonum(value, download_min_download_speed); + } + else if ( entry == "download.max_download_speed" ) + { + str::strtonum(value, download_max_download_speed); + } + else if ( entry == "download.max_silent_tries" ) + { + str::strtonum(value, download_max_silent_tries); + } else if ( entry == "vendordir" ) { cfg_vendor_path = Pathname(value); @@ -357,6 +377,11 @@ namespace zypp bool download_use_deltarpm; bool download_use_deltarpm_always; + int download_max_concurrent_connections; + int download_min_download_speed; + int download_max_download_speed; + int download_max_silent_tries; + bool solver_onlyRequires; Pathname solver_checkSystemFile; @@ -540,13 +565,24 @@ namespace zypp bool ZConfig::download_use_deltarpm_always() const { return download_use_deltarpm() && _pimpl->download_use_deltarpm_always; } + long ZConfig::download_max_concurrent_connections() const + { return _pimpl->download_max_concurrent_connections; } + + long ZConfig::download_min_download_speed() const + { return _pimpl->download_min_download_speed; } + + long ZConfig::download_max_download_speed() const + { return _pimpl->download_max_download_speed; } + + long ZConfig::download_max_silent_tries() const + { return _pimpl->download_max_silent_tries; } + bool ZConfig::solver_onlyRequires() const { return _pimpl->solver_onlyRequires; } Pathname ZConfig::solver_checkSystemFile() const { return _pimpl->solver_checkSystemFile; } - std::set<IdString> ZConfig::multiversion() const { return _pimpl->multiversion; } diff --git a/zypp/ZConfig.h b/zypp/ZConfig.h index 9052ae9..9b73393 100644 --- a/zypp/ZConfig.h +++ b/zypp/ZConfig.h @@ -56,6 +56,7 @@ namespace zypp class ZConfig : private base::NonCopyable { public: + /** Singleton ctor */ static ZConfig & instance(); @@ -153,6 +154,28 @@ namespace zypp */ unsigned repo_refresh_delay() const; + /** + * Maximum number of concurrent connections for a single transfer + */ + long download_max_concurrent_connections() const; + + /** + * Minimum download speed (bytes per second) + * until the connection is dropped + */ + long download_min_download_speed() const; + + /** + * Maximum download speed (bytes per second) + */ + long download_max_download_speed() const; + + /** + * Maximum silent tries + */ + long download_max_silent_tries() const; + + /** Whether to consider using a deltarpm when downloading a package. * Config option <tt>download.use_deltarpm (true)</tt> */ diff --git a/zypp/media/MediaAccess.cc b/zypp/media/MediaAccess.cc index 1030222..e429727 100644 --- a/zypp/media/MediaAccess.cc +++ b/zypp/media/MediaAccess.cc @@ -321,17 +321,8 @@ MediaAccess::release( const std::string & ejectDev ) // filename is interpreted relative to the attached url // and a path prefix is preserved to destination void -MediaAccess::provideFile( const Pathname & filename, bool cached, bool checkonly) const +MediaAccess::provideFile( const Pathname & filename ) const { - if ( cached ) { - PathInfo pi( localPath( filename ) ); - if ( pi.isExist() ) - return; - } - - if(checkonly) - ZYPP_THROW(MediaFileNotFoundException(url(), filename)); - if ( !_handler ) { ZYPP_THROW(MediaNotOpenException("provideFile(" + filename.asString() + ")")); } diff --git a/zypp/media/MediaAccess.h b/zypp/media/MediaAccess.h index 48dcd0d..8aa60cd 100644 --- a/zypp/media/MediaAccess.h +++ b/zypp/media/MediaAccess.h @@ -243,7 +243,7 @@ namespace zypp { * \throws MediaException * **/ - void provideFile( const Pathname & filename, bool cached = false, bool checkonly = false ) const; + void provideFile( const Pathname & filename ) const; /** * Remove filename below attach point IFF handler downloads files diff --git a/zypp/media/MediaAria2c.cc b/zypp/media/MediaAria2c.cc index 8490a83..e9d04e4 100644 --- a/zypp/media/MediaAria2c.cc +++ b/zypp/media/MediaAria2c.cc @@ -26,10 +26,12 @@ #include "zypp/Target.h" #include "zypp/ZYppFactory.h" +#include "zypp/media/TransferProgram.h" #include "zypp/media/MediaAria2c.h" #include "zypp/media/proxyinfo/ProxyInfos.h" #include "zypp/media/ProxyInfo.h" #include "zypp/media/MediaUserAuth.h" +#include "zypp/media/MediaCurl.h" #include "zypp/thread/Once.h" #include <cstdlib> #include @@ -169,12 +171,28 @@ void fillAriaCmdLine( const Pathname &ariapath, args.push_back("--follow-metalink=mem"); args.push_back("--check-integrity=true"); - // only present in recent aria - if ( Edition(ariaver) >= Edition("1.20") ) - args.push_back( "--use-head=false"); + // only present in recent aria lets find out the aria version + vector<string> fields; + // "aria2c version x.x" + str::split( ariaver, std::back_inserter(fields)); + if ( fields.size() == 3 ) + { + if ( Edition(fields[2]) >= Edition("1.1.2") ) + args.push_back( "--use-head=false"); + } + + if ( s.maxDownloadSpeed() > 0 ) + args.push_back(str::form("--max-download-limit=%ld", s.maxDownloadSpeed())); + if ( s.minDownloadSpeed() > 0 ) + args.push_back(str::form("--lowest-speed-limit=%ld", s.minDownloadSpeed())); + args.push_back(str::form("--max-tries=%ld", s.maxSilentTries())); + + if ( Edition(fields[2]) < Edition("1.2.0") ) + WAR << "aria2c is older than 1.2.0, some features may be disabled" << endl; + // TODO make this one configurable - args.push_back( "--max-concurrent-downloads=2"); + args.push_back(str::form("--max-concurrent-downloads=%ld", s.maxConcurrentConnections())); // add the anonymous id. for ( TransferSettings::Headers::const_iterator it = s.headersBegin(); @@ -232,74 +250,6 @@ void fillAriaCmdLine( const Pathname &ariapath, args.push_back(url.asString().c_str()); } -/** - * comannd line for curl. - * The argument list gets passed as reference - * and it is filled. - */ -void fillCurlCmdLine( const Pathname &curlpath, - const TransferSettings &s, - const Url &url, - ExternalProgram::Arguments &args ) -{ - args.push_back(curlpath.c_str()); - // only do a head request - args.push_back("-I"); - args.push_back("-A"); args.push_back(s.userAgentString()); - - // headers. - for ( TransferSettings::Headers::const_iterator it = s.headersBegin(); - it != s.headersEnd(); - ++it ) - { - args.push_back("-H"); - args.push_back(it->c_str()); - } - - args.push_back("--connect-timeout"); - args.push_back(str::numstring(s.timeout())); - - if ( s.username().empty() ) - { - if ( url.getScheme() == "ftp" ) - { - string id = "yast2:"; - id += VERSION; - args.push_back("--user"); - args.push_back(id); - DBG << "Anonymous FTP identification: '" << id << "'" << endl; - } - } - else - { - string userpass = s.username(); - - if ( s.password().size() ) - userpass += (":" + s.password()); - args.push_back("--user"); - args.push_back(userpass); - } - - if ( s.proxyEnabled() ) - { - args.push_back("--proxy"); - args.push_back(s.proxy()); - if ( ! s.proxyUsername().empty() ) - { - string userpass = s.proxyUsername(); - - if ( s.proxyPassword().size() ) - userpass += (":" + s.proxyPassword()); - args.push_back("--proxy-user"); - args.push_back(userpass); - } - } - - args.push_back("--url"); - args.push_back(url.asString().c_str()); -} - - static const char *const anonymousIdHeader() { // we need to add the release and identifier to the @@ -355,12 +305,11 @@ const char *const MediaAria2c::agentString() MediaAria2c::MediaAria2c( const Url & url_r, const Pathname & attach_point_hint_r ) - : MediaHandler( url_r, attach_point_hint_r, - "/", // urlpath at attachpoint - true ) // does_download + : MediaCurl( url_r, attach_point_hint_r ) { MIL << "MediaAria2c::MediaAria2c(" << url_r << ", " << attach_point_hint_r << ")" << endl; + /* if( !attachPoint().empty()) { PathInfo ainfo(attachPoint()); @@ -380,6 +329,7 @@ MediaAria2c::MediaAria2c( const Url & url_r, if( atemp != NULL) ::free(atemp); } + */ //At this point, we initialize aria2c path _aria2cPath = Pathname( whereisAria2c().asString() ); @@ -390,25 +340,8 @@ MediaAria2c::MediaAria2c( const Url & url_r, void MediaAria2c::attachTo (bool next) { - // clear last arguments - if ( next ) - ZYPP_THROW(MediaNotSupportedException(_url)); - - if ( !_url.isValid() ) - ZYPP_THROW(MediaBadUrlException(_url)); - - if( !isUseableAttachPoint(attachPoint())) - { - std::string mountpoint = createAttachPoint().asString(); - - if( mountpoint.empty()) - ZYPP_THROW( MediaBadAttachPointException(url())); - - setAttachPoint( mountpoint, true); - } - - disconnectFrom(); - + MediaCurl::attachTo(next); + _settings.setUserAgentString(agentString()); _settings.addHeader(anonymousIdHeader()); _settings.addHeader(distributionFlavorHeader()); @@ -427,25 +360,23 @@ void MediaAria2c::attachTo (bool next) } DBG << "Proxy: " << (_settings.proxy().empty() ? "-none-" : _settings.proxy()) << endl; - - MediaSourceRef media( new MediaSource(_url.getScheme(), _url.asString())); - setMediaSource(media); - } bool MediaAria2c::checkAttachPoint(const Pathname &apoint) const { - return MediaHandler::checkAttachPoint( apoint, true, true); + return MediaCurl::checkAttachPoint( apoint ); } void MediaAria2c::disconnectFrom() { + MediaCurl::disconnectFrom(); + } void MediaAria2c::releaseFrom( const std::string & ejectDev ) { - disconnect(); + MediaCurl::releaseFrom(ejectDev); } static Url getFileUrl(const Url & url, const Pathname & filename) @@ -573,147 +504,17 @@ void MediaAria2c::getFileCopy( const Pathname & filename , const Pathname & targ bool MediaAria2c::getDoesFileExist( const Pathname & filename ) const { - bool retry = false; - AuthData auth_data; - - do - { - try - { - return doGetDoesFileExist( filename ); - } - // authentication problem, retry with proper authentication data - catch (MediaUnauthorizedException & ex_r) - { - if(authenticate(ex_r.hint(), !retry)) - retry = true; - else - ZYPP_RETHROW(ex_r); - } - // unexpected exception - catch (MediaException & excpt_r) - { - ZYPP_RETHROW(excpt_r); - } - } - while (retry); - - return false; + return MediaCurl::getDoesFileExist(filename); } bool MediaAria2c::doGetDoesFileExist( const Pathname & filename ) const { - DBG << filename.asString() << endl; - callback::SendReport<DownloadProgressReport> report; - - Url fileurl(getFileUrl(_url, filename)); - bool retry = false; - - ExternalProgram::Arguments args; - - fillCurlCmdLine("/usr/bin/curl", _settings, fileurl, args); - - do - { - try - { - report->start(_url, fileurl.asString() ); - - ExternalProgram curl(args, ExternalProgram::Stderr_To_Stdout); - //Process response - for(std::string curlResponse( curl.receiveLine()); - curlResponse.length(); - curlResponse = curl.receiveLine()) - { - - if ( str::contains(curlResponse, "401 Authorization Required") ) - { - ZYPP_THROW(MediaUnauthorizedException( - _url, "Login failed.", "Login failed", "auth hint" - )); - } - - if ( str::contains(curlResponse, "404 Not Found") ) - return false; - - if ( str::contains(curlResponse, "200 OK") ) - return true; - } - - int code = curl.close(); - - switch (code) - { - case 0: break; - // connection problems - return true; - case 1: - case 2: - case 3: - case 7: - default: - ZYPP_THROW(MediaException(_url.asString())); - } - - - report->finish( _url , zypp::media::DownloadProgressReport::NO_ERROR, ""); - retry = false; - } - // retry with proper authentication data - catch (MediaUnauthorizedException & ex_r) - { - if(authenticate(ex_r.hint(), !retry)) - retry = true; - else - { - report->finish(fileurl, zypp::media::DownloadProgressReport::ACCESS_DENIED, ex_r.asUserHistory()); - ZYPP_RETHROW(ex_r); - } - - } - // unexpected exception - catch (MediaException & excpt_r) - { - // FIXME: error number fix - report->finish(fileurl, zypp::media::DownloadProgressReport::ERROR, excpt_r.asUserHistory()); - ZYPP_RETHROW(excpt_r); - } - } - while (retry); - - report->finish(fileurl, zypp::media::DownloadProgressReport::NO_ERROR, ""); - return true; + return MediaCurl::doGetDoesFileExist(filename); } - + void MediaAria2c::getDir( const Pathname & dirname, bool recurse_r ) const { - filesystem::DirContent content; - getDirInfo( content, dirname, /*dots*/false ); - - for ( filesystem::DirContent::const_iterator it = content.begin(); it != content.end(); ++it ) { - Pathname filename = dirname + it->name; - int res = 0; - - switch ( it->type ) { - case filesystem::FT_NOT_AVAIL: // old directory.yast contains no typeinfo at all - case filesystem::FT_FILE: - getFile( filename ); - break; - case filesystem::FT_DIR: // newer directory.yast contain at least directory info - if ( recurse_r ) { - getDir( filename, recurse_r ); - } else { - res = assert_dir( localPath( filename ) ); - if ( res ) { - WAR << "Ignore error (" << res << ") on creating local directory '" << localPath( filename ) << "'" << endl; - } - } - break; - default: - // don't provide devices, sockets, etc. - break; - } - } + MediaCurl::getDir(dirname, recurse_r); } bool MediaAria2c::authenticate(const std::string & availAuthTypes, bool firstTry) const diff --git a/zypp/media/MediaAria2c.h b/zypp/media/MediaAria2c.h index 9d58909..33d49f7 100644 --- a/zypp/media/MediaAria2c.h +++ b/zypp/media/MediaAria2c.h @@ -13,6 +13,7 @@ #define ZYPP_MEDIA_MEDIAARIA2C_H #include "zypp/media/MediaHandler.h" +#include "zypp/media/MediaCurl.h" #include "zypp/media/TransferSettings.h" #include "zypp/ZYppCallbacks.h" @@ -27,7 +28,7 @@ namespace zypp { * * @see MediaHandler **/ -class MediaAria2c : public MediaHandler { +class MediaAria2c : public MediaCurl { public: /** diff --git a/zypp/media/TransferSettings.cc b/zypp/media/TransferSettings.cc index e50a3a5..b206d1b 100644 --- a/zypp/media/TransferSettings.cc +++ b/zypp/media/TransferSettings.cc @@ -9,6 +9,7 @@ #include "zypp/base/NonCopyable.h" #include "zypp/ExternalProgram.h" #include "zypp/media/TransferSettings.h" +#include "zypp/ZConfig.h" using namespace std; @@ -27,6 +28,10 @@ public: : _useproxy(false) , _timeout(0) , _connect_timeout(0) + , _maxConcurrentConnections(ZConfig::instance().download_max_concurrent_connections()) + , _minDownloadSpeed(ZConfig::instance().download_min_download_speed()) + , _maxDownloadSpeed(ZConfig::instance().download_max_download_speed()) + , _maxSilentTries(ZConfig::instance().download_max_silent_tries()) {} virtual ~Impl() @@ -58,6 +63,11 @@ public: long _connect_timeout; Url _url; Pathname _targetdir; + + long _maxConcurrentConnections; + long _minDownloadSpeed; + long _maxDownloadSpeed; + long _maxSilentTries; }; TransferSettings::TransferSettings() @@ -171,6 +181,46 @@ long TransferSettings::connectTimeout() const return _impl->_connect_timeout; } +long TransferSettings::maxConcurrentConnections() const +{ + return _impl->_maxConcurrentConnections; +} + +void TransferSettings::setMaxConcurrentConnections(long v) +{ + _impl->_maxConcurrentConnections = v; +} + +long TransferSettings::minDownloadSpeed() const +{ + return _impl->_minDownloadSpeed; +} + +void TransferSettings::setMinDownloadSpeed(long v) +{ + _impl->_minDownloadSpeed = v; +} + +long TransferSettings::maxDownloadSpeed() const +{ + return _impl->_maxDownloadSpeed; +} + +void TransferSettings::setMaxDownloadSpeed(long v) +{ + _impl->_maxDownloadSpeed = v; +} + +long TransferSettings::maxSilentTries() const +{ + return _impl->_maxSilentTries; +} + +void TransferSettings::setMaxSilentTries(long v) +{ + _impl->_maxSilentTries = v; +} + } // ns media } // ns zypp diff --git a/zypp/media/TransferSettings.h b/zypp/media/TransferSettings.h index 3a513f0..1491bb7 100644 --- a/zypp/media/TransferSettings.h +++ b/zypp/media/TransferSettings.h @@ -1,6 +1,6 @@ -#ifndef TRANSFER_PROGRAM_H_ -#define TRANSFER_PROGRAM_H_ +#ifndef TRANSFER_SETTINGS_H_ +#define TRANSFER_SETTINGS_H_ #include <string> #include <vector> @@ -15,9 +15,7 @@ namespace media { /** - * Easy access to the transfer command line program no matter - * which one it is - * + * Holds transfer setting */ class TransferSettings { @@ -140,6 +138,48 @@ public: */ long timeout() const; + /** + * Maximum number of concurrent connections for a single transfer + */ + long maxConcurrentConnections() const; + + /** + * Set maximum number of concurrent connections for a single transfer + */ + void setMaxConcurrentConnections(long v); + + /** + * Minimum download speed (bytes per second) + * until the connection is dropped + */ + long minDownloadSpeed() const; + + /** + * Set minimum download speed (bytes per second) + * until the connection is dropped + */ + void setMinDownloadSpeed(long v); + + /** + * Maximum download speed (bytes per second) + */ + long maxDownloadSpeed() const; + + /** + * Set max download speed (bytes per second) + */ + void setMaxDownloadSpeed(long v); + + /** + * Maximum silent retries + */ + long maxSilentTries() const; + + /** + * Set maximum silent retries + */ + void setMaxSilentTries(long v); + protected: class Impl; RWCOW_pointer<Impl> _impl; -- To unsubscribe, e-mail: zypp-commit+unsubscribe@opensuse.org For additional commands, e-mail: zypp-commit+help@opensuse.org