[zypp-devel] [PATCH] Disable proxy for special repository

Repositories may be in different locations (inside/outside the company network), the outside repo needs proxy, but the internal repo should disable proxy. with out this patch libzypp can't work if inside repo and outside repo exist at the same time. This patch give user the option to disable proxy in repository file. Usage as follows: [home_xiaoqiang] name=local type=rpm-md baseurl=http://172.16.119.175 proxy=_none_ #Disable proxy #proxy=http://host:port #enable proxy enabled=1 --- zypp/Url.cc | 7 +++++++ zypp/Url.h | 10 ++++++++++ zypp/media/MediaAccess.cc | 13 ++++++++++--- zypp/media/MediaAria2c.cc | 4 ++-- zypp/media/MediaAria2c.h | 2 +- zypp/media/MediaCurl.cc | 5 ++++- zypp/media/MediaCurl.h | 3 ++- zypp/media/MediaMultiCurl.cc | 4 ++-- zypp/media/MediaMultiCurl.h | 2 +- zypp/parser/RepoFileReader.cc | 15 +++++++++++++-- zypp/url/UrlBase.cc | 8 ++++++++ zypp/url/UrlBase.h | 10 ++++++++++ 12 files changed, 70 insertions(+), 13 deletions(-) diff --git a/zypp/Url.cc b/zypp/Url.cc index 7ca3c61..81e8230 100644 --- a/zypp/Url.cc +++ b/zypp/Url.cc @@ -814,6 +814,13 @@ namespace zypp } // ----------------------------------------------------------------- + void + Url::removeQueryParam(const std::string ¶m) + { + m_impl->removeQueryParam(param); + } + + // ----------------------------------------------------------------- ViewOptions Url::getViewOptions() const { diff --git a/zypp/Url.h b/zypp/Url.h index e56bb17..a276e40 100644 --- a/zypp/Url.h +++ b/zypp/Url.h @@ -755,6 +755,16 @@ namespace zypp void setQueryParam(const std::string ¶m, const std::string &value); + /** + * \brief Delete the specified query parameter. + * \param param The decoded query parameter name. + * \throws url::UrlNotSupportedException if parameter parsing + * is not supported for a URL (scheme). + * \throws url::UrlDecodingException if the decoded result string + * would contain a '\\0' character. + */ + void + removeQueryParam(const std::string ¶m); // ----------------- /** diff --git a/zypp/media/MediaAccess.cc b/zypp/media/MediaAccess.cc index f0a769d..2f139bc 100644 --- a/zypp/media/MediaAccess.cc +++ b/zypp/media/MediaAccess.cc @@ -132,6 +132,7 @@ MediaAccess::open (const Url& url, const Pathname & preferred_attach_point) bool use_multicurl = true; const char *ariaenv = getenv( "ZYPP_ARIA2C" ); const char *multicurlenv = getenv( "ZYPP_MULTICURL" ); + bool disable_proxy = false; // if user disabled it manually if ( use_multicurl && multicurlenv && ( strcmp(multicurlenv, "0" ) == 0 ) ) { @@ -162,12 +163,18 @@ MediaAccess::open (const Url& url, const Pathname & preferred_attach_point) use_aria = false; } + Url curlUrl( url ); + if ( curlUrl.getQueryParam("noproxy") == "yes" ){ + curlUrl.removeQueryParam("noproxy"); + disable_proxy = true; + } + if ( use_aria ) - _handler = new MediaAria2c (url,preferred_attach_point); + _handler = new MediaAria2c (curlUrl,preferred_attach_point, disable_proxy); else if ( use_multicurl ) - _handler = new MediaMultiCurl (url,preferred_attach_point); + _handler = new MediaMultiCurl (curlUrl,preferred_attach_point, disable_proxy); else - _handler = new MediaCurl (url,preferred_attach_point); + _handler = new MediaCurl (curlUrl,preferred_attach_point, disable_proxy); } else { diff --git a/zypp/media/MediaAria2c.cc b/zypp/media/MediaAria2c.cc index 824b6de..1f0788a 100644 --- a/zypp/media/MediaAria2c.cc +++ b/zypp/media/MediaAria2c.cc @@ -231,8 +231,8 @@ const char *const MediaAria2c::agentString() MediaAria2c::MediaAria2c( const Url & url_r, - const Pathname & attach_point_hint_r ) - : MediaCurl( url_r, attach_point_hint_r ) + const Pathname & attach_point_hint_r, bool disable_proxy_r ) + : MediaCurl( url_r, attach_point_hint_r, disable_proxy_r ) { MIL << "MediaAria2c::MediaAria2c(" << url_r << ", " << attach_point_hint_r << ")" << endl; //Get aria2c version diff --git a/zypp/media/MediaAria2c.h b/zypp/media/MediaAria2c.h index 1169edf..ebfcc24 100644 --- a/zypp/media/MediaAria2c.h +++ b/zypp/media/MediaAria2c.h @@ -75,7 +75,7 @@ class MediaAria2c : public MediaCurl { public: MediaAria2c( const Url & url_r, - const Pathname & attach_point_hint_r ); + const Pathname & attach_point_hint_r, bool disable_proxy_r = false); virtual ~MediaAria2c() { try { release(); } catch(...) {} } diff --git a/zypp/media/MediaCurl.cc b/zypp/media/MediaCurl.cc index 086bdf1..531272b 100644 --- a/zypp/media/MediaCurl.cc +++ b/zypp/media/MediaCurl.cc @@ -396,11 +396,12 @@ static const char *const agentString() #define SET_OPTION_VOID(opt,val) SET_OPTION(opt,(void*)val) MediaCurl::MediaCurl( const Url & url_r, - const Pathname & attach_point_hint_r ) + const Pathname & attach_point_hint_r, bool disable_proxy_r ) : MediaHandler( url_r, attach_point_hint_r, "/", // urlpath at attachpoint true ), // does_download _curl( NULL ), + _noproxy(disable_proxy_r), _customHeaders(0L) { _curlError[0] = '\0'; @@ -1298,6 +1299,8 @@ void MediaCurl::doGetFileCopyFile( const Pathname & filename , const Pathname & string urlBuffer( curlUrl.asString()); CURLcode ret = curl_easy_setopt( _curl, CURLOPT_URL, urlBuffer.c_str() ); + if (_noproxy == true) + curl_easy_setopt(_curl, CURLOPT_NOPROXY, url.getHost().c_str()); if ( ret != 0 ) { ZYPP_THROW(MediaCurlSetOptException(url, _curlError)); } diff --git a/zypp/media/MediaCurl.h b/zypp/media/MediaCurl.h index bdd4396..efc12d6 100644 --- a/zypp/media/MediaCurl.h +++ b/zypp/media/MediaCurl.h @@ -95,7 +95,7 @@ class MediaCurl : public MediaHandler public: MediaCurl( const Url & url_r, - const Pathname & attach_point_hint_r ); + const Pathname & attach_point_hint_r, bool disable_proxy_r = false); virtual ~MediaCurl() { try { release(); } catch(...) {} } @@ -161,6 +161,7 @@ class MediaCurl : public MediaHandler std::string _currentCookieFile; static Pathname _cookieFile; + bool _noproxy; protected: CURL *_curl; diff --git a/zypp/media/MediaMultiCurl.cc b/zypp/media/MediaMultiCurl.cc index 8ae9294..eb413d8 100644 --- a/zypp/media/MediaMultiCurl.cc +++ b/zypp/media/MediaMultiCurl.cc @@ -1130,8 +1130,8 @@ multifetchrequest::run(std::vector<Url> &urllist) ////////////////////////////////////////////////////////////////////// -MediaMultiCurl::MediaMultiCurl(const Url &url_r, const Pathname & attach_point_hint_r) - : MediaCurl(url_r, attach_point_hint_r) +MediaMultiCurl::MediaMultiCurl(const Url &url_r, const Pathname & attach_point_hint_r, bool disable_proxy_r) + : MediaCurl(url_r, attach_point_hint_r, disable_proxy_r) { MIL << "MediaMultiCurl::MediaMultiCurl(" << url_r << ", " << attach_point_hint_r << ")" << endl; _multi = 0; diff --git a/zypp/media/MediaMultiCurl.h b/zypp/media/MediaMultiCurl.h index ce926fe..b525e9a 100644 --- a/zypp/media/MediaMultiCurl.h +++ b/zypp/media/MediaMultiCurl.h @@ -42,7 +42,7 @@ public: friend class multifetchrequest; friend class multifetchworker; - MediaMultiCurl(const Url &url_r, const Pathname & attach_point_hint_r); + MediaMultiCurl(const Url &url_r, const Pathname & attach_point_hint_r, bool disable_proxy_r = false); ~MediaMultiCurl(); virtual void doGetFileCopy( const Pathname & srcFilename, const Pathname & targetFilename, callback::SendReport<DownloadProgressReport> & _report, RequestOptions options = OPTION_NONE ) const; diff --git a/zypp/parser/RepoFileReader.cc b/zypp/parser/RepoFileReader.cc index 7a41431..b43e37f 100644 --- a/zypp/parser/RepoFileReader.cc +++ b/zypp/parser/RepoFileReader.cc @@ -44,6 +44,7 @@ namespace zypp { RepoInfo info; info.setAlias(*its); + Url url; for ( IniDict::entry_const_iterator it = dict.entriesBegin(*its); it != dict.entriesEnd(*its); @@ -57,7 +58,7 @@ namespace zypp else if ( it->first == "priority" ) info.setPriority( str::strtonum<unsigned>( it->second ) ); else if ( it->first == "baseurl" && !it->second.empty()) - info.addBaseUrl( Url(it->second) ); + url = it->second; else if ( it->first == "path" ) info.setPath( Pathname(it->second) ); else if ( it->first == "type" ) @@ -74,9 +75,19 @@ namespace zypp info.setKeepPackages( str::strToTrue( it->second ) ); else if ( it->first == "service" ) info.setService( it->second ); - else + else if ( it->first == "proxy" ) + { + if (it->second != "_none_" ) + { + int pos = it->second.find(":", 0); + url.setQueryParam("proxy", it->second.substr(0, pos)); + url.setQueryParam("proxyport", it->second.substr(pos + 1)); + } else + url.setQueryParam("noproxy", "yes"); + } else ERR << "Unknown attribute in [" << *its << "]: " << it->second << " ignored" << endl; } + info.addBaseUrl(url); info.setFilepath(file); MIL << info << endl; // add it to the list. diff --git a/zypp/url/UrlBase.cc b/zypp/url/UrlBase.cc index d42a032..c8578c1 100644 --- a/zypp/url/UrlBase.cc +++ b/zypp/url/UrlBase.cc @@ -1246,6 +1246,14 @@ namespace zypp setQueryStringMap(pmap); } + // --------------------------------------------------------------- + void + UrlBase::removeQueryParam(const std::string ¶m) + { + zypp::url::ParamMap pmap( getQueryStringMap(zypp::url::E_DECODED)); + pmap.erase(param); + setQueryStringMap(pmap); + } // --------------------------------------------------------------- std::string diff --git a/zypp/url/UrlBase.h b/zypp/url/UrlBase.h index 09be00b..d106024 100644 --- a/zypp/url/UrlBase.h +++ b/zypp/url/UrlBase.h @@ -862,6 +862,16 @@ namespace zypp virtual void setQueryParam(const std::string ¶m, const std::string &value); + /** + * \brief Delete the specified query parameter. + * \param param The decoded query parameter name. + * \throws url::UrlNotSupportedException if parameter parsing + * is not supported for a URL (scheme). + * \throws url::UrlDecodingException if the decoded result string + * would contain a '\\0' character. + */ + virtual void + removeQueryParam(const std::string ¶m); // ----------------- /** -- 1.7.2.2 -- To unsubscribe, e-mail: zypp-devel+unsubscribe@opensuse.org For additional commands, e-mail: zypp-devel+help@opensuse.org

Hi, thanks for the patch! Some comments: On Sat, Sep 18, 2010 at 05:21:39PM +0800, Zhang Qiang wrote:
Repositories may be in different locations (inside/outside the company network), the outside repo needs proxy, but the internal repo should disable proxy. with out this patch libzypp can't work if inside repo and outside repo exist at the same time.
This patch give user the option to disable proxy in repository file. Usage as follows: [home_xiaoqiang] name=local type=rpm-md baseurl=http://172.16.119.175 proxy=_none_ #Disable proxy #proxy=http://host:port #enable proxy enabled=1
It also gives the user the option to specify a proxy in the repository file (like with yum).
/** diff --git a/zypp/media/MediaAccess.cc b/zypp/media/MediaAccess.cc index f0a769d..2f139bc 100644 --- a/zypp/media/MediaAccess.cc +++ b/zypp/media/MediaAccess.cc @@ -132,6 +132,7 @@ MediaAccess::open (const Url& url, const Pathname & preferred_attach_point) bool use_multicurl = true; const char *ariaenv = getenv( "ZYPP_ARIA2C" ); const char *multicurlenv = getenv( "ZYPP_MULTICURL" ); + bool disable_proxy = false; // if user disabled it manually if ( use_multicurl && multicurlenv && ( strcmp(multicurlenv, "0" ) == 0 ) ) { @@ -162,12 +163,18 @@ MediaAccess::open (const Url& url, const Pathname & preferred_attach_point) use_aria = false; }
+ Url curlUrl( url ); + if ( curlUrl.getQueryParam("noproxy") == "yes" ){ + curlUrl.removeQueryParam("noproxy"); + disable_proxy = true; + }
Hmm, why handle "noproxy" different from "proxy"? I.e. why is there an extra disable_proxy argument, but all other settings (like "proxy" are detected later on in fillSettingsFromUrl()? [...]
{ _curlError[0] = '\0'; @@ -1298,6 +1299,8 @@ void MediaCurl::doGetFileCopyFile( const Pathname & filename , const Pathname & string urlBuffer( curlUrl.asString()); CURLcode ret = curl_easy_setopt( _curl, CURLOPT_URL, urlBuffer.c_str() ); + if (_noproxy == true) + curl_easy_setopt(_curl, CURLOPT_NOPROXY, url.getHost().c_str());
Shouldn't that simply be: curl_easy_setopt(_curl, CURLOPT_NOPROXY, "*");
[...] + if (it->second != "_none_" ) + { + int pos = it->second.find(":", 0); + url.setQueryParam("proxy", it->second.substr(0, pos)); + url.setQueryParam("proxyport", it->second.substr(pos + 1));
What if there is no ':' in the proxy string? find will return "npos" in that case. Cheers, Michael. -- Michael Schroeder mls@suse.de SUSE LINUX Products GmbH, GF Markus Rex, HRB 16746 AG Nuernberg main(_){while(_=~getchar())putchar(~_-1/(~(_|32)/13*2-11)*13);} -- To unsubscribe, e-mail: zypp-devel+unsubscribe@opensuse.org For additional commands, e-mail: zypp-devel+help@opensuse.org

On Monday 20 September 2010 10:45:58 Michael Schroeder wrote:
Hi, thanks for the patch! Some comments:
On Sat, Sep 18, 2010 at 05:21:39PM +0800, Zhang Qiang wrote:
Repositories may be in different locations (inside/outside the company network), the outside repo needs proxy, but the internal repo should disable proxy. with out this patch libzypp can't work if inside repo and outside repo exist at the same time.
This patch give user the option to disable proxy in repository file. Usage as follows: [home_xiaoqiang] name=local type=rpm-md baseurl=http://172.16.119.175 proxy=_none_ #Disable proxy #proxy=http://host:port #enable proxy enabled=1
It also gives the user the option to specify a proxy in the repository file (like with yum).
You should already be able to do this via the URL queryparams: https://server/path?proxy=foo.com&proxyport=42&proxyuser=me&proxypass=pw" Just evaluation of some 'noproxy' param is AFAIK missing.
Hmm, why handle "noproxy" different from "proxy"? I.e. why is there an extra disable_proxy argument, but all other settings (like "proxy" are detected later on in fillSettingsFromUrl()?
IMO right. Nevertheless we can think about supporting explicit .repo file options and just map them to queryparams. -- cu, Michael Andres +------------------------------------------------------------------+ Key fingerprint = 2DFA 5D73 18B1 E7EF A862 27AC 3FB8 9E3A 27C6 B0E4 +------------------------------------------------------------------+ Michael Andres ZYPP Development ma@suse.de SUSE LINUX Products GmbH, GF: Markus Rex, HRB 16746 (AG Nuernberg) Maxfeldstrasse 5, D-90409 Nuernberg, Germany, ++49 (0)911 - 740 53-0 +------------------------------------------------------------------+ -- To unsubscribe, e-mail: zypp-devel+unsubscribe@opensuse.org For additional commands, e-mail: zypp-devel+help@opensuse.org

On 09/20/2010 11:06 AM, Michael Andres wrote:
Hmm, why handle "noproxy" different from "proxy"? I.e. why is there an extra disable_proxy argument, but all other settings (like "proxy" are detected later on in fillSettingsFromUrl()?
IMO right. Nevertheless we can think about supporting explicit .repo file options and just map them to queryparams.
I think this is exactly what the patch does. It uses proxy=_none_ because yum uses that wird naming. I would keep the same name for compatibility. -- Duncan Mac-Vicar P. - Novell® Making IT Work As One™ SUSE LINUX Products GmbH, GF: Markus Rex, HRB 16746 (AG Nuernberg) -- To unsubscribe, e-mail: zypp-devel+unsubscribe@opensuse.org For additional commands, e-mail: zypp-devel+help@opensuse.org

Hi, thanks for the patch! Some comments: Thanks for your comments:)
baseurl=http://172.16.119.175 proxy=_none_ #Disable proxy #proxy=http://host:port #enable proxy enabled=1 It also gives the user the option to specify a proxy in the repository file (like with yum). [Zhang, Qiang] Yes.
/** diff --git a/zypp/media/MediaAccess.cc b/zypp/media/MediaAccess.cc index f0a769d..2f139bc 100644 --- a/zypp/media/MediaAccess.cc +++ b/zypp/media/MediaAccess.cc @@ -132,6 +132,7 @@ MediaAccess::open (const Url& url, const Pathname &
preferred_attach_point)
bool use_multicurl = true; const char *ariaenv = getenv( "ZYPP_ARIA2C" ); const char *multicurlenv = getenv( "ZYPP_MULTICURL" ); + bool disable_proxy = false; // if user disabled it manually if ( use_multicurl && multicurlenv && ( strcmp(multicurlenv, "0" )
== 0 ) )
{ @@ -162,12 +163,18 @@ MediaAccess::open (const Url& url, const Pathname &
preferred_attach_point)
use_aria = false; }
+ Url curlUrl( url ); + if ( curlUrl.getQueryParam("noproxy") == "yes" ){ + curlUrl.removeQueryParam("noproxy"); + disable_proxy = true; + }
Hmm, why handle "noproxy" different from "proxy"? I.e. why is there an extra disable_proxy argument, but all other settings (like "proxy" are detected later on in fillSettingsFromUrl()? [Zhang, Qiang] The "noproxy" param is just added to track the repo not to use proxy, so I think I need to remove this param before transfer to MediaCurl, as MediaCurl -> MediaHandler -> _url is a const type(const Url _url), so I can't remove param from MediaCurl.
So, I add an additional argument in the constructor of MediaCurl and add an additional variable in MediaCurl to note the proxy info for Url.
[...]
{ _curlError[0] = '\0'; @@ -1298,6 +1299,8 @@ void MediaCurl::doGetFileCopyFile( const Pathname & filename , const Pathname & string urlBuffer( curlUrl.asString()); CURLcode ret = curl_easy_setopt( _curl, CURLOPT_URL, urlBuffer.c_str() ); + if (_noproxy == true) + curl_easy_setopt(_curl, CURLOPT_NOPROXY, url.getHost().c_str());
Shouldn't that simply be: curl_easy_setopt(_curl, CURLOPT_NOPROXY, "*");
Yes, that's more easy and general.
[...] + if (it->second != "_none_" ) + { + int pos = it->second.find(":", 0); + url.setQueryParam("proxy", it->second.substr(0, pos)); + url.setQueryParam("proxyport", it->second.substr(pos + 1));
What if there is no ':' in the proxy string? find will return "npos" in that case.
[Zhang, Qiang] yeah, there's some issue. I needs write a simple reg to parse proxy. Attached is my new patch. Thanks Xiaoqiang

On Monday 20 September 2010 13:19:14 Zhang, Qiang Z wrote:
Hmm, why handle "noproxy" different from "proxy"? I.e. why is there an extra disable_proxy argument, but all other settings (like "proxy" are detected later on in fillSettingsFromUrl()?
[Zhang, Qiang] The "noproxy" param is just added to track the repo not to use proxy, so I think I need to remove this param before transfer to MediaCurl, as MediaCurl -> MediaHandler -> _url is a const type(const Url _url), so I can't remove param from MediaCurl.
So, I add an additional argument in the constructor of MediaCurl and add an additional variable in MediaCurl to note the proxy info for Url.
IMO it should be passed to MediaCurl via the Url: - no need to change the Media interfaces - no need for any extra handling outside MediaCurl If the original Url needs to be modified (e.g. to remove params), MediaCurls ctor could arrange this, or MediaCurl creates a copy and cleans it before passing it to libcurl. -- cu, Michael Andres +------------------------------------------------------------------+ Key fingerprint = 2DFA 5D73 18B1 E7EF A862 27AC 3FB8 9E3A 27C6 B0E4 +------------------------------------------------------------------+ Michael Andres ZYPP Development ma@suse.de SUSE LINUX Products GmbH, GF: Markus Rex, HRB 16746 (AG Nuernberg) Maxfeldstrasse 5, D-90409 Nuernberg, Germany, ++49 (0)911 - 740 53-0 +------------------------------------------------------------------+ -- To unsubscribe, e-mail: zypp-devel+unsubscribe@opensuse.org For additional commands, e-mail: zypp-devel+help@opensuse.org

On 09/20/2010 01:19 PM, Zhang, Qiang Z wrote:
curl_easy_setopt(_curl, CURLOPT_NOPROXY, "*"); Thanks Zhang for the patch.
I think the right implementation would look like this, as there is no need to set the values in each protocol, there is a TransferSettings class for that, Others, feedback? Duncan

On 09/20/2010 03:11 PM, Duncan Mac-Vicar P. wrote:
On 09/20/2010 01:19 PM, Zhang, Qiang Z wrote:
curl_easy_setopt(_curl, CURLOPT_NOPROXY, "*"); Thanks Zhang for the patch.
I think the right implementation would look like this, as there is no need to set the values in each protocol, there is a TransferSettings class for that,
Others, feedback?
Duncan Forgot to remove the noproxy setting, which is not needed as we use just proxy. I will fix that and update the patch. -- To unsubscribe, e-mail: zypp-devel+unsubscribe@opensuse.org For additional commands, e-mail: zypp-devel+help@opensuse.org

On 09/20/2010 03:16 PM, Duncan Mac-Vicar P. wrote:
On 09/20/2010 03:11 PM, Duncan Mac-Vicar P. wrote:
On 09/20/2010 01:19 PM, Zhang, Qiang Z wrote:
curl_easy_setopt(_curl, CURLOPT_NOPROXY, "*"); Thanks Zhang for the patch.
I think the right implementation would look like this, as there is no need to set the values in each protocol, there is a TransferSettings class for that,
Others, feedback?
Duncan Forgot to remove the noproxy setting, which is not needed as we use just proxy. I will fix that and update the patch.
New patch
participants (5)
-
Duncan Mac-Vicar P.
-
Michael Andres
-
Michael Schroeder
-
Zhang Qiang
-
Zhang, Qiang Z