Mailinglist Archive: opensuse-buildservice (216 mails)

< Previous Next >
Re: [opensuse-buildservice] local obs behind a proxy
On 2010-03-24 14:33:01 +0100, Christian wrote:
I want to use the 'remoturl' feature. (added PRJ openSUSE.org)
And my local obs is not able to connect to 'api.opensuse.org' directly.
Access to internet is only allowed via proxy (e.g. squid)

So is there a possibility to tell obs to use a '$http_proxy' ?

I just wrote a small patch which adds basic http proxy support to the
backend. To enable it add 'our $proxy = "http://<ip>:<port>";' to the
BSConfig.pm file. Afterwards all remoteurl requests will be sent to
proxy.
The current implementation doesn't support proxy-authorization yet
but it should be trivial to implement it (before concentrating on
those details I want to know if the current code is ok:) ).

Feedback, remarks, objections etc. are always welcome:)


Marcus
diff --git a/src/backend/BSRPC.pm b/src/backend/BSRPC.pm
index 50ac60a..2dcd5d2 100644
--- a/src/backend/BSRPC.pm
+++ b/src/backend/BSRPC.pm
@@ -54,6 +54,15 @@ sub urlencode {
return $url;
}

+sub parseuri {
+ my ($uri) = @_;
+ die("bad uri: $uri\n") unless $uri =~
/^(https?):\/\/(?:([^\/\@]*)\@)?([^\/:]+)(:\d+)?(\/.*)$/;
+ my ($proto, $auth, $host, $port, $path) = ($1, $2, $3, $4, $5);
+ my $hostport = $port ? "$host$port" : $host;
+ $port = substr($port || ($proto eq 'http' ? ":80" : ":443"), 1);
+ return ($proto, $auth, $host, $port, $path, $hostport);
+}
+
#
# handled paramters:
# timeout
@@ -137,19 +146,24 @@ sub rpc {
*S = $param->{'socket'};
$path = $uri;
} else {
- die("bad uri: $uri\n") unless $uri =~
/^(https?):\/\/(?:([^\/\@]*)\@)?([^\/:]+)(:\d+)?(\/.*)$/;
- my ($proto, $auth);
- ($proto, $auth, $host, $port, $path) = ($1, $2, $3, $4, $5);
- my $hostport = $port ? "$host$port" : $host;
- $port = substr($port || ($proto eq 'http' ? ":80" : ":443"), 1);
- if (!$hostlookupcache{$host}) {
- my $hostaddr = inet_aton($host);
- die("unknown host '$host'\n") unless $hostaddr;
- $hostlookupcache{$host} = $hostaddr;
+ my ($proto, $auth, $hostport);
+ ($proto, $auth, $host, $port, $path, $hostport) = parseuri($uri);
+ my $connecthost = $host;
+ my $connectport = $port;
+ if ($BSConfig::proxy && $param->{'proxy'}) {
+ my @r = parseuri($BSConfig::proxy);
+ $connecthost = $r[2];
+ $connectport = $r[3];
+ $path = $uri;
+ }
+ if (!$hostlookupcache{$connecthost}) {
+ my $hostaddr = inet_aton($connecthost);
+ die("unknown host '$connecthost'\n") unless $hostaddr;
+ $hostlookupcache{$connecthost} = $hostaddr;
}
socket(S, PF_INET, SOCK_STREAM, $tcpproto) || die("socket: $!\n");
setsockopt(S, SOL_SOCKET, SO_KEEPALIVE, pack("l",1));
- connect(S, sockaddr_in($port, $hostlookupcache{$host})) || die("connect to
$host:$port: $!\n");
+ connect(S, sockaddr_in($connectport, $hostlookupcache{$connecthost})) ||
die("connect to $connecthost:$connectport: $!\n");
unshift @xhdrs, "Connection: close";
unshift @xhdrs, "User-Agent: $useragent" unless !defined($useragent) ||
grep {/^user-agent:/si} @xhdrs;
unshift @xhdrs, "Host: $hostport" unless grep {/^host:/si} @xhdrs;
@@ -159,6 +173,21 @@ sub rpc {
unshift @xhdrs, "Authorization: Basic ".encode_base64($auth, '');
}
if ($proto eq 'https') {
+ if ($BSConfig::proxy && $param->{'proxy'}) {
+ print "proxy ssl: sending CONNECT request\n";
+ BSHTTP::swrite(\*S, "CONNECT $hostport HTTP/1.1\r\nHost:
$hostport\r\n\r\n");
+ my $ans = '';
+ do {
+ die("received truncated answer\n") if !sysread(S, $ans, 1024,
length($ans));
+ } while ($ans !~ /\n\r?\n/s);
+ die("bad answer\n") unless $ans =~
s/^HTTP\/\d+?\.\d+?\s+?(\d+[^\r\n]*)/Status: $1/s;
+ my $status = $1;
+ if ($status =~ /^200[^\d]/) {
+ print "proxy ssl: connection established (doing original req now)\n";
+ } else {
+ die("proxy ssl: CONNECT failed\n") unless $param->{'ignorestatus'};
+ }
+ }
if ($param->{'https'}) {
$param->{'https'}->(\*S);
} elsif ($tossl) {
diff --git a/src/backend/BSWatcher.pm b/src/backend/BSWatcher.pm
index 84122ad..bee58ad 100644
--- a/src/backend/BSWatcher.pm
+++ b/src/backend/BSWatcher.pm
@@ -630,6 +630,7 @@ sub rpc_recv_file {

sub rpc_recv_handler {
my ($ev) = @_;
+ print "rpc_recv_handler\n";
my $cs = 1024;
# needs to be bigger than the ssl package size...
$cs = 16384 if $ev->{'param'} && $ev->{'param'}->{'proto'} &&
$ev->{'param'}->{'proto'} eq 'https';
@@ -694,6 +695,18 @@ sub rpc_recv_handler {
}
}

+ if ($ev->{'rpcstate'} eq 'proxyconnect') {
+ print "proxy ssl: connection established (doing original req now)\n";
+ return unless initssl($ev);
+ $ev->{'recvbuf'} = '';
+ $ev->{'rpcstate'} = 'sending';
+ $ev->{'type'} = 'write';
+ $ev->{'handler'} = \&rpc_send_handler;
+ delete $ev->{'timeouthandler'};
+ BSEvents::add($ev, 0); # connect is finished so remove timeout
+ return
+ }
+
my $cl = $headers{'content-length'};
my $chunked = $headers{'transfer-encoding'} &&
lc($headers{'transfer-encoding'}) eq 'chunked' ? 1 : 0;

@@ -725,6 +738,7 @@ sub rpc_recv_handler {

sub rpc_send_handler {
my ($ev) = @_;
+ print "rpc_send_handler\n";
my $l = length($ev->{'sendbuf'});
return unless $l;
$l = 4096 if $l > 4096;
@@ -756,10 +770,29 @@ sub rpc_connect_timeout {
rpc_error($ev, "connect to $ev->{'rpcdest'}: timeout");
}

+sub initssl {
+ my ($ev) = @_;
+ if ($ev->{'param'} && $ev->{'param'}->{'proto'} && $ev->{'param'}->{'proto'}
eq 'https') {
+ print "switching to https\n";
+ fcntl($ev->{'fd'}, F_SETFL, 0); # in danger honor...
+ eval {
+ $ev->{'param'}->{'https'}->($ev->{'fd'});
+ };
+ fcntl($ev->{'fd'}, F_SETFL, O_NONBLOCK);
+ if ($@) {
+ my $err = $@;
+ $err =~ s/\n$//s;
+ rpc_error($ev, $err);
+ return;
+ }
+ }
+ return 1;
+}
+
sub rpc_connect_handler {
my ($ev) = @_;
my $err;
- #print "rpc_connect_handler\n";
+ print "rpc_connect_handler\n";
$err = getsockopt($ev->{'fd'}, SOL_SOCKET, SO_ERROR);
if (!defined($err)) {
$err = "getsockopt: $!";
@@ -776,21 +809,36 @@ sub rpc_connect_handler {
rpc_error($ev, $err);
return;
}
- #print "rpc_connect_handler: connected!\n";
- if ($ev->{'param'} && $ev->{'param'}->{'proto'} && $ev->{'param'}->{'proto'}
eq 'https') {
- print "switching to https\n";
- fcntl($ev->{'fd'}, F_SETFL, 0); # in danger honor...
- eval {
- $ev->{'param'}->{'https'}->($ev->{'fd'});
- };
- fcntl($ev->{'fd'}, F_SETFL, O_NONBLOCK);
- if ($@) {
- $err = $@;
- $err =~ s/\n$//s;
- rpc_error($ev, $err);
+ if (exists $ev->{'proxyconnect'} && $ev->{'param'} &&
$ev->{'param'}->{'proto'} && $ev->{'param'}->{'proto'} eq 'https') {
+ $ev->{'rpcstate'} = 'proxyconnect';
+# print "$ev->{'proxyconnect'}\n";
+ print "proxy ssl: sending CONNECT request\n";
+# $ev->{'proxyconnectbuf'} = "CONNECT $ev->{'proxyconnect'}
HTTP/1.1\r\n\r\n" unless exists $ev->{'proxyconnectbuf'};
+ $ev->{'proxyconnectbuf'} = "CONNECT $ev->{'proxyconnect'}
HTTP/1.1\r\nHost: $ev->{'proxyconnect'}\r\n\r\n" unless exists
$ev->{'proxyconnectbuf'};
+ my $r = syswrite($ev->{'fd'}, $ev->{'proxyconnectbuf'},
length($ev->{'proxyconnectbuf'}));
+ if (!defined($r)) {
+ if ($! == POSIX::EINTR || $! == POSIX::EWOULDBLOCK) {
+ BSEvents::add($ev);
+ return;
+ }
+ rpc_error($ev, "write error to $ev->{'proxyconnect'}: $!");
return;
}
+ if ($r != length($ev->{'proxyconnectbuf'})) {
+ $ev->{'proxyconnectbuf'} = substr($ev->{'proxyconnectbuf'}, $r) if $r;
+ BSEvents::add($ev);
+ return;
+ }
+ delete $ev->{'proxyconnectbuf'};
+ delete $ev->{'proxyconnect'};
+ $ev->{'type'} = 'read';
+ $ev->{'recvbuf'} = '';
+ $ev->{'handler'} = \&rpc_recv_handler;
+ BSEvents::add($ev);
+ return;
}
+ return unless initssl($ev);
+ #print "rpc_connect_handler: connected!\n";
$ev->{'rpcstate'} = 'sending';
delete $ev->{'timeouthandler'};
$ev->{'handler'} = \&rpc_send_handler;
@@ -799,6 +847,16 @@ sub rpc_connect_handler {

my $tcpproto = getprotobyname('tcp');

+sub parseuri {
+ my ($uri) = @_;
+ die("bad uri: $uri\n") unless $uri =~
/^(https?):\/\/(?:([^\/\@]*)\@)?([^\/:]+)(:\d+)?(\/.*)$/;
+ my ($proto, $auth, $host, $port, $path) = ($1, $2, $3, $4, $5);
+ my $hostport = $port ? "$host$port" : $host;
+ $port = substr($port || ($proto eq 'http' ? ":80" : ":443"), 1);
+ return ($proto, $auth, $host, $port, $path, $hostport);
+}
+
+
sub rpc {
my ($uri, $xmlargs, @args) = @_;

@@ -853,15 +911,20 @@ sub rpc {
}

# new rpc, create rpc event
- die("bad uri: $uri\n") unless $uri =~
/^(https?):\/\/(?:([^\/\@]*)\@)?([^\/:]+)(:\d+)?(\/.*)$/;
- my ($proto, $auth, $host, $port, $path) = ($1, $2, $3, $4, $5);
- my $hostport = $port ? "$host$port" : $host;
- $port = substr($port || ($proto eq 'http' ? ":80" : ":443"), 1);
- if (!$hostlookupcache{$host}) {
+ my ($proto, $auth, $host, $port, $path, $hostport) = parseuri($uri);
+ my $connecthost = $host;
+ my $connectport = $port;
+ if ($BSConfig::proxy && $param->{'proxy'}) {
+ my @r = parseuri($BSConfig::proxy);
+ $connecthost = $r[2];
+ $connectport = $r[3];
+ $path = $uri;
+ }
+ if (!$hostlookupcache{$connecthost}) {
# should do this async, but that's hard to do in perl
- my $hostaddr = inet_aton($host);
+ my $hostaddr = inet_aton($connecthost);
die("unknown host '$host'\n") unless $hostaddr;
- $hostlookupcache{$host} = $hostaddr;
+ $hostlookupcache{$connecthost} = $hostaddr;
}
unshift @xhdrs, "Connection: close";
unshift @xhdrs, "User-Agent: $BSRPC::useragent" unless
!defined($BSRPC::useragent) || grep {/^user-agent:/si} @xhdrs;
@@ -895,10 +958,11 @@ sub rpc {
$ev->{'rpcuri'} = $rpcuri;
$ev->{'rpcstate'} = 'connecting';
$ev->{'param'} = $param;
+ $ev->{'proxyconnect'} = $hostport if $BSConfig::proxy && $param->{'proxy'};
push @{$ev->{'joblist'}}, $jev;
$rpcs{$rpcuri} = $ev;
#print "new rpc $uri\n";
- if (!connect($fd, sockaddr_in($port, $hostlookupcache{$host}))) {
+ if (!connect($fd, sockaddr_in($connectport,
$hostlookupcache{$connecthost}))) {
if ($! == POSIX::EINPROGRESS) {
$ev->{'handler'} = \&rpc_connect_handler;
$ev->{'timeouthandler'} = \&rpc_connect_timeout;
diff --git a/src/backend/bs_repserver b/src/backend/bs_repserver
index dff6426..5638637 100755
--- a/src/backend/bs_repserver
+++ b/src/backend/bs_repserver
@@ -146,6 +146,7 @@ sub addrepo_remote {
my $param = {
'uri' =>
"$remoteproj->{'remoteurl'}/build/$remoteproj->{'remoteproject'}/$repoid/$arch/_repository",
'timeout' => 200,
+ 'proxy' => 1,
'receiver' => \&BSHTTP::cpio_receiver,
};
my $cpio = BSRPC::rpc($param, undef, "view=cache");
diff --git a/src/backend/bs_sched b/src/backend/bs_sched
index fb0cc7f..829527b 100755
--- a/src/backend/bs_sched
+++ b/src/backend/bs_sched
@@ -1646,6 +1646,7 @@ sub fetchremoteproj {
my $param = {
'uri' => "$proj->{'remoteurl'}/source/$proj->{'remoteproject'}/_meta",
'timeout' => 30,
+ 'proxy' => 1,
};
eval {
$rproj = BSRPC::rpc($param, $BSXML::proj);
@@ -1676,6 +1677,7 @@ sub fetchremoteconfig {
my $param = {
'uri' => "$proj->{'remoteurl'}/source/$proj->{'remoteproject'}/_config",
'timeout' => 30,
+ 'proxy' => 1,
};
eval {
$c = BSRPC::rpc($param);
@@ -1700,6 +1702,7 @@ sub addrepo_remote {
my $param = {
'uri' =>
"$remoteproj->{'remoteurl'}/build/$remoteproj->{'remoteproject'}/$repoid/$myarch/_repository",
'timeout' => 200,
+ 'proxy' => 1,
'receiver' => \&BSHTTP::cpio_receiver,
};
my $cpio;
@@ -2639,6 +2642,7 @@ sub checkaggregate {
my $param = {
'uri' =>
"$remoteprojs{$aprojid}->{'remoteurl'}/build/$remoteprojs{$aprojid}->{'remoteproject'}/$arepoid/$myarch/$apackid",
'timeout' => 20,
+ 'proxy' => 1,
};
my $binarylist;
eval {
@@ -2748,6 +2752,7 @@ sub rebuildaggregate {
'directory' => $jobdatadir,
'map' => "upload:",
'timeout' => 300,
+ 'proxy' => 1,
};
eval {
$cpio = BSRPC::rpc($param, undef, "view=cpio");
@@ -3109,6 +3114,7 @@ while(1) {
my $param = {
'uri' => "$remoteurl/lastevents",
'async' => 1,
+ 'proxy' => 1,
};
my @args;
push @args, "obsname=$BSConfig::obsname/$myarch" if $BSConfig::obsname;
diff --git a/src/backend/bs_srcserver b/src/backend/bs_srcserver
index b6e3fb4..dcc91cd 100755
--- a/src/backend/bs_srcserver
+++ b/src/backend/bs_srcserver
@@ -1725,7 +1725,7 @@ sub getproject {
my ($cgi, $projid) = @_;
# Read the project xml file
my $proj = checkprojrepoarch($projid, undef, undef, 1);
- $proj =
BSRPC::rpc("$proj->{'remoteurl'}/source/$proj->{'remoteproject'}/_meta",
$BSXML::proj) if $proj->{'remoteurl'};
+ $proj = BSRPC::rpc({'uri' =>
"$proj->{'remoteurl'}/source/$proj->{'remoteproject'}/_meta", 'proxy' => 1},
$BSXML::proj) if $proj->{'remoteurl'};
return ($proj, $BSXML::proj);
}

@@ -1835,7 +1835,7 @@ sub getpackagelist {
my ($cgi, $projid, $repoid, $arch) = @_;
my $proj = checkprojrepoarch($projid, $repoid, $arch, 1);
if ($proj->{'remoteurl'}) {
- return BSRPC::rpc("$proj->{'remoteurl'}/source/$proj->{'remoteproject'}",
$BSXML::dir), $BSXML::dir;
+ return BSRPC::rpc({'uri' =>
"$proj->{'remoteurl'}/source/$proj->{'remoteproject'}", 'proxy' => 1},
$BSXML::dir), $BSXML::dir;
}
my @packages = findpackages($projid);
my @plist = map {{'name' => $_}} @packages;
@@ -1846,7 +1846,7 @@ sub getpackage {
my ($cgi, $projid, $packid) = @_;
my $proj = checkprojrepoarch($projid, undef, undef, 1);
if ($proj->{'remoteurl'}) {
- my $pack =
BSRPC::rpc("$proj->{'remoteurl'}/source/$proj->{'remoteproject'}/$packid/_meta",
$BSXML::pack);
+ my $pack = BSRPC::rpc({'uri' =>
"$proj->{'remoteurl'}/source/$proj->{'remoteproject'}/$packid/_meta", 'proxy'
=> 1}, $BSXML::pack);
return ($pack, $BSXML::pack);
}
my $pack = readpack($projid, $packid);
@@ -1974,6 +1974,7 @@ sub fetchremoteproj {
my $param = {
'uri' => "$proj->{'remoteurl'}/source/$proj->{'remoteproject'}/_meta",
'timeout' => 60,
+ 'proxy' => 1,
};
my $rproj = BSRPC::rpc($param, $BSXML::proj);
return undef unless $rproj;
@@ -1991,6 +1992,7 @@ sub fetchremoteconfig {
my $param = {
'uri' => "$proj->{'remoteurl'}/source/$proj->{'remoteproject'}/_config",
'timeout' => 60,
+ 'proxy' => 1,
};
return BSRPC::rpc($param, undef);
}
@@ -2005,7 +2007,7 @@ sub remote_getrev {
my @args;
push @args, "expand";
push @args, "rev=$rev" if defined $rev;
- my $dir =
BSRPC::rpc("$proj->{'remoteurl'}/source/$proj->{'remoteproject'}/$packid",
$BSXML::dir, @args);
+ my $dir = BSRPC::rpc({'uri' =>
"$proj->{'remoteurl'}/source/$proj->{'remoteproject'}/$packid", 'proxy' => 1},
$BSXML::dir, @args);
die("$dir->{'error'}\n") if $dir->{'error'};
$rev = {};
$rev->{'rev'} = $dir->{'rev'} || $dir->{'srcmd5'};
@@ -2022,6 +2024,7 @@ sub remote_getrev {
'uri' =>
"$proj->{'remoteurl'}/source/$proj->{'remoteproject'}/$packid/$entry->{'name'}",
'filename' => "$uploaddir/$$",
'withmd5' => 1,
+ 'proxy' => 1,
'receiver' => \&BSHTTP::file_receiver,
};
my $res = BSRPC::rpc($param, undef, "rev=$rev->{'srcmd5'}");
@@ -2045,7 +2048,7 @@ sub remote_getrev {
last unless $li->{'srcmd5'} && !$li->{'error'};
my $ldir;
eval {
- $ldir = BSRPC::rpc("$proj->{'remoteurl'}/source/$lprojid/$lpackid",
$BSXML::dir, "rev=$li->{'srcmd5'}");
+ $ldir = BSRPC::rpc({'uri' =>
"$proj->{'remoteurl'}/source/$lprojid/$lpackid", 'proxy' => 1}, $BSXML::dir,
"rev=$li->{'srcmd5'}");
};
last if $@ || !$ldir;
$li = $ldir->{'linkinfo'};
@@ -3188,12 +3191,15 @@ sub getbinarylist {
'ignorestatus' => 1,
'receiver' => \&BSServer::reply_receiver,
};
- $param->{'uri'} =
"$proj->{'remoteurl'}/build/$proj->{'remoteproject'}/$repoid/$arch/$packid" if
$proj->{'remoteurl'};
+ if ($proj->{'remoteurl'}) {
+ $param->{'uri'} =
"$proj->{'remoteurl'}/build/$proj->{'remoteproject'}/$repoid/$arch/$packid";
+ $param->{'proxy'} = 1;
+ }
BSWatcher::rpc($param, undef, @args);
return undef;
}
my $uri = "$BSConfig::reposerver/build/$projid/$repoid/$arch/$packid";
- $uri =
"$proj->{'remoteurl'}/build/$proj->{'remoteproject'}/$repoid/$arch/$packid" if
$proj->{'remoteurl'};
+ $uri = {'uri' =>
"$proj->{'remoteurl'}/build/$proj->{'remoteproject'}/$repoid/$arch/$packid",
'proxy' => 1} if $proj->{'remoteurl'};
if ($view && $view eq 'binaryversions') {
push @args, 'nometa=1' if $cgi->{'nometa'};
my $bvl = BSWatcher::rpc($uri, $BSXML::binaryversionlist, @args);
@@ -4014,7 +4020,10 @@ sub worker_getbinaries {
}
my @binaries = split(',', $cgi->{'binaries'});
my ($remoteurl, $remoteprojid) = findremote($projid);
- my $binarylist =
BSWatcher::rpc("$remoteurl/build/$remoteprojid/$repoid/$arch/_repository",
$BSXML::binarylist, "view=names", map {"binary=$_"} @binaries);
+ # XXX: little hack
+ my $uri = {'uri' =>
"$remoteurl/build/$remoteprojid/$repoid/$arch/_repository"};
+ $uri->{'proxy'} = 1 if $remoteurl ne $BSConfig::reposerver;
+ my $binarylist = BSWatcher::rpc($uri, $BSXML::binarylist, "view=names", map
{"binary=$_"} @binaries);
return undef if $BSStdServer::isajax && !$binarylist;
my %binarylist;
for my $b (@{$binarylist->{'binary'} || []}) {
@@ -4076,6 +4085,7 @@ sub worker_getbinaries {
'directory' => $remotecache,
'map' => "upload$serialmd5:",
};
+ $param->{'proxy'} = 1 if $remoteurl ne $BSConfig::reposerver;
my $cpio;
if ($BSStdServer::isajax) {
$param->{'receiver'} = \&BSHTTP::file_receiver;
@@ -4135,7 +4145,9 @@ sub worker_getbinaryversions {
my ($cgi, $projid, $repoid, $arch) = @_;
my @binaries = split(',', $cgi->{'binaries'});
my ($remoteurl, $remoteprojid) = findremote($projid);
- my $bvl =
BSWatcher::rpc("$remoteurl/build/$remoteprojid/$repoid/$arch/_repository",
$BSXML::binaryversionlist, 'view=binaryversions', 'nometa=1', map {"binary=$_"}
@binaries);
+ my $uri = {'uri' =>
"$remoteurl/build/$remoteprojid/$repoid/$arch/_repository"};
+ $uri->{'proxy'} = 1 if $remoteurl ne $BSConfig::reposerver;
+ my $bvl = BSWatcher::rpc($uri, $BSXML::binaryversionlist,
'view=binaryversions', 'nometa=1', map {"binary=$_"} @binaries);
return ($bvl, $BSXML::binaryversionlist);
}

< Previous Next >
References