Hello community,
here is the log from the commit of package python-urlgrabber
checked in at Mon Oct 2 12:01:47 CEST 2006.
--------
--- python-urlgrabber/python-urlgrabber.changes 2006-09-21 14:26:51.000000000 +0200
+++ /mounts/work_src_done/STABLE/python-urlgrabber/python-urlgrabber.changes 2006-10-01 15:34:25.000000000 +0200
@@ -1,0 +2,6 @@
+Sun Oct 1 15:34:10 CEST 2006 - cthiel@suse.de
+
+- update to version 3.1.0
+ * various fixes
+
+-------------------------------------------------------------------
Old:
----
urlgrabber-2.9.10.tar.bz2
New:
----
urlgrabber-3.1.0.tar.bz2
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Other differences:
------------------
++++++ python-urlgrabber.spec ++++++
--- /var/tmp/diff_new_pack.beTh4i/_old 2006-10-02 12:01:31.000000000 +0200
+++ /var/tmp/diff_new_pack.beTh4i/_new 2006-10-02 12:01:31.000000000 +0200
@@ -1,5 +1,5 @@
#
-# spec file for package python-urlgrabber (Version 2.9.10)
+# spec file for package python-urlgrabber (Version 3.1.0)
#
# Copyright (c) 2006 SUSE LINUX Products GmbH, Nuernberg, Germany.
# This file and all modifications and additions to the pristine
@@ -12,8 +12,8 @@
Name: python-urlgrabber
BuildRequires: python-devel
-Version: 2.9.10
-Release: 6
+Version: 3.1.0
+Release: 1
Summary: A high-level cross-protocol url-grabber
Group: Development/Libraries/Python
License: LGPL
@@ -58,6 +58,9 @@
%{py_sitedir}/*.egg-info
%changelog -n python-urlgrabber
+* Sun Oct 01 2006 - cthiel@suse.de
+- update to version 3.1.0
+ * various fixes
* Thu Sep 21 2006 - cthiel@suse.de
- fix build with python 2.5
* Fri Aug 04 2006 - cthiel@suse.de
++++++ urlgrabber-2.9.10.tar.bz2 -> urlgrabber-3.1.0.tar.bz2 ++++++
diff -urN --exclude=CVS --exclude=.cvsignore --exclude=.svn --exclude=.svnignore old/urlgrabber-2.9.10/ChangeLog new/urlgrabber-3.1.0/ChangeLog
--- old/urlgrabber-2.9.10/ChangeLog 2006-07-20 22:17:32.000000000 +0200
+++ new/urlgrabber-3.1.0/ChangeLog 2006-09-22 02:59:04.000000000 +0200
@@ -1,3 +1,34 @@
+2006-09-22 Michael D. Stenner
+
+ * urlgrabber/__init__.py:
+
+ release 3.1.0
+
+2006-09-22 Michael D. Stenner
+
+ * test/grabberperf.py, urlgrabber/grabber.py,
+ urlgrabber/keepalive.py, urlgrabber/sslfactory.py:
+
+ Added James Bowes' SSL patch to use M2Crypto when available.
+
+2006-09-21 Michael D. Stenner
+
+ * ChangeLog:
+
+ updated ChangeLog
+
+2006-09-21 Michael D. Stenner
+
+ * urlgrabber/__init__.py:
+
+ release 3.0.0
+
+2006-07-20 Michael D. Stenner
+
+ * ChangeLog:
+
+ updated ChangeLog
+
2006-07-20 Michael D. Stenner
* urlgrabber/__init__.py:
diff -urN --exclude=CVS --exclude=.cvsignore --exclude=.svn --exclude=.svnignore old/urlgrabber-2.9.10/MANIFEST new/urlgrabber-3.1.0/MANIFEST
--- old/urlgrabber-2.9.10/MANIFEST 2006-07-20 22:17:41.000000000 +0200
+++ new/urlgrabber-3.1.0/MANIFEST 2006-09-22 02:59:14.000000000 +0200
@@ -20,3 +20,4 @@
urlgrabber/keepalive.py
urlgrabber/mirror.py
urlgrabber/progress.py
+urlgrabber/sslfactory.py
diff -urN --exclude=CVS --exclude=.cvsignore --exclude=.svn --exclude=.svnignore old/urlgrabber-2.9.10/PKG-INFO new/urlgrabber-3.1.0/PKG-INFO
--- old/urlgrabber-2.9.10/PKG-INFO 2006-07-20 22:17:42.000000000 +0200
+++ new/urlgrabber-3.1.0/PKG-INFO 2006-09-22 02:59:14.000000000 +0200
@@ -1,6 +1,6 @@
Metadata-Version: 1.0
Name: urlgrabber
-Version: 2.9.10
+Version: 3.1.0
Summary: A high-level cross-protocol url-grabber
Home-page: http://linux.duke.edu/projects/urlgrabber/
Author: Michael D. Stenner, Ryan Tomayko
diff -urN --exclude=CVS --exclude=.cvsignore --exclude=.svn --exclude=.svnignore old/urlgrabber-2.9.10/test/grabberperf.py new/urlgrabber-3.1.0/test/grabberperf.py
--- old/urlgrabber-2.9.10/test/grabberperf.py 2004-03-31 19:02:00.000000000 +0200
+++ new/urlgrabber-3.1.0/test/grabberperf.py 2006-09-22 02:58:05.000000000 +0200
@@ -123,8 +123,6 @@
print_result('raw', raw_times)
print_result('none', none_times)
- grabber.close_all()
-
def print_result(label, result_list):
format = '[%4s] mean: %6.3f ms, median: %6.3f ms, ' \
'min: %6.3f ms, max: %6.3f ms'
diff -urN --exclude=CVS --exclude=.cvsignore --exclude=.svn --exclude=.svnignore old/urlgrabber-2.9.10/urlgrabber/__init__.py new/urlgrabber-3.1.0/urlgrabber/__init__.py
--- old/urlgrabber-2.9.10/urlgrabber/__init__.py 2006-07-20 22:17:24.000000000 +0200
+++ new/urlgrabber-3.1.0/urlgrabber/__init__.py 2006-09-22 02:58:55.000000000 +0200
@@ -14,7 +14,7 @@
# Copyright 2002-2006 Michael D. Stenner, Ryan Tomayko
-# $Id: __init__.py,v 1.18 2006/07/20 20:17:24 mstenner Exp $
+# $Id: __init__.py,v 1.20 2006/09/22 00:58:55 mstenner Exp $
"""A high-level cross-protocol url-grabber.
@@ -44,8 +44,8 @@
automatically switching mirrors if there is a failure.
"""
-__version__ = '2.9.10'
-__date__ = '2006/07/20'
+__version__ = '3.1.0'
+__date__ = '2006/09/21'
__author__ = 'Michael D. Stenner , ' \
'Ryan Tomayko '
__url__ = 'http://linux.duke.edu/projects/urlgrabber/'
diff -urN --exclude=CVS --exclude=.cvsignore --exclude=.svn --exclude=.svnignore old/urlgrabber-2.9.10/urlgrabber/grabber.py new/urlgrabber-3.1.0/urlgrabber/grabber.py
--- old/urlgrabber-2.9.10/urlgrabber/grabber.py 2006-07-20 22:15:58.000000000 +0200
+++ new/urlgrabber-3.1.0/urlgrabber/grabber.py 2006-09-22 02:58:05.000000000 +0200
@@ -364,7 +364,7 @@
"""
-# $Id: grabber.py,v 1.47 2006/07/20 20:15:58 mstenner Exp $
+# $Id: grabber.py,v 1.48 2006/09/22 00:58:05 mstenner Exp $
import os
import os.path
@@ -385,6 +385,8 @@
except:
__version__ = '???'
+import sslfactory
+
auth_handler = urllib2.HTTPBasicAuthHandler( \
urllib2.HTTPPasswordMgrWithDefaultRealm())
@@ -403,10 +405,9 @@
# Just rename the module so it can't be imported.
import keepalive
from keepalive import HTTPHandler, HTTPSHandler
+ have_keepalive = True
except ImportError, msg:
- keepalive_handlers = ()
-else:
- keepalive_handlers = (HTTPHandler(), HTTPSHandler())
+ have_keepalive = False
try:
# add in range support conditionally too
@@ -455,10 +456,12 @@
global DEBUG
DEBUG = DBOBJ
- if keepalive_handlers and keepalive.DEBUG is None:
+ if have_keepalive and keepalive.DEBUG is None:
keepalive.DEBUG = DBOBJ
if have_range and byterange.DEBUG is None:
byterange.DEBUG = DBOBJ
+ if sslfactory.DEBUG is None:
+ sslfactory.DEBUG = DBOBJ
def _init_default_logger():
'''Examines the environment variable URLGRABBER_DEBUG and creates
@@ -581,11 +584,6 @@
def __init__(self, **kwargs):
self.__dict__.update(kwargs)
-def close_all():
- """close any open keepalive connections"""
- for handler in keepalive_handlers:
- handler.close_all()
-
def urlgrab(url, filename=None, **kwargs):
"""grab the file at <url> and make a local copy at <filename>
If filename is none, the basename of the url is used.
@@ -810,6 +808,8 @@
self.data = None
self.urlparser = URLParser()
self.quote = None
+ self.ssl_ca_cert = None
+ self.ssl_context = None
class URLGrabber:
"""Provides easy opening of URLs with a variety of options.
@@ -1007,14 +1007,14 @@
if hasattr(self.fo, name):
return getattr(self.fo, name)
raise AttributeError, name
-
+
def _get_opener(self):
"""Build a urllib2 OpenerDirector based on request options."""
if self.opts.opener:
return self.opts.opener
elif self._opener is None:
handlers = []
- need_keepalive_handler = (keepalive_handlers and self.opts.keepalive)
+ need_keepalive_handler = (have_keepalive and self.opts.keepalive)
need_range_handler = (range_handlers and \
(self.opts.range or self.opts.reget))
# if you specify a ProxyHandler when creating the opener
@@ -1043,16 +1043,20 @@
if not need_range_handler:
handlers.append( urllib2.FTPHandler() )
# -------------------------------------------------------
-
+
+ ssl_factory = sslfactory.get_factory(self.opts.ssl_ca_cert,
+ self.opts.ssl_context)
+
if need_keepalive_handler:
- handlers.extend( keepalive_handlers )
+ handlers.append(HTTPHandler())
+ handlers.append(HTTPSHandler(ssl_factory))
if need_range_handler:
handlers.extend( range_handlers )
handlers.append( auth_handler )
if self.opts.cache_openers:
- self._opener = CachedOpenerDirector(*handlers)
+ self._opener = CachedOpenerDirector(ssl_factory, *handlers)
else:
- self._opener = urllib2.build_opener(*handlers)
+ self._opener = ssl_factory.create_opener(*handlers)
# OK, I don't like to do this, but otherwise, we end up with
# TWO user-agent headers.
self._opener.addheaders = []
@@ -1297,13 +1301,15 @@
except: pass
_handler_cache = []
-def CachedOpenerDirector(*handlers):
+def CachedOpenerDirector(ssl_factory = None, *handlers):
for (cached_handlers, opener) in _handler_cache:
if cached_handlers == handlers:
for handler in opener.handlers:
handler.add_parent(opener)
return opener
- opener = urllib2.build_opener(*handlers)
+ if not ssl_factory:
+ ssl_factory = sslfactory.get_factory()
+ opener = ssl_factory.create_opener(*handlers)
_handler_cache.append( (handlers, opener) )
return opener
@@ -1418,7 +1424,6 @@
print 'success'
return
- close_all()
kwargs['checkfunc'] = (cfunc, ('hello',), {'there':'there'})
try: name = apply(retrygrab, (url, filename), kwargs)
except URLGrabError, e: print e
diff -urN --exclude=CVS --exclude=.cvsignore --exclude=.svn --exclude=.svnignore old/urlgrabber-2.9.10/urlgrabber/keepalive.py new/urlgrabber-3.1.0/urlgrabber/keepalive.py
--- old/urlgrabber-2.9.10/urlgrabber/keepalive.py 2006-07-20 22:15:58.000000000 +0200
+++ new/urlgrabber-3.1.0/urlgrabber/keepalive.py 2006-09-22 02:58:05.000000000 +0200
@@ -99,7 +99,7 @@
"""
-# $Id: keepalive.py,v 1.15 2006/07/20 20:15:58 mstenner Exp $
+# $Id: keepalive.py,v 1.16 2006/09/22 00:58:05 mstenner Exp $
import urllib2
import httplib
@@ -108,6 +108,8 @@
DEBUG = None
+import sslfactory
+
import sys
if sys.version_info < (2, 4): HANDLE_ERRORS = 1
else: HANDLE_ERRORS = 0
@@ -207,7 +209,7 @@
self._cm.remove(connection)
#### Transaction Execution
- def do_open(self, http_class, req):
+ def do_open(self, req):
host = req.get_host()
if not host:
raise urllib2.URLError('no host given')
@@ -228,7 +230,7 @@
h = self._cm.get_ready_conn(host)
else:
# no (working) free connections were found. Create a new one.
- h = http_class(host)
+ h = self._get_connection(host)
if DEBUG: DEBUG.info("creating new connection to %s (%d)",
host, id(h))
self._cm.add(host, h, 0)
@@ -321,19 +323,31 @@
if req.has_data():
h.send(data)
+ def _get_connection(self, host):
+ return NotImplementedError
+
class HTTPHandler(KeepAliveHandler, urllib2.HTTPHandler):
def __init__(self):
KeepAliveHandler.__init__(self)
def http_open(self, req):
- return self.do_open(HTTPConnection, req)
+ return self.do_open(req)
+
+ def _get_connection(self, host):
+ return HTTPConnection(host)
class HTTPSHandler(KeepAliveHandler, urllib2.HTTPSHandler):
- def __init__(self):
+ def __init__(self, ssl_factory=None):
KeepAliveHandler.__init__(self)
+ if not ssl_factory:
+ ssl_factory = sslfactory.get_factory()
+ self._ssl_factory = ssl_factory
def https_open(self, req):
- return self.do_open(HTTPSConnection, req)
+ return self.do_open(req)
+
+ def _get_connection(self, host):
+ return self._ssl_factory.get_https_connection(host)
class HTTPResponse(httplib.HTTPResponse):
# we need to subclass HTTPResponse in order to
diff -urN --exclude=CVS --exclude=.cvsignore --exclude=.svn --exclude=.svnignore old/urlgrabber-2.9.10/urlgrabber/sslfactory.py new/urlgrabber-3.1.0/urlgrabber/sslfactory.py
--- old/urlgrabber-2.9.10/urlgrabber/sslfactory.py 1970-01-01 01:00:00.000000000 +0100
+++ new/urlgrabber-3.1.0/urlgrabber/sslfactory.py 2006-09-22 02:58:05.000000000 +0200
@@ -0,0 +1,89 @@
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc.,
+# 59 Temple Place, Suite 330,
+# Boston, MA 02111-1307 USA
+
+# This file is part of urlgrabber, a high-level cross-protocol url-grabber
+
+import httplib
+import urllib2
+
+try:
+ from M2Crypto import SSL
+ from M2Crypto import httpslib
+ from M2Crypto import m2urllib2
+
+ have_m2crypto = True
+except ImportError:
+ have_m2crypto = False
+
+DEBUG = None
+
+if have_m2crypto:
+
+ class M2SSLFactory:
+
+ def __init__(self, ssl_ca_cert, ssl_context):
+ self.ssl_context = self._get_ssl_context(ssl_ca_cert, ssl_context)
+
+ def _get_ssl_context(self, ssl_ca_cert, ssl_context):
+ """
+ Create an ssl context using the CA cert file or ssl context.
+
+ The CA cert is used first if it was passed as an option. If not,
+ then the supplied ssl context is used. If no ssl context was supplied,
+ None is returned.
+ """
+ if ssl_ca_cert:
+ context = SSL.Context()
+ context.load_verify_locations(ssl_ca_cert)
+ context.set_verify(SSL.verify_peer, -1)
+ return context
+ else:
+ return ssl_context
+
+ def create_https_connection(self, host, response_class = None):
+ connection = httplib.HTTPSConnection(host, self.ssl_context)
+ if response_class:
+ connection.response_class = response_class
+ return connection
+
+ def create_opener(self, *handlers):
+ return m2urllib2.build_opener(self.ssl_context, *handlers)
+
+
+class SSLFactory:
+
+ def create_https_connection(self, host, response_class = None):
+ connection = httplib.HTTPSConnection(host)
+ if response_class:
+ connection.response_class = response_class
+ return connection
+
+ def create_opener(self, *handlers):
+ return urllib2.build_opener(*handlers)
+
+
+
+def get_factory(ssl_ca_cert = None, ssl_context = None):
+ """ Return an SSLFactory, based on if M2Crypto is available. """
+ if have_m2crypto:
+ return M2SSLFactory(ssl_ca_cert, ssl_context)
+ else:
+ # Log here if someone provides the args but we don't use them.
+ if ssl_ca_cert or ssl_context:
+ if DEBUG:
+ DEBUG.warning("SSL arguments supplied, but M2Crypto is not available. "
+ "Using Python SSL.")
+ return SSLFactory()
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Remember to have fun...
---------------------------------------------------------------------
To unsubscribe, e-mail: opensuse-commit+unsubscribe@opensuse.org
For additional commands, e-mail: opensuse-commit+help@opensuse.org