Hello community, here is the log from the commit of package python-portpicker for openSUSE:Factory checked in at 2019-05-22 15:41:09 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/python-portpicker (Old) and /work/SRC/openSUSE:Factory/.python-portpicker.new.5148 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Package is "python-portpicker" Wed May 22 15:41:09 2019 rev:2 rq:704702 version:1.3.1 Changes: -------- --- /work/SRC/openSUSE:Factory/python-portpicker/python-portpicker.changes 2015-12-09 22:31:35.000000000 +0100 +++ /work/SRC/openSUSE:Factory/.python-portpicker.new.5148/python-portpicker.changes 2019-05-22 15:42:02.318427055 +0200 @@ -1,0 +2,18 @@ +Mon May 20 15:18:38 UTC 2019 - pgajdos@suse.com + +- version update to 1.3.1 + * Fix a race condition in `pick_unused_port()` involving the free ports set. + * Adds an optional `portserver_address` parameter to `pick_unused_port()` so + that callers can specify their own regardless of `os.environ`. + * `pick_unused_port()` now raises `NoFreePortFoundError` when no available port + could be found rather than spinning in a loop trying forever. + * Fall back to `socket.AF_INET` when `socket.AF_UNIX` support is not available + to communicate with a portserver. + * Introduced `add_reserved_port()` and `return_port()` APIs to allow ports to + be recycled and allow users to bring ports of their own. + * Changed default port range to 15000-24999 to avoid ephemeral ports. + * Portserver bugfix. +- convert to single spec +- run test + +------------------------------------------------------------------- Old: ---- portpicker-1.1.0.tar.gz New: ---- portpicker-1.3.1.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ python-portpicker.spec ++++++ --- /var/tmp/diff_new_pack.gyRcto/_old 2019-05-22 15:42:04.218427047 +0200 +++ /var/tmp/diff_new_pack.gyRcto/_new 2019-05-22 15:42:04.234427047 +0200 @@ -1,7 +1,7 @@ # # spec file for package python-portpicker # -# Copyright (c) 2015 SUSE LINUX GmbH, Nuernberg, Germany. +# Copyright (c) 2019 SUSE LINUX GmbH, Nuernberg, Germany. # # All modifications and additions to the file contributed by third parties # remain the property of their copyright owners, unless otherwise agreed @@ -12,28 +12,26 @@ # license that conforms to the Open Source Definition (Version 1.9) # published by the Open Source Initiative. -# Please submit bugfixes or comments via http://bugs.opensuse.org/ +# Please submit bugfixes or comments via https://bugs.opensuse.org/ # -%define upstream_name portpicker +%{?!python_module:%define python_module() python-%{**} python3-%{**}} Name: python-portpicker -Version: 1.1.0 +Version: 1.3.1 Release: 0 Summary: A library to choose unique available network ports License: Apache-2.0 Group: Development/Libraries/Python -Url: https://github.com/google/python_portpicker -Source0: %{upstream_name}-%{version}.tar.gz -Requires: python -BuildRequires: python-setuptools -BuildRoot: %{_tmppath}/%{name}-%{version}-build - -%if 0%{?suse_version} && 0%{?suse_version} <= 1110 -%{!?python_sitelib: %global python_sitelib %(python -c "from distutils.sysconfig import get_python_lib; print get_python_lib()")} -%else +URL: https://github.com/google/python_portpicker +Source0: https://files.pythonhosted.org/packages/source/p/portpicker/portpicker-%{version}.tar.gz +BuildRequires: %{python_module setuptools} +BuildRequires: fdupes BuildArch: noarch -%endif +# SECTION test requirements +BuildRequires: %{python_module mock} +# /SECTION +%python_subpackages %description Portpicker provides an API to find and return an available network port for @@ -41,17 +39,23 @@ harnesses that launch local servers. %prep -%setup -q -n %{upstream_name}-%{version} +%setup -q -n portpicker-%{version} %build -python setup.py build +%python_build %install -python setup.py install --prefix=%{_prefix} --root=%{buildroot} +%python_install +%python_expand %fdupes %{buildroot}%{$python_sitelib} -%files -%defattr(-,root,root,-) -%doc CONTRIBUTING.md LICENSE README.md -%{python_sitelib}/%{upstream_name}* +%check +%python_expand PYTHONPATH=%{buildroot}%{$python_sitelib} $python src/tests/portpicker_test.py + +%files %{python_files} +%license LICENSE +%doc CONTRIBUTING.md README.md +%{python_sitelib}/* +# import asyncio +%python3_only %{_bindir}/portserver.py %changelog ++++++ portpicker-1.1.0.tar.gz -> portpicker-1.3.1.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/portpicker-1.1.0/ChangeLog.md new/portpicker-1.3.1/ChangeLog.md --- old/portpicker-1.1.0/ChangeLog.md 1970-01-01 01:00:00.000000000 +0100 +++ new/portpicker-1.3.1/ChangeLog.md 2019-03-05 17:20:26.000000000 +0100 @@ -0,0 +1,37 @@ +## 1.3.1 + + * Fix a race condition in `pick_unused_port()` involving the free ports set. + +## 1.3.0 + +* Adds an optional `portserver_address` parameter to `pick_unused_port()` so + that callers can specify their own regardless of `os.environ`. +* `pick_unused_port()` now raises `NoFreePortFoundError` when no available port + could be found rather than spinning in a loop trying forever. +* Fall back to `socket.AF_INET` when `socket.AF_UNIX` support is not available + to communicate with a portserver. + +## 1.2.0 + +* Introduced `add_reserved_port()` and `return_port()` APIs to allow ports to + be recycled and allow users to bring ports of their own. + +## 1.1.1 + +* Changed default port range to 15000-24999 to avoid ephemeral ports. +* Portserver bugfix. + +## 1.1.0 + +* Renamed portpicker APIs to use PEP8 style function names in code and docs. +* Legacy CapWords API name compatibility is maintained (and explicitly tested). + +## 1.0.1 + +* Code reindented to use 4 space indents and run through + [YAPF](https://github.com/google/yapf) for consistent style. +* Not packaged for release. + +## 1.0.0 + +* Original open source release. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/portpicker-1.1.0/MANIFEST.in new/portpicker-1.3.1/MANIFEST.in --- old/portpicker-1.1.0/MANIFEST.in 1970-01-01 01:00:00.000000000 +0100 +++ new/portpicker-1.3.1/MANIFEST.in 2017-10-09 19:32:19.000000000 +0200 @@ -0,0 +1,8 @@ +include src/port*.py +include src/tests/port*.py +include README.md +include LICENSE +include CONTRIBUTING.md +include ChangeLog.md +include setup.py +include test.sh diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/portpicker-1.1.0/PKG-INFO new/portpicker-1.3.1/PKG-INFO --- old/portpicker-1.1.0/PKG-INFO 2015-04-02 02:39:05.000000000 +0200 +++ new/portpicker-1.3.1/PKG-INFO 2019-03-05 17:54:53.000000000 +0100 @@ -1,11 +1,12 @@ Metadata-Version: 1.1 Name: portpicker -Version: 1.1.0 +Version: 1.3.1 Summary: A library to choose unique available network ports. Home-page: https://github.com/google/python_portpicker Author: Google Author-email: greg@krypto.org License: Apache 2.0 +Description-Content-Type: UNKNOWN Description: Portpicker provides an API to find and return an available network port for an application to bind to. Ideally suited for use from unittests or for test harnesses that launch local servers. @@ -19,7 +20,8 @@ Classifier: Programming Language :: Python :: 3 Classifier: Programming Language :: Python :: 3.3 Classifier: Programming Language :: Python :: 3.4 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 Classifier: Programming Language :: Python :: Implementation :: CPython Classifier: Programming Language :: Python :: Implementation :: Jython Classifier: Programming Language :: Python :: Implementation :: PyPy -Requires: mock(>=1.0) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/portpicker-1.1.0/README.md new/portpicker-1.3.1/README.md --- old/portpicker-1.1.0/README.md 2015-04-02 02:20:41.000000000 +0200 +++ new/portpicker-1.3.1/README.md 2019-01-15 22:11:37.000000000 +0100 @@ -1,6 +1,6 @@ # Python portpicker module -This module is useful finding unused network ports on a host. +This module is useful for finding unused network ports on a host. It supports both Python 2 and Python 3. This module provides a pure Python `pick_unused_port()` function. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/portpicker-1.1.0/setup.cfg new/portpicker-1.3.1/setup.cfg --- old/portpicker-1.1.0/setup.cfg 1970-01-01 01:00:00.000000000 +0100 +++ new/portpicker-1.3.1/setup.cfg 2019-03-05 17:54:53.000000000 +0100 @@ -0,0 +1,4 @@ +[egg_info] +tag_build = +tag_date = 0 + diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/portpicker-1.1.0/setup.py new/portpicker-1.3.1/setup.py --- old/portpicker-1.1.0/setup.py 2015-04-02 02:30:48.000000000 +0200 +++ new/portpicker-1.3.1/setup.py 2019-03-05 17:23:51.000000000 +0100 @@ -14,11 +14,13 @@ # """Simple distutils setup for the pure Python portpicker.""" -import distutils.core import sys import textwrap +import setuptools + + def main(): requires = [] scripts = [] @@ -31,9 +33,9 @@ # The example portserver implementation requires Python 3 and asyncio. scripts.append('src/portserver.py') - distutils.core.setup( + setuptools.setup( name='portpicker', - version='1.1.0', + version='1.3.1', description='A library to choose unique available network ports.', long_description=textwrap.dedent("""\ Portpicker provides an API to find and return an available network @@ -57,6 +59,8 @@ 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: Jython', 'Programming Language :: Python :: Implementation :: PyPy']) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/portpicker-1.1.0/src/portpicker.egg-info/PKG-INFO new/portpicker-1.3.1/src/portpicker.egg-info/PKG-INFO --- old/portpicker-1.1.0/src/portpicker.egg-info/PKG-INFO 1970-01-01 01:00:00.000000000 +0100 +++ new/portpicker-1.3.1/src/portpicker.egg-info/PKG-INFO 2019-03-05 17:54:53.000000000 +0100 @@ -0,0 +1,27 @@ +Metadata-Version: 1.1 +Name: portpicker +Version: 1.3.1 +Summary: A library to choose unique available network ports. +Home-page: https://github.com/google/python_portpicker +Author: Google +Author-email: greg@krypto.org +License: Apache 2.0 +Description-Content-Type: UNKNOWN +Description: Portpicker provides an API to find and return an available network + port for an application to bind to. Ideally suited for use from + unittests or for test harnesses that launch local servers. +Platform: POSIX +Classifier: Development Status :: 5 - Production/Stable +Classifier: License :: OSI Approved :: Apache Software License +Classifier: Intended Audience :: Developers +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 2 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.3 +Classifier: Programming Language :: Python :: 3.4 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: Implementation :: Jython +Classifier: Programming Language :: Python :: Implementation :: PyPy diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/portpicker-1.1.0/src/portpicker.egg-info/SOURCES.txt new/portpicker-1.3.1/src/portpicker.egg-info/SOURCES.txt --- old/portpicker-1.1.0/src/portpicker.egg-info/SOURCES.txt 1970-01-01 01:00:00.000000000 +0100 +++ new/portpicker-1.3.1/src/portpicker.egg-info/SOURCES.txt 2019-03-05 17:54:53.000000000 +0100 @@ -0,0 +1,15 @@ +CONTRIBUTING.md +ChangeLog.md +LICENSE +MANIFEST.in +README.md +setup.py +test.sh +src/portpicker.py +src/portserver.py +src/portpicker.egg-info/PKG-INFO +src/portpicker.egg-info/SOURCES.txt +src/portpicker.egg-info/dependency_links.txt +src/portpicker.egg-info/top_level.txt +src/tests/portpicker_test.py +src/tests/portserver_test.py \ No newline at end of file diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/portpicker-1.1.0/src/portpicker.egg-info/dependency_links.txt new/portpicker-1.3.1/src/portpicker.egg-info/dependency_links.txt --- old/portpicker-1.1.0/src/portpicker.egg-info/dependency_links.txt 1970-01-01 01:00:00.000000000 +0100 +++ new/portpicker-1.3.1/src/portpicker.egg-info/dependency_links.txt 2019-03-05 17:54:53.000000000 +0100 @@ -0,0 +1 @@ + diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/portpicker-1.1.0/src/portpicker.egg-info/top_level.txt new/portpicker-1.3.1/src/portpicker.egg-info/top_level.txt --- old/portpicker-1.1.0/src/portpicker.egg-info/top_level.txt 1970-01-01 01:00:00.000000000 +0100 +++ new/portpicker-1.3.1/src/portpicker.egg-info/top_level.txt 2019-03-05 17:54:53.000000000 +0100 @@ -0,0 +1 @@ +portpicker diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/portpicker-1.1.0/src/portpicker.py new/portpicker-1.3.1/src/portpicker.py --- old/portpicker-1.1.0/src/portpicker.py 2015-04-02 02:20:57.000000000 +0200 +++ new/portpicker-1.3.1/src/portpicker.py 2019-03-05 17:20:26.000000000 +0100 @@ -36,25 +36,65 @@ """ from __future__ import print_function + +import logging import os import random import socket import sys # The legacy Bind, IsPortFree, etc. names are not exported. -__all__ = ('bind', 'is_port_free', 'pick_unused_port', - 'get_port_from_port_server') +__all__ = ('bind', 'is_port_free', 'pick_unused_port', 'return_port', + 'add_reserved_port', 'get_port_from_port_server') _PROTOS = [(socket.SOCK_STREAM, socket.IPPROTO_TCP), (socket.SOCK_DGRAM, socket.IPPROTO_UDP)] +# Ports that are currently available to be given out. +_free_ports = set() + +# Ports that are reserved or from the portserver that may be returned. +_owned_ports = set() + +# Ports that we chose randomly that may be returned. +_random_ports = set() + + +class NoFreePortFoundError(Exception): + """Exception indicating that no free port could be found.""" + pass + + +def add_reserved_port(port): + """Add a port that was acquired by means other than the port server.""" + _free_ports.add(port) + + +def return_port(port): + """Return a port that is no longer being used so it can be reused.""" + if port in _random_ports: + _random_ports.remove(port) + elif port in _owned_ports: + _owned_ports.remove(port) + _free_ports.add(port) + elif port in _free_ports: + logging.info("Returning a port that was already returned: %s", port) + else: + logging.info("Returning a port that wasn't given by portpicker: %s", + port) + + def bind(port, socket_type, socket_proto): """Try to bind to a socket of the specified type, protocol, and port. This is primarily a helper function for PickUnusedPort, used to see if a particular port number is available. + For the port to be considered available, the kernel must support at least + one of (IPv6, IPv4), and the port must be available on each supported + family. + Args: port: The port number to bind to, or 0 to have the OS pick a free port. socket_type: The type of the socket (ex: socket.SOCK_STREAM). @@ -63,15 +103,24 @@ Returns: The port number on success or None on failure. """ - sock = socket.socket(socket.AF_INET, socket_type, socket_proto) - try: - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - sock.bind(('', port)) - return sock.getsockname()[1] - except socket.error: - return None - finally: - sock.close() + got_socket = False + for family in (socket.AF_INET6, socket.AF_INET): + try: + sock = socket.socket(family, socket_type, socket_proto) + got_socket = True + except socket.error: + continue + try: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind(('', port)) + if socket_type == socket.SOCK_STREAM: + sock.listen(1) + port = sock.getsockname()[1] + except socket.error: + return None + finally: + sock.close() + return port if got_socket else None Bind = bind # legacy API. pylint: disable=invalid-name @@ -84,31 +133,49 @@ Returns: boolean, whether it is free to use for both TCP and UDP """ - return (bind(port, _PROTOS[0][0], _PROTOS[0][1]) and - bind(port, _PROTOS[1][0], _PROTOS[1][1])) + return bind(port, *_PROTOS[0]) and bind(port, *_PROTOS[1]) IsPortFree = is_port_free # legacy API. pylint: disable=invalid-name -def pick_unused_port(pid=None): +def pick_unused_port(pid=None, portserver_address=None): """A pure python implementation of PickUnusedPort. Args: pid: PID to tell the portserver to associate the reservation with. If - None, - the current process's PID is used. + None, the current process's PID is used. + portserver_address: The address (path) of a unix domain socket + with which to connect to a portserver, a leading '@' + character indicates an address in the "abstract namespace". OR + On systems without socket.AF_UNIX, this is an AF_INET address. + If None, or no port is returned by the portserver at the provided + address, the environment will be checked for a PORTSERVER_ADDRESS + variable. If that is not set, no port server will be used. Returns: A port number that is unused on both TCP and UDP. + + Raises: + NoFreePortFoundError: No free port could be found. """ - port = None + try: # Instead of `if _free_ports:` to handle the race condition. + port = _free_ports.pop() + except KeyError: + pass + else: + _owned_ports.add(port) + return port # Provide access to the portserver on an opt-in basis. + if portserver_address: + port = get_port_from_port_server(portserver_address, pid=pid) + if port: + return port if 'PORTSERVER_ADDRESS' in os.environ: port = get_port_from_port_server(os.environ['PORTSERVER_ADDRESS'], pid=pid) - if not port: - return _pick_unused_port_without_server() - return port + if port: + return port + return _pick_unused_port_without_server() PickUnusedPort = pick_unused_port # legacy API. pylint: disable=invalid-name @@ -122,25 +189,33 @@ should not be called by code outside of this module. Returns: - A port number that is unused on both TCP and UDP. None on error. + A port number that is unused on both TCP and UDP. + + Raises: + NoFreePortFoundError: No free port could be found. """ # Try random ports first. rng = random.Random() for _ in range(10): - port = int(rng.randrange(32768, 60000)) + port = int(rng.randrange(15000, 25000)) if is_port_free(port): + _random_ports.add(port) return port - # Try OS-assigned ports next. + # Next, try a few times to get an OS-assigned port. # Ambrose discovered that on the 2.6 kernel, calling Bind() on UDP socket # returns the same port over and over. So always try TCP first. - while True: + for _ in range(10): # Ask the OS for an unused port. port = bind(0, _PROTOS[0][0], _PROTOS[0][1]) # Check if this port is unused on the other protocol. if port and bind(port, _PROTOS[1][0], _PROTOS[1][1]): + _random_ports.add(port) return port + # Give up. + raise NoFreePortFoundError() + def get_port_from_port_server(portserver_address, pid=None): """Request a free a port from a system-wide portserver. @@ -156,6 +231,7 @@ portserver_address: The address (path) of a unix domain socket with which to connect to the portserver. A leading '@' character indicates an address in the "abstract namespace." + On systems without socket.AF_UNIX, this is an AF_INET address. pid: The PID to tell the portserver to associate the reservation with. If None, the current process's PID is used. @@ -176,7 +252,11 @@ try: # Create socket. - sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + if hasattr(socket, 'AF_UNIX'): + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + else: + # fallback to AF_INET if this is not unix + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: # Connect to portserver. sock.connect(portserver_address) @@ -189,15 +269,19 @@ buf = sock.recv(1024) finally: sock.close() - except socket.error: - print('Socket error when connecting to portserver.', file=sys.stderr) + except socket.error as e: + print('Socket error when connecting to portserver:', e, + file=sys.stderr) return None try: - return int(buf.split(b'\n')[0]) + port = int(buf.split(b'\n')[0]) except ValueError: print('Portserver failed to find a port.', file=sys.stderr) return None + _owned_ports.add(port) + return port + GetPortFromPortServer = get_port_from_port_server # legacy API. pylint: disable=invalid-name diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/portpicker-1.1.0/src/portserver.py new/portpicker-1.3.1/src/portserver.py --- old/portpicker-1.1.0/src/portserver.py 2015-04-02 01:34:01.000000000 +0200 +++ new/portpicker-1.3.1/src/portserver.py 2017-10-09 19:32:19.000000000 +0200 @@ -38,6 +38,9 @@ log = None # Initialized to a logging.Logger by _configure_logging(). +_PROTOS = [(socket.SOCK_STREAM, socket.IPPROTO_TCP), + (socket.SOCK_DGRAM, socket.IPPROTO_UDP)] + def _get_process_command_line(pid): try: @@ -55,23 +58,51 @@ return 0 -def _port_is_available(port): - """Return False if the given network port is currently in use.""" - for socket_type, proto in ((socket.SOCK_STREAM, socket.IPPROTO_TCP), - (socket.SOCK_DGRAM, 0)): - sock = None +# TODO: Consider importing portpicker.bind() instead of duplicating the code. +def _bind(port, socket_type, socket_proto): + """Try to bind to a socket of the specified type, protocol, and port. + + For the port to be considered available, the kernel must support at least + one of (IPv6, IPv4), and the port must be available on each supported + family. + + Args: + port: The port number to bind to, or 0 to have the OS pick a free port. + socket_type: The type of the socket (ex: socket.SOCK_STREAM). + socket_proto: The protocol of the socket (ex: socket.IPPROTO_TCP). + + Returns: + The port number on success or None on failure. + """ + got_socket = False + for family in (socket.AF_INET6, socket.AF_INET): + try: + sock = socket.socket(family, socket_type, socket_proto) + got_socket = True + except socket.error: + continue try: - sock = socket.socket(socket.AF_INET, socket_type, proto) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) sock.bind(('', port)) if socket_type == socket.SOCK_STREAM: sock.listen(1) + port = sock.getsockname()[1] except socket.error: - return False + return None finally: - if sock: - sock.close() - return True + sock.close() + return port if got_socket else None + + +def _is_port_free(port): + """Check if specified port is free. + + Args: + port: integer, port to check + Returns: + boolean, whether it is free to use for both TCP and UDP + """ + return _bind(port, *_PROTOS[0]) and _bind(port, *_PROTOS[1]) def _should_allocate_port(pid): @@ -149,7 +180,7 @@ check_count += 1 if (candidate.start_time == 0 or candidate.start_time != _get_process_start_time(candidate.pid)): - if _port_is_available(candidate.pid): + if _is_port_free(candidate.port): candidate.pid = pid candidate.start_time = _get_process_start_time(pid) if not candidate.start_time: @@ -252,8 +283,8 @@ parser.add_argument( '--portserver_static_pool', type=str, - default='32768-60000', - help='Comma separated N-P Range(s) of ports to manage.') + default='15000-24999', + help='Comma separated N-P Range(s) of ports to manage (inclusive).') parser.add_argument( '--portserver_unix_socket_address', type=str, diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/portpicker-1.1.0/src/tests/portpicker_test.py new/portpicker-1.3.1/src/tests/portpicker_test.py --- old/portpicker-1.1.0/src/tests/portpicker_test.py 2015-04-02 02:16:51.000000000 +0200 +++ new/portpicker-1.3.1/src/tests/portpicker_test.py 2019-01-18 02:31:24.000000000 +0100 @@ -16,9 +16,11 @@ # """Unittests for the portpicker module.""" +from __future__ import print_function import os import random import socket +import sys import unittest try: @@ -40,6 +42,9 @@ def setUp(self): # So we can Bind even if portpicker.bind is stubbed out. self._bind = portpicker.bind + portpicker._owned_ports.clear() + portpicker._free_ports.clear() + portpicker._random_ports.clear() def testPickUnusedPortActuallyWorks(self): """This test can be flaky.""" @@ -65,6 +70,26 @@ @unittest.skipIf('PORTSERVER_ADDRESS' not in os.environ, 'no port server to test against') + def testPickUnusedCanSuccessfullyUsePortServerAddressKwarg(self): + + with mock.patch.object(portpicker, '_pick_unused_port_without_server'): + portpicker._pick_unused_port_without_server.side_effect = ( + Exception('eek!') + ) + + # Since _PickUnusedPortWithoutServer() raises an exception, and + # we've temporarily removed PORTSERVER_ADDRESS from os.environ, if + # we can successfully obtain a port, the portserver must be working. + addr = os.environ.pop('PORTSERVER_ADDRESS') + try: + port = portpicker.pick_unused_port(portserver_address=addr) + self.assertTrue(self.IsUnusedTCPPort(port)) + self.assertTrue(self.IsUnusedUDPPort(port)) + finally: + os.environ['PORTSERVER_ADDRESS'] = addr + + @unittest.skipIf('PORTSERVER_ADDRESS' not in os.environ, + 'no port server to test against') def testGetPortFromPortServer(self): """Exercise the get_port_from_port_server() helper function.""" for _ in range(10): @@ -90,6 +115,50 @@ server.sendall.assert_called_once_with(b'9876\n') self.assertEqual(port, 52768) + @mock.patch.dict(os.environ,{'PORTSERVER_ADDRESS': 'portserver'}) + def testReusesPortServerPorts(self): + server = mock.Mock() + server.recv.side_effect = [b'12345\n', b'23456\n', b'34567\n'] + with mock.patch.object(socket, 'socket', return_value=server): + self.assertEqual(portpicker.pick_unused_port(), 12345) + self.assertEqual(portpicker.pick_unused_port(), 23456) + portpicker.return_port(12345) + self.assertEqual(portpicker.pick_unused_port(), 12345) + + @mock.patch.dict(os.environ,{'PORTSERVER_ADDRESS': ''}) + def testDoesntReuseRandomPorts(self): + ports = set() + for _ in range(10): + port = portpicker.pick_unused_port() + ports.add(port) + portpicker.return_port(port) + self.assertGreater(len(ports), 5) # Allow some random reuse. + + def testReturnsReservedPorts(self): + with mock.patch.object(portpicker, '_pick_unused_port_without_server'): + portpicker._pick_unused_port_without_server.side_effect = ( + Exception('eek!')) + # Arbitrary port. In practice you should get this from somewhere + # that assigns ports. + reserved_port = 28465 + portpicker.add_reserved_port(reserved_port) + ports = set() + for _ in range(10): + port = portpicker.pick_unused_port() + ports.add(port) + portpicker.return_port(port) + self.assertEqual(len(ports), 1) + self.assertEqual(ports.pop(), reserved_port) + + @mock.patch.dict(os.environ,{'PORTSERVER_ADDRESS': ''}) + def testFallsBackToRandomAfterRunningOutOfReservedPorts(self): + # Arbitrary port. In practice you should get this from somewhere + # that assigns ports. + reserved_port = 23456 + portpicker.add_reserved_port(reserved_port) + self.assertEqual(portpicker.pick_unused_port(), reserved_port) + self.assertNotEqual(portpicker.pick_unused_port(), reserved_port) + def testRandomlyChosenPorts(self): # Unless this box is under an overwhelming socket load, this test # will heavily exercise the "pick a port randomly" part of the @@ -132,10 +201,67 @@ return None with mock.patch.object(portpicker, 'bind', bind_with_error): + got_at_least_one_port = False for _ in range(100): - port = portpicker._pick_unused_port_without_server() - self.assertTrue(self.IsUnusedTCPPort(port)) - self.assertTrue(self.IsUnusedUDPPort(port)) + try: + port = portpicker._pick_unused_port_without_server() + except portpicker.NoFreePortFoundError: + continue + else: + got_at_least_one_port = True + self.assertTrue(self.IsUnusedTCPPort(port)) + self.assertTrue(self.IsUnusedUDPPort(port)) + self.assertTrue(got_at_least_one_port) + + def testIsPortFree(self): + """This might be flaky unless this test is run with a portserver.""" + # The port should be free initially. + port = portpicker.pick_unused_port() + self.assertTrue(portpicker.is_port_free(port)) + + cases = [ + (socket.AF_INET, socket.SOCK_STREAM, None), + (socket.AF_INET6, socket.SOCK_STREAM, 0), + (socket.AF_INET6, socket.SOCK_STREAM, 1), + (socket.AF_INET, socket.SOCK_DGRAM, None), + (socket.AF_INET6, socket.SOCK_DGRAM, 0), + (socket.AF_INET6, socket.SOCK_DGRAM, 1), + ] + for (sock_family, sock_type, v6only) in cases: + # Occupy the port on a subset of possible protocols. + try: + sock = socket.socket(sock_family, sock_type, 0) + except socket.error: + print('Kernel does not support sock_family=%d' % sock_family, + file=sys.stderr) + # Skip this case, since we cannot occupy a port. + continue + + if not hasattr(socket, 'IPPROTO_IPV6'): + v6only = None + + if v6only is not None: + try: + sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, + v6only) + except socket.error: + print('Kernel does not support IPV6_V6ONLY=%d' % v6only, + file=sys.stderr) + # Don't care; just proceed with the default. + sock.bind(('', port)) + + # The port should be busy. + self.assertFalse(portpicker.is_port_free(port)) + sock.close() + + # Now it's free again. + self.assertTrue(portpicker.is_port_free(port)) + + def testIsPortFreeException(self): + port = portpicker.pick_unused_port() + with mock.patch.object(socket, 'socket') as mock_sock: + mock_sock.side_effect = socket.error('fake socket error', 0) + self.assertFalse(portpicker.is_port_free(port)) def testThatLegacyCapWordsAPIsExist(self): """The original APIs were CapWords style, 1.1 added PEP8 names.""" diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/portpicker-1.1.0/src/tests/portserver_test.py new/portpicker-1.3.1/src/tests/portserver_test.py --- old/portpicker-1.1.0/src/tests/portserver_test.py 2015-04-02 01:35:39.000000000 +0200 +++ new/portpicker-1.3.1/src/tests/portserver_test.py 2017-10-09 19:32:19.000000000 +0200 @@ -16,6 +16,7 @@ # """Tests for the example portserver.""" +from __future__ import print_function import asyncio import os import socket @@ -43,15 +44,49 @@ def test_get_process_start_time(self): self.assertGreater(portserver._get_process_start_time(os.getpid()), 0) - def test_port_is_available_true(self): + def test_is_port_free(self): """This might be flaky unless this test is run with a portserver.""" - # Insert Inception "we must go deeper" meme here. - self.assertTrue(portserver._port_is_available(self.port)) + # The port should be free initially. + self.assertTrue(portserver._is_port_free(self.port)) - def test_port_is_available_false(self): + cases = [ + (socket.AF_INET, socket.SOCK_STREAM, None), + (socket.AF_INET6, socket.SOCK_STREAM, 0), + (socket.AF_INET6, socket.SOCK_STREAM, 1), + (socket.AF_INET, socket.SOCK_DGRAM, None), + (socket.AF_INET6, socket.SOCK_DGRAM, 0), + (socket.AF_INET6, socket.SOCK_DGRAM, 1), + ] + for (sock_family, sock_type, v6only) in cases: + # Occupy the port on a subset of possible protocols. + try: + sock = socket.socket(sock_family, sock_type, 0) + except socket.error: + print('Kernel does not support sock_family=%d' % sock_family, + file=sys.stderr) + # Skip this case, since we cannot occupy a port. + continue + if v6only is not None: + try: + sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, + v6only) + except socket.error: + print('Kernel does not support IPV6_V6ONLY=%d' % v6only, + file=sys.stderr) + # Don't care; just proceed with the default. + sock.bind(('', self.port)) + + # The port should be busy. + self.assertFalse(portserver._is_port_free(self.port)) + sock.close() + + # Now it's free again. + self.assertTrue(portserver._is_port_free(self.port)) + + def test_is_port_free_exception(self): with mock.patch.object(socket, 'socket') as mock_sock: mock_sock.side_effect = socket.error('fake socket error', 0) - self.assertFalse(portserver._port_is_available(self.port)) + self.assertFalse(portserver._is_port_free(self.port)) def test_should_allocate_port(self): self.assertFalse(portserver._should_allocate_port(0)) @@ -140,23 +175,45 @@ self.assertRaises(ValueError, self.pool.add_port_to_free_pool, 0) self.assertRaises(ValueError, self.pool.add_port_to_free_pool, 65536) - @mock.patch.object(portserver, '_port_is_available') - def test_get_port_for_process_ok(self, mock_port_is_available): + @mock.patch.object(portserver, '_is_port_free') + def test_get_port_for_process_ok(self, mock_is_port_free): self.pool.add_port_to_free_pool(self.port) - mock_port_is_available.return_value = True + mock_is_port_free.return_value = True self.assertEqual(self.port, self.pool.get_port_for_process(os.getpid())) self.assertEqual(1, self.pool.ports_checked_for_last_request) - @mock.patch.object(portserver, '_port_is_available') - def test_get_port_for_process_none_left(self, mock_port_is_available): + @mock.patch.object(portserver, '_is_port_free') + def test_get_port_for_process_none_left(self, mock_is_port_free): self.pool.add_port_to_free_pool(self.port) self.pool.add_port_to_free_pool(22) - mock_port_is_available.return_value = False + mock_is_port_free.return_value = False self.assertEqual(2, self.pool.num_ports()) self.assertEqual(0, self.pool.get_port_for_process(os.getpid())) self.assertEqual(2, self.pool.num_ports()) self.assertEqual(2, self.pool.ports_checked_for_last_request) + @mock.patch.object(portserver, '_is_port_free') + @mock.patch.object(os, 'getpid') + def test_get_port_for_process_pid_eq_port(self, mock_getpid, mock_is_port_free): + self.pool.add_port_to_free_pool(12345) + self.pool.add_port_to_free_pool(12344) + mock_is_port_free.side_effect = lambda port: port == os.getpid() + mock_getpid.return_value = 12345 + self.assertEqual(2, self.pool.num_ports()) + self.assertEqual(12345, self.pool.get_port_for_process(os.getpid())) + self.assertEqual(2, self.pool.ports_checked_for_last_request) + + @mock.patch.object(portserver, '_is_port_free') + @mock.patch.object(os, 'getpid') + def test_get_port_for_process_pid_ne_port(self, mock_getpid, mock_is_port_free): + self.pool.add_port_to_free_pool(12344) + self.pool.add_port_to_free_pool(12345) + mock_is_port_free.side_effect = lambda port: port != os.getpid() + mock_getpid.return_value = 12345 + self.assertEqual(2, self.pool.num_ports()) + self.assertEqual(12344, self.pool.get_port_for_process(os.getpid())) + self.assertEqual(2, self.pool.ports_checked_for_last_request) + @mock.patch.object(portserver, '_get_process_command_line') @mock.patch.object(portserver, '_should_allocate_port')