April 2020 - openSUSE Commits - openSUSE Mailing Lists

commit python-libarchive-c for openSUSE:Leap:15.2
by root 01 May '20

01 May '20

Hello community, here is the log from the commit of package python-libarchive-c for openSUSE:Leap:15.2 checked in at 2020-04-30 18:51:40 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Leap:15.2/python-libarchive-c (Old) and /work/SRC/openSUSE:Leap:15.2/.python-libarchive-c.new.2738 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Package is "python-libarchive-c" Thu Apr 30 18:51:40 2020 rev:12 rq:794504 version:2.9 Changes: -------- --- /work/SRC/openSUSE:Leap:15.2/python-libarchive-c/python-libarchive-c.changes 2020-03-09 18:07:40.152886132 +0100 +++ /work/SRC/openSUSE:Leap:15.2/.python-libarchive-c.new.2738/python-libarchive-c.changes 2020-04-30 18:51:41.448630156 +0200 @@ -1,0 +2,15 @@ +Wed Apr 15 12:28:28 UTC 2020 - pgajdos(a)suse.com + +- version update to 2.9 + * Changes to the library: + optimized the ArchiveEntry class (#94) + added support for the zstd format (#90) + fixed the top-level __all__ list (7b97d8b) + updated the code to support Python 3.8 (#88) + improved the add_file_from_memory method (#85 and #86) + lowered the level of log messages from warning to info (#83) + * Tests: + dropped testing with Python 3.4, added 3.7 and 3.8 (#88) + fixed unicode handling in entry tests (#84) + +------------------------------------------------------------------- Old: ---- libarchive-c-2.8.tar.gz New: ---- libarchive-c-2.9.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ python-libarchive-c.spec ++++++ --- /var/tmp/diff_new_pack.dwxCj4/_old 2020-04-30 18:51:41.908631136 +0200 +++ /var/tmp/diff_new_pack.dwxCj4/_new 2020-04-30 18:51:41.912631145 +0200 @@ -1,7 +1,7 @@ # # spec file for package python-libarchive-c # -# Copyright (c) 2018 SUSE LINUX GmbH, Nuernberg, Germany. +# Copyright (c) 2020 SUSE LLC # # All modifications and additions to the file contributed by third parties # remain the property of their copyright owners, unless otherwise agreed @@ -17,9 +17,8 @@ %{?!python_module:%define python_module() python-%{**} python3-%{**}} -%bcond_without test Name: python-libarchive-c -Version: 2.8 +Version: 2.9 Release: 0 Summary: Python interface to libarchive License: CC0-1.0 @@ -31,11 +30,11 @@ BuildRequires: python-rpm-macros Requires: libarchive-devel BuildArch: noarch -%if %{with test} +# SECTION test requirements BuildRequires: %{python_module mock} BuildRequires: %{python_module pytest} BuildRequires: libarchive-devel -%endif +# /SECTION %python_subpackages %description @@ -52,10 +51,9 @@ %python_install %python_expand %fdupes %{buildroot}%{$python_sitelib} -%if %{with test} %check -%python_exec setup.py test -%endif +export LANG="en_US.UTF-8" +%pytest %files %{python_files} %doc README.rst ++++++ libarchive-c-2.8.tar.gz -> libarchive-c-2.9.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libarchive-c-2.8/.travis.yml new/libarchive-c-2.9/.travis.yml --- old/libarchive-c-2.8/.travis.yml 2018-06-10 15:01:15.000000000 +0200 +++ new/libarchive-c-2.9/.travis.yml 2019-10-20 13:18:23.000000000 +0200 @@ -2,9 +2,13 @@ matrix: include: - python: 3.5 - env: TOXENV=py27,py34,py35 + env: TOXENV=py27,py35 - python: 3.6 env: TOXENV=py36 + - python: 3.7 + env: TOXENV=py37 + - python: 3.8-dev + env: TOXENV=py38 branches: only: @@ -35,4 +39,4 @@ email: false sudo: required -dist: trusty +dist: xenial diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libarchive-c-2.8/PKG-INFO new/libarchive-c-2.9/PKG-INFO --- old/libarchive-c-2.8/PKG-INFO 2018-06-10 15:30:39.000000000 +0200 +++ new/libarchive-c-2.9/PKG-INFO 2019-10-20 14:40:31.000000000 +0200 @@ -1,6 +1,6 @@ -Metadata-Version: 1.0 +Metadata-Version: 2.1 Name: libarchive-c -Version: 2.8 +Version: 2.9 Summary: Python interface to libarchive Home-page: https://github.com/Changaco/python-libarchive-c Author: Changaco @@ -22,11 +22,21 @@ Compatibility ============= + python + ------ + python-libarchive-c is currently tested with python 2.7, 3.4, 3.5, and 3.6. If you find an incompatibility with older versions you can send us a small patch, but we won't accept big changes. + libarchive + ---------- + + python-libarchive-c may not work properly with obsolete versions of libarchive such as the ones included in MacOS. In that case you can install a recent version of libarchive (e.g. with ``brew install libarchive`` on MacOS) and use the ``LIBARCHIVE`` environment variable to point python-libarchive-c to it:: + + export LIBARCHIVE=/usr/local/Cellar/libarchive/3.3.3/lib/libarchive.13.dylib + Usage ===== @@ -68,3 +78,4 @@ Keywords: archive libarchive 7z tar bz2 zip gz Platform: UNKNOWN +Description-Content-Type: text/x-rst diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libarchive-c-2.8/README.rst new/libarchive-c-2.9/README.rst --- old/libarchive-c-2.8/README.rst 2018-06-10 13:44:41.000000000 +0200 +++ new/libarchive-c-2.9/README.rst 2018-12-06 11:57:08.000000000 +0100 @@ -14,11 +14,21 @@ Compatibility ============= +python +------ + python-libarchive-c is currently tested with python 2.7, 3.4, 3.5, and 3.6. If you find an incompatibility with older versions you can send us a small patch, but we won't accept big changes. +libarchive +---------- + +python-libarchive-c may not work properly with obsolete versions of libarchive such as the ones included in MacOS. In that case you can install a recent version of libarchive (e.g. with ``brew install libarchive`` on MacOS) and use the ``LIBARCHIVE`` environment variable to point python-libarchive-c to it:: + + export LIBARCHIVE=/usr/local/Cellar/libarchive/3.3.3/lib/libarchive.13.dylib + Usage ===== diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libarchive-c-2.8/libarchive/__init__.py new/libarchive-c-2.9/libarchive/__init__.py --- old/libarchive-c-2.8/libarchive/__init__.py 2018-06-10 15:01:15.000000000 +0200 +++ new/libarchive-c-2.9/libarchive/__init__.py 2019-10-20 13:18:23.000000000 +0200 @@ -6,10 +6,10 @@ ) from .write import custom_writer, fd_writer, file_writer, memory_writer -__all__ = [ +__all__ = [x.__name__ for x in ( ArchiveEntry, ArchiveError, extract_fd, extract_file, extract_memory, custom_reader, fd_reader, file_reader, memory_reader, stream_reader, custom_writer, fd_writer, file_writer, memory_writer -] +)] diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libarchive-c-2.8/libarchive/entry.py new/libarchive-c-2.9/libarchive/entry.py --- old/libarchive-c-2.8/libarchive/entry.py 2018-06-10 15:01:15.000000000 +0200 +++ new/libarchive-c-2.9/libarchive/entry.py 2019-10-20 13:18:23.000000000 +0200 @@ -24,6 +24,8 @@ class ArchiveEntry(object): + __slots__ = ('_archive_p', '_entry_p') + def __init__(self, archive_p, entry_p): self._archive_p = archive_p self._entry_p = entry_p diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libarchive-c-2.8/libarchive/ffi.py new/libarchive-c-2.9/libarchive/ffi.py --- old/libarchive-c-2.8/libarchive/ffi.py 2018-06-10 15:27:14.000000000 +0200 +++ new/libarchive-c-2.9/libarchive/ffi.py 2019-10-20 13:18:23.000000000 +0200 @@ -162,18 +162,18 @@ try: ffi('read_support_format_'+f_name, [c_archive_p], c_int, check_int) except AttributeError: # pragma: no cover - logger.warning('read format "%s" is not supported' % f_name) + logger.info('read format "%s" is not supported' % f_name) READ_FORMATS.remove(f_name) READ_FILTERS = set(( 'all', 'bzip2', 'compress', 'grzip', 'gzip', 'lrzip', 'lzip', 'lzma', - 'lzop', 'none', 'rpm', 'uu', 'xz', 'lz4' + 'lzop', 'none', 'rpm', 'uu', 'xz', 'lz4', 'zstd' )) for f_name in list(READ_FILTERS): try: ffi('read_support_filter_'+f_name, [c_archive_p], c_int, check_int) except AttributeError: # pragma: no cover - logger.warning('read filter "%s" is not supported' % f_name) + logger.info('read filter "%s" is not supported' % f_name) READ_FILTERS.remove(f_name) ffi('read_open', @@ -224,18 +224,18 @@ try: ffi('write_set_format_'+f_name, [c_archive_p], c_int, check_int) except AttributeError: # pragma: no cover - logger.warning('write format "%s" is not supported' % f_name) + logger.info('write format "%s" is not supported' % f_name) WRITE_FORMATS.remove(f_name) WRITE_FILTERS = set(( 'b64encode', 'bzip2', 'compress', 'grzip', 'gzip', 'lrzip', 'lzip', 'lzma', - 'lzop', 'uuencode', 'xz', 'lz4' + 'lzop', 'uuencode', 'xz', 'lz4', 'zstd' )) for f_name in list(WRITE_FILTERS): try: ffi('write_add_filter_'+f_name, [c_archive_p], c_int, check_int) except AttributeError: # pragma: no cover - logger.warning('write filter "%s" is not supported' % f_name) + logger.info('write filter "%s" is not supported' % f_name) WRITE_FILTERS.remove(f_name) ffi('write_open', diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libarchive-c-2.8/libarchive/write.py new/libarchive-c-2.9/libarchive/write.py --- old/libarchive-c-2.8/libarchive/write.py 2018-06-10 15:27:14.000000000 +0200 +++ new/libarchive-c-2.9/libarchive/write.py 2019-10-20 14:34:28.000000000 +0200 @@ -77,25 +77,40 @@ def add_file_from_memory( self, entry_path, entry_size, entry_data, - filetype=REGULAR_FILE, - permission=DEFAULT_UNIX_PERMISSION + filetype=REGULAR_FILE, permission=DEFAULT_UNIX_PERMISSION, + atime=None, mtime=None, ctime=None, birthtime=None, ): """"Add file from memory to archive. :param entry_path: where entry should be places in archive :type entry_path: str - :param entry_size: entire size of entry + :param entry_size: entire size of entry in bytes :type entry_size: int :param entry_data: content of entry - :type entry_data: iterable + :type entry_data: bytes or Iterable[bytes] :param filetype: which type of file: normal, symlink etc. should entry be created as :type filetype: octal number :param permission: with which permission should entry be created :type permission: octal number + :param atime: Last access time + :type atime: int seconds or tuple (int seconds, int nanoseconds) + :param mtime: Last modified time + :type mtime: int seconds or tuple (int seconds, int nanoseconds) + :param ctime: Creation time + :type ctime: int seconds or tuple (int seconds, int nanoseconds) + :param birthtime: Birth time (for archive formats that support it) + :type birthtime: int seconds or tuple (int seconds, int nanoseconds) """ archive_pointer = self._pointer + if isinstance(entry_data, bytes): + entry_data = (entry_data,) + elif isinstance(entry_data, str): + raise TypeError( + "entry_data: expected bytes, got %r" % type(entry_data) + ) + with new_archive_entry() as archive_entry_pointer: archive_entry = ArchiveEntry(None, archive_entry_pointer) @@ -103,6 +118,23 @@ entry_set_size(archive_entry_pointer, entry_size) entry_set_filetype(archive_entry_pointer, filetype) entry_set_perm(archive_entry_pointer, permission) + + if atime is not None: + if not isinstance(atime, tuple): + atime = (atime, 0) + archive_entry.set_atime(*atime) + if mtime is not None: + if not isinstance(mtime, tuple): + mtime = (mtime, 0) + archive_entry.set_mtime(*mtime) + if ctime is not None: + if not isinstance(ctime, tuple): + ctime = (ctime, 0) + archive_entry.set_ctime(*ctime) + if birthtime is not None: + if not isinstance(birthtime, tuple): + birthtime = (birthtime, 0) + archive_entry.set_birthtime(*birthtime) write_header(archive_pointer, archive_entry_pointer) for chunk in entry_data: diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libarchive-c-2.8/libarchive_c.egg-info/PKG-INFO new/libarchive-c-2.9/libarchive_c.egg-info/PKG-INFO --- old/libarchive-c-2.8/libarchive_c.egg-info/PKG-INFO 2018-06-10 15:30:39.000000000 +0200 +++ new/libarchive-c-2.9/libarchive_c.egg-info/PKG-INFO 2019-10-20 14:40:31.000000000 +0200 @@ -1,6 +1,6 @@ -Metadata-Version: 1.0 +Metadata-Version: 2.1 Name: libarchive-c -Version: 2.8 +Version: 2.9 Summary: Python interface to libarchive Home-page: https://github.com/Changaco/python-libarchive-c Author: Changaco @@ -22,11 +22,21 @@ Compatibility ============= + python + ------ + python-libarchive-c is currently tested with python 2.7, 3.4, 3.5, and 3.6. If you find an incompatibility with older versions you can send us a small patch, but we won't accept big changes. + libarchive + ---------- + + python-libarchive-c may not work properly with obsolete versions of libarchive such as the ones included in MacOS. In that case you can install a recent version of libarchive (e.g. with ``brew install libarchive`` on MacOS) and use the ``LIBARCHIVE`` environment variable to point python-libarchive-c to it:: + + export LIBARCHIVE=/usr/local/Cellar/libarchive/3.3.3/lib/libarchive.13.dylib + Usage ===== @@ -68,3 +78,4 @@ Keywords: archive libarchive 7z tar bz2 zip gz Platform: UNKNOWN +Description-Content-Type: text/x-rst diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libarchive-c-2.8/setup.cfg new/libarchive-c-2.9/setup.cfg --- old/libarchive-c-2.8/setup.cfg 2018-06-10 15:30:39.000000000 +0200 +++ new/libarchive-c-2.9/setup.cfg 2019-10-20 14:40:31.000000000 +0200 @@ -3,7 +3,7 @@ [flake8] exclude = .?*,env*/ -ignore = E226,E731 +ignore = E226,E731,W504 max-line-length = 80 [egg_info] diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libarchive-c-2.8/setup.py new/libarchive-c-2.9/setup.py --- old/libarchive-c-2.8/setup.py 2016-11-29 19:03:27.000000000 +0100 +++ new/libarchive-c-2.9/setup.py 2019-10-20 14:39:01.000000000 +0200 @@ -17,5 +17,6 @@ license='CC0', packages=find_packages(exclude=['tests']), long_description=open(join(dirname(__file__), 'README.rst')).read(), + long_description_content_type='text/x-rst', keywords='archive libarchive 7z tar bz2 zip gz', ) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libarchive-c-2.8/tests/__init__.py new/libarchive-c-2.9/tests/__init__.py --- old/libarchive-c-2.8/tests/__init__.py 2018-06-10 13:44:59.000000000 +0200 +++ new/libarchive-c-2.9/tests/__init__.py 2019-10-20 13:18:23.000000000 +0200 @@ -6,6 +6,10 @@ from os.path import abspath, dirname, join from stat import S_ISREG import tarfile +try: + from stat import filemode +except ImportError: # Python 2 + filemode = tarfile.filemode from libarchive import file_reader @@ -83,7 +87,7 @@ path += '/' # libarchive introduces prefixes such as h prefix for # hardlinks: tarfile does not, so we ignore the first char - mode = tarfile.filemode(entry.mode)[1:] + mode = filemode(entry.mode)[1:] yield { 'path': path, 'mtime': entry.mtime, diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libarchive-c-2.8/tests/test_entry.py new/libarchive-c-2.9/tests/test_entry.py --- old/libarchive-c-2.8/tests/test_entry.py 2018-06-10 13:44:59.000000000 +0200 +++ new/libarchive-c-2.9/tests/test_entry.py 2019-10-20 13:18:23.000000000 +0200 @@ -7,12 +7,15 @@ import locale from os import environ, stat from os.path import join +import unicodedata from libarchive import memory_reader, memory_writer from . import data_dir, get_entries, get_tarinfos +text_type = unicode if str is bytes else str # noqa: F821 + locale.setlocale(locale.LC_ALL, '') # needed for sane time stamp comparison @@ -93,4 +96,9 @@ for key in ignore: e1.pop(key) e2.pop(key) + # Normalize all unicode (can vary depending on the system) + for d in (e1, e2): + for key in d: + if isinstance(d[key], text_type): + d[key] = unicodedata.normalize('NFC', d[key]) assert e1 == e2 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libarchive-c-2.8/tests/test_rwx.py new/libarchive-c-2.9/tests/test_rwx.py --- old/libarchive-c-2.8/tests/test_rwx.py 2018-06-10 15:01:15.000000000 +0200 +++ new/libarchive-c-2.9/tests/test_rwx.py 2019-10-20 14:34:28.000000000 +0200 @@ -1,12 +1,16 @@ """Test reading, writing and extracting archives.""" from __future__ import division, print_function, unicode_literals + import io +import json import libarchive +from libarchive.entry import format_time from libarchive.extract import EXTRACT_OWNER, EXTRACT_PERM, EXTRACT_TIME from libarchive.write import memory_writer from mock import patch +import pytest from . import check_archive, in_dir, treestat @@ -115,24 +119,48 @@ assert not write_fail_mock.called -def test_adding_entry_from_memory(): - entry_path = 'this is path' - entry_data = 'content' - entry_size = len(entry_data) +(a)pytest.mark.parametrize( + 'archfmt,data_bytes', + [('zip', b'content'), + ('gnutar', b''), + ('pax', json.dumps({'a': 1, 'b': 2, 'c': 3}).encode()), + ('7zip', b'lorem\0ipsum')]) +def test_adding_entry_from_memory(archfmt, data_bytes): + entry_path = 'testfile.data' + entry_data = data_bytes + entry_size = len(data_bytes) blocks = [] + archfmt = 'zip' + has_birthtime = archfmt != 'zip' + + atime = (1482144741, 495628118) + mtime = (1482155417, 659017086) + ctime = (1482145211, 536858081) + btime = (1482144740, 495628118) if has_birthtime else None + def write_callback(data): blocks.append(data[:]) return len(data) - with libarchive.custom_writer(write_callback, 'zip') as archive: - archive.add_file_from_memory(entry_path, entry_size, entry_data) + with libarchive.custom_writer(write_callback, archfmt) as archive: + archive.add_file_from_memory( + entry_path, entry_size, entry_data, + atime=atime, mtime=mtime, ctime=ctime, birthtime=btime + ) buf = b''.join(blocks) with libarchive.memory_reader(buf) as memory_archive: for archive_entry in memory_archive: - assert entry_data.encode() == b''.join( - archive_entry.get_blocks() - ) + expected = entry_data + actual = b''.join(archive_entry.get_blocks()) + assert expected == actual assert archive_entry.path == entry_path + assert archive_entry.atime in (atime[0], format_time(*atime)) + assert archive_entry.mtime in (mtime[0], format_time(*mtime)) + assert archive_entry.ctime in (ctime[0], format_time(*ctime)) + if has_birthtime: + assert archive_entry.birthtime in ( + btime[0], format_time(*btime) + ) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libarchive-c-2.8/tox.ini new/libarchive-c-2.9/tox.ini --- old/libarchive-c-2.8/tox.ini 2018-06-10 15:01:15.000000000 +0200 +++ new/libarchive-c-2.9/tox.ini 2019-10-20 13:18:23.000000000 +0200 @@ -1,11 +1,11 @@ [tox] -envlist=py27,py34,py35,py36 +envlist=py27,py35,py36,py37,py38 skipsdist=True [testenv] passenv = LIBARCHIVE commands= - python -m pytest -vv --boxed --cov libarchive --cov-report term-missing {toxinidir}/tests {posargs} + python -m pytest -Wd -vv --boxed --cov libarchive --cov-report term-missing {toxinidir}/tests {posargs} flake8 {toxinidir} deps= flake8

1 0

commit python-py3status for openSUSE:Leap:15.2
by root 01 May '20

01 May '20

Hello community, here is the log from the commit of package python-py3status for openSUSE:Leap:15.2 checked in at 2020-04-30 18:51:38 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Leap:15.2/python-py3status (Old) and /work/SRC/openSUSE:Leap:15.2/.python-py3status.new.2738 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Package is "python-py3status" Thu Apr 30 18:51:38 2020 rev:14 rq:794483 version:3.28 Changes: -------- --- /work/SRC/openSUSE:Leap:15.2/python-py3status/python-py3status.changes 2020-03-15 13:35:51.902820333 +0100 +++ /work/SRC/openSUSE:Leap:15.2/.python-py3status.new.2738/python-py3status.changes 2020-04-30 18:51:39.220625408 +0200 @@ -1,0 +2,10 @@ +Wed Apr 15 09:16:38 UTC 2020 - Paolo Stivanin <info(a)paolostivanin.com> + +- Update to 3.28: + * IMPORTANT: drop support for EOL Python 2.6-3.4 + * clock module: add "locale" config parameter to change time representation + * mpd_status module: use currentsong command if possible + * networkmanager module: allow using the currently active AP in formats + * volume_status module: change amixer flag ordering fix + +------------------------------------------------------------------- Old: ---- py3status-3.27.tar.gz New: ---- py3status-3.28.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ python-py3status.spec ++++++ --- /var/tmp/diff_new_pack.0RzAwa/_old 2020-04-30 18:51:39.628626277 +0200 +++ /var/tmp/diff_new_pack.0RzAwa/_new 2020-04-30 18:51:39.632626286 +0200 @@ -16,9 +16,11 @@ # +%define skip_python2 1 + %{?!python_module:%define python_module() python-%{**} python3-%{**}} Name: python-py3status -Version: 3.27 +Version: 3.28 Release: 0 Summary: Python extensible i3status wrapper License: BSD-3-Clause ++++++ py3status-3.27.tar.gz -> py3status-3.28.tar.gz ++++++ ++++ 4201 lines of diff (skipped)

1 0

commit python-ldaptor for openSUSE:Leap:15.2
by root 01 May '20

01 May '20

Hello community, here is the log from the commit of package python-ldaptor for openSUSE:Leap:15.2 checked in at 2020-04-30 18:51:36 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Leap:15.2/python-ldaptor (Old) and /work/SRC/openSUSE:Leap:15.2/.python-ldaptor.new.2738 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Package is "python-ldaptor" Thu Apr 30 18:51:36 2020 rev:2 rq:794481 version:19.1.0 Changes: -------- --- /work/SRC/openSUSE:Leap:15.2/python-ldaptor/python-ldaptor.changes 2020-02-22 17:50:43.269562757 +0100 +++ /work/SRC/openSUSE:Leap:15.2/.python-ldaptor.new.2738/python-ldaptor.changes 2020-04-30 18:51:37.240621189 +0200 @@ -1,0 +2,18 @@ +Wed Apr 15 11:20:25 UTC 2020 - Marketa Calabkova <mcalabkova(a)suse.com> + +- Update to 19.1.0 + * Basic implementation of ``ldaptor.protocols.pureldap.LDAPSearchResultReference``. + * Explicit ``ldaptor.protocols.ldap.ldaperrors`` classes declaration was made + to allow syntax highlighting for this module. + * Example of using LDAP server with the database. Employees are store in the database table and retrieved + on server initialization. + * ``ldaptor.protocols.pureldap.LDAPPasswordModifyRequest`` string representation now contains + ``userIdentity``, ``oldPasswd`` and ``newPasswd`` attributes. Password attributes are represented as asterisks. + * ``ldaptor.protocols.pureldap.LDAPBindRequest`` string representation is now using asterisks to represent + ``auth`` attribute. + * ``NotImplementedError`` for ``ldaptor.protocols.pureldap.LDAPSearchResultReference`` was fixed. + * StartTLS regression bug was fixed: ``ldaptor.protocols.pureldap.LDAPStartTLSRequest.oid`` and + ``ldaptor.protocols.pureldap.LDAPStartTLSResponse.oid`` must be of bytes type. + * replace direct dependency on pyOpenSSL with Twisted[tls] + +------------------------------------------------------------------- Old: ---- ldaptor-19.0.0.tar.gz New: ---- ldaptor-19.1.0.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ python-ldaptor.spec ++++++ --- /var/tmp/diff_new_pack.97F4ft/_old 2020-04-30 18:51:37.540621828 +0200 +++ /var/tmp/diff_new_pack.97F4ft/_new 2020-04-30 18:51:37.540621828 +0200 @@ -1,7 +1,7 @@ # # spec file for package python-ldaptor # -# Copyright (c) 2019 SUSE LINUX GmbH, Nuernberg, Germany. +# Copyright (c) 2020 SUSE LLC # # All modifications and additions to the file contributed by third parties # remain the property of their copyright owners, unless otherwise agreed @@ -18,7 +18,7 @@ %{?!python_module:%define python_module() python-%{**} python3-%{**}} Name: python-ldaptor -Version: 19.0.0 +Version: 19.1.0 Release: 0 Summary: A Pure-Python Twisted library for LDAP License: MIT ++++++ ldaptor-19.0.0.tar.gz -> ldaptor-19.1.0.tar.gz ++++++ ++++ 1719 lines of diff (skipped)

1 0

commit python-argparse-manpage for openSUSE:Leap:15.2
by root 01 May '20

01 May '20

Hello community, here is the log from the commit of package python-argparse-manpage for openSUSE:Leap:15.2 checked in at 2020-04-30 18:51:33 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Leap:15.2/python-argparse-manpage (Old) and /work/SRC/openSUSE:Leap:15.2/.python-argparse-manpage.new.2738 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Package is "python-argparse-manpage" Thu Apr 30 18:51:33 2020 rev:3 rq:794280 version:1.3 Changes: -------- --- /work/SRC/openSUSE:Leap:15.2/python-argparse-manpage/python-argparse-manpage.changes 2020-03-09 18:02:31.204732761 +0100 +++ /work/SRC/openSUSE:Leap:15.2/.python-argparse-manpage.new.2738/python-argparse-manpage.changes 2020-04-30 18:51:34.792615972 +0200 @@ -1,0 +2,5 @@ +Tue Apr 14 14:52:26 UTC 2020 - Matej Cepl <mcepl(a)suse.com> + +- Don't use %python3_only command, but properly use alternatives. + +------------------------------------------------------------------- ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ python-argparse-manpage.spec ++++++ --- /var/tmp/diff_new_pack.rbqM6e/_old 2020-04-30 18:51:35.088616603 +0200 +++ /var/tmp/diff_new_pack.rbqM6e/_new 2020-04-30 18:51:35.092616612 +0200 @@ -32,6 +32,8 @@ BuildRequires: fdupes BuildRequires: python-rpm-macros Requires: python-setuptools +Requires(post): update-alternatives +Requires(postun): update-alternatives %python_subpackages %description @@ -51,16 +53,24 @@ %install %python_install +%python_clone -a %{buildroot}%{_bindir}/argparse-manpage +%python_clone -a %{buildroot}%{_mandir}/man1/argparse-manpage.1 %python_expand %fdupes %{buildroot}%{$python_sitelib} %check %pytest +%post +%{python_install_alternative argparse-manpage argparse-manpage.1} + +%postun +%python_uninstall_alternative argparse-manpage + %files %{python_files} %doc README* %license LICENSE %{python_sitelib}/* -%python3_only %{_bindir}/argparse-manpage -%python3_only %{_mandir}/man1/argparse-manpage.1%{?ext_man} +%python_alternative %{_bindir}/argparse-manpage +%python_alternative %{_mandir}/man1/argparse-manpage.1%{?ext_man} %changelog

1 0

commit python-cfscrape for openSUSE:Leap:15.2
by root 01 May '20

01 May '20

Hello community, here is the log from the commit of package python-cfscrape for openSUSE:Leap:15.2 checked in at 2020-04-30 18:51:31 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Leap:15.2/python-cfscrape (Old) and /work/SRC/openSUSE:Leap:15.2/.python-cfscrape.new.2738 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Package is "python-cfscrape" Thu Apr 30 18:51:31 2020 rev:17 rq:793882 version:2.1.1 Changes: -------- --- /work/SRC/openSUSE:Leap:15.2/python-cfscrape/python-cfscrape.changes 2020-03-09 18:03:31.244762567 +0100 +++ /work/SRC/openSUSE:Leap:15.2/.python-cfscrape.new.2738/python-cfscrape.changes 2020-04-30 18:51:33.648613535 +0200 @@ -1,0 +2,6 @@ +Fri Feb 28 00:26:13 UTC 2020 - Martin Herkt <9+suse(a)cirno.systems> + +- Update to 2.1.1 + * Updated to handle latest Cloudflare IUAM challenge + +------------------------------------------------------------------- Old: ---- cloudflare-scrape-2.0.8.tar.gz New: ---- cloudflare-scrape-2.1.1.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ python-cfscrape.spec ++++++ --- /var/tmp/diff_new_pack.9wralo/_old 2020-04-30 18:51:34.008614302 +0200 +++ /var/tmp/diff_new_pack.9wralo/_new 2020-04-30 18:51:34.012614310 +0200 @@ -1,7 +1,7 @@ # # spec file for package python-cfscrape # -# Copyright (c) 2019 SUSE LINUX GmbH, Nuernberg, Germany. +# Copyright (c) 2020 SUSE LLC # # All modifications and additions to the file contributed by third parties # remain the property of their copyright owners, unless otherwise agreed @@ -18,7 +18,7 @@ %{?!python_module:%define python_module() python-%{**} python3-%{**}} Name: python-cfscrape -Version: 2.0.8 +Version: 2.1.1 Release: 0 Summary: Python module to bypass Cloudflare's anti-bot page License: MIT @@ -66,9 +66,10 @@ %python_install %python_expand %fdupes %{buildroot}%{$python_sitelib} -%check -%python_exec setup.py develop --user -%python_exec -m pytest -v tests +# test suite requires internet access +# %%check +# %%python_exec setup.py develop --user +# %%python_exec -m pytest -v tests %files %{python_files} %license LICENSE ++++++ cloudflare-scrape-2.0.8.tar.gz -> cloudflare-scrape-2.1.1.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/cloudflare-scrape-2.0.8/.github/ISSUE_TEMPLATE/bug-report-template.md new/cloudflare-scrape-2.1.1/.github/ISSUE_TEMPLATE/bug-report-template.md --- old/cloudflare-scrape-2.0.8/.github/ISSUE_TEMPLATE/bug-report-template.md 2019-08-11 17:48:51.000000000 +0200 +++ new/cloudflare-scrape-2.1.1/.github/ISSUE_TEMPLATE/bug-report-template.md 2020-02-23 05:09:07.000000000 +0100 @@ -7,9 +7,26 @@ --- -Before creating an issue, first upgrade cfscrape with `pip install -U cfscrape` and see if you're still experiencing the problem. +Before creating an issue, first upgrade cfscrape with `pip install -U cfscrape` and see if you're still experiencing the problem. Please also confirm your Node version (`node --version` or `nodejs --version`) is version 10 or higher. -## Version number +Make sure the website you're having issues with is actually using anti-bot protection by Cloudflare and not a competitor like Imperva Incapsula or Sucuri. And if you're using an anonymizing proxy, a VPN, or Tor, Cloudflare often flags those IPs and may block you or present you with a captcha as a result. + +Please **confirm the following statements and check the boxes** before creating an issue: + +- [ ] I've upgraded cfscrape with `pip install -U cfscrape` +- [ ] I'm using Node version 10 or higher +- [ ] The site protection I'm having issues with is from Cloudflare +- [ ] I'm not using Tor, a VPN, or an anonymizing proxy + +## Python version number + +Run `python --version` and paste the output below: + +``` + +``` + +## cfscrape version number Run `pip show cfscrape` and paste the output below: @@ -17,7 +34,7 @@ ``` -## Code snippet experiencing the issue +## Code snippet involved with the issue ``` diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/cloudflare-scrape-2.0.8/.travis.yml new/cloudflare-scrape-2.1.1/.travis.yml --- old/cloudflare-scrape-2.0.8/.travis.yml 2019-08-11 17:48:51.000000000 +0200 +++ new/cloudflare-scrape-2.1.1/.travis.yml 2020-02-23 05:09:07.000000000 +0100 @@ -12,11 +12,28 @@ - python: '3.7' dist: xenial sudo: true + - env: cfscrape_node='4.5' + - env: cfscrape_node='node' + - os: osx + language: node_js + node_js: node + cache: + directories: + - $HOME/Library/Caches/Homebrew + - /usr/local/Homebrew cache: pip +before_install: + - | + if [ -n "${cfscrape_node}" ]; then + source ~/.nvm/nvm.sh + nvm install "${cfscrape_node}" + nvm use "${cfscrape_node}" + fi + install: - - node -e "console.log(process.versions);" + - node -p process.versions - make script: make ci diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/cloudflare-scrape-2.0.8/Makefile new/cloudflare-scrape-2.1.1/Makefile --- old/cloudflare-scrape-2.0.8/Makefile 2019-08-11 17:48:51.000000000 +0200 +++ new/cloudflare-scrape-2.1.1/Makefile 2020-02-23 05:09:07.000000000 +0100 @@ -4,6 +4,10 @@ pip install pipenv -U pipenv install --dev +requirements: + pipenv lock -r > requirements.txt + pipenv lock --dev -r > requirements-dev.txt + test: # This runs all of the tests, on both Python 2 and Python 3. pipenv run tox --parallel auto diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/cloudflare-scrape-2.0.8/Pipfile new/cloudflare-scrape-2.1.1/Pipfile --- old/cloudflare-scrape-2.0.8/Pipfile 2019-08-11 17:48:51.000000000 +0200 +++ new/cloudflare-scrape-2.1.1/Pipfile 2020-02-23 05:09:07.000000000 +0100 @@ -10,6 +10,8 @@ pytest = "*" # more_itertools is added to resolve a CI related issue with Pipenv + pytest more_itertools = { version = ">=4.0.0", markers = "python_version >= '2.7'" } +# mock is added to resolve a CI related issue with Pipenv + pytest +mock = "*" pytest-cov = "*" pytest-xdist = "*" pytest-forked = "*" diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/cloudflare-scrape-2.0.8/README.md new/cloudflare-scrape-2.1.1/README.md --- old/cloudflare-scrape-2.0.8/README.md 2019-08-11 17:48:51.000000000 +0200 +++ new/cloudflare-scrape-2.1.1/README.md 2020-02-23 05:09:07.000000000 +0100 @@ -29,7 +29,7 @@ Node.js dependency ============ -A currently supported version of [Node.js](https://nodejs.org/) is required to interpret Cloudflare's obfuscated JavaScript challenge. Releases older than 4.5 are definitely broken. +[Node.js](https://nodejs.org/) version 10 or above is required to interpret Cloudflare's obfuscated JavaScript challenge. Your machine may already have Node installed (check with `node -v`). If not, you can install it with `apt-get install nodejs` on Ubuntu >= 18.04 and Debian >= 9 and `brew install node` on macOS. Otherwise, you can get it from [Node's download page](https://nodejs.org/en/download/) or [their package manager installation page](https://nodejs.org/en/download/package-manager/). diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/cloudflare-scrape-2.0.8/appveyor.yml new/cloudflare-scrape-2.1.1/appveyor.yml --- old/cloudflare-scrape-2.0.8/appveyor.yml 2019-08-11 17:48:51.000000000 +0200 +++ new/cloudflare-scrape-2.1.1/appveyor.yml 2020-02-23 05:09:07.000000000 +0100 @@ -15,10 +15,23 @@ - PYTHON: 'C:/Python36-x64' - PYTHON: 'C:/Python37' - PYTHON: 'C:/Python37-x64' + cfscrape_node: '4.5' + - PYTHON: 'C:/Python37-x64' + cfscrape_node: 'node' install: + - ps: >- + If ($env:cfscrape_node -ne $null) { + If ($env:cfscrape_node -Match "node") { + Install-Product node "" + } + Else { + Install-Product node $env:cfscrape_node + } + } + - 'set PATH=%PYTHON%;%PYTHON%/Scripts;%PATH%' - 'python --version' - - 'node -e "console.log(process.versions);"' + - 'node -p process.versions' - 'pip -V' - 'pip install pipenv' - 'pipenv install --dev' diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/cloudflare-scrape-2.0.8/cfscrape/__init__.py new/cloudflare-scrape-2.1.1/cfscrape/__init__.py --- old/cloudflare-scrape-2.0.8/cfscrape/__init__.py 2019-08-11 17:48:51.000000000 +0200 +++ new/cloudflare-scrape-2.1.1/cfscrape/__init__.py 2020-02-23 05:09:07.000000000 +0100 @@ -20,7 +20,7 @@ from .user_agents import USER_AGENTS -__version__ = "2.0.8" +__version__ = "2.1.1" DEFAULT_USER_AGENT = random.choice(USER_AGENTS) @@ -96,6 +96,7 @@ # Define headers to force using an OrderedDict and preserve header order self.headers = headers + self.org_method = None self.mount("https://", CloudflareAdapter()) @@ -152,7 +153,13 @@ body = resp.text parsed_url = urlparse(resp.url) domain = parsed_url.netloc - submit_url = "%s://%s/cdn-cgi/l/chk_jschl" % (parsed_url.scheme, domain) + challenge_form = re.search(r'\<form.*?id=\"challenge-form\".*?\/form\>',body, flags=re.S).group(0) # find challenge form + method = re.search(r'method=\"(.*?)\"', challenge_form, flags=re.S).group(1) + if self.org_method is None: + self.org_method = resp.request.method + submit_url = "%s://%s%s" % (parsed_url.scheme, + domain, + re.search(r'action=\"(.*?)\"', challenge_form, flags=re.S).group(1).split('?')[0]) cloudflare_kwargs = copy.deepcopy(original_kwargs) @@ -160,13 +167,29 @@ headers["Referer"] = resp.url try: - params = cloudflare_kwargs["params"] = OrderedDict( - re.findall(r'name="(s|jschl_vc|pass)"(?: [^<>]*)? value="(.+?)"', body) - ) + cloudflare_kwargs["params"] = dict() + cloudflare_kwargs["data"] = dict() + if len(re.search(r'action=\"(.*?)\"', challenge_form, flags=re.S).group(1).split('?')) != 1: + for param in re.search(r'action=\"(.*?)\"', challenge_form, flags=re.S).group(1).split('?')[1].split('&'): + cloudflare_kwargs["params"].update({param.split('=')[0]:param.split('=')[1]}) + + for input_ in re.findall(r'\<input.*?(?:\/>|\<\/input\>)', challenge_form, flags=re.S): + if re.search(r'name=\"(.*?)\"',input_, flags=re.S).group(1) != 'jschl_answer': + if method == 'POST': + cloudflare_kwargs["data"].update({re.search(r'name=\"(.*?)\"',input_, flags=re.S).group(1): + re.search(r'value=\"(.*?)\"',input_, flags=re.S).group(1)}) + elif method == 'GET': + cloudflare_kwargs["params"].update({re.search(r'name=\"(.*?)\"',input_, flags=re.S).group(1): + re.search(r'value=\"(.*?)\"',input_, flags=re.S).group(1)}) + if method == 'POST': + for k in ("jschl_vc", "pass"): + if k not in cloudflare_kwargs["data"]: + raise ValueError("%s is missing from challenge form" % k) + elif method == 'GET': + for k in ("jschl_vc", "pass"): + if k not in cloudflare_kwargs["params"]: + raise ValueError("%s is missing from challenge form" % k) - for k in ("jschl_vc", "pass"): - if k not in params: - raise ValueError("%s is missing from challenge form" % k) except Exception as e: # Something is wrong with the page. # This may indicate Cloudflare has changed their anti-bot @@ -174,17 +197,19 @@ # please open a GitHub issue so I can update the code accordingly. raise ValueError( "Unable to parse Cloudflare anti-bot IUAM page: %s %s" - % (e.message, BUG_REPORT) + % (e, BUG_REPORT) ) # Solve the Javascript challenge answer, delay = self.solve_challenge(body, domain) - params["jschl_answer"] = answer + if method == 'POST': + cloudflare_kwargs["data"]["jschl_answer"] = answer + elif method == 'GET': + cloudflare_kwargs["params"]["jschl_answer"] = answer # Requests transforms any request into a GET after a redirect, # so the redirect has to be handled manually here to allow for # performing other types of requests even as the first request. - method = resp.request.method cloudflare_kwargs["allow_redirects"] = False # Cloudflare requires a delay before solving the challenge @@ -192,35 +217,51 @@ # Send the challenge response and handle the redirect manually redirect = self.request(method, submit_url, **cloudflare_kwargs) - redirect_location = urlparse(redirect.headers["Location"]) + if "Location" in redirect.headers: + redirect_location = urlparse(redirect.headers["Location"]) - if not redirect_location.netloc: - redirect_url = urlunparse( - ( - parsed_url.scheme, - domain, - redirect_location.path, - redirect_location.params, - redirect_location.query, - redirect_location.fragment, + if not redirect_location.netloc: + redirect_url = urlunparse( + ( + parsed_url.scheme, + domain, + redirect_location.path, + redirect_location.params, + redirect_location.query, + redirect_location.fragment, + ) ) - ) - return self.request(method, redirect_url, **original_kwargs) - return self.request(method, redirect.headers["Location"], **original_kwargs) + return self.request(method, redirect_url, **original_kwargs) + return self.request(method, redirect.headers["Location"], **original_kwargs) + elif "Set-Cookie" in redirect.headers: + if 'cf_clearance' in redirect.headers['Set-Cookie']: + resp = self.request(self.org_method, submit_url, cookies = redirect.cookies) + return resp + else: + return self.request(method, submit_url, **original_kwargs) + else: + resp = self.request(self.org_method, submit_url, **cloudflare_kwargs) + return resp + def solve_challenge(self, body, domain): try: + javascript = re.search(r'\<script type\=\"text\/javascript\"\>\n(.*?)\<\/script\>',body, flags=re.S).group(1) # find javascript + challenge, ms = re.search( r"setTimeout$function\(${\s*(var " r"s,t,o,p,b,r,e,a,k,i,n,g,f.+?\r?\n[\s\S]+?a\.value\s*=.+?)\r?\n" r"(?:[^{<>]*},\s*(\d{4,}))?", - body, + javascript, flags=re.S ).groups() # The challenge requires `document.getElementById` to get this content. # Future proofing would require escaping newlines and double quotes - innerHTML = re.search(r"<div(?: [^<>]*)? id=\"cf-dn.*?\">([^<>]*)", body) - innerHTML = innerHTML.group(1) if innerHTML else "" + innerHTML = '' + for i in javascript.split(';'): + if i.strip().split('=')[0].strip() == 'k': # from what i found out from pld example K var in + k = i.strip().split('=')[1].strip(' \'') # javafunction is for innerHTML this code to find it + innerHTML = re.search(r'\<div.*?id\=\"'+k+r'\".*?\>(.*?)\<\/div\>',body).group(1) #find innerHTML # Prefix the challenge with a fake document object. # Interpolate the domain, div contents, and JS challenge. @@ -274,11 +315,17 @@ """ % challenge ) + stderr = '' try: - result = subprocess.check_output( - ["node", "-e", js], stdin=subprocess.PIPE, stderr=subprocess.PIPE - ) + node = subprocess.Popen( + ["node", "-e", js], stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True + ) + result, stderr = node.communicate() + if node.returncode != 0: + stderr = "Node.js Exception:\n%s" % (stderr or None) + raise subprocess.CalledProcessError(node.returncode, "node -e ...", stderr) except OSError as e: if e.errno == 2: raise EnvironmentError( diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/cloudflare-scrape-2.0.8/tests/__init__.py new/cloudflare-scrape-2.1.1/tests/__init__.py --- old/cloudflare-scrape-2.0.8/tests/__init__.py 2019-08-11 17:48:51.000000000 +0200 +++ new/cloudflare-scrape-2.1.1/tests/__init__.py 2020-02-23 05:09:07.000000000 +0100 @@ -3,7 +3,7 @@ import responses import re -from requests.compat import urlencode +from requests.compat import urlencode, Morsel from collections import OrderedDict from os import path from io import open @@ -52,7 +52,7 @@ The request will error if it doesn't match a defined response. """ - def __init__(self, callback=lambda request: None, **kwargs): + def __init__(self, callback=lambda request: None, location=None, **kwargs): defaults = (('method', 'GET'), ('status', 302), ('headers', {'Location': '/'}), @@ -62,6 +62,9 @@ for k, v in defaults: kwargs.setdefault(k, v) + if location: + kwargs['headers']['Location'] = location + args = tuple(kwargs.pop(k) for k in ('status', 'headers', 'body')) kwargs['callback'] = lambda request: callback(request) or args @@ -89,6 +92,21 @@ super(DefaultResponse, self).__init__(**kwargs) +class CaptchaResponse(ChallengeResponse): + """Simulates a reCAPTCHA(v2) response from Cloudflare + + This would be the only response in current tests. + + Kwargs: + Keyword arguments used to override the defaults. + The request will error if it doesn't match a defined response. + """ + + def __init__(self, **kwargs): + kwargs.setdefault('status', 403) + super(CaptchaResponse, self).__init__(**kwargs) + + def fixtures(filename): """Read and cache a challenge fixture @@ -106,7 +124,7 @@ # This fancy decorator wraps tests so the responses will be mocked. # It could be called directly e.g. challenge_responses(*args)(test_func) -> wrapper -def challenge_responses(filename, jschl_answer): +def challenge_responses(filename, jschl_answer, redirect_to='/'): # This function is called with the test_func and returns a new wrapper. def challenge_responses_decorator(test): @responses.activate @@ -126,14 +144,43 @@ responses.add(ChallengeResponse(url=url, body=fixtures(filename))) - def onRedirect(request): + def on_redirect(request): # We don't register the last response unless the redirect occurs responses.add(DefaultResponse(url=url, body=requested_page)) - responses.add(RedirectResponse(url=submit_uri, callback=onRedirect)) + responses.add(RedirectResponse( + url=submit_uri, callback=on_redirect, location=redirect_to + )) return test(self, **cfscrape_kwargs) - # The following causes pytest to call the test wrapper once for each interpreter. return wrapper return challenge_responses_decorator + + +def recaptcha_responses(filename): + def recaptcha_responses_decorator(test): + @responses.activate + def wrapper(self): + responses.add(CaptchaResponse(url=url, body=fixtures(filename))) + + return test(self, **cfscrape_kwargs) + return wrapper + + return recaptcha_responses_decorator + + +def cloudflare_cookies(): + # Cloudflare cookie that should be set when challenge is presented + cfduid = Morsel() + cfduid.set('__cfduid', 'uid-1', 'uid-1') + cfduid['path'] = '/' + cfduid['domain'] = '.example-site.dev' + + # Cloudflare cookie that should be set when challenge is bypassed + cf_clearance = Morsel() + cf_clearance.set('cf_clearance', 'uid-2', 'uid-2') + cf_clearance['path'] = '/' + cf_clearance['domain'] = '.example-site.dev' + + return cfduid, cf_clearance diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/cloudflare-scrape-2.0.8/tests/fixtures/js_challenge_30_11_2019.html new/cloudflare-scrape-2.1.1/tests/fixtures/js_challenge_30_11_2019.html --- old/cloudflare-scrape-2.0.8/tests/fixtures/js_challenge_30_11_2019.html 1970-01-01 01:00:00.000000000 +0100 +++ new/cloudflare-scrape-2.1.1/tests/fixtures/js_challenge_30_11_2019.html 2020-02-23 05:09:07.000000000 +0100 @@ -0,0 +1,101 @@ +<!DOCTYPE HTML> +<html lang="en-US"> +<head> + <meta charset="UTF-8" /> + <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /> + <meta http-equiv="X-UA-Compatible" content="IE=Edge,chrome=1" /> + <meta name="robots" content="noindex, nofollow" /> + <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1" /> + <title>Just a moment...</title> + <style type="text/css"> + html, body {width: 100%; height: 100%; margin: 0; padding: 0;} + body {background-color: #ffffff; font-family: Helvetica, Arial, sans-serif; font-size: 100%;} + h1 {font-size: 1.5em; color: #404040; text-align: center;} + p {font-size: 1em; color: #404040; text-align: center; margin: 10px 0 0 0;} + #spinner {margin: 0 auto 30px auto; display: block;} + .attribution {margin-top: 20px;} + @-webkit-keyframes bubbles { 33%: { -webkit-transform: translateY(10px); transform: translateY(10px); } 66% { -webkit-transform: translateY(-10px); transform: translateY(-10px); } 100% { -webkit-transform: translateY(0); transform: translateY(0); } } + @keyframes bubbles { 33%: { -webkit-transform: translateY(10px); transform: translateY(10px); } 66% { -webkit-transform: translateY(-10px); transform: translateY(-10px); } 100% { -webkit-transform: translateY(0); transform: translateY(0); } } + .bubbles { background-color: #404040; width:15px; height: 15px; margin:2px; border-radius:100%; -webkit-animation:bubbles 0.6s 0.07s infinite ease-in-out; animation:bubbles 0.6s 0.07s infinite ease-in-out; -webkit-animation-fill-mode:both; animation-fill-mode:both; display:inline-block; } + </style> + + <script type="text/javascript"> + //<![CDATA[ + (function(){ + var a = function() {try{return !!window.addEventListener} catch(e) {return !1} }, + b = function(b, c) {a() ? document.addEventListener("DOMContentLoaded", b, c) : document.attachEvent("onreadystatechange", b)}; + b(function(){ + var a = document.getElementById('cf-content');a.style.display = 'block'; + setTimeout(function(){ + var s,t,o,p,b,r,e,a,k,i,n,g,f, hWuAPfm={"xVxyYj":+((!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+[])+(!+[]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(+[])+(!+[]+!![]+!![]+!![])+(+!![])+(!+[]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![])+(+!![]))/+((!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+[])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![])+(!+[]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]))}; + g = String.fromCharCode; + o = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/="; + e = function(s) { + s += "==".slice(2 - (s.length & 3)); + var bm, r = "", r1, r2, i = 0; + for (; i < s.length;) { + bm = o.indexOf(s.charAt(i++)) << 18 | o.indexOf(s.charAt(i++)) << 12 + | (r1 = o.indexOf(s.charAt(i++))) << 6 | (r2 = o.indexOf(s.charAt(i++))); + r += r1 === 64 ? g(bm >> 16 & 255) + : r2 === 64 ? g(bm >> 16 & 255, bm >> 8 & 255) + : g(bm >> 16 & 255, bm >> 8 & 255, bm & 255); + } + return r; + }; + t = document.createElement('div'); + t.innerHTML="<a href='/'>x</a>"; + t = t.firstChild.href;r = t.match(/https?:\/\//)[0]; + t = t.substr(r.length); t = t.substr(0,t.length-1); + a = document.getElementById('jschl-answer'); + f = document.getElementById('challenge-form'); + ;hWuAPfm.xVxyYj*=+((!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+[])+(!+[]+!![])+(+[])+(!+[]+!![]+!![]+!![])+(+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![])+(!+[]+!![]+!![]))/+((!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+[])+(!+[]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![])+(!+[]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]));hWuAPfm.xVxyYj*=+((!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+[])+(+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![])+(+[])+(!+[]+!![]+!![]+!![])+(!+[]+!![])+(!+[]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]))/+((+!![]+[])+(+!![])+(!+[]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]));hWuAPfm.xVxyYj+=+((!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+[])+(!+[]+!![])+(+[])+(!+[]+!![]+!![]+!![])+(+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![])+(!+[]+!![]+!![]))/+((!+[]+!![]+!![]+[])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![])+(!+[]+!![])+(!+[]+!![]+!![]+!![]+!![])+(!+[]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(+[]));a.value = (+hWuAPfm.xVxyYj + t.length).toFixed(10); '; 121' + f.action += location.hash; + f.submit(); + }, 4000); + }, false); + })(); + //]]> +</script> + + +</head> +<body> + <table width="100%" height="100%" cellpadding="20"> + <tr> + <td align="center" valign="middle"> + <div class="cf-browser-verification cf-im-under-attack"> + <noscript><h1 data-translate="turn_on_js" style="color:#bd2426;">Please turn JavaScript on and reload the page.</h1></noscript> + <div id="cf-content" style="display:none"> + + <div> + <div class="bubbles"></div> + <div class="bubbles"></div> + <div class="bubbles"></div> + </div> + <h1><span data-translate="checking_browser">Checking your browser before accessing</span> kissmanga.com.</h1> + + <p data-translate="process_is_automatic">This process is automatic. Your browser will redirect to your requested content shortly.</p> + <p data-translate="allow_5_secs">Please allow up to 5 seconds…</p> + </div> + + <form id="challenge-form" action="/Manga/The-Fairy-Captivity?__cf_chl_jschl_tk__=072b4332f1c34ea3a99d456a575ab0294c4868ec-1575019984-0-ASKQNetI_COO72dgnF_r2PC4LVc8vKexMyimbcpQjZIQqOwdo1Fh7PKubmgVlLUS58qVJBoAlUMzb0sWrn8vwtDRHKxVrFuTMcEYBinjd6tbp9aBJEZiQctvCPBqAJJKqKaMXOf-aldIFWEt5EegylBjixChwZxA4ihaAqyC59gBg36Wjnvs1lkUEou573kvOyDGcJTiVyzf3O9a86tyO2N-uF8xRuqfWhMYNPwoXTRhXxNRF6rinqmPuURCYej0fxLu0lf21UQUV6JQlfrWdA_UopDhB_KSHgYT1NAzrUcm" method="POST" enctype="application/x-www-form-urlencoded"> + <input type="hidden" name="r" value="901e5fc6103829c66a465ebaf9755c835e66f46c-1575019984-0-AUzvQW0hz4d1zGbwqui49PImeTFf+c4K/C6QEzvszNkWYIDBcEURSr4o/H5IdAWEHt8QBvt1k2LqiCygPXAg5VvhyPDUdk7+ngeTHRhJa/wqFRLtQDCJO8F9E5cpsDG4PjR83Paez4fsN7LfeVOg56jt4KZgSqgjomBNcBlsHuo/9c8FAQJtLCx/npoW2Pjsl8+FkojoziW4N1LWxlYd1qnjBgjmnxJvu8ZjdO9DEvXNCT7GZkNKhelSSO6PN6LWzJjyA+oKUN9TxeNjIVWPIL5dTFRA4vhRF5sfXNaJbpExqE9b37/vUyaaWZTGCL4U/dO+dnchaKwZiE+liztsPVpS1gxj+nUh8FjKN/QYpbcAJLOFAYsNQpM5THNGz9WANuF31jkhJT+w++EgL42B6uRFAYdutnqNnAwaZyR4Acph17v0pk4x/JEUtg4TvCG8MyW4g2S4u7Ak+drN5A/y6Li/rgiJKUJorGAE5BOhTR3IzkIsFhHJ+jZ2IMXLltPciTrbK3oN4tY33dfgwzEcTonCi9qB88Yuex/Sep9Mocmk/CSFo1PS44p+9kOFWkx9G1cCaGrifpbbOBMuTkVye5wPgDaXJRePSAc1FClCuxhskbadMST6DDvcRPwdziRlbalrCxe6f00HM131oP7aUkDropkfVlTvImfSh0nQLfjxayTZzOzb43SBe6/FKUBQig=="></input> + <input type="hidden" name="jschl_vc" value="c22565a73c49afb92b427eff230eb330"/> + <input type="hidden" name="pass" value="1575019988.841-6b8Fv8fEyM"/> + <input type="hidden" id="jschl-answer" name="jschl_answer"/> + </form> + +</div> + + + <div class="attribution"> + <a href="https://www.cloudflare.com/5xx-error-landing?utm_source=iuam" target="_blank" style="font-size: 12px;">DDoS protection by Cloudflare</a> + <br> + Ray ID: 53d393f93ae1c82f + </div> + </td> + + </tr> + </table> +</body> +</html> \ No newline at end of file diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/cloudflare-scrape-2.0.8/tests/test_adapters.py new/cloudflare-scrape-2.1.1/tests/test_adapters.py --- old/cloudflare-scrape-2.0.8/tests/test_adapters.py 2019-08-11 17:48:51.000000000 +0200 +++ new/cloudflare-scrape-2.1.1/tests/test_adapters.py 2020-02-23 05:09:07.000000000 +0100 @@ -27,7 +27,7 @@ # If pyOpenSSL is injected into urllib3, this should still work. try: assert isinstance(ssl_context, urllib3.contrib.pyopenssl.PyOpenSSLContext) - except: + except BaseException: assert isinstance(ssl_context, ssl.SSLContext) adapter.close() diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/cloudflare-scrape-2.0.8/tests/test_cfscrape.py new/cloudflare-scrape-2.1.1/tests/test_cfscrape.py --- old/cloudflare-scrape-2.0.8/tests/test_cfscrape.py 2019-08-11 17:48:51.000000000 +0200 +++ new/cloudflare-scrape-2.1.1/tests/test_cfscrape.py 2020-02-23 05:09:07.000000000 +0100 @@ -2,14 +2,22 @@ import pytest import cfscrape +import requests +import re +import os +import ssl +import responses +import subprocess from sure import expect -from . import challenge_responses, requested_page, url +from . import challenge_responses, recaptcha_responses, requested_page, url, \ + cloudflare_cookies, DefaultResponse, ChallengeResponse, fixtures, \ + cfscrape_kwargs class TestCloudflareScraper: - @challenge_responses(filename='js_challenge_10_04_2019.html', jschl_answer='18.8766915031') + @challenge_responses(filename='js_challenge_10_04_2019.html', jschl_answer='18.8766915031', redirect_to=url) def test_js_challenge_10_04_2019(self, **kwargs): scraper = cfscrape.CloudflareScraper(**kwargs) expect(scraper.get(url).content).to.equal(requested_page) @@ -39,3 +47,281 @@ def test_js_challenge_21_05_2015(self, **kwargs): scraper = cfscrape.CloudflareScraper(**kwargs) expect(scraper.get(url).content).to.equal(requested_page) + + @recaptcha_responses(filename='cf_recaptcha_15_04_2019.html') + def test_cf_recaptcha_15_04_2019(self, **kwargs): + scraper = cfscrape.CloudflareScraper(**kwargs) + message = re.compile(r'captcha challenge presented') + scraper.get.when.called_with(url) \ + .should.have.raised(cfscrape.CloudflareCaptchaError, message) + + v = ssl.OPENSSL_VERSION_NUMBER + ssl.OPENSSL_VERSION_NUMBER = 0x0090581f + try: + scraper = cfscrape.CloudflareScraper(**kwargs) + message = re.compile(r'OpenSSL version is lower than 1.1.1') + scraper.get.when.called_with(url) \ + .should.have.raised(cfscrape.CloudflareCaptchaError, message) + finally: + ssl.OPENSSL_VERSION_NUMBER = v + + @responses.activate + def test_js_challenge_unable_to_identify(self): + body = fixtures('js_challenge_10_04_2019.html') + body = body.replace(b'setTimeout', b'') + + responses.add(ChallengeResponse(url=url, body=body)) + + scraper = cfscrape.create_scraper(**cfscrape_kwargs) + message = re.compile(r'Unable to identify Cloudflare IUAM Javascript') + scraper.get.when.called_with(url) \ + .should.have.raised(ValueError, message) + + @responses.activate + def test_js_challenge_unexpected_answer(self): + body = fixtures('js_challenge_10_04_2019.html') + body = body.replace(b'\'; 121\'', b'a.value = "foobar"') + + responses.add(ChallengeResponse(url=url, body=body)) + + scraper = cfscrape.create_scraper(**cfscrape_kwargs) + message = re.compile(r'Cloudflare IUAM challenge returned unexpected answer') + scraper.get.when.called_with(url) \ + .should.have.raised(ValueError, message) + + @responses.activate + def test_js_challenge_missing_pass(self): + body = fixtures('js_challenge_10_04_2019.html') + body = body.replace(b'name="pass"', b'') + + responses.add(ChallengeResponse(url=url, body=body)) + + scraper = cfscrape.create_scraper(**cfscrape_kwargs) + message = re.compile(r'Unable to parse .* pass is missing from challenge form') + scraper.get.when.called_with(url) \ + .should.have.raised(ValueError, message) + + def test_js_challenge_subprocess_unknown_error(self, caplog): + def test(self, **kwargs): + __Popen = subprocess.Popen + + # Temporarily disable this method to generate an exception + subprocess.Popen = None + + try: + scraper = cfscrape.CloudflareScraper(**kwargs) + scraper.get.when.called_with(url) \ + .should.have.raised(TypeError) + caplog.text.should.match(re.compile(r'Error executing Cloudflare IUAM Javascript')) + finally: + subprocess.Popen = __Popen + + challenge_responses( + filename='js_challenge_10_04_2019.html', jschl_answer='18.8766915031' + )(test)(self) + + def test_js_challenge_subprocess_system_error(self, caplog): + def test(self, **kwargs): + __Popen = subprocess.Popen + + # Temporarily Mock subprocess method to raise an OSError + def mock(*args, **kwargs): + raise OSError('System Error') + + subprocess.Popen = mock + + try: + scraper = cfscrape.CloudflareScraper(**kwargs) + scraper.get.when.called_with(url) \ + .should.have.raised(OSError, re.compile(r'System Error')) + caplog.text.should.equal('') + finally: + subprocess.Popen = __Popen + + challenge_responses( + filename='js_challenge_10_04_2019.html', jschl_answer='18.8766915031' + )(test)(self) + + def test_js_challenge_subprocess_non_zero(self, caplog): + def test(self, **kwargs): + __Popen = subprocess.Popen + + # Temporarily Mock subprocess method to return non-zero exit code + def mock(*args, **kwargs): + def node(): pass + node.communicate = lambda: ('stdout', 'stderr') + node.returncode = 1 + return node + + subprocess.Popen = mock + + try: + scraper = cfscrape.CloudflareScraper(**kwargs) + message = re.compile(r'non-zero exit status') + scraper.get.when.called_with(url) \ + .should.have.raised(subprocess.CalledProcessError, message) + caplog.text.should.match(re.compile(r'Error executing Cloudflare IUAM Javascript')) + caplog.text.should_not.match(re.compile(r'Outdated Node.js detected')) + finally: + subprocess.Popen = __Popen + + challenge_responses( + filename='js_challenge_10_04_2019.html', jschl_answer='18.8766915031' + )(test)(self) + + def test_js_challenge_outdated_node(self, caplog): + def test(self, **kwargs): + __Popen = subprocess.Popen + + # Temporarily Mock subprocess method to return non-zero exit code + def mock(*args, **kwargs): + def node(): pass + node.communicate = lambda: ('stdout', 'Outdated Node.js detected') + node.returncode = 1 + return node + + subprocess.Popen = mock + + try: + scraper = cfscrape.CloudflareScraper(**kwargs) + message = re.compile(r'non-zero exit status') + scraper.get.when.called_with(url) \ + .should.have.raised(subprocess.CalledProcessError, message) + caplog.text.should_not.match(re.compile(r'Error executing Cloudflare IUAM Javascript')) + caplog.text.should.match(re.compile(r'Outdated Node.js detected')) + finally: + subprocess.Popen = __Popen + + challenge_responses( + filename='js_challenge_10_04_2019.html', jschl_answer='18.8766915031' + )(test)(self) + + @challenge_responses(filename='js_challenge_10_04_2019.html', jschl_answer='18.8766915031') + def test_js_challenge_environment_error(self, **kwargs): + __path = os.environ['PATH'] + # Temporarily unset PATH to hide Node.js + os.environ['PATH'] = '' + try: + scraper = cfscrape.CloudflareScraper(**kwargs) + message = re.compile(r'Missing Node.js runtime') + scraper.get.when.called_with(url) \ + .should.have.raised(EnvironmentError, message) + finally: + os.environ['PATH'] = __path + + @challenge_responses(filename='js_challenge_10_04_2019.html', jschl_answer='18.8766915031') + def test_get_cookie_string(self, **kwargs): + # get_cookie_string doesn't accept the delay kwarg. + # Set the delay in the Test class to speed up this test. + delay = kwargs.pop('delay', 0.1) + expected_ua = kwargs.setdefault('user_agent', 'custom-ua') + + cfduid, cf_clearance = cloudflare_cookies() + + # Use a class to workaround a `responses` bug where + # cookies aren't mocked correctly. + class Test(cfscrape.CloudflareScraper): + def __init__(self, *args, **kwargs): + kwargs.setdefault('delay', delay) + super(Test, self).__init__(*args, **kwargs) + + self.cookies.set('__cfduid', cfduid) + self.cookies.set('cf_clearance', cf_clearance) + + result = Test.get_cookie_string(url, **kwargs) + result.should.be.a('tuple') + result.should.have.length_of(2) + + cookie_arg, user_agent = result + + cookie_arg.should.be.a('str') + cookie_arg.should.contain('cf_clearance=%s' % cf_clearance.value) + cookie_arg.should.contain('__cfduid=%s' % cfduid.value) + + user_agent.should.equal(expected_ua) + + @challenge_responses(filename='js_challenge_10_04_2019.html', jschl_answer='18.8766915031') + def test_get_tokens(self, **kwargs): + # get_tokens doesn't accept the delay kwarg. + # Set the delay in the Test class to speed up this test. + delay = kwargs.pop('delay', 0.1) + expected_ua = kwargs.setdefault('user_agent', 'custom-ua') + + cfduid, cf_clearance = cloudflare_cookies() + + # Use a class to workaround a `responses` bug where + # cookies aren't mocked correctly. + class Test(cfscrape.CloudflareScraper): + def __init__(self, *args, **kwargs): + kwargs.setdefault('delay', delay) + super(Test, self).__init__(*args, **kwargs) + + self.cookies.set('__cfduid', cfduid) + self.cookies.set('cf_clearance', cf_clearance) + + tokens = Test.get_tokens(url, **kwargs) + tokens.should.be.a('tuple') + tokens.should.have.length_of(2) + + cookies, user_agent = tokens + + cookies.should.be.a('dict') + cookies.should.equal({ + 'cf_clearance': cf_clearance.value, + '__cfduid': cfduid.value + }) + + user_agent.should.equal(expected_ua) + + @challenge_responses(filename='js_challenge_10_04_2019.html', jschl_answer='18.8766915031') + def test_get_tokens_missing_cookie(self, **kwargs): + # get_tokens doesn't accept the delay kwarg. + delay = kwargs.pop('delay', 0.1) + + # Use derived class to set delay and test without cookies + class Test(cfscrape.CloudflareScraper): + def __init__(self, *args, **kwargs): + kwargs.setdefault('delay', delay) + super(Test, self).__init__(*args, **kwargs) + + message = re.compile(r'Unable to find Cloudflare cookies') + Test.get_tokens.when.called_with(url, **kwargs) \ + .should.have.raised(ValueError, message) + + @responses.activate + def test_get_tokens_request_error(self, caplog): + # get_tokens doesn't accept the delay kwarg. + kwargs = cfscrape_kwargs.copy() + kwargs.pop('delay', None) + + responses.add(DefaultResponse(url=url, status=500)) + cfscrape.get_tokens.when.called_with(url, **kwargs) \ + .should.have.raised(requests.HTTPError) + caplog.text.should.match(re.compile(r'Could not collect tokens')) + + @challenge_responses(filename='js_challenge_10_04_2019.html', jschl_answer='18.8766915031') + def test_cloudflare_is_bypassed(self, **kwargs): + # Use a class to workaround a `responses` bug where + # cookies aren't mocked correctly. + class Test(cfscrape.CloudflareScraper): + def __init__(self, *args, **kwargs): + super(Test, self).__init__(*args, **kwargs) + + cf_clearance = cloudflare_cookies()[1] + self.cookies.set('cf_clearance', cf_clearance) + + scraper = Test(**kwargs) + scraper.cloudflare_is_bypassed(url).should.be.ok + + def test_create_scraper_with_session(self): + session = requests.session() + session.headers = {'foo': 'bar'} + session.data = None + + scraper = cfscrape.create_scraper(sess=session) + scraper.headers.should.equal(session.headers) + scraper.should_not.have.property('data') + + session.data = {'bar': 'foo'} + scraper = cfscrape.create_scraper(sess=session) + scraper.data.should.equal(session.data)

1 0

commit python-imread for openSUSE:Leap:15.2
by root 01 May '20

01 May '20

Hello community, here is the log from the commit of package python-imread for openSUSE:Leap:15.2 checked in at 2020-04-30 18:51:32 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Leap:15.2/python-imread (Old) and /work/SRC/openSUSE:Leap:15.2/.python-imread.new.2738 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Package is "python-imread" Thu Apr 30 18:51:32 2020 rev:4 rq:794276 version:0.7.4 Changes: -------- --- /work/SRC/openSUSE:Leap:15.2/python-imread/python-imread.changes 2020-03-09 18:06:28.664850643 +0100 +++ /work/SRC/openSUSE:Leap:15.2/.python-imread.new.2738/python-imread.changes 2020-04-30 18:51:34.224614762 +0200 @@ -1,0 +2,7 @@ +Tue Apr 14 12:26:14 UTC 2020 - Marketa Calabkova <mcalabkova(a)suse.com> + +- update to 0.7.4 + * Add missing data to distribution + * Fix several memory access bugs in parsers (reported by Robert Scott) + +------------------------------------------------------------------- Old: ---- imread-0.7.1.tar.gz New: ---- imread-0.7.4.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ python-imread.spec ++++++ --- /var/tmp/diff_new_pack.puJJLC/_old 2020-04-30 18:51:34.564615487 +0200 +++ /var/tmp/diff_new_pack.puJJLC/_new 2020-04-30 18:51:34.568615495 +0200 @@ -1,7 +1,7 @@ # # spec file for package python-imread # -# Copyright (c) 2019 SUSE LINUX GmbH, Nuernberg, Germany. +# Copyright (c) 2020 SUSE LLC # # All modifications and additions to the file contributed by third parties # remain the property of their copyright owners, unless otherwise agreed @@ -18,12 +18,12 @@ %{?!python_module:%define python_module() python-%{**} python3-%{**}} Name: python-imread -Version: 0.7.1 +Version: 0.7.4 Release: 0 Summary: Image reading library License: MIT Group: Development/Languages/Python -Url: http://luispedro.org/software/imread +URL: http://luispedro.org/software/imread Source: https://files.pythonhosted.org/packages/source/i/imread/imread-%{version}.t… BuildRequires: %{python_module devel} BuildRequires: %{python_module numpy-devel} @@ -32,6 +32,7 @@ BuildRequires: fdupes BuildRequires: giflib-devel BuildRequires: libtiff-devel +BuildRequires: pkgconfig BuildRequires: python-rpm-macros BuildRequires: pkgconfig(libjpeg) BuildRequires: pkgconfig(libpng) @@ -39,8 +40,7 @@ # SECTION test requirements BuildRequires: %{python_module nose} # /SECTION -BuildRequires: python-numpy -BuildRoot: %{_tmppath}/%{name}-%{version}-build +Requires: python-numpy %python_subpackages %description @@ -70,7 +70,6 @@ %python_exec setup.py test %files %{python_files} -%defattr(-,root,root,-) %doc ChangeLog README.rst %license COPYING.MIT %{python_sitearch}/* ++++++ imread-0.7.1.tar.gz -> imread-0.7.4.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/imread-0.7.1/ChangeLog new/imread-0.7.4/ChangeLog --- old/imread-0.7.1/ChangeLog 2019-05-09 06:53:41.000000000 +0200 +++ new/imread-0.7.4/ChangeLog 2020-04-13 12:35:20.000000000 +0200 @@ -1,3 +1,12 @@ +Version 0.7.4 2020-04-14 by luispedro + * Add missing header files to distribution + +Version 0.7.3 2020-04-09 by luispedro + * Add missing test data to distribution + +Version 0.7.2 2020-03-24 by luispedro + * Fix several memory access bugs in parsers (reported by Robert Scott) + Version 0.7.1 2019-05-09 by luispedro * Fix 16-bit RGB/RGBA TIFF write (patch by Tomi Aarnio) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/imread-0.7.1/MANIFEST.in new/imread-0.7.4/MANIFEST.in --- old/imread-0.7.1/MANIFEST.in 2013-02-13 14:40:58.000000000 +0100 +++ new/imread-0.7.4/MANIFEST.in 2020-04-13 12:34:00.000000000 +0200 @@ -1,6 +1,7 @@ include README.rst include ChangeLog include COPYING.MIT -include imread/lib/*cpp +include imread/lib/*.cpp +include imread/lib/*.h include docs/source/*.rst -include imread/tests/data/* +recursive-include imread/tests/data *tiff *lsm *bmp diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/imread-0.7.1/PKG-INFO new/imread-0.7.4/PKG-INFO --- old/imread-0.7.1/PKG-INFO 2019-05-09 07:14:42.000000000 +0200 +++ new/imread-0.7.4/PKG-INFO 2020-04-13 13:17:07.000000000 +0200 @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: imread -Version: 0.7.1 +Version: 0.7.4 Summary: imread: Image reading library Home-page: http://luispedro.org/software/imread Author: Luis Pedro Coelho @@ -10,8 +10,8 @@ mahotas-imread: Read Image Files ================================ - .. image:: https://api.travis-ci.org/luispedro/imread.png - :target: https://travis-ci.org/luispedro/imread + .. image:: https://api.travis-ci.com/luispedro/imread.png + :target: https://travis-ci.com/luispedro/imread .. image:: https://anaconda.org/conda-forge/imread/badges/license.svg :target: http://opensource.org/licenses/MIT .. image:: https://anaconda.org/conda-forge/imread/badges/installer/conda.svg @@ -140,6 +140,14 @@ History ~~~~~~~ + Version 0.7.3 (2020-04-09) + -------------------------- + - Add missing test data to distribution + + Version 0.7.2 (2020-03-24) + -------------------------- + - Fix several memory access bugs in parsers (reported by Robert Scott) + Version 0.7.1 (2019-05-09) -------------------------- - Fix 16-bit RGB/RGBA TIFF write (patch by Tomi Aarnio) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/imread-0.7.1/README.rst new/imread-0.7.4/README.rst --- old/imread-0.7.1/README.rst 2019-05-09 07:00:05.000000000 +0200 +++ new/imread-0.7.4/README.rst 2020-04-09 16:33:46.000000000 +0200 @@ -2,8 +2,8 @@ mahotas-imread: Read Image Files ================================ -.. image:: https://api.travis-ci.org/luispedro/imread.png - :target: https://travis-ci.org/luispedro/imread +.. image:: https://api.travis-ci.com/luispedro/imread.png + :target: https://travis-ci.com/luispedro/imread .. image:: https://anaconda.org/conda-forge/imread/badges/license.svg :target: http://opensource.org/licenses/MIT .. image:: https://anaconda.org/conda-forge/imread/badges/installer/conda.svg @@ -132,6 +132,14 @@ History ~~~~~~~ +Version 0.7.3 (2020-04-09) +-------------------------- +- Add missing test data to distribution + +Version 0.7.2 (2020-03-24) +-------------------------- +- Fix several memory access bugs in parsers (reported by Robert Scott) + Version 0.7.1 (2019-05-09) -------------------------- - Fix 16-bit RGB/RGBA TIFF write (patch by Tomi Aarnio) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/imread-0.7.1/docs/source/history.rst new/imread-0.7.4/docs/source/history.rst --- old/imread-0.7.1/docs/source/history.rst 2019-05-09 06:54:29.000000000 +0200 +++ new/imread-0.7.4/docs/source/history.rst 2020-04-13 12:35:41.000000000 +0200 @@ -2,6 +2,18 @@ History ======= +Version 0.7.3 (2020-04-09) +-------------------------- +- Add missing header files to distribution + +Version 0.7.3 (2020-04-09) +-------------------------- +- Add missing test data to distribution + +Version 0.7.2 (2020-03-24) +-------------------------- +- Fix several memory access bugs in parsers (reported by Robert Scott) + Version 0.7.1 (2019-05-09) -------------------------- - Fix 16-bit RGB/RGBA TIFF write (patch by Tomi Aarnio) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/imread-0.7.1/imread/_imread.cpp new/imread-0.7.4/imread/_imread.cpp --- old/imread-0.7.1/imread/_imread.cpp 2019-05-09 06:52:45.000000000 +0200 +++ new/imread-0.7.4/imread/_imread.cpp 2019-05-12 17:44:29.000000000 +0200 @@ -1,4 +1,4 @@ -// Copyright 2012-2016 Luis Pedro Coelho <luis(a)luispedro.org> +// Copyright 2012-2019 Luis Pedro Coelho <luis(a)luispedro.org> // License: MIT (see COPYING.MIT file) @@ -89,7 +89,7 @@ return res; } -std::auto_ptr<byte_source> get_input(PyObject* filename_or_blob_object, const bool is_blob) { +std::unique_ptr<byte_source> get_input(PyObject* filename_or_blob_object, const bool is_blob) { if (is_blob) { size_t len; const char* data = get_blob(filename_or_blob_object, len); @@ -97,7 +97,7 @@ PyErr_SetString(PyExc_TypeError, "Expected blob of bytes"); throw py_exception_set(); } - return std::auto_ptr<byte_source>(new memory_source(reinterpret_cast<const byte*>(data), len)); + return std::unique_ptr<byte_source>(new memory_source(reinterpret_cast<const byte*>(data), len)); } else { const char* filename = get_cstring(filename_or_blob_object); if (!filename) throw py_exception_set(); @@ -114,7 +114,7 @@ PyErr_SetString(PyExc_OSError, ss.str().c_str()); throw py_exception_set(); } - return std::auto_ptr<byte_source>(new fd_source_sink(fd)); + return std::unique_ptr<byte_source>(new fd_source_sink(fd)); } } @@ -131,7 +131,7 @@ } try { - std::auto_ptr<byte_source> input = get_input(filename_or_blob_object, is_blob); + std::unique_ptr<byte_source> input = get_input(filename_or_blob_object, is_blob); const char* format = magic_format(&*input); if (!format) Py_RETURN_NONE; #if PY_MAJOR_VERSION >= 3 @@ -177,7 +177,7 @@ options_map opts = parse_options(optsDict); try { - std::auto_ptr<ImageFormat> format(get_format(formatstr)); + std::unique_ptr<ImageFormat> format(get_format(formatstr)); if (!format.get()) { std::stringstream ss; ss << "This format (" << formatstr << ") is unknown to imread"; @@ -201,9 +201,9 @@ } NumpyFactory factory; - std::auto_ptr<byte_source> input = get_input(filename_or_blob_object, is_blob); + std::unique_ptr<byte_source> input = get_input(filename_or_blob_object, is_blob); if (is_multi) { - std::auto_ptr<image_list> images = format->read_multi(input.get(), &factory, opts); + std::unique_ptr<image_list> images = format->read_multi(input.get(), &factory, opts); PyObject* output = PyList_New(images->size()); if (!output) return NULL; std::vector<Image*> pages = images->release(); @@ -213,7 +213,7 @@ } return output; } else { - std::auto_ptr<Image> output = format->read(input.get(), &factory, opts); + std::unique_ptr<Image> output = format->read(input.get(), &factory, opts); PyObject* final = PyTuple_New(2); if (!final) return NULL; PyTuple_SET_ITEM(final, 0, static_cast<NumpyImage&>(*output).releasePyObject()); @@ -266,7 +266,7 @@ } try { options_map opts = parse_options(optsDict); - std::auto_ptr<ImageFormat> format(get_format(formatstr)); + std::unique_ptr<ImageFormat> format(get_format(formatstr)); if (!format.get()) { std::stringstream ss; ss << "Handler not found for format '" << formatstr << "'"; @@ -290,7 +290,7 @@ throw CannotWriteError(ss.str()); } - std::auto_ptr<byte_sink> output(new fd_source_sink(fd)); + std::unique_ptr<byte_sink> output(new fd_source_sink(fd)); if (is_multi) { image_list array_list; @@ -304,7 +304,7 @@ // ~NumpyImage() will decrease the count Py_INCREF(array); NumpyImage* input = new NumpyImage(array); - array_list.push_back(std::auto_ptr<Image>(input)); + array_list.push_back(std::unique_ptr<Image>(input)); } format->write_multi(&array_list, output.get(), opts); } else { diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/imread-0.7.1/imread/imread_version.py new/imread-0.7.4/imread/imread_version.py --- old/imread-0.7.1/imread/imread_version.py 2019-05-09 06:53:57.000000000 +0200 +++ new/imread-0.7.4/imread/imread_version.py 2020-04-13 12:34:10.000000000 +0200 @@ -1 +1 @@ -__version__ = '0.7.1' +__version__ = '0.7.4' diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/imread-0.7.1/imread/lib/_bmp.cpp new/imread-0.7.4/imread/lib/_bmp.cpp --- old/imread-0.7.1/imread/lib/_bmp.cpp 2019-05-09 06:52:45.000000000 +0200 +++ new/imread-0.7.4/imread/lib/_bmp.cpp 2020-03-23 13:07:36.000000000 +0100 @@ -1,4 +1,4 @@ -// Copyright 2012-2013 Luis Pedro Coelho <luis(a)luispedro.org> +// Copyright 2012-2020 Luis Pedro Coelho <luis(a)luispedro.org> // License: MIT (see COPYING.MIT file) #define NO_IMPORT_ARRAY @@ -25,6 +25,9 @@ // awkward, but works correctly std::vector<byte>::const_iterator pbegin = color_table.begin(); for (int i = w-1; i >= 0; --i) { + if (color_table.size() < 4*row[i] + 3) { + throw CannotReadError("Malformed BMP file: color table is too small"); + } std::copy(pbegin + 4*row[i], pbegin + 4*row[i] + 3, row + 3*i); } } @@ -35,7 +38,7 @@ } } -std::auto_ptr<Image> BMPFormat::read(byte_source* src, ImageFactory* factory, const options_map&) { +std::unique_ptr<Image> BMPFormat::read(byte_source* src, ImageFactory* factory, const options_map&) { char magick[2]; if (src->read(reinterpret_cast<byte*>(magick), 2) != 2) { throw CannotReadError("imread.bmp: File is empty"); @@ -78,13 +81,13 @@ } const int depth = (bitsppixel == 16 ? -1 : 3); const int nbits = (bitsppixel == 16 ? 16 : 8); - std::auto_ptr<Image> output(factory->create(nbits, height, width, depth)); + std::unique_ptr<Image> output(factory->create(nbits, height, width, depth)); std::vector<byte> color_table; if (bitsppixel <= 8) { const uint32_t table_size = (n_colours == 0 ? pow2(bitsppixel) : n_colours); color_table.resize(table_size*4); - src->read_check(&color_table[0], table_size*4); + src->read_check(color_table.data(), table_size*4); } src->seek_absolute(offset); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/imread-0.7.1/imread/lib/_bmp.h new/imread-0.7.4/imread/lib/_bmp.h --- old/imread-0.7.1/imread/lib/_bmp.h 2019-05-09 06:52:45.000000000 +0200 +++ new/imread-0.7.4/imread/lib/_bmp.h 2019-05-12 17:44:29.000000000 +0200 @@ -1,4 +1,4 @@ -// Copyright 2012 Luis Pedro Coelho <luis(a)luispedro.org> +// Copyright 2012-2019 Luis Pedro Coelho <luis(a)luispedro.org> // License: MIT (see COPYING.MIT file) #ifndef LPC_BMP_H_INCLUDE_GUARD_THU_OCT_25_20_16_30_WEST_2012 @@ -11,7 +11,7 @@ bool can_read() const { return true; } bool can_write() const { return false; } - std::auto_ptr<Image> read(byte_source* src, ImageFactory* factory, const options_map& opts); + std::unique_ptr<Image> read(byte_source* src, ImageFactory* factory, const options_map& opts); }; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/imread-0.7.1/imread/lib/_jpeg.cpp new/imread-0.7.4/imread/lib/_jpeg.cpp --- old/imread-0.7.1/imread/lib/_jpeg.cpp 2019-05-09 06:52:45.000000000 +0200 +++ new/imread-0.7.4/imread/lib/_jpeg.cpp 2019-05-12 17:44:29.000000000 +0200 @@ -1,4 +1,4 @@ -// Copyright 2012-2014 Luis Pedro Coelho <luis(a)luispedro.org> +// Copyright 2012-2019 Luis Pedro Coelho <luis(a)luispedro.org> // License: MIT (see COPYING.MIT file) #include "base.h" #include "_jpeg.h" @@ -145,7 +145,7 @@ } // namespace -std::auto_ptr<Image> JPEGFormat::read(byte_source* src, ImageFactory* factory, const options_map&) { +std::unique_ptr<Image> JPEGFormat::read(byte_source* src, ImageFactory* factory, const options_map&) { jpeg_source_adaptor adaptor(src); jpeg_decompress_holder c; @@ -165,7 +165,7 @@ const int h = c.info.output_height; const int w = c.info.output_width; const int d = c.info.output_components; - std::auto_ptr<Image> output(factory->create(8, h, w, d)); + std::unique_ptr<Image> output(factory->create(8, h, w, d)); for (int r = 0; r != h; ++r) { byte* rowp = output->rowp_as<byte>(r); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/imread-0.7.1/imread/lib/_jpeg.h new/imread-0.7.4/imread/lib/_jpeg.h --- old/imread-0.7.1/imread/lib/_jpeg.h 2019-05-09 06:52:45.000000000 +0200 +++ new/imread-0.7.4/imread/lib/_jpeg.h 2019-05-12 17:44:29.000000000 +0200 @@ -1,4 +1,4 @@ -// Copyright 2012-2014 Luis Pedro Coelho <luis(a)luispedro.org> +// Copyright 2012-2019 Luis Pedro Coelho <luis(a)luispedro.org> // License: MIT (see COPYING.MIT file) #ifndef LPC_JPEG_H_INCLUDE_GUARD_THU_FEB__2_18_14_07_WET_2012 @@ -13,7 +13,7 @@ static bool match_format(byte_source* src) { return match_magic(src, "\xff\xd8\xff", 3); } - std::auto_ptr<Image> read(byte_source* src, ImageFactory* factory, const options_map& opts); + std::unique_ptr<Image> read(byte_source* src, ImageFactory* factory, const options_map& opts); void write(Image* input, byte_sink* output, const options_map& opts); }; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/imread-0.7.1/imread/lib/_lsm.cpp new/imread-0.7.4/imread/lib/_lsm.cpp --- old/imread-0.7.1/imread/lib/_lsm.cpp 2019-05-09 06:52:45.000000000 +0200 +++ new/imread-0.7.4/imread/lib/_lsm.cpp 2020-03-23 13:10:55.000000000 +0100 @@ -5,7 +5,7 @@ This is an open-source copyright as follows: Copyright (c) 2004-2008 BioImageXD Development Team - Copyright (C) 2012 Luis Pedro Coelho + Copyright (C) 2012-2020 Luis Pedro Coelho All rights reserved. @@ -154,7 +154,7 @@ ~LSMReader(); void PrintSelf(std::ostream& os, const char* indent=""); - std::auto_ptr<Image> read(ImageFactory* factory, const options_map&); + std::unique_ptr<Image> read(ImageFactory* factory, const options_map&); void readHeader(); int GetChannelColorComponent(int,int); @@ -167,10 +167,10 @@ private: + static int FindChannelNameStart(const byte *, int); + static int ReadChannelName(const byte *, int, byte *); + unsigned long ReadImageDirectory(byte_source *,unsigned long); - void SetChannelName(const char *,int); - int FindChannelNameStart(const char *, int); - int ReadChannelName(const char *, int, char *); int ReadChannelDataTypes(byte_source*, unsigned long); int ReadChannelColorsAndNames(byte_source *,unsigned long); int ReadTimeStampInformation(byte_source *,unsigned long); @@ -232,45 +232,51 @@ }; -int ReadFile(byte_source* s, unsigned long *pos,int size,void *buf,bool swap=false) +int ReadFile(byte_source* s, unsigned long *pos, int size, void *buf, bool swap=false) { - s->seek_absolute(*pos); - const unsigned ret = s->read(reinterpret_cast<byte*>(buf), size); + s->seek_absolute(*pos); + const unsigned ret = s->read(reinterpret_cast<byte*>(buf), size); #ifdef VTK_WORDS_BIGENDIAN if(swap) { vtkByteSwap::SwapLERange(buf,size); } #endif - *pos += ret; - return ret; + *pos += ret; + return ret; } -int ReadData(byte_source* s, unsigned long *pos,int size,char *buf) { - return ReadFile(s,pos,size,buf,1); -} - std::string read_str(byte_source* s, unsigned long* pos, const unsigned long len) { char* buf = new char[len]; - ReadData(s, pos, len, buf); + ReadFile(s, pos, len, buf, 1); std::string res(buf, len); delete [] buf; return res; } -unsigned char CharPointerToUnsignedChar(char *buf) -{ - return *((unsigned char*)(buf)); -} int CharPointerToInt(char *buf) { return *((int*)(buf)); } -unsigned int CharPointerToUnsignedInt(char *buf) -{ - return *((unsigned int*)(buf)); + +uint32_t parse_uint32_t(const byte *buf) { +#ifdef VTK_WORDS_BIGENDIAN + char buf2[4]; + ::memcpy(buf2, buf, 4); + vtkByteSwap::Swap4LE(buf2); + return *reinterpret_cast<const uint32_t*>(buf2); +#else + return *reinterpret_cast<const uint32_t*>(buf); +#endif +} + +uint32_t parse_uint32_t(const std::vector<byte> &buf) { + if (buf.size() < 4) { + throw CannotReadError("Malformed LSM file: expected 4 Bytes, cannot parse uint32_t"); + } + return parse_uint32_t(buf.data()); } short CharPointerToShort(char *buf) @@ -278,9 +284,22 @@ return *((short*)(buf)); } -unsigned short CharPointerToUnsignedShort(char *buf) -{ - return *((unsigned short*)(buf)); + +uint16_t parse_uint16_t(const byte *buf) { +#ifdef VTK_WORDS_BIGENDIAN + char buf2[2]; + buf2[0] = buf[1]; + buf2[1] = buf[0]; + return *reinterpret_cast<const uint16_t*>(buf2); +#else + return *reinterpret_cast<const uint16_t*>(buf); +#endif +} +uint16_t parse_uint16_t(const std::vector<byte>& buf) { + if (buf.size() < 2) { + throw CannotReadError("Failed to read short (size(vec) < 2)"); + } + return parse_uint16_t(buf.data()); } double CharPointerToDouble(char *buf) @@ -298,14 +317,10 @@ return CharPointerToInt(buff); } -unsigned int ReadUnsignedInt(byte_source* s, unsigned long *pos) -{ - char buff[4]; - ReadFile(s,pos,4,buff); -#ifdef VTK_WORDS_BIGENDIAN - vtkByteSwap::Swap4LE((unsigned int*)buff); -#endif - return CharPointerToUnsignedInt(buff); +unsigned int ReadUnsignedInt(byte_source* s, unsigned long *pos) { + byte buff[4]; + ReadFile(s, pos, 4, buff); + return parse_uint32_t(buff); } short ReadShort(byte_source* s, unsigned long *pos) @@ -318,14 +333,10 @@ return CharPointerToShort(buff); } -unsigned short ReadUnsignedShort(byte_source* s, unsigned long *pos) -{ - char buff[2]; - ReadFile(s,pos,2,buff); -#ifdef VTK_WORDS_BIGENDIAN - vtkByteSwap::Swap2LE((unsigned short*)buff); -#endif - return CharPointerToUnsignedShort(buff); +uint16_t ReadUnsignedShort(byte_source* s, unsigned long *pos) { + byte buff[2]; + ReadFile(s,pos,2,buff); + return parse_uint16_t(buff); } double ReadDouble(byte_source* s, unsigned long *pos) @@ -436,23 +447,15 @@ return this->channel_names_[chNum]; } -void LSMReader::SetChannelName(const char * name, const int chNum) -{ - const int n_channels = this->dimensions_[4]; - if(!name || chNum > n_channels) return; - this->channel_names_.resize(n_channels); - this->channel_names_[chNum] = std::string(name); -} -int LSMReader::FindChannelNameStart(const char *buf, const int length) { +int LSMReader::FindChannelNameStart(const byte *buf, const int length) { for (int i = 0; i < length; ++i) { - char ch = buf[i]; - if (ch > 32) return i; + if (buf[i] > 32) return i; } return length; } -int LSMReader::ReadChannelName(const char *nameBuff, const int length, char *buffer) { +int LSMReader::ReadChannelName(const byte *nameBuff, const int length, byte *buffer) { for (int i = 0; i < length; ++i) { buffer[i] = nameBuff[i]; if (!buffer[i]) return i; @@ -472,67 +475,64 @@ return 0; } -int LSMReader::ReadChannelColorsAndNames(byte_source* s, unsigned long start) -{ - int colNum,nameNum,sizeOfStructure,sizeOfNames,nameLength, nameSkip; - unsigned long colorOffset,nameOffset,pos; - char *nameBuff,*name,*tempBuff; - unsigned char component; - - pos = start; +int LSMReader::ReadChannelColorsAndNames(byte_source* s, const unsigned long start) { + unsigned long pos = start; // Read size of structure - sizeOfStructure = ReadInt(s,&pos); + const int sizeOfStructure = ReadInt(s, &pos); // Read number of colors - colNum = ReadInt(s,&pos); + const int n_cols = ReadInt(s, &pos); // Read number of names - nameNum = ReadInt(s,&pos); - sizeOfNames = sizeOfStructure - ( (10*4) + (colNum*4) ); - - nameBuff = new char[sizeOfNames+1]; - name = new char[sizeOfNames+1]; + const int n_names = ReadInt(s, &pos); + const int sizeOfNames = sizeOfStructure - ( (10*4) + (n_cols*4) ); - if(colNum != this->dimensions_[4]) { - // not great + if(n_cols != this->dimensions_[4]) { + throw CannotReadError("LSM file seems malformed (n_cols != dimensions_[4])"); } - if(nameNum != this->dimensions_[4]) { - // not great + if(n_names != this->dimensions_[4]) { + throw CannotReadError("LSM file seems malformed (n_names != dimensions_[4])"); } // Read offset to color info - colorOffset = ReadInt(s,&pos) + start; + unsigned long colorOffset = ReadInt(s, &pos) + start; // Read offset to name info - nameOffset = ReadInt(s,&pos) + start; + unsigned long nameOffset = ReadInt(s, &pos) + start; - this->channel_colors_.resize( 3 * (colNum+1)); + this->channel_colors_.resize( 3 * (n_cols+1)); // Read the colors for(int j = 0; j < this->dimensions_[4]; ++j) { char colorBuff[5]; - ReadFile(s,&colorOffset,4,colorBuff,1); + ReadFile(s, &colorOffset, 4, colorBuff, 1); for(int i=0;i<3;i++) { - component = CharPointerToUnsignedChar(colorBuff+i); - this->channel_colors_[i + 3*j] = component; + this->channel_colors_[i + 3*j] = static_cast<unsigned char>(colorBuff[i]); } } - ReadFile(s,&nameOffset,sizeOfNames,nameBuff,1); - nameLength = nameSkip = 0; - tempBuff = nameBuff; + std::vector<byte> nameBuff; + nameBuff.resize(sizeOfNames + 1); + std::vector<byte> name; + name.resize(sizeOfNames + 1); + + ReadFile(s, &nameOffset, sizeOfNames, nameBuff.data(), 1); + + this->channel_names_.resize(this->dimensions_[4]); + int nameStart = 0; for(int i = 0; i < this->dimensions_[4]; i++) { - nameSkip = this->FindChannelNameStart(tempBuff,sizeOfNames-nameSkip); - nameLength = this->ReadChannelName(tempBuff+nameSkip,sizeOfNames-nameSkip,name); + nameStart += FindChannelNameStart(nameBuff.data() + nameStart, + sizeOfNames-nameStart); + if (nameStart >= sizeOfNames) { + throw CannotReadError("LSM file malformed"); + } + const int nameLength = ReadChannelName(nameBuff.data()+nameStart, sizeOfNames-nameStart, name.data()); + nameStart += nameLength; - tempBuff += nameSkip + nameLength; - this->SetChannelName(name,i); + this->channel_names_[i] = std::string(reinterpret_cast<const char*>(name.data())); } - delete [] nameBuff; - delete [] name; return 0; } -int LSMReader::ReadTimeStampInformation(byte_source* s, unsigned long offset) -{ +int LSMReader::ReadTimeStampInformation(byte_source* s, unsigned long offset) { // position is 0 for non-timeseries files! if( offset == 0 ) return 0; @@ -551,9 +551,7 @@ * * */ -int LSMReader::ReadLSMSpecificInfo(byte_source* s, unsigned long pos) -{ - unsigned long offset; +int LSMReader::ReadLSMSpecificInfo(byte_source* s, unsigned long pos) { pos += 2 * 4; // skip over the start of the LSMInfo // first 4 byte entry if magic number @@ -645,7 +643,7 @@ // SKip Zeiss Vision KS-3D speific data pos += 4; // Read timestamp information - offset = ReadUnsignedInt(s,&pos); + unsigned long offset = ReadUnsignedInt(s, &pos); this->ReadTimeStampInformation(s,offset); return 1; @@ -765,156 +763,111 @@ } while (subblocksOpen); return 0; } -int LSMReader::AnalyzeTag(byte_source* s, unsigned long startPos) -{ - int value, dataSize; - char tempValue[4],tempValue2[4]; - char *actualValue = NULL; - const unsigned short tag = ReadUnsignedShort(s,&startPos); - const unsigned short type = ReadUnsignedShort(s,&startPos); - const unsigned short length = ReadUnsignedInt(s,&startPos); - ReadFile(s,&startPos,4,tempValue); - - for(int i=0;i<4;i++)tempValue2[i]=tempValue[i]; -#ifdef VTK_WORDS_BIGENDIAN - vtkByteSwap::Swap4LE((unsigned int*)tempValue2); -#endif - value = CharPointerToUnsignedInt(tempValue2); - - // if there is more than 4 bytes in value, - // value is an offset to the actual data - dataSize = TIFF_BYTES(type); - const unsigned long readSize = dataSize*length; - if(readSize > 4 && tag != TIF_CZ_LSMINFO) - { - actualValue = new char[readSize]; - startPos = value; - if(tag == TIF_STRIPOFFSETS ||tag == TIF_STRIPBYTECOUNTS) { - if( !ReadFile(s,&startPos,readSize,actualValue) ) { - throw CannotReadError("Failed to get strip offsets\n"); +int LSMReader::AnalyzeTag(byte_source* s, unsigned long startPos) { + std::vector<byte> valueData; + const unsigned short tag = ReadUnsignedShort(s, &startPos); + const unsigned short type = ReadUnsignedShort(s, &startPos); + const unsigned short length = ReadUnsignedInt(s, &startPos); + + valueData.resize(4); + ReadFile(s, &startPos, 4, valueData.data()); + + const int value = parse_uint32_t(valueData); + + // if there is more than 4 bytes in value, + // value is an offset to the actual data + const int dataSize = TIFF_BYTES(type); + const unsigned long readSize = dataSize*length; + if(readSize > 4 && tag != TIF_CZ_LSMINFO) { + valueData.resize(readSize); + startPos = value; + if(tag == TIF_STRIPOFFSETS ||tag == TIF_STRIPBYTECOUNTS) { + if( !ReadFile(s, &startPos, readSize, valueData.data()) ) { + throw CannotReadError("Failed to get strip offsets\n"); + } } } - } - else - { - actualValue = new char[4]; - for(int o=0;o<4;o++)actualValue[o] = tempValue[o]; - } - switch(tag) - { - case TIF_NEWSUBFILETYPE: - this->NewSubFileType = value; - break; - - case TIF_IMAGEWIDTH: -#ifdef VTK_WORDS_BIGENDIAN - vtkByteSwap::Swap4LE((unsigned int*)actualValue); -#endif - //this->dimensions_[0] = this->CharPointerToUnsignedInt(actualValue); - //this->dimensions_[0] = value; - break; - - case TIF_IMAGELENGTH: -#ifdef VTK_WORDS_BIGENDIAN - vtkByteSwap::Swap4LE((unsigned int*)actualValue); - //this->dimensions_[1] = this->CharPointerToUnsignedInt(actualValue); -#endif - //this->dimensions_[1] = value; - break; - - case TIF_BITSPERSAMPLE: -#ifdef VTK_WORDS_BIGENDIAN - vtkByteSwap::Swap2LE((unsigned short*)actualValue); -#endif - this->bits_per_sample_.resize(length); - unsigned short bits_per_sample_; - for(int i=0;i<length;i++) - { - bits_per_sample_ = CharPointerToUnsignedShort(actualValue + TIFF_BYTES(TIFF_SHORT)*i); - this->bits_per_sample_[i] = bits_per_sample_; - } - break; - - case TIF_COMPRESSION: -#ifdef VTK_WORDS_BIGENDIAN - vtkByteSwap::Swap2LE((unsigned short*)actualValue); -#endif - this->compression_ = CharPointerToUnsignedShort(actualValue); - break; - - case TIF_PHOTOMETRICINTERPRETATION: -#ifdef VTK_WORDS_BIGENDIAN - vtkByteSwap::Swap2LE((unsigned short*)actualValue); -#endif - this->PhotometricInterpretation = CharPointerToUnsignedShort(actualValue); - break; - - case TIF_STRIPOFFSETS: - this->strip_offset_.resize(length); -#ifdef VTK_WORDS_BIGENDIAN - vtkByteSwap::Swap4LERange((unsigned int*)actualValue,length); -#endif - if(length>1) { - for(int i=0;i<length;++i) - { - unsigned int* offsets = (unsigned int*)actualValue; - this->strip_offset_[i] = offsets[i]; + switch(tag) { + case TIF_NEWSUBFILETYPE: + this->NewSubFileType = value; + break; + + case TIF_IMAGEWIDTH: + //this->dimensions_[0] = parse_uint32_t(valueData); + //this->dimensions_[0] = value; + break; + + case TIF_IMAGELENGTH: + //this->dimensions_[1] = parse_uint32_t(valueData); + //this->dimensions_[1] = value; + break; + + case TIF_BITSPERSAMPLE: + if (valueData.size() < TIFF_BYTES(TIFF_SHORT) * length) { + throw CannotReadError("LSM file is malformed (TIF_BITSPERSAMPLE field is too short)"); } - } else { - this->strip_offset_[0] = value; - } - break; + this->bits_per_sample_.resize(length); + for(int i=0;i<length;i++) { + this->bits_per_sample_[i] = parse_uint16_t(valueData.data() + TIFF_BYTES(TIFF_SHORT)*i); + } + break; - case TIF_SAMPLESPERPIXEL: -#ifdef VTK_WORDS_BIGENDIAN - vtkByteSwap::Swap4LE((unsigned int*)actualValue); -#endif - this->sample_per_pixel_ = CharPointerToUnsignedInt(actualValue); - break; + case TIF_COMPRESSION: + this->compression_ = parse_uint16_t(valueData); + break; + + case TIF_PHOTOMETRICINTERPRETATION: + this->PhotometricInterpretation = parse_uint16_t(valueData); + break; + + case TIF_STRIPOFFSETS: + this->strip_offset_.resize(length); + if(length>1) { + if (length * sizeof(uint32_t) > valueData.size()) { + throw CannotReadError("LSM file is malformed (TIF_STRIPOFFSETS field is too short)"); + } + for(int i=0;i<length;++i) { + this->strip_offset_[i] = parse_uint32_t(valueData.data() + sizeof(uint32_t) * i); + } + } else { + this->strip_offset_[0] = value; + } + break; - case TIF_STRIPBYTECOUNTS: -#ifdef VTK_WORDS_BIGENDIAN - vtkByteSwap::Swap4LERange((unsigned int*)actualValue,length); -#endif - this->strip_byte_count_.resize(length); - if (length > 1) { - for(int i=0; i<length; ++i) { - this->strip_byte_count_[i] = CharPointerToUnsignedInt(actualValue + TIFF_BYTES(TIFF_LONG)*i); + case TIF_SAMPLESPERPIXEL: + this->sample_per_pixel_ = parse_uint32_t(valueData); + break; + + case TIF_STRIPBYTECOUNTS: + this->strip_byte_count_.resize(length); + if (length > 1) { + for(int i=0; i<length; ++i) { + if (valueData.size() < TIFF_BYTES(TIFF_LONG) * i + 4) { + throw CannotReadError(); + } + this->strip_byte_count_[i] = parse_uint32_t(valueData.data() + TIFF_BYTES(TIFF_LONG)*i); + } + } else { + this->strip_byte_count_[0] = value; } - } else { - this->strip_byte_count_[0] = value; - } - break; - case TIF_PLANARCONFIGURATION: -#ifdef VTK_WORDS_BIGENDIAN - vtkByteSwap::Swap2LE((unsigned short*)actualValue); -#endif - this->PlanarConfiguration = CharPointerToUnsignedShort(actualValue); - break; - case TIF_PREDICTOR: -#ifdef VTK_WORDS_BIGENDIAN - vtkByteSwap::Swap2LE((unsigned short*)actualValue); -#endif - this->Predictor = CharPointerToUnsignedShort(actualValue); - break; - case TIF_COLORMAP: -#ifdef VTK_WORDS_BIGENDIAN - vtkByteSwap::Swap4LE((unsigned int*)actualValue); -#endif - //this->ColorMapOffset = CharPointerToUnsignedInt(actualValue); - break; - case TIF_CZ_LSMINFO: + break; + case TIF_PLANARCONFIGURATION: + this->PlanarConfiguration = parse_uint16_t(valueData); + break; + case TIF_PREDICTOR: + this->Predictor = parse_uint16_t(valueData); + break; + case TIF_COLORMAP: + //this->ColorMapOffset = parse_uint32_t(valueData); + break; + case TIF_CZ_LSMINFO: - this->LSMSpecificInfoOffset = value; - break; + this->LSMSpecificInfoOffset = value; + break; } - if(actualValue) - { - delete [] actualValue; - } - return 0; + return 0; } @@ -1017,8 +970,8 @@ std::vector<unsigned char> decoded = lzw_decode(buffer, size); unsigned char* outbufp = &decoded[0]; - int width = this->dimensions_[0]; - int lines = size / (width*bytes); + const int width = this->dimensions_[0]; + const int lines = size / (width*bytes); for(int line = 0; line < lines; line++) { if(this->Predictor == 2) { @@ -1136,12 +1089,12 @@ } } -std::auto_ptr<Image> LSMReader::read(ImageFactory* factory, const options_map&) { +std::unique_ptr<Image> LSMReader::read(ImageFactory* factory, const options_map&) { this->readHeader(); const int dataType = this->GetDataTypeForChannel(0); // This could vary by channel! - std::auto_ptr<Image> output = factory->create( + std::unique_ptr<Image> output = factory->create( BYTES_BY_DATA_TYPE(dataType)*8, this->dimensions_[2], this->dimensions_[3], @@ -1178,7 +1131,7 @@ } // namespace -std::auto_ptr<Image> LSMFormat::read(byte_source* s, ImageFactory* factory, const options_map& opts) { +std::unique_ptr<Image> LSMFormat::read(byte_source* s, ImageFactory* factory, const options_map& opts) { LSMReader reader(s); return reader.read(factory, opts); } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/imread-0.7.1/imread/lib/_lsm.h new/imread-0.7.4/imread/lib/_lsm.h --- old/imread-0.7.1/imread/lib/_lsm.h 2019-05-09 06:52:45.000000000 +0200 +++ new/imread-0.7.4/imread/lib/_lsm.h 2019-05-12 17:44:29.000000000 +0200 @@ -64,7 +64,7 @@ public: bool can_read() const { return true; } - std::auto_ptr<Image> read(byte_source* src, ImageFactory* factory, const options_map& opts); + std::unique_ptr<Image> read(byte_source* src, ImageFactory* factory, const options_map& opts); }; #endif diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/imread-0.7.1/imread/lib/_png.cpp new/imread-0.7.4/imread/lib/_png.cpp --- old/imread-0.7.1/imread/lib/_png.cpp 2019-05-09 06:52:45.000000000 +0200 +++ new/imread-0.7.4/imread/lib/_png.cpp 2019-05-12 17:44:29.000000000 +0200 @@ -89,7 +89,7 @@ } } -std::auto_ptr<Image> PNGFormat::read(byte_source* src, ImageFactory* factory, const options_map& opts) { +std::unique_ptr<Image> PNGFormat::read(byte_source* src, ImageFactory* factory, const options_map& opts) { png_holder p(png_holder::read_mode); png_set_read_fn(p.png_ptr, src, read_from_source); p.create_info(); @@ -141,7 +141,7 @@ } } - std::auto_ptr<Image> output(factory->create(bit_depth, h, w, d)); + std::unique_ptr<Image> output(factory->create(bit_depth, h, w, d)); std::vector<png_bytep> rowps = allrows<png_byte>(*output); png_read_image(p.png_ptr, &rowps[0]); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/imread-0.7.1/imread/lib/_png.h new/imread-0.7.4/imread/lib/_png.h --- old/imread-0.7.1/imread/lib/_png.h 2019-05-09 06:52:45.000000000 +0200 +++ new/imread-0.7.4/imread/lib/_png.h 2019-05-12 17:44:29.000000000 +0200 @@ -1,4 +1,4 @@ -// Copyright 2012-2014 Luis Pedro Coelho <luis(a)luispedro.org> +// Copyright 2012-2019 Luis Pedro Coelho <luis(a)luispedro.org> // License: MIT (see COPYING.MIT file) #ifndef LPC_PNG_H_INCLUDE_GUARD_WED_FEB__1_16_34_50_WET_2012 @@ -9,7 +9,7 @@ bool can_write() const { return true; } static bool match_format(byte_source* src) { return match_magic(src, "\x89\x50\x4E\x47\x0D\x0A\x1A\x0A", 8); } - std::auto_ptr<Image> read(byte_source* src, ImageFactory* factory, const options_map& opts); + std::unique_ptr<Image> read(byte_source* src, ImageFactory* factory, const options_map& opts); void write(Image* input, byte_sink* output, const options_map& opts); }; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/imread-0.7.1/imread/lib/_tiff.cpp new/imread-0.7.4/imread/lib/_tiff.cpp --- old/imread-0.7.1/imread/lib/_tiff.cpp 2019-05-09 06:52:45.000000000 +0200 +++ new/imread-0.7.4/imread/lib/_tiff.cpp 2020-03-22 23:42:05.000000000 +0100 @@ -1,4 +1,4 @@ -// Copyright 2012-2015 Luis Pedro Coelho <luis(a)luispedro.org> +// Copyright 2012-2019 Luis Pedro Coelho <luis(a)luispedro.org> // License: MIT (see COPYING.MIT file) #define NO_IMPORT_ARRAY @@ -8,6 +8,7 @@ #include <sstream> #include <iostream> +#include <utility> #include <cstdio> #include <cstring> @@ -81,7 +82,11 @@ struct tif_holder { tif_holder(TIFF* tif) :tif(tif) - { } + { + if (!tif) { + throw CannotReadError(); + } + } ~tif_holder() { TIFFClose(tif); @@ -215,13 +220,13 @@ } // namespace -std::auto_ptr<image_list> STKFormat::read_multi(byte_source* src, ImageFactory* factory) { +std::unique_ptr<image_list> STKFormat::read_multi(byte_source* src, ImageFactory* factory) { shift_source moved(src); stk_extend ext; tiff_warn_error twe; tif_holder t = read_client(&moved); - std::auto_ptr<image_list> images(new image_list); + std::unique_ptr<image_list> images(new image_list); const uint32 h = tiff_get<uint32>(t, TIFFTAG_IMAGELENGTH); const uint32 w = tiff_get<uint32>(t, TIFFTAG_IMAGEWIDTH); @@ -242,7 +247,7 @@ // Monkey patch strip offsets. This is very hacky, but it seems to work! moved.shift(z * raw_strip_size); - std::auto_ptr<Image> output(factory->create(bits_per_sample, h, w, depth)); + std::unique_ptr<Image> output(factory->create(bits_per_sample, h, w, depth)); uint8_t* start = output->rowp_as<uint8_t>(0); for (int st = 0; st != n_strips; ++st) { const int offset = TIFFReadEncodedStrip(t.tif, st, start, strip_size); @@ -251,15 +256,15 @@ } start += offset; } - images->push_back(output); + images->push_back(std::move(output)); } return images; } -std::auto_ptr<image_list> TIFFFormat::do_read(byte_source* src, ImageFactory* factory, bool is_multi) { +std::unique_ptr<image_list> TIFFFormat::do_read(byte_source* src, ImageFactory* factory, bool is_multi) { tiff_warn_error twe; tif_holder t = read_client(src); - std::auto_ptr<image_list> images(new image_list); + std::unique_ptr<image_list> images(new image_list); do { const uint32 h = tiff_get<uint32>(t, TIFFTAG_IMAGELENGTH); const uint32 w = tiff_get<uint32>(t, TIFFTAG_IMAGEWIDTH); @@ -267,7 +272,7 @@ const uint16 bits_per_sample = tiff_get<uint16>(t, TIFFTAG_BITSPERSAMPLE); const int depth = nr_samples > 1 ? nr_samples : -1; - std::auto_ptr<Image> output = factory->create(bits_per_sample, h, w, depth); + std::unique_ptr<Image> output = factory->create(bits_per_sample, h, w, depth); if (ImageWithMetadata* metaout = dynamic_cast<ImageWithMetadata*>(output.get())) { std::string description = tiff_get<std::string>(t, TIFFTAG_IMAGEDESCRIPTION, ""); metaout->set_meta(description); @@ -277,7 +282,7 @@ throw CannotReadError("imread.imread._tiff: Error reading scanline"); } } - images->push_back(output); + images->push_back(std::move(output)); } while (is_multi && TIFFReadDirectory(t.tif)); return images; } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/imread-0.7.1/imread/lib/_tiff.h new/imread-0.7.4/imread/lib/_tiff.h --- old/imread-0.7.1/imread/lib/_tiff.h 2019-05-09 06:52:45.000000000 +0200 +++ new/imread-0.7.4/imread/lib/_tiff.h 2019-05-12 17:44:29.000000000 +0200 @@ -1,4 +1,4 @@ -// Copyright 2012 Luis Pedro Coelho <luis(a)luispedro.org> +// Copyright 2012-2019 Luis Pedro Coelho <luis(a)luispedro.org> // License: MIT (see COPYING.MIT file) #ifndef LPC_TIFF_INCLUDE_GUARD_Wed_Feb__8_19_02_16_WET_2012 @@ -14,20 +14,20 @@ bool can_write_multi() const { return true; } bool can_write_metadata() const { return true; } - std::auto_ptr<Image> read(byte_source* s, ImageFactory* f, const options_map& opts) { - std::auto_ptr<image_list> pages = this->do_read(s, f, false); + std::unique_ptr<Image> read(byte_source* s, ImageFactory* f, const options_map& opts) { + std::unique_ptr<image_list> pages = this->do_read(s, f, false); if (pages->size() != 1) throw ProgrammingError(); std::vector<Image*> ims = pages->release(); - return std::auto_ptr<Image>(ims[0]); + return std::unique_ptr<Image>(ims[0]); } - std::auto_ptr<image_list> read_multi(byte_source* s, ImageFactory* f, const options_map& opts) { + std::unique_ptr<image_list> read_multi(byte_source* s, ImageFactory* f, const options_map& opts) { return this->do_read(s, f, true); } void write(Image* input, byte_sink* output, const options_map& opts); void write_multi(image_list* input, byte_sink* output, const options_map& opts); private: - std::auto_ptr<image_list> do_read(byte_source* s, ImageFactory* f, bool is_multi); + std::unique_ptr<image_list> do_read(byte_source* s, ImageFactory* f, bool is_multi); void do_write(image_list* input, byte_sink* output, const options_map& opts, bool); }; @@ -38,7 +38,7 @@ bool can_read_multi() const { return true; } bool can_write() const { return false; } - std::auto_ptr<image_list> read_multi(byte_source* s, ImageFactory* f); + std::unique_ptr<image_list> read_multi(byte_source* s, ImageFactory* f); }; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/imread-0.7.1/imread/lib/_webp.cpp new/imread-0.7.4/imread/lib/_webp.cpp --- old/imread-0.7.1/imread/lib/_webp.cpp 2019-05-09 06:52:45.000000000 +0200 +++ new/imread-0.7.4/imread/lib/_webp.cpp 2019-05-12 17:44:29.000000000 +0200 @@ -1,4 +1,4 @@ -// Copyright 2012-2014 Luis Pedro Coelho <luis(a)luispedro.org> +// Copyright 2012-2019 Luis Pedro Coelho <luis(a)luispedro.org> // License: MIT (see COPYING.MIT file) #include "base.h" @@ -7,14 +7,14 @@ #include <webp/decode.h> -std::auto_ptr<Image> WebPFormat::read(byte_source* src, ImageFactory* factory, const options_map&) { +std::unique_ptr<Image> WebPFormat::read(byte_source* src, ImageFactory* factory, const options_map&) { std::vector<byte> data = full_data(*src); int w, h; int ok = WebPGetInfo(&data[0], data.size(), &w, &h); if (!ok) { throw CannotReadError("imread.imread._webp: File does not validate as WebP"); } - std::auto_ptr<Image> output(factory->create(8, h, w, 4)); + std::unique_ptr<Image> output(factory->create(8, h, w, 4)); const int stride = w*4; const uint8_t* p = WebPDecodeRGBAInto( &data[0], data.size(), diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/imread-0.7.1/imread/lib/_webp.h new/imread-0.7.4/imread/lib/_webp.h --- old/imread-0.7.1/imread/lib/_webp.h 2019-05-09 06:52:45.000000000 +0200 +++ new/imread-0.7.4/imread/lib/_webp.h 2019-05-12 17:44:29.000000000 +0200 @@ -1,4 +1,4 @@ -// Copyright 2012 Luis Pedro Coelho <luis(a)luispedro.org> +// Copyright 2012-2019 Luis Pedro Coelho <luis(a)luispedro.org> // License: MIT (see COPYING.MIT file) #ifndef LPC_WEBP_H_INCLUDE_GUARD_ @@ -8,7 +8,7 @@ bool can_read() const { return true; } bool can_write() const { return false; } - std::auto_ptr<Image> read(byte_source* src, ImageFactory* factory, const options_map&); + std::unique_ptr<Image> read(byte_source* src, ImageFactory* factory, const options_map&); }; #endif // LPC_WEBP_H_INCLUDE_GUARD_ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/imread-0.7.1/imread/lib/base.h new/imread-0.7.4/imread/lib/base.h --- old/imread-0.7.1/imread/lib/base.h 2019-05-09 06:52:45.000000000 +0200 +++ new/imread-0.7.4/imread/lib/base.h 2019-05-12 17:44:29.000000000 +0200 @@ -85,7 +85,7 @@ public: virtual ~Image() { } - virtual std::auto_ptr<Image> clone() const = 0; + virtual std::unique_ptr<Image> clone() const = 0; virtual void* rowp(int r) = 0; @@ -112,7 +112,7 @@ class ImageFactory { public: virtual ~ImageFactory() { } - virtual std::auto_ptr<Image> + virtual std::unique_ptr<Image> create(int nbits, int d0, int d1, int d2, int d3=-1, int d4=-1) = 0; protected: }; @@ -135,7 +135,7 @@ for (unsigned i = 0; i != content.size(); ++i) delete content[i]; } std::vector<Image*>::size_type size() const { return content.size(); } - void push_back(std::auto_ptr<Image> p) { content.push_back(p.release()); } + void push_back(std::unique_ptr<Image>&& p) { content.push_back(p.release()); } Image* at(const unsigned ix) const { return content.at(ix); } /// After release(), all of the pointers will be owned by the caller @@ -233,10 +233,10 @@ virtual bool can_write_multi() const { return false; } virtual bool can_write_metadata() const { return false; } - virtual std::auto_ptr<Image> read(byte_source* src, ImageFactory* factory, const options_map&) { + virtual std::unique_ptr<Image> read(byte_source* src, ImageFactory* factory, const options_map&) { throw NotImplementedError(); } - virtual std::auto_ptr<image_list> read_multi(byte_source* src, ImageFactory* factory, const options_map&) { + virtual std::unique_ptr<image_list> read_multi(byte_source* src, ImageFactory* factory, const options_map&) { throw NotImplementedError(); } virtual void write(Image* input, byte_sink* output, const options_map&) { diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/imread-0.7.1/imread/lib/errors.h new/imread-0.7.4/imread/lib/errors.h --- old/imread-0.7.1/imread/lib/errors.h 2013-10-04 08:27:30.000000000 +0200 +++ new/imread-0.7.4/imread/lib/errors.h 2020-03-20 15:56:53.000000000 +0100 @@ -1,4 +1,4 @@ -// Copyright 2012 Luis Pedro Coelho <luis(a)luispedro.org> +// Copyright 2012-2020 Luis Pedro Coelho <luis(a)luispedro.org> // License: MIT (see COPYING.MIT file) #ifndef LPC_ERRORS_H_INCLUDE_GUARD_WED_FEB__1_16_34_50_WET_2012 @@ -27,6 +27,7 @@ DECLARE_IMREAD_ERROR_TYPE(CannotReadError, "Read Error"); DECLARE_IMREAD_ERROR_TYPE(CannotWriteError, "Write Error"); +DECLARE_IMREAD_ERROR_TYPE(CannotSeekError, "Could not seek in file"); DECLARE_IMREAD_ERROR_TYPE(NotImplementedError, "Not Implemented"); DECLARE_IMREAD_ERROR_TYPE(ProgrammingError, "Programming Error"); DECLARE_IMREAD_ERROR_TYPE(OptionsError, "Options Error"); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/imread-0.7.1/imread/lib/file.h new/imread-0.7.4/imread/lib/file.h --- old/imread-0.7.1/imread/lib/file.h 2013-07-12 13:29:53.000000000 +0200 +++ new/imread-0.7.4/imread/lib/file.h 2020-03-20 15:52:42.000000000 +0100 @@ -1,4 +1,4 @@ -// Copyright 2012 Luis Pedro Coelho <luis(a)luispedro.org> +// Copyright 2012-2020 Luis Pedro Coelho <luis(a)luispedro.org> // License: MIT (see COPYING.MIT file) #ifndef LPC_FILE_H_INCLUDE_GUARD_WED_FEB__1_16_34_50_WET_2012 #define LPC_FILE_H_INCLUDE_GUARD_WED_FEB__1_16_34_50_WET_2012 @@ -23,9 +23,9 @@ return ::read(fd_, buffer, n); } virtual bool can_seek() const { return true; } - virtual size_t seek_absolute(size_t pos) { return ::lseek(fd_, pos, SEEK_SET); } - virtual size_t seek_relative(int delta) { return ::lseek(fd_, delta, SEEK_CUR); } - virtual size_t seek_end(int delta) { return ::lseek(fd_, delta, SEEK_END); } + virtual size_t seek_absolute(size_t pos) { return _check_seek(fd_, pos, SEEK_SET); } + virtual size_t seek_relative(int delta) { return _check_seek(fd_, delta, SEEK_CUR); } + virtual size_t seek_end(int delta) { return _check_seek(fd_, delta, SEEK_END); } virtual size_t write(const byte* buffer, size_t n) { @@ -33,6 +33,13 @@ } private: int fd_; + static off_t _check_seek(int fd, off_t offset, int whence) { + off_t r = ::lseek(fd, offset, whence); + if (r == static_cast<off_t>(-1)) { + throw CannotSeekError(); + } + return r; + } }; #endif // LPC_FILE_H_INCLUDE_GUARD_WED_FEB__1_16_34_50_WET_2012 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/imread-0.7.1/imread/lib/formats.cpp new/imread-0.7.4/imread/lib/formats.cpp --- old/imread-0.7.1/imread/lib/formats.cpp 2019-05-09 06:52:45.000000000 +0200 +++ new/imread-0.7.4/imread/lib/formats.cpp 2019-05-12 17:44:29.000000000 +0200 @@ -1,4 +1,4 @@ -// Copyright 2012-2014 Luis Pedro Coelho <luis(a)luispedro.org> +// Copyright 2012-2019 Luis Pedro Coelho <luis(a)luispedro.org> // License: MIT (see COPYING.MIT file) #include "formats.h" @@ -14,22 +14,22 @@ #include <cstring> -std::auto_ptr<ImageFormat> get_format(const char* format) { +std::unique_ptr<ImageFormat> get_format(const char* format) { using std::strcmp; - if (!strcmp(format, "png")) return std::auto_ptr<ImageFormat>(new PNGFormat); - if (!strcmp(format, "jpeg") || !strcmp(format, "jpg")) return std::auto_ptr<ImageFormat>(new JPEGFormat); - if (!strcmp(format, "lsm")) return std::auto_ptr<ImageFormat>(new LSMFormat); - if (!strcmp(format, "tiff") || !strcmp(format, "tif")) return std::auto_ptr<ImageFormat>(new TIFFFormat); + if (!strcmp(format, "png")) return std::unique_ptr<ImageFormat>(new PNGFormat); + if (!strcmp(format, "jpeg") || !strcmp(format, "jpg")) return std::unique_ptr<ImageFormat>(new JPEGFormat); + if (!strcmp(format, "lsm")) return std::unique_ptr<ImageFormat>(new LSMFormat); + if (!strcmp(format, "tiff") || !strcmp(format, "tif")) return std::unique_ptr<ImageFormat>(new TIFFFormat); #if IMREAD_EXCLUDE_WEBP - if (!strcmp(format, "webp")) return std::auto_ptr<ImageFormat>(0); + if (!strcmp(format, "webp")) return std::unique_ptr<ImageFormat>(nullptr); #else - if (!strcmp(format, "webp")) return std::auto_ptr<ImageFormat>(new WebPFormat); + if (!strcmp(format, "webp")) return std::unique_ptr<ImageFormat>(new WebPFormat); #endif - if (!strcmp(format, "stk")) return std::auto_ptr<ImageFormat>(new STKFormat); - if (!strcmp(format, "bmp")) return std::auto_ptr<ImageFormat>(new BMPFormat); - return std::auto_ptr<ImageFormat>(0); + if (!strcmp(format, "stk")) return std::unique_ptr<ImageFormat>(new STKFormat); + if (!strcmp(format, "bmp")) return std::unique_ptr<ImageFormat>(new BMPFormat); + return std::unique_ptr<ImageFormat>(nullptr); } const char* magic_format(byte_source* src) { diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/imread-0.7.1/imread/lib/formats.h new/imread-0.7.4/imread/lib/formats.h --- old/imread-0.7.1/imread/lib/formats.h 2019-05-09 06:52:45.000000000 +0200 +++ new/imread-0.7.4/imread/lib/formats.h 2019-05-12 17:44:29.000000000 +0200 @@ -1,4 +1,4 @@ -// Copyright 2012-2014 Luis Pedro Coelho <luis(a)luispedro.org> +// Copyright 2012-2019 Luis Pedro Coelho <luis(a)luispedro.org> // License: MIT (see COPYING.MIT file) #ifndef LPC_FORMATS_H_INCLUDE_GUARD_WED_FEB__1_16_34_50_WET_2012 @@ -6,7 +6,7 @@ #include "base.h" #include <memory> -std::auto_ptr<ImageFormat> get_format(const char*); +std::unique_ptr<ImageFormat> get_format(const char*); const char* magic_format(byte_source*); #endif // LPC_FORMATS_H_INCLUDE_GUARD_WED_FEB__1_16_34_50_WET_2012 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/imread-0.7.1/imread/lib/numpy.h new/imread-0.7.4/imread/lib/numpy.h --- old/imread-0.7.1/imread/lib/numpy.h 2019-05-09 06:52:45.000000000 +0200 +++ new/imread-0.7.4/imread/lib/numpy.h 2019-05-12 17:44:29.000000000 +0200 @@ -1,4 +1,4 @@ -// Copyright 2012-2015 Luis Pedro Coelho <luis(a)luispedro.org> +// Copyright 2012-2019 Luis Pedro Coelho <luis(a)luispedro.org> // License: MIT (see COPYING.MIT file) #ifndef LPC_NUMPY_H_INCLUDE_GUARD_WED_FEB__1_16_34_50_WET_2012 @@ -20,9 +20,9 @@ Py_XDECREF(array_); } - std::auto_ptr<Image> clone() const { + std::unique_ptr<Image> clone() const { Py_XINCREF(array_); - return std::auto_ptr<Image>(new NumpyImage(this->array_)); + return std::unique_ptr<Image>(new NumpyImage(this->array_)); } PyArrayObject* release() { @@ -86,7 +86,7 @@ class NumpyFactory : public ImageFactory { protected: - std::auto_ptr<Image> create(int nbits, int d0, int d1, int d2, int d3, int d4) { + std::unique_ptr<Image> create(int nbits, int d0, int d1, int d2, int d3, int d4) { npy_intp dims[5]; dims[0] = d0; dims[1] = d1; @@ -114,7 +114,7 @@ PyArrayObject* array = reinterpret_cast<PyArrayObject*>(PyArray_SimpleNew(nd, dims, dtype)); if (!array) throw std::bad_alloc(); try { - return std::auto_ptr<Image>(new NumpyImage(array)); + return std::unique_ptr<Image>(new NumpyImage(array)); } catch(...) { Py_DECREF(array); throw; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/imread-0.7.1/imread/tests/data/T new/imread-0.7.4/imread/tests/data/T --- old/imread-0.7.1/imread/tests/data/T 1970-01-01 01:00:00.000000000 +0100 +++ new/imread-0.7.4/imread/tests/data/T 2020-03-23 13:13:23.000000000 +0100 @@ -0,0 +1,7 @@ +bad-files/BMP/00r/00r06.bmp +bad-files/BMP/00r/00r03.bmp +bad-files/BMP/00r/00r01.bmp +bad-files/BMP/00r/00r00.bmp +bad-files/BMP/00r/00r02.bmp +bad-files/BMP/00r/00r04.bmp +bad-files/BMP/00r/00r05.bmp Binary files old/imread-0.7.1/imread/tests/data/bad-files/BMP/00r/00r00.bmp and new/imread-0.7.4/imread/tests/data/bad-files/BMP/00r/00r00.bmp differ Binary files old/imread-0.7.1/imread/tests/data/bad-files/BMP/00r/00r01.bmp and new/imread-0.7.4/imread/tests/data/bad-files/BMP/00r/00r01.bmp differ Binary files old/imread-0.7.1/imread/tests/data/bad-files/BMP/00r/00r02.bmp and new/imread-0.7.4/imread/tests/data/bad-files/BMP/00r/00r02.bmp differ Binary files old/imread-0.7.1/imread/tests/data/bad-files/BMP/00r/00r03.bmp and new/imread-0.7.4/imread/tests/data/bad-files/BMP/00r/00r03.bmp differ Binary files old/imread-0.7.1/imread/tests/data/bad-files/BMP/00r/00r04.bmp and new/imread-0.7.4/imread/tests/data/bad-files/BMP/00r/00r04.bmp differ Binary files old/imread-0.7.1/imread/tests/data/bad-files/BMP/00r/00r05.bmp and new/imread-0.7.4/imread/tests/data/bad-files/BMP/00r/00r05.bmp differ Binary files old/imread-0.7.1/imread/tests/data/bad-files/BMP/00r/00r06.bmp and new/imread-0.7.4/imread/tests/data/bad-files/BMP/00r/00r06.bmp differ Binary files old/imread-0.7.1/imread/tests/data/bad-files/LSM/00r/00r00.lsm and new/imread-0.7.4/imread/tests/data/bad-files/LSM/00r/00r00.lsm differ Binary files old/imread-0.7.1/imread/tests/data/bad-files/LSM/00r/00r01.lsm and new/imread-0.7.4/imread/tests/data/bad-files/LSM/00r/00r01.lsm differ Binary files old/imread-0.7.1/imread/tests/data/bad-files/LSM/00r/00r02.lsm and new/imread-0.7.4/imread/tests/data/bad-files/LSM/00r/00r02.lsm differ Binary files old/imread-0.7.1/imread/tests/data/bad-files/LSM/00r/00r03.lsm and new/imread-0.7.4/imread/tests/data/bad-files/LSM/00r/00r03.lsm differ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/imread-0.7.1/imread/tests/data/bad-files/LSM/00r/notes new/imread-0.7.4/imread/tests/data/bad-files/LSM/00r/notes --- old/imread-0.7.1/imread/tests/data/bad-files/LSM/00r/notes 1970-01-01 01:00:00.000000000 +0100 +++ new/imread-0.7.4/imread/tests/data/bad-files/LSM/00r/notes 2020-03-22 23:42:05.000000000 +0100 @@ -0,0 +1 @@ +AnalyzeTag doesn't check the value of `length` fits within the read buffer Binary files old/imread-0.7.1/imread/tests/data/bad-files/LSM/01r/01r00.lsm and new/imread-0.7.4/imread/tests/data/bad-files/LSM/01r/01r00.lsm differ Binary files old/imread-0.7.1/imread/tests/data/bad-files/LSM/01r/01r01.lsm and new/imread-0.7.4/imread/tests/data/bad-files/LSM/01r/01r01.lsm differ Binary files old/imread-0.7.1/imread/tests/data/bad-files/LSM/01r/01r02.lsm and new/imread-0.7.4/imread/tests/data/bad-files/LSM/01r/01r02.lsm differ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/imread-0.7.1/imread/tests/data/bad-files/LSM/01r/notes new/imread-0.7.4/imread/tests/data/bad-files/LSM/01r/notes --- old/imread-0.7.1/imread/tests/data/bad-files/LSM/01r/notes 1970-01-01 01:00:00.000000000 +0100 +++ new/imread-0.7.4/imread/tests/data/bad-files/LSM/01r/notes 2020-03-22 23:42:05.000000000 +0100 @@ -0,0 +1 @@ +ReadImageDirectory doesn't check there is sufficient space for numberOfTags in the read buffer Binary files old/imread-0.7.1/imread/tests/data/bad-files/LSM/02r/02r00.lsm and new/imread-0.7.4/imread/tests/data/bad-files/LSM/02r/02r00.lsm differ Binary files old/imread-0.7.1/imread/tests/data/bad-files/LSM/02r/02r01.lsm and new/imread-0.7.4/imread/tests/data/bad-files/LSM/02r/02r01.lsm differ Binary files old/imread-0.7.1/imread/tests/data/bad-files/LSM/02r/02r02.lsm and new/imread-0.7.4/imread/tests/data/bad-files/LSM/02r/02r02.lsm differ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/imread-0.7.1/imread/tests/data/bad-files/LSM/02r/notes new/imread-0.7.4/imread/tests/data/bad-files/LSM/02r/notes --- old/imread-0.7.1/imread/tests/data/bad-files/LSM/02r/notes 1970-01-01 01:00:00.000000000 +0100 +++ new/imread-0.7.4/imread/tests/data/bad-files/LSM/02r/notes 2020-03-22 23:42:05.000000000 +0100 @@ -0,0 +1 @@ +ReadChannelColorsAndNames doesn't check there is enough space for sizeOfNames in the read buffer Binary files old/imread-0.7.1/imread/tests/data/bad-files/LSM/03w/03w00.lsm and new/imread-0.7.4/imread/tests/data/bad-files/LSM/03w/03w00.lsm differ Binary files old/imread-0.7.1/imread/tests/data/bad-files/LSM/03w/03w01.lsm and new/imread-0.7.4/imread/tests/data/bad-files/LSM/03w/03w01.lsm differ Binary files old/imread-0.7.1/imread/tests/data/bad-files/LSM/03w/03w02.lsm and new/imread-0.7.4/imread/tests/data/bad-files/LSM/03w/03w02.lsm differ Binary files old/imread-0.7.1/imread/tests/data/bad-files/LSM/03w/03w03.lsm and new/imread-0.7.4/imread/tests/data/bad-files/LSM/03w/03w03.lsm differ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/imread-0.7.1/imread/tests/data/bad-files/LSM/03w/notes new/imread-0.7.4/imread/tests/data/bad-files/LSM/03w/notes --- old/imread-0.7.1/imread/tests/data/bad-files/LSM/03w/notes 1970-01-01 01:00:00.000000000 +0100 +++ new/imread-0.7.4/imread/tests/data/bad-files/LSM/03w/notes 2020-03-22 23:42:05.000000000 +0100 @@ -0,0 +1,3 @@ +ReadChannelColorsAndNames allows the file contents to control the size of the channel_colors_ vector (down to +a size of zero by providing a negative colnum?), which it can then cause to be written past the end of by +altering the value that ends up in dimensions[4]. Binary files old/imread-0.7.1/imread/tests/data/bad-files/TIFF/00r00.tiff and new/imread-0.7.4/imread/tests/data/bad-files/TIFF/00r00.tiff differ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/imread-0.7.1/imread/tests/test_bad.py new/imread-0.7.4/imread/tests/test_bad.py --- old/imread-0.7.1/imread/tests/test_bad.py 1970-01-01 01:00:00.000000000 +0100 +++ new/imread-0.7.4/imread/tests/test_bad.py 2020-03-23 13:14:28.000000000 +0100 @@ -0,0 +1,42 @@ +from nose.tools import raises +from imread import imread +from . import file_path + + +BAD_FILES = [ + 'bad-files/LSM/00r/00r01.lsm', + 'bad-files/LSM/00r/00r03.lsm', + 'bad-files/LSM/00r/00r02.lsm', + 'bad-files/LSM/00r/00r00.lsm', + + 'bad-files/LSM/01r/01r00.lsm', + 'bad-files/LSM/01r/01r01.lsm', + 'bad-files/LSM/01r/01r02.lsm', + + 'bad-files/LSM/02r/02r00.lsm', + 'bad-files/LSM/02r/02r01.lsm', + 'bad-files/LSM/02r/02r02.lsm', + + 'bad-files/LSM/03w/03w01.lsm', + 'bad-files/LSM/03w/03w00.lsm', + 'bad-files/LSM/03w/03w02.lsm', + 'bad-files/LSM/03w/03w03.lsm', + + 'bad-files/TIFF/00r00.tiff', + + 'bad-files/BMP/00r/00r00.bmp', + 'bad-files/BMP/00r/00r01.bmp', + 'bad-files/BMP/00r/00r02.bmp', + 'bad-files/BMP/00r/00r03.bmp', + 'bad-files/BMP/00r/00r04.bmp', + 'bad-files/BMP/00r/00r05.bmp', + 'bad-files/BMP/00r/00r06.bmp', + +] +def test_read(): + @raises(RuntimeError) + def read1(fname): + imread(file_path(fname)) + assert False + for fname in BAD_FILES: + read1(fname) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/imread-0.7.1/imread/tests/test_rois.py new/imread-0.7.4/imread/tests/test_rois.py --- old/imread-0.7.1/imread/tests/test_rois.py 2018-09-22 13:44:02.000000000 +0200 +++ new/imread-0.7.4/imread/tests/test_rois.py 2019-05-12 17:47:23.000000000 +0200 @@ -5,7 +5,7 @@ def test_rois_smoke(): - rois = ijrois.read_roi_zip('./imread/tests/data/rois.zip') + rois = ijrois.read_roi_zip(file_path('rois.zip')) assert len(rois) == 4 - r = ijrois.read_roi(open('./imread/tests/data/0186-0099.roi', 'rb')) + r = ijrois.read_roi(open(file_path('0186-0099.roi'), 'rb')) assert any([np.array_equal(ri, r) for ri in rois]) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/imread-0.7.1/imread.egg-info/PKG-INFO new/imread-0.7.4/imread.egg-info/PKG-INFO --- old/imread-0.7.1/imread.egg-info/PKG-INFO 2019-05-09 07:14:42.000000000 +0200 +++ new/imread-0.7.4/imread.egg-info/PKG-INFO 2020-04-13 13:17:06.000000000 +0200 @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: imread -Version: 0.7.1 +Version: 0.7.4 Summary: imread: Image reading library Home-page: http://luispedro.org/software/imread Author: Luis Pedro Coelho @@ -10,8 +10,8 @@ mahotas-imread: Read Image Files ================================ - .. image:: https://api.travis-ci.org/luispedro/imread.png - :target: https://travis-ci.org/luispedro/imread + .. image:: https://api.travis-ci.com/luispedro/imread.png + :target: https://travis-ci.com/luispedro/imread .. image:: https://anaconda.org/conda-forge/imread/badges/license.svg :target: http://opensource.org/licenses/MIT .. image:: https://anaconda.org/conda-forge/imread/badges/installer/conda.svg @@ -140,6 +140,14 @@ History ~~~~~~~ + Version 0.7.3 (2020-04-09) + -------------------------- + - Add missing test data to distribution + + Version 0.7.2 (2020-03-24) + -------------------------- + - Fix several memory access bugs in parsers (reported by Robert Scott) + Version 0.7.1 (2019-05-09) -------------------------- - Fix 16-bit RGB/RGBA TIFF write (patch by Tomi Aarnio) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/imread-0.7.1/imread.egg-info/SOURCES.txt new/imread-0.7.4/imread.egg-info/SOURCES.txt --- old/imread-0.7.1/imread.egg-info/SOURCES.txt 2019-05-09 07:14:42.000000000 +0200 +++ new/imread-0.7.4/imread.egg-info/SOURCES.txt 2020-04-13 13:17:07.000000000 +0200 @@ -43,6 +43,7 @@ imread/lib/numpy.h imread/lib/tools.h imread/tests/__init__.py +imread/tests/test_bad.py imread/tests/test_bmp.py imread/tests/test_error.py imread/tests/test_formats.py @@ -58,6 +59,7 @@ imread/tests/data/0186-0099.roi imread/tests/data/1_webp_a.webp imread/tests/data/GOOD.PNG +imread/tests/data/T imread/tests/data/arange512_16bit.png imread/tests/data/bit1.png imread/tests/data/diag.xcf @@ -75,4 +77,30 @@ imread/tests/data/rgba.png imread/tests/data/rois.zip imread/tests/data/stack.tiff -imread/tests/data/star1.bmp \ No newline at end of file +imread/tests/data/star1.bmp +imread/tests/data/bad-files/BMP/00r/00r00.bmp +imread/tests/data/bad-files/BMP/00r/00r01.bmp +imread/tests/data/bad-files/BMP/00r/00r02.bmp +imread/tests/data/bad-files/BMP/00r/00r03.bmp +imread/tests/data/bad-files/BMP/00r/00r04.bmp +imread/tests/data/bad-files/BMP/00r/00r05.bmp +imread/tests/data/bad-files/BMP/00r/00r06.bmp +imread/tests/data/bad-files/LSM/00r/00r00.lsm +imread/tests/data/bad-files/LSM/00r/00r01.lsm +imread/tests/data/bad-files/LSM/00r/00r02.lsm +imread/tests/data/bad-files/LSM/00r/00r03.lsm +imread/tests/data/bad-files/LSM/00r/notes +imread/tests/data/bad-files/LSM/01r/01r00.lsm +imread/tests/data/bad-files/LSM/01r/01r01.lsm +imread/tests/data/bad-files/LSM/01r/01r02.lsm +imread/tests/data/bad-files/LSM/01r/notes +imread/tests/data/bad-files/LSM/02r/02r00.lsm +imread/tests/data/bad-files/LSM/02r/02r01.lsm +imread/tests/data/bad-files/LSM/02r/02r02.lsm +imread/tests/data/bad-files/LSM/02r/notes +imread/tests/data/bad-files/LSM/03w/03w00.lsm +imread/tests/data/bad-files/LSM/03w/03w01.lsm +imread/tests/data/bad-files/LSM/03w/03w02.lsm +imread/tests/data/bad-files/LSM/03w/03w03.lsm +imread/tests/data/bad-files/LSM/03w/notes +imread/tests/data/bad-files/TIFF/00r00.tiff \ No newline at end of file diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/imread-0.7.1/setup.py new/imread-0.7.4/setup.py --- old/imread-0.7.1/setup.py 2019-05-09 06:59:09.000000000 +0200 +++ new/imread-0.7.4/setup.py 2020-03-31 12:21:51.000000000 +0200 @@ -1,9 +1,9 @@ # -*- coding: utf-8 -*- # vim: set ts=4 sts=4 sw=4 expandtab smartindent: -# Copyright (C) 2012-2014, Luis Pedro Coelho <luis(a)luispedro.org> +# Copyright (C) 2012-2020, Luis Pedro Coelho <luis(a)luispedro.org> # License: MIT -from __future__ import division, print_function +from glob import glob import platform import sys @@ -109,7 +109,10 @@ 'imread.tests': 'imread/tests', } package_data = { - 'imread.tests': ['data/*'], + 'imread.tests': ['data/*', + 'data/bad-files/*/*.tiff', + 'data/bad-files/BMP/*/*.bmp', + 'data/bad-files/LSM/*/*.lsm'] } classifiers = [

1 0

commit python-shodan for openSUSE:Leap:15.2
by root 01 May '20

01 May '20

Hello community, here is the log from the commit of package python-shodan for openSUSE:Leap:15.2 checked in at 2020-04-30 18:51:29 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Leap:15.2/python-shodan (Old) and /work/SRC/openSUSE:Leap:15.2/.python-shodan.new.2738 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Package is "python-shodan" Thu Apr 30 18:51:29 2020 rev:21 rq:794077 version:1.23.0 Changes: -------- --- /work/SRC/openSUSE:Leap:15.2/python-shodan/python-shodan.changes 2020-02-10 16:42:35.691751582 +0100 +++ /work/SRC/openSUSE:Leap:15.2/.python-shodan.new.2738/python-shodan.changes 2020-04-30 18:51:32.800611728 +0200 @@ -1,0 +2,19 @@ +Tue Apr 7 07:07:18 UTC 2020 - Sebastian Wagner <sebix+novell.com(a)sebix.at> + +- update to version 1.23.0: + - Add new CLI command: shodan alert domain + +------------------------------------------------------------------- +Fri Mar 27 18:31:29 UTC 2020 - Sebastian Wagner <sebix+novell.com(a)sebix.at> + +- update to version 1.22.1: + - vulns: Fix bug when converting data file to CSV using Python3 + +------------------------------------------------------------------- +Fri Mar 27 10:45:45 UTC 2020 - pgajdos(a)suse.com + +- version update to 1.22.0 + * New Streaming API method: /shodan/vulns/{vulns} to subscribe to IPs + that are vulnerable to an issue + +------------------------------------------------------------------- Old: ---- shodan-1.21.3.tar.gz New: ---- shodan-1.23.0.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ python-shodan.spec ++++++ --- /var/tmp/diff_new_pack.v2TytW/_old 2020-04-30 18:51:33.440613091 +0200 +++ /var/tmp/diff_new_pack.v2TytW/_new 2020-04-30 18:51:33.444613100 +0200 @@ -19,7 +19,7 @@ %{?!python_module:%define python_module() python-%{**} python3-%{**}} %{!?license: %global license %doc} Name: python-shodan -Version: 1.21.3 +Version: 1.23.0 Release: 0 Summary: Python library and command-line utility for Shodan License: MIT @@ -77,7 +77,7 @@ %files %{python_files} %doc AUTHORS README.rst -%%license LICENSE +%license LICENSE %python3_only %{_bindir}/shodan %{python_sitelib}/* %python3_only %{_mandir}/man1/shodan.1%{ext_man} ++++++ shodan-1.21.3.tar.gz -> shodan-1.23.0.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/shodan-1.21.3/CHANGELOG.md new/shodan-1.23.0/CHANGELOG.md --- old/shodan-1.21.3/CHANGELOG.md 2020-01-20 20:57:07.000000000 +0100 +++ new/shodan-1.23.0/CHANGELOG.md 2020-04-06 22:00:31.000000000 +0200 @@ -1,6 +1,18 @@ CHANGELOG ========= +1.23.0 +------ +* Add new CLI command: shodan alert domain + +1.22.1 +------ +* Fix bug when converting data file to CSV using Python3 + +1.22.0 +------ +* Add support for new vulnerability streaming endpoints + 1.21.3 ------ * Fix geo.json file converter diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/shodan-1.21.3/PKG-INFO new/shodan-1.23.0/PKG-INFO --- old/shodan-1.21.3/PKG-INFO 2020-01-20 21:22:45.000000000 +0100 +++ new/shodan-1.23.0/PKG-INFO 2020-04-06 22:10:29.000000000 +0200 @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: shodan -Version: 1.21.3 +Version: 1.23.0 Summary: Python library and command-line utility for Shodan (https://developer.shodan.io) Home-page: http://github.com/achillean/shodan-python/tree/master Author: John Matherly diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/shodan-1.21.3/docs/examples/gifcreator.rst new/shodan-1.23.0/docs/examples/gifcreator.rst --- old/shodan-1.21.3/docs/examples/gifcreator.rst 2016-01-17 04:32:15.000000000 +0100 +++ new/shodan-1.23.0/docs/examples/gifcreator.rst 2020-03-17 23:24:35.000000000 +0100 @@ -106,7 +106,7 @@ os.system('rm -f /tmp/gif-image-*.jpg') # Show a progress indicator - print result['ip_str'] + print(result['ip_str']) -The full code is also available on GitHub: https://gist.github.com/achillean/963eea552233d9550101 \ No newline at end of file +The full code is also available on GitHub: https://gist.github.com/achillean/963eea552233d9550101 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/shodan-1.21.3/docs/examples/query-summary.rst new/shodan-1.23.0/docs/examples/query-summary.rst --- old/shodan-1.21.3/docs/examples/query-summary.rst 2017-12-03 06:42:17.000000000 +0100 +++ new/shodan-1.23.0/docs/examples/query-summary.rst 2020-03-17 23:24:35.000000000 +0100 @@ -48,7 +48,7 @@ # Input validation if len(sys.argv) == 1: - print 'Usage: %s <search query>' % sys.argv[0] + print('Usage: %s <search query>' % sys.argv[0]) sys.exit(1) try: @@ -62,22 +62,22 @@ # And it also runs faster than doing a search(). result = api.count(query, facets=FACETS) - print 'Shodan Summary Information' - print 'Query: %s' % query - print 'Total Results: %s\n' % result['total'] + print('Shodan Summary Information') + print('Query: %s' % query) + print('Total Results: %s\n' % result['total']) # Print the summary info from the facets for facet in result['facets']: - print FACET_TITLES[facet] + print(FACET_TITLES[facet]) for term in result['facets'][facet]: - print '%s: %s' % (term['value'], term['count']) + print('%s: %s' % (term['value'], term['count'])) # Print an empty line between summary info - print '' + print('') - except Exception, e: - print 'Error: %s' % e + except Exception as e: + print('Error: %s' % e) sys.exit(1) """ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/shodan-1.21.3/docs/tutorial.rst new/shodan-1.23.0/docs/tutorial.rst --- old/shodan-1.21.3/docs/tutorial.rst 2018-03-16 22:31:41.000000000 +0100 +++ new/shodan-1.23.0/docs/tutorial.rst 2020-03-17 23:24:35.000000000 +0100 @@ -56,7 +56,7 @@ print('IP: {}'.format(result['ip_str'])) print(result['data']) print('') - except shodan.APIError, e: + except shodan.APIError as e: print('Error: {}'.format(e)) Stepping through the code, we first call the :py:func:`Shodan.search` method on the `api` object which diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/shodan-1.21.3/setup.py new/shodan-1.23.0/setup.py --- old/shodan-1.21.3/setup.py 2020-01-20 20:55:55.000000000 +0100 +++ new/shodan-1.23.0/setup.py 2020-04-06 21:29:29.000000000 +0200 @@ -2,12 +2,14 @@ from setuptools import setup + DEPENDENCIES = open('requirements.txt', 'r').read().split('\n') README = open('README.rst', 'r').read() + setup( name='shodan', - version='1.21.3', + version='1.23.0', description='Python library and command-line utility for Shodan (https://developer.shodan.io)', long_description=README, long_description_content_type='text/x-rst', diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/shodan-1.21.3/shodan/__main__.py new/shodan-1.23.0/shodan/__main__.py --- old/shodan-1.21.3/shodan/__main__.py 2020-01-20 20:55:45.000000000 +0100 +++ new/shodan-1.23.0/shodan/__main__.py 2020-03-18 00:28:27.000000000 +0100 @@ -635,7 +635,8 @@ @click.option('--alert', help='The network alert ID or "all" to subscribe to all network alerts on your account.', default=None, type=str) @click.option('--tags', help='A comma-separated list of tags to grab data on.', default=None, type=str) @click.option('--compresslevel', help='The gzip compression level (0-9; 0 = no compression, 9 = most compression', default=9, type=int) -def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, streamer, countries, asn, alert, tags, compresslevel): +(a)click.option('--vulns', help='A comma-separated list of vulnerabilities to grab data on.', default=None, type=str) +def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, streamer, countries, asn, alert, tags, compresslevel, vulns): """Stream data in real-time.""" # Setup the Shodan API key = get_api_key() @@ -663,9 +664,11 @@ stream_type.append('alert') if tags: stream_type.append('tags') + if vulns: + stream_type.append('vulns') if len(stream_type) > 1: - raise click.ClickException('Please use --ports, --countries, --tags OR --asn. You cant subscribe to multiple filtered streams at once.') + raise click.ClickException('Please use --ports, --countries, --tags, --vulns OR --asn. You cant subscribe to multiple filtered streams at once.') stream_args = None @@ -689,6 +692,9 @@ if tags: stream_args = tags.split(',') + + if vulns: + stream_args = vulns.split(',') # Flatten the list of stream types # Possible values are: @@ -710,6 +716,7 @@ 'countries': api.stream.countries(args, timeout=timeout), 'ports': api.stream.ports(args, timeout=timeout), 'tags': api.stream.tags(args, timeout=timeout), + 'vulns': api.stream.vulns(args, timeout=timeout), }.get(name, 'all') stream = _create_stream(stream_type, stream_args, timeout=timeout) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/shodan-1.21.3/shodan/cli/alert.py new/shodan-1.23.0/shodan/cli/alert.py --- old/shodan-1.21.3/shodan/cli/alert.py 2019-02-24 10:31:35.000000000 +0100 +++ new/shodan-1.23.0/shodan/cli/alert.py 2020-04-06 22:06:05.000000000 +0200 @@ -46,6 +46,35 @@ click.secho('Alert ID: {}'.format(alert['id']), fg='cyan') +(a)alert.command(name='domain') +(a)click.argument('domain', metavar='<domain>', type=str) +(a)click.option('--triggers', help='List of triggers to enable', default='malware,industrial_control_system,internet_scanner,iot,open_database,new_service,ssl_expired,vulnerable') +def alert_domain(domain, triggers): + """Create a network alert based on a domain name""" + key = get_api_key() + + api = shodan.Shodan(key) + try: + # Grab a list of IPs for the domain + domain = domain.lower() + click.secho('Looking up domain information...', dim=True) + info = api.dns.domain_info(domain, type='A') + domain_ips = set([record['value'] for record in info['data']]) + + # Create the actual alert + click.secho('Creating alert...', dim=True) + alert = api.create_alert('__domain: {}'.format(domain), list(domain_ips)) + + # Enable the triggers so it starts getting managed by Shodan Monitor + click.secho('Enabling triggers...', dim=True) + api.enable_alert_trigger(alert['id'], triggers) + except shodan.APIError as e: + raise click.ClickException(e.value) + + click.secho('Successfully created domain alert!', fg='green') + click.secho('Alert ID: {}'.format(alert['id']), fg='cyan') + + @alert.command(name='info') @click.argument('alert', metavar='<alert id>') def alert_info(alert): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/shodan-1.21.3/shodan/cli/converter/csvc.py new/shodan-1.23.0/shodan/cli/converter/csvc.py --- old/shodan-1.21.3/shodan/cli/converter/csvc.py 2019-02-11 03:12:57.000000000 +0100 +++ new/shodan-1.23.0/shodan/cli/converter/csvc.py 2020-03-27 18:11:51.000000000 +0100 @@ -54,7 +54,7 @@ # The "vulns" property can't be nicely flattened as-is so we turn # it into a list before processing the banner. if 'vulns' in banner: - banner['vulns'] = banner['vulns'].keys() + banner['vulns'] = list(banner['vulns'].keys()) # Python3 returns dict_keys so we neeed to cover that to a list try: row = [] diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/shodan-1.21.3/shodan/cli/converter/geojson.py new/shodan-1.23.0/shodan/cli/converter/geojson.py --- old/shodan-1.21.3/shodan/cli/converter/geojson.py 2020-01-20 20:49:17.000000000 +0100 +++ new/shodan-1.23.0/shodan/cli/converter/geojson.py 2020-04-06 22:06:15.000000000 +0200 @@ -48,5 +48,5 @@ }, } self.fout.write(dumps(feature) + ',') - except Exception as e: + except Exception: pass diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/shodan-1.21.3/shodan/cli/worldmap.py new/shodan-1.23.0/shodan/cli/worldmap.py --- old/shodan-1.21.3/shodan/cli/worldmap.py 2019-02-11 01:44:09.000000000 +0100 +++ new/shodan-1.23.0/shodan/cli/worldmap.py 2020-03-17 23:24:35.000000000 +0100 @@ -1,4 +1,3 @@ -#!/usr/bin/env python ''' F-Secure Virus World Map console edition diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/shodan-1.21.3/shodan/stream.py new/shodan-1.23.0/shodan/stream.py --- old/shodan-1.21.3/shodan/stream.py 2019-09-21 19:00:41.000000000 +0200 +++ new/shodan-1.23.0/shodan/stream.py 2020-03-18 00:22:04.000000000 +0100 @@ -134,3 +134,14 @@ stream = self._create_stream('/shodan/tags/%s' % ','.join(tags), timeout=timeout) for line in self._iter_stream(stream, raw): yield line + + def vulns(self, vulns, raw=False, timeout=None): + """ + A filtered version of the "banners" stream to only return banners that match the vulnerabilities of interest. + + :param vulns: A list of vulns to return banner data on. + :type vulns: string[] + """ + stream = self._create_stream('/shodan/vulns/%s' % ','.join(vulns), timeout=timeout) + for line in self._iter_stream(stream, raw): + yield line diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/shodan-1.21.3/shodan.egg-info/PKG-INFO new/shodan-1.23.0/shodan.egg-info/PKG-INFO --- old/shodan-1.21.3/shodan.egg-info/PKG-INFO 2020-01-20 21:22:45.000000000 +0100 +++ new/shodan-1.23.0/shodan.egg-info/PKG-INFO 2020-04-06 22:10:29.000000000 +0200 @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: shodan -Version: 1.21.3 +Version: 1.23.0 Summary: Python library and command-line utility for Shodan (https://developer.shodan.io) Home-page: http://github.com/achillean/shodan-python/tree/master Author: John Matherly

1 0

commit python-thespian for openSUSE:Leap:15.2
by root 01 May '20

01 May '20

Hello community, here is the log from the commit of package python-thespian for openSUSE:Leap:15.2 checked in at 2020-04-30 18:51:26 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Leap:15.2/python-thespian (Old) and /work/SRC/openSUSE:Leap:15.2/.python-thespian.new.2738 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Package is "python-thespian" Thu Apr 30 18:51:26 2020 rev:2 rq:786971 version:3.10.0 Changes: -------- --- /work/SRC/openSUSE:Leap:15.2/python-thespian/python-thespian.changes 2020-02-22 17:51:48.201693935 +0100 +++ /work/SRC/openSUSE:Leap:15.2/.python-thespian.new.2738/python-thespian.changes 2020-04-30 18:51:31.616609205 +0200 @@ -1,0 +2,7 @@ +Tue Mar 17 09:46:19 UTC 2020 - pgajdos(a)suse.com + +- version update to 3.10.0 + * transient_idle wakeups + * Adjustable Troupe limits, THESPIAN_BASE_IPADDR + +------------------------------------------------------------------- Old: ---- thespian-3.9.11.zip New: ---- thespian-3.10.0.zip ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ python-thespian.spec ++++++ --- /var/tmp/diff_new_pack.yfySTR/_old 2020-04-30 18:51:32.020610065 +0200 +++ /var/tmp/diff_new_pack.yfySTR/_new 2020-04-30 18:51:32.024610074 +0200 @@ -18,7 +18,7 @@ %{?!python_module:%define python_module() python-%{**} python3-%{**}} Name: python-thespian -Version: 3.9.11 +Version: 3.10.0 Release: 0 Summary: Python Actor concurrency library License: MIT @@ -69,6 +69,9 @@ %fdupes %{buildroot}%{$python_sitelib} } +%check +# neverending story + %files %{python_files} %license LICENSE.txt %doc README.rst doc/*.org

1 0

commit python-pipreqs for openSUSE:Leap:15.2
by root 01 May '20

01 May '20

Hello community, here is the log from the commit of package python-pipreqs for openSUSE:Leap:15.2 checked in at 2020-04-30 18:51:27 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Leap:15.2/python-pipreqs (Old) and /work/SRC/openSUSE:Leap:15.2/.python-pipreqs.new.2738 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Package is "python-pipreqs" Thu Apr 30 18:51:27 2020 rev:2 rq:789097 version:0.4.10 Changes: -------- --- /work/SRC/openSUSE:Leap:15.2/python-pipreqs/python-pipreqs.changes 2020-02-22 18:48:44.220279492 +0100 +++ /work/SRC/openSUSE:Leap:15.2/.python-pipreqs.new.2738/python-pipreqs.changes 2020-04-30 18:51:32.240610534 +0200 @@ -1,0 +2,6 @@ +Tue Mar 24 10:47:46 UTC 2020 - pgajdos(a)suse.com + +- version update to 0.4.10 + * bugfix: f.close() only required if open succeeded + +------------------------------------------------------------------- Old: ---- pipreqs-0.4.9.tar.gz New: ---- pipreqs-0.4.10.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ python-pipreqs.spec ++++++ --- /var/tmp/diff_new_pack.qWIoug/_old 2020-04-30 18:51:32.592611284 +0200 +++ /var/tmp/diff_new_pack.qWIoug/_new 2020-04-30 18:51:32.596611293 +0200 @@ -1,7 +1,7 @@ # # spec file for package python-pipreqs # -# Copyright (c) 2019 SUSE LINUX GmbH, Nuernberg, Germany. +# Copyright (c) 2020 SUSE LLC # # All modifications and additions to the file contributed by third parties # remain the property of their copyright owners, unless otherwise agreed @@ -18,18 +18,19 @@ %{?!python_module:%define python_module() python-%{**} python3-%{**}} Name: python-pipreqs -Version: 0.4.9 +Version: 0.4.10 Release: 0 Summary: Pip requirements generator based on imports in project License: Apache-2.0 Group: Development/Languages/Python URL: https://github.com/bndr/pipreqs Source: https://files.pythonhosted.org/packages/source/p/pipreqs/pipreqs-%{version}… +BuildRequires: %{python_module pytest} BuildRequires: %{python_module setuptools} -BuildRequires: %{python_module nose} BuildRequires: fdupes BuildRequires: python-rpm-macros Requires: python-docopt +Requires: python-setuptools Requires: python-yarg BuildArch: noarch # SECTION test requirements @@ -53,8 +54,8 @@ %python_expand %fdupes %{buildroot}%{$python_sitelib} %check -# Ignore four tests that require network access -%python_exec -m nose -e 'test_(get_imports_info|ignored_directory|init_)' +# Ignore tests that require network access +%pytest -k 'not (test_get_imports_info or test_ignored_directory or test_init or test_init_overwrite or teset_init_savepath or test_omit_version)' %files %{python_files} %doc AUTHORS.rst README.rst ++++++ pipreqs-0.4.9.tar.gz -> pipreqs-0.4.10.tar.gz ++++++ ++++ 1736 lines of diff (skipped)

1 0

commit libvirt for openSUSE:Leap:15.2
by root 01 May '20

01 May '20

Hello community, here is the log from the commit of package libvirt for openSUSE:Leap:15.2 checked in at 2020-04-30 18:51:23 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Leap:15.2/libvirt (Old) and /work/SRC/openSUSE:Leap:15.2/.libvirt.new.2738 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Package is "libvirt" Thu Apr 30 18:51:23 2020 rev:115 rq:799258 version:6.0.0 Changes: -------- --- /work/SRC/openSUSE:Leap:15.2/libvirt/libvirt.changes 2020-04-20 12:55:28.716727535 +0200 +++ /work/SRC/openSUSE:Leap:15.2/.libvirt.new.2738/libvirt.changes 2020-04-30 18:51:29.684605088 +0200 @@ -1,0 +2,23 @@ +Wed Apr 29 17:03:01 UTC 2020 - James Fehlig <jfehlig(a)suse.com> + +- qemu: Fix memory leak in qemuDomainGetStatsIOThread + CVE-2020-12430 + 9bf9e0ae-CVE-2020-12430.patch + bsc#1170765 + +------------------------------------------------------------------- +Tue Apr 21 17:45:36 UTC 2020 - James Fehlig <jfehlig(a)suse.com> + +- libxl: Add support for 'passthrough' and 'e820_host' settings + b7d6648d-conf-add-e820-host.patch, + 5749395b-libxl-e820-host.patch, + f3ef7daf-xenconfig-e820-host.patch, + 34077c1b-tests-check-e820-host.patch, + fadbaa23-conf-add-passthrough.patch, + 9529e007-libxl-passthrough.patch, + 9cb8bc6f-xenconfig-refactor-features.patch, + b523e225-xenconfig-passthrough.patch, + bed32525-tests-check-passthrough.patch + bsc#1167217 + +------------------------------------------------------------------- New: ---- 34077c1b-tests-check-e820-host.patch 5749395b-libxl-e820-host.patch 9529e007-libxl-passthrough.patch 9bf9e0ae-CVE-2020-12430.patch 9cb8bc6f-xenconfig-refactor-features.patch b523e225-xenconfig-passthrough.patch b7d6648d-conf-add-e820-host.patch bed32525-tests-check-passthrough.patch f3ef7daf-xenconfig-e820-host.patch fadbaa23-conf-add-passthrough.patch ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ libvirt.spec ++++++ --- /var/tmp/diff_new_pack.b9zLHq/_old 2020-04-30 18:51:30.876607628 +0200 +++ /var/tmp/diff_new_pack.b9zLHq/_new 2020-04-30 18:51:30.876607628 +0200 @@ -350,6 +350,16 @@ Patch9: 967f4eeb-xenconfig-event-channels.patch Patch10: 4cc90c2e-CVE-2020-10701.patch Patch11: 93b15ba0-qemu-fix-hang-in-p2p-xbzrle-compression-parallel-mig.patch +Patch12: b7d6648d-conf-add-e820-host.patch +Patch13: 5749395b-libxl-e820-host.patch +Patch14: f3ef7daf-xenconfig-e820-host.patch +Patch15: 34077c1b-tests-check-e820-host.patch +Patch16: fadbaa23-conf-add-passthrough.patch +Patch17: 9529e007-libxl-passthrough.patch +Patch18: 9cb8bc6f-xenconfig-refactor-features.patch +Patch19: b523e225-xenconfig-passthrough.patch +Patch20: bed32525-tests-check-passthrough.patch +Patch21: 9bf9e0ae-CVE-2020-12430.patch # Patches pending upstream review Patch100: libxl-dom-reset.patch Patch101: network-don-t-use-dhcp-authoritative-on-static-netwo.patch @@ -896,6 +906,16 @@ %patch9 -p1 %patch10 -p1 %patch11 -p1 +%patch12 -p1 +%patch13 -p1 +%patch14 -p1 +%patch15 -p1 +%patch16 -p1 +%patch17 -p1 +%patch18 -p1 +%patch19 -p1 +%patch20 -p1 +%patch21 -p1 %patch100 -p1 %patch101 -p1 %patch150 -p1 ++++++ 0001-Extract-stats-functions-from-the-qemu-driver.patch ++++++ --- /var/tmp/diff_new_pack.b9zLHq/_old 2020-04-30 18:51:30.920607722 +0200 +++ /var/tmp/diff_new_pack.b9zLHq/_new 2020-04-30 18:51:30.920607722 +0200 @@ -213,7 +213,7 @@ =================================================================== --- libvirt-6.0.0.orig/src/libvirt_private.syms +++ libvirt-6.0.0/src/libvirt_private.syms -@@ -714,6 +714,9 @@ virDomainConfNWFilterInstantiate; +@@ -716,6 +716,9 @@ virDomainConfNWFilterInstantiate; virDomainConfNWFilterTeardown; virDomainConfVMNWFilterTeardown; @@ -223,7 +223,7 @@ # conf/interface_conf.h virInterfaceDefFormat; -@@ -1691,6 +1694,7 @@ virCgroupGetMemoryUsage; +@@ -1693,6 +1696,7 @@ virCgroupGetMemoryUsage; virCgroupGetMemSwapHardLimit; virCgroupGetMemSwapUsage; virCgroupGetPercpuStats; ++++++ 34077c1b-tests-check-e820-host.patch ++++++ commit 34077c1b8b62bb61e76b908d0959216da0b3185f Author: Marek Marczykowski-Górecki <marmarek(a)invisiblethingslab.com> Date: Tue Apr 14 04:37:07 2020 +0200 tests: check e820_host option handling Signed-off-by: Marek Marczykowski-Górecki <marmarek(a)invisiblethingslab.com> Reviewed-by: Jim Fehlig <jfehlig(a)suse.com> Index: libvirt-6.0.0/tests/xlconfigdata/test-paravirt-e820_host.cfg =================================================================== --- /dev/null +++ libvirt-6.0.0/tests/xlconfigdata/test-paravirt-e820_host.cfg @@ -0,0 +1,13 @@ +name = "XenGuest1" +uuid = "45b60f51-88a9-47a8-a3b3-5e66d71b2283" +maxmem = 512 +memory = 512 +vcpus = 4 +e820_host = 1 +localtime = 0 +on_poweroff = "preserve" +on_reboot = "restart" +on_crash = "preserve" +vif = [ "mac=5a:36:0e:be:00:09" ] +bootloader = "/usr/bin/pygrub" +disk = [ "format=qcow2,vdev=xvda,access=rw,backendtype=qdisk,target=/var/lib/xen/images/debian/disk.qcow2" ] Index: libvirt-6.0.0/tests/xlconfigdata/test-paravirt-e820_host.xml =================================================================== --- /dev/null +++ libvirt-6.0.0/tests/xlconfigdata/test-paravirt-e820_host.xml @@ -0,0 +1,37 @@ +<domain type='xen'> + <name>XenGuest1</name> + <uuid>45b60f51-88a9-47a8-a3b3-5e66d71b2283</uuid> + <memory unit='KiB'>524288</memory> + <currentMemory unit='KiB'>524288</currentMemory> + <vcpu placement='static'>4</vcpu> + <bootloader>/usr/bin/pygrub</bootloader> + <os> + <type arch='x86_64' machine='xenpv'>linux</type> + </os> + <features> + <xen> + <e820_host state='on'/> + </xen> + </features> + <clock offset='utc' adjustment='reset'/> + <on_poweroff>preserve</on_poweroff> + <on_reboot>restart</on_reboot> + <on_crash>preserve</on_crash> + <devices> + <disk type='file' device='disk'> + <driver name='qemu' type='qcow2'/> + <source file='/var/lib/xen/images/debian/disk.qcow2'/> + <target dev='xvda' bus='xen'/> + </disk> + <controller type='xenbus' index='0'/> + <interface type='ethernet'> + <mac address='5a:36:0e:be:00:09'/> + </interface> + <console type='pty'> + <target type='xen' port='0'/> + </console> + <input type='mouse' bus='xen'/> + <input type='keyboard' bus='xen'/> + <memballoon model='xen'/> + </devices> +</domain> Index: libvirt-6.0.0/tests/xlconfigtest.c =================================================================== --- libvirt-6.0.0.orig/tests/xlconfigtest.c +++ libvirt-6.0.0/tests/xlconfigtest.c @@ -300,6 +300,7 @@ mymain(void) DO_TEST("vif-multi-ip"); DO_TEST("usb"); DO_TEST("usbctrl"); + DO_TEST("paravirt-e820_host"); testXLFreeDriver(driver); ++++++ 5749395b-libxl-e820-host.patch ++++++ commit 5749395bfd03fcdb55a8be6a69637a2acaaad2c9 Author: Marek Marczykowski-Górecki <marmarek(a)invisiblethingslab.com> Date: Tue Apr 14 04:37:05 2020 +0200 libxl: make use of e820_host feature Signed-off-by: Marek Marczykowski-Górecki <marmarek(a)invisiblethingslab.com> Reviewed-by: Jim Fehlig <jfehlig(a)suse.com> Index: libvirt-6.0.0/src/libxl/libxl_conf.c =================================================================== --- libvirt-6.0.0.orig/src/libxl/libxl_conf.c +++ libvirt-6.0.0/src/libxl/libxl_conf.c @@ -693,6 +693,20 @@ libxlMakeDomBuildInfo(virDomainDefPtr de b_info->u.pv.kernel = g_strdup(def->os.kernel); } b_info->u.pv.ramdisk = g_strdup(def->os.initrd); + + if (def->features[VIR_DOMAIN_FEATURE_XEN] == VIR_TRISTATE_SWITCH_ON) { + switch ((virTristateSwitch) def->xen_features[VIR_DOMAIN_XEN_E820_HOST]) { + case VIR_TRISTATE_SWITCH_ON: + libxl_defbool_set(&b_info->u.pv.e820_host, true); + break; + case VIR_TRISTATE_SWITCH_OFF: + libxl_defbool_set(&b_info->u.pv.e820_host, false); + break; + case VIR_TRISTATE_SWITCH_ABSENT: + case VIR_TRISTATE_SWITCH_LAST: + break; + } + } } /* only the 'xen' balloon device model is supported */ ++++++ 9529e007-libxl-passthrough.patch ++++++ commit 9529e0077a3f9002d6bc7f5504ca4583e5d985f1 Author: Jim Fehlig <jfehlig(a)suse.com> Date: Wed Apr 15 17:18:25 2020 -0600 libxl: make use of passthrough hypervisor feature Signed-off-by: Jim Fehlig <jfehlig(a)suse.com> Reviewed-by: Michal Privoznik <mprivozn(a)redhat.com> Index: libvirt-6.0.0/src/libxl/libxl_conf.c =================================================================== --- libvirt-6.0.0.orig/src/libxl/libxl_conf.c +++ libvirt-6.0.0/src/libxl/libxl_conf.c @@ -160,6 +160,27 @@ libxlMakeDomCreateInfo(libxl_ctx *ctx, c_info->type = LIBXL_DOMAIN_TYPE_PV; } +#ifdef LIBXL_HAVE_CREATEINFO_PASSTHROUGH + if (def->features[VIR_DOMAIN_FEATURE_XEN] == VIR_TRISTATE_SWITCH_ON) { + switch ((virTristateSwitch) def->xen_features[VIR_DOMAIN_XEN_PASSTHROUGH]) { + case VIR_TRISTATE_SWITCH_ON: + if (def->xen_passthrough_mode == VIR_DOMAIN_XEN_PASSTHROUGH_MODE_SYNC_PT) + c_info->passthrough = LIBXL_PASSTHROUGH_SYNC_PT; + else if (def->xen_passthrough_mode == VIR_DOMAIN_XEN_PASSTHROUGH_MODE_SHARE_PT) + c_info->passthrough = LIBXL_PASSTHROUGH_SHARE_PT; + else + c_info->passthrough = LIBXL_PASSTHROUGH_ENABLED; + break; + case VIR_TRISTATE_SWITCH_OFF: + c_info->passthrough = LIBXL_PASSTHROUGH_DISABLED; + break; + case VIR_TRISTATE_SWITCH_ABSENT: + case VIR_TRISTATE_SWITCH_LAST: + break; + } + } +#endif + c_info->name = g_strdup(def->name); if (def->nseclabels && ++++++ 9bf9e0ae-CVE-2020-12430.patch ++++++ commit 9bf9e0ae6af38c806f4672ca7b12a6b38d5a9581 Author: Peter Krempa <pkrempa(a)redhat.com> Date: Wed Feb 19 08:40:59 2020 +0100 qemuDomainGetStatsIOThread: Don't leak array with 0 iothreads qemuMonitorGetIOThreads returns a NULL-terminated list even when 0 iothreads are present. The caller didn't perform cleanup if there were 0 iothreads leaking the array. https://bugzilla.redhat.com/show_bug.cgi?id=1804548 Fixes: d1eac92784573559b6fd56836e33b215c89308e3 Reported-by: Jing Yan <jiyan(a)redhat.com> Signed-off-by: Peter Krempa <pkrempa(a)redhat.com> Reviewed-by: Ján Tomko <jtomko(a)redhat.com> Index: libvirt-6.0.0/src/qemu/qemu_driver.c =================================================================== --- libvirt-6.0.0.orig/src/qemu/qemu_driver.c +++ libvirt-6.0.0/src/qemu/qemu_driver.c @@ -21511,8 +21511,12 @@ qemuDomainGetStatsIOThread(virQEMUDriver if ((niothreads = qemuDomainGetIOThreadsMon(driver, dom, &iothreads)) < 0) return -1; - if (niothreads == 0) - return 0; + /* qemuDomainGetIOThreadsMon returns a NULL-terminated list, so we must free + * it even if it returns 0 */ + if (niothreads == 0) { + ret = 0; + goto cleanup; + } if (virTypedParamListAddUInt(params, niothreads, "iothread.count") < 0) goto cleanup; ++++++ 9cb8bc6f-xenconfig-refactor-features.patch ++++++ commit 9cb8bc6ff1ad80aba4d3c68756166a22e830d481 Author: Jim Fehlig <jfehlig(a)suse.com> Date: Wed Apr 15 21:48:42 2020 -0600 libxl: refactor cpu and hypervisor feature parser/formatter Signed-off-by: Jim Fehlig <jfehlig(a)suse.com> Reviewed-by: Michal Privoznik <mprivozn(a)redhat.com> Index: libvirt-6.0.0/src/libxl/xen_common.c =================================================================== --- libvirt-6.0.0.orig/src/libxl/xen_common.c +++ libvirt-6.0.0/src/libxl/xen_common.c @@ -493,15 +493,12 @@ xenParsePCIList(virConfPtr conf, virDoma static int -xenParseCPUFeatures(virConfPtr conf, - virDomainDefPtr def, - virDomainXMLOptionPtr xmlopt) +xenParseCPU(virConfPtr conf, + virDomainDefPtr def, + virDomainXMLOptionPtr xmlopt) { unsigned long count = 0; g_autofree char *cpus = NULL; - g_autofree char *tsc_mode = NULL; - int val = 0; - virDomainTimerDefPtr timer; if (xenConfigGetULong(conf, "vcpus", &count, 1) < 0) return -1; @@ -526,6 +523,17 @@ xenParseCPUFeatures(virConfPtr conf, if (cpus && (virBitmapParse(cpus, &def->cpumask, 4096) < 0)) return -1; + return 0; +} + + +static int +xenParseHypervisorFeatures(virConfPtr conf, virDomainDefPtr def) +{ + g_autofree char *tsc_mode = NULL; + virDomainTimerDefPtr timer; + int val = 0; + if (xenConfigGetString(conf, "tsc_mode", &tsc_mode, NULL) < 0) return -1; @@ -552,27 +560,26 @@ xenParseCPUFeatures(virConfPtr conf, if (def->os.type == VIR_DOMAIN_OSTYPE_HVM) { if (xenConfigGetBool(conf, "pae", &val, 1) < 0) return -1; - else if (val) def->features[VIR_DOMAIN_FEATURE_PAE] = VIR_TRISTATE_SWITCH_ON; + if (xenConfigGetBool(conf, "acpi", &val, 1) < 0) return -1; - else if (val) def->features[VIR_DOMAIN_FEATURE_ACPI] = VIR_TRISTATE_SWITCH_ON; + if (xenConfigGetBool(conf, "apic", &val, 1) < 0) return -1; - else if (val) def->features[VIR_DOMAIN_FEATURE_APIC] = VIR_TRISTATE_SWITCH_ON; + if (xenConfigGetBool(conf, "hap", &val, 1) < 0) return -1; - else if (!val) def->features[VIR_DOMAIN_FEATURE_HAP] = VIR_TRISTATE_SWITCH_OFF; + if (xenConfigGetBool(conf, "viridian", &val, 0) < 0) return -1; - else if (val) def->features[VIR_DOMAIN_FEATURE_VIRIDIAN] = VIR_TRISTATE_SWITCH_ON; @@ -1483,7 +1490,10 @@ xenParseConfigCommon(virConfPtr conf, if (xenParseEventsActions(conf, def) < 0) return -1; - if (xenParseCPUFeatures(conf, def, xmlopt) < 0) + if (xenParseCPU(conf, def, xmlopt) < 0) + return -1; + + if (xenParseHypervisorFeatures(conf, def) < 0) return -1; if (xenParseTimeOffset(conf, def) < 0) @@ -2115,7 +2125,7 @@ xenFormatCPUAllocation(virConfPtr conf, static int -xenFormatCPUFeatures(virConfPtr conf, virDomainDefPtr def) +xenFormatHypervisorFeatures(virConfPtr conf, virDomainDefPtr def) { size_t i; bool hvm = !!(def->os.type == VIR_DOMAIN_OSTYPE_HVM); @@ -2422,7 +2432,7 @@ xenFormatConfigCommon(virConfPtr conf, if (xenFormatCPUAllocation(conf, def) < 0) return -1; - if (xenFormatCPUFeatures(conf, def) < 0) + if (xenFormatHypervisorFeatures(conf, def) < 0) return -1; if (xenFormatTimeOffset(conf, def) < 0) ++++++ b523e225-xenconfig-passthrough.patch ++++++ commit b523e22521afe733165869c9e1ae18e88536acd6 Author: Jim Fehlig <jfehlig(a)suse.com> Date: Thu Apr 16 08:31:48 2020 -0600 xenconfig: Add support for 'passthrough' hypervisor feature Add support for xl.cfg(5) 'passthrough' option in the domXML-to-xenconfig configuration converter. Signed-off-by: Jim Fehlig <jfehlig(a)suse.com> Reviewed-by: Michal Privoznik <mprivozn(a)redhat.com> Index: libvirt-6.0.0/src/libvirt_private.syms =================================================================== --- libvirt-6.0.0.orig/src/libvirt_private.syms +++ libvirt-6.0.0/src/libvirt_private.syms @@ -642,6 +642,8 @@ virDomainWatchdogActionTypeToString; virDomainWatchdogDefFree; virDomainWatchdogModelTypeFromString; virDomainWatchdogModelTypeToString; +virDomainXenPassthroughModeTypeFromString; +virDomainXenPassthroughModeTypeToString; virDomainXMLOptionGetNamespace; virDomainXMLOptionGetSaveCookie; virDomainXMLOptionNew; Index: libvirt-6.0.0/src/libxl/xen_common.c =================================================================== --- libvirt-6.0.0.orig/src/libxl/xen_common.c +++ libvirt-6.0.0/src/libxl/xen_common.c @@ -530,14 +530,14 @@ xenParseCPU(virConfPtr conf, static int xenParseHypervisorFeatures(virConfPtr conf, virDomainDefPtr def) { - g_autofree char *tsc_mode = NULL; + g_autofree char *strval = NULL; virDomainTimerDefPtr timer; int val = 0; - if (xenConfigGetString(conf, "tsc_mode", &tsc_mode, NULL) < 0) + if (xenConfigGetString(conf, "tsc_mode", &strval, NULL) < 0) return -1; - if (tsc_mode) { + if (strval) { if (VIR_EXPAND_N(def->clock.timers, def->clock.ntimers, 1) < 0 || VIR_ALLOC(timer) < 0) return -1; @@ -547,16 +547,40 @@ xenParseHypervisorFeatures(virConfPtr co timer->tickpolicy = -1; timer->mode = VIR_DOMAIN_TIMER_MODE_AUTO; timer->track = -1; - if (STREQ_NULLABLE(tsc_mode, "always_emulate")) + if (STREQ_NULLABLE(strval, "always_emulate")) timer->mode = VIR_DOMAIN_TIMER_MODE_EMULATE; - else if (STREQ_NULLABLE(tsc_mode, "native")) + else if (STREQ_NULLABLE(strval, "native")) timer->mode = VIR_DOMAIN_TIMER_MODE_NATIVE; - else if (STREQ_NULLABLE(tsc_mode, "native_paravirt")) + else if (STREQ_NULLABLE(strval, "native_paravirt")) timer->mode = VIR_DOMAIN_TIMER_MODE_PARAVIRT; def->clock.timers[def->clock.ntimers - 1] = timer; } + if (xenConfigGetString(conf, "passthrough", &strval, NULL) < 0) + return -1; + + if (strval) { + if (STREQ(strval, "disabled")) { + def->features[VIR_DOMAIN_FEATURE_XEN] = VIR_TRISTATE_SWITCH_OFF; + def->xen_features[VIR_DOMAIN_XEN_PASSTHROUGH] = VIR_TRISTATE_SWITCH_OFF; + } else if (STREQ(strval, "enabled")) { + def->features[VIR_DOMAIN_FEATURE_XEN] = VIR_TRISTATE_SWITCH_ON; + def->xen_features[VIR_DOMAIN_XEN_PASSTHROUGH] = VIR_TRISTATE_SWITCH_ON; + } else if (STREQ(strval, "sync_pt")) { + def->features[VIR_DOMAIN_FEATURE_XEN] = VIR_TRISTATE_SWITCH_ON; + def->xen_features[VIR_DOMAIN_XEN_PASSTHROUGH] = VIR_TRISTATE_SWITCH_ON; + def->xen_passthrough_mode = VIR_DOMAIN_XEN_PASSTHROUGH_MODE_SYNC_PT; + } else if (STREQ(strval, "share_pt")) { + def->features[VIR_DOMAIN_FEATURE_XEN] = VIR_TRISTATE_SWITCH_ON; + def->xen_features[VIR_DOMAIN_XEN_PASSTHROUGH] = VIR_TRISTATE_SWITCH_ON; + def->xen_passthrough_mode = VIR_DOMAIN_XEN_PASSTHROUGH_MODE_SHARE_PT; + } else { + virReportError(VIR_ERR_CONF_SYNTAX, + _("Invalid passthrough mode %s"), strval); + } + } + if (def->os.type == VIR_DOMAIN_OSTYPE_HVM) { if (xenConfigGetBool(conf, "pae", &val, 1) < 0) return -1; @@ -2163,6 +2187,20 @@ xenFormatHypervisorFeatures(virConfPtr c } } + if (def->features[VIR_DOMAIN_FEATURE_XEN] == VIR_TRISTATE_SWITCH_ON) { + if (def->xen_features[VIR_DOMAIN_XEN_PASSTHROUGH] == VIR_TRISTATE_SWITCH_ON) { + if (def->xen_passthrough_mode == VIR_DOMAIN_XEN_PASSTHROUGH_MODE_SYNC_PT || + def->xen_passthrough_mode == VIR_DOMAIN_XEN_PASSTHROUGH_MODE_SHARE_PT) { + if (xenConfigSetString(conf, "passthrough", + virDomainXenPassthroughModeTypeToString(def->xen_passthrough_mode)) < 0) + return -1; + } else { + if (xenConfigSetString(conf, "passthrough", "enabled") < 0) + return -1; + } + } + } + for (i = 0; i < def->clock.ntimers; i++) { switch ((virDomainTimerNameType)def->clock.timers[i]->name) { case VIR_DOMAIN_TIMER_NAME_TSC: ++++++ b7d6648d-conf-add-e820-host.patch ++++++ commit b7d6648d436fe0a99d4faf0f99c88a27a7bfea33 Author: Marek Marczykowski-Górecki <marmarek(a)invisiblethingslab.com> Date: Tue Apr 14 04:37:04 2020 +0200 conf: add xen specific feature: e820_host e820_host is a Xen-specific option, only available for PV domains, that provides the domain a virtual e820 memory map based on the host one. It is enabled with a new Xen hypervisor feature, e.g. <features> <xen> <e820_host state='on'/> </xen> </features> e820_host is required when using PCI passthrough and is generally considered safe for any PV kernel. e820_host is silently ignored if set in HVM domain configuration. See xl.cfg(5) man page in the Xen documentation for more details. Signed-off-by: Marek Marczykowski-Górecki <marmarek(a)invisiblethingslab.com> Reviewed-by: Jim Fehlig <jfehlig(a)suse.com> Index: libvirt-6.0.0/docs/formatdomain.html.in =================================================================== --- libvirt-6.0.0.orig/docs/formatdomain.html.in +++ libvirt-6.0.0/docs/formatdomain.html.in @@ -2053,6 +2053,9 @@ <hidden state='on'/> <hint-dedicated state='on'/> </kvm> + <xen> + <e820_host state='on'/> + </xen> <pvspinlock state='on'/> <gic version='2'/> <ioapic driver='qemu'/> @@ -2234,6 +2237,23 @@ </tr> </table> </dd> + <dt><code>xen</code></dt> + <dd>Various features to change the behavior of the Xen hypervisor. + <table class="top_table"> + <tr> + <th>Feature</th> + <th>Description</th> + <th>Value</th> + <th>Since</th> + </tr> + <tr> + <td>e820_host</td> + <td>Expose the host e820 to the guest (PV only)</td> + <td>on, off</td> + <td><span class="since">6.3.0</span></td> + </tr> + </table> + </dd> <dt><code>pmu</code></dt> <dd>Depending on the <code>state</code> attribute (values <code>on</code>, <code>off</code>, default <code>on</code>) enable or disable the Index: libvirt-6.0.0/docs/schemas/domaincommon.rng =================================================================== --- libvirt-6.0.0.orig/docs/schemas/domaincommon.rng +++ libvirt-6.0.0/docs/schemas/domaincommon.rng @@ -5106,6 +5106,9 @@ <ref name="kvm"/> </optional> <optional> + <ref name="xen"/> + </optional> + <optional> <element name="privnet"> <empty/> </element> @@ -6074,6 +6077,19 @@ <ref name="featurestate"/> </element> </optional> + </interleave> + </element> + </define> + +  + <define name="xen"> + <element name="xen"> + <interleave> + <optional> + <element name="e820_host"> + <ref name="featurestate"/> + </element> + </optional> </interleave> </element> </define> Index: libvirt-6.0.0/src/conf/domain_conf.c =================================================================== --- libvirt-6.0.0.orig/src/conf/domain_conf.c +++ libvirt-6.0.0/src/conf/domain_conf.c @@ -172,6 +172,7 @@ VIR_ENUM_IMPL(virDomainFeature, "nested-hv", "msrs", "ccf-assist", + "xen", ); VIR_ENUM_IMPL(virDomainCapabilitiesPolicy, @@ -205,6 +206,11 @@ VIR_ENUM_IMPL(virDomainKVM, "hint-dedicated", ); +VIR_ENUM_IMPL(virDomainXen, + VIR_DOMAIN_XEN_LAST, + "e820_host" +); + VIR_ENUM_IMPL(virDomainMsrsUnknown, VIR_DOMAIN_MSRS_UNKNOWN_LAST, "ignore", @@ -20349,6 +20355,7 @@ virDomainDefParseXML(xmlDocPtr xml, case VIR_DOMAIN_FEATURE_HYPERV: case VIR_DOMAIN_FEATURE_KVM: case VIR_DOMAIN_FEATURE_MSRS: + case VIR_DOMAIN_FEATURE_XEN: def->features[val] = VIR_TRISTATE_SWITCH_ON; break; @@ -20659,6 +20666,51 @@ virDomainDefParseXML(xmlDocPtr xml, VIR_FREE(nodes); } + if (def->features[VIR_DOMAIN_FEATURE_XEN] == VIR_TRISTATE_SWITCH_ON) { + int feature; + int value; + if ((n = virXPathNodeSet("./features/xen/*", ctxt, &nodes)) < 0) + goto error; + + for (i = 0; i < n; i++) { + feature = virDomainXenTypeFromString((const char *)nodes[i]->name); + if (feature < 0) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, + _("unsupported Xen feature: %s"), + nodes[i]->name); + goto error; + } + + switch ((virDomainXen) feature) { + case VIR_DOMAIN_XEN_E820_HOST: + if (!(tmp = virXMLPropString(nodes[i], "state"))) { + virReportError(VIR_ERR_XML_ERROR, + _("missing 'state' attribute for " + "Xen feature '%s'"), + nodes[i]->name); + goto error; + } + + if ((value = virTristateSwitchTypeFromString(tmp)) < 0) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, + _("invalid value of state argument " + "for Xen feature '%s'"), + nodes[i]->name); + goto error; + } + + VIR_FREE(tmp); + def->xen_features[feature] = value; + break; + + /* coverity[dead_error_begin] */ + case VIR_DOMAIN_XEN_LAST: + break; + } + } + VIR_FREE(nodes); + } + if (def->features[VIR_DOMAIN_FEATURE_SMM] == VIR_TRISTATE_SWITCH_ON) { int rv = virDomainParseScaledValue("string(./features/smm/tseg)", "string(./features/smm/tseg/@unit)", @@ -22651,6 +22703,7 @@ virDomainDefFeaturesCheckABIStability(vi case VIR_DOMAIN_FEATURE_PRIVNET: case VIR_DOMAIN_FEATURE_HYPERV: case VIR_DOMAIN_FEATURE_KVM: + case VIR_DOMAIN_FEATURE_XEN: case VIR_DOMAIN_FEATURE_PVSPINLOCK: case VIR_DOMAIN_FEATURE_PMU: case VIR_DOMAIN_FEATURE_VMPORT: @@ -22822,6 +22875,30 @@ virDomainDefFeaturesCheckABIStability(vi } } + /* xen */ + if (src->features[VIR_DOMAIN_FEATURE_XEN] == VIR_TRISTATE_SWITCH_ON) { + for (i = 0; i < VIR_DOMAIN_XEN_LAST; i++) { + switch ((virDomainXen) i) { + case VIR_DOMAIN_XEN_E820_HOST: + if (src->xen_features[i] != dst->xen_features[i]) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, + _("State of Xen feature '%s' differs: " + "source: '%s', destination: '%s'"), + virDomainXenTypeToString(i), + virTristateSwitchTypeToString(src->xen_features[i]), + virTristateSwitchTypeToString(dst->xen_features[i])); + return false; + } + + break; + + /* coverity[dead_error_begin] */ + case VIR_DOMAIN_XEN_LAST: + break; + } + } + } + /* kvm */ if (src->features[VIR_DOMAIN_FEATURE_KVM] == VIR_TRISTATE_SWITCH_ON) { for (i = 0; i < VIR_DOMAIN_KVM_LAST; i++) { @@ -28304,6 +28381,31 @@ virDomainDefFormatFeatures(virBufferPtr virBufferAddLit(&childBuf, "</kvm>\n"); break; + case VIR_DOMAIN_FEATURE_XEN: + if (def->features[i] != VIR_TRISTATE_SWITCH_ON) + break; + + virBufferAddLit(&childBuf, "<xen>\n"); + virBufferAdjustIndent(&childBuf, 2); + for (j = 0; j < VIR_DOMAIN_XEN_LAST; j++) { + switch ((virDomainXen) j) { + case VIR_DOMAIN_XEN_E820_HOST: + if (def->xen_features[j]) + virBufferAsprintf(&childBuf, "<%s state='%s'/>\n", + virDomainXenTypeToString(j), + virTristateSwitchTypeToString( + def->xen_features[j])); + break; + + /* coverity[dead_error_begin] */ + case VIR_DOMAIN_XEN_LAST: + break; + } + } + virBufferAdjustIndent(&childBuf, -2); + virBufferAddLit(&childBuf, "</xen>\n"); + break; + case VIR_DOMAIN_FEATURE_CAPABILITIES: for (j = 0; j < VIR_DOMAIN_PROCES_CAPS_FEATURE_LAST; j++) { if (def->caps_features[j] != VIR_TRISTATE_SWITCH_ABSENT) Index: libvirt-6.0.0/src/conf/domain_conf.h =================================================================== --- libvirt-6.0.0.orig/src/conf/domain_conf.h +++ libvirt-6.0.0/src/conf/domain_conf.h @@ -1765,6 +1765,7 @@ typedef enum { VIR_DOMAIN_FEATURE_NESTED_HV, VIR_DOMAIN_FEATURE_MSRS, VIR_DOMAIN_FEATURE_CCF_ASSIST, + VIR_DOMAIN_FEATURE_XEN, VIR_DOMAIN_FEATURE_LAST } virDomainFeature; @@ -1811,6 +1812,12 @@ typedef enum { } virDomainMsrsUnknown; typedef enum { + VIR_DOMAIN_XEN_E820_HOST = 0, + + VIR_DOMAIN_XEN_LAST +} virDomainXen; + +typedef enum { VIR_DOMAIN_CAPABILITIES_POLICY_DEFAULT = 0, VIR_DOMAIN_CAPABILITIES_POLICY_ALLOW, VIR_DOMAIN_CAPABILITIES_POLICY_DENY, @@ -2429,6 +2436,7 @@ struct _virDomainDef { int hyperv_features[VIR_DOMAIN_HYPERV_LAST]; int kvm_features[VIR_DOMAIN_KVM_LAST]; int msrs_features[VIR_DOMAIN_MSRS_LAST]; + int xen_features[VIR_DOMAIN_XEN_LAST]; unsigned int hyperv_spinlocks; int hyperv_stimer_direct; virGICVersion gic_version; @@ -3467,6 +3475,7 @@ VIR_ENUM_DECL(virDomainGraphicsSpiceMous VIR_ENUM_DECL(virDomainGraphicsVNCSharePolicy); VIR_ENUM_DECL(virDomainHyperv); VIR_ENUM_DECL(virDomainKVM); +VIR_ENUM_DECL(virDomainXen); VIR_ENUM_DECL(virDomainMsrsUnknown); VIR_ENUM_DECL(virDomainRNGModel); VIR_ENUM_DECL(virDomainRNGBackend); ++++++ bed32525-tests-check-passthrough.patch ++++++ commit bed325253681fa61537b01be821538479fa9df60 Author: Jim Fehlig <jfehlig(a)suse.com> Date: Fri Apr 17 14:19:16 2020 -0600 tests: check conversion of passthrough hypervisor feature Add a new test to check the 'mode' attribute of the passthrough element and augment an existing, related test to check enablement of the passthrough element only. Signed-off-by: Jim Fehlig <jfehlig(a)suse.com> Reviewed-by: Michal Privoznik <mprivozn(a)redhat.com> Index: libvirt-6.0.0/tests/xlconfigdata/test-fullvirt-hypervisor-features.cfg =================================================================== --- /dev/null +++ libvirt-6.0.0/tests/xlconfigdata/test-fullvirt-hypervisor-features.cfg @@ -0,0 +1,26 @@ +name = "XenGuest2" +uuid = "c7a5fdb2-cdaf-9455-926a-d65c16db1809" +maxmem = 579 +memory = 394 +vcpus = 1 +pae = 1 +acpi = 1 +apic = 1 +viridian = 0 +passthrough = "share_pt" +rtc_timeoffset = 0 +localtime = 0 +on_poweroff = "destroy" +on_reboot = "restart" +on_crash = "restart" +device_model = "/usr/lib/xen/bin/qemu-system-i386" +sdl = 0 +vnc = 1 +vncunused = 1 +vnclisten = "127.0.0.1" +vif = [ "mac=00:16:3e:66:92:9c,bridge=xenbr1,script=vif-bridge,model=e1000" ] +parallel = "none" +serial = "none" +builder = "hvm" +boot = "c" +disk = [ "format=raw,vdev=hda,access=rw,backendtype=phy,target=/dev/HostVG/XenGuest2" ] Index: libvirt-6.0.0/tests/xlconfigdata/test-fullvirt-hypervisor-features.xml =================================================================== --- /dev/null +++ libvirt-6.0.0/tests/xlconfigdata/test-fullvirt-hypervisor-features.xml @@ -0,0 +1,50 @@ +<domain type='xen'> + <name>XenGuest2</name> + <uuid>c7a5fdb2-cdaf-9455-926a-d65c16db1809</uuid> + <memory unit='KiB'>592896</memory> + <currentMemory unit='KiB'>403456</currentMemory> + <vcpu placement='static'>1</vcpu> + <os> + <type arch='x86_64' machine='xenfv'>hvm</type> + <loader type='rom'>/usr/lib/xen/boot/hvmloader</loader> + <boot dev='hd'/> + </os> + <features> + <acpi/> + <apic/> + <pae/> + <xen> + <passthrough state='on' mode='share_pt'/> + </xen> + </features> + <clock offset='variable' adjustment='0' basis='utc'/> + <on_poweroff>destroy</on_poweroff> + <on_reboot>restart</on_reboot> + <on_crash>restart</on_crash> + <devices> + <emulator>/usr/lib/xen/bin/qemu-system-i386</emulator> + <disk type='block' device='disk'> + <driver name='phy' type='raw'/> + <source dev='/dev/HostVG/XenGuest2'/> + <target dev='hda' bus='ide'/> + <address type='drive' controller='0' bus='0' target='0' unit='0'/> + </disk> + <controller type='xenbus' index='0'/> + <controller type='ide' index='0'/> + <interface type='bridge'> + <mac address='00:16:3e:66:92:9c'/> + <source bridge='xenbr1'/> + <script path='vif-bridge'/> + <model type='e1000'/> + </interface> + <input type='mouse' bus='ps2'/> + <input type='keyboard' bus='ps2'/> + <graphics type='vnc' port='-1' autoport='yes' listen='127.0.0.1'> + <listen type='address' address='127.0.0.1'/> + </graphics> + <video> + <model type='cirrus' vram='8192' heads='1' primary='yes'/> + </video> + <memballoon model='xen'/> + </devices> +</domain> Index: libvirt-6.0.0/tests/xlconfigdata/test-paravirt-e820_host.cfg =================================================================== --- libvirt-6.0.0.orig/tests/xlconfigdata/test-paravirt-e820_host.cfg +++ libvirt-6.0.0/tests/xlconfigdata/test-paravirt-e820_host.cfg @@ -4,6 +4,7 @@ maxmem = 512 memory = 512 vcpus = 4 e820_host = 1 +passthrough = "enabled" localtime = 0 on_poweroff = "preserve" on_reboot = "restart" Index: libvirt-6.0.0/tests/xlconfigdata/test-paravirt-e820_host.xml =================================================================== --- libvirt-6.0.0.orig/tests/xlconfigdata/test-paravirt-e820_host.xml +++ libvirt-6.0.0/tests/xlconfigdata/test-paravirt-e820_host.xml @@ -11,6 +11,7 @@ <features> <xen> <e820_host state='on'/> + <passthrough state='on'/> </xen> </features> <clock offset='utc' adjustment='reset'/> Index: libvirt-6.0.0/tests/xlconfigtest.c =================================================================== --- libvirt-6.0.0.orig/tests/xlconfigtest.c +++ libvirt-6.0.0/tests/xlconfigtest.c @@ -301,6 +301,9 @@ mymain(void) DO_TEST("usb"); DO_TEST("usbctrl"); DO_TEST("paravirt-e820_host"); +#ifdef LIBXL_HAVE_CREATEINFO_PASSTHROUGH + DO_TEST("fullvirt-hypervisor-features"); +#endif testXLFreeDriver(driver); ++++++ f3ef7daf-xenconfig-e820-host.patch ++++++ commit f3ef7dafc8a60e6e21ac31cbbc019aa76a34e523 Author: Marek Marczykowski-Górecki <marmarek(a)invisiblethingslab.com> Date: Tue Apr 14 04:37:06 2020 +0200 xenconfig: parse e820_host option Signed-off-by: Marek Marczykowski-Górecki <marmarek(a)invisiblethingslab.com> Reviewed-by: Jim Fehlig <jfehlig(a)suse.com> Index: libvirt-6.0.0/src/libxl/xen_common.c =================================================================== --- libvirt-6.0.0.orig/src/libxl/xen_common.c +++ libvirt-6.0.0/src/libxl/xen_common.c @@ -592,6 +592,13 @@ xenParseCPUFeatures(virConfPtr conf, def->clock.timers[def->clock.ntimers - 1] = timer; } + } else { + if (xenConfigGetBool(conf, "e820_host", &val, 0) < 0) { + return -1; + } else if (val) { + def->features[VIR_DOMAIN_FEATURE_XEN] = VIR_TRISTATE_SWITCH_ON; + def->xen_features[VIR_DOMAIN_XEN_E820_HOST] = VIR_TRISTATE_SWITCH_ON; + } } return 0; @@ -2138,6 +2145,12 @@ xenFormatCPUFeatures(virConfPtr conf, vi (def->features[VIR_DOMAIN_FEATURE_VIRIDIAN] == VIR_TRISTATE_SWITCH_ON) ? 1 : 0) < 0) return -1; + } else { + if (def->features[VIR_DOMAIN_FEATURE_XEN] == VIR_TRISTATE_SWITCH_ON) { + if (def->xen_features[VIR_DOMAIN_XEN_E820_HOST] == VIR_TRISTATE_SWITCH_ON) + if (xenConfigSetInt(conf, "e820_host", 1) < 0) + return -1; + } } for (i = 0; i < def->clock.ntimers; i++) { ++++++ fadbaa23-conf-add-passthrough.patch ++++++ commit fadbaa23757ff9dca329bdb8d3447c27599f6884 Author: Jim Fehlig <jfehlig(a)suse.com> Date: Wed Apr 15 16:34:54 2020 -0600 conf: add xen hypervisor feature 'passthrough' 'passthrough' is Xen-Specific guest configuration option new to Xen 4.13 that enables IOMMU mappings for a guest and hence whether it supports PCI passthrough. The default is disabled. See the xl.cfg(5) man page and xen.git commit babde47a3fe for more details. The default state of disabled prevents hotlugging PCI devices. However, if the guest configuration contains a PCI passthrough device at time of creation, libxl will automatically enable 'passthrough' and subsequent hotplugging of PCI devices will also be possible. It is not possible to unconditionally enable 'passthrough' since it would introduce a migration incompatibility due to guest ABI change. Instead, introduce another Xen hypervisor feature that can be used to enable guest PCI passthrough <features> <xen> <passthrough state='on'/> </xen> </features> To allow finer control over how IOMMU maps to guest P2M table, the passthrough element also supports a 'mode' attribute with values restricted to snyc_pt and share_pt, similar to xl.cfg(5) 'passthrough' setting . Signed-off-by: Jim Fehlig <jfehlig(a)suse.com> Reviewed-by: Michal Privoznik <mprivozn(a)redhat.com> Index: libvirt-6.0.0/docs/formatdomain.html.in =================================================================== --- libvirt-6.0.0.orig/docs/formatdomain.html.in +++ libvirt-6.0.0/docs/formatdomain.html.in @@ -2055,6 +2055,7 @@ </kvm> <xen> <e820_host state='on'/> + <passthrough state='on' mode='share_pt'/> </xen> <pvspinlock state='on'/> <gic version='2'/> @@ -2252,6 +2253,12 @@ <td>on, off</td> <td><span class="since">6.3.0</span></td> </tr> + <tr> + <td>passthrough</td> + <td>Enable IOMMU mappings allowing PCI passthrough)</td> + <td>on, off; mode - optional string sync_pt or share_pt</td> + <td><span class="since">6.3.0</span></td> + </tr> </table> </dd> <dt><code>pmu</code></dt> Index: libvirt-6.0.0/docs/schemas/domaincommon.rng =================================================================== --- libvirt-6.0.0.orig/docs/schemas/domaincommon.rng +++ libvirt-6.0.0/docs/schemas/domaincommon.rng @@ -6090,6 +6090,18 @@ <ref name="featurestate"/> </element> </optional> + <optional> + <element name="passthrough"> + <ref name="featurestate"/> + <optional> + <attribute name="mode"> + <data type="string"> + <param name='pattern'>(sync_pt|share_pt)</param> + </data> + </attribute> + </optional> + </element> + </optional> </interleave> </element> </define> Index: libvirt-6.0.0/src/conf/domain_conf.c =================================================================== --- libvirt-6.0.0.orig/src/conf/domain_conf.c +++ libvirt-6.0.0/src/conf/domain_conf.c @@ -208,7 +208,15 @@ VIR_ENUM_IMPL(virDomainKVM, VIR_ENUM_IMPL(virDomainXen, VIR_DOMAIN_XEN_LAST, - "e820_host" + "e820_host", + "passthrough", +); + +VIR_ENUM_IMPL(virDomainXenPassthroughMode, + VIR_DOMAIN_XEN_PASSTHROUGH_MODE_LAST, + "default", + "sync_pt", + "share_pt", ); VIR_ENUM_IMPL(virDomainMsrsUnknown, @@ -20669,6 +20677,8 @@ virDomainDefParseXML(xmlDocPtr xml, if (def->features[VIR_DOMAIN_FEATURE_XEN] == VIR_TRISTATE_SWITCH_ON) { int feature; int value; + g_autofree char *ptval = NULL; + if ((n = virXPathNodeSet("./features/xen/*", ctxt, &nodes)) < 0) goto error; @@ -20681,27 +20691,53 @@ virDomainDefParseXML(xmlDocPtr xml, goto error; } + if (!(tmp = virXMLPropString(nodes[i], "state"))) { + virReportError(VIR_ERR_XML_ERROR, + _("missing 'state' attribute for " + "Xen feature '%s'"), + nodes[i]->name); + goto error; + } + + if ((value = virTristateSwitchTypeFromString(tmp)) < 0) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, + _("invalid value of state argument " + "for Xen feature '%s'"), + nodes[i]->name); + goto error; + } + + VIR_FREE(tmp); + def->xen_features[feature] = value; + switch ((virDomainXen) feature) { case VIR_DOMAIN_XEN_E820_HOST: - if (!(tmp = virXMLPropString(nodes[i], "state"))) { - virReportError(VIR_ERR_XML_ERROR, - _("missing 'state' attribute for " - "Xen feature '%s'"), - nodes[i]->name); - goto error; - } + break; - if ((value = virTristateSwitchTypeFromString(tmp)) < 0) { + case VIR_DOMAIN_XEN_PASSTHROUGH: + if (value != VIR_TRISTATE_SWITCH_ON) + break; + + if ((ptval = virXMLPropString(nodes[i], "mode"))) { + int mode = virDomainXenPassthroughModeTypeFromString(ptval); + + if (mode < 0) { virReportError(VIR_ERR_CONFIG_UNSUPPORTED, - _("invalid value of state argument " - "for Xen feature '%s'"), - nodes[i]->name); + _("unsupported mode '%s' for Xen passthrough feature"), + ptval); goto error; } - VIR_FREE(tmp); - def->xen_features[feature] = value; - break; + if (mode != VIR_DOMAIN_XEN_PASSTHROUGH_MODE_SYNC_PT && + mode != VIR_DOMAIN_XEN_PASSTHROUGH_MODE_SHARE_PT) { + virReportError(VIR_ERR_XML_ERROR, "%s", + _("'mode' attribute for Xen feature " + "'passthrough' must be 'sync_pt' or 'share_pt'")); + goto error; + } + def->xen_passthrough_mode = mode; + } + break; /* coverity[dead_error_begin] */ case VIR_DOMAIN_XEN_LAST: @@ -22878,18 +22914,28 @@ virDomainDefFeaturesCheckABIStability(vi /* xen */ if (src->features[VIR_DOMAIN_FEATURE_XEN] == VIR_TRISTATE_SWITCH_ON) { for (i = 0; i < VIR_DOMAIN_XEN_LAST; i++) { + if (src->xen_features[i] != dst->xen_features[i]) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, + _("State of Xen feature '%s' differs: " + "source: '%s', destination: '%s'"), + virDomainXenTypeToString(i), + virTristateSwitchTypeToString(src->xen_features[i]), + virTristateSwitchTypeToString(dst->xen_features[i])); + return false; + } switch ((virDomainXen) i) { case VIR_DOMAIN_XEN_E820_HOST: - if (src->xen_features[i] != dst->xen_features[i]) { + break; + + case VIR_DOMAIN_XEN_PASSTHROUGH: + if (src->xen_passthrough_mode != dst->xen_passthrough_mode) { virReportError(VIR_ERR_CONFIG_UNSUPPORTED, - _("State of Xen feature '%s' differs: " + _("'mode' of Xen passthrough feature differs: " "source: '%s', destination: '%s'"), - virDomainXenTypeToString(i), - virTristateSwitchTypeToString(src->xen_features[i]), - virTristateSwitchTypeToString(dst->xen_features[i])); + virDomainXenPassthroughModeTypeToString(src->xen_passthrough_mode), + virDomainXenPassthroughModeTypeToString(dst->xen_passthrough_mode)); return false; } - break; /* coverity[dead_error_begin] */ @@ -28388,13 +28434,30 @@ virDomainDefFormatFeatures(virBufferPtr virBufferAddLit(&childBuf, "<xen>\n"); virBufferAdjustIndent(&childBuf, 2); for (j = 0; j < VIR_DOMAIN_XEN_LAST; j++) { + if (def->xen_features[j] == VIR_TRISTATE_SWITCH_ABSENT) + continue; + + virBufferAsprintf(&childBuf, "<%s state='%s'", + virDomainXenTypeToString(j), + virTristateSwitchTypeToString( + def->xen_features[j])); + switch ((virDomainXen) j) { case VIR_DOMAIN_XEN_E820_HOST: - if (def->xen_features[j]) - virBufferAsprintf(&childBuf, "<%s state='%s'/>\n", - virDomainXenTypeToString(j), - virTristateSwitchTypeToString( - def->xen_features[j])); + virBufferAddLit(&childBuf, "/>\n"); + break; + case VIR_DOMAIN_XEN_PASSTHROUGH: + if (def->xen_features[j] != VIR_TRISTATE_SWITCH_ON) { + virBufferAddLit(&childBuf, "/>\n"); + break; + } + if (def->xen_passthrough_mode == VIR_DOMAIN_XEN_PASSTHROUGH_MODE_SYNC_PT || + def->xen_passthrough_mode == VIR_DOMAIN_XEN_PASSTHROUGH_MODE_SHARE_PT) { + virBufferEscapeString(&childBuf, " mode='%s'/>\n", + virDomainXenPassthroughModeTypeToString(def->xen_passthrough_mode)); + } else { + virBufferAddLit(&childBuf, "/>\n"); + } break; /* coverity[dead_error_begin] */ Index: libvirt-6.0.0/src/conf/domain_conf.h =================================================================== --- libvirt-6.0.0.orig/src/conf/domain_conf.h +++ libvirt-6.0.0/src/conf/domain_conf.h @@ -1813,11 +1813,20 @@ typedef enum { typedef enum { VIR_DOMAIN_XEN_E820_HOST = 0, + VIR_DOMAIN_XEN_PASSTHROUGH, VIR_DOMAIN_XEN_LAST } virDomainXen; typedef enum { + VIR_DOMAIN_XEN_PASSTHROUGH_MODE_DEFAULT = 0, + VIR_DOMAIN_XEN_PASSTHROUGH_MODE_SYNC_PT, + VIR_DOMAIN_XEN_PASSTHROUGH_MODE_SHARE_PT, + + VIR_DOMAIN_XEN_PASSTHROUGH_MODE_LAST +} virDomainXenPassthroughMode; + +typedef enum { VIR_DOMAIN_CAPABILITIES_POLICY_DEFAULT = 0, VIR_DOMAIN_CAPABILITIES_POLICY_ALLOW, VIR_DOMAIN_CAPABILITIES_POLICY_DENY, @@ -2437,6 +2446,7 @@ struct _virDomainDef { int kvm_features[VIR_DOMAIN_KVM_LAST]; int msrs_features[VIR_DOMAIN_MSRS_LAST]; int xen_features[VIR_DOMAIN_XEN_LAST]; + int xen_passthrough_mode; unsigned int hyperv_spinlocks; int hyperv_stimer_direct; virGICVersion gic_version; @@ -3476,6 +3486,7 @@ VIR_ENUM_DECL(virDomainGraphicsVNCShareP VIR_ENUM_DECL(virDomainHyperv); VIR_ENUM_DECL(virDomainKVM); VIR_ENUM_DECL(virDomainXen); +VIR_ENUM_DECL(virDomainXenPassthroughMode); VIR_ENUM_DECL(virDomainMsrsUnknown); VIR_ENUM_DECL(virDomainRNGModel); VIR_ENUM_DECL(virDomainRNGBackend); ++++++ libxl-set-cach-mode.patch ++++++ --- /var/tmp/diff_new_pack.b9zLHq/_old 2020-04-30 18:51:31.108608122 +0200 +++ /var/tmp/diff_new_pack.b9zLHq/_new 2020-04-30 18:51:31.108608122 +0200 @@ -7,7 +7,7 @@ =================================================================== --- libvirt-6.0.0.orig/src/libxl/libxl_conf.c +++ libvirt-6.0.0/src/libxl/libxl_conf.c -@@ -871,6 +871,30 @@ libxlDiskSetDiscard(libxl_device_disk *x +@@ -906,6 +906,30 @@ libxlDiskSetDiscard(libxl_device_disk *x #endif } @@ -38,7 +38,7 @@ static char * libxlMakeNetworkDiskSrcStr(virStorageSourcePtr src, const char *username, -@@ -1112,6 +1136,7 @@ libxlMakeDisk(virDomainDiskDefPtr l_disk +@@ -1147,6 +1171,7 @@ libxlMakeDisk(virDomainDiskDefPtr l_disk x_disk->is_cdrom = l_disk->device == VIR_DOMAIN_DISK_DEVICE_CDROM ? 1 : 0; if (libxlDiskSetDiscard(x_disk, l_disk->discard) < 0) return -1; ++++++ libxl-support-block-script.patch ++++++ --- /var/tmp/diff_new_pack.b9zLHq/_old 2020-04-30 18:51:31.120608148 +0200 +++ /var/tmp/diff_new_pack.b9zLHq/_new 2020-04-30 18:51:31.124608156 +0200 @@ -11,7 +11,7 @@ =================================================================== --- libvirt-6.0.0.orig/src/libxl/libxl_conf.c +++ libvirt-6.0.0/src/libxl/libxl_conf.c -@@ -871,6 +871,22 @@ libxlDiskSetDiscard(libxl_device_disk *x +@@ -906,6 +906,22 @@ libxlDiskSetDiscard(libxl_device_disk *x #endif } @@ -34,7 +34,7 @@ static void libxlDiskSetCacheMode(libxl_device_disk *x_disk, int cachemode) { -@@ -1011,6 +1027,7 @@ libxlMakeNetworkDiskSrc(virStorageSource +@@ -1046,6 +1062,7 @@ libxlMakeNetworkDiskSrc(virStorageSource int libxlMakeDisk(virDomainDiskDefPtr l_disk, libxl_device_disk *x_disk) { @@ -42,7 +42,7 @@ const char *driver = virDomainDiskGetDriver(l_disk); int format = virDomainDiskGetFormat(l_disk); int actual_type = virStorageSourceGetActualType(l_disk->src); -@@ -1026,7 +1043,7 @@ libxlMakeDisk(virDomainDiskDefPtr l_disk +@@ -1061,7 +1078,7 @@ libxlMakeDisk(virDomainDiskDefPtr l_disk if (libxlMakeNetworkDiskSrc(l_disk->src, &x_disk->pdev_path) < 0) return -1; } else { @@ -51,7 +51,7 @@ } x_disk->vdev = g_strdup(l_disk->dst); -@@ -1137,6 +1154,9 @@ libxlMakeDisk(virDomainDiskDefPtr l_disk +@@ -1172,6 +1189,9 @@ libxlMakeDisk(virDomainDiskDefPtr l_disk if (libxlDiskSetDiscard(x_disk, l_disk->discard) < 0) return -1; libxlDiskSetCacheMode(x_disk, l_disk->cachemode); ++++++ suse-bump-xen-version.patch ++++++ --- /var/tmp/diff_new_pack.b9zLHq/_old 2020-04-30 18:51:31.144608199 +0200 +++ /var/tmp/diff_new_pack.b9zLHq/_new 2020-04-30 18:51:31.144608199 +0200 @@ -31,7 +31,7 @@ =================================================================== --- libvirt-6.0.0.orig/src/libxl/libxl_conf.c +++ libvirt-6.0.0/src/libxl/libxl_conf.c -@@ -1727,7 +1727,7 @@ libxlDriverConfigNew(void) +@@ -1762,7 +1762,7 @@ libxlDriverConfigNew(void) { libxlDriverConfigPtr cfg; char ebuf[1024]; ++++++ suse-libxl-disable-autoballoon.patch ++++++ --- /var/tmp/diff_new_pack.b9zLHq/_old 2020-04-30 18:51:31.160608233 +0200 +++ /var/tmp/diff_new_pack.b9zLHq/_new 2020-04-30 18:51:31.160608233 +0200 @@ -31,7 +31,7 @@ =================================================================== --- libvirt-6.0.0.orig/src/libxl/libxl_conf.c +++ libvirt-6.0.0/src/libxl/libxl_conf.c -@@ -1703,15 +1703,12 @@ libxlMakeBuildInfoVfb(virPortAllocatorRa +@@ -1738,15 +1738,12 @@ libxlMakeBuildInfoVfb(virPortAllocatorRa /* * Get domain0 autoballoon configuration. Honor user-specified * setting in libxl.conf first. If not specified, autoballooning @@ -48,7 +48,7 @@ int res; res = virConfGetValueBool(conf, "autoballoon", &cfg->autoballoon); -@@ -1720,15 +1717,8 @@ libxlGetAutoballoonConf(libxlDriverConfi +@@ -1755,15 +1752,8 @@ libxlGetAutoballoonConf(libxlDriverConfi else if (res == 1) return 0; ++++++ suse-xen-ovmf-loaders.patch ++++++ --- /var/tmp/diff_new_pack.b9zLHq/_old 2020-04-30 18:51:31.184608284 +0200 +++ /var/tmp/diff_new_pack.b9zLHq/_new 2020-04-30 18:51:31.184608284 +0200 @@ -9,7 +9,7 @@ =================================================================== --- libvirt-6.0.0.orig/src/libxl/libxl_conf.c +++ libvirt-6.0.0/src/libxl/libxl_conf.c -@@ -1780,6 +1780,15 @@ libxlDriverConfigNew(void) +@@ -1815,6 +1815,15 @@ libxlDriverConfigNew(void) goto error; }

1 0