commit python-openqa_review for openSUSE:Factory
Hello community, here is the log from the commit of package python-openqa_review for openSUSE:Factory checked in at 2017-02-07 12:08:20 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/python-openqa_review (Old) and /work/SRC/openSUSE:Factory/.python-openqa_review.new (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Package is "python-openqa_review" Changes: -------- --- /work/SRC/openSUSE:Factory/python-openqa_review/python-openqa_review.changes 2017-02-03 18:57:56.900896765 +0100 +++ /work/SRC/openSUSE:Factory/.python-openqa_review.new/python-openqa_review.changes 2017-02-07 12:08:20.668222109 +0100 @@ -1,0 +2,18 @@ +Mon Feb 06 17:09:31 UTC 2017 - opensuse-packaging@opensuse.org + +- Update to version 1.5.0: + * Post issue reminder comments from daily review script + * tumblesle_release: Add AMQP notification support for 'suse_msg' + * tumblesle_release: Add documentation for AMQP notifications + * tumblesle_release: Prevent spammy repetition of AMQP notifications + * Get rid of duplicate update_jekyll + * More tumblesle notifications + * Fix tumblesle script not checking for new builds / updated job results + * Make tumblesle_release notification connection more resilient + * tumblesle_release: Only store in notify_seen on success + * Recommend safer installation proposals + * Ensure valid python certificates with certifi + * Be more specific about the error when file not found in cache + * Ensure all data is cached on save in openqa_review_osd_daily_email + +------------------------------------------------------------------- Old: ---- python-openqa_review-1.4.1.tar.gz New: ---- python-openqa_review-1.5.0.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ python-openqa_review.spec ++++++ --- /var/tmp/diff_new_pack.wSZm0W/_old 2017-02-07 12:08:21.188148544 +0100 +++ /var/tmp/diff_new_pack.wSZm0W/_new 2017-02-07 12:08:21.188148544 +0100 @@ -18,7 +18,7 @@ %define short_name openqa_review Name: python-%{short_name} -Version: 1.4.1 +Version: 1.5.0 Release: 0 Summary: A review helper script for openQA License: MIT ++++++ python-openqa_review-1.4.1.tar.gz -> python-openqa_review-1.5.0.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openqa_review-1.4.1/README.md new/openqa_review-1.5.0/README.md --- old/openqa_review-1.4.1/README.md 2017-01-20 14:23:49.000000000 +0100 +++ new/openqa_review-1.5.0/README.md 2017-02-06 18:06:38.000000000 +0100 @@ -6,11 +6,19 @@ ## Usage -* Install requirements and package, e.g. +* Install requirements and package, using an isolated Python environment + such as [VirtualEnv](http://docs.python-guide.org/en/latest/dev/virtualenvs/). ``` -sudo pip install -r requirements.txt -sudo pip install . +pip install -r requirements.txt +pip install . +``` + +or if you are using openSUSE distribution, it is recommended to use `zypper`, +e.g.: + +``` +zypper in python-openqa_review ``` * Call `openqa-review` from PATH, e.g. with `--help` diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openqa_review-1.4.1/bin/openqa_review_osd_daily_email new/openqa_review-1.5.0/bin/openqa_review_osd_daily_email --- old/openqa_review-1.4.1/bin/openqa_review_osd_daily_email 2017-01-20 14:23:49.000000000 +0100 +++ new/openqa_review-1.5.0/bin/openqa_review_osd_daily_email 2017-02-06 18:06:38.000000000 +0100 @@ -9,7 +9,8 @@ load_args="${load_args:-"--load --load-dir=${tmp}"}" openqa_review_email_args="${openqa_review_email_args:-"${load_args}"}" openqa_review_html_args="${openqa_review_html_args:-"${load_args} --report-links"}" -openqa_review_save_args="${openqa_review_save_args:-"--report-links --save --save-dir ${tmp}"}" +openqa_review_save_args="${openqa_review_save_args:-"--report-links --reminder-comment-on-issues --dry-run --save --save-dir ${tmp}"}" +openqa_review_reminder_args="${openqa_review_reminder_args:-"${load_args} --reminder-comment-on-issues"}" openqa_review="${openqa_review:-"$(which openqa-review)"}" TPL="${TPL:-"dashboard_files/dashboard.html.in"}" # the awk call is to ignore the "InsecureWarning" and the following blank line @@ -48,3 +49,5 @@ $(<$TPL) EOF " > ${html_target} + +${openqa_review} $openqa_review_args $openqa_review_reminder_args > /dev/null diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openqa_review-1.4.1/bin/tumblesle-release-server-12sp3-x86_64 new/openqa_review-1.5.0/bin/tumblesle-release-server-12sp3-x86_64 --- old/openqa_review-1.4.1/bin/tumblesle-release-server-12sp3-x86_64 2017-01-20 14:23:49.000000000 +0100 +++ new/openqa_review-1.5.0/bin/tumblesle-release-server-12sp3-x86_64 2017-02-06 18:06:38.000000000 +0100 @@ -1 +1 @@ -flock -n /tmp/tumblesle_release.lock -c "tumblesle-release --openqa-host http://openqa.suse.de --group-id 55 --product 'Server' --src openqa:/var/lib/openqa/factory/ --match '*SP3*Server*x86_64*' --match-hdd 'SLES-12-SP3-x86_64*' -vvvv --post-release-hook /home/tumblesle/bin/update_jekyll $@" 2>&1 | tee -a /var/log/tumblesle/tumblesle-release.log +flock -n /tmp/tumblesle_release.lock -c "tumblesle-release --openqa-host http://openqa.suse.de --group-id 55 --product 'Server' --src openqa:/var/lib/openqa/factory/ --match '*SP3*Server*x86_64*' --match-hdd 'SLES-12-SP3-x86_64*' -vvvv --post-release-hook /home/tumblesle/bin/update_jekyll --seen-maxlen=24 $@" 2>&1 | tee -a /var/log/tumblesle/tumblesle-release.log diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openqa_review-1.4.1/bin/update_jekyll new/openqa_review-1.5.0/bin/update_jekyll --- old/openqa_review-1.4.1/bin/update_jekyll 2017-01-20 14:23:49.000000000 +0100 +++ new/openqa_review-1.5.0/bin/update_jekyll 2017-02-06 18:06:38.000000000 +0100 @@ -1,5 +1,7 @@ #!/bin/sh -e -old=$(find /srv/www/tumblesle -maxdepth 1 | head -n -3 | grep '[0-9]\+$') -chmod -R +w ${old} -rm -rf ${old} +old=$(find /srv/www/tumblesle -maxdepth 1 | head -n -3 | grep '[0-9]\+$') || true +if [ "$old" != "" ]; then + chmod -R +w ${old} + rm -rf ${old} +fi sudo -u wwwrun /srv/jekyll-source/.git/hooks/post-merge diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openqa_review-1.4.1/openqa_review/browser.py new/openqa_review-1.5.0/openqa_review/browser.py --- old/openqa_review-1.4.1/openqa_review/browser.py 2017-01-20 14:23:49.000000000 +0100 +++ new/openqa_review-1.5.0/openqa_review/browser.py 2017-02-06 18:06:38.000000000 +0100 @@ -25,6 +25,11 @@ pass +class CacheNotFoundError(DownloadError): + """content could not retrieved from cache.""" + pass + + def url_to_filename(url): """ Convert URL to a valid, unambigous filename. @@ -88,7 +93,7 @@ msg = "Request to %s was not successful, file %s not found" % (url, filename) log.info(msg) # as 'load' simulates downloading we also have to simulate an appropriate error - raise DownloadError(msg) + raise CacheNotFoundError(msg) else: # pragma: no cover raise content = json.loads(raw) if as_json else raw diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openqa_review-1.4.1/openqa_review/openqa_review.py new/openqa_review-1.5.0/openqa_review/openqa_review.py --- old/openqa_review-1.4.1/openqa_review/openqa_review.py 2017-01-20 14:23:49.000000000 +0100 +++ new/openqa_review-1.5.0/openqa_review/openqa_review.py 2017-02-06 18:06:38.000000000 +0100 @@ -614,6 +614,7 @@ def add_comment(self, comment): """Add a comment to an issue with RPC/REST operations.""" + log.info("Posting a comment on %s ticket [%s](%s)" % (self.issue_type, self.bugref, self.bugref_href)) if self.issue_type == 'bugzilla': self.bugzilla_browser.json_rpc_post('/jsonrpc.cgi', 'Bug.add_comment', { "id": self.bugid, diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openqa_review-1.4.1/openqa_review/tumblesle_release.py new/openqa_review-1.5.0/openqa_review/tumblesle_release.py --- old/openqa_review-1.4.1/openqa_review/tumblesle_release.py 2017-01-20 14:23:49.000000000 +0100 +++ new/openqa_review-1.5.0/openqa_review/tumblesle_release.py 2017-02-06 18:06:38.000000000 +0100 @@ -16,6 +16,10 @@ directory structure is available on. The source, i.e. where to copy assets from, can be remote and the path can be specified in an rsync-compatible way, e.g. "openqa:/var/lib/openqa/factory/" + +Notifications over AMQP are sent out if a host has been configured +appropriately in the configuration file (see config file example). The +notifications are serialized in JSON strings. """ # Python 2 and 3: easiest option @@ -29,11 +33,13 @@ import fnmatch import glob import logging +import json import os.path +import pika import re import sys import time -from collections import defaultdict +from collections import defaultdict, deque from configparser import ConfigParser from subprocess import check_call @@ -56,6 +62,8 @@ # example scenario: openSUSE-Tumbleweed-DVD-x86_64-gnome@64bit whitelist = arm7l-foo,bar@uefi +#[notification] +#host = localhost """ @@ -101,6 +109,32 @@ log.info("Whitelist content for %s: %s" % (self.args.product, self.whitelist)) self.release_info_path = os.path.join(self.args.dest, self.args.release_file) self.browser = Browser(args, args.openqa_host) + if not config.has_section('notification'): + return + notify_host = config.get('notification', 'host', fallback='kazhua.suse.de') + self.notify_connection = pika.BlockingConnection(pika.ConnectionParameters(host=notify_host)) + self.notify_channel = self.notify_connection.channel() + self.notify_channel.exchange_declare(exchange='pubsub', type='topic', passive=True, durable=True) + self.notify_topic = 'suse.tumblesle' + self.notify_seen = deque(maxlen=args.seen_maxlen) + + def __del__(self): + """Cleanup notification objects.""" + if not hasattr(self, 'notify_connection'): + return + self.notify_connection.close() + + def notify(self, message, topic='info'): + """Send notification over messaging bus.""" + if not hasattr(self, 'notify_channel'): + log.debug("No notification channel enabled, discarding notify.") + return + body = json.dumps(message) + if body in self.notify_seen: + log.debug("notification message already sent out recently, not resending: %s" % body) + return + self.notify_channel.basic_publish(exchange='pubsub', routing_key='.'.join([self.notify_topic, topic]), body=body) + self.notify_seen.append(body) def run(self, do_run=True): """Continously run while 'do_run' is True, check for last build and release if satisfying.""" @@ -128,7 +162,7 @@ def is_matching_iso(i): return 'iso' in i['type'] and 'Staging' not in i['name'] and re.match(match_re, i['name']) log.debug("Finding most recent ISO matching regex '%s'" % match_re) - assets = self.browser.get_json('/api/v1/assets')['assets'] + assets = self.browser.get_json('/api/v1/assets', cache=self.args.load)['assets'] isos = [i['name'] for i in assets if is_matching_iso(i)] return isos @@ -136,7 +170,7 @@ """Retrieve jobs for current group by build id, returns dict with result as keys.""" group_id = int(self.args.group_id) log.debug("Getting jobs in build %s ..." % build) - jobs_build = self.browser.get_json('/api/v1/jobs?state=done&latest=1&build=%s&group_id=%s' % (build, group_id))['jobs'] + jobs_build = self.browser.get_json('/api/v1/jobs?state=done&latest=1&build=%s&group_id=%s' % (build, group_id), cache=self.args.load)['jobs'] jobs_in_build_product = [i for i in jobs_build if i['group_id'] == group_id] jobs_by_result = defaultdict(list) for job in jobs_in_build_product: @@ -215,6 +249,7 @@ new_fixed = sets['released'].difference(sets['last']) log.info("Regression in new build %s, new failures: %s" % (build['last'], ', '.join(new_failures))) log.debug("new fixed: %s" % ', '.join(new_fixed)) + self.notify({'build': build['last'], 'new_failures': list(new_failures)}, topic='regression') # # assuming every job in released_failed is in whitelist # if len(hard_failed) > previous_hard_failed: @@ -280,6 +315,7 @@ self.update_symlinks(build_dest) self.update_release_info() log.debug("Release DONE") + self.notify({'build': self.release_build}, topic='release') if self.args.post_release_hook: log.debug("Calling post_release_hook '%s'" % self.args.post_release_hook) check_call(self.args.post_release_hook) @@ -343,6 +379,12 @@ parser.add_argument('--post-release-hook', help="Specify application path for a post-release hook which is called after every successful release", default=None) + parser.add_argument('--seen-maxlen', type=int, + help="""The length of the 'seen' buffer for notifications. Any AMQP notification is stored in a FIFO and + before sending it is checked if the notification was already sent out recently with same content. + Together with '--sleeptime' the interval under which the same message would be resent can be configured, + e.g. maxlen*sleeptime = minimum time of reappearence (s)""", + default=500) add_load_save_args(parser) return parser.parse_args() diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openqa_review-1.4.1/setup.py new/openqa_review-1.5.0/setup.py --- old/openqa_review-1.4.1/setup.py 2017-01-20 14:23:49.000000000 +0100 +++ new/openqa_review-1.5.0/setup.py 2017-02-06 18:06:38.000000000 +0100 @@ -34,6 +34,11 @@ "humanfriendly", "requests", "PyYAML", + "pika", + "certifi", + ], + test_require=[ + "pytest-mock", ], author="Oliver kurz", author_email="okurz@suse.com", diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openqa_review-1.4.1/tests/test_tumblesle_release.py new/openqa_review-1.5.0/tests/test_tumblesle_release.py --- old/openqa_review-1.4.1/tests/test_tumblesle_release.py 2017-01-20 14:23:49.000000000 +0100 +++ new/openqa_review-1.5.0/tests/test_tumblesle_release.py 2017-02-06 18:06:38.000000000 +0100 @@ -73,6 +73,7 @@ args.load_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'tumblesle/0046_0056_new_release') args.dest = '/tmp/' args.post_release_hook = None + args.seen_maxlen = 1 # Enable saving and disable loading if you want to add new test data downloaded from hosts # args.save = True # args.save_dir = args.load_dir @@ -175,3 +176,20 @@ tr = tumblesle_release.TumblesleRelease(args) tr.check_last_builds() assert tr.release_build == '0215' + + +def test_notifications_are_sent(args, mocker): + with TumblesleDirectory(args) as tmp_dir: + config_path = os.path.join(tmp_dir, 'config_file') + with open(config_path, 'a') as config: + config.write(""" +[notification] +host = localhost +""") + mocker.patch('pika.BlockingConnection') + tr = tumblesle_release.TumblesleRelease(args) + tr.one_run() + assert '{"build": "0056"}' in tr.notify_seen + # this will yield the same message and is therefore not sent out again + tr.one_run() + assert '{"build": "0056"}' in tr.notify_seen diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openqa_review-1.4.1/tox.ini new/openqa_review-1.5.0/tox.ini --- old/openqa_review-1.4.1/tox.ini 2017-01-20 14:23:49.000000000 +0100 +++ new/openqa_review-1.5.0/tox.ini 2017-02-06 18:06:38.000000000 +0100 @@ -51,12 +51,14 @@ deps = -rrequirements.txt pytest pytest-cov + pytest-mock [testenv] commands = py.test {posargs:-m 'not webtest'} deps = -rrequirements.txt pytest + pytest-mock [testenv:webtests] # This testenv could be called explicitly, e.g. by 'tox -e webtests'. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openqa_review-1.4.1/update_jekyll new/openqa_review-1.5.0/update_jekyll --- old/openqa_review-1.4.1/update_jekyll 2017-01-20 14:23:49.000000000 +0100 +++ new/openqa_review-1.5.0/update_jekyll 1970-01-01 01:00:00.000000000 +0100 @@ -1,7 +0,0 @@ -#!/bin/sh -e -old=$(find /srv/www/tumblesle -maxdepth 1 | head -n -3 | grep '[0-9]\+$') || true -if [ "$old" != "" ]; then - chmod -R +w ${old} - rm -rf ${old} -fi -sudo -u wwwrun /srv/jekyll-source/.git/hooks/post-merge
participants (1)
-
root@hilbertn.suse.de