Hello community, here is the log from the commit of package python-beniget for openSUSE:Factory checked in at 2020-08-31 16:50:41 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/python-beniget (Old) and /work/SRC/openSUSE:Factory/.python-beniget.new.3399 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Package is "python-beniget" Mon Aug 31 16:50:41 2020 rev:4 rq:830622 version:0.3.0 Changes: -------- --- /work/SRC/openSUSE:Factory/python-beniget/python-beniget.changes 2019-09-13 15:03:24.253281151 +0200 +++ /work/SRC/openSUSE:Factory/.python-beniget.new.3399/python-beniget.changes 2020-08-31 16:51:03.928378137 +0200 @@ -1,0 +2,6 @@ +Mon Aug 31 04:08:36 UTC 2020 - Steve Kowalik <steven.kowalik@suse.com> + +- Update to 0.3.0: + * Require gast 0.4.0 + +------------------------------------------------------------------- Old: ---- beniget-0.2.0.tar.gz New: ---- beniget-0.3.0.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ python-beniget.spec ++++++ --- /var/tmp/diff_new_pack.47jL7Q/_old 2020-08-31 16:51:04.744378532 +0200 +++ /var/tmp/diff_new_pack.47jL7Q/_new 2020-08-31 16:51:04.744378532 +0200 @@ -1,7 +1,7 @@ # # spec file for package python-beniget # -# Copyright (c) 2019 SUSE LINUX GmbH, Nuernberg, Germany. +# Copyright (c) 2020 SUSE LLC # # All modifications and additions to the file contributed by third parties # remain the property of their copyright owners, unless otherwise agreed @@ -18,20 +18,19 @@ %{?!python_module:%define python_module() python-%{**} python3-%{**}} Name: python-beniget -Version: 0.2.0 +Version: 0.3.0 Release: 0 Summary: Module to extract semantic information about static Python code License: BSD-3-Clause -Group: Development/Languages/Python URL: https://github.com/serge-sans-paille/beniget/ Source: https://files.pythonhosted.org/packages/source/b/beniget/beniget-%{version}.tar.gz BuildRequires: %{python_module setuptools} BuildRequires: fdupes BuildRequires: python-rpm-macros -Requires: python-gast >= 0.3.0 +Requires: python-gast >= 0.4.0 BuildArch: noarch # SECTION test requirements -BuildRequires: %{python_module gast >= 0.3.0} +BuildRequires: %{python_module gast >= 0.4.0} # /SECTION %python_subpackages ++++++ beniget-0.2.0.tar.gz -> beniget-0.3.0.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/beniget-0.2.0/PKG-INFO new/beniget-0.3.0/PKG-INFO --- old/beniget-0.2.0/PKG-INFO 2019-09-07 17:51:31.000000000 +0200 +++ new/beniget-0.3.0/PKG-INFO 2020-08-07 23:52:40.000000000 +0200 @@ -1,6 +1,6 @@ Metadata-Version: 1.2 Name: beniget -Version: 0.2.0 +Version: 0.3.0 Summary: Extract semantic information about static Python code Home-page: https://github.com/serge-sans-paille/beniget/ Author: serge-sans-paille @@ -9,7 +9,8 @@ Description: A static analyzer for Python2 and Python3 code. - Beniget provides a static over-approximation of the global and local definitions inside Python Module/Class/Function. + Beniget provides a static over-approximation of the global and + local definitions inside Python Module/Class/Function. It can also compute def-use chains from each definition. Platform: UNKNOWN Classifier: Development Status :: 4 - Beta diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/beniget-0.2.0/beniget/beniget.py new/beniget-0.3.0/beniget/beniget.py --- old/beniget-0.2.0/beniget/beniget.py 2019-09-07 17:50:27.000000000 +0200 +++ new/beniget-0.3.0/beniget/beniget.py 2020-08-07 23:52:10.000000000 +0200 @@ -1,6 +1,5 @@ from collections import defaultdict, OrderedDict from contextlib import contextmanager -from copy import deepcopy import sys import gast as ast @@ -75,7 +74,8 @@ raise ValueError("{} has no parent of type {}".format(node, cls)) def parentFunction(self, node): - return self.parentInstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + return self.parentInstance(node, (ast.FunctionDef, + ast.AsyncFunctionDef)) def parentStmt(self, node): return self.parentInstance(node, ast.stmt) @@ -101,7 +101,9 @@ If the node associated to this Def has a name, returns this name. Otherwise returns its type """ - if isinstance(self.node, (ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)): + if isinstance(self.node, (ast.ClassDef, + ast.FunctionDef, + ast.AsyncFunctionDef)): return self.node.name elif isinstance(self.node, ast.Name): return self.node.id @@ -128,7 +130,8 @@ else: nodes[self] = len(nodes) return "{} -> ({})".format( - self.node, ", ".join(u._repr(nodes.copy()) for u in self._users) + self.node, ", ".join(u._repr(nodes.copy()) + for u in self._users) ) def __str__(self): @@ -140,7 +143,8 @@ else: nodes[self] = len(nodes) return "{} -> ({})".format( - self.name(), ", ".join(u._str(nodes.copy()) for u in self._users) + self.name(), ", ".join(u._str(nodes.copy()) + for u in self._users) ) @@ -189,9 +193,13 @@ c -> (c -> (Call -> ())) """ - def __init__(self): + def __init__(self, filename=None): + """ + - filename: str, included in error messages if specified + """ self.chains = {} self.locals = defaultdict(list) + self.filename = filename # deep copy of builtins, to remain reentrant self._builtins = {k: Def(v) for k, v in Builtins.items()} @@ -210,9 +218,12 @@ # be defined in another path of the control flow (esp. in loop) self._undefs = [] - # stack of current node holding definitions (class, module, function...) + # stack of current node holding definitions: class, module, function... self._currenthead = [] + self._breaks = [] + self._continues = [] + # dead code levels self.deadcode = 0 @@ -221,7 +232,8 @@ def dump_definitions(self, node, ignore_builtins=True): if isinstance(node, ast.Module) and not ignore_builtins: builtins = {d for d in self._builtins.values()} - return sorted(d.name() for d in self.locals[node] if d not in builtins) + return sorted(d.name() + for d in self.locals[node] if d not in builtins) else: return sorted(d.name() for d in self.locals[node]) @@ -233,7 +245,12 @@ def unbound_identifier(self, name, node): if hasattr(node, "lineno"): - location = " at {}:{}".format(node.lineno, node.col_offset) + filename = "{}:".format( + "<unknown>" if self.filename is None else self.filename + ) + location = " at {}{}:{}".format(filename, + node.lineno, + node.col_offset) else: location = "" print("W: unbound identifier '{}'{}".format(name, location)) @@ -249,7 +266,7 @@ stars = [] for d in reversed(self._definitions): if name in d: - return d[name] if not stars else stars + d[name] + return d[name] if not stars else stars + list(d[name]) if "*" in d: stars.extend(d["*"]) @@ -280,7 +297,7 @@ for undef_name, _undefs in self._undefs[-1].items(): if undef_name in self._definitions[-1]: for newdef in self._definitions[-1][undef_name]: - for undef, stars in _undefs: + for undef, _ in _undefs: for user in undef.users(): newdef.add_user(user) else: @@ -335,7 +352,8 @@ # handle function bodies for fnode, ctx in self._defered[-1]: - visitor = getattr(self, "visit_{}".format(type(fnode).__name__)) + visitor = getattr(self, + "visit_{}".format(type(fnode).__name__)) defs, self._definitions = self._definitions, ctx visitor(fnode, step=DefinitionStep) self._definitions = defs @@ -367,13 +385,18 @@ else: self._definitions[-1][name] = ordered_set(dnode_or_dnodes) + @staticmethod + def add_to_definition(definition, name, dnode_or_dnodes): + if isinstance(dnode_or_dnodes, Def): + definition[name].add(dnode_or_dnodes) + else: + definition[name].update(dnode_or_dnodes) + def extend_definition(self, name, dnode_or_dnodes): if self.deadcode: return - if isinstance(dnode_or_dnodes, Def): - self._definitions[-1][name].add(dnode_or_dnodes) - else: - self._definitions[-1][name].update(dnode_or_dnodes) + DefUseChains.add_to_definition(self._definitions[-1], name, + dnode_or_dnodes) def visit_FunctionDef(self, node, step=DeclarationStep): if step is DeclarationStep: @@ -393,6 +416,9 @@ definitions.pop() self._defered[-1].append((node, definitions)) elif step is DefinitionStep: + # function is not considered as defined when evaluating returns + if node.returns: + self.visit(node.returns) with self.DefinitionContext(node): self.visit(node.args) self.process_body(node.body) @@ -420,12 +446,23 @@ if node.value: self.visit(node.value) + def visit_Break(self, _): + for k, v in self._definitions[-1].items(): + DefUseChains.add_to_definition(self._breaks[-1], k, v) + self._definitions[-1].clear() + + def visit_Continue(self, _): + for k, v in self._definitions[-1].items(): + DefUseChains.add_to_definition(self._continues[-1], k, v) + self._definitions[-1].clear() + def visit_Delete(self, node): for target in node.targets: self.visit(target) def visit_Assign(self, node): - dvalue = self.visit(node.value) + # link is implicit through ctx + self.visit(node.value) for target in node.targets: self.visit(target) @@ -448,7 +485,12 @@ if node.target.id in self._promoted_locals[-1]: self.extend_definition(node.target.id, dtarget) else: + loaded_from = [d.name() for d in self.defs(node.target)] self.set_definition(node.target.id, dtarget) + # If we augassign from a value that comes from '*', let's use + # this node as the definition point. + if '*' in loaded_from: + self.locals[self._currenthead[-1]].append(dtarget) else: self.visit(node.target).add_user(dvalue) @@ -461,12 +503,8 @@ def visit_For(self, node): self.visit(node.iter) - # process else clause in the case of an early break - self._undefs.append(defaultdict(list)) - self._definitions.append(self._definitions[-1].copy()) - self.process_body(node.orelse) - self._definitions.pop() # drop defs because they don't dominate body - self._undefs.pop() + self._breaks.append(defaultdict(ordered_set)) + self._continues.append(defaultdict(ordered_set)) self._undefs.append(defaultdict(list)) self._definitions.append(self._definitions[-1].copy()) @@ -474,20 +512,34 @@ self.process_body(node.body) self.process_undefs() + continue_defs = self._continues.pop() + for d, u in continue_defs.items(): + self.extend_definition(d, u) + self._continues.append(defaultdict(ordered_set)) + # extra round to ``emulate'' looping self.visit(node.target) self.process_body(node.body) - # reprocess else clause in case of late break + # process else clause in case of late break self._definitions.append(defaultdict(ordered_set)) self.process_body(node.orelse) orelse_defs = self._definitions.pop() + break_defs = self._breaks.pop() + continue_defs = self._continues.pop() + body_defs = self._definitions.pop() for d, u in orelse_defs.items(): self.extend_definition(d, u) + for d, u in continue_defs.items(): + self.extend_definition(d, u) + + for d, u in break_defs.items(): + self.extend_definition(d, u) + for d, u in body_defs.items(): self.extend_definition(d, u) @@ -497,6 +549,8 @@ self._definitions.append(self._definitions[-1].copy()) self._undefs.append(defaultdict(list)) + self._breaks.append(defaultdict(ordered_set)) + self._continues.append(defaultdict(ordered_set)) self.process_body(node.orelse) @@ -509,6 +563,11 @@ self.process_undefs() + continue_defs = self._continues.pop() + for d, u in continue_defs.items(): + self.extend_definition(d, u) + self._continues.append(defaultdict(ordered_set)) + # extra round to simulate loop self.visit(node.test) self.process_body(node.body) @@ -521,6 +580,14 @@ orelse_defs = self._definitions.pop() body_defs = self._definitions.pop() + break_defs = self._breaks.pop() + continue_defs = self._continues.pop() + + for d, u in continue_defs.items(): + self.extend_definition(d, u) + + for d, u in break_defs.items(): + self.extend_definition(d, u) for d, u in orelse_defs.items(): self.extend_definition(d, u) @@ -798,7 +865,7 @@ if node.annotation is not None: self.visit(node.annotation) - elif isinstance(node.ctx, ast.Load): + elif isinstance(node.ctx, (ast.Load, ast.Del)): node_in_chains = node in self.chains if node_in_chains: dnode = self.chains[node] @@ -808,17 +875,7 @@ d.add_user(dnode) if not node_in_chains: self.chains[node] = dnode - - elif isinstance(node.ctx, ast.Del): - dnode = self.chains.setdefault(node, Def(node)) - self.set_definition(node.id, []) - # should we also remove node.id from locals? - # for d in self._definitions[-1][node.id]: - # try: - # self.locals[self._currenthead[-1]].remove(d) - # except ValueError: - # pass - # del self._definitions[-1][node.id] + # currently ignore the effect of a del else: raise NotImplementedError() return dnode @@ -862,14 +919,6 @@ self.visit(node.step).add_user(dnode) return dnode - def visit_ExtSlice(self, node): - dnode = self.chains.setdefault(node, Def(node)) - for dim in node.dims: - self.visit(dim).add_user(dnode) - return dnode - - visit_Index = visit_Await - # misc def visit_comprehension(self, node): @@ -947,20 +996,6 @@ if __name__ == "__main__": import sys - class DefUseChainsX(DefUseChains): - def __init__(self, filename): - super(DefUseChainsX, self).__init__() - self.filename = filename - - def unbound_identifier(self, name, node): - if hasattr(node, "lineno"): - location = " at {}:{}:".format(node.lineno, node.col_offset) - else: - location = "" - print( - "W: unbound identifier '{}'{}{}".format(name, location, self.filename) - ) - class Beniget(ast.NodeVisitor): def __init__(self, filename, module): super(Beniget, self).__init__() @@ -970,7 +1005,7 @@ self.ancestors = Ancestors() self.ancestors.visit(module) - self.defuses = DefUseChainsX(self.filename) + self.defuses = DefUseChains(self.filename) self.defuses.visit(module) self.visit(module) @@ -1005,7 +1040,8 @@ if self.filename.endswith("__init__.py"): return self.check_unused( - node, skipped_types=(ast.FunctionDef, ast.ClassDef, ast.Name) + node, skipped_types=(ast.FunctionDef, ast.AsyncFunctionDef, + ast.ClassDef, ast.Name) ) def visit_FunctionDef(self, node): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/beniget-0.2.0/beniget.egg-info/PKG-INFO new/beniget-0.3.0/beniget.egg-info/PKG-INFO --- old/beniget-0.2.0/beniget.egg-info/PKG-INFO 2019-09-07 17:51:31.000000000 +0200 +++ new/beniget-0.3.0/beniget.egg-info/PKG-INFO 2020-08-07 23:52:39.000000000 +0200 @@ -1,6 +1,6 @@ Metadata-Version: 1.2 Name: beniget -Version: 0.2.0 +Version: 0.3.0 Summary: Extract semantic information about static Python code Home-page: https://github.com/serge-sans-paille/beniget/ Author: serge-sans-paille @@ -9,7 +9,8 @@ Description: A static analyzer for Python2 and Python3 code. - Beniget provides a static over-approximation of the global and local definitions inside Python Module/Class/Function. + Beniget provides a static over-approximation of the global and + local definitions inside Python Module/Class/Function. It can also compute def-use chains from each definition. Platform: UNKNOWN Classifier: Development Status :: 4 - Beta diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/beniget-0.2.0/beniget.egg-info/requires.txt new/beniget-0.3.0/beniget.egg-info/requires.txt --- old/beniget-0.2.0/beniget.egg-info/requires.txt 2019-09-07 17:51:31.000000000 +0200 +++ new/beniget-0.3.0/beniget.egg-info/requires.txt 2020-08-07 23:52:39.000000000 +0200 @@ -1 +1 @@ -gast>=0.3.0 +gast~=0.4.0 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/beniget-0.2.0/requirements.txt new/beniget-0.3.0/requirements.txt --- old/beniget-0.2.0/requirements.txt 2019-09-07 17:50:27.000000000 +0200 +++ new/beniget-0.3.0/requirements.txt 2020-08-07 23:52:10.000000000 +0200 @@ -1 +1 @@ -gast >= 0.3.0 +gast ~= 0.4.0 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/beniget-0.2.0/setup.py new/beniget-0.3.0/setup.py --- old/beniget-0.2.0/setup.py 2019-09-07 17:51:00.000000000 +0200 +++ new/beniget-0.3.0/setup.py 2020-08-07 23:52:10.000000000 +0200 @@ -9,13 +9,14 @@ setup( name="beniget", # gast, beniget! - version="0.2.0", + version="0.3.0", packages=["beniget"], description="Extract semantic information about static Python code", long_description=""" A static analyzer for Python2 and Python3 code. -Beniget provides a static over-approximation of the global and local definitions inside Python Module/Class/Function. +Beniget provides a static over-approximation of the global and +local definitions inside Python Module/Class/Function. It can also compute def-use chains from each definition.""", author="serge-sans-paille", author_email="serge.guelton@telecom-bretagne.eu", diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/beniget-0.2.0/tests/chains.py new/beniget-0.3.0/tests/chains.py --- old/beniget-0.2.0/tests/chains.py 2019-09-07 17:50:27.000000000 +0200 +++ new/beniget-0.3.0/tests/chains.py 2020-05-25 16:37:33.000000000 +0200 @@ -1,9 +1,25 @@ +from contextlib import contextmanager from unittest import TestCase, skipIf import gast as ast import beniget +import io import sys +@contextmanager +def captured_output(): + if sys.version_info.major >= 3: + new_out, new_err = io.StringIO(), io.StringIO() + else: + new_out, new_err = io.BytesIO(), io.BytesIO() + old_out, old_err = sys.stdout, sys.stderr + try: + sys.stdout, sys.stderr = new_out, new_err + yield sys.stdout, sys.stderr + finally: + sys.stdout, sys.stderr = old_out, old_err + + class TestDefUseChains(TestCase): def checkChains(self, code, ref): class StrictDefUseChains(beniget.DefUseChains): @@ -62,6 +78,18 @@ code, ["m -> (m -> (BinOp -> ()))", "i -> ()", "m -> (m -> (BinOp -> ()))"] ) + def test_continue_in_loop(self): + code = "for i in [1, 2]:\n if i: m = 1; continue\n m = 1\nm" + self.checkChains( + code, ['i -> (i -> ())', 'm -> (m -> ())', 'm -> (m -> ())'] + ) + + def test_break_in_loop(self): + code = "for i in [1, 2]:\n if i: m = 1; continue\n m = 1\nm" + self.checkChains( + code, ['i -> (i -> ())', 'm -> (m -> ())', 'm -> (m -> ())'] + ) + def test_augassign(self): code = "a = 1; a += 2; a" self.checkChains(code, ['a -> (a -> (a -> ()))']) @@ -146,10 +174,9 @@ code, ['I -> (I -> ())', 'J -> (J -> ())', - 'J -> (J -> ())', 'i -> (i -> (Compare -> ()), i -> ())', - 'I -> (I -> ())'] - , + 'I -> (I -> ())', + 'J -> (J -> ())'] ) def test_simple_while(self): @@ -171,6 +198,38 @@ ['i -> (i -> ())', 'i -> ()']) + def test_while_cond_break(self): + code = "i = 8\nwhile 1:\n if i: i=1;break\ni" + self.checkChains( + code, + ['i -> (i -> (), i -> ())', 'i -> (i -> ())']) + + def test_nested_while(self): + code = ''' +done = 1 +while done: + + while done: + if 1: + done = 1 + break + + if 1: + break''' + + + self.checkChains( + code, + ['done -> (done -> (), done -> ())', + 'done -> (done -> (), done -> ())'] + ) + + def test_while_cond_continue(self): + code = "i = 8\nwhile 1:\n if i: i=1;continue\ni" + self.checkChains( + code, + ['i -> (i -> (), i -> ())', 'i -> (i -> (), i -> ())']) + def test_complex_while_orelse(self): code = "I = J = i = 0\nwhile i:\n if i < 3: I = i\nelse:\n if 1: J = I\nJ" self.checkChains( @@ -184,6 +243,14 @@ ], ) + def test_while_orelse_break(self): + code = "I = 0\nwhile I:\n if 1: I = 1; break\nelse: I" + self.checkChains( + code, + ['I -> (I -> (), I -> ())', + 'I -> ()'], + ) + def test_while_nested_break(self): code = "i = 8\nwhile i:\n if i: break\n i = 3\ni" self.checkChains( @@ -278,10 +345,45 @@ self.checkChains(code, ["decorator -> (decorator -> (C -> ()))", "C -> ()"]) @skipIf(sys.version_info.major < 3, "Python 3 syntax") + def test_functiondef_returns(self): + code = "x = 1\ndef foo() -> x: pass" + self.checkChains(code, ['x -> (x -> ())', 'foo -> ()']) + + @skipIf(sys.version_info.major < 3, "Python 3 syntax") def test_class_annotation(self): code = "type_ = int\ndef foo(bar: type_): pass" self.checkChains(code, ["type_ -> (type_ -> ())", "foo -> ()"]) + def check_unbound_identifier_message(self, code, expected_messages, filename=None): + node = ast.parse(code) + c = beniget.DefUseChains(filename) + with captured_output() as (out, err): + c.visit(node) + produced_messages = out.getvalue().strip().split("\n") + + self.assertEqual(len(expected_messages), len(produced_messages)) + for expected, produced in zip(expected_messages, produced_messages): + self.assertIn(expected, produced, "actual message contains expected message") + + def test_unbound_identifier_message_format(self): + code = "foo(1)\nbar(2)" + self.check_unbound_identifier_message(code, ["<unknown>:1", "<unknown>:2"]) + self.check_unbound_identifier_message(code, ["foo.py:1", "foo.py:2"], filename="foo.py") + + def test_star_import_with_conditional_redef(self): + code = ''' +from math import * + +if 1: + def pop(): + cos() +cos = pop()''' + self.checkChains(code, [ + '* -> (cos -> (Call -> ()))', + 'pop -> (pop -> (Call -> ()))', + 'cos -> (cos -> (Call -> ()))' + ]) + class TestUseDefChains(TestCase): def checkChains(self, code, ref): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/beniget-0.2.0/tests/definitions.py new/beniget-0.3.0/tests/definitions.py --- old/beniget-0.2.0/tests/definitions.py 2019-06-28 16:02:48.000000000 +0200 +++ new/beniget-0.3.0/tests/definitions.py 2020-06-05 22:45:23.000000000 +0200 @@ -50,6 +50,10 @@ code = "class C:pass\ndel C" self.checkGlobals(code, ["C"]) + def testDelClassDefReDef(self): + code = "class C:pass\ndel C\nclass C:pass" + self.checkGlobals(code, ["C", "C"]) + def testNestedClassDef(self): code = "class C:\n class D: pass" self.checkGlobals(code, ["C"]) @@ -218,6 +222,10 @@ code = "from foo import *" self.checkGlobals(code, ["*"]) + def testGlobalImportFromStarRedefine(self): + code = "from foo import *\nx+=1" + self.checkGlobals(code, ["*", "x"]) + def testGlobalImportsFrom(self): code = "from foo import bar, man" self.checkGlobals(code, ["bar", "man"])