Script 'mail_helper' called by obssrc Hello community, here is the log from the commit of package python-pyp for openSUSE:Factory checked in at 2023-03-01 16:14:24 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/python-pyp (Old) and /work/SRC/openSUSE:Factory/.python-pyp.new.31432 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Package is "python-pyp" Wed Mar 1 16:14:24 2023 rev:2 rq:1068349 version:1.1.0 Changes: -------- --- /work/SRC/openSUSE:Factory/python-pyp/python-pyp.changes 2021-09-10 23:41:51.366594168 +0200 +++ /work/SRC/openSUSE:Factory/.python-pyp.new.31432/python-pyp.changes 2023-03-01 16:14:51.154848178 +0100 @@ -1,0 +2,30 @@ +Wed Feb 22 06:20:21 UTC 2023 - Daniel Garcia <daniel.garcia@suse.com> + +- Use release from github, the new release was created there + (gh#hauntsaninja/pyp#33) + +------------------------------------------------------------------- +Tue Feb 21 16:35:23 UTC 2023 - Daniel Garcia <daniel.garcia@suse.com> + +- Use release from pypi, the last release it not tagged in github, but + add a new source with the github repo to get the tests that are not + in the pypi release. (gh#hauntsaninja/pyp#33) +- Update to 1.1.0: + * Fix AST construction on Python 3.11 + * Constructed ASTs now have a more convincing end_lineno + * Test coverage for fallback unparsing, other test improvements + * Now packaged by flit +- [v1.0.0] + * Configuration now allows the use of magic variables, effectively + allowing you to define your own magic variables. See README.md for + details + * Explicit printing in used config functions will now disable + automatic printing + * Config definitions can now use things defined from wildcard + imports. Automatic imports now work in config as well + * Removed s as a magic variable. If you miss it, you can redefine it + in your config using s = x + * Implement correct scoping semantics for comprehensions, including + with assignment expressions + +------------------------------------------------------------------- Old: ---- pyp-0.3.4.tar.gz New: ---- pyp-1.1.0.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ python-pyp.spec ++++++ --- /var/tmp/diff_new_pack.Eqaib4/_old 2023-03-01 16:14:51.622850598 +0100 +++ /var/tmp/diff_new_pack.Eqaib4/_new 2023-03-01 16:14:51.630850640 +0100 @@ -1,7 +1,7 @@ # # spec file for package python-pyp # -# Copyright (c) 2021 SUSE LLC +# Copyright (c) 2023 SUSE LLC # Copyright (c) 2020-2021 LISA GmbH, Bingen, Germany # # All modifications and additions to the file contributed by third parties @@ -17,10 +17,9 @@ # -%{?!python_module:%define python_module() python-%{**} python3-%{**}} %define skip_python2 1 Name: python-pyp -Version: 0.3.4 +Version: 1.1.0 Release: 0 Summary: Python at the shell License: MIT @@ -29,17 +28,19 @@ Source0: https://github.com/hauntsaninja/pyp/archive/v%{version}.tar.gz#/pyp-%{version}.tar.gz BuildRequires: %{python_module astunparse} BuildRequires: %{python_module base} -BuildRequires: %{python_module setuptools} +BuildRequires: %{python_module flit-core} +BuildRequires: %{python_module pip} +BuildRequires: %{python_module wheel} BuildRequires: fdupes BuildRequires: python-rpm-macros # testing requirements BuildRequires: bc -BuildRequires: jq BuildRequires: %{python_module pytest} +BuildRequires: jq Requires: python-astunparse BuildArch: noarch Requires(post): update-alternatives -Requires(postun): update-alternatives +Requires(postun):update-alternatives %python_subpackages %description @@ -48,14 +49,14 @@ See README.md or https://github.com/hauntsaninja/pyp for examples. %prep -%setup -q -n pyp-%{version} +%autosetup -p1 -n pyp-%{version} sed -i '/^#!\//, 1d' pyp.py %build -%python_build +%pyproject_wheel %install -%python_install +%pyproject_install %python_clone -a %{buildroot}%{_bindir}/pyp %python_expand %fdupes %{buildroot}%{$python_sitelib} @@ -74,7 +75,9 @@ %files %{python_files} %license LICENSE %doc *.md -%{python_sitelib}/ +%{python_sitelib}/pyp.py +%{python_sitelib}/pypyp-%{version}*-info +%pycache_only %{python_sitelib}/__pycache__ %python_alternative %{_bindir}/pyp %changelog ++++++ pyp-0.3.4.tar.gz -> pyp-1.1.0.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/pyp-0.3.4/.github/workflows/tests.yml new/pyp-1.1.0/.github/workflows/tests.yml --- old/pyp-0.3.4/.github/workflows/tests.yml 2021-09-09 03:44:01.000000000 +0200 +++ new/pyp-1.1.0/.github/workflows/tests.yml 2023-01-12 10:45:46.000000000 +0100 @@ -1,8 +1,6 @@ name: Tests -on: - push: - pull_request: +on: [push, pull_request, workflow_dispatch] permissions: contents: read @@ -11,15 +9,15 @@ lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 - run: pip install tox - run: tox -e lint mypy: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 - run: pip install tox - run: tox -e mypy tests: @@ -28,12 +26,14 @@ fail-fast: false matrix: include: - - {python: '3.6', tox: py36} + - {python: '3.7', tox: py37} - {python: '3.8', tox: py38} - {python: '3.9', tox: py39} + - {python: '3.10', tox: py310} + - {python: '3.11', tox: py311} steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} - run: pip install tox @@ -43,9 +43,8 @@ env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 - run: pip install tox coveralls - run: tox -e coverage - run: coveralls --service=github - diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/pyp-0.3.4/CHANGELOG.md new/pyp-1.1.0/CHANGELOG.md --- old/pyp-0.3.4/CHANGELOG.md 2021-09-09 03:44:01.000000000 +0200 +++ new/pyp-1.1.0/CHANGELOG.md 2023-01-12 10:45:46.000000000 +0100 @@ -1,5 +1,22 @@ # Changelog +## [v1.1.0] + +- Fix AST construction on Python 3.11 +- Constructed ASTs now have a more convincing `end_lineno` +- Test coverage for fallback unparsing, other test improvements +- Now packaged by `flit` + +## [v1.0.0] + +- Configuration now allows the use of magic variables, effectively allowing you to define your own +magic variables. See README.md for details +- Explicit printing in used config functions will now disable automatic printing +- Config definitions can now use things defined from wildcard imports. Automatic imports now work +in config as well +- Removed `s` as a magic variable. If you miss it, you can redefine it in your config using `s = x` +- Implement correct scoping semantics for comprehensions, including with assignment expressions + ## [v0.3.4] - Reduce reconstructed traceback's reliance on CPython implementation details diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/pyp-0.3.4/CONTRIBUTING.md new/pyp-1.1.0/CONTRIBUTING.md --- old/pyp-0.3.4/CONTRIBUTING.md 2021-09-09 03:44:01.000000000 +0200 +++ new/pyp-1.1.0/CONTRIBUTING.md 2023-01-12 10:45:46.000000000 +0100 @@ -16,9 +16,10 @@ ## Making a release - Update the changelog +- Update the version in `CHANGELOG.md` - Update the version in `__version__` -- Update the version in `setup.py` +- Update the version in `pyproject.toml` - `rm -rf dist` -- `python setup.py sdist bdist_wheel` +- `python -m build .` - `twine upload dist/*` - Tag the release on Github diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/pyp-0.3.4/FAQ.md new/pyp-1.1.0/FAQ.md --- old/pyp-0.3.4/FAQ.md 2021-09-09 03:44:01.000000000 +0200 +++ new/pyp-1.1.0/FAQ.md 2023-01-12 10:45:46.000000000 +0100 @@ -3,6 +3,7 @@ ### Contents - [I'm running into issues with newlines / complicated statements](#im-running-into-issues-with-newlines--complicated-statements) +- [What's in your config?](#whats-in-your-config) - [What are pyp's dependencies?](#what-are-pyps-dependencies) - [Can I customise the shebang on the output of `--script`?](#can-i-customise-the-shebang-on-the-output-of---script) - [The output of `--explain` is a little weirdly formatted](#the-output-of---explain-is-a-little-weirdly-formatted) @@ -48,6 +49,21 @@ $ pyp --explain $'if int(x) >= 100:\n x += " is big"\n print(x)\nelse:\n print(x + " is small")' ``` +#### What's in your config? + +My config is pretty simple: +```py +import numpy as np + +n = int(x) +j = json.loads(stdin) +f = x.split() +# like f, but returns None if index is of bounds +ff = defaultdict(lambda: None, dict(enumerate(x.split()))) + +d = defaultdict(list) +``` + #### What are pyp's dependencies? If run on Python 3.9 or later, pyp has no dependencies. @@ -80,12 +96,12 @@ processes start in parallel, so this is zero extra wall time if your piped input has any latency. Here's a benchmark that should basically just be measuring the fixed costs of start up and AST -transformation (run on my old, not powerful laptop): +transformation (run on my laptop): ``` -$ hyperfine -w 10 -m 100 'pyp x' +hyperfine -w 10 -m 100 'pyp x' Benchmark #1: pyp x - Time (mean �� ��): 81.5 ms �� 1.4 ms [User: 60.3 ms, System: 15.9 ms] - Range (min ��� max): 78.6 ms ��� 84.8 ms 100 runs + Time (mean �� ��): 56.3 ms �� 1.0 ms [User: 41.2 ms, System: 11.4 ms] + Range (min ��� max): 53.9 ms ��� 60.3 ms 100 runs ``` One note here, as mentioned in the README, is that if you use wildcard imports (`from x import *`) @@ -100,13 +116,13 @@ ``` $ hyperfine -w 3 -m 10 "seq 1 999999 | pyp 'sum(map(int, stdin))'" Benchmark #1: seq 1 999999 | pyp 'sum(map(int, stdin))' - Time (mean �� ��): 490.9 ms �� 4.3 ms [User: 848.8 ms, System: 26.0 ms] - Range (min ��� max): 487.4 ms ��� 502.3 ms 10 runs + Time (mean �� ��): 258.2 ms �� 5.6 ms [User: 422.3 ms, System: 17.0 ms] + Range (min ��� max): 252.1 ms ��� 270.7 ms 11 runs $ hyperfine -w 3 -m 10 'seq 1 999999 | awk "{s += $0} END {print s}"' Benchmark #1: seq 1 999999 | awk "{s += $0} END {print s}" - Time (mean �� ��): 754.6 ms �� 4.8 ms [User: 1.152 s, System: 0.013 s] - Range (min ��� max): 748.9 ms ��� 763.2 ms 10 runs + Time (mean �� ��): 405.3 ms �� 3.4 ms [User: 599.6 ms, System: 5.5 ms] + Range (min ��� max): 399.4 ms ��� 410.9 ms 10 runs ``` More seriously, random micro benchmark aside, pyp should be fast enough that you shouldn't worry @@ -119,8 +135,8 @@ ``` $ hyperfine -w 3 -m 10 "seq 1 999999 | pyp 'sum(map(int, lines))'" Benchmark #1: seq 1 999999 | pyp 'sum(map(int, lines))' - Time (mean �� ��): 848.5 ms �� 146.2 ms [User: 1.157 s, System: 0.087 s] - Range (min ��� max): 748.4 ms ��� 1239.1 ms 10 runs + Time (mean �� ��): 378.9 ms �� 3.2 ms [User: 530.2 ms, System: 38.5 ms] + Range (min ��� max): 375.4 ms ��� 384.6 ms 10 runs ``` #### Can I use pyp with PyPy? diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/pyp-0.3.4/README.md new/pyp-1.1.0/README.md --- old/pyp-0.3.4/README.md 2021-09-09 03:44:01.000000000 +0200 +++ new/pyp-1.1.0/README.md 2023-01-12 10:45:46.000000000 +0100 @@ -24,7 +24,7 @@ for many common shell utilities. For a cheatsheet / tldr, run `pyp --help`. #### pyp can easily be used to apply Python code to each line in the input. -Just use one of the magic variables `x`, `l`, `s` or `line` to refer to the current line. +Just use one of the magic variables `x`, `l` or `line` to refer to the current line. ```sh # pyp like cut @@ -146,7 +146,7 @@ class PotentiallyUsefulClass: ... ``` -When attempting to define undefined names, pyp will statically* analyse this file as a source of +When attempting to define undefined names, pyp will statically\* analyse this file as a source of possible definitions. This means that if you don't use `tf`, we won't import `tensorflow`! And of course, `--explain` will show you exactly what gets run (and hence what doesn't!): @@ -174,6 +174,28 @@ names, though we skip this in the happy path. If this matters to you, definitely don't `from tensorflow import *` in your config! </sub> +#### pyp lets you configure your own magic! + +If definitions in your config file depend on magic variables, pyp will substitute them in the +way that makes sense. For example, put the following in your config... +```py +n = int(x) +f = x.split() +j = json.load(stdin) + +import pandas as pd +csv = pd.read_csv(stdin) +``` + +...to make pyp easier than ever for your custom use cases: +```sh +ps | pyp 'f[3]' + +cat commits.json | pyp 'j[0]["commit"]["author"]' + +< cities.csv pyp 'csv.to_string()' +``` + #### I have questions! There's additional documentation and examples at [FAQ](https://github.com/hauntsaninja/pyp/blob/master/FAQ.md). @@ -215,6 +237,7 @@ - Some of them have specialised support for things like JSON input or running shell commands. - Some of them expose the input in interesting ways with custom line / file / stream objects. - Some of them have more advanced options for error handling. +- None of them have powerful configuration like pyp. - None of them have anything like `--explain`. For whatever it's worth, I've listed these projects in approximate order of my personal preference. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/pyp-0.3.4/pyp.py new/pyp-1.1.0/pyp.py --- old/pyp-0.3.4/pyp.py 2021-09-09 03:44:01.000000000 +0200 +++ new/pyp-1.1.0/pyp.py 2023-01-12 10:45:46.000000000 +0100 @@ -12,7 +12,7 @@ from typing import Any, Dict, Iterator, List, Optional, Set, Tuple, cast __all__ = ["pypprint"] -__version__ = "0.3.4" +__version__ = "1.1.0" def pypprint(*args, **kwargs): # type: ignore @@ -46,12 +46,14 @@ An undefined name is any name that is loaded before it is defined (in any scope). Notes: a) we ignore deletes, b) used builtins will appear in undefined names, c) this logic - doesn't fully support comprehension / nonlocal / global / late-binding scopes. + doesn't fully support nonlocal / global / late-binding scopes. """ def __init__(self, *trees: ast.AST) -> None: self._scopes: List[Set[str]] = [set()] + self._comprehension_scopes: List[int] = [] + self.undefined: Set[str] = set() self.wildcard_imports: List[str] = [] for tree in trees: @@ -72,8 +74,8 @@ def generic_visit(self, node: ast.AST) -> None: def order(f_v: Tuple[str, Any]) -> int: - # This ordering fixes comprehensions, loops, assignments - return {"generators": -2, "iter": -2, "value": -1}.get(f_v[0], 0) + # This ordering fixes comprehensions, dict comps, loops, assignments + return {"generators": -3, "iter": -3, "key": -2, "value": -1}.get(f_v[0], 0) # Adapted from ast.NodeVisitor.generic_visit, but re-orders traversal a little for _, value in sorted(ast.iter_fields(node), key=order): @@ -102,6 +104,18 @@ self.undefined.add(node.target.id) self.generic_visit(node) + def visit_NamedExpr(self, node: Any) -> None: + self.visit(node.value) + # PEP 572 has weird scoping rules + assert isinstance(node.target, ast.Name) + assert isinstance(node.target.ctx, ast.Store) + scope_index = len(self._scopes) - 1 + comp_index = len(self._comprehension_scopes) - 1 + while comp_index >= 0 and scope_index == self._comprehension_scopes[comp_index]: + scope_index -= 1 + comp_index -= 1 + self._scopes[scope_index].add(node.target.id) + def visit_alias(self, node: ast.alias) -> None: if node.name != "*": self._scopes[-1].add(node.asname if node.asname is not None else node.name) @@ -159,6 +173,18 @@ self.flexible_visit(node.body) self._scopes[-1].remove(node.name) + def visit_comprehension_helper(self, node: Any) -> None: + self._comprehension_scopes.append(len(self._scopes)) + self._scopes.append(set()) + self.generic_visit(node) + self._scopes.pop() + self._comprehension_scopes.pop() + + visit_ListComp = visit_comprehension_helper + visit_SetComp = visit_comprehension_helper + visit_GeneratorExp = visit_comprehension_helper + visit_DictComp = visit_comprehension_helper + def dfs_walk(node: ast.AST) -> Iterator[ast.AST]: """Helper to iterate over an AST depth-first.""" @@ -169,6 +195,21 @@ yield node +MAGIC_VARS = { + "index": {"i", "idx", "index"}, + "loop": {"line", "x", "l"}, + "input": {"lines", "stdin"}, +} + + +def is_magic_var(name: str) -> bool: + return any(name in vars for vars in MAGIC_VARS.values()) + + +class PypError(Exception): + pass + + def get_config_contents() -> str: """Returns the empty string if no config file is specified.""" config_file = os.environ.get("PYP_CONFIG_PATH") @@ -181,10 +222,6 @@ raise PypError(f"Config file not found at PYP_CONFIG_PATH={config_file}") from e -class PypError(Exception): - pass - - class PypConfig: """PypConfig is responsible for handling user configuration. @@ -210,6 +247,7 @@ self.parts: List[ast.stmt] = config_ast.body # Maps from a name to index of config part that defines it self.name_to_def: Dict[str, int] = {} + self.def_to_names: Dict[int, List[str]] = defaultdict(list) # Maps from index of config part to undefined names it needs self.requires: Dict[int, Set[str]] = defaultdict(set) # Modules from which automatic imports work without qualification, ordered by AST encounter @@ -218,7 +256,7 @@ self.shebang: str = "#!/usr/bin/env python3" if config_contents.startswith("#!"): self.shebang = "\n".join( - itertools.takewhile(lambda l: l.startswith("#"), config_contents.splitlines()) + itertools.takewhile(lambda line: line.startswith("#"), config_contents.splitlines()) ) top_level: Tuple[Any, ...] = (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef) @@ -236,7 +274,10 @@ for name in f.top_level_defined: if self.name_to_def.get(name, index) != index: raise PypError(f"Config has multiple definitions of {repr(name)}") + if is_magic_var(name): + raise PypError(f"Config cannot redefine built-in magic variable {repr(name)}") self.name_to_def[name] = index + self.def_to_names[index].append(name) self.requires[index] = f.undefined self.wildcard_imports.extend(f.wildcard_imports) @@ -276,6 +317,10 @@ self.defined: Set[str] = f.top_level_defined self.undefined: Set[str] = f.undefined self.wildcard_imports: List[str] = f.wildcard_imports + # We'll always use sys in ``build_input``, so add it to undefined. + # This lets config define it or lets us automatically import it later + # (If before defines it, we'll just let it override the import...) + self.undefined.add("sys") self.define_pypprint = define_pypprint self.config = config @@ -283,6 +328,51 @@ # The print statement ``build_output`` will add, if it determines it needs to. self.implicit_print: Optional[ast.Call] = None + def build_missing_config(self) -> None: + """Modifies the AST to define undefined names defined in config.""" + config_definitions: Set[str] = set() + attempt_to_define = set(self.undefined) + while attempt_to_define: + can_define = attempt_to_define & set(self.config.name_to_def) + # The things we can define might in turn require some definitions, so update the things + # we need to attempt to define and loop + attempt_to_define = set() + for name in can_define: + config_definitions.update(self.config.def_to_names[self.config.name_to_def[name]]) + attempt_to_define.update(self.config.requires[self.config.name_to_def[name]]) + # We don't need to attempt to define things we've already decided we need to define + attempt_to_define -= config_definitions + + config_indices = {self.config.name_to_def[name] for name in config_definitions} + + # Run basically the same thing in reverse to see which dependencies stem from magic vars + before_config_indices = set(config_indices) + derived_magic_indices = { + i for i in config_indices if any(map(is_magic_var, self.config.requires[i])) + } + derived_magic_names = set() + + while derived_magic_indices: + before_config_indices -= derived_magic_indices + derived_magic_names |= { + name for i in derived_magic_indices for name in self.config.def_to_names[i] + } + derived_magic_indices = { + i for i in before_config_indices if self.config.requires[i] & derived_magic_names + } + magic_config_indices = config_indices - before_config_indices + + before_config_defs = [self.config.parts[i] for i in sorted(before_config_indices)] + magic_config_defs = [self.config.parts[i] for i in sorted(magic_config_indices)] + + self.before_tree.body = before_config_defs + self.before_tree.body + self.tree.body = magic_config_defs + self.tree.body + + for i in config_indices: + self.undefined.update(self.config.requires[i]) + self.defined |= config_definitions + self.undefined -= config_definitions + def define(self, name: str) -> None: """Defines a name.""" self.defined.add(name) @@ -322,7 +412,7 @@ ) ): # ...then recursively look for a standalone expression - return inner(body[-1].body, use_pypprint) # type: ignore + return inner(body[-1].body, use_pypprint) return False if isinstance(body[-1].value, ast.Name): @@ -371,11 +461,6 @@ How we do this depends on which magic variables are used. """ - MAGIC_VARS = { - "index": {"i", "idx", "index"}, - "loop": {"line", "x", "l", "s"}, - "input": {"lines", "stdin"}, - } possible_vars = {typ: names & self.undefined for typ, names in MAGIC_VARS.items()} if (possible_vars["loop"] or possible_vars["index"]) and possible_vars["input"]: @@ -391,9 +476,6 @@ names_str = ", ".join(names) raise PypError(f"Multiple candidates for {typ} variable: {names_str}") - # We'll use sys here no matter what; add it to undefined so we import it later - self.undefined.add("sys") - if possible_vars["loop"] or possible_vars["index"]: # We'll loop over stdin and define loop / index variables idx_var = possible_vars["index"].pop() if possible_vars["index"] else None @@ -430,32 +512,12 @@ else: no_pipe_assertion = ast.parse( "assert sys.stdin.isatty() or not sys.stdin.read(), " - '''"The command doesn't process input, but input is present"''' + """"The command doesn't process input, but input is present. """ + '''Maybe you meant to use a magic variable like `stdin` or `x`?"''' ) self.tree.body = no_pipe_assertion.body + self.tree.body self.use_pypprint_for_implicit_print() - def build_missing_config(self) -> None: - """Modifies the AST to define undefined names defined in config.""" - config_definitions: Set[str] = set() - attempt_to_define = set(self.undefined) - while attempt_to_define: - can_define = attempt_to_define & set(self.config.name_to_def) - config_definitions.update(can_define) - # The things we can define might in turn require some definitions, so update the things - # we need to attempt to define and loop - attempt_to_define = set() - for name in can_define: - attempt_to_define.update(self.config.requires[self.config.name_to_def[name]]) - # We don't need to attempt to define things we've already decided we need to define - attempt_to_define -= config_definitions - - config_indices = sorted({self.config.name_to_def[name] for name in config_definitions}) - self.before_tree.body = [ - self.config.parts[i] for i in config_indices - ] + self.before_tree.body - self.undefined -= config_definitions - def build_missing_imports(self) -> None: """Modifies the AST to import undefined names.""" self.undefined -= set(dir(__import__("builtins"))) @@ -506,9 +568,9 @@ def build(self) -> ast.Module: """Returns a transformed AST.""" + self.build_missing_config() self.build_output() self.build_input() - self.build_missing_config() self.build_missing_imports() ret = ast.parse("") @@ -519,6 +581,8 @@ if isinstance(node, ast.stmt): i += 1 node.lineno = i + if sys.version_info >= (3, 8): + node.end_lineno = i return ast.fix_missing_locations(ret) @@ -533,8 +597,14 @@ return cast(str, astunparse.unparse(tree)) except ImportError: pass - if short_fallback: - return f"# {ast.dump(tree)} # --explain has instructions to make this readable" + return ( + fallback_unparse(tree) + if not short_fallback + else f"# {ast.dump(tree)} # --explain has instructions to make this readable" + ) + + +def fallback_unparse(tree: ast.AST) -> str: return f""" from ast import * tree = fix_missing_locations({ast.dump(tree)}) @@ -558,6 +628,8 @@ try: exec(compile(tree, filename="<pyp>", mode="exec"), {}) except Exception as e: + # On error, reconstruct a traceback into the generated code + # Also add some diagnostics for ModuleNotFoundError and NameError try: line_to_node: Dict[int, ast.AST] = {} for node in dfs_walk(tree): @@ -577,6 +649,8 @@ ) for fs in tb_except.stack: if fs.filename == "<pyp>": + if fs.lineno is None: + raise AssertionError("When would this happen?") fs._line = code_for_line(fs.lineno) # type: ignore[attr-defined] fs.lineno = "PYP_REDACTED" # type: ignore[assignment] @@ -617,7 +691,7 @@ "Easily run Python at the shell!\n\n" "For help and examples, see https://github.com/hauntsaninja/pyp\n\n" "Cheatsheet:\n" - "- Use `line`, `x`, `l`, or `s` for a line in the input. Use `i`, `idx` or `index` " + "- Use `x`, `l` or `line` for a line in the input. Use `i`, `idx` or `index` " "for the index\n" "- Use `lines` to get a list of rstripped lines\n" "- Use `stdin` to get sys.stdin\n" diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/pyp-0.3.4/pyproject.toml new/pyp-1.1.0/pyproject.toml --- old/pyp-0.3.4/pyproject.toml 2021-09-09 03:44:01.000000000 +0200 +++ new/pyp-1.1.0/pyproject.toml 2023-01-12 10:45:46.000000000 +0100 @@ -1,2 +1,37 @@ +[project] +name = "pypyp" +version = "1.1.0" +authors = [{name = "Shantanu Jain"}, {email = "hauntsaninja@gmail.com"}] +description = "Easily run Python at the shell! Magical, but never mysterious." +readme = "README.md" +license = {file = "LICENSE"} +classifiers = [ + "Environment :: Console", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Topic :: Software Development", + "Topic :: Utilities", +] +requires-python = ">=3.6" +dependencies = ["astunparse; python_version<'3.9'"] + +[project.scripts] +pyp = "pyp:main" + +[project.urls] +homepage = "https://github.com/hauntsaninja/pyp" +repository = "https://github.com/hauntsaninja/pyp" +changelog = "https://github.com/hauntsaninja/pyp/blob/master/CHANGELOG.md" + +[tool.flit.module] +name = "pyp" + +[build-system] +requires = ["flit_core>=3.4"] +build-backend = "flit_core.buildapi" + [tool.black] line-length = 100 +skip-magic-trailing-comma = true diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/pyp-0.3.4/setup.py new/pyp-1.1.0/setup.py --- old/pyp-0.3.4/setup.py 2021-09-09 03:44:01.000000000 +0200 +++ new/pyp-1.1.0/setup.py 1970-01-01 01:00:00.000000000 +0100 @@ -1,28 +0,0 @@ -from setuptools import setup - -with open("README.md", "r") as f: - long_description = f.read() - -setup( - name="pypyp", - version="0.3.4", - author="Shantanu Jain", - author_email="hauntsaninja@gmail.com", - description="Easily run Python at the shell! Magical, but never mysterious.", - long_description=long_description, - long_description_content_type="text/markdown", - url="https://github.com/hauntsaninja/pyp", - classifiers=[ - "Environment :: Console", - "Intended Audience :: Developers", - "License :: OSI Approved :: MIT License", - "Operating System :: OS Independent", - "Programming Language :: Python :: 3", - "Topic :: Software Development", - "Topic :: Utilities", - ], - py_modules=["pyp"], - entry_points={"console_scripts": ["pyp=pyp:main"]}, - install_requires=["astunparse; python_version<'3.9'"], - python_requires=">=3.6", -) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/pyp-0.3.4/tests/test_find_names.py new/pyp-1.1.0/tests/test_find_names.py --- old/pyp-0.3.4/tests/test_find_names.py 2021-09-09 03:44:01.000000000 +0200 +++ new/pyp-1.1.0/tests/test_find_names.py 2023-01-12 10:45:46.000000000 +0100 @@ -125,10 +125,21 @@ check_find_names("f((f := lambda x: x))", {"f"}, {"f"}) check_find_names("f((f := lambda x: (x, y)))", {"f"}, {"f", "y"}) check_find_names("if (x := 1): print(x)", {"x"}, {"print"}) - check_find_names("(y for x in xx if (y := x) == 'foo')", {"x", "y"}, {"xx"}) check_find_names("x: (x := 1) = 2", {"x"}, set()) check_find_names("f'{(x := 1)} {x}'", {"x"}, set()) check_find_names("class A((A := object)): ...", {"A"}, {"object"}) + + check_find_names("[(y := x) for x in xx]", {"y"}, {"xx"}) + check_find_names("(y for x in xx if (y := x) == 'foo')", {"y"}, {"xx"}) + check_find_names("[[(y := z) for z in x] for x in xx]", {"y"}, {"xx"}) + check_find_names("[[[(y := z) for z in x] for x in xx] for x in xx]", {"y"}, {"xx"}) + check_find_names("(lambda: [[(y := z) for z in x] for x in xx])()", set(), {"xx"}) + check_find_names("[lambda: [[(y := z) for z in x] for x in xx] for x in xx]", set(), {"xx"}) + check_find_names("[(lambda a=(x := 5): a) for _ in range(5)]", {"x"}, {"range"}) + + check_find_names("{(x := y): (y := 1) for _ in range(5)}", {"x", "y"}, {"y", "range"}) + check_find_names("{(x := 1): (y := x) for _ in range(5)}", {"x", "y"}, {"range"}) + if sys.version_info >= (3, 9): check_find_names( "d1 = lambda i: i\n@(d2 := d1)\n@(d3 := d2)\ndef f(): ...", @@ -144,14 +155,14 @@ def test_comprehensions(): - check_find_names("(x for x in y)", {"x"}, {"y"}) - check_find_names("(x for x in x)", {"x"}, {"x"}) - check_find_names("(x for xx in xxx for x in xx)", {"x", "xx"}, {"xxx"}) - check_find_names("(x for x in xx for xx in xxx)", {"x", "xx"}, {"xx", "xxx"}) - check_find_names("(x for x in xx if x > 0)", {"x"}, {"xx"}) - check_find_names("[x for x in xx if x > 0]", {"x"}, {"xx"}) - check_find_names("{x for x in xx if x > 0}", {"x"}, {"xx"}) - check_find_names("{x: x for x in xx if x > 0}", {"x"}, {"xx"}) + check_find_names("(x for x in y)", set(), {"y"}) + check_find_names("(x for x in x)", set(), {"x"}) + check_find_names("(x for xx in xxx for x in xx)", set(), {"xxx"}) + check_find_names("(x for x in xx for xx in xxx)", set(), {"xx", "xxx"}) + check_find_names("(x for x in xx if x > 0)", set(), {"xx"}) + check_find_names("[x for x in xx if x > 0]", set(), {"xx"}) + check_find_names("{x for x in xx if x > 0}", set(), {"xx"}) + check_find_names("{x: x for x in xx if x > 0}", set(), {"xx"}) def test_args(): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/pyp-0.3.4/tests/test_pyp.py new/pyp-1.1.0/tests/test_pyp.py --- old/pyp-0.3.4/tests/test_pyp.py 2021-09-09 03:44:01.000000000 +0200 +++ new/pyp-1.1.0/tests/test_pyp.py 2023-01-12 10:45:46.000000000 +0100 @@ -145,7 +145,7 @@ run_pyp("pyp 'print(x); print(len(lines))'") with pytest.raises(pyp.PypError, match="Multiple candidates for loop"): - run_pyp("pyp 'print(x); print(s)'") + run_pyp("pyp 'print(x); print(l)'") with pytest.raises(pyp.PypError, match="Multiple candidates for input"): run_pyp("pyp 'stdin; lines'") @@ -198,15 +198,21 @@ # Check the entire output, end to end pyp_error = run_cmd("pyp 'def f(): 1/0' 'f()'", check=False) - message = lambda x, y: ( # noqa - "error: Code raised the following exception, consider using --explain to investigate:\n\n" - "Possible reconstructed traceback (most recent call last):\n" - ' File "<pyp>", in <module>\n' - " output = f()\n" - ' File "<pyp>", in f\n' - f" {x}1 / 0{y}\n" - "ZeroDivisionError: division by zero\n" + message = lambda po, pc: ( # noqa + ( + "error: Code raised the following exception, " + "consider using --explain to investigate:\n\n" + "Possible reconstructed traceback (most recent call last):\n" + ' File "<pyp>", in <module>\n' + " output = f()\n" + ) + + (" ^^^^^^^^^^^\n" if sys.version_info >= (3, 11) else "") + + (' File "<pyp>", in f\n' f" {po}1 / 0{pc}\n") + + (" \n" if sys.version_info >= (3, 11) else "") + + ("ZeroDivisionError: division by zero\n") ) + print(repr(pyp_error)) + print(repr(message("", ""))) assert pyp_error == message("(", ")") or pyp_error == message("", "") # Test tracebacks involving statements with nested child statements @@ -219,7 +225,9 @@ "pyp --explain -b 'd = defaultdict(list)' 'user, pid, *_ = x.split()' " """'d[user].append(pid)' -a 'del d["root"]' -a d""" ) - script = r""" + po = "" if sys.version_info >= (3, 11) else "(" + pc = "" if sys.version_info >= (3, 11) else ")" + script = rf""" #!/usr/bin/env python3 from collections import defaultdict import sys @@ -227,7 +235,7 @@ d = defaultdict(list) for x in sys.stdin: x = x.rstrip('\n') - (user, pid, *_) = x.split() + {po}user, pid, *_{pc} = x.split() d[user].append(pid) del d['root'] if d is not None: @@ -277,7 +285,7 @@ #!/usr/bin/env python3 from shlex import split import sys -assert sys.stdin.isatty() or not sys.stdin.read(), "The command doesn't process input, but input is present" +assert sys.stdin.isatty() or not sys.stdin.read(), "The command doesn't process input, but input is present. Maybe you meant to use a magic variable like `stdin` or `x`?" from shlex import * split """ # noqa @@ -287,7 +295,7 @@ #!/usr/bin/env python3 from shlex import split import sys -assert sys.stdin.isatty() or not sys.stdin.read(), "The command doesn't process input, but input is present" +assert sys.stdin.isatty() or not sys.stdin.read(), "The command doesn't process input, but input is present. Maybe you meant to use a magic variable like `stdin` or `x`?" from os.path import * from shlex import * split @@ -298,6 +306,16 @@ ) +def test_fallback_unparse(): + original_code = """ +x = 2 + 3 +x = x * x +print((lambda: x)()) +""" + code = pyp.fallback_unparse(ast.parse(original_code)) + assert subprocess.check_output([sys.executable, "-c", code]).decode().strip() == "25" + + # ==================== # Config tests # ==================== @@ -329,7 +347,7 @@ import sys import numpy as np from scipy.linalg import eigvals -assert sys.stdin.isatty() or not sys.stdin.read(), "The command doesn't process input, but input is present" +assert sys.stdin.isatty() or not sys.stdin.read(), "The command doesn't process input, but input is present. Maybe you meant to use a magic variable like `stdin` or `x`?" eigvals(np.array([[0.0, -1.0], [1.0, 0.0]])) """ # noqa compare_scripts( @@ -363,6 +381,98 @@ """ compare_scripts(run_pyp(["--explain", "stdin; smallarray(); pass"]), script4) + # test using wildcard imports in config + config_mock.return_value = """ +from typing import * +any = Any + """ + script5 = """ +#!/usr/bin/env python3 +from typing import Any +import sys +any = Any +stdin = sys.stdin +stdin +any +""" + compare_scripts(run_pyp(["--explain", "stdin; any; pass"]), script5) + + +@patch("pyp.get_config_contents") +def test_config_magic_vars(config_mock): + config_mock.return_value = "n = int(x)\nj = json.loads(stdin)\ndef upfront(): pass" + + script1 = """ +#!/usr/bin/env python3 +import json +import sys +from pyp import pypprint +stdin = sys.stdin +j = json.loads(stdin) +output = j[0] +if output is not None: + pypprint(output) +""" + compare_scripts(run_pyp(["--explain", "j[0]"]), script1) + + script2 = r""" +#!/usr/bin/env python3 +import sys +for x in sys.stdin: + x = x.rstrip('\n') + n = int(x) + if n is not None: + print(n) +""" + compare_scripts(run_pyp(["--explain", "n"]), script2) + + config_mock.return_value = """ +f = lambda x: x +n = int(x) +o = f(n) + 1 +p = f(o) + 3 +q = f(p) + 5 +""" + assert run_pyp("p", input="0\n7") == "4\n11\n" + assert run_pyp("q", input="0\n7") == "9\n16\n" + + script3 = r""" +#!/usr/bin/env python3 +import sys +f = lambda x: x +for x in sys.stdin: + x = x.rstrip('\n') + n = int(x) + o = f(n) + 1 + p = f(o) + 3 + q = f(p) + 5 + if q is not None: + print(q) +""" + compare_scripts(run_pyp(["--explain", "q"]), script3) + + config_mock.return_value = """ +ilines = (z.rstrip() for z in stdin) +class Indexable: + ... +idxgen = Indexable(ilines) +""" + script4 = r""" +#!/usr/bin/env python3 +import sys +from pyp import pypprint + +class Indexable: + ... +stdin = sys.stdin +ilines = (z.rstrip() for z in stdin) +idxgen = Indexable(ilines) +output = idxgen[1] +if output is not None: + pypprint(output) +""" + compare_scripts(run_pyp(["--explain", "idxgen[1]"]), script4) + @patch("pyp.get_config_contents") def test_config_invalid(config_mock): @@ -388,6 +498,19 @@ run_pyp("missing") assert isinstance(e.value.__cause__, ImportError) + config_mock.return_value = "x = 8" + with pytest.raises(pyp.PypError, match=r"Config.*cannot redefine built-in.*'x'"): + run_pyp("x") + + config_mock.return_value = "stdin = 5" + with pytest.raises(pyp.PypError, match=r"Config.*cannot redefine built-in.*'stdin'"): + run_pyp("type(stdin).__name__") + + # See test_config_scope for more + config_mock.return_value = "def f(x): stdin = 5" + run_pyp("x") + run_pyp("stdin") + @patch("pyp.get_config_contents") def test_config_shebang(config_mock): @@ -412,11 +535,39 @@ @patch("pyp.get_config_contents") +def test_config_automatic_import(config_mock): + config_mock.return_value = "j = json" + script1 = """ +#!/usr/bin/env python3 +import json +import sys +j = json +assert sys.stdin.isatty() or not sys.stdin.read(), "The command doesn't process input, but input is present. Maybe you meant to use a magic variable like `stdin` or `x`?" +j +""" # noqa + compare_scripts(run_pyp(["--explain", "j; pass"]), script1) + + config_mock.return_value = "from typing import *\nL = List" + script2 = """ +#!/usr/bin/env python3 +from typing import List +import sys +L = List +assert sys.stdin.isatty() or not sys.stdin.read(), "The command doesn't process input, but input is present. Maybe you meant to use a magic variable like `stdin` or `x`?" +L +""" # noqa + compare_scripts(run_pyp(["--explain", "L; pass"]), script2) + + +@patch("pyp.get_config_contents") def test_config_scope(config_mock): config_mock.return_value = """ -def f(x): contextlib = 5 -class A: - def asyncio(self): ... +def f(x, stdin, asyncio): + contextlib = 5 + import asyncio +class A(asyncio): + contextlib = 55 + def asyncio(self, asyncio): ... """ script = """ #!/usr/bin/env python3 @@ -437,11 +588,10 @@ config_mock.return_value = "range = 5" assert run_pyp("print(range)") == "5\n" - # shadowing a magic variable - config_mock.return_value = "stdin = 5" - assert run_pyp("type(stdin).__name__") == "StringIO\n" - config_mock.return_value = "x = 8" - assert run_pyp("x") == "" + # shadowing print + config_mock.return_value = "print = lambda p: p" + assert run_pyp("x", input="9") == "9\n" + assert run_pyp("print(x)", input="9") == "" # shadowing a wildcard import config_mock.return_value = "from typing import *\nList = 5" @@ -457,6 +607,12 @@ @patch("pyp.get_config_contents") +def test_config_automatic_print(config_mock): + config_mock.return_value = "def tnirp(p): print(''.join(reversed(p)))" + assert run_pyp("tnirp(x)", input="tnirp") == "print\n" + + +@patch("pyp.get_config_contents") def test_config_recursive(config_mock): config_mock.return_value = "def f(x): return g(x)\ndef g(x): return f(x)" script = """ @@ -490,7 +646,7 @@ import ast import sys {if_block} -assert sys.stdin.isatty() or not sys.stdin.read(), "The command doesn't process input, but input is present" +assert sys.stdin.isatty() or not sys.stdin.read(), "The command doesn't process input, but input is present. Maybe you meant to use a magic variable like `stdin` or `x`?" unparse(ast.parse('x')) """ # noqa compare_scripts(run_pyp(["--explain", "unparse(ast.parse('x')); pass"]), script1) @@ -508,11 +664,14 @@ import sys import ast {except_block} -assert sys.stdin.isatty() or not sys.stdin.read(), "The command doesn't process input, but input is present" +assert sys.stdin.isatty() or not sys.stdin.read(), "The command doesn't process input, but input is present. Maybe you meant to use a magic variable like `stdin` or `x`?" unparse(ast.parse('x')) """ # noqa compare_scripts(run_pyp(["--explain", "unparse(ast.parse('x')); pass"]), script2) + config_mock.return_value = "foo = False\nif foo: y = 5\nelse: y = 10" + assert run_pyp("y") == "10\n" + @pytest.mark.xfail(reason="We don't currently support this") @patch("pyp.get_config_contents") @@ -534,7 +693,7 @@ import sys import ast {except_block} -assert sys.stdin.isatty() or not sys.stdin.read(), "The command doesn't process input, but input is present" +assert sys.stdin.isatty() or not sys.stdin.read(), "The command doesn't process input, but input is present. Maybe you meant to use a magic variable like `stdin` or `x`?" unparse(ast.parse('x')) """ # noqa compare_scripts(run_pyp(["--explain", "unparse(ast.parse('x')); pass"]), script3) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/pyp-0.3.4/tox.ini new/pyp-1.1.0/tox.ini --- old/pyp-0.3.4/tox.ini 2021-09-09 03:44:01.000000000 +0200 +++ new/pyp-1.1.0/tox.ini 2023-01-12 10:45:46.000000000 +0100 @@ -1,12 +1,12 @@ [tox] skipsdist = True -envlist = py36, py39, lint, mypy +envlist = py39, py311, lint, mypy [testenv] deps = pytest commands = pip install -e . - pytest + pytest {posargs} [testenv:lint] deps = @@ -20,12 +20,15 @@ isort --diff --check --quiet . [testenv:mypy] -deps = mypy -commands = mypy --strict -m pyp +deps = mypy>=0.991 +commands = + mypy --strict -m pyp --python-version 3.6 + mypy --strict -m pyp --python-version 3.11 [coverage:report] exclude_lines = - def unparse + raise AssertionError + def unparse\( if __name__ == "__main__": [testenv:coverage]