From 7876a87a2da8eb66504a34c36aa49f5c23148046 Mon Sep 17 00:00:00 2001 From: Alexander Loechel Date: Sat, 18 Oct 2025 11:31:30 +0300 Subject: [PATCH 01/40] Type Annotations for RestrictedPython --- .meta.toml | 2 +- setup.py | 8 +- src/RestrictedPython/Utilities.py | 4 +- src/RestrictedPython/_compat.py | 2 + src/RestrictedPython/compile.py | 72 ++++---- src/RestrictedPython/transformer.py | 248 +++++++++++++++------------- tox.ini | 4 +- 7 files changed, 184 insertions(+), 156 deletions(-) diff --git a/.meta.toml b/.meta.toml index c5b68d7f..3ba9ecbd 100644 --- a/.meta.toml +++ b/.meta.toml @@ -47,7 +47,7 @@ testenv-additional = [ " coverage combine", " coverage html", " coverage report -m --fail-under=100", - "depends = py39,py310,py311,py311-datetime,py312,py313,py314,coverage", + "depends = py39,py310,py311,py312,py313,py314,py314-datetime,coverage", ] coverage-command = "pytest --cov=src --cov=tests --cov-report= tests {posargs}" coverage-setenv = [ diff --git a/setup.py b/setup.py index 45bf79d3..a7d2ec2b 100644 --- a/setup.py +++ b/setup.py @@ -39,17 +39,20 @@ def read(*rnames): 'Programming Language :: Python', 'Operating System :: OS Independent', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', 'Programming Language :: Python :: 3.12', 'Programming Language :: Python :: 3.13', + 'Programming Language :: Python :: 3.14', 'Programming Language :: Python :: Implementation :: CPython', 'Topic :: Security', + 'Typing :: Typed', ], keywords='restricted execution security untrusted code', author='Zope Foundation and Contributors', author_email='zope-dev@zope.dev', + maintainer='Zope Foundation, Plone Foundation and Contributors', + maintainer_email='security@plone.org', project_urls={ "Documentation": "https://restrictedpython.readthedocs.io/", "Source": "https://github.com/zopefoundation/RestrictedPython", @@ -59,9 +62,10 @@ def read(*rnames): packages=find_packages('src'), package_dir={'': 'src'}, install_requires=[], - python_requires=">=3.9, <3.15", + python_requires=">=3.10, <3.15", extras_require={ 'test': ['pytest', 'pytest-mock'], + 'typecheck': ['mypy', 'typeshed'], 'docs': ['Sphinx', 'furo'], }, include_package_data=True, diff --git a/src/RestrictedPython/Utilities.py b/src/RestrictedPython/Utilities.py index 26d73d15..e07f8e49 100644 --- a/src/RestrictedPython/Utilities.py +++ b/src/RestrictedPython/Utilities.py @@ -14,6 +14,7 @@ import math import random import string +from collections.abc import Iterable utility_builtins = {} @@ -75,7 +76,8 @@ def test(*args): utility_builtins['test'] = test -def reorder(s, with_=None, without=()): +def reorder(s: Iterable, with_: Iterable | None = None, + without: Iterable | None = None) -> Iterable: # s, with_, and without are sequences treated as sets. # The result is subtract(intersect(s, with_), without), # unless with_ is None, in which case it is subtract(s, without). diff --git a/src/RestrictedPython/_compat.py b/src/RestrictedPython/_compat.py index 2d85cc46..2c5641e2 100644 --- a/src/RestrictedPython/_compat.py +++ b/src/RestrictedPython/_compat.py @@ -6,5 +6,7 @@ IS_PY310_OR_GREATER = _version.major == 3 and _version.minor >= 10 IS_PY311_OR_GREATER = _version.major == 3 and _version.minor >= 11 IS_PY312_OR_GREATER = _version.major == 3 and _version.minor >= 12 +IS_PY313_OR_GREATER = _version.major == 3 and _version.minor >= 13 +IS_PY314_OR_GREATER = _version.major == 3 and _version.minor >= 14 IS_CPYTHON = platform.python_implementation() == 'CPython' diff --git a/src/RestrictedPython/compile.py b/src/RestrictedPython/compile.py index 3253b8c9..0172023a 100644 --- a/src/RestrictedPython/compile.py +++ b/src/RestrictedPython/compile.py @@ -1,11 +1,19 @@ import ast import warnings from collections import namedtuple +from os import PathLike +from typing import Any +from typing import Literal +from typing import TypeAlias from RestrictedPython._compat import IS_CPYTHON from RestrictedPython.transformer import RestrictingNodeTransformer +# Temporary workaround for missing _typeshed +ReadableBuffer: TypeAlias = bytes | bytearray + + CompileResult = namedtuple( 'CompileResult', 'code, errors, warnings, used_names') syntax_error_template = ( @@ -18,12 +26,12 @@ def _compile_restricted_mode( - source, - filename='', - mode="exec", - flags=0, - dont_inherit=False, - policy=RestrictingNodeTransformer): + source: str | ReadableBuffer | ast.Module | ast.Expression | ast.Interactive, + filename: str | ReadableBuffer | PathLike[Any] = '', + mode: Literal["exec", "eval", "single"] = "exec", + flags: int = 0, + dont_inherit: bool = False, + policy: ast.NodeTransformer = RestrictingNodeTransformer) -> CompileResult: if not IS_CPYTHON: warnings.warn_explicit( @@ -78,11 +86,11 @@ def _compile_restricted_mode( def compile_restricted_exec( - source, - filename='', - flags=0, - dont_inherit=False, - policy=RestrictingNodeTransformer): + source: str | ReadableBuffer | ast.Module | ast.Expression | ast.Interactive, + filename: str | ReadableBuffer | PathLike[Any] = '', + flags: int = 0, + dont_inherit: bool = False, + policy: ast.NodeTransformer = RestrictingNodeTransformer) -> CompileResult: """Compile restricted for the mode `exec`.""" return _compile_restricted_mode( source, @@ -94,11 +102,11 @@ def compile_restricted_exec( def compile_restricted_eval( - source, - filename='', - flags=0, - dont_inherit=False, - policy=RestrictingNodeTransformer): + source: str | ReadableBuffer | ast.Module | ast.Expression | ast.Interactive, + filename: str | ReadableBuffer | PathLike[Any] = '', + flags: int = 0, + dont_inherit: bool = False, + policy: ast.NodeTransformer = RestrictingNodeTransformer) -> CompileResult: """Compile restricted for the mode `eval`.""" return _compile_restricted_mode( source, @@ -110,11 +118,11 @@ def compile_restricted_eval( def compile_restricted_single( - source, - filename='', - flags=0, - dont_inherit=False, - policy=RestrictingNodeTransformer): + source: str | ReadableBuffer | ast.Module | ast.Expression | ast.Interactive, + filename: str | ReadableBuffer | PathLike[Any] = '', + flags: int = 0, + dont_inherit: bool = False, + policy: ast.NodeTransformer = RestrictingNodeTransformer) -> CompileResult: """Compile restricted for the mode `single`.""" return _compile_restricted_mode( source, @@ -128,12 +136,12 @@ def compile_restricted_single( def compile_restricted_function( p, # parameters body, - name, - filename='', + name: str, + filename: str | ReadableBuffer | PathLike[Any] = '', globalize=None, # List of globals (e.g. ['here', 'context', ...]) - flags=0, - dont_inherit=False, - policy=RestrictingNodeTransformer): + flags: int = 0, + dont_inherit: bool = False, + policy: ast.NodeTransformer = RestrictingNodeTransformer) -> CompileResult: """Compile a restricted code object for a function. Documentation see: @@ -181,12 +189,12 @@ def compile_restricted_function( def compile_restricted( - source, - filename='', - mode='exec', - flags=0, - dont_inherit=False, - policy=RestrictingNodeTransformer): + source: str | ReadableBuffer | ast.Module | ast.Expression | ast.Interactive, + filename: str | ReadableBuffer | PathLike[Any] = '', + mode: str = 'exec', + flags: int = 0, + dont_inherit: bool = False, + policy: ast.NodeTransformer = RestrictingNodeTransformer) -> CompileResult: """Replacement for the built-in compile() function. policy ... `ast.NodeTransformer` class defining the restrictions. diff --git a/src/RestrictedPython/transformer.py b/src/RestrictedPython/transformer.py index 7fd30164..5ecc5f63 100644 --- a/src/RestrictedPython/transformer.py +++ b/src/RestrictedPython/transformer.py @@ -113,7 +113,7 @@ # When new ast nodes are generated they have no 'lineno', 'end_lineno', # 'col_offset' and 'end_col_offset'. This function copies these fields from the # incoming node: -def copy_locations(new_node, old_node): +def copy_locations(new_node: ast.AST, old_node: ast.AST) -> None: assert 'lineno' in new_node._attributes new_node.lineno = old_node.lineno @@ -151,7 +151,11 @@ def new_print_scope(self): class RestrictingNodeTransformer(ast.NodeTransformer): - def __init__(self, errors=None, warnings=None, used_names=None): + def __init__(self, + errors: list[str] | None = None, + warnings: list[str] | None = None, + used_names: dict[str, + str] | None = None): super().__init__() self.errors = [] if errors is None else errors self.warnings = [] if warnings is None else warnings @@ -168,26 +172,26 @@ def __init__(self, errors=None, warnings=None, used_names=None): self.print_info = PrintInfo() - def gen_tmp_name(self): + def gen_tmp_name(self) -> str: # 'check_name' ensures that no variable is prefixed with '_'. # => Its safe to use '_tmp..' as a temporary variable. name = '_tmp%i' % self._tmp_idx self._tmp_idx += 1 return name - def error(self, node, info): + def error(self, node: ast.AST, info: str) -> None: """Record a security error discovered during transformation.""" lineno = getattr(node, 'lineno', None) self.errors.append( f'Line {lineno}: {info}') - def warn(self, node, info): - """Record a security error discovered during transformation.""" + def warn(self, node: ast.AST, info: str) -> None: + """Record a security warning discovered during transformation.""" lineno = getattr(node, 'lineno', None) self.warnings.append( f'Line {lineno}: {info}') - def guard_iter(self, node): + def guard_iter(self, node: ast.AST) -> ast.AST: """ Converts: for x in expr @@ -218,10 +222,10 @@ def guard_iter(self, node): node.iter = new_iter return node - def is_starred(self, ob): + def is_starred(self, ob: ast.AST) -> bool: return isinstance(ob, ast.Starred) - def gen_unpack_spec(self, tpl): + def gen_unpack_spec(self, tpl: ast.Tuple) -> ast.Dict: """Generate a specification for 'guarded_unpack_sequence'. This spec is used to protect sequence unpacking. @@ -297,14 +301,18 @@ def gen_unpack_spec(self, tpl): return spec - def protect_unpack_sequence(self, target, value): + def protect_unpack_sequence( + self, + target: ast.Tuple, + value: ast.AST) -> ast.Call: spec = self.gen_unpack_spec(target) return ast.Call( func=ast.Name('_unpack_sequence_', ast.Load()), args=[value, spec, ast.Name('_getiter_', ast.Load())], keywords=[]) - def gen_unpack_wrapper(self, node, target): + def gen_unpack_wrapper(self, node: ast.AST, + target: ast.Tuple) -> tuple[ast.Name, ast.Try]: """Helper function to protect tuple unpacks. node: used to copy the locations for the new nodes. @@ -353,13 +361,13 @@ def gen_unpack_wrapper(self, node, target): return (tmp_target, cleanup) - def gen_none_node(self): + def gen_none_node(self) -> ast.NameConstant: return ast.NameConstant(value=None) - def gen_del_stmt(self, name_to_del): + def gen_del_stmt(self, name_to_del: str) -> ast.Delete: return ast.Delete(targets=[ast.Name(name_to_del, ast.Del())]) - def transform_slice(self, slice_): + def transform_slice(self, slice_: ast.AST) -> ast.AST: """Transform slices into function parameters. ast.Slice nodes are only allowed within a ast.Subscript node. @@ -398,7 +406,7 @@ def transform_slice(self, slice_): args=args, keywords=[]) - elif isinstance(slice_, ast.ExtSlice): + elif isinstance(slice_, ast.Tuple): dims = ast.Tuple([], ast.Load()) for item in slice_.dims: dims.elts.append(self.transform_slice(item)) @@ -408,7 +416,11 @@ def transform_slice(self, slice_): # Index, Slice and ExtSlice are only defined Slice types. raise NotImplementedError(f"Unknown slice type: {slice_}") - def check_name(self, node, name, allow_magic_methods=False): + def check_name( + self, + node: ast.AST, + name: str, + allow_magic_methods: bool = False) -> None: """Check names if they are allowed. If ``allow_magic_methods is True`` names in `ALLOWED_FUNC_NAMES` @@ -433,7 +445,7 @@ def check_name(self, node, name, allow_magic_methods=False): elif name in FORBIDDEN_FUNC_NAMES: self.error(node, f'"{name}" is a reserved name.') - def check_function_argument_names(self, node): + def check_function_argument_names(self, node: ast.FunctionDef) -> None: for arg in node.args.args: self.check_name(node, arg.arg) @@ -446,7 +458,7 @@ def check_function_argument_names(self, node): for arg in node.args.kwonlyargs: self.check_name(node, arg.arg) - def check_import_names(self, node): + def check_import_names(self, node: ast.ImportFrom | ast.Import) -> ast.AST: """Check the names being imported. This is a protection against rebinding dunder names like @@ -463,7 +475,7 @@ def check_import_names(self, node): return self.node_contents_visit(node) - def inject_print_collector(self, node, position=0): + def inject_print_collector(self, node: ast.AST, position: int = 0) -> None: print_used = self.print_info.print_used printed_used = self.print_info.printed_used @@ -496,7 +508,7 @@ def inject_print_collector(self, node, position=0): # Special Functions for an ast.NodeTransformer - def generic_visit(self, node): + def generic_visit(self, node: ast.AST) -> ast.AST: """Reject ast nodes which do not have a corresponding `visit_` method. This is needed to prevent new ast nodes from new Python versions to be @@ -511,18 +523,18 @@ def generic_visit(self, node): ) self.not_allowed(node) - def not_allowed(self, node): + def not_allowed(self, node: ast.AST) -> None: self.error( node, f'{node.__class__.__name__} statements are not allowed.') - def node_contents_visit(self, node): + def node_contents_visit(self, node: ast.AST) -> ast.AST: """Visit the contents of a node.""" return super().generic_visit(node) # ast for Literals - def visit_Constant(self, node): + def visit_Constant(self, node: ast.Constant) -> ast.Constant | None: """Allow constant literals with restriction for Ellipsis. Constant replaces Num, Str, Bytes, NameConstant and Ellipsis in @@ -539,37 +551,37 @@ def visit_Constant(self, node): return return self.node_contents_visit(node) - def visit_Interactive(self, node): + def visit_Interactive(self, node: ast.Interactive) -> ast.AST: """Allow single mode without restrictions.""" return self.node_contents_visit(node) - def visit_List(self, node): + def visit_List(self, node: ast.List) -> ast.AST: """Allow list literals without restrictions.""" return self.node_contents_visit(node) - def visit_Tuple(self, node): + def visit_Tuple(self, node: ast.Tuple) -> ast.AST: """Allow tuple literals without restrictions.""" return self.node_contents_visit(node) - def visit_Set(self, node): + def visit_Set(self, node: ast.Set) -> ast.AST: """Allow set literals without restrictions.""" return self.node_contents_visit(node) - def visit_Dict(self, node): + def visit_Dict(self, node: ast.Dict) -> ast.AST: """Allow dict literals without restrictions.""" return self.node_contents_visit(node) - def visit_FormattedValue(self, node): + def visit_FormattedValue(self, node: ast.FormattedValue) -> ast.AST: """Allow f-strings without restrictions.""" return self.node_contents_visit(node) - def visit_JoinedStr(self, node): + def visit_JoinedStr(self, node: ast.JoinedStr) -> ast.AST: """Allow joined string without restrictions.""" return self.node_contents_visit(node) # ast for Variables - def visit_Name(self, node): + def visit_Name(self, node: ast.Name) -> ast.Name | None: """Prevents access to protected names. Converts use of the name 'printed' to this expression: '_print()' @@ -603,25 +615,25 @@ def visit_Name(self, node): self.check_name(node, node.id) return node - def visit_Load(self, node): + def visit_Load(self, node: ast.Load) -> ast.Load | None: """ """ return self.node_contents_visit(node) - def visit_Store(self, node): + def visit_Store(self, node: ast.Store) -> ast.AST: """ """ return self.node_contents_visit(node) - def visit_Del(self, node): + def visit_Del(self, node: ast.Del) -> ast.Del: """ """ return self.node_contents_visit(node) - def visit_Starred(self, node): + def visit_Starred(self, node: ast.Starred) -> ast.AST: """ """ @@ -629,18 +641,18 @@ def visit_Starred(self, node): # Expressions - def visit_Expression(self, node): + def visit_Expression(self, node: ast.Expression) -> ast.AST: """Allow Expression statements without restrictions. They are in the AST when using the `eval` compile mode. """ return self.node_contents_visit(node) - def visit_Expr(self, node): + def visit_Expr(self, node: ast.Expr) -> ast.AST: """Allow Expr statements (any expression) without restrictions.""" return self.node_contents_visit(node) - def visit_UnaryOp(self, node): + def visit_UnaryOp(self, node: ast.UnaryOp) -> ast.AST: """ UnaryOp (Unary Operations) is the overall element for: * Not --> which should be allowed @@ -649,135 +661,135 @@ def visit_UnaryOp(self, node): """ return self.node_contents_visit(node) - def visit_UAdd(self, node): + def visit_UAdd(self, node: ast.UAdd) -> ast.AST: """Allow positive notation of variables. (e.g. +var)""" return self.node_contents_visit(node) - def visit_USub(self, node): + def visit_USub(self, node: ast.USub) -> ast.AST: """Allow negative notation of variables. (e.g. -var)""" return self.node_contents_visit(node) - def visit_Not(self, node): + def visit_Not(self, node: ast.Not) -> ast.AST: """Allow the `not` operator.""" return self.node_contents_visit(node) - def visit_Invert(self, node): + def visit_Invert(self, node: ast.Invert) -> ast.AST: """Allow `~` expressions.""" return self.node_contents_visit(node) - def visit_BinOp(self, node): + def visit_BinOp(self, node: ast.BinOp) -> ast.AST: """Allow binary operations.""" return self.node_contents_visit(node) - def visit_Add(self, node): + def visit_Add(self, node: ast.Add) -> ast.AST: """Allow `+` expressions.""" return self.node_contents_visit(node) - def visit_Sub(self, node): + def visit_Sub(self, node: ast.Sub) -> ast.AST: """Allow `-` expressions.""" return self.node_contents_visit(node) - def visit_Mult(self, node): + def visit_Mult(self, node: ast.Mult) -> ast.AST: """Allow `*` expressions.""" return self.node_contents_visit(node) - def visit_Div(self, node): + def visit_Div(self, node: ast.Div) -> ast.AST: """Allow `/` expressions.""" return self.node_contents_visit(node) - def visit_FloorDiv(self, node): + def visit_FloorDiv(self, node: ast.FloorDiv) -> ast.AST: """Allow `//` expressions.""" return self.node_contents_visit(node) - def visit_Mod(self, node): + def visit_Mod(self, node: ast.Mod) -> ast.AST: """Allow `%` expressions.""" return self.node_contents_visit(node) - def visit_Pow(self, node): + def visit_Pow(self, node: ast.Pow) -> ast.AST: """Allow `**` expressions.""" return self.node_contents_visit(node) - def visit_LShift(self, node): + def visit_LShift(self, node: ast.LShift) -> ast.AST: """Allow `<<` expressions.""" return self.node_contents_visit(node) - def visit_RShift(self, node): + def visit_RShift(self, node: ast.RShift) -> ast.AST: """Allow `>>` expressions.""" return self.node_contents_visit(node) - def visit_BitOr(self, node): + def visit_BitOr(self, node: ast.BitOr) -> ast.AST: """Allow `|` expressions.""" return self.node_contents_visit(node) - def visit_BitXor(self, node): + def visit_BitXor(self, node: ast.BitXor) -> ast.AST: """Allow `^` expressions.""" return self.node_contents_visit(node) - def visit_BitAnd(self, node): + def visit_BitAnd(self, node: ast.BitAnd) -> ast.AST: """Allow `&` expressions.""" return self.node_contents_visit(node) - def visit_MatMult(self, node): + def visit_MatMult(self, node: ast.MatMult) -> ast.AST: """Allow multiplication (`@`).""" return self.node_contents_visit(node) - def visit_BoolOp(self, node): + def visit_BoolOp(self, node: ast.BoolOp) -> ast.AST: """Allow bool operator without restrictions.""" return self.node_contents_visit(node) - def visit_And(self, node): + def visit_And(self, node: ast.And) -> ast.AST: """Allow bool operator `and` without restrictions.""" return self.node_contents_visit(node) - def visit_Or(self, node): + def visit_Or(self, node: ast.Or) -> ast.AST: """Allow bool operator `or` without restrictions.""" return self.node_contents_visit(node) - def visit_Compare(self, node): + def visit_Compare(self, node: ast.Compare) -> ast.AST: """Allow comparison expressions without restrictions.""" return self.node_contents_visit(node) - def visit_Eq(self, node): + def visit_Eq(self, node: ast.Eq) -> ast.AST: """Allow == expressions.""" return self.node_contents_visit(node) - def visit_NotEq(self, node): + def visit_NotEq(self, node: ast.NotEq) -> ast.AST: """Allow != expressions.""" return self.node_contents_visit(node) - def visit_Lt(self, node): + def visit_Lt(self, node: ast.Lt) -> ast.AST: """Allow < expressions.""" return self.node_contents_visit(node) - def visit_LtE(self, node): + def visit_LtE(self, node: ast.LtE) -> ast.AST: """Allow <= expressions.""" return self.node_contents_visit(node) - def visit_Gt(self, node): + def visit_Gt(self, node: ast.Gt) -> ast.AST: """Allow > expressions.""" return self.node_contents_visit(node) - def visit_GtE(self, node): + def visit_GtE(self, node: ast.GtE) -> ast.AST: """Allow >= expressions.""" return self.node_contents_visit(node) - def visit_Is(self, node): + def visit_Is(self, node: ast.Is) -> ast.AST: """Allow `is` expressions.""" return self.node_contents_visit(node) - def visit_IsNot(self, node): + def visit_IsNot(self, node: ast.IsNot) -> ast.AST: """Allow `is not` expressions.""" return self.node_contents_visit(node) - def visit_In(self, node): + def visit_In(self, node: ast.In) -> ast.AST: """Allow `in` expressions.""" return self.node_contents_visit(node) - def visit_NotIn(self, node): + def visit_NotIn(self, node: ast.NotIn) -> ast.AST: """Allow `not in` expressions.""" return self.node_contents_visit(node) - def visit_Call(self, node): + def visit_Call(self, node: ast.Call) -> ast.AST: """Checks calls with '*args' and '**kwargs'. Note: The following happens only if '*args' or '**kwargs' is used. @@ -819,17 +831,17 @@ def visit_Call(self, node): copy_locations(node.func, node.args[0]) return node - def visit_keyword(self, node): + def visit_keyword(self, node: ast.keyword) -> ast.AST: """ """ return self.node_contents_visit(node) - def visit_IfExp(self, node): + def visit_IfExp(self, node: ast.IfExp) -> ast.AST: """Allow `if` expressions without restrictions.""" return self.node_contents_visit(node) - def visit_Attribute(self, node): + def visit_Attribute(self, node: ast.Attribute) -> ast.AST: """Checks and mutates attribute access/assignment. 'a.b' becomes '_getattr_(a, "b")' @@ -885,7 +897,7 @@ def visit_Attribute(self, node): # Subscripting - def visit_Subscript(self, node): + def visit_Subscript(self, node: ast.Subscript) -> ast.AST: """Transforms all kinds of subscripts. 'foo[bar]' becomes '_getitem_(foo, bar)' @@ -930,19 +942,19 @@ def visit_Subscript(self, node): raise NotImplementedError( f"Unknown ctx type: {type(node.ctx)}") - def visit_Index(self, node): + def visit_Index(self, node: ast.Index) -> ast.AST: """ """ return self.node_contents_visit(node) - def visit_Slice(self, node): + def visit_Slice(self, node: ast.Slice) -> ast.AST: """ """ return self.node_contents_visit(node) - def visit_ExtSlice(self, node): + def visit_ExtSlice(self, node: ast.ExtSlice) -> ast.AST: """ """ @@ -950,31 +962,31 @@ def visit_ExtSlice(self, node): # Comprehensions - def visit_ListComp(self, node): + def visit_ListComp(self, node: ast.ListComp) -> ast.AST: """ """ return self.node_contents_visit(node) - def visit_SetComp(self, node): + def visit_SetComp(self, node: ast.SetComp) -> ast.AST: """ """ return self.node_contents_visit(node) - def visit_GeneratorExp(self, node): + def visit_GeneratorExp(self, node: ast.GeneratorExp) -> ast.AST: """ """ return self.node_contents_visit(node) - def visit_DictComp(self, node): + def visit_DictComp(self, node: ast.DictComp) -> ast.AST: """ """ return self.node_contents_visit(node) - def visit_comprehension(self, node): + def visit_comprehension(self, node: ast.comprehension) -> ast.AST: """ """ @@ -982,7 +994,7 @@ def visit_comprehension(self, node): # Statements - def visit_Assign(self, node): + def visit_Assign(self, node: ast.Assign) -> ast.AST: """ """ @@ -1031,7 +1043,7 @@ def visit_Assign(self, node): return new_nodes - def visit_AugAssign(self, node): + def visit_AugAssign(self, node: ast.AugAssign) -> ast.AST: """Forbid certain kinds of AugAssign According to the language reference (and ast.c) the following nodes @@ -1082,75 +1094,75 @@ def visit_AugAssign(self, node): raise NotImplementedError( f"Unknown target type: {type(node.target)}") - def visit_Raise(self, node): + def visit_Raise(self, node: ast.Raise) -> ast.AST: """Allow `raise` statements without restrictions.""" return self.node_contents_visit(node) - def visit_Assert(self, node): + def visit_Assert(self, node: ast.Assert) -> ast.AST: """Allow assert statements without restrictions.""" return self.node_contents_visit(node) - def visit_Delete(self, node): + def visit_Delete(self, node: ast.Delete) -> ast.AST: """Allow `del` statements without restrictions.""" return self.node_contents_visit(node) - def visit_Pass(self, node): + def visit_Pass(self, node: ast.Pass) -> ast.AST: """Allow `pass` statements without restrictions.""" return self.node_contents_visit(node) # Imports - def visit_Import(self, node): + def visit_Import(self, node: ast.Import) -> ast.AST: """Allow `import` statements with restrictions. See check_import_names.""" return self.check_import_names(node) - def visit_ImportFrom(self, node): + def visit_ImportFrom(self, node: ast.ImportFrom) -> ast.AST: """Allow `import from` statements with restrictions. See check_import_names.""" return self.check_import_names(node) - def visit_alias(self, node): + def visit_alias(self, node: ast.alias) -> ast.AST: """Allow `as` statements in import and import from statements.""" return self.node_contents_visit(node) # Control flow - def visit_If(self, node): + def visit_If(self, node: ast.If) -> ast.AST: """Allow `if` statements without restrictions.""" return self.node_contents_visit(node) - def visit_For(self, node): + def visit_For(self, node: ast.For) -> ast.AST: """Allow `for` statements with some restrictions.""" return self.guard_iter(node) - def visit_While(self, node): + def visit_While(self, node: ast.While) -> ast.AST: """Allow `while` statements.""" return self.node_contents_visit(node) - def visit_Break(self, node): + def visit_Break(self, node: ast.Break) -> ast.AST: """Allow `break` statements without restrictions.""" return self.node_contents_visit(node) - def visit_Continue(self, node): + def visit_Continue(self, node: ast.Continue) -> ast.AST: """Allow `continue` statements without restrictions.""" return self.node_contents_visit(node) - def visit_Try(self, node): + def visit_Try(self, node: ast.Try) -> ast.AST: """Allow `try` without restrictions.""" return self.node_contents_visit(node) - def visit_TryStar(self, node): + def visit_TryStar(self, node: ast.AST) -> ast.AST: """Disallow `ExceptionGroup` due to a potential sandbox escape.""" self.not_allowed(node) - def visit_ExceptHandler(self, node): + def visit_ExceptHandler(self, node: ast.ExceptHandler) -> ast.AST: """Protect exception handlers.""" node = self.node_contents_visit(node) self.check_name(node, node.name) return node - def visit_With(self, node): + def visit_With(self, node: ast.With) -> ast.AST: """Protect tuple unpacking on with statements.""" node = self.node_contents_visit(node) @@ -1165,13 +1177,13 @@ def visit_With(self, node): return node - def visit_withitem(self, node): + def visit_withitem(self, node: ast.withitem) -> ast.AST: """Allow `with` statements (context managers) without restrictions.""" return self.node_contents_visit(node) # Function and class definitions - def visit_FunctionDef(self, node): + def visit_FunctionDef(self, node: ast.FunctionDef) -> ast.AST: """Allow function definitions (`def`) with some restrictions.""" self.check_name(node, node.name, allow_magic_methods=True) self.check_function_argument_names(node) @@ -1181,44 +1193,44 @@ def visit_FunctionDef(self, node): self.inject_print_collector(node) return node - def visit_Lambda(self, node): + def visit_Lambda(self, node: ast.Lambda) -> ast.AST: """Allow lambda with some restrictions.""" self.check_function_argument_names(node) return self.node_contents_visit(node) - def visit_arguments(self, node): + def visit_arguments(self, node: ast.arguments) -> ast.AST: """ """ return self.node_contents_visit(node) - def visit_arg(self, node): + def visit_arg(self, node: ast.arg) -> ast.AST: """ """ return self.node_contents_visit(node) - def visit_Return(self, node): + def visit_Return(self, node: ast.Return) -> ast.AST: """Allow `return` statements without restrictions.""" return self.node_contents_visit(node) - def visit_Yield(self, node): + def visit_Yield(self, node: ast.Yield) -> ast.AST: """Allow `yield`statements without restrictions.""" return self.node_contents_visit(node) - def visit_YieldFrom(self, node): + def visit_YieldFrom(self, node: ast.YieldFrom) -> ast.AST: """Allow `yield`statements without restrictions.""" return self.node_contents_visit(node) - def visit_Global(self, node): + def visit_Global(self, node: ast.Global) -> ast.AST: """Allow `global` statements without restrictions.""" return self.node_contents_visit(node) - def visit_Nonlocal(self, node): + def visit_Nonlocal(self, node: ast.Nonlocal) -> ast.AST: """Deny `nonlocal` statements.""" self.not_allowed(node) - def visit_ClassDef(self, node): + def visit_ClassDef(self, node: ast.ClassDef) -> ast.AST: """Check the name of a class definition.""" self.check_name(node, node.name) node = self.node_contents_visit(node) @@ -1235,7 +1247,7 @@ class {0.name}(metaclass=__metaclass__): new_class_node.decorator_list = node.decorator_list return new_class_node - def visit_Module(self, node): + def visit_Module(self, node: ast.Module) -> ast.AST: """Add the print_collector (only if print is used) at the top.""" node = self.node_contents_visit(node) @@ -1253,25 +1265,25 @@ def visit_Module(self, node): # Async und await - def visit_AsyncFunctionDef(self, node): + def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> ast.AST: """Deny async functions.""" self.not_allowed(node) - def visit_Await(self, node): + def visit_Await(self, node: ast.Await) -> ast.AST: """Deny async functionality.""" self.not_allowed(node) - def visit_AsyncFor(self, node): + def visit_AsyncFor(self, node: ast.AsyncFor) -> ast.AST: """Deny async functionality.""" self.not_allowed(node) - def visit_AsyncWith(self, node): + def visit_AsyncWith(self, node: ast.AsyncWith) -> ast.AST: """Deny async functionality.""" self.not_allowed(node) # Assignment expressions (walrus operator ``:=``) # New in 3.8 - def visit_NamedExpr(self, node): + def visit_NamedExpr(self, node: ast.NamedExpr) -> ast.AST: """Allow assignment expressions under some circumstances.""" # while the grammar requires ``node.target`` to be a ``Name`` # the abstract syntax is more permissive and allows an ``expr``. diff --git a/tox.ini b/tox.ini index 4015b3a7..1f3fce15 100644 --- a/tox.ini +++ b/tox.ini @@ -13,7 +13,7 @@ envlist = py314 docs coverage - py311-datetime + py314-datetime combined-coverage [testenv] @@ -92,7 +92,7 @@ commands = pre-commit run --all-files --show-diff-on-failure [testenv:docs] -basepython = python3 +basepython = python3.14 skip_install = false commands_pre = commands = From 1cd61981dcb7ef4730a9765141ace742690b49d5 Mon Sep 17 00:00:00 2001 From: Alexander Loechel Date: Sat, 18 Oct 2025 11:36:48 +0300 Subject: [PATCH 02/40] isinstance check with ExtSlice and Tuple as for older Python Versions --- src/RestrictedPython/transformer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/RestrictedPython/transformer.py b/src/RestrictedPython/transformer.py index 5ecc5f63..990bdfa1 100644 --- a/src/RestrictedPython/transformer.py +++ b/src/RestrictedPython/transformer.py @@ -406,7 +406,7 @@ def transform_slice(self, slice_: ast.AST) -> ast.AST: args=args, keywords=[]) - elif isinstance(slice_, ast.Tuple): + elif isinstance(slice_, (ast.Tuple, ast.ExtSlice)): dims = ast.Tuple([], ast.Load()) for item in slice_.dims: dims.elts.append(self.transform_slice(item)) From e2599abe8144feb3aa4c5c77c7f5d041e939088d Mon Sep 17 00:00:00 2001 From: Alexander Loechel Date: Sat, 18 Oct 2025 11:41:47 +0300 Subject: [PATCH 03/40] liniting --- src/RestrictedPython/compile.py | 44 ++++++++++++++++++++------------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/src/RestrictedPython/compile.py b/src/RestrictedPython/compile.py index 0172023a..83050a8e 100644 --- a/src/RestrictedPython/compile.py +++ b/src/RestrictedPython/compile.py @@ -1,5 +1,9 @@ import ast import warnings +from ast import Expression +from ast import Interactive +from ast import Module +from ast import NodeTransformer from collections import namedtuple from os import PathLike from typing import Any @@ -26,12 +30,13 @@ def _compile_restricted_mode( - source: str | ReadableBuffer | ast.Module | ast.Expression | ast.Interactive, + source: str | ReadableBuffer | Module | Expression | Interactive, filename: str | ReadableBuffer | PathLike[Any] = '', mode: Literal["exec", "eval", "single"] = "exec", flags: int = 0, dont_inherit: bool = False, - policy: ast.NodeTransformer = RestrictingNodeTransformer) -> CompileResult: + policy: NodeTransformer = RestrictingNodeTransformer, +) -> CompileResult: if not IS_CPYTHON: warnings.warn_explicit( @@ -47,13 +52,13 @@ def _compile_restricted_mode( dont_inherit=dont_inherit) elif issubclass(policy, RestrictingNodeTransformer): c_ast = None - allowed_source_types = [str, ast.Module] + allowed_source_types = [str, Module] if not issubclass(type(source), tuple(allowed_source_types)): raise TypeError('Not allowed source type: ' '"{0.__class__.__name__}".'.format(source)) c_ast = None # workaround for pypy issue https://bitbucket.org/pypy/pypy/issues/2552 - if isinstance(source, ast.Module): + if isinstance(source, Module): c_ast = source else: try: @@ -86,11 +91,12 @@ def _compile_restricted_mode( def compile_restricted_exec( - source: str | ReadableBuffer | ast.Module | ast.Expression | ast.Interactive, + source: str | ReadableBuffer | Module | Expression | Interactive, filename: str | ReadableBuffer | PathLike[Any] = '', flags: int = 0, dont_inherit: bool = False, - policy: ast.NodeTransformer = RestrictingNodeTransformer) -> CompileResult: + policy: NodeTransformer = RestrictingNodeTransformer, +) -> CompileResult: """Compile restricted for the mode `exec`.""" return _compile_restricted_mode( source, @@ -102,11 +108,12 @@ def compile_restricted_exec( def compile_restricted_eval( - source: str | ReadableBuffer | ast.Module | ast.Expression | ast.Interactive, + source: str | ReadableBuffer | Module | Expression | Interactive, filename: str | ReadableBuffer | PathLike[Any] = '', flags: int = 0, dont_inherit: bool = False, - policy: ast.NodeTransformer = RestrictingNodeTransformer) -> CompileResult: + policy: NodeTransformer = RestrictingNodeTransformer, +) -> CompileResult: """Compile restricted for the mode `eval`.""" return _compile_restricted_mode( source, @@ -118,11 +125,12 @@ def compile_restricted_eval( def compile_restricted_single( - source: str | ReadableBuffer | ast.Module | ast.Expression | ast.Interactive, + source: str | ReadableBuffer | Module | Expression | Interactive, filename: str | ReadableBuffer | PathLike[Any] = '', flags: int = 0, dont_inherit: bool = False, - policy: ast.NodeTransformer = RestrictingNodeTransformer) -> CompileResult: + policy: NodeTransformer = RestrictingNodeTransformer, +) -> CompileResult: """Compile restricted for the mode `single`.""" return _compile_restricted_mode( source, @@ -141,7 +149,8 @@ def compile_restricted_function( globalize=None, # List of globals (e.g. ['here', 'context', ...]) flags: int = 0, dont_inherit: bool = False, - policy: ast.NodeTransformer = RestrictingNodeTransformer) -> CompileResult: + policy: ast.NodeTransformer = RestrictingNodeTransformer, +) -> CompileResult: """Compile a restricted code object for a function. Documentation see: @@ -189,12 +198,13 @@ def compile_restricted_function( def compile_restricted( - source: str | ReadableBuffer | ast.Module | ast.Expression | ast.Interactive, - filename: str | ReadableBuffer | PathLike[Any] = '', - mode: str = 'exec', - flags: int = 0, - dont_inherit: bool = False, - policy: ast.NodeTransformer = RestrictingNodeTransformer) -> CompileResult: + source: str | ReadableBuffer | Module | Expression | Interactive, + filename: str | ReadableBuffer | PathLike[Any] = '', + mode: str = 'exec', + flags: int = 0, + dont_inherit: bool = False, + policy: NodeTransformer = RestrictingNodeTransformer, +) -> CompileResult: """Replacement for the built-in compile() function. policy ... `ast.NodeTransformer` class defining the restrictions. From 4915d8e21705ea697c557daf45a7f17057be0606 Mon Sep 17 00:00:00 2001 From: Alexander Loechel Date: Sat, 18 Oct 2025 12:05:09 +0300 Subject: [PATCH 04/40] Remove Python 3.9 as it end of life --- .github/workflows/tests.yml | 15 +++++++-------- .meta.toml | 2 +- tox.ini | 1 - 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8cb5ba8f..5abf4912 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -24,20 +24,19 @@ jobs: - ["windows", "windows-latest"] config: # [Python version, tox env] - - ["3.11", "release-check"] - - ["3.9", "py39"] + - ["3.14", "release-check"] - ["3.10", "py310"] - ["3.11", "py311"] - ["3.12", "py312"] - ["3.13", "py313"] - ["3.14", "py314"] - - ["3.11", "docs"] - - ["3.11", "coverage"] - - ["3.11", "py311-datetime"] + - ["3.14", "docs"] + - ["3.14", "coverage"] + - ["3.14", "py314-datetime"] exclude: - - { os: ["windows", "windows-latest"], config: ["3.11", "release-check"] } - - { os: ["windows", "windows-latest"], config: ["3.11", "docs"] } - - { os: ["windows", "windows-latest"], config: ["3.11", "coverage"] } + - { os: ["windows", "windows-latest"], config: ["3.14", "release-check"] } + - { os: ["windows", "windows-latest"], config: ["3.14", "docs"] } + - { os: ["windows", "windows-latest"], config: ["3.14", "coverage"] } runs-on: ${{ matrix.os[1] }} if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name diff --git a/.meta.toml b/.meta.toml index 3ba9ecbd..f7000fbe 100644 --- a/.meta.toml +++ b/.meta.toml @@ -47,7 +47,7 @@ testenv-additional = [ " coverage combine", " coverage html", " coverage report -m --fail-under=100", - "depends = py39,py310,py311,py312,py313,py314,py314-datetime,coverage", + "depends = py310,py311,py312,py313,py314,py314-datetime,coverage", ] coverage-command = "pytest --cov=src --cov=tests --cov-report= tests {posargs}" coverage-setenv = [ diff --git a/tox.ini b/tox.ini index 1f3fce15..71e73aa6 100644 --- a/tox.ini +++ b/tox.ini @@ -5,7 +5,6 @@ minversion = 3.18 envlist = release-check lint - py39 py310 py311 py312 From 3299a8bb301c16bfe36b9908bc0cbf6bf7b52b04 Mon Sep 17 00:00:00 2001 From: Alexander Loechel Date: Sat, 18 Oct 2025 12:09:34 +0300 Subject: [PATCH 05/40] Remove License Cassifier, as they are deprecated --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index a7d2ec2b..6e19db74 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,6 @@ def read(*rnames): long_description_content_type='text/x-rst', classifiers=[ 'Development Status :: 6 - Mature', - 'License :: OSI Approved :: Zope Public License', 'Programming Language :: Python', 'Operating System :: OS Independent', 'Programming Language :: Python :: 3', From 4d56e39735fdc56f850dc1195d3c8ad25b776a16 Mon Sep 17 00:00:00 2001 From: Alexander Loechel Date: Sat, 18 Oct 2025 12:16:17 +0300 Subject: [PATCH 06/40] Add Comment for TryStar Annotation --- src/RestrictedPython/Utilities.py | 2 +- src/RestrictedPython/transformer.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/RestrictedPython/Utilities.py b/src/RestrictedPython/Utilities.py index e07f8e49..b966b4e2 100644 --- a/src/RestrictedPython/Utilities.py +++ b/src/RestrictedPython/Utilities.py @@ -77,7 +77,7 @@ def test(*args): def reorder(s: Iterable, with_: Iterable | None = None, - without: Iterable | None = None) -> Iterable: + without: Iterable = ()) -> Iterable: # s, with_, and without are sequences treated as sets. # The result is subtract(intersect(s, with_), without), # unless with_ is None, in which case it is subtract(s, without). diff --git a/src/RestrictedPython/transformer.py b/src/RestrictedPython/transformer.py index 990bdfa1..abd9aa6f 100644 --- a/src/RestrictedPython/transformer.py +++ b/src/RestrictedPython/transformer.py @@ -1153,7 +1153,10 @@ def visit_Try(self, node: ast.Try) -> ast.AST: return self.node_contents_visit(node) def visit_TryStar(self, node: ast.AST) -> ast.AST: - """Disallow `ExceptionGroup` due to a potential sandbox escape.""" + """Disallow `ExceptionGroup` due to a potential sandbox escape. + + TODO: Type Annotation for node when dropping support for Python < 3.11 should be ast.TryStar. + """ self.not_allowed(node) def visit_ExceptHandler(self, node: ast.ExceptHandler) -> ast.AST: From 935a960b9a6e881f68e849c67364de0cda491008 Mon Sep 17 00:00:00 2001 From: Alexander Loechel Date: Sat, 18 Oct 2025 12:16:41 +0300 Subject: [PATCH 07/40] Add Comment for TryStar Annotation --- src/RestrictedPython/transformer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/RestrictedPython/transformer.py b/src/RestrictedPython/transformer.py index abd9aa6f..bcd87c5e 100644 --- a/src/RestrictedPython/transformer.py +++ b/src/RestrictedPython/transformer.py @@ -1155,7 +1155,8 @@ def visit_Try(self, node: ast.Try) -> ast.AST: def visit_TryStar(self, node: ast.AST) -> ast.AST: """Disallow `ExceptionGroup` due to a potential sandbox escape. - TODO: Type Annotation for node when dropping support for Python < 3.11 should be ast.TryStar. + TODO: Type Annotation for node when dropping support + for Python < 3.11 should be ast.TryStar. """ self.not_allowed(node) From e67da675036549242322289bc8844df535d8aff6 Mon Sep 17 00:00:00 2001 From: Alexander Loechel Date: Sat, 18 Oct 2025 12:16:58 +0300 Subject: [PATCH 08/40] Add Comment for TryStar Annotation --- src/RestrictedPython/transformer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/RestrictedPython/transformer.py b/src/RestrictedPython/transformer.py index bcd87c5e..126f1ff4 100644 --- a/src/RestrictedPython/transformer.py +++ b/src/RestrictedPython/transformer.py @@ -1155,7 +1155,7 @@ def visit_Try(self, node: ast.Try) -> ast.AST: def visit_TryStar(self, node: ast.AST) -> ast.AST: """Disallow `ExceptionGroup` due to a potential sandbox escape. - TODO: Type Annotation for node when dropping support + TODO: Type Annotation for node when dropping support for Python < 3.11 should be ast.TryStar. """ self.not_allowed(node) From 2a3d72829062af83148fe93a917ba9fb796a12eb Mon Sep 17 00:00:00 2001 From: Alexander Loechel Date: Sat, 18 Oct 2025 12:19:04 +0300 Subject: [PATCH 09/40] Add Changelog Entry --- CHANGES.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 8823a783..041bc755 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -5,6 +5,9 @@ Changes ---------------- - Nothing changed yet. +- Drop Support for Python 3.9 as it was EOL on 2025-10. +- Added basis support of Python 3.14 +- Added Type Annotations 8.1a1.dev0 (2025-03-20) From 337379507d03a7a5a707c211090a0a033abfd5f0 Mon Sep 17 00:00:00 2001 From: Alexander Loechel Date: Sat, 18 Oct 2025 12:51:33 +0300 Subject: [PATCH 10/40] Base for Python 3.14 Updates --- docs/contributing/ast/python3_14.ast | 194 +++++++++++++++++++++ docs/contributing/changes_from313to314.rst | 5 + 2 files changed, 199 insertions(+) create mode 100644 docs/contributing/ast/python3_14.ast create mode 100644 docs/contributing/changes_from313to314.rst diff --git a/docs/contributing/ast/python3_14.ast b/docs/contributing/ast/python3_14.ast new file mode 100644 index 00000000..662f4f17 --- /dev/null +++ b/docs/contributing/ast/python3_14.ast @@ -0,0 +1,194 @@ +-- Python 3.14 AST +-- ASDL's 4 builtin types are: +-- identifier, int, string, constant + +module Python version "3.14" +{ + mod = Module(stmt* body, type_ignore* type_ignores) + | Interactive(stmt* body) + | Expression(expr body) + | FunctionType(expr* argtypes, expr returns) + + stmt = FunctionDef(identifier name, + arguments args, + stmt* body, + expr* decorator_list, + expr? returns, + string? type_comment, + type_param* type_params) + | AsyncFunctionDef(identifier name, + arguments args, + stmt* body, + expr* decorator_list, + expr? returns, + string? type_comment, + type_param* type_params) + + | ClassDef(identifier name, + expr* bases, + keyword* keywords, + stmt* body, + expr* decorator_list, + type_param* type_params) + | Return(expr? value) + + | Delete(expr* targets) + | Assign(expr* targets, expr value, string? type_comment) + | TypeAlias(expr name, type_param* type_params, expr value) + | AugAssign(expr target, operator op, expr value) + -- 'simple' indicates that we annotate simple name without parens + | AnnAssign(expr target, expr annotation, expr? value, int simple) + + -- use 'orelse' because else is a keyword in target languages + | For(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment) + | AsyncFor(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment) + | While(expr test, stmt* body, stmt* orelse) + | If(expr test, stmt* body, stmt* orelse) + | With(withitem* items, stmt* body, string? type_comment) + | AsyncWith(withitem* items, stmt* body, string? type_comment) + + | Match(expr subject, match_case* cases) + + | Raise(expr? exc, expr? cause) + | Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody) + | TryStar(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody) + | Assert(expr test, expr? msg) + + | Import(alias* names) + | ImportFrom(identifier? module, alias* names, int? level) + + | Global(identifier* names) + | Nonlocal(identifier* names) + | Expr(expr value) + | Pass + | Break + | Continue + + -- col_offset is the byte offset in the utf8 string the parser uses + attributes (int lineno, int col_offset, int? end_lineno, int? end_col_offset) + + -- BoolOp() can use left & right? + expr = BoolOp(boolop op, expr* values) + | NamedExpr(expr target, expr value) + | BinOp(expr left, operator op, expr right) + | UnaryOp(unaryop op, expr operand) + | Lambda(arguments args, expr body) + | IfExp(expr test, expr body, expr orelse) + | Dict(expr* keys, expr* values) + | Set(expr* elts) + | ListComp(expr elt, comprehension* generators) + | SetComp(expr elt, comprehension* generators) + | DictComp(expr key, expr value, comprehension* generators) + | GeneratorExp(expr elt, comprehension* generators) + -- the grammar constrains where yield expressions can occur + | Await(expr value) + | Yield(expr? value) + | YieldFrom(expr value) + -- need sequences for compare to distinguish between + -- x < 4 < 3 and (x < 4) < 3 + | Compare(expr left, cmpop* ops, expr* comparators) + | Call(expr func, expr* args, keyword* keywords) + | FormattedValue(expr value, int conversion, expr? format_spec) + | JoinedStr(expr* values) + | Constant(constant value, string? kind) + + -- the following expression can appear in assignment context + | Attribute(expr value, identifier attr, expr_context ctx) + | Subscript(expr value, expr slice, expr_context ctx) + | Starred(expr value, expr_context ctx) + | Name(identifier id, expr_context ctx) + | List(expr* elts, expr_context ctx) + | Tuple(expr* elts, expr_context ctx) + + -- can appear only in Subscript + | Slice(expr? lower, expr? upper, expr? step) + + -- col_offset is the byte offset in the utf8 string the parser uses + attributes (int lineno, int col_offset, int? end_lineno, int? end_col_offset) + + expr_context = Load + | Store + | Del + + boolop = And + | Or + + operator = Add + | Sub + | Mult + | MatMult + | Div + | Mod + | Pow + | LShift + | RShift + | BitOr + | BitXor + | BitAnd + | FloorDiv + + unaryop = Invert + | Not + | UAdd + | USub + + cmpop = Eq + | NotEq + | Lt + | LtE + | Gt + | GtE + | Is + | IsNot + | In + | NotIn + + comprehension = (expr target, expr iter, expr* ifs, int is_async) + + excepthandler = ExceptHandler(expr? type, identifier? name, stmt* body) + attributes (int lineno, int col_offset, int? end_lineno, int? end_col_offset) + + arguments = (arg* posonlyargs, + arg* args, + arg? vararg, + arg* kwonlyargs, + expr* kw_defaults, + arg? kwarg, + expr* defaults) + + arg = (identifier arg, expr? annotation, string? type_comment) + attributes (int lineno, int col_offset, int? end_lineno, int? end_col_offset) + + -- keyword arguments supplied to call (NULL identifier for **kwargs) + keyword = (identifier? arg, expr value) + attributes (int lineno, int col_offset, int? end_lineno, int? end_col_offset) + + -- import name with optional 'as' alias. + alias = (identifier name, identifier? asname) + attributes (int lineno, int col_offset, int? end_lineno, int? end_col_offset) + + withitem = (expr context_expr, expr? optional_vars) + + match_case = (pattern pattern, expr? guard, stmt* body) + + pattern = MatchValue(expr value) + | MatchSingleton(constant value) + | MatchSequence(pattern* patterns) + | MatchMapping(expr* keys, pattern* patterns, identifier? rest) + | MatchClass(expr cls, pattern* patterns, identifier* kwd_attrs, pattern* kwd_patterns) + + | MatchStar(identifier? name) + -- The optional "rest" MatchMapping parameter handles capturing extra mapping keys + + | MatchAs(pattern? pattern, identifier? name) + | MatchOr(pattern* patterns) + + attributes (int lineno, int col_offset, int end_lineno, int end_col_offset) + + type_ignore = TypeIgnore(int lineno, string tag) + + type_param = TypeVar(identifier name, expr? bound, expr? default_value) + | ParamSpec(identifier name, expr? default_value) + | TypeVarTuple(identifier name, expr? default_value) + attributes (int lineno, int col_offset, int end_lineno, int end_col_offset) +} diff --git a/docs/contributing/changes_from313to314.rst b/docs/contributing/changes_from313to314.rst new file mode 100644 index 00000000..2690a880 --- /dev/null +++ b/docs/contributing/changes_from313to314.rst @@ -0,0 +1,5 @@ +Changes from Python 3.13 to Python 3.14 +--------------------------------------- + +.. literalinclude:: ast/python3_14.ast + :diff: ast/python3_13.ast From f86d8951aef75e19e6d8783b0c11d1c5117c7657 Mon Sep 17 00:00:00 2001 From: Alexander Loechel Date: Sat, 18 Oct 2025 13:49:08 +0300 Subject: [PATCH 11/40] Update docs for Python 3.14 --- docs/contributing/ast/python3_14.ast | 4 +++- docs/contributing/index.rst | 7 ++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/contributing/ast/python3_14.ast b/docs/contributing/ast/python3_14.ast index 662f4f17..2c1e0b78 100644 --- a/docs/contributing/ast/python3_14.ast +++ b/docs/contributing/ast/python3_14.ast @@ -74,7 +74,7 @@ module Python version "3.14" | UnaryOp(unaryop op, expr operand) | Lambda(arguments args, expr body) | IfExp(expr test, expr body, expr orelse) - | Dict(expr* keys, expr* values) + | Dict(expr?* keys, expr* values) | Set(expr* elts) | ListComp(expr elt, comprehension* generators) | SetComp(expr elt, comprehension* generators) @@ -89,7 +89,9 @@ module Python version "3.14" | Compare(expr left, cmpop* ops, expr* comparators) | Call(expr func, expr* args, keyword* keywords) | FormattedValue(expr value, int conversion, expr? format_spec) + | Interpolation(expr value, constant str, int conversion, expr? format_spec) | JoinedStr(expr* values) + | TemplateStr(expr* values) | Constant(constant value, string? kind) -- the following expression can appear in assignment context diff --git a/docs/contributing/index.rst b/docs/contributing/index.rst index 23e93b22..1a5cbdc9 100644 --- a/docs/contributing/index.rst +++ b/docs/contributing/index.rst @@ -103,6 +103,7 @@ A (modified style) Copy of all Abstract Grammar Definitions for the Python versi changes_from310to311 changes_from311to312 changes_from312to313 + changes_from313to314 .. _understand: @@ -235,11 +236,11 @@ Technical Backgrounds - Links to External Documentation * AST Grammar of Python (`Status of Python Versions`_) + * `Python 3.14 AST`_ (EOL 2030-10) * `Python 3.13 AST`_ (EOL 2029-10) * `Python 3.12 AST`_ (EOL 2028-10) * `Python 3.11 AST`_ (EOL 2027-10) * `Python 3.10 AST`_ (EOL 2026-10) - * `Python 3.9 AST`_ (EOL 2025-10) * `AST NodeVistiors Class`_ * `AST NodeTransformer Class`_ @@ -257,6 +258,8 @@ Todos .. _`What's new in Python`: https://docs.python.org/3/whatsnew/ +.. _`What's new in Python 3.14`: https://docs.python.org/3.14/whatsnew/3.14.html + .. _`What's new in Python 3.13`: https://docs.python.org/3.13/whatsnew/3.13.html .. _`What's new in Python 3.12`: https://docs.python.org/3.12/whatsnew/3.12.html @@ -281,6 +284,8 @@ Todos .. _`Python 3 AST`: https://docs.python.org/3/library/ast.html#abstract-grammar +.. _`Python 3.14 AST`: https://docs.python.org/3.14/library/ast.html#abstract-grammar + .. _`Python 3.13 AST`: https://docs.python.org/3.13/library/ast.html#abstract-grammar .. _`Python 3.12 AST`: https://docs.python.org/3.12/library/ast.html#abstract-grammar From b4ceb35e4c842ec06fefd2728986989e98728c2e Mon Sep 17 00:00:00 2001 From: Alexander Loechel Date: Sat, 18 Oct 2025 14:08:34 +0300 Subject: [PATCH 12/40] add provisional visit_TempalteStr and visit_Interpolation to transformer to start looking into it --- src/RestrictedPython/transformer.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/RestrictedPython/transformer.py b/src/RestrictedPython/transformer.py index 126f1ff4..cffce5ac 100644 --- a/src/RestrictedPython/transformer.py +++ b/src/RestrictedPython/transformer.py @@ -575,6 +575,26 @@ def visit_FormattedValue(self, node: ast.FormattedValue) -> ast.AST: """Allow f-strings without restrictions.""" return self.node_contents_visit(node) + def visit_TemplateStr(self, node: ast.AST) -> ast.AST: + """Allow template strings without restrictions. + + TODO: Review security implications of template strings. + TODO: Change Type Annotation to ast.TemplateStr when + Support for Python 3.13 is dropped. + """ + return self.not_allowed(node) + # return self.node_contents_visit(node) + + def visit_InterpolatedStr(self, node: ast.AST) -> ast.AST: + """Allow interpolated strings without restrictions. + + TODO: Review security implications of interpolated strings. + TODO: Change Type Annotation to ast.InterpolatedStr when + Support for Python 3.13 is dropped. + """ + return self.not_allowed(node) + # return self.node_contents_visit(node) + def visit_JoinedStr(self, node: ast.JoinedStr) -> ast.AST: """Allow joined string without restrictions.""" return self.node_contents_visit(node) From 43941ee0bb52b0225258403fe1d20b6951c2a56e Mon Sep 17 00:00:00 2001 From: Alexander Loechel Date: Sat, 18 Oct 2025 16:41:07 +0300 Subject: [PATCH 13/40] Disable t-strings --- src/RestrictedPython/transformer.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/src/RestrictedPython/transformer.py b/src/RestrictedPython/transformer.py index cffce5ac..d88fbd9a 100644 --- a/src/RestrictedPython/transformer.py +++ b/src/RestrictedPython/transformer.py @@ -576,23 +576,30 @@ def visit_FormattedValue(self, node: ast.FormattedValue) -> ast.AST: return self.node_contents_visit(node) def visit_TemplateStr(self, node: ast.AST) -> ast.AST: - """Allow template strings without restrictions. - - TODO: Review security implications of template strings. + """Template strings are not allowed by default. + Even so, that template strings can be useful in context of Template Engines + A Template String itself is not executed itself, but it contain expressions + and need additional template rendering logic applied to it to be useful. + Those rendering logic would be affected by RestrictedPython as well. + + TODO: Deeper review of security implications of template strings. TODO: Change Type Annotation to ast.TemplateStr when Support for Python 3.13 is dropped. """ - return self.not_allowed(node) + self.warn(node, 'TemplateStr statements are not yet allowed, please use f-strings or a real template engine instead.') + self.not_allowed(node) # return self.node_contents_visit(node) - def visit_InterpolatedStr(self, node: ast.AST) -> ast.AST: - """Allow interpolated strings without restrictions. - - TODO: Review security implications of interpolated strings. - TODO: Change Type Annotation to ast.InterpolatedStr when + def visit_Interpolation(self, node: ast.AST) -> ast.AST: + """Interpolations are not allowed by default. + As Interpolations are part of Template Strings, they will not be reached in + context of RestrictedPython as Template Strings are not allowed. + + TODO: Deeper review of security implications of interpolated strings. + TODO: Change Type Annotation to ast.Interpolation when Support for Python 3.13 is dropped. """ - return self.not_allowed(node) + self.not_allowed(node) # return self.node_contents_visit(node) def visit_JoinedStr(self, node: ast.JoinedStr) -> ast.AST: From f0e2a060f682822c86ae9976549d110ddc20e956 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci-lite[bot]" <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Date: Sat, 18 Oct 2025 13:41:44 +0000 Subject: [PATCH 14/40] Apply pre-commit code formatting --- src/RestrictedPython/transformer.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/RestrictedPython/transformer.py b/src/RestrictedPython/transformer.py index d88fbd9a..0093921c 100644 --- a/src/RestrictedPython/transformer.py +++ b/src/RestrictedPython/transformer.py @@ -576,25 +576,27 @@ def visit_FormattedValue(self, node: ast.FormattedValue) -> ast.AST: return self.node_contents_visit(node) def visit_TemplateStr(self, node: ast.AST) -> ast.AST: - """Template strings are not allowed by default. + """Template strings are not allowed by default. Even so, that template strings can be useful in context of Template Engines - A Template String itself is not executed itself, but it contain expressions + A Template String itself is not executed itself, but it contain expressions and need additional template rendering logic applied to it to be useful. - Those rendering logic would be affected by RestrictedPython as well. - + Those rendering logic would be affected by RestrictedPython as well. + TODO: Deeper review of security implications of template strings. TODO: Change Type Annotation to ast.TemplateStr when Support for Python 3.13 is dropped. """ - self.warn(node, 'TemplateStr statements are not yet allowed, please use f-strings or a real template engine instead.') + self.warn( + node, + 'TemplateStr statements are not yet allowed, please use f-strings or a real template engine instead.') self.not_allowed(node) # return self.node_contents_visit(node) def visit_Interpolation(self, node: ast.AST) -> ast.AST: """Interpolations are not allowed by default. - As Interpolations are part of Template Strings, they will not be reached in + As Interpolations are part of Template Strings, they will not be reached in context of RestrictedPython as Template Strings are not allowed. - + TODO: Deeper review of security implications of interpolated strings. TODO: Change Type Annotation to ast.Interpolation when Support for Python 3.13 is dropped. From 26a218cb93377855ff57a20eab4874fea68e0a45 Mon Sep 17 00:00:00 2001 From: Alexander Loechel Date: Sat, 18 Oct 2025 17:08:29 +0300 Subject: [PATCH 15/40] reactivate Template-Strings --- src/RestrictedPython/transformer.py | 23 ++++--- tests/transformer/test_tstring.py | 103 ++++++++++++++++++++++++++++ 2 files changed, 116 insertions(+), 10 deletions(-) create mode 100644 tests/transformer/test_tstring.py diff --git a/src/RestrictedPython/transformer.py b/src/RestrictedPython/transformer.py index 0093921c..d092167d 100644 --- a/src/RestrictedPython/transformer.py +++ b/src/RestrictedPython/transformer.py @@ -577,9 +577,10 @@ def visit_FormattedValue(self, node: ast.FormattedValue) -> ast.AST: def visit_TemplateStr(self, node: ast.AST) -> ast.AST: """Template strings are not allowed by default. - Even so, that template strings can be useful in context of Template Engines - A Template String itself is not executed itself, but it contain expressions - and need additional template rendering logic applied to it to be useful. + Even so, that template strings can be useful in context of Template + Engines. A Template String itself is not executed itself, but it + contain expressions and need additional template rendering logic + applied to it to be useful. Those rendering logic would be affected by RestrictedPython as well. TODO: Deeper review of security implications of template strings. @@ -588,21 +589,23 @@ def visit_TemplateStr(self, node: ast.AST) -> ast.AST: """ self.warn( node, - 'TemplateStr statements are not yet allowed, please use f-strings or a real template engine instead.') - self.not_allowed(node) - # return self.node_contents_visit(node) + 'TemplateStr statements are not yet allowed, ' + 'please use f-strings or a real template engine instead.') + # self.not_allowed(node) + return self.node_contents_visit(node) def visit_Interpolation(self, node: ast.AST) -> ast.AST: """Interpolations are not allowed by default. - As Interpolations are part of Template Strings, they will not be reached in - context of RestrictedPython as Template Strings are not allowed. + As Interpolations are part of Template Strings, they will not be + reached in the context of RestrictedPython as Template Strings + ‚‚are not allowed. TODO: Deeper review of security implications of interpolated strings. TODO: Change Type Annotation to ast.Interpolation when Support for Python 3.13 is dropped. """ - self.not_allowed(node) - # return self.node_contents_visit(node) + # self.not_allowed(node) + return self.node_contents_visit(node) def visit_JoinedStr(self, node: ast.JoinedStr) -> ast.AST: """Allow joined string without restrictions.""" diff --git a/tests/transformer/test_tstring.py b/tests/transformer/test_tstring.py new file mode 100644 index 00000000..c53a3df9 --- /dev/null +++ b/tests/transformer/test_tstring.py @@ -0,0 +1,103 @@ +from string.templatelib import Template + +import pytest + +from RestrictedPython import compile_restricted_exec +from RestrictedPython._compat import IS_PY314_OR_GREATER +from RestrictedPython.Eval import default_guarded_getattr +from RestrictedPython.Eval import default_guarded_getiter +from RestrictedPython.PrintCollector import PrintCollector + + +@pytest.mark.skipif( + not IS_PY314_OR_GREATER, + reason="t-strings were added in Python 3.14.", +) +def test_transform(): + """It compiles a function call successfully and returns the used name.""" + + result = compile_restricted_exec('a = t"{max([1, 2, 3])}"') + assert result.errors == () + assert result.warnings == [ + 'Line 1: TemplateStr statements are not yet allowed, please use f-strings or a real template engine instead.'] # NOQA: E501 + assert result.code is not None + loc = {} + exec(result.code, {}, loc) + template = loc['a'] + assert isinstance(template, Template) + assert template.values == (3, ) + assert result.used_names == {'max': True} + + +@pytest.mark.skipif( + not IS_PY314_OR_GREATER, + reason="t-strings were added in Python 3.14.", +) +def test_visit_invalid_variable_name(): + """Accessing private attributes is forbidden. + + This is just a smoke test to validate that restricted exec is used + in the run-time evaluation of t-strings. + """ + result = compile_restricted_exec('t"{__init__}"') + assert result.errors == ( + 'Line 1: "__init__" is an invalid variable name because it starts with "_"', # NOQA: E501 + ) + + +t_string_self_documenting_expressions_example = """ +from datetime import date +from string.templatelib import Template, Interpolation + +user = 'eric_idle' +member_since = date(1975, 7, 31) + +def render_template(template: Template) -> str: + result = '' + for part in template: + if isinstance(part, Interpolation): + if isinstance(part.value, str): + result += part.value.upper() + else: + result += str(part.value) + else: + result += part.lower() + return result + +print(render_template(t'The User {user} is a member since {member_since}')) +""" + + +@pytest.mark.skipif( + not IS_PY314_OR_GREATER, + reason="t-strings were added in Python 3.14.", +) +def test_t_string_self_documenting_expressions(): + """Checks if t-string self-documenting expressions is checked.""" + result = compile_restricted_exec( + t_string_self_documenting_expressions_example, + ) + # assert result.errors == ( + # 'Line 13: TemplateStr statements are not allowed.', + # ) + # assert result.warnings == [ + # 'Line 13: TemplateStr statements are not yet allowed, please use ' + # 'f-strings or a real template engine instead.', + # "Line None: Prints, but never reads 'printed' variable." + # ] + # assert result.code is None + assert result.errors == () + assert result.warnings == [ + 'Line 20: TemplateStr statements are not yet allowed, ' + 'please use f-strings or a real template engine instead.', + "Line None: Prints, but never reads 'printed' variable."] + assert result.code is not None + + glb = { + '_print_': PrintCollector, + '_getattr_': default_guarded_getattr, + '_getiter_': default_guarded_getiter, + '_inplacevar_': lambda x: x, + } + exec(result.code, glb) + assert glb['_print']() == "user='eric_idle' member_since=datetime.date(1975, 7, 31)\n" # NOQA: E501 From a4e189ed945988c62c77d9f52ac2de2c77644be8 Mon Sep 17 00:00:00 2001 From: Alexander Loechel Date: Sat, 18 Oct 2025 17:38:13 +0300 Subject: [PATCH 16/40] Update Documentation for TemplateStr and Interploation --- src/RestrictedPython/transformer.py | 22 +++++++++++----------- tests/transformer/test_tstring.py | 4 ++-- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/RestrictedPython/transformer.py b/src/RestrictedPython/transformer.py index d092167d..78c67156 100644 --- a/src/RestrictedPython/transformer.py +++ b/src/RestrictedPython/transformer.py @@ -576,14 +576,13 @@ def visit_FormattedValue(self, node: ast.FormattedValue) -> ast.AST: return self.node_contents_visit(node) def visit_TemplateStr(self, node: ast.AST) -> ast.AST: - """Template strings are not allowed by default. - Even so, that template strings can be useful in context of Template - Engines. A Template String itself is not executed itself, but it - contain expressions and need additional template rendering logic - applied to it to be useful. + """Template strings are allowed by default. + + As Template strings are a very basic template mechanism, that needs + additional rendering logic to be useful, they are not blocked by + default. Those rendering logic would be affected by RestrictedPython as well. - TODO: Deeper review of security implications of template strings. TODO: Change Type Annotation to ast.TemplateStr when Support for Python 3.13 is dropped. """ @@ -595,12 +594,13 @@ def visit_TemplateStr(self, node: ast.AST) -> ast.AST: return self.node_contents_visit(node) def visit_Interpolation(self, node: ast.AST) -> ast.AST: - """Interpolations are not allowed by default. - As Interpolations are part of Template Strings, they will not be - reached in the context of RestrictedPython as Template Strings - ‚‚are not allowed. + """Interpolations are allowed by default. + As Interpolations are part of Template Strings, they are needed + to be reached in the context of RestrictedPython as Template Strings + are allowed. As a user has to provide additional rendering logic + to make use of Template Strings, the security implications of + Interpolations are limited in the context of RestrictedPython. - TODO: Deeper review of security implications of interpolated strings. TODO: Change Type Annotation to ast.Interpolation when Support for Python 3.13 is dropped. """ diff --git a/tests/transformer/test_tstring.py b/tests/transformer/test_tstring.py index c53a3df9..7aa187e5 100644 --- a/tests/transformer/test_tstring.py +++ b/tests/transformer/test_tstring.py @@ -97,7 +97,7 @@ def test_t_string_self_documenting_expressions(): '_print_': PrintCollector, '_getattr_': default_guarded_getattr, '_getiter_': default_guarded_getiter, - '_inplacevar_': lambda x: x, + '_inplacevar_': lambda x, y, z: y + z, } exec(result.code, glb) - assert glb['_print']() == "user='eric_idle' member_since=datetime.date(1975, 7, 31)\n" # NOQA: E501 + assert glb['_print']() == "the user ERIC_IDLE is a member since 1975-07-31\n" # NOQA: E501 From fd0328a5cfaef06f6c93c273c0e62b7735719c42 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci-lite[bot]" <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Date: Sat, 18 Oct 2025 14:38:56 +0000 Subject: [PATCH 17/40] Apply pre-commit code formatting --- src/RestrictedPython/transformer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/RestrictedPython/transformer.py b/src/RestrictedPython/transformer.py index 78c67156..2826b3ac 100644 --- a/src/RestrictedPython/transformer.py +++ b/src/RestrictedPython/transformer.py @@ -595,7 +595,7 @@ def visit_TemplateStr(self, node: ast.AST) -> ast.AST: def visit_Interpolation(self, node: ast.AST) -> ast.AST: """Interpolations are allowed by default. - As Interpolations are part of Template Strings, they are needed + As Interpolations are part of Template Strings, they are needed to be reached in the context of RestrictedPython as Template Strings are allowed. As a user has to provide additional rendering logic to make use of Template Strings, the security implications of From aceca07c2fd8246aec37f0ced75351905353e201 Mon Sep 17 00:00:00 2001 From: Alexander Loechel Date: Sat, 18 Oct 2025 17:41:49 +0300 Subject: [PATCH 18/40] conditional import --- tests/transformer/test_tstring.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/transformer/test_tstring.py b/tests/transformer/test_tstring.py index 7aa187e5..de0ed356 100644 --- a/tests/transformer/test_tstring.py +++ b/tests/transformer/test_tstring.py @@ -1,5 +1,3 @@ -from string.templatelib import Template - import pytest from RestrictedPython import compile_restricted_exec @@ -9,6 +7,10 @@ from RestrictedPython.PrintCollector import PrintCollector +if IS_PY314_OR_GREATER: + from string.templatelib import Template + + @pytest.mark.skipif( not IS_PY314_OR_GREATER, reason="t-strings were added in Python 3.14.", From 3533c0f9d09601d1d8cb2a6cc5f9c58e3d11fd74 Mon Sep 17 00:00:00 2001 From: Alexander Loechel Date: Sat, 18 Oct 2025 17:45:08 +0300 Subject: [PATCH 19/40] fix coverage numbers --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 89faa35f..5fa3c8e2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ branch = true source = ["RestrictedPython"] [tool.coverage.report] -fail_under = 97.3 +fail_under = 97.0 precision = 2 ignore_errors = true show_missing = true From e7503315ad2050c6cf94fdfbf96934020087697d Mon Sep 17 00:00:00 2001 From: Alexander Loechel Date: Sat, 18 Oct 2025 18:17:35 +0300 Subject: [PATCH 20/40] readd Python 3.9 support --- .github/workflows/tests.yml | 1 + .meta.toml | 2 +- setup.py | 2 +- src/RestrictedPython/compile.py | 13 ++++++++++--- tox.ini | 1 + 5 files changed, 14 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5abf4912..dd13cef3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -25,6 +25,7 @@ jobs: config: # [Python version, tox env] - ["3.14", "release-check"] + - ["3.9", "py39"] - ["3.10", "py310"] - ["3.11", "py311"] - ["3.12", "py312"] diff --git a/.meta.toml b/.meta.toml index f7000fbe..3ba9ecbd 100644 --- a/.meta.toml +++ b/.meta.toml @@ -47,7 +47,7 @@ testenv-additional = [ " coverage combine", " coverage html", " coverage report -m --fail-under=100", - "depends = py310,py311,py312,py313,py314,py314-datetime,coverage", + "depends = py39,py310,py311,py312,py313,py314,py314-datetime,coverage", ] coverage-command = "pytest --cov=src --cov=tests --cov-report= tests {posargs}" coverage-setenv = [ diff --git a/setup.py b/setup.py index 6e19db74..c01597e1 100644 --- a/setup.py +++ b/setup.py @@ -61,7 +61,7 @@ def read(*rnames): packages=find_packages('src'), package_dir={'': 'src'}, install_requires=[], - python_requires=">=3.10, <3.15", + python_requires=">=3.9, <3.15", extras_require={ 'test': ['pytest', 'pytest-mock'], 'typecheck': ['mypy', 'typeshed'], diff --git a/src/RestrictedPython/compile.py b/src/RestrictedPython/compile.py index 83050a8e..07fd180a 100644 --- a/src/RestrictedPython/compile.py +++ b/src/RestrictedPython/compile.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import ast import warnings from ast import Expression @@ -8,15 +10,20 @@ from os import PathLike from typing import Any from typing import Literal -from typing import TypeAlias from RestrictedPython._compat import IS_CPYTHON +from RestrictedPython._compat import IS_PY310_OR_GREATER from RestrictedPython.transformer import RestrictingNodeTransformer -# Temporary workaround for missing _typeshed -ReadableBuffer: TypeAlias = bytes | bytearray +if IS_PY310_OR_GREATER: + from typing import TypeAlias + # Temporary workaround for missing _typeshed + ReadableBuffer: TypeAlias = bytes | bytearray +else: + from typing_extensions import TypeAlias # type: ignore[import] + ReadableBuffer: TypeAlias = bytes | bytearray # type: ignore[no-redef] CompileResult = namedtuple( 'CompileResult', 'code, errors, warnings, used_names') diff --git a/tox.ini b/tox.ini index 71e73aa6..1f3fce15 100644 --- a/tox.ini +++ b/tox.ini @@ -5,6 +5,7 @@ minversion = 3.18 envlist = release-check lint + py39 py310 py311 py312 From 5b8285cdf901c6912b4082db9aa546f75f29a5c0 Mon Sep 17 00:00:00 2001 From: Jens Vagelpohl Date: Sun, 19 Oct 2025 16:29:19 +0200 Subject: [PATCH 21/40] - updating package files with zope/meta and fixing tests --- .github/workflows/tests.yml | 15 +++++++-------- .meta.toml | 3 ++- .pre-commit-config.yaml | 2 +- pyproject.toml | 2 +- setup.py | 2 +- src/RestrictedPython/compile.py | 2 +- tox.ini | 7 +++---- 7 files changed, 16 insertions(+), 17 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3717ce65..3dc22730 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -24,20 +24,19 @@ jobs: - ["windows", "windows-latest"] config: # [Python version, tox env] - - ["3.14", "release-check"] - - ["3.9", "py39"] + - ["3.11", "release-check"] - ["3.10", "py310"] - ["3.11", "py311"] - ["3.12", "py312"] - ["3.13", "py313"] - ["3.14", "py314"] - - ["3.14", "docs"] - - ["3.14", "coverage"] - - ["3.14", "py314-datetime"] + - ["3.11", "docs"] + - ["3.11", "coverage"] + - ["3.11", "py311-datetime"] exclude: - - { os: ["windows", "windows-latest"], config: ["3.14", "release-check"] } - - { os: ["windows", "windows-latest"], config: ["3.14", "docs"] } - - { os: ["windows", "windows-latest"], config: ["3.14", "coverage"] } + - { os: ["windows", "windows-latest"], config: ["3.11", "release-check"] } + - { os: ["windows", "windows-latest"], config: ["3.11", "docs"] } + - { os: ["windows", "windows-latest"], config: ["3.11", "coverage"] } runs-on: ${{ matrix.os[1] }} if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name diff --git a/.meta.toml b/.meta.toml index e1f29d4d..35b7ea04 100644 --- a/.meta.toml +++ b/.meta.toml @@ -2,7 +2,7 @@ # https://github.com/zopefoundation/meta/tree/master/config/pure-python [meta] template = "pure-python" -commit-id = "72252845" +commit-id = "9d049229" [python] with-pypy = false @@ -11,6 +11,7 @@ with-sphinx-doctests = true with-windows = true with-future-python = true with-macos = false +oldest-python = "3.10" [tox] use-flake8 = true diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cbb2541a..5f86332c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,7 +15,7 @@ repos: rev: v3.21.0 hooks: - id: pyupgrade - args: [--py39-plus] + args: [--py310-plus] - repo: https://github.com/isidentical/teyit rev: 0.4.3 hooks: diff --git a/pyproject.toml b/pyproject.toml index 5fa3c8e2..7b94ef64 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ branch = true source = ["RestrictedPython"] [tool.coverage.report] -fail_under = 97.0 +fail_under = 97.2 precision = 2 ignore_errors = true show_missing = true diff --git a/setup.py b/setup.py index 5a3f97ff..dddd6d9d 100644 --- a/setup.py +++ b/setup.py @@ -61,7 +61,7 @@ def read(*rnames): packages=find_packages('src'), package_dir={'': 'src'}, install_requires=[], - python_requires=">=3.9, <3.15", + python_requires=">=3.10, <3.15", extras_require={ 'test': ['pytest', 'pytest-mock'], 'typecheck': ['mypy', 'typeshed'], diff --git a/src/RestrictedPython/compile.py b/src/RestrictedPython/compile.py index 07fd180a..56f3553c 100644 --- a/src/RestrictedPython/compile.py +++ b/src/RestrictedPython/compile.py @@ -22,7 +22,7 @@ # Temporary workaround for missing _typeshed ReadableBuffer: TypeAlias = bytes | bytearray else: - from typing_extensions import TypeAlias # type: ignore[import] + from typing import TypeAlias # type: ignore[import] ReadableBuffer: TypeAlias = bytes | bytearray # type: ignore[no-redef] CompileResult = namedtuple( diff --git a/tox.ini b/tox.ini index 1f3fce15..990f2601 100644 --- a/tox.ini +++ b/tox.ini @@ -5,7 +5,6 @@ minversion = 3.18 envlist = release-check lint - py39 py310 py311 py312 @@ -13,7 +12,7 @@ envlist = py314 docs coverage - py314-datetime + py311-datetime combined-coverage [testenv] @@ -52,7 +51,7 @@ commands = coverage combine coverage html coverage report -m --fail-under=100 -depends = py39,py310,py311,py311-datetime,py312,py313,py314,coverage +depends = py310,py311,py311-datetime,py312,py313,py314,coverage [testenv:setuptools-latest] basepython = python3 @@ -92,7 +91,7 @@ commands = pre-commit run --all-files --show-diff-on-failure [testenv:docs] -basepython = python3.14 +basepython = python3 skip_install = false commands_pre = commands = From ac53335291d03319dee9ba9786d67a28aebb2f65 Mon Sep 17 00:00:00 2001 From: Jens Vagelpohl Date: Sun, 19 Oct 2025 16:34:12 +0200 Subject: [PATCH 22/40] - fix last test --- .meta.toml | 2 +- setup.py | 1 + src/RestrictedPython/compile.py | 12 +++--------- tests/transformer/test_tstring.py | 1 - 4 files changed, 5 insertions(+), 11 deletions(-) diff --git a/.meta.toml b/.meta.toml index 35b7ea04..f9192772 100644 --- a/.meta.toml +++ b/.meta.toml @@ -48,7 +48,7 @@ testenv-additional = [ " coverage combine", " coverage html", " coverage report -m --fail-under=100", - "depends = py39,py310,py311,py312,py313,py314,py314-datetime,coverage", + "depends = py310,py311,py311-datetime,py312,py313,py314,coverage", ] coverage-command = "pytest --cov=src --cov=tests --cov-report= tests {posargs}" coverage-setenv = [ diff --git a/setup.py b/setup.py index dddd6d9d..b8b6f650 100644 --- a/setup.py +++ b/setup.py @@ -35,6 +35,7 @@ def read(*rnames): long_description_content_type='text/x-rst', classifiers=[ 'Development Status :: 6 - Mature', + 'License :: OSI Approved :: Zope Public License', 'Programming Language :: Python', 'Operating System :: OS Independent', 'Programming Language :: Python :: 3', diff --git a/src/RestrictedPython/compile.py b/src/RestrictedPython/compile.py index 56f3553c..8d29f3a9 100644 --- a/src/RestrictedPython/compile.py +++ b/src/RestrictedPython/compile.py @@ -10,20 +10,14 @@ from os import PathLike from typing import Any from typing import Literal +from typing import TypeAlias from RestrictedPython._compat import IS_CPYTHON -from RestrictedPython._compat import IS_PY310_OR_GREATER from RestrictedPython.transformer import RestrictingNodeTransformer -if IS_PY310_OR_GREATER: - from typing import TypeAlias - - # Temporary workaround for missing _typeshed - ReadableBuffer: TypeAlias = bytes | bytearray -else: - from typing import TypeAlias # type: ignore[import] - ReadableBuffer: TypeAlias = bytes | bytearray # type: ignore[no-redef] +# Temporary workaround for missing _typeshed +ReadableBuffer: TypeAlias = bytes | bytearray CompileResult = namedtuple( 'CompileResult', 'code, errors, warnings, used_names') diff --git a/tests/transformer/test_tstring.py b/tests/transformer/test_tstring.py index 87cfd968..3a2bf2d5 100644 --- a/tests/transformer/test_tstring.py +++ b/tests/transformer/test_tstring.py @@ -79,7 +79,6 @@ def test_t_string_self_documenting_expressions(): t_string_self_documenting_expressions_example, ) assert result.errors == () - assert result.warnings == [] assert result.code is not None glb = { From 0e1408cfe9ec30396c56297cd381ed41b424c9a6 Mon Sep 17 00:00:00 2001 From: Jens Vagelpohl Date: Mon, 20 Oct 2025 16:27:38 +0200 Subject: [PATCH 23/40] - expand change log entry to be more clear. --- CHANGES.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 0f72aed6..9a50d664 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,7 +4,8 @@ Changes 8.2 (unreleased) ---------------- -- Added Type Annotations +- Add type annotations to the package code. + For clarification, restricted Python code does not support type annotations. 8.1 (2025-10-19) From 0a292d336e08bda0aedbca71aa7d58f7803f6528 Mon Sep 17 00:00:00 2001 From: zedzhen <59135268+zedzhen@users.noreply.github.com> Date: Tue, 2 Jun 2026 12:17:21 +0300 Subject: [PATCH 24/40] fix return type --- src/RestrictedPython/compile.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/RestrictedPython/compile.py b/src/RestrictedPython/compile.py index 8d29f3a9..19c75d69 100644 --- a/src/RestrictedPython/compile.py +++ b/src/RestrictedPython/compile.py @@ -8,6 +8,7 @@ from ast import NodeTransformer from collections import namedtuple from os import PathLike +from types import CodeType from typing import Any from typing import Literal from typing import TypeAlias @@ -205,7 +206,7 @@ def compile_restricted( flags: int = 0, dont_inherit: bool = False, policy: NodeTransformer = RestrictingNodeTransformer, -) -> CompileResult: +) -> CodeType: """Replacement for the built-in compile() function. policy ... `ast.NodeTransformer` class defining the restrictions. From 444760ed6854a1ca1136ccaca5541acb12ef34ed Mon Sep 17 00:00:00 2001 From: zedzhen <59135268+zedzhen@users.noreply.github.com> Date: Tue, 2 Jun 2026 12:29:54 +0300 Subject: [PATCH 25/40] style update (autopep8) --- src/RestrictedPython/transformer.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/RestrictedPython/transformer.py b/src/RestrictedPython/transformer.py index 40262a0f..7480664d 100644 --- a/src/RestrictedPython/transformer.py +++ b/src/RestrictedPython/transformer.py @@ -370,10 +370,10 @@ def gen_del_stmt(self, name_to_del: str) -> ast.Delete: return ast.Delete(targets=[ast.Name(name_to_del, ast.Del())]) def check_name( - self, - node: ast.AST, - name: str, - allow_magic_methods: bool = False) -> None: + self, + node: ast.AST, + name: str, + allow_magic_methods: bool = False) -> None: """Check names if they are allowed. If ``allow_magic_methods is True`` names in `ALLOWED_FUNC_NAMES` From 0e92a0b34568eedd7228de491245d92b6980bae7 Mon Sep 17 00:00:00 2001 From: zedzhen <59135268+zedzhen@users.noreply.github.com> Date: Tue, 2 Jun 2026 12:30:29 +0300 Subject: [PATCH 26/40] add type hints for RestrictingNodeTransformer attributes --- src/RestrictedPython/transformer.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/RestrictedPython/transformer.py b/src/RestrictedPython/transformer.py index 7480664d..c3377ddf 100644 --- a/src/RestrictedPython/transformer.py +++ b/src/RestrictedPython/transformer.py @@ -152,6 +152,9 @@ def new_print_scope(self): class RestrictingNodeTransformer(ast.NodeTransformer): + errors: list[str] + warnings: list[str] + used_names: dict[str, str] def __init__(self, errors: list[str] | None = None, From be5ad38c358130bff9c12500e7ee6b92f92fa040 Mon Sep 17 00:00:00 2001 From: zedzhen <59135268+zedzhen@users.noreply.github.com> Date: Tue, 2 Jun 2026 12:31:30 +0300 Subject: [PATCH 27/40] update type hints for RestrictingNodeTransformer methods --- src/RestrictedPython/transformer.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/RestrictedPython/transformer.py b/src/RestrictedPython/transformer.py index c3377ddf..dcd7fd6b 100644 --- a/src/RestrictedPython/transformer.py +++ b/src/RestrictedPython/transformer.py @@ -159,8 +159,7 @@ class RestrictingNodeTransformer(ast.NodeTransformer): def __init__(self, errors: list[str] | None = None, warnings: list[str] | None = None, - used_names: dict[str, - str] | None = None): + used_names: dict[str, str] | None = None): super().__init__() self.errors = [] if errors is None else errors self.warnings = [] if warnings is None else warnings @@ -490,7 +489,7 @@ def node_contents_visit(self, node: ast.AST) -> ast.AST: # ast for Literals - def visit_Constant(self, node: ast.Constant) -> ast.Constant | None: + def visit_Constant(self, node: ast.Constant) -> ast.Constant: """Allow constant literals. Constant replaces Num, Str, Bytes, NameConstant and Ellipsis in From 37a4beceafd14c7bbaee19810cee7657a10f956b Mon Sep 17 00:00:00 2001 From: zedzhen <59135268+zedzhen@users.noreply.github.com> Date: Tue, 2 Jun 2026 12:42:10 +0300 Subject: [PATCH 28/40] update type hint for `policy` argument --- src/RestrictedPython/compile.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/RestrictedPython/compile.py b/src/RestrictedPython/compile.py index 19c75d69..a142aa1b 100644 --- a/src/RestrictedPython/compile.py +++ b/src/RestrictedPython/compile.py @@ -37,7 +37,7 @@ def _compile_restricted_mode( mode: Literal["exec", "eval", "single"] = "exec", flags: int = 0, dont_inherit: bool = False, - policy: NodeTransformer = RestrictingNodeTransformer, + policy: type[NodeTransformer] | None = RestrictingNodeTransformer, ) -> CompileResult: if not IS_CPYTHON: @@ -97,7 +97,7 @@ def compile_restricted_exec( filename: str | ReadableBuffer | PathLike[Any] = '', flags: int = 0, dont_inherit: bool = False, - policy: NodeTransformer = RestrictingNodeTransformer, + policy: type[NodeTransformer] | None = RestrictingNodeTransformer, ) -> CompileResult: """Compile restricted for the mode `exec`.""" return _compile_restricted_mode( @@ -114,7 +114,7 @@ def compile_restricted_eval( filename: str | ReadableBuffer | PathLike[Any] = '', flags: int = 0, dont_inherit: bool = False, - policy: NodeTransformer = RestrictingNodeTransformer, + policy: type[NodeTransformer] | None = RestrictingNodeTransformer, ) -> CompileResult: """Compile restricted for the mode `eval`.""" return _compile_restricted_mode( @@ -131,7 +131,7 @@ def compile_restricted_single( filename: str | ReadableBuffer | PathLike[Any] = '', flags: int = 0, dont_inherit: bool = False, - policy: NodeTransformer = RestrictingNodeTransformer, + policy: type[NodeTransformer] | None = RestrictingNodeTransformer, ) -> CompileResult: """Compile restricted for the mode `single`.""" return _compile_restricted_mode( @@ -151,7 +151,7 @@ def compile_restricted_function( globalize=None, # List of globals (e.g. ['here', 'context', ...]) flags: int = 0, dont_inherit: bool = False, - policy: ast.NodeTransformer = RestrictingNodeTransformer, + policy: type[NodeTransformer] | None = RestrictingNodeTransformer, ) -> CompileResult: """Compile a restricted code object for a function. @@ -205,7 +205,7 @@ def compile_restricted( mode: str = 'exec', flags: int = 0, dont_inherit: bool = False, - policy: NodeTransformer = RestrictingNodeTransformer, + policy: type[NodeTransformer] | None = RestrictingNodeTransformer, ) -> CodeType: """Replacement for the built-in compile() function. From 55985ad898ab6c44d1ad1def143964ec12a04b65 Mon Sep 17 00:00:00 2001 From: zedzhen <59135268+zedzhen@users.noreply.github.com> Date: Tue, 2 Jun 2026 12:42:34 +0300 Subject: [PATCH 29/40] add type hints for RestrictingNodeTransformer.visit_Interpolation --- src/RestrictedPython/transformer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/RestrictedPython/transformer.py b/src/RestrictedPython/transformer.py index dcd7fd6b..bc1a09ad 100644 --- a/src/RestrictedPython/transformer.py +++ b/src/RestrictedPython/transformer.py @@ -535,7 +535,7 @@ def visit_TemplateStr(self, node: ast.AST) -> ast.AST: """ return self.node_contents_visit(node) - def visit_Interpolation(self, node): + def visit_Interpolation(self, node: ast.AST) -> ast.AST: """Interpolations are allowed by default. As Interpolations are part of Template Strings, they are needed From 99fd0beebdd5f4bb0f496f82fb709fea359c9863 Mon Sep 17 00:00:00 2001 From: zedzhen <59135268+zedzhen@users.noreply.github.com> Date: Tue, 2 Jun 2026 12:50:50 +0300 Subject: [PATCH 30/40] update CHANGES.rst --- CHANGES.rst | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index b0a9391d..c3df6302 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,7 +4,8 @@ Changes 8.3 (unreleased) ---------------- -- Nothing changed yet. +- Add type annotations to the package code. + For clarification, restricted Python code does not support type annotations. 8.3a1.dev0 (2026-05-29) @@ -16,9 +17,6 @@ Changes 8.2 (2026-05-29) ---------------- -- Add type annotations to the package code. - For clarification, restricted Python code does not support type annotations. - - Remove documentation that appears to promote unsupported direct guards usage. - Move package metadata from setup.py to pyproject.toml. From fd28e281481d63f9c4ecb3f39bf5e18313c1c1ae Mon Sep 17 00:00:00 2001 From: zedzhen <59135268+zedzhen@users.noreply.github.com> Date: Tue, 2 Jun 2026 13:00:33 +0300 Subject: [PATCH 31/40] add types to CompileResult changing the return type in compile_restricted_function to match the rest of the functions --- src/RestrictedPython/compile.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/RestrictedPython/compile.py b/src/RestrictedPython/compile.py index a142aa1b..01d95776 100644 --- a/src/RestrictedPython/compile.py +++ b/src/RestrictedPython/compile.py @@ -6,11 +6,13 @@ from ast import Interactive from ast import Module from ast import NodeTransformer -from collections import namedtuple +from collections.abc import Mapping +from collections.abc import Sequence from os import PathLike from types import CodeType from typing import Any from typing import Literal +from typing import NamedTuple from typing import TypeAlias from RestrictedPython._compat import IS_CPYTHON @@ -20,8 +22,14 @@ # Temporary workaround for missing _typeshed ReadableBuffer: TypeAlias = bytes | bytearray -CompileResult = namedtuple( - 'CompileResult', 'code, errors, warnings, used_names') + +class CompileResult(NamedTuple): + code: CodeType | None + errors: Sequence[str] + warnings: Sequence[str] + used_names: Mapping[str, str] + + syntax_error_template = ( 'Line {lineno}: {type}: {msg} at statement: {statement!r}') @@ -168,7 +176,7 @@ def compile_restricted_function( msg=v.msg, statement=v.text.strip() if v.text else None) return CompileResult( - code=None, errors=(error,), warnings=(), used_names=()) + code=None, errors=(error,), warnings=(), used_names={}) # The compiled code is actually executed inside a function # (that is called when the code is called) so reading and assigning to a From 501299a0ae243f6239fd9bcdd07fa4974207bd48 Mon Sep 17 00:00:00 2001 From: zedzhen <59135268+zedzhen@users.noreply.github.com> Date: Tue, 2 Jun 2026 13:32:21 +0300 Subject: [PATCH 32/40] fix type hint for used_names --- src/RestrictedPython/compile.py | 2 +- src/RestrictedPython/transformer.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/RestrictedPython/compile.py b/src/RestrictedPython/compile.py index 01d95776..cf0035f9 100644 --- a/src/RestrictedPython/compile.py +++ b/src/RestrictedPython/compile.py @@ -27,7 +27,7 @@ class CompileResult(NamedTuple): code: CodeType | None errors: Sequence[str] warnings: Sequence[str] - used_names: Mapping[str, str] + used_names: Mapping[str, bool] syntax_error_template = ( diff --git a/src/RestrictedPython/transformer.py b/src/RestrictedPython/transformer.py index bc1a09ad..e2518fcc 100644 --- a/src/RestrictedPython/transformer.py +++ b/src/RestrictedPython/transformer.py @@ -154,12 +154,12 @@ def new_print_scope(self): class RestrictingNodeTransformer(ast.NodeTransformer): errors: list[str] warnings: list[str] - used_names: dict[str, str] + used_names: dict[str, bool] def __init__(self, errors: list[str] | None = None, warnings: list[str] | None = None, - used_names: dict[str, str] | None = None): + used_names: dict[str, bool] | None = None): super().__init__() self.errors = [] if errors is None else errors self.warnings = [] if warnings is None else warnings From b56ad855c758333e004806f16048e54dbd1df207 Mon Sep 17 00:00:00 2001 From: zedzhen <59135268+zedzhen@users.noreply.github.com> Date: Tue, 2 Jun 2026 13:41:22 +0300 Subject: [PATCH 33/40] add None to the return type hints --- src/RestrictedPython/transformer.py | 191 ++++++++++++++-------------- 1 file changed, 96 insertions(+), 95 deletions(-) diff --git a/src/RestrictedPython/transformer.py b/src/RestrictedPython/transformer.py index e2518fcc..133544cb 100644 --- a/src/RestrictedPython/transformer.py +++ b/src/RestrictedPython/transformer.py @@ -463,7 +463,7 @@ def inject_print_collector(self, node: ast.AST, position: int = 0) -> None: # Special Functions for an ast.NodeTransformer - def generic_visit(self, node: ast.AST) -> ast.AST: + def generic_visit(self, node: ast.AST) -> ast.AST | None: """Reject ast nodes which do not have a corresponding `visit_` method. This is needed to prevent new ast nodes from new Python versions to be @@ -489,7 +489,7 @@ def node_contents_visit(self, node: ast.AST) -> ast.AST: # ast for Literals - def visit_Constant(self, node: ast.Constant) -> ast.Constant: + def visit_Constant(self, node: ast.Constant) -> ast.AST | None: """Allow constant literals. Constant replaces Num, Str, Bytes, NameConstant and Ellipsis in @@ -498,31 +498,31 @@ def visit_Constant(self, node: ast.Constant) -> ast.Constant: """ return self.node_contents_visit(node) - def visit_Interactive(self, node: ast.Interactive) -> ast.AST: + def visit_Interactive(self, node: ast.Interactive) -> ast.AST | None: """Allow single mode without restrictions.""" return self.node_contents_visit(node) - def visit_List(self, node: ast.List) -> ast.AST: + def visit_List(self, node: ast.List) -> ast.AST | None: """Allow list literals without restrictions.""" return self.node_contents_visit(node) - def visit_Tuple(self, node: ast.Tuple) -> ast.AST: + def visit_Tuple(self, node: ast.Tuple) -> ast.AST | None: """Allow tuple literals without restrictions.""" return self.node_contents_visit(node) - def visit_Set(self, node: ast.Set) -> ast.AST: + def visit_Set(self, node: ast.Set) -> ast.AST | None: """Allow set literals without restrictions.""" return self.node_contents_visit(node) - def visit_Dict(self, node: ast.Dict) -> ast.AST: + def visit_Dict(self, node: ast.Dict) -> ast.AST | None: """Allow dict literals without restrictions.""" return self.node_contents_visit(node) - def visit_FormattedValue(self, node: ast.FormattedValue) -> ast.AST: + def visit_FormattedValue(self, node: ast.FormattedValue) -> ast.AST | None: """Allow f-strings without restrictions.""" return self.node_contents_visit(node) - def visit_TemplateStr(self, node: ast.AST) -> ast.AST: + def visit_TemplateStr(self, node: ast.AST) -> ast.AST | None: """Template strings are allowed by default. As Template strings are a very basic template mechanism, that needs @@ -535,7 +535,7 @@ def visit_TemplateStr(self, node: ast.AST) -> ast.AST: """ return self.node_contents_visit(node) - def visit_Interpolation(self, node: ast.AST) -> ast.AST: + def visit_Interpolation(self, node: ast.AST) -> ast.AST | None: """Interpolations are allowed by default. As Interpolations are part of Template Strings, they are needed @@ -549,7 +549,7 @@ def visit_Interpolation(self, node: ast.AST) -> ast.AST: """ return self.node_contents_visit(node) - def visit_JoinedStr(self, node: ast.JoinedStr) -> ast.AST: + def visit_JoinedStr(self, node: ast.JoinedStr) -> ast.AST | None: """Allow joined string without restrictions.""" return self.node_contents_visit(node) @@ -595,19 +595,19 @@ def visit_Load(self, node: ast.Load) -> ast.Load | None: """ return self.node_contents_visit(node) - def visit_Store(self, node: ast.Store) -> ast.AST: + def visit_Store(self, node: ast.Store) -> ast.AST | None: """ """ return self.node_contents_visit(node) - def visit_Del(self, node: ast.Del) -> ast.Del: + def visit_Del(self, node: ast.Del) -> ast.AST | None: """ """ return self.node_contents_visit(node) - def visit_Starred(self, node: ast.Starred) -> ast.AST: + def visit_Starred(self, node: ast.Starred) -> ast.AST | None: """ """ @@ -615,18 +615,18 @@ def visit_Starred(self, node: ast.Starred) -> ast.AST: # Expressions - def visit_Expression(self, node: ast.Expression) -> ast.AST: + def visit_Expression(self, node: ast.Expression) -> ast.AST | None: """Allow Expression statements without restrictions. They are in the AST when using the `eval` compile mode. """ return self.node_contents_visit(node) - def visit_Expr(self, node: ast.Expr) -> ast.AST: + def visit_Expr(self, node: ast.Expr) -> ast.AST | None: """Allow Expr statements (any expression) without restrictions.""" return self.node_contents_visit(node) - def visit_UnaryOp(self, node: ast.UnaryOp) -> ast.AST: + def visit_UnaryOp(self, node: ast.UnaryOp) -> ast.AST | None: """ UnaryOp (Unary Operations) is the overall element for: * Not --> which should be allowed @@ -635,135 +635,135 @@ def visit_UnaryOp(self, node: ast.UnaryOp) -> ast.AST: """ return self.node_contents_visit(node) - def visit_UAdd(self, node: ast.UAdd) -> ast.AST: + def visit_UAdd(self, node: ast.UAdd) -> ast.AST | None: """Allow positive notation of variables. (e.g. +var)""" return self.node_contents_visit(node) - def visit_USub(self, node: ast.USub) -> ast.AST: + def visit_USub(self, node: ast.USub) -> ast.AST | None: """Allow negative notation of variables. (e.g. -var)""" return self.node_contents_visit(node) - def visit_Not(self, node: ast.Not) -> ast.AST: + def visit_Not(self, node: ast.Not) -> ast.AST | None: """Allow the `not` operator.""" return self.node_contents_visit(node) - def visit_Invert(self, node: ast.Invert) -> ast.AST: + def visit_Invert(self, node: ast.Invert) -> ast.AST | None: """Allow `~` expressions.""" return self.node_contents_visit(node) - def visit_BinOp(self, node: ast.BinOp) -> ast.AST: + def visit_BinOp(self, node: ast.BinOp) -> ast.AST | None: """Allow binary operations.""" return self.node_contents_visit(node) - def visit_Add(self, node: ast.Add) -> ast.AST: + def visit_Add(self, node: ast.Add) -> ast.AST | None: """Allow `+` expressions.""" return self.node_contents_visit(node) - def visit_Sub(self, node: ast.Sub) -> ast.AST: + def visit_Sub(self, node: ast.Sub) -> ast.AST | None: """Allow `-` expressions.""" return self.node_contents_visit(node) - def visit_Mult(self, node: ast.Mult) -> ast.AST: + def visit_Mult(self, node: ast.Mult) -> ast.AST | None: """Allow `*` expressions.""" return self.node_contents_visit(node) - def visit_Div(self, node: ast.Div) -> ast.AST: + def visit_Div(self, node: ast.Div) -> ast.AST | None: """Allow `/` expressions.""" return self.node_contents_visit(node) - def visit_FloorDiv(self, node: ast.FloorDiv) -> ast.AST: + def visit_FloorDiv(self, node: ast.FloorDiv) -> ast.AST | None: """Allow `//` expressions.""" return self.node_contents_visit(node) - def visit_Mod(self, node: ast.Mod) -> ast.AST: + def visit_Mod(self, node: ast.Mod) -> ast.AST | None: """Allow `%` expressions.""" return self.node_contents_visit(node) - def visit_Pow(self, node: ast.Pow) -> ast.AST: + def visit_Pow(self, node: ast.Pow) -> ast.AST | None: """Allow `**` expressions.""" return self.node_contents_visit(node) - def visit_LShift(self, node: ast.LShift) -> ast.AST: + def visit_LShift(self, node: ast.LShift) -> ast.AST | None: """Allow `<<` expressions.""" return self.node_contents_visit(node) - def visit_RShift(self, node: ast.RShift) -> ast.AST: + def visit_RShift(self, node: ast.RShift) -> ast.AST | None: """Allow `>>` expressions.""" return self.node_contents_visit(node) - def visit_BitOr(self, node: ast.BitOr) -> ast.AST: + def visit_BitOr(self, node: ast.BitOr) -> ast.AST | None: """Allow `|` expressions.""" return self.node_contents_visit(node) - def visit_BitXor(self, node: ast.BitXor) -> ast.AST: + def visit_BitXor(self, node: ast.BitXor) -> ast.AST | None: """Allow `^` expressions.""" return self.node_contents_visit(node) - def visit_BitAnd(self, node: ast.BitAnd) -> ast.AST: + def visit_BitAnd(self, node: ast.BitAnd) -> ast.AST | None: """Allow `&` expressions.""" return self.node_contents_visit(node) - def visit_MatMult(self, node: ast.MatMult) -> ast.AST: + def visit_MatMult(self, node: ast.MatMult) -> ast.AST | None: """Allow multiplication (`@`).""" return self.node_contents_visit(node) - def visit_BoolOp(self, node: ast.BoolOp) -> ast.AST: + def visit_BoolOp(self, node: ast.BoolOp) -> ast.AST | None: """Allow bool operator without restrictions.""" return self.node_contents_visit(node) - def visit_And(self, node: ast.And) -> ast.AST: + def visit_And(self, node: ast.And) -> ast.AST | None: """Allow bool operator `and` without restrictions.""" return self.node_contents_visit(node) - def visit_Or(self, node: ast.Or) -> ast.AST: + def visit_Or(self, node: ast.Or) -> ast.AST | None: """Allow bool operator `or` without restrictions.""" return self.node_contents_visit(node) - def visit_Compare(self, node: ast.Compare) -> ast.AST: + def visit_Compare(self, node: ast.Compare) -> ast.AST | None: """Allow comparison expressions without restrictions.""" return self.node_contents_visit(node) - def visit_Eq(self, node: ast.Eq) -> ast.AST: + def visit_Eq(self, node: ast.Eq) -> ast.AST | None: """Allow == expressions.""" return self.node_contents_visit(node) - def visit_NotEq(self, node: ast.NotEq) -> ast.AST: + def visit_NotEq(self, node: ast.NotEq) -> ast.AST | None: """Allow != expressions.""" return self.node_contents_visit(node) - def visit_Lt(self, node: ast.Lt) -> ast.AST: + def visit_Lt(self, node: ast.Lt) -> ast.AST | None: """Allow < expressions.""" return self.node_contents_visit(node) - def visit_LtE(self, node: ast.LtE) -> ast.AST: + def visit_LtE(self, node: ast.LtE) -> ast.AST | None: """Allow <= expressions.""" return self.node_contents_visit(node) - def visit_Gt(self, node: ast.Gt) -> ast.AST: + def visit_Gt(self, node: ast.Gt) -> ast.AST | None: """Allow > expressions.""" return self.node_contents_visit(node) - def visit_GtE(self, node: ast.GtE) -> ast.AST: + def visit_GtE(self, node: ast.GtE) -> ast.AST | None: """Allow >= expressions.""" return self.node_contents_visit(node) - def visit_Is(self, node: ast.Is) -> ast.AST: + def visit_Is(self, node: ast.Is) -> ast.AST | None: """Allow `is` expressions.""" return self.node_contents_visit(node) - def visit_IsNot(self, node: ast.IsNot) -> ast.AST: + def visit_IsNot(self, node: ast.IsNot) -> ast.AST | None: """Allow `is not` expressions.""" return self.node_contents_visit(node) - def visit_In(self, node: ast.In) -> ast.AST: + def visit_In(self, node: ast.In) -> ast.AST | None: """Allow `in` expressions.""" return self.node_contents_visit(node) - def visit_NotIn(self, node: ast.NotIn) -> ast.AST: + def visit_NotIn(self, node: ast.NotIn) -> ast.AST | None: """Allow `not in` expressions.""" return self.node_contents_visit(node) - def visit_Call(self, node: ast.Call) -> ast.AST: + def visit_Call(self, node: ast.Call) -> ast.AST | None: """Checks calls with '*args' and '**kwargs'. Note: The following happens only if '*args' or '**kwargs' is used. @@ -805,17 +805,17 @@ def visit_Call(self, node: ast.Call) -> ast.AST: copy_locations(node.func, node.args[0]) return node - def visit_keyword(self, node: ast.keyword) -> ast.AST: + def visit_keyword(self, node: ast.keyword) -> ast.AST | None: """ """ return self.node_contents_visit(node) - def visit_IfExp(self, node: ast.IfExp) -> ast.AST: + def visit_IfExp(self, node: ast.IfExp) -> ast.AST | None: """Allow `if` expressions without restrictions.""" return self.node_contents_visit(node) - def visit_Attribute(self, node: ast.Attribute) -> ast.AST: + def visit_Attribute(self, node: ast.Attribute) -> ast.AST | None: """Checks and mutates attribute access/assignment. 'a.b' becomes '_getattr_(a, "b")' @@ -871,7 +871,7 @@ def visit_Attribute(self, node: ast.Attribute) -> ast.AST: # Subscripting - def visit_Subscript(self, node: ast.Subscript) -> ast.AST: + def visit_Subscript(self, node: ast.Subscript) -> ast.AST | None: """Transforms all kinds of subscripts. 'foo[bar]' becomes '_getitem_(foo, bar)' @@ -916,7 +916,7 @@ def visit_Subscript(self, node: ast.Subscript) -> ast.AST: raise NotImplementedError( f"Unknown ctx type: {type(node.ctx)}") - def visit_Slice(self, node: ast.Slice) -> ast.AST: + def visit_Slice(self, node: ast.Slice) -> ast.AST | None: """ """ @@ -924,31 +924,31 @@ def visit_Slice(self, node: ast.Slice) -> ast.AST: # Comprehensions - def visit_ListComp(self, node: ast.ListComp) -> ast.AST: + def visit_ListComp(self, node: ast.ListComp) -> ast.AST | None: """ """ return self.node_contents_visit(node) - def visit_SetComp(self, node: ast.SetComp) -> ast.AST: + def visit_SetComp(self, node: ast.SetComp) -> ast.AST | None: """ """ return self.node_contents_visit(node) - def visit_GeneratorExp(self, node: ast.GeneratorExp) -> ast.AST: + def visit_GeneratorExp(self, node: ast.GeneratorExp) -> ast.AST | None: """ """ return self.node_contents_visit(node) - def visit_DictComp(self, node: ast.DictComp) -> ast.AST: + def visit_DictComp(self, node: ast.DictComp) -> ast.AST | None: """ """ return self.node_contents_visit(node) - def visit_comprehension(self, node: ast.comprehension) -> ast.AST: + def visit_comprehension(self, node: ast.comprehension) -> ast.AST | None: """ """ @@ -956,7 +956,7 @@ def visit_comprehension(self, node: ast.comprehension) -> ast.AST: # Statements - def visit_Assign(self, node: ast.Assign) -> ast.AST: + def visit_Assign(self, node: ast.Assign) -> ast.AST | None: """ """ @@ -1005,7 +1005,7 @@ def visit_Assign(self, node: ast.Assign) -> ast.AST: return new_nodes - def visit_AugAssign(self, node: ast.AugAssign) -> ast.AST: + def visit_AugAssign(self, node: ast.AugAssign) -> ast.AST | None: """Forbid certain kinds of AugAssign According to the language reference (and ast.c) the following nodes @@ -1056,65 +1056,65 @@ def visit_AugAssign(self, node: ast.AugAssign) -> ast.AST: raise NotImplementedError( f"Unknown target type: {type(node.target)}") - def visit_Raise(self, node: ast.Raise) -> ast.AST: + def visit_Raise(self, node: ast.Raise) -> ast.AST | None: """Allow `raise` statements without restrictions.""" return self.node_contents_visit(node) - def visit_Assert(self, node: ast.Assert) -> ast.AST: + def visit_Assert(self, node: ast.Assert) -> ast.AST | None: """Allow assert statements without restrictions.""" return self.node_contents_visit(node) - def visit_Delete(self, node: ast.Delete) -> ast.AST: + def visit_Delete(self, node: ast.Delete) -> ast.AST | None: """Allow `del` statements without restrictions.""" return self.node_contents_visit(node) - def visit_Pass(self, node: ast.Pass) -> ast.AST: + def visit_Pass(self, node: ast.Pass) -> ast.AST | None: """Allow `pass` statements without restrictions.""" return self.node_contents_visit(node) # Imports - def visit_Import(self, node: ast.Import) -> ast.AST: + def visit_Import(self, node: ast.Import) -> ast.AST | None: """Allow `import` statements with restrictions. See check_import_names.""" return self.check_import_names(node) - def visit_ImportFrom(self, node: ast.ImportFrom) -> ast.AST: + def visit_ImportFrom(self, node: ast.ImportFrom) -> ast.AST | None: """Allow `import from` statements with restrictions. See check_import_names.""" return self.check_import_names(node) - def visit_alias(self, node: ast.alias) -> ast.AST: + def visit_alias(self, node: ast.alias) -> ast.AST | None: """Allow `as` statements in import and import from statements.""" return self.node_contents_visit(node) # Control flow - def visit_If(self, node: ast.If) -> ast.AST: + def visit_If(self, node: ast.If) -> ast.AST | None: """Allow `if` statements without restrictions.""" return self.node_contents_visit(node) - def visit_For(self, node: ast.For) -> ast.AST: + def visit_For(self, node: ast.For) -> ast.AST | None: """Allow `for` statements with some restrictions.""" return self.guard_iter(node) - def visit_While(self, node: ast.While) -> ast.AST: + def visit_While(self, node: ast.While) -> ast.AST | None: """Allow `while` statements.""" return self.node_contents_visit(node) - def visit_Break(self, node: ast.Break) -> ast.AST: + def visit_Break(self, node: ast.Break) -> ast.AST | None: """Allow `break` statements without restrictions.""" return self.node_contents_visit(node) - def visit_Continue(self, node: ast.Continue) -> ast.AST: + def visit_Continue(self, node: ast.Continue) -> ast.AST | None: """Allow `continue` statements without restrictions.""" return self.node_contents_visit(node) - def visit_Try(self, node: ast.Try) -> ast.AST: + def visit_Try(self, node: ast.Try) -> ast.AST | None: """Allow `try` without restrictions.""" return self.node_contents_visit(node) - def visit_TryStar(self, node: ast.AST) -> ast.AST: + def visit_TryStar(self, node: ast.AST) -> ast.AST | None: """Disallow `ExceptionGroup` due to a potential sandbox escape. TODO: Type Annotation for node when dropping support @@ -1122,13 +1122,13 @@ def visit_TryStar(self, node: ast.AST) -> ast.AST: """ self.not_allowed(node) - def visit_ExceptHandler(self, node: ast.ExceptHandler) -> ast.AST: + def visit_ExceptHandler(self, node: ast.ExceptHandler) -> ast.AST | None: """Protect exception handlers.""" node = self.node_contents_visit(node) self.check_name(node, node.name) return node - def visit_With(self, node: ast.With) -> ast.AST: + def visit_With(self, node: ast.With) -> ast.AST | None: """Protect tuple unpacking on with statements.""" node = self.node_contents_visit(node) @@ -1143,13 +1143,13 @@ def visit_With(self, node: ast.With) -> ast.AST: return node - def visit_withitem(self, node: ast.withitem) -> ast.AST: + def visit_withitem(self, node: ast.withitem) -> ast.AST | None: """Allow `with` statements (context managers) without restrictions.""" return self.node_contents_visit(node) # Function and class definitions - def visit_FunctionDef(self, node: ast.FunctionDef) -> ast.AST: + def visit_FunctionDef(self, node: ast.FunctionDef) -> ast.AST | None: """Allow function definitions (`def`) with some restrictions.""" self.check_name(node, node.name, allow_magic_methods=True) self.check_function_argument_names(node) @@ -1159,44 +1159,44 @@ def visit_FunctionDef(self, node: ast.FunctionDef) -> ast.AST: self.inject_print_collector(node) return node - def visit_Lambda(self, node: ast.Lambda) -> ast.AST: + def visit_Lambda(self, node: ast.Lambda) -> ast.AST | None: """Allow lambda with some restrictions.""" self.check_function_argument_names(node) return self.node_contents_visit(node) - def visit_arguments(self, node: ast.arguments) -> ast.AST: + def visit_arguments(self, node: ast.arguments) -> ast.AST | None: """ """ return self.node_contents_visit(node) - def visit_arg(self, node: ast.arg) -> ast.AST: + def visit_arg(self, node: ast.arg) -> ast.AST | None: """ """ return self.node_contents_visit(node) - def visit_Return(self, node: ast.Return) -> ast.AST: + def visit_Return(self, node: ast.Return) -> ast.AST | None: """Allow `return` statements without restrictions.""" return self.node_contents_visit(node) - def visit_Yield(self, node: ast.Yield) -> ast.AST: + def visit_Yield(self, node: ast.Yield) -> ast.AST | None: """Allow `yield`statements without restrictions.""" return self.node_contents_visit(node) - def visit_YieldFrom(self, node: ast.YieldFrom) -> ast.AST: + def visit_YieldFrom(self, node: ast.YieldFrom) -> ast.AST | None: """Allow `yield`statements without restrictions.""" return self.node_contents_visit(node) - def visit_Global(self, node: ast.Global) -> ast.AST: + def visit_Global(self, node: ast.Global) -> ast.AST | None: """Allow `global` statements without restrictions.""" return self.node_contents_visit(node) - def visit_Nonlocal(self, node: ast.Nonlocal) -> ast.AST: + def visit_Nonlocal(self, node: ast.Nonlocal) -> ast.AST | None: """Deny `nonlocal` statements.""" self.not_allowed(node) - def visit_ClassDef(self, node: ast.ClassDef) -> ast.AST: + def visit_ClassDef(self, node: ast.ClassDef) -> ast.AST | None: """Check the name of a class definition.""" self.check_name(node, node.name) node = self.node_contents_visit(node) @@ -1213,7 +1213,7 @@ class {0.name}(metaclass=__metaclass__): new_class_node.decorator_list = node.decorator_list return new_class_node - def visit_Module(self, node: ast.Module) -> ast.AST: + def visit_Module(self, node: ast.Module) -> ast.AST | None: """Add the print_collector (only if print is used) at the top.""" node = self.node_contents_visit(node) @@ -1231,25 +1231,26 @@ def visit_Module(self, node: ast.Module) -> ast.AST: # Async und await - def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> ast.AST: + def visit_AsyncFunctionDef( + self, node: ast.AsyncFunctionDef) -> ast.AST | None: """Deny async functions.""" self.not_allowed(node) - def visit_Await(self, node: ast.Await) -> ast.AST: + def visit_Await(self, node: ast.Await) -> ast.AST | None: """Deny async functionality.""" self.not_allowed(node) - def visit_AsyncFor(self, node: ast.AsyncFor) -> ast.AST: + def visit_AsyncFor(self, node: ast.AsyncFor) -> ast.AST | None: """Deny async functionality.""" self.not_allowed(node) - def visit_AsyncWith(self, node: ast.AsyncWith) -> ast.AST: + def visit_AsyncWith(self, node: ast.AsyncWith) -> ast.AST | None: """Deny async functionality.""" self.not_allowed(node) # Assignment expressions (walrus operator ``:=``) # New in 3.8 - def visit_NamedExpr(self, node: ast.NamedExpr) -> ast.AST: + def visit_NamedExpr(self, node: ast.NamedExpr) -> ast.AST | None: """Allow assignment expressions under some circumstances.""" # while the grammar requires ``node.target`` to be a ``Name`` # the abstract syntax is more permissive and allows an ``expr``. From 2dd1fb56d2c3682f51946069c76a4cfc7d168169 Mon Sep 17 00:00:00 2001 From: zedzhen <59135268+zedzhen@users.noreply.github.com> Date: Tue, 2 Jun 2026 13:44:42 +0300 Subject: [PATCH 34/40] add py.typed --- src/RestrictedPython/py.typed | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 src/RestrictedPython/py.typed diff --git a/src/RestrictedPython/py.typed b/src/RestrictedPython/py.typed new file mode 100644 index 00000000..e69de29b From ea394dc634794735190ad07a127bc6c0bcfa2e0c Mon Sep 17 00:00:00 2001 From: zedzhen <59135268+zedzhen@users.noreply.github.com> Date: Wed, 3 Jun 2026 09:46:52 +0300 Subject: [PATCH 35/40] use "normal import" --- src/RestrictedPython/Eval.py | 2 +- src/RestrictedPython/Utilities.py | 8 +-- src/RestrictedPython/compile.py | 83 +++++++++++++++---------------- 3 files changed, 46 insertions(+), 47 deletions(-) diff --git a/src/RestrictedPython/Eval.py b/src/RestrictedPython/Eval.py index 408b25ae..e9626ca5 100644 --- a/src/RestrictedPython/Eval.py +++ b/src/RestrictedPython/Eval.py @@ -14,7 +14,7 @@ import ast -from .compile import compile_restricted_eval +from RestrictedPython.compile import compile_restricted_eval nltosp = str.maketrans('\r\n', ' ') diff --git a/src/RestrictedPython/Utilities.py b/src/RestrictedPython/Utilities.py index b966b4e2..8efc89e3 100644 --- a/src/RestrictedPython/Utilities.py +++ b/src/RestrictedPython/Utilities.py @@ -11,10 +11,10 @@ # ############################################################################## +import collections.abc import math import random import string -from collections.abc import Iterable utility_builtins = {} @@ -76,8 +76,10 @@ def test(*args): utility_builtins['test'] = test -def reorder(s: Iterable, with_: Iterable | None = None, - without: Iterable = ()) -> Iterable: +def reorder( + s: collections.abc.Iterable, + with_: collections.abc.Iterable | None = None, + without: collections.abc.Iterable = ()) -> collections.abc.Iterable: # s, with_, and without are sequences treated as sets. # The result is subtract(intersect(s, with_), without), # unless with_ is None, in which case it is subtract(s, without). diff --git a/src/RestrictedPython/compile.py b/src/RestrictedPython/compile.py index cf0035f9..ef566604 100644 --- a/src/RestrictedPython/compile.py +++ b/src/RestrictedPython/compile.py @@ -1,51 +1,48 @@ from __future__ import annotations import ast +import collections.abc +import os +import types +import typing import warnings -from ast import Expression -from ast import Interactive -from ast import Module -from ast import NodeTransformer -from collections.abc import Mapping -from collections.abc import Sequence -from os import PathLike -from types import CodeType -from typing import Any -from typing import Literal -from typing import NamedTuple -from typing import TypeAlias from RestrictedPython._compat import IS_CPYTHON from RestrictedPython.transformer import RestrictingNodeTransformer # Temporary workaround for missing _typeshed -ReadableBuffer: TypeAlias = bytes | bytearray +ReadableBuffer: typing.TypeAlias = bytes | bytearray -class CompileResult(NamedTuple): - code: CodeType | None - errors: Sequence[str] - warnings: Sequence[str] - used_names: Mapping[str, bool] +class CompileResult(typing.NamedTuple): + code: types.CodeType | None + errors: collections.abc.Sequence[str] + warnings: collections.abc.Sequence[str] + used_names: collections.abc.Mapping[str, bool] syntax_error_template = ( - 'Line {lineno}: {type}: {msg} at statement: {statement!r}') + 'Line {lineno}: {type}: {msg} at statement: {statement!r}' +) NOT_CPYTHON_WARNING = ( 'RestrictedPython is only supported on CPython: use on other Python ' 'implementations may create security issues.' ) +_T_source: typing.TypeAlias = ( + str | ReadableBuffer | ast.Module | ast.Expression | ast.Interactive +) + def _compile_restricted_mode( - source: str | ReadableBuffer | Module | Expression | Interactive, - filename: str | ReadableBuffer | PathLike[Any] = '', - mode: Literal["exec", "eval", "single"] = "exec", + source: _T_source, + filename: str | ReadableBuffer | os.PathLike[typing.Any] = '', + mode: typing.Literal["exec", "eval", "single"] = "exec", flags: int = 0, dont_inherit: bool = False, - policy: type[NodeTransformer] | None = RestrictingNodeTransformer, + policy: type[ast.NodeTransformer] | None = RestrictingNodeTransformer, ) -> CompileResult: if not IS_CPYTHON: @@ -62,13 +59,13 @@ def _compile_restricted_mode( dont_inherit=dont_inherit) elif issubclass(policy, RestrictingNodeTransformer): c_ast = None - allowed_source_types = [str, Module] + allowed_source_types = [str, ast.Module] if not issubclass(type(source), tuple(allowed_source_types)): raise TypeError('Not allowed source type: ' '"{0.__class__.__name__}".'.format(source)) c_ast = None # workaround for pypy issue https://bitbucket.org/pypy/pypy/issues/2552 - if isinstance(source, Module): + if isinstance(source, ast.Module): c_ast = source else: try: @@ -101,11 +98,11 @@ def _compile_restricted_mode( def compile_restricted_exec( - source: str | ReadableBuffer | Module | Expression | Interactive, - filename: str | ReadableBuffer | PathLike[Any] = '', + source: _T_source, + filename: str | ReadableBuffer | os.PathLike[typing.Any] = '', flags: int = 0, dont_inherit: bool = False, - policy: type[NodeTransformer] | None = RestrictingNodeTransformer, + policy: type[ast.NodeTransformer] | None = RestrictingNodeTransformer, ) -> CompileResult: """Compile restricted for the mode `exec`.""" return _compile_restricted_mode( @@ -118,11 +115,11 @@ def compile_restricted_exec( def compile_restricted_eval( - source: str | ReadableBuffer | Module | Expression | Interactive, - filename: str | ReadableBuffer | PathLike[Any] = '', + source: _T_source, + filename: str | ReadableBuffer | os.PathLike[typing.Any] = '', flags: int = 0, dont_inherit: bool = False, - policy: type[NodeTransformer] | None = RestrictingNodeTransformer, + policy: type[ast.NodeTransformer] | None = RestrictingNodeTransformer, ) -> CompileResult: """Compile restricted for the mode `eval`.""" return _compile_restricted_mode( @@ -135,11 +132,11 @@ def compile_restricted_eval( def compile_restricted_single( - source: str | ReadableBuffer | Module | Expression | Interactive, - filename: str | ReadableBuffer | PathLike[Any] = '', + source: _T_source, + filename: str | ReadableBuffer | os.PathLike[typing.Any] = '', flags: int = 0, dont_inherit: bool = False, - policy: type[NodeTransformer] | None = RestrictingNodeTransformer, + policy: type[ast.NodeTransformer] | None = RestrictingNodeTransformer, ) -> CompileResult: """Compile restricted for the mode `single`.""" return _compile_restricted_mode( @@ -155,11 +152,11 @@ def compile_restricted_function( p, # parameters body, name: str, - filename: str | ReadableBuffer | PathLike[Any] = '', + filename: str | ReadableBuffer | os.PathLike[typing.Any] = '', globalize=None, # List of globals (e.g. ['here', 'context', ...]) flags: int = 0, dont_inherit: bool = False, - policy: type[NodeTransformer] | None = RestrictingNodeTransformer, + policy: type[ast.NodeTransformer] | None = RestrictingNodeTransformer, ) -> CompileResult: """Compile a restricted code object for a function. @@ -208,13 +205,13 @@ def compile_restricted_function( def compile_restricted( - source: str | ReadableBuffer | Module | Expression | Interactive, - filename: str | ReadableBuffer | PathLike[Any] = '', - mode: str = 'exec', - flags: int = 0, - dont_inherit: bool = False, - policy: type[NodeTransformer] | None = RestrictingNodeTransformer, -) -> CodeType: + source: _T_source, + filename: str | ReadableBuffer | os.PathLike[typing.Any] = '', + mode: str = 'exec', + flags: int = 0, + dont_inherit: bool = False, + policy: type[ast.NodeTransformer] | None = RestrictingNodeTransformer, +) -> types.CodeType: """Replacement for the built-in compile() function. policy ... `ast.NodeTransformer` class defining the restrictions. From b1d1303c6035b4961959d72ddcededf27cac6d0d Mon Sep 17 00:00:00 2001 From: zedzhen <59135268+zedzhen@users.noreply.github.com> Date: Wed, 3 Jun 2026 10:26:46 +0300 Subject: [PATCH 36/40] add list[ast.AST] to the return type hints --- src/RestrictedPython/transformer.py | 198 ++++++++++++++-------------- 1 file changed, 102 insertions(+), 96 deletions(-) diff --git a/src/RestrictedPython/transformer.py b/src/RestrictedPython/transformer.py index 133544cb..39204c27 100644 --- a/src/RestrictedPython/transformer.py +++ b/src/RestrictedPython/transformer.py @@ -21,6 +21,7 @@ import ast import contextlib import textwrap +import typing # For AugAssign the operator must be converted to a string. @@ -111,10 +112,13 @@ "cr_origin", ]) +_T_visit_return: typing.TypeAlias = ast.AST | list[ast.AST] | None # When new ast nodes are generated they have no 'lineno', 'end_lineno', # 'col_offset' and 'end_col_offset'. This function copies these fields from the # incoming node: + + def copy_locations(new_node: ast.AST, old_node: ast.AST) -> None: assert 'lineno' in new_node._attributes new_node.lineno = old_node.lineno @@ -489,7 +493,7 @@ def node_contents_visit(self, node: ast.AST) -> ast.AST: # ast for Literals - def visit_Constant(self, node: ast.Constant) -> ast.AST | None: + def visit_Constant(self, node: ast.Constant) -> _T_visit_return: """Allow constant literals. Constant replaces Num, Str, Bytes, NameConstant and Ellipsis in @@ -498,31 +502,33 @@ def visit_Constant(self, node: ast.Constant) -> ast.AST | None: """ return self.node_contents_visit(node) - def visit_Interactive(self, node: ast.Interactive) -> ast.AST | None: + def visit_Interactive(self, node: ast.Interactive) -> _T_visit_return: """Allow single mode without restrictions.""" return self.node_contents_visit(node) - def visit_List(self, node: ast.List) -> ast.AST | None: + def visit_List(self, node: ast.List) -> _T_visit_return: """Allow list literals without restrictions.""" return self.node_contents_visit(node) - def visit_Tuple(self, node: ast.Tuple) -> ast.AST | None: + def visit_Tuple(self, node: ast.Tuple) -> _T_visit_return: """Allow tuple literals without restrictions.""" return self.node_contents_visit(node) - def visit_Set(self, node: ast.Set) -> ast.AST | None: + def visit_Set(self, node: ast.Set) -> _T_visit_return: """Allow set literals without restrictions.""" return self.node_contents_visit(node) - def visit_Dict(self, node: ast.Dict) -> ast.AST | None: + def visit_Dict(self, node: ast.Dict) -> _T_visit_return: """Allow dict literals without restrictions.""" return self.node_contents_visit(node) - def visit_FormattedValue(self, node: ast.FormattedValue) -> ast.AST | None: + def visit_FormattedValue( + self, + node: ast.FormattedValue) -> _T_visit_return: """Allow f-strings without restrictions.""" return self.node_contents_visit(node) - def visit_TemplateStr(self, node: ast.AST) -> ast.AST | None: + def visit_TemplateStr(self, node: ast.AST) -> _T_visit_return: """Template strings are allowed by default. As Template strings are a very basic template mechanism, that needs @@ -535,7 +541,7 @@ def visit_TemplateStr(self, node: ast.AST) -> ast.AST | None: """ return self.node_contents_visit(node) - def visit_Interpolation(self, node: ast.AST) -> ast.AST | None: + def visit_Interpolation(self, node: ast.AST) -> _T_visit_return: """Interpolations are allowed by default. As Interpolations are part of Template Strings, they are needed @@ -549,13 +555,13 @@ def visit_Interpolation(self, node: ast.AST) -> ast.AST | None: """ return self.node_contents_visit(node) - def visit_JoinedStr(self, node: ast.JoinedStr) -> ast.AST | None: + def visit_JoinedStr(self, node: ast.JoinedStr) -> _T_visit_return: """Allow joined string without restrictions.""" return self.node_contents_visit(node) # ast for Variables - def visit_Name(self, node: ast.Name) -> ast.Name | None: + def visit_Name(self, node: ast.Name) -> _T_visit_return: """Prevents access to protected names. Converts use of the name 'printed' to this expression: '_print()' @@ -589,25 +595,25 @@ def visit_Name(self, node: ast.Name) -> ast.Name | None: self.check_name(node, node.id) return node - def visit_Load(self, node: ast.Load) -> ast.Load | None: + def visit_Load(self, node: ast.Load) -> _T_visit_return: """ """ return self.node_contents_visit(node) - def visit_Store(self, node: ast.Store) -> ast.AST | None: + def visit_Store(self, node: ast.Store) -> _T_visit_return: """ """ return self.node_contents_visit(node) - def visit_Del(self, node: ast.Del) -> ast.AST | None: + def visit_Del(self, node: ast.Del) -> _T_visit_return: """ """ return self.node_contents_visit(node) - def visit_Starred(self, node: ast.Starred) -> ast.AST | None: + def visit_Starred(self, node: ast.Starred) -> _T_visit_return: """ """ @@ -615,18 +621,18 @@ def visit_Starred(self, node: ast.Starred) -> ast.AST | None: # Expressions - def visit_Expression(self, node: ast.Expression) -> ast.AST | None: + def visit_Expression(self, node: ast.Expression) -> _T_visit_return: """Allow Expression statements without restrictions. They are in the AST when using the `eval` compile mode. """ return self.node_contents_visit(node) - def visit_Expr(self, node: ast.Expr) -> ast.AST | None: + def visit_Expr(self, node: ast.Expr) -> _T_visit_return: """Allow Expr statements (any expression) without restrictions.""" return self.node_contents_visit(node) - def visit_UnaryOp(self, node: ast.UnaryOp) -> ast.AST | None: + def visit_UnaryOp(self, node: ast.UnaryOp) -> _T_visit_return: """ UnaryOp (Unary Operations) is the overall element for: * Not --> which should be allowed @@ -635,135 +641,135 @@ def visit_UnaryOp(self, node: ast.UnaryOp) -> ast.AST | None: """ return self.node_contents_visit(node) - def visit_UAdd(self, node: ast.UAdd) -> ast.AST | None: + def visit_UAdd(self, node: ast.UAdd) -> _T_visit_return: """Allow positive notation of variables. (e.g. +var)""" return self.node_contents_visit(node) - def visit_USub(self, node: ast.USub) -> ast.AST | None: + def visit_USub(self, node: ast.USub) -> _T_visit_return: """Allow negative notation of variables. (e.g. -var)""" return self.node_contents_visit(node) - def visit_Not(self, node: ast.Not) -> ast.AST | None: + def visit_Not(self, node: ast.Not) -> _T_visit_return: """Allow the `not` operator.""" return self.node_contents_visit(node) - def visit_Invert(self, node: ast.Invert) -> ast.AST | None: + def visit_Invert(self, node: ast.Invert) -> _T_visit_return: """Allow `~` expressions.""" return self.node_contents_visit(node) - def visit_BinOp(self, node: ast.BinOp) -> ast.AST | None: + def visit_BinOp(self, node: ast.BinOp) -> _T_visit_return: """Allow binary operations.""" return self.node_contents_visit(node) - def visit_Add(self, node: ast.Add) -> ast.AST | None: + def visit_Add(self, node: ast.Add) -> _T_visit_return: """Allow `+` expressions.""" return self.node_contents_visit(node) - def visit_Sub(self, node: ast.Sub) -> ast.AST | None: + def visit_Sub(self, node: ast.Sub) -> _T_visit_return: """Allow `-` expressions.""" return self.node_contents_visit(node) - def visit_Mult(self, node: ast.Mult) -> ast.AST | None: + def visit_Mult(self, node: ast.Mult) -> _T_visit_return: """Allow `*` expressions.""" return self.node_contents_visit(node) - def visit_Div(self, node: ast.Div) -> ast.AST | None: + def visit_Div(self, node: ast.Div) -> _T_visit_return: """Allow `/` expressions.""" return self.node_contents_visit(node) - def visit_FloorDiv(self, node: ast.FloorDiv) -> ast.AST | None: + def visit_FloorDiv(self, node: ast.FloorDiv) -> _T_visit_return: """Allow `//` expressions.""" return self.node_contents_visit(node) - def visit_Mod(self, node: ast.Mod) -> ast.AST | None: + def visit_Mod(self, node: ast.Mod) -> _T_visit_return: """Allow `%` expressions.""" return self.node_contents_visit(node) - def visit_Pow(self, node: ast.Pow) -> ast.AST | None: + def visit_Pow(self, node: ast.Pow) -> _T_visit_return: """Allow `**` expressions.""" return self.node_contents_visit(node) - def visit_LShift(self, node: ast.LShift) -> ast.AST | None: + def visit_LShift(self, node: ast.LShift) -> _T_visit_return: """Allow `<<` expressions.""" return self.node_contents_visit(node) - def visit_RShift(self, node: ast.RShift) -> ast.AST | None: + def visit_RShift(self, node: ast.RShift) -> _T_visit_return: """Allow `>>` expressions.""" return self.node_contents_visit(node) - def visit_BitOr(self, node: ast.BitOr) -> ast.AST | None: + def visit_BitOr(self, node: ast.BitOr) -> _T_visit_return: """Allow `|` expressions.""" return self.node_contents_visit(node) - def visit_BitXor(self, node: ast.BitXor) -> ast.AST | None: + def visit_BitXor(self, node: ast.BitXor) -> _T_visit_return: """Allow `^` expressions.""" return self.node_contents_visit(node) - def visit_BitAnd(self, node: ast.BitAnd) -> ast.AST | None: + def visit_BitAnd(self, node: ast.BitAnd) -> _T_visit_return: """Allow `&` expressions.""" return self.node_contents_visit(node) - def visit_MatMult(self, node: ast.MatMult) -> ast.AST | None: + def visit_MatMult(self, node: ast.MatMult) -> _T_visit_return: """Allow multiplication (`@`).""" return self.node_contents_visit(node) - def visit_BoolOp(self, node: ast.BoolOp) -> ast.AST | None: + def visit_BoolOp(self, node: ast.BoolOp) -> _T_visit_return: """Allow bool operator without restrictions.""" return self.node_contents_visit(node) - def visit_And(self, node: ast.And) -> ast.AST | None: + def visit_And(self, node: ast.And) -> _T_visit_return: """Allow bool operator `and` without restrictions.""" return self.node_contents_visit(node) - def visit_Or(self, node: ast.Or) -> ast.AST | None: + def visit_Or(self, node: ast.Or) -> _T_visit_return: """Allow bool operator `or` without restrictions.""" return self.node_contents_visit(node) - def visit_Compare(self, node: ast.Compare) -> ast.AST | None: + def visit_Compare(self, node: ast.Compare) -> _T_visit_return: """Allow comparison expressions without restrictions.""" return self.node_contents_visit(node) - def visit_Eq(self, node: ast.Eq) -> ast.AST | None: + def visit_Eq(self, node: ast.Eq) -> _T_visit_return: """Allow == expressions.""" return self.node_contents_visit(node) - def visit_NotEq(self, node: ast.NotEq) -> ast.AST | None: + def visit_NotEq(self, node: ast.NotEq) -> _T_visit_return: """Allow != expressions.""" return self.node_contents_visit(node) - def visit_Lt(self, node: ast.Lt) -> ast.AST | None: + def visit_Lt(self, node: ast.Lt) -> _T_visit_return: """Allow < expressions.""" return self.node_contents_visit(node) - def visit_LtE(self, node: ast.LtE) -> ast.AST | None: + def visit_LtE(self, node: ast.LtE) -> _T_visit_return: """Allow <= expressions.""" return self.node_contents_visit(node) - def visit_Gt(self, node: ast.Gt) -> ast.AST | None: + def visit_Gt(self, node: ast.Gt) -> _T_visit_return: """Allow > expressions.""" return self.node_contents_visit(node) - def visit_GtE(self, node: ast.GtE) -> ast.AST | None: + def visit_GtE(self, node: ast.GtE) -> _T_visit_return: """Allow >= expressions.""" return self.node_contents_visit(node) - def visit_Is(self, node: ast.Is) -> ast.AST | None: + def visit_Is(self, node: ast.Is) -> _T_visit_return: """Allow `is` expressions.""" return self.node_contents_visit(node) - def visit_IsNot(self, node: ast.IsNot) -> ast.AST | None: + def visit_IsNot(self, node: ast.IsNot) -> _T_visit_return: """Allow `is not` expressions.""" return self.node_contents_visit(node) - def visit_In(self, node: ast.In) -> ast.AST | None: + def visit_In(self, node: ast.In) -> _T_visit_return: """Allow `in` expressions.""" return self.node_contents_visit(node) - def visit_NotIn(self, node: ast.NotIn) -> ast.AST | None: + def visit_NotIn(self, node: ast.NotIn) -> _T_visit_return: """Allow `not in` expressions.""" return self.node_contents_visit(node) - def visit_Call(self, node: ast.Call) -> ast.AST | None: + def visit_Call(self, node: ast.Call) -> _T_visit_return: """Checks calls with '*args' and '**kwargs'. Note: The following happens only if '*args' or '**kwargs' is used. @@ -805,17 +811,17 @@ def visit_Call(self, node: ast.Call) -> ast.AST | None: copy_locations(node.func, node.args[0]) return node - def visit_keyword(self, node: ast.keyword) -> ast.AST | None: + def visit_keyword(self, node: ast.keyword) -> _T_visit_return: """ """ return self.node_contents_visit(node) - def visit_IfExp(self, node: ast.IfExp) -> ast.AST | None: + def visit_IfExp(self, node: ast.IfExp) -> _T_visit_return: """Allow `if` expressions without restrictions.""" return self.node_contents_visit(node) - def visit_Attribute(self, node: ast.Attribute) -> ast.AST | None: + def visit_Attribute(self, node: ast.Attribute) -> _T_visit_return: """Checks and mutates attribute access/assignment. 'a.b' becomes '_getattr_(a, "b")' @@ -871,7 +877,7 @@ def visit_Attribute(self, node: ast.Attribute) -> ast.AST | None: # Subscripting - def visit_Subscript(self, node: ast.Subscript) -> ast.AST | None: + def visit_Subscript(self, node: ast.Subscript) -> _T_visit_return: """Transforms all kinds of subscripts. 'foo[bar]' becomes '_getitem_(foo, bar)' @@ -916,7 +922,7 @@ def visit_Subscript(self, node: ast.Subscript) -> ast.AST | None: raise NotImplementedError( f"Unknown ctx type: {type(node.ctx)}") - def visit_Slice(self, node: ast.Slice) -> ast.AST | None: + def visit_Slice(self, node: ast.Slice) -> _T_visit_return: """ """ @@ -924,31 +930,31 @@ def visit_Slice(self, node: ast.Slice) -> ast.AST | None: # Comprehensions - def visit_ListComp(self, node: ast.ListComp) -> ast.AST | None: + def visit_ListComp(self, node: ast.ListComp) -> _T_visit_return: """ """ return self.node_contents_visit(node) - def visit_SetComp(self, node: ast.SetComp) -> ast.AST | None: + def visit_SetComp(self, node: ast.SetComp) -> _T_visit_return: """ """ return self.node_contents_visit(node) - def visit_GeneratorExp(self, node: ast.GeneratorExp) -> ast.AST | None: + def visit_GeneratorExp(self, node: ast.GeneratorExp) -> _T_visit_return: """ """ return self.node_contents_visit(node) - def visit_DictComp(self, node: ast.DictComp) -> ast.AST | None: + def visit_DictComp(self, node: ast.DictComp) -> _T_visit_return: """ """ return self.node_contents_visit(node) - def visit_comprehension(self, node: ast.comprehension) -> ast.AST | None: + def visit_comprehension(self, node: ast.comprehension) -> _T_visit_return: """ """ @@ -956,7 +962,7 @@ def visit_comprehension(self, node: ast.comprehension) -> ast.AST | None: # Statements - def visit_Assign(self, node: ast.Assign) -> ast.AST | None: + def visit_Assign(self, node: ast.Assign) -> _T_visit_return: """ """ @@ -1005,7 +1011,7 @@ def visit_Assign(self, node: ast.Assign) -> ast.AST | None: return new_nodes - def visit_AugAssign(self, node: ast.AugAssign) -> ast.AST | None: + def visit_AugAssign(self, node: ast.AugAssign) -> _T_visit_return: """Forbid certain kinds of AugAssign According to the language reference (and ast.c) the following nodes @@ -1056,65 +1062,65 @@ def visit_AugAssign(self, node: ast.AugAssign) -> ast.AST | None: raise NotImplementedError( f"Unknown target type: {type(node.target)}") - def visit_Raise(self, node: ast.Raise) -> ast.AST | None: + def visit_Raise(self, node: ast.Raise) -> _T_visit_return: """Allow `raise` statements without restrictions.""" return self.node_contents_visit(node) - def visit_Assert(self, node: ast.Assert) -> ast.AST | None: + def visit_Assert(self, node: ast.Assert) -> _T_visit_return: """Allow assert statements without restrictions.""" return self.node_contents_visit(node) - def visit_Delete(self, node: ast.Delete) -> ast.AST | None: + def visit_Delete(self, node: ast.Delete) -> _T_visit_return: """Allow `del` statements without restrictions.""" return self.node_contents_visit(node) - def visit_Pass(self, node: ast.Pass) -> ast.AST | None: + def visit_Pass(self, node: ast.Pass) -> _T_visit_return: """Allow `pass` statements without restrictions.""" return self.node_contents_visit(node) # Imports - def visit_Import(self, node: ast.Import) -> ast.AST | None: + def visit_Import(self, node: ast.Import) -> _T_visit_return: """Allow `import` statements with restrictions. See check_import_names.""" return self.check_import_names(node) - def visit_ImportFrom(self, node: ast.ImportFrom) -> ast.AST | None: + def visit_ImportFrom(self, node: ast.ImportFrom) -> _T_visit_return: """Allow `import from` statements with restrictions. See check_import_names.""" return self.check_import_names(node) - def visit_alias(self, node: ast.alias) -> ast.AST | None: + def visit_alias(self, node: ast.alias) -> _T_visit_return: """Allow `as` statements in import and import from statements.""" return self.node_contents_visit(node) # Control flow - def visit_If(self, node: ast.If) -> ast.AST | None: + def visit_If(self, node: ast.If) -> _T_visit_return: """Allow `if` statements without restrictions.""" return self.node_contents_visit(node) - def visit_For(self, node: ast.For) -> ast.AST | None: + def visit_For(self, node: ast.For) -> _T_visit_return: """Allow `for` statements with some restrictions.""" return self.guard_iter(node) - def visit_While(self, node: ast.While) -> ast.AST | None: + def visit_While(self, node: ast.While) -> _T_visit_return: """Allow `while` statements.""" return self.node_contents_visit(node) - def visit_Break(self, node: ast.Break) -> ast.AST | None: + def visit_Break(self, node: ast.Break) -> _T_visit_return: """Allow `break` statements without restrictions.""" return self.node_contents_visit(node) - def visit_Continue(self, node: ast.Continue) -> ast.AST | None: + def visit_Continue(self, node: ast.Continue) -> _T_visit_return: """Allow `continue` statements without restrictions.""" return self.node_contents_visit(node) - def visit_Try(self, node: ast.Try) -> ast.AST | None: + def visit_Try(self, node: ast.Try) -> _T_visit_return: """Allow `try` without restrictions.""" return self.node_contents_visit(node) - def visit_TryStar(self, node: ast.AST) -> ast.AST | None: + def visit_TryStar(self, node: ast.AST) -> _T_visit_return: """Disallow `ExceptionGroup` due to a potential sandbox escape. TODO: Type Annotation for node when dropping support @@ -1122,13 +1128,13 @@ def visit_TryStar(self, node: ast.AST) -> ast.AST | None: """ self.not_allowed(node) - def visit_ExceptHandler(self, node: ast.ExceptHandler) -> ast.AST | None: + def visit_ExceptHandler(self, node: ast.ExceptHandler) -> _T_visit_return: """Protect exception handlers.""" node = self.node_contents_visit(node) self.check_name(node, node.name) return node - def visit_With(self, node: ast.With) -> ast.AST | None: + def visit_With(self, node: ast.With) -> _T_visit_return: """Protect tuple unpacking on with statements.""" node = self.node_contents_visit(node) @@ -1143,13 +1149,13 @@ def visit_With(self, node: ast.With) -> ast.AST | None: return node - def visit_withitem(self, node: ast.withitem) -> ast.AST | None: + def visit_withitem(self, node: ast.withitem) -> _T_visit_return: """Allow `with` statements (context managers) without restrictions.""" return self.node_contents_visit(node) # Function and class definitions - def visit_FunctionDef(self, node: ast.FunctionDef) -> ast.AST | None: + def visit_FunctionDef(self, node: ast.FunctionDef) -> _T_visit_return: """Allow function definitions (`def`) with some restrictions.""" self.check_name(node, node.name, allow_magic_methods=True) self.check_function_argument_names(node) @@ -1159,44 +1165,44 @@ def visit_FunctionDef(self, node: ast.FunctionDef) -> ast.AST | None: self.inject_print_collector(node) return node - def visit_Lambda(self, node: ast.Lambda) -> ast.AST | None: + def visit_Lambda(self, node: ast.Lambda) -> _T_visit_return: """Allow lambda with some restrictions.""" self.check_function_argument_names(node) return self.node_contents_visit(node) - def visit_arguments(self, node: ast.arguments) -> ast.AST | None: + def visit_arguments(self, node: ast.arguments) -> _T_visit_return: """ """ return self.node_contents_visit(node) - def visit_arg(self, node: ast.arg) -> ast.AST | None: + def visit_arg(self, node: ast.arg) -> _T_visit_return: """ """ return self.node_contents_visit(node) - def visit_Return(self, node: ast.Return) -> ast.AST | None: + def visit_Return(self, node: ast.Return) -> _T_visit_return: """Allow `return` statements without restrictions.""" return self.node_contents_visit(node) - def visit_Yield(self, node: ast.Yield) -> ast.AST | None: + def visit_Yield(self, node: ast.Yield) -> _T_visit_return: """Allow `yield`statements without restrictions.""" return self.node_contents_visit(node) - def visit_YieldFrom(self, node: ast.YieldFrom) -> ast.AST | None: + def visit_YieldFrom(self, node: ast.YieldFrom) -> _T_visit_return: """Allow `yield`statements without restrictions.""" return self.node_contents_visit(node) - def visit_Global(self, node: ast.Global) -> ast.AST | None: + def visit_Global(self, node: ast.Global) -> _T_visit_return: """Allow `global` statements without restrictions.""" return self.node_contents_visit(node) - def visit_Nonlocal(self, node: ast.Nonlocal) -> ast.AST | None: + def visit_Nonlocal(self, node: ast.Nonlocal) -> _T_visit_return: """Deny `nonlocal` statements.""" self.not_allowed(node) - def visit_ClassDef(self, node: ast.ClassDef) -> ast.AST | None: + def visit_ClassDef(self, node: ast.ClassDef) -> _T_visit_return: """Check the name of a class definition.""" self.check_name(node, node.name) node = self.node_contents_visit(node) @@ -1213,7 +1219,7 @@ class {0.name}(metaclass=__metaclass__): new_class_node.decorator_list = node.decorator_list return new_class_node - def visit_Module(self, node: ast.Module) -> ast.AST | None: + def visit_Module(self, node: ast.Module) -> _T_visit_return: """Add the print_collector (only if print is used) at the top.""" node = self.node_contents_visit(node) @@ -1232,25 +1238,25 @@ def visit_Module(self, node: ast.Module) -> ast.AST | None: # Async und await def visit_AsyncFunctionDef( - self, node: ast.AsyncFunctionDef) -> ast.AST | None: + self, node: ast.AsyncFunctionDef) -> _T_visit_return: """Deny async functions.""" self.not_allowed(node) - def visit_Await(self, node: ast.Await) -> ast.AST | None: + def visit_Await(self, node: ast.Await) -> _T_visit_return: """Deny async functionality.""" self.not_allowed(node) - def visit_AsyncFor(self, node: ast.AsyncFor) -> ast.AST | None: + def visit_AsyncFor(self, node: ast.AsyncFor) -> _T_visit_return: """Deny async functionality.""" self.not_allowed(node) - def visit_AsyncWith(self, node: ast.AsyncWith) -> ast.AST | None: + def visit_AsyncWith(self, node: ast.AsyncWith) -> _T_visit_return: """Deny async functionality.""" self.not_allowed(node) # Assignment expressions (walrus operator ``:=``) # New in 3.8 - def visit_NamedExpr(self, node: ast.NamedExpr) -> ast.AST | None: + def visit_NamedExpr(self, node: ast.NamedExpr) -> _T_visit_return: """Allow assignment expressions under some circumstances.""" # while the grammar requires ``node.target`` to be a ``Name`` # the abstract syntax is more permissive and allows an ``expr``. From 23ec4811696e220685ce0f1359180e2f85a13152 Mon Sep 17 00:00:00 2001 From: zedzhen <59135268+zedzhen@users.noreply.github.com> Date: Wed, 3 Jun 2026 17:51:28 +0300 Subject: [PATCH 37/40] add config for mypy --- pyproject.toml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 2c442e16..502dd97b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -88,3 +88,23 @@ directory = "parts/htmlcov" [tool.setuptools.dynamic] readme = {file = ["README.rst", "CHANGES.rst"]} +[tool.mypy] +mypy_path = "src" +packages = ["RestrictedPython"] +python_version = "3.10" +warn_unreachable = true +implicit_reexport = false +strict = true + +[[tool.mypy.overrides]] +module = ["DateTime"] +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = ["RestrictedPython.Guards"] +check_untyped_defs = false +disallow_untyped_defs = false + +[[tool.mypy.overrides]] +module = ["RestrictedPython.transformer"] +warn_no_return = false From 838d222e216e9331d40601ffd41b0093daa8cefe Mon Sep 17 00:00:00 2001 From: zedzhen <59135268+zedzhen@users.noreply.github.com> Date: Wed, 3 Jun 2026 17:53:55 +0300 Subject: [PATCH 38/40] replace `compile(flags=ast.PyCF_ONLY_AST)` to `ast.parse` --- src/RestrictedPython/Eval.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/RestrictedPython/Eval.py b/src/RestrictedPython/Eval.py index e9626ca5..8f5d6eff 100644 --- a/src/RestrictedPython/Eval.py +++ b/src/RestrictedPython/Eval.py @@ -70,11 +70,10 @@ def prepRestrictedCode(self): def prepUnrestrictedCode(self): if self.ucode is None: - exp_node = compile( + exp_node = ast.parse( self.expr, '', - 'eval', - ast.PyCF_ONLY_AST) + 'eval') co = compile(exp_node, '', 'eval') From 2913cb6c0412fbcf418a06437e4ae6480f638b3d Mon Sep 17 00:00:00 2001 From: zedzhen <59135268+zedzhen@users.noreply.github.com> Date: Wed, 3 Jun 2026 18:02:28 +0300 Subject: [PATCH 39/40] update type hints --- src/RestrictedPython/Eval.py | 41 +++++++++++------ src/RestrictedPython/Guards.py | 2 +- src/RestrictedPython/Limits.py | 27 +++++++++-- src/RestrictedPython/PrintCollector.py | 8 ++-- src/RestrictedPython/Utilities.py | 27 +++++++---- src/RestrictedPython/_types.py | 8 ++++ src/RestrictedPython/compile.py | 46 ++++++++++--------- src/RestrictedPython/transformer.py | 63 +++++++++++++++++--------- 8 files changed, 150 insertions(+), 72 deletions(-) create mode 100644 src/RestrictedPython/_types.py diff --git a/src/RestrictedPython/Eval.py b/src/RestrictedPython/Eval.py index 8f5d6eff..a9179648 100644 --- a/src/RestrictedPython/Eval.py +++ b/src/RestrictedPython/Eval.py @@ -13,7 +13,11 @@ """Restricted Python Expressions.""" import ast +import collections +import types +import typing +from RestrictedPython._types import _cast_not_none from RestrictedPython.compile import compile_restricted_eval @@ -22,13 +26,21 @@ # No restrictions. default_guarded_getattr = getattr +_T = typing.TypeVar('_T') +_TK = typing.TypeVar('_TK', contravariant=True) +_TV = typing.TypeVar('_TV', covariant=True) -def default_guarded_getitem(ob, index): + +class _GetItem(typing.Protocol[_TK, _TV]): + def __getitem__(self, key: _TK) -> _TV: ... + + +def default_guarded_getitem(ob: _GetItem[_TK, _TV], index: _TK) -> _TV: # No restrictions. return ob[index] -def default_guarded_getiter(ob): +def default_guarded_getiter(ob: _T) -> _T: # No restrictions. return ob @@ -36,17 +48,18 @@ def default_guarded_getiter(ob): class RestrictionCapableEval: """A base class for restricted code.""" - globals = {'__builtins__': None} + globals: dict[str, typing.Any] = {'__builtins__': None} + # restricted - rcode = None + rcode: types.CodeType | None = None # unrestricted - ucode = None + ucode: types.CodeType | None = None # Names used by the expression - used = None + used: tuple[str, ...] | None = None - def __init__(self, expr): + def __init__(self, expr: str): """Create a restricted expression where: @@ -60,7 +73,7 @@ def __init__(self, expr): # Catch syntax errors. self.prepUnrestrictedCode() - def prepRestrictedCode(self): + def prepRestrictedCode(self) -> None: if self.rcode is None: result = compile_restricted_eval(self.expr, '') if result.errors: @@ -68,7 +81,7 @@ def prepRestrictedCode(self): self.used = tuple(result.used_names) self.rcode = result.code - def prepUnrestrictedCode(self): + def prepUnrestrictedCode(self) -> None: if self.ucode is None: exp_node = ast.parse( self.expr, @@ -89,7 +102,9 @@ def prepUnrestrictedCode(self): self.ucode = co - def eval(self, mapping): + def eval(self, + mapping: collections.abc.Mapping[str, + typing.Any]) -> typing.Any: # This default implementation is probably not very useful. :-( # This is meant to be overridden. self.prepRestrictedCode() @@ -102,11 +117,11 @@ def eval(self, mapping): global_scope.update(self.globals) - for name in self.used: + for name in _cast_not_none(self.used): if (name not in global_scope) and (name in mapping): global_scope[name] = mapping[name] - return eval(self.rcode, global_scope) + return eval(_cast_not_none(self.rcode), global_scope) - def __call__(self, **kw): + def __call__(self, **kw: typing.Any) -> typing.Any: return self.eval(kw) diff --git a/src/RestrictedPython/Guards.py b/src/RestrictedPython/Guards.py index d7c1b9c3..eb9cc0f4 100644 --- a/src/RestrictedPython/Guards.py +++ b/src/RestrictedPython/Guards.py @@ -220,7 +220,7 @@ def guard(ob): return guard -full_write_guard = _full_write_guard() +full_write_guard = _full_write_guard() # type: ignore[no-untyped-call] def guarded_setattr(object, name, value): diff --git a/src/RestrictedPython/Limits.py b/src/RestrictedPython/Limits.py index e133ec70..4c933ad3 100644 --- a/src/RestrictedPython/Limits.py +++ b/src/RestrictedPython/Limits.py @@ -10,11 +10,28 @@ # FOR A PARTICULAR PURPOSE # ############################################################################## +import collections.abc +import typing -limited_builtins = {} +limited_builtins: dict[str, typing.Any] = {} -def limited_range(iFirst, *args): + +@typing.overload +def limited_range(iFirst: int) -> collections.abc.Sequence[int]: ... + + +@typing.overload +def limited_range(iStart: int, iEnd: int, / + ) -> collections.abc.Sequence[int]: ... + + +@typing.overload +def limited_range(iStart: int, iEnd: int, iStep: int, / + ) -> collections.abc.Sequence[int]: ... + + +def limited_range(iFirst: int, *args: int) -> collections.abc.Sequence[int]: # limited range function from Martijn Pieters RANGELIMIT = 1000 if not len(args): @@ -41,8 +58,10 @@ def limited_range(iFirst, *args): limited_builtins['range'] = limited_range +_T = typing.TypeVar('_T') + -def limited_list(seq): +def limited_list(seq: collections.abc.Iterable[_T]) -> list[_T]: if isinstance(seq, str): raise TypeError('cannot convert string to list') return list(seq) @@ -51,7 +70,7 @@ def limited_list(seq): limited_builtins['list'] = limited_list -def limited_tuple(seq): +def limited_tuple(seq: collections.abc.Iterable[_T]) -> tuple[_T, ...]: if isinstance(seq, str): raise TypeError('cannot convert string to tuple') return tuple(seq) diff --git a/src/RestrictedPython/PrintCollector.py b/src/RestrictedPython/PrintCollector.py index d28a7ab6..0528e38c 100644 --- a/src/RestrictedPython/PrintCollector.py +++ b/src/RestrictedPython/PrintCollector.py @@ -15,17 +15,17 @@ class PrintCollector: """Collect written text, and return it when called.""" - def __init__(self, _getattr_=None): + def __init__(self, _getattr_=None): # type: ignore[no-untyped-def] self.txt = [] self._getattr_ = _getattr_ - def write(self, text): + def write(self, text: str) -> None: self.txt.append(text) - def __call__(self): + def __call__(self) -> str: return ''.join(self.txt) - def _call_print(self, *objects, **kwargs): + def _call_print(self, *objects, **kwargs): # type: ignore[no-untyped-def] if kwargs.get('file', None) is None: kwargs['file'] = self else: diff --git a/src/RestrictedPython/Utilities.py b/src/RestrictedPython/Utilities.py index 8efc89e3..6a269591 100644 --- a/src/RestrictedPython/Utilities.py +++ b/src/RestrictedPython/Utilities.py @@ -15,18 +15,20 @@ import math import random import string +import types +import typing -utility_builtins = {} +utility_builtins: dict[str, typing.Any] = {} class _AttributeDelegator: - def __init__(self, mod, *excludes): + def __init__(self, mod: types.ModuleType, *excludes: str): """delegate attribute lookups outside *excludes* to module *mod*.""" self.__mod = mod self.__excludes = excludes - def __getattr__(self, attr): + def __getattr__(self, attr: str) -> typing.Any: if attr in self.__excludes: raise NotImplementedError( f"{self.__mod.__name__}.{attr} is not safe") @@ -51,7 +53,7 @@ def __getattr__(self, attr): pass -def same_type(arg1, *args): +def same_type(arg1: object, *args: object) -> bool: """Compares the class or type of two or more objects.""" t = getattr(arg1, '__class__', type(arg1)) for arg in args: @@ -62,8 +64,10 @@ def same_type(arg1, *args): utility_builtins['same_type'] = same_type +_T = typing.TypeVar('_T') -def test(*args): + +def test(*args: _T) -> _T | None: length = len(args) for i in range(1, length, 2): if args[i - 1]: @@ -71,15 +75,22 @@ def test(*args): if length % 2: return args[-1] + return None utility_builtins['test'] = test +_TK = typing.TypeVar('_TK') +_TV = typing.TypeVar('_TV') +_T_in: typing.TypeAlias = collections.abc.Iterable[_TK | tuple[_TK, _TV]] +_T_out: typing.TypeAlias = list[tuple[_TK, _TK | _TV]] + def reorder( - s: collections.abc.Iterable, - with_: collections.abc.Iterable | None = None, - without: collections.abc.Iterable = ()) -> collections.abc.Iterable: + s: _T_in[_TK, _TV], + with_: collections.abc.Iterable[typing.Any] | None = None, + without: collections.abc.Iterable[typing.Any] = () +) -> _T_out[_TK, _TV]: # s, with_, and without are sequences treated as sets. # The result is subtract(intersect(s, with_), without), # unless with_ is None, in which case it is subtract(s, without). diff --git a/src/RestrictedPython/_types.py b/src/RestrictedPython/_types.py new file mode 100644 index 00000000..9cc3c0f7 --- /dev/null +++ b/src/RestrictedPython/_types.py @@ -0,0 +1,8 @@ +import typing + + +_T = typing.TypeVar('_T') + + +def _cast_not_none(var: _T | None) -> _T: + return var # type: ignore[return-value] diff --git a/src/RestrictedPython/compile.py b/src/RestrictedPython/compile.py index ef566604..8b2d1049 100644 --- a/src/RestrictedPython/compile.py +++ b/src/RestrictedPython/compile.py @@ -8,6 +8,7 @@ import warnings from RestrictedPython._compat import IS_CPYTHON +from RestrictedPython._types import _cast_not_none from RestrictedPython.transformer import RestrictingNodeTransformer @@ -31,14 +32,14 @@ class CompileResult(typing.NamedTuple): 'implementations may create security issues.' ) -_T_source: typing.TypeAlias = ( - str | ReadableBuffer | ast.Module | ast.Expression | ast.Interactive -) +_T_ast_compilable: typing.TypeAlias = ( + ast.Module | ast.Expression | ast.Interactive) +_T_source: typing.TypeAlias = str | ReadableBuffer | _T_ast_compilable def _compile_restricted_mode( source: _T_source, - filename: str | ReadableBuffer | os.PathLike[typing.Any] = '', + filename: str | bytes | os.PathLike[typing.Any] = '', mode: typing.Literal["exec", "eval", "single"] = "exec", flags: int = 0, dont_inherit: bool = False, @@ -50,26 +51,27 @@ def _compile_restricted_mode( NOT_CPYTHON_WARNING, RuntimeWarning, 'RestrictedPython', 0) byte_code = None - collected_errors = [] - collected_warnings = [] - used_names = {} + collected_errors: list[str] = [] + collected_warnings: list[str] = [] + used_names: dict[str, bool] = {} if policy is None: # Unrestricted Source Checks byte_code = compile(source, filename, mode=mode, flags=flags, dont_inherit=dont_inherit) elif issubclass(policy, RestrictingNodeTransformer): - c_ast = None allowed_source_types = [str, ast.Module] if not issubclass(type(source), tuple(allowed_source_types)): raise TypeError('Not allowed source type: ' '"{0.__class__.__name__}".'.format(source)) - c_ast = None + c_ast: _T_ast_compilable | None = None # workaround for pypy issue https://bitbucket.org/pypy/pypy/issues/2552 if isinstance(source, ast.Module): c_ast = source else: try: - c_ast = ast.parse(source, filename, mode) + c_ast = typing.cast( + _T_ast_compilable, ast.parse( + source, filename, mode)) except (TypeError, ValueError) as e: collected_errors.append(str(e)) except SyntaxError as v: @@ -99,7 +101,7 @@ def _compile_restricted_mode( def compile_restricted_exec( source: _T_source, - filename: str | ReadableBuffer | os.PathLike[typing.Any] = '', + filename: str | bytes | os.PathLike[typing.Any] = '', flags: int = 0, dont_inherit: bool = False, policy: type[ast.NodeTransformer] | None = RestrictingNodeTransformer, @@ -116,7 +118,7 @@ def compile_restricted_exec( def compile_restricted_eval( source: _T_source, - filename: str | ReadableBuffer | os.PathLike[typing.Any] = '', + filename: str | bytes | os.PathLike[typing.Any] = '', flags: int = 0, dont_inherit: bool = False, policy: type[ast.NodeTransformer] | None = RestrictingNodeTransformer, @@ -133,7 +135,7 @@ def compile_restricted_eval( def compile_restricted_single( source: _T_source, - filename: str | ReadableBuffer | os.PathLike[typing.Any] = '', + filename: str | bytes | os.PathLike[typing.Any] = '', flags: int = 0, dont_inherit: bool = False, policy: type[ast.NodeTransformer] | None = RestrictingNodeTransformer, @@ -149,11 +151,12 @@ def compile_restricted_single( def compile_restricted_function( - p, # parameters - body, + p: str, # parameters + body: str | ReadableBuffer | ast.Module | ast.Interactive, name: str, - filename: str | ReadableBuffer | os.PathLike[typing.Any] = '', - globalize=None, # List of globals (e.g. ['here', 'context', ...]) + filename: str | bytes | os.PathLike[typing.Any] = '', + # List of globals (e.g. ['here', 'context', ...]) + globalize: list[str] | None = None, flags: int = 0, dont_inherit: bool = False, policy: type[ast.NodeTransformer] | None = RestrictingNodeTransformer, @@ -190,7 +193,7 @@ def compile_restricted_function( assert isinstance(function_ast, ast.FunctionDef) function_ast.name = name - wrapper_ast.body[0].body = body_ast.body + function_ast.body = body_ast.body wrapper_ast = ast.fix_missing_locations(wrapper_ast) result = _compile_restricted_mode( @@ -206,7 +209,7 @@ def compile_restricted_function( def compile_restricted( source: _T_source, - filename: str | ReadableBuffer | os.PathLike[typing.Any] = '', + filename: str | bytes | os.PathLike[typing.Any] = '', mode: str = 'exec', flags: int = 0, dont_inherit: bool = False, @@ -221,7 +224,8 @@ def compile_restricted( result = _compile_restricted_mode( source, filename=filename, - mode=mode, + mode=mode, # type: ignore[arg-type] + # https://github.com/zopefoundation/RestrictedPython/issues/318 flags=flags, dont_inherit=dont_inherit, policy=policy) @@ -234,4 +238,4 @@ def compile_restricted( ) if result.errors: raise SyntaxError(result.errors) - return result.code + return _cast_not_none(result.code) diff --git a/src/RestrictedPython/transformer.py b/src/RestrictedPython/transformer.py index 39204c27..f02f6eaf 100644 --- a/src/RestrictedPython/transformer.py +++ b/src/RestrictedPython/transformer.py @@ -19,7 +19,9 @@ import ast +import collections import contextlib +import sys import textwrap import typing @@ -112,14 +114,20 @@ "cr_origin", ]) -_T_visit_return: typing.TypeAlias = ast.AST | list[ast.AST] | None +_T_visit_return: typing.TypeAlias = ast.AST | typing.Iterable[ast.AST] | None +_T_pos_ast: typing.TypeAlias = ( + ast.stmt | ast.expr | ast.excepthandler | ast.arg | ast.keyword | ast.alias + | ast.pattern) +if sys.version_info >= (3, 12): + _T_pos_ast: typing.TypeAlias = _T_pos_ast | ast.type_param +_T = typing.TypeVar('_T', bound=ast.AST) # When new ast nodes are generated they have no 'lineno', 'end_lineno', # 'col_offset' and 'end_col_offset'. This function copies these fields from the # incoming node: -def copy_locations(new_node: ast.AST, old_node: ast.AST) -> None: +def copy_locations(new_node: _T_pos_ast, old_node: _T_pos_ast) -> None: assert 'lineno' in new_node._attributes new_node.lineno = old_node.lineno @@ -136,12 +144,12 @@ def copy_locations(new_node: ast.AST, old_node: ast.AST) -> None: class PrintInfo: - def __init__(self): + def __init__(self) -> None: self.print_used = False self.printed_used = False @contextlib.contextmanager - def new_print_scope(self): + def new_print_scope(self) -> collections.abc.Iterator[None]: old_print_used = self.print_used old_printed_used = self.printed_used @@ -199,7 +207,7 @@ def warn(self, node: ast.AST, info: str) -> None: self.warnings.append( f'Line {lineno}: {info}') - def guard_iter(self, node: ast.AST) -> ast.AST: + def guard_iter(self, node: ast.For | ast.comprehension) -> _T_visit_return: """ Converts: for x in expr @@ -230,7 +238,9 @@ def guard_iter(self, node: ast.AST) -> ast.AST: node.iter = new_iter return node - def is_starred(self, ob: ast.AST) -> bool: + def is_starred(self, ob: ast.AST) -> typing.TypeGuard[ast.Starred]: + # TODO: Change Type Annotation to typing.TypeIs[ast.Starred] when + # Support for Python 3.12 is dropped. return isinstance(ob, ast.Starred) def gen_unpack_spec(self, tpl: ast.Tuple) -> ast.Dict: @@ -281,7 +291,8 @@ def gen_unpack_spec(self, tpl: ast.Tuple) -> ast.Dict: spec = ast.Dict(keys=[], values=[]) spec.keys.append(ast.Constant('childs')) - spec.values.append(ast.Tuple([], ast.Load())) + val0 = ast.Tuple([], ast.Load()) + spec.values.append(val0) # starred elements in a sequence do not contribute into the min_len. # For example a, b, *c = g @@ -302,7 +313,7 @@ def gen_unpack_spec(self, tpl: ast.Tuple) -> ast.Dict: el = ast.Tuple([], ast.Load()) el.elts.append(ast.Constant(idx - offset)) el.elts.append(self.gen_unpack_spec(val)) - spec.values[0].elts.append(el) + val0.elts.append(el) spec.keys.append(ast.Constant('min_len')) spec.values.append(ast.Constant(min_len)) @@ -312,14 +323,15 @@ def gen_unpack_spec(self, tpl: ast.Tuple) -> ast.Dict: def protect_unpack_sequence( self, target: ast.Tuple, - value: ast.AST) -> ast.Call: + value: ast.expr) -> ast.Call: spec = self.gen_unpack_spec(target) return ast.Call( func=ast.Name('_unpack_sequence_', ast.Load()), args=[value, spec, ast.Name('_getiter_', ast.Load())], keywords=[]) - def gen_unpack_wrapper(self, node: ast.AST, + def gen_unpack_wrapper(self, + node: ast.stmt, target: ast.Tuple) -> tuple[ast.Name, ast.Try]: """Helper function to protect tuple unpacks. @@ -356,8 +368,9 @@ def gen_unpack_wrapper(self, node: ast.AST, # arg = converter # finally: # del tmp_arg - try_body = [ast.Assign(targets=[target], value=converter)] - finalbody = [self.gen_del_stmt(tmp_name)] + try_body: list[ast.stmt] = [ast.Assign( + targets=[target], value=converter)] + finalbody: list[ast.stmt] = [self.gen_del_stmt(tmp_name)] cleanup = ast.Try( body=try_body, finalbody=finalbody, handlers=[], orelse=[]) @@ -377,8 +390,8 @@ def gen_del_stmt(self, name_to_del: str) -> ast.Delete: def check_name( self, - node: ast.AST, - name: str, + node: _T_pos_ast, + name: str | None, allow_magic_methods: bool = False) -> None: """Check names if they are allowed. @@ -404,7 +417,9 @@ def check_name( elif name in FORBIDDEN_FUNC_NAMES: self.error(node, f'"{name}" is a reserved name.') - def check_function_argument_names(self, node: ast.FunctionDef) -> None: + def check_function_argument_names( + self, + node: ast.FunctionDef | ast.AsyncFunctionDef | ast.Lambda) -> None: for arg in node.args.args: self.check_name(node, arg.arg) @@ -434,7 +449,10 @@ def check_import_names(self, node: ast.ImportFrom | ast.Import) -> ast.AST: return self.node_contents_visit(node) - def inject_print_collector(self, node: ast.AST, position: int = 0) -> None: + def inject_print_collector( + self, + node: ast.Module | ast.FunctionDef, + position: int = 0) -> None: print_used = self.print_info.print_used printed_used = self.print_info.printed_used @@ -467,7 +485,8 @@ def inject_print_collector(self, node: ast.AST, position: int = 0) -> None: # Special Functions for an ast.NodeTransformer - def generic_visit(self, node: ast.AST) -> ast.AST | None: + def generic_visit(self, # type: ignore[override] + node: ast.AST) -> _T_visit_return: """Reject ast nodes which do not have a corresponding `visit_` method. This is needed to prevent new ast nodes from new Python versions to be @@ -487,9 +506,9 @@ def not_allowed(self, node: ast.AST) -> None: node, f'{node.__class__.__name__} statements are not allowed.') - def node_contents_visit(self, node: ast.AST) -> ast.AST: + def node_contents_visit(self, node: _T) -> _T: """Visit the contents of a node.""" - return super().generic_visit(node) + return super().generic_visit(node) # type: ignore[return-value] # ast for Literals @@ -570,6 +589,7 @@ def visit_Name(self, node: ast.Name) -> _T_visit_return: node = self.node_contents_visit(node) if isinstance(node.ctx, ast.Load): + new_node: _T_pos_ast if node.id == 'printed': self.print_info.printed_used = True new_node = ast.Call( @@ -1213,7 +1233,8 @@ def visit_ClassDef(self, node: ast.ClassDef) -> _T_visit_return: class {0.name}(metaclass=__metaclass__): pass '''.format(node)) - new_class_node = ast.parse(CLASS_DEF).body[0] + new_class_node = typing.cast( + ast.ClassDef, ast.parse(CLASS_DEF).body[0]) new_class_node.body = node.body new_class_node.bases = node.bases new_class_node.decorator_list = node.decorator_list @@ -1268,7 +1289,7 @@ def visit_NamedExpr(self, node: ast.NamedExpr) -> _T_visit_return: node = self.node_contents_visit(node) # this checks ``node.target`` target = node.target if not isinstance(target, ast.Name): - self.error( + self.error( # type: ignore[unreachable] node, "Assignment expressions are only allowed for simple targets") return node From 22ced9021e7a89677d95bf0cccc701758f489c1e Mon Sep 17 00:00:00 2001 From: zedzhen <59135268+zedzhen@users.noreply.github.com> Date: Wed, 3 Jun 2026 18:09:24 +0300 Subject: [PATCH 40/40] add mypy check in pre-commit --- .pre-commit-config.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d2d50290..ec5b0d50 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,3 +27,8 @@ repos: - id: flake8 additional_dependencies: - flake8-debugger == 4.1.2 + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v2.1.0 + hooks: + - id: mypy + pass_filenames: false