Module opshin.optimize.optimize_union_expansion
Expand source code
from _ast import Call, FunctionDef
from ast import *
from dataclasses import dataclass, field
from itertools import product
from typing import Any, List, Optional
from ..util import CompilingNodeTransformer, NameSupply
from .optimize_remove_deadconds import OptimizeRemoveDeadConditions
from copy import deepcopy
"""
Expand union types
"""
def _sanitize_type_key(raw: str) -> str:
return (
raw.replace(" ", "")
.replace("__", "___")
.replace("[", "_l_")
.replace("]", "_r_")
.replace(",", "_c_")
.replace(".", "_d_")
)
def type_to_key(typ: expr) -> str:
try:
raw = unparse(typ)
except Exception:
return "UnknownType"
return _sanitize_type_key(raw)
def type_to_specialization_key(typ: Any) -> str:
if isinstance(typ, expr):
if isinstance(typ, Name):
return _sanitize_type_key(typ.id)
return type_to_key(typ)
concrete_typ = getattr(typ, "typ", typ)
if hasattr(concrete_typ, "record") and hasattr(concrete_typ.record, "orig_name"):
return _sanitize_type_key(concrete_typ.record.orig_name)
if hasattr(concrete_typ, "python_type"):
return _sanitize_type_key(concrete_typ.python_type())
return _sanitize_type_key(str(concrete_typ))
@dataclass(frozen=True)
class UnionExpansionVariant:
id: str
@dataclass
class UnionExpansion:
specialized_argument_positions: tuple[int, ...]
variants: dict[tuple[str, ...], UnionExpansionVariant] = field(default_factory=dict)
@staticmethod
def _type_key(argument_types: list[Any]) -> tuple[str, ...]:
return tuple(type_to_specialization_key(typ) for typ in argument_types)
def register(
self,
specialized_argument_types: list[Any],
variant: UnionExpansionVariant,
) -> bool:
key = self._type_key(specialized_argument_types)
if key in self.variants:
return False
self.variants[key] = variant
return True
def variant_for(self, argument_types: list[Any]) -> Optional[UnionExpansionVariant]:
specialized_types = [
argument_types[i] for i in self.specialized_argument_positions
]
return self.variants.get(self._type_key(specialized_types))
class RewriteKnownIsinstanceChecks(CompilingNodeTransformer):
def __init__(self, arg_types: dict[str, str]):
self.arg_types = arg_types
def visit_Call(self, node: Call) -> Any:
node = self.generic_visit(node)
if (
isinstance(node.func, Name)
and node.func.id == "isinstance"
and len(node.args) == 2
):
arg, typ = node.args
if isinstance(arg, Name) and isinstance(typ, Name):
known_type = self.arg_types.get(arg.id)
if known_type is not None:
typ_str = getattr(typ, "id", type_to_key(typ))
return Constant(value=(known_type == typ_str))
return node
class OptimizeUnionExpansion(CompilingNodeTransformer):
step = "Expanding Unions"
def __init__(self):
self.current_class_name: Optional[str] = None
def visit(self, node):
previous_class_name = self.current_class_name
if isinstance(node, ClassDef):
self.current_class_name = node.name
try:
if isinstance(node, Module):
self.name_supply = NameSupply.from_tree(node, "union")
if hasattr(node, "body") and isinstance(node.body, list):
node.body = self.visit_sequence(node.body)
if hasattr(node, "orelse") and isinstance(node.orelse, list):
node.orelse = self.visit_sequence(node.orelse)
if hasattr(node, "finalbody") and isinstance(node.finalbody, list):
node.finalbody = self.visit_sequence(node.finalbody)
return super().visit(node)
finally:
self.current_class_name = previous_class_name
def specialization_key(self, typ: expr) -> str:
if (
isinstance(typ, Name)
and typ.id == "Self"
and self.current_class_name is not None
):
return _sanitize_type_key(self.current_class_name)
if isinstance(typ, Constant) and isinstance(typ.value, str):
return _sanitize_type_key(typ.value)
return type_to_specialization_key(typ)
def is_Union_annotation(self, ann: expr):
if isinstance(ann, Subscript) and isinstance(ann.value, Name):
if ann.value.id == "Union":
return ann.slice.elts
return False
def _union_arg_positions(self, stmt: FunctionDef) -> list[int]:
positions = []
for i, arg in enumerate(stmt.args.args):
if self.is_Union_annotation(arg.annotation):
positions.append(i)
return positions
def _specialize_function(
self,
stmt: FunctionDef,
union_positions: list[int],
union_type_options: list[list[expr]],
) -> tuple[List[FunctionDef], UnionExpansion]:
new_functions = []
expansion = UnionExpansion(tuple(union_positions))
for concrete_types in product(*union_type_options):
new_f = deepcopy(stmt)
# Calls are first type-checked against the unspecialized function,
# which supplies omitted defaults. Specialized variants are an
# internal dispatch target and must not independently re-check a
# default against every narrowed union member.
new_f.args.defaults = []
known_union_types = {}
specialization_keys = []
for i, typ in zip(union_positions, concrete_types):
concrete_type = deepcopy(typ)
new_f.args.args[i].annotation = concrete_type
type_key = self.specialization_key(concrete_type)
known_union_types[new_f.args.args[i].arg] = type_key
specialization_keys.append(type_key)
variant = UnionExpansionVariant(self.name_supply.fresh_name())
if not expansion.register(specialization_keys, variant):
continue
new_f.name = variant.id
new_f.union_expansion_variant = variant
new_f = RewriteKnownIsinstanceChecks(known_union_types).visit(new_f)
new_f = OptimizeRemoveDeadConditions().visit(new_f)
new_functions.append(new_f)
return new_functions, expansion
def visit_sequence(self, body):
new_body = []
for stmt in body:
if not isinstance(stmt, FunctionDef):
new_body.append(stmt)
continue
union_positions = self._union_arg_positions(stmt)
if not union_positions:
new_body.append(stmt)
continue
union_type_options = [
self.is_Union_annotation(stmt.args.args[i].annotation)
for i in union_positions
]
new_funcs, expansion = self._specialize_function(
stmt, union_positions, union_type_options
)
stmt.union_expansion = expansion
new_body.append(stmt)
new_body.extend(new_funcs)
return new_body
Functions
def type_to_key(typ: ast.expr) ‑> str-
Expand source code
def type_to_key(typ: expr) -> str: try: raw = unparse(typ) except Exception: return "UnknownType" return _sanitize_type_key(raw) def type_to_specialization_key(typ: Any) ‑> str-
Expand source code
def type_to_specialization_key(typ: Any) -> str: if isinstance(typ, expr): if isinstance(typ, Name): return _sanitize_type_key(typ.id) return type_to_key(typ) concrete_typ = getattr(typ, "typ", typ) if hasattr(concrete_typ, "record") and hasattr(concrete_typ.record, "orig_name"): return _sanitize_type_key(concrete_typ.record.orig_name) if hasattr(concrete_typ, "python_type"): return _sanitize_type_key(concrete_typ.python_type()) return _sanitize_type_key(str(concrete_typ))
Classes
class OptimizeUnionExpansion-
A :class:
NodeVisitorsubclass that walks the abstract syntax tree and allows modification of nodes.The
NodeTransformerwill walk the AST and use the return value of the visitor methods to replace or remove the old node. If the return value of the visitor method isNone, the node will be removed from its location, otherwise it is replaced with the return value. The return value may be the original node in which case no replacement takes place.Here is an example transformer that rewrites all occurrences of name lookups (
foo) todata['foo']::class RewriteName(NodeTransformer):
def visit_Name(self, node): return Subscript( value=Name(id='data', ctx=Load()), slice=Constant(value=node.id), ctx=node.ctx )Keep in mind that if the node you're operating on has child nodes you must either transform the child nodes yourself or call the :meth:
generic_visitmethod for the node first.For nodes that were part of a collection of statements (that applies to all statement nodes), the visitor may also return a list of nodes rather than just a single node.
Usually you use the transformer like this::
node = YourTransformer().visit(node)
Expand source code
class OptimizeUnionExpansion(CompilingNodeTransformer): step = "Expanding Unions" def __init__(self): self.current_class_name: Optional[str] = None def visit(self, node): previous_class_name = self.current_class_name if isinstance(node, ClassDef): self.current_class_name = node.name try: if isinstance(node, Module): self.name_supply = NameSupply.from_tree(node, "union") if hasattr(node, "body") and isinstance(node.body, list): node.body = self.visit_sequence(node.body) if hasattr(node, "orelse") and isinstance(node.orelse, list): node.orelse = self.visit_sequence(node.orelse) if hasattr(node, "finalbody") and isinstance(node.finalbody, list): node.finalbody = self.visit_sequence(node.finalbody) return super().visit(node) finally: self.current_class_name = previous_class_name def specialization_key(self, typ: expr) -> str: if ( isinstance(typ, Name) and typ.id == "Self" and self.current_class_name is not None ): return _sanitize_type_key(self.current_class_name) if isinstance(typ, Constant) and isinstance(typ.value, str): return _sanitize_type_key(typ.value) return type_to_specialization_key(typ) def is_Union_annotation(self, ann: expr): if isinstance(ann, Subscript) and isinstance(ann.value, Name): if ann.value.id == "Union": return ann.slice.elts return False def _union_arg_positions(self, stmt: FunctionDef) -> list[int]: positions = [] for i, arg in enumerate(stmt.args.args): if self.is_Union_annotation(arg.annotation): positions.append(i) return positions def _specialize_function( self, stmt: FunctionDef, union_positions: list[int], union_type_options: list[list[expr]], ) -> tuple[List[FunctionDef], UnionExpansion]: new_functions = [] expansion = UnionExpansion(tuple(union_positions)) for concrete_types in product(*union_type_options): new_f = deepcopy(stmt) # Calls are first type-checked against the unspecialized function, # which supplies omitted defaults. Specialized variants are an # internal dispatch target and must not independently re-check a # default against every narrowed union member. new_f.args.defaults = [] known_union_types = {} specialization_keys = [] for i, typ in zip(union_positions, concrete_types): concrete_type = deepcopy(typ) new_f.args.args[i].annotation = concrete_type type_key = self.specialization_key(concrete_type) known_union_types[new_f.args.args[i].arg] = type_key specialization_keys.append(type_key) variant = UnionExpansionVariant(self.name_supply.fresh_name()) if not expansion.register(specialization_keys, variant): continue new_f.name = variant.id new_f.union_expansion_variant = variant new_f = RewriteKnownIsinstanceChecks(known_union_types).visit(new_f) new_f = OptimizeRemoveDeadConditions().visit(new_f) new_functions.append(new_f) return new_functions, expansion def visit_sequence(self, body): new_body = [] for stmt in body: if not isinstance(stmt, FunctionDef): new_body.append(stmt) continue union_positions = self._union_arg_positions(stmt) if not union_positions: new_body.append(stmt) continue union_type_options = [ self.is_Union_annotation(stmt.args.args[i].annotation) for i in union_positions ] new_funcs, expansion = self._specialize_function( stmt, union_positions, union_type_options ) stmt.union_expansion = expansion new_body.append(stmt) new_body.extend(new_funcs) return new_bodyAncestors
- CompilingNodeTransformer
- TypedNodeTransformer
- ast.NodeTransformer
- ast.NodeVisitor
Class variables
var step-
Inherited from:
CompilingNodeTransformer.stepThe type of the None singleton.
Methods
def is_Union_annotation(self, ann: ast.expr)-
Expand source code
def is_Union_annotation(self, ann: expr): if isinstance(ann, Subscript) and isinstance(ann.value, Name): if ann.value.id == "Union": return ann.slice.elts return False def specialization_key(self, typ: ast.expr) ‑> str-
Expand source code
def specialization_key(self, typ: expr) -> str: if ( isinstance(typ, Name) and typ.id == "Self" and self.current_class_name is not None ): return _sanitize_type_key(self.current_class_name) if isinstance(typ, Constant) and isinstance(typ.value, str): return _sanitize_type_key(typ.value) return type_to_specialization_key(typ) def visit(self, node)-
Inherited from:
CompilingNodeTransformer.visitVisit a node.
Expand source code
def visit(self, node): previous_class_name = self.current_class_name if isinstance(node, ClassDef): self.current_class_name = node.name try: if isinstance(node, Module): self.name_supply = NameSupply.from_tree(node, "union") if hasattr(node, "body") and isinstance(node.body, list): node.body = self.visit_sequence(node.body) if hasattr(node, "orelse") and isinstance(node.orelse, list): node.orelse = self.visit_sequence(node.orelse) if hasattr(node, "finalbody") and isinstance(node.finalbody, list): node.finalbody = self.visit_sequence(node.finalbody) return super().visit(node) finally: self.current_class_name = previous_class_name def visit_sequence(self, body)-
Expand source code
def visit_sequence(self, body): new_body = [] for stmt in body: if not isinstance(stmt, FunctionDef): new_body.append(stmt) continue union_positions = self._union_arg_positions(stmt) if not union_positions: new_body.append(stmt) continue union_type_options = [ self.is_Union_annotation(stmt.args.args[i].annotation) for i in union_positions ] new_funcs, expansion = self._specialize_function( stmt, union_positions, union_type_options ) stmt.union_expansion = expansion new_body.append(stmt) new_body.extend(new_funcs) return new_body
class RewriteKnownIsinstanceChecks (arg_types: dict[str, str])-
A :class:
NodeVisitorsubclass that walks the abstract syntax tree and allows modification of nodes.The
NodeTransformerwill walk the AST and use the return value of the visitor methods to replace or remove the old node. If the return value of the visitor method isNone, the node will be removed from its location, otherwise it is replaced with the return value. The return value may be the original node in which case no replacement takes place.Here is an example transformer that rewrites all occurrences of name lookups (
foo) todata['foo']::class RewriteName(NodeTransformer):
def visit_Name(self, node): return Subscript( value=Name(id='data', ctx=Load()), slice=Constant(value=node.id), ctx=node.ctx )Keep in mind that if the node you're operating on has child nodes you must either transform the child nodes yourself or call the :meth:
generic_visitmethod for the node first.For nodes that were part of a collection of statements (that applies to all statement nodes), the visitor may also return a list of nodes rather than just a single node.
Usually you use the transformer like this::
node = YourTransformer().visit(node)
Expand source code
class RewriteKnownIsinstanceChecks(CompilingNodeTransformer): def __init__(self, arg_types: dict[str, str]): self.arg_types = arg_types def visit_Call(self, node: Call) -> Any: node = self.generic_visit(node) if ( isinstance(node.func, Name) and node.func.id == "isinstance" and len(node.args) == 2 ): arg, typ = node.args if isinstance(arg, Name) and isinstance(typ, Name): known_type = self.arg_types.get(arg.id) if known_type is not None: typ_str = getattr(typ, "id", type_to_key(typ)) return Constant(value=(known_type == typ_str)) return nodeAncestors
- CompilingNodeTransformer
- TypedNodeTransformer
- ast.NodeTransformer
- ast.NodeVisitor
Class variables
var step-
Inherited from:
CompilingNodeTransformer.stepThe type of the None singleton.
Methods
def visit(self, node)-
Inherited from:
CompilingNodeTransformer.visitVisit a node.
def visit_Call(self, node: ast.Call) ‑> Any-
Expand source code
def visit_Call(self, node: Call) -> Any: node = self.generic_visit(node) if ( isinstance(node.func, Name) and node.func.id == "isinstance" and len(node.args) == 2 ): arg, typ = node.args if isinstance(arg, Name) and isinstance(typ, Name): known_type = self.arg_types.get(arg.id) if known_type is not None: typ_str = getattr(typ, "id", type_to_key(typ)) return Constant(value=(known_type == typ_str)) return node
class UnionExpansion (specialized_argument_positions: tuple[int, ...], variants: dict[tuple[str, ...], UnionExpansionVariant] = <factory>)-
UnionExpansion(specialized_argument_positions: tuple[int, …], variants: dict[tuple[str, …], opshin.optimize.optimize_union_expansion.UnionExpansionVariant] =
) Expand source code
@dataclass class UnionExpansion: specialized_argument_positions: tuple[int, ...] variants: dict[tuple[str, ...], UnionExpansionVariant] = field(default_factory=dict) @staticmethod def _type_key(argument_types: list[Any]) -> tuple[str, ...]: return tuple(type_to_specialization_key(typ) for typ in argument_types) def register( self, specialized_argument_types: list[Any], variant: UnionExpansionVariant, ) -> bool: key = self._type_key(specialized_argument_types) if key in self.variants: return False self.variants[key] = variant return True def variant_for(self, argument_types: list[Any]) -> Optional[UnionExpansionVariant]: specialized_types = [ argument_types[i] for i in self.specialized_argument_positions ] return self.variants.get(self._type_key(specialized_types))Instance variables
var specialized_argument_positions : tuple[int, ...]-
The type of the None singleton.
var variants : dict[tuple[str, ...], UnionExpansionVariant]-
The type of the None singleton.
Methods
def register(self, specialized_argument_types: list[typing.Any], variant: UnionExpansionVariant) ‑> bool-
Expand source code
def register( self, specialized_argument_types: list[Any], variant: UnionExpansionVariant, ) -> bool: key = self._type_key(specialized_argument_types) if key in self.variants: return False self.variants[key] = variant return True def variant_for(self, argument_types: list[typing.Any]) ‑> UnionExpansionVariant | None-
Expand source code
def variant_for(self, argument_types: list[Any]) -> Optional[UnionExpansionVariant]: specialized_types = [ argument_types[i] for i in self.specialized_argument_positions ] return self.variants.get(self._type_key(specialized_types))
class UnionExpansionVariant (id: str)-
UnionExpansionVariant(id: str)
Expand source code
@dataclass(frozen=True) class UnionExpansionVariant: id: strInstance variables
var id : str-
The type of the None singleton.