Parser
Parser
"""
import typing
import typing as t
if t.TYPE_CHECKING:
import typing_extensions as te
_statement_keywords = frozenset(
[
"for",
"if",
"block",
"extends",
"print",
"macro",
"include",
"from",
"import",
"set",
"with",
"autoescape",
]
)
_compare_operators = frozenset(["eq", "ne", "lt", "lteq", "gt", "gteq"])
class Parser:
"""This is the central parsing class Jinja uses. It's passed to
extensions and can be used to parse expressions or statements.
"""
def __init__(
self,
environment: "Environment",
source: str,
name: [Link][str] = None,
filename: [Link][str] = None,
state: [Link][str] = None,
) -> None:
[Link] = environment
[Link] = environment._tokenize(source, name, filename, state)
[Link] = name
[Link] = filename
[Link] = False
[Link]: [Link][
str, [Link][[Parser], [Link][[Link], [Link][[Link]]]]
] = {}
for extension in environment.iter_extensions():
for tag in [Link]:
[Link][tag] = [Link]
self._last_identifier = 0
self._tag_stack: [Link][str] = []
self._end_token_stack: [Link][[Link][str, ...]] = []
def fail(
self,
msg: str,
lineno: [Link][int] = None,
exc: [Link][TemplateSyntaxError] = TemplateSyntaxError,
) -> "[Link]":
"""Convenience method that raises `exc` with the message, passed
line number or last line number as well as the current name and
filename.
"""
if lineno is None:
lineno = [Link]
raise exc(msg, lineno, [Link], [Link])
def _fail_ut_eof(
self,
name: [Link][str],
end_token_stack: [Link][[Link][str, ...]],
lineno: [Link][int],
) -> "[Link]":
expected: [Link][str] = set()
for exprs in end_token_stack:
[Link](map(describe_token_expr, exprs))
if end_token_stack:
currently_looking: [Link][str] = " or ".join(
map(repr, map(describe_token_expr, end_token_stack[-1]))
)
else:
currently_looking = None
if name is None:
message = ["Unexpected end of template."]
else:
message = [f"Encountered unknown tag {name!r}."]
if currently_looking:
if name is not None and name in expected:
[Link](
"You probably made a nesting mistake. Jinja is expecting this tag,"
f" but currently looking for {currently_looking}."
)
else:
[Link](
f"Jinja was looking for the following tags: {currently_looking}."
)
if self._tag_stack:
[Link](
"The innermost block that needs to be closed is"
f" {self._tag_stack[-1]!r}."
)
def fail_unknown_tag(
self, name: str, lineno: [Link][int] = None
) -> "[Link]":
"""Called if the parser encounters an unknown tag. Tries to fail
with a human readable error message that could help to identify
the problem.
"""
self._fail_ut_eof(name, self._end_token_stack, lineno)
def fail_eof(
self,
end_tokens: [Link][[Link][str, ...]] = None,
lineno: [Link][int] = None,
) -> "[Link]":
"""Like fail_unknown_tag but for end of template situations."""
stack = list(self._end_token_stack)
if end_tokens is not None:
[Link](end_tokens)
self._fail_ut_eof(None, stack, lineno)
def is_tuple_end(
self, extra_end_rules: [Link][[Link][str, ...]] = None
) -> bool:
"""Are we at the end of a tuple?"""
if [Link] in ("variable_end", "block_end", "rparen"):
return True
elif extra_end_rules is not None:
return [Link].test_any(extra_end_rules) # type: ignore
return False
def parse_statements(
self, end_tokens: [Link][str, ...], drop_needle: bool = False
) -> [Link][[Link]]:
"""Parse multiple statements into a list until one of the end tokens
is reached. This is used to parse the body of statements as it also
parses template data if appropriate. The parser checks first if the
current token is a colon and skips it if there is one. Then it checks
for the block end and parses until if one of the `end_tokens` is
reached. Per default the active token in the stream at the end of
the call is the matched end token. If this is not wanted `drop_needle`
can be set to `True` and the end token is removed.
"""
# the first token may be a colon for python compatibility
[Link].skip_if("colon")
if drop_needle:
next([Link])
return result
[Link].skip_if("name:" + [Link])
return node
def parse_import_context(
self, node: _ImportInclude, default: bool
) -> _ImportInclude:
if [Link].test_any(
"name:with", "name:without"
) and [Link]().test("name:context"):
node.with_context = next([Link]).value == "with"
[Link]()
else:
node.with_context = default
return node
while True:
if [Link]:
[Link]("comma")
if [Link] == "name":
if parse_context():
break
target = self.parse_assign_target(name_only=True)
if [Link]("_"):
[Link](
"names starting with an underline can not be imported",
[Link],
exc=TemplateAssertionError,
)
if [Link].skip_if("name:as"):
alias = self.parse_assign_target(name_only=True)
[Link](([Link], [Link]))
else:
[Link]([Link])
if parse_context() or [Link] != "comma":
break
else:
[Link]("name")
if not hasattr(node, "with_context"):
node.with_context = False
return node
call_node = self.parse_expression()
if not isinstance(call_node, [Link]):
[Link]("expected call", [Link])
[Link] = call_node
[Link] = self.parse_statements(("name:endcall",), drop_needle=True)
return node
@[Link]
def parse_assign_target(
self, with_tuple: bool = ..., name_only: "[Link][True]" = ...
) -> [Link]: ...
@[Link]
def parse_assign_target(
self,
with_tuple: bool = True,
name_only: bool = False,
extra_end_rules: [Link][[Link][str, ...]] = None,
with_namespace: bool = False,
) -> [Link][[Link], [Link], [Link]]: ...
def parse_assign_target(
self,
with_tuple: bool = True,
name_only: bool = False,
extra_end_rules: [Link][[Link][str, ...]] = None,
with_namespace: bool = False,
) -> [Link][[Link], [Link], [Link]]:
"""Parse an assignment target. As Jinja allows assignments to
tuples, this function can parse all allowed assignment targets. Per
default assignments to tuples are parsed, that can be disable however
by setting `with_tuple` to `False`. If only assignments to names are
wanted `name_only` can be set to `True`. The `extra_end_rules`
parameter is forwarded to the tuple parsing function. If
`with_namespace` is enabled, a namespace assignment may be parsed.
"""
target: [Link]
if name_only:
token = [Link]("name")
target = [Link]([Link], "store", lineno=[Link])
else:
if with_tuple:
target = self.parse_tuple(
simplified=True,
extra_end_rules=extra_end_rules,
with_namespace=with_namespace,
)
else:
target = self.parse_primary(with_namespace=with_namespace)
target.set_ctx("store")
if not target.can_assign():
[Link](
f"can't assign to {type(target).__name__.lower()!r}", [Link]
)
while [Link].skip_if("name:if"):
expr2 = self.parse_or()
if [Link].skip_if("name:else"):
expr3 = self.parse_condexpr()
else:
expr3 = None
expr1 = [Link](expr2, expr1, expr3, lineno=lineno)
lineno = [Link]
return expr1
if token_type == "sub":
next([Link])
node = [Link](self.parse_unary(False), lineno=lineno)
elif token_type == "add":
next([Link])
node = [Link](self.parse_unary(False), lineno=lineno)
else:
node = self.parse_primary()
node = self.parse_postfix(node)
if with_filter:
node = self.parse_filter_expr(node)
return node
def parse_tuple(
self,
simplified: bool = False,
with_condexpr: bool = True,
extra_end_rules: [Link][[Link][str, ...]] = None,
explicit_parentheses: bool = False,
with_namespace: bool = False,
) -> [Link][[Link], [Link]]:
"""Works like `parse_expression` but if multiple expressions are
delimited by a comma a :class:`~[Link]` node is created.
This method could also return a regular expression instead of a tuple
if no commas where found.
Because tuples do not require delimiters and may end in a bogus comma
an extra hint is needed that marks the end of a tuple. For example
for loops support tuples between `for` and `in`. In that case the
`extra_end_rules` is set to ``['name:in']``.
else:
args: [Link][[Link]] = []
is_tuple = False
while True:
if args:
[Link]("comma")
if self.is_tuple_end(extra_end_rules):
break
[Link](parse())
if [Link] == "comma":
is_tuple = True
else:
break
lineno = [Link]
if not is_tuple:
if args:
return args[0]
def parse_subscript(
self, node: [Link]
) -> [Link][[Link], [Link]]:
token = next([Link])
arg: [Link]
if [Link] == "dot":
attr_token = [Link]
next([Link])
if attr_token.type == "name":
return [Link](
node, attr_token.value, "load", lineno=[Link]
)
elif attr_token.type != "integer":
[Link]("expected name or number", attr_token.lineno)
arg = [Link](attr_token.value, lineno=attr_token.lineno)
return [Link](node, arg, "load", lineno=[Link])
if [Link] == "lbracket":
args: [Link][[Link]] = []
while [Link] != "rbracket":
if args:
[Link]("comma")
[Link](self.parse_subscribed())
[Link]("rbracket")
if len(args) == 1:
arg = args[0]
else:
arg = [Link](args, "load", lineno=[Link])
return [Link](node, arg, "load", lineno=[Link])
[Link]("expected subscript expression", [Link])
if [Link] == "colon":
next([Link])
args = [None]
else:
node = self.parse_expression()
if [Link] != "colon":
return node
next([Link])
args = [node]
if [Link] == "colon":
[Link](None)
elif [Link] not in ("rbracket", "comma"):
[Link](self.parse_expression())
else:
[Link](None)
if [Link] == "colon":
next([Link])
if [Link] not in ("rbracket", "comma"):
[Link](self.parse_expression())
else:
[Link](None)
else:
[Link](None)
def parse_call_args(
self,
) -> [Link][
[Link][[Link]],
[Link][[Link]],
[Link][[Link]],
[Link][[Link]],
]:
token = [Link]("lparen")
args = []
kwargs = []
dyn_args = None
dyn_kwargs = None
require_comma = False
if [Link] == "mul":
ensure(dyn_args is None and dyn_kwargs is None)
next([Link])
dyn_args = self.parse_expression()
elif [Link] == "pow":
ensure(dyn_kwargs is None)
next([Link])
dyn_kwargs = self.parse_expression()
else:
if (
[Link] == "name"
and [Link]().type == "assign"
):
# Parsing a kwarg
ensure(dyn_kwargs is None)
key = [Link]
[Link](2)
value = self.parse_expression()
[Link]([Link](key, value, lineno=[Link]))
else:
# Parsing an arg
ensure(dyn_args is None and dyn_kwargs is None and not kwargs)
[Link](self.parse_expression())
require_comma = True
[Link]("rparen")
return args, kwargs, dyn_args, dyn_kwargs
def parse_filter(
self, node: [Link][[Link]], start_inline: bool = False
) -> [Link][[Link]]:
while [Link] == "pipe" or start_inline:
if not start_inline:
next([Link])
token = [Link]("name")
name = [Link]
while [Link] == "dot":
next([Link])
name += "." + [Link]("name").value
if [Link] == "lparen":
args, kwargs, dyn_args, dyn_kwargs = self.parse_call_args()
else:
args = []
kwargs = []
dyn_args = dyn_kwargs = None
node = [Link](
node, name, args, kwargs, dyn_args, dyn_kwargs, lineno=[Link]
)
start_inline = False
return node
def subparse(
self, end_tokens: [Link][[Link][str, ...]] = None
) -> [Link][[Link]]:
body: [Link][[Link]] = []
data_buffer: [Link][[Link]] = []
add_data = data_buffer.append
try:
while [Link]:
token = [Link]
if [Link] == "data":
if [Link]:
add_data([Link]([Link], lineno=[Link]))
next([Link])
elif [Link] == "variable_begin":
next([Link])
add_data(self.parse_tuple(with_condexpr=True))
[Link]("variable_end")
elif [Link] == "block_begin":
flush_data()
next([Link])
if end_tokens is not None and [Link].test_any(
*end_tokens
):
return body
rv = self.parse_statement()
if isinstance(rv, list):
[Link](rv)
else:
[Link](rv)
[Link]("block_end")
else:
raise AssertionError("internal parsing error")
flush_data()
finally:
if end_tokens is not None:
self._end_token_stack.pop()
return body