Merge pull request #1183 from exponea/1138-fix-unintended-lstrip-behavior-change

Fix unintended lstrip_blocks behavior

Author: davidism

CHANGES.rst

@@ -25,6 +25,8 @@ Unreleased
     :pr:`1178`
 -   The special ``namespace()`` assignment object in templates works in
     async environments. :issue:`1180`
+-   Fix whitespace being removed before tags in the middle of lines when
+    ``lstrip_blocks`` is enabled. :issue:`1138`
 
 
 Version 2.11.1

src/jinja2/lexer.py

@@ -682,6 +682,7 @@ def tokeniter(self, source, name, filename=None, state=None):
         balancing_stack = []
         lstrip_unless_re = self.lstrip_unless_re
         newlines_stripped = 0
+        line_starting = True
 
         while 1:
             # tokenizer loop
@@ -731,11 +732,11 @@ def tokeniter(self, source, name, filename=None, state=None):
                         ):
                             # The start of text between the last newline and the tag.
                             l_pos = text.rfind("\n") + 1
-
-                            # If there's only whitespace between the newline and the
-                            # tag, strip it.
-                            if not lstrip_unless_re.search(text, l_pos):
-                                groups = (text[:l_pos],) + groups[1:]
+                            if l_pos > 0 or line_starting:
+                                # If there's only whitespace between the newline and the
+                                # tag, strip it.
+                                if not lstrip_unless_re.search(text, l_pos):
+                                    groups = (text[:l_pos],) + groups[1:]
 
                     for idx, token in enumerate(tokens):
                         # failure group
@@ -794,6 +795,8 @@ def tokeniter(self, source, name, filename=None, state=None):
                         yield lineno, tokens, data
                     lineno += data.count("\n")
 
+                line_starting = m.group()[-1:] == "\n"
+
                 # fetch new position into new variable so that we can check
                 # if there is a internal parsing error which would result
                 # in an infinite loop

tests/test_lexnparse.py

@@ -729,6 +729,98 @@ def test_lstrip_angle_bracket_compact(self, env):
         )
         assert tmpl.render(seq=range(5)) == "".join("%s\n" % x for x in range(5))
 
+    def test_lstrip_blocks_outside_with_new_line(self):
+        env = Environment(lstrip_blocks=True, trim_blocks=False)
+        tmpl = env.from_string(
+            "  {% if kvs %}(\n"
+            "   {% for k, v in kvs %}{{ k }}={{ v }} {% endfor %}\n"
+            "  ){% endif %}"
+        )
+        out = tmpl.render(kvs=[("a", 1), ("b", 2)])
+        assert out == "(\na=1 b=2 \n  )"
+
+    def test_lstrip_trim_blocks_outside_with_new_line(self):
+        env = Environment(lstrip_blocks=True, trim_blocks=True)
+        tmpl = env.from_string(
+            "  {% if kvs %}(\n"
+            "   {% for k, v in kvs %}{{ k }}={{ v }} {% endfor %}\n"
+            "  ){% endif %}"
+        )
+        out = tmpl.render(kvs=[("a", 1), ("b", 2)])
+        assert out == "(\na=1 b=2   )"
+
+    def test_lstrip_blocks_inside_with_new_line(self):
+        env = Environment(lstrip_blocks=True, trim_blocks=False)
+        tmpl = env.from_string(
+            "  ({% if kvs %}\n"
+            "   {% for k, v in kvs %}{{ k }}={{ v }} {% endfor %}\n"
+            "  {% endif %})"
+        )
+        out = tmpl.render(kvs=[("a", 1), ("b", 2)])
+        assert out == "  (\na=1 b=2 \n)"
+
+    def test_lstrip_trim_blocks_inside_with_new_line(self):
+        env = Environment(lstrip_blocks=True, trim_blocks=True)
+        tmpl = env.from_string(
+            "  ({% if kvs %}\n"
+            "   {% for k, v in kvs %}{{ k }}={{ v }} {% endfor %}\n"
+            "  {% endif %})"
+        )
+        out = tmpl.render(kvs=[("a", 1), ("b", 2)])
+        assert out == "  (a=1 b=2 )"
+
+    def test_lstrip_blocks_without_new_line(self):
+        env = Environment(lstrip_blocks=True, trim_blocks=False)
+        tmpl = env.from_string(
+            "  {% if kvs %}"
+            "   {% for k, v in kvs %}{{ k }}={{ v }} {% endfor %}"
+            "  {% endif %}"
+        )
+        out = tmpl.render(kvs=[("a", 1), ("b", 2)])
+        assert out == "   a=1 b=2   "
+
+    def test_lstrip_trim_blocks_without_new_line(self):
+        env = Environment(lstrip_blocks=True, trim_blocks=True)
+        tmpl = env.from_string(
+            "  {% if kvs %}"
+            "   {% for k, v in kvs %}{{ k }}={{ v }} {% endfor %}"
+            "  {% endif %}"
+        )
+        out = tmpl.render(kvs=[("a", 1), ("b", 2)])
+        assert out == "   a=1 b=2   "
+
+    def test_lstrip_blocks_consume_after_without_new_line(self):
+        env = Environment(lstrip_blocks=True, trim_blocks=False)
+        tmpl = env.from_string(
+            "  {% if kvs -%}"
+            "   {% for k, v in kvs %}{{ k }}={{ v }} {% endfor -%}"
+            "  {% endif -%}"
+        )
+        out = tmpl.render(kvs=[("a", 1), ("b", 2)])
+        assert out == "a=1 b=2 "
+
+    def test_lstrip_trim_blocks_consume_before_without_new_line(self):
+        env = Environment(lstrip_blocks=False, trim_blocks=False)
+        tmpl = env.from_string(
+            "  {%- if kvs %}"
+            "   {%- for k, v in kvs %}{{ k }}={{ v }} {% endfor -%}"
+            "  {%- endif %}"
+        )
+        out = tmpl.render(kvs=[("a", 1), ("b", 2)])
+        assert out == "a=1 b=2 "
+
+    def test_lstrip_trim_blocks_comment(self):
+        env = Environment(lstrip_blocks=True, trim_blocks=True)
+        tmpl = env.from_string(" {# 1 space #}\n  {# 2 spaces #}    {# 4 spaces #}")
+        out = tmpl.render()
+        assert out == " " * 4
+
+    def test_lstrip_trim_blocks_raw(self):
+        env = Environment(lstrip_blocks=True, trim_blocks=True)
+        tmpl = env.from_string("{{x}}\n{%- raw %} {% endraw -%}\n{{ y }}")
+        out = tmpl.render(x=1, y=2)
+        assert out == "1 2"
+
     def test_php_syntax_with_manual(self, env):
         env = Environment(
             "<?", "?>", "<?=", "?>", "<!--", "-->", lstrip_blocks=True, trim_blocks=True