diff --git a/src/main/java/com/hubspot/jinjava/Jinjava.java b/src/main/java/com/hubspot/jinjava/Jinjava.java index 5c06e3d7a..8ef3f2153 100644 --- a/src/main/java/com/hubspot/jinjava/Jinjava.java +++ b/src/main/java/com/hubspot/jinjava/Jinjava.java @@ -20,7 +20,6 @@ import com.hubspot.jinjava.el.ExtendedSyntaxBuilder; import com.hubspot.jinjava.el.TruthyTypeConverter; import com.hubspot.jinjava.el.ext.eager.EagerExtendedSyntaxBuilder; -import com.hubspot.jinjava.interpret.AutoCloseableSupplier.AutoCloseableImpl; import com.hubspot.jinjava.interpret.Context; import com.hubspot.jinjava.interpret.FatalTemplateErrorsException; import com.hubspot.jinjava.interpret.InterpretException; @@ -246,14 +245,10 @@ public RenderResult renderForResult( context = new Context(copyGlobalContext(), bindings, renderConfig.getDisabled()); } - try ( - AutoCloseableImpl interpreterAutoCloseable = JinjavaInterpreter - .closeablePushCurrent( - globalConfig.getInterpreterFactory().newInstance(this, context, renderConfig) - ) - .get() - ) { - JinjavaInterpreter interpreter = interpreterAutoCloseable.value(); + try { + JinjavaInterpreter interpreter = globalConfig + .getInterpreterFactory() + .newInstance(this, context, renderConfig); try { String result = interpreter.render(template); return new RenderResult( diff --git a/src/main/java/com/hubspot/jinjava/interpret/JinjavaInterpreter.java b/src/main/java/com/hubspot/jinjava/interpret/JinjavaInterpreter.java index 0c2f5b145..ed643e06b 100644 --- a/src/main/java/com/hubspot/jinjava/interpret/JinjavaInterpreter.java +++ b/src/main/java/com/hubspot/jinjava/interpret/JinjavaInterpreter.java @@ -355,170 +355,184 @@ public String render(Node root, boolean processExtendRoots) { * @return rendered result */ private String render(Node root, boolean processExtendRoots, long renderLimit) { - OutputList output = new OutputList( - RenderLimitUtils.clampProvidedRenderLimitToConfig(renderLimit, config) - ); - for (Node node : root.getChildren()) { - lineNumber = node.getLineNumber(); - position = node.getStartPosition(); - String renderStr = node.getMaster().getImage(); - try { - if (node instanceof ExpressionNode && context.doesRenderStackContain(renderStr)) { - // This is a circular rendering. Stop rendering it here. - addError( - new TemplateError( - ErrorType.WARNING, - ErrorReason.EXCEPTION, - ErrorItem.TAG, - "Rendering cycle detected: '" + renderStr + "'", - null, - getLineNumber(), - node.getStartPosition(), - null, - BasicTemplateErrorCategory.IMPORT_CYCLE_DETECTED, - ImmutableMap.of("string", renderStr) - ) - ); - output.addNode(new RenderedOutputNode(renderStr)); - } else { - OutputNode out; - try ( - AutoCloseableImpl closeable = context - .closeablePushRenderStack(renderStr) - .get() + boolean pushed = false; + //noinspection ErrorProne + if (JinjavaInterpreter.getCurrent() != this) { + JinjavaInterpreter.pushCurrent(this); + pushed = true; + } + try { + OutputList output = new OutputList( + RenderLimitUtils.clampProvidedRenderLimitToConfig(renderLimit, config) + ); + for (Node node : root.getChildren()) { + lineNumber = node.getLineNumber(); + position = node.getStartPosition(); + String renderStr = node.getMaster().getImage(); + try { + if ( + node instanceof ExpressionNode && context.doesRenderStackContain(renderStr) ) { - try { - out = node.render(this); - } catch (DeferredValueException e) { - context.handleDeferredNode(node); - out = new RenderedOutputNode(node.getMaster().getImage()); + // This is a circular rendering. Stop rendering it here. + addError( + new TemplateError( + ErrorType.WARNING, + ErrorReason.EXCEPTION, + ErrorItem.TAG, + "Rendering cycle detected: '" + renderStr + "'", + null, + getLineNumber(), + node.getStartPosition(), + null, + BasicTemplateErrorCategory.IMPORT_CYCLE_DETECTED, + ImmutableMap.of("string", renderStr) + ) + ); + output.addNode(new RenderedOutputNode(renderStr)); + } else { + OutputNode out; + try ( + AutoCloseableImpl closeable = context + .closeablePushRenderStack(renderStr) + .get() + ) { + try { + out = node.render(this); + } catch (DeferredValueException e) { + context.handleDeferredNode(node); + out = new RenderedOutputNode(node.getMaster().getImage()); + } } + output.addNode(out); } - output.addNode(out); - } - } catch (OutputTooBigException e) { - addError(TemplateError.fromOutputTooBigException(e)); - return output.getValue(); - } catch (CollectionTooBigException e) { - addError( - new TemplateError( - ErrorType.FATAL, - ErrorReason.COLLECTION_TOO_BIG, - ErrorItem.OTHER, - ExceptionUtils.getMessage(e), - null, - -1, - -1, - e, - BasicTemplateErrorCategory.UNKNOWN, - ImmutableMap.of() - ) - ); - return output.getValue(); - } - } - DynamicRenderedOutputNode pathSetter = new DynamicRenderedOutputNode(); - output.addNode(pathSetter); - Optional basePath = context.getCurrentPathStack().peek(); - StringBuilder ignoredOutput = new StringBuilder(); - // render all extend parents, keeping the last as the root output - if (processExtendRoots) { - Set extendPaths = new HashSet<>(); - Optional extendPath = context.getExtendPathStack().peek(); - int numDeferredTokensBefore = 0; - while (!extendParentRoots.isEmpty()) { - if (extendPaths.contains(extendPath.orElse(""))) { + } catch (OutputTooBigException e) { + addError(TemplateError.fromOutputTooBigException(e)); + return output.getValue(); + } catch (CollectionTooBigException e) { addError( - TemplateError.fromException( - new ExtendsTagCycleException( - extendPath.orElse(""), - context.getExtendPathStack().getTopLineNumber(), - context.getExtendPathStack().getTopStartPosition() - ) + new TemplateError( + ErrorType.FATAL, + ErrorReason.COLLECTION_TOO_BIG, + ErrorItem.OTHER, + ExceptionUtils.getMessage(e), + null, + -1, + -1, + e, + BasicTemplateErrorCategory.UNKNOWN, + ImmutableMap.of() ) ); - break; + return output.getValue(); } - extendPaths.add(extendPath.orElse("")); - try ( - AutoCloseableImpl> closeableCurrentPath = - context - .getCurrentPathStack() - .closeablePush( - extendPath.orElse(""), - context.getExtendPathStack().getTopLineNumber(), - context.getExtendPathStack().getTopStartPosition() + } + DynamicRenderedOutputNode pathSetter = new DynamicRenderedOutputNode(); + output.addNode(pathSetter); + Optional basePath = context.getCurrentPathStack().peek(); + StringBuilder ignoredOutput = new StringBuilder(); + // render all extend parents, keeping the last as the root output + if (processExtendRoots) { + Set extendPaths = new HashSet<>(); + Optional extendPath = context.getExtendPathStack().peek(); + int numDeferredTokensBefore = 0; + while (!extendParentRoots.isEmpty()) { + if (extendPaths.contains(extendPath.orElse(""))) { + addError( + TemplateError.fromException( + new ExtendsTagCycleException( + extendPath.orElse(""), + context.getExtendPathStack().getTopLineNumber(), + context.getExtendPathStack().getTopStartPosition() + ) ) - .get() - ) { - String currentPath = closeableCurrentPath - .value() - .unwrapOrElseThrow(Function.identity()); - Node parentRoot = extendParentRoots.removeFirst(); - if (context.getDeferredTokens().size() > numDeferredTokensBefore) { - ignoredOutput.append( - output - .getNodes() - .stream() - .filter(node -> node instanceof RenderedOutputNode) - .map(OutputNode::getValue) - .collect(Collectors.joining()) ); + break; } - numDeferredTokensBefore = context.getDeferredTokens().size(); - output = new OutputList(config.getMaxOutputSize()); - output.addNode(pathSetter); - boolean hasNestedExtends = false; - for (Node node : parentRoot.getChildren()) { - lineNumber = node.getLineNumber() - 1; // The line number is off by one when rendering the extend parent - position = node.getStartPosition(); - try { - OutputNode out = node.render(this); - output.addNode(out); - if (isExtendsTag(node)) { - hasNestedExtends = true; + extendPaths.add(extendPath.orElse("")); + try ( + AutoCloseableImpl> closeableCurrentPath = + context + .getCurrentPathStack() + .closeablePush( + extendPath.orElse(""), + context.getExtendPathStack().getTopLineNumber(), + context.getExtendPathStack().getTopStartPosition() + ) + .get() + ) { + String currentPath = closeableCurrentPath + .value() + .unwrapOrElseThrow(Function.identity()); + Node parentRoot = extendParentRoots.removeFirst(); + if (context.getDeferredTokens().size() > numDeferredTokensBefore) { + ignoredOutput.append( + output + .getNodes() + .stream() + .filter(node -> node instanceof RenderedOutputNode) + .map(OutputNode::getValue) + .collect(Collectors.joining()) + ); + } + numDeferredTokensBefore = context.getDeferredTokens().size(); + output = new OutputList(config.getMaxOutputSize()); + output.addNode(pathSetter); + boolean hasNestedExtends = false; + for (Node node : parentRoot.getChildren()) { + lineNumber = node.getLineNumber() - 1; // The line number is off by one when rendering the extend parent + position = node.getStartPosition(); + try { + OutputNode out = node.render(this); + output.addNode(out); + if (isExtendsTag(node)) { + hasNestedExtends = true; + } + } catch (OutputTooBigException e) { + addError(TemplateError.fromOutputTooBigException(e)); + return output.getValue(); } - } catch (OutputTooBigException e) { - addError(TemplateError.fromOutputTooBigException(e)); - return output.getValue(); } + Optional currentExtendPath = context.getExtendPathStack().pop(); + extendPath = + hasNestedExtends ? currentExtendPath : context.getExtendPathStack().peek(); + basePath = Optional.of(currentPath); } - Optional currentExtendPath = context.getExtendPathStack().pop(); - extendPath = - hasNestedExtends ? currentExtendPath : context.getExtendPathStack().peek(); - basePath = Optional.of(currentPath); } } - } - int numDeferredTokensBefore = context.getDeferredTokens().size(); - resolveBlockStubs(output); - if (context.getDeferredTokens().size() > numDeferredTokensBefore) { - pathSetter.setValue( - EagerReconstructionUtils.buildBlockOrInlineSetTag( - RelativePathResolver.CURRENT_PATH_CONTEXT_KEY, - basePath, - this - ) - ); - } + int numDeferredTokensBefore = context.getDeferredTokens().size(); + resolveBlockStubs(output); + if (context.getDeferredTokens().size() > numDeferredTokensBefore) { + pathSetter.setValue( + EagerReconstructionUtils.buildBlockOrInlineSetTag( + RelativePathResolver.CURRENT_PATH_CONTEXT_KEY, + basePath, + this + ) + ); + } - if (ignoredOutput.length() > 0) { - return ( - EagerReconstructionUtils.labelWithNotes( - EagerReconstructionUtils.wrapInTag( - ignoredOutput.toString(), - DoTag.TAG_NAME, - this, - false - ), - IGNORED_OUTPUT_FROM_EXTENDS_NOTE, - this - ) + - output.getValue() - ); + if (ignoredOutput.length() > 0) { + return ( + EagerReconstructionUtils.labelWithNotes( + EagerReconstructionUtils.wrapInTag( + ignoredOutput.toString(), + DoTag.TAG_NAME, + this, + false + ), + IGNORED_OUTPUT_FROM_EXTENDS_NOTE, + this + ) + + output.getValue() + ); + } + return output.getValue(); + } finally { + if (pushed) { + JinjavaInterpreter.popCurrent(); + } } - return output.getValue(); } private void resolveBlockStubs(OutputList output) { diff --git a/src/main/java/com/hubspot/jinjava/lib/tag/FromTag.java b/src/main/java/com/hubspot/jinjava/lib/tag/FromTag.java index 4e2013728..0e2e89691 100644 --- a/src/main/java/com/hubspot/jinjava/lib/tag/FromTag.java +++ b/src/main/java/com/hubspot/jinjava/lib/tag/FromTag.java @@ -114,13 +114,7 @@ public String interpret(TagNode tagNode, JinjavaInterpreter interpreter) { .getInterpreterFactory() .newInstance(interpreter); child.getContext().put(Context.IMPORT_RESOURCE_PATH_KEY, templateFile); - try ( - AutoCloseableImpl a = JinjavaInterpreter - .closeablePushCurrent(child) - .get() - ) { - child.render(node); - } + child.render(node); interpreter.addAllChildErrors(templateFile, child.getErrorsCopy()); diff --git a/src/main/java/com/hubspot/jinjava/lib/tag/ImportTag.java b/src/main/java/com/hubspot/jinjava/lib/tag/ImportTag.java index 8c73d4947..8fbc7a83a 100644 --- a/src/main/java/com/hubspot/jinjava/lib/tag/ImportTag.java +++ b/src/main/java/com/hubspot/jinjava/lib/tag/ImportTag.java @@ -123,14 +123,7 @@ public String interpret(TagNode tagNode, JinjavaInterpreter interpreter) { .getInterpreterFactory() .newInstance(interpreter); child.getContext().put(Context.IMPORT_RESOURCE_PATH_KEY, templateFile); - - try ( - AutoCloseableImpl a = JinjavaInterpreter - .closeablePushCurrent(child) - .get() - ) { - child.render(node.value()); - } + child.render(node.value()); interpreter.addAllChildErrors(templateFile, child.getErrorsCopy()); diff --git a/src/main/java/com/hubspot/jinjava/lib/tag/eager/EagerFromTag.java b/src/main/java/com/hubspot/jinjava/lib/tag/eager/EagerFromTag.java index 2e2f4f0d5..7a0ed06c6 100644 --- a/src/main/java/com/hubspot/jinjava/lib/tag/eager/EagerFromTag.java +++ b/src/main/java/com/hubspot/jinjava/lib/tag/eager/EagerFromTag.java @@ -120,13 +120,7 @@ public String getEagerTagImage(TagToken tagToken, JinjavaInterpreter interpreter .newInstance(interpreter); child.getContext().put(Context.IMPORT_RESOURCE_PATH_KEY, templateFile); String output; - try ( - AutoCloseableImpl a = JinjavaInterpreter - .closeablePushCurrent(child) - .get() - ) { - output = child.render(node); - } + output = child.render(node); interpreter.addAllChildErrors(templateFile, child.getErrorsCopy()); diff --git a/src/main/java/com/hubspot/jinjava/lib/tag/eager/EagerImportTag.java b/src/main/java/com/hubspot/jinjava/lib/tag/eager/EagerImportTag.java index e6fc2f945..70117f3c6 100644 --- a/src/main/java/com/hubspot/jinjava/lib/tag/eager/EagerImportTag.java +++ b/src/main/java/com/hubspot/jinjava/lib/tag/eager/EagerImportTag.java @@ -85,14 +85,9 @@ public String getEagerTagImage(TagToken tagToken, JinjavaInterpreter interpreter .newInstance(interpreter); child.getContext().put(Context.IMPORT_RESOURCE_PATH_KEY, templateFile); String output; - try ( - AutoCloseableImpl a = JinjavaInterpreter - .closeablePushCurrent(child) - .get() - ) { - eagerImportingStrategy.setup(child); - output = child.render(node.value()); - } + eagerImportingStrategy.setup(child); + output = child.render(node.value()); + interpreter.addAllChildErrors(templateFile, child.getErrorsCopy()); Map childBindings = child.getContext().getSessionBindings(); diff --git a/src/main/java/com/hubspot/jinjava/tree/TreeParser.java b/src/main/java/com/hubspot/jinjava/tree/TreeParser.java index 9904e7c7a..56f11003c 100644 --- a/src/main/java/com/hubspot/jinjava/tree/TreeParser.java +++ b/src/main/java/com/hubspot/jinjava/tree/TreeParser.java @@ -38,6 +38,7 @@ import com.hubspot.jinjava.tree.parse.TokenScanner; import com.hubspot.jinjava.tree.parse.TokenScannerSymbols; import com.hubspot.jinjava.tree.parse.UnclosedToken; +import com.hubspot.jinjava.tree.parse.WhitespaceControlParser; import org.apache.commons.lang3.StringUtils; public class TreeParser { @@ -45,6 +46,7 @@ public class TreeParser { private final PeekingIterator scanner; private final JinjavaInterpreter interpreter; private final TokenScannerSymbols symbols; + private final WhitespaceControlParser whitespaceControlParser; private Node parent; @@ -53,6 +55,10 @@ public TreeParser(JinjavaInterpreter interpreter, String input) { Iterators.peekingIterator(new TokenScanner(input, interpreter.getConfig())); this.interpreter = interpreter; this.symbols = interpreter.getConfig().getTokenScannerSymbols(); + this.whitespaceControlParser = + interpreter.getConfig().getLegacyOverrides().isParseWhitespaceControlStrictly() + ? WhitespaceControlParser.STRICT + : WhitespaceControlParser.LENIENT; } public Node buildTree() { @@ -178,7 +184,8 @@ private Node text(TextToken textToken) { StringUtils.stripEnd(textToken.getImage(), "\t "), textToken.getLineNumber(), textToken.getStartPosition(), - symbols + symbols, + whitespaceControlParser ); } } diff --git a/src/main/java/com/hubspot/jinjava/tree/parse/ExpressionToken.java b/src/main/java/com/hubspot/jinjava/tree/parse/ExpressionToken.java index 2587b06d8..d8d9996d5 100644 --- a/src/main/java/com/hubspot/jinjava/tree/parse/ExpressionToken.java +++ b/src/main/java/com/hubspot/jinjava/tree/parse/ExpressionToken.java @@ -29,7 +29,17 @@ public ExpressionToken( int startPosition, TokenScannerSymbols symbols ) { - super(image, lineNumber, startPosition, symbols); + this(image, lineNumber, startPosition, symbols, WhitespaceControlParser.LENIENT); + } + + public ExpressionToken( + String image, + int lineNumber, + int startPosition, + TokenScannerSymbols symbols, + WhitespaceControlParser whitespaceControlParser + ) { + super(image, lineNumber, startPosition, symbols, whitespaceControlParser); } @Override diff --git a/src/main/java/com/hubspot/jinjava/tree/parse/NoteToken.java b/src/main/java/com/hubspot/jinjava/tree/parse/NoteToken.java index 33b1375af..3f5360e67 100644 --- a/src/main/java/com/hubspot/jinjava/tree/parse/NoteToken.java +++ b/src/main/java/com/hubspot/jinjava/tree/parse/NoteToken.java @@ -25,7 +25,17 @@ public NoteToken( int startPosition, TokenScannerSymbols symbols ) { - super(image, lineNumber, startPosition, symbols); + this(image, lineNumber, startPosition, symbols, WhitespaceControlParser.LENIENT); + } + + public NoteToken( + String image, + int lineNumber, + int startPosition, + TokenScannerSymbols symbols, + WhitespaceControlParser whitespaceControlParser + ) { + super(image, lineNumber, startPosition, symbols, whitespaceControlParser); } @Override diff --git a/src/main/java/com/hubspot/jinjava/tree/parse/TagToken.java b/src/main/java/com/hubspot/jinjava/tree/parse/TagToken.java index 17d380b47..a737dd96c 100644 --- a/src/main/java/com/hubspot/jinjava/tree/parse/TagToken.java +++ b/src/main/java/com/hubspot/jinjava/tree/parse/TagToken.java @@ -31,7 +31,17 @@ public TagToken( int startPosition, TokenScannerSymbols symbols ) { - super(image, lineNumber, startPosition, symbols); + this(image, lineNumber, startPosition, symbols, WhitespaceControlParser.LENIENT); + } + + public TagToken( + String image, + int lineNumber, + int startPosition, + TokenScannerSymbols symbols, + WhitespaceControlParser whitespaceControlParser + ) { + super(image, lineNumber, startPosition, symbols, whitespaceControlParser); } @Override diff --git a/src/main/java/com/hubspot/jinjava/tree/parse/TextToken.java b/src/main/java/com/hubspot/jinjava/tree/parse/TextToken.java index c43b6b57c..23288c5dc 100644 --- a/src/main/java/com/hubspot/jinjava/tree/parse/TextToken.java +++ b/src/main/java/com/hubspot/jinjava/tree/parse/TextToken.java @@ -27,7 +27,17 @@ public TextToken( int startPosition, TokenScannerSymbols symbols ) { - super(image, lineNumber, startPosition, symbols); + this(image, lineNumber, startPosition, symbols, WhitespaceControlParser.LENIENT); + } + + public TextToken( + String image, + int lineNumber, + int startPosition, + TokenScannerSymbols symbols, + WhitespaceControlParser whitespaceControlParser + ) { + super(image, lineNumber, startPosition, symbols, whitespaceControlParser); } public void mergeImageAndContent(TextToken otherToken) { diff --git a/src/main/java/com/hubspot/jinjava/tree/parse/Token.java b/src/main/java/com/hubspot/jinjava/tree/parse/Token.java index d33908bff..ee526f5bb 100644 --- a/src/main/java/com/hubspot/jinjava/tree/parse/Token.java +++ b/src/main/java/com/hubspot/jinjava/tree/parse/Token.java @@ -15,9 +15,6 @@ **********************************************************************/ package com.hubspot.jinjava.tree.parse; -import com.hubspot.jinjava.JinjavaConfig; -import com.hubspot.jinjava.LegacyOverrides; -import com.hubspot.jinjava.interpret.JinjavaInterpreter; import com.hubspot.jinjava.interpret.UnexpectedTokenException; import java.io.Serializable; @@ -32,6 +29,7 @@ public abstract class Token implements Serializable { protected final int lineNumber; protected final int startPosition; private final TokenScannerSymbols symbols; + private final WhitespaceControlParser whitespaceControlParser; private boolean leftTrim; private boolean rightTrim; @@ -41,12 +39,14 @@ public Token( String image, int lineNumber, int startPosition, - TokenScannerSymbols symbols + TokenScannerSymbols symbols, + WhitespaceControlParser whitespaceControlParser ) { this.image = image; this.lineNumber = lineNumber; this.startPosition = startPosition; this.symbols = symbols; + this.whitespaceControlParser = whitespaceControlParser; parse(); } @@ -89,25 +89,14 @@ public void setRightTrimAfterEnd(boolean rightTrimAfterEnd) { * @return the content stripped of any whitespace control characters. */ protected final String handleTrim(String unwrapped) { - boolean parseWhitespaceControlStrictly = JinjavaInterpreter - .getCurrentMaybe() - .map(JinjavaInterpreter::getConfig) - .map(JinjavaConfig::getLegacyOverrides) - .map(LegacyOverrides::isParseWhitespaceControlStrictly) - .orElse(false); - - WhitespaceControlParser parser = parseWhitespaceControlStrictly - ? WhitespaceControlParser.STRICT - : WhitespaceControlParser.LENIENT; - String result = unwrapped; - if (parser.hasLeftTrim(result)) { + if (whitespaceControlParser.hasLeftTrim(result)) { setLeftTrim(true); - result = parser.stripLeft(result); + result = whitespaceControlParser.stripLeft(result); } - if (parser.hasRightTrim(result)) { + if (whitespaceControlParser.hasRightTrim(result)) { setRightTrim(true); - result = parser.stripRight(result); + result = whitespaceControlParser.stripRight(result); } return result; } @@ -132,18 +121,43 @@ public String toString() { static Token newToken( int tokenKind, TokenScannerSymbols symbols, + WhitespaceControlParser whitespaceControlParser, String image, int lineNumber, int startPosition ) { if (tokenKind == symbols.getFixed()) { - return new TextToken(image, lineNumber, startPosition, symbols); + return new TextToken( + image, + lineNumber, + startPosition, + symbols, + whitespaceControlParser + ); } else if (tokenKind == symbols.getNote()) { - return new NoteToken(image, lineNumber, startPosition, symbols); + return new NoteToken( + image, + lineNumber, + startPosition, + symbols, + whitespaceControlParser + ); } else if (tokenKind == symbols.getExprStart()) { - return new ExpressionToken(image, lineNumber, startPosition, symbols); + return new ExpressionToken( + image, + lineNumber, + startPosition, + symbols, + whitespaceControlParser + ); } else if (tokenKind == symbols.getTag()) { - return new TagToken(image, lineNumber, startPosition, symbols); + return new TagToken( + image, + lineNumber, + startPosition, + symbols, + whitespaceControlParser + ); } else { throw new UnexpectedTokenException( String.valueOf((char) tokenKind), diff --git a/src/main/java/com/hubspot/jinjava/tree/parse/TokenScanner.java b/src/main/java/com/hubspot/jinjava/tree/parse/TokenScanner.java index c336969d3..e189e6df8 100644 --- a/src/main/java/com/hubspot/jinjava/tree/parse/TokenScanner.java +++ b/src/main/java/com/hubspot/jinjava/tree/parse/TokenScanner.java @@ -38,7 +38,8 @@ public class TokenScanner extends AbstractIterator { private char inQuote = 0; private int currLine = 1; private int lastNewlinePos = 0; - private TokenScannerSymbols symbols; + private final TokenScannerSymbols symbols; + private final WhitespaceControlParser whitespaceControlParser; public TokenScanner(String input, JinjavaConfig config) { this.config = config; @@ -58,6 +59,10 @@ public TokenScanner(String input, JinjavaConfig config) { lastNewlinePos = 0; symbols = config.getTokenScannerSymbols(); + whitespaceControlParser = + config.getLegacyOverrides().isParseWhitespaceControlStrictly() + ? WhitespaceControlParser.STRICT + : WhitespaceControlParser.LENIENT; } private Token getNextToken() { @@ -232,12 +237,14 @@ private Token getEndToken() { String.valueOf(is, tokenStart, tokenLength), currLine, tokenStart - lastNewlinePos + 1, - symbols + symbols, + whitespaceControlParser ); } return Token.newToken( type, symbols, + whitespaceControlParser, String.valueOf(is, tokenStart, tokenLength), currLine, tokenStart - lastNewlinePos + 1 @@ -248,6 +255,7 @@ private Token newToken(int kind) { Token t = Token.newToken( kind, symbols, + whitespaceControlParser, String.valueOf(is, lastStart, tokenLength), currLine, lastStart - lastNewlinePos + 1 @@ -276,7 +284,14 @@ private Token newToken(int kind) { } if (inRaw > 0 && t.getType() != symbols.getFixed()) { - return Token.newToken(symbols.getFixed(), symbols, t.image, currLine, tokenStart); + return Token.newToken( + symbols.getFixed(), + symbols, + whitespaceControlParser, + t.image, + currLine, + tokenStart + ); } return t; diff --git a/src/main/java/com/hubspot/jinjava/tree/parse/UnclosedToken.java b/src/main/java/com/hubspot/jinjava/tree/parse/UnclosedToken.java index 4c1b768b6..76e5ca1b4 100644 --- a/src/main/java/com/hubspot/jinjava/tree/parse/UnclosedToken.java +++ b/src/main/java/com/hubspot/jinjava/tree/parse/UnclosedToken.java @@ -8,6 +8,16 @@ public UnclosedToken( int startPosition, TokenScannerSymbols symbols ) { - super(image, lineNumber, startPosition, symbols); + this(image, lineNumber, startPosition, symbols, WhitespaceControlParser.LENIENT); + } + + public UnclosedToken( + String image, + int lineNumber, + int startPosition, + TokenScannerSymbols symbols, + WhitespaceControlParser whitespaceControlParser + ) { + super(image, lineNumber, startPosition, symbols, whitespaceControlParser); } }