From: Fabien Potencier Date: Fri, 26 Nov 2010 09:35:49 +0000 (+0100) Subject: rewrote expression parser with a new algorithm to limit the depth of nested calls X-Git-Url: http://git.silmor.de/gitweb/?a=commitdiff_plain;h=9d8583efcc236f7c8bc357ef0c122c9246ef5115;p=web%2Fkonrad%2Ftwig.git rewrote expression parser with a new algorithm to limit the depth of nested calls This change has no impact for the end user, but many internal benefits: * less nested calls (was the primary reason for the change as xdebug limits the depth of nested calls); * less code; * code is much cleaner (I have removed all the parsing "hacks"); * faster; * more flexible (we will now be able to expose an API to add new operators); * easier to maintain. --- diff --git a/CHANGELOG b/CHANGELOG index f9f56fa..24f70d6 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -11,8 +11,11 @@ Backward incompatibilities: * the urlencode filter had been renamed to url_encode * the include tag now merges the passed variables with the current context by default (the old behavior is still possible by adding the "only" keyword) - * Moved Exceptions to Twig_Error_* (Twig_SyntaxError/Twig_RuntimeError are now Twig_Error_Syntax/Twig_Error_Runtime) + * moved Exceptions to Twig_Error_* (Twig_SyntaxError/Twig_RuntimeError are now Twig_Error_Syntax/Twig_Error_Runtime) + * removed support for {{ 1 < i < 3 }} (use {{ i > 1 and i < 3 }} instead) + * added the ** (power) operator + * changed the algorithm used for parsing expressions * added the spaceless tag * removed trim_blocks option * added support for is*() methods for attributes (foo.bar now looks for foo->getBar() or foo->isBar()) diff --git a/doc/02-Twig-for-Template-Designers.markdown b/doc/02-Twig-for-Template-Designers.markdown index 37c2359..5602c8a 100644 --- a/doc/02-Twig-for-Template-Designers.markdown +++ b/doc/02-Twig-for-Template-Designers.markdown @@ -1006,13 +1006,6 @@ combine multiple expressions: The following comparison operators are supported in any expression: `==`, `!=`, `<`, `>`, `>=`, and `<=`. ->**TIP** ->Besides PHP classic comparison operators, Twig also supports a shortcut ->notation when you want to test a value in a range: -> -> [twig] -> {% if 1 < foo < 4 %}foo is between 1 and 4{% endif %} - ### Other Operators The following operators are very useful but don't fit into any of the other diff --git a/lib/Twig/ExpressionParser.php b/lib/Twig/ExpressionParser.php index e5d530a..80f8d1d 100644 --- a/lib/Twig/ExpressionParser.php +++ b/lib/Twig/ExpressionParser.php @@ -13,240 +13,132 @@ /** * Parses expressions. * + * This parser implements a "Precedence climbing" algorithm. + * + * @see http://www.engr.mun.ca/~theo/Misc/exp_parsing.htm + * @see http://en.wikipedia.org/wiki/Operator-precedence_parser + * * @package twig * @author Fabien Potencier */ class Twig_ExpressionParser { + const OPERATOR_LEFT = 1; + const OPERATOR_RIGHT = 2; + protected $parser; + protected $unaryOperators; + protected $binaryOperators; public function __construct(Twig_Parser $parser) { $this->parser = $parser; + $this->unaryOperators = $this->getUnaryOperators(); + $this->binaryOperators = $this->getBinaryOperators(); } - public function parseExpression() + public function getUnaryOperators() { - return $this->parseConditionalExpression(); + return array( + 'not' => array('precedence' => 50, 'class' => 'Twig_Node_Expression_Unary_Not'), + '-' => array('precedence' => 50, 'class' => 'Twig_Node_Expression_Unary_Neg'), + '+' => array('precedence' => 50, 'class' => 'Twig_Node_Expression_Unary_Pos'), + ); } - public function parseConditionalExpression() + public function getBinaryOperators() { - $lineno = $this->parser->getCurrentToken()->getLine(); - $expr1 = $this->parseOrExpression(); - while ($this->parser->getStream()->test(Twig_Token::OPERATOR_TYPE, '?')) { - $this->parser->getStream()->next(); - $expr2 = $this->parseOrExpression(); - $this->parser->getStream()->expect(Twig_Token::OPERATOR_TYPE, ':'); - $expr3 = $this->parseConditionalExpression(); - $expr1 = new Twig_Node_Expression_Conditional($expr1, $expr2, $expr3, $lineno); - $lineno = $this->parser->getCurrentToken()->getLine(); - } - - return $expr1; - } - - public function parseOrExpression() - { - $lineno = $this->parser->getCurrentToken()->getLine(); - $left = $this->parseAndExpression(); - while ($this->parser->getStream()->test('or')) { - $this->parser->getStream()->next(); - $right = $this->parseAndExpression(); - $left = new Twig_Node_Expression_Binary_Or($left, $right, $lineno); - $lineno = $this->parser->getCurrentToken()->getLine(); - } - - return $left; + return array( + 'or' => array('precedence' => 10, 'class' => 'Twig_Node_Expression_Binary_Or', 'associativity' => self::OPERATOR_LEFT), + 'and' => array('precedence' => 15, 'class' => 'Twig_Node_Expression_Binary_And', 'associativity' => self::OPERATOR_LEFT), + '==' => array('precedence' => 20, 'class' => 'Twig_Node_Expression_Binary_Equal', 'associativity' => self::OPERATOR_LEFT), + '!=' => array('precedence' => 20, 'class' => 'Twig_Node_Expression_Binary_NotEqual', 'associativity' => self::OPERATOR_LEFT), + '<' => array('precedence' => 20, 'class' => 'Twig_Node_Expression_Binary_Less', 'associativity' => self::OPERATOR_LEFT), + '>' => array('precedence' => 20, 'class' => 'Twig_Node_Expression_Binary_Greater', 'associativity' => self::OPERATOR_LEFT), + '>=' => array('precedence' => 20, 'class' => 'Twig_Node_Expression_Binary_GreaterEqual', 'associativity' => self::OPERATOR_LEFT), + '<=' => array('precedence' => 20, 'class' => 'Twig_Node_Expression_Binary_LessEqual', 'associativity' => self::OPERATOR_LEFT), + 'not in' => array('precedence' => 20, 'class' => 'Twig_Node_Expression_Binary_NotIn', 'associativity' => self::OPERATOR_LEFT), + 'in' => array('precedence' => 20, 'class' => 'Twig_Node_Expression_Binary_In', 'associativity' => self::OPERATOR_LEFT), + '+' => array('precedence' => 30, 'class' => 'Twig_Node_Expression_Binary_Add', 'associativity' => self::OPERATOR_LEFT), + '-' => array('precedence' => 30, 'class' => 'Twig_Node_Expression_Binary_Sub', 'associativity' => self::OPERATOR_LEFT), + '~' => array('precedence' => 40, 'class' => 'Twig_Node_Expression_Binary_Concat', 'associativity' => self::OPERATOR_LEFT), + '*' => array('precedence' => 60, 'class' => 'Twig_Node_Expression_Binary_Mul', 'associativity' => self::OPERATOR_LEFT), + '/' => array('precedence' => 60, 'class' => 'Twig_Node_Expression_Binary_Div', 'associativity' => self::OPERATOR_LEFT), + '//' => array('precedence' => 60, 'class' => 'Twig_Node_Expression_Binary_FloorDiv', 'associativity' => self::OPERATOR_LEFT), + '%' => array('precedence' => 60, 'class' => 'Twig_Node_Expression_Binary_Mod', 'associativity' => self::OPERATOR_LEFT), + 'is' => array('precedence' => 100, 'callable' => array($this, 'parseTestExpression'), 'associativity' => self::OPERATOR_LEFT), + 'is not' => array('precedence' => 100, 'callable' => array($this, 'parseNotTestExpression'), 'associativity' => self::OPERATOR_LEFT), + '..' => array('precedence' => 110, 'class' => 'Twig_Node_Expression_Binary_Range', 'associativity' => self::OPERATOR_LEFT), + '**' => array('precedence' => 200, 'class' => 'Twig_Node_Expression_Binary_Power', 'associativity' => self::OPERATOR_RIGHT), + ); } - public function parseAndExpression() + public function parseExpression($precedence = 0) { - $lineno = $this->parser->getCurrentToken()->getLine(); - $left = $this->parseCompareExpression(); - while ($this->parser->getStream()->test('and')) { + $expr = $this->getPrimary(); + $token = $this->parser->getCurrentToken(); + while ($this->isBinary($token) && $this->binaryOperators[$token->getValue()]['precedence'] >= $precedence) { + $op = $this->binaryOperators[$token->getValue()]; $this->parser->getStream()->next(); - $right = $this->parseCompareExpression(); - $left = new Twig_Node_Expression_Binary_And($left, $right, $lineno); - $lineno = $this->parser->getCurrentToken()->getLine(); - } - return $left; - } - - public function parseCompareExpression() - { - static $operators = array('==', '!=', '<', '>', '>=', '<='); - $lineno = $this->parser->getCurrentToken()->getLine(); - $expr = $this->parseAddExpression(); - $ops = array(); - $negated = false; - while ( - $this->parser->getStream()->test(Twig_Token::OPERATOR_TYPE, $operators) - || - ($this->parser->getStream()->test(Twig_Token::NAME_TYPE, 'not') && $this->parser->getStream()->look()->test(Twig_Token::NAME_TYPE, 'in')) - || - $this->parser->getStream()->test(Twig_Token::NAME_TYPE, 'in') - ) { - $this->parser->getStream()->rewind(); - if ($this->parser->getStream()->test(Twig_Token::NAME_TYPE, 'not')) { - $negated = true; - $this->parser->getStream()->next(); + if (isset($op['callable'])) { + $expr = call_user_func($op['callable'], $expr); + } else { + $expr1 = $this->parseExpression(self::OPERATOR_LEFT === $op['associativity'] ? $op['precedence'] + 1 : $op['precedence']); + $class = $op['class']; + $expr = new $class($expr, $expr1, $token->getLine()); } - $ops[] = new Twig_Node_Expression_Constant($this->parser->getStream()->next()->getValue(), $lineno); - $ops[] = $this->parseAddExpression(); - } - if (empty($ops)) { - return $expr; - } - - $node = new Twig_Node_Expression_Compare($expr, new Twig_Node($ops), $lineno); - - if ($negated) { - $node = new Twig_Node_Expression_Unary_Not($node, $lineno); - } - - return $node; - } - - public function parseAddExpression() - { - $lineno = $this->parser->getCurrentToken()->getLine(); - $left = $this->parseSubExpression(); - while ($this->parser->getStream()->test(Twig_Token::OPERATOR_TYPE, '+')) { - $this->parser->getStream()->next(); - $right = $this->parseSubExpression(); - $left = new Twig_Node_Expression_Binary_Add($left, $right, $lineno); - $lineno = $this->parser->getCurrentToken()->getLine(); - } - - return $left; - } - - public function parseSubExpression() - { - $lineno = $this->parser->getCurrentToken()->getLine(); - $left = $this->parseConcatExpression(); - while ($this->parser->getStream()->test(Twig_Token::OPERATOR_TYPE, '-')) { - $this->parser->getStream()->next(); - $right = $this->parseConcatExpression(); - $left = new Twig_Node_Expression_Binary_Sub($left, $right, $lineno); - $lineno = $this->parser->getCurrentToken()->getLine(); + $token = $this->parser->getCurrentToken(); } - return $left; + return $this->parseConditionalExpression($expr); } - public function parseConcatExpression() + protected function getPrimary() { - $lineno = $this->parser->getCurrentToken()->getLine(); - $left = $this->parseMulExpression(); - while ($this->parser->getStream()->test(Twig_Token::OPERATOR_TYPE, '~')) { - $this->parser->getStream()->next(); - $right = $this->parseMulExpression(); - $left = new Twig_Node_Expression_Binary_Concat($left, $right, $lineno); - $lineno = $this->parser->getCurrentToken()->getLine(); - } - - return $left; - } + $token = $this->parser->getCurrentToken(); - public function parseMulExpression() - { - $lineno = $this->parser->getCurrentToken()->getLine(); - $left = $this->parseDivExpression(); - while ($this->parser->getStream()->test(Twig_Token::OPERATOR_TYPE, '*')) { + if ($this->isUnary($token)) { + $operator = $this->unaryOperators[$token->getValue()]; $this->parser->getStream()->next(); - $right = $this->parseDivExpression(); - $left = new Twig_Node_Expression_Binary_Mul($left, $right, $lineno); - $lineno = $this->parser->getCurrentToken()->getLine(); - } - - return $left; - } + $expr = $this->parseExpression($operator['precedence']); + $class = $operator['class']; - public function parseDivExpression() - { - $lineno = $this->parser->getCurrentToken()->getLine(); - $left = $this->parseFloorDivExpression(); - while ($this->parser->getStream()->test(Twig_Token::OPERATOR_TYPE, '/')) { + return new $class($expr, $token->getLine()); + } elseif ($token->test(Twig_Token::PUNCTUATION_TYPE, '(')) { $this->parser->getStream()->next(); - $right = $this->parseModExpression(); - $left = new Twig_Node_Expression_Binary_Div($left, $right, $lineno); - $lineno = $this->parser->getCurrentToken()->getLine(); - } + $expr = $this->parseExpression(); + $this->parser->getStream()->expect(Twig_Token::PUNCTUATION_TYPE, ')'); - return $left; - } - - public function parseFloorDivExpression() - { - $lineno = $this->parser->getCurrentToken()->getLine(); - $left = $this->parseModExpression(); - while ($this->parser->getStream()->test(Twig_Token::OPERATOR_TYPE, '//')) { - $this->parser->getStream()->next(); - $right = $this->parseModExpression(); - $left = new Twig_Node_Expression_Binary_FloorDiv($left, $right, $lineno); - $lineno = $this->parser->getCurrentToken()->getLine(); + return $expr; } - return $left; + return $this->parsePrimaryExpression(); } - public function parseModExpression() + protected function parseConditionalExpression($expr) { - $lineno = $this->parser->getCurrentToken()->getLine(); - $left = $this->parseUnaryExpression(); - while ($this->parser->getStream()->test(Twig_Token::OPERATOR_TYPE, '%')) { + while ($this->parser->getStream()->test(Twig_Token::PUNCTUATION_TYPE, '?')) { $this->parser->getStream()->next(); - $right = $this->parseUnaryExpression(); - $left = new Twig_Node_Expression_Binary_Mod($left, $right, $lineno); - $lineno = $this->parser->getCurrentToken()->getLine(); - } + $expr2 = $this->parseExpression(); + $this->parser->getStream()->expect(Twig_Token::PUNCTUATION_TYPE, ':'); + $expr3 = $this->parseExpression(); - return $left; - } - - public function parseUnaryExpression() - { - if ($this->parser->getStream()->test('not')) { - return $this->parseNotExpression(); - } - if ($this->parser->getCurrentToken()->getType() == Twig_Token::OPERATOR_TYPE) { - switch ($this->parser->getCurrentToken()->getValue()) { - case '-': - return $this->parseNegExpression(); - case '+': - return $this->parsePosExpression(); - } + $expr = new Twig_Node_Expression_Conditional($expr, $expr2, $expr3, $this->parser->getCurrentToken()->getLine()); } - return $this->parsePrimaryExpression(); + return $expr; } - public function parseNotExpression() + protected function isUnary(Twig_Token $token) { - $token = $this->parser->getStream()->next(); - $node = $this->parseUnaryExpression(); - - return new Twig_Node_Expression_Unary_Not($node, $token->getLine()); + return $this->parser->getStream()->test(Twig_Token::OPERATOR_TYPE) && isset($this->unaryOperators[$token->getValue()]); } - public function parseNegExpression() + protected function isBinary(Twig_Token $token) { - $token = $this->parser->getStream()->next(); - $node = $this->parseUnaryExpression(); - - return new Twig_Node_Expression_Unary_Neg($node, $token->getLine()); - } - - public function parsePosExpression() - { - $token = $this->parser->getStream()->next(); - $node = $this->parseUnaryExpression(); - - return new Twig_Node_Expression_Unary_Pos($node, $token->getLine()); + return $this->parser->getStream()->test(Twig_Token::OPERATOR_TYPE) && isset($this->binaryOperators[$token->getValue()]); } public function parsePrimaryExpression($assignment = false) @@ -281,12 +173,8 @@ class Twig_ExpressionParser break; default: - if ($token->test(Twig_Token::OPERATOR_TYPE, '[')) { + if ($token->test(Twig_Token::PUNCTUATION_TYPE, '[')) { $node = $this->parseArrayExpression(); - } elseif ($token->test(Twig_Token::OPERATOR_TYPE, '(')) { - $this->parser->getStream()->next(); - $node = $this->parseExpression(); - $this->parser->getStream()->expect(Twig_Token::OPERATOR_TYPE, ')'); } else { throw new Twig_Error_Syntax(sprintf('Unexpected token "%s" of value "%s"', Twig_Token::getTypeAsString($token->getType()), $token->getValue()), $token->getLine()); } @@ -302,15 +190,15 @@ class Twig_ExpressionParser public function parseArrayExpression() { $stream = $this->parser->getStream(); - $stream->expect(Twig_Token::OPERATOR_TYPE, '['); + $stream->expect(Twig_Token::PUNCTUATION_TYPE, '['); $elements = array(); - while (!$stream->test(Twig_Token::OPERATOR_TYPE, ']')) { + while (!$stream->test(Twig_Token::PUNCTUATION_TYPE, ']')) { if (!empty($elements)) { - $stream->expect(Twig_Token::OPERATOR_TYPE, ','); + $stream->expect(Twig_Token::PUNCTUATION_TYPE, ','); // trailing ,? - if ($stream->test(Twig_Token::OPERATOR_TYPE, ']')) { - $stream->expect(Twig_Token::OPERATOR_TYPE, ']'); + if ($stream->test(Twig_Token::PUNCTUATION_TYPE, ']')) { + $stream->expect(Twig_Token::PUNCTUATION_TYPE, ']'); return new Twig_Node_Expression_Array($elements, $this->parser->getCurrentToken()->getLine()); } @@ -323,7 +211,7 @@ class Twig_ExpressionParser $stream->test(Twig_Token::NUMBER_TYPE) ) { - if ($stream->look()->test(Twig_Token::OPERATOR_TYPE, ':')) { + if ($stream->look()->test(Twig_Token::PUNCTUATION_TYPE, ':')) { // hash $key = $stream->next()->getValue(); $stream->next(); @@ -337,7 +225,7 @@ class Twig_ExpressionParser $elements[] = $this->parseExpression(); } - $stream->expect(Twig_Token::OPERATOR_TYPE, ']'); + $stream->expect(Twig_Token::PUNCTUATION_TYPE, ']'); return new Twig_Node_Expression_Array($elements, $this->parser->getCurrentToken()->getLine()); } @@ -346,19 +234,14 @@ class Twig_ExpressionParser { while (1) { $token = $this->parser->getCurrentToken(); - if ($token->getType() == Twig_Token::OPERATOR_TYPE) { - if ('..' == $token->getValue()) { - $node = $this->parseRangeExpression($node); - } elseif ('.' == $token->getValue() || '[' == $token->getValue()) { + if ($token->getType() == Twig_Token::PUNCTUATION_TYPE) { + if ('.' == $token->getValue() || '[' == $token->getValue()) { $node = $this->parseSubscriptExpression($node); } elseif ('|' == $token->getValue()) { $node = $this->parseFilterExpression($node); } else { break; } - } elseif ($token->getType() == Twig_Token::NAME_TYPE && 'is' == $token->getValue()) { - $node = $this->parseTestExpression($node); - break; } else { break; } @@ -367,44 +250,21 @@ class Twig_ExpressionParser return $node; } + public function parseNotTestExpression($node) + { + return new Twig_Node_Expression_Unary_Not($this->parseTestExpression($node), $this->parser->getCurrentToken()->getLine()); + } + public function parseTestExpression($node) { $stream = $this->parser->getStream(); - $token = $stream->next(); - $lineno = $token->getLine(); - - $negated = false; - if ($stream->test('not')) { - $stream->next(); - $negated = true; - } - $name = $stream->expect(Twig_Token::NAME_TYPE); - $arguments = null; - if ($stream->test(Twig_Token::OPERATOR_TYPE, '(')) { + if ($stream->test(Twig_Token::PUNCTUATION_TYPE, '(')) { $arguments = $this->parseArguments($node); } - $test = new Twig_Node_Expression_Test($node, $name->getValue(), $arguments, $lineno); - - if ($negated) { - $test = new Twig_Node_Expression_Unary_Not($test, $lineno); - } - - return $test; - } - - public function parseRangeExpression($node) - { - $token = $this->parser->getStream()->next(); - $lineno = $token->getLine(); - $end = $this->parseExpression(); - - $name = new Twig_Node_Expression_Constant('range', $lineno); - $arguments = new Twig_Node(array($end)); - - return new Twig_Node_Expression_Filter($node, $name, $arguments, $lineno); + return new Twig_Node_Expression_Test($node, $name->getValue(), $arguments, $this->parser->getCurrentToken()->getLine()); } public function parseSubscriptExpression($node) @@ -418,7 +278,7 @@ class Twig_ExpressionParser if ($token->getType() == Twig_Token::NAME_TYPE || $token->getType() == Twig_Token::NUMBER_TYPE) { $arg = new Twig_Node_Expression_Constant($token->getValue(), $lineno); - if ($this->parser->getStream()->test(Twig_Token::OPERATOR_TYPE, '(')) { + if ($this->parser->getStream()->test(Twig_Token::PUNCTUATION_TYPE, '(')) { $type = Twig_Node_Expression_GetAttr::TYPE_METHOD; $arguments = $this->parseArguments(); } else { @@ -431,7 +291,7 @@ class Twig_ExpressionParser $type = Twig_Node_Expression_GetAttr::TYPE_ARRAY; $arg = $this->parseExpression(); - $this->parser->getStream()->expect(Twig_Token::OPERATOR_TYPE, ']'); + $this->parser->getStream()->expect(Twig_Token::PUNCTUATION_TYPE, ']'); } return new Twig_Node_Expression_GetAttr($node, $arg, $arguments, $type, $lineno); @@ -452,7 +312,7 @@ class Twig_ExpressionParser $token = $this->parser->getStream()->expect(Twig_Token::NAME_TYPE); $name = new Twig_Node_Expression_Constant($token->getValue(), $token->getLine()); - if (!$this->parser->getStream()->test(Twig_Token::OPERATOR_TYPE, '(')) { + if (!$this->parser->getStream()->test(Twig_Token::PUNCTUATION_TYPE, '(')) { $arguments = new Twig_Node(); } else { $arguments = $this->parseArguments(); @@ -460,7 +320,7 @@ class Twig_ExpressionParser $node = new Twig_Node_Expression_Filter($node, $name, $arguments, $token->getLine(), $tag); - if (!$this->parser->getStream()->test(Twig_Token::OPERATOR_TYPE, '|')) { + if (!$this->parser->getStream()->test(Twig_Token::PUNCTUATION_TYPE, '|')) { break; } @@ -473,16 +333,16 @@ class Twig_ExpressionParser public function parseArguments() { $parser = $this->parser->getStream(); - $parser->expect(Twig_Token::OPERATOR_TYPE, '('); + $parser->expect(Twig_Token::PUNCTUATION_TYPE, '('); $args = array(); - while (!$parser->test(Twig_Token::OPERATOR_TYPE, ')')) { + while (!$parser->test(Twig_Token::PUNCTUATION_TYPE, ')')) { if (!empty($args)) { - $parser->expect(Twig_Token::OPERATOR_TYPE, ','); + $parser->expect(Twig_Token::PUNCTUATION_TYPE, ','); } $args[] = $this->parseExpression(); } - $parser->expect(Twig_Token::OPERATOR_TYPE, ')'); + $parser->expect(Twig_Token::PUNCTUATION_TYPE, ')'); return new Twig_Node($args); } @@ -493,17 +353,16 @@ class Twig_ExpressionParser $targets = array(); while (true) { if (!empty($targets)) { - $this->parser->getStream()->expect(Twig_Token::OPERATOR_TYPE, ','); + $this->parser->getStream()->expect(Twig_Token::PUNCTUATION_TYPE, ','); } - if ($this->parser->getStream()->test(Twig_Token::OPERATOR_TYPE, ')') || + if ($this->parser->getStream()->test(Twig_Token::PUNCTUATION_TYPE, ')') || $this->parser->getStream()->test(Twig_Token::VAR_END_TYPE) || - $this->parser->getStream()->test(Twig_Token::BLOCK_END_TYPE) || - $this->parser->getStream()->test('in')) + $this->parser->getStream()->test(Twig_Token::BLOCK_END_TYPE)) { break; } $targets[] = $this->parsePrimaryExpression(true); - if (!$this->parser->getStream()->test(Twig_Token::OPERATOR_TYPE, ',')) { + if (!$this->parser->getStream()->test(Twig_Token::PUNCTUATION_TYPE, ',')) { break; } } @@ -518,16 +377,16 @@ class Twig_ExpressionParser $is_multitarget = false; while (true) { if (!empty($targets)) { - $this->parser->getStream()->expect(Twig_Token::OPERATOR_TYPE, ','); + $this->parser->getStream()->expect(Twig_Token::PUNCTUATION_TYPE, ','); } - if ($this->parser->getStream()->test(Twig_Token::OPERATOR_TYPE, ')') || + if ($this->parser->getStream()->test(Twig_Token::PUNCTUATION_TYPE, ')') || $this->parser->getStream()->test(Twig_Token::VAR_END_TYPE) || $this->parser->getStream()->test(Twig_Token::BLOCK_END_TYPE)) { break; } $targets[] = $this->parseExpression(); - if (!$this->parser->getStream()->test(Twig_Token::OPERATOR_TYPE, ',')) { + if (!$this->parser->getStream()->test(Twig_Token::PUNCTUATION_TYPE, ',')) { break; } $is_multitarget = true; diff --git a/lib/Twig/Extension/Core.php b/lib/Twig/Extension/Core.php index 4c81fd5..e19fff5 100644 --- a/lib/Twig/Extension/Core.php +++ b/lib/Twig/Extension/Core.php @@ -63,7 +63,6 @@ class Twig_Extension_Core extends Twig_Extension 'reverse' => new Twig_Filter_Function('twig_reverse_filter'), 'length' => new Twig_Filter_Function('twig_length_filter', array('needs_environment' => true)), 'sort' => new Twig_Filter_Function('twig_sort_filter'), - 'in' => new Twig_Filter_Function('twig_in_filter'), 'range' => new Twig_Filter_Function('twig_range_filter'), 'cycle' => new Twig_Filter_Function('twig_cycle_filter'), diff --git a/lib/Twig/Lexer.php b/lib/Twig/Lexer.php index 4dd18b5..5a62cbc 100644 --- a/lib/Twig/Lexer.php +++ b/lib/Twig/Lexer.php @@ -27,15 +27,19 @@ class Twig_Lexer implements Twig_LexerInterface protected $filename; protected $env; protected $options; + protected $operatorRegex; + + // All unary and binary operators defined in Twig_ExpressionParser plus the = sign + protected $operators = array('=', 'not', 'or', 'and', '==', '!=', '<', '>', '>=', '<=', 'not in', 'in', '+', '-', '~', '*', '/', '//', '%', 'is', 'is not', '..', '**'); const POSITION_DATA = 0; const POSITION_BLOCK = 1; const POSITION_VAR = 2; - const REGEX_NAME = '/[A-Za-z_][A-Za-z0-9_]*/A'; - const REGEX_NUMBER = '/[0-9]+(?:\.[0-9]+)?/A'; - const REGEX_STRING = '/(?:"([^"\\\\]*(?:\\\\.[^"\\\\]*)*)"|\'([^\'\\\\]*(?:\\\\.[^\'\\\\]*)*)\')/Asm'; - const REGEX_OPERATOR = '/<=? | >=? | [!=]= | = | \/\/ | \.\. | [(){}.,%*\/+~|-] | \[ | \] | \? | \:/Ax'; + const REGEX_NAME = '/[A-Za-z_][A-Za-z0-9_]*/A'; + const REGEX_NUMBER = '/[0-9]+(?:\.[0-9]+)?/A'; + const REGEX_STRING = '/(?:"([^"\\\\]*(?:\\\\.[^"\\\\]*)*)"|\'([^\'\\\\]*(?:\\\\.[^\'\\\\]*)*)\')/Asm'; + const REGEX_PUNCTUATION = '/[\[\](){}?:.,|]/A'; public function __construct(Twig_Environment $env = null, array $options = array()) { @@ -48,6 +52,13 @@ class Twig_Lexer implements Twig_LexerInterface 'tag_block' => array('{%', '%}'), 'tag_variable' => array('{{', '}}'), ), $options); + + $this->operatorRegex = $this->getOperatorRegex(); + } + + public function sortByLength($a, $b) + { + return strlen($a) > strlen($b) ? -1 : 1; } /** @@ -270,10 +281,10 @@ class Twig_Lexer implements Twig_LexerInterface } // first parse operators - if (preg_match(self::REGEX_OPERATOR, $this->code, $match, null, $this->cursor)) { - $this->moveCursor($match[0]); + if (preg_match($this->operatorRegex, $this->code, $match, null, $this->cursor)) { + $this->moveCursor(trim($match[0], ' ()')); - return new Twig_Token(Twig_Token::OPERATOR_TYPE, $match[0], $this->lineno); + return new Twig_Token(Twig_Token::OPERATOR_TYPE, trim($match[0], ' ()'), $this->lineno); } // now names else if (preg_match(self::REGEX_NAME, $this->code, $match, null, $this->cursor)) { @@ -291,6 +302,13 @@ class Twig_Lexer implements Twig_LexerInterface return new Twig_Token(Twig_Token::NUMBER_TYPE, $value, $this->lineno); } + // punctuation + else if (preg_match(self::REGEX_PUNCTUATION, $this->code, $match, null, $this->cursor)) { + $this->moveCursor($match[0]); + $this->moveLineNo($match[0]); + + return new Twig_Token(Twig_Token::PUNCTUATION_TYPE, $match[0], $this->lineno); + } // and finally strings else if (preg_match(self::REGEX_STRING, $this->code, $match, null, $this->cursor)) { $this->moveCursor($match[0]); @@ -314,4 +332,21 @@ class Twig_Lexer implements Twig_LexerInterface $this->cursor += strlen($text); } + protected function getOperatorRegex() + { + usort($this->operators, array($this, 'sortByLength')); + $operators = array(); + foreach ($this->operators as $operator) { + $last = ord(substr($operator, -1)); + // an operator that ends with a character must be followed by + // a whitespace or a parenthese + if (($last >= 65 && $last <= 90) || ($last >= 97 && $last <= 122)) { + $operators[] = preg_quote($operator, '/').'(?:[ \(\)])'; + } else { + $operators[] = preg_quote($operator, '/'); + } + } + + return '/'.implode('|', $operators).'/A'; + } } diff --git a/lib/Twig/Node/Expression/Binary/Equal.php b/lib/Twig/Node/Expression/Binary/Equal.php new file mode 100644 index 0000000..d28140e --- /dev/null +++ b/lib/Twig/Node/Expression/Binary/Equal.php @@ -0,0 +1,17 @@ +raw('=='); + } +} diff --git a/lib/Twig/Node/Expression/Binary/Greater.php b/lib/Twig/Node/Expression/Binary/Greater.php new file mode 100644 index 0000000..3cc8e1c --- /dev/null +++ b/lib/Twig/Node/Expression/Binary/Greater.php @@ -0,0 +1,17 @@ +raw('>'); + } +} diff --git a/lib/Twig/Node/Expression/Binary/GreaterEqual.php b/lib/Twig/Node/Expression/Binary/GreaterEqual.php new file mode 100644 index 0000000..7fe974b --- /dev/null +++ b/lib/Twig/Node/Expression/Binary/GreaterEqual.php @@ -0,0 +1,17 @@ +raw('>='); + } +} diff --git a/lib/Twig/Node/Expression/Binary/In.php b/lib/Twig/Node/Expression/Binary/In.php new file mode 100644 index 0000000..b59544e --- /dev/null +++ b/lib/Twig/Node/Expression/Binary/In.php @@ -0,0 +1,33 @@ +raw('twig_in_filter(') + ->subcompile($this->getNode('left')) + ->raw(', ') + ->subcompile($this->getNode('right')) + ->raw(')') + ; + } + + public function operator($compiler) + { + return $compiler->raw('in'); + } +} diff --git a/lib/Twig/Node/Expression/Binary/Less.php b/lib/Twig/Node/Expression/Binary/Less.php new file mode 100644 index 0000000..42fc5df --- /dev/null +++ b/lib/Twig/Node/Expression/Binary/Less.php @@ -0,0 +1,17 @@ +raw('<'); + } +} diff --git a/lib/Twig/Node/Expression/Binary/LessEqual.php b/lib/Twig/Node/Expression/Binary/LessEqual.php new file mode 100644 index 0000000..0f4b52c --- /dev/null +++ b/lib/Twig/Node/Expression/Binary/LessEqual.php @@ -0,0 +1,17 @@ +raw('<='); + } +} diff --git a/lib/Twig/Node/Expression/Binary/NotEqual.php b/lib/Twig/Node/Expression/Binary/NotEqual.php new file mode 100644 index 0000000..84bee67 --- /dev/null +++ b/lib/Twig/Node/Expression/Binary/NotEqual.php @@ -0,0 +1,17 @@ +raw('!='); + } +} diff --git a/lib/Twig/Node/Expression/Binary/NotIn.php b/lib/Twig/Node/Expression/Binary/NotIn.php new file mode 100644 index 0000000..486e879 --- /dev/null +++ b/lib/Twig/Node/Expression/Binary/NotIn.php @@ -0,0 +1,33 @@ +raw('!twig_in_filter(') + ->subcompile($this->getNode('left')) + ->raw(', ') + ->subcompile($this->getNode('right')) + ->raw(')') + ; + } + + public function operator($compiler) + { + return $compiler->raw('not in'); + } +} diff --git a/lib/Twig/Node/Expression/Binary/Power.php b/lib/Twig/Node/Expression/Binary/Power.php new file mode 100644 index 0000000..4eede5e --- /dev/null +++ b/lib/Twig/Node/Expression/Binary/Power.php @@ -0,0 +1,33 @@ +raw('pow(') + ->subcompile($this->getNode('left')) + ->raw(', ') + ->subcompile($this->getNode('right')) + ->raw(')') + ; + } + + public function operator($compiler) + { + return $compiler->raw('**'); + } +} diff --git a/lib/Twig/Node/Expression/Binary/Range.php b/lib/Twig/Node/Expression/Binary/Range.php new file mode 100644 index 0000000..b7f1e91 --- /dev/null +++ b/lib/Twig/Node/Expression/Binary/Range.php @@ -0,0 +1,33 @@ +raw('twig_range_filter(') + ->subcompile($this->getNode('left')) + ->raw(', ') + ->subcompile($this->getNode('right')) + ->raw(')') + ; + } + + public function operator($compiler) + { + return $compiler->raw('..'); + } +} diff --git a/lib/Twig/Node/Expression/Compare.php b/lib/Twig/Node/Expression/Compare.php deleted file mode 100644 index c87be9a..0000000 --- a/lib/Twig/Node/Expression/Compare.php +++ /dev/null @@ -1,61 +0,0 @@ - $expr, 'ops' => $ops), array(), $lineno); - } - - public function compile($compiler) - { - if ('in' === $this->getNode('ops')->getNode('0')->getAttribute('value')) { - return $this->compileIn($compiler); - } - - $this->getNode('expr')->compile($compiler); - - $nbOps = count($this->getNode('ops')); - for ($i = 0; $i < $nbOps; $i += 2) { - if ($i > 0) { - $compiler->raw(' && ($tmp'.($i / 2)); - } - - $compiler->raw(' '.$this->getNode('ops')->getNode($i)->getAttribute('value').' '); - - if ($i != $nbOps - 2) { - $compiler - ->raw('($tmp'.(($i / 2) + 1).' = ') - ->subcompile($this->getNode('ops')->getNode($i + 1)) - ->raw(')') - ; - } else { - $compiler->subcompile($this->getNode('ops')->getNode($i + 1)); - } - } - - for ($j = 1; $j < $i / 2; $j++) { - $compiler->raw(')'); - } - } - - protected function compileIn($compiler) - { - $compiler - ->raw('twig_in_filter(') - ->subcompile($this->getNode('expr')) - ->raw(', ') - ->subcompile($this->getNode('ops')->getNode(1)) - ->raw(')') - ; - } -} diff --git a/lib/Twig/Token.php b/lib/Twig/Token.php index 0b4e0cb..8bbe8c0 100644 --- a/lib/Twig/Token.php +++ b/lib/Twig/Token.php @@ -25,6 +25,7 @@ class Twig_Token const NUMBER_TYPE = 6; const STRING_TYPE = 7; const OPERATOR_TYPE = 8; + const PUNCTUATION_TYPE = 9; public function __construct($type, $value, $lineno) { @@ -112,6 +113,9 @@ class Twig_Token case self::OPERATOR_TYPE: $name = 'OPERATOR_TYPE'; break; + case self::PUNCTUATION_TYPE: + $name = 'PUNCTUATION_TYPE'; + break; default: throw new Twig_Error_Syntax(sprintf('Token of type %s does not exist.', $type)); } diff --git a/lib/Twig/TokenParser/For.php b/lib/Twig/TokenParser/For.php index 758fb4a..4fec054 100644 --- a/lib/Twig/TokenParser/For.php +++ b/lib/Twig/TokenParser/For.php @@ -22,7 +22,7 @@ class Twig_TokenParser_For extends Twig_TokenParser { $lineno = $token->getLine(); $targets = $this->parser->getExpressionParser()->parseAssignmentExpression(); - $this->parser->getStream()->expect('in'); + $this->parser->getStream()->expect(Twig_Token::OPERATOR_TYPE, 'in'); $seq = $this->parser->getExpressionParser()->parseExpression(); $withLoop = true; diff --git a/test/Twig/Tests/Fixtures/expressions/comparison.test b/test/Twig/Tests/Fixtures/expressions/comparison.test index 4ed77e5..726b850 100644 --- a/test/Twig/Tests/Fixtures/expressions/comparison.test +++ b/test/Twig/Tests/Fixtures/expressions/comparison.test @@ -5,7 +5,6 @@ Twig supports comparison operators (==, !=, <, >, >=, <=) {{ 1 < 2 }}/{{ 1 < 1 }}/{{ 1 <= 2 }}/{{ 1 <= 1 }} {{ 1 == 1 }}/{{ 1 == 2 }} {{ 1 != 1 }}/{{ 1 != 2 }} -{{ 1 < 2 < 3 }}/{{ 1 < 2 < 3 < 4 }} --DATA-- return array() --EXPECT-- @@ -13,4 +12,3 @@ return array() 1//1/1 1/ /1 -1/1 diff --git a/test/Twig/Tests/Fixtures/filters/in.test b/test/Twig/Tests/Fixtures/filters/in.test deleted file mode 100644 index 4d6e337..0000000 --- a/test/Twig/Tests/Fixtures/filters/in.test +++ /dev/null @@ -1,27 +0,0 @@ ---TEST-- -"in" filter ---TEMPLATE-- -{{ 1|in([1, 2, 3]) }} -{{ 5|in([1, 2, 3]) }} -{{ 'cd'|in('abcde') }} -{{ 'ad'|in('abcde') }} -{{ 'foo'|in(foo) }} -{{ 'a'|in(foo|keys) }} ---DATA-- -class ItemsIteratorForInFilter implements Iterator -{ - protected $values = array('a' => 'foo', 'b' => 'bar'); - public function current() { return current($this->values); } - public function key() { return key($this->values); } - public function next() { return next($this->values); } - public function rewind() { return reset($this->values); } - public function valid() { return false !== current($this->values); } -} -return array('foo' => new ItemsIteratorForInFilter()) ---EXPECT-- -1 - -1 - -1 -1 diff --git a/test/Twig/Tests/Node/Expression/CompareTest.php b/test/Twig/Tests/Node/Expression/CompareTest.php deleted file mode 100644 index fb98ff8..0000000 --- a/test/Twig/Tests/Node/Expression/CompareTest.php +++ /dev/null @@ -1,72 +0,0 @@ -', 0), - new Twig_Node_Expression_Constant(2, 0), - ), array(), 0); - $node = new Twig_Node_Expression_Compare($expr, $ops, 0); - - $this->assertEquals($expr, $node->getNode('expr')); - $this->assertEquals($ops, $node->getNode('ops')); - } - - /** - * @covers Twig_Node_Expression_Compare::compile - * @covers Twig_Node_Expression_Compare::compileIn - * @dataProvider getTests - */ - public function testCompile($node, $source, $environment = null) - { - parent::testCompile($node, $source, $environment); - } - - public function getTests() - { - $tests = array(); - - $expr = new Twig_Node_Expression_Constant(1, 0); - $ops = new Twig_Node(array( - new Twig_Node_Expression_Constant('>', 0), - new Twig_Node_Expression_Constant(2, 0), - ), array(), 0); - $node = new Twig_Node_Expression_Compare($expr, $ops, 0); - $tests[] = array($node, '1 > 2'); - - $ops = new Twig_Node(array( - new Twig_Node_Expression_Constant('>', 0), - new Twig_Node_Expression_Constant(2, 0), - new Twig_Node_Expression_Constant('<', 0), - new Twig_Node_Expression_Constant(4, 0), - ), array(), 0); - $node = new Twig_Node_Expression_Compare($expr, $ops, 0); - $tests[] = array($node, '1 > ($tmp1 = 2) && ($tmp1 < 4)'); - - $ops = new Twig_Node(array( - new Twig_Node_Expression_Constant('in', 0), - new Twig_Node_Expression_Constant(2, 0), - ), array(), 0); - $node = new Twig_Node_Expression_Compare($expr, $ops, 0); - $tests[] = array($node, 'twig_in_filter(1, 2)'); - - return $tests; - } -}