From 648c843523bcb131d96f37435f650a6909713442 Mon Sep 17 00:00:00 2001 From: nikic <+@ni-po.com> Date: Mon, 20 Dec 2010 16:41:30 +0100 Subject: [PATCH] TokenStream changes: * remove unused look() and rewind() * do not shift tokens, but increase a pointer --- doc/advanced.rst | 17 +++--- lib/Twig/TokenStream.php | 119 +++++++++++++---------------------- test/Twig/Tests/TokenStreamTest.php | 47 +------------- 3 files changed, 54 insertions(+), 129 deletions(-) diff --git a/doc/advanced.rst b/doc/advanced.rst index 9634b1b..9df1dd5 100644 --- a/doc/advanced.rst +++ b/doc/advanced.rst @@ -347,16 +347,17 @@ tag. It should return a ``Twig_Node`` instance that represents the node (the The parsing process is simplified thanks to a bunch of methods you can call from the token stream (``$this->parser->getStream()``): -* ``test()``: Tests the type and optionally the value of the next token and - returns it. +* ``getCurrent()``: Gets the current token in the stream. -* ``expect()``: Expects a token and returns it (like ``test()``) or throw a - syntax error if not found (the second argument is the expected value of the - token). +* ``next()``: Moves to the next token in the stream, *but returns the old one*. -* ``look()``: Looks a the next token. This is how you can have a look at the - next token without consuming it (after you are done with ``look()``, you - must use ``rewind()``). +* ``test($type)``, ``test($value)`` or ``test($type, $value)``: Determines whether + the current token is of a particular type or value (or both). The value may be an + array of several possible values. + +* ``expect($type[, $value[, $message]])``: If the current token isn't of the given + type/value a syntax error is thrown. Otherwise, if the type and value are correct, + the token is returned and the stream moves to the next token. Parsing expressions is done by calling the ``parseExpression()`` like we did for the ``set`` tag. diff --git a/lib/Twig/TokenStream.php b/lib/Twig/TokenStream.php index 081a279..d8e8edf 100644 --- a/lib/Twig/TokenStream.php +++ b/lib/Twig/TokenStream.php @@ -11,104 +11,54 @@ */ class Twig_TokenStream { - protected $pushed; - protected $originalTokens; protected $tokens; - protected $eof; protected $current; protected $filename; - public function __construct(array $tokens, $filename) + /** + * @param array $tokens Array of tokens + * @param string $filename Name which $tokens are associated with + */ + public function __construct(array $tokens, $filename = null) { - $this->pushed = array(); - $this->originalTokens = $tokens; - $this->tokens = $tokens; - $this->filename = $filename; - $this->next(); + $this->tokens = $tokens; + $this->current = 0; + $this->filename = $filename; } public function __toString() { - $repr = ''; - foreach ($this->originalTokens as $token) { - $repr .= $token."\n"; - } - - return $repr; - } - - public function push($token) - { - $this->pushed[] = $token; + return implode("\n", $this->tokens); } /** * Sets the pointer to the next token and returns the old one. * - * @param Boolean $fromStack Whether to get a token from the stack or not - */ - public function next($fromStack = true) - { - if ($fromStack && !empty($this->pushed)) { - $old = array_shift($this->pushed); - $token = array_shift($this->pushed); - } else { - $old = $this->current; - $token = array_shift($this->tokens); - } - - if (null === $token) { - throw new Twig_Error_Syntax('Unexpected end of template', -1); - } - - $this->current = $token; - - $this->eof = $token->getType() === Twig_Token::EOF_TYPE; - - return $old; - } - - /** - * Looks at the next token. + * @return Twig_Token */ - public function look() + public function next() { - $old = $this->next(false); - $new = $this->current; - $this->push($old); - $this->push($new); - - return $new; - } - - /** - * Rewinds the pushed tokens. - */ - public function rewind() - { - $tokens = array(); - while ($this->pushed) { - $tokens[] = array_shift($this->pushed); - array_shift($this->pushed); + if (!isset($this->tokens[++$this->current])) { + throw new Twig_Error_Syntax('Unexpected end of template'); } - $this->tokens = array_merge($tokens, array($this->current), $this->tokens); - - $this->next(); + return $this->tokens[$this->current - 1]; } /** - * Expects a token (like $token->test()) and returns it or throw a syntax error. + * test()s a token and returns it or throws a syntax error. + * + * @return Twig_Token */ - public function expect($primary, $secondary = null, $message = null) + public function expect($type, $value = null, $message = null) { - $token = $this->current; - if (!$token->test($primary, $secondary)) { + $token = $this->tokens[$this->current]; + if (!$token->test($type, $value)) { throw new Twig_Error_Syntax(sprintf('%sUnexpected token "%s" of value "%s" ("%s" expected%s)', $message ? $message.'. ' : '', Twig_Token::typeToEnglish($token->getType()), $token->getValue(), - Twig_Token::typeToEnglish($primary), $secondary ? sprintf(' with value "%s"', $secondary) : ''), - $this->current->getLine() + Twig_Token::typeToEnglish($type), $value ? sprintf(' with value "%s"', $value) : ''), + $token->getLine() ); } $this->next(); @@ -117,23 +67,40 @@ class Twig_TokenStream } /** - * Forwards that call to the current token. + * test() current token + * + * @return bool */ public function test($primary, $secondary = null) { - return $this->current->test($primary, $secondary); + return $this->tokens[$this->current]->test($primary, $secondary); } + /** + * Checks if end of stream was reached + * + * @return bool + */ public function isEOF() { - return $this->eof; + return $this->tokens[$this->current]->getType() === Twig_Token::EOF_TYPE; } + /** + * Gets the current token + * + * @return Twig_Token + */ public function getCurrent() { - return $this->current; + return $this->tokens[$this->current]; } + /** + * Gets the filename associated with this stream + * + * @return string + */ public function getFilename() { return $this->filename; diff --git a/test/Twig/Tests/TokenStreamTest.php b/test/Twig/Tests/TokenStreamTest.php index 341773c..794a037 100644 --- a/test/Twig/Tests/TokenStreamTest.php +++ b/test/Twig/Tests/TokenStreamTest.php @@ -29,56 +29,13 @@ class Twig_Tests_TokenStreamTest extends PHPUnit_Framework_TestCase public function testNext() { - $stream = new Twig_TokenStream(self::$tokens, '', false); + $stream = new Twig_TokenStream(self::$tokens); $repr = array(); while (!$stream->isEOF()) { $token = $stream->next(); $repr[] = $token->getValue(); } - $this->assertEquals('1, 2, 3, 4, 5, 6, 7', implode(', ', $repr), '->next() returns the next token in the stream'); - } - - public function testLook() - { - $stream = new Twig_TokenStream(self::$tokens, '', false); - $this->assertEquals(2, $stream->look()->getValue(), '->look() returns the next token'); - $repr = array(); - while (!$stream->isEOF()) { - $token = $stream->next(); - - $repr[] = $token->getValue(); - } - $this->assertEquals('1, 2, 3, 4, 5, 6, 7', implode(', ', $repr), '->look() pushes the token to the stack'); - - $stream = new Twig_TokenStream(self::$tokens, '', false); - $this->assertEquals(2, $stream->look()->getValue(), '->look() returns the next token'); - $this->assertEquals(3, $stream->look()->getValue(), '->look() can be called several times to look more than one upcoming token'); - $this->assertEquals(4, $stream->look()->getValue(), '->look() can be called several times to look more than one upcoming token'); - $this->assertEquals(5, $stream->look()->getValue(), '->look() can be called several times to look more than one upcoming token'); - $repr = array(); - while (!$stream->isEOF()) { - $token = $stream->next(); - - $repr[] = $token->getValue(); - } - $this->assertEquals('1, 2, 3, 4, 5, 6, 7', implode(', ', $repr), '->look() pushes the token to the stack'); - } - - public function testRewind() - { - $stream = new Twig_TokenStream(self::$tokens, '', false); - $this->assertEquals(2, $stream->look()->getValue(), '->look() returns the next token'); - $this->assertEquals(3, $stream->look()->getValue(), '->look() can be called several times to look more than one upcoming token'); - $this->assertEquals(4, $stream->look()->getValue(), '->look() can be called several times to look more than one upcoming token'); - $this->assertEquals(5, $stream->look()->getValue(), '->look() can be called several times to look more than one upcoming token'); - $stream->rewind(); - $repr = array(); - while (!$stream->isEOF()) { - $token = $stream->next(false); - - $repr[] = $token->getValue(); - } - $this->assertEquals('1, 2, 3, 4, 5, 6, 7', implode(', ', $repr), '->rewind() pushes all pushed tokens to the token array'); + $this->assertEquals('1, 2, 3, 4, 5, 6, 7', implode(', ', $repr), '->next() advances the pointer and returns the current token'); } } -- 1.7.2.5