From: nikic <+@ni-po.com>
Date: Mon, 20 Dec 2010 15:41:30 +0000 (+0100)
Subject: TokenStream changes:
X-Git-Url: http://git.silmor.de/gitweb/?a=commitdiff_plain;h=648c843523bcb131d96f37435f650a6909713442;p=konrad%2Ftwig.git

TokenStream changes:

 * remove unused look() and rewind()
 * do not shift tokens, but increase a pointer
---

diff --git a/doc/advanced.rst b/doc/advanced.rst
index 9634b1b..9df1dd5 100644
--- a/doc/advanced.rst
+++ b/doc/advanced.rst
@@ -347,16 +347,17 @@ tag. It should return a ``Twig_Node`` instance that represents the node (the
 The parsing process is simplified thanks to a bunch of methods you can call
 from the token stream (``$this->parser->getStream()``):
 
-* ``test()``: Tests the type and optionally the value of the next token and
-  returns it.
+* ``getCurrent()``: Gets the current token in the stream.
 
-* ``expect()``: Expects a token and returns it (like ``test()``) or throw a
-  syntax error if not found (the second argument is the expected value of the
-  token).
+* ``next()``: Moves to the next token in the stream, *but returns the old one*.
 
-* ``look()``: Looks a the next token. This is how you can have a look at the
-  next token without consuming it (after you are done with ``look()``, you
-  must use ``rewind()``).
+* ``test($type)``, ``test($value)`` or ``test($type, $value)``: Determines whether
+  the current token is of a particular type or value (or both). The value may be an
+  array of several possible values.
+
+* ``expect($type[, $value[, $message]])``: If the current token isn't of the given
+  type/value a syntax error is thrown. Otherwise, if the type and value are correct,
+  the token is returned and the stream moves to the next token.
 
 Parsing expressions is done by calling the ``parseExpression()`` like we did for
 the ``set`` tag.
diff --git a/lib/Twig/TokenStream.php b/lib/Twig/TokenStream.php
index 081a279..d8e8edf 100644
--- a/lib/Twig/TokenStream.php
+++ b/lib/Twig/TokenStream.php
@@ -11,104 +11,54 @@
  */
 class Twig_TokenStream
 {
-    protected $pushed;
-    protected $originalTokens;
     protected $tokens;
-    protected $eof;
     protected $current;
     protected $filename;
 
-    public function __construct(array $tokens, $filename)
+    /**
+     * @param array  $tokens   Array of tokens
+     * @param string $filename Name which $tokens are associated with
+     */
+    public function __construct(array $tokens, $filename = null)
     {
-        $this->pushed = array();
-        $this->originalTokens = $tokens;
-        $this->tokens = $tokens;
-        $this->filename = $filename;
-        $this->next();
+        $this->tokens     = $tokens;
+        $this->current    = 0;
+        $this->filename   = $filename;
     }
 
     public function __toString()
     {
-        $repr = '';
-        foreach ($this->originalTokens as $token) {
-            $repr .= $token."\n";
-        }
-
-        return $repr;
-    }
-
-    public function push($token)
-    {
-        $this->pushed[] = $token;
+        return implode("\n", $this->tokens);
     }
 
     /**
      * Sets the pointer to the next token and returns the old one.
      *
-     * @param Boolean $fromStack Whether to get a token from the stack or not
-     */
-    public function next($fromStack = true)
-    {
-        if ($fromStack && !empty($this->pushed)) {
-            $old = array_shift($this->pushed);
-            $token = array_shift($this->pushed);
-        } else {
-            $old = $this->current;
-            $token = array_shift($this->tokens);
-        }
-
-        if (null === $token) {
-            throw new Twig_Error_Syntax('Unexpected end of template', -1);
-        }
-
-        $this->current = $token;
-
-        $this->eof = $token->getType() === Twig_Token::EOF_TYPE;
-
-        return $old;
-    }
-
-    /**
-     * Looks at the next token.
+     * @return Twig_Token
      */
-    public function look()
+    public function next()
     {
-        $old = $this->next(false);
-        $new = $this->current;
-        $this->push($old);
-        $this->push($new);
-
-        return $new;
-    }
-
-    /**
-     * Rewinds the pushed tokens.
-     */
-    public function rewind()
-    {
-        $tokens = array();
-        while ($this->pushed) {
-            $tokens[] = array_shift($this->pushed);
-            array_shift($this->pushed);
+        if (!isset($this->tokens[++$this->current])) {
+            throw new Twig_Error_Syntax('Unexpected end of template');
         }
 
-        $this->tokens = array_merge($tokens, array($this->current), $this->tokens);
-
-        $this->next();
+        return $this->tokens[$this->current - 1];
     }
 
     /**
-     * Expects a token (like $token->test()) and returns it or throw a syntax error.
+     * test()s a token and returns it or throws a syntax error.
+     *
+     * @return Twig_Token
      */
-    public function expect($primary, $secondary = null, $message = null)
+    public function expect($type, $value = null, $message = null)
     {
-        $token = $this->current;
-        if (!$token->test($primary, $secondary)) {
+        $token = $this->tokens[$this->current];
+        if (!$token->test($type, $value)) {
             throw new Twig_Error_Syntax(sprintf('%sUnexpected token "%s" of value "%s" ("%s" expected%s)',
                 $message ? $message.'. ' : '',
                 Twig_Token::typeToEnglish($token->getType()), $token->getValue(),
-                Twig_Token::typeToEnglish($primary), $secondary ? sprintf(' with value "%s"', $secondary) : ''),
-                $this->current->getLine()
+                Twig_Token::typeToEnglish($type), $value ? sprintf(' with value "%s"', $value) : ''),
+                $token->getLine()
             );
         }
         $this->next();
@@ -117,23 +67,40 @@ class Twig_TokenStream
     }
 
     /**
-     * Forwards that call to the current token.
+     * test() current token
+     *
+     * @return bool
      */
     public function test($primary, $secondary = null)
     {
-        return $this->current->test($primary, $secondary);
+        return $this->tokens[$this->current]->test($primary, $secondary);
     }
 
+    /**
+     * Checks if end of stream was reached
+     *
+     * @return bool
+     */
     public function isEOF()
     {
-        return $this->eof;
+        return $this->tokens[$this->current]->getType() === Twig_Token::EOF_TYPE;
     }
 
+    /**
+     * Gets the current token
+     *
+     * @return Twig_Token
+     */
     public function getCurrent()
     {
-        return $this->current;
+        return $this->tokens[$this->current];
     }
 
+    /**
+     * Gets the filename associated with this stream
+     *
+     * @return string
+     */
     public function getFilename()
     {
         return $this->filename;
diff --git a/test/Twig/Tests/TokenStreamTest.php b/test/Twig/Tests/TokenStreamTest.php
index 341773c..794a037 100644
--- a/test/Twig/Tests/TokenStreamTest.php
+++ b/test/Twig/Tests/TokenStreamTest.php
@@ -29,56 +29,13 @@ class Twig_Tests_TokenStreamTest extends PHPUnit_Framework_TestCase
 
     public function testNext()
     {
-        $stream = new Twig_TokenStream(self::$tokens, '', false);
+        $stream = new Twig_TokenStream(self::$tokens);
         $repr = array();
         while (!$stream->isEOF()) {
             $token = $stream->next();
 
             $repr[] = $token->getValue();
         }
-        $this->assertEquals('1, 2, 3, 4, 5, 6, 7', implode(', ', $repr), '->next() returns the next token in the stream');
-    }
-
-    public function testLook()
-    {
-        $stream = new Twig_TokenStream(self::$tokens, '', false);
-        $this->assertEquals(2, $stream->look()->getValue(), '->look() returns the next token');
-        $repr = array();
-        while (!$stream->isEOF()) {
-            $token = $stream->next();
-
-            $repr[] = $token->getValue();
-        }
-        $this->assertEquals('1, 2, 3, 4, 5, 6, 7', implode(', ', $repr), '->look() pushes the token to the stack');
-
-        $stream = new Twig_TokenStream(self::$tokens, '', false);
-        $this->assertEquals(2, $stream->look()->getValue(), '->look() returns the next token');
-        $this->assertEquals(3, $stream->look()->getValue(), '->look() can be called several times to look more than one upcoming token');
-        $this->assertEquals(4, $stream->look()->getValue(), '->look() can be called several times to look more than one upcoming token');
-        $this->assertEquals(5, $stream->look()->getValue(), '->look() can be called several times to look more than one upcoming token');
-        $repr = array();
-        while (!$stream->isEOF()) {
-            $token = $stream->next();
-
-            $repr[] = $token->getValue();
-        }
-        $this->assertEquals('1, 2, 3, 4, 5, 6, 7', implode(', ', $repr), '->look() pushes the token to the stack');
-    }
-
-    public function testRewind()
-    {
-        $stream = new Twig_TokenStream(self::$tokens, '', false);
-        $this->assertEquals(2, $stream->look()->getValue(), '->look() returns the next token');
-        $this->assertEquals(3, $stream->look()->getValue(), '->look() can be called several times to look more than one upcoming token');
-        $this->assertEquals(4, $stream->look()->getValue(), '->look() can be called several times to look more than one upcoming token');
-        $this->assertEquals(5, $stream->look()->getValue(), '->look() can be called several times to look more than one upcoming token');
-        $stream->rewind();
-        $repr = array();
-        while (!$stream->isEOF()) {
-            $token = $stream->next(false);
-
-            $repr[] = $token->getValue();
-        }
-        $this->assertEquals('1, 2, 3, 4, 5, 6, 7', implode(', ', $repr), '->rewind() pushes all pushed tokens to the token array');
+        $this->assertEquals('1, 2, 3, 4, 5, 6, 7', implode(', ', $repr), '->next() advances the pointer and returns the current token');
     }
 }