The parsing process is simplified thanks to a bunch of methods you can call
from the token stream (``$this->parser->getStream()``):
-* ``test()``: Tests the type and optionally the value of the next token and
- returns it.
+* ``getCurrent()``: Gets the current token in the stream.
-* ``expect()``: Expects a token and returns it (like ``test()``) or throw a
- syntax error if not found (the second argument is the expected value of the
- token).
+* ``next()``: Moves to the next token in the stream, *but returns the old one*.
-* ``look()``: Looks a the next token. This is how you can have a look at the
- next token without consuming it (after you are done with ``look()``, you
- must use ``rewind()``).
+* ``test($type)``, ``test($value)`` or ``test($type, $value)``: Determines whether
+ the current token is of a particular type or value (or both). The value may be an
+ array of several possible values.
+
+* ``expect($type[, $value[, $message]])``: If the current token isn't of the given
+ type/value a syntax error is thrown. Otherwise, if the type and value are correct,
+ the token is returned and the stream moves to the next token.
Parsing expressions is done by calling the ``parseExpression()`` like we did for
the ``set`` tag.
*/
class Twig_TokenStream
{
- protected $pushed;
- protected $originalTokens;
protected $tokens;
- protected $eof;
protected $current;
protected $filename;
- public function __construct(array $tokens, $filename)
+ /**
+ * @param array $tokens Array of tokens
+ * @param string $filename Name which $tokens are associated with
+ */
+ public function __construct(array $tokens, $filename = null)
{
- $this->pushed = array();
- $this->originalTokens = $tokens;
- $this->tokens = $tokens;
- $this->filename = $filename;
- $this->next();
+ $this->tokens = $tokens;
+ $this->current = 0;
+ $this->filename = $filename;
}
public function __toString()
{
- $repr = '';
- foreach ($this->originalTokens as $token) {
- $repr .= $token."\n";
- }
-
- return $repr;
- }
-
- public function push($token)
- {
- $this->pushed[] = $token;
+ return implode("\n", $this->tokens);
}
/**
* Sets the pointer to the next token and returns the old one.
*
- * @param Boolean $fromStack Whether to get a token from the stack or not
- */
- public function next($fromStack = true)
- {
- if ($fromStack && !empty($this->pushed)) {
- $old = array_shift($this->pushed);
- $token = array_shift($this->pushed);
- } else {
- $old = $this->current;
- $token = array_shift($this->tokens);
- }
-
- if (null === $token) {
- throw new Twig_Error_Syntax('Unexpected end of template', -1);
- }
-
- $this->current = $token;
-
- $this->eof = $token->getType() === Twig_Token::EOF_TYPE;
-
- return $old;
- }
-
- /**
- * Looks at the next token.
+ * @return Twig_Token
*/
- public function look()
+ public function next()
{
- $old = $this->next(false);
- $new = $this->current;
- $this->push($old);
- $this->push($new);
-
- return $new;
- }
-
- /**
- * Rewinds the pushed tokens.
- */
- public function rewind()
- {
- $tokens = array();
- while ($this->pushed) {
- $tokens[] = array_shift($this->pushed);
- array_shift($this->pushed);
+ if (!isset($this->tokens[++$this->current])) {
+ throw new Twig_Error_Syntax('Unexpected end of template');
}
- $this->tokens = array_merge($tokens, array($this->current), $this->tokens);
-
- $this->next();
+ return $this->tokens[$this->current - 1];
}
/**
- * Expects a token (like $token->test()) and returns it or throw a syntax error.
+ * test()s a token and returns it or throws a syntax error.
+ *
+ * @return Twig_Token
*/
- public function expect($primary, $secondary = null, $message = null)
+ public function expect($type, $value = null, $message = null)
{
- $token = $this->current;
- if (!$token->test($primary, $secondary)) {
+ $token = $this->tokens[$this->current];
+ if (!$token->test($type, $value)) {
throw new Twig_Error_Syntax(sprintf('%sUnexpected token "%s" of value "%s" ("%s" expected%s)',
$message ? $message.'. ' : '',
Twig_Token::typeToEnglish($token->getType()), $token->getValue(),
- Twig_Token::typeToEnglish($primary), $secondary ? sprintf(' with value "%s"', $secondary) : ''),
- $this->current->getLine()
+ Twig_Token::typeToEnglish($type), $value ? sprintf(' with value "%s"', $value) : ''),
+ $token->getLine()
);
}
$this->next();
}
/**
- * Forwards that call to the current token.
+ * test() current token
+ *
+ * @return bool
*/
public function test($primary, $secondary = null)
{
- return $this->current->test($primary, $secondary);
+ return $this->tokens[$this->current]->test($primary, $secondary);
}
+ /**
+ * Checks if end of stream was reached
+ *
+ * @return bool
+ */
public function isEOF()
{
- return $this->eof;
+ return $this->tokens[$this->current]->getType() === Twig_Token::EOF_TYPE;
}
+ /**
+ * Gets the current token
+ *
+ * @return Twig_Token
+ */
public function getCurrent()
{
- return $this->current;
+ return $this->tokens[$this->current];
}
+ /**
+ * Gets the filename associated with this stream
+ *
+ * @return string
+ */
public function getFilename()
{
return $this->filename;
public function testNext()
{
- $stream = new Twig_TokenStream(self::$tokens, '', false);
+ $stream = new Twig_TokenStream(self::$tokens);
$repr = array();
while (!$stream->isEOF()) {
$token = $stream->next();
$repr[] = $token->getValue();
}
- $this->assertEquals('1, 2, 3, 4, 5, 6, 7', implode(', ', $repr), '->next() returns the next token in the stream');
- }
-
- public function testLook()
- {
- $stream = new Twig_TokenStream(self::$tokens, '', false);
- $this->assertEquals(2, $stream->look()->getValue(), '->look() returns the next token');
- $repr = array();
- while (!$stream->isEOF()) {
- $token = $stream->next();
-
- $repr[] = $token->getValue();
- }
- $this->assertEquals('1, 2, 3, 4, 5, 6, 7', implode(', ', $repr), '->look() pushes the token to the stack');
-
- $stream = new Twig_TokenStream(self::$tokens, '', false);
- $this->assertEquals(2, $stream->look()->getValue(), '->look() returns the next token');
- $this->assertEquals(3, $stream->look()->getValue(), '->look() can be called several times to look more than one upcoming token');
- $this->assertEquals(4, $stream->look()->getValue(), '->look() can be called several times to look more than one upcoming token');
- $this->assertEquals(5, $stream->look()->getValue(), '->look() can be called several times to look more than one upcoming token');
- $repr = array();
- while (!$stream->isEOF()) {
- $token = $stream->next();
-
- $repr[] = $token->getValue();
- }
- $this->assertEquals('1, 2, 3, 4, 5, 6, 7', implode(', ', $repr), '->look() pushes the token to the stack');
- }
-
- public function testRewind()
- {
- $stream = new Twig_TokenStream(self::$tokens, '', false);
- $this->assertEquals(2, $stream->look()->getValue(), '->look() returns the next token');
- $this->assertEquals(3, $stream->look()->getValue(), '->look() can be called several times to look more than one upcoming token');
- $this->assertEquals(4, $stream->look()->getValue(), '->look() can be called several times to look more than one upcoming token');
- $this->assertEquals(5, $stream->look()->getValue(), '->look() can be called several times to look more than one upcoming token');
- $stream->rewind();
- $repr = array();
- while (!$stream->isEOF()) {
- $token = $stream->next(false);
-
- $repr[] = $token->getValue();
- }
- $this->assertEquals('1, 2, 3, 4, 5, 6, 7', implode(', ', $repr), '->rewind() pushes all pushed tokens to the token array');
+ $this->assertEquals('1, 2, 3, 4, 5, 6, 7', implode(', ', $repr), '->next() advances the pointer and returns the current token');
}
}
projects / web/konrad/twig.git / commitdiff