From: Fabien Potencier <fabien.potencier@gmail.com>
Date: Fri, 28 May 2010 13:28:22 +0000 (+0200)
Subject: added a grammar to easily describe a tag syntax with a string
X-Git-Url: http://git.silmor.de/gitweb/?a=commitdiff_plain;h=0ca17abc7129eeba1c63f330ac4d76a96e553192;p=web%2Fkonrad%2Ftwig.git

added a grammar to easily describe a tag syntax with a string
---

diff --git a/CHANGELOG b/CHANGELOG
index 540dc84..045848b 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -5,6 +5,7 @@ Backward incompatibilities:
  * removed the sandboxed attribute of the include tag (use the new sandbox tag instead)
  * refactored the Node system (if you have custom nodes, you will have to update them to use the new API)
 
+ * added a grammar sub-framework to ease the creation of custom tags
  * fixed the for tag for large arrays (some loop variables are now only available for arrays and objects that implement the Countable interface)
  * removed the Twig_Resource::resolveMissingFilter() method
  * fixed the filter tag which did not apply filtering to included files
diff --git a/lib/Twig/Grammar.php b/lib/Twig/Grammar.php
new file mode 100644
index 0000000..a9c6e3d
--- /dev/null
+++ b/lib/Twig/Grammar.php
@@ -0,0 +1,30 @@
+<?php
+
+/*
+ * This file is part of Twig.
+ *
+ * (c) 2010 Fabien Potencier
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+abstract class Twig_Grammar implements Twig_GrammarInterface
+{
+    protected $name;
+    protected $parser;
+
+    public function __construct($name)
+    {
+        $this->name = $name;
+    }
+
+    public function setParser(Twig_ParserInterface $parser)
+    {
+        $this->parser = $parser;
+    }
+
+    public function getName()
+    {
+        return $this->name;
+    }
+}
diff --git a/lib/Twig/Grammar/Arguments.php b/lib/Twig/Grammar/Arguments.php
new file mode 100644
index 0000000..d0d3f91
--- /dev/null
+++ b/lib/Twig/Grammar/Arguments.php
@@ -0,0 +1,22 @@
+<?php
+
+/*
+ * This file is part of Twig.
+ *
+ * (c) 2010 Fabien Potencier
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+class Twig_Grammar_Arguments extends Twig_Grammar
+{
+    public function __toString()
+    {
+        return sprintf('<%s:arguments>', $this->name);
+    }
+
+    public function parse(Twig_Token $token)
+    {
+        return $this->parser->getExpressionParser()->parseArguments();
+    }
+}
diff --git a/lib/Twig/Grammar/Array.php b/lib/Twig/Grammar/Array.php
new file mode 100644
index 0000000..7605e9e
--- /dev/null
+++ b/lib/Twig/Grammar/Array.php
@@ -0,0 +1,22 @@
+<?php
+
+/*
+ * This file is part of Twig.
+ *
+ * (c) 2010 Fabien Potencier
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+class Twig_Grammar_Array extends Twig_Grammar
+{
+    public function __toString()
+    {
+        return sprintf('<%s:array>', $this->name);
+    }
+
+    public function parse(Twig_Token $token)
+    {
+        return $this->parser->getExpressionParser()->parseArrayExpression();
+    }
+}
diff --git a/lib/Twig/Grammar/Body.php b/lib/Twig/Grammar/Body.php
new file mode 100644
index 0000000..081d43d
--- /dev/null
+++ b/lib/Twig/Grammar/Body.php
@@ -0,0 +1,39 @@
+<?php
+
+/*
+ * This file is part of Twig.
+ *
+ * (c) 2010 Fabien Potencier
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+class Twig_Grammar_Body extends Twig_Grammar
+{
+    protected $end;
+
+    public function __construct($name, $end = null)
+    {
+        parent::__construct($name);
+
+        $this->end = null === $end ? 'end'.$name : $end;
+    }
+
+    public function __toString()
+    {
+        return sprintf('<%s:body>', $this->name);
+    }
+
+    public function parse(Twig_Token $token)
+    {
+        $stream = $this->parser->getStream();
+        $stream->expect(Twig_Token::BLOCK_END_TYPE);
+
+        return $this->parser->subparse(array($this, 'decideBlockEnd'), true);
+    }
+
+    public function decideBlockEnd($token)
+    {
+        return $token->test($this->end);
+    }
+}
diff --git a/lib/Twig/Grammar/Boolean.php b/lib/Twig/Grammar/Boolean.php
new file mode 100644
index 0000000..02a6070
--- /dev/null
+++ b/lib/Twig/Grammar/Boolean.php
@@ -0,0 +1,24 @@
+<?php
+
+/*
+ * This file is part of Twig.
+ *
+ * (c) 2010 Fabien Potencier
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+class Twig_Grammar_Boolean extends Twig_Grammar
+{
+    public function __toString()
+    {
+        return sprintf('<%s:boolean>', $this->name);
+    }
+
+    public function parse(Twig_Token $token)
+    {
+        $this->parser->getStream()->expect(Twig_Token::NAME_TYPE, array('true', 'false'));
+
+        return new Twig_Node_Expression_Constant('true' === $token->getValue() ? true : false, $token->getLine());
+    }
+}
diff --git a/lib/Twig/Grammar/Constant.php b/lib/Twig/Grammar/Constant.php
new file mode 100644
index 0000000..9ea584b
--- /dev/null
+++ b/lib/Twig/Grammar/Constant.php
@@ -0,0 +1,24 @@
+<?php
+
+/*
+ * This file is part of Twig.
+ *
+ * (c) 2010 Fabien Potencier
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+class Twig_Grammar_Constant extends Twig_Grammar
+{
+    public function __toString()
+    {
+        return $this->name;
+    }
+
+    public function parse(Twig_Token $token)
+    {
+        $this->parser->getStream()->expect($this->name);
+
+        return $this->name;
+    }
+}
diff --git a/lib/Twig/Grammar/Expression.php b/lib/Twig/Grammar/Expression.php
new file mode 100644
index 0000000..43012f3
--- /dev/null
+++ b/lib/Twig/Grammar/Expression.php
@@ -0,0 +1,22 @@
+<?php
+
+/*
+ * This file is part of Twig.
+ *
+ * (c) 2010 Fabien Potencier
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+class Twig_Grammar_Expression extends Twig_Grammar
+{
+    public function __toString()
+    {
+        return sprintf('<%s>', $this->name);
+    }
+
+    public function parse(Twig_Token $token)
+    {
+        return $this->parser->getExpressionParser()->parseExpression();
+    }
+}
diff --git a/lib/Twig/Grammar/Number.php b/lib/Twig/Grammar/Number.php
new file mode 100644
index 0000000..0ce3f16
--- /dev/null
+++ b/lib/Twig/Grammar/Number.php
@@ -0,0 +1,24 @@
+<?php
+
+/*
+ * This file is part of Twig.
+ *
+ * (c) 2010 Fabien Potencier
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+class Twig_Grammar_Number extends Twig_Grammar
+{
+    public function __toString()
+    {
+        return sprintf('<%s:number>', $this->name);
+    }
+
+    public function parse(Twig_Token $token)
+    {
+        $this->parser->getStream()->expect(Twig_Token::NUMBER_TYPE);
+
+        return new Twig_Node_Expression_Constant($token->getValue(), $token->getLine());
+    }
+}
diff --git a/lib/Twig/Grammar/Optional.php b/lib/Twig/Grammar/Optional.php
new file mode 100644
index 0000000..a16dbfb
--- /dev/null
+++ b/lib/Twig/Grammar/Optional.php
@@ -0,0 +1,64 @@
+<?php
+
+/*
+ * This file is part of Twig.
+ *
+ * (c) 2010 Fabien Potencier
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+class Twig_Grammar_Optional extends Twig_Grammar
+{
+    protected $grammar;
+
+    public function __construct()
+    {
+        $this->grammar = array();
+        foreach (func_get_args() as $grammar) {
+            $this->addGrammar($grammar);
+        }
+    }
+
+    public function __toString()
+    {
+        $repr = array();
+        foreach ($this->grammar as $grammar) {
+            $repr[] = (string) $grammar;
+        }
+
+        return sprintf('[%s]', implode(' ', $repr));
+    }
+
+    public function addGrammar(Twig_GrammarInterface $grammar)
+    {
+        $this->grammar[] = $grammar;
+    }
+
+    public function parse(Twig_Token $token)
+    {
+        // test if we have the optional element before consuming it
+        if ($this->grammar[0] instanceof Twig_Grammar_Constant) {
+            if (!$this->parser->getStream()->test($this->grammar[0]->getName())) {
+                return array();
+            }
+        } elseif ($this->grammar[0] instanceof Twig_Grammar_Name) {
+            if (!$this->parser->getStream()->test(Twig_Token::NAME_TYPE)) {
+                return array();
+            }
+        } elseif ($this->parser->getStream()->test(Twig_Token::BLOCK_END_TYPE)) {
+            // if this is not a Constant or a Name, it must be the last element of the tag
+
+            return array();
+        }
+
+        $elements = array();
+        foreach ($this->grammar as $grammar) {
+            $grammar->setParser($this->parser);
+
+            $elements[$grammar->getName()] = $grammar->parse($token);
+        }
+
+        return $elements;
+    }
+}
diff --git a/lib/Twig/Grammar/Tag.php b/lib/Twig/Grammar/Tag.php
new file mode 100644
index 0000000..2aad31a
--- /dev/null
+++ b/lib/Twig/Grammar/Tag.php
@@ -0,0 +1,56 @@
+<?php
+
+/*
+ * This file is part of Twig.
+ *
+ * (c) 2010 Fabien Potencier
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+class Twig_Grammar_Tag extends Twig_Grammar
+{
+    protected $grammar;
+
+    public function __construct()
+    {
+        $this->grammar = array();
+        foreach (func_get_args() as $grammar) {
+            $this->addGrammar($grammar);
+        }
+    }
+
+    public function __toString()
+    {
+        $repr = array();
+        foreach ($this->grammar as $grammar) {
+            $repr[] = (string) $grammar;
+        }
+
+        return implode(' ', $repr);
+    }
+
+    public function addGrammar(Twig_GrammarInterface $grammar)
+    {
+        $this->grammar[] = $grammar;
+    }
+
+    public function parse(Twig_Token $token)
+    {
+        $elements = array();
+        foreach ($this->grammar as $grammar) {
+            $grammar->setParser($this->parser);
+
+            $element = $grammar->parse($token);
+            if (is_array($element)) {
+                $elements = array_merge($elements, $element);
+            } else {
+                $elements[$grammar->getName()] = $element;
+            }
+        }
+
+        $this->parser->getStream()->expect(Twig_Token::BLOCK_END_TYPE);
+
+        return $elements;
+    }
+}
diff --git a/lib/Twig/GrammarInterface.php b/lib/Twig/GrammarInterface.php
new file mode 100644
index 0000000..5a71b7e
--- /dev/null
+++ b/lib/Twig/GrammarInterface.php
@@ -0,0 +1,18 @@
+<?php
+
+/*
+ * This file is part of Twig.
+ *
+ * (c) 2010 Fabien Potencier
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+interface Twig_GrammarInterface
+{
+    public function setParser(Twig_ParserInterface $parser);
+
+    public function parse(Twig_Token $token);
+
+    public function getName();
+}
diff --git a/lib/Twig/SimpleTokenParser.php b/lib/Twig/SimpleTokenParser.php
new file mode 100644
index 0000000..42e78da
--- /dev/null
+++ b/lib/Twig/SimpleTokenParser.php
@@ -0,0 +1,86 @@
+<?php
+
+/*
+ * This file is part of Twig.
+ *
+ * (c) 2010 Fabien Potencier
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+abstract class Twig_SimpleTokenParser extends Twig_TokenParser
+{
+    /**
+     * Parses a token and returns a node.
+     *
+     * @param Twig_Token $token A Twig_Token instance
+     *
+     * @return Twig_NodeInterface A Twig_NodeInterface instance
+     */
+    public function parse(Twig_Token $token)
+    {
+        $grammar = $this->getGrammar();
+        if (!is_object($grammar)) {
+            $grammar = static::parseGrammar($grammar);
+        }
+
+        $grammar->setParser($this->parser);
+        $values = $grammar->parse($token);
+
+        return $this->getNode($values, $token->getLine());
+    }
+
+    /**
+     * Gets the grammar as an object or as a string.
+     *
+     * @return string|Twig_Grammar A Twig_Grammar instance or a string
+     */
+    abstract protected function getGrammar();
+
+    /**
+     * Gets the nodes based on the parsed values.
+     *
+     * @param array   $values An array of values
+     * @param integer $line   The parser line
+     */
+    abstract protected function getNode(array $values, $line);
+
+    static public function parseGrammar($str, $main = true)
+    {
+        static $cursor;
+
+        if (true === $main) {
+            $cursor = 0;
+            $grammar = new Twig_Grammar_Tag();
+        } else {
+            $grammar = new Twig_Grammar_Optional();
+        }
+
+        while ($cursor < strlen($str)) {
+            if (preg_match('/\s+/A', $str, $match, null, $cursor)) {
+                $cursor += strlen($match[0]);
+            } elseif (preg_match('/<(\w+)(?:\:(\w+))?>/A', $str, $match, null, $cursor)) {
+                $class = sprintf('Twig_Grammar_%s', ucfirst(isset($match[2]) ? $match[2] : 'Expression'));
+                if (!class_exists($class)) {
+                    throw new InvalidArgumentException(sprintf('Unable to understand "%s" in grammar (%s class does not exist)', $match[0], $class));
+                }
+                $grammar->addGrammar(new $class($match[1]));
+                $cursor += strlen($match[0]);
+            } elseif (preg_match('/(\w+)/A', $str, $match, null, $cursor)) {
+                $grammar->addGrammar(new Twig_Grammar_Constant($match[1]));
+                $cursor += strlen($match[0]);
+            } elseif (preg_match('/\[/A', $str, $match, null, $cursor)) {
+                $cursor += strlen($match[0]);
+                $grammar->addGrammar(static::parseGrammar($str, false));
+            } elseif (true !== $main && preg_match('/\]/A', $str, $match, null, $cursor)) {
+                $cursor += strlen($match[0]);
+
+                return $grammar;
+            } else {
+                throw new InvalidArgumentException(sprintf('Unable to parse grammar "%s" near "...%s..."', $str, substr($str, $cursor, 10)));
+            }
+        }
+
+        return $grammar;
+    }
+}
diff --git a/test/Twig/Tests/Grammar/ArgumentsTest.php b/test/Twig/Tests/Grammar/ArgumentsTest.php
new file mode 100644
index 0000000..1578600
--- /dev/null
+++ b/test/Twig/Tests/Grammar/ArgumentsTest.php
@@ -0,0 +1,19 @@
+<?php
+
+/*
+ * This file is part of Twig.
+ *
+ * (c) Fabien Potencier
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+class Twig_Tests_Grammar_ArgumentsTest extends PHPUnit_Framework_TestCase
+{
+    public function testMagicToString()
+    {
+        $grammar = new Twig_Grammar_Arguments('foo');
+        $this->assertEquals('<foo:arguments>', (string) $grammar);
+    }
+}
diff --git a/test/Twig/Tests/Grammar/ArrayTest.php b/test/Twig/Tests/Grammar/ArrayTest.php
new file mode 100644
index 0000000..7af1774
--- /dev/null
+++ b/test/Twig/Tests/Grammar/ArrayTest.php
@@ -0,0 +1,19 @@
+<?php
+
+/*
+ * This file is part of Twig.
+ *
+ * (c) Fabien Potencier
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+class Twig_Tests_Grammar_ArrayTest extends PHPUnit_Framework_TestCase
+{
+    public function testMagicToString()
+    {
+        $grammar = new Twig_Grammar_Array('foo');
+        $this->assertEquals('<foo:array>', (string) $grammar);
+    }
+}
diff --git a/test/Twig/Tests/Grammar/BodyTest.php b/test/Twig/Tests/Grammar/BodyTest.php
new file mode 100644
index 0000000..d342675
--- /dev/null
+++ b/test/Twig/Tests/Grammar/BodyTest.php
@@ -0,0 +1,19 @@
+<?php
+
+/*
+ * This file is part of Twig.
+ *
+ * (c) Fabien Potencier
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+class Twig_Tests_Grammar_BodyTest extends PHPUnit_Framework_TestCase
+{
+    public function testMagicToString()
+    {
+        $grammar = new Twig_Grammar_Body('foo');
+        $this->assertEquals('<foo:body>', (string) $grammar);
+    }
+}
diff --git a/test/Twig/Tests/Grammar/BooleanTest.php b/test/Twig/Tests/Grammar/BooleanTest.php
new file mode 100644
index 0000000..6cae563
--- /dev/null
+++ b/test/Twig/Tests/Grammar/BooleanTest.php
@@ -0,0 +1,19 @@
+<?php
+
+/*
+ * This file is part of Twig.
+ *
+ * (c) Fabien Potencier
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+class Twig_Tests_Grammar_BooleanTest extends PHPUnit_Framework_TestCase
+{
+    public function testMagicToString()
+    {
+        $grammar = new Twig_Grammar_Boolean('foo');
+        $this->assertEquals('<foo:boolean>', (string) $grammar);
+    }
+}
diff --git a/test/Twig/Tests/Grammar/ConstantTest.php b/test/Twig/Tests/Grammar/ConstantTest.php
new file mode 100644
index 0000000..f293c8b
--- /dev/null
+++ b/test/Twig/Tests/Grammar/ConstantTest.php
@@ -0,0 +1,19 @@
+<?php
+
+/*
+ * This file is part of Twig.
+ *
+ * (c) Fabien Potencier
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+class Twig_Tests_Grammar_ConstantTest extends PHPUnit_Framework_TestCase
+{
+    public function testMagicToString()
+    {
+        $grammar = new Twig_Grammar_Constant('foo');
+        $this->assertEquals('foo', (string) $grammar);
+    }
+}
diff --git a/test/Twig/Tests/Grammar/ExpressionTest.php b/test/Twig/Tests/Grammar/ExpressionTest.php
new file mode 100644
index 0000000..2919196
--- /dev/null
+++ b/test/Twig/Tests/Grammar/ExpressionTest.php
@@ -0,0 +1,19 @@
+<?php
+
+/*
+ * This file is part of Twig.
+ *
+ * (c) Fabien Potencier
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+class Twig_Tests_Grammar_ExpressionTest extends PHPUnit_Framework_TestCase
+{
+    public function testMagicToString()
+    {
+        $grammar = new Twig_Grammar_Expression('foo');
+        $this->assertEquals('<foo>', (string) $grammar);
+    }
+}
diff --git a/test/Twig/Tests/Grammar/NumberTest.php b/test/Twig/Tests/Grammar/NumberTest.php
new file mode 100644
index 0000000..4749803
--- /dev/null
+++ b/test/Twig/Tests/Grammar/NumberTest.php
@@ -0,0 +1,19 @@
+<?php
+
+/*
+ * This file is part of Twig.
+ *
+ * (c) Fabien Potencier
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+class Twig_Tests_Grammar_NumberTest extends PHPUnit_Framework_TestCase
+{
+    public function testMagicToString()
+    {
+        $grammar = new Twig_Grammar_Number('foo');
+        $this->assertEquals('<foo:number>', (string) $grammar);
+    }
+}
diff --git a/test/Twig/Tests/Grammar/OptionalTest.php b/test/Twig/Tests/Grammar/OptionalTest.php
new file mode 100644
index 0000000..049f501
--- /dev/null
+++ b/test/Twig/Tests/Grammar/OptionalTest.php
@@ -0,0 +1,19 @@
+<?php
+
+/*
+ * This file is part of Twig.
+ *
+ * (c) Fabien Potencier
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+class Twig_Tests_Grammar_OptionalTest extends PHPUnit_Framework_TestCase
+{
+    public function testMagicToString()
+    {
+        $grammar = new Twig_Grammar_Optional(new Twig_Grammar_Constant('foo'), new Twig_Grammar_Number('bar'));
+        $this->assertEquals('[foo <bar:number>]', (string) $grammar);
+    }
+}
diff --git a/test/Twig/Tests/Grammar/TagTest.php b/test/Twig/Tests/Grammar/TagTest.php
new file mode 100644
index 0000000..1441e14
--- /dev/null
+++ b/test/Twig/Tests/Grammar/TagTest.php
@@ -0,0 +1,23 @@
+<?php
+
+/*
+ * This file is part of Twig.
+ *
+ * (c) Fabien Potencier
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+class Twig_Tests_Grammar_TagTest extends PHPUnit_Framework_TestCase
+{
+    public function testMagicToString()
+    {
+        $grammar = new Twig_Grammar_Tag(
+            new Twig_Grammar_Constant('foo'),
+            new Twig_Grammar_Number('bar'),
+            new Twig_Grammar_Optional(new Twig_Grammar_Constant('foo'), new Twig_Grammar_Number('bar'))
+        );
+        $this->assertEquals('foo <bar:number> [foo <bar:number>]', (string) $grammar);
+    }
+}
diff --git a/test/Twig/Tests/SimpleTokenParser.php b/test/Twig/Tests/SimpleTokenParser.php
new file mode 100644
index 0000000..567d133
--- /dev/null
+++ b/test/Twig/Tests/SimpleTokenParser.php
@@ -0,0 +1,48 @@
+<?php
+
+/*
+ * This file is part of Twig.
+ *
+ * (c) Fabien Potencier
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+class SimpleTokenParser extends Twig_SimpleTokenParser
+{
+    protected $tag;
+    protected $grammar;
+
+    public function __construct($tag, $grammar)
+    {
+        $this->tag = $tag;
+        $this->grammar = $grammar;
+    }
+
+    public function getGrammar()
+    {
+        return $this->grammar;
+    }
+
+    public function getTag()
+    {
+        return $this->tag;
+    }
+
+    public function getNode(array $values, $line)
+    {
+        $nodes = array();
+        $nodes[] = new Twig_Node_Print(new Twig_Node_Expression_Constant('|', $line), $line);
+        foreach ($values as $value) {
+            if ($value instanceof Twig_NodeInterface) {
+                $nodes[] = new Twig_Node_Print($value, $line);
+            } else {
+                $nodes[] = new Twig_Node_Print(new Twig_Node_Expression_Constant($value, $line), $line);
+            }
+            $nodes[] = new Twig_Node_Print(new Twig_Node_Expression_Constant('|', $line), $line);
+        }
+
+        return new Twig_Node($nodes);
+    }
+}
\ No newline at end of file
diff --git a/test/Twig/Tests/SimpleTokenParserTest.php b/test/Twig/Tests/SimpleTokenParserTest.php
new file mode 100644
index 0000000..b15638f
--- /dev/null
+++ b/test/Twig/Tests/SimpleTokenParserTest.php
@@ -0,0 +1,101 @@
+<?php
+
+/*
+ * This file is part of Twig.
+ *
+ * (c) Fabien Potencier
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+class Twig_Tests_SimpleTokenParserTest extends PHPUnit_Framework_TestCase
+{
+    /**
+     * @dataProvider getTests
+     */
+    public function testParseGrammar($str, $grammar)
+    {
+        $this->assertEquals($grammar, Twig_SimpleTokenParser::parseGrammar($str), '::parseGrammar() parses a grammar');
+    }
+
+    public function testParseGrammarExceptions()
+    {
+        try {
+            Twig_SimpleTokenParser::parseGrammar('<foo:foo>');
+            $this->fail();
+        } catch (Exception $e) {
+            $this->assertEquals('InvalidArgumentException', get_class($e));
+        }
+
+        try {
+            Twig_SimpleTokenParser::parseGrammar('<foo:foo');
+            $this->fail();
+        } catch (Exception $e) {
+            $this->assertEquals('InvalidArgumentException', get_class($e));
+        }
+
+        try {
+            Twig_SimpleTokenParser::parseGrammar('<foo:foo> (with');
+            $this->fail();
+        } catch (Exception $e) {
+            $this->assertEquals('InvalidArgumentException', get_class($e));
+        }
+    }
+
+    public function getTests()
+    {
+        return array(
+            array('', new Twig_Grammar_Tag()),
+            array('const', new Twig_Grammar_Tag(
+                new Twig_Grammar_Constant('const')
+            )),
+            array('   const   ', new Twig_Grammar_Tag(
+                new Twig_Grammar_Constant('const')
+            )),
+            array('<expr>', new Twig_Grammar_Tag(
+                new Twig_Grammar_Expression('expr')
+            )),
+            array('<expr:expression>', new Twig_Grammar_Tag(
+                new Twig_Grammar_Expression('expr')
+            )),
+            array('   <expr:expression>   ', new Twig_Grammar_Tag(
+                new Twig_Grammar_Expression('expr')
+            )),
+            array('<nb:number>', new Twig_Grammar_Tag(
+                new Twig_Grammar_Number('nb')
+            )),
+            array('<bool:boolean>', new Twig_Grammar_Tag(
+                new Twig_Grammar_Boolean('bool')
+            )),
+            array('<content:body>', new Twig_Grammar_Tag(
+                new Twig_Grammar_Body('content')
+            )),
+            array('<expr:expression> [with <arguments:array>]', new Twig_Grammar_Tag(
+                new Twig_Grammar_Expression('expr'),
+                new Twig_Grammar_Optional(
+                    new Twig_Grammar_Constant('with'),
+                    new Twig_Grammar_Array('arguments')
+                )
+            )),
+            array('  <expr:expression>   [  with   <arguments:array> ]  ', new Twig_Grammar_Tag(
+                new Twig_Grammar_Expression('expr'),
+                new Twig_Grammar_Optional(
+                    new Twig_Grammar_Constant('with'),
+                    new Twig_Grammar_Array('arguments')
+                )
+            )),
+            array('<expr:expression> [with <arguments:array> [or <optional:expression>]]', new Twig_Grammar_Tag(
+                new Twig_Grammar_Expression('expr'),
+                new Twig_Grammar_Optional(
+                    new Twig_Grammar_Constant('with'),
+                    new Twig_Grammar_Array('arguments'),
+                    new Twig_Grammar_Optional(
+                        new Twig_Grammar_Constant('or'),
+                        new Twig_Grammar_Expression('optional')
+                    )
+                )
+            )),
+        );
+    }
+}
diff --git a/test/Twig/Tests/grammarTest.php b/test/Twig/Tests/grammarTest.php
new file mode 100644
index 0000000..11bd95a
--- /dev/null
+++ b/test/Twig/Tests/grammarTest.php
@@ -0,0 +1,63 @@
+<?php
+
+/*
+ * This file is part of Twig.
+ *
+ * (c) Fabien Potencier
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+require_once dirname(__FILE__).'/SimpleTokenParser.php';
+
+class grammarTest extends PHPUnit_Framework_TestCase
+{
+    /**
+     * @dataProvider getTests
+     */
+    public function testGrammar($tag, $grammar, $template, $output, $exception)
+    {
+        $twig = new Twig_Environment(new Twig_Loader_String(), array('cache' => false));
+        $twig->addTokenParser(new SimpleTokenParser($tag, $grammar));
+
+        $ok = true;
+        try {
+            $template = $twig->loadTemplate($template);
+        } catch (Exception $e) {
+            $ok = false;
+
+            if (false === $exception) {
+                $this->fail('Exception not expected');
+            } else {
+                $this->assertEquals($exception, get_class($e));
+            }
+        }
+
+        if ($ok) {
+            if (false !== $exception) {
+                $this->fail(sprintf('Exception "%s" expected', $exception));
+            }
+
+            $actual = $template->render(array());
+            $this->assertEquals($output, $actual);
+        }
+    }
+
+    public function getTests()
+    {
+        return array(
+            array('foo1', '', '{% foo1 %}', '|', false),
+            array('foo2', '', '{% foo2 "bar" %}', '|', 'Twig_SyntaxError'),
+            array('foo3', '<foo>', '{% foo3 "bar" %}', '|bar|', false),
+            array('foo4', '<foo>', '{% foo4 1 + 2 %}', '|3|', false),
+            array('foo5', '<foo:expression>', '{% foo5 1 + 2 %}', '|3|', false),
+            array('foo6', '<foo:array>', '{% foo6 1 + 2 %}', '|3|', 'Twig_SyntaxError'),
+            array('foo7', '<foo>', '{% foo7 %}', '|3|', 'Twig_SyntaxError'),
+            array('foo8', '<foo:array>', '{% foo8 [1, 2] %}', '|Array|', false),
+            array('foo9', '<foo> with <bar>', '{% foo9 "bar" with "foobar" %}', '|bar|with|foobar|', false),
+            array('foo10', '<foo> [with <bar>]', '{% foo10 "bar" with "foobar" %}', '|bar|with|foobar|', false),
+            array('foo11', '<foo> [with <bar>]', '{% foo11 "bar" %}', '|bar|', false),
+        );
+    }
+}