Tokenizer: Identifiers inside '[' and ']'

VAR.ExpressionEvaluator.Tests/TokenizerTests.cs

@@ -1,5 +1,5 @@
-using Microsoft.VisualStudio.TestTools.UnitTesting;
+using System.IO;
-using System.IO;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
 
 namespace VAR.ExpressionEvaluator.Tests
 {
@@ -92,7 +92,7 @@ namespace VAR.ExpressionEvaluator.Tests
         [TestMethod()]
         public void Tokenizer__Identifiers()
         {
-            var testString = "null true false _$variable1 $variable2";
+            var testString = "null true false _$variable1 $variable2 [;{}#]";
             var t = new Tokenizer(new StringReader(testString));
 
             // "null"
@@ -120,6 +120,11 @@ namespace VAR.ExpressionEvaluator.Tests
             Assert.AreEqual(t.Text, "$variable2");
             t.NextToken();
 
+            // ";{}#"
+            Assert.AreEqual(t.Token, Token.Identifier);
+            Assert.AreEqual(t.Text, ";{}#");
+            t.NextToken();
+
             Assert.AreEqual(t.Token, Token.EOF);
         }
 

VAR.ExpressionEvaluator/Tokenizer.cs

@@ -181,6 +181,41 @@ namespace VAR.ExpressionEvaluator
                     if (_currentChar == '\0') { break; }
                 }
                 _text = sb.ToString();
+                _currentToken = Token.Identifier;
+            }
+            else if (_currentChar == '[')
+            {
+                char stringEndsWith = ']';
+
+                NextChar();
+                StringBuilder sbString = new StringBuilder();
+                while (_currentChar != stringEndsWith && _currentChar != '\0')
+                {
+                    if (_currentChar != '\\')
+                    {
+                        sbString.Append(_currentChar);
+                    }
+                    else
+                    {
+                        NextChar();
+                        if (_currentChar == stringEndsWith)
+                        {
+                            sbString.Append(stringEndsWith);
+                        }
+                        else
+                        {
+                            // FIXME: Other escaped characters
+                            sbString.Append(_currentChar);
+                        }
+                    }
+                    NextChar();
+                }
+                NextChar();
+                _text = sbString.ToString();
+                _currentToken = Token.Identifier;
+            }
+            if (_currentToken == Token.Identifier)
+            {
                 string textLowercase = _text.ToLower();
                 if (textLowercase == "and")
                 {