diff --git a/VAR.ExpressionEvaluator.Tests/TokenizerTests.cs b/VAR.ExpressionEvaluator.Tests/TokenizerTests.cs index a65dc22..19b6bb8 100644 --- a/VAR.ExpressionEvaluator.Tests/TokenizerTests.cs +++ b/VAR.ExpressionEvaluator.Tests/TokenizerTests.cs @@ -1,5 +1,5 @@ -using Microsoft.VisualStudio.TestTools.UnitTesting; -using System.IO; +using System.IO; +using Microsoft.VisualStudio.TestTools.UnitTesting; namespace VAR.ExpressionEvaluator.Tests { @@ -92,7 +92,7 @@ namespace VAR.ExpressionEvaluator.Tests [TestMethod()] public void Tokenizer__Identifiers() { - var testString = "null true false _$variable1 $variable2"; + var testString = "null true false _$variable1 $variable2 [;{}#]"; var t = new Tokenizer(new StringReader(testString)); // "null" @@ -120,6 +120,11 @@ namespace VAR.ExpressionEvaluator.Tests Assert.AreEqual(t.Text, "$variable2"); t.NextToken(); + // ";{}#" + Assert.AreEqual(t.Token, Token.Identifier); + Assert.AreEqual(t.Text, ";{}#"); + t.NextToken(); + Assert.AreEqual(t.Token, Token.EOF); } diff --git a/VAR.ExpressionEvaluator/Tokenizer.cs b/VAR.ExpressionEvaluator/Tokenizer.cs index aba0f05..2c912d4 100644 --- a/VAR.ExpressionEvaluator/Tokenizer.cs +++ b/VAR.ExpressionEvaluator/Tokenizer.cs @@ -181,6 +181,41 @@ namespace VAR.ExpressionEvaluator if (_currentChar == '\0') { break; } } _text = sb.ToString(); + _currentToken = Token.Identifier; + } + else if (_currentChar == '[') + { + char stringEndsWith = ']'; + + NextChar(); + StringBuilder sbString = new StringBuilder(); + while (_currentChar != stringEndsWith && _currentChar != '\0') + { + if (_currentChar != '\\') + { + sbString.Append(_currentChar); + } + else + { + NextChar(); + if (_currentChar == stringEndsWith) + { + sbString.Append(stringEndsWith); + } + else + { + // FIXME: Other escaped characters + sbString.Append(_currentChar); + } + } + NextChar(); + } + NextChar(); + _text = sbString.ToString(); + _currentToken = Token.Identifier; + } + if (_currentToken == Token.Identifier) + { string textLowercase = _text.ToLower(); if (textLowercase == "and") {