From 379583607b0f240613312de679dec94f81f55242 Mon Sep 17 00:00:00 2001
From: "Valeriano A.R" <valen.alfonso@gmail.com>
Date: Mon, 2 Dec 2019 07:56:12 +0100
Subject: [PATCH] Tokenicer: Better identifier tokening.

---
 VAR.ExpressionEvaluator.Tests/ParserTests.cs  | 11 ++++++++++
 .../TokenizerTests.cs                         | 14 +++++++++++--
 VAR.ExpressionEvaluator/Tokenizer.cs          | 20 ++++++++++++++++---
 3 files changed, 40 insertions(+), 5 deletions(-)

diff --git a/VAR.ExpressionEvaluator.Tests/ParserTests.cs b/VAR.ExpressionEvaluator.Tests/ParserTests.cs
index 7124d30..0c8e785 100644
--- a/VAR.ExpressionEvaluator.Tests/ParserTests.cs
+++ b/VAR.ExpressionEvaluator.Tests/ParserTests.cs
@@ -171,6 +171,17 @@ namespace VAR.ExpressionEvaluator.Tests
             Assert.AreEqual(50m, result);
         }
 
+        [TestMethod()]
+        public void Variables__Var1DivideVar2()
+        {
+            EvaluationContext evaluationContex = new EvaluationContext();
+            evaluationContex.SetVariable("_v1", 100);
+            evaluationContex.SetVariable("$v2", 20);
+            string expression = "_v1 / $v2";
+            object result = Parser.EvaluateString(expression, evaluationContex);
+            Assert.AreEqual(5m, result);
+        }
+
         #endregion Variables
 
         #region Funcitions
diff --git a/VAR.ExpressionEvaluator.Tests/TokenizerTests.cs b/VAR.ExpressionEvaluator.Tests/TokenizerTests.cs
index 0127905..a65dc22 100644
--- a/VAR.ExpressionEvaluator.Tests/TokenizerTests.cs
+++ b/VAR.ExpressionEvaluator.Tests/TokenizerTests.cs
@@ -90,9 +90,9 @@ namespace VAR.ExpressionEvaluator.Tests
         }
 
         [TestMethod()]
-        public void Tokenizer__Keywords()
+        public void Tokenizer__Identifiers()
         {
-            var testString = "null true false";
+            var testString = "null true false _$variable1 $variable2";
             var t = new Tokenizer(new StringReader(testString));
 
             // "null"
@@ -110,6 +110,16 @@ namespace VAR.ExpressionEvaluator.Tests
             Assert.AreEqual(t.Text, "false");
             t.NextToken();
 
+            // "_$variable1"
+            Assert.AreEqual(t.Token, Token.Identifier);
+            Assert.AreEqual(t.Text, "_$variable1");
+            t.NextToken();
+
+            // "$variable2"
+            Assert.AreEqual(t.Token, Token.Identifier);
+            Assert.AreEqual(t.Text, "$variable2");
+            t.NextToken();
+
             Assert.AreEqual(t.Token, Token.EOF);
         }
 
diff --git a/VAR.ExpressionEvaluator/Tokenizer.cs b/VAR.ExpressionEvaluator/Tokenizer.cs
index c0512fd..aba0f05 100644
--- a/VAR.ExpressionEvaluator/Tokenizer.cs
+++ b/VAR.ExpressionEvaluator/Tokenizer.cs
@@ -171,10 +171,10 @@ namespace VAR.ExpressionEvaluator
             }
 
             // Identifier
-            if (char.IsLetter(_currentChar))
+            if (IsIdentifierStartLetter(_currentChar))
             {
                 var sb = new StringBuilder();
-                while (char.IsLetterOrDigit(_currentChar) || _currentChar == '_')
+                while (IsIdentifierLetter(_currentChar))
                 {
                     sb.Append(_currentChar);
                     NextChar();
@@ -216,7 +216,11 @@ namespace VAR.ExpressionEvaluator
                     else
                     {
                         NextChar();
-                        if (_currentChar == '\\')
+                        if (_currentChar == stringEndsWith)
+                        {
+                            sbString.Append(stringEndsWith);
+                        }
+                        else if (_currentChar == '\\')
                         {
                             sbString.Append('\\');
                         }
@@ -260,5 +264,15 @@ namespace VAR.ExpressionEvaluator
 
             throw new InvalidDataException(string.Format("Unexpected character: {0} at {1}", _currentChar, _currentPosition));
         }
+
+        private static bool IsIdentifierLetter(char c)
+        {
+            return char.IsLetterOrDigit(c) || c == '_' || c == '$' || c == '#' || c == '@';
+        }
+
+        private static bool IsIdentifierStartLetter(char c)
+        {
+            return char.IsLetter(c) || c == '_' || c == '$' || c == '#' || c == '@';
+        }
     }
 }