From 2bd7e89cb357299f3a2a6fb21d92a6ffbfec2227 Mon Sep 17 00:00:00 2001 From: "Valeriano A.R" Date: Sun, 1 Dec 2019 01:08:11 +0100 Subject: [PATCH] Tokenizer. --- .gitignore | 3 + .../Properties/AssemblyInfo.cs | 15 ++ .../TokenizerTests.cs | 178 ++++++++++++++++ .../VAR.ExpressionEvaluator.Tests.csproj | 90 ++++++++ VAR.ExpressionEvaluator.sln | 31 +++ .../Properties/AssemblyInfo.cs | 15 ++ VAR.ExpressionEvaluator/Tokenizer.cs | 200 ++++++++++++++++++ .../VAR.ExpressionEvaluator.csproj | 48 +++++ 8 files changed, 580 insertions(+) create mode 100644 .gitignore create mode 100644 VAR.ExpressionEvaluator.Tests/Properties/AssemblyInfo.cs create mode 100644 VAR.ExpressionEvaluator.Tests/TokenizerTests.cs create mode 100644 VAR.ExpressionEvaluator.Tests/VAR.ExpressionEvaluator.Tests.csproj create mode 100644 VAR.ExpressionEvaluator.sln create mode 100644 VAR.ExpressionEvaluator/Properties/AssemblyInfo.cs create mode 100644 VAR.ExpressionEvaluator/Tokenizer.cs create mode 100644 VAR.ExpressionEvaluator/VAR.ExpressionEvaluator.csproj diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a01cabe --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +/.vs/* +*/bin/* +*/obj/* diff --git a/VAR.ExpressionEvaluator.Tests/Properties/AssemblyInfo.cs b/VAR.ExpressionEvaluator.Tests/Properties/AssemblyInfo.cs new file mode 100644 index 0000000..aef7bdf --- /dev/null +++ b/VAR.ExpressionEvaluator.Tests/Properties/AssemblyInfo.cs @@ -0,0 +1,15 @@ +using System.Reflection; +using System.Runtime.InteropServices; + +[assembly: AssemblyTitle("VAR.ExpressionEvaluator.Tests")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("VAR.ExpressionEvaluator.Tests")] +[assembly: AssemblyCopyright("Copyright © VAR 2019")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] +[assembly: ComVisible(false)] +[assembly: Guid("3195f0e0-9489-49ce-8fc2-627a73f80cc7")] +[assembly: AssemblyVersion("0.1.0.0")] +[assembly: AssemblyFileVersion("0.1.0.0")] diff --git a/VAR.ExpressionEvaluator.Tests/TokenizerTests.cs b/VAR.ExpressionEvaluator.Tests/TokenizerTests.cs new file mode 100644 index 0000000..aef37aa --- /dev/null +++ b/VAR.ExpressionEvaluator.Tests/TokenizerTests.cs @@ -0,0 +1,178 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; +using System.IO; + +namespace VAR.ExpressionEvaluator.Tests +{ + [TestClass()] + public class TokenizerTests + { + [TestMethod()] + public void Tokenizer__Plus() + { + var testString = "10 + 20"; + var t = new Tokenizer(new StringReader(testString)); + + // "10" + Assert.AreEqual(t.Token, Token.Number); + Assert.AreEqual(t.Number, 10); + t.NextToken(); + + // "+" + Assert.AreEqual(t.Token, Token.Plus); + t.NextToken(); + + // "20" + Assert.AreEqual(t.Token, Token.Number); + Assert.AreEqual(t.Number, 20); + t.NextToken(); + + Assert.AreEqual(t.Token, Token.EOF); + } + + [TestMethod()] + public void Tokenizer__PlusMinusAndDecimal() + { + var testString = "10 + 20 - 30.123"; + var t = new Tokenizer(new StringReader(testString)); + + // "10" + Assert.AreEqual(t.Token, Token.Number); + Assert.AreEqual(t.Number, 10); + t.NextToken(); + + // "+" + Assert.AreEqual(t.Token, Token.Plus); + t.NextToken(); + + // "20" + Assert.AreEqual(t.Token, Token.Number); + Assert.AreEqual(t.Number, 20); + t.NextToken(); + + // "-" + Assert.AreEqual(t.Token, Token.Minus); + t.NextToken(); + + // "20" + Assert.AreEqual(t.Token, Token.Number); + Assert.AreEqual(t.Number, 30.123m); + t.NextToken(); + + Assert.AreEqual(t.Token, Token.EOF); + } + + [TestMethod()] + public void Tokenizer__SimpleString() + { + var testString = "\"Hello World\""; + var t = new Tokenizer(new StringReader(testString)); + + // "Hello World" + Assert.AreEqual(t.Token, Token.String); + Assert.AreEqual(t.Text, "Hello World"); + t.NextToken(); + + Assert.AreEqual(t.Token, Token.EOF); + } + + [TestMethod()] + public void Tokenizer__StringWithEscaping() + { + var testString = "\"Hello \\\"World\\\"\""; + var t = new Tokenizer(new StringReader(testString)); + + // "Hello \"World\"" + Assert.AreEqual(t.Token, Token.String); + Assert.AreEqual(t.Text, "Hello \"World\""); + t.NextToken(); + + Assert.AreEqual(t.Token, Token.EOF); + } + + [TestMethod()] + public void Tokenizer__Keywords() + { + var testString = "null true false"; + var t = new Tokenizer(new StringReader(testString)); + + // "null" + Assert.AreEqual(t.Token, Token.Keyword); + Assert.AreEqual(t.Text, "null"); + t.NextToken(); + + // "true" + Assert.AreEqual(t.Token, Token.Keyword); + Assert.AreEqual(t.Text, "true"); + t.NextToken(); + + // "false" + Assert.AreEqual(t.Token, Token.Keyword); + Assert.AreEqual(t.Text, "false"); + t.NextToken(); + + Assert.AreEqual(t.Token, Token.EOF); + } + + [TestMethod()] + public void Tokenizer__AllTogether() + { + var testString = "(10 + 20) * -30.123 + \"Hello \\\"World\\\"\" = false"; + var t = new Tokenizer(new StringReader(testString)); + + // "(" + Assert.AreEqual(t.Token, Token.ParentesisStart); + t.NextToken(); + + // "10" + Assert.AreEqual(t.Token, Token.Number); + Assert.AreEqual(t.Number, 10); + t.NextToken(); + + // "+" + Assert.AreEqual(t.Token, Token.Plus); + t.NextToken(); + + // "20" + Assert.AreEqual(t.Token, Token.Number); + Assert.AreEqual(t.Number, 20); + t.NextToken(); + + // ")" + Assert.AreEqual(t.Token, Token.ParentesisEnd); + t.NextToken(); + + // "*" + Assert.AreEqual(t.Token, Token.Multiply); + t.NextToken(); + + // "-" + Assert.AreEqual(t.Token, Token.Minus); + t.NextToken(); + + // "20" + Assert.AreEqual(t.Token, Token.Number); + Assert.AreEqual(t.Number, 30.123m); + t.NextToken(); + + // "+" + Assert.AreEqual(t.Token, Token.Plus); + t.NextToken(); + + // "Hello \"World\"" + Assert.AreEqual(t.Token, Token.String); + Assert.AreEqual(t.Text, "Hello \"World\""); + t.NextToken(); + + // "=" + Assert.AreEqual(t.Token, Token.Equals); + t.NextToken(); + + // "false" + Assert.AreEqual(t.Token, Token.Keyword); + Assert.AreEqual(t.Text, "false"); + t.NextToken(); + + Assert.AreEqual(t.Token, Token.EOF); + } + } +} \ No newline at end of file diff --git a/VAR.ExpressionEvaluator.Tests/VAR.ExpressionEvaluator.Tests.csproj b/VAR.ExpressionEvaluator.Tests/VAR.ExpressionEvaluator.Tests.csproj new file mode 100644 index 0000000..2954f8f --- /dev/null +++ b/VAR.ExpressionEvaluator.Tests/VAR.ExpressionEvaluator.Tests.csproj @@ -0,0 +1,90 @@ + + + + Debug + AnyCPU + {3195F0E0-9489-49CE-8FC2-627A73F80CC7} + Library + Properties + VAR.ExpressionEvaluator.Tests + VAR.ExpressionEvaluator.Tests + v4.6.1 + 512 + {3AC096D0-A1C2-E12C-1390-A8335801FDAB};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC} + 10.0 + $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion) + $(ProgramFiles)\Common Files\microsoft shared\VSTT\$(VisualStudioVersion)\UITestExtensionPackages + False + UnitTest + + + + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + + + + + + + + + + + + + + + + + + + + + {74635F68-55B1-4819-84A3-9EA818D396D9} + VAR.ExpressionEvaluator + + + + + + + False + + + False + + + False + + + False + + + + + + + + \ No newline at end of file diff --git a/VAR.ExpressionEvaluator.sln b/VAR.ExpressionEvaluator.sln new file mode 100644 index 0000000..e6ee684 --- /dev/null +++ b/VAR.ExpressionEvaluator.sln @@ -0,0 +1,31 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 15 +VisualStudioVersion = 15.0.28307.902 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "VAR.ExpressionEvaluator", "VAR.ExpressionEvaluator\VAR.ExpressionEvaluator.csproj", "{74635F68-55B1-4819-84A3-9EA818D396D9}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "VAR.ExpressionEvaluator.Tests", "VAR.ExpressionEvaluator.Tests\VAR.ExpressionEvaluator.Tests.csproj", "{3195F0E0-9489-49CE-8FC2-627A73F80CC7}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {74635F68-55B1-4819-84A3-9EA818D396D9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {74635F68-55B1-4819-84A3-9EA818D396D9}.Debug|Any CPU.Build.0 = Debug|Any CPU + {74635F68-55B1-4819-84A3-9EA818D396D9}.Release|Any CPU.ActiveCfg = Release|Any CPU + {74635F68-55B1-4819-84A3-9EA818D396D9}.Release|Any CPU.Build.0 = Release|Any CPU + {3195F0E0-9489-49CE-8FC2-627A73F80CC7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3195F0E0-9489-49CE-8FC2-627A73F80CC7}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3195F0E0-9489-49CE-8FC2-627A73F80CC7}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3195F0E0-9489-49CE-8FC2-627A73F80CC7}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {E00226E1-0D92-4FA3-8164-79D4F9B6320C} + EndGlobalSection +EndGlobal diff --git a/VAR.ExpressionEvaluator/Properties/AssemblyInfo.cs b/VAR.ExpressionEvaluator/Properties/AssemblyInfo.cs new file mode 100644 index 0000000..57bf608 --- /dev/null +++ b/VAR.ExpressionEvaluator/Properties/AssemblyInfo.cs @@ -0,0 +1,15 @@ +using System.Reflection; +using System.Runtime.InteropServices; + +[assembly: AssemblyTitle("VAR.ExpressionEvaluator")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("VAR.ExpressionEvaluator")] +[assembly: AssemblyCopyright("Copyright © VAR 2019")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] +[assembly: ComVisible(false)] +[assembly: Guid("74635f68-55b1-4819-84a3-9ea818d396d9")] +[assembly: AssemblyVersion("0.1.0.0")] +[assembly: AssemblyFileVersion("0.1.0.0")] diff --git a/VAR.ExpressionEvaluator/Tokenizer.cs b/VAR.ExpressionEvaluator/Tokenizer.cs new file mode 100644 index 0000000..f63562c --- /dev/null +++ b/VAR.ExpressionEvaluator/Tokenizer.cs @@ -0,0 +1,200 @@ +using System.Globalization; +using System.IO; +using System.Text; + +namespace VAR.ExpressionEvaluator +{ + public enum Token + { + EOF, + Plus, + Minus, + Division, + Multiply, + Equals, + ParentesisStart, + ParentesisEnd, + Keyword, + String, + Number, + } + + public class Tokenizer + { + private TextReader _reader; + private int _currentPosition = 0; + private char _currentChar; + private Token _currentToken; + private string _text; + private decimal? _number; + + + public Tokenizer(TextReader reader) + { + _reader = reader; + _currentPosition = -1; + NextChar(); + NextToken(); + } + + public Token Token + { + get { return _currentToken; } + } + + public string Text + { + get { return _text; } + } + + public decimal? Number + { + get { return _number; } + } + + private void NextChar() + { + int ch = _reader.Read(); + if (ch < 0) + { + _currentChar = '\0'; + return; + } + _currentChar = (char)ch; + _currentPosition++; + } + + private void SkipWhite() + { + while (char.IsWhiteSpace(_currentChar)) + { + NextChar(); + } + } + + public void NextToken() + { + _currentToken = Token.EOF; + _text = null; + _number = null; + + SkipWhite(); + + // Special characters + switch (_currentChar) + { + case '\0': + return; + + case '+': + NextChar(); + _currentToken = Token.Plus; + return; + + case '-': + NextChar(); + _currentToken = Token.Minus; + return; + + case '/': + NextChar(); + _currentToken = Token.Division; + return; + + case '*': + NextChar(); + _currentToken = Token.Multiply; + return; + + case '(': + NextChar(); + _currentToken = Token.ParentesisStart; + return; + + case ')': + NextChar(); + _currentToken = Token.ParentesisEnd; + return; + + case '=': + NextChar(); + _currentToken = Token.Equals; + return; + } + + // Keywords + if (char.IsLetter(_currentChar)) + { + var sb = new StringBuilder(); + while (char.IsLetterOrDigit(_currentChar) || _currentChar == '_') + { + sb.Append(_currentChar); + NextChar(); + if (_currentChar == '\0') { break; } + } + _text = sb.ToString(); + _currentToken = Token.Keyword; + return; + } + + // String + if (_currentChar == '"' || _currentChar == '\'') + { + char stringEndsWith = _currentChar; + NextChar(); + StringBuilder sbString = new StringBuilder(); + while (_currentChar != stringEndsWith && _currentChar != '\0') + { + if (_currentChar != '\\') + { + sbString.Append(_currentChar); + } + else + { + NextChar(); + if (_currentChar == '\\') + { + sbString.Append('\\'); + } + else if (_currentChar == 't') + { + sbString.Append('\t'); + } + else if (_currentChar == 'n') + { + sbString.Append('\n'); + } + else + { + // FIXME: Other escaped characters + sbString.Append(_currentChar); + } + } + NextChar(); + } + NextChar(); + _text = sbString.ToString(); + _currentToken = Token.String; + return; + } + + // Numbers + if (char.IsDigit(_currentChar) || _currentChar == '.') + { + var sbNumber = new StringBuilder(); + bool haveDecimalPoint = false; + while (char.IsDigit(_currentChar) || (!haveDecimalPoint && _currentChar == '.')) + { + sbNumber.Append(_currentChar); + haveDecimalPoint = _currentChar == '.'; + NextChar(); + } + _number = decimal.Parse(sbNumber.ToString(), CultureInfo.InvariantCulture); + _currentToken = Token.Number; + return; + } + + throw new InvalidDataException(string.Format("Unexpected character: {0} at {1}", _currentChar, _currentPosition)); + } + } +} diff --git a/VAR.ExpressionEvaluator/VAR.ExpressionEvaluator.csproj b/VAR.ExpressionEvaluator/VAR.ExpressionEvaluator.csproj new file mode 100644 index 0000000..6d97afc --- /dev/null +++ b/VAR.ExpressionEvaluator/VAR.ExpressionEvaluator.csproj @@ -0,0 +1,48 @@ + + + + + Debug + AnyCPU + {74635F68-55B1-4819-84A3-9EA818D396D9} + Library + Properties + VAR.ExpressionEvaluator + VAR.ExpressionEvaluator + v4.6.1 + 512 + true + + + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + + + + + + + + + + + + + + + \ No newline at end of file