VAR/VAR.ExpressionEvaluator
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Tokenizer.

Browse Source
This commit is contained in:
Valeriano A.R.
2019-12-01 01:08:11 +01:00
commit 2bd7e89cb3
8 changed files with 580 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored Normal file
View File

@@ -0,0 +1,3 @@
/.vs/*
*/bin/*
*/obj/*

15
VAR.ExpressionEvaluator.Tests/Properties/AssemblyInfo.cs Normal file
View File

@@ -0,0 +1,15 @@
using System.Reflection;
using System.Runtime.InteropServices;
[assembly: AssemblyTitle("VAR.ExpressionEvaluator.Tests")]
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("")]
[assembly: AssemblyProduct("VAR.ExpressionEvaluator.Tests")]
[assembly: AssemblyCopyright("Copyright © VAR 2019")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
[assembly: ComVisible(false)]
[assembly: Guid("3195f0e0-9489-49ce-8fc2-627a73f80cc7")]
[assembly: AssemblyVersion("0.1.0.0")]
[assembly: AssemblyFileVersion("0.1.0.0")]

178
VAR.ExpressionEvaluator.Tests/TokenizerTests.cs Normal file
View File

@@ -0,0 +1,178 @@
using Microsoft.VisualStudio.TestTools.UnitTesting;
using System.IO;
namespace VAR.ExpressionEvaluator.Tests
{
[TestClass()]
public class TokenizerTests
{
[TestMethod()]
public void Tokenizer__Plus()
{
var testString = "10 + 20";
var t = new Tokenizer(new StringReader(testString));
// "10"
Assert.AreEqual(t.Token, Token.Number);
Assert.AreEqual(t.Number, 10);
t.NextToken();
// "+"
Assert.AreEqual(t.Token, Token.Plus);
t.NextToken();
// "20"
Assert.AreEqual(t.Token, Token.Number);
Assert.AreEqual(t.Number, 20);
t.NextToken();
Assert.AreEqual(t.Token, Token.EOF);
}
[TestMethod()]
public void Tokenizer__PlusMinusAndDecimal()
{
var testString = "10 + 20 - 30.123";
var t = new Tokenizer(new StringReader(testString));
// "10"
Assert.AreEqual(t.Token, Token.Number);
Assert.AreEqual(t.Number, 10);
t.NextToken();
// "+"
Assert.AreEqual(t.Token, Token.Plus);
t.NextToken();
// "20"
Assert.AreEqual(t.Token, Token.Number);
Assert.AreEqual(t.Number, 20);
t.NextToken();
// "-"
Assert.AreEqual(t.Token, Token.Minus);
t.NextToken();
// "20"
Assert.AreEqual(t.Token, Token.Number);
Assert.AreEqual(t.Number, 30.123m);
t.NextToken();
Assert.AreEqual(t.Token, Token.EOF);
}
[TestMethod()]
public void Tokenizer__SimpleString()
{
var testString = "\"Hello World\"";
var t = new Tokenizer(new StringReader(testString));
// "Hello World"
Assert.AreEqual(t.Token, Token.String);
Assert.AreEqual(t.Text, "Hello World");
t.NextToken();
Assert.AreEqual(t.Token, Token.EOF);
}
[TestMethod()]
public void Tokenizer__StringWithEscaping()
{
var testString = "\"Hello \\\"World\\\"\"";
var t = new Tokenizer(new StringReader(testString));
// "Hello \"World\""
Assert.AreEqual(t.Token, Token.String);
Assert.AreEqual(t.Text, "Hello \"World\"");
t.NextToken();
Assert.AreEqual(t.Token, Token.EOF);
}
[TestMethod()]
public void Tokenizer__Keywords()
{
var testString = "null true false";
var t = new Tokenizer(new StringReader(testString));
// "null"
Assert.AreEqual(t.Token, Token.Keyword);
Assert.AreEqual(t.Text, "null");
t.NextToken();
// "true"
Assert.AreEqual(t.Token, Token.Keyword);
Assert.AreEqual(t.Text, "true");
t.NextToken();
// "false"
Assert.AreEqual(t.Token, Token.Keyword);
Assert.AreEqual(t.Text, "false");
t.NextToken();
Assert.AreEqual(t.Token, Token.EOF);
}
[TestMethod()]
public void Tokenizer__AllTogether()
{
var testString = "(10 + 20) * -30.123 + \"Hello \\\"World\\\"\" = false";
var t = new Tokenizer(new StringReader(testString));
// "("
Assert.AreEqual(t.Token, Token.ParentesisStart);
t.NextToken();
// "10"
Assert.AreEqual(t.Token, Token.Number);
Assert.AreEqual(t.Number, 10);
t.NextToken();
// "+"
Assert.AreEqual(t.Token, Token.Plus);
t.NextToken();
// "20"
Assert.AreEqual(t.Token, Token.Number);
Assert.AreEqual(t.Number, 20);
t.NextToken();
// ")"
Assert.AreEqual(t.Token, Token.ParentesisEnd);
t.NextToken();
// "*"
Assert.AreEqual(t.Token, Token.Multiply);
t.NextToken();
// "-"
Assert.AreEqual(t.Token, Token.Minus);
t.NextToken();
// "20"
Assert.AreEqual(t.Token, Token.Number);
Assert.AreEqual(t.Number, 30.123m);
t.NextToken();
// "+"
Assert.AreEqual(t.Token, Token.Plus);
t.NextToken();
// "Hello \"World\""
Assert.AreEqual(t.Token, Token.String);
Assert.AreEqual(t.Text, "Hello \"World\"");
t.NextToken();
// "="
Assert.AreEqual(t.Token, Token.Equals);
t.NextToken();
// "false"
Assert.AreEqual(t.Token, Token.Keyword);
Assert.AreEqual(t.Text, "false");
t.NextToken();
Assert.AreEqual(t.Token, Token.EOF);
}
}
}

90
VAR.ExpressionEvaluator.Tests/VAR.ExpressionEvaluator.Tests.csproj Normal file
View File

@@ -0,0 +1,90 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="15.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
<ProjectGuid>{3195F0E0-9489-49CE-8FC2-627A73F80CC7}</ProjectGuid>
<OutputType>Library</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>VAR.ExpressionEvaluator.Tests</RootNamespace>
<AssemblyName>VAR.ExpressionEvaluator.Tests</AssemblyName>
<TargetFrameworkVersion>v4.6.1</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment>
<ProjectTypeGuids>{3AC096D0-A1C2-E12C-1390-A8335801FDAB};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}</ProjectTypeGuids>
<VisualStudioVersion Condition="'$(VisualStudioVersion)' == ''">10.0</VisualStudioVersion>
<VSToolsPath Condition="'$(VSToolsPath)' == ''">$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)</VSToolsPath>
<ReferencePath>$(ProgramFiles)\Common Files\microsoft shared\VSTT\$(VisualStudioVersion)\UITestExtensionPackages</ReferencePath>
<IsCodedUITest>False</IsCodedUITest>
<TestProjectType>UnitTest</TestProjectType>
<TargetFrameworkProfile />
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
<OutputPath>bin\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<DebugType>pdbonly</DebugType>
<Optimize>true</Optimize>
<OutputPath>bin\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<ItemGroup>
<Reference Include="System" />
</ItemGroup>
<Choose>
<When Condition="('$(VisualStudioVersion)' == '10.0' or '$(VisualStudioVersion)' == '') and '$(TargetFrameworkVersion)' == 'v3.5'">
<ItemGroup>
<Reference Include="Microsoft.VisualStudio.QualityTools.UnitTestFramework, Version=10.1.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL" />
</ItemGroup>
</When>
<Otherwise>
<ItemGroup>
<Reference Include="Microsoft.VisualStudio.QualityTools.UnitTestFramework" />
</ItemGroup>
</Otherwise>
</Choose>
<ItemGroup>
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="TokenizerTests.cs" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\VAR.ExpressionEvaluator\VAR.ExpressionEvaluator.csproj">
<Project>{74635F68-55B1-4819-84A3-9EA818D396D9}</Project>
<Name>VAR.ExpressionEvaluator</Name>
</ProjectReference>
</ItemGroup>
<Choose>
<When Condition="'$(VisualStudioVersion)' == '10.0' And '$(IsCodedUITest)' == 'True'">
<ItemGroup>
<Reference Include="Microsoft.VisualStudio.QualityTools.CodedUITestFramework, Version=10.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<Private>False</Private>
</Reference>
<Reference Include="Microsoft.VisualStudio.TestTools.UITest.Common, Version=10.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<Private>False</Private>
</Reference>
<Reference Include="Microsoft.VisualStudio.TestTools.UITest.Extension, Version=10.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<Private>False</Private>
</Reference>
<Reference Include="Microsoft.VisualStudio.TestTools.UITesting, Version=10.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<Private>False</Private>
</Reference>
</ItemGroup>
</When>
</Choose>
<Import Project="$(VSToolsPath)\TeamTest\Microsoft.TestTools.targets" Condition="Exists('$(VSToolsPath)\TeamTest\Microsoft.TestTools.targets')" />
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild">
</Target>
<Target Name="AfterBuild">
</Target>
-->
</Project>

31
VAR.ExpressionEvaluator.sln Normal file
View File

@@ -0,0 +1,31 @@
Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio 15
VisualStudioVersion = 15.0.28307.902
MinimumVisualStudioVersion = 10.0.40219.1
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "VAR.ExpressionEvaluator", "VAR.ExpressionEvaluator\VAR.ExpressionEvaluator.csproj", "{74635F68-55B1-4819-84A3-9EA818D396D9}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "VAR.ExpressionEvaluator.Tests", "VAR.ExpressionEvaluator.Tests\VAR.ExpressionEvaluator.Tests.csproj", "{3195F0E0-9489-49CE-8FC2-627A73F80CC7}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Release|Any CPU = Release|Any CPU
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{74635F68-55B1-4819-84A3-9EA818D396D9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{74635F68-55B1-4819-84A3-9EA818D396D9}.Debug|Any CPU.Build.0 = Debug|Any CPU
{74635F68-55B1-4819-84A3-9EA818D396D9}.Release|Any CPU.ActiveCfg = Release|Any CPU
{74635F68-55B1-4819-84A3-9EA818D396D9}.Release|Any CPU.Build.0 = Release|Any CPU
{3195F0E0-9489-49CE-8FC2-627A73F80CC7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{3195F0E0-9489-49CE-8FC2-627A73F80CC7}.Debug|Any CPU.Build.0 = Debug|Any CPU
{3195F0E0-9489-49CE-8FC2-627A73F80CC7}.Release|Any CPU.ActiveCfg = Release|Any CPU
{3195F0E0-9489-49CE-8FC2-627A73F80CC7}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {E00226E1-0D92-4FA3-8164-79D4F9B6320C}
EndGlobalSection
EndGlobal

15
VAR.ExpressionEvaluator/Properties/AssemblyInfo.cs Normal file
View File

@@ -0,0 +1,15 @@
using System.Reflection;
using System.Runtime.InteropServices;
[assembly: AssemblyTitle("VAR.ExpressionEvaluator")]
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("")]
[assembly: AssemblyProduct("VAR.ExpressionEvaluator")]
[assembly: AssemblyCopyright("Copyright © VAR 2019")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
[assembly: ComVisible(false)]
[assembly: Guid("74635f68-55b1-4819-84a3-9ea818d396d9")]
[assembly: AssemblyVersion("0.1.0.0")]
[assembly: AssemblyFileVersion("0.1.0.0")]

200
VAR.ExpressionEvaluator/Tokenizer.cs Normal file
View File

@@ -0,0 +1,200 @@
using System.Globalization;
using System.IO;
using System.Text;
namespace VAR.ExpressionEvaluator
{
public enum Token
{
EOF,
Plus,
Minus,
Division,
Multiply,
Equals,
ParentesisStart,
ParentesisEnd,
Keyword,
String,
Number,
}
public class Tokenizer
{
private TextReader _reader;
private int _currentPosition = 0;
private char _currentChar;
private Token _currentToken;
private string _text;
private decimal? _number;
public Tokenizer(TextReader reader)
{
_reader = reader;
_currentPosition = -1;
NextChar();
NextToken();
}
public Token Token
{
get { return _currentToken; }
}
public string Text
{
get { return _text; }
}
public decimal? Number
{
get { return _number; }
}
private void NextChar()
{
int ch = _reader.Read();
if (ch < 0)
{
_currentChar = '\0';
return;
}
_currentChar = (char)ch;
_currentPosition++;
}
private void SkipWhite()
{
while (char.IsWhiteSpace(_currentChar))
{
NextChar();
}
}
public void NextToken()
{
_currentToken = Token.EOF;
_text = null;
_number = null;
SkipWhite();
// Special characters
switch (_currentChar)
{
case '\0':
return;
case '+':
NextChar();
_currentToken = Token.Plus;
return;
case '-':
NextChar();
_currentToken = Token.Minus;
return;
case '/':
NextChar();
_currentToken = Token.Division;
return;
case '*':
NextChar();
_currentToken = Token.Multiply;
return;
case '(':
NextChar();
_currentToken = Token.ParentesisStart;
return;
case ')':
NextChar();
_currentToken = Token.ParentesisEnd;
return;
case '=':
NextChar();
_currentToken = Token.Equals;
return;
}
// Keywords
if (char.IsLetter(_currentChar))
{
var sb = new StringBuilder();
while (char.IsLetterOrDigit(_currentChar) || _currentChar == '_')
{
sb.Append(_currentChar);
NextChar();
if (_currentChar == '\0') { break; }
}
_text = sb.ToString();
_currentToken = Token.Keyword;
return;
}
// String
if (_currentChar == '"' || _currentChar == '\'')
{
char stringEndsWith = _currentChar;
NextChar();
StringBuilder sbString = new StringBuilder();
while (_currentChar != stringEndsWith && _currentChar != '\0')
{
if (_currentChar != '\\')
{
sbString.Append(_currentChar);
}
else
{
NextChar();
if (_currentChar == '\\')
{
sbString.Append('\\');
}
else if (_currentChar == 't')
{
sbString.Append('\t');
}
else if (_currentChar == 'n')
{
sbString.Append('\n');
}
else
{
// FIXME: Other escaped characters
sbString.Append(_currentChar);
}
}
NextChar();
}
NextChar();
_text = sbString.ToString();
_currentToken = Token.String;
return;
}
// Numbers
if (char.IsDigit(_currentChar) || _currentChar == '.')
{
var sbNumber = new StringBuilder();
bool haveDecimalPoint = false;
while (char.IsDigit(_currentChar) || (!haveDecimalPoint && _currentChar == '.'))
{
sbNumber.Append(_currentChar);
haveDecimalPoint = _currentChar == '.';
NextChar();
}
_number = decimal.Parse(sbNumber.ToString(), CultureInfo.InvariantCulture);
_currentToken = Token.Number;
return;
}
throw new InvalidDataException(string.Format("Unexpected character: {0} at {1}", _currentChar, _currentPosition));
}
}
}

48
VAR.ExpressionEvaluator/VAR.ExpressionEvaluator.csproj Normal file
View File

@@ -0,0 +1,48 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
<ProjectGuid>{74635F68-55B1-4819-84A3-9EA818D396D9}</ProjectGuid>
<OutputType>Library</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>VAR.ExpressionEvaluator</RootNamespace>
<AssemblyName>VAR.ExpressionEvaluator</AssemblyName>
<TargetFrameworkVersion>v4.6.1</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment>
<Deterministic>true</Deterministic>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
<OutputPath>bin\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<DebugType>pdbonly</DebugType>
<Optimize>true</Optimize>
<OutputPath>bin\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<ItemGroup>
<Reference Include="System" />
<Reference Include="System.Core" />
<Reference Include="System.Xml.Linq" />
<Reference Include="System.Data.DataSetExtensions" />
<Reference Include="Microsoft.CSharp" />
<Reference Include="System.Data" />
<Reference Include="System.Net.Http" />
<Reference Include="System.Xml" />
</ItemGroup>
<ItemGroup>
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="Tokenizer.cs" />
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
</Project>