Tokenizer.
This commit is contained in:
15
VAR.ExpressionEvaluator/Properties/AssemblyInfo.cs
Normal file
15
VAR.ExpressionEvaluator/Properties/AssemblyInfo.cs
Normal file
@@ -0,0 +1,15 @@
|
||||
using System.Reflection;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
[assembly: AssemblyTitle("VAR.ExpressionEvaluator")]
|
||||
[assembly: AssemblyDescription("")]
|
||||
[assembly: AssemblyConfiguration("")]
|
||||
[assembly: AssemblyCompany("")]
|
||||
[assembly: AssemblyProduct("VAR.ExpressionEvaluator")]
|
||||
[assembly: AssemblyCopyright("Copyright © VAR 2019")]
|
||||
[assembly: AssemblyTrademark("")]
|
||||
[assembly: AssemblyCulture("")]
|
||||
[assembly: ComVisible(false)]
|
||||
[assembly: Guid("74635f68-55b1-4819-84a3-9ea818d396d9")]
|
||||
[assembly: AssemblyVersion("0.1.0.0")]
|
||||
[assembly: AssemblyFileVersion("0.1.0.0")]
|
||||
200
VAR.ExpressionEvaluator/Tokenizer.cs
Normal file
200
VAR.ExpressionEvaluator/Tokenizer.cs
Normal file
@@ -0,0 +1,200 @@
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using System.Text;
|
||||
|
||||
namespace VAR.ExpressionEvaluator
|
||||
{
|
||||
public enum Token
|
||||
{
|
||||
EOF,
|
||||
Plus,
|
||||
Minus,
|
||||
Division,
|
||||
Multiply,
|
||||
Equals,
|
||||
ParentesisStart,
|
||||
ParentesisEnd,
|
||||
Keyword,
|
||||
String,
|
||||
Number,
|
||||
}
|
||||
|
||||
public class Tokenizer
|
||||
{
|
||||
private TextReader _reader;
|
||||
private int _currentPosition = 0;
|
||||
private char _currentChar;
|
||||
private Token _currentToken;
|
||||
private string _text;
|
||||
private decimal? _number;
|
||||
|
||||
|
||||
public Tokenizer(TextReader reader)
|
||||
{
|
||||
_reader = reader;
|
||||
_currentPosition = -1;
|
||||
NextChar();
|
||||
NextToken();
|
||||
}
|
||||
|
||||
public Token Token
|
||||
{
|
||||
get { return _currentToken; }
|
||||
}
|
||||
|
||||
public string Text
|
||||
{
|
||||
get { return _text; }
|
||||
}
|
||||
|
||||
public decimal? Number
|
||||
{
|
||||
get { return _number; }
|
||||
}
|
||||
|
||||
private void NextChar()
|
||||
{
|
||||
int ch = _reader.Read();
|
||||
if (ch < 0)
|
||||
{
|
||||
_currentChar = '\0';
|
||||
return;
|
||||
}
|
||||
_currentChar = (char)ch;
|
||||
_currentPosition++;
|
||||
}
|
||||
|
||||
private void SkipWhite()
|
||||
{
|
||||
while (char.IsWhiteSpace(_currentChar))
|
||||
{
|
||||
NextChar();
|
||||
}
|
||||
}
|
||||
|
||||
public void NextToken()
|
||||
{
|
||||
_currentToken = Token.EOF;
|
||||
_text = null;
|
||||
_number = null;
|
||||
|
||||
SkipWhite();
|
||||
|
||||
// Special characters
|
||||
switch (_currentChar)
|
||||
{
|
||||
case '\0':
|
||||
return;
|
||||
|
||||
case '+':
|
||||
NextChar();
|
||||
_currentToken = Token.Plus;
|
||||
return;
|
||||
|
||||
case '-':
|
||||
NextChar();
|
||||
_currentToken = Token.Minus;
|
||||
return;
|
||||
|
||||
case '/':
|
||||
NextChar();
|
||||
_currentToken = Token.Division;
|
||||
return;
|
||||
|
||||
case '*':
|
||||
NextChar();
|
||||
_currentToken = Token.Multiply;
|
||||
return;
|
||||
|
||||
case '(':
|
||||
NextChar();
|
||||
_currentToken = Token.ParentesisStart;
|
||||
return;
|
||||
|
||||
case ')':
|
||||
NextChar();
|
||||
_currentToken = Token.ParentesisEnd;
|
||||
return;
|
||||
|
||||
case '=':
|
||||
NextChar();
|
||||
_currentToken = Token.Equals;
|
||||
return;
|
||||
}
|
||||
|
||||
// Keywords
|
||||
if (char.IsLetter(_currentChar))
|
||||
{
|
||||
var sb = new StringBuilder();
|
||||
while (char.IsLetterOrDigit(_currentChar) || _currentChar == '_')
|
||||
{
|
||||
sb.Append(_currentChar);
|
||||
NextChar();
|
||||
if (_currentChar == '\0') { break; }
|
||||
}
|
||||
_text = sb.ToString();
|
||||
_currentToken = Token.Keyword;
|
||||
return;
|
||||
}
|
||||
|
||||
// String
|
||||
if (_currentChar == '"' || _currentChar == '\'')
|
||||
{
|
||||
char stringEndsWith = _currentChar;
|
||||
NextChar();
|
||||
StringBuilder sbString = new StringBuilder();
|
||||
while (_currentChar != stringEndsWith && _currentChar != '\0')
|
||||
{
|
||||
if (_currentChar != '\\')
|
||||
{
|
||||
sbString.Append(_currentChar);
|
||||
}
|
||||
else
|
||||
{
|
||||
NextChar();
|
||||
if (_currentChar == '\\')
|
||||
{
|
||||
sbString.Append('\\');
|
||||
}
|
||||
else if (_currentChar == 't')
|
||||
{
|
||||
sbString.Append('\t');
|
||||
}
|
||||
else if (_currentChar == 'n')
|
||||
{
|
||||
sbString.Append('\n');
|
||||
}
|
||||
else
|
||||
{
|
||||
// FIXME: Other escaped characters
|
||||
sbString.Append(_currentChar);
|
||||
}
|
||||
}
|
||||
NextChar();
|
||||
}
|
||||
NextChar();
|
||||
_text = sbString.ToString();
|
||||
_currentToken = Token.String;
|
||||
return;
|
||||
}
|
||||
|
||||
// Numbers
|
||||
if (char.IsDigit(_currentChar) || _currentChar == '.')
|
||||
{
|
||||
var sbNumber = new StringBuilder();
|
||||
bool haveDecimalPoint = false;
|
||||
while (char.IsDigit(_currentChar) || (!haveDecimalPoint && _currentChar == '.'))
|
||||
{
|
||||
sbNumber.Append(_currentChar);
|
||||
haveDecimalPoint = _currentChar == '.';
|
||||
NextChar();
|
||||
}
|
||||
_number = decimal.Parse(sbNumber.ToString(), CultureInfo.InvariantCulture);
|
||||
_currentToken = Token.Number;
|
||||
return;
|
||||
}
|
||||
|
||||
throw new InvalidDataException(string.Format("Unexpected character: {0} at {1}", _currentChar, _currentPosition));
|
||||
}
|
||||
}
|
||||
}
|
||||
48
VAR.ExpressionEvaluator/VAR.ExpressionEvaluator.csproj
Normal file
48
VAR.ExpressionEvaluator/VAR.ExpressionEvaluator.csproj
Normal file
@@ -0,0 +1,48 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
|
||||
<PropertyGroup>
|
||||
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
|
||||
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
|
||||
<ProjectGuid>{74635F68-55B1-4819-84A3-9EA818D396D9}</ProjectGuid>
|
||||
<OutputType>Library</OutputType>
|
||||
<AppDesignerFolder>Properties</AppDesignerFolder>
|
||||
<RootNamespace>VAR.ExpressionEvaluator</RootNamespace>
|
||||
<AssemblyName>VAR.ExpressionEvaluator</AssemblyName>
|
||||
<TargetFrameworkVersion>v4.6.1</TargetFrameworkVersion>
|
||||
<FileAlignment>512</FileAlignment>
|
||||
<Deterministic>true</Deterministic>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
|
||||
<DebugSymbols>true</DebugSymbols>
|
||||
<DebugType>full</DebugType>
|
||||
<Optimize>false</Optimize>
|
||||
<OutputPath>bin\Debug\</OutputPath>
|
||||
<DefineConstants>DEBUG;TRACE</DefineConstants>
|
||||
<ErrorReport>prompt</ErrorReport>
|
||||
<WarningLevel>4</WarningLevel>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
|
||||
<DebugType>pdbonly</DebugType>
|
||||
<Optimize>true</Optimize>
|
||||
<OutputPath>bin\Release\</OutputPath>
|
||||
<DefineConstants>TRACE</DefineConstants>
|
||||
<ErrorReport>prompt</ErrorReport>
|
||||
<WarningLevel>4</WarningLevel>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<Reference Include="System" />
|
||||
<Reference Include="System.Core" />
|
||||
<Reference Include="System.Xml.Linq" />
|
||||
<Reference Include="System.Data.DataSetExtensions" />
|
||||
<Reference Include="Microsoft.CSharp" />
|
||||
<Reference Include="System.Data" />
|
||||
<Reference Include="System.Net.Http" />
|
||||
<Reference Include="System.Xml" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<Compile Include="Properties\AssemblyInfo.cs" />
|
||||
<Compile Include="Tokenizer.cs" />
|
||||
</ItemGroup>
|
||||
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
|
||||
</Project>
|
||||
Reference in New Issue
Block a user