CsvFieldIndexer: Add tests to Save and Load

This commit is contained in:
2024-02-18 16:30:45 +01:00
parent 9429751e65
commit 52cb729c0e
4 changed files with 153 additions and 81 deletions

View File

@@ -252,4 +252,34 @@ public class CsvFieldIndexerTests
#endregion Search #endregion Search
#region Save & Load
[Fact]
public void Save__TwoLinesWithTwoQuotedColumnsTwoMatchesSave__LoadsCorrectly()
{
// --- Arrange
StringReader sr = new(
"""
"Hélló","Wórld"
"Hélló","Wórld"
""");
// --- Act
CsvFieldIndexer indexer = new();
indexer.GenerateIndex(sr);
MemoryStream stream = new();
indexer.Save(stream);
byte[] savedData = stream.ToArray();
CsvFieldIndexer indexer2 = new();
MemoryStream stream2 = new(savedData);
bool loadResult = indexer2.Load(stream2);
// --- Assert
Assert.True(loadResult);
Assert.Equal(indexer.Index, indexer2.Index);
Assert.Equal(indexer.FieldIndex, indexer2.FieldIndex);
}
#endregion Save & Load
} }

View File

@@ -7,6 +7,10 @@ namespace CsvLib;
public class CsvFieldIndexer public class CsvFieldIndexer
{ {
#region Declarations
private const byte FileFormatVersion = 1;
private bool _insideString; private bool _insideString;
private Encoding _currentEncoding = Encoding.Default; private Encoding _currentEncoding = Encoding.Default;
@@ -15,6 +19,10 @@ public class CsvFieldIndexer
private readonly char _quoteChar; private readonly char _quoteChar;
private readonly char _escapeChar; private readonly char _escapeChar;
#endregion Declarations
#region Life cycle
public CsvFieldIndexer(char separator = ',', char quoteChar = '"', char escapeChar = '\\') public CsvFieldIndexer(char separator = ',', char quoteChar = '"', char escapeChar = '\\')
{ {
_separator = separator; _separator = separator;
@@ -22,6 +30,10 @@ public class CsvFieldIndexer
_escapeChar = escapeChar; _escapeChar = escapeChar;
} }
#endregion Life cycle
#region Properties
private List<long> _index = new(); private List<long> _index = new();
public List<long> Index { get { return _index; } } public List<long> Index { get { return _index; } }
@@ -30,6 +42,10 @@ public class CsvFieldIndexer
public List<List<long>> FieldIndex { get { return _fieldIndex; } } public List<List<long>> FieldIndex { get { return _fieldIndex; } }
#endregion Properties
#region Parsing
private void DummyParser(string line) private void DummyParser(string line)
{ {
for (int i = 0; i < line.Length; i++) for (int i = 0; i < line.Length; i++)
@@ -114,6 +130,10 @@ public class CsvFieldIndexer
return fieldPositions; return fieldPositions;
} }
#endregion Parsing
#region GenerateIndex
private void GenerateIndex(string file) private void GenerateIndex(string file)
{ {
using FileStream stream = new(file, FileMode.Open); using FileStream stream = new(file, FileMode.Open);
@@ -149,17 +169,14 @@ public class CsvFieldIndexer
} }
} }
private const byte FileFormatVersion = 1; #endregion GenerateIndex
private void SaveFile(string indexFile) #region Save
{
if (File.Exists(indexFile)) public void Save(Stream streamOut)
{
File.Delete(indexFile);
}
Stream streamOut = File.Open(indexFile, FileMode.Create);
using (BinaryWriter binWriter = new(streamOut))
{ {
using BinaryWriter binWriter = new(streamOut);
binWriter.Write((byte)'C'); binWriter.Write((byte)'C');
binWriter.Write((byte)'S'); binWriter.Write((byte)'S');
binWriter.Write((byte)'V'); binWriter.Write((byte)'V');
@@ -182,19 +199,23 @@ public class CsvFieldIndexer
} }
} }
} }
private void SaveFile(string indexFile)
{
if (File.Exists(indexFile))
{
File.Delete(indexFile);
}
Stream streamOut = File.Open(indexFile, FileMode.Create);
Save(streamOut);
streamOut.Close(); streamOut.Close();
} }
private bool LoadFile(string indexFile) #endregion Save
{
if (File.Exists(indexFile) == false) #region Load
{
return false; public bool Load(Stream streamIn)
}
List<long> tempIndex;
List<List<long>> tempFieldIndex;
Stream streamIn = File.Open(indexFile, FileMode.Open);
try
{ {
using BinaryReader binReader = new(streamIn); using BinaryReader binReader = new(streamIn);
@@ -207,7 +228,7 @@ public class CsvFieldIndexer
if (fileVersion != FileFormatVersion) { return false; } if (fileVersion != FileFormatVersion) { return false; }
int numIndexes = binReader.ReadInt32(); int numIndexes = binReader.ReadInt32();
tempIndex = new List<long>(numIndexes); List<long> tempIndex = new(numIndexes);
for (int i = 0; i < numIndexes; i++) for (int i = 0; i < numIndexes; i++)
{ {
long value = binReader.ReadInt64(); long value = binReader.ReadInt64();
@@ -215,7 +236,7 @@ public class CsvFieldIndexer
} }
int numFieldIndexes = binReader.ReadInt32(); int numFieldIndexes = binReader.ReadInt32();
tempFieldIndex = new List<List<long>>(numFieldIndexes); List<List<long>> tempFieldIndex = new(numFieldIndexes);
for (int j = 0; j < numFieldIndexes; j++) for (int j = 0; j < numFieldIndexes; j++)
{ {
int numCurrentFieldIndexes = binReader.ReadInt32(); int numCurrentFieldIndexes = binReader.ReadInt32();
@@ -227,6 +248,22 @@ public class CsvFieldIndexer
} }
tempFieldIndex.Add(currentFieldIndex); tempFieldIndex.Add(currentFieldIndex);
} }
_index = tempIndex;
_fieldIndex = tempFieldIndex;
return true;
}
private bool LoadFile(string indexFile)
{
if (File.Exists(indexFile) == false)
{
return false;
}
Stream streamIn = File.Open(indexFile, FileMode.Open);
try
{
if (Load(streamIn) == false) return false;
} }
catch (Exception) catch (Exception)
{ {
@@ -237,8 +274,6 @@ public class CsvFieldIndexer
{ {
streamIn.Close(); streamIn.Close();
} }
_index = tempIndex;
_fieldIndex = tempFieldIndex;
return true; return true;
} }
@@ -263,20 +298,9 @@ public class CsvFieldIndexer
} }
} }
public List<long> Search(string fileName, string textToSearch, Action<float>? notifyProgress = null) #endregion Load
{
List<long> index; #region Search
using FileStream streamIn = new(fileName, FileMode.Open);
try
{
index = Search(streamIn, textToSearch, notifyProgress);
}
finally
{
streamIn.Close();
}
return index;
}
public List<long> Search(Stream streamIn, string textToSearch, Action<float>? notifyProgress = null) public List<long> Search(Stream streamIn, string textToSearch, Action<float>? notifyProgress = null)
{ {
@@ -294,7 +318,7 @@ public class CsvFieldIndexer
if (tsElapsed.TotalMilliseconds > 200) if (tsElapsed.TotalMilliseconds > 200)
{ {
datePrevious = DateTime.UtcNow; datePrevious = DateTime.UtcNow;
notifyProgress?.Invoke(j/(float)_fieldIndex.Count); notifyProgress?.Invoke(j / (float)_fieldIndex.Count);
} }
long offset = _fieldIndex[j][i]; long offset = _fieldIndex[j][i];
@@ -318,4 +342,21 @@ public class CsvFieldIndexer
return newIndexes; return newIndexes;
} }
public List<long> SearchFile(string fileName, string textToSearch, Action<float>? notifyProgress = null)
{
List<long> index;
using FileStream streamIn = new(fileName, FileMode.Open);
try
{
index = Search(streamIn, textToSearch, notifyProgress);
}
finally
{
streamIn.Close();
}
return index;
}
#endregion Search
} }

View File

@@ -1,4 +1,5 @@
<wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation"> <wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
<s:Boolean x:Key="/Default/Environment/Filtering/ExcludeCoverageFilters/=_002A_002ETests_003B_002A_003B_002A_003B_002A/@EntryIndexedValue">True</s:Boolean>
<s:String x:Key="/Default/Environment/Hierarchy/Build/BuildTool/CustomBuildToolPath/@EntryValue">/usr/share/dotnet/sdk/7.0.107/MSBuild.dll</s:String> <s:String x:Key="/Default/Environment/Hierarchy/Build/BuildTool/CustomBuildToolPath/@EntryValue">/usr/share/dotnet/sdk/7.0.107/MSBuild.dll</s:String>
<s:Int64 x:Key="/Default/Environment/Hierarchy/Build/BuildTool/MsbuildVersion/@EntryValue">4294967293</s:Int64> <s:Int64 x:Key="/Default/Environment/Hierarchy/Build/BuildTool/MsbuildVersion/@EntryValue">4294967293</s:Int64>
<s:Boolean x:Key="/Default/UserDictionary/Words/=Avalonia/@EntryIndexedValue">True</s:Boolean></wpf:ResourceDictionary> <s:Boolean x:Key="/Default/UserDictionary/Words/=Avalonia/@EntryIndexedValue">True</s:Boolean></wpf:ResourceDictionary>

View File

@@ -95,7 +95,7 @@ public partial class MainWindow : Window
CsvFieldIndexer csvIndexer = new(); CsvFieldIndexer csvIndexer = new();
csvIndexer.LoadIndexOfFile(_loadedFile); csvIndexer.LoadIndexOfFile(_loadedFile);
List<long> newIndexes = csvIndexer.Search(_loadedFile, textToSearch); List<long> newIndexes = csvIndexer.SearchFile(_loadedFile, textToSearch);
_index = newIndexes; _index = newIndexes;
_totalRegs = _index.Count - 1; _totalRegs = _index.Count - 1;