mirror of
https://github.com/XFox111/GZipCompression.git
synced 2026-04-22 06:16:18 +03:00
Merging to .NET Core 3.1 (#2)
* Code refactoring * Rebuilt project for .NET Core 3.1 * Updated code comments
This commit is contained in:
+7
-7
@@ -1,9 +1,9 @@
|
|||||||
|
|
||||||
Microsoft Visual Studio Solution File, Format Version 12.00
|
Microsoft Visual Studio Solution File, Format Version 12.00
|
||||||
# Visual Studio Version 16
|
# Visual Studio Version 16
|
||||||
VisualStudioVersion = 16.0.29009.5
|
VisualStudioVersion = 16.0.30104.148
|
||||||
MinimumVisualStudioVersion = 10.0.40219.1
|
MinimumVisualStudioVersion = 10.0.40219.1
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GZipTest", "GZipTest\GZipTest.csproj", "{BE563CBF-0E92-4BD8-8157-D6EEBFE9535F}"
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GZipTest", "GZipTest\GZipTest.csproj", "{4D9BAFDB-056F-44F7-951D-F78607D195EE}"
|
||||||
EndProject
|
EndProject
|
||||||
Global
|
Global
|
||||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||||
@@ -11,15 +11,15 @@ Global
|
|||||||
Release|Any CPU = Release|Any CPU
|
Release|Any CPU = Release|Any CPU
|
||||||
EndGlobalSection
|
EndGlobalSection
|
||||||
GlobalSection(ProjectConfigurationPlatforms) = postSolution
|
GlobalSection(ProjectConfigurationPlatforms) = postSolution
|
||||||
{BE563CBF-0E92-4BD8-8157-D6EEBFE9535F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
{4D9BAFDB-056F-44F7-951D-F78607D195EE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||||
{BE563CBF-0E92-4BD8-8157-D6EEBFE9535F}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
{4D9BAFDB-056F-44F7-951D-F78607D195EE}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||||
{BE563CBF-0E92-4BD8-8157-D6EEBFE9535F}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
{4D9BAFDB-056F-44F7-951D-F78607D195EE}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||||
{BE563CBF-0E92-4BD8-8157-D6EEBFE9535F}.Release|Any CPU.Build.0 = Release|Any CPU
|
{4D9BAFDB-056F-44F7-951D-F78607D195EE}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||||
EndGlobalSection
|
EndGlobalSection
|
||||||
GlobalSection(SolutionProperties) = preSolution
|
GlobalSection(SolutionProperties) = preSolution
|
||||||
HideSolutionNode = FALSE
|
HideSolutionNode = FALSE
|
||||||
EndGlobalSection
|
EndGlobalSection
|
||||||
GlobalSection(ExtensibilityGlobals) = postSolution
|
GlobalSection(ExtensibilityGlobals) = postSolution
|
||||||
SolutionGuid = {ED3D3F32-26C1-434B-B48B-1EE336E8B1F0}
|
SolutionGuid = {AC70D379-6CF0-4834-9C6A-34C1457C1F32}
|
||||||
EndGlobalSection
|
EndGlobalSection
|
||||||
EndGlobal
|
EndGlobal
|
||||||
|
|||||||
@@ -1,6 +0,0 @@
|
|||||||
<?xml version="1.0" encoding="utf-8"?>
|
|
||||||
<configuration>
|
|
||||||
<startup>
|
|
||||||
|
|
||||||
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.7.2"/></startup>
|
|
||||||
</configuration>
|
|
||||||
@@ -5,19 +5,15 @@ using System.IO.Compression;
|
|||||||
|
|
||||||
namespace GZipTest
|
namespace GZipTest
|
||||||
{
|
{
|
||||||
class CompressionModule : ProcessingModule
|
class CompressionModule : ProcessingModuleBase
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Reading uncompressed source file
|
/// Reading uncompressed source file
|
||||||
/// </summary>
|
/// </summary>
|
||||||
internal override void Read()
|
protected override void Read()
|
||||||
{
|
|
||||||
try
|
|
||||||
{
|
|
||||||
using (FileStream input = File.OpenRead(source)) //Opening reading stream
|
|
||||||
{
|
{
|
||||||
|
using FileStream input = File.OpenRead(source); // Opening reading stream
|
||||||
segmentCount = (long)Math.Ceiling((double)input.Length / 1048576); // segmentCount field will be used to display progress bar
|
segmentCount = (long)Math.Ceiling((double)input.Length / 1048576); // segmentCount field will be used to display progress bar
|
||||||
length = input.Length; //This variable will be used in post analysis
|
|
||||||
|
|
||||||
for (int i = 0; input.Position < input.Length; i++)
|
for (int i = 0; input.Position < input.Length; i++)
|
||||||
{
|
{
|
||||||
@@ -36,23 +32,17 @@ namespace GZipTest
|
|||||||
readBuffer.Enqueue(new KeyValuePair<int, byte[]>(i, block)); // Adding read block to compression queue. Each block must contain its position number since compression is multi thread
|
readBuffer.Enqueue(new KeyValuePair<int, byte[]>(i, block)); // Adding read block to compression queue. Each block must contain its position number since compression is multi thread
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
catch (Exception e)
|
|
||||||
{
|
|
||||||
ReportError(this, $"Error occured in Reading thread. Served blocks: {served}", e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
internal override void ProcessOne()
|
protected override void ProcessOne()
|
||||||
{
|
{
|
||||||
if (!readBuffer.TryDequeue(out KeyValuePair<int, byte[]> block)) // Extracting read block
|
if (!readBuffer.TryDequeue(out KeyValuePair<int, byte[]> block)) // Extracting read block
|
||||||
return;
|
return;
|
||||||
|
|
||||||
processed.WaitOne(); // Waiting for empty place for compressed block
|
processed.WaitOne(); // Waiting for empty place for compressed block
|
||||||
|
|
||||||
using (MemoryStream stream = new MemoryStream()) //Instatiating memory stream which will contain compressed block
|
using MemoryStream stream = new MemoryStream(); // Instatiating memory stream which will contain compressed block
|
||||||
using (GZipStream compressor = new GZipStream(stream, CompressionMode.Compress)) //Instantiating compression stream
|
using GZipStream compressor = new GZipStream(stream, CompressionMode.Compress); // Instantiating compression stream
|
||||||
{
|
|
||||||
compressor.Write(block.Value, 0, block.Value.Length); // Compressing block
|
compressor.Write(block.Value, 0, block.Value.Length); // Compressing block
|
||||||
compressor.Close();
|
compressor.Close();
|
||||||
|
|
||||||
@@ -65,10 +55,7 @@ namespace GZipTest
|
|||||||
zippedMeta.CopyTo(newBlock, fileMeta.Length);
|
zippedMeta.CopyTo(newBlock, fileMeta.Length);
|
||||||
compressedBlock.CopyTo(newBlock, fileMeta.Length + 4);
|
compressedBlock.CopyTo(newBlock, fileMeta.Length + 4);
|
||||||
|
|
||||||
processedBuffer.TryAdd( //Processing block and adding it to write queue keeping its position number
|
processedBuffer.TryAdd(block.Key, newBlock); // Processing block and adding it to write queue keeping its position number
|
||||||
block.Key,
|
|
||||||
newBlock);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -5,17 +5,15 @@ using System.IO.Compression;
|
|||||||
|
|
||||||
namespace GZipTest
|
namespace GZipTest
|
||||||
{
|
{
|
||||||
class DecompressionModule : ProcessingModule
|
class DecompressionModule : ProcessingModuleBase
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Reading compressed source file
|
/// Reading compressed source file
|
||||||
/// </summary>
|
/// </summary>
|
||||||
internal override void Read()
|
protected override void Read()
|
||||||
{
|
|
||||||
try
|
|
||||||
{
|
|
||||||
using (FileStream input = File.OpenRead(source)) //Opening reading stream
|
|
||||||
{
|
{
|
||||||
|
using FileStream input = File.OpenRead(source); // Opening reading stream
|
||||||
|
|
||||||
byte[] segmentMeta = new byte[8]; // Reading first 8 bytes to determine total count of blocks
|
byte[] segmentMeta = new byte[8]; // Reading first 8 bytes to determine total count of blocks
|
||||||
input.Read(segmentMeta, 0, 8);
|
input.Read(segmentMeta, 0, 8);
|
||||||
segmentCount = BitConverter.ToInt64(segmentMeta, 0); // segmentCount field will be used to display progress bar
|
segmentCount = BitConverter.ToInt64(segmentMeta, 0); // segmentCount field will be used to display progress bar
|
||||||
@@ -39,30 +37,21 @@ namespace GZipTest
|
|||||||
readBuffer.Enqueue(new KeyValuePair<int, byte[]>(i, block)); // Adding read block to compression queue. Each block must contain its position number since compression is multi thread
|
readBuffer.Enqueue(new KeyValuePair<int, byte[]>(i, block)); // Adding read block to compression queue. Each block must contain its position number since compression is multi thread
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
catch (Exception e)
|
|
||||||
{
|
|
||||||
ReportError(this, $"Error occured in Reading thread. Served blocks: {served}", e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
internal override void ProcessOne()
|
protected override void ProcessOne()
|
||||||
{
|
{
|
||||||
if (!readBuffer.TryDequeue(out KeyValuePair<int, byte[]> block)) // Extracting read block
|
if (!readBuffer.TryDequeue(out KeyValuePair<int, byte[]> block)) // Extracting read block
|
||||||
return;
|
return;
|
||||||
|
|
||||||
processed.WaitOne(); // Waiting for empty place for compressed block
|
processed.WaitOne(); // Waiting for empty place for compressed block
|
||||||
|
|
||||||
using (MemoryStream stream = new MemoryStream(block.Value)) //Instantiating memory stream with compressed block data
|
using MemoryStream stream = new MemoryStream(block.Value); // Instantiating memory stream with compressed block data
|
||||||
using (GZipStream compressor = new GZipStream(stream, CompressionMode.Decompress)) //Instantiating decompressor stream
|
using GZipStream compressor = new GZipStream(stream, CompressionMode.Decompress); // Instantiating decompressor stream
|
||||||
using (MemoryStream destination = new MemoryStream()) //Instantiating memory stream which will contain decompressed block
|
using MemoryStream destination = new MemoryStream(); // Instantiating memory stream which will contain decompressed block
|
||||||
{
|
|
||||||
compressor.CopyTo(destination); // Decompressing block
|
compressor.CopyTo(destination); // Decompressing block
|
||||||
|
|
||||||
processedBuffer.TryAdd( //Processing block and adding it to write queue keeping its position number
|
processedBuffer.TryAdd(block.Key, destination.ToArray()); // Processing block and adding it to write queue keeping its position number
|
||||||
block.Key,
|
|
||||||
destination.ToArray());
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
+11
-56
@@ -1,60 +1,15 @@
|
|||||||
<?xml version="1.0" encoding="utf-8"?>
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
<Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
|
||||||
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
|
|
||||||
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
|
|
||||||
<ProjectGuid>{BE563CBF-0E92-4BD8-8157-D6EEBFE9535F}</ProjectGuid>
|
|
||||||
<OutputType>Exe</OutputType>
|
<OutputType>Exe</OutputType>
|
||||||
<RootNamespace>GZipTest</RootNamespace>
|
<TargetFramework>netcoreapp3.1</TargetFramework>
|
||||||
<AssemblyName>GZipTest</AssemblyName>
|
<Authors>Michael "XFox" Gordeev</Authors>
|
||||||
<TargetFrameworkVersion>v4.7.2</TargetFrameworkVersion>
|
<Version>2.0.0</Version>
|
||||||
<FileAlignment>512</FileAlignment>
|
<Copyright>©2020 Michael "XFox" Gordeev</Copyright>
|
||||||
<AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
|
<RepositoryUrl>https://github.com/XFox111/GZipCompression.git</RepositoryUrl>
|
||||||
<Deterministic>true</Deterministic>
|
<PackageProjectUrl>https://github.com/XFox111/GZipCompression</PackageProjectUrl>
|
||||||
<TargetFrameworkProfile />
|
<RepositoryType>git</RepositoryType>
|
||||||
|
<Description>Multi-thread console program which is used to compress/decompress files using block-by-block compression. I've done this project as an entrance test for Software developer position in Veeam Software</Description>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
|
|
||||||
<PlatformTarget>AnyCPU</PlatformTarget>
|
|
||||||
<DebugSymbols>true</DebugSymbols>
|
|
||||||
<DebugType>full</DebugType>
|
|
||||||
<Optimize>false</Optimize>
|
|
||||||
<OutputPath>bin\Debug\</OutputPath>
|
|
||||||
<DefineConstants>DEBUG;TRACE</DefineConstants>
|
|
||||||
<ErrorReport>prompt</ErrorReport>
|
|
||||||
<WarningLevel>4</WarningLevel>
|
|
||||||
<Prefer32Bit>false</Prefer32Bit>
|
|
||||||
</PropertyGroup>
|
|
||||||
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
|
|
||||||
<PlatformTarget>AnyCPU</PlatformTarget>
|
|
||||||
<DebugType>pdbonly</DebugType>
|
|
||||||
<Optimize>true</Optimize>
|
|
||||||
<OutputPath>bin\Release\</OutputPath>
|
|
||||||
<DefineConstants>TRACE</DefineConstants>
|
|
||||||
<ErrorReport>prompt</ErrorReport>
|
|
||||||
<WarningLevel>4</WarningLevel>
|
|
||||||
<Prefer32Bit>false</Prefer32Bit>
|
|
||||||
</PropertyGroup>
|
|
||||||
<ItemGroup>
|
|
||||||
<Reference Include="System" />
|
|
||||||
<Reference Include="System.Core" />
|
|
||||||
<Reference Include="System.Xml.Linq" />
|
|
||||||
<Reference Include="System.Data.DataSetExtensions" />
|
|
||||||
<Reference Include="Microsoft.CSharp" />
|
|
||||||
<Reference Include="System.Data" />
|
|
||||||
<Reference Include="System.Net.Http" />
|
|
||||||
<Reference Include="System.Xml" />
|
|
||||||
</ItemGroup>
|
|
||||||
<ItemGroup>
|
|
||||||
<Compile Include="CompressionModule.cs" />
|
|
||||||
<Compile Include="DecompressionModule.cs" />
|
|
||||||
<Compile Include="IProcessingModule.cs" />
|
|
||||||
<Compile Include="ProcessingModule.cs" />
|
|
||||||
<Compile Include="Program.cs" />
|
|
||||||
<Compile Include="Properties\AssemblyInfo.cs" />
|
|
||||||
</ItemGroup>
|
|
||||||
<ItemGroup>
|
|
||||||
<None Include="App.config" />
|
|
||||||
</ItemGroup>
|
|
||||||
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
|
|
||||||
</Project>
|
</Project>
|
||||||
@@ -1,18 +1,20 @@
|
|||||||
using System;
|
namespace GZipTest
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.IO;
|
|
||||||
using System.Linq;
|
|
||||||
using System.Text;
|
|
||||||
using System.Threading.Tasks;
|
|
||||||
|
|
||||||
namespace GZipTest
|
|
||||||
{
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// Processing module interface
|
||||||
|
/// </summary>
|
||||||
interface IProcessingModule
|
interface IProcessingModule
|
||||||
{
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// Indicates wether the module is processing a file
|
||||||
|
/// </summary>
|
||||||
|
bool IsWorking { get; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Starts the job
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="input">Source file path (relative or absolute)</param>
|
||||||
|
/// <param name="output">Destination file path (relative or absolute)</param>
|
||||||
void Run(string input, string output);
|
void Run(string input, string output);
|
||||||
void Stop();
|
|
||||||
event ProgressChangedEventHandler ProgressChanged;
|
|
||||||
event EventHandler Complete;
|
|
||||||
event ErrorEventHandler ErrorOccured;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,146 +0,0 @@
|
|||||||
using System;
|
|
||||||
using System.Collections.Concurrent;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.IO;
|
|
||||||
using System.Linq;
|
|
||||||
using System.Threading;
|
|
||||||
|
|
||||||
namespace GZipTest
|
|
||||||
{
|
|
||||||
/// <summary>
|
|
||||||
/// Delegate void used to inform UI thread about changed progress
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="done">Amount of blocks that have been done</param>
|
|
||||||
/// <param name="totalSegments">Amount of total blocks</param>
|
|
||||||
public delegate void ProgressChangedEventHandler(long done, long totalSegments);
|
|
||||||
|
|
||||||
public abstract class ProcessingModule : IProcessingModule
|
|
||||||
{
|
|
||||||
public event ProgressChangedEventHandler ProgressChanged;
|
|
||||||
public event EventHandler Complete;
|
|
||||||
public event ErrorEventHandler ErrorOccured;
|
|
||||||
|
|
||||||
internal Thread readingThread, writingThread;
|
|
||||||
internal Thread[] compressionThreads = new Thread[Math.Max(1, Environment.ProcessorCount - 2)]; //If we have ability to use more than 3 threads we add more threads that will proccess blocks (because this operation takes the biggest amount of resources)
|
|
||||||
|
|
||||||
internal Semaphore processed; //Semaphore will help us to maintain RAM and use minimum of it
|
|
||||||
|
|
||||||
internal ConcurrentQueue<KeyValuePair<int, byte[]>> readBuffer = new ConcurrentQueue<KeyValuePair<int, byte[]>>(); //We use queue for reading and processing blocks since FIFO method is more efficient here
|
|
||||||
internal ConcurrentDictionary<int, byte[]> processedBuffer = new ConcurrentDictionary<int, byte[]>(); //And use dictionary for writing since we need blocks to be placed in order
|
|
||||||
|
|
||||||
//These variables are used for tracking progress
|
|
||||||
internal long segmentCount = 0;
|
|
||||||
internal long served = 0;
|
|
||||||
internal long length;
|
|
||||||
|
|
||||||
//Source and output file paths
|
|
||||||
internal string source, result;
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Initializing workflow
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="input">Source file path</param>
|
|
||||||
/// <param name="output">Destination file path</param>
|
|
||||||
public void Run(string input, string output)
|
|
||||||
{
|
|
||||||
//Setting files paths
|
|
||||||
source = input;
|
|
||||||
result = output;
|
|
||||||
|
|
||||||
//Instantiating threads
|
|
||||||
readingThread = new Thread(Read);
|
|
||||||
writingThread = new Thread(Write);
|
|
||||||
|
|
||||||
for (int i = 0; i < compressionThreads.Length; i++)
|
|
||||||
compressionThreads[i] = new Thread(Process);
|
|
||||||
|
|
||||||
foreach (Thread i in compressionThreads)
|
|
||||||
i.Priority = ThreadPriority.Highest; //Since compression is the slowest operation it must be marked as high priority task
|
|
||||||
|
|
||||||
//Semaphore will indicate how many blocks can be now written.
|
|
||||||
//There can be max 5 blocks for each compression thread because there's no reason for more.
|
|
||||||
//5 block in a row mean that compressing algorithm is faster than writing algorithm so there's no need to process more block until these are done
|
|
||||||
processed = new Semaphore(compressionThreads.Length * 5, compressionThreads.Length * 5);
|
|
||||||
|
|
||||||
//Starting threads
|
|
||||||
readingThread.Start();
|
|
||||||
foreach (Thread i in compressionThreads)
|
|
||||||
i.Start();
|
|
||||||
writingThread.Start();
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Instantly terminates all threads and cleans up stuff
|
|
||||||
/// </summary>
|
|
||||||
public void Stop()
|
|
||||||
{
|
|
||||||
//Terminating threads
|
|
||||||
readingThread.Abort();
|
|
||||||
foreach (Thread thread in compressionThreads)
|
|
||||||
thread.Abort();
|
|
||||||
writingThread.Abort();
|
|
||||||
|
|
||||||
//Collecting garbage (Yours' Cap)
|
|
||||||
GC.Collect();
|
|
||||||
}
|
|
||||||
internal void ReportError(object sender, string message, Exception ex) => ErrorOccured?.Invoke(sender, new ErrorEventArgs(new Exception(message, ex)));
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Reading source file
|
|
||||||
/// </summary>
|
|
||||||
internal abstract void Read();
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Processes one block. This method is used in Read and Write threads
|
|
||||||
/// </summary>
|
|
||||||
internal abstract void ProcessOne();
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Processing read block
|
|
||||||
/// </summary>
|
|
||||||
internal void Process()
|
|
||||||
{
|
|
||||||
try
|
|
||||||
{
|
|
||||||
while (readingThread.IsAlive || readBuffer.Count > 0) //The task will be alive as long as reading is in progress or there's stil any unprocessed blocks
|
|
||||||
ProcessOne();
|
|
||||||
}
|
|
||||||
catch (Exception e)
|
|
||||||
{
|
|
||||||
ReportError(this, $"Error occured in Compression thread. Served blocks: {served}", e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Writing processed block to disk
|
|
||||||
/// </summary>
|
|
||||||
internal void Write()
|
|
||||||
{
|
|
||||||
try
|
|
||||||
{
|
|
||||||
using (FileStream stream = new FileStream(result, FileMode.Create, FileAccess.Write)) //Instantiating writing stream
|
|
||||||
{
|
|
||||||
while (compressionThreads.Any(i => i.IsAlive) || processedBuffer.Count > 0) //The task will be alive as long as compression is in progress or there's stil any unwritten block
|
|
||||||
{
|
|
||||||
if (!processedBuffer.TryRemove((int)served, out byte[] block)) //Extracting block that need to be written next
|
|
||||||
{
|
|
||||||
if (readBuffer.Count > 0) //Helping processing thread to do its job
|
|
||||||
ProcessOne();
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
stream.Write(block, 0, block.Length); //Writing block to the file
|
|
||||||
processed.Release(); //Informing compression threads that they can continue
|
|
||||||
|
|
||||||
ProgressChanged?.Invoke(++served, segmentCount); //Updating progress bar
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Complete?.Invoke(length / 1024 / 1024, null);
|
|
||||||
}
|
|
||||||
catch (Exception e)
|
|
||||||
{
|
|
||||||
ReportError(this, $"Error occured in writing thread. Blocks served: {served}", e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,142 @@
|
|||||||
|
using System;
|
||||||
|
using System.Collections.Concurrent;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.IO;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Threading;
|
||||||
|
|
||||||
|
namespace GZipTest
|
||||||
|
{
|
||||||
|
abstract class ProcessingModuleBase : IProcessingModule
|
||||||
|
{
|
||||||
|
#region Variables
|
||||||
|
Thread readingThread, writingThread;
|
||||||
|
readonly Thread[] compressionThreads = new Thread[Math.Max(1, Environment.ProcessorCount - 2)]; // If we have ability to use more than 3 threads we add more threads that will proccess blocks (because this operation takes the biggest amount of resources)
|
||||||
|
|
||||||
|
protected Semaphore processed; // Semaphore will help us to maintain RAM and use minimum of it
|
||||||
|
|
||||||
|
protected ConcurrentQueue<KeyValuePair<int, byte[]>> readBuffer = new ConcurrentQueue<KeyValuePair<int, byte[]>>(); // We use queue for reading and processing blocks since FIFO method is more efficient here
|
||||||
|
protected ConcurrentDictionary<int, byte[]> processedBuffer = new ConcurrentDictionary<int, byte[]>(); // And use dictionary for writing since we need blocks to be placed in order
|
||||||
|
|
||||||
|
// These variables are used for tracking progress
|
||||||
|
protected long segmentCount = 1;
|
||||||
|
long served = 0;
|
||||||
|
|
||||||
|
// Source and output file paths
|
||||||
|
protected string source, result;
|
||||||
|
|
||||||
|
readonly DateTime start = DateTime.Now;
|
||||||
|
|
||||||
|
public bool IsWorking { get; private set; }
|
||||||
|
#endregion
|
||||||
|
|
||||||
|
#region Methods
|
||||||
|
/// <summary>
|
||||||
|
/// Initializing workflow
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="input">Source file path</param>
|
||||||
|
/// <param name="output">Destination file path</param>
|
||||||
|
public void Run(string input, string output)
|
||||||
|
{
|
||||||
|
IsWorking = true;
|
||||||
|
// Setting files paths
|
||||||
|
source = input;
|
||||||
|
result = output;
|
||||||
|
|
||||||
|
// Instantiating threads
|
||||||
|
readingThread = new Thread(Read);
|
||||||
|
writingThread = new Thread(Write);
|
||||||
|
|
||||||
|
for (int i = 0; i < compressionThreads.Length; i++)
|
||||||
|
compressionThreads[i] = new Thread(Process)
|
||||||
|
{
|
||||||
|
Priority = ThreadPriority.Highest // Since compression is the slowest operation it must be marked as high priority task
|
||||||
|
};
|
||||||
|
|
||||||
|
// Semaphore will indicate how many blocks can be now written.
|
||||||
|
// There can be max 5 blocks for each compression thread because there's no reason for more.
|
||||||
|
// 5 block in a row mean that compressing algorithm is faster than writing algorithm so there's no need to process more block until these are done
|
||||||
|
processed = new Semaphore(compressionThreads.Length * 5, compressionThreads.Length * 5);
|
||||||
|
|
||||||
|
// Starting threads
|
||||||
|
readingThread.Start();
|
||||||
|
foreach (Thread i in compressionThreads)
|
||||||
|
i.Start();
|
||||||
|
writingThread.Start();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Reads source file
|
||||||
|
/// </summary>
|
||||||
|
protected abstract void Read();
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Processes one block. This method is used in Read and Write threads
|
||||||
|
/// </summary>`
|
||||||
|
protected abstract void ProcessOne();
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Processing read block
|
||||||
|
/// </summary>
|
||||||
|
void Process()
|
||||||
|
{
|
||||||
|
while (readingThread.IsAlive || readBuffer.Count > 0) // The task will be alive as long as reading is in progress or there's stil any unprocessed blocks
|
||||||
|
ProcessOne();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Writes processed block to disk
|
||||||
|
/// </summary>
|
||||||
|
void Write()
|
||||||
|
{
|
||||||
|
using (FileStream stream = new FileStream(result, FileMode.Create, FileAccess.Write)) // Instantiating writing stream
|
||||||
|
{
|
||||||
|
while (compressionThreads.Any(i => i.IsAlive) || processedBuffer.Count > 0) // The task will be alive as long as compression is in progress or there's stil any unwritten block
|
||||||
|
{
|
||||||
|
if (!processedBuffer.TryRemove((int)served, out byte[] block)) // Extracting block that need to be written next
|
||||||
|
{
|
||||||
|
if (readBuffer.Count > 0) // Helping processing thread to do its job
|
||||||
|
ProcessOne();
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
stream.Write(block, 0, block.Length); // Writing block to the file
|
||||||
|
processed.Release(); // Informing compression threads that they can continue
|
||||||
|
|
||||||
|
served++; // Updating counter
|
||||||
|
|
||||||
|
SetProgress();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
TimeSpan elapsed = DateTime.Now - start;
|
||||||
|
Console.WriteLine($"\nDone\nFile processing is completed within within {elapsed.Minutes} minutes {elapsed.Seconds} seconds");
|
||||||
|
IsWorking = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Draws a progress bar in output console and displays some information
|
||||||
|
/// </summary>
|
||||||
|
void SetProgress()
|
||||||
|
{
|
||||||
|
TimeSpan elapsed = DateTime.Now - start;
|
||||||
|
//Border braces
|
||||||
|
Console.CursorLeft = 0;
|
||||||
|
Console.Write("[");
|
||||||
|
Console.CursorLeft = 21;
|
||||||
|
Console.Write("]");
|
||||||
|
|
||||||
|
//Progress bar
|
||||||
|
for (int i = 0; i < served * 20 / segmentCount; i++)
|
||||||
|
{
|
||||||
|
Console.CursorLeft = i + 1;
|
||||||
|
Console.Write("■");
|
||||||
|
}
|
||||||
|
|
||||||
|
//Percentage
|
||||||
|
Console.CursorLeft = 23;
|
||||||
|
Console.Write($"{served * 100 / segmentCount}% {served} ({segmentCount * 5} MB) of {segmentCount} ({segmentCount * 5} MB) blocks [{elapsed:hh\\:mm\\:ss}]");
|
||||||
|
}
|
||||||
|
#endregion
|
||||||
|
}
|
||||||
|
}
|
||||||
+35
-61
@@ -5,18 +5,16 @@ namespace GZipTest
|
|||||||
{
|
{
|
||||||
class Program
|
class Program
|
||||||
{
|
{
|
||||||
static DateTime start = DateTime.Now;
|
|
||||||
static IProcessingModule module;
|
static IProcessingModule module;
|
||||||
|
|
||||||
static int Main(string[] args)
|
static int Main(string[] args)
|
||||||
{
|
|
||||||
try
|
|
||||||
{
|
{
|
||||||
// Validating input parameters
|
// Validating input parameters
|
||||||
if (args.Length != 3)
|
if (args.Length < 1) // If there's no parameters provided, display help
|
||||||
throw new InvalidDataException("Invalid parameters set.\nUsage: NewWinRar.exe [compress|decompress] [source file name] [destination file name]");
|
{
|
||||||
if (!File.Exists(args[1]))
|
DisplayHelp();
|
||||||
throw new FileNotFoundException("The source file not found. Make sure it is place in the program's directory and has the same name. Stating extension is required");
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
// Instatiating module
|
// Instatiating module
|
||||||
switch (args[0].ToLower())
|
switch (args[0].ToLower())
|
||||||
@@ -29,73 +27,49 @@ namespace GZipTest
|
|||||||
Console.WriteLine("Unpacking file...");
|
Console.WriteLine("Unpacking file...");
|
||||||
module = new DecompressionModule();
|
module = new DecompressionModule();
|
||||||
break;
|
break;
|
||||||
|
case "help":
|
||||||
|
DisplayHelp();
|
||||||
|
return 0;
|
||||||
default:
|
default:
|
||||||
throw new InvalidDataException("Invalid parameter. The first parameter must be 'compress' or 'decompress'");
|
throw new InvalidDataException("Invalid parameter. The first parameter must be 'compress', 'decompress' or 'help'");
|
||||||
}
|
}
|
||||||
|
|
||||||
//Subscribing to events
|
if (args.Length < 3)
|
||||||
module.ProgressChanged += SetProgress;
|
throw new InvalidDataException("Target file or destination file path missing. Type 'help' to get usage information");
|
||||||
module.Complete += Complete;
|
if (!File.Exists(args[1]))
|
||||||
module.ErrorOccured += Module_ErrorOccured;
|
throw new FileNotFoundException("The source file not found. Check provided path and try again. Stating extension is required");
|
||||||
|
|
||||||
//Executing module
|
//Executing module
|
||||||
module.Run(args[1], args[2]);
|
module.Run(input: args[1],
|
||||||
|
output: args[2]);
|
||||||
|
|
||||||
|
while (module.IsWorking); // Get UI thread busy while in progress
|
||||||
|
|
||||||
|
Console.WriteLine("Press any key to continue...");
|
||||||
|
Console.ReadKey();
|
||||||
|
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
//Catching errors and displaying them
|
|
||||||
catch (Exception e)
|
|
||||||
{
|
|
||||||
Console.Error.WriteLine($"\n\n{e.ToString()}\n" + e.InnerException != null && e.InnerException != e ? $"\n{e.InnerException.ToString()}\n" : "");
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private static void Module_ErrorOccured(object sender, ErrorEventArgs e)
|
|
||||||
{
|
|
||||||
Console.Error.WriteLine("Error has occured. Threads tremination initiated");
|
|
||||||
Console.Error.WriteLine($"\n\n{e.GetException().ToString()}\n");
|
|
||||||
module.Complete -= Complete;
|
|
||||||
Console.WriteLine("Press any key to continue...");
|
|
||||||
Console.ReadKey();
|
|
||||||
module.Stop();
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Displays complete message and post analysis
|
/// Displays program descriptions and usage instructions
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="size">Represents original file size in MB</param>
|
static void DisplayHelp()
|
||||||
/// <param name="e">Not used</param>
|
|
||||||
private static void Complete(object size, EventArgs e)
|
|
||||||
{
|
{
|
||||||
TimeSpan elapsed = DateTime.Now - start;
|
Console.WriteLine("Compresses or decompresses files. Compressed archives cannot be opened with other archivers.\n");
|
||||||
Console.WriteLine($"\nDone\nProcessed {size} MB within {elapsed.Minutes} minutes {elapsed.Seconds} seconds\nPress any key to continue...");
|
Console.WriteLine("USAGE:\n" +
|
||||||
|
"GZipTest [OPERATION] [SOURCE] [TARGET]\n");
|
||||||
|
|
||||||
|
Console.WriteLine("Parameters:");
|
||||||
|
Console.WriteLine("OPERATION \t Operation which will be executed by the program - compression or decompression. Required.");
|
||||||
|
Console.WriteLine("\t Valid values: compress | decompress | help");
|
||||||
|
|
||||||
|
Console.WriteLine("\nSOURCE \t\t Relative or absolute path to file which will be processed by the program. Required.");
|
||||||
|
|
||||||
|
Console.WriteLine("\nTARGET \t\t Relative or absolute path to destination file which will be created after the program work. Required.");
|
||||||
|
|
||||||
|
Console.WriteLine("\nPress any key to continue...");
|
||||||
Console.ReadKey();
|
Console.ReadKey();
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Displaying progress bar which represents current workflow position
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="percentage">Integer from 0 to 100. Represents amount of completed work</param>
|
|
||||||
public static void SetProgress(long done, long totalSegments)
|
|
||||||
{
|
|
||||||
TimeSpan elapsed = DateTime.Now - start;
|
|
||||||
//Border braces
|
|
||||||
Console.CursorLeft = 0;
|
|
||||||
Console.Write("[");
|
|
||||||
Console.CursorLeft = 21;
|
|
||||||
Console.Write("]");
|
|
||||||
|
|
||||||
//Progress bar
|
|
||||||
for (int i = 0; i < done * 20 / totalSegments; i++)
|
|
||||||
{
|
|
||||||
Console.CursorLeft = i + 1;
|
|
||||||
Console.Write("■");
|
|
||||||
}
|
|
||||||
|
|
||||||
//Percentage
|
|
||||||
Console.CursorLeft = 23;
|
|
||||||
Console.Write($"{done * 100 / totalSegments}% {done} of {totalSegments} blocks [{elapsed.ToString(@"hh\:mm\:ss")}]");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,36 +0,0 @@
|
|||||||
using System.Reflection;
|
|
||||||
using System.Runtime.CompilerServices;
|
|
||||||
using System.Runtime.InteropServices;
|
|
||||||
|
|
||||||
// General Information about an assembly is controlled through the following
|
|
||||||
// set of attributes. Change these attribute values to modify the information
|
|
||||||
// associated with an assembly.
|
|
||||||
[assembly: AssemblyTitle("NewWinRar")]
|
|
||||||
[assembly: AssemblyDescription("")]
|
|
||||||
[assembly: AssemblyConfiguration("")]
|
|
||||||
[assembly: AssemblyCompany("")]
|
|
||||||
[assembly: AssemblyProduct("NewWinRar")]
|
|
||||||
[assembly: AssemblyCopyright("Copyright © 2019")]
|
|
||||||
[assembly: AssemblyTrademark("")]
|
|
||||||
[assembly: AssemblyCulture("")]
|
|
||||||
|
|
||||||
// Setting ComVisible to false makes the types in this assembly not visible
|
|
||||||
// to COM components. If you need to access a type in this assembly from
|
|
||||||
// COM, set the ComVisible attribute to true on that type.
|
|
||||||
[assembly: ComVisible(false)]
|
|
||||||
|
|
||||||
// The following GUID is for the ID of the typelib if this project is exposed to COM
|
|
||||||
[assembly: Guid("c4bc63bf-c2a8-4057-b746-662f9dcf5a6b")]
|
|
||||||
|
|
||||||
// Version information for an assembly consists of the following four values:
|
|
||||||
//
|
|
||||||
// Major Version
|
|
||||||
// Minor Version
|
|
||||||
// Build Number
|
|
||||||
// Revision
|
|
||||||
//
|
|
||||||
// You can specify all the values or you can default the Build and Revision Numbers
|
|
||||||
// by using the '*' as shown below:
|
|
||||||
// [assembly: AssemblyVersion("1.0.*")]
|
|
||||||
[assembly: AssemblyVersion("1.0.0.0")]
|
|
||||||
[assembly: AssemblyFileVersion("1.0.0.0")]
|
|
||||||
Reference in New Issue
Block a user