Finished and polished project (added files)

GZipTest.sln

@@ -0,0 +1,25 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 16
+VisualStudioVersion = 16.0.29009.5
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GZipTest", "GZipTest\GZipTest.csproj", "{BE563CBF-0E92-4BD8-8157-D6EEBFE9535F}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{BE563CBF-0E92-4BD8-8157-D6EEBFE9535F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{BE563CBF-0E92-4BD8-8157-D6EEBFE9535F}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{BE563CBF-0E92-4BD8-8157-D6EEBFE9535F}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{BE563CBF-0E92-4BD8-8157-D6EEBFE9535F}.Release|Any CPU.Build.0 = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {ED3D3F32-26C1-434B-B48B-1EE336E8B1F0}
+	EndGlobalSection
+EndGlobal

GZipTest/App.config

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+    <startup> 
+        
+    <supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.7.2"/></startup>
+</configuration>

GZipTest/CompressionModule.cs

@@ -0,0 +1,74 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.IO.Compression;
+
+namespace GZipTest
+{
+    class CompressionModule : ProcessingModule
+    {
+        /// <summary>
+        /// Reading uncompressed source file
+        /// </summary>
+        internal override void Read()
+        {
+            try
+            {
+                using (FileStream input = File.OpenRead(source))                                       //Opening reading stream
+                {
+                    segmentCount = (long)Math.Ceiling((double)input.Length / 1048576);                  //segmentCount field will be used to display progress bar
+                    length = input.Length;                                                              //This variable will be used in post analysis
+
+                    for (int i = 0; input.Position < input.Length; i++)
+                    {
+                        if (readBuffer.Count >= 5 * Environment.ProcessorCount)                         //Helping compression thread if there's too many unprocessed blocks
+                        {
+                            ProcessOne();
+                            i--;
+                            continue;
+                        }
+
+                        int blockSize = (int)Math.Min(1048576, input.Length - input.Position);          //Determining new block size. Either 1MB or count of the last bytes
+
+                        byte[] block = new byte[blockSize];                                             //Instantiating empty block
+                        input.Read(block, 0, blockSize);                                                //Reading next block
+
+                        readBuffer.Enqueue(new KeyValuePair<int, byte[]>(i, block));                    //Adding read block to compression queue. Each block must contain its position number since compression is multi thread
+                    }
+                }
+            }
+            catch (Exception e)
+            {
+                ReportError(this, $"Error occured in Reading thread. Served blocks: {served}", e);
+            }
+        }
+
+        internal override void ProcessOne()
+        {
+            if (!readBuffer.TryDequeue(out KeyValuePair<int, byte[]> block))                                //Extracting read block
+                return;
+
+            processed.WaitOne();                                                                            //Waiting for empty place for compressed block
+
+            using (MemoryStream stream = new MemoryStream())                                                //Instatiating memory stream which will contain compressed block
+            using (GZipStream compressor = new GZipStream(stream, CompressionMode.Compress))                //Instantiating compression stream
+            {
+                compressor.Write(block.Value, 0, block.Value.Length);                                       //Compressing block
+                compressor.Close();
+
+                byte[] compressedBlock = stream.ToArray();                                                  //Getting compressed block
+                byte[] fileMeta = block.Key == 0 ? BitConverter.GetBytes(segmentCount) : new byte[0];       //If it's the first block in a file we write info about total block count (that will be used to count progress)
+                byte[] zippedMeta = BitConverter.GetBytes(compressedBlock.Length);                          //Creating compressed block length info
+
+                byte[] newBlock = new byte[fileMeta.Length + 4 + compressedBlock.Length];                   //Merging arrays
+                fileMeta.CopyTo(newBlock, 0);
+                zippedMeta.CopyTo(newBlock, fileMeta.Length);
+                compressedBlock.CopyTo(newBlock, fileMeta.Length + 4);
+
+                processedBuffer.TryAdd(                                                                     //Processing block and adding it to write queue keeping its position number
+                    block.Key,
+                    newBlock);
+            }
+        }
+    }
+}

GZipTest/DecompressionModule.cs

@@ -0,0 +1,68 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.IO.Compression;
+
+namespace GZipTest
+{
+    class DecompressionModule : ProcessingModule
+    {
+        /// <summary>
+        /// Reading compressed source file
+        /// </summary>
+        internal override void Read()
+        {
+            try
+            {
+                using (FileStream input = File.OpenRead(source))                                       //Opening reading stream
+                {
+                    byte[] segmentMeta = new byte[8];                                                   //Reading first 8 bytes to determine total count of blocks
+                    input.Read(segmentMeta, 0, 8);
+                    segmentCount = BitConverter.ToInt64(segmentMeta, 0);                                //segmentCount field will be used to display progress bar
+
+                    for (int i = 0; input.Position < input.Length; i++)
+                    {
+                        if (readBuffer.Count >= 5 * Environment.ProcessorCount)                         //Helping decompression thread if there's too many unprocessed blocks
+                        {
+                            ProcessOne();
+                            i--;
+                            continue;
+                        }
+
+                        byte[] meta = new byte[4];                                                      //Reading first 4 bytes to determine block's length
+                        input.Read(meta, 0, 4);
+                        int blockSize = BitConverter.ToInt32(meta, 0);
+
+                        byte[] block = new byte[blockSize];                                             //Instantiating empty block
+                        input.Read(block, 0, blockSize);                                                //Reading next block
+
+                        readBuffer.Enqueue(new KeyValuePair<int, byte[]>(i, block));                    //Adding read block to compression queue. Each block must contain its position number since compression is multi thread
+                    }
+                }
+            }
+            catch (Exception e)
+            {
+                ReportError(this, $"Error occured in Reading thread. Served blocks: {served}", e);
+            }
+        }
+
+        internal override void ProcessOne()
+        {
+            if (!readBuffer.TryDequeue(out KeyValuePair<int, byte[]> block))                    //Extracting read block
+                return;
+
+            processed.WaitOne();                                                                //Waiting for empty place for compressed block
+
+            using (MemoryStream stream = new MemoryStream(block.Value))                         //Instantiating memory stream with compressed block data
+            using (GZipStream compressor = new GZipStream(stream, CompressionMode.Decompress))  //Instantiating decompressor stream
+            using (MemoryStream destination = new MemoryStream())                               //Instantiating memory stream which will contain decompressed block
+            {
+                compressor.CopyTo(destination);                                                 //Decompressing block
+
+                processedBuffer.TryAdd(                                                         //Processing block and adding it to write queue keeping its position number
+                    block.Key,
+                    destination.ToArray());
+            }
+        }
+    }
+}

GZipTest/GZipTest.csproj

@@ -0,0 +1,60 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
+  <PropertyGroup>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+    <ProjectGuid>{BE563CBF-0E92-4BD8-8157-D6EEBFE9535F}</ProjectGuid>
+    <OutputType>Exe</OutputType>
+    <RootNamespace>GZipTest</RootNamespace>
+    <AssemblyName>GZipTest</AssemblyName>
+    <TargetFrameworkVersion>v4.7.2</TargetFrameworkVersion>
+    <FileAlignment>512</FileAlignment>
+    <AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
+    <Deterministic>true</Deterministic>
+    <TargetFrameworkProfile />
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
+    <PlatformTarget>AnyCPU</PlatformTarget>
+    <DebugSymbols>true</DebugSymbols>
+    <DebugType>full</DebugType>
+    <Optimize>false</Optimize>
+    <OutputPath>bin\Debug\</OutputPath>
+    <DefineConstants>DEBUG;TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+    <Prefer32Bit>false</Prefer32Bit>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
+    <PlatformTarget>AnyCPU</PlatformTarget>
+    <DebugType>pdbonly</DebugType>
+    <Optimize>true</Optimize>
+    <OutputPath>bin\Release\</OutputPath>
+    <DefineConstants>TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+    <Prefer32Bit>false</Prefer32Bit>
+  </PropertyGroup>
+  <ItemGroup>
+    <Reference Include="System" />
+    <Reference Include="System.Core" />
+    <Reference Include="System.Xml.Linq" />
+    <Reference Include="System.Data.DataSetExtensions" />
+    <Reference Include="Microsoft.CSharp" />
+    <Reference Include="System.Data" />
+    <Reference Include="System.Net.Http" />
+    <Reference Include="System.Xml" />
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Include="CompressionModule.cs" />
+    <Compile Include="DecompressionModule.cs" />
+    <Compile Include="IProcessingModule.cs" />
+    <Compile Include="ProcessingModule.cs" />
+    <Compile Include="Program.cs" />
+    <Compile Include="Properties\AssemblyInfo.cs" />
+  </ItemGroup>
+  <ItemGroup>
+    <None Include="App.config" />
+  </ItemGroup>
+  <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
+</Project>

GZipTest/IProcessingModule.cs

@@ -0,0 +1,18 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace GZipTest
+{
+    interface IProcessingModule
+    {
+        void Run(string input, string output);
+        void Stop();
+        event ProgressChangedEventHandler ProgressChanged;
+        event EventHandler Complete;
+        event ErrorEventHandler ErrorOccured;
+    }
+}

GZipTest/ProcessingModule.cs

@@ -0,0 +1,146 @@
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Threading;
+
+namespace GZipTest
+{
+    /// <summary>
+    /// Delegate void used to inform UI thread about changed progress
+    /// </summary>
+    /// <param name="done">Amount of blocks that have been done</param>
+    /// <param name="totalSegments">Amount of total blocks</param>
+    public delegate void ProgressChangedEventHandler(long done, long totalSegments);
+
+    public abstract class ProcessingModule : IProcessingModule
+    {
+        public event ProgressChangedEventHandler ProgressChanged;
+        public event EventHandler Complete;
+        public event ErrorEventHandler ErrorOccured;
+
+        internal Thread readingThread, writingThread;
+        internal Thread[] compressionThreads = new Thread[Math.Max(1, Environment.ProcessorCount - 2)];  //If we have ability to use more than 3 threads we add more threads that will proccess blocks (because this operation takes the biggest amount of resources)
+
+        internal Semaphore processed;      //Semaphore will help us to maintain RAM and use minimum of it
+
+        internal ConcurrentQueue<KeyValuePair<int, byte[]>> readBuffer = new ConcurrentQueue<KeyValuePair<int, byte[]>>();   //We use queue for reading and processing blocks since FIFO method is more efficient here
+        internal ConcurrentDictionary<int, byte[]> processedBuffer = new ConcurrentDictionary<int, byte[]>();                //And use dictionary for writing since we need blocks to be placed in order
+
+        //These variables are used for tracking progress
+        internal long segmentCount = 0;
+        internal long served = 0;
+        internal long length;
+
+        //Source and output file paths
+        internal string source, result;
+
+        /// <summary>
+        /// Initializing workflow
+        /// </summary>
+        /// <param name="input">Source file path</param>
+        /// <param name="output">Destination file path</param>
+        public void Run(string input, string output)
+        {
+            //Setting files paths 
+            source = input;
+            result = output;
+
+            //Instantiating threads
+            readingThread = new Thread(Read);
+            writingThread = new Thread(Write);
+
+            for (int i = 0; i < compressionThreads.Length; i++)
+                compressionThreads[i] = new Thread(Process);
+
+            foreach (Thread i in compressionThreads)
+                i.Priority = ThreadPriority.Highest;    //Since compression is the slowest operation it must be marked as high priority task
+
+            //Semaphore will indicate how many blocks can be now written. 
+            //There can be max 5 blocks for each compression thread because there's no reason for more. 
+            //5 block in a row mean that compressing algorithm is faster than writing algorithm so there's no need to process more block until these are done
+            processed = new Semaphore(compressionThreads.Length * 5, compressionThreads.Length * 5);
+
+            //Starting threads
+            readingThread.Start();
+            foreach (Thread i in compressionThreads)
+                i.Start();
+            writingThread.Start();
+        }
+
+        /// <summary>
+        /// Instantly terminates all threads and cleans up stuff
+        /// </summary>
+        public void Stop()
+        {
+            //Terminating threads
+            readingThread.Abort();
+            foreach (Thread thread in compressionThreads)
+                thread.Abort();
+            writingThread.Abort();
+
+            //Collecting garbage (Yours' Cap)
+            GC.Collect();
+        }
+        internal void ReportError(object sender, string message, Exception ex) => ErrorOccured?.Invoke(sender, new ErrorEventArgs(new Exception(message, ex)));
+
+        /// <summary>
+        /// Reading source file
+        /// </summary>
+        internal abstract void Read();
+
+        /// <summary>
+        /// Processes one block. This method is used in Read and Write threads
+        /// </summary>
+        internal abstract void ProcessOne();
+
+        /// <summary>
+        /// Processing read block
+        /// </summary>
+        internal void Process()
+        {
+            try
+            {
+                while (readingThread.IsAlive || readBuffer.Count > 0)                                               //The task will be alive as long as reading is in progress or there's stil any unprocessed blocks
+                    ProcessOne();
+            }
+            catch (Exception e)
+            {
+                ReportError(this, $"Error occured in Compression thread. Served blocks: {served}", e);
+            }
+        }
+
+        /// <summary>
+        /// Writing processed block to disk
+        /// </summary>
+        internal void Write()
+        {
+            try
+            {
+                using (FileStream stream = new FileStream(result, FileMode.Create, FileAccess.Write))   //Instantiating writing stream
+                {
+                    while (compressionThreads.Any(i => i.IsAlive) || processedBuffer.Count > 0)         //The task will be alive as long as compression is in progress or there's stil any unwritten block
+                    {
+                        if (!processedBuffer.TryRemove((int)served, out byte[] block))                  //Extracting block that need to be written next 
+                        {
+                            if (readBuffer.Count > 0)                                                   //Helping processing thread to do its job
+                                ProcessOne();
+                            continue;
+                        }
+
+                        stream.Write(block, 0, block.Length);                                           //Writing block to the file
+                        processed.Release();                                                            //Informing compression threads that they can continue
+
+                        ProgressChanged?.Invoke(++served, segmentCount);                                //Updating progress bar
+                    }
+                }
+                Complete?.Invoke(length / 1024 / 1024, null);
+            }
+            catch (Exception e)
+            {
+                ReportError(this, $"Error occured in writing thread. Blocks served: {served}", e);
+            }
+        }
+    }
+}

GZipTest/Program.cs

@@ -0,0 +1,101 @@
+using System;
+using System.IO;
+
+namespace GZipTest
+{
+    class Program
+    {
+        static DateTime start = DateTime.Now;
+        static IProcessingModule module;
+
+        static int Main(string[] args)
+        {
+            try
+            {
+                //Validating input parameters
+                if (args.Length != 3)
+                    throw new InvalidDataException("Invalid parameters set.\nUsage: NewWinRar.exe [compress|decompress] [source file name] [destination file name]");
+                if (!File.Exists(args[1]))
+                    throw new FileNotFoundException("The source file not found. Make sure it is place in the program's directory and has the same name. Stating extension is required");
+
+                //Instatiating module
+                switch (args[0].ToLower())
+                {
+                    case "compress":
+                        Console.WriteLine("Compressing file...");
+                        module = new CompressionModule();
+                        break;
+                    case "decompress":
+                        Console.WriteLine("Unpacking file...");
+                        module = new DecompressionModule();
+                        break;
+                    default:
+                        throw new InvalidDataException("Invalid parameter. The first parameter must be 'compress' or 'decompress'");
+                }
+
+                //Subscribing to events
+                module.ProgressChanged += SetProgress;
+                module.Complete += Complete;
+                module.ErrorOccured += Module_ErrorOccured;
+
+                //Executing module
+                module.Run(args[1], args[2]);
+
+                return 0;
+            }
+            //Catching errors and displaying them
+            catch (Exception e)
+            {
+                Console.Error.WriteLine($"\n\n{e.ToString()}\n" + e.InnerException != null && e.InnerException != e ? $"\n{e.InnerException.ToString()}\n" : "");
+                return 1;
+            }
+        }
+
+        private static void Module_ErrorOccured(object sender, ErrorEventArgs e)
+        {
+            Console.Error.WriteLine("Error has occured. Threads tremination initiated");
+            Console.Error.WriteLine($"\n\n{e.GetException().ToString()}\n");
+            module.Complete -= Complete;
+            Console.WriteLine("Press any key to continue...");
+            Console.ReadKey();
+            module.Stop();
+        }
+
+        /// <summary>
+        /// Displays complete message and post analysis
+        /// </summary>
+        /// <param name="size">Represents original file size in MB</param>
+        /// <param name="e">Not used</param>
+        private static void Complete(object size, EventArgs e)
+        {
+            TimeSpan elapsed = DateTime.Now - start;
+            Console.WriteLine($"\nDone\nProcessed {size} MB within {elapsed.Minutes} minutes {elapsed.Seconds} seconds\nPress any key to continue...");
+            Console.ReadKey();
+        }
+
+        /// <summary>
+        /// Displaying progress bar which represents current workflow position
+        /// </summary>
+        /// <param name="percentage">Integer from 0 to 100. Represents amount of completed work</param>
+        public static void SetProgress(long done, long totalSegments)
+        {
+            TimeSpan elapsed = DateTime.Now - start;
+            //Border braces
+            Console.CursorLeft = 0;
+            Console.Write("[");
+            Console.CursorLeft = 21;
+            Console.Write("]");
+
+            //Progress bar
+            for (int i = 0; i < done * 20 / totalSegments; i++)
+            {
+                Console.CursorLeft = i + 1;
+                Console.Write("■");
+            }
+
+            //Percentage
+            Console.CursorLeft = 23;
+            Console.Write($"{done * 100 / totalSegments}%   {done} of {totalSegments} blocks [{elapsed.ToString(@"hh\:mm\:ss")}]");
+        }
+    }
+}

GZipTest/Properties/AssemblyInfo.cs

@@ -0,0 +1,36 @@
+using System.Reflection;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+
+// General Information about an assembly is controlled through the following
+// set of attributes. Change these attribute values to modify the information
+// associated with an assembly.
+[assembly: AssemblyTitle("NewWinRar")]
+[assembly: AssemblyDescription("")]
+[assembly: AssemblyConfiguration("")]
+[assembly: AssemblyCompany("")]
+[assembly: AssemblyProduct("NewWinRar")]
+[assembly: AssemblyCopyright("Copyright ©  2019")]
+[assembly: AssemblyTrademark("")]
+[assembly: AssemblyCulture("")]
+
+// Setting ComVisible to false makes the types in this assembly not visible
+// to COM components.  If you need to access a type in this assembly from
+// COM, set the ComVisible attribute to true on that type.
+[assembly: ComVisible(false)]
+
+// The following GUID is for the ID of the typelib if this project is exposed to COM
+[assembly: Guid("c4bc63bf-c2a8-4057-b746-662f9dcf5a6b")]
+
+// Version information for an assembly consists of the following four values:
+//
+//      Major Version
+//      Minor Version
+//      Build Number
+//      Revision
+//
+// You can specify all the values or you can default the Build and Revision Numbers
+// by using the '*' as shown below:
+// [assembly: AssemblyVersion("1.0.*")]
+[assembly: AssemblyVersion("1.0.0.0")]
+[assembly: AssemblyFileVersion("1.0.0.0")]