Hello,

Below is code that compares 122 text files (there are two copies of each file, making 61 old copies and 61 new copies). These are data files from a database and can be extremely large. I am having a problem when I hit my largest file (471,483 KB and it grows every day depending on what is added to it). In my last test run the two files were 471,483 KB and 485,359 KB). I compare these files to only extract the new data to a new text file that i write in another direcotry. Is there a way i can handle/free up memory to handle this exception? Thanks in advance.
Code:
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;

namespace ConsoleApplication1
{
    class Program
    {
        static void Main(string[] args)
        {

            Console.WriteLine("Comparing text files to find new data...");

            //these are the folders that have the data
            // i perform two loops to go through both of them
            string folderUsing;

            try
            {
                //to iterate over both folders
                for (int i = 1; i <= 2; i++)
                {
                    if (i == 1)
                    {
                        folderUsing = "dataset1";
                    }
                    else
                    {
                        folderUsing = "dataset2";
                    }

                    Console.WriteLine();
                    Console.WriteLine("Using data from {0}", folderUsing);

                    //New appended data location, this folder will always be there
                    string folderSending = "C:\\Reporting\\" + folderUsing + "\\Uploads\\";

                    //New Data location, this folder will always be there
                    string folderNewData = "C:\\Reporting\\" + folderUsing + "\\NewData";

                    //Old files get moved here, need to create this folder if not one already
                    //this acts as my backup
                    string folderArchive = "C:\\Reporting\\" + folderUsing + "\\" + DateTime.Now.ToString("yyyyMMdd") + "\\";

                    //Prior files location, this folder will always be there
                    string folderPrior = "C:\\Reporting\\" + folderUsing + "\\";

                    //Create the new data folder if it does not exist
                    if (!Directory.Exists(folderArchive))
                    {
                        Directory.CreateDirectory(folderArchive);
                    }

                    //Prelim test, make sure all files in new dir are in old dir
                    //if there is a new file move it into the new directory immediately
                    string[] fileEntriesNew = Directory.GetFiles(folderNewData);
                    foreach (string fileName in fileEntriesNew)
                    {

                        string newFileName = Path.GetFileName(fileName);
                        FileInfo newFileInfo = new FileInfo(fileName);
                        FileInfo oldFileInfo = new FileInfo(folderPrior + newFileName);

                        //Check old dir name for similar file, if it does not exist, copy entire file over
                        if (!File.Exists(folderPrior + newFileName))
                        {
                            Console.WriteLine("{0} does not exist.", folderPrior + newFileName);
                            Console.WriteLine("{0} will be copied over.", fileName);
                            Console.WriteLine("Press enter to continue");
                            Console.ReadLine();
                            File.Move(fileName, folderSending);
                        }
                        else
                        {
                            Console.WriteLine();
                            Console.WriteLine("Comparing files {0}, {1} KB", newFileName, newFileInfo.Length / 1024);

                            //the new file should never be less than the old file size
                            if (newFileInfo.Length < oldFileInfo.Length)
                            {
                                Console.WriteLine("Possible error: The new file is smaller than the old file");
                                Console.WriteLine("Press enter to continue");
                                Console.ReadLine();
                            }

                            //if there are two similar files open them both
                            if (File.Exists(folderPrior + newFileName))
                            {

                                Console.WriteLine("Reading files");
                                //IEnumerable data sources (arrays), this will get all lines in newText not in oldText
                                string[] newText = File.ReadAllLines(fileName);
                                string[] oldText = File.ReadAllLines(folderPrior + newFileName);

                                //The query based on the data sources
                                IEnumerable<string> differenceQuery = newText.Except(oldText);

                                //Any will get any differences and I do not want the first line to be blank
                                if (differenceQuery.Any() && differenceQuery.First() != "")
                                {

                                    Console.WriteLine("Outputting new lines");

                                    using (StreamWriter fsSending = new StreamWriter(folderSending + newFileName, true))
                                    {

                                        foreach (string newLine in differenceQuery)
                                        {
                                            fsSending.WriteLine(newLine);
                                        }

                                    }
                                }

                            }
                        }

                    }

                    //this will move new files in to the old dir and old files into the archive dir
                    Console.WriteLine();
                    Console.WriteLine("Moving files to appropriate destination");

                    string[] fileEntriesOld = Directory.GetFiles(folderNewData);
                    foreach (string fileName in fileEntriesOld)
                    {
                        string fileMove = Path.GetFileName(fileName);
                        File.Move(folderPrior + fileMove, folderArchive + fileMove);
                        File.Move(fileName, folderPrior + fileMove);
                    }

                }

            }

            catch (Exception e)
            {
                Console.WriteLine("The process failed: {0}", e.ToString());
                throw;
            }
            finally
            {
                Console.WriteLine();
                Console.WriteLine("Program finished");
            }

        }
    }
}