Loader.cpp

/*
 *  Loader.cpp
 *  xwall
 *
 *  Created by Tim Medcalf on 18/10/2010.
 *  Copyright 2010 ErgoThis Ltd. All rights reserved.
 *
 */

#include "Loader.h"
#include "Global.h"
#include <vector>
#include <fstream>
#include <iostream>
#include "TextPreprocessor.h"
#include "TextFreq.h"
#include <map>

vector<TextFreq> Loader::load_file(string &thisfile)
{
	//irrespective of the type of file
	// we need to load the lines of the file
	//into a list
	vector<string>source_text;// = new vector<string>;
	string line;
	ifstream inputfile (thisfile.c_str());
	if (inputfile.is_open())
	{
		while (! inputfile.eof() )
		{
			getline(inputfile,line);
			source_text.push_back(line);
		}
		inputfile.close();
	}
	else 
	{
		cout << "Unable to open file" << endl;
    exit(EXIT_FAILURE);
	}
	
	return (settings.input_type == ip_text) ?  load_text_file(source_text) : load_frequency_file(source_text);
	
}

vector<TextFreq> Loader::load_text_file(vector<string> &source_text)
{
	if (!settings.quiet)  { cout << "Cleaning Text..." << endl; }
	TextPreprocessor clean_text;
	vector<string>allwords;
  for (int i = 0; i<source_text.size();i++)
	{
		//get a list of words from current line
		vector<string> linewords = clean_text.getwords(source_text[i]);
		//copy them on to the end of the full words list
		allwords.insert(allwords.end(), linewords.begin(), linewords.end());
	}
	//delete clean_text;
	if (!settings.quiet) { cout << "(" << allwords.size() << " total words)" << endl;}
	
	//now lets sort through the list identifying unique words and storing their frequency	
	if (!settings.quiet)  { cout << "Ranking text..." << endl; }
	
	vector<TextFreq> wordfreqs = ranktext(allwords);
	if (!settings.quiet) { cout << "(" << wordfreqs.size() << " distinct words)" << endl; }

  trim_words(wordfreqs);
  if (settings.frequency_list_output_file.length() > 0)
  {
    //need to loop through the freq list and save the output
    ofstream outputfile (settings.frequency_list_output_file.c_str(), ios_base::trunc);
    if (outputfile.is_open())
    {
      for(vector<TextFreq>::iterator itf = wordfreqs.begin(); itf < wordfreqs.end(); itf++)
      {
        outputfile << (*itf).text.c_str() << "=" << (*itf).freq << endl;
      }
      outputfile.close();
    }    
  }
	return wordfreqs;
	
}

vector<TextFreq> Loader::load_frequency_file(vector<string> &source_text)
{
  WordFilter filterer;
	vector<TextFreq> wordfreqs;
  
	size_t pos;
	for (int i = 0; i<source_text.size();i++)
	{
		string tmpstr = source_text[i];
		pos = tmpstr.find("=");
		string text = tmpstr.substr(0,pos);
		int freq = atoi(tmpstr.substr(pos+1).c_str());
		if ((freq > 0) && ((!settings.remove_common_words) || (filterer.allowed(text))))
		{
			TextFreq newword = TextFreq(text);
			newword.freq = freq;
			wordfreqs.push_back(newword);
		}
	}
	//lets sort the list based on the word count
	sort(wordfreqs.begin(),wordfreqs.end(), sortbyoccurrences());
  trim_words(wordfreqs);
  if (settings.frequency_list_output_file.length() > 0)
  {
    //need to loop through the freq list and save the output
    ofstream outputfile (settings.frequency_list_output_file.c_str(), ios_base::trunc);
    if (outputfile.is_open())
    {
      for(vector<TextFreq>::iterator itf = wordfreqs.begin(); itf < wordfreqs.end(); itf++)
      {
        outputfile << (*itf).text.c_str() << "=" << (*itf).freq << endl;
      }
      outputfile.close();
    }    
  }
	return wordfreqs;
}

void Loader::trim_words(vector<TextFreq> &wordfreqs)
{
	if (settings.maximum_words == 0) { settings.maximum_words = (int)wordfreqs.size();}
	
	//make sure max_words is not bigger than the actual amount of words!
	settings.maximum_words = min(settings.maximum_words, (int)wordfreqs.size());
	
	if (!settings.quiet) { cout << "Using the " << settings.maximum_words << " most common unique words."; }
	if (!settings.quiet) { settings.allow_extra_words ? cout << " (plus any extra words that fit in)" << endl : cout << endl; }
	

	//clear out words we don't need
	if ((!settings.allow_extra_words) && (wordfreqs.size() > settings.maximum_words))
	{ 
		//if we're only going to use a set amount of words, we might as well get rid of the rest
		wordfreqs.erase(wordfreqs.begin() + settings.maximum_words ,wordfreqs.end());
	}
}


vector<TextFreq> Loader::ranktext(vector<string> &allwords)
{
  //create a map to store words as we scan through the list
  //if we've already found a word, just increase the count
  
  //create a map to store it
	map<string,TextFreq> mapwords;
	map<string,TextFreq>::iterator itmap;
	//pair<string,TextFreq> tmppair;
	//now we need to iterate through the list and count up the dups
	for(vector<string>::iterator it = allwords.begin(); it < allwords.end(); it++) {
    string tmpstr = *it;
		itmap = mapwords.find(tmpstr);
		//have we found the word?
		if (itmap == mapwords.end())
		{
			//nope - we need to add it then
			TextFreq newword = TextFreq(tmpstr);
			mapwords.insert(pair<string,TextFreq>(newword.text,newword));
		}
		else 
		{
			itmap->second.inc();
		}
	}
	
	//okay - we need to turn the map into a straight list of words
	vector<TextFreq> words;
	//vector<TextFreq>::iterator itcw;
	
	words.reserve(mapwords.size());
	for (itmap = mapwords.begin(); itmap != mapwords.end(); itmap++) {
		words.push_back(itmap->second);
	}
	
	//lets sort the list based on the word count
	sort(words.begin(),words.end(), sortbyoccurrences());

	return words;
}