#! /usr/bin/perl
#
# Read the text files from ../Transcriptions and extract Ngram tables
# from the Informal and Retold styles. Store Tables in 
# ../DatabaseFiles/Ngrams.
#
# Use: 
# ConstructNgramTables <SpeakerCode1> <SpeakerCode2> ...
#
###############################################################################
#
# Copyright R.J.J.H. van Son © 2000, 2001
#
# Author Rob van Son
# Institute of Phonetic Sciences & ACLC
# University of Amsterdam
# Herengracht 338
# NL-1016CG Amsterdam, The Netherlands
# Email: Rob.van.Son@hum.uva.nl
#        rob.van.son@workmail.com
# WWW  : http://www.fon.hum.uva.nl/rob/
# mail:  Institute of Phonetic Sciences
#        University of Amsterdam
#        Herengracht 338
#        NL-1016CG Amsterdam
#        The Netherlands
#        tel +31 205252183
#        fax +31 205252197
#
# License for use and disclaimers
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
#
#
#######################################################
#

$HomeDir = '../../..';
require "$HomeDir/Links.pl";
require "$Scripts/SplitText.pl";

my $Separator = '/';
my $DatabaseFiles = "$DatabaseTables${Separator}Ngrams${Separator}";

my $TableName = "Ngram";

# List of styles to use
my %StyleList = (
'VI' => 1,
'FR' => 49,
'VR' => 98
);

my $PunctuationChars = '(^|\s)[\,\:\;\`\'\-\"](\s|$)';

#
# Split a line of text into words
my $PunctuationChars = '(^|\s)[\,\:\;\`\'\-](\s|$)';
sub SplitIntoWords      # ($line) -> @LineOfWords;
{
    my $Line = shift || " ";
    
    my @LineOfWords = (" ");
    my @AllWords = ();
    
    # Empty line contains a single space
    return @LineOfWords unless $Line =~ /\S/;
    
    # Split line to words
    $Line =~ s/([\'\`]s)\s+(morg|ocht|mid|avond|nacht|here|graven|hert)/\1\-\2/g;
    @LineOfWords = split(" ", $Line);
    
    while(@LineOfWords)
    {
        if($LineOfWords[0] =~ /\</)
        {
            # Brackets are currently removed
            my @Brackets = ();
            while($LineOfWords[0] !~ /\>/)
            { 
                push(@Brackets, shift(@LineOfWords));
            };
            push(@Brackets, shift(@LineOfWords));
        }
        else
        {
            push(@AllWords, $LineOfWords[0]);
            shift(@LineOfWords);
        };
    };
    
    return @AllWords;
};

#
# The column headers and types
my @ColumnHeaders1gram = (
'word [TEXT] word value (text)',
'Frequency [INT4] Number of occurrences',
'LogFreq [FLOAT8] Logarithm (base 2) of the Frequency'
);

my @ColumnHeaders2gram = (
'word [TEXT] word value (text)',
'prevword [TEXT] previous word value (text)',
'Frequency [INT4] Number of occurrences',
'LogFreq [FLOAT8] Logarithm (base 2) of the Frequency'
);

my @ColumnHeaders3gram = (
'word [TEXT] word value (text)',
'prevword [TEXT] previous word value (text)',
'nextword [TEXT] next word value (text)',
'Frequency [INT4] Number of occurrences',
'LogFreq [FLOAT8] Logarithm (base 2) of the Frequency'
);

# Make sure ARGV contains non-empty items
@ARGV = grep(/\S/, @ARGV);

# If @ARGV is empty, do all
@ARGV = ('*') unless @ARGV;

my $SpeakerID = "";

# Handle all speakers in the arguments
foreach $SpeakerID (@ARGV)
{
    # Get all files
    my @FileList = glob("$TranscriptionsDir/$SpeakerID${Separator}*.txt");

    # Get information
    my $FilePath;
    foreach $FilePath (@FileList)
    {
	$FilePath =~ m@([^$Separator]+)\.txt$@;
	my $FileName = $1;
	if($FileName =~ /^([FM]\d+[A-Za-z]+)(\d+)(([FVP])([A-Za-z]+)\d*)(\_(.+))?$/)
	{
	    # Split the Item ID into its components
	    my $CurrentSpeakerID = $1;
	    my $CurrentRecording = $2;
	    my $CurrentChunk = $3;
	    my $CurrentType = $4;
	    my $CurrentStyle = $5;
	    my $Author = $6 ? $6 : 'NULL';

	    # Make Other Values readable
	    my $CurrentSex = $1 if $CurrentSpeakerID =~ /^([A-Z])/i;
	    my $CurrentAge = $1 if $CurrentSpeakerID =~ /(\d+)/;
	    my $CurrentTextDescr = $RealNames{$CurrentType} ? $RealNames{$CurrentType} : $CurrentType;
	    $CurrentTextDescr .= $RealNames{$CurrentStyle} ? $RealNames{$CurrentStyle} : $CurrentStyle;
	
	    # Get information on file contents
	    my $CharacterCount = 0;
	    my $WordCount = 0;
	    my $LineCount = 0;
	    my $SentenceCount = 0;
	    
	    # Skip unwanted styles
	    next unless $StyleList{$CurrentStyle};
	    
	    open(TEXTFILE, $FilePath) || die "$FilePath: $!\n";
	    # Split files into sentences
	    my $AllSentences = SplitAllLines(<TEXTFILE>);

	    my $PreviousWord = '*';
	    my $Word;
	    my $NextWord = '*'
	    foreach $Sentence (@$AllSentences)
	    {
		# Skip empty lines
		next unless $Sentence =~ /\S/;
		next if $Sentence =~ /^\#/;
		
		chomp($Sentence); # Remove EOL
		
		# Cut sentence into single words (taking care of 's )
		my @WordSentence = SplitIntoWords($Sentence);
		
		# Fill tables
		my $PreviousWord = '*';
		my $i;
		for($i=0; $i < scalar(@WordSentence); ++$i)
		{
		};
	    };
	    
	    # Store Database information
	    my $CurrentDatabaseFile = "${CurrentSpeakerID}$TableName.txt";
	    # Create a database file if it doesn't exist
	    unless(-e "$DatabaseFiles$CurrentDatabaseFile")
	    {
		open(DATABASE, ">$DatabaseFiles$CurrentDatabaseFile") 
		|| die "$DatabaseFiles$CurrentDatabaseFile: $!\n";
		print DATABASE "\#> ", join("\n\#> ", @ColumnHeaders), "\n# \n# $TableName\n# \n";

		# Print the copyright license
		if(open(LICENSE, "<./licenseterms.txt"))
		{
			my $Date = localtime();
			$Date =~ /\s([\d]+)\s*$/;
			my $Year = $1;
			while(<LICENSE>)
			{
				if(/\©/ && ! /$Year/)
				{
					s/(\d)\s{6}(\s+\#)/\1, $Year\2/ig;
				};
				print DATABASE $_;
			};
			close(LICENSE)
		};

		close(DATABASE);
	    };
	    
	    my $CurrentRecord = ""
	    ."$FileName\t"
	    ."$CurrentSpeakerID\t"
	    ."$CurrentSex\t"
	    ."$CurrentAge\t"
	    ."$CurrentType\t"
	    ."$CurrentStyle\t"
	    ."$CurrentTextDescr\t"
	    ."$CurrentRecording\t"
	    ."$CurrentChunk\t"
	    ."$CurrentChunkNumber\t"
	    ."$Author\t"
	    ."$SentenceCount\t$LineCount\t$WordCount\t$CharacterCount\t"
	    ."\n";
	    
	    
	    # Open the correct DatabaseFile and add the current record
	    open(DATABASE, ">>$DatabaseFiles$CurrentDatabaseFile") 
	    || die "$DatabaseFiles$CurrentDatabaseFile: $!\n";
	    print DATABASE $CurrentRecord;
	    close(DATABASE);
	};
	
    };
    
    
};

