#! /usr/bin/perl
#
# Celex2CGNLexicon.pl converts a classical Celex tab-delimited word transcription
# list (gnu zipped!) to the transcription used by CGN. It adds broad feature value 
# transcriptions, i.e., Manner ('V'owel, 'G'lide, 'N'asal, 'S'top, 'F'ricative, 
# 'O'ther), Articulator ('O'ral, 'L'abial, 'C'oronal, 'D'orsal, 'B'ack, 'G'lottal)
# and voice ('+', '-').
# 
# A "word" can consist of two separate orthograqphic words, e.g., 'Agnus Dei'.
# All words are printed in LOWERCASE.
#
# use:
# Celex2CGNLexicon.pl <SourceLexicon.gz> 
#
# SourceLexicon can be compressed (gzip) or not. It can also be a pipe,
# i.e., STDIN ('-');
#
# It assumes the following content in the input:
# OrthographicWord	Transcription	StressPattern	Frequency	POS
#
# Empty fields are left empty.
#
# This program is very straightforeward. It reads a large transcription dictionary
# and just replaces the Celex transcription with a tab-delimited list of:
# 1) A CGN compatible phoneme transcription
# 2) as 1) but without the syllable boundaries (-) and stress markers (")
# 3) A transcription in terms of Manner of Articulation
# 4) A transcription in terms of Primary Articulator
# 5) A transcription in terms of Voicing
#
# The file is preceded by a Copyright Notice and a list of column names
# types and descriptions.
#          
#
# Author: Rob van Son 
#         email:
#	  Rob.van.Son@hum.uva.nl
# 	  rob.van.son@workmail.com (more private)
# 	  mail:
# 	  Institute of Phonetic Sciences
# 	  University of Amsterdam
# 	  Herengracht 338
# 	  NL-1016CG Amsterdam
# 	  The Netherlands
# 	  tel +31 205252183
# 	  fax +31 205252197
# 	  
# Date:   January 28, 2002
# Version:1.0
# Env:    Perl 5.6
#
################################################################################
#                                                                              #
#          LICENCE                                                             #
#                                                                              #
#          This program is free software; you can redistribute it and/or       #
#          modify it under the terms of the GNU General Public License         #
#          as published by the Free Software Foundation; either version 2      #
#          of the License, or (at your option) any later version.              #
#                                                                              #
#          This program is distributed in the hope that it will be useful,     #
#          but WITHOUT ANY WARRANTY; without even the implied warranty of      #
#          MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the       #
#          GNU General Public License for more details.                        #
#                                                                              #
#          You should have received a copy of the GNU General Public License   #
#          along with this program; if not, write to the Free Software         #
#          Foundation, Inc., 59 Temple Place - Suite 330,                      #
#          Boston, MA  02111-1307, USA.                                        #
#                                                                              #
################################################################################
#
# The user definable transcription table
######################################################
#                                                    #
#                                                    #
#    Conversion Table to go from Celex to CGN labels #
#                                                    #
#                                                    #
######################################################
my %SAMPAtoCGNtable =
(
'p' => 'p',
'b' => 'b',
't' => 't',
'd' => 'd',
'k' => 'k',
'g' => 'g',
'f' => 'f',
'v' => 'v',
's' => 's',
'z' => 'z',
'S' => 'S',
'Z' => 'Z',
'x' => 'x',
'G' => 'G',
'h' => 'h',
'N' => 'N',
'm' => 'm',
'n' => 'n',
'nj' => 'J',
'l' => 'l',
'r' => 'r',
'w' => 'w',
'j' => 'j',
'I' => 'I',
'E' => 'E',
'A' => 'A',
'O' => 'O',
'}' => 'Y',
'i:' => 'i',
'y:' => 'y',
'e:' => 'e',
'|:' => '2',
'a:' => 'a',
'o:' => 'o',
'u:' => 'u',
'@' => '@',
'EI' => 'E+',
'/I' => '9+',
'Au' => 'O+',
'E:' => 'E:',
'/:' => '9:',
'Q:' => 'O:',
'E~' => 'E~',
'A~' => 'A~',
'O~' => 'O~',
'Y~' => 'Y~'
);

# Convert SAMPA phoneme labels into CGN phoneme lables
sub SAMPAtoCGN	# (SAMPAstring) -> CGNstring
{
	my $string = shift || "";
	$string =~ 
	s/([\/\@AEIOQIYiyaeou29\|\}]{2}|\S[\+\~\:]?)/$SAMPAtoCGNtable{"$1"} ? $SAMPAtoCGNtable{"$1"} : $1;/iseg;
	return $string;
};

# Celex POS to CGN pos
my %ConvertPOS=	# {CelexPosName} -> CGNPosName
(
'A' => 'ADJ',
'ADV' => 'BW',
'ART'  => 'LID',
'C' => 'VG',
'EXP' => 'N',
'I' => 'TSW',
'N' => 'N',
'NUM' => 'TW',
'PREP'  => 'VZ',
'PRON' => 'VNW',
'V' => 'WW'
);

#
#
#
$HomeDir = '..';
require "$HomeDir/Links.pl";
require "$Scripts/CGN_phonemes.pl";  # For the phoneme classes
#
# The Column Headers list will add the corresponding column headers.
my @LexiconColumnHeaders = (
'Number [INT4] Ordinal number of word in list (a kind of ID)',
'Word [Text] Orthographic transcription of word', 
'CGN [TEXT] Phonemically transcribed word (CGN format)', 
'Phonemes [TEXT] Phonemically transcribed word WITHOUT syllable boundaries or stress markers (CGN format)', 
'Manner [TEXT] Manner of Articulation', 
'Articulator [TEXT] Primary Articulator', 
'Voicing [TEXT] Voiced (V) / Unvoiced (U)',
'STRESSPAT [TEXT] Syllable Stress (1/0) pattern', 
'FREQUENCY [INT4] Number of occurrences', 
'CLASS [CHAR(4)] Part of Speech category'
);


#
# Get Input File and open it
#
my $InputFileName = shift(@ARGV) || '-';
if($InputFileName =~ /\.gz$/)
{
	open(INPUT, "gunzip -c '$InputFileName' | ") || die "gunzip -c '$InputFileName': $!\n";
}
else
{
	open(INPUT, "<$InputFileName") || die "$InputFileName: $!\n";
};



# Print the column headers
print STDOUT "\#> ", join("\n\#> ", @LexiconColumnHeaders), "\n";

# Print the copyright license
if(open(LICENSE, "<$HomeDir/DatabaseFiles/scripts/ConstructTables/licenseterms.txt"))
{
    my $Date = localtime();
    $Date =~ /\s([\d]+)\s*$/;
    my $Year = $1;
    while(<LICENSE>)
    {
	    if(/\©/ && ! /$Year/)
	    {
		    s/(\d)\s{6}(\s+\#)/\1, $Year\2/ig;
	    };
	    print STDOUT $_;
    };
    close(LICENSE)
};

# Keep track of the summ frequency
my $SummFrequency = 0;
# Start at 1
my $WordCounter = 1;
while(<INPUT>)
{
	#Skip comments
	next if /^\s*\#[\s\#]/;
	
	#Skip empty lines
	next unless /\S/;
	chomp;
	
	# Split the input into tab-delimited fields
	($Orthography, $Celex, $Stress, $Frequency, $CelexPOS) = split(/\t/);

	# Convert Celex POS to the CGN version (more or less)
	$POS = exists($ConvertPOS{$CelexPOS}) ? $ConvertPOS{$CelexPOS} : $CelexPOS;
	
	# Calculate the summ frequency
	$SummFrequency += $Frequency;
	
	# Start outputline
	my @OutputLine = ();
	push(@OutputLine, $WordCounter, $Orthography);
	
	# Convert $Celex into CGN
	my $CGN = SAMPAtoCGN($Celex);
	my $NoSyllCGN = $CGN;
	$NoSyllCGN =~ s/[\-\"]//g;
	
	# Construct Manner of Articulation, Articulator, and voicing
	my $Manner = CGNtoManner($NoSyllCGN);
	my $Articulator = CGNtoArticulator($NoSyllCGN);
	my $Voicing = CGNtoVoicing($NoSyllCGN);
	
	#
	push(@OutputLine, $CGN, $NoSyllCGN, $Manner, $Articulator, $Voicing);
	
	# Add other entries
	$Frequency = 0 unless $Frequency > 0;
	push(@OutputLine, $Stress, $Frequency, $POS);
	
	# Print them
	print STDOUT join("\t", @OutputLine), "\n";
	
	++$WordCounter;
};

# Print the summ frequency
print STDOUT "\# Summ Frequency = $SummFrequency\n";
