#! ./perl
#
# NormalText.pl normalizes orthographic texts by removing abreviations. 
# and number. Lines starting with '# ' are
# ignored.
#
# use:
# NormalText.pl <file1> <file2> ...
# cat <file1> <file2> ... | SplitText.pl
#
#          
#
# Author: Rob van Son 
#         email:
#         rob@fon.hum.uva.nl (temporary) 
#	  Rob.van.Son@hum.uva.nl (future) 
# 	  rob.van.son@workmail.com (more private)
# 	  mail:
# 	  Institute of Phonetic Sciences
# 	  University of Amsterdam
# 	  Herengracht 338
# 	  NL-1016CG Amsterdam
# 	  The Netherlands
# 	  tel +31 205252183
# 	  fax +31 205252197
# 	  
# Date:   July 6, 1999
# Version:1.0
# Env:    Perl 5.002
#
################################################################################
#                                                                              #
#          LICENCE                                                             #
#                                                                              #
#          This program is free software; you can redistribute it and/or       #
#          modify it under the terms of the GNU General Public License         #
#          as published by the Free Software Foundation; either version 2      #
#          of the License, or (at your option) any later version.              #
#                                                                              #
#          This program is distributed in the hope that it will be useful,     #
#          but WITHOUT ANY WARRANTY; without even the implied warranty of      #
#          MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the       #
#          GNU General Public License for more details.                        #
#                                                                              #
#          You should have received a copy of the GNU General Public License   #
#          along with this program; if not, write to the Free Software         #
#          Foundation, Inc., 59 Temple Place - Suite 330,                      #
#          Boston, MA  02111-1307, USA.                                        #
#                                                                              #
################################################################################
#
my %NumberList = (
'-' => 'min',
'+' => 'plus',
',' => 'komma',
'0' => 'nul',
'1' => 'eeen',
'2' => 'twee',
'3' => 'drie',
'4' => 'vier',
'5' => 'vijf',
'6' => 'zes',
'7' => 'zeven',
'8' => 'acht',
'9' => 'negen',
'10' => 'tien',
'11' => 'elf',
'12' => 'twaalf',
'13' => 'dertien',
'14' => 'veertien',
'15' => 'vijftien',
'16' => 'zestien',
'17' => 'zeventien',
'18' => 'achttien',
'19' => 'negentien',
'20' => 'twintig',
'30' => 'dertig',
'40' => 'veertig',
'50' => 'vijftig',
'60' => 'zestig',
'70' => 'zeventig',
'80' => 'tachtig',
'90' => 'negentig',
'100' => 'honderd',
'1000' => 'duizend',
'10000' => 'tienduizend',
'1000000' => 'miljoen',
'1000000000' => 'miljard',
'1000000000000' => 'biljoen',
);

my %CurrencyList = (
'HFL' => 'gulden',
'FL' => 'gulden',
'USD'  => 'dollar',
'UKP'  => 'pond',
'DM' => 'mark',
'FF' => 'frank',
'BF' => 'frank',
'EUR' => 'euro'
);
my $CurrencySearch = join("|", keys(%CurrencyList));
my %CurrencyPart = (
'HFL' => 'cent',
'FL' => 'cent',
'USD'  => 'dollarcent',
'UKP'  => 'pennies',
'DM' => 'pfennig',
'FF' => 'centimes',
'BF' => 'centimes',
'EUR' => 'cent'
);

sub ExpandLongNumber  # (LongInteger) -> Text
{
    my $Integer = shift;
    my @LongNumber = ();
    my $Log = "";

    #
    # . separates thousands, "1xxx" is expanded as "1x honderd xx"
    while((!$Log && ($Integer =~ /^((1\d)((\d)(\d)))$/isg)) ||
	  ($Integer =~ /\.?((\d)?((\d?)(\d)))$/isg))
    {
	$Integer = $`;
	my @Hundred = ($NumberList{$1}) || (); 
	if(@Hundred)
	{
	    pop(@Hundred) if length($Log) < 6 && $1 <= 1;
	    push(@Hundred, $NumberList{'1'.$Log}) if $Log;
	    unshift(@LongNumber, join("-", @Hundred));
	    $Log .= "000";
	    next;
	};
	@Hundred = ($NumberList{$2}) if $4 ne "" && $2 > 1;
	push(@Hundred, "honderd") if $4 ne "" && $2 > 0;

	if($4 && $NumberList{$3})
	{
	    push(@Hundred, $NumberList{$3});
	}
	else
	{
	    my $tens = $4 ne "" ? $4 : $2;
	    push(@Hundred, $NumberList{$5}) if $5 > 0;
	    push(@Hundred, "en") if $5 > 0 && $tens > 0;
	    push(@Hundred, $NumberList{$tens.'0'}) if $tens > 0;
	};
	push(@Hundred, $NumberList{'1'.$Log}) if $Log;
	unshift(@LongNumber, join("-", @Hundred));
	$Log .= "000";
    };
    return join(" ", @LongNumber);
}

sub NormalizeLine   # ($Line) -> $Line
{
    my $newline = shift;  # Start of text

    return $newline if $newline =~ /^\s*\#/;
    
    # Transform odd quotes
    $newline =~ s/[\ą\Ś]/\'/sg;
    $newline =~ s/\,\,|\'\'/\"/g;

    # Expand abreviations
    $newline =~ s/\bo\.?a\b/onder andere/isg;	# o.a.
    $newline =~ s/\bm\.?n\b/met name/isg;	# m.n.
    $newline =~ s/\bn\.?b\b/nota bene/isg;	# n.b.
    $newline =~ s/\bv\.?w\b/vorige week/isg;	# v.w.
    $newline =~ s/\bv\.?v\b/vice versa/isg;	# v.v.
    $newline =~ s/\betc\./etcetera/isg;		# etc.
    $newline =~ s/\benz\./enzovoorts/isg;	# enz.
    $newline =~ s/\bj\.?l\b/jongstleden/isg;	# j.l.
    $newline =~ s/\bv\.l\.n\.r\b/van links naar rechts/isg;	# v.l.n.r.
    $newline =~ s/\bjr(\.|\b)/junior/isg;	# jr.
    $newline =~ s/\ba\.?s\b/aanstaande/isg;	# a.s.
    $newline =~ s/\bincl\./inclusief/isg;	# incl.
    $newline =~ s/\bp\.?p(\.|\b)/per persoon/isg;	# p.p.
    $newline =~ s/\bp\.?d(\.|\b)/per dag/isg;	# p.d.
    $newline =~ s/\bp\.?w(\.|\b)/per week/isg;	# p.w.
    $newline =~ s/\bp\.?m(\.|\b)/per maand/isg;	# p.m.
    $newline =~ s/\bp\.?j(\.|\b)/per jaar/isg;	# p.j.
    $newline =~ s/\bing(\.|\b)/ingenieur /isg;	# ing.
    $newline =~ s/\bir(\.|\b)/ingenieur /isg;	# ir.
    $newline =~ s/\bmr(\.|\b)/meester /isg;	# mr.
    $newline =~ s/\bdr(\.|\b)/doctor /isg;	# dr.
    $newline =~ s/\bdrs(\.|\b)/doctorandus/isg;	# drs.
    $newline =~ s/\bt\/m\b/tot en met/isg;	# t/m
    $newline =~ s/\bprof(\.|\b)/professor /isg;	# prof.
    $newline =~ s/\bred\./redactie/isg;		# red.
    $newline =~ s/\btel[\.\:\s]/telefoon /isg;	# tel.
    $newline =~ s/\bnr[\.\:\s]/nummer /isg;	# nr.
    $newline =~ s/\bVS\b/Verenigde Staten/g;	# VS
    $newline =~ s/(\b|\d)\%/ procent/g;		# %
    $newline =~ s/\ź/ fl. /g;    		# Fl symbol
    $newline =~ s/\$/USD/g;        		# US dollar symbol
    $newline =~ s/\Ł/UKP/g;        		# UK pound symbol

    # Currency
    $newline =~ s/\b($CurrencySearch)\.?\s*[0]+\,(\d\d)\b/"$2 ".$CurrencyPart{uc($1)}/eisg;
    $newline =~ s/\b($CurrencySearch)\.?\s*([\.\d]*\d)\,\0\0\b/"$2 ".$CurrencyList{uc($1)}/eisg;
    $newline =~ s/\b($CurrencySearch)\.?\s*([\.\d]*\d)\,\-/"$2 ".$CurrencyList{uc($1)}/eisg;
    $newline =~ s/\b($CurrencySearch)\.?\s*([\.\d]*\d)\,(\d\d)\b/"$2 ".$CurrencyList{uc($1)}." $3"/eisg;
    $newline =~ s/\b($CurrencySearch)\.?\s*([\.\d]*\d)(\,[^\d]|[^\,\d]|$)/"$2 ".$CurrencyList{uc($1)}.$3/eisg;
    # Replace $ etc back
    $newline =~ s/\bUSD([^a-zA-Z]|$)/\$$1/sg;        	# US dollar symbol
    $newline =~ s/\bUKP([^a-zA-Z]|$)/\Ł$1/sg;        	# UK pound symbol

    # Time
    $newline =~ s/\b([012]?\d)\:(00)\s*(h|uur|u)?\b/$1 uur/isg;
    $newline =~ s/\b([012]?\d)\:([0-6]\d)\s*(h|uur|u)?\b/$1 uur $2/isg;

    # (Dutch) Telephone numbers
    $newline =~ s/(\b0|\+\d\d? )(\d\d)\s*\-?\s*(\d\d\d)(\d\d)(\d\d)\b/$1 $2 $3 $4 $5 /isg;

    # Expand numbers
    while($newline =~ /([\-\+]?)([\d\.\,]*\d)/)
    {
	my $Prev = $`;
	my $Post = $';
	my $Sign = $1;
	my $Number = $2;
	my $Transcription = "";

	my $Integer = $Number;
	my $Separator = "";
	my $Fraction = "";
	# The decimal sign is ','
	if($Number =~ /[\,]/)
	{
	    $Integer = $`;
	    $Separator = $&;
	    $Fraction = $';
	};

	$Number = ("$NumberList{$Sign} " || "");

	if($Transcription = $NumberList{$Integer})
	{
	    $Number .= " $Transcription";
	}
	else
	{
	    $Number .= ExpandLongNumber($Integer);
	}


	# After the decimal Separator
        $Number .= " $NumberList{$Separator} " if $Separator;

        my  @Parts = ();
	# Double places are expanded as two digit-numbers
        $Parts[0] = ExpandLongNumber($1), ($Fraction = $`) if $Fraction =~ /^(\d\d?)$/isg;

	# More digits are expanded digit-by-digit
        while($Fraction =~ s/([\d])$//isg)
        {
   	    unshift(@Parts, $NumberList{$1}); 
	};
	$Number .= join(" ", @Parts);

	$newline = $Prev.$Number.$Post;
    };
    
    # Handle special cases
    $newline =~ s/\b[\ée]én\b/eeen/g;
    $newline =~ s/\=/ is /g;
    $newline =~ s/\&/ en /g;

    # Ready
    return $newline;
} 

#
# Do not execute inside a require
unless(caller())
{
while(<>)
{
    # Skip "Comments"
    ((print STDOUT $_) && next) if /^\s*\#/;

    $newline = NormalizeLine($_);  # Start of text
    
    print STDOUT "$newline";
    
};
};

1; # Make require happy