#! ./perl
#
# SplitText.pl splits orthographic texts into sentences. The
# output is a sentence a line. Lines starting with '# ' are
# ignored.
#
# use:
# SplitText.pl <file1> <file2> ...
# cat <file1> <file2> ... | SplitText.pl
#
# Author: Rob van Son 
#         email:
#         rob@fon.hum.uva.nl (temporary) 
#	  Rob.van.Son@hum.uva.nl (future) 
# 	  rob.van.son@workmail.com (more private)
# 	  mail:
# 	  Institute of Phonetic Sciences
# 	  University of Amsterdam
# 	  Herengracht 338
# 	  NL-1016CG Amsterdam
# 	  The Netherlands
# 	  tel +31 205252183
# 	  fax +31 205252197
# 	  
# Date:   July 6, 1999
# Version:1.0
# Env:    Perl 5.002
#
#
#          
################################################################################
#                                                                              #
#	Changes                                                                #
#	           							       #
#	14 Dec 1999 - Added support for 't and 'n type of clitics at           #
#		      sentence start                                           #
#                                                                              #
################################################################################
#                                                                              
#                                                                              
#          								
################################################################################
#                                                                              #
#          LICENCE                                                             #
#                                                                              #
#          This program is free software; you can redistribute it and/or       #
#          modify it under the terms of the GNU General Public License         #
#          as published by the Free Software Foundation; either version 2      #
#          of the License, or (at your option) any later version.              #
#                                                                              #
#          This program is distributed in the hope that it will be useful,     #
#          but WITHOUT ANY WARRANTY; without even the implied warranty of      #
#          MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the       #
#          GNU General Public License for more details.                        #
#                                                                              #
#          You should have received a copy of the GNU General Public License   #
#          along with this program; if not, write to the Free Software         #
#          Foundation, Inc., 59 Temple Place - Suite 330,                      #
#          Boston, MA  02111-1307, USA.                                        #
#                                                                              #
################################################################################
#
my @SquoteReplaceList = (avonds, morgens, middags, nachts, herens);
my $SreplaceList = join("|", );
sub SplitAllLines	# (@AllLines) -> \@AllSentences;
{
    my @AllLines = @_;
    my @AllSentences = ();
    my $line = "";
    while(@AllLines || $Line)
    {
        my $newline = shift(@AllLines);
        # Skip "Comments"
        push(@AllSentences, $newline) && next if $newline =~ /^\s*\#\s+/;

        # Split sentences
        $line .= $newline.' ';

        unless($newline =~ /\S/)	# Empty lines signal end-of-sentence
        { 
    	$line = "";
    	push(@AllSentences,"\n");
        };
        # A sentence ends with lowercase letters, a period and either an uppercase or newline
	# (or 't, 's, 'r, 'k or 'n and a capital)
        while($line =~ /(([^A-Z][^A-Z]|Uh|En|[\)\'\"])[\.\?\!\;\:]+[\"\']?)($|\s+(('[stnkr]\s|d'r|[\"\'\(]?[A-Z])|\s*$))/sg)
        {
    	    $sentence = "$`$1";# copy sentence
	    $line = $4.$';	   # update line

	    # Clean up sentence
	
	    $sentence =~ s/\b([a-z]|zo)\'([a-z])/$1\#\#\#\#$2/isg;	# protect <d'r> type of quotes
	    $sentence =~ s/(^|\s)\'([a-z]|ns)\b/$1\#\#\#\#$2/sg;	# protect <'s> type of quotes
	    $sentence =~ s/([a-z]+)\'(s)\b/$1\#\#\#\#$2/sg;		# protect <massa's> type of quotes

	    $sentence =~ s/([\)\(\.\,\'\"\?\!\;\:\]\[])/ $1 /isg;
	    $sentence =~ s/^\s+//g;	
	    $sentence =~ s/\s+/ /g;	

	    $sentence =~ s/\#\#\#\#/\'/g;   # correct 's 'm etc 

	    $sentence =~ s/^(\'[stn]\s[A-Z][a-z])/lc($1)/eg;
	    $sentence =~ s/^([A-Z][a-z])/lc($1)/eg;

	    #
	    # Handle some special cases: 's avonds, 's morgens etc. These are single words with double lexical
	    # entries
	    $sentence =~ s/(^|\s)\'s ($SreplaceList)/"$1\'s\-".lc($2)/ieg;

	    push(@AllSentences, "$sentence\n");
        };
	
    };
    return \@AllSentences;
}

#
#
# Do not execute inside a require
unless(caller())
{
my @AllLines = <>;

my $AllSentences = SplitAllLines(@AllLines) if @AllLines;
@AllLines = @$AllSentences if $AllSentences;
print STDOUT @AllLines;
};

1; # Make require happy
