#! /usr/bin/perl
#
print STDOUT << 'HELP' unless scalar(@ARGV) > 0 && !grep(/^[\-]+h/i, @ARGV);
Construct a corpus distribution that can be written to cdrom 
with speech from one microphone

Use:
ConstructSLcdrom.pl [-g fileglob] <Speaker> [fm|hm] [destination]

Example:
> ConstructSLcdrom.pl -g 'F[RT]1[A-Z]' F20N

This will download the complete repository using cvs
and the read and retold versions of the first, Fixed story 
(i.e., 'the north wind and the sun') into the current directory.  

Copies all necessary files conforming to *<fileglob>* from a single <Speaker> 
and a single microphone (channel) to the SLcdrom directory for distribution.
It includes audio (.aifc) sentences and Center of Gravity files
plus the complete contents of the CVS repository. 

This is completely Web Based. It uses the DataServer 
(http://www.fon.hum.uva.nl/IFAcorpus) and the CVS source
(pserver:anonymous@uvafon.hum.uva.nl:/u/cvs) both
from "./Links.pl" to download ('cvs' and 'wget').
You need wget (http://www.wget.org/)!

NOTE:
Before you use this program, please log in to the CVS server (see Links.pl):
> cvs -d:$CVSsource login
$CVSpassword

Example:
> cvs -d:pserver:anonymous@uvafon.hum.uva.nl:/u/cvs login
anonymous

HELP
@ARGV = grep(!/^[\-]+h/i, @ARGV);
#
# The CVS downloading is rather slow and generates an enourmous 
# avalange of error messages if the CVS module has already been
# downloaded. As it won't update an existing file if it
# exists, there is no point in trying, so CVS will only run
# if the Speaker realted directories do NOT exists.
#
###############################################################################
#
# Copyright R.J.J.H. van Son © 2000, 2001
#
# Author Rob van Son
# Institute of Phonetic Sciences & ACLC
# University of Amsterdam
# Herengracht 338
# NL-1016CG Amsterdam, The Netherlands
# Email: Rob.van.Son@hum.uva.nl
#        rob.van.son@workmail.com
# WWW  : http://www.fon.hum.uva.nl/rob/
# mail:  Institute of Phonetic Sciences
#        University of Amsterdam
#        Herengracht 338
#        NL-1016CG Amsterdam
#        The Netherlands
#        tel +31 205252183
#        fax +31 205252197
#
# License for use and disclaimers
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
#
#
#######################################################
#
$HomeDir = $ENV{'PWD'};
my $DefaultCDromDir = '';
if(-e './Links.pl')
{
	require './Links.pl' || die "require: $!\n";
	$DefaultCDromDir = '.';
}
elsif(-e './SLcorpus/Links.pl')
{
	require './SLcorpus/Links.pl' || die "require: $!\n";
	$DefaultCDromDir = '.';
}
else
{
	require '../Links.pl' || die "require: $!\n";
	$DefaultCDromDir = '';
};

my $ModuleName = 'SLcorpus';
my $ChunkModuleName = "$ModuleName/Labels/chunks";
my $LabelModuleName = "$ModuleName/Labels/sentences";

# For wget support
my $SourceSpeechServer = "$DataServer/$Sentences";
my $SourceCoGServer = "$DataServer/$CenterOfGravity";

my $SourceSpeechDir = '../../SLspeech/sentences';
my $SourceCoGDir = '../../SLspeech/CenterOfGravity';

# First look for a restricted file glob
my $glob = "";
if($ARGV[0] =~ /[\-]+g(lob)?$/i)
{
	shift(@ARGV);
	$glob = shift(@ARGV);
};
# Get the other arguments
my $Speaker = shift(@ARGV) || die "No speaker to distribute\n"
				."Use: ConstructSLcdrom.pl [-g fileglob] <Speaker> [fm|hm] [destination]\n";
my $Microphone = shift(@ARGV) || 'fm';  # Default is the fixed microphone
my $SLcdromDir = shift(@ARGV) || $DefaultCDromDir;  # Default directory

die "$SLcdromDir: Destination directory ambiguous (does it exists? Are you recursing inside the corpus)\n"
	."Use: ConstructSLcdrom.pl [-g fileglob] <Speaker> [fm|hm] [destination]\n" 
    unless $SLcdromDir && -d "$SLcdromDir";

# Start README file
my @StartLines = ();
@StartLines = `cat $SLcdromDir/README|grep '\#'|grep -vF '$Speaker $Microphone $glob'` if -e "$SLcdromDir/README";
open(README, ">$SLcdromDir/README") || die "$SLcdromDir/README: $!\n";
print README @StartLines;
print README "# This distribution contains speech from the IFA Spoken Language Corpus\n" unless scalar(@StartLines);
print README "# Speaker Microphone File-Glob (pattern)\n" unless scalar(@StartLines);
print README "# $Speaker $Microphone $glob\n";	# DO NOT USE TABS ON THIS LINE
close(README);


# Unpack the corpus info if it has been compressed and remove the tar
`cd $SLcdromDir;tar -xzf SLcorpus.tgz` if -e "$SLcdromDir/SLcorpus.tgz";
`cd $SLcdromDir;rm ./SLcorpus.tgz` if  -e "$SLcdromDir/SLcorpus.tgz" && -d "$SLcdromDir/SLcorpus";

# First, extract the latest CVS tree (if it hasn't been compressed)
# DONOT forget to login as anonymous before you run the script!
`cd $SLcdromDir; cvs -q -d:$CVSsource checkout -f '$ModuleName'` unless -d "$SLcdromDir/$ModuleName/CVS";
# Ensure that all Speaker (and other) Label-directories are indeed updated!
`cd $SLcdromDir; cvs -Q -d:$CVSsource update -f '$ModuleName'`;

# Add comment to README
open(README, ">>$SLcdromDir/README") || die "$SLcdromDir/README: $!\n";
print README "./SLcorpus:\tThe complete IFA Spoken Language Corpus\n";
close(README);

# Copy licensing info
`cp --update $SLcdromDir/SLcorpus/*.txt $SLcdromDir/`;
`cp --update $SLcdromDir/SLcorpus/*.html $SLcdromDir/`;
`cp --update $SLcdromDir/SLcorpus/*.jpg $SLcdromDir/`;
`cp --update $SLcdromDir/SLcorpus/scripts/ConstructSLcdrom.pl $SLcdromDir/`;

# Add comment to README
open(README, ">>$SLcdromDir/README") || die "$SLcdromDir/README: $!\n";
my @LicenseInfo = `cd $SLcdromDir; ls -1 ./*.txt ./*.html ./*.jpg`; 
map {chomp} @LicenseInfo; # Remove newlines
print README join(":\tProject and Licensing information\n", @LicenseInfo);
print README ":\tProject and Licensing information\n"; # Last line
print README "./ConstructSLcdrom.pl:\tPerl script to construct or update a CDrom distribution (the script that created this file)\n";
close(README);

# Copy Links
`cp --update $SLcdromDir/SLcorpus/Links.pl $CreateDir` if -e "$CreateDir/Links.pl";


# copy Praat directory
my $CreateDir = "$SLcdromDir/Praat";
my $DirExists = -d "$CreateDir";
($return_value = mkdir("$CreateDir", 0744) || die "$CreateDir: $!\n") unless $DirExists;

`cp --update --recursive $SLcdromDir/SLcorpus/Praat/* $CreateDir`;

# Add comment to README
open(README, ">>$SLcdromDir/README") || die "$SLcdromDir/README: $!\n";
print README "./Praat:\tThe praat speech tool (multiple platforms)\n";
close(README);

# copy Ogg Vorbis directory
my $CreateDir = "$SLcdromDir/Ogg";
my $DirExists = -d "$CreateDir";
($return_value = mkdir("$CreateDir", 0744) || die "$CreateDir: $!\n") unless $DirExists;

`cp --update --recursive $SLcdromDir/SLcorpus/Ogg/* $CreateDir`;

# Add comment to README
open(README, ">>$SLcdromDir/README") || die "$SLcdromDir/README: $!\n";
print README "./Ogg:\tSource code and installation of the Ogg Vorbis audio compression software\n";
close(README);

# copy Label directories
my $CreateDir = "$SLcdromDir/Labels";
my $DirExists = -d "$CreateDir";
($return_value = mkdir("$CreateDir", 0744) || die "$CreateDir: $!\n") unless $DirExists;
# Speaker subdir
$CreateDir .= "/$Speaker";
$DirExists = -d "$CreateDir";
($return_value = mkdir("$CreateDir", 0744) || die "$CreateDir: $!\n") unless $DirExists;
# label subdirs
$DirExists = -d "$CreateDir/phoneme";
($return_value = mkdir("$CreateDir/phoneme", 0744) || die "$CreateDir/phoneme: $!\n") unless $DirExists;
$DirExists = -d "$CreateDir/prephoneme";
($return_value = mkdir("$CreateDir/prephoneme", 0744) || die "$CreateDir/prephoneme: $!\n") unless $DirExists;

`cp --update --recursive $SLcdromDir/SLcorpus/Labels/sentences/$Speaker/phoneme/*${glob}*.phoneme $SLcdromDir/Labels/$Speaker/phoneme`;
`cp --update --recursive $SLcdromDir/SLcorpus/Labels/sentences/$Speaker/prephoneme/*${glob}*.prephoneme $SLcdromDir/Labels/$Speaker/prephoneme`;

# Add comment to README
open(README, ">>$SLcdromDir/README") || die "$SLcdromDir/README: $!\n";
print README "./Labels:\tLabel files for all audio (speech) files\n";
close(README);

# Create Data directory
my $CreateDir = "$SLcdromDir/SLspeech";
$DirExists = -d "$CreateDir";
($return_value = mkdir("$CreateDir", 0744) || die "$CreateDir: $!\n") unless $DirExists;

# Create Chunk (Ogg Vorbis) directory
my $CreateChunkDir = "$CreateDir/ogg";
$DirExists = -d "$CreateChunkDir";
($return_value = mkdir("$CreateChunkDir", 0744) || die "$CreateChunkDir: $!\n") unless $DirExists;

$CreateChunkDir .= "/$Speaker";
$DirExists = -d "$CreateChunkDir";
($return_value = mkdir("$CreateChunkDir", 0744) || die "$CreateChunkDir: $!\n") unless $DirExists;

# Create Speech directories
my $CreateSoundDir = "$CreateDir/sentences";
$DirExists = -d "$CreateSoundDir";
($return_value = mkdir("$CreateSoundDir", 0744) || die "$CreateSoundDir: $!\n") unless $DirExists;

$CreateSoundDir .= "/$Microphone";
$DirExists = -d "$CreateSoundDir";
($return_value = mkdir("$CreateSoundDir", 0744) || die "$CreateSoundDir: $!\n") unless $DirExists;

$CreateSoundDir .= "/$Speaker";
$DirExists = -d "$CreateSoundDir";
($return_value = mkdir("$CreateSoundDir", 0744) || die "$CreateSoundDir: $!\n") unless $DirExists;

# Create CoG directories
my $CreateCoGDir = "$CreateDir/CenterOfGravity";
$DirExists = -d "$CreateCoGDir";
($return_value = mkdir("$CreateCoGDir", 0744) || die "$CreateCoGDir: $!\n") unless $DirExists;

$CreateCoGDir .= "/$Microphone";
$DirExists = -d "$CreateCoGDir";
($return_value = mkdir("$CreateCoGDir", 0744) || die "$CreateCoGDir: $!\n") unless $DirExists;

$CreateCoGDir .= "/$Speaker";
$DirExists = -d "$CreateCoGDir";
($return_value = mkdir("$CreateCoGDir", 0744) || die "$CreateCoGDir: $!\n") unless $DirExists;

# Open Praat file for inspection
my $InspectStartLines = "#!praat\n# Praat script to inspect all files on the distribution\n# \n" 
 unless -s "$SLcdromDir/InspectFiles.praat" > 0;
open(INSPECT, ">>$SLcdromDir/InspectFiles.praat") || die "$SLcdromDir/InspectFiles.praat: $!\n";
print INSPECT $InspectStartLines if $InspectStartLines;

# Copy Speech files
my $Extention = 'aifc';
my $Extention2 = 'cog';
my $SoundPath = "$CreateSoundDir/";
$SoundPath =~ s@^\./@@g;
my $CoGPath = "$CreateCoGDir/";
$CoGPath =~ s@^\./@@g;
my @FileList = glob("$SLcdromDir/Labels/$Speaker/prephoneme/*${glob}*.prephoneme");
foreach $FilePath (@FileList)
{
		$FilePath =~ m@/([^/]+)/prephoneme/([^/]+)\.prephoneme$@;
		my $CurrentSpeaker = $1;
		my $FileName = $2;
		die "$FilePath does not contain a filename\n" unless $FileName =~ /\w/;
		my $CurrentSource = "$SourceSpeechServer/$Microphone/$CurrentSpeaker/${FileName}_$Microphone.$Extention";
		my $CurrentTarget = "$SoundPath${FileName}_$Microphone.$Extention";
		
		my $CurrentCoGSource = "$SourceCoGServer/$Microphone/$CurrentSpeaker/${FileName}_$Microphone.$Extention2";
		my $CurrentCoGTarget = "$CoGPath${FileName}_$Microphone.$Extention2";
		
		# 
		# Get the matching label name (phoneme, if it exists)
		my $CurrentLabel = $FilePath;
		$CurrentLabel =~ s/prephoneme/phoneme/g;
		$CurrentLabel =~ s/\.phoneme$/_*\.phoneme/g;
		$CurrentLabel = (glob($CurrentLabel))[0];
		$CurrentLabel = $FilePath unless "$CurrentLabel" && -e "$CurrentLabel";
		$CurrentLabel =~ s/^$SLcdromDir/\./g;
		
		# Get Label file
		unless(-e "$CurrentTarget" && -e "$CurrentCoGTarget")
		{
			print INSPECT "Read from file... $CurrentLabel\n";
			print INSPECT "Rename... ${FileName}_label\n";
		};
		
		
		my $PrintPause = 0;
		# Center of Gravity Skip existing files
		unless(-e "$CurrentCoGTarget")
		{
			print STDERR "Current copy: $CurrentCoGSource -> $CurrentCoGTarget\n";
			`wget -q -O $CurrentCoGTarget $CurrentCoGSource`;
			die "$CurrentCoGSource -> $CurrentCoGTarget: $!\n" unless -s "$CurrentCoGTarget" > 0;
		
			# Fill the Praat script for inspection
			print INSPECT "Read from file... $CurrentCoGTarget\n";
			print INSPECT "Rename... ${FileName}_CoG\n";
			print INSPECT "plus TextGrid ${FileName}_label\n";
			print INSPECT "Edit\n";
			
			++$PrintPause;
		};
		
		# Sounds Skip existing files
		unless(-e "$CurrentTarget")
		{
			print STDERR "Current copy: $CurrentSource -> $CurrentTarget\n";
			`wget -q -O $CurrentTarget $CurrentSource`;
			die "$CurrentSource -> $CurrentTarget: $!\n" unless -s "$CurrentTarget" > 0;
		
			# Fill the Praat script for inspection
			print INSPECT "Open long sound file... $CurrentTarget\n";
			print INSPECT "Rename... ${FileName}_speech\n";
			print INSPECT "plus TextGrid ${FileName}_label\n";
			print INSPECT "Edit\n";

			++$PrintPause;
		};
		
		# Get Label file
		if($PrintPause)
		{
			print INSPECT "pause Next utterance\n";
			print INSPECT "select all\n";
			print INSPECT "Remove\n";
			print INSPECT "\n";
		};
};
close(INSPECT);

# Add comment to README
open(README, ">>$SLcdromDir/README") || die "$SLcdromDir/README: $!\n";
print README "./SLspeech/sentences:\tAll audio (speech) files\n";
print README "./SLspeech/CenterOfGravity:\tAll Center of Gravity (cog) files\n";
print README "./InspectFiles.praat:\tPraat script to inspect all speech files (for all platforms)\n";
close(README);

#
# If the size of the data becomes too large to fit on a CDrom, compress the SLcorpus tree.
#
# Determine total size of CDrom data
my $totalSize = `du -cms $SLcdromDir | grep total`;
$totalSize =~ s/[^\d\-]*$//ig;

# Don't re-compress the corpus (shouldn't be possible, but is tested anyhow)
unless( -e "$SLcdromDir/SLcorpus.tgz")
{
	# Compress SLcorpus if it is too large (don't remove yet)
	`cd $SLcdromDir;tar -czf SLcorpus.tgz ./SLcorpus` if $totalSize > 620;
	
	# Remove SLcorpus dir-tree and add comment to README
	if(-e "$SLcdromDir/SLcorpus.tgz")
	{
		`cd $SLcdromDir;rm -rf ./SLcorpus`; # Be carefull, the tar can fail
		# Add comment
		open(README, ">>$SLcdromDir/README") || die "$SLcdromDir/README: $!\n";
		print README "./SLcorpus.tgz:\tA compressed tar archive of SLcorpus (unpack with: tar -xvzf ./SLcorpus.tgz)\n";
		close(README);
	};
};
