#! /usr/bin/perl
#
# Calculate the entropy and the Kullback-Leibler
# distance with Zipf's distribution for a 
# Standard Word List (frequency counts on position
# 9) or a default word list (frequency counts on
# position 1). In both situations the Word position
# is 2.
#
# Use:
# ./WordListEntropy.pl <Path> ...
#
# Path: the path to the word List or -.
#
###############################################################################
#
# Copyright R.J.J.H. van Son © 2002
#
# Author Rob van Son
# Institute of Phonetic Sciences & ACLC
# University of Amsterdam
# Herengracht 338
# NL-1016CG Amsterdam, The Netherlands
# Email: Rob.van.Son@hum.uva.nl
#        rob.van.son@workmail.com
# WWW  : http://www.fon.hum.uva.nl/rob/
# mail:  Institute of Phonetic Sciences
#        University of Amsterdam
#        Herengracht 338
#        NL-1016CG Amsterdam
#        The Netherlands
#        tel +31 205252183
#        fax +31 205252197
#
# License for use and disclaimers
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
#
#
#######################################################
#
#
my @WordList=();
while(<>)
{
	next if /^\s*\#/;
	chomp;
	split(/\t/);
$_[8] = 0 if $_[8] == 1;
	if(scalar(@_) >= 9)
	{
		push(@WordList, "$_[8]\t$_[1]");
	}
	else
	{
		push(@WordList, "$_[0]\t$_[1]");
	};
}
@WordList = sort {$b <=> $a} @WordList;
foreach $line (@WordList)
{
	($count, $word)=split(/\t/, $line);
	++$zerotypes unless $count;
	next unless $count;
	$Currentcount = $count; 
	++$types;
	$tokens += $Currentcount;
	$sumCiLogCi += $Currentcount*log($Currentcount);
	$sumI += 1.0/$types;
	$sumCilogI += $Currentcount*log($types);
};
$H = (log($tokens) - $sumCiLogCi / $tokens)/log(2);
$KLdist = (log($sumI) + $sumCilogI / $tokens)/log(2) - $H;
print "# zerotype = $zerotypes, types = $types, tokens = $tokens\n";
print "# H = $H bits/type, Kullback-Leibler(Zipfs) = $KLdist\n";
print "Cross H(Zipfs dist) = ", $KLdist + $H, " bits/type\n";
