\documentclass{beamer}
%\documentclass[handout]{beamer}
%\documentclass[trans]{beamer}
%\documentclass[article]{beamer}
\usepackage{comment}
%\usepackage{relsize}
\usepackage[latin1]{inputenc}
\usepackage{amsmath}
\usepackage{tipa}
%\usepackage[T1]{fontenc}
%\usepackage[pdftex]{graphicx}
\usepackage{multimedia}
\usepackage{hyperref}
\usepackage{pifont}
\usepackage{multibib}
\newcites{monday,tuesday,wednesday,thursday,friday}
{Further Reading,Further Reading,Further Reading,Further Reading,Further Reading}

%http://www.csd.uwo.ca/~jgorski/personal/beamerthemes/
%\usetheme[]{Warsaw}
%\usetheme{Berkeley}
%\usetheme{Berlin}
%\usetheme{Bergen}
%\usetheme{Berlin}Antibes,Bergen,Boadilla,Copenhagen,Darmstadt
%\usetheme{Boadilla}
%\usetheme{Antibes} %heel kaal
%\usetheme{Bergen} links rechts
%\usetheme{Dresden} %nice
%\usetheme{Frankfurt} %ok
%\usetheme[right, hideothersubsections]{Goettingen} % overzicht rechts
%\usetheme[right, hideothersubsections]{Hannover} % als Goetngen met overzicht links
%\usetheme{Ilmenau}
%\usetheme{JuanLesPins}
%\usetheme{Luebeck} % overzicht boven, auteur beneden
%\usetheme{Madrid} % alleen navigatie onder
%\usetheme{Malmoe} % nav boven
%\usetheme{Marburg} % donkere navigatiebalk rechtslectures
%\usetheme{Montpellier} % matig
\usetheme[right,hideothersubsections]{PaloAlto}% nav links
%\usetheme[right]{PaloAlto}% nav links
%\usetheme{Pittsburgh}% alleen standaard nav
%\usetheme{Rochester}% helder, alleen standaard nav
%\usetheme{Singapore} %helder ok nav boven+#sheets indicatie
%\usetheme{Szeged} %helder ok nav boven+#sheets indicatie

%%%%%innerthemes: title and part pages, itemize, enumerate, desciption, block, theorem/proof, figures, tables, footnotes and bibl. entries 
%\useinnertheme{default} %
%\useinnertheme{circles} % 
%\useinnertheme{rectangles} %
\useinnertheme{rounded} %
%\useinnertheme{inmargin} %structure on left (like blocktitles) and normal info like block bodies on right

%%%%%outerthemes: head and footline, sidebars, logo, frame title
%\useoutertheme{infolines}%
%\useoutertheme{miniframes}%
%\useoutertheme{smoothbars}%
%\useoutertheme[right,height=0pt,hideothersubsections]{sidebar}%options:height=dimension, hideothersubsections, hideallsubsections, left, right, width=dimension
%\useoutertheme{split}%
%\useoutertheme{shadow}%
%\useoutertheme{tree}%option:[hooks]
%\useoutertheme{smoothtree}%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%Transparency Effects
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\setbeamercovered{transparent}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%Color Themes:
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%
%\usecolourtheme [option]{structure} %options: rgb={} or RGB={} sets structure foreground colour
\usecolortheme{sidebartab} %highlights current entry in table of contents
%\usecolortheme[RGB={25,25,112}]{structure} %72,61,139
\usecolortheme[RGB={39,64,139}]{structure} %72,61,139


%\usecolortheme[RGB={176,48,96}]{alerted text}

%%%%%%%innercolorthemes: changes palette colors
%\usecolortheme{lily} %used to restore default colours 
%\usecolortheme{orchid}
\usecolortheme{rose}

%%%%%%%outercolourthemes:
\usecolortheme{dolphin}
%\usecolortheme{whale}
%\usecolortheme{seahorse}

%%%%%%complete colorthemes:
%\usecolortheme{albatross}
%\usecolortheme{overlystylish}
%\usecolortheme{beetle}
%\usecolortheme{crane}
%\usecolortheme{dove}
%\usecolortheme{fly}
%\usecolortheme{seagull}

\setbeamercolor{frametitle}{bg=structure.fg!10!white,fg=structure.fg!75!black}
\setbeamercolor{block title}{fg=structure.fg!75!black}

%\setbeamercolor{outline}{fg=structure}
\setbeamercolor{sidebar}{bg=structure.fg!50!white}
\setbeamercolor{section in sidebar}{fg=structure}
\setbeamercolor{subsection in sidebar}{fg=structure}
\setbeamercolor{subsection in sidebar}{fg=black}
\setbeamercolor{alerted text}{fg=purple}
\setbeamercolor{sidebartab}{fg=alerted text}
\setbeamercolor{logo}{bg=structure.fg!50!white}
\setbeamercolor{alerted text}{bg=white}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%Font Themes
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%usefonttheme{deafault} 
%usefonttheme{professionalfonts} 
%usefonttheme{} 


\logo{\includegraphics[height=1cm]{Pictures/IFAlogo.pdf}}

\title[Information in Speech]{Information in Spoken Language}
\subtitle
{A quantitative approach}

\author[]{Rob van Son}
\date{2006 / 01 / 01}
\institute{Chair of Phonetic Sciences\\ACLC\\
  University of Amsterdam}
\date[Short Occasion] % (optional)
{LOT winterschool 2006
\begin{figure}[b]
\includegraphics[width=0.5\textwidth]{Pictures/aclclogo.pdf}
\end{figure}
}

%
% The obligatyright notices and license texts
% Implemented as Latex macros
% Add copyright notice and GPL
\newcommand{\multicite}[1]{\cite{#1}}
\newcommand{\copyrightauthor}[0]{R.J.J.H. van Son}
\newcommand{\GNUcite}[0]{\multicite{GPL}}
\newcommand{\licensenumber}[0]{License}
% Add  week number
\newcommand{\copyrightnotice}[0]
{Copyright \copyright 2005,2006 \copyrightauthor, \hyperlink{\licensenumber}{GNU General Public License} \GNUcite}

% Insert complete GNU GPL license frames. Add weeknumber
\newcommand{\insertgnugpl}[0]
{
\begin{frame}[allowframebreaks,allowdisplaybreaks,label=\licensenumber]
\frametitle{\insertsubsection}
\copyrightnotice \\
\vskip20pt
\begin{scriptsize}
\begin{quote}
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
\end{quote}
\end{scriptsize}

\end{frame}

\begin{frame}[allowframebreaks,allowdisplaybreaks]
\frametitle{\insertsubsection}

\begin{tiny}

\begin{center}
{\parindent 0in

Version 2, June 1991

Copyright \copyright\ 1989, 1991 Free Software Foundation, Inc.

\bigskip

51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA

\bigskip

Everyone is permitted to copy and distribute verbatim copies
of this license document, but changing it is not allowed.
}
\end{center}

\begin{center}
{\bf\large Preamble}
\end{center}


The licenses for most software are designed to take away your freedom to
share and change it.  By contrast, the GNU General Public License is
intended to guarantee your freedom to share and change free software---to
make sure the software is free for all its users.  This General Public
License applies to most of the Free Software Foundation's software and to
any other program whose authors commit to using it.  (Some other Free
Software Foundation software is covered by the GNU Library General Public
License instead.)  You can apply it to your programs, too.

When we speak of free software, we are referring to freedom, not price.
Our General Public Licenses are designed to make sure that you have the
freedom to distribute copies of free software (and charge for this service
if you wish), that you receive source code or can get it if you want it,
that you can change the software or use pieces of it in new free programs;
and that you know you can do these things.

To protect your rights, we need to make restrictions that forbid anyone to
deny you these rights or to ask you to surrender the rights.  These
restrictions translate to certain responsibilities for you if you
distribute copies of the software, or if you modify it.

For example, if you distribute copies of such a program, whether gratis or
for a fee, you must give the recipients all the rights that you have.  You
must make sure that they, too, receive or can get the source code.  And
you must show them these terms so they know their rights.

We protect your rights with two steps: (1) copyright the software, and (2)
offer you this license which gives you legal permission to copy,
distribute and/or modify the software.

Also, for each author's protection and ours, we want to make certain that
everyone understands that there is no warranty for this free software.  If
the software is modified by someone else and passed on, we want its
recipients to know that what they have is not the original, so that any
problems introduced by others will not reflect on the original authors'
reputations.

Finally, any free program is threatened constantly by software patents.
We wish to avoid the danger that redistributors of a free program will
individually obtain patent licenses, in effect making the program
proprietary.  To prevent this, we have made it clear that any patent must
be licensed for everyone's free use or not licensed at all.

The precise terms and conditions for copying, distribution and
modification follow.

\begin{center}
{\small \sc Terms and Conditions For Copying, Distribution and
  Modification}
\end{center}


%\renewcommand{\theenumi}{\alpha{enumi}}
\begin{enumerate}

\addtocounter{enumi}{-1}

\item 

This License applies to any program or other work which contains a notice
placed by the copyright holder saying it may be distributed under the
terms of this General Public License.  The ``Program'', below, refers to
any such program or work, and a ``work based on the Program'' means either
the Program or any derivative work under copyright law: that is to say, a
work containing the Program or a portion of it, either verbatim or with
modifications and/or translated into another language.  (Hereinafter,
translation is included without limitation in the term ``modification''.)
Each licensee is addressed as ``you''.

Activities other than copying, distribution and modification are not
covered by this License; they are outside its scope.  The act of
running the Program is not restricted, and the output from the Program
is covered only if its contents constitute a work based on the
Program (independent of having been made by running the Program).
Whether that is true depends on what the Program does.

\item You may copy and distribute verbatim copies of the Program's source
  code as you receive it, in any medium, provided that you conspicuously
  and appropriately publish on each copy an appropriate copyright notice
  and disclaimer of warranty; keep intact all the notices that refer to
  this License and to the absence of any warranty; and give any other
  recipients of the Program a copy of this License along with the Program.

You may charge a fee for the physical act of transferring a copy, and you
may at your option offer warranty protection in exchange for a fee.

\item

You may modify your copy or copies of the Program or any portion
of it, thus forming a work based on the Program, and copy and
distribute such modifications or work under the terms of Section 1
above, provided that you also meet all of these conditions:

\begin{enumerate}
\begin{tiny}

\item 

You must cause the modified files to carry prominent notices stating that
you changed the files and the date of any change.

\item

You must cause any work that you distribute or publish, that in
whole or in part contains or is derived from the Program or any
part thereof, to be licensed as a whole at no charge to all third
parties under the terms of this License.

\item
If the modified program normally reads commands interactively
when run, you must cause it, when started running for such
interactive use in the most ordinary way, to print or display an
announcement including an appropriate copyright notice and a
notice that there is no warranty (or else, saying that you provide
a warranty) and that users may redistribute the program under
these conditions, and telling the user how to view a copy of this
License.  (Exception: if the Program itself is interactive but
does not normally print such an announcement, your work based on
the Program is not required to print an announcement.)

\end{tiny}
\end{enumerate}

These requirements apply to the modified work as a whole.  If
identifiable sections of that work are not derived from the Program,
and can be reasonably considered independent and separate works in
themselves, then this License, and its terms, do not apply to those
sections when you distribute them as separate works.  But when you
distribute the same sections as part of a whole which is a work based
on the Program, the distribution of the whole must be on the terms of
this License, whose permissions for other licensees extend to the
entire whole, and thus to each and every part regardless of who wrote it.

Thus, it is not the intent of this section to claim rights or contest
your rights to work written entirely by you; rather, the intent is to
exercise the right to control the distribution of derivative or
collective works based on the Program.

In addition, mere aggregation of another work not based on the Program
with the Program (or with a work based on the Program) on a volume of
a storage or distribution medium does not bring the other work under
the scope of this License.

\item
You may copy and distribute the Program (or a work based on it,
under Section 2) in object code or executable form under the terms of
Sections 1 and 2 above provided that you also do one of the following:

\begin{enumerate}
\begin{tiny}

\item

Accompany it with the complete corresponding machine-readable
source code, which must be distributed under the terms of Sections
1 and 2 above on a medium customarily used for software interchange; or,

\item

Accompany it with a written offer, valid for at least three
years, to give any third party, for a charge no more than your
cost of physically performing source distribution, a complete
machine-readable copy of the corresponding source code, to be
distributed under the terms of Sections 1 and 2 above on a medium
customarily used for software interchange; or,

\item

Accompany it with the information you received as to the offer
to distribute corresponding source code.  (This alternative is
allowed only for noncommercial distribution and only if you
received the program in object code or executable form with such
an offer, in accord with Subsection b above.)

\end{tiny}
\end{enumerate}

The source code for a work means the preferred form of the work for
making modifications to it.  For an executable work, complete source
code means all the source code for all modules it contains, plus any
associated interface definition files, plus the scripts used to
control compilation and installation of the executable.  However, as a
special exception, the source code distributed need not include
anything that is normally distributed (in either source or binary
form) with the major components (compiler, kernel, and so on) of the
operating system on which the executable runs, unless that component
itself accompanies the executable.

If distribution of executable or object code is made by offering
access to copy from a designated place, then offering equivalent
access to copy the source code from the same place counts as
distribution of the source code, even though third parties are not
compelled to copy the source along with the object code.

\item
You may not copy, modify, sublicense, or distribute the Program
except as expressly provided under this License.  Any attempt
otherwise to copy, modify, sublicense or distribute the Program is
void, and will automatically terminate your rights under this License.
However, parties who have received copies, or rights, from you under
this License will not have their licenses terminated so long as such
parties remain in full compliance.

\item
You are not required to accept this License, since you have not
signed it.  However, nothing else grants you permission to modify or
distribute the Program or its derivative works.  These actions are
prohibited by law if you do not accept this License.  Therefore, by
modifying or distributing the Program (or any work based on the
Program), you indicate your acceptance of this License to do so, and
all its terms and conditions for copying, distributing or modifying
the Program or works based on it.

\item
Each time you redistribute the Program (or any work based on the
Program), the recipient automatically receives a license from the
original licensor to copy, distribute or modify the Program subject to
these terms and conditions.  You may not impose any further
restrictions on the recipients' exercise of the rights granted herein.
You are not responsible for enforcing compliance by third parties to
this License.

\item
If, as a consequence of a court judgment or allegation of patent
infringement or for any other reason (not limited to patent issues),
conditions are imposed on you (whether by court order, agreement or
otherwise) that contradict the conditions of this License, they do not
excuse you from the conditions of this License.  If you cannot
distribute so as to satisfy simultaneously your obligations under this
License and any other pertinent obligations, then as a consequence you
may not distribute the Program at all.  For example, if a patent
license would not permit royalty-free redistribution of the Program by
all those who receive copies directly or indirectly through you, then
the only way you could satisfy both it and this License would be to
refrain entirely from distribution of the Program.

If any portion of this section is held invalid or unenforceable under
any particular circumstance, the balance of the section is intended to
apply and the section as a whole is intended to apply in other
circumstances.

It is not the purpose of this section to induce you to infringe any
patents or other property right claims or to contest validity of any
such claims; this section has the sole purpose of protecting the
integrity of the free software distribution system, which is
implemented by public license practices.  Many people have made
generous contributions to the wide range of software distributed
through that system in reliance on consistent application of that
system; it is up to the author/donor to decide if he or she is willing
to distribute software through any other system and a licensee cannot
impose that choice.

This section is intended to make thoroughly clear what is believed to
be a consequence of the rest of this License.

\item
If the distribution and/or use of the Program is restricted in
certain countries either by patents or by copyrighted interfaces, the
original copyright holder who places the Program under this License
may add an explicit geographical distribution limitation excluding
those countries, so that distribution is permitted only in or among
countries not thus excluded.  In such case, this License incorporates
the limitation as if written in the body of this License.

\item
The Free Software Foundation may publish revised and/or new versions
of the General Public License from time to time.  Such new versions will
be similar in spirit to the present version, but may differ in detail to
address new problems or concerns.

Each version is given a distinguishing version number.  If the Program
specifies a version number of this License which applies to it and ``any
later version'', you have the option of following the terms and conditions
either of that version or of any later version published by the Free
Software Foundation.  If the Program does not specify a version number of
this License, you may choose any version ever published by the Free Software
Foundation.

\item
If you wish to incorporate parts of the Program into other free
programs whose distribution conditions are different, write to the author
to ask for permission.  For software which is copyrighted by the Free
Software Foundation, write to the Free Software Foundation; we sometimes
make exceptions for this.  Our decision will be guided by the two goals
of preserving the free status of all derivatives of our free software and
of promoting the sharing and reuse of software generally.

\begin{center}
{\small\sc
No Warranty
}
\end{center}

\item
{\sc Because the program is licensed free of charge, there is no warranty
for the program, to the extent permitted by applicable law.  Except when
otherwise stated in writing the copyright holders and/or other parties
provide the program ``as is'' without warranty of any kind, either expressed
or implied, including, but not limited to, the implied warranties of
merchantability and fitness for a particular purpose.  The entire risk as
to the quality and performance of the program is with you.  Should the
program prove defective, you assume the cost of all necessary servicing,
repair or correction.}

\item
{\sc In no event unless required by applicable law or agreed to in writing
will any copyright holder, or any other party who may modify and/or
redistribute the program as permitted above, be liable to you for damages,
including any general, special, incidental or consequential damages arising
out of the use or inability to use the program (including but not limited
to loss of data or data being rendered inaccurate or losses sustained by
you or third parties or a failure of the program to operate with any other
programs), even if such holder or other party has been advised of the
possibility of such damages.}

\end{enumerate}
\end{tiny}

\begin{center}
{\small\sc End of Terms and Conditions}
\end{center}


\pagebreak[2]

\small{Appendix: How to Apply These Terms to Your New Programs}

\begin{tiny}
If you develop a new program, and you want it to be of the greatest
possible use to the public, the best way to achieve this is to make it
free software which everyone can redistribute and change under these
terms.

  To do so, attach the following notices to the program.  It is safest to
  attach them to the start of each source file to most effectively convey
  the exclusion of warranty; and each file should have at least the
  ``copyright'' line and a pointer to where the full notice is found.

\begin{quote}
one line to give the program's name and a brief idea of what it does. \\
Copyright (C) yyyy  name of author \\

This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
\end{quote}

Also add information on how to contact you by electronic and paper mail.

If the program is interactive, make it output a short notice like this
when it starts in an interactive mode:

\begin{quote}
Gnomovision version 69, Copyright (C) yyyy  name of author \\
Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. \\
This is free software, and you are welcome to redistribute it
under certain conditions; type `show c' for details.
\end{quote}


The hypothetical commands {\tt show w} and {\tt show c} should show the
appropriate parts of the General Public License.  Of course, the commands
you use may be called something other than {\tt show w} and {\tt show c};
they could even be mouse-clicks or menu items---whatever suits your
program.

You should also get your employer (if you work as a programmer) or your
school, if any, to sign a ``copyright disclaimer'' for the program, if
necessary.  Here is a sample; alter the names:

\begin{quote}
Yoyodyne, Inc., hereby disclaims all copyright interest in the program \\
`Gnomovision' (which makes passes at compilers) written by James Hacker. \\

signature of Ty Coon, 1 April 1989 \\
Ty Coon, President of Vice
\end{quote}


This General Public License does not permit incorporating your program
into proprietary programs.  If your program is a subroutine library, you
may consider it more useful to permit linking proprietary applications
with the library.  If this is what you want to do, use the GNU Library
General Public License instead of this License.
\end{tiny}
\end{frame}
}

%\AtBeginLecture{\frame{\Large Today's Lecture: \insertlecture}}
\AtBeginSection[] %or \AtBeginSubsecton{} 
{
  \begin{frame}<beamer>
    \frametitle{\insertsubsection}
    \tableofcontents[hideothersubsections]
    \flushbottom{\tiny{\copyrightnotice}}
  \end{frame}
}
%
% Print Lectures in part
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\includeonlylecture{monday} 
% \includeonlylecture{tuesday} 
% \includeonlylecture{wednesday} 
% \includeonlylecture{thursday} 
% \includeonlylecture{friday} 
% \includeonlylecture{Bibliography} 

\begin{document}
\frame{\titlepage}

\section*{Outline}

%\frame{
%	\frametitle{\insertsubsection}
%	\tableofcontents [hideallsubsections] %[pausesections]
%	}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\lecture{Introduction}{monday}
\renewcommand{\multicite}[1]{{\tiny\nocite{#1}\citemonday{#1}}}
\renewcommand{\licensenumber}[0]{License1}

\part{Introduction to Information Theory}
\section{Introduction to Information Theory}

\subsection{Introduction}
\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{Statistics is the bookkeeping of information}
\begin{itemize}[<+->]
\item Language is about communication
\item Communication implies a message
\item A message is only useful if it is ``surprising'' to some extend
\item That is, the receiver must be uncertain about the content of the message
\item Information and probability quantify uncertainty
\item Information is the more fundamental concept
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{Probability is}
\begin{itemize}[<+->]
\item A measure of the frequency of outcomes
\item A measure of chance given what is known
\item A number between 0 and 1 (inclusive)
\item A measure of our knowledge (or ignorance)
\item Boring?
\end{itemize}
\end{block}

\flushbottom{
\begin{scriptsize}
\multicite{BavaudChappelierKohlas2005}
\multicite{Schneider1999}
\multicite{DJCM03}
\end{scriptsize}
}
\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Axioms}

\begin{block}{Probability: if $E_1, \dots, E_n$ are
possible outcomes of an observation, then $P(E_i)$ is the probability
of outcome $E_i$ iff}
\begin{enumerate}[<+-| alert@+>]
\item $0 \leq P(E_i) \leq 1$
\item $P(E_1 \vee \dots \vee E_i \vee \dots \vee E_n)=1$
\item Additivity:
        $P(E_1 \vee E_2)=P(E_1)+P(E_2)$ \\ 
	where $E_1$ and $E_2$ are mutually exclusive.
\item Countable additivity: \\
     $P(\bigcup_{i=1}^n E_i) = \sum_{i=1}^{n}P(E_i)$ 
     for $n=1, 2, \dots, N$ \\
     where $E_1, E_2, \dots$ are mutually exclusive. 
\end{enumerate}
\end{block}


\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Conventions}

Take three observables: \textit{\underline{C}olor}, 
\textit{\underline{F}lower}, and \textit{\underline{P}lace}

\begin{block}{The following conventions will be used:}
\begin{enumerate}[<+->]
\item $P(C=\textcolor<1>{red}{Red})$: the probability of seeing a 
          \textit{\textcolor<1>{red}{Red flower}}
\item $P(C=\textcolor<2>{red}{Red}, F=Rose)$: 
          the probability of seeing a  \textit{\textcolor<2>{red}{Red Rose}}
\item $P(C=\textcolor<3>{red}{Red}|F=Rose)$: 
          the probability of seeing a  \textit{\textcolor<3>{red}{Red Flower}},
	  given that the flower is a \textit{Rose}
\item $P(C=\textcolor<4>{red}{Red}, F=Rose | P=Flower\ Shop) \leq 1$
\item $P(\textcolor<5>{red}{Red}  \vee  \textcolor<5>{blue}{Blue}) = 
          P(\textcolor<5>{red}{Red})  +  P(\textcolor<5>{blue}{Blue})$ \\
          Basic sum rule for probabilities
\item \alert<6>{$P(C, F | P)  = P(C|F , P) \cdot P(F | P) = P(F | C , P) \cdot P(C | P)$} \\
	   The basic product rule for probabilities
\end{enumerate}
\end{block}


\end{frame}

\subsection{Probability distributions}
\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{Useful distributions, called Probability Density Functions (pdf)}
\begin{itemize}[<+->]
\item Uniform distribution, discrete and uniform
\item Poisson distribution
\item Normal (Gaussian) distribution
\item Zipf distribution
\item Mean value, $\mu$, is called Expected value $\mu = E[x] = \int^{+\infty}_{-\infty} x \cdot P(x) dx$
\item Distribution width is called $Standard\ Deviation$ 
      which is defined as $\sigma = \sqrt{E[(x-E(x))^2]}$
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Uniform Discrete}

\begin{block}{$N$, equally probable and equally spaced values 
               $\{E_1, \dots, E_n\}$ (possibly if 
	       $N \rightarrow \infty$)}
\begin{itemize}[<+-| alert@+>]
\item Each category, $E_i$, has the same probability
\item $P(E_i) = 1/N$
\item Example: Dice $\{1,  \dots, 6\}$ and coins $\{Head, Tail\}$
\item Most basic distribution
\item Default if only the number of values is known
\item Mean $\mu = \dfrac{1}{N}\sum^N_{i=1} E_i = \dfrac{1}{2} (E_1 + E_N)$
\item Variance $\sigma^2 = \dfrac{1}{12} (E_N - E_1)^2$
\end{itemize}
\end{block}

\begin{footnotesize}
\url{http://en.wikipedia.org/wiki/Uniform_distribution_\%28discrete\%29},
\url{http://gwydir.demon.co.uk/jo/probability/diceinfo.htm}
\end{footnotesize}
\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Uniform Continuous}

\begin{block}{Equally probable values in interval $[a,b]$}
\begin{itemize}[<+-| alert@+>]
\item $Pdf(x) = \dfrac{1}{b-a}$
\item Most basic distribution (continuous case)
\item Default if only the range is known
\item Mean $\mu = \dfrac{a+b}{2}$
\item Variance $\sigma^2 = \dfrac{(b-a)^2}{12}$
\end{itemize}
\end{block}

\begin{footnotesize}
\url{http://en.wikipedia.org/wiki/Uniform_distribution_\%28continuous\%29}
\end{footnotesize}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Poisson}

\begin{center}
\parbox{0.99\textwidth}{
\parbox{0.6\textwidth}{
\includegraphics[width=0.4\textwidth]{Pictures/800px-Poisson_distribution_PMF}
}
\parbox{0.35\textwidth}{
$Pdf(k;\lambda) = \dfrac{e^{-\lambda}\lambda^k}{k!}$ \\
$k$: count, $\lambda$: rate
}
}
\end{center} 

\begin{block}{Rare events occuring with a fixed rate $\lambda$}
\begin{itemize}[<+->]
\item Mushrooms per meter of forest, typing errors per page, radio-active decay
\item Average and variance are identical $\mu = \sigma^2 = \lambda$
\item Default if only an average is known
\end{itemize}
\end{block}
\begin{footnotesize}
\url{http://en.wikipedia.org/wiki/Poisson_distribution}
\end{footnotesize}
\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Normal or Gaussian}

\begin{center}
\parbox{0.99\textwidth}{
\parbox{0.5\textwidth}{
\includegraphics[width=0.4\textwidth]{Pictures/Normal_distribution_pdf}
}
\parbox{0.45\textwidth}{
\begin{footnotesize}$Pdf(x; \mu, \sigma) = \dfrac{e^{\frac{-(x-\mu)^2}{2\sigma^2}}}{\sigma\sqrt{2\pi}}$ \\
\vskip1em
$x$: observable\\ $\mu$: Average\\ $\sigma^2$: variance
\end{footnotesize}
}
}
\end{center} 

\begin{block}{General measurements}
\begin{itemize}[<+->]
\item Many physical and physiological measurements, counting
\item Default if both an average and a variance are known
\item A sum of a large number of independent variables is approximately normal
      (under certain conditions)
\end{itemize}
\end{block}
\begin{footnotesize}
\url{http://en.wikipedia.org/wiki/Normal_distribution}
\end{footnotesize}
\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Zipf}

\begin{center}
\parbox{0.99\textwidth}{
\parbox{0.5\textwidth}{
\includegraphics[width=0.4\textwidth]{Pictures/Zipf_distribution_PMF}}
\parbox{0.45\textwidth}{
\begin{footnotesize}
$Pdf(k; s, N) = \dfrac{\dfrac{1}{k^s}}{\sum_{n=1}^{N} \dfrac{1}{n^s}}$ \\
$k$: rank; $s$: exponent\\ $N$: number of elements\\
note logarithmic scales
\end{footnotesize}
}
}
\end{center} 

\begin{block}{Product of frequency and rank is constant: $f_i \approx C \cdot \dfrac{1}{r_i}$}
\begin{itemize}[<+->]
\item Word frequencies, city sizes, high incomes, earthquake sizes
\item Default with power laws
\item For word frequencies, $s \approx 1$
\end{itemize}
\end{block}
\begin{footnotesize}
\url{http://en.wikipedia.org/wiki/Zipf_distribution}
\multicite{dover-2004}\multicite{kawamura-2002}
\end{footnotesize}
\end{frame}

\subsection{Bayesian probabilities}
\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{Incorporating knowledge}
\begin{itemize}[<+->]
\item Probability is predicting outcomes from knowledge
\item Explicitely formulate knowledge as probabilities
\item Formalize the \alert<3>{\textit<4->{probability of a hypothesis}}
\item Destinguish \alert<4>{\textit<5->{a priori}} (knowledge) and 
      \alert<4>{\textit<5->{a posteriori}} (observations) probabilities
\item Determine the information content of a single observation
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{footnotesize}
\begin{eqnarray*}
P(Data, Hypothesis) & = & P(Hypothesis | Data) \cdot P(Data) \\
                    & = & P(Data | Hypothesis) \cdot P(Hypothesis) \\
		    & \Leftrightarrow & \\
P(Hypothesis | Data) & = & \dfrac{\alert<1>{
                         \textbf<2->{P}(\textbf<2->{Data} | \textbf<2->{Hypothesis})}
                         \cdot \alert<2>{\textbf<3->{P(Hypothesis)}}}
                        {\alert<3,4>{\textbf<4->{P(Data)}}}
\end{eqnarray*} 
\end{footnotesize}

\begin{block}{Express $P(Hypothesis|Data)$:}
\begin{itemize}[<+->]
\item As a function of the measurements
\item And the \alert<2>{\textit<3->{a priori}} probability of the hypothesis
\item Normalized by the \alert<3>{\textit<4->{a priori}} 
      probability of the data
\item The normalization probability can often be ignored, as it will be
      identical for all hypotheses
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Toy example}

\begin{block}{Where has Watson most likely been: \textit{Market},
              \textit{Garden}, \textit{Meadow}, \textit{Park}?}
\begin{itemize}[<+->]
\item Watson caries a \textcolor{yellow!80!black}{Yellow} \textit{Buttercup}
\item He divides his walks equally along these ``\textit{places}'' (uniform prior)
\item Which is most likely, obtaining a \textcolor{yellow!80!black}{Yellow} \textit{Buttercup} 
      in a \textit{Market}, a \textit{Garden}, a \textit{Meadow}, or a \textit{Park}?
\item In formula:
\end{itemize}
\end{block}
\uncover<4>{
\begin{eqnarray*}
\underset{p}{argmax} P(p|\textcolor{yellow!80!black}{Y}, B) 
              = \underset{p}{argmax}P(\textcolor{yellow!80!black}{Y}, B|p)\cdot P(p) \\
      where\ p \in \{Market, Garden, Meadow, Park\}
\end{eqnarray*} 
}
\end{frame}

\subsection{Information and probabilities}
\begin{frame}
\frametitle{\insertsubsection: Surprise!}

\begin{block}{Information is a quantification of \textit{surprise}}
\begin{itemize}[<+->]
\item Information depends on probability $p_i$
\item A more surprising observation, ie, a lower $p_i$, caries more information
\item Information should be additive, two CD's can carry twice the information of one CD
\item Define information in observation $O_i$ with probability $p_i$ as 
      \alert<4>{$h(p_i) = -\log_2 p_i$}
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Uncertainty}

\begin{block}{The uncertainty is the average information content and is called
              \textit{Entropy}, $H(p_1, p_2, \dots, p_n)$. \textit{Entropy} should be:}
\begin{itemize}[<+->]
\item \alert<1>{\textit<2->{Independent}} 
      of the labeling, ie, numbering, of $p_i$
\item \alert<2>{\textit<3->{Decomposable}}, splitting a
       category in two gives:
       $H^\prime(p^\prime_1, p^{\prime\prime}_1, \dots) = H(p_1, \dots) + 
               p_1\cdot H(\frac{p^\prime_1}{p_1}, \frac{p^{\prime\prime}_1}{p_1})$
\item \alert<3>{\textit<4->{Continuous}},
      a small change in the probabilities should result in a small 
      change in \textit{entropy}
\item \alert<4>{\textit<5->{Monotonic}}, for a uniform distibution of $n$ items,
      entropy increases monotonically with the number of categories $n \ge 1$
\item $\Rightarrow H(p_1, p_2, \dots, p_n) = -\sum^n_{i=1} p_i \log_2(p_i)$
\end{itemize}
\end{block}

\begin{scriptsize}See chapter 1 of \multicite{BavaudChappelierKohlas2005}\end{scriptsize}
\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{Probability distributions have \textit{entropies}: Examples}
\begin{itemize}[<+->]
\item Discrete Uniform distribution: 
      \alert<1>{$H(\dfrac{1}{N}) = \log_2(N)$}
\item Continuous Uniform distribution  
      \alert<2>{$[a, b]$: $H(\dfrac{1}{b-a}) = \log_2(b-a)$}
\item Poisson distribution: 
      \alert<3>{$H(k; \lambda) =
	\lambda[\dfrac{1}{\ln(2)}-\log_2(\lambda)]+e^{-\lambda}
	\sum^\infty_{k=0}\dfrac{\lambda^k\log_2(k!)}{k!}$}
\item Normal (Gaussian) distribution: 
       \alert<4>{$H(x; \mu, \sigma) = \log_2(\sigma \sqrt{2\pi e})$}
\item Zipf distribution ($C_{N,s} = \sum_{k=1}^{N} \dfrac{1}{k^s}$): \\ 
      \alert<5>{$H(k; s, N) = 
      \dfrac{s}{C_{N,s}} 
      \sum^N_{k=1} \dfrac{\log_2(k)}{k^s} + \log_2(C_{N,s})$}
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Measuring}

\begin{block}{Information is the reduction of uncertainty
$\underset{parameter}{X} \overset{channel}{\Rightarrow} \underset{observation}{Y}$}
\begin{itemize}[<+->]
\item Entropy in $X$ before the observation: \alert<1>{$H(X)$}
\item Entropy after the observation of value of $Y$: \alert<2>{$H(X|Y)$}
\item Average information gained through observing $Y$: \alert<3>{$I(X|Y) = H(X) - H(X|Y)$}
\item If there is \textit{no} uncertainty left after observing $Y$, ie, $H(X|Y)=0$:
      \alert<4>{$I(X|Y) = H(X)$}
\item If $X$ and $Y$ are independent, ie, $H(X|Y) = H(X)$, then \alert<5>{$I(X|Y) = 0$}
\item Always, \alert<6>{$H(X|Y) \le H(X) \Rightarrow I(X|Y) \le H(X)$ }
\item It is common to use $H(\cdot)$ as a synonym of $I(\cdot)$
\end{itemize}
\end{block}

\begin{scriptsize}See chapter 1 of \multicite{BavaudChappelierKohlas2005}\end{scriptsize}
\end{frame}

\subsection{Relative entropy}
\begin{frame}
\frametitle{\insertsubsection: $KL(p:q) = H(p,q) - H(p)$}
 
\begin{eqnarray*}
 KL(p:q) = \sum_i p_i \log_2 \dfrac{p_i}{q_i}\ & \bigvee\ 
  \int_{-\infty}^\infty p(x) \log_2 \dfrac{p(x)}{q(x)} dx \\
  discontinuous & continuous
\end{eqnarray*} 
$ H(p,q) = \sum_i p_i \log_2 q_i$: Cross Entropy

\begin{block}{Kullback-Leibler distance}
\begin{itemize}[<+->]
\item A non-symmetric divergence: $KL(p:q) \ne KL(q:p)$
\item Measures ``distance'' between prob. distributions
\item Information gain between \alert<3>{\textit<4->{Prior}} 
      and \alert<3>{\textit<4->{Posterior}} distribution
\item Example: Word distributions as a distance between 
      document types
\end{itemize}
\end{block}

\begin{footnotesize}
\url{http://en.wikipedia.org/wiki/Kullback-Leibler_distance}
\end{footnotesize}
\end{frame}

\subsection{Compression}
\begin{frame}
\frametitle{\insertsubsection: Minimum size}

\begin{block}{
Entropy, $H(A)$ can be understood as the minimal number of bits needed to 
fully \textit{specify} $A$ 
\alert<1>{\textit<2->{given a known production process}}
}
\begin{itemize}[<+->]
\item In an \alert<1>{\textit<2->{unknown process}}, $K(A)$ replaces $H(A)$ as the information content
\item $K(A)$: Minimum number of bits to  \alert<2>{\textit<3->{reconstruct}} $A$
\item $K(A)$ is the theoretical lower limit of \alert<3>{\textit<4->{compression size}} $C(A)$
\item Practical (lossless) compression packages, $C(A)$, eg, ZIP, GZIP, BZIP2 etc.
      never reach this limit
\end{itemize}
\end{block}
\begin{scriptsize}
$K(A)$ is called the Kolmogorov complexity \multicite{PVitanyi2005}\multicite{chater-2001}
\end{scriptsize}
\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Similarity metric}

\begin{eqnarray}
NCD(A,B) &=& \dfrac{min\{C(A|B), C(B|A)\}}{max\{C(A), C(B)\}} \notag \\
                  &=& \dfrac{C(AB) - min\{C(A), C(B)\}}{max\{C(A), C(B)\}} \notag 
\end{eqnarray} 
NCD: Normalized Compression Distance

\begin{block}{Similarity by compression}
\begin{itemize}[<+->]
\item Always $C(AB) \le C(A) + C(B)$ ($+ constant$)
\item Estimate entropy by suitable ``long range'' compression
\item $K(text) \le C(text)$ in bits
\end{itemize}
\end{block}
\begin{scriptsize}
\url{http://www.complearn.org/ncd.html}
\multicite{Chenetal2004}\multicite{PVitanyi2005}\multicite{chater-2001}
\end{scriptsize}
\end{frame}

\subsection{Markov Chains}
\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{Words and letters never follow each other at random}
\begin{itemize}[<+->]
\item The simplest language ``model'' predicts the next word based on the previous word
\item Markov chain: $P(w_{i+1}|w_i) = \dfrac{P(w_{i+1}, w_i)}{P(w_i)}$ 
\item Can be extended to more words
\item Large amounts of text are needed to determine $P(w_{i+1}, w_i)$ reliably
\item Example Markov text:  \\
\textit{Step which one could go be grabbed. People to Do that my the former Netscape brand's fortunes that means indent command to The user visible displays a.}
\end{itemize}
\end{block}
\begin{scriptsize}
\url{http://en.wikipedia.org/wiki/Markov_chain} \\
Generate texts: \url{http://www.jwz.org/dadadodo/}
\end{scriptsize}
\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Language models}

\begin{block}{With Markov chains, or N-grams, the probability of 
             a sequence can be calculated}
\begin{itemize}[<+->]
\item What is the probability of encounting a sentence $(w_1, \dots, w_n)$?
\item A human style language model is not known
\item Use \textit{N-gram} Markov chains
\item $P(w_1, \dots, w_n) = \prod^n_{i=1} P(w_i|w_1, \dots, w{i-1})$ (exact)
\item $P(w_1, \dots, w_n) \approx \prod^n_{i=1} P(w_i|w_{i-N+1}, \dots, w_{i-1})$ 
      (\textit{N-gram} approximation)
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Perplexity}

\begin{eqnarray}
P_X(Model) &=& 2^{H_{X}(w_i|W_{1 \dots i-1})}  \notag \\
H_X(w_i|W_{1 \dots i-1}) &=& -\sum_{\{W\}} P_{observed}(w_i|\dots) \log P_{model}(w_i|\dots) \notag 
\end{eqnarray} 
$ H_X(\cdot)$: Cross Entropy
\begin{block}{Perplexity: ``average'' number of choices
             for the next word}
\begin{itemize}[<+->]
\item Matches observed with modelled word order
\item A better language model has a lower perplexity
\item For an \textit{N-gram} Markov chain the perplexity is well defined
\item Using the model entropy io. the cross entropy estimates the quality
         of the model on the training corpus
\end{itemize}
\end{block}

\end{frame}

\subsection{Maximum Entropy}
\begin{frame}
\frametitle{\insertsubsection}
\begin{eqnarray}
find \; p^* &=& \underset{p\in C}{argmax}\; H(p) \notag \\
              &=& \underset{p\in C}{argmax} \left( -\sum_{x,y} \tilde{p}(x)p(y|x) \log p(y|x) \right) \notag
\end{eqnarray} 

\begin{block}{Which model, $p^*$, fits my data best and by what criterium?}
\begin{itemize}[<+->]
\item Quantify all constraints (knowledge) and determine the set of possible distributions $p \in C$
\item Determine the average entropy, $H(y|x)$, over the observed (measured) probabilities $\tilde{p}(x)$ 
\item The \alert<3>{best} distribution, $p^*$ has the highest entropy
\end{itemize}
\end{block}

\begin{scriptsize}
\multicite{AB}\multicite{AB96}
\multicite{ALB96}
\multicite{max}
\multicite{RR96}
\end{scriptsize}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Kangaroo example}
\begin{center}
$\frac{1}{3}$ of all kangaroos have \textcolor{blue}{blue} eyes and $\frac{1}{3}$ are left handed
\begin{tabular}{l | c c | r}
blue  & Left  & Handed & \\
eyed &  true & false& tot \\
\hline 
true &  $x$             &  $\frac{1}{3}-x$ & $\frac{1}{3}$ \\
false &  $\frac{1}{3}-x$   &  $\frac{1}{3}+x$ & $\frac{2}{3}$ \\
\hline
tot   &       $\frac{1}{3}$ & $\frac{2}{3}$ & 1 \\
\end{tabular}  
\end{center}

\begin{block}{How many are both \textcolor{blue}{blue} eyed and left handed?}
\begin{itemize}
\item All $0 \le x \le \frac{1}{3}$ are possible
\item $H(x =  \frac{1}{9}) \approx 1.84$ has maximum entropy
\item $x=\frac{1}{9}$ is the only solution with uncorrelated eye color and handedness
\end{itemize}
\end{block}

\end{frame}

\subsection{Bibliography}
\begin{frame}[allowframebreaks]
\frametitle{\insertsubsection}
\nocitemonday{AbelardBayes}
\nocite{AbelardBayes}
\nocitemonday{MND03}
\nocite{MND03}
\nocitemonday{ALB96}
\nocite{ALB96}
\nocitemonday{AB}
\nocite{AB}
\nocitemonday{AB96}
\nocite{AB96}
\nocitemonday{max}
\nocite{max}
\nocitemonday{Jaynes1990}
\nocite{Jaynes1990}
\nocitemonday{DJM95}
\nocite{DJM95}
\nocitemonday{NS96}
\nocite{NS96}
\nocitemonday{Schneider1999}
\nocite{Schneider1999}
\nocitemonday{StromBayes}
\nocite{StromBayes}
\nocitemonday{TC05}
\nocite{TC05}
\nocitemonday{Thornburg}
\nocite{Thornburg}
\nocitemonday{BavaudChappelierKohlas2005}
\nocite{BavaudChappelierKohlas2005}
\nocitemonday{Jaynes1995}
\nocite{Jaynes1995}
\nocitemonday{DJCM03}
\nocite{DJCM03}
\nocitemonday{PVitanyi2005}
\nocite{PVitanyi2005}
\nocitemonday{RR94}
\nocite{RR94}

\bibliographystylemonday{plainnat}
\tiny{
\bibliographymonday{LOTwinterschool2006}
}
\end{frame}


% Insert License text
\insertgnugpl


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\lecture{Introduction}{tuesday}
\renewcommand{\multicite}[1]{\nocite{#1}\citetuesday{#1}}
\renewcommand{\licensenumber}[0]{License2}

\section{Symbolic Information in language}

\subsection{Introduction}
\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{Language is understood in symbolic form}
\begin{itemize}[<+->]
\item Speech is transcribed into phonemes and words
\item Somehow, phonemes seem to matter for understanding
\item What information is carried by a single phoneme?
\item Answer depends on the information in words
\item The predictability of words
\item Word recognition by humans
\item What does a single phoneme contribute?
\end{itemize}
\end{block}

\end{frame}

\subsection{Information in the Lexicon}
\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{\alert<1>{\textit<2->{Content}} and \alert<1>{\textit<2->{function}} words}
\begin{itemize}[<+->]
\item A basic distinction
\item Content words have an independent, lexical, meaning
\item Function words have little, if any, lexical meaning, but chiefly indicate a grammatical relationship
\item Function words form a closed class (around 1000 words)
\item Content words belong to an open class
\item Function words have a high frequency/low information content
\item But how to treat the information content of words itself?
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{Word frequencies follow a Zipf distribution}
\begin{itemize}[<+->]
\item 8 Jane Austens novels (1811-1817)
\item 14,817 word forms on 801,183 words (tokens)
\item Most frequent 10: \textit{the to and of a her I in was it}
      (180,464 tokens)
\item 4,571 words occur only once, 1,992 twice
\item $H = 9.306$ bits/form
\item Kullback-Leibler(Words, Zipfs dist) = 0.173 bits/form \\
      (average information difference)
\end{itemize}
\end{block}

\begin{footnotesize}
\url{http://en.wikipedia.org/wiki/Zipf_distribution}
\multicite{dover-2004}\multicite{kawamura-2002}\multicite{Gutenberg}
\end{footnotesize}
\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{tabular}{l | r r r r}
Author & N & Forms & $H$ & KL(Zipf) \\
\hline
Henry Fielding  & 957,777 & \alert<1>{20,689} &  \alert<2>{9.516} & \alert<3>{0.141} \\
Jane Austen  & 801,183 & \alert<1>{14,817} & \alert<2>{9.306} & \alert<3>{0.173} \\
Bront\"e sisters & 892,779 & \alert<1>{24,658} & \alert<2>{9.828} & \alert<3>{0.108} \\
Plato (transl.) & 1,445,372 & \alert<1>{21,095} & \alert<2>{9.173} & \alert<3>{0.129} \\
\hline
\end{tabular}

\begin{block}{Works of several authors 1750-1900?}
\begin{itemize}[<+->]
\item Large variation in number of distinct word forms (vocabulary)
\item Information per word ($H$) roughly constant ($2^H \approx 890$)
\item Difference with Zipf's distribution $\le 2\%$
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{tabular}{l | r r r r}
          & Fielding  & Austen  & Bront\"e & Plato \\
\hline
Henry Fielding  & \alert<1>{\textit<2->{0.141}} & 0.311 & 0.377 & \alert<4>{0.627} \\
Jane Austen     & 0.323  & \alert<1>{\textit<2->{0.173}} & 0.326 & \alert<4>{0.767} \\
Bront\"e sisters & 0.323  & 0.270 & \alert<1>{\textit<2->{0.10}8} & \alert<4>{0.712} \\
Plato (transl.) & \alert<4>{0.581}  & \alert<4>{0.588} & \alert<4>{0.651} & \alert<1>{\textit<2->{0.129}} \\
\hline 
\end{tabular} 

\begin{block}{Kullback-Leibler divergence or Cross Entropy}
\begin{itemize}[<+->]
\item Diagonal terms are distance to Zipf distribution
\item $KL(q,p) = H(p) - H_X(q,p) \approx $ information difference per word
\item Averaged over one of the distributions
\item Multi-authored translations of Plato differ from novels
\item All works differ more from each other than from Zipf's distr.
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{tabular}{l | r r r r}
          & Austen  & Bront\"e & Plato \\
\hline
Henry Fielding  & 0.489 & 0.460 & 0.447 \\
Jane Austen     &  & 0.526 & 0.536 \\
Bront\"e sisters &  &  & 0.470 \\
\hline 
\end{tabular} 

\begin{block}{Normalized Compression Distance (NCD)}
\begin{itemize}[<+->]
\item $NCD(x,y) = \frac{C(xy)-min(C(x),C(y))}{max(C(x),C(y))}$
\item $C(x) ~$ is number of distinct word forms in text
\item Half the words in the larger corpus do not appear in smaller corpus
\item Jane Austen's works had the smallest ``vocabulary'' $\Rightarrow$ largest distances
\end{itemize}
\end{block}

\end{frame}

\subsection{Markov models}
\begin{frame}
\frametitle{\insertsubsection: Predictability}

\alert<1>{
\begin{equation}
P(w_{i+1}|w_i) = \dfrac{P(w_{i+1}, w_i)}{P(w_i)}
\end{equation} 
} 

\alert<2>{
\begin{equation}
P(w_1, \dots, w_n) \approx \prod^n_{i=1} P(w_i|w_{i-N+1}, \dots, w_{i-1})
\end{equation} 
}

\begin{block}{Extremely useful technology}
\begin{itemize}[<+->]
\item Probability of the next word, given the previous words
\item Probability of a sentence (N-gram)
\item Most likely sentence (eg, using Viterbi algorithm)
\item Humans do not use anything like it
\end{itemize}
\end{block}
\begin{tiny}\multicite{JurafskyMartin2000}\end{tiny}
\end{frame}

\subsection{Hidden Markov Models}

\begin{frame}
\frametitle{\insertsubsection}

\begin{equation}
P(\alert<2>{S_i}|O_i, \alert<2>{S_{i-1}}) = \alert<3>{P(O_i|\alert<2>{S_i})
         \cdot P(\alert<2>{S_i}|\alert<2>{S_{i-1}})} \notag
\end{equation} 

\begin{block}{HMM technology is extremely important for language (re-)search}
\begin{itemize} [<+->]
\item Split the problem in an Observation part and a
       Model part
\item The ``states'', \alert<2>{$S_i$}, are not visible
\item However, they can be estimated using the observation probabilities
      \alert<3>{$P(O_i|S_i)$} and the transition probabilities 
      \alert<3>{$P(S_i|S_{i-1})$}
\item Standard in Automatic Speech Recognition (ASR)
\end{itemize}
\end{block}

\begin{tiny}\multicite{JurafskyMartin2000}\end{tiny}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Pronunciation networks}

\includegraphics[width=0.9\textwidth]{Pictures/fig07-05} 

\begin{block}{Construct phone state models for each word in the dictionary}
\begin{itemize} [<+->]
\item Possible pronunciations for each word have to be encoded in the dictionary
\item Transition probabilities are "trained" from the frequency of occurrence 
      of the pronunciation in the corpus
\end{itemize}
\end{block}

\begin{tiny}\multicite{JurafskyMartin2000}\end{tiny}
\end{frame}


\begin{frame}
\frametitle{\insertsubsection: Hidden states}

\includegraphics[width=0.9\textwidth]{Pictures/fig07-03} 

\begin{block}{Word models: simple phone state model for \textit{need}}
\begin{itemize} [<+->]
\item Each transition has a probability
\item \alert<2>{start} and \alert<2>{end} are special states
\item Each state \emph{or} each transition has associated sound observations with a distinct probability density function (PDF)
\end{itemize}
\end{block}
\begin{tiny}\multicite{JurafskyMartin2000}\end{tiny}
\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Observation probabilities}

\includegraphics[width=0.7\textwidth]{Pictures/fig07-04} 

\begin{block}{Observed are sound "spectra" for time "frames"}
\begin{itemize} [<+->]
\item Observation sequences have a probability
\item Calculate this probability for each possible word
\item Probabilities of observation $O_i$ calculated from all possible underlying states
\item Chose word \emph{sequence} with the highest overall probability
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Phone networks}

\includegraphics[width=0.9\textwidth]{Pictures/HMMphoneNetwork} 

\begin{block}{Phone models are concatenated into utterance networks}
\begin{itemize} [<+->]
\item Each word model is itself a Markov finite state network of phone models
\item Phones and word are connected through the \emph{start} and \emph{end} states (not shown)
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{Hidden Markov Models are very useful for:}
\begin{itemize} [<+->]
\item Automatic Speech Recognition
\item Automatic POS tagging: $P(POS_i|Word_i, POS_{i-1}) = P(Word_i|POS_i) \cdot P(POS_i| POS_{i-1})$
\item Accent assignment \begin{tiny}\multicite{PanMcKeown1999}\end{tiny}
\item Automatic translation
\item Speech Understanding with Semantic HMMs $P(Message_i|Words) = 
      P(Words|Message_i) \cdot P(Message_i|Message_{i-1})$ 
      \begin{tiny}\multicite{Buehleretal2005}\end{tiny}
\item Still, humans do \alert<6>{not} use (Hidden) Markov Models
\end{itemize}

\end{block}

\end{frame}


\subsection{Predictability in context}
\begin{frame}
\frametitle{\insertsubsection}

Movies\dots \uncover<2->{pop-corn} \\
Cats and\dots \uncover<3->{dogs} \\
A stitch in time saves\dots \uncover<4->{nine} \\
Good morning,\dots \uncover<5->{how are you} \\

\begin{block}{No real model of human grammar but there are regularities
in human (re-)cognition}
\begin{itemize}[<+->]
\item Priming: people expect words given associates
\item Semantic web: words in proximity tend to be cognates
\item Words group in ``documents''
\item Words tend to cluster inside ``texts''
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: In-document frequency}

\begin{block}{Distinguish global and local word frequencies: 
             \begin{footnotesize}
	     Term Frequency times Inverse Document Frequency (TF*IDF)
	     \end{footnotesize}}
\begin{itemize}[<+->]
\item Humans expect certain words based on topic, form, and style
\item Break down corpus into small units, eg, articles, of roughly equal size
\item Determine global word frequency
\item Divide it by the fraction of documents the word is used in
\item In-document frequency accounts statistically for both \textit{priming}
      and \textit{semantic webs}
\end{itemize}
\end{block}
\begin{scriptsize}\multicite{PanMcKeown1999}\end{scriptsize}
\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Context distinctiveness}

CD = Kullback-Leibler distance between local
context distribution and global word distribution \\
\begin{scriptsize}\multicite{McDonaldShill2001}\end{scriptsize}

\begin{block}{Predictability from direct context}
\begin{itemize}[<+->]
\item Use a statistical predictor
\item Determine information about a word in its context
\item $\Rightarrow$ Kulback-Leibler divergence
\item Bag-of-words technique, eg, 10 word contexts 
	\begin{scriptsize}\multicite{SproatvanSanten1998}\end{scriptsize}
\item Difference in distribution around a word \alert<5>{\textit<6->{predicts}} the word
\item Subtract context divergence from word \alert<6>{\textit<7->{information}}
\item Context increases ``perceived'' frequency
\end{itemize}
\end{block}

\end{frame}

\subsection{Human word recognition}
\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{Basic aspects of word recognition}
\begin{itemize}[<+->]
\item Prelexical: Sound $\rightarrow$ Phonemic Symbols
\item Postlexical:
\item $\rightarrow$ Activation
\item $\rightarrow$ Competition
\item $\rightarrow$ Selection
\end{itemize}
\end{block}

\begin{scriptsize}
\multicite{Cutler1997SpeechComm}
\multicite{NorrisMcQueenCutler200BehBrainSc}
\multicite{McQueenCutlerNorris2003SpeechComm}
\end{scriptsize}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Postlexical stage - Activation}

\begin{footnotesize}
\begin{tabular}{l | l r r r}
Phon. & ML Word & I(W) & Matching N & \textit{Dutch} \\
\hline
\textipa{b} & bij & 7.882 & 122,629 & terms \\
\textipa{bo:} & bovendien & 11.658 & 5772 & terms \\
\textipa{bo:m} & bomen & 14.831 & 798 & terms \\
\hline
\end{tabular} 
\end{footnotesize}

\begin{block}{After recognizing a new phoneme, words are activated}
\begin{itemize}[<+->]
\item \alert<1>{\textit<2->{All}} matching words are activated
\item Frequent words are activated more than rare words
\item Non-matching phonemes decrease or end the activation of a word
\item A word end is identified after an impossible continuation
\item Word boundaries are also indicated by prosodic markers
\item Simple model ignores morphological structure
\end{itemize}
\end{block}
\begin{scriptsize}
\multicite{Cutler1997SpeechComm}
\multicite{NorrisMcQueenCutler200BehBrainSc}
\multicite{McQueenCutlerNorris2003SpeechComm}
\end{scriptsize}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Postlexical stage - Competition}

\begin{block}{Words ``compete'' for activation}
\begin{itemize}[<+->]
\item Recognition is blocked as long as there are multiple candidates left
\item Only when there is a single word left, is it recognized
\item Words can occur inside other words
\item $\Rightarrow$ wait for word boundary 
      \begin{tiny}\multicite{Cutler1997SpeechComm}\end{tiny}
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Postlexical stage - Selection}

\begin{block}{Competition blocks selection}
\begin{itemize}[<+->]
\item If there is only a single candidate left, chose it
\item If there is no single perfect candidate, chose the best
      (eg, \textit{cigaret} for \textipa{/SIg@rEt/})
\item Ganong effect: All candidates that fit an incomplete phoneme
      remain activated \\
      (eg, \textipa{/(g/k)ot/} $\rightarrow$ \textit{Goat} or \textit{Coat})
\item Without enough phonetic ``evidence'', the most likely word is chosen
      $\rightarrow$ phonemic restoration
\item People are only rarely aware of what \textit{phonemes} have actually been spoken
\end{itemize}
\end{block}

\end{frame}

\subsection{Phonemic information}
\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{What difference a phoneme makes?}
\begin{itemize}[<+->]
\item Combine Information theory with Human word recognition
\item Words matching a phoneme onset with and without the new phoneme added
\item $I(phon|onset) = H(W|onset+\star) - H(W|onset+phon)$
\item $H(W|S)$: Entropy of words matching $S$
\item Use a \alert<5>{\textit<6->{large}} 
      (automatically) transcribed corpus, eg, $350 Mword$
       \textit{Twente News Corpus} \tiny{\multicite{ordelman2002}}
\item Sensitive to vocabulary structure
\item But: Psychologically implausible?
\item Compound words, $> 1$ million word-forms?
\item No data available on this measure
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Alternative}

\begin{eqnarray}
I(phon|onset) &=& -\log_2 P(phon|onset) \notag \\
                 &=& -\log_2 \dfrac{TokenCount(onset+phon)}{TokenCount(onset+\star)}\notag
\end{eqnarray}

\begin{block}{Information in phoneme given preceding onset}
\begin{itemize}[<+->]
\item Insensitive to vocabulary structure
\item Can use smaller ($\approx 30 Mw$) corpora
\item Adaptable
\item Psychological plausibility???
\item Data are available
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: In-document frequency}


\begin{equation}
I^\prime(phon|onset, w) = -\log_2 \dfrac{TokenCount(onset+phon)+ D(w)}
                                        {TokenCount(onset+\star)+D(w)}\notag
\end{equation}
\begin{footnotesize}
$D(w) = (TF(w)\cdot IDF(w) - TF(w))\cdot TotalCount$
$TF(w)$: Term frequency of $w$
\end{footnotesize}

\begin{block}{Include statistical predictability of in-document frequency}
\begin{itemize}[<+->]
\item In the text, the correct word will be ``predictable''
\item Perceived ``frequency'' of the correct word, $w$, 
      is the in-document frequency, $TF(w)\cdot IDF(w)$
\item Assume only the correct word, $w$, is boosted
\item Replace global frequency of $w$ by in-document frequency
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Context distinctiveness}

\begin{equation}
I^{\prime\prime}(phon|onset, w) = -\log_2 \dfrac{TokenCount(onset+phon)+ D(w)}
                                        {TokenCount(onset+\star)+D(w)}\notag
\end{equation}
\begin{footnotesize}
$D(w) = (TF(w)\cdot2^{CD(w)} - TF(w))\cdot TotalCount$\\
$CD(w) = KL(LocalDistr(w),GlobalDistr)$\\
$TF(w)$: Term frequency of $w$
\end{footnotesize}

\begin{block}{Focus on direct context of $w$: $LocalDistr(w)$}
\begin{itemize}[<+->]
\item $CD(w)$: Kullback-Leibler distance between local and global distribution
\item Perceived frequency is $TF(w)\cdot 2^{CD(w)}$
\item Data \textit{are} available
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Context distinctiveness}

\begin{block}{Example: \textipa{/o/} in Dutch ``boom'' (\textit{tree}) using CELEX/CGN}
\begin{footnotesize}
\begin{itemize}[<+->]
\item Word tokens starting with \textipa{/bo/}: \alert<1>{67,710} \begin{scriptsize}(1,172 CELEX entries)\end{scriptsize}
\item The same for \textipa{/b./}: 	\alert<2>{1,544,483} \begin{scriptsize}(26,186 CELEX entries)\end{scriptsize}
\item $\alert<11>{I} =-\log_2(\dfrac{67710}{1544483}) = \alert<3,11>{4.51}$ bit
\item Relative CGN frequency of \textit{boom}: 	\alert<4>{$5.05\cdot 10^{-5}$}
\item Context Distinctiveness: $CD(boom) =  \alert<5>{4.53}$ bit
\item Relative frequency in context: 	$2^{CD(boom)}\cdot 5.05\cdot10^{-5}= \alert<6>{1.2\cdot10^{-3}}$
\item CELEX word count of \textit{boom}: 	\alert<7>{2,226 }  \begin{scriptsize}(smoothed count)\end{scriptsize}
\item Context-corrected CELEX count: 	$\alert<8>{45,402}\  (=1.2\cdot 10^{-3}\cdot39\cdot10^6)$
\item Correction term: $D(boom)=  45,402 - 2,226 = \alert<9>{43,176}$
\item $\alert<11>{I^{\prime\prime}}=-\log_2([67710+43176]/[1544483+43176]) = \alert<10,11>{3.84}$
\item That is, \alert<11>{$I^{\prime\prime} < I_{}$}, so context reduces lexical uncertainty.
\end{itemize}
\end{footnotesize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection}


\begin{center}
\includegraphics[height=0.5\textheight]{Pictures/RvSProceedings25Fig1} 
\begin{scriptsize}\textit{Segments (x1000)}\end{scriptsize}
\end{center}

\begin{block}{$I^{\prime\prime}(phon|contex, w)$ versus the position in the word}
\begin{itemize}[<+->]
\item Only syllables without a \textipa{/@/}
\item Strong decline after a few positions part of model
\item All manners of articulation carry the same information
\end{itemize}
\end{block}

\end{frame}

\subsection{Bibliography}
\begin{frame}[allowframebreaks]
\frametitle{\insertsubsection}

\nocitetuesday{Cutler1997SpeechComm}
\nocite{Cutler1997SpeechComm}
\nocitetuesday{Cutler1997SpeechComm}
\nocitetuesday{RDUD96}
\nocite{RDUD96}
\nocite{RJKosinski2003}
\nocitetuesday{McDonaldShill2001}
\nocite{McDonaldShill2001}
\nocitetuesday{McQueenCutlerNorris2003SpeechComm}
\nocite{McQueenCutlerNorris2003SpeechComm}
\nocitetuesday{NorrisMcQueenCutler200BehBrainSc}
\nocite{NorrisMcQueenCutler200BehBrainSc}
\nocitetuesday{Owensetal1997}
\nocite{Owensetal1997}
\nocitetuesday{VSP03}
\nocite{VSP03}
\nocitetuesday{VanSonVanSanten2005}
\nocite{VanSonVanSanten2005}

\bibliographystyletuesday{plainnat}
\tiny{
\bibliographytuesday{LOTwinterschool2006}
}
\end{frame}


% Insert License text
\insertgnugpl


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\lecture{Introduction}{wednesday}
\renewcommand{\multicite}[1]{\nocite{#1}\citewednesday{#1}}
\renewcommand{\licensenumber}[0]{License3}

\section{Acoustic information in speech}

\subsection{Introduction}
\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{Spoken language is mostly sound: Speech}
\begin{itemize}[<+->]
\item To understand speech we must understand speaking and hearing
\item Hearing limits the information we can distill from sound
\item Articulation limits what we can produce within these limits
\item What information can be extracted from speech sounds?
\item How can we study this?
\item How can we study human speech processing?
\end{itemize}
\end{block}

\end{frame}

\subsection{Acoustics of speech}
\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{The speech chain constist of \alert{articulation}, 
              \alert{sound}, and \alert{hearing}}
\begin{itemize}[<+->]
\item The speech chain is \alert<1>{\textit<2->{not}} symbolic
\item Articulation is preceeded by formulation
\item Hearing is followed by recognition
\item Both formulation and recognition center around 
      \alert<4>{\textit<5->{phonemic segments}} and \alert<4>{\textit<5->{words}}
\item However, speech does contain \alert<5>{\textit<6->{neither}}, just sound
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Oscillogram}
\center{
\movie [externalviewer]{
\includegraphics[width=0.6\textwidth]{Pictures/ErWasEensEenOudKasteel}
}{Sounds/F28G1FT2A1+6_fm.wav}
}
\begin{block}{"Er was eens een oud kasteel"}
\begin{itemize} [<+->]
\item Display of presure versus time
\item Words are aligned with sound
\item Using computer readable (SAMPA) phoneme symbols
\end{itemize}
\end{block}
\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Digital sound and band-width}
\center{\includegraphics[width=0.45\textwidth]{Pictures/s-quantisized}}
\begin{block}{1.5 ms of an \textipa{/s/} sound from "was"}
\begin{itemize} [<+->]
\item Samples taken at 44.1 kHz (CD audio)
\item Quantisize at 16 bit ($\approx 65000$ amplitude levels)
\item Maximum audio frequency 22.05 kHz (Nyquist frequency)
      but generally \emph{much} less
\item Dynamic Range $\approx 96 db$ ($\approx 6 db / bit$)
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Amplitude and sound level}
\center{\movie [externalviewer]{
\includegraphics[width=0.5\textwidth]{Pictures/KasteelIntensity}}
{Sounds/F28G1FT2A6_fm_Kasteel.wav}
}
\begin{block}{Intensity contour of "Kasteel"}
\begin{itemize} [<+->]
\item Intensity versus amplitude
\item Intensity in db ($10 \cdot log_{10}(Sound Energy)$)
\item Intensity you hear is not the intensity you measure
	$\Rightarrow$ correct for human perception ($dBA$)
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Durations}
\center{\movie [externalviewer]{
\includegraphics[width=0.45\textwidth]{Pictures/KasteelPhonemes}}
{Sounds/F28G1FT2A6_fm_Kasteel.wav}
}
\begin{block}{Phoneme segmentation of "Kasteel"}
\begin{itemize} [<+->]
\item Determine the boundaries of words, syllables and phonemes
\item Use waveform, ear, and spectrum
\item Segmentation is ambiguous and laborious
\item Start with automatic segmentation (for speed)
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: The perception of tones: $F_0$}
\center
{
\movie [externalviewer]
{\includegraphics[width=0.40\textwidth]{Pictures/125Hztone}}
{Sounds/125Hztone.wav}
\movie [externalviewer]
{\includegraphics[width=0.48\textwidth]{Pictures/3tonecomplex125Hz}}
{Sounds/3tonecomplex125Hz.wav}
}
\begin{block}{Pitch or $F_0$ is the \emph{perception} of a harmonic
             sequence. Generally, perceived $pitch$ is the:}
\begin{itemize} [<+->]
\item frequency of a pure tone (top, 125 Hz)
\item distance between the components in a mixture of harmonic tones
      (eg, 125 Hz)
\item closest harmonic fit in complex sounds (bells)
\end{itemize}
\end{block}
\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Measuring $F_0$}
\center
{
\movie [externalviewer]
{\includegraphics[width=0.5\textwidth]{Pictures/F28G1FT2A_Pitch}}
{Sounds/F28G1FT2A1+6_fm.wav}
\movie [externalviewer]
{\includegraphics[width=0.5\textwidth]{Pictures/F28G1FT2A_PitchTier}}
{Sounds/F28G1FT2A1+6_fmHum.wav}
}
\begin{block}{The best $F_0$ candidates are determined}
\begin{itemize} [<+->]
\item from the possible repeat frequencies using an autocorrelation function
\item from the best fitting harmonics using a \emph{harmonic sieve}
\end{itemize}
\end{block}
\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Pitch contours}
\center
{
\movie [externalviewer]
{\includegraphics[width=0.8\textwidth]{Pictures/F28G1FT2A_PitchTier}}
{Sounds/F28G1FT2A_fm.wav}

\movie [externalviewer]
{\Large \textcolor{blue}{Hummed sound}}
{Sounds/F28G1FT2A_fmHum.wav}
}
\begin{block}{$F_0$ makes the melody, or intonation, of an utterance}
\begin{itemize} [<+->]
\item There is a general decrease of $F_0$ over an utterance: The \emph{declination}
\item $F_0$ movements indicate emphasized words: pitch \emph{accents}
\item $F_0$ movements and \emph{declination resets} indicate boundaries
\end{itemize}
\end{block}
\end{frame}

\subsection{Hearing}

\begin{frame}
 \frametitle{\insertsubsection: The human ear}
\begin{figure}[t]
\includegraphics[height=0.4\textheight]{Pictures/Ear-anatomy-text-small.png}
\includegraphics[height=0.4\textheight]{Pictures/Cochlea-crosssection.png}
\end{figure}

\begin{block}{The anatomy of the human ear (l) and cochlea (r)}
\begin{itemize}[<+->]
\item Sound enters the ear canal (left)
\item The ear drum transfers the sound over the ossicles to the oval window into the cochlea
\item The cochlea (right) converts sound into nerve exitations
\end{itemize}
\end{block}
\url{http://en.wikipedia.org/wiki/Human_ear}
\end{frame}

\begin{frame}[label=Cochlea]
\frametitle{\insertsubsection: The basilar membrane}
\center
{
  \includegraphics[width=0.6\textwidth]{Pictures/lec6_fig1}
}

\begin{block}{Frequency map of the basilar membrane from 
       \begin{scriptsize}\multicite{Moo03} \nocite{Moo03}\end{scriptsize}}
\begin{itemize} [<+->]
\item The ear analyses sounds roughly into 
      $Log \left( Power \left( Frequency \right) \right)$ 
      vs. $Log \left(  Frequency \right)$
\item Speech is analyzed in the same way
\item Use power spectra of sounds
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Loudness}
\center
{
  \includegraphics[height=0.4\textheight]{Pictures/FletcherMunson_ELC.png}
}

\begin{block}{Relation between perceived loudness and frequency }
\begin{itemize} [<+->]
\item The absolute threshold varies with frequency
\item Best hearing between 800Hz and 6000Hz
\item Human speech needs 400Hz - 3400Hz (telephone speech)
\item Loudness JND is around 1 dB (soft sounds)
\end{itemize}
\end{block}
\begin{scriptsize}\url{http://en.wikipedia.org/wiki/Threshold_of_hearing}\end{scriptsize}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Critical bands}

\parbox{0.99\textwidth}{
\parbox{0.5\textwidth}{
\center
{
  \includegraphics[width=0.45\textwidth]{Pictures/JNDcritbndfig_5.png} 
}
}
\parbox{0.45\textwidth}{
\begin{itemize} [<+->]
\item Sounds grouped in \textit{critical bands} (CB)
\item CB is around 30 times JND
\item One signal per critical band
\item 3 CB per Octave
\item Time resolution $\approx 25 ms$
\end{itemize}
}
}

\begin{scriptsize}\url{http://www.music.gla.ac.uk/~george/audio/psy/psy.html}\end{scriptsize}
\end{frame}

\begin{frame}[label=E_Spectrum]

\frametitle{\insertsubsection: Example of \textipa{/E/}}
\center
{
\hyperlink{E_FormantSpectrum}{
\includegraphics[width=0.8\textwidth]{Pictures/E_spectrum}
}
}

Note the harmonic structure and the "bumps"
\end{frame}

\begin{frame}

\frametitle{\insertsubsection: Example of \textipa{/n/}}
\begin{center}
\includegraphics[width=0.8\textwidth]{Pictures/n_spectrum}
\end{center}

Note the harmonic structure and the low level of high frequencies
\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Example of \textipa{/s/}}
\begin{center}
\includegraphics[width=0.8\textwidth]{Pictures/s_spectrum}
\end{center}

Note the noisy structure and the broad bandwidth
\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{How much information can the ear extract at a minimum from speech?}
\begin{itemize} [<+->]
\item Frequencies: 400Hz - 3400Hz $\rightarrow$ 3 octaves $\approx 9$ 
      CBs
\item Dynamic range $\approx 30dB$ per CB, $\rightarrow 5$ bit / CB
\item Time resolution $\approx 25 ms$
\item Information available: $\dfrac{9 CB \cdot 5 bit}{0.025 sec} \approx 1800$ bit/second
\item Just an order of magnitude estimation
\end{itemize}
\end{block}
\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{How much information is actually needed?}
\begin{itemize} [<+->]
\item Concatenated phoneme transitions (diphones) are perfectly intelligble 
\item There are $\approx 1600$ diphones ($40 \cdot 40$ allophones)
\item $1600$ diphones fit easily in $11$ bit
\item Articulation rate $\le 15$ phones/second 
\item Information needed for recognition: $11 \cdot 15 = 165$ bits/second
\item Compare to $1800$ bits available, ample redundancy
\end{itemize}
\end{block}
\end{frame}

\subsection{Speaking}
\begin{frame}
\frametitle{\insertsubsection: Articulators}
\parbox{0.99\textwidth}{
\parbox{0.5\textwidth}{
\center
{
  \includegraphics[width=0.5\textwidth]{Pictures/HeadCrossSection.png}\\
  Vocal trackt and primary articulators
}
}
\parbox{0.45\textwidth}{
\begin{itemize} [<+->]
\item The lungs pump air through the glottal folds
\item The glottal folds can vibrate (or not)
\item The oral cavity can be constricted at many points
\item Alternatively, air can flow through the nasal cavity
\end{itemize}
}
}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Source Filter model of speech}
\begin{center}
\includegraphics[height=0.3\textwidth]{Pictures/OralTrackt}
\includegraphics[width=0.6\textwidth]{Pictures/VoiceSource_fig2}
\end{center}

Sound enters the oral cavity (vocal tract) from below and is filtered 
by the resonances of the cavity

\end{frame}


\begin{frame}[label=E_FormantSpectrum]
\frametitle{\insertsubsection: Resonances and formants}

\begin{center}
\hyperlink{E_Spectrum}{
\includegraphics[width=0.7\textwidth]{Pictures/E_formantspectrum}
}
\end{center}

Oral cavity filter function of \textipa{/E/} (LPC model). 
Peaks are formants $F_1$ and $F2$.

The resonances of the vocal tract are called Formants, and numbered
from below, i.e., $F_1$, $F_2$, $F_3$, $\cdots$. Normally,
the first three are sufficient to describe (voiced) speech.

\end{frame}


\begin{frame}
\frametitle{\insertsubsection: Vowel Formant space}

\begin{center}
\includegraphics[width=0.5\textwidth]{Pictures/Proc24IFAcorpusFig1}
\end{center}

Vowel formant space of Dutch.

Only two formant values, $F_1$ and $F_2$, suffice to identify
a vowel (in the ideal case). However, in normal speech, there
is so much overlap and variation that it remains almost 
impossible.

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Reduction}

\begin{center}
\includegraphics[width=0.4\textwidth]{Pictures/VowelReduction}
\end{center}

\begin{block}{Articulation takes effort and humans are ``lazy''} 
\begin{itemize}[<+->]
\item Unimportant, unstressed, items are articulated ``less well''
\item Leads to \alert<2>{\textit<3->{less distinctive}} sound segments
\item Visible as a smaller vowel triangle (see above)
\item Also found in consonants
\end{itemize}
\end{block}

\end{frame}

\subsection{Spectrograms}

\begin{frame}
\frametitle{\insertsubsection}

\begin{center}
\includegraphics[width=0.5\textwidth]{Pictures/KasteelSpectrogram}
\end{center}

\begin{block}{A spectrogram shows the development of the spectrum in
              time (darker is more power)} 
\begin{itemize}[<+->]
\item A spectrogram shows the harmonics
\item Vowels, fricatives, and plosives are visible
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Narrow versus Wide band}

\begin{center}
\includegraphics[width=0.45\textwidth]{Pictures/KasteelNarrow100ms}
\includegraphics[width=0.45\textwidth]{Pictures/KasteelWide10ms}
\end{center}

\begin{block}{Two views on spectrograms} 
\begin{itemize}[<+->]
\item Narrow-band (top): High frequency resolution, low time resolution
\item Wide-band (bottom): Low frequency resolution, high time resolution
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Formant and Pitch tracking}

\begin{center}
\includegraphics[width=0.7\textwidth]{Pictures/KasteelPitchFormants}
\end{center}

Formants (red dots) and Pitch (blue line) can be automatically 
determined and plotted into a spectrogram.

\end{frame}


\begin{frame}
\frametitle{\insertsubsection: Noise and bursts}

\begin{center}
\includegraphics[width=0.7\textwidth]{Pictures/st_NoiseAndBurst}
\end{center}

Fricatives are visible as gray noise patches. Plosives as 
a silent part followed by a noisy burst.

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Spectrogram reading}

\begin{center}
\includegraphics[width=0.7\textwidth]{Pictures/KasteelPitchFormants}
\end{center}

It is actually possible, after a few weeks training, to read spectrograms.
All the information needed to "understand" the speech is in the spectrogram 
\multicite{LC97}.
\end{frame}

\subsection{Corpora}

\begin{frame}
\frametitle{\insertsubsection: Transliteration}

\begin{block}{Before anything can be done with speech, it has to be
              written down and transcribed} 
\begin{itemize}[<+->]
\item Write out orthographically what was said (and check it)
\item Align chunks of text roughly with the stretches of speech
\item Transcribe the text automatically into phonemes using a lexicon
\item Split the orthographic/phonemic text into words
\item Align the words/phonemes automatically with the speech
\item Add automatic Part-of-Speech tags and Syntax
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Transcription}

\begin{block}{Human annotator transcriptions: Difficult and expensive} 
\begin{itemize}[<+->]
\item Accents, stress, and boundaries (always ambiguous)
\item Handcorrected word-boundaries
\item Handcorrected phoneme-boundaries (always ambiguous)
\item Check Part-of-Speech tags
\item Check Syntax
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: \\Identifying and annotating phonemes}

\center{\movie [externalviewer]{
\includegraphics[width=0.5\textwidth]{Pictures/KasteelPhonemes}}
{Sounds/F28G1FT2A6_fm_Kasteel.wav}
}

\begin{block}{Phonemes are not pearls on a string} 
\begin{itemize}[<+->]
\item Phonemes always overlap and are extremely variable
\item A phoneme you hear can appear absent in the waveform
\item It is often unclear what phonemes were uttered
\item Sometimes, even the order is unclear
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: \\ToBI like systems for intonation transcription}

\begin{center}
\includegraphics[width=0.45\textwidth]{Pictures/ToDIvoorb}
\movie [externalviewer]{\includegraphics[width=0.45\textwidth]{Pictures/ToDI052}}
{Sounds/ToDI052.wav}
\end{center}

\begin{small}
\begin{block}{ToDI symbols (IP: Intonational Phrase)
\begin{scriptsize}\multicite{ToDI2003}\end{scriptsize}}

\begin{tabular}{|l l l|}
\hline
High & Low & description\\
\hline
H*	&L*	&high/low accent\\
H	&L	&upward/downward movement after L*/H*\\
H\%	&L\%	&rising/low ending of IP\\
\%H	&\%L	&high/low beginning of IP\\
\%HL	&       &Initial falling pitch not marking accent\\
\%	&       &half-completed fall/rise at end of IP\\
!H*	&       &downstepped H*\\
\hline
\end{tabular}
\end{block}

\end{small}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{There is no data like more data}
\begin{itemize} [<+->]
\item Speech and Language are extremely complex
\item Large amounts of data are necessary to model them
\item ``The best application/research is the one with the largest corpus''
\item 10-1000 hours of speech recordings needed
\item $10^8$ - $10^9$ word text corpus needed
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: for Speech and Language research}

\emph{A language corpus is a documented collection of coherent text,
speech, video, and transcriptions and annotations of these}

\begin{block}{Requirements}
\begin{itemize} [<+->]
\item Meta data (fixed): Information on the items
\item Normalization (fixed): All items must adhere to certain guidelines
\item Data (fixed): Immutable text or speech records
\item Transcriptions and annotations (cumulative): 
		Added value of interpretations and analysis
\item Storage, distribution, access, and software (volatile): Practical use
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Spoken Dutch Corpus (CGN) \begin{scriptsize}\multicite{cgn1.0}\end{scriptsize}}

\begin{block}{Contents ($\dfrac{2}{3}$ Dutch, $\dfrac{1}{3}$ Flemish)}
\begin{itemize} [<+->]
\item 500 hours (5,650,000 words) recorded in The Netherlands
\item 300 hours (3,250,000 words) in Flanders
\item 9 million words
\item 4250 speakers
\item 15 Styles/genres (formal reading to informal multilogues)
\item Field recordings with Sony Minidisk
\item 16/16 and 8/8 kHz/bit encoding
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: CGN Annotations}

\begin{small}
\begin{block}{Annotations and transcriptions}
\begin{itemize} [<+->]
\item Orthographic transcription (the full 8,900,000 words)
\item Manually verified POS tagging and lemmatization (all)
\item Lexicon and identification of multi word units (all)
\item Automatic time alignment and phonetic transcription at the word level (all)
\item Manually verified broad phonetic transcription (1,000,000 words)
\item Manually verified time alignment at the word level (1,000,000 words)
\item Syntactic annotation (1,000,000 words)
\item Two independent prosodic annotations (250,000 words)
\end{itemize}
\end{block}
\end{small}

\end{frame}
\subsection{Bibliography}
\begin{frame}[allowframebreaks]
\frametitle{\insertsubsection}

\nocitewednesday{vanBeinum1987}
\nocite{vanBeinum1987}
\nocitetuesday{Sacra1996}
\nocite{Sacra1996}
\nocitetuesday{Posner2005PLoS}
\nocite{Posner2005PLoS}
\nocitetuesday{SigmanDehaene2005PLoS}
\nocite{SigmanDehaene2005PLoS}

\bibliographystylewednesday{plainnat}
\tiny{
\bibliographywednesday{LOTwinterschool2006}
}


\end{frame}


% Insert License text
\insertgnugpl


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\lecture{Introduction}{thursday}
\renewcommand{\multicite}[1]{\nocite{#1}\citethursday{#1}}
\renewcommand{\licensenumber}[0]{License4}

\section{Information Structure}

\subsection{Introduction}
\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{There is more in speech than lexical information}
\begin{itemize}[<+->]
\item Speech is \alert<1>{\textit<2->{sequential}}!
\item Lexical information is unevenly distributed
\item Words have to be isolated to be recognized
\item Some words are less important than others
\item Words come in groups
\item Semantic and syntactic structure should be reflected in the speech
\item These are questions about recognition and processing
\item How to study them?
\end{itemize}
\end{block}

\end{frame}

\subsection{Reaction time experiments}
\begin{frame}
\frametitle{\insertsubsection: \\
             How do humans process speech?}
\begin{center}
\includegraphics[width=0.8\textwidth]{Pictures/ReactionTimeExp}
\end{center}

\begin{block}{Time the difference between stimulus and response}
\begin{itemize}[<+->]
\item Model perception, processing, and motor reaction
\item Manipulate one of the steps using the stimulus
\item Find timing difference
\item Extremely sensitive probe
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Model}

\begin{figure}
	\includegraphics[height=0.40\textheight]{Pictures/PCMmodel.pdf}
\end{figure} 

\begin{block}{Three temporal stages in Reactions to Stimuli:}
\begin{itemize}[<+->]
\item Perceptual  ($P$) and motor ($M$) component
\item Both with deterministic response-times ($t_p$ and $t_m$)
\item Central {\usebeamercolor[fg]{structure}\textbf{decision making component}} ($C$)
characterized by a random walk to a decision threshold
\end{itemize}
\end{block}
	
\begin{scriptsize}
\multicite{Posner2005PLoS}
\multicite{SigmanDehaene2005PLoS}
\end{scriptsize}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Decision component}
	
\begin{figure}
	\includegraphics[height=0.50\textheight]{Pictures/PCMmodelCcomponent.pdf}
\end{figure} 	

\begin{block}{Random walk in the Central Component}
\begin{itemize}[<+->]
\item Deterministic $t_{0} = t_{p} + t_{m}$
\item Variable $RT = t_{0} + Random Walk$
\item Integration time $\tau = \frac{1}{\alpha}$
\end{itemize}
 \end{block}

\begin{scriptsize}
\multicite{Posner2005PLoS}
\multicite{SigmanDehaene2005PLoS}
\end{scriptsize}

\end{frame}
		
\begin{frame}
\frametitle{\insertsubsection: Relative integration time $\tau_{1} / \tau_{2}$}
	
\begin{figure}
	\includegraphics[height=0.45\textheight]{Pictures/PCMmodelCcomponent.pdf}
\end{figure} 	

\begin{block}{$\frac{\tau_{1}}{\tau_{2}}$ determined from RT \textbf{variances}}
\begin{itemize}[<+->]
\item  $Mean(RT) = t_{0} + \tau$
\item  $Var(RT) = \sigma^{2}\tau^{3}$
         \hspace{\stretch{1}}\small{(where $\sigma$ is a modeling parameter)}
\item $\frac{\tau_{1}}{\tau_{2}} = \sqrt[3]{\frac{S^2_{1}}{S^2_{2}}}$
          \hspace{\stretch{1}}\small{with ($S^2$ = variance)}
\end{itemize}
 \end{block}
 
\begin{scriptsize}
\multicite{Posner2005PLoS}
\multicite{SigmanDehaene2005PLoS}
\end{scriptsize}

\end{frame}

\subsection{Syllable and Lexical stress}
\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{Stressed syllables are special}
\begin{itemize}[<+->]
\item English and Dutch words tend to start with a stressed syllable 
\item The stressed syllable tends to be the most complex ($=$ informative)
\item Words can be identified fairly well with only the stressed 
      syllable and its position in the word
\item Eg, in CELEX, only 15 Dutch words with a total of 577 tokens have the
      structure "*-kOn-*-*", like 'verkondiging' (\textit{proclamation})
\item This is also true if only the broad phoneme classes are used 
\end{itemize}
\end{block}

\begin{scriptsize}
\multicite{CutlerCarter1987}
\multicite{Zue1985}
\end{scriptsize}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection}
\begin{tabular}{l | r r | l r r}
\hline
Stress & Words & Tokens & no \textipa{/x@ -/} & Words & Tokens \\
\hline
$+--$  & 143,031 & 2,326,745 & & \alert<3>{150,553} & \alert<3>{2,792,217} \\
$-+-$  & 60,970 & 2,385,914 & & \alert<3>{54,605} & \alert<3>{1,944,693} \\
$--+$  & 37,750 & 1,132,019 & & \alert<3>{36,593} & \alert<3>{1,107,768} \\
\hline
\end{tabular}
Dutch stress position in words of 3 and more syllables (CELEX)

\begin{block}{If some syllables contain more information, it would be advantageous to
              emphasize them}
\begin{itemize}[<+->]
\item \alert<1>{\textit<2->{Speed up recognition}} by starting words 
       with a \textit{distinctive} stressed syllable
\item Most English, and Dutch, words start with a stressed, or informative, syllable
\item Even stronger with common prefixes, eg, \textipa{/x@ -/}, removed
\end{itemize}
\end{block}

\begin{scriptsize}
\multicite{CutlerCarter1987}
\multicite{Cutler1997SpeechComm}
\multicite{Zue1985}
\end{scriptsize}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{What distinguishes lexical stress?
             (in English and Dutch)}
\begin{itemize}[<+->]
\item Less reduction (and coarticulation)
\item Longer phoneme durations
\item Louder
\item In short: \alert<4>{\textit<5->{Emphasis}}
\item \alert<5>{No} pitch marking
\end{itemize}
\end{block}

\begin{scriptsize}
\multicite{Sluijter1995}
\multicite{Sluijter1996}
\multicite{Sluijter1995PhD}
\multicite{Sluijter1997}
\end{scriptsize}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{Question: Do listeners actually use lexical stress for word recognition?}
\begin{itemize}[<+->]
\item Not all languages have lexical stress
\item Among others, \textit{English} and \textit{Dutch} have lexical stress
\item Experiment: Lexical competition increases RT
\item But \alert<4>{\textit<5->{only}} if stress pattern is correct
\item English listeners do \alert<5>{\textit<6->{not}} use lexical stress
\item In English, vowel quality (full vs schwa) replaces stress
\item Dutch listeners \alert<7>{\textit<8>{do}} use lexical stress
\end{itemize}
\end{block}

\begin{scriptsize}
\multicite{Cutler1997SpeechComm}
\end{scriptsize}

\end{frame}


\begin{frame}
\frametitle{\insertsubsection:\\ 
             Example: Word Spotting in Dutch}

\begin{center}
Press button when you hear a word\\
Present words and measure Reaction Times to spotting the embedded word, 
e.g., \textbf{zee} (\textit{sea}):
\end{center}
\begin{tabular}{l r l r}
\hline
	& RT & & RT \\
\hline
lu\alert{ZEE} & $<600ms$ & \alert{LU}zee & $<600ms$ \\
mu\alert{ZEE} & \underline{$>700ms$} & \alert{MU}zee & $\approx 650ms$ \\
\hline
\end{tabular} 

\begin{block}{Explanation:}
\textbf{muzee} starts an existing Dutch word, \textbf{museum},
if the stress is right. \textbf{luzee} does \textit{not}. 
Listeners can\textit{not} recognize a word while there are still alternatives.
\end{block}

\begin{scriptsize}\multicite{Cutler1997SpeechComm}\end{scriptsize}
\end{frame}

\subsection{Word segmentation}
\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{Word recognition works phoneme by phoneme}
\begin{itemize}[<+->]
\item Word boundary markers would prevent word-in-word ambiguity
\item The \alert<2>{\textit<3->{segmentation problem}}
\item Primary cue is rithm
\item Rithmic groups are isolated and used as ``first guess''
\item Three options (examples):
\item Mora timed: Japanese
\item Syllable timed: French
\item Stress timed: English, Dutch
\end{itemize}
\end{block}

\begin{scriptsize}\multicite{Cutler1997SpeechComm}\end{scriptsize}
\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{center}
\begin{figure}
	\alert<1>{\includegraphics[width=0.25\textwidth]{Pictures/Cutler97Fig1}}
	\alert<2>{\includegraphics[width=0.25\textwidth]{Pictures/Cutler97Fig2}}
	\alert<3>{\includegraphics[width=0.25\textwidth]{Pictures/Cutler97Fig3}}\\
\begin{footnotesize}
Fragment detection response times (msec) of \alert<1>{\textit<2->{English (left)}}, 
\alert<2>{\textit<3->{Japanese (center)}},
and \alert<3>{\textit<4->{French (right)}} listeners to CV and CVC targets in 
words with closed initial syllables. \multicite{Cutler1997SpeechComm}
\end{footnotesize}
\end{figure}
\end{center}

\begin{block}{Single rithmic units are identifed faster}
\begin{itemize}[<+->]
\item \alert<1>{\textit<2->{English}} (\textit{ba/bal} in \textit{balcony}):
          \alert<1>{\textit<2->{No}} effect (same stress)
\item \alert<2>{\textit<3->{Japanese}} (\textit{ta/tan} in \textit{tanshi}): 
          Faster for \alert<2>{\textit<3->{mora}}
\item \alert<3>{\textit<4->{French}} (\textit{ba/bal} in \textit{balcon}): 
          Faster for \alert<3>{\textit<4->{syllable}}
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Vowel harmony}
\begin{center}
\begin{figure}
	\alert<1>{\includegraphics[width=0.25\textwidth]{Pictures/Cutler97Fig17}}
	\alert<2>{\includegraphics[width=0.25\textwidth]{Pictures/Cutler97Fig18}}\\
\begin{footnotesize}
Word spotting response times (msec) of Finnish listeners
to Finnish words with preceding (left) or following (right) 
contexts containing harmonically mismatching versus matching vowels.
 \multicite{Cutler1997SpeechComm}
\end{footnotesize}
\end{figure}
\end{center}

\begin{block}{Finish vowel harmony: \textipa{/a\ o\ u/} and \textipa{/\ae\ \oe\ y/} are
             excluded from the same word}
A vowel harmony mismatch marks the start of a new word (left). 
The end, however, is ignored (right)
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{center}
\includegraphics[width=0.35\textwidth]{Pictures/VanSonVanSanten05Fig6a.png}
\includegraphics[width=0.35\textwidth]{Pictures/VanSonVanSanten05Fig6b.png}
\end{center}

\begin{block}{Marking word boundaries and stressed syllables
             \begin{scriptsize}US English\end{scriptsize}}
\begin{itemize}[<+->]
\item Emphasize initial and reduce final consonants
\item Reduce unstressed consonants
\item Both in duration and spectrally (CoG)
\item Coronals, \textipa{/s z t d n l/}, are special
\end{itemize}
\end{block}

\begin{scriptsize}\multicite{VanSonVanSanten2005}\end{scriptsize}
\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Prosodic boundaries}
\begin{center}
\begin{figure}
	\includegraphics[width=0.6\textwidth]{Pictures/FougeronKeating97Fig5}
\end{figure}
\end{center}

\begin{block}{Strengthening of prosodic boundaries}
\begin{itemize}[<+->]
\item More linguo-palatal contact after a boundary than before
\item In \textit{CV CV CV CV} utterances
\item Start is strong, end is weak
\end{itemize}
\end{block}

\begin{scriptsize}
\multicite{fougeron:3728}
\end{scriptsize}
\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{Other prosodic boundaries are also articulatory marked}
\begin{itemize}[<+->]
\item Levels: Word, intermediate phrase, intonational phrase, and utterance
\item Higher levels marked by pitch ``movements''
\item Domain initial phonemes are \alert<3>{\textit<4->{emphasized}}
\item Domain final phonemes are \alert<4>{\textit<5->{de}}-emphasized
\item Emphasis by \alert<5>{\textit<6->{less reduction}} etc.
\item De-emphasis by \alert<6>{\textit<7->{more reduction}} etc.
\item Also very efficient:
\item The first phoneme(s) of a domain will be \alert<8>{\textit<9->{informative}}
\item The last phoneme(s) of a domain are often \alert<9>{\textit<10->{redundant}}
\end{itemize}
\end{block}

\begin{scriptsize}
\multicite{AylettTurk2004}
\multicite{fougeron:3728}
\multicite{VanSonVanSanten2005}
\end{scriptsize}
\end{frame}

\subsection{Focus and prominence}
\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{In any utterance, some words convey new, 
        important information, and others give less or 
        repeat old information}
\begin{itemize}[<+->]
\item The focus refers to the new, unpredictable words
\item The words in focus are often marked in prosody and articulation
\item These words are made \alert<3>{\textit<4->{prominent}}
\item Prominence means \textit{longer}, \textit{louder}, \textit{less reduced}
\item Prominent words can also be accented ($F_0$ movements)
\item In general: Informative $\rightarrow$ Emphasized
\item Pitch accents mark prominent words
\item What is informative depends on the speakers and context
\end{itemize}
\end{block}

\begin{scriptsize}
\multicite{AylettTurk2004}
\end{scriptsize}
\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Foreground/Background}

\begin{block}{In discourse, utterances contain \textit{background} (known) 
              and \textit{foreground} (new) information}
\begin{itemize}[<+->]
\item The \textit{background} sets the stage
\item The \textit{foreground} puts the players on the stage
\item \textit{Foreground} is goal directed
\item ``\textit{Walking in the park, I saw a dog}''
\item Background: \textit{I was walking in the park and saw something}
\item Foreground: \textit{a dog}
\item \textit{Foreground} more specific than \textit{background}
\item Default: \textit{Background} first, \textit{foreground} last (English)
\item \textit{Foreground} will be in focus
\end{itemize}
\end{block}

\begin{scriptsize}
\multicite{AylettTurk2004}
\multicite{Streefkerketal2001}
\multicite{Streefkerk2002PhD}
\end{scriptsize}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Given and New information}

\begin{block}{Given information is redundant, new information important}
\begin{itemize}[<+->]
\item The prominence of a word decreases with it's ``givenness''
\item New information is almost always prominently put in focus
\item There is a lexical dimension, ie, repeated words
\item There is also a semantic dimension, ie, repeated concepts
\item The latter involves priming, eg, names and roles
\item That is, refering to a \textit{president} primes 
       her/his \textit{name} as given
\item Prosody marks new information and de-emphasizes given information
\end{itemize}
\end{block}

\begin{scriptsize}
\multicite{AylettTurk2004}
\end{scriptsize}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: \textit{information structure}}

\begin{center}
\begin{figure}
	\includegraphics[width=0.35\textwidth]{Pictures/AylettTurk2004Fig2}
\end{figure}
\end{center}

\begin{block}{The smooth signal redundancy hypothesis 
              \begin{scriptsize}\multicite{AylettTurk2004}\end{scriptsize}}
\begin{itemize}[<+->]
\item Redundancy in the language affects prosody
\item Prosody \textit{reduces} redundant articulation
\item Acoustic reduction $\rightarrow$ Robust and efficient communication 
\item But: Is prosody the \alert<4>{\textit<5>{only}} \textit{compressor}?
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: \textit{information structure}}

\begin{center}
\begin{figure}
	\includegraphics[width=0.5\textwidth]{Pictures/AylettTurk2004Fig3}
\end{figure}
\end{center}

\begin{block}{The smooth signal redundancy hypothesis combined with traditional 
              phonology }
\begin{itemize}[<+->]
\item Language and lexicon evolution favor efficient structures
\item Speaker might evaluate intermediate structures for comprehension?
\item Still: Is prosody the \alert<3>{\textit<4>{only}} \textit{compressor}?
\end{itemize}
\end{block}

\begin{scriptsize}\multicite{AylettTurk2004}\end{scriptsize}
\end{frame}

\subsection{Bibliography}
\begin{frame}[allowframebreaks]
\frametitle{\insertsubsection}

\nocitethursday{CutlerCarter1987}
\nocite{CutlerCarter1987}
\nocitethursday{Cutleretal1997}
\nocite{Cutleretal1997}
\nocitethursday{Sluijter1996}
\nocite{Sluijter1996}
\nocitethursday{Sluijter1997}
\nocite{Sluijter1997}
\nocitethursday{Streefkerketal2001}
\nocite{Streefkerketal2001}
\nocitethursday{Sluijter1995PhD}
\nocite{Sluijter1995PhD}
\nocitethursday{Streefkerk2002PhD}
\nocite{Streefkerk2002PhD}

\bibliographystylethursday{plainnat}
\tiny{
\bibliographythursday{LOTwinterschool2006}
}

\end{frame}


% Insert License text
\insertgnugpl

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\lecture{Efficiency in speech}{friday}
\renewcommand{\multicite}[1]{\nocite{#1}\citefriday{#1}}
\renewcommand{\licensenumber}[0]{License5}

\section{Efficiency in speech}

\subsection{Introduction}
\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{Speech is a communication channel}
\begin{itemize}[<+->]
\item Communication channels have a bandwidth
\item A communication channel should be efficient
\item Efficiency can be achieved on several levels:
\item Encoding
\item Compression (acoustic, phoneme, word)
\item Fault tolerance
\item In short: Spend effort and bandwidth according to communicative importance
\end{itemize}
\end{block}

\end{frame}

\subsection{Efficiency}
\begin{frame}
\frametitle{\insertsubsection: Efficiency}

\begin{block}{Effort vs Bits}
\begin{itemize}[<+->]
\item For recognition, words and phonemes should be as distinct as possible
\item Distinction is determined by:
\item Duration: longer means more time to recognize
\item Loudness: Dynamic range should be large
\item Spectal width: Use whole available spectrum
\item Articulation: Precise, consistent articulation
\item All of these cost more effort
\item For speaking, making distinctions is difficult
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{Measure distinction and effort according to communicative importance}
\begin{itemize}[<+->]
\item Limited to lexical word recognition
\item Assume importance are just lexical bits
\item Unimportant parts (low bits) should be:
\item Shorter
\item Less loud
\item Less spectral width
\item Lax articulation (more coarticulation)
\item That is: \alert<8>{Unimportant $\rightarrow$ reduced}
\item Where \textit{Reduction $==$ Compression}
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{Efficient speech is a testable hypothesis }
\begin{itemize}[<+->]
\item A correlation between Reduction (compression) and Bit-load
\item Two hypotheses:
\item \textit{Smooth signal redundancy hypothesis} 
      \begin{scriptsize}\multicite{AylettTurk2004}\end{scriptsize}
\item $\rightarrow$ Prosody structure implements compression
\item \textit{Phoneme level reduction}
\item $\rightarrow$ Compression at the lowest levels, down to the phoneme
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Recapitulation \textit{information structure}}

\begin{center}
\begin{figure}
	\includegraphics[width=0.35\textwidth]{Pictures/AylettTurk2004Fig2}
\end{figure}
\end{center}

\begin{block}{The smooth signal redundancy hypothesis 
              \begin{scriptsize}\multicite{AylettTurk2004}\end{scriptsize}}
\begin{itemize}[<+->]
\item Redundancy in the language affects prosody
\item Prosody \textit{reduces} redundant articulation
\item Determine the distribution of lexical information over prosodic and
      non-prosodic features
\end{itemize}
\end{block}

\end{frame}

\subsection{Vowel inventories}
\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{At the lowest level, speech is encoded in phonemic segments.
              Phonemes are the first candidate for compression}
\begin{itemize}[<+->]
\item Efficient use of available perceptual/articulatory
      space
\item In vowel formant ($F_1/F_2$) space:
\item Distribute vowels to be far apart
\item Spread them evenly
\item Use complex articulations only if space is full
\item Start with corner vowels \textipa{/i a u/}
\item Construct symmetric systems
\item If full, start to use \textit{quantity} (duration), 
      \textit{diphthongs}, \textit{nasalization}
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\parbox{\textwidth}{
\parbox{0.54\textwidth}{
\includegraphics[width=0.5\textwidth]{Pictures/SchwartzEtAl97Fig2}
}
\parbox{0.44\textwidth}{
Acoustic vowel space
\begin{itemize}[<+->]
\item Acoustic properties of all monotphthongs
\item Probably no language has them all
\item Distinguishing them all is ``difficult''
\item Both in \textit{speaking} and \textit{hearing}
\end{itemize}
}
}
\begin{scriptsize}
\multicite{dB00}
\multicite{SBV+97}
\multicite{TenBosch1995}
\end{scriptsize}
\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\parbox{\textwidth}{
\parbox{0.54\textwidth}{
\includegraphics[width=0.5\textwidth]{Pictures/SchwartzEtAl97Fig9}
}
\parbox{0.44\textwidth}{
Distribution of vowels versus inventory size over languages
\begin{itemize}[<+->]
\item 5 vowel inventory is most popular (100 languages)
\item For every size, the most even distribution is overwhelmingly chosen
\end{itemize}
}
}
\begin{scriptsize}
\multicite{dB00}
\multicite{SBV+97}
\multicite{TenBosch1995}
\end{scriptsize}
\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{Languages try to optimize vowel spaces}
\begin{itemize}[<+->]
\item Comparable effects seem to be working in consonants
\item Eg: \textipa{/x s f/} are are distributed over spectrum and loudness
\item However, consonants are more difficult to research
\end{itemize}
\end{block}

\end{frame}

\subsection{Word length}
\begin{frame}
\frametitle{\insertsubsection: Number of phonemes}

\begin{footnotesize}
\begin{tabular}{l | r r l}
$I = -log_2(frequency)$         & $R$ &  $N$ & Token Count\\
\hline
\alert<1>{$-log_2(TF)$} & \alert<1>{0.300} &  69,501 & $>10$ (CELEX)\\
\alert<2>{$-log_2(TF)$}  & \alert<2>{0.186} &  259,984 & $>10$ (TNC) \\
\alert<2>{$-log_2(TF\cdot IDF)$} & \alert<2>{0.319} & 259,984 & $>10$ (TNC) \\
\alert<2>{$-log_2(TF)-CD$} & \alert<2>{0.379} & 259,984 & $>10$ (TNC) \\
\alert<3>{$-log_2(TF)$} & \alert<3>{0.686} & 11,337 & all (IFA, usage) \\
\hline
\end{tabular} \\
\end{footnotesize}

\begin{footnotesize}
$TF$: Term Frequency, $TF\cdot IDF$: $TF$ time Inverse Document Frequency,
$CD$: Context Distinctiveness
\begin{scriptsize}\multicite{IFA2003}\multicite{ordelman2002}\end{scriptsize}
\end{footnotesize}


\begin{block}{An efficient coding of words would make high-frequency words 
             short and low frequency words long}
\begin{itemize}[<+->]
\item The correlation is rather high in the small CELEX wordlist
\item The TNC (350 Mword) shows the correlation increases with context
\item On a token basis, ie word occurences, 
      almost half the variance in length is explained
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Duration}

$R$ ($N$) for Read and Spontaneous speech
\begin{footnotesize}
\begin{tabular}{l | l l l}
Style        & all & $I \leq 10$ & $I>10$\\
\hline
Read         & 0.720 (\textit{29,269}) & 0.428 (\textit{14,739}) & 0.489 (\textit{14,530})\\
Spontaneous  & 0.609 (\textit{11,348}) & 0.180 (\textit{5,832}) & 0.484 (\textit{5,516})\\
\hline
\end{tabular} \\
\end{footnotesize}
\begin{footnotesize}Correlation between $-\log_2()$ CELEX word frequencies and word duration\end{footnotesize}

\begin{block}{An efficient coding of words would make high-frequency words 
             short and low frequency words long}
\begin{itemize}[<+->]
\item Duration is distributed efficiently
\item Rare words tend to be longer
\item The correlation is marked weaker if high/low frequency words are distinguished
\item To a large part, this can be obtained by a function/content 
      word encoding (not shown)
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Conclusion}

\begin{block}{Word length is correlated to word frequency}
\begin{itemize}[<+->]
\item High frequency words tend to be shorter, both in phonemes and duration
\item There is a large distinction between \textit{Function} and \textit{Content} words
\item The correlation is weaker within these clases
\item This tendency increases if context is taken into account
\item Word length is to some extend efficient
\end{itemize}
\end{block}

\end{frame}

\subsection{Phoneme reduction}

\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{Efficient speech would also mean that redundant phonemes are reduced}
\begin{itemize}[<+->]
\item Less informative phonemes should be:
\item Shorter
\item Weaker (= less loud)
\item Spectrally reduced
\item Leads to \alert<5>{\textit<6->{less distinctive}} sound segments
\item Vowels: Less contrast in $F_1/F_2$ formants
\item Consonants: Weakening of higher frequencies (CoG)
\item When testing, account for speaker and phoneme identity,
      prominence, lexical stress, and position in the syllable
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Vowels}

\begin{center}
\includegraphics[width=0.4\textwidth]{Pictures/VowelReduction}
\end{center}

\begin{block}{Articulation takes effort and humans are ``lazy'' (efficient)} 
\begin{itemize}[<+->]
\item Unimportant, unstressed, items are articulated ``less well''
\item Visible as a smaller vowel triangle (see above)
\item Less \textit{contrast} between vowels: $F_1/F_2$ distances
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Segmental information (recapitulation)}

\begin{equation}
I^{\prime\prime}(phon|onset, w) = -\log_2 \dfrac{TokenCount(onset+phon)+ D(w)}
                                        {TokenCount(onset+\star)+D(w)}\notag
\end{equation}
\begin{footnotesize}
$D(w) = (TF(w)\cdot2^{CD(w)} - TF(w))\cdot TotalCount$\\
$CD(w) = KL(LocalDistr(w),GlobalDistr)$\\
$TF(w)$: Term frequency of $w$
\end{footnotesize}

\begin{block}{Focus on direct context of $w$: $LocalDistr(w)$}
\begin{itemize}[<+->]
\item $CD(w)$: Kullback-Leibler distance between local and global distribution
\item Perceived frequency is $TF(w)\cdot 2^{CD(w)}$
\item Data \textit{are} available
\end{itemize}
\end{block}

\begin{scriptsize}\multicite{VSP03}\end{scriptsize}
\end{frame}

\subsection{Information Structure}
\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{Prosodic and structural factors determine reduction}
\begin{itemize}[<+->]
\item See whether the distribution of phonemic information over 
      prosodic factors explains (correlates with) reduction
\item Reduction: Duration and Mean Frequency, ie, Spectral Center of Gravity
\item Does reduction really mirror the distribution of information over phonemes?
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Distribution of information}

\parbox{\textwidth}{
\parbox{0.54\textwidth}{
	\includegraphics[width=0.5\textwidth]{Pictures/VanSonPols03Fig4}
}
\parbox{0.44\textwidth}{
Explained variance of $I^{\prime\prime}$
\begin{itemize}[<+->]
\item Phoneme position and identity: 81\% (discounted below)
\item Number of syllables, prominence and lexical stress: 34\% (of what is left)
\item Cluster length and syllable part (onset/coda): 8.7\% ``
\item Word and syllable position: 4.3\% ``
\end{itemize}
}}

\begin{scriptsize}\multicite{VSP03}\end{scriptsize}
\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Reduction}
\begin{center}
\begin{figure}
	\includegraphics[width=0.4\textwidth]{Pictures/VanSonPols03Fig5}
	\includegraphics[width=0.4\textwidth]{Pictures/VanSonPols03Fig6}
\end{figure}
\end{center}

\begin{block}{Explained variance of duration and ``mean frequency''}
\begin{itemize}[<+->]
\item Order of factors the same as for information
\item In general, the distribution of segmental information is reflected in the reduction 
\end{itemize}
\end{block}

\begin{scriptsize}\multicite{VSP03}\end{scriptsize}

\end{frame}

\subsection{Beyond information structure}

\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{Information structure, ie, prosody, can account for 
             a lot of the efficiency}
\begin{itemize}[<+->]
\item But, does there remain room for reduction after accounting for all prosodic factors?
\item Correlated \textit{information content} with \textit{reduction} after 
      accounting for all the above factors
\end{itemize}
\end{block}

\end{frame}


\begin{frame}
\frametitle{\insertsubsection: Vowels}

\begin{center}
\includegraphics[width=0.7\textwidth]{Pictures/VanSonPols03Fig2.png}
\end{center}
\begin{block}{Correlation between $I^{\prime\prime}_{CD}$ vs. duration and $F_1/F_2$ contrast}
\begin{itemize}[<+->]
\item There is a correlation for all speaking conditions and most prominence levels
\item Correlations are small but consistent
\end{itemize}
\end{block}

\begin{scriptsize}
\multicite{VSP03}
\end{scriptsize}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Consonants}

\begin{center}
\includegraphics[width=0.7\textwidth]{Pictures/VanSonPols03Fig3.png}
\end{center}

\begin{block}{Correlation between $I^{\prime\prime}_{CD}$ and duration/Center of Gravity}
\begin{itemize}[<+->]
\item Vowels are best distinguished
\item Correlations are small but consistent
\end{itemize}
\end{block}

\begin{scriptsize}
\multicite{VSP03}
\end{scriptsize}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Conclusion}

\begin{block}{Phoneme reduction seems to be correlated (somewhat) to
             the information content}
\begin{itemize}[<+->]
\item The correlations are small, after accounting for prosody
\item There is efficiency obtained beyond the information structure
\item Is there a more definite test?
\item Look at specific effects of phonemic predictability
\end{itemize}
\end{block}

\end{frame}

\subsection{Stress and Position}
\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{Model duration and spectral reduction of intervocalic consonants in US English}
\begin{itemize}[<+->]
\item Remove all prosodic variation and focus on three factors:
\item Position in the word
\item Syllable stress
\item Primary articulator ($\approx$ place of articulation)
\item $\rightarrow$ Labials \textipa{/f v p b m w/}
\item $\rightarrow$ Coronals \textipa{/s z t d n l/}
\item $\rightarrow$ Post-Coronals \textipa{/k g r/} (ie, the rest)
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{center}
\includegraphics[width=0.35\textwidth]{Pictures/VanSonVanSanten05Fig6a.png}
\includegraphics[width=0.35\textwidth]{Pictures/VanSonVanSanten05Fig6b.png}
\end{center}

\begin{block}{A model to predict duration and CoG from the position in 
              the word and lexical stress
              \begin{scriptsize}\multicite{VanSonVanSanten2005}\end{scriptsize}}
\begin{itemize}[<+->]
\item In general, \textit{stressed} or \textit{initial} is less reduced
\item Articulator can be factored out (modelled)
\item There are the odd \alert<3>{\textit<4>{de-emphasized}} coronals: Flaps and taps
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{center}
\includegraphics[width=0.9\textwidth]{Pictures/VanSonVanSanten05Tab2.png}
\end{center}

\begin{block}{Coronal consonants are very frequent in unstressed/word-final position, $C_1$: intervocalic, $C_N$: clusters (based on US English CELEX word list)}
\begin{itemize}[<+->]
\item Strong: All word initial and stressed word medial
\item Weak: Unstressed word medial and all word final
\item Weak labials or Post-coronals contribute $\geq 2$ bits 
\item Weak coronals contribute $< 1$ bit
\end{itemize}
\end{block}
\begin{scriptsize}\multicite{VanSonVanSanten2005}\end{scriptsize}
\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{center}
\includegraphics[width=0.35\textwidth]{Pictures/VanSonVanSanten05Fig8a.png}
\includegraphics[width=0.35\textwidth]{Pictures/VanSonVanSanten05Fig8b.png}
\end{center}

\begin{block}{Correlate variation (reduction) versus information content}
\begin{itemize}[<+->]
\item Most dots lie on a straight line
\item Special behavior of weak coronals \textipa{/sztdnl/} understandable
\item Consonant reduction explainable from information content
\end{itemize}
\end{block}
\begin{scriptsize}\multicite{VanSonVanSanten2005}\end{scriptsize}
\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Conclusion}

\begin{block}{Again, phoneme reduction seems to be correlated to
             the information content}
\begin{itemize}[<+->]
\item This time, information structure cannot explain this
\item There is efficiency beyond the information structure
\item On the whole, speech seems to be efficiently organized
\item Do speakers actually attend to efficiency?
\item Unlikely, efficiency might result from speakers being listeners too
\item If things are obvious for the speaker, they tend to be obvious for the listener
\item Stress things you find non-obvious, and you will be efficient?
\end{itemize}
\end{block}

\end{frame}

\subsection{Bibliography}
\begin{frame}[allowframebreaks]
\frametitle{\insertsubsection}

\nocitefriday{AylettTurk2004}
\nocite{AylettTurk2004}
\nocitefriday{Lieberman1963}
\nocite{Lieberman1963} 
\nocitefriday{VanSonVanSanten2005}
\nocite{VanSonVanSanten2005} 
\nocitefriday{AylettPhD}
\nocite{AylettPhD}
\nocitefriday{VanSonVanSanten2005}
\nocite{VanSonVanSanten2005} 


\bibliographystylefriday{plainnat}
\tiny{
\bibliographyfriday{LOTwinterschool2006}
}

\end{frame}

\lecture{Appendix}{Bibliography}
\subsection{Bibliography}
\renewcommand{\refname}{Bibliography}
\begin{frame}[allowframebreaks]
\frametitle{\insertsubsection}
\tiny{
\bibliographystyle{plainnat}
\bibliography{LOTwinterschool2006}
}
\end{frame}	

\begin{frame}
\frametitle{\insertsubsection}
\end{frame}

\begin{frame}
\frametitle{\insertsubsection}
\end{frame}


% Insert License text
\insertgnugpl

\end{document}
