\documentclass{beamer}
%\documentclass[handout]{beamer}
%\documentclass[trans]{beamer}
%\documentclass[article]{beamer}
%\usepackage{natbib}
\usepackage{comment}
\usepackage{relsize}
\usepackage{pifont}
\usepackage{tipa}
%\usepackage{relsize}
\usepackage[english]{babel}
%\usepackage[latin1]{inputenc}
\usepackage{amsmath}
\usepackage{amsfonts}
\usepackage{amssymb}
\usepackage{fancyhdr}
%\usepackage[T1]{fontenc}
%\usepackage[pdftex]{graphicx}
\usepackage{multimedia}
\usepackage{hyperref}
\usepackage{multibib}

% To print only specific lectures, look for
% \includeonlylecture below

% Extra definities van David Weenink
\newcommand{\timit}{\textsc{timit}}
\newcommand{\Timit}{\textsc{Timit}}
\newcommand{\praat}{\textsc{p\textsmaller{raat}}}
\newcommand{\Praat}{\textlarger{P}raat}
\newcommand{\pc}[1]{\texttt{\textsmaller{#1}}}
\newcommand{\hc}[1]{\cite[#1]{harrington-etal:99}}
\newcommand{\assignment}[1]{Assignment #1: }
\newcommand{\assignmentdiscussion}[1]{Discussion of assignment #1.}
\newcommand{\preparation}{Preparation: }

\newcites{Sthree,Sfour,Sfive,Ssix,Sseven,Seight}
{Further Reading,Further Reading,Further Reading,Further Reading,Further Reading, Further Reading}

%http://www.csd.uwo.ca/~jgorski/personal/beamerthemes/
%\usetheme[]{Warsaw}
%\usetheme{Berkeley}
%\usetheme{Berlin}
%\usetheme{Bergen}
%\usetheme{Berlin}Antibes,Bergen,Boadilla,Copenhagen,Darmstadt
%\usetheme{Boadilla}
%\usetheme{Antibes} %heel kaal
%\usetheme{Bergen} links rechts
%\usetheme{Dresden} %nice
%\usetheme{Frankfurt} %ok
\usetheme[right, hideothersubsections]{Goettingen} % overzicht rechts
%\usetheme[right, hideothersubsections]{Hannover} % als Goetngen met overzicht links
%\usetheme{Ilmenau}
%\usetheme{JuanLesPins}
%\usetheme{Luebeck} % overzicht boven, auteur beneden
%\usetheme{Madrid} % alleen navigatie onder
%\usetheme{Malmoe} % nav boven
%\usetheme{Marburg} % donkere navigatiebalk rechts
%\usetheme{Montpellier} % matig
%\usetheme[right]{PaloAlto}% nav links
%\usetheme{Pittsburgh}% alleen standaard nav
%\usetheme{Rochester}% helder, alleen standaard nav
%\usetheme{Singapore} %helder ok nav boven+#sheets indicatie
%\usetheme{Szeged} %helder ok nav boven+#sheets indicatie

%%%%%innerthemes: title and part pages, itemize, enumerate, desciption, block, theorem/proof, figures, tables, footnotes and bibl. entries 
%\useinnertheme{default} %
%\useinnertheme{circles} % 
%\useinnertheme{rectangles} %
\useinnertheme{rounded} %
%\useinnertheme{inmargin} %structure on left (like blocktitles) and normal info like block bodies on right

%%%%%outerthemes: head and footline, sidebars, logo, frame title
%\useoutertheme{infolines}%
%\useoutertheme{miniframes}%
%\useoutertheme{smoothbars}%
%\useoutertheme[right,height=0pt,hideothersubsections]{sidebar}%options:height=dimension, hideothersubsections, hideallsubsections, left, right, width=dimension
%\useoutertheme{split}%
%\useoutertheme{shadow}%
%\useoutertheme{tree}%option:[hooks]
%\useoutertheme{smoothtree}%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%Color Themes:
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\usecolortheme{sidebartab} %highlights current entry in table of contents
\usecolortheme[RGB={25,25,112}]{structure} %72,61,139

\setbeamercovered{transparent}
%\usecolortheme{seahorse}
\usecolortheme{rose}
\usecolortheme{dolphin}
\setbeamercolor{frametitle}{bg=structure.fg!20!white}

% Complete color themes:  albatross, beetle, crane, dove, fly, seagull
% Inner color themes: lily, orchid, rose
% Outer color themes:  whale, seahorse, dolphin(?)


%\usefonttheme[onlylarge]{structuresmallcapsserif}
%\usefonttheme[onlysmall]{structurebold}
%\setbeamerfont{title}{shape=\itshape,family=\rmfamily}
%\setbeamercolor{title}{fg=red!80!black, bg=red!20!white}

%\usepackage{natbib} % Doesn't work in beamer, such a shame

%\newcommand{\timit}{\textsc{timit}}
%\newcommand{\Timit}{\textsc{Timit}}
%opening
\title[Speech recognition and synthesis]{Speech recognition and synthesis}

\author[]{David Weenink and R.J.J.H. van Son}
\date{First semester 2005}
\institute{Institute of Phonetic Sciences\\ACLC\\
University of Amsterdam}
\date[Short Occasion] % (optional)
{ASR \& TTS
\begin{figure}[b]
\includegraphics[width=0.5\textwidth]{Pictures/aclclogo.pdf}
\end{figure}
}

\logo{\includegraphics[height=1cm]{Pictures/IFAlogo.pdf}}

%\setbeamercolor{frametitle}{fg=red}
% \setbeamertemplate{frametitle}
% {
% \begin{centering}
% \textbf{\insertframetitle}
% \par
% \end{centering}
% }

% Delete this, if you do not want the table of contents to pop up at
% the beginning of each subsection:

%begin{comment}
% \AtBeginSection[] %or \AtBeginSubsecton{} 
% {
% \begin{frame}<beamer>
% \frametitle{Outline}
% \tableofcontents[hideothersubsections]
% \end{frame}
% }
%end{comment}
%
% The obligatyright notices and license texts
% Implemented as Latex macros
% Add copyright notice and GPL
\newcommand{\multicite}[1]{\cite{#1}}
\newcommand{\copyrightauthor}[0]{David Weenink and R.J.J.H. van Son}
\newcommand{\GNUcite}[0]{\multicite{GPL}}
\newcommand{\licensenumber}[0]{License}
% Add  week number
\newcommand{\copyrightnotice}[0]
{Copyright \copyright 2005 \copyrightauthor, \hyperlink{\licensenumber}{GNU General Public License} \GNUcite}

% Insert complete GNU GPL license frames. Add weeknumber
\newcommand{\insertgnugpl}[0]
{
\begin{frame}[allowframebreaks,allowdisplaybreaks,label=\licensenumber]
\frametitle{Copyright License}
\copyrightnotice \\
\vskip20pt
\begin{scriptsize}
\begin{quote}
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
\end{quote}
\end{scriptsize}

\end{frame}

\begin{frame}[allowframebreaks,allowdisplaybreaks]
\frametitle{The GNU General Public License}

\begin{tiny}

\begin{center}
{\parindent 0in

Version 2, June 1991

Copyright \copyright\ 1989, 1991 Free Software Foundation, Inc.

\bigskip

51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA

\bigskip

Everyone is permitted to copy and distribute verbatim copies
of this license document, but changing it is not allowed.
}
\end{center}

\begin{center}
{\bf\large Preamble}
\end{center}


The licenses for most software are designed to take away your freedom to
share and change it.  By contrast, the GNU General Public License is
intended to guarantee your freedom to share and change free software---to
make sure the software is free for all its users.  This General Public
License applies to most of the Free Software Foundation's software and to
any other program whose authors commit to using it.  (Some other Free
Software Foundation software is covered by the GNU Library General Public
License instead.)  You can apply it to your programs, too.

When we speak of free software, we are referring to freedom, not price.
Our General Public Licenses are designed to make sure that you have the
freedom to distribute copies of free software (and charge for this service
if you wish), that you receive source code or can get it if you want it,
that you can change the software or use pieces of it in new free programs;
and that you know you can do these things.

To protect your rights, we need to make restrictions that forbid anyone to
deny you these rights or to ask you to surrender the rights.  These
restrictions translate to certain responsibilities for you if you
distribute copies of the software, or if you modify it.

For example, if you distribute copies of such a program, whether gratis or
for a fee, you must give the recipients all the rights that you have.  You
must make sure that they, too, receive or can get the source code.  And
you must show them these terms so they know their rights.

We protect your rights with two steps: (1) copyright the software, and (2)
offer you this license which gives you legal permission to copy,
distribute and/or modify the software.

Also, for each author's protection and ours, we want to make certain that
everyone understands that there is no warranty for this free software.  If
the software is modified by someone else and passed on, we want its
recipients to know that what they have is not the original, so that any
problems introduced by others will not reflect on the original authors'
reputations.

Finally, any free program is threatened constantly by software patents.
We wish to avoid the danger that redistributors of a free program will
individually obtain patent licenses, in effect making the program
proprietary.  To prevent this, we have made it clear that any patent must
be licensed for everyone's free use or not licensed at all.

The precise terms and conditions for copying, distribution and
modification follow.

\begin{center}
{\small \sc Terms and Conditions For Copying, Distribution and
  Modification}
\end{center}


%\renewcommand{\theenumi}{\alpha{enumi}}
\begin{enumerate}

\addtocounter{enumi}{-1}

\item 

This License applies to any program or other work which contains a notice
placed by the copyright holder saying it may be distributed under the
terms of this General Public License.  The ``Program'', below, refers to
any such program or work, and a ``work based on the Program'' means either
the Program or any derivative work under copyright law: that is to say, a
work containing the Program or a portion of it, either verbatim or with
modifications and/or translated into another language.  (Hereinafter,
translation is included without limitation in the term ``modification''.)
Each licensee is addressed as ``you''.

Activities other than copying, distribution and modification are not
covered by this License; they are outside its scope.  The act of
running the Program is not restricted, and the output from the Program
is covered only if its contents constitute a work based on the
Program (independent of having been made by running the Program).
Whether that is true depends on what the Program does.

\item You may copy and distribute verbatim copies of the Program's source
  code as you receive it, in any medium, provided that you conspicuously
  and appropriately publish on each copy an appropriate copyright notice
  and disclaimer of warranty; keep intact all the notices that refer to
  this License and to the absence of any warranty; and give any other
  recipients of the Program a copy of this License along with the Program.

You may charge a fee for the physical act of transferring a copy, and you
may at your option offer warranty protection in exchange for a fee.

\item

You may modify your copy or copies of the Program or any portion
of it, thus forming a work based on the Program, and copy and
distribute such modifications or work under the terms of Section 1
above, provided that you also meet all of these conditions:

\begin{enumerate}
\begin{tiny}

\item 

You must cause the modified files to carry prominent notices stating that
you changed the files and the date of any change.

\item

You must cause any work that you distribute or publish, that in
whole or in part contains or is derived from the Program or any
part thereof, to be licensed as a whole at no charge to all third
parties under the terms of this License.

\item
If the modified program normally reads commands interactively
when run, you must cause it, when started running for such
interactive use in the most ordinary way, to print or display an
announcement including an appropriate copyright notice and a
notice that there is no warranty (or else, saying that you provide
a warranty) and that users may redistribute the program under
these conditions, and telling the user how to view a copy of this
License.  (Exception: if the Program itself is interactive but
does not normally print such an announcement, your work based on
the Program is not required to print an announcement.)

\end{tiny}
\end{enumerate}

These requirements apply to the modified work as a whole.  If
identifiable sections of that work are not derived from the Program,
and can be reasonably considered independent and separate works in
themselves, then this License, and its terms, do not apply to those
sections when you distribute them as separate works.  But when you
distribute the same sections as part of a whole which is a work based
on the Program, the distribution of the whole must be on the terms of
this License, whose permissions for other licensees extend to the
entire whole, and thus to each and every part regardless of who wrote it.

Thus, it is not the intent of this section to claim rights or contest
your rights to work written entirely by you; rather, the intent is to
exercise the right to control the distribution of derivative or
collective works based on the Program.

In addition, mere aggregation of another work not based on the Program
with the Program (or with a work based on the Program) on a volume of
a storage or distribution medium does not bring the other work under
the scope of this License.

\item
You may copy and distribute the Program (or a work based on it,
under Section 2) in object code or executable form under the terms of
Sections 1 and 2 above provided that you also do one of the following:

\begin{enumerate}
\begin{tiny}

\item

Accompany it with the complete corresponding machine-readable
source code, which must be distributed under the terms of Sections
1 and 2 above on a medium customarily used for software interchange; or,

\item

Accompany it with a written offer, valid for at least three
years, to give any third party, for a charge no more than your
cost of physically performing source distribution, a complete
machine-readable copy of the corresponding source code, to be
distributed under the terms of Sections 1 and 2 above on a medium
customarily used for software interchange; or,

\item

Accompany it with the information you received as to the offer
to distribute corresponding source code.  (This alternative is
allowed only for noncommercial distribution and only if you
received the program in object code or executable form with such
an offer, in accord with Subsection b above.)

\end{tiny}
\end{enumerate}

The source code for a work means the preferred form of the work for
making modifications to it.  For an executable work, complete source
code means all the source code for all modules it contains, plus any
associated interface definition files, plus the scripts used to
control compilation and installation of the executable.  However, as a
special exception, the source code distributed need not include
anything that is normally distributed (in either source or binary
form) with the major components (compiler, kernel, and so on) of the
operating system on which the executable runs, unless that component
itself accompanies the executable.

If distribution of executable or object code is made by offering
access to copy from a designated place, then offering equivalent
access to copy the source code from the same place counts as
distribution of the source code, even though third parties are not
compelled to copy the source along with the object code.

\item
You may not copy, modify, sublicense, or distribute the Program
except as expressly provided under this License.  Any attempt
otherwise to copy, modify, sublicense or distribute the Program is
void, and will automatically terminate your rights under this License.
However, parties who have received copies, or rights, from you under
this License will not have their licenses terminated so long as such
parties remain in full compliance.

\item
You are not required to accept this License, since you have not
signed it.  However, nothing else grants you permission to modify or
distribute the Program or its derivative works.  These actions are
prohibited by law if you do not accept this License.  Therefore, by
modifying or distributing the Program (or any work based on the
Program), you indicate your acceptance of this License to do so, and
all its terms and conditions for copying, distributing or modifying
the Program or works based on it.

\item
Each time you redistribute the Program (or any work based on the
Program), the recipient automatically receives a license from the
original licensor to copy, distribute or modify the Program subject to
these terms and conditions.  You may not impose any further
restrictions on the recipients' exercise of the rights granted herein.
You are not responsible for enforcing compliance by third parties to
this License.

\item
If, as a consequence of a court judgment or allegation of patent
infringement or for any other reason (not limited to patent issues),
conditions are imposed on you (whether by court order, agreement or
otherwise) that contradict the conditions of this License, they do not
excuse you from the conditions of this License.  If you cannot
distribute so as to satisfy simultaneously your obligations under this
License and any other pertinent obligations, then as a consequence you
may not distribute the Program at all.  For example, if a patent
license would not permit royalty-free redistribution of the Program by
all those who receive copies directly or indirectly through you, then
the only way you could satisfy both it and this License would be to
refrain entirely from distribution of the Program.

If any portion of this section is held invalid or unenforceable under
any particular circumstance, the balance of the section is intended to
apply and the section as a whole is intended to apply in other
circumstances.

It is not the purpose of this section to induce you to infringe any
patents or other property right claims or to contest validity of any
such claims; this section has the sole purpose of protecting the
integrity of the free software distribution system, which is
implemented by public license practices.  Many people have made
generous contributions to the wide range of software distributed
through that system in reliance on consistent application of that
system; it is up to the author/donor to decide if he or she is willing
to distribute software through any other system and a licensee cannot
impose that choice.

This section is intended to make thoroughly clear what is believed to
be a consequence of the rest of this License.

\item
If the distribution and/or use of the Program is restricted in
certain countries either by patents or by copyrighted interfaces, the
original copyright holder who places the Program under this License
may add an explicit geographical distribution limitation excluding
those countries, so that distribution is permitted only in or among
countries not thus excluded.  In such case, this License incorporates
the limitation as if written in the body of this License.

\item
The Free Software Foundation may publish revised and/or new versions
of the General Public License from time to time.  Such new versions will
be similar in spirit to the present version, but may differ in detail to
address new problems or concerns.

Each version is given a distinguishing version number.  If the Program
specifies a version number of this License which applies to it and ``any
later version'', you have the option of following the terms and conditions
either of that version or of any later version published by the Free
Software Foundation.  If the Program does not specify a version number of
this License, you may choose any version ever published by the Free Software
Foundation.

\item
If you wish to incorporate parts of the Program into other free
programs whose distribution conditions are different, write to the author
to ask for permission.  For software which is copyrighted by the Free
Software Foundation, write to the Free Software Foundation; we sometimes
make exceptions for this.  Our decision will be guided by the two goals
of preserving the free status of all derivatives of our free software and
of promoting the sharing and reuse of software generally.

\begin{center}
{\small\sc
No Warranty
}
\end{center}

\item
{\sc Because the program is licensed free of charge, there is no warranty
for the program, to the extent permitted by applicable law.  Except when
otherwise stated in writing the copyright holders and/or other parties
provide the program ``as is'' without warranty of any kind, either expressed
or implied, including, but not limited to, the implied warranties of
merchantability and fitness for a particular purpose.  The entire risk as
to the quality and performance of the program is with you.  Should the
program prove defective, you assume the cost of all necessary servicing,
repair or correction.}

\item
{\sc In no event unless required by applicable law or agreed to in writing
will any copyright holder, or any other party who may modify and/or
redistribute the program as permitted above, be liable to you for damages,
including any general, special, incidental or consequential damages arising
out of the use or inability to use the program (including but not limited
to loss of data or data being rendered inaccurate or losses sustained by
you or third parties or a failure of the program to operate with any other
programs), even if such holder or other party has been advised of the
possibility of such damages.}

\end{enumerate}
\end{tiny}

\begin{center}
{\small\sc End of Terms and Conditions}
\end{center}


\pagebreak[2]

\small{Appendix: How to Apply These Terms to Your New Programs}

\begin{tiny}
If you develop a new program, and you want it to be of the greatest
possible use to the public, the best way to achieve this is to make it
free software which everyone can redistribute and change under these
terms.

  To do so, attach the following notices to the program.  It is safest to
  attach them to the start of each source file to most effectively convey
  the exclusion of warranty; and each file should have at least the
  ``copyright'' line and a pointer to where the full notice is found.

\begin{quote}
one line to give the program's name and a brief idea of what it does. \\
Copyright (C) yyyy  name of author \\

This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
\end{quote}

Also add information on how to contact you by electronic and paper mail.

If the program is interactive, make it output a short notice like this
when it starts in an interactive mode:

\begin{quote}
Gnomovision version 69, Copyright (C) yyyy  name of author \\
Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. \\
This is free software, and you are welcome to redistribute it
under certain conditions; type `show c' for details.
\end{quote}


The hypothetical commands {\tt show w} and {\tt show c} should show the
appropriate parts of the General Public License.  Of course, the commands
you use may be called something other than {\tt show w} and {\tt show c};
they could even be mouse-clicks or menu items---whatever suits your
program.

You should also get your employer (if you work as a programmer) or your
school, if any, to sign a ``copyright disclaimer'' for the program, if
necessary.  Here is a sample; alter the names:

\begin{quote}
Yoyodyne, Inc., hereby disclaims all copyright interest in the program \\
`Gnomovision' (which makes passes at compilers) written by James Hacker. \\

signature of Ty Coon, 1 April 1989 \\
Ty Coon, President of Vice
\end{quote}


This General Public License does not permit incorporating your program
into proprietary programs.  If your program is a subroutine library, you
may consider it more useful to permit linking proprietary applications
with the library.  If this is what you want to do, use the GNU Library
General Public License instead of this License.
\end{tiny}
\end{frame}
}

% \AtBeginLecture{\frame{\Large Today's Lecture: \insertlecture}}
\AtBeginSection[] %or \AtBeginSubsecton{} 
{
  \begin{frame}<beamer>
    \frametitle{\inserttitle}
    \tableofcontents[hideothersubsections]
    \flushbottom{\tiny{\copyrightnotice}}
  \end{frame}
}
%
% Print Lectures in part
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% \includeonlylecture{week01} 
% \includeonlylecture{week02} 
% \includeonlylecture{week03} 
% \includeonlylecture{week04} 
% \includeonlylecture{week05} 
% \includeonlylecture{week06} 
% \includeonlylecture{week07} 
% \includeonlylecture{week08} 
% \includeonlylecture{week09} 
% \includeonlylecture{week10} 
\includeonlylecture{week11} 
% \includeonlylecture{week12} 

\begin{document}
\lecture{Outline}{outline}

\frame{\titlepage}

\section*{Outline}

\frame{
	\frametitle{\inserttitle}
	\tableofcontents[hideallsubsections] %[pausesections]
}

%%
%%% David Weenink
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% let use Timit as a database and practise with large scale analyses by means of scripts.
% Formant analysis, deficits advantages, classification,
% Bandfilter analyses,
%
% Artificial vowel
% 1. Create PitchTier, 2. Edit/Add Point..., 3. To PointProcess, 4. To Sound,
% 5 Create FormantTier, Edit 6. select Sound+FormantTier: Filter
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%\part{Inleiding}
\lecture{Inleiding}{week01}
\renewcommand{\multicite}[1]{\cite{#1}}
\renewcommand{\copyrightauthor}[0]{David Weenink}
\renewcommand{\licensenumber}[0]{License1}

\section{Inleiding}

\begin{comment}
\frame
{
Speech waveform, sinusoids, amplitude and phase spectra, dB's. \\
Introduction computer program \praat.\\
Reference: \hc{ch. 1 \& 2}.\\
Get \praat{} from \url{http://www.fon.hum.uva.nl/praat}.\\ 
\assignment{1} Familiarization with microphone, sound-card and mixer.
}
\end{comment}

\subsection{Administrativa}
\frame
{ \frametitle{Administrativa}
\begin{itemize}
\item Colleges: donderdags van 12 -- 14
\item Practicum: donderdags 14.30-16.30
\item Wekelijkse opdrachten \emph{Alle opdrachten moeten voldoende voor tentamen}
\item Communication via \url{http://blackboard.ic.uva.nl/}
\begin{itemize}
\item Announcements
\item Mail
\item Assignments
\item \dots
\end{itemize}

\end{itemize}

}

\subsection{OS and Software}
\frame
{\frametitle{OS en programmatuur}
\begin{itemize}
\item \Praat{} voor analyses: \url{http://www.praat.org}
\item OS: Linux /Windows/Macintosh
\end{itemize}

}

\subsection{Inhoud}

\frame
{ \frametitle{Deze cursus}
\begin{itemize}
\item Daniel Jurafsky \& James H. Martin (2000),\\
\emph{Speech and Language Processing}, \\
Prentice Hall.
\item Kennismaking met (eigen) spraak
\item Spraakanalyses met \praat
\item Aspecten van Spraaktechnolgie
\begin{itemize}
\item Synthese
\item Herkenning
\item Grote databases
\end{itemize}
\item Synthese via festival
\end{itemize}

}

\subsection{De golfvorm}
\frame
{ \frametitle{\Timit{} voorbeeld: sa1 van spreker mjsw0}
\begin{columns}
\begin{column}{0.6\textwidth}
\begin{figure}
\includegraphics[heigth=0.9\textheight,width=0.9\textwidth]{Pictures/timit_label_example_mjsw0_sa1}
\end{figure}
\end{column}
\begin{column}{0.4\textwidth}
\timit{} Zin \texttt{sa1}:\\
She had your dark suit in greasy wash water all year
\end{column}
\end{columns}
}


\frame
{ \frametitle{Decompositie van complexe signalen}
Spraak golfvorm is te complex\\
Ontbinding via Fourier-transformatie

}


\subsection{Elementaire (basis) signalen}
\frame
{ \frametitle{Frequency and phase}
\begin{columns}
\begin{column}{0.55\textwidth}
\begin{figure}
\includegraphics[heigth=0.9\textheight,width=0.9\textwidth]{Pictures/frequency_and_phase}
\end{figure}
\end{column}
\begin{column}{0.45\textwidth}
Mathematical functions:
\begin{itemize}
\item[(a)] $0.9\sin (2\pi100t)$
\item[(b)] $0.9\sin (2\pi200t)$
\item[(c)] $0.9\sin (2\pi200t+\pi/2)$
\item[(c)] $0.9\cos (2\pi200t)$
\end{itemize}
\end{column}
\end{columns}

}

\subsection{Het computerprogramma \praat}
\frame
{ \frametitle{Het computerprogramma \praat}
\begin{block}{Schaamteloos uit de folder}
\begin{description}
\item [flexibel] Interactief \& scriptbaar
\item [portabel] Linux, Macintosh, Windows \& Unix versies
\item [herbruikbaar] Voor Fonetiek, Fonologie, Statistiek
\item [bruikbaar] Algemeen aanvaarde principes van mens-machineinteractie
\item [onderhoudbaar] Lichtgewicht, goed onderhoudbare kode (object ge\"ori\"enteerd)
\end{description}
\end{block}
}

\frame
{ \frametitle{Flexibel}
	\begin{itemize}
	\item Scripting
	\item History mechanisme
	\item Buttons...
	\item Dynamisch menu 
	\end{itemize}
}

\frame
{ \frametitle{Portabel}
	\begin{itemize}
	\item ANSI-C
	\item Speciale macros maken lineaire overerving mogelijk
	\item Andere macros genereren bij elke klasse automatisch lees, schrijf en kopi\"eer code
	\item Vensteromgeving is Motif met emulaties voor Windows \& Mac
	\item aangepaste audio
	\end{itemize}
}
\frame
{ \frametitle{(Her)bruikbaar-1}
\begin{block}{Objectge\"ori\"enteerd ontwerp met klasses}
	\begin{description}
	\item [Algemene klasses] Sound, Long\-Sound, Matrix, PointProcess, Strings, TableOfReal, Permutation, Distributions, Eigen, SVD, Minimizer, Polygon
	\item [Periodiciteitsanalyse] Pitch, Intensity, Harmonicity
	\item [Spectrale analyses] Spectrum, Spectrogram, Formant, \\
	LPC, Cepstrum, CC, LFCC, MFCC, Excitation, Cochleagram, LTAS
	\item [Labeling] TextGrid, IntervalTier, PointTier
	\end{description}
\end{block}
}
\frame
{ \frametitle{(Her)bruikbaar-2}
	\begin{description}
	\item [MDS] (Dis)similarity, Distance, Configuration, Procrustus, Confusion, ScalarProduct
	\item [Multivariaat] SSCP, Covariance, Correlation, PCA, Discriminant, ClassificationTable, ContingencyTable
	\item [Neurale netten] Pattern, Categories, FFNet, ART, ART2A, FuzzyART, ARTMAP, CategoryART
	\item [Optimality theory] OTGrammar
	\end{description}

}

\frame
{ \frametitle{Bruikbaar}
\begin{block}{Bruikbaarheid door konsekwente syntax}
	\begin{itemize}
	\item Opschrift op knoppen: Hoofdletter
	\item All\'e\'en interactie knoppen hebben ''...''
	\item Acties zonder nieuw object: \emph{Werkwoord}
	\item Acties met nieuw object: ''To ''
	\item Acties in script: opschriften op knop
	\item Muisselectie in script: \emph{select}, \emph{plus} en \emph{minus}
	\item Scripting: for, while, if, etc.
	\item Dynamisch menu: <type> help, Edit, Draw, Query, Modify
	\end{itemize}
\end{block}

}


\frame
{ \frametitle{Elementaire signalen via \praat}
Menu: \pc{New/Sound}
\begin{itemize}
\item[(a)] \pc{Create Sound... s1 0 0.03 22050 0.9*sin(2*pi*100*x)}
\item[(b)] \pc{Create Sound... s2 0 0.03 22050 0.9*sin(2*pi*200*x)}
\item[(c)] \pc{Create Sound... s3 0 0.03 22050}\\
\pc{... 0.9*sin(2*pi*200*x+pi/2)}
\item[(c)] \pc{Create Sound... s4 0 0.03 22050 0.9*cos(2*pi*200*x)}

\end{itemize}
}
\subsection{Fourier transformatie}
\frame
{ \frametitle{Decompositie: Fourier transform}
\begin{block}{Nieuwe basis}
\begin{enumerate}
\item sum of sines and cosines \\
$s(t)=\sum_{k=0} (a_k \sin(2\pi k f_0 t)+b_k \cos(2\pi k f_0 t))$
\item sum of sines with phases\\
$s(t)=\sum_{k=0} (c_k \sin(2\pi k f_0 t+\phi_k)$
(Immers: $a\sin\alpha+b\cos\alpha=c\sin(\alpha+\phi)$,\\
met $\alpha$ en $\phi$ functies van $a$ and $b$)
\end{enumerate}
\end{block}
\begin{block}{In \praat}
\pc{select Sound s1}\\
\pc{To Spectrum... no}\footnote{Option \pc{yes}: perform Fast Fourier Transform.}\\
\pc{Edit}
\end{block}
}

\frame
{ \frametitle{Logarithmische compressie, Decibel}
\begin{block}{Decibel}
dB's: $10 \log\frac{P(ower)}{P_{ref}}$\\
Omdat P $\sim$ Amplitude$^2$\\
dB's: $20 \log\frac{A(mplitude)}{A_{ref}}$\\
\end{block}

}

% Insert License text
\insertgnugpl

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%\part{Spreken en verstaan}
\lecture{Spreken en verstaan}{week02}
\renewcommand{\multicite}[1]{\cite{#1}}
\renewcommand{\copyrightauthor}[0]{David Weenink}
\renewcommand{\licensenumber}[0]{License2}

\section{Spreken en verstaan}

\subsection{De spraakketen}
\frame
{ \frametitle{De spraakketen}
Van idee naar geluid naar perceptie naar idee naar geluid...
}

\subsection{Fonetiek en fonologie}
\frame
{ \frametitle{Fonetiek en fonologie}

\begin{itemize}
\item Fonetiek:
\begin{itemize}
\item fysieke proces van spreken en verstaan\\
productie, signaaleigenschappen, verschillen...
\item analyse spraaksignaal
\end{itemize}

\item Fonologie: klanken onderdeel van systeem\\
\begin{itemize}
\item welke zijn betekenisonderscheidend
\item hoe is het klanksysteem?
\item welke combinaties mogen?
\end{itemize}
\end{itemize}
Fonetisch: \textipa{[A]}\\
Fonologisch: \textipa{/A/}
}

\subsection{Spreken}
\frame
{ \frametitle{De spraakorganen}
\begin{figure}
\includegraphics[heigth=0.9\textheight,width=0.9\textwidth]{Pictures/dwarsdoorsnede_hoofd.png}
\end{figure}
}

\subsection{Verstaan}
\frame
{ \frametitle{Het oor}
\begin{figure}
\includegraphics[heigth=0.9\textheight,width=0.9\textwidth]{Pictures/dwarsdoorsnede_oor.png}
\end{figure}
}
\subsection{Spraakklanken}
\frame
{ \frametitle{Spraakklanken}
\begin{block}{Twee categori\"en}
\begin{itemize}
\item vocalen /klinkers\\
nauwelijks vernauwing in mond-keel kanaal
\item consonanten / medeklinkers\\
Wel vernauwing
\end{itemize}

\end{block}
\begin{block}{Verder onderscheid op basis van}
\begin{itemize}
\item Manier van articulatie
\item plaats van articulatie
\item stemgeving
\end{itemize}
\end{block}
}

\frame
{ \frametitle{Manier van articulatie}
\begin{itemize}
\item plosieven (plofklanken): p, t, k\\
volledige afsluiting met overdruk, release
\item fricatieven (wrijfklanken): f, s\\
bijna volledige vernauwing
\item liquidae (vloeiklanken): r, l\\
lucht langs zijkant tong
\item nasalen (neusklanken): m, n\\
lucht door de neus
\item halfklinkers (glijklanken): w, j\\
nauwelijks vernauwing
\end{itemize}

}

\frame
{ \frametitle{Plaats van articulatie}
p, t, k zijn verschillend
\begin{itemize}
\item labiale klanken: b\\
bij de bovenlip
\item dentale klanken: d\\
bij de boventanden
\item alveolaire klanken: s\\
tandkas achter de boventanden
\item palatale klanken: j\\
harde gehemelte
\item velaire klanken: k\\
zachte gehemelte
\item uvulaire klanken: huig-r\\
bij de huig
\item glottale klanken: h\\
bij de stemspleet
\end{itemize}
dentaal+alveolair+palataal =coronale klanken\\
velair+uvulair+glottaal = dorsale klanken
}

\frame
{ \frametitle{Stemgeving}
\begin{itemize}
\item stemhebbend: b, d, g
\item stemloos: p, t, k
\end{itemize}

}


\frame
{ \frametitle{Manier en plaats van articulatie: Consonanten/medeklinkers}
\begin{figure}
\includegraphics[heigth=0.9\textheight,width=0.9\textwidth]{Pictures/nederlandse-consonanten-manner.png}
\end{figure}

}

\frame
{ \frametitle{Vocalen/klinkers}
\begin{block}{Articulatieplaats}
\begin{itemize}
\item positie van de tongrug
\begin{itemize}
\item voor-achter: /\textipa{i}/, /\textipa{u}/
\item hoog-laag of gesloten-open: /\textipa{u}/ - /\textscripta/ 
\end{itemize}
\item lippen: /\textipa{i}/, /\textipa{y}/
\end{itemize}
\end{block}
}

\frame
{ \frametitle{}
\begin{figure}
\includegraphics[heigth=0.9\textheight,width=0.9\textwidth]{Pictures/nederlandse-klinkers.png}
\end{figure}

}

\frame
{ \frametitle{Spraaksynthese}
\begin{block}{Manieren}
\begin{itemize}
\item nabootsing van menselijke articulatie
\item recorder: opslag van alle zinnen
\item opbouw uit kleinere stukjes
\begin{itemize}
\item difoonsynthese extra = \textipa{[\*\#E][Ek][ks][st][ra:][a:\*\#]}\\\
beregeling van luidhuid, foneemduur (beklemtoonde wordt langer) en toonhoogte (vraagzin)
\end{itemize}
\item totaal via syntheseregels

\end{itemize}

\end{block}
}
\begin{comment}
\frame
{ \frametitle{}
Menselijke spraakproductie en spraakperceptie
Fonemen, woorden, zinnen
Duur, luidheid, klankkleur/timbre
discrete tekst op papier versus continue spraaksignaal
woordklemtoon, zinsaccent
toonhoogteverloop, melodie, ritme
fluisteren
mannen, vrouwen, kinderen
spraak- en taalverwerving door kinderen en als tweede taal door volwassenen

}
\end{comment}

% Insert License text
\insertgnugpl


%%% R.J.J.H. van Son

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%\part{Measuring Speech}
\lecture{Measuring Speech}{week03}
\renewcommand{\multicite}[1]{\nocite{#1}\citeSthree{#1}}
\renewcommand{\copyrightauthor}[0]{R.J.J.H. van Son}
\renewcommand{\licensenumber}[0]{License3}

\section{Measuring Speech}

\subsection{Introduction}
\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{All technology starts with quantitative modelling}
\begin{itemize} [<+->]
\item Speech technology is about speech \alert<1>{sounds}
\item Only limited knowledge of human speech production 
      and perception is necessary for modeling speech sounds
\item In practice, knowledge about human speech is
      only used implicitely
\end{itemize}
\end{block}

\end{frame}

\subsection{Waveforms}
\begin{frame}
\frametitle{\insertsubsection: Oscillogram}
\center{
\movie [externalviewer]{
\includegraphics[width=0.7\textwidth]{Pictures/ErWasEensEenOudKasteel}
}{Sounds/F28G1FT2A1+6_fm.wav}
}
\begin{block}{"Er was eens een oud kasteel"}
\begin{itemize} [<+->]
\item Display of presure versus time
\item Words are aligned with sound
\item Using computer readable (SAMPA) phoneme symbols
\end{itemize}
\end{block}
\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Digital sound and band-width}
\center{\includegraphics[width=0.55\textwidth]{Pictures/s-quantisized}}
\begin{block}{1.5 ms of an \textipa{/s/} sound from "was"}
\begin{itemize} [<+->]
\item Samples taken at 44.1 kHz (CD audio)
\item Quantisize at 16 bit ($\approx 65000$ amplitude levels)
\item Maximum audio frequency 22.05 kHz (Nyquist frequency)
      but generally \emph{much} less
\item Dynamic Range $\approx 96 db$ ($\approx 6 db / bit$)
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Amplitude and sound level}
\center{\movie [externalviewer]{
\includegraphics[width=0.6\textwidth]{Pictures/KasteelIntensity}}
{Sounds/F28G1FT2A6_fm_Kasteel.wav}
}
\begin{block}{Intensity contour of "Kasteel"}
\begin{itemize} [<+->]
\item Intensity versus amplitude
\item Intensity in db ($10 \cdot log_{10}(Sound Energy)$)
\item Intensity you hear is not the intensity you measure
	$\Rightarrow$ correct for human perception ($dBA$)
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Durations}
\center{\movie [externalviewer]{
\includegraphics[width=0.6\textwidth]{Pictures/KasteelPhonemes}}
{Sounds/F28G1FT2A6_fm_Kasteel.wav}
}
\begin{block}{Phoneme segmentation of "Kasteel"}
\begin{itemize} [<+->]
\item Determine the boundaries of words, syllables and phonemes
\item Use waveform, ear, and spectrum
\item Segmentation is ambiguous and laborious
\item Start with automatic segmentation (for speed)
\end{itemize}
\end{block}

\end{frame}

\subsection{Pitch and F0}
\begin{frame}
\frametitle{\insertsubsection: The perception of tones: $F_0$}
\center
{
\movie [externalviewer]
{\includegraphics[width=0.40\textwidth]{Pictures/125Hztone}}
{Sounds/125Hztone.wav}
\movie [externalviewer]
{\includegraphics[width=0.48\textwidth]{Pictures/3tonecomplex125Hz}}
{Sounds/3tonecomplex125Hz.wav}
}
\begin{block}{Pitch or $F_0$ is the \emph{perception} of a harmonic
             sequence. Generally, perceived $pitch$ is the:}
\begin{itemize} [<+->]
\item frequency of a pure tone (top, 125 Hz)
\item distance between the components in a mixture of harmonic tones
      (eg, 125 Hz)
\item closest harmonic fit in complex sounds (bells)
\end{itemize}
\end{block}
\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Measuring $F_0$}
\center
{
\movie [externalviewer]
{\includegraphics[width=0.6\textwidth]{Pictures/F28G1FT2A_Pitch}}
{Sounds/F28G1FT2A1+6_fm.wav}
\movie [externalviewer]
{\includegraphics[width=0.6\textwidth]{Pictures/F28G1FT2A_PitchTier}}
{Sounds/F28G1FT2A1+6_fmHum.wav}
}
\begin{block}{The best $F_0$ candidates are determined}
\begin{itemize} [<+->]
\item from the possible repeat frequencies using an autocorrelation function
\item from the best fitting harmonics using a \emph{harmonic sieve}
\end{itemize}
\end{block}
\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Pitch contours}
\center
{
\movie [externalviewer]
{\includegraphics[width=0.8\textwidth]{Pictures/F28G1FT2A_PitchTier}}
{Sounds/F28G1FT2A_fm.wav}

\movie [externalviewer]
{\Large \textcolor{blue}{Hummed sound}}
{Sounds/F28G1FT2A_fmHum.wav}
}
\begin{block}{$F_0$ makes the melody, or intonation, of an utterance}
\begin{itemize} [<+->]
\item There is a general decrease of $F_0$ over an utterance: The \emph{declination}
\item $F_0$ movements indicate emphasized words: pitch \emph{accents}
\item $F_0$ movements and \emph{declination resets} indicate boundaries
\end{itemize}
\end{block}
\end{frame}

\subsection{Spectrum}

\begin{frame}[label=Cochlea]
\frametitle{\insertsubsection: The Ear (again)}
\center
{
\hyperlink{InnerEar}{
  \includegraphics[width=0.8\textwidth]{Pictures/lec6_fig1}
}
}

\begin{block}{Frequency map of the cochlea from \citeSthree{Moo03} \nocite{Moo03}}
\begin{itemize} [<+->]
\item The ear analyses sounds roughly into 
      $Log \left( Power \left( Frequency \right) \right)$ 
      vs. $Log \left(  Frequency \right)$
\item Speech is analyzed in the same way
\item Use power spectra of sounds
\end{itemize}
\end{block}

\end{frame}

\begin{frame}[label=E_Spectrum]

\frametitle{\insertsubsection: Example of \textipa{/E/}}
\center
{
\hyperlink{E_FormantSpectrum}{
\includegraphics[width=0.99\textwidth]{Pictures/E_spectrum}
}
}

Note the harmonic structure and the "bumps"
\end{frame}

\begin{frame}

\frametitle{\insertsubsection: Example of \textipa{/n/}}
\begin{center}
\includegraphics[width=0.99\textwidth]{Pictures/n_spectrum}
\end{center}

Note the harmonic structure and the low level of high frequencies
\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Example of \textipa{/s/}}
\begin{center}
\includegraphics[width=0.99\textwidth]{Pictures/s_spectrum}
\end{center}

Note the noisy structure and the broad bandwidth
\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Source Filter model of speech}
\begin{center}
\includegraphics[height=0.3\textwidth]{Pictures/OralTrackt.pdf}
\includegraphics[width=0.6\textwidth]{Pictures/VoiceSource_fig2.png}
\end{center}

Sound enters the oral cavity (vocal tract) from below and is filtered 
by the resonances of the cavity

\end{frame}


\begin{frame}[label=E_FormantSpectrum]
\frametitle{\insertsubsection: Resonances and formants}

\begin{center}
\hyperlink{E_Spectrum}{
\includegraphics[width=0.7\textwidth]{Pictures/E_formantspectrum}
}
\end{center}

Oral cavity filter function of \textipa{/E/} (LPC model). 
Peaks are formants $F_1$ and $F2$.

The resonances of the vocal tract are called Formants, and numbered
from below, i.e., $F_1$, $F_2$, $F_3$, $\cdots$. Normally,
the first three are sufficient to describe (voiced) speech.

\end{frame}


\begin{frame}
\frametitle{\insertsubsection: Vowel Formant space}

\begin{center}
\includegraphics[width=0.5\textwidth]{Pictures/Proc24IFAcorpusFig1}
\end{center}

Vowel formant space of Dutch.

Only two formant values, $F_1$ and $F_2$, suffice to identify
a vowel (in the ideal case). However, in normal speech, there
is so much overlap and variation that it remains almost 
impossible.

\end{frame}

\subsection{Spectrograms}

\begin{frame}
\frametitle{\insertsubsection}

\begin{center}
\includegraphics[width=0.7\textwidth]{Pictures/KasteelSpectrogram}
\end{center}

\begin{block}{A spectrogram shows the development of the spectrum in
              time (darker is more power)} 
\begin{itemize}[<+->]
\item A spectrogram shows the harmonics
\item Vowels, fricatives, and plosives are visible
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Narrow versus Wide band}

\begin{center}
\includegraphics[width=0.5\textwidth]{Pictures/KasteelNarrow100ms}
\includegraphics[width=0.5\textwidth]{Pictures/KasteelWide10ms}
\end{center}

\begin{block}{Two views on spectrograms} 
\begin{itemize}[<+->]
\item Narrow-band (top): High frequency resolution, low time resolution
\item Wide-band (bottom): Low frequency resolution, high time resolution
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Formant and Pitch tracking}

\begin{center}
\includegraphics[width=0.85\textwidth]{Pictures/KasteelPitchFormants}
\end{center}

Formants (red dots) and Pitch (blue line) can be automatically 
determined and plotted into a spectrogram.

\end{frame}


\begin{frame}
\frametitle{\insertsubsection: Noise and bursts}

\begin{center}
\includegraphics[width=0.85\textwidth]{Pictures/st_NoiseAndBurst}
\end{center}

Fricatives are visible as gray noise patches. Plosives as 
a silent part followed by a noisy burst.

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Spectrogram reading}

\begin{center}
\includegraphics[width=0.85\textwidth]{Pictures/KasteelPitchFormants}
\end{center}

It is actually possible, after a few weeks training, to read spectrograms.
All the information needed to "understand" the speech is in the spectrogram 
\citeSthree{LC97}.
\end{frame}

\subsection{Transcription}

\begin{frame}
\frametitle{\insertsubsection: Transliteration}

\begin{block}{Before anything can be done with speech, it has to be
              written down and transcribed} 
\begin{itemize}[<+->]
\item Write out orthographically what was said (and check it)
\item Align chunks of text roughly with the stretches of speech
\item Transcribe the text automatically into phonemes using a lexicon
\item Split the orthographic/phonemic text into words
\item Align the words/phonemes automatically with the speech
\item Add automatic Part-of-Speech tags and Syntax
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Transcription}

\begin{block}{Human annotator transcriptions: Difficult and expensive} 
\begin{itemize}[<+->]
\item Accents, stress, and boundaries (always ambiguous)
\item Handcorrected word-boundaries
\item Handcorrected phoneme-boundaries (always ambiguous)
\item Check Part-of-Speech tags
\item Check Syntax
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: \\Identifying and annotating phonemes}

\center{\movie [externalviewer]{
\includegraphics[width=0.5\textwidth]{Pictures/KasteelPhonemes}}
{Sounds/F28G1FT2A6_fm_Kasteel.wav}
}

\begin{block}{Phonemes are not pearls on a string} 
\begin{itemize}[<+->]
\item Phonemes always overlap and are extremely variable
\item A phoneme you hear can appear absent in the waveform
\item It is often unclear what phonemes were uttered
\item Sometimes, even the order is unclear
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: \\ToBI like systems for intonation transcription}

\begin{center}
\includegraphics[width=0.45\textwidth]{Pictures/ToDIvoorb}
\movie [externalviewer]{\includegraphics[width=0.45\textwidth]{Pictures/ToDI052}}
{Sounds/ToDI052.wav}
\end{center}

\begin{small}
\begin{block}{ToDI symbols \cite{ToDI2003} \nociteSthree{ToDI2003} 
 (IP: Intonational Phrase)}
\begin{tabular}{|l l l|}
\hline
High & Low & description\\
\hline
H*	&L*	&high/low accent\\
H	&L	&upward/downward movement after L*/H*\\
H\%	&L\%	&rising/low ending of IP\\
\%H	&\%L	&high/low beginning of IP\\
\%HL	&       &Initial falling pitch not marking accent\\
\%	&       &half-completed fall/rise at end of IP\\
!H*	&       &downstepped H*\\
\hline
\end{tabular}
\end{block}

\end{small}

\end{frame}

\subsection{Assignment}

\begin{frame}
\frametitle{\insertsubsection - 3 Spectrogram and spectrum}

\begin{block}{See BlackBoard for full description} 
\begin{itemize}[<+->]
\item Use a recorded sentence (assignment 2).
\item Determine durations of all vowels (max 10)
\item Determine the spectrum of a single (monophthong) vowel
\item Calculate the spectrogram
\item Draw waveform, spectrum, and spectrogram
\end{itemize}
\end{block}

\end{frame}


\subsection{Bibliography}

\begin{frame}[allowframebreaks]
\frametitle{Further Reading}
\nociteSthree{BoersmaWeenink}
\nocite{BoersmaWeenink}
\nociteSthree{Boersma2001}
\nocite{Boersma2001}
\nociteSthree{LC97}
\nociteSthree{LC97}
\nocite{LC97}
\nociteSthree{ToDI2003}
\nocite{ToDI2003}
\nociteSthree{Filip1995}
\nocite{Filip1995}
\bibliographystyleSthree{plainnat}
\tiny{
\bibliographySthree{SpraakherkenningEnSynthese}
}
\end{frame}

\appendix
\begin{frame}
\frametitle{Appendix A}

\end{frame}

\begin{frame}[label=InnerEar]
\frametitle{The inner ear}

\begin{center}
\hyperlink{Cochlea}{
\includegraphics[heigth=0.9\textheight,width=0.9\textwidth]{Pictures/dwarsdoorsnede_oor.png}
}
\end{center}

\end{frame}

% Insert License text
\insertgnugpl


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%\part{Measuring Speech}
\lecture{Speech Corpora, labeling and segmentation}{week04}
\renewcommand{\multicite}[1]{\nocite{#1}\citeSfour{#1}}
\renewcommand{\copyrightauthor}[0]{R.J.J.H. van Son}
\renewcommand{\licensenumber}[0]{License4}

\section{Speech Corpora, labeling and segmentation}

\subsection{Introduction}
\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{There is no data like more data}
\begin{itemize} [<+->]
\item Speech and Language are extremely complex
\item Large amounts of data are necessary to model them
\item ``The best application is the one with the largest corpus''
\item 10-1000 hours of speech recordings needed
\item $10^8$ - $10^9$ word text corpus needed
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Corpora for Speech and Language Technology}

\emph{A language corpus is a documented collection of coherent text,
speech, video, and transcriptions and annotations of these}

\begin{block}{Requirements}
\begin{itemize} [<+->]
\item Meta data (fixed)
\item Normalization (fixed)
\item Data (fixed)
\item Transcriptions and annotations (cumulative)
\item Storage, distribution, access, and software (volatile)
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Corpora for Speech and Language Technology}

\begin{block}{Requirements}
\begin{itemize}
\item \alert{Meta data (fixed): Information on the items}
	\begin{itemize} [<+->]
	\item Bibliographic/biographic information (author, speaker) 
	\item Dates
	\item Origin, eg, place of publishing, recording
	\item Language variant
	\item Genre and style
	\item Recording trail, post-processing, and formats
	\item \alert {Access criteria, Copyrights, Privacy\&Ethical guidelines}
	\item \dots
	\end{itemize}
\item Normalization (fixed)
\item Data (fixed)
\item Transcriptions and annotations (cumulative)
\item Storage, distribution, access, and software (volatile)
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Corpora for Speech and Language Technology}

\begin{block}{Requirements}
\begin{itemize}
\item Meta data (fixed)
\item \alert {Normalization (fixed): All items must adhere to certain guidelines}
	\begin{itemize} [<+->]
	\item Inclusion/selection criteria
	\item Recording and text formats
	\item Spelling rules, orthographic normalization
	\item Storage formats (sample frequencies, file formats)
	\item \dots
	\end{itemize}
\item Data (fixed)
\item Transcriptions and annotations (cumulative)
\item Storage, distribution, access, and software (volatile)
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Corpora for Speech and Language Technology}

\begin{block}{Requirements}
\begin{itemize}
\item Meta data (fixed)
\item Normalization (fixed)
\item \alert{Data (fixed): Immutable text or speech records}
	\begin{itemize} [<+->]
	\item Broadcast recordings
	\item Speech recordings
	\item Video recordings
	\item Original text
	\item Transliterations of speech (correctable)
	\item \dots
	\end{itemize}
\item Transcriptions and annotations (cumulative)
\item Storage, distribution, access, and software (volatile)
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Corpora for Speech and Language Technology}

\begin{block}{Requirements}
\begin{itemize}
\item Meta data (fixed)
\item Normalization (fixed)
\item Data (fixed)
\item \alert {Transcriptions and annotations (cumulative): 
		Added value of interpretations and analysis}
	\begin{itemize} [<+->]
	\item Orthographic transcription (transliteration) of speech
	\item Paragraph and sentence boundaries
	\item Phonemic transcription
	\item Prosodic transcription (eg, ToBI)
	\item Part-of-Speech tagging
	\item Lemmatization
	\item Syntactic trees (tree-bank)
	\item \dots
	\end{itemize}
\item Storage, distribution, access, and software (volatile)
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Corpora for Speech and Language Technology}

\begin{block}{Requirements}
\begin{itemize}
\item Meta data (fixed)
\item Normalization (fixed)
\item Data (fixed)
\item Transcriptions and annotations (cumulative)
\item \alert{Storage, distribution, access, and software (volatile): Practical usage}
	\begin{itemize} [<+->]
	\item Digital storage, what and where
	\item On-line and/or media distribution (DVD)
	\item Access policies (pricing, licenses)
	\item Exploration software
	\item Database tables
	\item DBMS
	\item Updates and policy
	\item \dots
	\end{itemize}
\end{itemize}
\end{block}

\end{frame}

\subsection{Language corpora}
\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{Example corpora and their sizes}
\begin{itemize} [<+->]
\item IFA Corpus: 50 thousand words (5$\dfrac{1}{2}$ hours)
	\citeSfour{IFA2003}\nocite{IFA2003}
\item Spoken Dutch Corpus (CGN): 9 million words (800 hours)
	\citeSfour{cgn1.0}\nocite{cgn1.0}
\item British National Corpus (BNC): 100 million words
	\citeSfour{BNC1997}\nocite{BNC1997}
\item Twente journal corpus: 300 million words (Dutch) 
	\citeSfour{ordelman2002}\nocite{ordelman2002}
\item Tilburg text corpus: 600 million words (Dutch, unpublished?)
\item COSMAS corpus archive: 1.8 billion words (German)
	\citeSfour{IDS05}\nocite{IDS05}
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: CGN \citeSfour{cgn1.0}}

\begin{block}{Contents ($\dfrac{2}{3}$ Dutch, $\dfrac{1}{3}$ Flemish)}
\begin{itemize} [<+->]
\item 500 hours (5,650,000 words) recorded in The Netherlands
\item 300 hours (3,250,000 words) in Flanders
\item 4250 speakers
\item 15 Styles/genres
\item Field recordings with Sony Minidisk
\item 16/16 and 8/8 kHz/bit encoding
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: CGN Styles and Genres}

\begin{small}
\begin{block}{CGN: 9 million words from 800 hours of speech}
\begin{tabular}{|r r l|}
\hline
Hour & kWords & Style\\
\hline
225 & 2,626 & spontaneous conversations ('face-to-face')\\
51  &  565 & interviews with teachers of Dutch (VNC)\\
92  &  1,209 & spontaneous telephone dialogues\\
64  & 853 & spontaneous telephone dialogues\\
11  & 136 & simulated business negotiations\\
64  & 790 & interviews/discussions/debates (broadcast)\\
36  & 360 & discussions/debates/meetings (non-broad.)\\
44  & 405 & lessons recorded in the classroom\\
21  & 208 & live (eg sport) commentaries (broadcast)\\
17  & 186 & newsreports/reportages (broadcast)\\
36  & 368 & news (broadcast)\\
15  & 146 & commentaries/columns/reviews (broadcast)\\
2  & 18 & ceremonious speeches/sermons\\
16  & 141 & lectures/seminars\\
104  & 903 & read speech (read books)\\
\hline
\end{tabular}
\end{block}
\end{small}
\end{frame}

\begin{frame}
\frametitle{\insertsubsection: CGN Annotations}

\begin{small}
\begin{block}{Annotations and transcriptions}
\begin{itemize} [<+->]
\item Orthographic transcription (the full 8,900,000 words)
\item Manually verified POS tagging and lemmatization (all)
\item Lexicon and identification of multi word units (all)
\item Automatic time alignment and phonetic transcription at the word level (all)
\item Manually verified broad phonetic transcription (1,000,000 words)
\item Manually verified time alignment at the word level (1,000,000 words)
\item Syntactic annotation (1,000,000 words)
\item Two independent prosodic annotations (250,000 words)
\end{itemize}
\end{block}
\end{small}

\end{frame}

\subsection{Use of corpora in Speech Technology}
\begin{frame}
\frametitle{\insertsubsection: Research}

\begin{block}{Phonetic, prosodic and syntactic research}
\begin{itemize} [<+->]
\item Phoneme durations
\item Stress and Accent placement
\item Intonation and expressive speech (emotions)
\item Part-of-Speech tagging
\item Prosodic and syntactic boundaries
\item Phoneme assimilation (eg, word boundaries)
\item Pronunciation variation
\item Morphological decomposition
\item Visual speech
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: TTS Modeling}

\begin{block}{Text to Speech synthesis}
\begin{itemize} [<+->]
\item Produce accentuation and boundaries from text
\item Produce phoneme durations from text
\item Grapheme-to-phoneme conversion (lexicon)
\item Chunk words into groups (punctuation)
\item Decompose words into components (compound words)
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: ASR Modeling}

\begin{block}{Automatic Speech Recognition}
\begin{itemize} [<+->]
\item Hidden Markov Model training
\item Language model (smoothed N-grams)
\item Pronunciation variation
\item Tree bank training (syntactic probabilities)
\end{itemize}
\end{block}

\end{frame}

\subsection{Annotation, Segmentation, and labeling}
\begin{frame}
\frametitle{\insertsubsection: Orthography}

\begin{block}{Manual Orthographic transcription (transliteration) 
              is used to automatically generate}
\begin{itemize} [<+->]
\item Tokens (words) $\Rightarrow$ Word alignment
\item Phonemic transcription $\Rightarrow$ Phone alignment
\item POS tags
\end{itemize}
\end{block}

All annotations and segmentation can be manually verified
(at great cost)

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: POS tagging}

\begin{block}{POS tags are used to automatically generate}
\begin{itemize} [<+->]
\item Lexical stress
\item Syntactic structure
\item Lemmas
\item Prosodic structure (ToBI) $\Rightarrow$ 
      \alert{currently only by hand}
\end{itemize}
\end{block}

All annotations and segmentation can be manually verified
(at great cost)

\end{frame}

\subsection{Phonetic symbols}
\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{Speech corpora needed an unambiguous digital encoding
  of IPA symbols (now there is UNICODE)}
\begin{itemize} [<+->]
\item Language specific encodings
	\begin{itemize} [<+->]
	\item 1 character ASCII encodings + diacritics (SAMPA)
	\item 2 character ASCII encodings (SWITCHBOARD)
	\end{itemize}
\item Complete IPA encodings
	\begin{itemize} [<+->]
	\item 2 character ASCII encoding  (Worldbet \citeSfour{Hieronymus1994}\nocite{Hieronymus1994})
	\item Control encodings (LaTeX Tipa, Praat)
	\end{itemize}
\item Currently, control encodings are impractical for manual labeling
\item  Note that mapping sounds to the IPA is \emph{not} trivial
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: CGN's SAMPA encoding}

\begin{block}{SAMPA vs. Worldbet, eg, Vowels}
\begin{small}
\begin{tabular}{|l l l l l|}
\hline
IPA & CGN & Wbet & Example & Word\\
\hline
\textipa{I} & I & `I ' & lIp & lip\\
\textipa{E} & E & `E ' &lEx & leg\\
\textipa{A} & A & `A ' & lAt & lat\\
\textipa{O} & O & `$>$ ' & bOm & bom\\
\textipa{Y} & Y & `ux' & pYt & put\\
\textipa{i} & i & `i ' & lip & liep\\
\textipa{y} & y & `y ' & byr & buur\\
\textipa{e} & e & `e ' & lex & leeg\\
\textipa{\textbaro} & 2 & `7 ' & d2k & deuk\\
\textipa{a} & a & `a ' & lat & laat\\
\textipa{o} & o & `o ' & bom & boom\\
\textipa{u} & u & `u ' & buk & boek\\
\textipa{@} & @ & `\& ` & x@-lE+k & gelijk\\
\textipa{Ei} & E+ & `Ei' & wE+s & wijs\\
\textipa{\textbaro y} & 9+ & `8y' & h9+s & huis\\
\textipa{Ou} & O+ & `Ou' & kO+t & koud\\
\hline
\end{tabular}
\end{small}
\end{block}

\end{frame}

\subsection{Assignment}
\begin{frame}
\frametitle{\insertsubsection: 4 Segmentation and labeling}

\begin{block}{Label and segment your utterance on the word and 
segment (phoneme) level}
\begin{itemize} [<+->]
\item Open sound in praat
\item Create a Word tiers
\item Add the (aligned) words to the tier
\item Copy to a Phoneme tier
\item Then add (split into) the phonemes
\end{itemize}
\end{block}

\end{frame}


\subsection{Bibliography}

\begin{frame}[allowframebreaks]
\frametitle{Further Reading}
\nociteSfour{BoersmaWeenink}
\nocite{BoersmaWeenink}
\nociteSfour{Boersma2001}
\nocite{Boersma2001}
\bibliographystyleSfour{plainnat}
\tiny{
\bibliographySfour{SpraakherkenningEnSynthese}
}
\end{frame}

\appendix
\begin{frame}
\frametitle{Appendix A}

\end{frame}



% Insert License text
\insertgnugpl


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%\part{Measuring Speech}
\lecture{Automatic Text-To-Speech synthesis}{week05}
\renewcommand{\multicite}[1]{\nocite{#1}\citeSfive{#1}}
\renewcommand{\copyrightauthor}[0]{R.J.J.H. van Son}
\renewcommand{\licensenumber}[0]{License5}

\section{Automatic Text-To-Speech synthesis}

\subsection{Introduction}
\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{Uses of speech synthesis by computer}
\begin{itemize} [<+->]
\item Read aloud existing text, eg, news, email and stories
\item Communicate volatile data as speech, eg, weather reports, query results
\item The computer part of interactive dialogs
\end{itemize}
\end{block}

\uncover<4->{The building block is a Text-to-Speech system that can handle
standard text with a Speech Synthesis (XML) markup. The TTS system
has to be able to generate acceptable speech from plain text, 
but can improve the quality using the markup tags}

\end{frame}

\subsection{Computer Speech}
\begin{frame}
\frametitle{\insertsubsection: Generating the sound}

\begin{center}
Speech Synthesizers can be classified on the way they generate speech sounds.
This determines the type, and amount, of data that have to be collected.
\end{center}

\begin{block}{Speech Synthesis}
\begin{itemize} [<+->]
\item Articulatory models
\item Rules (formant synthesis)
\item Diphone concatenation
\item Unit selection
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Articulatory models}

\begin{center}
\includegraphics[width=0.45\textwidth]{Pictures/Articulation.pdf}
\includegraphics[width=0.45\textwidth]{Pictures/ArticulatorySynthesisEditor.jpg}
\end{center}

\begin{block}{Characteristics (\movie [externalviewer]{\textcolor{blue}
{/\textipa{Er@}/ from Praat}}{Sounds/ArticulatoryExample_ere.wav})
\nocite{Boersma1998}\citeSfive{Boersma1998}}
\begin{itemize} [<+->]
\item Quantitative Source-Filter model of vocal tract
\item Solve Navier-Stokes equations for air-flow
\item Needs hard-to-get articulatory data
\end{itemize}
\end{block}

\end{frame}


\begin{frame}
\frametitle{\insertsubsection: Rule, or formant, based synthesis}


\begin{center}
\movie [externalviewer]{\includegraphics[width=0.40\textwidth]{Pictures/Klatt_img011.jpg}
\includegraphics[width=0.45\textwidth]{Pictures/KlattCascade_img008.jpg}}
{Sounds/KlattExample.wav}\\
Klatt synthesizer \nocite{SRL,Spr}\citeSfive{Spr,SRL}
\end{center}

\begin{block}{Characteristics (
\movie [externalviewer]{\textcolor{blue}{YorkTalk}}{Sounds/YorkTalkSentence.wav}
\nocite{Moh05}\citeSfive{Moh05})}
\begin{itemize} [<+->]
\item Recreate sounds using source and resonances
\item Model formant tracks by rules
\item Endless tuning, no data driven modeling possible
\end{itemize}
\end{block}

\end{frame}


\begin{frame}
\frametitle{\insertsubsection: Diphone synthesis}


\begin{center}
\movie [externalviewer]{\includegraphics[width=0.80\textwidth]{Pictures/MinderDiphone.jpg}}
{Sounds/MinderDiphone.wav}
\end{center}

\begin{block}{Characteristics (
\movie [externalviewer]{\textcolor{blue}{Spengi, Philips/IPO}}
 {Sounds/NoordenwindSpengi.wav}
\nocite{Moh05}\citeSfive{Moh05})}
\begin{itemize} [<+->]
\item Concatenative synthesis: Glue phoneme-phoneme transitions
\item Good quality, but requires all phoneme combinations to be present
\item Sound encoding must allow intonation changes
\end{itemize}
\end{block}

\end{frame}


\begin{frame}
\frametitle{\insertsubsection: Non-uniform unit selection}

\begin{center}
Generalize diphone synthesis to use larger, non-uniform,
units like: diphones, multiphones (clusters),
demi-syllables, syllables, words, and short phrases
\end{center}

\begin{block}{Characteristics (
\movie [externalviewer]{\textcolor{blue}{Festival}}
 {Sounds/Festival_unitsel.wav}
 \nocite{BL03}\citeSfive{BL03})}
\begin{itemize} [<+->]
\item Requires large annotated speech corpora (\~GByte range)
\item Corpus must be well annotated and searchable
\item Efficient statistical search algorithms to optimize 
      unit selection based on prosody and concatenation costs
\item More speech in corpus $\Rightarrow$ Better synthesis
\item But also $\Rightarrow$ More work to find the best combination
\end{itemize}
\end{block}

\end{frame}


\begin{frame}
\frametitle{\insertsubsection: Text-to-Speech}

\begin{block}{Text in Speech out: Processing ``steps''}
\begin{itemize} [<+->]
\item Text normalization
\item Grapheme Phoneme conversion
\item Accent placement
\item Duration generation
\item Intonation generation
\item Speech Generation
\end{itemize}
\end{block}

\end{frame}

\subsection{Text preprocessing}
\begin{frame}
\frametitle{\insertsubsection: Normalize texts}

\begin{scriptsize}\begin{block}{Text should contain only pronounceable tokens}
\parbox{\textwidth}{
\parbox{0.45\textwidth}{\begin{itemize} [<+->]
\item Abbreviations
\item Dates
\item Times
\item Telephone numbers
\end{itemize}
}
\parbox{0.45\textwidth}{\begin{itemize} [<+->]
\item Money
\item Street Addresses
\item General numbers
\item Special characters
\end{itemize}
}
}
\end{block}
\end{scriptsize}

\begin{scriptsize}
Join Kerry Stratton \alert<8>{\&} his guest chamber orchestra as they bring the music of the Italian Maestro to life on our stage. Tickets \alert<5>{\alert<8>{\$}\alert<7>{46.00}}

\alert<7>{5} Easy Ways to Order Tickets\\
\alert<1>{\textbf{A}} Visit our Box Office \alert<8>{(}map\alert<8>{)} 
\alert<1>{Mon} through \alert<1>{Sat}, \alert<3>{11:00} \alert<1,3>{a.m.} 
to \alert<3>{6:00} \alert<1,3>{p.m.}
Summer Hours: \alert<2>{July 4} to \alert<1,2>{Sept} \alert<2>{2, 2005} - 
\alert<3>{11:00} \alert<1,3>{a.m.} to \alert<3>{4:30} \alert<1,3>{p.m.}\\
\alert<1>{\textbf{B}} Call our Box Office at \alert<4>{905-305-SHOW \alert<8>{(}7469\alert<8>{)}} or Toll Free at \alert<4>{1-866-768-8801} \alert<8>{(}not available in \alert<7>{416\alert<8>{/}647} area codes\alert<8>{)}.\\
\alert<1>{\textbf{C}} Fax your order form to \alert<4>{905-415-7538}.\\
\alert<1>{\textbf{D}} Return your completed order form with payment to: 
Markham Theatre, \alert<6>{171 Town Centre \alert<1>{Blvd.}, Markham, \alert<1>{ON}, \alert<1>{L3R 8G5}}.\\
\alert<1>{\textbf{E}} Online ticket sales are currently only available for Single Tickets beginning \alert<2>{September 13, 2005}.
\end{scriptsize}

\end{frame}

\subsection{Grapheme to Phoneme conversion}
\begin{frame}
\frametitle{\insertsubsection: \\ By dictionary and rules}

\begin{block}{Tokenize the text and look up the words in a
pronunciation dictionary. If not found, use rules}
\begin{itemize} [<+->]
\item Dictionary entries: \texttt{("dictionary" nil (d ih1 k sh ax n eh1 r iy0))}
\item Rules: \texttt{( LC [ alpha ] RC => beta )}
	\begin{itemize} [<+->]
	\item \texttt{( \# [ c h ] r => k )}  ``ch'' word initially in English
	\item \texttt{( \# [ c h ] => ch )}  ``ch'' word initially in English
	\item \texttt{( [ c ] => k )}   default rule for ``c''
	\end{itemize}
\end{itemize}
\end{block}

\uncover<6->{After all words have been converted, there is a second pass to catch changes at word boundaries and general effects of running speech}

\end{frame}

\subsection{Morphological decomposition}
\begin{frame}
\frametitle{\insertsubsection: Out-of-Vocabulary words}

\begin{block}{Compound words and other words not in the dictionary are common}
\begin{itemize} [<+->]
\item Compound words are common in many languages, eg, German, Dutch, Finnish, Turkish 
\item Compound word consist of lexical words that are connected with infixes, eg, \textit{-s-}
      and surrounded by affixes, eg, \textit{a-}, \textit{in-}, \textit{-ed}
\item Compounding or affixes can change the pronunciation and orthography
      of a word component, eg, \textit{Kunst} $\rightarrow$ \textit{K\"unst+ler} )
\item Parse complex words with a statistical weighted finite-state transducer (WFST)
      \citeSfive{BMobius1998}\nocite{BMobius1998}
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: German examples}

\begin{center}
\begin{Large}\emph{Unerfindlichkeitsunterstellung}\\ ``allegation of incomprehensibility''
\end{Large}
\end{center}

\begin{small}WFST states: \textbf{START PREFIX ROOT INFIX SUFFIX END}\end{small}

\begin{block}{German decompositions \citeSfive{BMobius1998}\nocite{BMobius1998}}
\begin{itemize} [<+->]
\item \emph{gener+ator} ``generator''
\item \emph{honor+ar} ``fee'' 
\item \emph{Schwind+sucht} ``consumption''
\item \emph{Arbeit+s+amt} ``employment agency''
\item \emph{Sonne+n+schein} ``sunshine''
\item \emph{Un+er+find+lich+keit+s+unter+stel+lung} ``allegation of incomprehensibility''
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Decomposition}

\includegraphics[width=0.95\textwidth]{Pictures/WordFormingAffixesMoebius.jpg}

Use a dictionary and include a morphological compound list with pronunciations.
\citeSfive{BMobius1998}\nocite{BMobius1998}

\end{frame}

\subsection{Lexical stress and sentence accent}
\begin{frame}
\frametitle{\insertsubsection: Prominence}

\begin{block}{In every utterance, one or more words are more prominent than others. They are:}
\begin{itemize} [<+->]
\item Accented, ie, carry a pitch movement
\item Longer
\item Louder
\item Less reduced
\end{itemize}
\end{block}

\begin{block}{Prominence is determined by}
\begin{itemize} [<+->]
\item Word type, function words are almost never prominent
\item Word frequency, rare words are prominent more often
\item New information is prominent, given is not
\item Not too many prominent words in a row
\end{itemize}
\end{block}

\uncover<9->{There are rules for assigning prominence, but they need good POS tagging. 
Just accenting every content words works too}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Syllable stress}

\begin{block}{In every word, one or more syllables are more prominent than others. They are:}
\begin{itemize} [<+->]
\item Longer
\item Louder
\item Less reduced
\end{itemize}
\end{block}

\begin{block}{Syllable stress is determined by}
\begin{itemize} [<+->]
\item The lexicon or language (lexical/fixed stress positions)
\item Syllable weight, ``heavy'' syllable are stressed
\item No stressed syllables in a row
\item Informative syllables are stressed
\end{itemize}
\end{block}

\uncover<8->{Mostly, you can get away with either the lexicon, or fixed positions.
 Syllable stress shifts in compound words. Morphological decomposition gives rules 
 for these shifts}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Phrase boundaries}

\begin{block}{Intonation covers utterances of a few words at a time (around 5-7). 
      Breaking up sentences at acceptable places is difficult}
\begin{itemize} [<+->]
\item Use punctuation
\item Guess boundaries on POS tags (HMM style)
\item Do a partial syntactic parse and use phrases 
\end{itemize}
\end{block}

\uncover<4->{In general, it is difficult to go beyond punctuation and some simple
heuristics without syntactic parsing}

\end{frame}

\subsection{Duration}
\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{Phoneme duration is determined by:}
\begin{itemize} [<+->]
\item Phoneme identity
\item Surrounding phonemes
\item Sentence accent/prominence
\item Syllable stress
\item Syllable length and position (Onset, Coda)
\item Word length
\item Phrase/sentence boundary position
\item \dots
\end{itemize}
\end{block}

\uncover<9->{These factors are used to construct statistical models from 
annotated speech corpora.
Golden standard is Correlation and Regression Trees (CART). 
But many other statistical methods are used}

\end{frame}

\subsection{Intonation}
\begin{frame}
\frametitle{\insertsubsection}

\begin{center}
\movie [externalviewer]{\includegraphics[width=0.45\textwidth]{Pictures/ToDIvoorb}}
{Sounds/ToDI052.wav}
\end{center}

\begin{block}{With the durations known, the pitch contour can be calculated}
\begin{itemize} [<+->]
\item Speaker and style determine the pitch range
\item Give each accent a pitch movement shape and size
\item Assign each vowel its target $F_0$ value
\item Interpolate the values into a valid contour
\item Assign each phoneme it's $F_0$ values
\end{itemize}
\end{block}

\end{frame}

\subsection{Acoustic realization, PSOLA, MBROLA}
\begin{frame}
\frametitle{\insertsubsection}

\movie [externalviewer]{\includegraphics[width=\textwidth]{Pictures/MBEPSOLA}}
{Sounds/MBROLAchante.wav}

\begin{block}{Multi Band Excitation (Time Domain) Pitch Synchronous Overlap Add 
\nocite{MBROLA}\citeSfive{MBROLA}}
\begin{itemize} [<+->]
\item Mark all pitch periods (blue pulses in \emph{Praat})
\item Fixed periods for voiceless speech
\item Window speech around each mark
\item To lengthen/shorten a sound, reduplicate/delete periods
\item To increase/decrease $F_0$, shorten/lengthen times between periods
\item Synthesize sound by summing windowed periods at their correct time position
\end{itemize}
\end{block}

\end{frame}

\subsection{Nextens}
\begin{frame}
\frametitle{\insertsubsection}

\begin{center}
`Nederlandse Extensie voor Tekst naar Spraak'
or `Dutch Extension for Text to Speech'
\textcolor{blue}{\movie [externalviewer]{example}{Sounds/Nextens_speech.wav}}
\end{center}	     

\begin{block}{Nextens runs on top of Festival 
\multicite{Nextens2003,Festvox}}
\begin{itemize} [<+->]
\item New Dutch voices in Festival
\item Nintens GUI (io, commandline in Festival)
\item Available for non-commercial use (\emph{not} Open Source)
\item Developed at the Radboud University and the University of Tilburg (Joop Kerkhof, Erwin Marsi, and others)
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Annotation interface}

\begin{center}
\includegraphics[height=0.80\textheight]{Pictures/Nextens_annotation}
\end{center}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: $F_0$ interface}

\begin{center}
\includegraphics[width=0.90\textwidth]{Pictures/Nextens_f0}
\end{center}


\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Parameters interface}

\begin{center}
\includegraphics[height=0.80\textheight]{Pictures/Nextens_parameters}
\end{center}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Commandline interface}

\begin{center}
\includegraphics[height=0.80\textheight]{Pictures/Nextens_commandline}
\end{center}

\end{frame}

\subsection{Assignment}
\begin{frame}
\frametitle{\insertsubsection: Week 5 Using Nextens}

\begin{block}{See CD or Nextens web-site for information \nocite{Nextens2003}\citeSfive{Nextens2003}}
\begin{itemize} [<+->]
\item Install Nextens (see info)
\item Start Nintens
\item Under file menu, select `add sentence'
\item Generate a sentence
\item Inspect tabs, especially the ToDI tab
\end{itemize}
\end{block}

\end{frame}

\subsection{Bibliography}

\begin{frame}[allowframebreaks]
\frametitle{Further Reading}
\nociteSfive{BoersmaWeenink}
\nocite{BoersmaWeenink}
\nociteSfive{Boersma2001}
\nocite{Boersma2001}
\nociteSfive{HLTsurvey}
\nocite{HLTsurvey}
\bibliographystyleSfive{plainnat}
\tiny{
\bibliographySfive{SpraakherkenningEnSynthese}
}
\end{frame}

\appendix
\begin{frame}
\frametitle{Appendix A}

\end{frame}


% Insert License text
\insertgnugpl


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%\part{Measuring Speech}
\lecture{More about TTS and evaluation}{week06}
\renewcommand{\multicite}[1]{\nocite{#1}\citeSfive{#1}}
\renewcommand{\copyrightauthor}[0]{R.J.J.H. van Son}
\renewcommand{\licensenumber}[0]{License6}

\section{More about TTS and evaluation}

\subsection{Introduction}
\begin{frame}
\frametitle{\insertsubsection}

\begin{center}
R\&D in general purpose TTS systems is almost completely directed towards concatenative synthesis.
Special purpose systems for speech research, visual speech generation, and small footprint
applications still use Articulatory Synthesis
or rule based systems (eg, \tiny{\multicite{BoersmaWeenink,Moh05}})
\end{center}

\begin{block}{Developping concatenative TTS systems \tiny{\multicite{BL03b}}}
\begin{itemize} [<+->]
\item A strength is that it produces 
      natural sounding speech from recorded human speech
\item A weakness is that its quality
      totally depends on the qualities of the original recorded
      voice
\item Evaluation must separate voice characteristics and system
      characteristics
\end{itemize}
\end{block}

\end{frame}

\subsection{Recording a voice}
\begin{frame}
\frametitle{\insertsubsection: Speaker selection}

\begin{block}{Characteristics of a ``good speaker''\texttrademark 
\tiny{\multicite{BL03b}}}
\begin{itemize} [<+->]
\item Availability and willingness (long recording times)
\item Clear voice
\item Consistent speaking (variability is bad)
\item Will form the personality of the synthesis
\item Will sign over all rights to you:
	\begin{itemize} [<+-| alert@+>]
  	\item free for any use
	\item free to distribute to anyone but cannot be used for commercial purposes without
		further contract.
	\item research use only (does this allow public demos?)
	\item fully proprietary
	\end{itemize}
\item Note: The style of speaking determines the style of the synthesis
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Diphone database}

\begin{block}{Diphone lists ($\approx$1600 diphones)}
\begin{itemize} [<+->]
\item Choose phoneset
\item Construct diphone list in nonsense words, eg 
      [\emph{pau t aa \underline{b aa b} aa pau}]
\item Add special or foreign phonemes and clusters
\item Synthesize prompts as sounds for presentation
	\begin{itemize} [<+->]
	\item Text is ambiguous
	\item Consistent prosody
	\item Consistent pronunciation
	\end{itemize}	
\item Record words under the \alert<8>{best} of circumstances
\item Label and align phones (automatically)
\item Extract pitch marks (electroglottogram)
\item Build parameter files, eg, \tiny{\multicite{MBROLA}}
\item Build and test database itself
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Unit database}

\begin{block}{Unit selection TTS is based on general speech, 
              prosodic variation is good}
\begin{itemize} [<+->]
\item Size: phone, diphone, demi-syllable 
\item Type: phone, phone+stress, phone+word
\item Concatenate units ``in context'', eg, 
      stressed vs unstressed or word-initial vs -final phones
\item Select units that fit requirements best
\item Could use general speech corpus, but this generally 
      lacks coverage and consistency
\item Best to record a specially designed database
\end{itemize}
\end{block}

\end{frame}


\begin{frame}
\frametitle{\insertsubsection: Constructing a unit database}

\begin{block}{Use a general language corpus with utterances
that cover all relevant phenomena (Festival \tiny{\multicite{BL03b}})}
\begin{itemize} [<+->]
\item Design the prompts (greedy algorithms)
\item Record the prompts (\alert<2>{best} of circumstances)
\item Autolabel the prompts
\item Build utterance structures for recorded utterances
\item Extract pitchmarks and build LPC coefficients (electroglottogram)
\item Build a unit based synthesizer from the utterances
\item Test and tune
\end{itemize}
\end{block}

\end{frame}

\subsection{Processing a voice}
\begin{frame}
\frametitle{\insertsubsection: Autolabeling}

\begin{block}{Use the prompts to label and segment}
\begin{itemize} [<+->]
\item The prompts have known labeling and segmentation
\item Align the prompts to the recordings, eg, dynamic time 
      warping or forced ASR alignment
\item Can even be done when synthesized prompts are from a TTS
      of a different language
\item If segmentation goes wrong, verify by hand
\item Determine syllable stress and sentence accent
      from prompt specification
\item Feed labels into utterance structures etc.
\end{itemize}
\end{block}

\end{frame}

\subsection{Speech characteristics}
\begin{frame}
\frametitle{\insertsubsection: Expressive speech}

\begin{block}{Consistent pronunciation means little expression.
Add different styles (professional speaker/actor) \tiny{\multicite{BNS02}}}
\begin{itemize} [<+->]
\item Use appropriate style for task, eg, news, weather, stories
\item Message has more effect in correct emotional state
\item Very important when working for children
\item Basic states: anger, happiness, sadness and neutral
\item Prosodic models must be specific for each emotional state
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Changing speaker characteristics
(not yet feasible)}

\begin{block}{Each different ``voice'' needs a separate speaker. Only
what has been recorded can be spoken}
\begin{itemize} [<+->]
\item Change voice characteristics to create a different speaker, 
      eg, man to woman to child (Praat allows this as a demo)
\item Change voice to a different language variant or style
\item Add new (level of) ``expressiveness''
\item Emotional state can be manipulated to some degree in prosody alone
\item Techniques from rule based synthesis are needed to change complex traits,
      like stress and emotional states, reliably
\end{itemize}
\end{block}

\end{frame}

\subsection{Evaluation}
\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{Evaluation is the mother of progress}
\begin{itemize} [<+->]
\item Evaluate modules separately
\item Construct rigorous and uniform evaluation procedures and criteria
\item Separate diagnostic tests from full system evaluations
\item Compare different system
\item Standardize external input: Voice, texts, use
\item TTS is evaluated by listeners
\begin{itemize} [<+->]
\item Self selected volunteers (eg, internet)
\item Paid naive listeners (eg, students)
\item Paid target groups (eg, office workers, K12 children)
\item TTS developers (Tit-for-Tat evaluation)
\item External Experts
\end{itemize}

\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Criteria}

\begin{block}{What can be evaluated (full system evaluation)}
\begin{itemize} [<+->]
\item Intelligibility at phoneme and word level
\item Naturalness and pleasantness
\item Intonation and prosody
\item Stress positions and breaks
\item Long text rendering (eg, intonation variation)
\item Task appropriateness, ie, correct style
\item Voice and style selection in multi-speaker story telling
     (when feasible)
\end{itemize}
\end{block}

\end{frame}

\subsection{Blizzard challenge 2005}
\begin{frame}
\frametitle{\insertsubsection}

\begin{center}
Aim: Find better synthesis techniques by comparing systems on the same data
\end{center}

\begin{block}{Evaluating corpus-based speech synthesis on common datasets \tiny{\multicite{BT05,Ben05}}}
\begin{itemize} [<+->]
\item Effort to start international comparative evaluation of TTS systems \nocite{Jenolan98}\nociteSfive{Jenolan98}
\item Which approaches work, which don't
\item Distribute common unit database, generate full TTS system within two weeks
\item Evaluate common texts, 250 sentences from 5 genres
\item Prevent ``cheating'' where needed
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Speech}

\begin{block}{Common speech databases}
\begin{itemize} [<+->]
\item CMU ARCTIC databases: 2 old + 2 new voices
\item 1200 phonetically balanced sentences (5-15 words)
\item Project Gutenberg novels (prose style) \tiny{\multicite{Gutenberg}}
\item All words in CMUDICT \tiny{\multicite{Festvox}}
\item Eg, \textit{They were three hundred yards apart.}
\item Automatically segmented and labeled \tiny{\multicite{Festvox}}
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Evaluation}

\begin{block}{5 text genres, 50 sentences each}
\begin{itemize} [<+->]
\item Novels, same stories as original sentences\\
	\textit{Joe Garland lives like a good fellow.}
\item News, standard press-wire\\
	\textit{The two countries agreed to resolve any conflict through \dots Interfax said.}
\item Conversation, human side of spoken dialog system\\
	\textit{Yeah, I guess it will and something downtown please.}
\item Phonetically confusable sentences\\
	\textit{Now we will say cold/colt again.}
\item Semantically unpredictable sentences (SUS)\\
	\textit{The unsure steaks overcame the zippy rudder.}
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Listeners}

\begin{block}{Listener groups (and number who completed all tests) \tiny{\multicite{Ben05}}}
\begin{itemize} [<+->]
\item Speech experts, each participant provided 10 local experts (50)
\item Volunteers over the web (60, unpaid)
\item US undergraduates (58, paid)
\item It proved to be difficult to get enough listeners ($\approx 100$)
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Test types}

\begin{block}{Test types}
Mean opinion scores on a five point scale for \textit{Novels}, \textit{News}, and \textit{Conversation}
and Word Error Rate for \textit{Phonetically Confusable} and \textit{Semantically Unpredictable Sentences}
 \tiny{\multicite{Ben05}}
\end{block}

\end{frame}


\subsection{Assignment}
\begin{frame}
\frametitle{\insertsubsection: Week 6}

\begin{block}{Locate speech synthesis sites on the web}
\begin{itemize} [<+->]
\item Listen to TTS examples
\item Locate sites where you can enter your own text
\item Select real sentences from the web, try out ``difficult'' ones
\item Compare them to Nextens and Festival
\item What goes well? What doesn't?
\item Describe the differences.
\end{itemize}
\end{block}

\end{frame}

\subsection{Bibliography}

\begin{frame}[allowframebreaks]
\frametitle{Further Reading}
\nociteSfive{BoersmaWeenink}
\nocite{BoersmaWeenink}
\nociteSfive{Boersma2001}
\nocite{Boersma2001}
\bibliographystyleSfive{plainnat}
\tiny{
\bibliographySfive{SpraakherkenningEnSynthese}
}
\end{frame}

\appendix
\begin{frame}
\frametitle{Appendix A}

\end{frame}

% Insert License text
\insertgnugpl


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%\part{Measuring Speech}
\lecture{Automatic Speech Recognition}{week07}
\renewcommand{\multicite}[1]{\nocite{#1}\citeSsix{#1}}
\renewcommand{\copyrightauthor}[0]{R.J.J.H. van Son}
\renewcommand{\licensenumber}[0]{License7}

\section{Automatic Speech Recognition}

\subsection{Introduction}
\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{Speech recognition in Human Machine interaction}
\begin{itemize} [<+->]
\item A full interaction requires human input
\item Input with speech is often faster and easier than with text or pointers
	\begin{itemize} [<+->]
	\item Over the phone
	\item With large or unlimited choice, eg, person and place names
	\item Free text, eg, dictation messages
	\item With hands occupied, eg, while driving
	\end{itemize}
\item Sometimes speech input is ineffective
	\begin{itemize} [<+->]
	\item In a noisy surrounding, eg, a train station
	\item With small menu based selections
	\item Large variation in speakers, eg, second language speakers
	\item Tasks that are difficult to describe verbally, eg, routing on a map
	\end{itemize}
\end{itemize}
\tiny{Many pictures (and their copyrights) are from }\tiny{\multicite{JurafskyMartin2000}}
\end{block}


\end{frame}

\subsection{Automatic Speech Recognition}
\begin{frame}
\frametitle{\insertsubsection}

\includegraphics[width=0.9\textwidth]{Pictures/fig07-01}

\begin{block}{ASR is a database retrieval problem}
\begin{itemize} [<+->]
\item A speech recognizer is a clever example database
\item The problem is: How to retrieve the most likely words from the acoustic signal
\item Break down into two problems: Get the most likely 
	\begin{itemize} [<+->]
	\item word candidates given the sound
	\item word sequence given the available word candidates
	\end{itemize}
\item Currently both problems are solved stochastically
\end{itemize}
\end{block}

\end{frame}

\subsection{Speech Input}
\begin{frame}
\frametitle{\insertsubsection: How to partition the ASR problem}

What is the most likely word sequence given the observed sound:
\begin{multline*}
\underset{Words}{argmax}\ P\left(Words | Observation\right)  = \\
\underset{Words}{argmax} \dfrac{P\left(Observation|Words\right)\cdot P\left(Words\right)}
				{P\left(Observation\right)}
\end{multline*}

\begin{block}{Split this into two separate tasks}
\begin{itemize} [<+->]
\item $P\left(Observation\right)$ is a normalization constant, independent of word recognition (ignore it)
\item $P\left(Observation|Words\right)$ is the acoustic $likelihood$ of the words
\item $P\left(Words\right)$ is the $prior$ of the word sequence (ie, the language model)
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: An overview of ASR}

\includegraphics[width=0.85\textwidth]{Pictures/fig07-02}

\begin{block}{Sound waveform to word sequence}
\begin{itemize} [<+->]
\item Encode the waveform into Spectral Features
\item Determine word likelyhoods $P\left(Sound|Words\right)$ for each word
\item Determine word sequence probability $P\left(Words\right)$ for each sequence
\end{itemize}
\end{block}

\end{frame}

\subsection{Language Prior}
\begin{frame}
\frametitle{\insertsubsection: $P\left(Words\right)$}

\begin{center}
\includegraphics[height=0.50\textheight]{Pictures/farewell}
\end{center}

\begin{block}{\textit{Farewell} Finite State example \\ 
  \begin{footnotesize}every arrow has a probability\end{footnotesize}}
\begin{itemize} [<+->]
\item The probability of observing an utterance
\item Example from \begin{tiny}\url{http://www.geocities.com/SoHo/Square/3472/nounphrase.html}\end{tiny}
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Word sequences}

Estimate $P\left(Words\right) = $
\[P\left(w_1, \dots, w_n\right) = \prod^n_{i=1} P\left(w_i|w_1 \dots w_{i-1}\right)\]

\begin{block}{Approximate $P\left(Words\right)$ by modeling 
 $P\left(w_i|w_1 \dots w_{i-1}\right) \approx$}
\begin{itemize} [<+->]
\item $P\left(w_i|State_\alpha\right)$: Finite State Grammar
\item $P\left(w_i|w_{i-n+1} \dots w_{i-1}\right)$: N-gram
\item $\sum_\alpha P\left(w_i|Tree_\alpha\left(w_1 \dots w_{i-1}\right)\right)\cdot 
P\left(Tree_\alpha\left(w_1 \dots w_{i-1}\right)\right)$: 
Context Free Grammar with (lexicalized) tree structures build from 
$\left(w_1 \dots w_{i-1}\right)$
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: N-grams}

\begin{block}{Collect \textit{word}, \textit{word-pair} and 
              \textit{word-triplet} 
              frequencies \begin{tiny}\multicite{Goodman2001}\end{tiny}}
\begin{itemize} [<+->]
\item Impossible to get frequencies of all possible bi/trigrams
\item Construct smoothed probability distributions 
\item Special "states" for sentence start and "end"
\item $P\left(Words\right) \approx P\left(w_i|w_{i-2}, w_{i-1}\right)$
\item Use interpolation or backoff, eg, $P\left(w_i|w_{i-2}, w_{i-1}\right) 
       \approx \alpha \cdot P\left(w_i|w_{i-1}\right)$
      if the tri-gram $\left(w_{i-2}, w_{i-1}, w_i\right)$ was not observed
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Data Oriented Parsing (CFG Example) \begin{tiny}\multicite{Sambur2003}\end{tiny}}

\begin{center}
\includegraphics[width=0.80\textwidth]{Pictures/ECML03}
\end{center}


\begin{block}{Subtree have occurrence and insertion probabilities}
\begin{itemize} [<+->]
\item Requires a treebank with frequencies
\item Correct normalization of probabilities
\item Computationally expensive, like all probabilistic CF parsers
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Grammar Perplexity}

\[Perplexity\left(\mathfrak{G}\right) = 2^{H\left(\mathfrak{G}\right)}\]
where
\begin{multline*}
H\left(\mathfrak{G}\right) =
\sum_{w_i} - P\left(w_i|w_1 \dots w_{i-1}\right) \cdot
\log_2 P\left(w_i|w_1 \dots w_{i-1}\right)
\end{multline*}

\begin{block}{For a tri-gram grammar this corresponds to:}
\begin{itemize} [<+->]
\item $P\left(w_i|w_{i-2}, w_{i-1}\right) = \dfrac{P\left(w_{i-2}, w_{i-1}, w_i\right)}
{P\left(w_{i-2}, w_{i-1}\right)}$
\item Perplexity corresponds to the difficulty of predicting the next word
\item A lower perplexity is better for ASR
\end{itemize}
\end{block}

\end{frame}

\subsection{Spectral analysis}
\begin{frame}
\frametitle{\insertsubsection: FFT, LPC, PLP, MFCC, filter-banks}
\begin{tabular}{c c}

\parbox{0.45\textwidth}{
\includegraphics[width=0.5\textwidth]{Pictures/FFTmelsmoothed}} 
&
\parbox{0.45\textwidth}{
\begin{itemize} [<+->]
\item Need a spectral representation
\item FFT: too noisy
\item LPC: wrong sensitivity
\item Resolution of the ear (Mel Freq, PLP, Filter banks)
\item Sound level in dB (PLP, Filter banks)
\item Spectral shape (MFCC)
\end{itemize}
}
\end{tabular} 

\end{frame}

\subsection{Hidden Markov Models}

\begin{frame}
\frametitle{\insertsubsection: Markov chains}

\includegraphics[width=0.9\textwidth]{Pictures/fig07-03} 

\begin{block}{Word models: simple phone state model for \textit{need}}
\begin{itemize} [<+->]
\item Each transition has a probability
\item \alert<2>{start} and \alert<2>{end} are special states
\item Each state \emph{or} each transition has associated sound observations with a distinct probability density function (PDF)
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Observation probabilities}

\includegraphics[width=0.9\textwidth]{Pictures/fig07-04} 

\begin{block}{Observed are sound "spectra" for time "frames"}
\begin{itemize} [<+->]
\item Observation sequences have a probability
\item Calculate this probability for each possible word
\item Probabilities of observation $O_i$ calculated from all possible underlying states
\item Chose word \emph{sequence} with the highest overall probability
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Pronunciation networks}

\includegraphics[width=0.9\textwidth]{Pictures/fig07-05} 

\begin{block}{Construct phone state models for each word in the dictionary}
\begin{itemize} [<+->]
\item The possible pronunciations for each word have to be encoded in the dictionary
\item The transition probabilities are "trained" from the frequency of occurrence of the pronunciation in the speech corpus
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Phone networks}

\includegraphics[width=0.9\textwidth]{Pictures/HMMphoneNetwork} 

\begin{block}{Phone models are concatenated into utterance networks}
\begin{itemize} [<+->]
\item Each word model is itself a Markov finite state network of phone models
\item Phones and word are connected through the \emph{start} and \emph{end} states (not shown)
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Context Sensitive Phone latices}

\begin{block}{Phone models are constructed of subphone states in context}
\begin{itemize} [<+->]
\item Each phone model is itself a Markov finite state network
\item For each phoneme context separate phone models are constructed
\item Each sub-phone context sensitive state can have it's own observation PDF
\item For the sake of reducing training, the observation PDF's of different states are \textit{tied} (ie, made identical)
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Context Sensitive Phone latices \begin{tiny}\multicite{CSLUtoolkit}\end{tiny}}

\includegraphics[width=0.95\textwidth]{Pictures/CSLUcontext_model} 

\end{frame}


\subsection{Evaluation}
\begin{frame}
\frametitle{\insertsubsection: NIST, DARPA, hubs and spokes}

\begin{block}{The National Institute of Standards (NIST) and the DARPA program
organize evaluation "contests" for ASR systems}
\begin{itemize} [<+->]
\item Tests contain mandatory core components \alert<1>{\textit{hubs}}
\item Tests contain optional specialized components \alert<2>{\textit{spokes}}
\item Tests evolve to include not only Speech-to-Text but also who spoke
when, speaker localization etc.
\item Includes varying speech material and conditions
\item Contestants get training materials from the organization
\item After time for training, contestants receive test speech and have to return the results
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: NIST results \tiny{\multicite{Pallett2003}}}

\includegraphics[width=0.9\textwidth]{Pictures/historybench-July04-v0} 

\begin{block}{}
\begin{itemize} [<+->]
\item WER (vertical) go down over time
\item More complex tasks introduced over time
\end{itemize}
\end{block}

\end{frame}

\subsection{Assignment}
\begin{frame}
\frametitle{\insertsubsection: Week 7}

\begin{block}{Use DTW to match speech samples}
\begin{itemize} [<+->]
\item Record or collect different realizations (eg, normal/fast) of the same utterances
\item Use praat (Formant \& LPC -, to MFCC...) to create \textit{Mel Frequency based Cepstral Coefficients} 
\item Generate a dynamic time warp (To DTW..., match start and end and use \textit{no slope restrictions})
\item Paint it
\item Use the same technique to select a spoken number from a sequence of numbers. Note that there can be problems from matching the other numbers
\end{itemize}
\end{block}

\end{frame}

\subsection{Bibliography}

\begin{frame}[allowframebreaks]
\frametitle{Further Reading}
See chapter 7.1, 7.2, 7.5 \tiny{\multicite{JurafskyMartin2000}}

\nociteSsix{BoersmaWeenink}
\nocite{BoersmaWeenink}
\nociteSsix{Boersma2001}
\nocite{Boersma2001}
\nociteSsix{JurafskyMartin2000}
\nocite{JurafskyMartin2000}
\nociteSsix{ISIPASR}
\nocite{ISIPASR}
\nociteSsix{CSLUtoolkit}
\nocite{CSLUtoolkit}
\bibliographystyleSsix{plainnat}
\tiny{
\bibliographySsix{SpraakherkenningEnSynthese}
}
\end{frame}

\appendix
\begin{frame}
\frametitle{Appendix A}

\end{frame}


% Insert License text
\insertgnugpl


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%\part{Measuring Speech}
\lecture{More about ASR}{week08}
\renewcommand{\multicite}[1]{\nocite{#1}\citeSsix{#1}}
\renewcommand{\copyrightauthor}[0]{R.J.J.H. van Son}
\renewcommand{\licensenumber}[0]{License8}

\section{More about ASR}

\subsection{Introduction}
\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{Two technologies are needed to make the HMM framework practical}
\begin{itemize} [<+->]
\item Decoder technology to find the $\underset{Words}{argmax}\ P(Observation|Words) \cdot P(Words)$
\item Determining the stochastic parameters of the HMM state automaton, ie, training
\end{itemize}
\end{block}

\tiny{Many pictures (and their copyrights) are from }\tiny{\multicite{JurafskyMartin2000}}

\end{frame}

\subsection{Dynamic programming}
\begin{frame}
\frametitle{\insertsubsection}

\begin{center}
\includegraphics[height=0.45\textheight]{Pictures/fig05-04}
\end{center} 

\begin{block}{Look for best alignment: Minimum edit distance}
\begin{itemize} [<+->]
\item Delete
\item Insert
\item Substitute
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{center}
\includegraphics[height=0.4\textheight]{Pictures/fig05-05}
\end{center} 

\begin{block}{Fill a matrix with cumulative edit distances, $distance[i, j] =$ min of}
\begin{itemize} [<+->]
\item $distance[i-1, j] + \text{insert-cost}(target_i)$
\item $distance[i-1, j-1] + \text{substitution-cost}(source_j, target_i)$
\item $distance[i, j-1] + \text{deletion-cost}(source_j)$
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{center}
\includegraphics[height=0.35\textheight]{Pictures/fig05-06}
\end{center} 

\begin{block}{Trace back the choices of the minimal distance (bold numbers)}
\begin{itemize} [<+->]
\item This finds the globally minimal cost path
\item Full search unwieldy for large and complex matrices 
\item In general, searches are pruned to exclude paths that deviate far from the diagonal: Beam search
\end{itemize}
\end{block}

\end{frame}

\subsection{Viterbi algorithm}
\begin{frame}
\frametitle{\insertsubsection}

\begin{center}
\includegraphics[width=0.75\textwidth]{Pictures/fig07-05}
\end{center} 

\begin{block}{Simplified pronunciation networks \tiny{\multicite{JurafskyMartin2000}}}
\begin{itemize} [<+->]
\item Each word is modeled as a Finite State Machine
\item Individual phoneme HMMs are trained from a corpus that does not contain all the words
\item A pronunciation dictionary contains all word models
\item Transition probabilities are "trained" from a transcribed speech corpus
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection}


\begin{tabular}{c c}

\parbox{0.45\textwidth}{
\begin{center}
\includegraphics[width=0.45\textwidth]{Pictures/fig07-06}
\end{center} 

Viterbi algorithm result \textit{``for I need a''} \tiny{\multicite{JurafskyMartin2000}}
}
&
\parbox{0.45\textwidth}{
\begin{itemize} [<+->]
\item Whole sequence on \textbf{X} axis
\item All word models on the other axis
\item Switch to (any) new word after reaching the end of the current word
\item Word switching cost based on the language model
\end{itemize}
}
\end{tabular}
\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{center}
\includegraphics[width=0.95\textwidth]{Pictures/fig07-07} 
\end{center}

\begin{block}{Bigram probabilities \tiny{\multicite{JurafskyMartin2000}}}
\begin{itemize} [<+->]
\item Word switching in Viterbi searches uses probabilities
\item Switch to a new word with bigram probability cost
\item Does not work with trigram probabilities
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{center}
\includegraphics[height=0.40\textheight]{Pictures/fig07-08}
\end{center} 

\begin{block}{Single pronunciation automaton for \textit{I}, \textit{need}, \textit{on}, and \textit{the}
 \tiny{\multicite{JurafskyMartin2000}}}
\begin{itemize} [<+->]
\item Bigram probabilities connect the word models
\item Merge \textbf{start} and \textbf{end} states of connected words
\item Need for \textit{pruning} is apparent
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{center}
\includegraphics[height=0.50\textheight]{Pictures/fig07-09}
\end{center} 

\begin{block}{Extended version of the edit distance \tiny{\multicite{JurafskyMartin2000}}}
\begin{itemize} [<+->]
\item $a[s,s^\prime] = P(s \rightarrow s^\prime)$
\item $b_{s^\prime}(o_t) = P(o_t | s^\prime)$
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{center}
\includegraphics[height=0.50\textheight]{Pictures/fig07-10}
\end{center} 

\begin{block}{Individual state columns in Viterbi algorithm \tiny{\multicite{JurafskyMartin2000}}}
\begin{itemize} [<+->]
\item The actual entries for the Automaton
\item Note the problems for a 20,000 word dictionary
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Subphones revisited \tiny{\multicite{JurafskyMartin2000}}}

\begin{center}
\includegraphics[width=0.95\textwidth]{Pictures/fig07-11}
\end{center} 

\begin{block}{Use structured, context sensitive phone units}
\begin{itemize} [<+->]
\item Single phone units perform bad due to coarticulation
\item \textit{Begin} differs from \textit{End} (eg, /\textipa{d}/)
\item 60 context dependent triphones $\Rightarrow 60^3 = 216000$ models
\item Cluster contexts,eg, on manner and place of articulation
\end{itemize}
\end{block}

\end{frame}

\subsection{Other approaches to decoding}
\begin{frame}
\frametitle{\insertsubsection: Introduction}

\begin{block}{The standard HMM model has limitations}
\begin{itemize} [<+->]
\item Viterbi decoder penalizes multiple pronunciations
\item Viterbi decoder does not work for anything more complex than bigram
\item It is not possible to include other linguistic knowledge
	\begin{itemize} [<+->]
	\item Phoneme duration (HMM have a Poison distribution)
	\item Intonation
	\item Semantics
	\item Speaker identification
	\item Expressive speech tags
	\item Task related knowledge
	\end{itemize}
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{center}
\includegraphics[width=0.85\textwidth]{Pictures/fig07-12}
\end{center} 

\begin{block}{Two stage N-best decoding \tiny{\multicite{JurafskyMartin2000}}}
\begin{itemize} [<+->]
\item Keep N-best utterance list or word lattice
\item Rescore the probabilities with the extra knowledge
	\begin{itemize} [<+->]
	\item A trigram or higher grammar
	\item Phoneme duration probability \tiny{Chapt 7 \multicite{WangXue1997}}
	\item Parallel Intonation and Accent detector (HMM)
            \tiny{example without N-best \multicite{Taylor1998}}
	\item Include semantic or task related knowledge
	\item Multiple speakers and expressive speech tags
	\end{itemize}
\item Look up best path through rescored word lattice
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: $A^*$}

\begin{center}
\includegraphics[height=0.45\textheight]{Pictures/fig07-13} 
\end{center}

\begin{block}{Stack, or $A^*$, decoding \tiny{\multicite{JurafskyMartin2000}}}
\begin{itemize} [<+->]
\item Viterbi uses \alert<1>{best} path upto position $t$ to get to $t+1$
\item $A^*$ uses \alert<2>{complete} forward algorithm (exact likelihoods)
\item $A^*$ searches potential utterances \alert<3>{best-first}
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: $A^*$}

\begin{center}
\includegraphics[width=0.85\textwidth]{Pictures/fig07-14} 
\end{center}

\begin{block}{Stack decoding \tiny{\multicite{JurafskyMartin2000}}}
\begin{itemize} [<+->]
\item At each point, the $A^*$ looks for the most likely next word
\item Acoustic likelihood is part the criterium
\item Use the forward probability of preceding words
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: $A^*$}

\begin{center}
\alert<1>{\includegraphics[width=0.25\textwidth]{Pictures/fig07-15}}
\alert<2>{\includegraphics[width=0.30\textwidth]{Pictures/fig07-16}}
\alert<3>{\includegraphics[width=0.40\textwidth]{Pictures/fig07-17}}
\end{center} 

\begin{block}{\textit{If music be the food of love}
\tiny{\multicite{JurafskyMartin2000}}}
\begin{itemize} [<+->]
\item \alert<1>{\textit{``Start Alice''}} has highest score: \alert<1>{40}
\item \alert<2>{\textit{``Start if''}} has highest score: \alert<2>{30}
\item \alert<3>{\textit{``Start if music''}} has highest score: \alert<3>{32}
\end{itemize}
\end{block}

\end{frame}


\begin{frame}
\frametitle{\insertsubsection: $A^*$}
\begin{block}{Remarks}
\begin{itemize} [<+->]
\item Use fast match heuristics for selecting next words
\item Longer utterances have lower probabilities, score should correct for this
\item $A^*$ evaluation function: $f^*(p) = g(p) + h^*(p)$
\item $g(partial\ path) = P(O|Words)\cdot P(Words)$, ie, the likelihood until now
\item $h^*(p)$ something that correlates with number of words in the rest of the utterance
\item Defining a good $h^*(p)$ is an interesting (unsolved) problem
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: $A^*$ fast match}

\begin{center}
\includegraphics[height=0.4\textheight]{Pictures/fig07-18}
\end{center}

\begin{block}{A tree structured lexicon from SPHINX \tiny{\multicite{Gou}\multicite{JurafskyMartin2000}}}
\begin{itemize} [<+->]
\item Need to get forward probabilities of potential continuations \alert<1>{\textit{fast}}
\item Tree lexicon shares forward probabilities between words
\item Allows early pruning of search trees
\end{itemize}
\end{block}

\end{frame}

\subsection{Training acoustic models}
\begin{frame}
\frametitle{\insertsubsection: Introduction}
\begin{block}{Determine $P(Observation|Words)$, ie, the transition probability between phone states $a_{ij}$ and the acoustic likelihood of the speech vectors $b_j(o_k)$}
\begin{itemize} [<+->]
\item Large, ``transcribed'' speech corpus (on text level)
\item Coverage of speakers and language types
\item Recorded under the same conditions as intended use, eg, over the phone or in a driving car
\item Use the same microphone etc.
\item Using a simulated task (Wizard of Oz or Green curtain) to elicit the same kind of speech
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{center}
\includegraphics[height=0.4\textheight]{Pictures/fig07-04}
\end{center}

\begin{block}{If all states were known \tiny{\multicite{JurafskyMartin2000}}}
\begin{itemize} [<+->]
\item $a_{ij} = \dfrac{\# S_{ij}}{\# S_{i*}}$ (count transitions and states)
\item $b_{i}(O_k) = \dfrac{\# (O_k \& S_i)}{\# S_{i}}$ (for discrete $O_k$)
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{center}
\includegraphics[height=0.35\textheight]{Pictures/SphinxmanAcoustHMM}
\end{center}

\begin{block}{If observations are continuous vectors \tiny{\multicite{ManSphinx}}}
\begin{itemize} [<+->]
\item $b_{i}(O_t) \Rightarrow N\{\hat{\mu_{i}}, \hat{\Sigma_{i}}\}$
\item $\hat{\mu}_{i} = \dfrac{1}{T_{i}}\sum^{T_i}_{t=1} O_t$
\item $\hat{\Sigma}_{i} = \dfrac{1}{T_{i}} \sum^{T_i}_{t=1}[(O_t - \hat{\mu_{i}})^\prime(O_t - \hat{\mu_{i}})]$
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{center}
\includegraphics[height=0.30\textheight]{Pictures/figD-03}
\end{center}

\begin{block}{States have to be estimated. Use an iterative procedure \tiny{App D \multicite{JurafskyMartin2000}}}
\begin{itemize} [<+->]
\item Run the recognizer on the corpus with the known words
\item Calculate $\hat{a}_{ij}=\frac{expected\ \# S_i \rightarrow S_j}{expected\ \# S_i \rightarrow S_{*}}$
\item Calculate $\hat{b}_{j}(v_k)=\frac{expected\ \# S_j\ observing\ v_k}{expected\ \# S_j}$
\item Update all values and start again
\end{itemize}
\end{block}

\end{frame}

\subsection{FLOSS resources}
\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{Free and Open Source ASR systems}
\begin{itemize} [<+->]
\item SPHINX (CMU) \begin{tiny}\multicite{Gou} \multicite{Singh}\end{tiny}
\item CMU Statistical Language Modeling Toolkit \begin{tiny}\multicite{Rosenfeld}\end{tiny}
\item CMU Pronouncing Dictionary \begin{tiny}\multicite{LenzoPD}\end{tiny}
\item Internet-Accessible Speech Recognition Technology project (ISIP, Mississippi State University) \begin{tiny}\multicite{ISIPASR}\end{tiny}
\item Open Mind Speech \begin{tiny}\multicite{OpenMind}\end{tiny}
\end{itemize}
\end{block}

\end{frame}

\subsection{Assignment}
\begin{frame}
\frametitle{\insertsubsection: Week 8}

\begin{scriptsize}
\begin{block}{Construct your own language model}
\begin{itemize} [<+->]
\item Download texts from the internet, eg, \multicite{Gutenberg}
\item Use a single author or a single genre
\item Use \texttt{--help} to see instructions of the programs
\item Construct unigram and bigram word tables with \textit{Ngramcount.pl}
    \url{http://www.fon.hum.uva.nl/rob/Courses/Taaltechnologien/Ngramcount.pl}
%	\begin{itemize} [<+->]
	\item perl Ngramcount.pl 1 $<$filename1$>$ $<$filename2$>$ ... $>$ unigramtable.txt 
	\item perl Ngramcount.pl 2 $<$filename1$>$ $<$filename2$>$ ... $>$ bigramtable.txt
	\item Inspect the table files. What are the most frequent words and bigrams?
%	\end{itemize}
\item Calculate the probabilities of sentences with \textit{ngramprobability.pl}
    \url{http://www.fon.hum.uva.nl/rob/Courses/Taaltechnologien/ngramprobability.pl}
%	\begin{itemize} [<+->]
	\item perl ngramprobability.pl --count 5 --verbose bigramtable.txt "$<$sentence$>$"
	\item Enter some sentences and inspect the resulting probabilities
	\item Experiment with the \texttt{--count} option. Try \texttt{--count -1} on a sentence that contains unknown word combinations
%	\end{itemize}
\end{itemize}
\end{block}
\end{scriptsize}
\end{frame}

\subsection{Bibliography}

\begin{frame}[allowframebreaks]
\frametitle{Further Reading}
\nociteSsix{BoersmaWeenink}
\nocite{BoersmaWeenink}
\nociteSsix{Boersma2001}
\nocite{Boersma2001}
\bibliographystyleSsix{plainnat}
\tiny{
\bibliographySsix{SpraakherkenningEnSynthese}
}
\end{frame}

\appendix
\begin{frame}
\frametitle{Appendix A}

\end{frame}


% Insert License text
\insertgnugpl


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%\part{Measuring Speech}
\lecture{Dialog systems}{week09}
\renewcommand{\multicite}[1]{\nocite{#1}\citeSseven{#1}}
\renewcommand{\copyrightauthor}[0]{R.J.J.H. van Son}
\renewcommand{\licensenumber}[0]{License9}

\section{Dialog systems}

\subsection{Introduction}
\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{Speech recognition and synthesis are most useful if combined into a full Human-Machine dialog system}
\begin{itemize} [<+->]
\item Human conversations are extremely efficient and effective interactions
\item Spoken dialogs are not like a command-line Question-Answer query session
\item Conversations include ``control'' signals at \textit{low} (pre-verbal) and 
	\textit{high} levels 
\item Humans speak in \alert<4>{\textit<5->{turns}}
\item In simple automated systems, interactions must be restricted and well structured
\end{itemize}
\end{block}

\tiny{Many pictures (and their copyrights) are from }\tiny{\multicite{JurafskyMartin2000}}
\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{In conversations, timing is everything}
\begin{itemize} [<+->]
\item Human dialogs are composed of game-like \alert<1>{\textit<2->{moves}}
\item \alert<2>{\textit<3->{Turn}} distribution is crucial for effective 
	Human-Machine interactions
    \begin{itemize} [<+->]
    \item \alert<3>{\textit<4->{who}} speaks next
    \item \alert<4>{\textit<5->{when}} should the next speaker start
    \end{itemize}
\item Central to human conversations is \alert<5>{\textit<6->{projection}}
\item \alert<6>{\textit<7->{Projection}} is the ability to predict the
    \begin{itemize} [<+->]
    \item \alert<7>{\textit<8->{timing}} of turns
    \item \alert<8>{\textit<9->{type}} of upcoming moves
    \end{itemize}
\end{itemize}
\end{block}

\end{frame}

\subsection{Turns}
\begin{frame}
\frametitle{\insertsubsection}
\begin{block}{What defines a turn?}
\begin{itemize} [<+->]
\item A \alert<1>{\textit<2->{single}} move in the conversation ``game''
\item Ends with the \alert<2>{\textit<3->{end}} of the last utterance
\item Utterance \alert<3>{\textit<4->{completes}} a move
\item Does \alert<4>{\textit<5->{not}} end in a level tone
\item Does not end in a \alert<5>{\textit<6->{filled}} pause (eg, ``uuhh'')
\item Can be followed by a \alert<6>{\textit<7->{silent pause}}
\end{itemize}
\end{block}
The end of a turn is a \alert<7>{\textit<1-6,8->{TRP}}, a \textit<7>{Transition Relevance Place}. 

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: TRPs}

\begin{block}{Turns and Turn taking. At each TRP of each turn:}
\begin{itemize} [<+->]
\item If during this turn the current speaker has selected \alert<1>{\textit<2->{A}} as the next speaker then \alert<1>{\textit<2->{A}} must speak next
\item If the current speaker does not select the next speaker, \alert<2>{\textit<3->{any}} other speaker may take the next turn
\item If no one else takes the next turn, the \alert<3>{\textit<4->{current}} speaker may take the next turn
\end{itemize}
\end{block}

\end{frame}

\subsection{Speech acts}
\begin{frame}
\frametitle{\insertsubsection}

Conversational \textit{moves} are build from \textit{speech acts}

\begin{block}{Basic speech acts}
\begin{itemize} [<+->]
\item \alert<1>{\textbf<2->{Assertives}} \\
 \textit<2->{suggesting, putting forward, swearing, boasting, concluding}
\item \alert<2>{\textbf<3->{Directives}} \\
 \textit<3->{asking, ordering, requesting, inviting, advising, begging}
\item \alert<3>{\textbf<4->{Commissives}} \\
 \textit<4->{promissing, planning, vowing, betting, opposing}
\item \alert<4>{\textbf<5->{Expressives}} \\
 \textit<5->{thanking, apologizing, welcoming, deploring}
\item \alert<5>{\textbf<6->{Declarations}} \textit<6->{changing the world by speech, eg,\\ 
 ``I resign'', ``You're fired''}
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{Basic control tasks, handle conversation flow}
\begin{itemize} [<+->]
\item \alert<1>{\textbf<2->{Attention}} \textit<2->{someone is listening}
    \begin{itemize} [<+->]
    \item Visually, by looking
    \item By using \textit{minimal responses} whenever possible 
    \end{itemize}
\item \alert<4>{\textbf<5->{Acknowledgment}} \textit<5->{move is received}
\item \alert<5>{\textbf<6->{Grounding}} \textit<6->{move is integrated, or not}
    \begin{itemize} [<+->]
    \item \textit{Okay}, etc.
    \item By minimal responses
    \item By (partially) repeating previous move
    \item By a relevant next move
    \end{itemize}
\item \alert<10>{\textbf<11->{Assessing}} \textit<11->{move is judged}
\item \alert<11>{\textbf<12->{Relevant move}} \textit<12->{just start a relevant turn}
\item \alert<12>{\textit<13->{New turn}}
       can subsume \alert<12>{\textit<13->{Assessing}} 
       can subsume \alert<12>{\textit<13->{Grounding}} 
       can subsume \alert<12>{\textit<13->{Acknowledgment}} 
       can subsume \alert<12>{\textit<13->{Attention}}
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{Timing of responses}
\begin{itemize} [<+->]
\item Respond immediately
\item If a \alert<2>{\textit<3->{complex}} response cannot be given in time, 
	switch to a \alert<2>{\textit<3->{simpler, faster}} response type
\item If all else fails, start with an \alert<3>{\textit<4->{Uhhhh}} placeholder
\item Signal problems with a \alert<4>{\textit<5->{delayed}} response
\item Eg, an immediate repeat signals \alert<5>{\textit<6->{acknowledgment}}, a delayed repeat 
      asks for \alert<5>{\textit<6->{confirmation}}
\item If refusal or repair is dispreferred insert \alert<6>{\textit<7->{significant silence}}
\end{itemize}
\end{block}

\end{frame}

\subsection{Minimal responses}
\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{Also: Backchannels, continuer, acknowledgment tokens}
\begin{itemize} [<+->]
\item \alert<1>{\textit<2->{Uh}}, \alert<1>{\textit<2->{Uhm}}, 
	\alert<1>{\textit<2->{HmmHmm}}, \alert<1>{\textit<2->{Yes}}, 
	\alert<1>{\textit<2->{Sure}}, etc.
\item Perform the basic control tasks
\item Do \alert<3>{\textit<4->{not}} take a turn
\item Do \alert<4>{\textit<5->{not}} interrupt the speaker
\item Are semantically, or even lexically, \alert<5>{\textit<6->{empty}}
\item \alert<6>{Keep the conversation going smoothly}
\item \alert<7>{Without visual ``feedback'', eg, on the phone, a lack of 
      audible minimal responses interrupts the conversation} 
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Timing}

\begin{center}
\includegraphics[width=0.70\textwidth]{Pictures/SigDial2005PaperFigure2}
\end{center}

\begin{block}{\textcolor{blue}{\textit{Natural}} and \textit{elicited}
              minimal responses}
\begin{footnotesize}
\begin{itemize} [<+->]
\item Responses start directly after the TRP, even for the unintelligible signals 
       (\alert<1>{$\approx 200 ms$}).
\item Preparations (the \textit{early responses}) start \alert<2>{\textit<3->{before}} the
      utterance ends
\end{itemize}
\end{footnotesize}
\end{block}
 
\begin{footnotesize}\textit{Early responses} are laryngial preparation signals.
\textcolor{green}{\textit{Intonation Only}} responses are unintelligible \textit{uh} 
sounds
\end{footnotesize}
\tiny{\multicite{WesselingVanSon2005a}\multicite{WvS2005b}}

\end{frame}

\subsection{Conversations}
\begin{frame}
\frametitle{\insertsubsection: Implicatures}

Conversations contain rules of inference

\begin{block}{Conversational Maxims of Grice}
\begin{itemize} [<+->]
\item \alert<1>{\textbf<2->{Quantity}}: Be \alert<1>{\textit<2->{exactly}} as informative as required
    \begin{itemize} [<+->]
    \item Not \alert<2>{\textit<3->{less}} informative
    \item Not \alert<3>{\textit<4->{more}} informative
    \end{itemize}
\item \alert<4>{\textbf<5->{Quality}}: Speak the \alert<4>{\textit<5->{truth}}
    \begin{itemize} [<+->]
    \item Do not say what you believe is \alert<5>{\textit<6->{false}}
    \item Do not say that for which you lack \alert<6>{\textit<7->{evidence}}
    \end{itemize}
\item \alert<7>{\textbf<8->{Relevance}}: Be relevant
\item \alert<8>{\textbf<9->{Manner}}: Be \alert<8>{\textit<9->{perspicuous}}
    \begin{itemize} [<+->]
    \item Avoid \alert<9>{\textit<10->{obscurity}}
    \item Avoid \alert<10>{\textit<11->{ambiguity}}
    \item Be \alert<11>{\textit<12->{brief}}
    \item Be \alert<12>{\textit<13->{orderly}}
    \end{itemize}
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Practical dialogs}

General conversations are much too complex. 
Limit \textit{Automatic Dialog Systems} to practical dialogues

\begin{block}{Dialogues that are focused on a concrete task, eg,}
\begin{itemize} [<+-| alert@+>]
\item Task-oriented
\item Information seeking
\item Advice and tutoring
\item Command and control
\end{itemize}
\end{block}
\tiny{\multicite{ABD+01}}
\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Adjacency pairs}

\begin{block}{Practical dialogues contain many controlled turn switches, 
              called Adjacency pairs}
\begin{itemize} [<+-| alert@+>]
\item Question $\Rightarrow$ Answer
\item Proposal $\Rightarrow$ Acceptance/Rejection
\item Apology $\Rightarrow$ Acceptance/Rejection
\item Summons $\Rightarrow$ Answer
\end{itemize}
\end{block}
\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Example dialogue}

\begin{center}
\includegraphics[height=0.45\textheight]{Pictures/fig19-04}
\end{center}

\begin{block}{}
\begin{footnotesize}
\begin{itemize} [<+-| alert@+>]
\item No real minimal responses
\item \textit{Uh Hmm} as an \textit{Acknowledgment}
\item \textit{OK}, \textit{Right}, and repeating dates
      as \textit{Grounding}
\item A lot of \textit{Question-Answering} pairs
\item A lot of \textit{Implicatures}
\end{itemize}
\end{footnotesize}
\end{block}
\tiny{\multicite{JurafskyMartin2000}}
\end{frame}

\subsection{Automatic Dialog System basics}
\begin{frame}
\frametitle{\insertsubsection}

\begin{center}
\includegraphics[width=0.65\textwidth]{Pictures/fig19-05}
\end{center}

\begin{block}{Three part system}
\begin{itemize} [<+-| alert@+>]
\item Speech recognition and understanding
	\begin{itemize} [<+-| alert@+>]
	\item ASR front end with adapted language model
	\item NLP back end for task related semantic parsing
	\end{itemize}
\item Language generation and speech synthesis
	\begin{itemize} [<+-| alert@+>]
	\item TTS output, can be simple phrase concatenation
	\item Frame based or simple grammar sentence generator
	\end{itemize}
\item Dialog management
	\begin{itemize} [<+-| alert@+>]
	\item Task related manager
	\item Task Database back-end
	\end{itemize}
\end{itemize}
\end{block}
\tiny{\multicite{JurafskyMartin2000}}
\end{frame}

\subsection{Recognizer}
\begin{frame}
\frametitle{\insertsubsection}

\begin{center}
\includegraphics[height=0.50\textheight]{Pictures/fig19-06}
\end{center}

\begin{block}{Recognizer must deliver semantic message}
\begin{itemize} [<+->]
\item Semantic context-free grammar (SCFG) for TINA
\item Mixes words and concepts
\item Hand written rules
\end{itemize}
\end{block}
\tiny{\multicite{JurafskyMartin2000}}
\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{center}
\includegraphics[width=0.80\textwidth]{Pictures/fig19-07}
\end{center}

\begin{block}{HMM concept grammar}
\begin{itemize} [<+->]
\item $\underset{C}{argmax}\ P(C|W) = \underset{C}{argmax}\ P(W|C)\cdot P(C)$
\item $P(W|C) = \underset{i=2,N}{\prod} P(w_i|w_{i-N+1},\dots,w_{i-1},c_i)$
\item $P(C) = \underset{i=2,M}{\prod} P(c_i|c_{i-M+1},\dots,w_{i-1})$
\item Trained on a concept-labeled corpus
\end{itemize}
\end{block}
\tiny{\multicite{JurafskyMartin2000}}
\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{center}
\includegraphics[height=0.45\textheight]{Pictures/fig19-08}
\end{center}

\begin{block}{Data fragmentation problem}
\begin{itemize} [<+->]
\item Identical names can be different concepts
\item Eg, cities as \textit{origin} and \textit{destination}
\item Use a modified SCFG for $P(C)$
\item Add SCFG rules for concepts, ie, non-terminals
\end{itemize}
\end{block}
\tiny{\multicite{JurafskyMartin2000}}
\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{center}
\includegraphics[height=0.40\textheight]{Pictures/fig19-09}
\end{center}

\begin{block}{$P(C)$: Probabilistic finite state concept network}
\begin{itemize} [<+->]
\item Enter and Exit states
\item Each arrow has a probability
\item Circles indicate origin, destination, flight indicator, airline, etc.
\end{itemize}
\end{block}
\tiny{\multicite{JurafskyMartin2000}}
\end{frame}

\subsection{Speech Generator}
\begin{frame}
\frametitle{\insertsubsection}

\begin{center}
\includegraphics[width=0.90\textwidth]{Pictures/fig19-10}
\end{center}

\begin{block}{Concept to speech}
\begin{itemize} [<+->]
\item The database manager generates an abstract message
\item Modelled into a sentence structure
\item Surface form, ie, the words, are generated
\item Prosody generated from words and content, 
\item Fed into a TTS system
\end{itemize}
\end{block}
\tiny{\multicite{JurafskyMartin2000}}
\end{frame}

\subsection{Dialog management}
\begin{frame}
\frametitle{\insertsubsection}

\begin{center}
\includegraphics[height=0.45\textheight]{Pictures/fig19-11}
\end{center}

\begin{block}{Finite state automata}
\begin{itemize} [<+->]
\item Simple dialog states
\item Good for form filling dialogues (frames)
\item Can handle frame switching (stochastically)
\end{itemize}
\end{block}
\tiny{\multicite{JurafskyMartin2000}}
\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{center}
\includegraphics[width=0.90\textwidth]{Pictures/fig19-12}
\end{center}

\begin{block}{Who takes the initiative}
\begin{itemize} [<+-| alert@+>]
\item Machine prompts all user actions $\Rightarrow$ Finite state script
\item User asks questions $\Rightarrow$ Single frame
\item Machine allows some user initiatives $\Rightarrow$ Frame switching
\item Negotiation $\Rightarrow$ Plan based models
\end{itemize}
\end{block}
\tiny{\multicite{JurafskyMartin2000}\multicite{ABD+01}}
\end{frame}

\subsection{Bibliography}

\begin{frame}[allowframebreaks]
\frametitle{Further Reading}
\bibliographystyleSseven{plainnat}
\tiny{
\bibliographySseven{SpraakherkenningEnSynthese}
}
\end{frame}

\appendix
\begin{frame}
\frametitle{Appendix A}

\end{frame}


% Insert License text
\insertgnugpl


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%\part{Measuring Speech}
\lecture{More on dialog systems}{week10}
\renewcommand{\multicite}[1]{\nocite{#1}\citeSseven{#1}}
\renewcommand{\copyrightauthor}[0]{R.J.J.H. van Son}
\renewcommand{\licensenumber}[0]{License10}

\section{More on dialog systems}

\subsection{Introduction}
\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{Successful \textit{Automatic Dialog Systems} must}
\begin{itemize} [<+->]
\item Handle numerous different users
\item Incite effective user expectations
\item Fail gracefully (eg, with human back-up)
\item Allow multimodal interaction, if at all possible
\item Allow user initiative
\item \textit{Automatic Dialog Systems} are as much an 
 \textit{ergonomic} as a \textit{speech} technology problem
\end{itemize}
\end{block}

\tiny{Many pictures (and their copyrights) are from }\tiny{\multicite{ABD+01}}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{center}
\includegraphics[width=0.65\textwidth]{Pictures/fig19-05}
\end{center}

\begin{block}{Automatic Dialog Systems have the combined limitations of:}
\begin{itemize} [<+->]
\item \alert<1>{\textbf<2->{ASR + NLP}}: The real bottleneck
\item \alert<2>{\textbf<3->{NLG + TTS}}: Normally not a problem
\item \alert<3>{\textbf<4->{Dialog management + database}}: A bottleneck in complex tasks
\end{itemize}
\end{block}
\tiny{\multicite{JurafskyMartin2000}}
\end{frame}

\subsection{Conversational Human-Computer Interaction}
\begin{frame}
\frametitle{\insertsubsection: Practical dialogs}

General conversations are much too complex. 
Limit \textit{Automatic Dialog Systems} to practical dialogues

\begin{block}{Dialogues that are focused on a concrete task, eg,}
\begin{itemize} [<+-| alert@+>]
\item Task-oriented
\item Information seeking
\item Advice and tutoring
\item Command and control
\end{itemize}
\end{block}
\tiny{\multicite{ABD+01}}
\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{The Practical Dialogue Hypothesis}
The conversational competence required for practical dialogues, 
while still complex, is significantly simpler to achieve than 
general human conversational competence
\end{block}

\tiny{\multicite{ABD+01}}
\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{The Domain-Independence Hypothesis}
Within the genre of practical dialogue, the bulk of the 
complexity in the language interpretation and dialogue 
management is independent of the task being performed
\end{block}

\tiny{\multicite{ABD+01}}
\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{center}
\includegraphics[width=0.9\textwidth]{Pictures/Aetal2001Fig1}
\end{center}

\begin{block}{Dialogue and task complexity}
\begin{itemize} [<+-| alert@+>]
\item Practical Dialogues
\item Frame based (form-filling) is currently most used
\item Set of frames complex due to switch (going back)
\item Plan and Agent based require model-of-the-world
\end{itemize}
\end{block}

\tiny{\multicite{ABD+01}}
\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{center}
\includegraphics[width=0.9\textwidth]{Pictures/Aetal2001Fig2}
\end{center}

\begin{block}{Context for a train information task}
\begin{itemize} [<+-| alert@+>]
\item Frame based dialogue system
\item Fill in forms, send query when ready
\item Simple and robust
\item Simplifies ASR+NLP tasks (pattern matching)
\end{itemize}
\end{block}

\tiny{\multicite{ABD+01}}
\end{frame}

\subsection{Spoken Dialogue Systems}
\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{Challenges for Dialogue Systems}
\begin{itemize} [<+-| alert@+>]
\item Parsing Language in Practical Dialogues
\item Integrating Dialogue and Task Performance
\item Intention Recognition
\item Mixed Initiative Dialogue
\end{itemize}
\end{block}

\tiny{\multicite{ABD+01}}
\end{frame}

\begin{frame}[label=ChallengesSDS]
\frametitle{\insertsubsection: Challenges}

\begin{block}{Parsing Language in Practical Dialogues}
\begin{itemize} [<+-| alert@+>]
\item Detailed semantic, ``deep'', representation
\item Broad coverage NL grammars fail due to ambiguity
\item Semantic restrictions could work
\item Add domain-specific restrictions for tasks
\item Apply \hyperlink{MaximsGrice}{\textit{Grice's Maxims}}
\item Parsing based on \textit{Speech Acts}
\end{itemize}
\end{block}

\tiny{\multicite{ABD+01}}
\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Challenges}

\begin{center}
Integrating Dialogue and Task Performance
\end{center}
\parbox{\textwidth}
{
	\parbox{0.60\textwidth}
	{
		\begin{itemize} [<+-| alert@+>]
		\item Complex tasks based on Agents
		\item Abstract \textbf{problem-solving} model:
		\item \textbf{Objectives}: The way we want the world to be
		\item \textbf{Solutions}: Courses of action to achieve objectives
		\item \textbf{Resources}: Objects and abstractions available
		\item \textbf{Situations}: The way the world currently might be
		\end{itemize}
	}
	\parbox{0.39\textwidth}
	{
		\begin{center}
		\includegraphics[width=0.39\textwidth]{Pictures/Aetal2001Fig4}
		\begin{scriptsize}Agent based architecture\end{scriptsize}
		\end{center}
	}
}

\tiny{\multicite{ABD+01}}
\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Challenges}

\begin{block}{Intention Recognition}
\begin{itemize} [<+-| alert@+>]
\item Determine the goal of the user
\item Can switch with every utterance
\item Use implicatures
\item Extrapolate from preceding actions
\item Interpolate from ``parent'' (sub-)goals
\item Is a probabilistic framework possible?
\end{itemize}
\end{block}

\tiny{\multicite{ABD+01}}
\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Challenges}

\begin{block}{Mixed Initiative Dialogue}
\begin{itemize} [<+-| alert@+>]
\item Finite-state: typically fixed system-initiative
\item Frame based: Fixed user/system-initiative (eg, Query-Answer)
\item Fixed user-initiative: User does not know what information is needed
\item Solution: Limited mixed initiative
\item Full mixed-initiative: Both user and system can \textit{barge-in}
\item Mixed-initiative: Weight costs of interruption and system goals
\end{itemize}
\end{block}

\tiny{\multicite{ABD+01}}
\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{footnotesize}
\begin{quotation}
\begin{enumerate}
\item USR: We need to \alert<4>{\textit<4>{get the woman in Penfield to \alert<2>{\textit<2>{Strong}}}}
\item SYS: \alert<1>{\textit<1>{OK}}
\item USR: What \alert<2>{\textit<2>{vehicles}} are available?
\item SYS: There are \alert<2>{\textit<2>{ambulances}} in Pittsford and Webster
\item USR: \alert<1>{OK}. Use one from Pittsford
\item SYS: \alert<3,5>{\textit<3,5>{Do you know that Route 96 is 
		blocked due to construction?}}
\item USR: \alert<1>{\textit<1>{Oh}}
\item USR: Let's use the interstate instead
\item SYS: OK. I'll \alert<4>{\textit<4>{dispatch the crew}}
\end{enumerate}
\end{quotation} 
\end{footnotesize}

\begin{block}{A short example of a practical dialog}
\begin{itemize} [<+-| alert@+>]
\item Both speakers use acknowledgements (\textit{OK, Oh})
\item Semantic representation
\item Intention recognition
\item Goal driven
\item System interrupts dialog (barge-in)
\end{itemize}
\end{block}

\tiny{\multicite{ABD+01}}
\end{frame}

\subsection{TRIPS}
\begin{frame}
\frametitle{\insertsubsection}

\begin{center}
\includegraphics[width=0.65\textwidth]{Pictures/Aetal2001Fig5}
\end{center}

\begin{block}{Interacting with TRIPS}
\begin{itemize} [<+->]
\item Multi modal interaction with current state shown
\item Emergency Response System
\end{itemize}
\end{block}

\tiny{\multicite{ABD+01}}
\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{center}
\includegraphics[width=0.65\textwidth]{Pictures/Aetal2001Fig6}
\end{center}

\begin{block}{TRIPS system architecture}
\begin{itemize} [<+-| alert@+>]
\item Interpretation
\item Generation
\item Behavior
\end{itemize}
\end{block}

\tiny{\multicite{ABD+01}}
\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

% \begin{center}
% \includegraphics[heigth=0.33\textheigth]{Pictures/Aetal2001Fig7}
% \end{center}

\begin{footnotesize}
\begin{quotation}
\begin{enumerate}
\item USR: \alert<1>{\textit<2->{We}} need to get \alert<2>{\textit<3->{the woman in Penfield}} to \alert<3>{\textit<4->{Strong}}
\end{enumerate}
\end{quotation} 
\end{footnotesize}
\begin{block}{Reference resolution}
\begin{itemize} [<+-| alert@+>]
\item SS1: The set consisting of \textit{USR }and \textit{SYS} (general dialogue setting)
\item WOM1: The Injured woman in Penfield previously discussed (discourse history)
\item Strong Memorial Hospital (general world knowledge)
\end{itemize}
\end{block}

\tiny{\multicite{ABD+01}}
\end{frame}

\subsection{OVIS}
\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{Public Transport Information System}
\begin{itemize} [<+-| alert@+>]
\item Deliver train travel information (station-to-station)
\item Telephone based application
\item Speech only
\item Replaced existing human based service
\item Based on an existing German system (Philips Aachen)
\item Has been in active service (still is)
\item Frame-based
\end{itemize}
\end{block}

\tiny{\multicite{Ovis}}
\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{Spoken Dialogue System (SDS) components}
\begin{enumerate} [<+-| alert@+>]
\item Continuous HMM based Speech Recognition (CSR)
\item Natural Language Processing (NLP)
\item Dialogue Management (DM)
\item Text-To-Speech (TTS)
\end{enumerate}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

Skip \textit{Wizard-of-Oz} or \textit{Green-curtain} scenarios and build a working 
system from scratch.

\begin{block}{Stages to build and train SDS}
\begin{enumerate} [<+-| alert@+>]
\item Make a first version of the SDS with available data
     (which need not be application-specific)
\item Ask a limited group of people to use this system, and store the dialogues
\item Use the recorded data (which are application-specific) to improve the SDS
\item Gradually increase the data and the number of users
\item Repeat steps [2], [3], and [4] until the system works satisfactorily
\end{enumerate}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Continuous Speech Recognition}

\begin{block}{Start training with the Polyphone multi-speaker corpus}
\begin{itemize} [<+-| alert@+>]
\item 2500 utterances
\item Read speech
\item Semi-spontaneous (read) speech
\item Recorded over the phone
\item For each speaker, 5 out 50 \textit{Polyphone} sentences selected
\item Phonetically rich sentences (\textit{all} Dutch phonemes)
\item 50 Dutch phone models (2 for each of /\textipa{r}/ and /\textipa{l}/)
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Pronunciation lexicon}

\begin{block}{Phoneme representations}
\begin{itemize} [<+-| alert@+>]
\item Names of stations from the ONOMASTICA database
\item Lemma forms of other words from the CELEX database
\item Remaining generated by a grapheme-to-phoneme converter
\item Pronunciation variation initially not modelled
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: NLP and DM}

\begin{block}{NLP and DM taken from German original}
\begin{itemize} [<+-| alert@+>]
\item Date and time conventions adapted
\item Interface with different train table format (eg, start of \textit{tomorrow})
\item Adaptations for user preferences, eg, train numbers
\item Collect volunteer queries from keyboard simulation
\item Form based database query system with feed-back
\item Allows user to correct the system
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: TTS}

\begin{block}{Speech generation (TTS)}
\begin{itemize} [<+-| alert@+>]
\item German original could not be used
\item Concatenate utterance fragments
\item Female voice
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Training}

\begin{center}
\begin{tabular}{l r l l}
\hline
Database & utterances & source & duration (hours:min)\\
\hline
\alert<1>{DB0} & \alert<1>{2500} & \alert<1>{Polyphone} & \alert<1>{4:42}\\
\alert<3>{DB1} & \alert<3>{1301} & \alert<3>{application} & \alert<3>{0:41}\\
DB2 & 5496 & application & 3:47\\
DB3 & 6401 & application & 4:35\\
DB4 & 8000 & application & 5:55\\
DB5 & 10003 & application & 7:20\\
\hline
\end{tabular}

\end{center}
\begin{block}{Databases used during development of the SDS}
\begin{itemize} [<+-| alert@+>]
\item Start with the \textit{Polyphone} database (DB0)
\item Collect volunteer responses from this system
\item Retrain the system with the new speech and repeat
\item DB1-5 are incremental, ie, DB5 contains all of DB4 etc.
\end{itemize}
\end{block}

\tiny{\multicite{Ovis}}
\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Training}


\begin{center}
\includegraphics[width=0.65\textwidth]{Pictures/OVISFig1} \\
\begin{scriptsize}number of utterances in database\end{scriptsize}
\end{center}

\begin{block}{Out-of-vocabulary words per utterance vs. corpus size}
\begin{itemize} [<+-| alert@+>]
\item Number of OOV words is small
\item DB0-DB3 small number of users
\item After DB3 (6401 utterances) new users recruited
\end{itemize}
\end{block}

\tiny{\multicite{Ovis}}
\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Training}

\begin{scriptsize}
\begin{center}
\begin{tabular}{l r r r r r r r}
\hline
System  & \alert<1,2>{P0 + L0} & \alert<1>{P02 + L0} & \alert<2,3>{P02 + L2} \\
\hline
WG - WER & \alert<1,2>{20.59} &  \alert<1>{18.36}  &   \alert<2,3>{6.72}  \\
WG - SER & \alert<1,2>{40.00} &  \alert<1>{36.60}  &  \alert<2,3>{16.00}  \\
BS - WER & \alert<1,2>{39.87} &  \alert<1>{31.45}  &  \alert<2,3>{14.73}  \\
BS - SER & \alert<1,2>{65.00} &  \alert<1>{54.20}  &  \alert<2,3>{28.00}  \\
\hline
\end{tabular}
\end{center}
\end{scriptsize}

\begin{block}{Performance level for different phoneme models (Pi) and language models (Lj).
Evaluation is done with test database 1}
\begin{itemize} [<+-| alert@+>]
\item Training phoneme models on both DB0 (polyphone) and DB2 (application) reduced error rates
\item Training language model on DB2 (application) reduced errors more
\item Application specific data is more important for language modelling than phoneme modelling
\end{itemize}
\end{block}

\tiny{\multicite{Ovis}}
\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Training}

\begin{scriptsize}
\begin{center}
\begin{tabular}{l r r r r r r r}
\hline
System   & \alert<1>{P02 + L2} & P03 + L2 & \alert<1>{P03 + L3} & \alert<2>{P3 + L2} 
         & \alert<2>{P3 + L3} \\
\hline
WG - WER & \alert<1>{6.72}  & 6.94  &   \alert<1>{6.94}  &   \alert<2>{6.94} &   \alert<2>{6.94} \\
WG - SER & \alert<1>{16.00}  & 15.20  &  \alert<1,3>{15.60}  &  \alert<2>{16.20} &  \alert<2,3>{15.40} \\
BS - WER & \alert<1>{14.73}  & 15.43  &  \alert<1,3>{15.70}  &  \alert<2>{16.41} &  \alert<2,3>{14.84} \\
BS - SER & \alert<1>{28.00} & 29.00  &  \alert<1,3>{28.60}  &  \alert<2>{26.00} &  \alert<2,3>{26.40} \\
\hline
\end{tabular}
\end{center}
\end{scriptsize}

\begin{block}{Performance level for different phoneme models (P02/3 vs P3) and language models (L2 vs L3).
Evaluation is done with test database 1}
\begin{itemize} [<+-| alert@+>]
\item Increasing DB size from 5496 to 6401 utterances had little effect
\item Leaving out Polyphone data (DB0) hardly had an effect
\item Leaving out DB0 even decreased WER a little
\end{itemize}
\end{block}

\tiny{WG: word-graph, BS: best sentence, \multicite{Ovis}}
\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Training}

\begin{center}
\begin{tabular}{l | r | r r r }
\hline
 testDB &   old    &       & new & \\
 System & P3 + L3 &  \alert<1>{P3 + L3} & \alert<2>{P4 + L4} & \alert<3>{P5 + L5} \\
\hline
WG - WER &  6.94  &  \alert<1>{ 8.87} &   \alert<2>{6.81} &   \alert<3>{6.69} \\
WG - SER & 15.40  &  \alert<1>{17.80} &  \alert<2>{14.40} &  \alert<3>{13.80} \\
BS - WER & 14.84  &  \alert<1>{15.27} &  \alert<2>{12.93} &  \alert<3>{14.02} \\
BS - SER & 26.40  &  \alert<1>{25.40} &  \alert<2>{24.20} &  \alert<3>{24.60} \\
\hline
\end{tabular}
\end{center}

\begin{block}{Performance levels for different phoneme models (Pi) and language models (Lj).
Evaluation is done with test database 1 (column 2: old) and 2 (columns 3-5: new)}
\begin{itemize} [<+-| alert@+>]
\item Test database 2 induced more errors
\item DB4 (8,000 utterances) had lower WER again
\item Increase to 10,000 utterances (DB5) had little effect
\end{itemize}
\end{block}

\tiny{WG: word-graph, BS: best sentence, \multicite{Ovis}}
\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{Pronunciation variation and non-speech sounds}
\begin{itemize} [<+-| alert@+>]
\item A single pronunciation per word gives problems
\item Eg, \textipa{/GEld@rOp/} vs. \textipa{/GEldrOp/} and \textipa{/Ams@dAm/} vs. \textipa{/Amst@rdAm/}
\item Different sources causes inconsistencies
\item People use several different variants
\item Variant in lexicon not the ``best'' one
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Conclusions}

\begin{block}{It actually worked!}
\begin{itemize} [<+-| alert@+>]
\item Adapt an existing frame-based system
\item Bootstrap on actual usage
\item Collect and train more
\item Use robust DM
\item Use human fall-back
\end{itemize}
\end{block}

\end{frame}

\subsection{Bibliography}

\begin{frame}[allowframebreaks]
\frametitle{Further Reading}
\nociteSseven{BoersmaWeenink}
\nocite{BoersmaWeenink}
\nociteSseven{Boersma2001}
\nocite{Boersma2001}
\bibliographystyleSseven{plainnat}
\tiny{
\bibliographySseven{SpraakherkenningEnSynthese}
}
\end{frame}

\appendix
\begin{frame}[label=MaximsGrice]
\frametitle{Appendix A: Implicatures}

Conversations contain rules of inference

\begin{block}{Conversational Maxims of Grice}
\begin{itemize} [<+->]
\item \alert<1>{\textbf<2->{Quantity}}: Be \alert<1>{\textit<2->{exactly}} as informative as required
    \begin{itemize} [<+->]
    \item Not \alert<2>{\textit<3->{less}} informative
    \item Not \alert<3>{\textit<4->{more}} informative
    \end{itemize}
\item \alert<4>{\textbf<5->{Quality}}: Speak the \alert<4>{\textit<5->{truth}}
    \begin{itemize} [<+->]
    \item Do not say what you believe is \alert<5>{\textit<6->{false}}
    \item Do not say that for which you lack \alert<6>{\textit<7->{evidence}}
    \end{itemize}
\item \alert<7>{\textbf<8->{Relevance}}: Be relevant
\item \alert<8>{\textbf<9->{Manner}}: Be \alert<8>{\textit<9->{perspicuous}}
    \begin{itemize} [<+->]
    \item Avoid \alert<9>{\textit<10->{obscurity}}
    \item Avoid \alert<10>{\textit<11->{ambiguity}}
    \item Be \alert<11>{\textit<12->{brief}}
    \item Be \alert<12>{\textit<13->{orderly}}
    \end{itemize}
\end{itemize}
\end{block}
\hyperlink{ChallengesSDS}{Back to Challenges}

\end{frame}


% Insert License text
\insertgnugpl


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%\part{Measuring Speech}
\lecture{Examples: Student's projects}{week11}
\renewcommand{\multicite}[1]{\nocite{#1}\citeSeight{#1}}
\renewcommand{\copyrightauthor}[0]{R.J.J.H. van Son}
\renewcommand{\licensenumber}[0]{License11}

\section{Examples: Student's projects}

\subsection{Introduction}
\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{Speech technology for ``disadvantaged'' languages}
\begin{itemize} [<+->]
\item Language barriers limit access to digital resources
\item Speech technology needed for access to services, eg, phone services
\item Language often part of national, cultural, and political identity
\item Lack of Language and Speech technology will put communities 
      at a disadvantage
\item Many speech technology projects for ``minority'' languages
      started by single ``students'' of the language
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{Basic speech technology projects}
\begin{itemize} [<+->]
\item Demonstration TTS or ASR systems \textit{can} be build by a single person
\item All tools available on the internet for free
\item Basic systems for a new language take around 3-6 person months
\item Systems and work are modular
\item Systems should be constructed iteratively
\item Start with an existing system, and change it gradually
\item If digital resources are available, \alert<7>{use them!}
\end{itemize}
\end{block}
\flushbottom{
\begin{scriptsize}
See \url{http://www.fon.hum.uva.nl/IFA-publications/Others/Other_papers.html}
\end{scriptsize}
}
\end{frame}

\subsection{Example 1: A basic Frisian TTS}
\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{Master's thesis}
\begin{itemize} [<+->]
\item No speech technology available for Frisian
\item Language community is organized scientifically
\item There is ``political'' demand for Frisian Language Technology
\item Student is a native speaker
\item 4 Month thesis project
\item Dutch diphones (no time to create Frisian set)
\item Aim: ``bootstrap'' the development of a TTS system
\end{itemize}
\end{block}
\flushbottom{\tiny{\multicite{DPvS05}}}
\end{frame}

\begin{frame}
\frametitle{\insertsubsection: West Frisian dialects in the Netherlands}

\begin{center}
\includegraphics[width=0.50\textwidth]{Pictures/Dijkstra2005Map1}
\end{center}

\begin{block}{West Germanic language (Indo-European)}
\begin{itemize} [<+->]
\item Main dialects: Klaaifrysk, W\`aldfrysk, and S\^ud-Westhoeksk
\item Standard Frisian based on Klaaifrys
\item Official status since 1970 
\end{itemize}
\end{block}

\flushbottom{\tiny{\multicite{DPvS05}}}
\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{Total population of \textit{Friesland} $>634,000$
              \begin{tiny}\multicite{GorterJonkman1995}\end{tiny}}
\begin{itemize} [<+->]
\item 55\% native speakers (350,000)
\item 74\% understands Frisian (470,000)
\item 65\% reads Frisian (410,000)
\item 17\% writes in Frisian (110,000)
\end{itemize}
\end{block}

\flushbottom{\tiny{\multicite{DPvS05}}}
\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{Start with an existing system, and change it gradually}
\begin{itemize} [<+->]
\item \textit{Frisian} is close to \textit{Dutch} in many respects
\item \textit{Nextens} and those that build it were available
\item Contacts with the \textit{Fryske Akademy} could supply language help
\item A digital pronunciation lexicon could be ``borrowed''
\item Technical and community support were available
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{Language resources and tools}
\begin{itemize} [<+->]
\item \textit{Fryske Akademy}
\item MBROLA \tiny{\multicite{MBROLA}}
\item Nextens \tiny{\multicite{Nextens2003}}
\item Festival \tiny{\multicite{BL03}}
\item Pre-publication of ``Frysk H\^anwurdboek'' (Concise dictionary)
\item Worldbet \tiny{\multicite{Hieronymus1994}}
\item Enthusiasm from everyone
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Nextens}

\begin{block}{The architecture of NeXTeNS (Festival):}
\begin{itemize} [<+->]
\item \textbf{Token Module}: Tokenization
\item \textbf{POS Module}: Part-Of-Speech tagging
\item \textbf{Syntactic Module}: Syntax parsing (\textit{disabled})
\item \textbf{Phrasing Module}: Phrase break prediction
\item \textbf{Intonation Module}: Sentence accents
\item \textbf{Tune Module}: Tune choice needed for ToDI
\item \textbf{Word Module}: Grapheme-to-phoneme conversion
\item \textbf{Pauses Module}: Insertion of pause segments
\item \textbf{Postlexical Module}: Anything left over
\item \textbf{Duration Module}: Segment and pause-durations
\item \textbf{Fundamental frequency control}: ToDI $\Rightarrow$ utterance
\item \textbf{Waveform synthesis}
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{Building a Frisian TTS}
\begin{itemize} [<+->]
\item Construct Frisian Worldbet phonetic alphabet \tiny{\multicite{Hieronymus1994}}
\item Convert pronunciation lexicon to Worldbet
\item Phrasing, Tune, Pause: Use Dutch (small adaptations)
\item Tokenization: Enter Frisian numbers and abbreviations
\item POS: Translated Dutch \textit{Function} wordlist
\item POS: Use only \textit{Content/Function} word difference
\item Intonation: Accent every other \textit{Content} word
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{Building a Frisian TTS: Word module}
\begin{itemize} [<+->]
\item Pronunciation lexicon
\item Letter-to-Sound rules, eg, \\
      \texttt{(VOWEL [- g] VOICEDC = - G)}
\item Syllable stress rules, ie, strong/weak syllables
\item Map complex sounds, eg, nasalized vowels and triphthongs
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{Building a Frisian TTS: Other modules}
\begin{itemize} [<+->]
\item Postlexical: Adapted Dutch rules
\item Postlexical: Map Frisian worldbet to Dutch SAMPA symbols
\item Duration: Shorten schwa, change duration long vowels
\item Fundamental frequency: Adapt Dutch ToDI module
\item Waveform synthesis: Map each ``Frisian'' phone to the ``nearest'' Dutch MBROLA phone
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Evaluation}

\begin{center}
\includegraphics[width=0.50\textwidth]{Pictures/Dijkstra2005Fig1}
\end{center}

\begin{block}{Mean judgments for 20 test sentences}
\begin{itemize} [<+->]
\item End evaluation over WWW with 32 native subjects
\item 10 short ($<$ 13 words) 10 long ($\ge$ 13 words)
\item Example of \textcolor{blue}{\movie [externalviewer]{short}{Sounds/fryssutt11.wav}}
      and \textcolor{blue}{\movie [externalviewer]{long}{Sounds/fryssutt9.wav}} 
      sentence
\item 6 qualities on a 5 point scale (higher is better/more rapid)
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection}
\begin{footnotesize}
\begin{tabular}{l|r r r}
  & \alert<3>{short} $N\approx 331$ & \alert<3>{long} $N\approx 331$ & total $N\approx 662$  \\
\hline
intelligibility & \alert<3>{2.57} (1.25) & \alert<3>{2.80} (1.24) & \alert<1>{2.69} (1.25) \\
quality         & \alert<3>{2.51} (0.99) & \alert<3>{2.50} (0.97) & \alert<1>{2.50} (0.98) \\
\alert<2>{naturalness} & \alert<2,3>{2.31} (0.97) & \alert<2,3>{2.22} (0.97) & 
      \alert<1,2>{2.27} (0.97) \\
lexical stress  & \alert<3>{2.67} (1.05) & \alert<3>{2.58} (0.99) & \alert<1>{2.64} (1.02) \\
sentence melody & \alert<3>{2.79} (0.99) & \alert<3>{2.64} (1.02) & \alert<1>{2.72} (1.01) \\
speaking rate   & \alert<3>{3.30} (0.65) & \alert<3>{3.35} (0.71) & 3.33 (0.68) \\
\hline
\end{tabular} 
\end{footnotesize}

\begin{block}{Mean judgments (standard deviation)}
\begin{itemize} [<+->]
\item Mean ratings \alert<1>{\textit<2->{below 3}} (mid-point)
\item \alert<2>{\textit<3->{Naturalness rated}} lowest
\item Sentence length did not change ratings
\item Ratings were above \alert<4>{\textit<5->{1!}}
\item \alert<5>{Note}: This was done using a \alert<5>{Dutch} diphone set
\end{itemize}
\end{block}

\flushbottom{\scriptsize{
Judgments on a 5 point scale, higher is better. For speaking rate higher is more rapid
}}
\end{frame}

\subsection{Example 2: Digit recognition in two languages}
\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{Kinyarwanda: Official language of Rwanda}
\begin{itemize} [<+->]
\item Niger-Congo Language \url{http://www.nvtc.gov/lotw/months/september/niger.html}
\item 7-8 million native speakers
\item Many Rwandese are monolingual
\item Recognizer build by \textit{Muhirwe Jackson} for his Master of Science thesis \begin{tiny}\multicite{JacksonM2005}\end{tiny}
\item Computer Science of Makerere University, Kampala, Uganda
\item Implements the tutorial digit recognizer from HTK
\end{itemize}
\end{block}
\flushbottom{\tiny{
\multicite{JacksonM2005}\\
\multicite{HTKbook}\\
\multicite{HTKdistr}}}
\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{Dutch: Official language of the Netherlands}
\begin{itemize} [<+->]
\item West Germanic language, 21 million native speakers
\item Masters of Science course for AI students
\item University of Amsterdam
\item Speech Technology project
\item 1 month, 6 students \begin{tiny}\multicite{SpTechProj2004}\end{tiny}
\item Implements the tutorial telephone application from HTK
\end{itemize}
\end{block}
\flushbottom{\tiny{
\multicite{SpTechProj2004}\\
\multicite{HTKbook}\\
\multicite{HTKdistr}}}
\end{frame}

\subsection{Building a basic ASR system}
\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{Tasks}
\begin{enumerate} [<+->]
\item Building the task grammar
\item Constructing a dictionary for the models
\item Recording the data.
\item Creating transcription files for training data
\item Encoding the data (feature processing)
\item (Re-) training the acoustic models
\item Evaluating the recognizers against the test data
\item Reporting recognition results
\end{enumerate}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{scriptsize}
\texttt{\# Kinyarwanda \\
\alert<1>{\$digit=RIMWE | KABIRI | GATATU | KANE | GATANU | 
           GATANDATU | KARINDWI | UMUNANI | ICYENDA | ZERO;} \\
\alert<2>{(SENT-START \alert<3>{\alert<3>{[}} \$digit \alert<3>{\alert<3>{]}} SENT-END)} \\
\medskip
\# Dutch \\
\alert<1>{\$digit = EEN | TWEE | DRIE | VIER | VIJF |
          ZES | ZEVEN | ACHT | NEGEN | NUL;}\\
\alert<1>{\$name = \alert<3>{[} ROB \alert<3>{]} (VAN SON) |
\alert<3>{[} FRANS \alert<3>{]} ADRIAANS |
\alert<3>{[} TOM \alert<3>{]} LENTZ |
\alert<3>{[} MARIJN | MARINUS \alert<3>{]} KOOLEN |
\alert<3>{[} ORK \alert<3>{]} (DE ROOIJ) |
\alert<3>{[} MARKUS \alert<3>{]} HEUKELOM |
\alert<3>{[} DAAN \alert<3>{]} VREESWIJK;} \\
\alert<2>{( SENT-START ( DRAAI <\$digit> | BEL \$name) SENT-END )} \\
}
\end{scriptsize}

\begin{block}{Task Grammars}
\begin{itemize} [<+->]
\item Define digits and names
\item Define grammar on vocabular
\item Square brackets enclose optional items
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{Construct pronunciation dictionary}
\begin{itemize} [<+->]
\item Make a word list of all words in the training corpus or a suitable text corpus
\item Transcribe the words by hand or use a TTS system (eg, \textit{Nextens})
\item Feed the lexicon to HTK \tiny{\multicite{HTKdistr}}
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{Generate prompts and record utterances}
\begin{itemize} [<+->]
\item Use task grammar to generate random prompts
\item Record as many users as possible reading the prompts
\item Better, subjects repeat synthesized (TTS) prompts
\item Transcribe all prompts and all sentences in the corpus
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{Training}
\begin{itemize} [<+->]
\item Transcribe and (feature) encode utterances
\item Feed as much speech as possible to the HTK training
\item Kinyarwanda uses 3 male and 3 female speakers, 150 sentences 
\item Words were hand-labeled
\item Dutch uses 1000 labeled sentences from the IFAcorpus (4 male, 4 female speakers)
\item Dutch recorded 150 task sentences from 4 male speakers (total 600)
\item Recorded utterances were transcribed automatically
\item Put all files in correct format and fire up HTK training \tiny{\multicite{HTKdistr}}
\end{itemize}
\end{block}

\end{frame}

\subsection{ASR evaluation}
\begin{frame}
\frametitle{\insertsubsection: Kinyarwanda}

\begin{tabular}{l|r r r}
\hline
Subject & Words correct & Substitution errors & Percentage \\
\hline
Subject 1 & 9 & 1 & 90\% \\
Subject 2 & 8 & 2 & 80\% \\
Subject 3 & 8 & 2 & 80\% \\
Subject 4 & 8 & 2 & 80\% \\
\hline
\end{tabular}

\begin{block}{Live data recognition results}
\begin{itemize} [<+->]
\item 4 New subjects
\item Read out all 10 numbers
\item HTK self-test results (\textit{not} live): 
\item Sentence Recognition Rate: 92.00\% (N=50)
\item Word Recognition Rate: 94.87\% (N=156)
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Dutch}

\begin{tabular}{l | l}
\hline
TRAINED ON & TESTED ON \\
\hline
IFA + Domain & Domain, training speakers \\
IFA + Domain & Domain, `unknown' speaker \\
IFA + Domain & New sentences, training speakers \\
IFA + Domain & New sentences, new speaker  \\
\hline
\end{tabular}

\begin{block}{Testing procedures}
\begin{itemize} [<+->]
\item Two corpora: IFA corpus and Domain corpus
\item Testing using randomly selected sentences
\item Test set not used during training
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Dutch}

\begin{tabular}{l | r r }
\hline
Left  &    WORD          &  SENTENCE \\
Out \% & RECOGNITION (\%) & RECOGNITION (\%) \\
\hline
10 & 99.71 & 91.38 \\
20 & 99.46 & 92.31 \\
50 & 99.67 & 89.93 \\
80 & 99.66 & 89.18  \\
\hline
\end{tabular}

\begin{block}{Testing on random sentences}
\begin{itemize} [<+->]
\item Leave out random sentences and train
\item Test randomly selected sentences
\item Smaller training set affects Sentence Recognition most
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{\insertsubsection: Dutch}

\begin{footnotesize}
\begin{tabular}{l | r r }
\hline
LEFT OUT & WORD  & SENTENCE \\
SPEAKER & RECOGNITION (\%) & RECOGNITION (\%) \\
\hline
Tom   & 99.57 & 85.71 \\
Markus & 99.78 & 72.60 \\
Ork & 99.43 & 89.13 \\
Frans & 99.78 & 81.63 \\
\hline \hline
LEFT OUT   & WORD & SENTENCE \\
PERCENTAGE & RECOGNITION (\%) & RECOGNITION (\%) \\
\hline
12 & 99.41 & 92.86 \\
25 & 99.80 & 90.57 \\
50 & 99.84 & 89.35 \\
\hline
\end{tabular}\\
Top: Testing on a new speaker, Bottom: Testing on new sentences\\
\end{footnotesize}

\begin{block}{New speakers are worse than new sentences}
\begin{itemize} [<+->]
\item More speakers needed for independence
\item Sentence recognition drops sharply
\item New speaker \textit{and} new sentences \\
      Recognition: Word - 99.57\%, Sent - 84.35\%
\end{itemize}
\end{block}

\end{frame}

\subsection{Conclusion}
\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{Simple TTS and ASR can be done in a few months}
\begin{itemize} [<+->]
\item Free tools are available
\item People like it when their language is used
\item Recording speech is the most laborous step
\item More speech is better, as is more text
\item Pronunciation dictionaries are crucial
\end{itemize}
\end{block}

\end{frame}

\subsection{Bibliography}

\begin{frame}[allowframebreaks]
\frametitle{Further Reading}
\bibliographystyleSeight{plainnat}
\tiny{
\bibliographySeight{SpraakherkenningEnSynthese}
}
\end{frame}

\appendix
\begin{frame}
\frametitle{Appendix A}

\end{frame}


% Insert License text
\insertgnugpl


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%\part{Measuring Speech}
\lecture{Students Choice}{week12}
\renewcommand{\multicite}[1]{\nocite{#1}\citeSeight{#1}}
\renewcommand{\copyrightauthor}[0]{R.J.J.H. van Son}
\renewcommand{\licensenumber}[0]{License12}

\section{Students Choice}

\subsection{Introduction}
\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{}
\begin{itemize} [<+->]
\item text
\end{itemize}
\end{block}

\end{frame}

\subsection{}
\begin{frame}
\frametitle{\insertsubsection}

\begin{block}{}
\begin{itemize} [<+->]
\item text
\end{itemize}
\end{block}

\end{frame}

\subsection{Assignment}
\begin{frame}
\frametitle{\insertsubsection: }

\begin{block}{}
\begin{itemize} [<+->]
\item text
\end{itemize}
\end{block}

\end{frame}

\subsection{Bibliography}

\begin{frame}[allowframebreaks]
\frametitle{Further Reading}
\nociteSeight{BoersmaWeenink}
\nocite{BoersmaWeenink}
\nociteSeight{Boersma2001}
\nocite{Boersma2001}
\bibliographystyleSeight{plainnat}
\tiny{
\bibliographySeight{SpraakherkenningEnSynthese}
}
\end{frame}

\appendix
\begin{frame}
\frametitle{Appendix A}

\end{frame}


% Insert License text
\insertgnugpl


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\lecture{bibliography}{bibliography}
\renewcommand{\multicite}[1]{\cite{#1}}
\renewcommand{\copyrightauthor}[0]{David Weenink and R.J.J.H. van Son}
\renewcommand{\licensenumber}[0]{LicenseAll}

\renewcommand{\refname}{Bibliography}
\begin{frame}[allowframebreaks]
\frametitle{Bibliography}
\tiny{
\bibliographystyle{plainnat}
\bibliography{SpraakherkenningEnSynthese}
}
\end{frame}	

% Insert License text
\insertgnugpl


\end{document}
