/* praat_FFNet_init.c
 *
 * Copyright (C) 1994-2002 David Weenink
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or (at
 * your option) any later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 */

/*
 djmw 20020408 GPL
 djmw 20020408 added FFNet_help
*/

#include <math.h>
#include "praat.h"
#include "Minimizers.h"
#include "FFNet_Eigen.h"
#include "FFNet_Matrix.h"
#include "FFNet_Pattern.h"
#include "FFNet_Activation_Categories.h"
#include "FFNet_Pattern_Activation.h"
#include "FFNet_Pattern_Categories.h"

FORM (FFNet_createIrisExample, "Create iris example", "FFNet")
	LABEL ("", "For the feedforward neural net")
	INTEGER ("Number of hidden nodes", "0")
	OK
DO
	long nHidden = GET_INTEGER ("Number of hidden nodes");
	REQUIRE (nHidden >= 0, "Number of hidden nodes must be greate than or equal to zero.")
	NEW (FFNet_createIrisExample (nHidden))
END

/**************** New FFNet ***************************/

FORM (FFNet_create, "Create a Feedforward Neural Net", "New: Create Feedforward Net...")
	WORD ("Name", "1001")
    NATURAL ("Number of input units", "1")
    NATURAL ("Number of output units", "1")
    INTEGER ("Number of units in 1-st hidden layer", "0")
    INTEGER ("Number of units in 2-nd hidden layer", "0")
    BOOLEAN ("Outputs are linear", 0)
    OK
DO
    long noLayers = 0, noUnitsInLayer[5],
	noInputs = GET_INTEGER ("Number of input units"),
	noOutputs = GET_INTEGER ("Number of output units"),
	noHidden1 = GET_INTEGER ("Number of units in 1-st hidden layer"),
	noHidden2 = GET_INTEGER ("Number of units in 2-nd hidden layer");
    REQUIRE (noHidden1 >= 0, "Number of hidden units >= 0.")
    REQUIRE (noHidden2 >= 0, "Number of hidden units >= 0.")
    noUnitsInLayer[noLayers] = noInputs;
    if (noHidden1 > 0) noUnitsInLayer[++noLayers] = noHidden1;
    if (noHidden2 > 0) noUnitsInLayer[++noLayers] = noHidden2;
    noUnitsInLayer[++noLayers] = noOutputs;
    if (! praat_new (FFNet_create (noLayers, noUnitsInLayer, NULL, 
		GET_INTEGER ("Outputs are linear"), FFNet_NONLIN_SIGMOID),
		GET_STRING ("Name"))) return 0;
END

/**************** New Pattern ***************************/

FORM (Pattern_create, "Create a Pattern with zeroes", 0)
	WORD ("Name", "1x1")
    NATURAL ("Dimension of a pattern", "1")
    NATURAL ("Number of patterns", "1")
    OK
DO
	if (! praat_new (Pattern_create (GET_INTEGER ("Number of patterns"), 
		GET_INTEGER ("Dimension of a pattern")), GET_STRING ("Name"))) return 0;
END

/**************** New Categories ***************************/

FORM (Categories_create, "Create Categories", "")
	WORD ("Name", "empty")
	OK
DO
	if (! praat_new (Categories_create (), GET_STRING ("Name"))) return 0;
END

DIRECT (FFNet_help)
	Melder_help ("FFNet");
END

FORM (FFNet_reset, "Reset the FFNet", "FFNet: Reset...")
    POSITIVE ("Range for weights", "0.1")
    LABEL (""," (weights will be random numbers in interval [-range, +range])")
    OK
DO
    WHERE (SELECTED)
    {
    	FFNet_reset (OBJECT, GET_REAL ("Range for weights"));
		praat_dataChanged (OBJECT);
	}
END

FORM (FFNet_selectBiasesInLayer, "Select all weights to biases in a layer", 0)
	NATURAL ("Layer", "1")
	OK
DO
	WHERE (SELECTED) 
	{
		FFNet_selectBiasesInLayer (OBJECT, GET_INTEGER ("Layer"));
		praat_dataChanged (OBJECT);
	}
END

DIRECT (FFNet_selectAllWeights)
    WHERE (SELECTED)
    {
    	FFNet_selectAllWeights (OBJECT);
		praat_dataChanged (OBJECT);
	}
END

DIRECT (FFNet_drawTopology)
    EVERY_DRAW (FFNet_drawTopology (OBJECT, GRAPHICS))
END

FORM (FFNet_drawWeightsToLayer, "Draw weights to layer", "FFNet: Draw weights to layer...")
    NATURAL ("Connections to layer", "1")
	RADIO ("Scale", 1)
	RADIOBUTTON ("by maximum of all weights to layer")
	RADIOBUTTON ("by maximum weight from 'from-unit'")
	RADIOBUTTON ("by maximum weight to 'to-unit'")
    BOOLEAN ("Garnish", 1)
    OK
DO
    EVERY_DRAW (FFNet_drawWeightsToLayer (OBJECT, GRAPHICS, GET_INTEGER ("Connections to layer"),
    	GET_INTEGER ("Scale"), GET_INTEGER ("Garnish")))
END

FORM (FFNet_drawCostHistory, "Draw cost history", "FFNet: Draw cost history...")
    INTEGER ("From iteration", "0")
    INTEGER ("To iteration", "0")
    REAL ("From cost", "0.0")
    REAL ("To cost", "0.0")
    BOOLEAN ("Garnish", 1)
    OK
DO
    EVERY_DRAW (FFNet_drawCostHistory (OBJECT, GRAPHICS,
	GET_INTEGER ("From iteration"), GET_INTEGER ("To iteration"),
	GET_REAL ("From cost"), GET_REAL ("To cost"), GET_INTEGER ("Garnish")))
END

FORM (FFNet_weightsToMatrix, "Weights that connect to layer to Matrix ", 0)
    NATURAL ("Layer", "1")
    OK
DO
    EVERY_TO (FFNet_weightsToMatrix (OBJECT, GET_INTEGER ("Layer"), 0))
END

/******************* FFNet && Activation *************************************/

FORM (FFNet_Activation_to_Categories, "From Neural Net Activation to Categories", 0)
	RADIO ("Kind of labeling", 1)
	RADIOBUTTON ("Winner-takes-all")
	RADIOBUTTON ("Stochastic")
	OK
DO
	char name [200];
	praat_name2 (name, classFFNet, classActivation);
	if (! praat_new (FFNet_Activation_to_Categories (ONLY (classFFNet), ONLY (classActivation),
		GET_INTEGER ("Kind of labeling")), name)) return 0;
END

/******************* FFNet && Eigen ******************************************/

FORM (FFNet_Eigen_drawIntersection, "Intersection of hyperplanes with eigenspace", 0)
    INTEGER ("X-component", "1")
    INTEGER ("Y-component", "2")
    REAL ("xmin", "0.0")
    REAL ("xmax", "0.0")
    REAL ("ymin", "0.0")
    REAL ("ymax", "0.0")
    OK
DO
    long pcx = GET_INTEGER ("X-component"), pcy = GET_INTEGER ("Y-component");
    REQUIRE (pcx != 0 && pcy != 0, "X and Y component must differ from 0.")
    praat_picture_open ();
    FFNet_Eigen_drawIntersection (ONLY (classFFNet), ONLY (classEigen),
    	GRAPHICS, pcx, pcy, GET_REAL ("xmin"), GET_REAL ("xmax"),
    	GET_REAL ("ymin"), GET_REAL ("ymax"));
    praat_picture_close ();
END

/************************* FFNet && Categories **********************************/

DIRECT (FFNet_Categories_to_Activation)
	NEW (FFNet_Categories_to_Activation (ONLY (classFFNet), ONLY (classCategories)))
END

/************************* FFNet && Matrix **********************************/

FORM (FFNet_weightsFromMatrix, "Replace weights by values from Matrix", 0)
    NATURAL ("Layer", "1")
    OK
DO
    NEW (FFNet_weightsFromMatrix (ONLY (classFFNet), ONLY (classMatrix),
    	GET_INTEGER ("Layer")));
END

/************************* FFNet && Pattern **********************************/

FORM (FFNet_Pattern_drawActivation, "Draw an activation", 0)
	NATURAL ("Pattern (row) number", "1");
	OK
DO
	EVERY_DRAW (FFNet_Pattern_drawActivation (ONLY (classFFNet), ONLY (classPattern),
		GRAPHICS, GET_INTEGER ("Pattern")))
END

FORM (FFNet_Pattern_to_Activation, "To activations in layer", 0)
	NATURAL ("Layer", "1")
	OK
DO
	char name [200];
	praat_name2 (name, classFFNet, classPattern);
	NEW (FFNet_Pattern_to_Activation (ONLY (classFFNet),
		ONLY (classPattern), GET_INTEGER ("Layer")))
END

FORM (FFNet_Pattern_to_Categories, "Classify Pattern input to Categories", "FFNet & Pattern: To Categories...")
	RADIO ("Kind of labeling", 1)
	RADIOBUTTON ("Winner-takes-all")
	RADIOBUTTON ("Stochastic")
	OK
DO
	char name [200];
	praat_name2 (name, classFFNet, classPattern);
	if (! praat_new (FFNet_Pattern_to_Categories (ONLY (classFFNet),
		ONLY (classPattern), GET_INTEGER ("Kind of labeling")), name)) return 0;
END

/*********** FFNet Pattern Activation **********************************/

FORM (FFNet_Pattern_Activation_learnPR, "Learn with Polak Ribiere's conjugate gradient algorithm", 0)
    NATURAL ("Maximum number of epochs", "100")
    POSITIVE ("Tolerance of minimizer", "1e-7")
	RADIO ("Cost function", 1)
	RADIOBUTTON ("minimum squared error")
	RADIOBUTTON ("minimum cross entropy")
    OK
DO
    return FFNet_Pattern_Activation_learnPR (ONLY (classFFNet), ONLY (classPattern),
    	ONLY (classActivation), GET_INTEGER ("Maximum number of epochs"),
		GET_REAL ("Tolerance of minimizer"), NULL, GET_INTEGER ("Cost function")); 
END

FORM (FFNet_Pattern_Activation_learnFP, "Learn with Fletcher Powell algorithm", 0)
    NATURAL ("Maximum number of epochs", "100")
    POSITIVE ("Tolerance of minimizer", "1e-7")
	RADIO ("Cost function", 1)
	RADIOBUTTON ("minimum squared error")
	RADIOBUTTON ("minimum cross entropy")
    OK
DO
    return FFNet_Pattern_Activation_learnFP (ONLY (classFFNet), ONLY (classPattern),
    	ONLY (classActivation), GET_INTEGER ("Maximum number of epochs"),
		GET_REAL ("Tolerance of minimizer"), NULL, GET_INTEGER ("Cost function")); 
END

FORM (FFNet_Pattern_Activation_learnSD, "Learn with steepest descent algorithm", 0)
    NATURAL ("Maximum number of epochs", "100")
    POSITIVE ("Tolerance of minimizer", "1e-7")
    LABEL ("Specifics", "Specific for this minimization")
    POSITIVE ("Learning rate", "0.1")
    REAL ("Momentum", "0.9")
	RADIO ("Cost function", 1)
	RADIOBUTTON ("minimum squared error")
	RADIOBUTTON ("minimum cross entropy")
    OK
DO
	struct structSteepestDescentMinimizer_parameters p;
    p.eta = GET_REAL ("Learning rate");
    p.momentum = GET_REAL ("Momentum");
    return FFNet_Pattern_Activation_learnSD (ONLY (classFFNet), ONLY (classPattern),
    	ONLY (classActivation), GET_INTEGER ("Maximum number of epochs"),
		GET_REAL ("Tolerance of minimizer"), & p, GET_INTEGER ("Cost function")); 
END

FORM (FFNet_Pattern_Activation_learnSM, 
	"Learn with conjugate gradient algorithm", 0)
    NATURAL ("Maximum number of epochs", "100")
    POSITIVE ("Tolerance of minimizer", "1e-7")
	RADIO ("Cost function", 1)
	RADIOBUTTON ("minimum squared error")
	RADIOBUTTON ("minimum cross entropy")
    OK
DO
    return FFNet_Pattern_Activation_learnSM (ONLY (classFFNet), 
		ONLY (classPattern), ONLY (classActivation), 
		GET_INTEGER ("Maximum number of epochs"),
		GET_REAL ("Tolerance of minimizer"), NULL, 
		GET_INTEGER ("Cost function")); 
END


/*********** FFNet Pattern Categories **********************************/


FORM (Pattern_Categories_to_FFNet, "Create a Feedforward Neural Net", 
	"Pattern & Categories: To Feedforward Net...")
    INTEGER ("Number of units in 1-st hidden layer", "0")
    INTEGER ("Number of units in 2-nd hidden layer", "0")
    BOOLEAN ("Outputs are linear", 0)
    OK
DO
	Pattern p = ONLY (classPattern); 
	Categories ucategories = NULL, l = ONLY (classCategories);
	long nLayers = 0, nOutputs, nUnitsInLayer[5]; FFNet ffnet = NULL;
	long nHidden1 = GET_INTEGER ("Number of units in 1-st hidden layer");
	long nHidden2 = GET_INTEGER ("Number of units in 2-nd hidden layer");
    REQUIRE (nHidden1 >= 0, "Number of hidden units >= 0.")
    REQUIRE (nHidden2 >= 0, "Number of hidden units >= 0.")
	REQUIRE ( (ucategories = Categories_selectUniqueItems (l, 1)),
		"No memory for categories.")
	nOutputs = ucategories->size;
	REQUIRE (nOutputs > 1, "Number of categories > 1.")
	nUnitsInLayer[0] = p->nx;
    if (nHidden1 > 0) nUnitsInLayer[++nLayers] = nHidden1;
    if (nHidden2 > 0) nUnitsInLayer[++nLayers] = nHidden2;
    nUnitsInLayer[++nLayers] = nOutputs;
    ffnet = FFNet_create (nLayers, nUnitsInLayer, ucategories, 
		GET_INTEGER ("Outputs are linear"), FFNet_NONLIN_SIGMOID);
	forget (ucategories);
	if (! praat_new (ffnet, NULL)) return 0;
END

FORM (FFNet_Pattern_Categories_learnPR, "Learn with Polak Ribiere's conjugate "
	"gradient algorithm", "Pattern & FFNet & Categories: Learn...")
    NATURAL ("Maximum number of epochs", "100")
    POSITIVE ("Tolerance of minimizer", "1e-7")
	RADIO ("Cost function", 1)
	RADIOBUTTON ("minimum squared error")
	RADIOBUTTON ("minimum cross entropy")
    OK
DO
    return FFNet_Pattern_Categories_learnPR (ONLY (classFFNet), 
		ONLY (classPattern), ONLY (classCategories), 
		GET_INTEGER ("Maximum number of epochs"),
		GET_REAL ("Tolerance of minimizer"), NULL, 
		GET_INTEGER ("Cost function")); 
END

FORM (FFNet_Pattern_Categories_learnFP, "Learn with Fletcher Powell algorithm", 
	"Pattern & FFNet & Categories: Learn...")
    NATURAL ("Maximum number of epochs", "100")
    POSITIVE ("Tolerance of minimizer", "1e-7")
	RADIO ("Cost function", 1)
	RADIOBUTTON ("minimum squared error")
	RADIOBUTTON ("minimum cross entropy")
    OK
DO
    return FFNet_Pattern_Categories_learnFP (ONLY (classFFNet), 
		ONLY (classPattern), ONLY (classCategories), 
		GET_INTEGER ("Maximum number of epochs"),
		GET_REAL ("Tolerance of minimizer"), NULL, 
		GET_INTEGER ("Cost function")); 
END

FORM (FFNet_Pattern_Categories_learnSM, "Learn with conjugate gradient "
	"algorithm", "Pattern & FFNet & Categories: Learn...")
    NATURAL ("Maximum number of epochs", "100")
    POSITIVE ("Tolerance of minimizer", "1e-7")
	RADIO ("Cost function", 1)
	RADIOBUTTON ("minimum squared error")
	RADIOBUTTON ("minimum cross entropy")
    OK
DO
    return FFNet_Pattern_Categories_learnSM (ONLY (classFFNet), 
		ONLY (classPattern), ONLY (classCategories), 
		GET_INTEGER ("Maximum number of epochs"),
		GET_REAL ("Tolerance of minimizer"), NULL, 
		GET_INTEGER ("Cost function")); 
END

FORM (FFNet_Pattern_Categories_learnSD, "Learn with steepest descent algorithm",
	"Pattern & FFNet & Categories: Learn slow...")
    NATURAL ("Maximum number of epochs", "100")
    POSITIVE ("Tolerance of minimizer", "1e-7")
    LABEL ("Specifics", "Specific for this minimization")
    POSITIVE ("Learning rate", "0.1")
    REAL ("Momentum", "0.9")
	RADIO ("Cost function", 1)
	RADIOBUTTON ("minimum squared error")
	RADIOBUTTON ("minimum cross entropy")
    OK
DO
	struct structSteepestDescentMinimizer_parameters p;
	p.eta = GET_REAL ("Learning rate");
    p.momentum = GET_REAL ("Momentum");
    return FFNet_Pattern_Categories_learnSD (ONLY (classFFNet), 
		ONLY (classPattern), ONLY (classCategories), 
		GET_INTEGER ("Maximum number of epochs"),
		GET_REAL ("Tolerance of minimizer"), &p, 
		GET_INTEGER ("Cost function")); 
END

void praat_uvafon_FFNet_init (void);
void praat_uvafon_FFNet_init (void)
{
	Thing_recognizeClassesByName (classFFNet, NULL);
	
    praat_addMenuCommand ("Objects", "New", "Neural nets", 0, 0, 0);
	praat_addMenuCommand ("Objects", "New", "FFNet tutorial", 0, 1,
		DO_FFNet_help);
    praat_addMenuCommand ("Objects", "New", "-- FFNet --", 0, 1, 0);		
    praat_addMenuCommand ("Objects", "New", "Create Feedforward Net...", 
		0, 1, DO_FFNet_create);
    praat_addMenuCommand ("Objects", "New", "Create Pattern with zeroes...",	
		0, 1, DO_Pattern_create);
    praat_addMenuCommand ("Objects", "New", "Create empty Categories...", 0, 1,
		DO_Categories_create);
    praat_addMenuCommand ("Objects", "New", "Create iris example...", 
		0, 1, DO_FFNet_createIrisExample);
        
	praat_addAction1 (classFFNet, 0, "FFNet help", 0, 0, DO_FFNet_help);
	praat_addAction1 (classFFNet, 0, "Draw", 0, 0, 0);
	praat_addAction1 (classFFNet, 0, "Draw topology", 0, 0, 
		DO_FFNet_drawTopology);
	praat_addAction1 (classFFNet, 0, "Draw weights to layer...", 0, 0, 
		DO_FFNet_drawWeightsToLayer);
	praat_addAction1 (classFFNet, 0, "Draw cost history...", 0, 0, 
		DO_FFNet_drawCostHistory);
	praat_addAction1 (classFFNet, 0, "Modify", 0, 0, 0);
	praat_addAction1 (classFFNet, 1, "Reset...", 0, 0, DO_FFNet_reset);
	praat_addAction1 (classFFNet, 0, "Select biases...", 0, 0, 
		DO_FFNet_selectBiasesInLayer);
	praat_addAction1 (classFFNet, 0, "Select all weights", 0, 0, 
		DO_FFNet_selectAllWeights);
	praat_addAction1 (classFFNet, 0, "Extract", 0, 0, 0);
	praat_addAction1 (classFFNet, 0, "Weights to Matrix...", 0, 0, 
		DO_FFNet_weightsToMatrix);
	
	praat_addAction2 (classFFNet, 1, classActivation, 1, "Analyse", 0, 0, 0);
	praat_addAction2 (classFFNet, 1, classActivation, 1, "To Categories...", 
		0, 0, DO_FFNet_Activation_to_Categories);
	
	praat_addAction2 (classFFNet, 1, classEigen, 1, "Draw", 0, 0, 0);
	praat_addAction2 (classFFNet, 1, classEigen, 1, 
		"Draw hyperplane intersections", 0, 0, DO_FFNet_Eigen_drawIntersection);
	
	praat_addAction2 (classFFNet, 1, classCategories, 1, "Analyse", 0, 0, 0);
	praat_addAction2 (classFFNet, 1, classCategories, 1, "To Activation", 0, 0, 
		DO_FFNet_Categories_to_Activation);
	
	praat_addAction2 (classFFNet, 1, classMatrix, 1, "Modify", 0, 0, 0);
	praat_addAction2 (classFFNet, 1, classMatrix, 1, "Weights from Matrix...", 
		0, 0, DO_FFNet_weightsFromMatrix);
	
	praat_addAction2 (classFFNet, 1, classPattern, 1, "Draw", 0, 0, 0);
	praat_addAction2 (classFFNet, 1, classPattern, 1, "Draw activation...", 
		0, 0, DO_FFNet_Pattern_drawActivation);
	praat_addAction2 (classFFNet, 1, classPattern, 1, "Analyse", 0, 0, 0);
	praat_addAction2 (classFFNet, 1, classPattern, 1, "To Activation...", 0, 0, 
		DO_FFNet_Pattern_to_Activation);
	praat_addAction2 (classFFNet, 1, classPattern, 1, 
		"To Categories (classify)...", 0, 0, DO_FFNet_Pattern_to_Categories);
	
	praat_addAction2 (classPattern, 1, classCategories, 1, 
		"To Feedforward Net...", 0, 0, DO_Pattern_Categories_to_FFNet);
	
	praat_addAction (classFFNet, 1, classPattern, 1, classActivation, 1, 
		"Learn", 0, 0, 0);
	praat_addAction (classFFNet, 1, classPattern, 1, classActivation, 1, 
		"Learn (PR)...", 0, 0, DO_FFNet_Pattern_Activation_learnPR);
	praat_addAction (classFFNet, 1, classPattern, 1, classActivation, 1, 
		"Learn (FP)...", 0, 0, DO_FFNet_Pattern_Activation_learnFP);
	praat_addAction (classFFNet, 1, classPattern, 1, classActivation, 1, 
		"Learn (SM)...", 0, 0, DO_FFNet_Pattern_Activation_learnSM);
	praat_addAction (classFFNet, 1, classPattern, 1, classActivation, 1, 
		"Learn slow...", 0, 0, DO_FFNet_Pattern_Activation_learnSD);
	
	praat_addAction (classFFNet, 1, classPattern, 1, classCategories, 1, 
		"Learn", 0, 0, 0);
	praat_addAction (classFFNet, 1, classPattern, 1, classCategories, 1, 
		"Learn (PR)...", 0, 0, DO_FFNet_Pattern_Categories_learnPR);
	praat_addAction (classFFNet, 1, classPattern, 1, classCategories, 1, 
		"Learn (FP)...", 0, 0, DO_FFNet_Pattern_Categories_learnFP);
	praat_addAction (classFFNet, 1, classPattern, 1, classCategories, 1, 
		"Learn (SM)...", 0, 0, DO_FFNet_Pattern_Categories_learnSM);
	praat_addAction (classFFNet, 1, classPattern, 1, classCategories, 1, 
		"Learn slow...", 0, 0, DO_FFNet_Pattern_Categories_learnSD);
    
    INCLUDE_MANPAGES (manual_FFNet_init)
	
}