#!/usr/bin/perl
#
# Extract all URLs from the bbl files and wget them into
# the Literature subdirectory
#
push(@ARGV, './') unless @ARGV;

my $GetSiteCommand = 
"wget -N --mirror --no-parent --page-requisites  --non-verbose --convert-links --html-extension ";
my @ignorelist = ('http://www.plos.org/', 'http://www.abelard.org', 'http://homepages.inf.ed.ac.uk/s0450736/');
foreach my $directory (@ARGV)
{
    # Create Literature subdirectory
    $directory .= '/' unless $directory =~ m!/$!;
    
    # Get all bbl files
    my @filelist = glob("{$directory}*.bbl");
    foreach $file (@filelist)
    {
        $file =~ m!(^|/)([^/]+)\.[^\.]+$!;
        my $basename = ($2 || $file);
        my $targetdir = "${directory}Literature/$basename";
        `mkdir -pv $targetdir`;
        
        # open bbl file and read all URLs
        my @URLlines = ();
        my $currentURL = "";
        open(BBLFILE, $file) || die "$file: $!\n";
        while(<BBLFILE>)    # Watch out for misconfigured files
        {
            chomp;
            # Ignore non URL lines
            next if ($_ !~ /\\url\{/) && (length($currentURL) == 0);
            $currentURL .= $_;
            $currentURL =~ s/\%\s*$//g;
            $currentURL =~ s/\s+$//g;
            
            # get the URL if it is complete
            if($currentURL =~  /\\url\{([^\}]+)\}/)
            {
                push(@URLlines, $1);
                $currentURL = "";
            }
        };
        close(BBLFILE);
        foreach my $url (@URLlines)
        {
            next if grep(/^$url$/, @ignorelist);
            `$GetSiteCommand --quiet --directory-prefix=${targetdir}/ $url`;
        };
    };
};

