#!/usr/bin/perl -w
# Australian TV Guide XMLTV grabber by Damon Searle
# Derived from a yahoo XMLTV grabber by Ron Kellam which was itself...
# Derived from original code by Justin Hawkins
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

# 30 Oct 2004
#  Damon Searle <djsearle@netspace.net.au>
#  - wrote first version
#  - gets data from NineMSN as a backup. Its not that fancy,
# 31 Oct 2004
#  Fred Donelly <fdonelly@hotmail.com>
#  - added an option so that the output file can be specified on the
#    command line and from the quick test I gave it, it now works with
#    mythfilldatabase.
#  - $offset set to +1000 at the top and then had "+1000" set in a
#  output string further down rather than the variable
# 4 Nov 2004
#  Paul Andreassen <paulx@andreassen.com.au>
#  - learned some perl and now wants to go back to python
#  - added and then reduced status info
#  - retry on failure to getstore
#  - changed cache to '/var/local/tv_grab_au'
#  - added threading for each day
# 5 Nov 2004
#  - improved threading with use of queue
#  Eyal Lebedinsky <eyal@eyal.emu.id.au>
#  - easier location selection
# 8 Nov 2004
#  Paul
#  - fixed pid=0 bug
#  - did some merging, I hate merging
# 9 Nov 2004
#  Rob Hill <rob@dot.net.au>
#  - added Sydney
# 10 Nov 2004
#  Mary Wright <mwright@taz-devil.dyndns.org>
#  - digital info for Sydney
#  Paul
#  - more cleanup and improved error checking
#  - used mirror instead of getstore to get any updates
#  - mirror didn't work replaced with own smarts to check for updates to times
# 11 Nov 2004
#  - added program name in check
# 13 Nov 2004
#  - added freesd for Brisbane
# 14 Nov 2004
#  - --configure to exit nicely
#  - if no program data then skip program nicely, mainly for foxtel data
#  - added foxtel channels
# 17 Nov 2004
#  Michael Cheshire <michael-mlists@cheshire.id.au>
#  - added remaining foxtel channels
#  Eyal Lebedinsky <eyal@eyal.emu.id.au>
#  - Fix misspelling Unknows -> Unknown
#  - Note: is Sydney now is on summer time +1100?
# 28 Nov 2004
#  Paul
#  - converts &amp; -> &, &quot; -> "
#  - option to change all & to and
#  - added data from yesterday to fill 0:00 to 6:00
#  - deletes old cache days, may miss some if not run for a while
# 1 Dec 2004
#  Mark Spieth with changes by Paul
#  - offset
#  - melbourne
#  - rated
# 10 Dec 2004
#  Paul
#  - added Queensland
#  - another attempt at offset calculation
#  - start and stop time cleanup
# 11 Dec 2004
#  - more logical offset calculation
#  - popular browser pretence
# 13 Dec 2004
#  - simple fix for no pids
#  - added location and cache-dir options
#  - added channel renaming option
# 21 Dec 2004
#  - connection cache activated

use strict;
use Getopt::Long;
use XMLTV;
use LWP::Simple;
use LWP::Simple qw($ua);
use Date::Manip;   # timezone setup and calculations aren't correct
use Time::Local;
use File::Path;
use threads;
use Thread::Queue;

# Instructions:
# Select your region and source.
# If your location isn't listed below, go to
# http://tvguide.ninemsn.com.au/guide/ select your area
# look at the last number in the URL before ".asp" and set
# the region variable below. Then put the channel names as listed
# on the tv guide site into the variables below.
# Then set your XMLTV ids from the database in the XMLTVID_URL variable.
#
# If it doesn't work with mythfilldatabase, try:
# ./tv_grab_au
# mythfilldatabase --file 1 -1 /var/local/tv_grab_au/guide.xml

# pick your region
#
#my $location = "Canberra";
#my $location = "Brisbane";
#my $location = "Queensland";
my $location = "Sydney";
#my $location = "Melbourne";
#my $location = "Australia";

# pick your source
#
my $source = "free";
#my $source = "freesd";
#my $source = "freehd";
#my $source = "foxtel";

# choose the XMLID URL suffix that mythtv knows
#
my $XMLTVID_URL = "d1.com.au";
#my $XMLTVID_URL = "tv_grab_au.com.au";

# change to how you think it should work
my $days_to_grab = 8;
my $threads = 5;
my $retrys = 3;
my $seconds_before_retry = 2;
my $convert_ands = 0;
my $days_to_clean = 7;
my $cache_dir = "/var/local/tv_grab_au";
$ua->agent("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)");
#$ua->agent("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.5) Gecko/20041107 Firefox/1.0");
#$ua->agent("tv_grab_au/2004.12.13 (Linux; Unix; en-AU) Perl/5.8.6";
#execute 'export PERL_LWP_USE_HTTP_10=1' before script for http 1.0 support

# Options
my $opt_days; my $opt_output; my $opt_configfile; my $opt_configure = 0;
my $opt_location; my $opt_cachedir; my $opt_rename = 0; my %rename_channels;

GetOptions('days=i'   => \$opt_days,
   'output=s'   => \$opt_output,
   'config-file=s'   => \$opt_configfile,
   'configure'   => \$opt_configure,
   'location=s'   => \$opt_location,
   'cache-dir=s'   => \$opt_cachedir,
   'rename'   => \$opt_rename,
);
if ($opt_days) {
   $days_to_grab = $opt_days
}
if (!($opt_output)) {
   $opt_output = $cache_dir . "/guide.xml";
}
# $opt_configfile should probably do something ('/home/mythtv/.mythtv/tv_grab_au.xmltv')
if ($opt_configure == 1) {
        print "configuration must be done in this script at $0\n";
        exit (0);
}
if ($opt_location) {
   $location = $opt_location;
}
if ($opt_cachedir) {
   $cache_dir = $opt_cachedir;
}
if ($opt_rename == 1) {
   $rename_channels{"ABC VIC"}="ABC TV Melbourne";
   $rename_channels{"Channel Seven Melbourne"}="7 Digital";
   $rename_channels{"SBS Melbourne"}="SBS DIGITAL 1";
   $rename_channels{"Network TEN Melbourne"}="TEN Digital";
   $rename_channels{"Channel Nine Melbourne"}="Nine Digital";
}
print "grabing $location $source for $days_to_grab days into $opt_output\n";

# Setup
&Date_Init("TZ=UT");   # Date::Manip timezone setup and calculations aren't correct so disable
my $today = &UnixDate("today", "%Y%m%d");
my $yesterday = &DateCalc($today, "-1 day");

my $conn_cache ||= { total_capacity => ($threads + 2) };
$ua->conn_cache($conn_cache);

my $guide_url = "http://tvguide.ninemsn.com.au/guide/";
my $details_url = "http://tvguide.ninemsn.com.au/cu/default.asp?pid=";
my $XMLTV_prefix = $source . "." . $location . ".";
my $XMLTV_suffix = "." . $XMLTVID_URL;

my $region; my %channels;
if ("Canberra" eq $location) {
   $region = "126";
   if ("free" eq $source) {
      $channels{"ABC NSW"}="2";
      $channels{"Prime Southern"}="PrimS";
      $channels{"SBS Sydney"}="SBS";
      $channels{"Southern Cross TEN Capital"}="10Cap";
      $channels{"WIN Television NSW"}="WIN"
   } elsif ("freesd" eq $source or "freehd" eq $source) {
      $channels{"ABC NSW"}="2";
      $channels{"Prime Southern"}="7";
      $channels{"SBS Sydney"}="SBS";
      $channels{"Southern Cross TEN Capital"}="10";
      $channels{"WIN Television NSW"}="9"
   } else {
      print "Unknown source '$source' for $location\n";
      exit (1);
   }
} elsif ("Brisbane" eq $location) {
   $region = "75";
   if (("free" eq $source)||("freesd" eq $source)) {
      $channels{"ABC QLD"}="2";
      $channels{"Channel Seven Brisbane"}="7";
      $channels{"SBS Queensland"}="SBS";
      $channels{"Network TEN Brisbane"}="10";
      $channels{"Channel Nine Brisbane Metro"}="9";
   } else {
      print "Unknown source '$source' for $location\n";
      exit (1);
   }
} elsif ("Queensland" eq $location) {
   $region = "79";
   if (("free" eq $source)||("freesd" eq $source)) {
      $channels{"ABC QLD"}="2";
      $channels{"Channel Seven Queensland"}="7";
      $channels{"SBS Queensland"}="SBS";
      $channels{"Southern Cross TEN Queensland"}="10";
      $channels{"WIN Television QLD"}="9";
   } else {
      print "Unknown source '$source' for $location\n";
      exit (1);
   }
} elsif ("Sydney" eq $location) {
   $region = "73";
   if ($source =~ m/^(free|freesd|freehd)$/) {
      $channels{"ABC NSW"}="2";
      $channels{"ABC2"}="21";
      $channels{"Channel Seven Sydney"}="7";
      $channels{"SBS Sydney"}="SBS";
      $channels{"SBS News"}="33";
      $channels{"Network TEN Sydney"}="10";
      $channels{"Channel Nine Sydney"}="9";
   } else {
      print "Unknown source '$source' for $location\n";
      exit (1);
   }
} elsif ("Melbourne" eq $location) {
   $region = "94";
   if (("free" eq $source)||("freesd" eq $source)||("freehd" eq $source)) {
      $channels{"ABC VIC"}="2";
      $channels{"Channel Seven Melbourne"}="7";
      $channels{"SBS Melbourne"}="SBS";
      $channels{"Network TEN Melbourne"}="10";
      $channels{"Channel Nine Melbourne"}="9";
   } else {
      print "Unknown source '$source' for $location\n";
      exit (1);
   }
} elsif ("Adelaide" eq $location) {
   $region = "81";
   if (("free" eq $source)||("freesd" eq $source)) {
      $channels{"ABC SA"}="2";
      $channels{"Channel Seven Adelaide"}="7";
      $channels{"SBS SA"}="SBS";
      $channels{"Network TEN Adelaide"}="10";
      $channels{"Channel Nine Adelaide"}="9";
   } else {
      print "Unknown source '$source' for $location\n";
      exit (1);
   }
} elsif ("Australia" eq $location) {
   $region = "123";
   if ("foxtel" eq $source) {
      $channels{"Arena TV"}="Arena";
      $channels{"BBC World"}="BBC";
      $channels{"Cartoon Network"}="Cartoon";
      $channels{"Channel [V]"}="Red";
      $channels{"CNBC"}="CNBC";
      $channels{"CNN"}="CNN";
      $channels{"Discovery Channel"}="Disc";
      $channels{"FOX News"}="FoxFNC";
      $channels{"FOX8"}="FOX";
      $channels{"MAX"}="FoxMMX";
      $channels{"National Geographic Channel"}="NatGe";
      $channels{"Nickelodeon"}="Nick";
      $channels{"Showtime"}="Show";
      $channels{"Showtime 2"}="FoxSH2";
      $channels{"Sky News"}="SkyNews";
      $channels{"TV1"}="TV1";
      $channels{"UKTV"}="UKTV";
      $channels{"Showtime Greats"}="ShowGreats";
      $channels{"World Movies"}="wmov";
      $channels{"WCH"}="WCH";
      $channels{"TVSN"}="TVSN";
      $channels{"Sky Racing"}="SkyRa";
      $channels{"Ovation"}="Ovation";
      $channels{"Disney Channel"}="Disney";
      $channels{"Animal Planet"}="Animal";
      $channels{"The Comedy Channel"}="Com";
      $channels{"The LifeStyle Channel"}="Lifes";
      $channels{"FOX Sports 1"}="FoxFS1";
      $channels{"Movie One"}="Movie1";
      $channels{"TCM"}="TCM";
      $channels{"MTV"}="MTV";
      $channels{"FOX Sports 2"}="FoxSP2";
      $channels{"FOX Footy Channel"}="FFC";
      $channels{"Movie Extra"}="MovieEx";
      $channels{"Hallmark Channel"}="Hall";
      $channels{"The History Channel"}="FoxHST";
      $channels{"ESPN"}="ESPN";
      $channels{"FOX Classics"}="FoxCLA";
      $channels{"Movie Greats"}="MovieGr";
   } else {
      print "Unknown source '$source' for $location\n";
      exit (1);
   }
} else {
   print "Unknown location '$location'\n";
   exit (1);
}

my $prog_ref; my $chan_ref;
foreach my $channel (keys %channels) {
   my $name = $channel;
   if (exists($rename_channels{$channel})) {
      $name = $rename_channels{$channel};
   }
   $$chan_ref{$channel} = {
      'id' => $XMLTV_prefix . $channels{$channel} . $XMLTV_suffix,
      'display-name' => [ [ $name, undef ]]
   };
}

# Program
print "starting $threads threads\n";
my @thrlist;
my $datepids = Thread::Queue->new;
for (my $thread=0; $thread<$threads; $thread++) {
        push @thrlist, threads->new(\&fetch_details);
}

print "loading queue\n";
my $currentday = $yesterday;
my $day_counter = 0;
while ($day_counter <= $days_to_grab) {
   my $date = &UnixDate($currentday, "%d%m%Y");
   my @day_lines = get_day($date,1);
   if (@day_lines == 0) {
      $currentday = &DateCalc($currentday, "+ 1 day");   
      $day_counter++;
           next;
   }

   my @pids;
   my @rowspans;
   my @names;
   foreach my $line (@day_lines) {
      foreach my $link (split /\n|tr|TR|TD|tr/, $line ) {
         if ($link =~ /<A href=.+cu\/default.asp/) {

# <td rowspan=5 onmouseover=mOvr(this) onmouseout=mOut(this)
#     onclick=mClk(this) class=tv><A href=/cu/default.asp?pid=12395882
#     onclick="pop(12395882);return false;"
#     target=_new class=tv>Thunderstone</a></TD>

            $link =~ s/.+rowspan=//;

            my $rowspan = $link;
            #$rowspan =~ s/.+rowspan=//;
            $rowspan =~ s/ .+//;

            my $name = $link;
            $name =~ s/.+target=_new class=tv>//g;
            $name =~ s/<\/a>.+//g;
         
            $link =~ s/.+pid=//g;
            #$link =~ s/".+//g; #"
            $link =~ s/ .+//;
            if (($rowspan =~ /\d+/) and ($link =~ /\d\d+/))
            {
               push @pids, $link;
               push @rowspans, $rowspan;
               push @names, $name;               
            }
         }
      }
   }

   if (changed_guide($date,@pids,@rowspans,@names)) {
      for (my $count=0; $count <= $#pids; $count++) {
         $datepids->enqueue($date . "-" . $pids[$count]);
      }
   }

   $day_counter++;
   $currentday = &DateCalc($currentday, "+ 1 day");
}

for (my $thread=0; $thread<$threads; $thread++) {
   $datepids->enqueue(0 . "-" . 0);
}
print "queue is complete\n";

foreach my $thr (@thrlist) {
    $thr->join;
}
print "all threads done\n";

print "building xml structure\n";
$currentday = $yesterday;
$day_counter = 0;
while ($day_counter <= $days_to_grab) {
   my @pids;
   my $date = &UnixDate($currentday, "%d%m%Y");

   my $guide_prn_file = $cache_dir . "/" . $date . "/guide.prn";
   if (open(PRN, $guide_prn_file)) {
      my @prn = split />/, <PRN>;
      close(PRN);
      
      if ($#prn > 1) {
         my $pidlast = ($#prn + 1)/3 - 1;
         @pids=@prn[0..$pidlast];
      } else {
         print "no pids in $guide_prn_file\n";
         @pids=();
      }
   } else {
      print "can't read $guide_prn_file\n";
      @pids=()
   }
   
   my $retry = 0;      
   foreach my $pid (@pids) {
      my @details = get_details($date, $pid);
      if (@details == 0) {
              next;
      }
      
      my $show_details_table = "";
      my $use_line = 0;
      my $close_html = 0;
      foreach my $line (@details) {
         if ($line =~ /bgColor=#f7f3e8/) {
            $use_line = 0;
         }
         if ($use_line == 1) {
            $show_details_table .= $line;
         }
         if ($line =~ /bgcolor=#ffffff/) {
            $use_line = 1;
         }
         if ($line =~ /<\/HTML>/) {
            $close_html = 1;
         }
          }

      if ($close_html == 0) {
         my $name = $cache_dir . "/" . $date . "/" . $pid . ".html";
         if ($retry++ >= $retrys) {
            print "giving up on truncated $name\n";
            $retry=0;
            next;
         }
         unlink $name;
         push @pids, $pid;
         print "t";   # truncated
         sleep($seconds_before_retry);
         next;
      }
      $retry=0;

      if ((length $show_details_table) == 0) {
         print "m";   # missing: can't do anything about this
         next;
      }
          
          $show_details_table =~ s/<[^>]*>/\n/g;
      $show_details_table =~ s/\&nbsp\;//g;
      #$show_details_table =~ s/<BR>|<TR>|<TD><B><b><\/B><\/b>/\n/g;
      #$show_details_table =~ s/Genre://g;
      #$show_details_table =~ s/Rated:/\n/g;

#my $short_details = $show_details_table;
#$short_details =~ s/\r+/ /g;
#$short_details =~ s/\n+/ /g;
#$short_details =~ s/ +/ /g;
#
#if ($show_details_table =~ m/Ozzy/)
#{
   #print "\n******************** Found Ozzy in: '$short_details'\n"
#}
      #$show_details_table =~ s/\&quot\;/"/g;

      #if ($convert_ands == 1) {
          #$show_details_table =~ s/( |)\&amp\;( |)/ and /g;
          #$show_details_table =~ s/( |)\&( |)/ and /g;
      #} else {
          #$show_details_table =~ s/\&amp\;/\&/g;
      #}
      
      my $count = 0;      
      my $time;
      my $channel = "";      
      my $title1 = "";
      my $title2 = "";
      my $duration;
      my $rating = "";
      my $genre = "";
      my $descr = "";

      #print $show_details_table. "\n\n\n";
      foreach my $line (split /\n/, $show_details_table) {
         if ($count == 4) {
            #print "Time: " . $line . "\n";
            $time = $line;
         }
         elsif ($count == 7) {
            $channel = $line;
            #print "Channel: " . $line . "\n";
         }
         elsif ($count == 19) {
            $title1 = $line;
            #print "Program: " . $line . "\n";
         }
         elsif ($count == 20) {
            $line =~ s/ - //g;
            $title2 = $line;
            #print "Subtitle: " . $line . "\n";
         }
         elsif ($count == 21) {
            $line =~ s/\D//g;
            $duration = $line;
            #print "Run time: " . $line . "\n";
         }
         elsif ($count == 22) {
            $line =~ s/^.*: (.*)\).*$/$1/;
            if ($line =~ /[a-zA-Z]/)
            {
               $rating = $line;
            }
            #print "Rating: " . $line . "\n";
         }
         elsif ($count == 26) {
            $line =~ s/ //g;
            $genre = $line;
            #print "Genre: " . $line . "\n";
         }
         elsif ($count == 28 && $line =~ /[a-zA-Z]/) {
            $descr = $line;
            #print "Description: " . $line . "\n";
         }
         #print $count .": " . $line . "\n";
         ++$count;
      }
      
      if (defined $channels{$channel}) {
         $channel = $XMLTV_prefix . $channels{$channel} . $XMLTV_suffix;
      } else {
         print "unknown channel $channel\n";
         next;
      }

      my $start=&ParseDate(&UnixDate($currentday,"%Y%m%d ") . $time);
      if (&Date_Cmp($time,"6:00") < 0) {
         $start=&DateCalc($start,"+ 1 day");
      }
      
      my $stop=&DateCalc($start,"+ " . $duration . " minutes");
      
      if (&Date_Cmp($start,$today) < 0) {   # skip programs that start before today
         next;
      }

      # Date::Manip timezone setup and calculations aren't correct so don't use
      $start = &UnixDate($start, "%q ") . get_TZ_Offset($start);
      $stop  = &UnixDate($stop, "%q ") . get_TZ_Offset($stop);
      
      my $a_prog = {
         channel => $channel,
         start   => $start,
         stop    => $stop,
         title   => [ [ $title1, undef ] ]
      };
      
      $descr =~ s/^\s+//;
      $descr =~ s/\s+$//;

      if ($title2) { $$a_prog{'sub-title'} = [ [ $title2, undef ] ]; }
      if ($rating) { $$a_prog{rating}      = [ [ $rating, "CTVA", undef ] ]; }      
      if ($genre)  { $$a_prog{category}    = [ [ $genre, undef ] ]; }
      if ($descr)  { $$a_prog{desc}        = [ [ $descr, undef ] ]; }
                 
      push @$prog_ref, $a_prog;
   }
   
   $currentday = &DateCalc($currentday, "+ 1 day");
   $day_counter++;
}

my $data = [
    'ISO-8859-1',
    {
     'source-info-name'    => 'http://tvguide.ninemsn.com.au/',
     'generator-info-name' => 'NineMSN grabber',
     'generator-info-url'  => '',
     'generator-info-name' => "XMLTV - tv_grab_au NineMSN v0.2"
    },
    $chan_ref,
    $prog_ref
];

print "writing file\n";
my $fh = new IO::File ">$opt_output";
XMLTV::write_data($data, OUTPUT=>$fh);

print "cleaning $days_to_clean days\n";
$currentday = &DateCalc($yesterday, "-1 day");
$day_counter = 0;
while ($day_counter < $days_to_clean) {
   remove_day(&UnixDate($currentday, "%d%m%Y"));

   $currentday = &DateCalc($currentday, "-1 day");
   $day_counter++;
}
print "done\n";

# subroutines
sub get_TZ_Offset
{
   # Date::Manip timezone setup and calculations aren't correct so don't use
        my @date=UnixDate(shift,("%S","%M","%H","%d","%m","%Y"));
   $date[4] -= 1;   # stupid month fix
   my $offset=timegm(@date)-timelocal(@date);
   return sprintf "+%d%02d",$offset/3600,($offset/60)%60;
}

sub mygetstore($$)
{
   my ($url, $file) = @_;

   system("wget -q -O $file '$url'");
}

sub get_day
{
   my $date = shift;
   my $force = shift;
   
   my $url = $guide_url . $date . "_" . $region . ".asp";
   my $guide_dir = $cache_dir . "/" . $date;
   my $guide_file = $guide_dir . "/guide.html";
   mkpath ($guide_dir);

   for (my $retry=0; (($force==1) || (!(-e $guide_file))) && is_error(mygetstore($url, $guide_file)) && ($retry<$retrys); $retry++) {
      print ".";
      sleep($seconds_before_retry);
   }
   
   my @guide_lines;
   if (open(GUIDE, $guide_file)) {
      @guide_lines = <GUIDE>;
      close(GUIDE);
   } else {
      @guide_lines = ();
      print "giving up on $guide_file\n";
   }
   return @guide_lines;
}

sub remove_day
{
   my $date = shift;

   my $guide_dir = $cache_dir . "/" . $date;
   my @files= glob($guide_dir . "/*");
   unlink @files;

   return rmdir $guide_dir;
}

sub get_details
{
   my $date = shift;
   my $program_id = shift;
   
   my $url = $details_url . $program_id;
   my $guide_dir = $cache_dir . "/" . $date;
   my $details_file = $guide_dir . "/" . $program_id . ".html";
   mkpath ($guide_dir);
   
   for (my $retry=0; (!(-e $details_file)) && is_error(mygetstore($url, $details_file)) && ($retry<$retrys); $retry++) {
      print ".";
      sleep($seconds_before_retry);
   }

   my @details_lines;
   if (open(DETAILS, $details_file)) {
      @details_lines = <DETAILS>;
      close(DETAILS);
   } else {
      @details_lines = ();
      print "giving up on $details_file\n";
   }
   return @details_lines;
}

sub fetch_details
{
   my $datepid=$datepids->dequeue;
   my @datepidl=split /-/, $datepid;
   my $date = $datepidl[0];
   my $pid = $datepidl[1];

   while (($date!=0) and ($pid!=0)) {
      my $guide_dir = $cache_dir . "/" . $date;
      mkpath ($guide_dir);
   
      my $url = $details_url . $pid;
      my $details_file = $guide_dir . "/" . $pid . ".html";

      for (my $retry=0; is_error(mygetstore($url, $details_file)) && ($retry<$retrys); $retry++) {
         sleep($seconds_before_retry);
      }

      $datepid=$datepids->dequeue;
      @datepidl=split /-/, $datepid;
      $date = $datepidl[0];
      $pid = $datepidl[1];
   }
}

sub changed_guide
{
   my $date = shift;
   my @pidsrowspansnames = @_;
   
   my $guide_prn_file = $cache_dir . "/" . $date . "/guide.prn";
   if (open(PRN, $guide_prn_file)) {
      my @prn = split />/, <PRN>;
      close(PRN);

      if (($#prn > 1) and ($#prn == $#pidsrowspansnames)) {
         my $count;
         my $diff = ((($#prn+1)*2)/3)-1;
         for ($count=0; ($count <= $diff) && ($prn[$count]==$pidsrowspansnames[$count]); $count++)
         { }
         
         if ($count==($diff+1)) {
            for (; ($count <= $#prn) && ($prn[$count] eq $pidsrowspansnames[$count]); $count++)
            { }
         
                if ($count==($#prn+1)) {
               print "$date unchanged\n";
               return 0;
            }
         }
      }
   }

   print "$date downloading\n";

   if (open(PRN, ">", $guide_prn_file)) {
      for (my $count=0; $count<$#pidsrowspansnames; $count++) {
         print PRN "$pidsrowspansnames[$count]>";
      }
      if ($#pidsrowspansnames >= 0)
      {
         print PRN "$pidsrowspansnames[$#pidsrowspansnames]";
      }
      close(PRN);
   } else {
      print "can't open for writing $guide_prn_file\n";
   }

   return 1;
}