democracy now importer almost done

2 files changed, 69 insertions, 231 deletions

diff --git a/rhautoimport-dn b/rhautoimport-dn
index 9e54460..de41f37 100755
--- a/rhautoimport-dn
+++ b/rhautoimport-dn
@@ -31,7 +31,6 @@ use XML::Feed::Enclosure;
 use URI::URL;
 use IO::Handle;
 use IPC::Open3;
-use RHRD::rddb;
 
 use lib '/usr/local/share/rhautoimport/';
 use rhautoimport;
@@ -40,12 +39,9 @@ my $STAT_FILE = $ENV{'HOME'} . "/rhautoimport-dn.stat";
 my $FILES_RSS_URL = "http://www.democracynow.org/podcast-stations.xml";
 my $HEADLINES_RSS_URL = "http://www.democracynow.org/podcast.xml";
 my $IGNORE_HEADLINE = 0;
+my $RD_GROUP = "democracyn";
 my $PV_ID = '111';
 
-binmode(STDIN, ":utf8");
-binmode(STDOUT, ":utf8");
-binmode(STDERR, ":utf8");
-
 if($#ARGV >= 0 && $ARGV[0] eq 'last') {
   print "!!!This is the last attempt, there won't be a retry on error!!!\n"
 }
@@ -55,24 +51,6 @@ if($#ARGV >= 0 && $ARGV[0] eq 'noheadline') {
   $STAT_FILE = $ENV{'HOME'} . "/rhautoimport-dn-noheadline.stat";
 }
 
-my $user = `/usr/bin/id -un`;
-$user =~ s/\n//;
-my $group = "democracyn";
-
-my ($dbh, $errorstring) = RHRD::rddb::opendb();
-if(!defined $dbh) {
-  print "$errorstring\n";
-  exit 1;
-}
-my @allowed_dbs = RHRD::rddb::get_dropboxes($dbh, $user, $group);
-if(!defined $allowed_dbs[0] && defined $allowed_dbs[2]) {
-  print "$allowed_dbs[2]\n";
-  exit 1;
-}
-
-my $dropbox = "";
-my $to_cart = 0;
-
 my @today = Date::Calc::Today();
 my @yesterday = Date::Calc::Add_Delta_Days($today[0], $today[1], $today[2], -1);
 my @tomorrow = Date::Calc::Add_Delta_Days($today[0], $today[1], $today[2], 1);
@@ -84,21 +62,30 @@ if(localtime->hour >= 12) {
   @broadcast_day = @today;
   @import_day = @tomorrow;
 }
+my $dow = Date::Calc::Day_of_Week(@import_day);
+$dow = 0 unless $dow < 7;
 
 print "day of original broadcast: " . Date::Calc::Date_to_Text(@broadcast_day) . "\n";
 print "day of Radio Helsinki broadcast: " . Date::Calc::Date_to_Text(@import_day) . "\n";
 
+my $show_id = -1;
+my $show_title = "";
+
+my @allowed_dbs = rhautoimport::get_dropboxes($RD_GROUP);
+if(!defined $allowed_dbs[0] && defined $allowed_dbs[1]) {
+  print "$allowed_dbs[1]\n";
+  exit 1;
+}
+
 for my $href ( @allowed_dbs ) {
-  my $dow = Date::Calc::Day_of_Week(@import_day);
-  if ($dow == int(substr($href->{'NAME'},0,2))) {
-    $dropbox = $href->{'PATH'};
-    $to_cart = $href->{'TO_CART'};
+  if($href->{'TYPE'} eq "show" && $dow == $href->{'SHOWDOW'}) {
+    $show_title = $href->{'SHOWTITLE'};
+    $show_id = $href->{'SHOWID'};
   }
 }
 
-if($dropbox eq "") {
+if($show_id < 0) {
   print "no dropbox for day in question\n";
-  RHRD::rddb::closedb($dbh);
   exit 0;
 }
 
@@ -111,44 +98,33 @@ print " -> $FILES_RSS_URL\n";
 my $files_feed = XML::Feed->parse(URI->new($FILES_RSS_URL))
   or die "Error fetching feed: " . XML::Feed->errstr;
 
+my $uri = "";
 my $file = "";
-my $out_file = "";
 for my $entry ($files_feed->entries) {
   if($entry->enclosure && $entry->enclosure->type eq "audio/mpeg") {
     if($entry->enclosure->url =~ /^https?:\/\/.*\.democracynow\.org\/dn([0-9-]+)-1\.mp3$/) {
       next if($yc ne $1);
 
-      my $url = new URI::URL($entry->enclosure->url);
-      my @path = $url->path_components;
+      $uri = new URI::URL($entry->enclosure->url);
+      my @path = $uri->path_components;
 
       my $current_file = `cat $STAT_FILE`;
       if($current_file eq $path[-1]) {
         print "Already downloaded file of day in question\n";
-        RHRD::rddb::closedb($dbh);
         exit 0;
       }
-      $out_file = $path[-1];
-      if(!rhautoimport::check_file_extension($out_file)) {
-        print "\n\nThe extension of the matching file '". $out_file . "' seems to be wrong - manual import necessary!!!\n";
-        RHRD::rddb::closedb($dbh);
+      $file = $path[-1];
+      if(!rhautoimport::check_file_extension($file)) {
+        print "\n\nThe extension of the matching file '". $file . "' seems to be wrong - manual import necessary!!!\n";
         exit 1;
       }
 
-      print $1 . ": downloading " . $entry->enclosure->url . " (" . $entry->enclosure->length . " Bytes) .. ";
-
-      $file = "/tmp/" . $out_file;
-      system("wget", "--quiet", "--no-check-certificate", $url, "-O", $file);
-      if( $! != 0) {
-        die "wget returned with error: " . $!;
-      }
-      print "ok\n";
-
       last;
     }
   }
 }
-if($file eq "" || !(-e "$file")) {
-  print "No Entry found from day in qeustion or download error - ";
+if($uri eq "") {
+  print "No Entry found from day in question or download error - ";
   if($#ARGV >= 0 && $ARGV[0] eq 'last') {
     print "giving up, manual import necessary!!!\n";
   } else {
@@ -193,39 +169,16 @@ unless($IGNORE_HEADLINE) {
   print "\n";
 }
 
-print "converting $file ... ";
-my $infile = $file;
-$file =~ s/\.mp3$/.wav/;
-system("ffmpeg", "-i", $infile, "-acodec", "pcm_s16le", $file);
-if( $! != 0) {
-  die "ffmpeg returned with error: " . $!;
-}
-print "ok\n";
-unlink $infile;
-
-print "will import $file to dropbox $dropbox (cart=$to_cart)\n\n";
-
-my $error_cb = sub {
-  my ($text) = @_;
+print "will import '$uri' to show $show_id, $show_title\n\n";
 
-  print "\n$text";
-  return 0;
-};
-
-rhautoimport::check_key_file() or die "Import Key not found, use rhautoimport-create-id to create one\n";
-
-my $ret;
-my $log;
-$log = rhautoimport::clear_carts($dbh, $group, $to_cart);
-my $import_log;
-($ret, $import_log) = rhautoimport::import_single($file, $dropbox, $user, 0, $error_cb);
-$log .= $import_log;
-
-RHRD::rddb::closedb($dbh);
-
-unlink $file;
+my ($ret, $log) = rhautoimport::clear_carts($show_id);
+$log = '' unless defined $log;
+if($ret == 0) {
+   ($ret, my $import_log) = rhautoimport::import_uri($show_id, $uri);
+   $log .= $import_log if defined $import_log;
+}
 
-if(!$ret) {
+if($ret) {
   print "\nImport Error:\n";
   print $log;
   exit 1;
@@ -233,7 +186,7 @@ if(!$ret) {
 
 unlink($STAT_FILE);
 open(my $fhs, '>', $STAT_FILE);
-print $fhs $out_file;
+print $fhs $file;
 close($fhs);
 
 exit 0;
diff --git a/rhautoimport.pm b/rhautoimport.pm
index 76e3893..4bf9568 100644
--- a/rhautoimport.pm
+++ b/rhautoimport.pm
@@ -25,19 +25,31 @@ use strict;
 
 package rhautoimport;
 
-use IO::Handle;
-use IPC::Open3;
-use File::Spec;
 use File::Basename;
-use URI::Escape;
+use IPC::Open3;
+use IO::Handle;
 use LWP::Simple;
 use XML::Feed;
+use RHRD::rddb;
 
-my $ssh_host = "airplay";
-my $ssh_user = "rhautoimport";
-my $ssh_key_file = "$ENV{'HOME'}/.rhautoimport/import.key";
-my $ssh_dir = "/programm/.rhautoimport";
-my $rdimport_wrapper = "/usr/local/bin/dropbox_newfile.pl";
+use constant {
+  RD_USER => 'autoimport',
+};
+
+
+sub get_dropboxes
+{
+  my ($groupname) = @_;
+
+  my ($dbh, $status, $errorstring) = RHRD::rddb::opendb();
+  if(!defined $dbh) {
+    return ($dbh, $errorstring);
+  }
+  my @allowed_dbs = RHRD::rddb::get_dropboxes($dbh, RD_USER, $groupname);
+  RHRD::rddb::closedb($dbh);
+
+  return @allowed_dbs;
+}
 
 sub fetch_parse_rss
 {
@@ -59,162 +71,35 @@ sub fetch_parse_rss
   return XML::Feed->parse(\$xml);
 }
 
-sub clear_carts
+# TODO: use rhimportd for this?
+sub check_file_extension
 {
-  my ($dbh, $group, $to_cart, $progress_cb) = @_;
+  my ($file) = @_;
 
-  my ($low_cart, $high_cart) = ($to_cart, $to_cart);
-  if($to_cart == 0) {
-    ($low_cart, $high_cart) = get_cart_range($dbh, $group);
-  }
-
-  my $log = "clearing Carts $low_cart - $high_cart\n";
-  print $log;
-  my $cart = $low_cart;
-  while($cart <= $high_cart) {
-    $progress_cb->($cart - $low_cart, $high_cart + 1 - $low_cart, "deleting $cart") if($progress_cb);
-
-    my $sql = qq{select CUT_NAME from CUTS where CART_NUMBER='$cart';};
-    my $sth = $dbh->prepare($sql);
-    $sth->execute();
-    while(my $cut_name = $sth->fetchrow_array()) {
-      delete_file("/var/snd/$cut_name.wav");
-      $log .= " - deleting file /var/snd/$cut_name.wav";
+  my $ext = uc((fileparse($file, qr/\.[^.]*/))[2]);
+  foreach (".MP3", ".OGG", ".FLAC", ".WAV") {
+    if($ext eq $_) {
+      return 1;
     }
-    $sth->finish();
-
-    $sql = qq{delete from CUTS where CART_NUMBER='$cart';};
-    $sth = $dbh->prepare($sql);
-    $sth->execute();
-    $sth->finish();
-
-    $sql = qq{delete from CART where NUMBER='$cart';};
-    $sth = $dbh->prepare($sql);
-    $sth->execute();
-    $sth->finish();
-
-    $cart++;
-  }
-
-  return $log;
-}
-
-sub import_single
-{
-  my ($file, $dropbox, $user, $progress_cb, $error_cb) = @_;
-  print "Starting import from file $file to $dropbox\n";
-  $progress_cb->(0, 1, $file) if($progress_cb);
-  prepare_import($user);
-  my ($ret, $out) = import_file($file, $dropbox, $user, $error_cb);
-  if($ret) {
-    $progress_cb->(1, 1, "Import abgeschlossen!") if($progress_cb);
-  } else {
-    $progress_cb->(0, 1, "Import abgebrochen!") if($progress_cb);
   }
-  return ($ret, $out);
+  return 0;
 }
 
-sub scp_put_file
-{
-  my ($file, $user, $host, $path) = @_;
-  my @cmd = ( 'scp', '-prqB', '-i', $ssh_key_file, $file, "$user\@$host:$path/" );
-  my ($reader, $writer, $error ) = ( new IO::Handle, new IO::Handle, new IO::Handle );
-  $writer->autoflush(1);
-  local $SIG{CHLD} = 'DEFAULT';
-  my $pid = open3($writer, $reader, $error, @cmd);
-  binmode($reader, ":utf8");
-  binmode($writer, ":utf8");
-  binmode($error, ":utf8");
-  waitpid $pid, 0;
-  my $errstr = "";
-  if ( $? >> 8 ) {
-    $errstr = join('', <$error>);
-  }
-  return $errstr;
-}
-
-sub ssh_exec_command
-{
-  my ($command) = @_;
-  my @cmd = ( 'ssh' , '-q', '-o', 'BatchMode=yes', '-i', $ssh_key_file, "$ssh_user\@$ssh_host" , $command );
-  my ($reader, $writer, $error ) = ( new IO::Handle, new IO::Handle, new IO::Handle );
-  $writer->autoflush(1);
-  local $SIG{CHLD} = 'DEFAULT';
-  my $pid = open3($writer, $reader, $error, @cmd);
-  binmode($reader, ":utf8");
-  binmode($writer, ":utf8");
-  binmode($error, ":utf8");
-  waitpid $pid, 0;
-  my $out = join('', <$reader>);
-  my $errstr = "";
-  if ( $? >> 8 ) {
-    $errstr = join('', <$error>);
-    $errstr = "unkown error" if($errstr eq "");
-  }
-  return ($out, $errstr);
-}
-
-sub prepare_import
-{
-  my ($user) = @_;
-
-  ### create user directory
-  ssh_exec_command("umask 002; mkdir -p \"$ssh_dir/$user\"");
-}
-
-sub import_file
+sub clear_carts
 {
-  my ($file, $dropbox, $user, $error_cb) = @_;
+  my ($show_id) = @_;
 
-  print " - importing $file to $dropbox .. ";
-  $| = 1;
-
-  ### copy file to master server
-  my $err = scp_put_file($file, $ssh_user, $ssh_host, "$ssh_dir/$user");
-  if($err ne "") {
-    print "Transfer Error\n";
-    if($error_cb) {
-      return $error_cb->($err);
-    }
-    return 0;
-  }
-  print "transferred .. ";
-
-  ### remotely call rdimport
-  my ($volume, $directories, $remote_file) = File::Spec->splitpath($file);
-  $remote_file = "$ssh_dir/$user/$remote_file";
-  my ($out ,$error) = ssh_exec_command("$rdimport_wrapper --path \"$dropbox\" --file \"$remote_file\"");
-  my $lastline = $1 if $out =~ /\n(.*)$/;
-  if($error ne "" || $lastline !~ /^[0-9:\- ]+:\s+Deleted file/) {
-    print "Import Error\n";
-    delete_file($remote_file);
-    if($error_cb) {
-      if($error) {
-        return $error_cb->("Import Fehler: $error");
-      }
-      else {
-        return $error_cb->("Import Fehler: \n$out");
-      }
-    }
-    return 0;
-  }
-
-  print "imported OK\n";
-  return (1, $out);
+  return (1, "clear_carts not implemented...\n");
 }
 
-sub delete_file
+sub import_uri
 {
-  my ($filename) = @_;
+  my ($show_id, $uri) = @_;
 
-  print " - deleting $filename\n";
-  my ($out ,$err) = ssh_exec_command("rm \"$filename\"");
-  if($err ne "") {
-    return 0;
-  }
-  return 1;
+  return (1, "import_uri not implemented...\n");
 }
 
+# TODO: use rhimportd for this?
 sub pv_add_note
 {
   my ( $title, $text, $id, $date, $type, $index ) = @_;