importer for democracy-now now uses rhimportd

2 files changed, 113 insertions, 24 deletions

diff --git a/rhautoimport-dn b/rhautoimport-dn
index 61d68d1..693611c 100755
--- a/rhautoimport-dn
+++ b/rhautoimport-dn
@@ -78,9 +78,10 @@ if(!defined $allowed_dbs[0] && defined $allowed_dbs[1]) {
 }
 
 for my $href ( @allowed_dbs ) {
-  if($href->{'TYPE'} eq "show" && $dow == $href->{'SHOWDOW'}) {
+  if($dow == $href->{'SHOWDOW'}) {
     $show_title = $href->{'SHOWTITLE'};
     $show_id = $href->{'SHOWID'};
+    last;
   }
 }
 
@@ -95,7 +96,7 @@ my $yc = sprintf("%04d-%02d%02d", $broadcast_day[0], $broadcast_day[1], $broadca
 print "looking for files from date $y in RSS Feed\n";
 print " -> $FILES_RSS_URL\n";
 
-my $files_feed = XML::Feed->parse(URI->new($FILES_RSS_URL))
+my $files_feed = rhautoimport::fetch_parse_rss($FILES_RSS_URL)
   or die "Error fetching feed: " . XML::Feed->errstr;
 
 my $uri = "";
@@ -138,7 +139,7 @@ unless($IGNORE_HEADLINE) {
   print "\nlooking for headlines from date $y in RSS Feed\n";
   print " -> $HEADLINES_RSS_URL\n";
 
-  my $headlines_feed = XML::Feed->parse(URI->new($HEADLINES_RSS_URL))
+  my $headlines_feed = rhautoimport::fetch_parse_rss($HEADLINES_RSS_URL)
     or die "Error fetching feed: " . XML::Feed->errstr;
 
   for my $entry ($headlines_feed->entries) {
@@ -146,7 +147,7 @@ unless($IGNORE_HEADLINE) {
       my $sum_title = $entry->title;
       my $sum_text = $entry->summary->body . "\n<br />\n<a href=\"" . $entry->link . "\">" . $entry->link . "</a>\n";
 
-      if($entry->id =~ /tag:democracynow.org,([0-9-]+):media\/mp3download\/[^\/]+/) {
+      if($entry->id =~ /tag:democracynow.org,([0-9-]+):media\/mp3podcast\/[^\/]+/) {
         next if($y ne $1);
 
         print "summary:\n" . $sum_title . "\n\n" . $sum_text . "\n";
@@ -171,18 +172,14 @@ unless($IGNORE_HEADLINE) {
 
 print "will import '$uri' to show $show_id, $show_title\n\n";
 
-my ($ret, $log) = rhautoimport::clear_carts($show_id);
-$log = '' unless defined $log;
-if($ret == 0) {
-   ($ret, my $import_log) = rhautoimport::import_uri($show_id, $uri);
-   $log .= $import_log if defined $import_log;
-}
-
+my ($ret, $log) = rhautoimport::import_uri($show_id, $uri->as_string, "delete");
 if($ret) {
-  print "\nImport Error:\n";
+  print "\nImport Error:\n\n";
   print $log;
   exit 1;
 }
+print "\nImport Success:\n\n";
+print $log;
 
 unlink($STAT_FILE);
 open(my $fhs, '>', $STAT_FILE);
diff --git a/rhautoimport.pm b/rhautoimport.pm
index de5e373..f866940 100644
--- a/rhautoimport.pm
+++ b/rhautoimport.pm
@@ -26,14 +26,20 @@ use strict;
 package rhautoimport;
 
 use File::Basename;
+use File::Temp;
+use File::Slurp;
 use IPC::Open3;
 use IO::Handle;
 use LWP::Simple;
 use XML::Feed;
 use RHRD::rddb;
+use JSON;
+
 
 use constant {
   RD_USER => 'autoimport',
+  RHIMPORTD_WATCH_DIR => '/run/rhimportd/watch',
+  RHIMPORTD_TIMEOUT => 10
 };
 
 
@@ -41,12 +47,12 @@ sub get_dropboxes
 {
   my ($groupname) = @_;
 
-  my ($dbh, $status, $errorstring) = RHRD::rddb::opendb();
-  if(!defined $dbh) {
-    return ($dbh, $errorstring);
+  my ($ctx, $status, $errorstring) = RHRD::rddb::init();
+  if(!defined $ctx) {
+    return ($ctx, $errorstring);
   }
-  my @allowed_dbs = RHRD::rddb::get_dropboxes($dbh, RD_USER, $groupname);
-  RHRD::rddb::closedb($dbh);
+  my @allowed_dbs = RHRD::rddb::get_dropboxes($ctx, RD_USER, $groupname, 'show');
+  RHRD::rddb::destroy($ctx);
 
   return @allowed_dbs;
 }
@@ -71,7 +77,6 @@ sub fetch_parse_rss
   return XML::Feed->parse(\$xml);
 }
 
-# TODO: use rhimportd for this?
 sub check_file_extension
 {
   my ($file) = @_;
@@ -85,21 +90,108 @@ sub check_file_extension
   return 0;
 }
 
-sub clear_carts
+sub parse_result
 {
-  my ($show_id) = @_;
+  my ($donefile) = @_;
+
+  my $log = "";
+  my $result_json;
+  eval {
+    $result_json = read_file($donefile, binmode => ':utf8');
+    1;
+  } or do {
+    $log .= "!! error reading import result file !!\n";
+    unlink($donefile);
+    return (1, $log)
+  };
+
+  my $result;
+  eval {
+    $result = from_json($result_json);
+    1;
+  } or do {
+    $log .= "\n!! error parsing import result !!\n";
+    unlink($donefile);
+    return (1, $log);
+  };
+
+  my $response_code = %$result{'RESPONSE_CODE'};
+  my $error_string = %$result{'ERROR_STRING'};
+  my $cart = %$result{'CART_NUMBER'};
+  $cart = 0 unless defined($cart);
+  my $cut = %$result{'CUT_NUMBER'};
+  $cut = 0 unless defined($cut);
+  my $source_file = %$result{'SOURCE_FILE'};
+
+  if(!defined($response_code) || $response_code != 200) {
+    if(defined($error_string)) {
+      $log .= "rhimportd returnd error: " . $error_string . "\n";
+    } else {
+      $log .= "rhimportd return invalid result !!\n";
+    }
+    unlink($donefile);
+    return (1, $log);
+  }
+
+  $log .= "rhimportd returned: OK (Cart: " . $cart . ", Cut: " . $cut;
+  $log .= ", source file: '" . $source_file . "'" if defined($source_file);
+  $log .= ")\n";
 
-  return (1, "clear_carts not implemented...\n");
+  unlink($donefile);
+  return (0, $log);
 }
 
 sub import_uri
 {
-  my ($show_id, $uri) = @_;
+  my ($show_id, $uri, $file_policy) = @_;
+
+  my %request;
+  $request{'LOGIN_NAME'} = RD_USER;
+  $request{'SHOW_ID'} = $show_id;
+  $request{'CLEAR_SHOW_CARTS'} = JSON::true;
+	$request{'SOURCE_URI'} = $uri;
+	$request{'SOURCE_FILE_POLICY'} = $file_policy if $file_policy;
+
+  my $fh = File::Temp->new(TEMPLATE => 'rhautoimport-XXXXXXXX',
+                           DIR      => RHIMPORTD_WATCH_DIR,
+                           SUFFIX   => '.new');
+  return (1, "can't create request file in watchdir\n") unless($fh);
+
+  my $requestfile = $fh->filename;
+  my $log = "request file written to '$requestfile' ... ";
+  print $fh to_json(\%request);
+  print $fh "\n";
+  close $fh;
+
+  $requestfile =~ s/.new$//;
+  my $runningfile = $requestfile . ".running";
+  my $donefile = $requestfile . ".done";
+
+  $log .= "waiting for rhimportd to respond\n";
+  for my $i (1..RHIMPORTD_TIMEOUT) {
+    sleep(1);
+    last if(-e $runningfile || -e $donefile);
+
+    if($i >= RHIMPORTD_TIMEOUT) {
+      $log .= "timout waiting for rhimportd ... is it running?\n";
+      unlink($requestfile);
+      return (1, $log);
+    }
+  }
+
+  $log .= "import is running ... waiting for result\n";
+  my $duration = 0;
+  while(1) {
+    last if(-e $donefile);
+    sleep(1);
+    $duration++;
+  }
+  $log .= "import is done ... took " . $duration . " seconds!\n";
 
-  return (1, "import_uri not implemented...\n");
+  my ($ret, $tmplog) = parse_result($donefile);
+  return $ret, $log . $tmplog;
 }
 
-# TODO: use rhimportd for this?
 sub pv_add_note
 {
   my ( $title, $text, $id, $date, $type, $index ) = @_;