From f6caff53c677882be78457589e2d48690472ac71 Mon Sep 17 00:00:00 2001
From: Christian Pointner <equinox@spreadspace.org>
Date: Sat, 12 Mar 2016 03:59:25 +0100
Subject: importer for democracy-now now uses rhimportd
diff --git a/rhautoimport-dn b/rhautoimport-dn
index 61d68d1..693611c 100755
--- a/rhautoimport-dn
+++ b/rhautoimport-dn
@@ -78,9 +78,10 @@ if(!defined $allowed_dbs[0] && defined $allowed_dbs[1]) {
}
for my $href ( @allowed_dbs ) {
- if($href->{'TYPE'} eq "show" && $dow == $href->{'SHOWDOW'}) {
+ if($dow == $href->{'SHOWDOW'}) {
$show_title = $href->{'SHOWTITLE'};
$show_id = $href->{'SHOWID'};
+ last;
}
}
@@ -95,7 +96,7 @@ my $yc = sprintf("%04d-%02d%02d", $broadcast_day[0], $broadcast_day[1], $broadca
print "looking for files from date $y in RSS Feed\n";
print " -> $FILES_RSS_URL\n";
-my $files_feed = XML::Feed->parse(URI->new($FILES_RSS_URL))
+my $files_feed = rhautoimport::fetch_parse_rss($FILES_RSS_URL)
or die "Error fetching feed: " . XML::Feed->errstr;
my $uri = "";
@@ -138,7 +139,7 @@ unless($IGNORE_HEADLINE) {
print "\nlooking for headlines from date $y in RSS Feed\n";
print " -> $HEADLINES_RSS_URL\n";
- my $headlines_feed = XML::Feed->parse(URI->new($HEADLINES_RSS_URL))
+ my $headlines_feed = rhautoimport::fetch_parse_rss($HEADLINES_RSS_URL)
or die "Error fetching feed: " . XML::Feed->errstr;
for my $entry ($headlines_feed->entries) {
@@ -146,7 +147,7 @@ unless($IGNORE_HEADLINE) {
my $sum_title = $entry->title;
my $sum_text = $entry->summary->body . "\n<br />\n<a href=\"" . $entry->link . "\">" . $entry->link . "</a>\n";
- if($entry->id =~ /tag:democracynow.org,([0-9-]+):media\/mp3download\/[^\/]+/) {
+ if($entry->id =~ /tag:democracynow.org,([0-9-]+):media\/mp3podcast\/[^\/]+/) {
next if($y ne $1);
print "summary:\n" . $sum_title . "\n\n" . $sum_text . "\n";
@@ -171,18 +172,14 @@ unless($IGNORE_HEADLINE) {
print "will import '$uri' to show $show_id, $show_title\n\n";
-my ($ret, $log) = rhautoimport::clear_carts($show_id);
-$log = '' unless defined $log;
-if($ret == 0) {
- ($ret, my $import_log) = rhautoimport::import_uri($show_id, $uri);
- $log .= $import_log if defined $import_log;
-}
-
+my ($ret, $log) = rhautoimport::import_uri($show_id, $uri->as_string, "delete");
if($ret) {
- print "\nImport Error:\n";
+ print "\nImport Error:\n\n";
print $log;
exit 1;
}
+print "\nImport Success:\n\n";
+print $log;
unlink($STAT_FILE);
open(my $fhs, '>', $STAT_FILE);
diff --git a/rhautoimport.pm b/rhautoimport.pm
index de5e373..f866940 100644
--- a/rhautoimport.pm
+++ b/rhautoimport.pm
@@ -26,14 +26,20 @@ use strict;
package rhautoimport;
use File::Basename;
+use File::Temp;
+use File::Slurp;
use IPC::Open3;
use IO::Handle;
use LWP::Simple;
use XML::Feed;
use RHRD::rddb;
+use JSON;
+
use constant {
RD_USER => 'autoimport',
+ RHIMPORTD_WATCH_DIR => '/run/rhimportd/watch',
+ RHIMPORTD_TIMEOUT => 10
};
@@ -41,12 +47,12 @@ sub get_dropboxes
{
my ($groupname) = @_;
- my ($dbh, $status, $errorstring) = RHRD::rddb::opendb();
- if(!defined $dbh) {
- return ($dbh, $errorstring);
+ my ($ctx, $status, $errorstring) = RHRD::rddb::init();
+ if(!defined $ctx) {
+ return ($ctx, $errorstring);
}
- my @allowed_dbs = RHRD::rddb::get_dropboxes($dbh, RD_USER, $groupname);
- RHRD::rddb::closedb($dbh);
+ my @allowed_dbs = RHRD::rddb::get_dropboxes($ctx, RD_USER, $groupname, 'show');
+ RHRD::rddb::destroy($ctx);
return @allowed_dbs;
}
@@ -71,7 +77,6 @@ sub fetch_parse_rss
return XML::Feed->parse(\$xml);
}
-# TODO: use rhimportd for this?
sub check_file_extension
{
my ($file) = @_;
@@ -85,21 +90,108 @@ sub check_file_extension
return 0;
}
-sub clear_carts
+sub parse_result
{
- my ($show_id) = @_;
+ my ($donefile) = @_;
+
+ my $log = "";
+ my $result_json;
+ eval {
+ $result_json = read_file($donefile, binmode => ':utf8');
+ 1;
+ } or do {
+ $log .= "!! error reading import result file !!\n";
+ unlink($donefile);
+ return (1, $log)
+ };
+
+ my $result;
+ eval {
+ $result = from_json($result_json);
+ 1;
+ } or do {
+ $log .= "\n!! error parsing import result !!\n";
+ unlink($donefile);
+ return (1, $log);
+ };
+
+ my $response_code = %$result{'RESPONSE_CODE'};
+ my $error_string = %$result{'ERROR_STRING'};
+ my $cart = %$result{'CART_NUMBER'};
+ $cart = 0 unless defined($cart);
+ my $cut = %$result{'CUT_NUMBER'};
+ $cut = 0 unless defined($cut);
+ my $source_file = %$result{'SOURCE_FILE'};
+
+ if(!defined($response_code) || $response_code != 200) {
+ if(defined($error_string)) {
+ $log .= "rhimportd returnd error: " . $error_string . "\n";
+ } else {
+ $log .= "rhimportd return invalid result !!\n";
+ }
+ unlink($donefile);
+ return (1, $log);
+ }
+
+ $log .= "rhimportd returned: OK (Cart: " . $cart . ", Cut: " . $cut;
+ $log .= ", source file: '" . $source_file . "'" if defined($source_file);
+ $log .= ")\n";
- return (1, "clear_carts not implemented...\n");
+ unlink($donefile);
+ return (0, $log);
}
sub import_uri
{
- my ($show_id, $uri) = @_;
+ my ($show_id, $uri, $file_policy) = @_;
+
+ my %request;
+ $request{'LOGIN_NAME'} = RD_USER;
+ $request{'SHOW_ID'} = $show_id;
+ $request{'CLEAR_SHOW_CARTS'} = JSON::true;
+ $request{'SOURCE_URI'} = $uri;
+ $request{'SOURCE_FILE_POLICY'} = $file_policy if $file_policy;
+
+ my $fh = File::Temp->new(TEMPLATE => 'rhautoimport-XXXXXXXX',
+ DIR => RHIMPORTD_WATCH_DIR,
+ SUFFIX => '.new');
+ return (1, "can't create request file in watchdir\n") unless($fh);
+
+ my $requestfile = $fh->filename;
+ my $log = "request file written to '$requestfile' ... ";
+ print $fh to_json(\%request);
+ print $fh "\n";
+ close $fh;
+
+ $requestfile =~ s/.new$//;
+ my $runningfile = $requestfile . ".running";
+ my $donefile = $requestfile . ".done";
+
+ $log .= "waiting for rhimportd to respond\n";
+ for my $i (1..RHIMPORTD_TIMEOUT) {
+ sleep(1);
+ last if(-e $runningfile || -e $donefile);
+
+ if($i >= RHIMPORTD_TIMEOUT) {
+ $log .= "timout waiting for rhimportd ... is it running?\n";
+ unlink($requestfile);
+ return (1, $log);
+ }
+ }
+
+ $log .= "import is running ... waiting for result\n";
+ my $duration = 0;
+ while(1) {
+ last if(-e $donefile);
+ sleep(1);
+ $duration++;
+ }
+ $log .= "import is done ... took " . $duration . " seconds!\n";
- return (1, "import_uri not implemented...\n");
+ my ($ret, $tmplog) = parse_result($donefile);
+ return $ret, $log . $tmplog;
}
-# TODO: use rhimportd for this?
sub pv_add_note
{
my ( $title, $text, $id, $date, $type, $index ) = @_;
--
cgit v0.10.2