From f6caff53c677882be78457589e2d48690472ac71 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Sat, 12 Mar 2016 03:59:25 +0100 Subject: importer for democracy-now now uses rhimportd diff --git a/rhautoimport-dn b/rhautoimport-dn index 61d68d1..693611c 100755 --- a/rhautoimport-dn +++ b/rhautoimport-dn @@ -78,9 +78,10 @@ if(!defined $allowed_dbs[0] && defined $allowed_dbs[1]) { } for my $href ( @allowed_dbs ) { - if($href->{'TYPE'} eq "show" && $dow == $href->{'SHOWDOW'}) { + if($dow == $href->{'SHOWDOW'}) { $show_title = $href->{'SHOWTITLE'}; $show_id = $href->{'SHOWID'}; + last; } } @@ -95,7 +96,7 @@ my $yc = sprintf("%04d-%02d%02d", $broadcast_day[0], $broadcast_day[1], $broadca print "looking for files from date $y in RSS Feed\n"; print " -> $FILES_RSS_URL\n"; -my $files_feed = XML::Feed->parse(URI->new($FILES_RSS_URL)) +my $files_feed = rhautoimport::fetch_parse_rss($FILES_RSS_URL) or die "Error fetching feed: " . XML::Feed->errstr; my $uri = ""; @@ -138,7 +139,7 @@ unless($IGNORE_HEADLINE) { print "\nlooking for headlines from date $y in RSS Feed\n"; print " -> $HEADLINES_RSS_URL\n"; - my $headlines_feed = XML::Feed->parse(URI->new($HEADLINES_RSS_URL)) + my $headlines_feed = rhautoimport::fetch_parse_rss($HEADLINES_RSS_URL) or die "Error fetching feed: " . XML::Feed->errstr; for my $entry ($headlines_feed->entries) { @@ -146,7 +147,7 @@ unless($IGNORE_HEADLINE) { my $sum_title = $entry->title; my $sum_text = $entry->summary->body . "\n
\nlink . "\">" . $entry->link . "\n"; - if($entry->id =~ /tag:democracynow.org,([0-9-]+):media\/mp3download\/[^\/]+/) { + if($entry->id =~ /tag:democracynow.org,([0-9-]+):media\/mp3podcast\/[^\/]+/) { next if($y ne $1); print "summary:\n" . $sum_title . "\n\n" . $sum_text . "\n"; @@ -171,18 +172,14 @@ unless($IGNORE_HEADLINE) { print "will import '$uri' to show $show_id, $show_title\n\n"; -my ($ret, $log) = rhautoimport::clear_carts($show_id); -$log = '' unless defined $log; -if($ret == 0) { - ($ret, my $import_log) = rhautoimport::import_uri($show_id, $uri); - $log .= $import_log if defined $import_log; -} - +my ($ret, $log) = rhautoimport::import_uri($show_id, $uri->as_string, "delete"); if($ret) { - print "\nImport Error:\n"; + print "\nImport Error:\n\n"; print $log; exit 1; } +print "\nImport Success:\n\n"; +print $log; unlink($STAT_FILE); open(my $fhs, '>', $STAT_FILE); diff --git a/rhautoimport.pm b/rhautoimport.pm index de5e373..f866940 100644 --- a/rhautoimport.pm +++ b/rhautoimport.pm @@ -26,14 +26,20 @@ use strict; package rhautoimport; use File::Basename; +use File::Temp; +use File::Slurp; use IPC::Open3; use IO::Handle; use LWP::Simple; use XML::Feed; use RHRD::rddb; +use JSON; + use constant { RD_USER => 'autoimport', + RHIMPORTD_WATCH_DIR => '/run/rhimportd/watch', + RHIMPORTD_TIMEOUT => 10 }; @@ -41,12 +47,12 @@ sub get_dropboxes { my ($groupname) = @_; - my ($dbh, $status, $errorstring) = RHRD::rddb::opendb(); - if(!defined $dbh) { - return ($dbh, $errorstring); + my ($ctx, $status, $errorstring) = RHRD::rddb::init(); + if(!defined $ctx) { + return ($ctx, $errorstring); } - my @allowed_dbs = RHRD::rddb::get_dropboxes($dbh, RD_USER, $groupname); - RHRD::rddb::closedb($dbh); + my @allowed_dbs = RHRD::rddb::get_dropboxes($ctx, RD_USER, $groupname, 'show'); + RHRD::rddb::destroy($ctx); return @allowed_dbs; } @@ -71,7 +77,6 @@ sub fetch_parse_rss return XML::Feed->parse(\$xml); } -# TODO: use rhimportd for this? sub check_file_extension { my ($file) = @_; @@ -85,21 +90,108 @@ sub check_file_extension return 0; } -sub clear_carts +sub parse_result { - my ($show_id) = @_; + my ($donefile) = @_; + + my $log = ""; + my $result_json; + eval { + $result_json = read_file($donefile, binmode => ':utf8'); + 1; + } or do { + $log .= "!! error reading import result file !!\n"; + unlink($donefile); + return (1, $log) + }; + + my $result; + eval { + $result = from_json($result_json); + 1; + } or do { + $log .= "\n!! error parsing import result !!\n"; + unlink($donefile); + return (1, $log); + }; + + my $response_code = %$result{'RESPONSE_CODE'}; + my $error_string = %$result{'ERROR_STRING'}; + my $cart = %$result{'CART_NUMBER'}; + $cart = 0 unless defined($cart); + my $cut = %$result{'CUT_NUMBER'}; + $cut = 0 unless defined($cut); + my $source_file = %$result{'SOURCE_FILE'}; + + if(!defined($response_code) || $response_code != 200) { + if(defined($error_string)) { + $log .= "rhimportd returnd error: " . $error_string . "\n"; + } else { + $log .= "rhimportd return invalid result !!\n"; + } + unlink($donefile); + return (1, $log); + } + + $log .= "rhimportd returned: OK (Cart: " . $cart . ", Cut: " . $cut; + $log .= ", source file: '" . $source_file . "'" if defined($source_file); + $log .= ")\n"; - return (1, "clear_carts not implemented...\n"); + unlink($donefile); + return (0, $log); } sub import_uri { - my ($show_id, $uri) = @_; + my ($show_id, $uri, $file_policy) = @_; + + my %request; + $request{'LOGIN_NAME'} = RD_USER; + $request{'SHOW_ID'} = $show_id; + $request{'CLEAR_SHOW_CARTS'} = JSON::true; + $request{'SOURCE_URI'} = $uri; + $request{'SOURCE_FILE_POLICY'} = $file_policy if $file_policy; + + my $fh = File::Temp->new(TEMPLATE => 'rhautoimport-XXXXXXXX', + DIR => RHIMPORTD_WATCH_DIR, + SUFFIX => '.new'); + return (1, "can't create request file in watchdir\n") unless($fh); + + my $requestfile = $fh->filename; + my $log = "request file written to '$requestfile' ... "; + print $fh to_json(\%request); + print $fh "\n"; + close $fh; + + $requestfile =~ s/.new$//; + my $runningfile = $requestfile . ".running"; + my $donefile = $requestfile . ".done"; + + $log .= "waiting for rhimportd to respond\n"; + for my $i (1..RHIMPORTD_TIMEOUT) { + sleep(1); + last if(-e $runningfile || -e $donefile); + + if($i >= RHIMPORTD_TIMEOUT) { + $log .= "timout waiting for rhimportd ... is it running?\n"; + unlink($requestfile); + return (1, $log); + } + } + + $log .= "import is running ... waiting for result\n"; + my $duration = 0; + while(1) { + last if(-e $donefile); + sleep(1); + $duration++; + } + $log .= "import is done ... took " . $duration . " seconds!\n"; - return (1, "import_uri not implemented...\n"); + my ($ret, $tmplog) = parse_result($donefile); + return $ret, $log . $tmplog; } -# TODO: use rhimportd for this? sub pv_add_note { my ( $title, $text, $id, $date, $type, $index ) = @_; -- cgit v0.10.2