From 81531b51fe70fee446d22d3505d04709cd7b5b8a Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Sat, 26 Sep 2015 22:40:24 +0200 Subject: renamed rhimport to rhautoimport diff --git a/Makefile b/Makefile index 7202ab4..d1efa33 100644 --- a/Makefile +++ b/Makefile @@ -1,22 +1,22 @@ ## -## rhimport +## rhautoimport ## ## Copyright (C) 2009-2015 Christian Pointner ## -## This file is part of rhimport. +## This file is part of rhautoimport. ## -## rhimport is free software: you can redistribute it and/or modify +## rhautoimport is free software: you can redistribute it and/or modify ## it under the terms of the GNU General Public License as published by ## the Free Software Foundation, either version 3 of the License, or ## any later version. ## -## rhimport is distributed in the hope that it will be useful, +## rhautoimport is distributed in the hope that it will be useful, ## but WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ## GNU General Public License for more details. ## ## You should have received a copy of the GNU General Public License -## along with rhimport. If not, see . +## along with rhautoimport. If not, see . ## ifneq ($(MAKECMDGOALS),distclean) @@ -25,7 +25,7 @@ endif .PHONY: clean distclean -EXECUTABLE := rhimport +EXECUTABLE := rhautoimport IMPORTER := btl dn fe fw mz nw oi o94n rs ra sm sv tr ut zf zffe zfw EXECUTABLES := $(IMPORTER:%=$(EXECUTABLE)-%) @@ -44,23 +44,20 @@ install: $(INSTALL_TARGETS) install-bin: $(INSTALL) -d $(DESTDIR)$(BINDIR) - $(INSTALL) -m 755 $(EXECUTABLE) $(DESTDIR)$(BINDIR) - @sed s#/usr/local/share/rhimport/#$(SHAREDIR)/$(EXECUTABLE)/#g -i $(DESTDIR)$(BINDIR)/$(EXECUTABLE) $(INSTALL) -m 755 $(EXECUTABLES) $(DESTDIR)$(BINDIR) @for exe in $(EXECUTABLES); do \ - sed s#/usr/local/share/rhimport/#$(SHAREDIR)/$(EXECUTABLE)/#g -i $(DESTDIR)$(BINDIR)/$$exe ; \ + sed s#/usr/local/share/rhautoimport/#$(SHAREDIR)/$(EXECUTABLE)/#g -i $(DESTDIR)$(BINDIR)/$$exe ; \ done install-share: $(INSTALL) -d $(DESTDIR)$(SHAREDIR)/$(EXECUTABLE) - $(INSTALL) -m 644 rhimport.pm $(DESTDIR)$(SHAREDIR)/$(EXECUTABLE) + $(INSTALL) -m 644 rhautoimport.pm $(DESTDIR)$(SHAREDIR)/$(EXECUTABLE) uninstall: remove remove: $(REMOVE_TARGETS) remove-bin: - rm -f $(DESTDIR)$(BINDIR)/$(EXECUTABLE) rm -f $(DESTDIR)$(BINDIR)/$(EXECUTABLES) remove-share: diff --git a/configure b/configure index 5b2df89..3184c68 100755 --- a/configure +++ b/configure @@ -1,24 +1,24 @@ #!/bin/sh # # -# rhimport +# rhautoimport # # Copyright (C) 2009-2015 Christian Pointner # -# This file is part of rhimport. +# This file is part of rhautoimport. # -# rhimport is free software: you can redistribute it and/or modify +# rhautoimport is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # any later version. # -# rhimport is distributed in the hope that it will be useful, +# rhautoimport is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with rhimport. If not, see . +# along with rhautoimport. If not, see . # PREFIX='/usr/local' diff --git a/fetch-feeds-frn.pl b/fetch-feeds-frn.pl index ff8cd16..bae84a1 100755 --- a/fetch-feeds-frn.pl +++ b/fetch-feeds-frn.pl @@ -1,24 +1,24 @@ #!/usr/bin/perl -w # # -# rhimport +# rhautoimport # # Copyright (C) 2009-2015 Christian Pointner # -# This file is part of rhimport. +# This file is part of rhautoimport. # -# rhimport is free software: you can redistribute it and/or modify +# rhautoimport is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # any later version. # -# rhimport is distributed in the hope that it will be useful, +# rhautoimport is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with rhimport. If not, see . +# along with rhautoimport. If not, see . # use strict; diff --git a/fetch-files-cba.pl b/fetch-files-cba.pl index a812b1c..04db78e 100755 --- a/fetch-files-cba.pl +++ b/fetch-files-cba.pl @@ -1,24 +1,24 @@ #!/usr/bin/perl -w # # -# rhimport +# rhautoimport # # Copyright (C) 2009-2015 Christian Pointner # -# This file is part of rhimport. +# This file is part of rhautoimport. # -# rhimport is free software: you can redistribute it and/or modify +# rhautoimport is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # any later version. # -# rhimport is distributed in the hope that it will be useful, +# rhautoimport is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with rhimport. If not, see . +# along with rhautoimport. If not, see . # use strict; diff --git a/rhautoimport-btl b/rhautoimport-btl new file mode 100755 index 0000000..98252d3 --- /dev/null +++ b/rhautoimport-btl @@ -0,0 +1,177 @@ +#!/usr/bin/perl -w +# +# +# rhautoimport +# +# Copyright (C) 2009-2015 Christian Pointner +# +# This file is part of rhautoimport. +# +# rhautoimport is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# rhautoimport is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with rhautoimport. If not, see . +# + +use strict; +use Date::Calc; +use XML::Feed; +use XML::Feed::Entry; +use XML::Feed::Content; +use XML::Feed::Enclosure; +use File::Fetch; +use IO::Handle; +use IPC::Open3; +use HTML::Entities; +use LWP::Simple; +use RHRD::rddb; + +use lib '/usr/local/share/rhautoimport/'; +use rhautoimport; + +my $STAT_FILE = $ENV{'HOME'} . "/rhautoimport-btl.stat"; +my $RSS_URL = "http://www.btlonline.org/rss/btl128.xml"; +my $PV_ID = '221'; + +binmode(STDIN, ":utf8"); +binmode(STDOUT, ":utf8"); +binmode(STDERR, ":utf8"); + +if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "!!!This is the last attempt, there won't be a retry on error!!!\n" +} + +my $user = `/usr/bin/id -un`; +$user =~ s/\n//; +my $group = "betweenlin"; + +my ($dbh, $errorstring) = RHRD::rddb::opendb(); +if(!defined $dbh) { + print "$errorstring\n"; + exit 1; +} +my @allowed_dbs = RHRD::rddb::get_dropboxes($dbh, $user, $group); +if(!defined $allowed_dbs[0] && defined $allowed_dbs[2]) { + print "$allowed_dbs[2]\n"; + exit 1; +} + +if(scalar(@allowed_dbs) != 1) { + print "found more or less than one Dropbox for this group?!\n"; + RHRD::rddb::closedb($dbh); + exit 1; +} +my $dropbox = $allowed_dbs[0]->{'PATH'}; +my $to_cart = $allowed_dbs[0]->{'TO_CART'}; + + +my @today = Date::Calc::Today(); +my @import_date = Date::Calc::Add_Delta_Days(@today, 7); +@import_date = Date::Calc::Standard_to_Business(@import_date); +$import_date[2] = 2; +my @broadcast_date = @import_date; +$broadcast_date[2] = 5; + +@import_date = Date::Calc::Business_to_Standard(@import_date); +@broadcast_date = Date::Calc::Business_to_Standard(@broadcast_date); + +print "today: " . Date::Calc::Date_to_Text(@today) . "\n"; +print "day of next Radio Helsinki broadcast: " . Date::Calc::Date_to_Text(@import_date) . "\n"; +print "ending of current broadcast cycle: " . Date::Calc::Date_to_Text(@broadcast_date) . "\n"; + +my $id = sprintf("%04d-%02d-%02d", @import_date); +my $bd = sprintf("%04d-%02d-%02d", @broadcast_date); +my $bdfile = sprintf("%02d%02d%02d-btlv128\.mp3", $broadcast_date[0]%100, $broadcast_date[1], $broadcast_date[2]); + +print "looking for files like '$bdfile' in RSS Feed\n"; +print " -> $RSS_URL\n"; + +my $feed = XML::Feed->parse(URI->new($RSS_URL)) + or die "Error fetching feed: " . XML::Feed->errstr; + +my $file = ""; +my $out_file = ""; +for my $entry ($feed->entries) { + if($entry->enclosure && ($entry->enclosure->type eq "audio/mpeg" || entry->enclosure->type eq "audio/mp3")) { + next unless $entry->enclosure->url =~ /$bdfile/; + + my $sum_title = decode_entities($entry->title); + $sum_title =~ s/ \(128 kbps\)$//; + my $sum_text = decode_entities($entry->content->body); + + my $ff = File::Fetch->new(uri => $entry->enclosure->url); + my $current_file = `cat $STAT_FILE`; + if($current_file eq $ff->output_file) { + print "Already downloaded file of today\n"; + RHRD::rddb::closedb($dbh); + exit 0; + } + $out_file = $ff->output_file; + if(!rhautoimport::check_file_extension($out_file)) { + print "\n\nThe extension of the matching file '". $out_file . "' seems to be wrong - manual import necessary!!!\n"; + RHRD::rddb::closedb($dbh); + exit 1; + } + + print $bdfile . ": downloading " . $entry->enclosure->url . " (" . $entry->enclosure->length . " Bytes) .. "; + $file = $ff->fetch( to => '/tmp' ) or die $ff->error; + print "ok\n"; + + print "summary:\n" . $sum_title . "\n\n" . $sum_text . "\n"; + + rhautoimport::pv_add_note($sum_title, $sum_text, $PV_ID, $id, "1"); + + last; + } +} +if($file eq "" || !(-e "$file")) { + print "No Entry found for " . Date::Calc::Date_to_Text(@broadcast_date) . " or download error - "; + if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "giving up, manual import necessary!!!\n"; + } else { + print "will retry later\n"; + } + exit 1; +} + +print "will import $file to dropbox $dropbox (cart=$to_cart)\n"; + +my $error_cb = sub { + my ($text) = @_; + + print "\n$text"; + return 0; +}; + +rhautoimport::check_key_file() or die "Import Key not found, use rhautoimport-create-id to create one\n"; + +my $ret; +my $log = rhautoimport::clear_carts($dbh, $group, $to_cart); +my $import_log; +($ret, $import_log) = rhautoimport::import_single($file, $dropbox, $user, 0, $error_cb); +$log .= $import_log; + +RHRD::rddb::closedb($dbh); + +unlink $file; + +if(!$ret) { + print "\nImport Error:\n"; + print $log; + exit 1; +} + +unlink($STAT_FILE); +open(my $fhs, '>', $STAT_FILE); +print $fhs "$out_file"; +close($fhs); + +exit 0; diff --git a/rhautoimport-dn b/rhautoimport-dn new file mode 100755 index 0000000..9e54460 --- /dev/null +++ b/rhautoimport-dn @@ -0,0 +1,239 @@ +#!/usr/bin/perl -w +# +# +# rhautoimport +# +# Copyright (C) 2009-2015 Christian Pointner +# +# This file is part of rhautoimport. +# +# rhautoimport is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# rhautoimport is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with rhautoimport. If not, see . +# + +use strict; +use Date::Calc; +use Time::localtime; +use XML::Feed; +use XML::Feed::Entry; +use XML::Feed::Content; +use XML::Feed::Enclosure; +use URI::URL; +use IO::Handle; +use IPC::Open3; +use RHRD::rddb; + +use lib '/usr/local/share/rhautoimport/'; +use rhautoimport; + +my $STAT_FILE = $ENV{'HOME'} . "/rhautoimport-dn.stat"; +my $FILES_RSS_URL = "http://www.democracynow.org/podcast-stations.xml"; +my $HEADLINES_RSS_URL = "http://www.democracynow.org/podcast.xml"; +my $IGNORE_HEADLINE = 0; +my $PV_ID = '111'; + +binmode(STDIN, ":utf8"); +binmode(STDOUT, ":utf8"); +binmode(STDERR, ":utf8"); + +if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "!!!This is the last attempt, there won't be a retry on error!!!\n" +} + +if($#ARGV >= 0 && $ARGV[0] eq 'noheadline') { + $IGNORE_HEADLINE = 1; + $STAT_FILE = $ENV{'HOME'} . "/rhautoimport-dn-noheadline.stat"; +} + +my $user = `/usr/bin/id -un`; +$user =~ s/\n//; +my $group = "democracyn"; + +my ($dbh, $errorstring) = RHRD::rddb::opendb(); +if(!defined $dbh) { + print "$errorstring\n"; + exit 1; +} +my @allowed_dbs = RHRD::rddb::get_dropboxes($dbh, $user, $group); +if(!defined $allowed_dbs[0] && defined $allowed_dbs[2]) { + print "$allowed_dbs[2]\n"; + exit 1; +} + +my $dropbox = ""; +my $to_cart = 0; + +my @today = Date::Calc::Today(); +my @yesterday = Date::Calc::Add_Delta_Days($today[0], $today[1], $today[2], -1); +my @tomorrow = Date::Calc::Add_Delta_Days($today[0], $today[1], $today[2], 1); + +my @broadcast_day = @yesterday; +my @import_day = @today; +if(localtime->hour >= 12) { + print "It's past noon, assuming to import file from today\n"; + @broadcast_day = @today; + @import_day = @tomorrow; +} + +print "day of original broadcast: " . Date::Calc::Date_to_Text(@broadcast_day) . "\n"; +print "day of Radio Helsinki broadcast: " . Date::Calc::Date_to_Text(@import_day) . "\n"; + +for my $href ( @allowed_dbs ) { + my $dow = Date::Calc::Day_of_Week(@import_day); + if ($dow == int(substr($href->{'NAME'},0,2))) { + $dropbox = $href->{'PATH'}; + $to_cart = $href->{'TO_CART'}; + } +} + +if($dropbox eq "") { + print "no dropbox for day in question\n"; + RHRD::rddb::closedb($dbh); + exit 0; +} + +my $y = sprintf("%04d-%02d-%02d", $broadcast_day[0], $broadcast_day[1], $broadcast_day[2]); +my $yc = sprintf("%04d-%02d%02d", $broadcast_day[0], $broadcast_day[1], $broadcast_day[2]); + +print "looking for files from date $y in RSS Feed\n"; +print " -> $FILES_RSS_URL\n"; + +my $files_feed = XML::Feed->parse(URI->new($FILES_RSS_URL)) + or die "Error fetching feed: " . XML::Feed->errstr; + +my $file = ""; +my $out_file = ""; +for my $entry ($files_feed->entries) { + if($entry->enclosure && $entry->enclosure->type eq "audio/mpeg") { + if($entry->enclosure->url =~ /^https?:\/\/.*\.democracynow\.org\/dn([0-9-]+)-1\.mp3$/) { + next if($yc ne $1); + + my $url = new URI::URL($entry->enclosure->url); + my @path = $url->path_components; + + my $current_file = `cat $STAT_FILE`; + if($current_file eq $path[-1]) { + print "Already downloaded file of day in question\n"; + RHRD::rddb::closedb($dbh); + exit 0; + } + $out_file = $path[-1]; + if(!rhautoimport::check_file_extension($out_file)) { + print "\n\nThe extension of the matching file '". $out_file . "' seems to be wrong - manual import necessary!!!\n"; + RHRD::rddb::closedb($dbh); + exit 1; + } + + print $1 . ": downloading " . $entry->enclosure->url . " (" . $entry->enclosure->length . " Bytes) .. "; + + $file = "/tmp/" . $out_file; + system("wget", "--quiet", "--no-check-certificate", $url, "-O", $file); + if( $! != 0) { + die "wget returned with error: " . $!; + } + print "ok\n"; + + last; + } + } +} +if($file eq "" || !(-e "$file")) { + print "No Entry found from day in qeustion or download error - "; + if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "giving up, manual import necessary!!!\n"; + } else { + print "will retry later\n"; + } + exit 1; +} + + +unless($IGNORE_HEADLINE) { + print "\nlooking for headlines from date $y in RSS Feed\n"; + print " -> $HEADLINES_RSS_URL\n"; + + my $headlines_feed = XML::Feed->parse(URI->new($HEADLINES_RSS_URL)) + or die "Error fetching feed: " . XML::Feed->errstr; + + for my $entry ($headlines_feed->entries) { + if($entry->enclosure && $entry->enclosure->type eq "audio/mpeg") { + my $sum_title = $entry->title; + my $sum_text = $entry->summary->body . "\n
\nlink . "\">" . $entry->link . "\n"; + + if($entry->id =~ /tag:democracynow.org,([0-9-]+):media\/mp3download\/[^\/]+/) { + next if($y ne $1); + + print "summary:\n" . $sum_title . "\n\n" . $sum_text . "\n"; + rhautoimport::pv_add_note($sum_title, $sum_text, $PV_ID, sprintf("%04d-%02d-%02d", @import_day), "1", 0); + print "\n"; + rhautoimport::pv_add_note($sum_title, $sum_text, $PV_ID, sprintf("%04d-%02d-%02d", @import_day), "1", 1); + print "\n"; + + last; + } + } + } +} else { + print "\nignoring headlines for they are not available by now\n"; + my $dow_string = Date::Calc::Day_of_Week_to_Text(Date::Calc::Day_of_Week(@broadcast_day)); + my $sum_title = sprintf("Democracy Now! %04d-%02d-%02d %s", @broadcast_day, $dow_string); + my $sum_text = ""; + print "summary:\n" . $sum_title . "\n\n" . $sum_text . "\n"; + rhautoimport::pv_add_note($sum_title, $sum_text, $PV_ID, sprintf("%04d-%02d-%02d", @broadcast_day), "1", 2); + print "\n"; +} + +print "converting $file ... "; +my $infile = $file; +$file =~ s/\.mp3$/.wav/; +system("ffmpeg", "-i", $infile, "-acodec", "pcm_s16le", $file); +if( $! != 0) { + die "ffmpeg returned with error: " . $!; +} +print "ok\n"; +unlink $infile; + +print "will import $file to dropbox $dropbox (cart=$to_cart)\n\n"; + +my $error_cb = sub { + my ($text) = @_; + + print "\n$text"; + return 0; +}; + +rhautoimport::check_key_file() or die "Import Key not found, use rhautoimport-create-id to create one\n"; + +my $ret; +my $log; +$log = rhautoimport::clear_carts($dbh, $group, $to_cart); +my $import_log; +($ret, $import_log) = rhautoimport::import_single($file, $dropbox, $user, 0, $error_cb); +$log .= $import_log; + +RHRD::rddb::closedb($dbh); + +unlink $file; + +if(!$ret) { + print "\nImport Error:\n"; + print $log; + exit 1; +} + +unlink($STAT_FILE); +open(my $fhs, '>', $STAT_FILE); +print $fhs $out_file; +close($fhs); + +exit 0; diff --git a/rhautoimport-fe b/rhautoimport-fe new file mode 100755 index 0000000..9c7738d --- /dev/null +++ b/rhautoimport-fe @@ -0,0 +1,204 @@ +#!/usr/bin/perl -w +# +# +# rhautoimport +# +# Copyright (C) 2009-2015 Christian Pointner +# +# This file is part of rhautoimport. +# +# rhautoimport is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# rhautoimport is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with rhautoimport. If not, see . +# + +use strict; +use Date::Calc; +use XML::Feed; +use XML::Feed::Entry; +use XML::Feed::Content; +use XML::Feed::Enclosure; +use File::Fetch; +use IO::Handle; +use IPC::Open3; +use HTML::Entities; +use File::Touch; +use RHRD::rddb; + +use lib '/usr/local/share/rhautoimport/'; +use rhautoimport; + +my $STAT_FILE = $ENV{'HOME'} . "/rhautoimport-fe.stat"; +my $NOTE_FILE = $ENV{'HOME'} . "/rhautoimport-fe.last_note"; +my $NEW_FILE = $ENV{'HOME'} . "/rhautoimport-fe.is_new"; +my $RSS_URL = "http://freie-radios.net/portal/podcast.php?serie=208&rss"; +#my $PV_ID = '359'; + +binmode(STDIN, ":utf8"); +binmode(STDOUT, ":utf8"); +binmode(STDERR, ":utf8"); + +if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "!!!This is the last attempt, there won't be a retry on error!!!\n" +} + +my @today = Date::Calc::Today(); +my $dow = Date::Calc::Day_of_Week(@today); +my @tomorrow = Date::Calc::Add_Delta_Days($today[0], $today[1], $today[2], 1); + +my @tmp = Date::Calc::Standard_to_Business(@today); +$tmp[2] = 1; +my @this_mon = Date::Calc::Business_to_Standard(@tmp); +my @next_mon = Date::Calc::Add_Delta_Days($this_mon[0], $this_mon[1], $this_mon[2], 7); + +my @broadcast_day = @today; +my @import_day = @tomorrow; +if ($dow == 5) { + @import_day = @next_mon; +} + +print "day of original broadcast: " . Date::Calc::Date_to_Text(@broadcast_day) . "\n"; +print "day of Radio Helsinki broadcast: " . Date::Calc::Date_to_Text(@import_day) . "\n"; + +my $user = `/usr/bin/id -un`; +$user =~ s/\n//; +my $group = "vonunten"; +$dow = Date::Calc::Day_of_Week(@import_day); +$group .= "Mo" if($dow == 1); +$group .= "Di" if($dow == 2); +$group .= "Mi" if($dow == 3); +$group .= "Do" if($dow == 4); +$group .= "Fr" if($dow == 5); + +my ($dbh, $errorstring) = RHRD::rddb::opendb(); +if(!defined $dbh) { + print "$errorstring\n"; + exit 1; +} +my @allowed_dbs = RHRD::rddb::get_dropboxes($dbh, $user, $group); +if(!defined $allowed_dbs[0] && defined $allowed_dbs[2]) { + print "$allowed_dbs[2]\n"; + exit 1; +} +my $dropbox = ""; +my $to_cart = 0; + +if(scalar(@allowed_dbs) == 0) { + print "no dropbox for day in question\n"; + RHRD::rddb::closedb($dbh); + exit 0; +} +if(scalar(@allowed_dbs) != 1) { + print "found more or less than one Dropbox for this group?!\n"; + RHRD::rddb::closedb($dbh); + exit 1; +} +$dropbox = $allowed_dbs[0]->{'PATH'}; +$to_cart = $allowed_dbs[0]->{'TO_CART'}; + +my $regexp = sprintf('%04d%02d%02d.*\.mp3', $broadcast_day[0], $broadcast_day[1], $broadcast_day[2]); +print "looking for file from " . Date::Calc::Date_to_Text(@broadcast_day) . " in RSS Feed\n"; +print " -> $RSS_URL\n"; + +my $feed = XML::Feed->parse(URI->new($RSS_URL)) + or die "Error fetching feed: " . XML::Feed->errstr; + +my $file = ""; +my $out_file = ""; +for my $entry ($feed->entries) { + if($entry->enclosure && $entry->enclosure->type eq "audio/mpeg") { + next unless $entry->enclosure->url =~ /$regexp/; + next unless $entry->enclosure->length >= (20*1024*1024); + + my $sum_title = decode_entities($entry->title); + my $sum_text = decode_entities($entry->content->body); + + my $ff = File::Fetch->new(uri => $entry->enclosure->url); + my $current_file = `cat $STAT_FILE`; + if($current_file eq $ff->output_file) { + print "Already downloaded file of today\n"; + RHRD::rddb::closedb($dbh); + exit 0; + } + $out_file = $ff->output_file; + if(!rhautoimport::check_file_extension($out_file)) { + print "\n\nThe extension of the matching file '". $out_file . "' seems to be wrong - manual import necessary!!!\n"; + RHRD::rddb::closedb($dbh); + exit 1; + } + + print $regexp . ": downloading " . $entry->enclosure->url . " (" . $entry->enclosure->length . " Bytes) .. "; + $file = $ff->fetch( to => '/tmp' ) or die $ff->error; + print "ok\n"; + + print "summary:\n" . $sum_title . "\n\n" . $sum_text . "\n"; + + # if($#ARGV >= 0 && $ARGV[0] eq 'nopv') { + # print "not adding note to PV\n"; + # } else { + # rhautoimport::pv_add_note($sum_title, $sum_text, $PV_ID, sprintf("%04d-%02d-%02d", $import_day[0], $import_day[1], $import_day[2]), "1"); + # } + # print "\n"; + + unlink($NOTE_FILE); + open(my $fhs, '>', $NOTE_FILE); + binmode($fhs, ":utf8"); + print $fhs $sum_title . "\n\n" . $sum_text . "\n"; + close($fhs); + + last; + } +} +if($file eq "" || !(-e "$file")) { + print "No Entry found for " . Date::Calc::Date_to_Text(@broadcast_day) . " or download error - "; + if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "giving up, manual import necessary!!!\n"; + } else { + print "will retry later\n"; + } + exit 1; +} + +print "will import $file to dropbox $dropbox (cart=$to_cart)\n"; + +my $error_cb = sub { + my ($text) = @_; + + print "\n$text"; + return 0; +}; + +rhautoimport::check_key_file() or die "Import Key not found, use rhautoimport-create-id to create one\n"; + +my $ret; +my $log = rhautoimport::clear_carts($dbh, $group, $to_cart); +my $import_log; +($ret, $import_log) = rhautoimport::import_single($file, $dropbox, $user, 0, $error_cb); +$log .= $import_log; + +RHRD::rddb::closedb($dbh); + +unlink $file; + +if(!$ret) { + print "\nImport Error:\n"; + print $log; + exit 1; +} + +unlink($STAT_FILE); +open(my $fhs, '>', $STAT_FILE); +print $fhs $out_file; +close($fhs); +touch($NEW_FILE); + +exit 0; diff --git a/rhautoimport-fw b/rhautoimport-fw new file mode 100755 index 0000000..d1914b6 --- /dev/null +++ b/rhautoimport-fw @@ -0,0 +1,219 @@ +#!/usr/bin/perl -w +# +# +# rhautoimport +# +# Copyright (C) 2009-2015 Christian Pointner +# +# This file is part of rhautoimport. +# +# rhautoimport is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# rhautoimport is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with rhautoimport. If not, see . +# + +use strict; +use Date::Calc; +use XML::Feed; +use XML::Feed::Entry; +use XML::Feed::Content; +use XML::Feed::Enclosure; +use File::Fetch; +use IO::Handle; +use IPC::Open3; +use HTML::Entities; +use RHRD::rddb; + +use lib '/usr/local/share/rhautoimport/'; +use rhautoimport; + +my $STAT_FILE = $ENV{'HOME'} . "/rhautoimport-fw.stat"; +my $LAST_FILE = $ENV{'HOME'} . "/rhautoimport-fw.last"; +my $RSS_URL = "http://cba.fro.at/seriesrss/263965?c=Kfs2IoV2Wmd"; +$XML::Feed::MULTIPLE_ENCLOSURES=1; +my $PV_ID = '382'; + +binmode(STDIN, ":utf8"); +binmode(STDOUT, ":utf8"); +binmode(STDERR, ":utf8"); + +if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "!!!This is the last attempt, there won't be a retry on error!!!\n" +} + +my $user = `/usr/bin/id -un`; +$user =~ s/\n//; +my $group = "freiewelle"; + +my ($dbh, $errorstring) = RHRD::rddb::opendb(); +if(!defined $dbh) { + print "$errorstring\n"; + exit 1; +} +my @allowed_dbs = RHRD::rddb::get_dropboxes($dbh, $user, $group); +if(!defined $allowed_dbs[0] && defined $allowed_dbs[2]) { + print "$allowed_dbs[2]\n"; + exit 1; +} + +my $idx = 0; +my $idx_reb = 1; +if(scalar(@allowed_dbs) != 2) { + print "found more or less than 2 Dropboxes for this group?!\n"; + RHRD::rddb::closedb($dbh); + exit 1; +} +my $dropbox = $allowed_dbs[$idx]->{'PATH'}; +my $to_cart = $allowed_dbs[$idx]->{'TO_CART'}; +my $dropbox_reb = $allowed_dbs[$idx_reb]->{'PATH'}; +my $to_cart_reb = $allowed_dbs[$idx_reb]->{'TO_CART'}; + +my @import_date = Date::Calc::Standard_to_Business(Date::Calc::Today()); +if($import_date[2] != 1) { + $import_date[2] = 1; + @import_date = Date::Calc::Business_to_Standard(@import_date); + @import_date = Date::Calc::Add_Delta_Days(@import_date, 7); +} else { + @import_date = Date::Calc::Business_to_Standard(@import_date); +} +my @import_date_reb = Date::Calc::Add_Delta_Days(@import_date, 4); + +print "day of next broadcast: " . Date::Calc::Date_to_Text(@import_date) . "\n"; +print "day of next rebroadcast: " . Date::Calc::Date_to_Text(@import_date_reb) . "\n"; + +my $id = sprintf("%04d-%02d-%02d", @import_date); +my $bnum = `cat $LAST_FILE`; +$bnum += 1; + +my $current_stat = `cat $STAT_FILE`; +my ($current_id, $current_file) = $current_stat =~ m/^(.*)\n(.*)/; +if($current_id eq $id) { + print "Already downloaded current file\n"; + RHRD::rddb::closedb($dbh); + exit 0; +} + +print "looking for file with number $bnum in RSS Feed\n"; +print " -> $RSS_URL\n"; + +my $feed = XML::Feed->parse(URI->new($RSS_URL)) + or die "Error fetching feed: " . XML::Feed->errstr; + +my $bdexp = sprintf("^(\\d+)", $bnum); + + +my $file = ""; +my $out_file = ""; +my $i = 0; +for my $entry ($feed->entries) { + $i++; + next unless $entry->enclosure; + my $j = 0; + for my $enclosure($entry->enclosure) { + $j++; + if($enclosure->type eq "audio/mpeg" || $enclosure->type eq "audio/ogg") { + print "$i/$j: (" . $enclosure->type . ", " . $enclosure->length . ") " . $enclosure->url . "\n"; + my $url = $enclosure->url; + if($url =~ /^(.*)_cut(\.[^\.]+)$/) { + $url = $1 . $2; + } + + my $ff = File::Fetch->new(uri => $url); + next unless (uc($ff->output_file) =~ /$bdexp/); # file not from correct day + next unless ($bnum == $1); + + my $sum_title = decode_entities($entry->title); + my $sum_text = decode_entities($entry->content->body); + + my $current_file = `cat $STAT_FILE`; + if($current_file eq $ff->output_file) { + print "Already downloaded current file\n"; + RHRD::rddb::closedb($dbh); + exit 0; + } + $out_file = $ff->output_file; + if(!rhautoimport::check_file_extension($out_file)) { + print "\n\nThe extension of the matching file '". $out_file . "' seems to be wrong - manual import necessary!!!\n"; + RHRD::rddb::closedb($dbh); + exit 1; + } + + print " --> " . $bnum . ": downloading " . $url . " (" . $enclosure->length . " Bytes) .. "; + $file = $ff->fetch( to => '/tmp' ) or die $ff->error; + print "ok\n"; + + print "summary:\n" . $sum_title . "\n\n" . $sum_text . "\n"; + + rhautoimport::pv_add_note($sum_title, $sum_text, $PV_ID, sprintf("%04d-%02d-%02d", $import_date[0], $import_date[1], $import_date[2]), "1"); + print "\n"; + rhautoimport::pv_add_note($sum_title, $sum_text, $PV_ID, sprintf("%04d-%02d-%02d", $import_date_reb[0], $import_date_reb[1], $import_date_reb[2]), "2"); + print "\n"; + + last; + } + } + last if $file ne ""; +} + +if($file eq "" || !(-e "$file")) { + print "No Entry found or download error - "; + if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "giving up, manual import necessary!!!\n"; + } else { + print "will retry later\n"; + } + exit 1; +} + +print "will import $file to dropbox $dropbox (cart=$to_cart)\n"; +print "will import $file to rebroadcast dropbox $dropbox_reb (cart=$to_cart_reb)\n\n"; + +my $error_cb = sub { + my ($text) = @_; + + print "\n$text"; + return 0; +}; + +rhautoimport::check_key_file() or die "Import Key not found, use rhautoimport-create-id to create one\n"; + +my $ret; +my $log = rhautoimport::clear_carts($dbh, $group, $to_cart); +my $import_log; +($ret, $import_log) = rhautoimport::import_single($file, $dropbox, $user, 0, $error_cb); +$log .= $import_log; +$import_log = rhautoimport::clear_carts($dbh, $group, $to_cart_reb); +$log .= $import_log; +($ret, $import_log) = rhautoimport::import_single($file, $dropbox_reb, $user, 0, $error_cb); +$log .= $import_log; + +RHRD::rddb::closedb($dbh); + +unlink $file; + +if(!$ret) { + print "\nImport Error:\n"; + print $log; + exit 1; +} + +unlink($STAT_FILE); +open(my $fhs, '>', $STAT_FILE); +print $fhs "$id\n$out_file"; +close($fhs); + +unlink($LAST_FILE); +open($fhs, '>', $LAST_FILE); +print $fhs "$bnum"; +close($fhs); + +exit 0; diff --git a/rhautoimport-mz b/rhautoimport-mz new file mode 100755 index 0000000..fa50a69 --- /dev/null +++ b/rhautoimport-mz @@ -0,0 +1,247 @@ +#!/usr/bin/perl -w +# +# +# rhautoimport +# +# Copyright (C) 2009-2015 Christian Pointner +# +# This file is part of rhautoimport. +# +# rhautoimport is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# rhautoimport is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with rhautoimport. If not, see . +# + +use strict; +use Switch; +use Date::Calc; +use Time::localtime; +use IO::Handle; +use IPC::Open3; +use File::Slurp; +use RHRD::rddb; + +use lib '/usr/local/share/rhautoimport/'; +use rhautoimport; + + +my @STATIC_FILES = ($ENV{'HOME'} . "/frontex", $ENV{'HOME'} . "/maribor" ); + +my $STAT_FILE = $ENV{'HOME'} . "/rhautoimport-mz.stat"; +my $ZF_NOTE_FILE = $ENV{'HOME'} . "/rhautoimport-zf.last_note"; +my $ZF_NEW_FILE = $ENV{'HOME'} . "/rhautoimport-zf.is_new"; +my $PV_ID = '352'; + +binmode(STDIN, ":utf8"); +binmode(STDOUT, ":utf8"); +binmode(STDERR, ":utf8"); + +if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "!!!This is the last attempt, there won't be a retry on error!!!\n" +} + +my $user = `/usr/bin/id -un`; +$user =~ s/\n//; + +my @today = Date::Calc::Today(); +my @next_day = Date::Calc::Add_Delta_Days(@today, 1); +my $dow = Date::Calc::Day_of_Week(@today); +if($dow == 5) { + @next_day = Date::Calc::Add_Delta_Days(@today, 3); +} + +print "today: " . Date::Calc::Date_to_Text(@today) . "\n"; +print "next day would be: " . Date::Calc::Date_to_Text(@next_day) . "\n"; +my @import_day = @next_day; +if(localtime->hour < 12) { + print "It's forenoon, assuming to import for today\n"; + @import_day = @today; +} +print "day of broadcast: " . Date::Calc::Date_to_Text(@import_day) . "\n"; + + +$dow = Date::Calc::Day_of_Week(@import_day); +my $group = ""; +switch($dow) { + case 1 { $group = "mahlzeitMo" } + case 2 { $group = "mahlzeitDi" } + case 3 { $group = "mahlzeitMi" } + case 4 { $group = "mahlzeitDo" } + case 5 { $group = "mahlzeitFr" } + else { print("invalid day of week?!\n"); exit 1; } +} + +my ($dbh, $errorstring) = RHRD::rddb::opendb(); +if(!defined $dbh) { + print "$errorstring\n"; + exit 1; +} +my @allowed_dbs = RHRD::rddb::get_dropboxes($dbh, $user, $group); +if(!defined $allowed_dbs[0] && defined $allowed_dbs[2]) { + print "$allowed_dbs[2]\n"; + exit 1; +} + +if(scalar(@allowed_dbs) != 1) { + print "found no or more than one Dropboxes for this group?!\n"; + RHRD::rddb::closedb($dbh); + exit 1; +} +my $dropbox = $allowed_dbs[0]->{'PATH'}; + + +my $import_date = sprintf("%04d-%02d-%02d", @import_day); +my $current_date = `cat $STAT_FILE`; +if($current_date eq $import_date) { + print "Already downloaded file of day in question\n"; + RHRD::rddb::closedb($dbh); + exit 0; +} + +unless(-e $ZF_NEW_FILE) { + print "zip-fm not imported yet - "; + if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "giving up, rebroadcasting last show"; + } else { + print "will retry later\n"; + exit 0, + } +} + + +sub read_first_line { + my ($file) = @_; + my @lines = read_file($file, binmode => ':utf8', err_mode => 'quiet'); + return "" unless ($lines[0]); + + chomp($lines[0]); + return $lines[0]; +}; + +sub get_note { + my ($file) = @_; + my @lines = read_file($file, binmode => ':utf8', err_mode => 'quiet'); + return "" unless ($lines[0]); + + chomp($lines[0]); + $lines[0] = "

$lines[0]

\n"; + return join('',@lines); +}; + +sub find_next_file { + my ($dir, $current) = @_; + + my %files = (); + opendir(my $dh, $dir) or die $!; + while (my $file = readdir($dh)) { + next if ($file =~ m/^\./); + next if ($file =~ m/_notes$/); + $files{$file} = 1; + } + closedir($dh); + + my $num = scalar keys %files; + foreach my $file (sort keys %files) { + if($file gt $current) { + return $num, $file; + } + $num--; + } + + return 0, ""; +}; + +my $sum_title = "zip-fm (WH)"; +my $sum_text = get_note($ZF_NOTE_FILE) . "
\n"; + +print "\nsearching local files:\n"; + +my @files = (); +my %next_files = (); +foreach my $dir (@STATIC_FILES) { + my $file = read_first_line("$dir/.current"); + if($file eq "") { + print " $dir: skipped !!! (no .current file or no more files)\n"; + next; + } + my ($remaining, $next_file) = find_next_file($dir, $file); + if($remaining >= 1) { + print " $dir: adding $file\n (next: $next_file, $remaining files remaining)\n"; + } else { + print " $dir: adding $file\n (this was the last file!!!!)\n"; + } + $next_files{$dir} = $next_file; + + $sum_title = "$sum_title, " . read_first_line("$dir/.sum_title"); + $sum_text = $sum_text . "
\n" . get_note("$dir/$file" . "_notes"); + foreach my $ext ("flac", "wav", "ogg", "mp3") { + push(@files, "$dir/.jingle.$ext") if(-e "$dir/.jingle.$ext"); + } + push(@files, "$dir/$file"); +} +$sum_text = $sum_text . "

\n"; + +print "\nsummary:\n" . $sum_title . "\n\n" . $sum_text . "\n"; +rhautoimport::pv_add_note($sum_title, $sum_text, $PV_ID, sprintf("%04d-%02d-%02d", @import_day), "1"); +print "\n"; + + +print "will import " . join(", ", @files) . " to dropbox $dropbox\n"; + + +my $error_cb = sub { + my ($text) = @_; + + print "\n$text"; + return 0; +}; + +rhautoimport::check_key_file() or die "Import Key not found, use rhautoimport-create-id to create one\n"; + +my $ret = 1; +my $log = rhautoimport::clear_carts($dbh, $group, 0); +my ($cart, $high_cart) = rhautoimport::get_cart_range($dbh, $group); +foreach my $file (@files) { + if($cart > $high_cart) { + print "not enough carts - will ignore remaining files!!!"; + last; + } + print " importing $file to cart $cart\n"; + my $import_log; + ($ret, $import_log) = rhautoimport::import_single($file, $dropbox, $user, 0, $error_cb); + $log .= $import_log; + last unless $ret; + + $cart++; +} + +RHRD::rddb::closedb($dbh); + +if(!$ret) { + print "\nImport Error:\n"; + print $log; + exit 1; +} + +unlink($STAT_FILE); +open(my $fhs, '>', $STAT_FILE); +print $fhs $import_date; +close($fhs); + +unlink($ZF_NEW_FILE); +foreach my $dir (@STATIC_FILES) { + open(my $fhs, '>', "$dir/.current"); + print $fhs $next_files{$dir} if ($next_files{$dir}); + close($fhs); +} + +exit 0; diff --git a/rhautoimport-nw b/rhautoimport-nw new file mode 100755 index 0000000..3d6656b --- /dev/null +++ b/rhautoimport-nw @@ -0,0 +1,190 @@ +#!/usr/bin/perl -w +# +# +# rhautoimport +# +# Copyright (C) 2009-2015 Christian Pointner +# +# This file is part of rhautoimport. +# +# rhautoimport is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# rhautoimport is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with rhautoimport. If not, see . +# + +use strict; +use Date::Calc; +use XML::Feed; +use XML::Feed::Entry; +use XML::Feed::Content; +use XML::Feed::Enclosure; +use File::Fetch; +use IO::Handle; +use IPC::Open3; +use HTML::Entities; +use RHRD::rddb; + +use lib '/usr/local/share/rhautoimport/'; +use rhautoimport; + +my $STAT_FILE = $ENV{'HOME'} . "/rhautoimport-nw.stat"; +my $RSS_URL = "http://cba.fro.at/seriesrss/262514?c=Kfs2IoV2Wmd"; +$XML::Feed::MULTIPLE_ENCLOSURES=1; +my $PV_ID = '73'; + +binmode(STDIN, ":utf8"); +binmode(STDOUT, ":utf8"); +binmode(STDERR, ":utf8"); + +if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "!!!This is the last attempt, there won't be a retry on error!!!\n" +} + +my $user = `/usr/bin/id -un`; +$user =~ s/\n//; +my $group = "netwatcher"; + +my ($dbh, $errorstring) = RHRD::rddb::opendb(); +if(!defined $dbh) { + print "$errorstring\n"; + exit 1; +} +my @allowed_dbs = RHRD::rddb::get_dropboxes($dbh, $user, $group); +if(!defined $allowed_dbs[0] && defined $allowed_dbs[2]) { + print "$allowed_dbs[2]\n"; + exit 1; +} + +if(scalar(@allowed_dbs) != 1) { + print "found more or less than 1 Dropbox for this group?!\n"; + RHRD::rddb::closedb($dbh); + exit 1; +} +my $dropbox = $allowed_dbs[0]->{'PATH'}; +my $to_cart = $allowed_dbs[0]->{'TO_CART'}; + +my @today = Date::Calc::Today(); +my @import_date = @today; +if( Date::Calc::Day_of_Week(@today) > 3 ) { + @import_date = Date::Calc::Add_Delta_Days(@import_date, 7); +} +if( Date::Calc::Day_of_Week(@today) != 3 ) { + @import_date = Date::Calc::Standard_to_Business(@import_date); + $import_date[2] = 3; + @import_date = Date::Calc::Business_to_Standard(@import_date); +} +my @broadcast_day = Date::Calc::Add_Delta_Days(@import_date, -5); +print "day of original broadcast: " . Date::Calc::Date_to_Text(@broadcast_day) . "\n"; +print "day of Radio Helsinki broadcast: " . Date::Calc::Date_to_Text(@import_date) . "\n"; + +my $id = sprintf("%04d-%02d-%02d", @import_date); +my $bd = sprintf("%04d-%02d-%02d", @broadcast_day); +my $bdfile = sprintf("NETWATCHER%04d%02d%02d(.*)\.MP3", @broadcast_day); +print "looking for files from $bd in RSS Feed\n"; +print " -> $RSS_URL\n"; + +my $feed = XML::Feed->parse(URI->new($RSS_URL)) + or die "Error fetching feed: " . XML::Feed->errstr; + +my $file = ""; +my $out_file = ""; +my $i = 0; +for my $entry ($feed->entries) { + $i++; + next unless $entry->enclosure; + my $j = 0; + for my $enclosure($entry->enclosure) { + $j++; + if($enclosure->type eq "audio/mpeg" || $enclosure->type eq "audio/ogg") { + print "$i/$j: (" . $enclosure->type . ", " . $enclosure->length . ") " . $enclosure->url . "\n"; + my $url = $enclosure->url; + if($url =~ /^(.*)_cut(\.[^\.]+)$/) { + $url = $1 . $2; + } + + my $ff = File::Fetch->new(uri => $url); + next unless (uc($ff->output_file) =~ $bdfile); # file not from correct day + + my $sum_title = decode_entities($entry->title); + my $sum_text = decode_entities($entry->content->body); + + my $current_file = `cat $STAT_FILE`; + if($current_file eq $ff->output_file) { + print "Already downloaded file of today\n"; + RHRD::rddb::closedb($dbh); + exit 0; + } + $out_file = $ff->output_file; + if(!rhautoimport::check_file_extension($out_file)) { + print "\n\nThe extension of the matching file '". $out_file . "' seems to be wrong - manual import necessary!!!\n"; + RHRD::rddb::closedb($dbh); + exit 1; + } + + print " --> " . $bdfile . ": downloading " . $url . " (" . $enclosure->length . " Bytes) .. "; + $file = $ff->fetch( to => '/tmp' ) or die $ff->error; + print "ok\n"; + + print "summary:\n" . $sum_title . "\n\n" . $sum_text . "\n"; + + rhautoimport::pv_add_note($sum_title, $sum_text, $PV_ID, $id, "1"); + print "\n"; + + last; + } + } + last if $file ne ""; +} + +if($file eq "" || !(-e "$file")) { + print "No Entry found from $bd or download error - "; + if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "giving up, manual import necessary!!!\n"; + } else { + print "will retry later\n"; + } + exit 1; +} + +print "will import $file to dropbox $dropbox (cart=$to_cart)\n"; + +my $error_cb = sub { + my ($text) = @_; + + print "\n$text"; + return 0; +}; + +rhautoimport::check_key_file() or die "Import Key not found, use rhautoimport-create-id to create one\n"; + +my $ret; +my $log = rhautoimport::clear_carts($dbh, $group, $to_cart); +my $import_log; +($ret, $import_log) = rhautoimport::import_single($file, $dropbox, $user, 0, $error_cb); +$log .= $import_log; + +RHRD::rddb::closedb($dbh); + +unlink $file; + +if(!$ret) { + print "\nImport Error:\n"; + print $log; + exit 1; +} + +unlink($STAT_FILE); +open(my $fhs, '>', $STAT_FILE); +print $fhs $out_file; +close($fhs); + +exit 0; diff --git a/rhautoimport-o94n b/rhautoimport-o94n new file mode 100755 index 0000000..c3f8ab6 --- /dev/null +++ b/rhautoimport-o94n @@ -0,0 +1,209 @@ +#!/usr/bin/perl -w +# +# +# rhautoimport +# +# Copyright (C) 2009-2015 Christian Pointner +# +# This file is part of rhautoimport. +# +# rhautoimport is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# rhautoimport is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with rhautoimport. If not, see . +# + +use strict; +use Date::Calc; +use XML::Feed; +use XML::Feed::Entry; +use XML::Feed::Content; +use XML::Feed::Enclosure; +use File::Fetch; +use IO::Handle; +use IPC::Open3; +use HTML::Entities; +use RHRD::rddb; + +use lib '/usr/local/share/rhautoimport/'; +use rhautoimport; + +my $STAT_FILE = $ENV{'HOME'} . "/rhautoimport-o94n.stat"; +my $RSS_URL = "http://cba.fro.at/seriesrss/264131?c=Kfs2IoV2Wmd"; +$XML::Feed::MULTIPLE_ENCLOSURES=1; +my $PV_ID = '430'; + +binmode(STDIN, ":utf8"); +binmode(STDOUT, ":utf8"); +binmode(STDERR, ":utf8"); + +if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "!!!This is the last attempt, there won't be a retry on error!!!\n" +} + +my $user = `/usr/bin/id -un`; +$user =~ s/\n//; +my $group = "o94"; + +my ($dbh, $errorstring) = RHRD::rddb::opendb(); +if(!defined $dbh) { + print "$errorstring\n"; + exit 1; +} +my @allowed_dbs = RHRD::rddb::get_dropboxes($dbh, $user, $group); +if(!defined $allowed_dbs[0] && defined $allowed_dbs[2]) { + print "$allowed_dbs[2]\n"; + exit 1; +} + + +my $dropbox = ''; +my $to_cart = 0; + +for my $db (@allowed_dbs) { + if($db->{'PATH'} =~ /^\/programm\/02/) { + $to_cart = $db->{'TO_CART'}; + $dropbox = $db->{'PATH'}; + last; + } +} +if($to_cart == 0) { + print "no dropbox found for Tuesday?!\n"; + RHRD::rddb::closedb($dbh); + exit 1; +} + + +my @today = Date::Calc::Today(); +my @yesterday = Date::Calc::Add_Delta_Days(@today, -1); +my @tomorrow = Date::Calc::Add_Delta_Days(@today, 1); +my $dow = Date::Calc::Day_of_Week(@today); + +my @broadcast_day; +my @import_date; +if($dow == 1) { + @broadcast_day = @today; + @import_date = @tomorrow; +} elsif ($dow == 2) { + @broadcast_day = @yesterday; + @import_date = @today; +} else { + print "Wrong Day! Check cron script!\n"; + exit 1; +} + +print "day of original broadcast: " . Date::Calc::Date_to_Text(@broadcast_day) . "\n"; +print "day of Radio Helsinki broadcast: " . Date::Calc::Date_to_Text(@import_date) . "\n"; + +my $id = sprintf("%04d-%02d-%02d", @import_date); +my $bd = sprintf("%04d-%02d-%02d", @broadcast_day); +my $bdfile = sprintf("^NACHRICHTEN%04d%02d%02d", @broadcast_day); + +print "looking for files from $bd in RSS Feed\n"; +print " -> $RSS_URL\n"; + +my $feed = XML::Feed->parse(URI->new($RSS_URL)) + or die "Error fetching feed: " . XML::Feed->errstr; + +my $file = ""; +my $out_file = ""; +my $i = 0; +for my $entry ($feed->entries) { + $i++; + next unless $entry->enclosure; + my $j = 0; + for my $enclosure($entry->enclosure) { + $j++; + if($enclosure->type eq "audio/mpeg" || $enclosure->type eq "audio/ogg") { + print "$i/$j: (" . $enclosure->type . ", " . $enclosure->length . ") " . $enclosure->url . "\n"; + my $url = $enclosure->url; + if($url =~ /^(.*)_cut(\.[^\.]+)$/) { + $url = $1 . $2; + } + + my $ff = File::Fetch->new(uri => $url); + next unless (uc($ff->output_file) =~ $bdfile); # file not from correct day + + my $sum_title = decode_entities($entry->title); + $sum_title =~ s/\s*\(ganze sendung\)//i; + my $sum_text = decode_entities($entry->content->body); + + my $current_file = `cat $STAT_FILE`; + if($current_file eq $ff->output_file) { + print "Already downloaded file of today\n"; + RHRD::rddb::closedb($dbh); + exit 0; + } + $out_file = $ff->output_file; + if(!rhautoimport::check_file_extension($out_file)) { + print "\n\nThe extension of the matching file '". $out_file . "' seems to be wrong - manual import necessary!!!\n"; + RHRD::rddb::closedb($dbh); + exit 1; + } + + print " --> " . $bdfile . ": downloading " . $url . " (" . $enclosure->length . " Bytes) .. "; + $file = $ff->fetch( to => '/tmp' ) or die $ff->error; + print "ok\n"; + + print "summary:\n" . $sum_title . "\n\n" . $sum_text . "\n"; + + rhautoimport::pv_add_note($sum_title, $sum_text, $PV_ID, sprintf("%04d-%02d-%02d", @import_date), "1"); + print "\n"; + + last; + } + } + last if $file ne ""; +} + +if($file eq "" || !(-e "$file")) { + print "No Entry found from $bd or download error - "; + if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "giving up, manual import necessary!!!\n"; + } else { + print "will retry later\n"; + } + exit 1; +} + +print "will import $file to dropbox $dropbox (cart=$to_cart)\n"; + +my $error_cb = sub { + my ($text) = @_; + + print "\n$text"; + return 0; +}; + +rhautoimport::check_key_file() or die "Import Key not found, use rhautoimport-create-id to create one\n"; + +my $ret; +my $log = rhautoimport::clear_carts($dbh, $group, $to_cart); +my $import_log; +($ret, $import_log) = rhautoimport::import_single($file, $dropbox, $user, 0, $error_cb); +$log .= $import_log; + +RHRD::rddb::closedb($dbh); + +unlink $file; + +if(!$ret) { + print "\nImport Error:\n"; + print $log; + exit 1; +} + +unlink($STAT_FILE); +open(my $fhs, '>', $STAT_FILE); +print $fhs $out_file; +close($fhs); + +exit 0; diff --git a/rhautoimport-oi b/rhautoimport-oi new file mode 100755 index 0000000..1e637c8 --- /dev/null +++ b/rhautoimport-oi @@ -0,0 +1,182 @@ +#!/usr/bin/perl -w +# +# +# rhautoimport +# +# Copyright (C) 2009-2015 Christian Pointner +# +# This file is part of rhautoimport. +# +# rhautoimport is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# rhautoimport is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with rhautoimport. If not, see . +# + +use strict; +use Date::Calc; +use XML::Feed; +use XML::Feed::Entry; +use XML::Feed::Content; +use XML::Feed::Enclosure; +use File::Fetch; +use IO::Handle; +use IPC::Open3; +use HTML::Entities; +use RHRD::rddb; + +use lib '/usr/local/share/rhautoimport/'; +use rhautoimport; + +my $STAT_FILE = $ENV{'HOME'} . "/rhautoimport-oi.stat"; +my $LAST_FILE = $ENV{'HOME'} . "/rhautoimport-oi.last"; +my $RSS_URL = "http://npla.de/onda/feed/serie=3"; +my $PV_ID = '220'; + +binmode(STDIN, ":utf8"); +binmode(STDOUT, ":utf8"); +binmode(STDERR, ":utf8"); + +if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "!!!This is the last attempt, there won't be a retry on error!!!\n" +} + +my $user = `/usr/bin/id -un`; +$user =~ s/\n//; +my $group = "ondainfo"; + +my ($dbh, $errorstring) = RHRD::rddb::opendb(); +if(!defined $dbh) { + print "$errorstring\n"; + exit 1; +} +my @allowed_dbs = RHRD::rddb::get_dropboxes($dbh, $user, $group); +if(!defined $allowed_dbs[0] && defined $allowed_dbs[2]) { + print "$allowed_dbs[2]\n"; + exit 1; +} + +if(scalar(@allowed_dbs) != 1) { + print "found more or less than one Dropbox for this group?!\n"; + RHRD::rddb::closedb($dbh); + exit 1; +} +my $dropbox = $allowed_dbs[0]->{'PATH'}; +my $to_cart = $allowed_dbs[0]->{'TO_CART'}; + + +my @today = Date::Calc::Today(); +my @import_date = Date::Calc::Add_Delta_Days(@today, 7); +@import_date = Date::Calc::Standard_to_Business(@import_date); +$import_date[2] = 2; +@import_date = Date::Calc::Business_to_Standard(@import_date); + +my $broadcast_num = `cat $LAST_FILE`; +$broadcast_num += 1; + +print "today: " . Date::Calc::Date_to_Text(@today) . "\n"; +print "day of next Radio Helsinki broadcast: " . Date::Calc::Date_to_Text(@import_date) . "\n"; +print "Number of next broadcast: " . $broadcast_num . "\n"; + +my $id = sprintf("%04d-%02d-%02d", @import_date); +my $bdfile = sprintf('.*onda.?info_%d.*\.mp3$', $broadcast_num); + +my $current_stat = `cat $STAT_FILE`; +my ($current_id, $current_file) = $current_stat =~ m/^(.*)\n(.*)/; +if($current_id eq $id) { + print "Already downloaded current file\n"; + RHRD::rddb::closedb($dbh); + exit 0; +} + +print "looking for files like '$bdfile' in RSS Feed\n"; +print " -> $RSS_URL\n"; + +my $feed = rhautoimport::fetch_parse_rss($RSS_URL) + or die "Error fetching feed: " . XML::Feed->errstr; + +my $file = ""; +my $out_file = ""; +for my $entry ($feed->entries) { + if($entry->enclosure) { + print $entry->enclosure->url . "\n"; + next unless $entry->enclosure->url =~ /$bdfile/; + + my $sum_title = decode_entities($entry->title); + my $sum_text = decode_entities($entry->content->body); + + my $ff = File::Fetch->new(uri => $entry->enclosure->url); + $out_file = $ff->output_file; + if(!rhautoimport::check_file_extension($out_file)) { + print "\n\nThe extension of the matching file '". $out_file . "' seems to be wrong - manual import necessary!!!\n"; + RHRD::rddb::closedb($dbh); + exit 1; + } + + print $broadcast_num . ": downloading " . $entry->enclosure->url . " (" . $entry->enclosure->length . " Bytes) .. "; + $file = $ff->fetch( to => '/tmp' ) or die $ff->error; + print "ok\n"; + + print "summary:\n" . $sum_title . "\n\n" . $sum_text . "\n"; + + rhautoimport::pv_add_note($sum_title, $sum_text, $PV_ID, $id, "1"); + + last; + } +} +if($file eq "" || !(-e "$file")) { + print "No Entry found with " . $broadcast_num . " or download error - "; + if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "giving up, manual import necessary!!!\n"; + } else { + print "will retry later\n"; + } + exit 1; +} + +print "will import $file to dropbox $dropbox (cart=$to_cart)\n"; + +my $error_cb = sub { + my ($text) = @_; + + print "\n$text"; + return 0; +}; + +rhautoimport::check_key_file() or die "Import Key not found, use rhautoimport-create-id to create one\n"; + +my $ret; +my $log = rhautoimport::clear_carts($dbh, $group, $to_cart); +my $import_log; +($ret, $import_log) = rhautoimport::import_single($file, $dropbox, $user, 0, $error_cb); +$log .= $import_log; + +RHRD::rddb::closedb($dbh); + +unlink $file; + +if(!$ret) { + print "\nImport Error:\n"; + print $log; + exit 1; +} + +unlink($STAT_FILE); +open(my $fhs, '>', $STAT_FILE); +print $fhs "$id\n$out_file"; +close($fhs); + +unlink($LAST_FILE); +open($fhs, '>', $LAST_FILE); +print $fhs "$broadcast_num"; +close($fhs); + +exit 0; diff --git a/rhautoimport-ra b/rhautoimport-ra new file mode 100755 index 0000000..3018429 --- /dev/null +++ b/rhautoimport-ra @@ -0,0 +1,199 @@ +#!/usr/bin/perl -w +# +# +# rhautoimport +# +# Copyright (C) 2009-2015 Christian Pointner +# +# This file is part of rhautoimport. +# +# rhautoimport is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# rhautoimport is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with rhautoimport. If not, see . +# + +use strict; +use Date::Calc; +use XML::Feed; +use XML::Feed::Entry; +use XML::Feed::Content; +use XML::Feed::Enclosure; +use File::Fetch; +use IO::Handle; +use IPC::Open3; +use HTML::Entities; +use RHRD::rddb; + +use lib '/usr/local/share/rhautoimport/'; +use rhautoimport; + +my $STAT_FILE = $ENV{'HOME'} . "/rhautoimport-ra.stat"; +my $RSS_URL = "http://cba.fro.at/seriesrss/262430?c=Kfs2IoV2Wmd"; +$XML::Feed::MULTIPLE_ENCLOSURES=1; +my $PV_ID = '76'; + +binmode(STDIN, ":utf8"); +binmode(STDOUT, ":utf8"); +binmode(STDERR, ":utf8"); + +if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "!!!This is the last attempt, there won't be a retry on error!!!\n" +} + +my $user = `/usr/bin/id -un`; +$user =~ s/\n//; +my $group = "RadioATTAC"; + +my ($dbh, $errorstring) = RHRD::rddb::opendb(); +if(!defined $dbh) { + print "$errorstring\n"; + exit 1; +} +my @allowed_dbs = RHRD::rddb::get_dropboxes($dbh, $user, $group); +if(!defined $allowed_dbs[0] && defined $allowed_dbs[2]) { + print "$allowed_dbs[2]\n"; + exit 1; +} + +my $idx = 0; +my $idx_reb = 1; +if(scalar(@allowed_dbs) != 2) { + print "found more or less than 2 Dropboxes for this group?!\n"; + RHRD::rddb::closedb($dbh); + exit 1; +} +my $dropbox = $allowed_dbs[$idx]->{'PATH'}; +my $to_cart = $allowed_dbs[$idx]->{'TO_CART'}; +my $dropbox_reb = $allowed_dbs[$idx_reb]->{'PATH'}; +my $to_cart_reb = $allowed_dbs[$idx_reb]->{'TO_CART'}; + +my @today = Date::Calc::Standard_to_Business(Date::Calc::Today()); +$today[2] = 1; +my @broadcast_day = Date::Calc::Business_to_Standard(@today); +$today[2] = 3; +my @import_date = Date::Calc::Business_to_Standard(@today); +my @import_date_reb = Date::Calc::Add_Delta_Days($import_date[0], $import_date[1], $import_date[2], 3); +print "day of original broadcast: " . Date::Calc::Date_to_Text(@broadcast_day) . "\n"; +print "day of Radio Helsinki broadcast: " . Date::Calc::Date_to_Text(@import_date) . "\n"; +print "day of Radio Helsinki rebroadcast: " . Date::Calc::Date_to_Text(@import_date_reb) . "\n"; + +my $bd = sprintf("%04d-%02d-%02d", $broadcast_day[0], $broadcast_day[1], $broadcast_day[2]); +print "looking for files from $bd in RSS Feed\n"; +print " -> $RSS_URL\n"; + +my $feed = XML::Feed->parse(URI->new($RSS_URL)) + or die "Error fetching feed: " . XML::Feed->errstr; + +my $bdexp = sprintf("^(\\d+)\\..*\\s*0?%d\\s*[.,]\\s*0?%d\\s*[.,]\\s*%4d", $broadcast_day[2], $broadcast_day[1], $broadcast_day[0]); + + +my $file = ""; +my $out_file = ""; +my $i = 0; +for my $entry ($feed->entries) { + $i++; + next unless $entry->enclosure; + my $j = 0; + for my $enclosure($entry->enclosure) { + $j++; + if($enclosure->type eq "audio/mpeg" || $enclosure->type eq "audio/ogg") { + print "$i/$j: (" . $enclosure->type . ", " . $enclosure->length . ") " . $enclosure->url . "\n"; + my $url = $enclosure->url; + if($url =~ /^(.*)_cut(\.[^\.]+)$/) { + $url = $1 . $2; + } + + my $sum_title = decode_entities($entry->title); + next unless $sum_title =~ /$bdexp/; + my $sum_text = decode_entities($entry->content->body); + my $bnum = $1; + + my $ff = File::Fetch->new(uri => $url); + next unless $bnum eq substr($ff->output_file, 0, 3); + my $current_file = `cat $STAT_FILE`; + if($current_file eq $ff->output_file) { + print "Already downloaded file of today\n"; + RHRD::rddb::closedb($dbh); + exit 0; + } + $out_file = $ff->output_file; + if(!rhautoimport::check_file_extension($out_file)) { + print "\n\nThe extension of the matching file '". $out_file . "' seems to be wrong - manual import necessary!!!\n"; + RHRD::rddb::closedb($dbh); + exit 1; + } + + print " --> " . $bnum . ": downloading " . $url . " (" . $enclosure->length . " Bytes) .. "; + $file = $ff->fetch( to => '/tmp' ) or die $ff->error; + print "ok\n"; + + print "summary:\n" . $sum_title . "\n\n" . $sum_text . "\n"; + + rhautoimport::pv_add_note($sum_title, $sum_text, $PV_ID, sprintf("%04d-%02d-%02d", $import_date[0], $import_date[1], $import_date[2]), "1"); + print "\n"; + rhautoimport::pv_add_note($sum_title, $sum_text, $PV_ID, sprintf("%04d-%02d-%02d", $import_date_reb[0], $import_date_reb[1], $import_date_reb[2]), "2"); + print "\n"; + + last; + } + } + last if $file ne ""; +} + +if($file eq "" || !(-e "$file")) { + print "No Entry found from $bd or download error - "; + if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "giving up, manual import necessary!!!\n"; + } else { + print "will retry later\n"; + } + exit 1; +} + +print "will import $file to dropbox $dropbox (cart=$to_cart)\n"; +print "will import $file to rebroadcast dropbox $dropbox_reb (cart=$to_cart_reb)\n\n"; + +my $error_cb = sub { + my ($text) = @_; + + print "\n$text"; + return 0; +}; + +rhautoimport::check_key_file() or die "Import Key not found, use rhautoimport-create-id to create one\n"; + +my $ret; +my $log = rhautoimport::clear_carts($dbh, $group, $to_cart); +my $import_log; +($ret, $import_log) = rhautoimport::import_single($file, $dropbox, $user, 0, $error_cb); +$log .= $import_log; +$import_log = rhautoimport::clear_carts($dbh, $group, $to_cart_reb); +$log .= $import_log; +($ret, $import_log) = rhautoimport::import_single($file, $dropbox_reb, $user, 0, $error_cb); +$log .= $import_log; + +RHRD::rddb::closedb($dbh); + +unlink $file; + +if(!$ret) { + print "\nImport Error:\n"; + print $log; + exit 1; +} + +unlink($STAT_FILE); +open(my $fhs, '>', $STAT_FILE); +print $fhs $out_file; +close($fhs); + +exit 0; diff --git a/rhautoimport-rs b/rhautoimport-rs new file mode 100755 index 0000000..99a3b19 --- /dev/null +++ b/rhautoimport-rs @@ -0,0 +1,226 @@ +#!/usr/bin/perl -w +# +# +# rhautoimport +# +# Copyright (C) 2009-2015 Christian Pointner +# +# This file is part of rhautoimport. +# +# rhautoimport is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# rhautoimport is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with rhautoimport. If not, see . +# + +use strict; +use Date::Calc; +use XML::Feed; +use XML::Feed::Entry; +use XML::Feed::Content; +use XML::Feed::Enclosure; +use File::Fetch; +use IO::Handle; +use IPC::Open3; +use HTML::Entities; +use LWP::Simple; +use RHRD::rddb; + +use lib '/usr/local/share/rhautoimport/'; +use rhautoimport; + +my $STAT_FILE = $ENV{'HOME'} . "/rhautoimport-rs.stat"; +my $RSS_URL = "http://cba.fro.at/seriesrss/262433?c=Kfs2IoV2Wmd"; +$XML::Feed::MULTIPLE_ENCLOSURES=1; +my $PV_ID = '75'; + +binmode(STDIN, ":utf8"); +binmode(STDOUT, ":utf8"); +binmode(STDERR, ":utf8"); + +if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "!!!This is the last attempt, there won't be a retry on error!!!\n" +} + +my $user = `/usr/bin/id -un`; +$user =~ s/\n//; +my $group = "radioStimm"; +my %week_table = ( 1 => 2, 2 => 6, 3 => 2, 4 => 6 ); + +my ($dbh, $errorstring) = RHRD::rddb::opendb(); +if(!defined $dbh) { + print "$errorstring\n"; + exit 1; +} +my @allowed_dbs = RHRD::rddb::get_dropboxes($dbh, $user, $group); +if(!defined $allowed_dbs[0] && defined $allowed_dbs[2]) { + print "$allowed_dbs[2]\n"; + exit 1; +} + +my $dropbox = ""; +my $to_cart = 0; +foreach(@allowed_dbs) { + if($_->{'NAME'} =~ /^autoimport/) { + $dropbox = $_->{'PATH'}; + $to_cart = $_->{'TO_CART'}; + } +} +if(!$to_cart) { + print "no dropbox found!\n"; + RHRD::rddb::closedb($dbh); + exit 1; +} + +my $curweek = RHRD::utils::get_rd_week(); +my $nextweek = $curweek == 4 ? 1 : $curweek + 1; + +my @today = Date::Calc::Today(); +print "today: " . Date::Calc::Date_to_Text(@today) . "\n"; + +my $dow = Date::Calc::Day_of_Week(@today); +my @import_date = Date::Calc::Standard_to_Business(@today); + +if($dow <= $week_table{$curweek}) { + print "We are in week $curweek - broadcast is on " . Date::Calc::Day_of_Week_to_Text($week_table{$curweek}) . "\n"; + $import_date[2] = $week_table{$curweek}; + @import_date = Date::Calc::Business_to_Standard($import_date[0], $import_date[1], $import_date[2]); +} else { + print "We are in week $curweek - broadcast was on " . Date::Calc::Day_of_Week_to_Text($week_table{$curweek}) . "\n"; + print " next week is $nextweek - broadcast is on " . Date::Calc::Day_of_Week_to_Text($week_table{$nextweek}) . "\n"; + $import_date[2] = $week_table{$nextweek}; + @import_date = Date::Calc::Business_to_Standard(@import_date); + @import_date = Date::Calc::Add_Delta_Days(@import_date, 7); +} +print "day of next Radio Helsinki broadcast: " . Date::Calc::Date_to_Text(@import_date) . "\n"; + +my @broadcast_date = Date::Calc::Nth_Weekday_of_Month_Year($import_date[0], $import_date[1], 2, 1); +if(Date::Calc::Delta_Days(@broadcast_date, @import_date) > 0) { + my @tmp_date = Date::Calc::Nth_Weekday_of_Month_Year($import_date[0], $import_date[1], 2, 3); + if(Date::Calc::Delta_Days(@tmp_date, @import_date) > 0) { + @broadcast_date = @tmp_date; + } +} else { + my @tmp_date = Date::Calc::Add_Delta_YM(@broadcast_date, 0, -1); + @broadcast_date = Date::Calc::Nth_Weekday_of_Month_Year($tmp_date[0], $tmp_date[1], 2, 3); +} +print "day of latest original broadcast before next Radio Helsinki broadcast: " . Date::Calc::Date_to_Text(@broadcast_date) . "\n"; + +if(Date::Calc::Delta_Days(@broadcast_date, @today) <= 0) { + print "File won't be available by now!\n"; + RHRD::rddb::closedb($dbh); + exit 0; +} + +my $id = sprintf("%04d-%02d-%02d", $import_date[0], $import_date[1], $import_date[2]); +my $bd = sprintf("%04d-%02d-%02d", $broadcast_date[0], $broadcast_date[1], $broadcast_date[2]); +my $bdfile = sprintf("RAST%04d%02d%02d(CBA)?(.*)\.MP3", $broadcast_date[0], $broadcast_date[1], $broadcast_date[2]); + +print "looking for files from $bd in RSS Feed\n"; +print " -> $RSS_URL\n"; + +my $feed = XML::Feed->parse(URI->new($RSS_URL)) + or die "Error fetching feed: " . XML::Feed->errstr; + +my $file = ""; +my $out_file = ""; +my $i = 0; +for my $entry ($feed->entries) { + $i++; + next unless $entry->enclosure; + my $j = 0; + for my $enclosure($entry->enclosure) { + $j++; + if($enclosure->type eq "audio/mpeg" || $enclosure->type eq "audio/ogg") { + print "$i/$j: (" . $enclosure->type . ", " . $enclosure->length . ") " . $enclosure->url . "\n"; + my $url = $enclosure->url; + if($url =~ /^(.*)_cut(\.[^\.]+)$/) { + $url = $1 . $2; + } + + my $ff = File::Fetch->new(uri => $url); + next unless (uc($ff->output_file) =~ $bdfile); # file not from correct day + + my $sum_title = decode_entities($entry->title); + my $sum_text = decode_entities($entry->content->body); + + my $current_stat = `cat $STAT_FILE`; + my ($current_id, $current_file) = $current_stat =~ m/^(.*)\n(.*)/; + if($current_id eq $id && $current_file eq $ff->output_file) { + print "Already downloaded file of today\n"; + RHRD::rddb::closedb($dbh); + exit 0; + } + $out_file = $ff->output_file; + if(!rhautoimport::check_file_extension($out_file)) { + print "\n\nThe extension of the matching file '". $out_file . "' seems to be wrong - manual import necessary!!!\n"; + RHRD::rddb::closedb($dbh); + exit 1; + } + + print " --> " . $bdfile . ": downloading " . $url . " (" . $enclosure->length . " Bytes) .. "; + $file = $ff->fetch( to => '/tmp' ) or die $ff->error; + print "ok\n"; + + print "summary:\n" . $sum_title . "\n\n" . $sum_text . "\n"; + + rhautoimport::pv_add_note($sum_title, $sum_text, $PV_ID, $id, "1"); + print "\n"; + + last; + } + } + last if $file ne ""; +} + +if($file eq "" || !(-e "$file")) { + print "No Entry found from $bd or download error - "; + if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "giving up, manual import necessary!!!\n"; + } else { + print "will retry later\n"; + } + exit 1; +} + +print "will import $file to dropbox $dropbox (cart=$to_cart)\n"; + +my $error_cb = sub { + my ($text) = @_; + + print "\n$text"; + return 0; +}; + +rhautoimport::check_key_file() or die "Import Key not found, use rhautoimport-create-id to create one\n"; + +my $ret; +my $log = rhautoimport::clear_carts($dbh, $group, $to_cart); +my $import_log; +($ret, $import_log) = rhautoimport::import_single($file, $dropbox, $user, 0, $error_cb); +$log .= $import_log; + +RHRD::rddb::closedb($dbh); + +unlink $file; + +if(!$ret) { + print "\nImport Error:\n"; + print $log; + exit 1; +} + +unlink($STAT_FILE); +open(my $fhs, '>', $STAT_FILE); +print $fhs "$id\n$out_file"; +close($fhs); + +exit 0; diff --git a/rhautoimport-sm b/rhautoimport-sm new file mode 100755 index 0000000..6678d35 --- /dev/null +++ b/rhautoimport-sm @@ -0,0 +1,213 @@ +#!/usr/bin/perl -w +# +# +# rhautoimport +# +# Copyright (C) 2009-2015 Christian Pointner +# +# This file is part of rhautoimport. +# +# rhautoimport is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# rhautoimport is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with rhautoimport. If not, see . +# + +use strict; +use Date::Calc; +use XML::Feed; +use XML::Feed::Entry; +use XML::Feed::Content; +use XML::Feed::Enclosure; +use File::Fetch; +use IO::Handle; +use IPC::Open3; +use HTML::Entities; +use LWP::Simple; +use RHRD::rddb; + +use lib '/usr/local/share/rhautoimport/'; +use rhautoimport; + +my $STAT_FILE = $ENV{'HOME'} . "/rhautoimport-sm.stat"; +my $RSS_URL = "http://cba.fro.at/seriesrss/263673?c=Kfs2IoV2Wmd"; +$XML::Feed::MULTIPLE_ENCLOSURES=1; +my $PV_ID = '304'; + +binmode(STDIN, ":utf8"); +binmode(STDOUT, ":utf8"); +binmode(STDERR, ":utf8"); + +if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "!!!This is the last attempt, there won't be a retry on error!!!\n" +} + +my $user = `/usr/bin/id -un`; +$user =~ s/\n//; +my $group = "soundsmove"; + +my ($dbh, $errorstring) = RHRD::rddb::opendb(); +if(!defined $dbh) { + print "$errorstring\n"; + exit 1; +} +my @allowed_dbs = RHRD::rddb::get_dropboxes($dbh, $user, $group); +if(!defined $allowed_dbs[0] && defined $allowed_dbs[2]) { + print "$allowed_dbs[2]\n"; + exit 1; +} + +if(scalar(@allowed_dbs) != 1) { + print "found more or less than one Dropbox for this group?!\n"; + RHRD::rddb::closedb($dbh); + exit 1; +} +my $dropbox = $allowed_dbs[0]->{'PATH'}; +my $to_cart = $allowed_dbs[0]->{'TO_CART'}; + +my $curweek = RHRD::utils::get_rd_week(); + +my @today = Date::Calc::Today(); +my $dow = Date::Calc::Day_of_Week(@today); +if(($curweek != 1 && $curweek != 4) || + ($curweek == 1 && $dow > 3) || + ($curweek == 4 && $dow <= 3)) { + print "File won't be available by now!\n"; + RHRD::rddb::closedb($dbh); + exit 0; +} +print "today: " . Date::Calc::Date_to_Text(@today) . "\n"; + +my @import_date = Date::Calc::Standard_to_Business(@today); +$import_date[2] = 3; +@import_date = Date::Calc::Business_to_Standard(@import_date); +if($curweek == 4) { + @import_date = Date::Calc::Add_Delta_Days(@import_date, 7); +} +print "day of next Radio Helsinki broadcast: " . Date::Calc::Date_to_Text(@import_date) . "\n"; + +my @broadcast_date = Date::Calc::Nth_Weekday_of_Month_Year($import_date[0], $import_date[1], 3, 1); +if(Date::Calc::Delta_Days(@broadcast_date, @import_date) <= 0) { + my @tmp_date = Date::Calc::Add_Delta_YM(@broadcast_date, 0, -1); + @broadcast_date = Date::Calc::Nth_Weekday_of_Month_Year($tmp_date[0], $tmp_date[1], 3, 1); +} +print "day of latest original broadcast before next Radio Helsinki broadcast: " . Date::Calc::Date_to_Text(@broadcast_date) . "\n"; + +if(Date::Calc::Delta_Days(@broadcast_date, @today) <= 0) { + print "File won't be available by now!\n"; + RHRD::rddb::closedb($dbh); + exit 0; +} + +my $id = sprintf("%04d-%02d-%02d", @import_date); +my $bd = sprintf("%04d-%02d-%02d", @broadcast_date); +my $bdfile = sprintf("%04d-?%02d-?%02d", @broadcast_date); + +print "looking for files from $bd in RSS Feed\n"; +print " -> $RSS_URL\n"; + +my $feed = XML::Feed->parse(URI->new($RSS_URL)) + or die "Error fetching feed: " . XML::Feed->errstr; + +my $file = ""; +my $out_file = ""; +my $i = 0; +for my $entry ($feed->entries) { + $i++; + next unless $entry->enclosure; + my $j = 0; + for my $enclosure($entry->enclosure) { + $j++; + if($enclosure->type eq "audio/mpeg" || $enclosure->type eq "audio/ogg") { + print "$i/$j: (" . $enclosure->type . ", " . $enclosure->length . ") " . $enclosure->url . "\n"; + my $url = $enclosure->url; + if($url =~ /^(.*)_cut(\.[^\.]+)$/) { + $url = $1 . $2; + } + + my $ff = File::Fetch->new(uri => $url); + next unless (uc($ff->output_file) =~ $bdfile); # file not from correct day + + my $sum_title = decode_entities($entry->title); + my $sum_text = decode_entities($entry->content->body); + + my $current_stat = `cat $STAT_FILE`; + my ($current_id, $current_file) = $current_stat =~ m/^(.*)\n(.*)/; + if($current_id eq $id && $current_file eq $ff->output_file) { + print "Already downloaded file of today\n"; + RHRD::rddb::closedb($dbh); + exit 0; + } + $out_file = $ff->output_file; + if(!rhautoimport::check_file_extension($out_file)) { + print "\n\nThe extension of the matching file '". $out_file . "' seems to be wrong - manual import necessary!!!\n"; + RHRD::rddb::closedb($dbh); + exit 1; + } + + print " --> " . $bdfile . ": downloading " . $url . " (" . $enclosure->length . " Bytes) .. "; + $file = $ff->fetch( to => '/tmp' ) or die $ff->error; + print "ok\n"; + + print "summary:\n" . $sum_title . "\n\n" . $sum_text . "\n"; + + rhautoimport::pv_add_note($sum_title, $sum_text, $PV_ID, $id, "1"); + print "\n"; + + last; + } + } + last if $file ne ""; +} + +if($file eq "" || !(-e "$file")) { + print "No Entry found from $bd or download error - "; + if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "giving up, manual import necessary!!!\n"; + } else { + print "will retry later\n"; + } + exit 1; +} + +print "will import $file to dropbox $dropbox (cart=$to_cart)\n"; + +my $error_cb = sub { + my ($text) = @_; + + print "\n$text"; + return 0; +}; + +rhautoimport::check_key_file() or die "Import Key not found, use rhautoimport-create-id to create one\n"; + +my $ret; +my $log = rhautoimport::clear_carts($dbh, $group, $to_cart); +my $import_log; +($ret, $import_log) = rhautoimport::import_single($file, $dropbox, $user, 0, $error_cb); +$log .= $import_log; + +RHRD::rddb::closedb($dbh); + +unlink $file; + +if(!$ret) { + print "\nImport Error:\n"; + print $log; + exit 1; +} + +unlink($STAT_FILE); +open(my $fhs, '>', $STAT_FILE); +print $fhs "$id\n$out_file"; +close($fhs); + +exit 0; diff --git a/rhautoimport-sv b/rhautoimport-sv new file mode 100755 index 0000000..12c975c --- /dev/null +++ b/rhautoimport-sv @@ -0,0 +1,212 @@ +#!/usr/bin/perl -w +# +# +# rhautoimport +# +# Copyright (C) 2009-2015 Christian Pointner +# +# This file is part of rhautoimport. +# +# rhautoimport is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# rhautoimport is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with rhautoimport. If not, see . +# + +use strict; +use Date::Calc; +use XML::Feed; +use XML::Feed::Entry; +use XML::Feed::Content; +use XML::Feed::Enclosure; +use File::Fetch; +use IO::Handle; +use IPC::Open3; +use HTML::Entities; +use LWP::Simple; +use RHRD::rddb; + +use lib '/usr/local/share/rhautoimport/'; +use rhautoimport; + +my $STAT_FILE = $ENV{'HOME'} . "/rhautoimport-sv.stat"; +my $RSS_URL = "http://cba.fro.at/seriesrss/262457?c=Kfs2IoV2Wmd"; +$XML::Feed::MULTIPLE_ENCLOSURES=1; +my $PV_ID = '396'; + +binmode(STDIN, ":utf8"); +binmode(STDOUT, ":utf8"); +binmode(STDERR, ":utf8"); + +if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "!!!This is the last attempt, there won't be a retry on error!!!\n" +} + +my $user = `/usr/bin/id -un`; +$user =~ s/\n//; +my $group = "svetjevas"; + +my ($dbh, $errorstring) = RHRD::rddb::opendb(); +if(!defined $dbh) { + print "$errorstring\n"; + exit 1; +} +my @allowed_dbs = RHRD::rddb::get_dropboxes($dbh, $user, $group); +if(!defined $allowed_dbs[0] && defined $allowed_dbs[2]) { + print "$allowed_dbs[2]\n"; + exit 1; +} + +if(scalar(@allowed_dbs) != 1) { + print "found more or less than one Dropbox for this group?!\n"; + RHRD::rddb::closedb($dbh); + exit 1; +} +my $dropbox = $allowed_dbs[0]->{'PATH'}; +my $to_cart = $allowed_dbs[0]->{'TO_CART'}; + +my $curweek = RHRD::utils::get_rd_week(); + +my @today = Date::Calc::Today(); +my $dow = Date::Calc::Day_of_Week(@today); +if(($curweek != 3 && $curweek != 4) || + ($curweek == 4 && $dow > 2)) { + print "File won't be available by now!\n"; + RHRD::rddb::closedb($dbh); + exit 0; +} +print "today: " . Date::Calc::Date_to_Text(@today) . "\n"; + +my @import_date = Date::Calc::Standard_to_Business(@today); +$import_date[2] = 2; +@import_date = Date::Calc::Business_to_Standard(@import_date); +if($curweek == 3) { + @import_date = Date::Calc::Add_Delta_Days(@import_date, 7); +} +print "day of next Radio Helsinki broadcast: " . Date::Calc::Date_to_Text(@import_date) . "\n"; + +my @broadcast_date = Date::Calc::Nth_Weekday_of_Month_Year($import_date[0], $import_date[1], 7, 1); +if(Date::Calc::Delta_Days(@broadcast_date, @import_date) <= 0) { + my @tmp_date = Date::Calc::Add_Delta_YM(@broadcast_date, 0, -1); + @broadcast_date = Date::Calc::Nth_Weekday_of_Month_Year($tmp_date[0], $tmp_date[1], 7, 1); +} +print "day of latest original broadcast before next Radio Helsinki broadcast: " . Date::Calc::Date_to_Text(@broadcast_date) . "\n"; + +if(Date::Calc::Delta_Days(@broadcast_date, @today) <= 0) { + print "File won't be available by now!\n"; + RHRD::rddb::closedb($dbh); + exit 0; +} + +my $id = sprintf("%04d-%02d-%02d", @import_date); +my $bd = sprintf("%04d-%02d-%02d", @broadcast_date); +my $bdfile = sprintf("SEOI%02d%02d", $broadcast_date[2], $broadcast_date[1]); + +print "looking for files from $bd in RSS Feed\n"; +print " -> $RSS_URL\n"; + +my $feed = XML::Feed->parse(URI->new($RSS_URL)) + or die "Error fetching feed: " . XML::Feed->errstr; + +my $file = ""; +my $out_file = ""; +my $i = 0; +for my $entry ($feed->entries) { + $i++; + next unless $entry->enclosure; + my $j = 0; + for my $enclosure($entry->enclosure) { + $j++; + if($enclosure->type eq "audio/mpeg" || $enclosure->type eq "audio/ogg") { + print "$i/$j: (" . $enclosure->type . ", " . $enclosure->length . ") " . $enclosure->url . "\n"; + my $url = $enclosure->url; + if($url =~ /^(.*)_cut(\.[^\.]+)$/) { + $url = $1 . $2; + } + + my $ff = File::Fetch->new(uri => $url); + next unless (uc($ff->output_file) =~ $bdfile); # file not from correct day + + my $sum_title = decode_entities($entry->title); + my $sum_text = decode_entities($entry->content->body); + + my $current_stat = `cat $STAT_FILE`; + my ($current_id, $current_file) = $current_stat =~ m/^(.*)\n(.*)/; + if($current_id eq $id && $current_file eq $ff->output_file) { + print "Already downloaded file of today\n"; + RHRD::rddb::closedb($dbh); + exit 0; + } + $out_file = $ff->output_file; + if(!rhautoimport::check_file_extension($out_file)) { + print "\n\nThe extension of the matching file '". $out_file . "' seems to be wrong - manual import necessary!!!\n"; + RHRD::rddb::closedb($dbh); + exit 1; + } + + print " --> " . $bdfile . ": downloading " . $url . " (" . $enclosure->length . " Bytes) .. "; + $file = $ff->fetch( to => '/tmp' ) or die $ff->error; + print "ok\n"; + + print "summary:\n" . $sum_title . "\n\n" . $sum_text . "\n"; + + rhautoimport::pv_add_note($sum_title, $sum_text, $PV_ID, $id, "1"); + print "\n"; + + last; + } + } + last if $file ne ""; +} + +if($file eq "" || !(-e "$file")) { + print "No Entry found from $bd or download error - "; + if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "giving up, manual import necessary!!!\n"; + } else { + print "will retry later\n"; + } + exit 1; +} + +print "will import $file to dropbox $dropbox (cart=$to_cart)\n"; + +my $error_cb = sub { + my ($text) = @_; + + print "\n$text"; + return 0; +}; + +rhautoimport::check_key_file() or die "Import Key not found, use rhautoimport-create-id to create one\n"; + +my $ret; +my $log = rhautoimport::clear_carts($dbh, $group, $to_cart); +my $import_log; +($ret, $import_log) = rhautoimport::import_single($file, $dropbox, $user, 0, $error_cb); +$log .= $import_log; + +RHRD::rddb::closedb($dbh); + +unlink $file; + +if(!$ret) { + print "\nImport Error:\n"; + print $log; + exit 1; +} + +unlink($STAT_FILE); +open(my $fhs, '>', $STAT_FILE); +print $fhs "$id\n$out_file"; +close($fhs); + +exit 0; diff --git a/rhautoimport-tr b/rhautoimport-tr new file mode 100755 index 0000000..d467658 --- /dev/null +++ b/rhautoimport-tr @@ -0,0 +1,192 @@ +#!/usr/bin/perl -w +# +# +# rhautoimport +# +# Copyright (C) 2009-2015 Christian Pointner +# +# This file is part of rhautoimport. +# +# rhautoimport is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# rhautoimport is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with rhautoimport. If not, see . +# + +use strict; +use Date::Calc; +use XML::Feed; +use XML::Feed::Entry; +use XML::Feed::Content; +use XML::Feed::Enclosure; +use File::Fetch; +use IO::Handle; +use IPC::Open3; +use HTML::Entities; +use RHRD::rddb; + +use lib '/usr/local/share/rhautoimport/'; +use rhautoimport; + +my $STAT_FILE = $ENV{'HOME'} . "/rhautoimport-tr.stat"; +my $RSS_URL = "http://cba.fro.at/series/tierrechtsradio/feed?c=Kfs2IoV2Wmd"; +$XML::Feed::MULTIPLE_ENCLOSURES=1; +my $PV_ID = '506'; + +binmode(STDIN, ":utf8"); +binmode(STDOUT, ":utf8"); +binmode(STDERR, ":utf8"); + +if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "!!!This is the last attempt, there won't be a retry on error!!!\n" +} + +my $user = `/usr/bin/id -un`; +$user =~ s/\n//; +my $group = "tierrecht"; + +my ($dbh, $errorstring) = RHRD::rddb::opendb(); +if(!defined $dbh) { + print "$errorstring\n"; + exit 1; +} +my @allowed_dbs = RHRD::rddb::get_dropboxes($dbh, $user, $group); +if(!defined $allowed_dbs[0] && defined $allowed_dbs[2]) { + print "$allowed_dbs[2]\n"; + exit 1; +} + +my $idx = 0; +if(scalar(@allowed_dbs) != 1) { + print "found more or less than 1 Dropbox for this group?!\n"; + RHRD::rddb::closedb($dbh); + exit 1; +} +my $dropbox = $allowed_dbs[$idx]->{'PATH'}; +my $to_cart = $allowed_dbs[$idx]->{'TO_CART'}; + +my @today = Date::Calc::Standard_to_Business(Date::Calc::Today()); +if($today[2] == 5) { + print "please don't run this script on fridays!\n"; + RHRD::rddb::closedb($dbh); + exit 1; +} +my @broadcast_day = Date::Calc::Business_to_Standard($today[0], $today[1], 5); +if($today[2] < 5) { + @broadcast_day = Date::Calc::Add_Delta_Days($broadcast_day[0], $broadcast_day[1], $broadcast_day[2], -7); +} +my @import_date = Date::Calc::Add_Delta_Days($broadcast_day[0], $broadcast_day[1], $broadcast_day[2], 7); + +print "day of original broadcast: " . Date::Calc::Date_to_Text(@broadcast_day) . "\n"; +print "day of Radio Helsinki broadcast: " . Date::Calc::Date_to_Text(@import_date) . "\n"; + +my $bd = sprintf("%04d-%02d-%02d", $broadcast_day[0], $broadcast_day[1], $broadcast_day[2]); +my $bdfile = sprintf("^%04d%02d%02d", $broadcast_day[0], $broadcast_day[1], $broadcast_day[2]); + +print "looking for files from $bd in RSS Feed\n"; +print " -> $RSS_URL\n"; + +my $feed = XML::Feed->parse(URI->new($RSS_URL)) + or die "Error fetching feed: " . XML::Feed->errstr; + +my $file = ""; +my $out_file = ""; +my $i = 0; +for my $entry ($feed->entries) { + $i++; + next unless $entry->enclosure; + my $j = 0; + for my $enclosure($entry->enclosure) { + $j++; + if($enclosure->type eq "audio/mpeg" || $enclosure->type eq "audio/ogg") { + print "$i/$j: (" . $enclosure->type . ", " . $enclosure->length . ") " . $enclosure->url . "\n"; + my $url = $enclosure->url; + if($url =~ /^(.*)_cut(\.[^\.]+)$/) { + $url = $1 . $2; + } + + my $ff = File::Fetch->new(uri => $url); + next unless (uc($ff->output_file) =~ $bdfile); # file not from correct day + + my $sum_title = decode_entities($entry->title); + my $sum_text = decode_entities($entry->content->body); + + my $current_file = `cat $STAT_FILE`; + if($current_file eq $ff->output_file) { + print "Already downloaded file of today\n"; + RHRD::rddb::closedb($dbh); + exit 0; + } + $out_file = $ff->output_file; + if(!rhautoimport::check_file_extension($out_file)) { + print "\n\nThe extension of the matching file '". $out_file . "' seems to be wrong - manual import necessary!!!\n"; + RHRD::rddb::closedb($dbh); + exit 1; + } + + print " --> " . $bdfile . ": downloading " . $url . " (" . $enclosure->length . " Bytes) .. "; + $file = $ff->fetch( to => '/tmp' ) or die $ff->error; + print "ok\n"; + + print "summary:\n" . $sum_title . "\n\n" . $sum_text . "\n"; + + rhautoimport::pv_add_note($sum_title, $sum_text, $PV_ID, sprintf("%04d-%02d-%02d", $import_date[0], $import_date[1], $import_date[2]), "1"); + print "\n"; + + last; + } + } + last if $file ne ""; +} + +if($file eq "" || !(-e "$file")) { + print "No Entry found from $bd or download error - "; + if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "giving up, manual import necessary!!!\n"; + } else { + print "will retry later\n"; + } + exit 1; +} + +print "will import $file to dropbox $dropbox (cart=$to_cart)\n"; + +my $error_cb = sub { + my ($text) = @_; + + print "\n$text"; + return 0; +}; + +rhautoimport::check_key_file() or die "Import Key not found, use rhautoimport-create-id to create one\n"; + +my $ret; +my $log = rhautoimport::clear_carts($dbh, $group, $to_cart); +my $import_log; +($ret, $import_log) = rhautoimport::import_single($file, $dropbox, $user, 0, $error_cb); +$log .= $import_log; + +RHRD::rddb::closedb($dbh); + +unlink $file; + +if(!$ret) { + print "\nImport Error:\n"; + print $log; + exit 1; +} + +unlink($STAT_FILE); +open(my $fhs, '>', $STAT_FILE); +print $fhs $out_file; +close($fhs); + +exit 0; diff --git a/rhautoimport-ut b/rhautoimport-ut new file mode 100755 index 0000000..22b27d9 --- /dev/null +++ b/rhautoimport-ut @@ -0,0 +1,199 @@ +#!/usr/bin/perl -w +# +# +# rhautoimport +# +# Copyright (C) 2009-2015 Christian Pointner +# +# This file is part of rhautoimport. +# +# rhautoimport is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# rhautoimport is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with rhautoimport. If not, see . +# + +use strict; +use Date::Calc; +use XML::Feed; +use XML::Feed::Entry; +use XML::Feed::Content; +use XML::Feed::Enclosure; +use File::Fetch; +use IO::Handle; +use IPC::Open3; +use HTML::Entities; +use RHRD::rddb; + +use lib '/usr/local/share/rhautoimport/'; +use rhautoimport; + +my $STAT_FILE = $ENV{'HOME'} . "/rhautoimport-ut.stat"; +my $RSS_URL = "http://cba.fro.at/seriesrss/264016?c=Kfs2IoV2Wmd"; +$XML::Feed::MULTIPLE_ENCLOSURES=1; +my $PV_ID = '393'; + +binmode(STDIN, ":utf8"); +binmode(STDOUT, ":utf8"); +binmode(STDERR, ":utf8"); + +if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "!!!This is the last attempt, there won't be a retry on error!!!\n" +} + +my $user = `/usr/bin/id -un`; +$user =~ s/\n//; +my $group = "uton"; + +my ($dbh, $errorstring) = RHRD::rddb::opendb(); +if(!defined $dbh) { + print "$errorstring\n"; + exit 1; +} +my @allowed_dbs = RHRD::rddb::get_dropboxes($dbh, $user, $group); +if(!defined $allowed_dbs[0] && defined $allowed_dbs[2]) { + print "$allowed_dbs[2]\n"; + exit 1; +} + +my $idx = 0; +my $idx_reb = 1; +if(scalar(@allowed_dbs) != 2) { + print "found more or less than 2 Dropboxes for this group?!\n"; + RHRD::rddb::closedb($dbh); + exit 1; +} +my $dropbox = $allowed_dbs[$idx]->{'PATH'}; +my $to_cart = $allowed_dbs[$idx]->{'TO_CART'}; +my $dropbox_reb = $allowed_dbs[$idx_reb]->{'PATH'}; +my $to_cart_reb = $allowed_dbs[$idx_reb]->{'TO_CART'}; + +my @import_date = Date::Calc::Standard_to_Business(Date::Calc::Today()); +my $dow = $import_date[2]; +$import_date[2] = 3; +@import_date = Date::Calc::Business_to_Standard(@import_date); +@import_date = Date::Calc::Add_Delta_Days(@import_date, 7) if ($dow > 3); + +my @broadcast_day = Date::Calc::Add_Delta_Days(@import_date, -106); +my @import_date_reb = Date::Calc::Add_Delta_Days(@import_date, 2); +print "day of original broadcast: " . Date::Calc::Date_to_Text(@broadcast_day) . "\n"; +print "day of Radio Helsinki broadcast: " . Date::Calc::Date_to_Text(@import_date) . "\n"; +print "day of Radio Helsinki rebroadcast: " . Date::Calc::Date_to_Text(@import_date_reb) . "\n"; + +my $bd = sprintf("%04d-%02d-%02d", $broadcast_day[0], $broadcast_day[1], $broadcast_day[2]); +print "looking for files from $bd in RSS Feed\n"; +print " -> $RSS_URL\n"; + +my $feed = XML::Feed->parse(URI->new($RSS_URL)) + or die "Error fetching feed: " . XML::Feed->errstr; + +my $bdexp = sprintf("^UTON%02d%02d%02d", $broadcast_day[0] % 100, $broadcast_day[1], $broadcast_day[2]); + +my $file = ""; +my $out_file = ""; +my $i = 0; +for my $entry ($feed->entries) { + $i++; + next unless $entry->enclosure; + my $j = 0; + for my $enclosure($entry->enclosure) { + $j++; + if($enclosure->type eq "audio/mpeg" || $enclosure->type eq "audio/ogg") { + print "$i/$j: (" . $enclosure->type . ", " . $enclosure->length . ") " . $enclosure->url . "\n"; + my $url = $enclosure->url; + if($url =~ /^(.*)_cut(\.[^\.]+)$/) { + $url = $1 . $2; + } + + my $ff = File::Fetch->new(uri => $url); + next unless (uc($ff->output_file) =~ $bdexp); # file not from correct day + + my $sum_title = decode_entities($entry->title); + my $sum_text = decode_entities($entry->content->body); + + my $current_file = `cat $STAT_FILE`; + if($current_file eq $ff->output_file) { + print "Already downloaded file of today\n"; + RHRD::rddb::closedb($dbh); + exit 0; + } + $out_file = $ff->output_file; + if(!rhautoimport::check_file_extension($out_file)) { + print "\n\nThe extension of the matching file '". $out_file . "' seems to be wrong - manual import necessary!!!\n"; + RHRD::rddb::closedb($dbh); + exit 1; + } + + print " --> " . $bdexp . ": downloading " . $url . " (" . $enclosure->length . " Bytes) .. "; + $file = $ff->fetch( to => '/tmp' ) or die $ff->error; + print "ok\n"; + + print "summary:\n" . $sum_title . "\n\n" . $sum_text . "\n"; + + rhautoimport::pv_add_note($sum_title, $sum_text, $PV_ID, sprintf("%04d-%02d-%02d", $import_date[0], $import_date[1], $import_date[2]), "1"); + print "\n"; + rhautoimport::pv_add_note($sum_title, $sum_text, $PV_ID, sprintf("%04d-%02d-%02d", $import_date_reb[0], $import_date_reb[1], $import_date_reb[2]), "2"); + print "\n"; + + last; + } + } + last if $file ne ""; +} + +if($file eq "" || !(-e "$file")) { + print "No Entry found from $bd or download error - "; + if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "giving up, manual import necessary!!!\n"; + } else { + print "will retry later\n"; + } + exit 1; +} + +print "will import $file to dropbox $dropbox (cart=$to_cart)\n"; +print "will import $file to rebroadcast dropbox $dropbox_reb (cart=$to_cart_reb)\n\n"; + +my $error_cb = sub { + my ($text) = @_; + + print "\n$text"; + return 0; +}; + +rhautoimport::check_key_file() or die "Import Key not found, use rhautoimport-create-id to create one\n"; + +my $ret; +my $log = rhautoimport::clear_carts($dbh, $group, $to_cart); +my $import_log; +($ret, $import_log) = rhautoimport::import_single($file, $dropbox, $user, 0, $error_cb); +$log .= $import_log; +$import_log = rhautoimport::clear_carts($dbh, $group, $to_cart_reb); +$log .= $import_log; +($ret, $import_log) = rhautoimport::import_single($file, $dropbox_reb, $user, 0, $error_cb); +$log .= $import_log; + +RHRD::rddb::closedb($dbh); + +unlink $file; + +if(!$ret) { + print "\nImport Error:\n"; + print $log; + exit 1; +} + +unlink($STAT_FILE); +open(my $fhs, '>', $STAT_FILE); +print $fhs $out_file; +close($fhs); + +exit 0; diff --git a/rhautoimport-zf b/rhautoimport-zf new file mode 100755 index 0000000..264232c --- /dev/null +++ b/rhautoimport-zf @@ -0,0 +1,193 @@ +#!/usr/bin/perl -w +# +# +# rhautoimport +# +# Copyright (C) 2009-2015 Christian Pointner +# +# This file is part of rhautoimport. +# +# rhautoimport is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# rhautoimport is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with rhautoimport. If not, see . +# + +use strict; +use Date::Calc; +use XML::Feed; +use XML::Feed::Entry; +use XML::Feed::Content; +use XML::Feed::Enclosure; +use File::Fetch; +use IO::Handle; +use IPC::Open3; +use HTML::Entities; +use File::Touch; +use RHRD::rddb; + +use lib '/usr/local/share/rhautoimport/'; +use rhautoimport; + +my $STAT_FILE = $ENV{'HOME'} . "/rhautoimport-zf.stat"; +my $NOTE_FILE = $ENV{'HOME'} . "/rhautoimport-zf.last_note"; +my $NEW_FILE = $ENV{'HOME'} . "/rhautoimport-zf.is_new"; +my $RSS_URL = "http://freie-radios.net/portal/podcast.php?serie=53&rss"; +my $PV_ID = '300'; + +binmode(STDIN, ":utf8"); +binmode(STDOUT, ":utf8"); +binmode(STDERR, ":utf8"); + +if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "!!!This is the last attempt, there won't be a retry on error!!!\n" +} + +my $user = `/usr/bin/id -un`; +$user =~ s/\n//; +my $group = "zipfm"; + +my ($dbh, $errorstring) = RHRD::rddb::opendb(); +if(!defined $dbh) { + print "$errorstring\n"; + exit 1; +} +my @allowed_dbs = RHRD::rddb::get_dropboxes($dbh, $user, $group); +if(!defined $allowed_dbs[0] && defined $allowed_dbs[2]) { + print "$allowed_dbs[2]\n"; + exit 1; +} + + +my $dropbox = ""; +my $to_cart = 0; +my $dropbox_reb = ""; +my $to_cart_reb = 0; + +my @today = Date::Calc::Today(); +my $dow = Date::Calc::Day_of_Week(@today); +my @tomorrow = Date::Calc::Add_Delta_Days(@today, 1); + +my @broadcast_day = @today; +my @rebroadcast_day = @tomorrow; +print "day of broadcast: " . Date::Calc::Date_to_Text(@broadcast_day) . "\n"; +print "day of rebroadcast: " . Date::Calc::Date_to_Text(@rebroadcast_day) . "\n" unless $dow == 5; + +for my $href ( @allowed_dbs ) { + if ($dow == int(substr($href->{'NAME'},0,2))) { + $dropbox = $href->{'PATH'}; + $to_cart = $href->{'TO_CART'}; + } +} + +if($dropbox eq "") { + print "no dropbox for day in question\n"; + RHRD::rddb::closedb($dbh); + exit 0; +} + +my $regexp = sprintf("%04d%02d%02d", $broadcast_day[0], $broadcast_day[1], $broadcast_day[2]); +print "looking for file from " . Date::Calc::Date_to_Text(@broadcast_day) . " in RSS Feed\n"; +print " -> $RSS_URL\n"; + +my $feed = XML::Feed->parse(URI->new($RSS_URL)) + or die "Error fetching feed: " . XML::Feed->errstr; + +my $file = ""; +my $out_file = ""; +for my $entry ($feed->entries) { + if($entry->enclosure && $entry->enclosure->type eq "audio/mpeg") { + next unless $entry->enclosure->url =~ /$regexp/; + + my $sum_title = decode_entities($entry->title); + my $sum_text = decode_entities($entry->content->body); + + my $ff = File::Fetch->new(uri => $entry->enclosure->url); + my $current_file = `cat $STAT_FILE`; + if($current_file eq $ff->output_file) { + print "Already downloaded file of today\n"; + RHRD::rddb::closedb($dbh); + exit 0; + } + $out_file = $ff->output_file; + if(!rhautoimport::check_file_extension($out_file)) { + print "\n\nThe extension of the matching file '". $out_file . "' seems to be wrong - manual import necessary!!!\n"; + RHRD::rddb::closedb($dbh); + exit 1; + } + + print $regexp . ": downloading " . $entry->enclosure->url . " (" . $entry->enclosure->length . " Bytes) .. "; + $file = $ff->fetch( to => '/tmp' ) or die $ff->error; + print "ok\n"; + + print "summary:\n" . $sum_title . "\n\n" . $sum_text . "\n"; + + if($#ARGV >= 0 && $ARGV[0] eq 'nopv') { + print "not adding note to PV for regular entry - only for rebroadcast\n"; + } else { + rhautoimport::pv_add_note($sum_title, $sum_text, $PV_ID, sprintf("%04d-%02d-%02d", @broadcast_day), "1", 1); + } + rhautoimport::pv_add_note($sum_title, $sum_text, $PV_ID, sprintf("%04d-%02d-%02d", @rebroadcast_day), "2", 0) unless $dow == 5; + print "\n"; + + unlink($NOTE_FILE); + open(my $fhs, '>', $NOTE_FILE); + binmode($fhs, ":utf8"); + print $fhs $sum_title . "\n\n" . $sum_text . "\n"; + close($fhs); + + last; + } +} +if($file eq "" || !(-e "$file")) { + print "No Entry found for " . Date::Calc::Date_to_Text(@broadcast_day) . " or download error - "; + if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "giving up, manual import necessary!!!\n"; + } else { + print "will retry later\n"; + } + exit 1; +} + +print "will import $file to dropbox $dropbox (cart=$to_cart)\n"; + +my $error_cb = sub { + my ($text) = @_; + + print "\n$text"; + return 0; +}; + +rhautoimport::check_key_file() or die "Import Key not found, use rhautoimport-create-id to create one\n"; + +my $ret; +my $log = rhautoimport::clear_carts($dbh, $group, $to_cart); +my $import_log; +($ret, $import_log) = rhautoimport::import_single($file, $dropbox, $user, 0, $error_cb); +$log .= $import_log; + +RHRD::rddb::closedb($dbh); + +unlink $file; + +if(!$ret) { + print "\nImport Error:\n"; + print $log; + exit 1; +} + +unlink($STAT_FILE); +open(my $fhs, '>', $STAT_FILE); +print $fhs $out_file; +close($fhs); +touch($NEW_FILE); + +exit 0; diff --git a/rhautoimport-zffe b/rhautoimport-zffe new file mode 100755 index 0000000..eb4c1a1 --- /dev/null +++ b/rhautoimport-zffe @@ -0,0 +1,118 @@ +#!/usr/bin/perl -w +# +# +# rhautoimport +# +# Copyright (C) 2009-2015 Christian Pointner +# +# This file is part of rhautoimport. +# +# rhautoimport is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# rhautoimport is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with rhautoimport. If not, see . +# + +use strict; +use Switch; +use Date::Calc; +use Time::localtime; +use IO::Handle; +use IPC::Open3; +use File::Slurp; + +use lib '/usr/local/share/rhautoimport/'; +use rhautoimport; + +my $STAT_FILE = $ENV{'HOME'} . "/rhautoimport-zffe.stat"; +my $ZF_NOTE_FILE = $ENV{'HOME'} . "/rhautoimport-zf.last_note"; +my $ZF_NEW_FILE = $ENV{'HOME'} . "/rhautoimport-zf.is_new"; +my $FE_NOTE_FILE = $ENV{'HOME'} . "/rhautoimport-fe.last_note"; +my $FE_NEW_FILE = $ENV{'HOME'} . "/rhautoimport-fe.is_new"; +my $PV_ID = '359'; + +binmode(STDIN, ":utf8"); +binmode(STDOUT, ":utf8"); +binmode(STDERR, ":utf8"); + +if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "!!!This is the last attempt, there won't be a retry on error!!!\n" +} + +my @today = Date::Calc::Today(); +my $dow = Date::Calc::Day_of_Week(@today); +my @tomorrow = Date::Calc::Add_Delta_Days($today[0], $today[1], $today[2], 1); + +my @tmp = Date::Calc::Standard_to_Business(@today); +$tmp[2] = 1; +my @this_mon = Date::Calc::Business_to_Standard(@tmp); +my @next_mon = Date::Calc::Add_Delta_Days($this_mon[0], $this_mon[1], $this_mon[2], 7); + +my @import_day = @tomorrow; +if ($dow == 5) { + @import_day = @next_mon; +} + +my $import_date = sprintf("%04d-%02d-%02d", @import_day); +my $current_date = `cat $STAT_FILE`; +if($current_date eq $import_date) { + print "Already created note of today\n"; + exit 0; +} + +unless(-e $ZF_NEW_FILE) { + print "zip-fm not yet imported - "; + if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "giving up, rebroadcasting last show"; + } else { + print "will retry later\n"; + exit 0, + } +} + +unless(-e $FE_NEW_FILE) { + print "focus europa not yet imported - "; + if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "giving up, rebroadcasting last show"; + } else { + print "will retry later\n"; + exit 0, + } +} + +sub get_note { + my ($file) = @_; + my @lines = read_file($file, binmode => ':utf8', err_mode => 'quiet'); + return "" unless ($lines[0]); + + chomp($lines[0]); + $lines[0] = "

$lines[0]

\n"; + return join('',@lines); +}; + +my $sum_title = "focus europa, zip-fm (WH)"; +my $sum_text = get_note($FE_NOTE_FILE) . "
\n"; +$sum_text = $sum_text . "
\n" . get_note($ZF_NOTE_FILE) . "
\n"; +$sum_text = $sum_text . "

\n"; + +print "\nsummary:\n" . $sum_title . "\n\n" . $sum_text . "\n"; +rhautoimport::pv_add_note($sum_title, $sum_text, $PV_ID, sprintf("%04d-%02d-%02d", @import_day), "1"); +print "\n"; + +unlink($STAT_FILE); +open(my $fhs, '>', $STAT_FILE); +print $fhs $import_date; +close($fhs); + +unlink($ZF_NEW_FILE); +unlink($FE_NEW_FILE); + +exit 0; diff --git a/rhautoimport-zfw b/rhautoimport-zfw new file mode 100755 index 0000000..9cdb169 --- /dev/null +++ b/rhautoimport-zfw @@ -0,0 +1,195 @@ +#!/usr/bin/perl -w +# +# +# rhautoimport +# +# Copyright (C) 2009-2015 Christian Pointner +# +# This file is part of rhautoimport. +# +# rhautoimport is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# rhautoimport is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with rhautoimport. If not, see . +# + +use strict; +use Date::Calc; +use Time::localtime; +use XML::Feed; +use XML::Feed::Entry; +use XML::Feed::Content; +use XML::Feed::Enclosure; +use File::Fetch; +use IO::Handle; +use IPC::Open3; +use HTML::Entities; +use RHRD::rddb; + +use lib '/usr/local/share/rhautoimport/'; +use rhautoimport; + +my $STAT_FILE = $ENV{'HOME'} . "/rhautoimport-zfw.stat"; +my $RSS_URL = "http://cba.fro.at/seriesrss/263764?c=Kfs2IoV2Wmd"; +$XML::Feed::MULTIPLE_ENCLOSURES=1; +my $PV_ID = '360'; + +binmode(STDIN, ":utf8"); +binmode(STDOUT, ":utf8"); +binmode(STDERR, ":utf8"); + +if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "!!!This is the last attempt, there won't be a retry on error!!!\n" +} + +my $user = `/usr/bin/id -un`; +$user =~ s/\n//; +my $group = "zipfmwla"; + +my ($dbh, $errorstring) = RHRD::rddb::opendb(); +if(!defined $dbh) { + print "$errorstring\n"; + exit 1; +} +my @allowed_dbs = RHRD::rddb::get_dropboxes($dbh, $user, $group); +if(!defined $allowed_dbs[0] && defined $allowed_dbs[2]) { + print "$allowed_dbs[2]\n"; + exit 1; +} + +if(scalar(@allowed_dbs) != 1) { + print "found no or more than one Dropboxes for this group?!\n"; + RHRD::rddb::closedb($dbh); + exit 1; +} +my $dropbox = $allowed_dbs[0]->{'PATH'}; +my $to_cart = $allowed_dbs[0]->{'TO_CART'}; + + +my @today = Date::Calc::Today(); +my @yesterday = Date::Calc::Add_Delta_Days($today[0], $today[1], $today[2], -1); +my @tomorrow = Date::Calc::Add_Delta_Days($today[0], $today[1], $today[2], 1); + +my @broadcast_day = @yesterday; +my @import_day = @today; +if(localtime->hour >= 12) { + print "It's past noon, assuming to import file from today\n"; + @broadcast_day = @today; + @import_day = @tomorrow; +} + +print "day of original broadcast: " . Date::Calc::Date_to_Text(@broadcast_day) . "\n"; +print "day of Radio Helsinki broadcast: " . Date::Calc::Date_to_Text(@import_day) . "\n"; + +my $id = sprintf("%04d-%02d-%02d", $import_day[0], $import_day[1], $import_day[2]); +my $bd = sprintf("%04d-%02d-%02d", $broadcast_day[0], $broadcast_day[1], $broadcast_day[2]); +print "looking for files from $bd in RSS Feed\n"; +print " -> $RSS_URL\n"; + +my $feed = XML::Feed->parse(URI->new($RSS_URL)) + or die "Error fetching feed: " . XML::Feed->errstr; + +my $bdfile = sprintf("%04d%02d%02d", $broadcast_day[0], $broadcast_day[1], $broadcast_day[2]); + +my $file = ""; +my $out_file = ""; +my $i = 0; +for my $entry ($feed->entries) { + $i++; + next unless $entry->enclosure; + my $j = 0; + for my $enclosure($entry->enclosure) { + $j++; + if($enclosure->type eq "audio/mpeg" || $enclosure->type eq "audio/ogg") { + print "$i/$j: (" . $enclosure->type . ", " . $enclosure->length . ") " . $enclosure->url . "\n"; + my $url = $enclosure->url; + if($url =~ /^(.*)_cut(\.[^\.]+)$/) { + $url = $1 . $2; + } + + my $ff = File::Fetch->new(uri => $url); + next unless (uc($ff->output_file) =~ $bdfile); # file not from correct day + + my ($sum_title, $sum_title_rest) = split(/:/, decode_entities($entry->title), 2); + my $sum_text = decode_entities($entry->content->body); + + my $current_stat = `cat $STAT_FILE`; + my ($current_id, $current_file) = $current_stat =~ m/^(.*)\n(.*)/; + if($current_id eq $id && $current_file eq $ff->output_file) { + print "Already downloaded file of today\n"; + RHRD::rddb::closedb($dbh); + exit 0; + } + $out_file = $ff->output_file; + if(!rhautoimport::check_file_extension($out_file)) { + print "\n\nThe extension of the matching file '". $out_file . "' seems to be wrong - manual import necessary!!!\n"; + RHRD::rddb::closedb($dbh); + exit 1; + } + + print " --> " . $bdfile . ": downloading " . $url . " (" . $enclosure->length . " Bytes) .. "; + $file = $ff->fetch( to => '/tmp' ) or die $ff->error; + print "ok\n"; + + print "summary:\n" . $sum_title . "\n\n" . $sum_text . "\n"; + + rhautoimport::pv_add_note($sum_title, $sum_text, $PV_ID, $id, "1"); + print "\n"; + + last; + } + } + last if $file ne ""; +} + +if($file eq "" || !(-e "$file")) { + print "No Entry found from $bd or download error - "; + if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "giving up, manual import necessary!!!\n"; + } else { + print "will retry later\n"; + } + exit 1; +} + +print "will import $file to dropbox $dropbox (cart=$to_cart)\n"; + +my $error_cb = sub { + my ($text) = @_; + + print "\n$text"; + return 0; +}; + +rhautoimport::check_key_file() or die "Import Key not found, use rhautoimport-create-id to create one\n"; + +my $ret; +my $log = rhautoimport::clear_carts($dbh, $group, $to_cart); +my $import_log; +($ret, $import_log) = rhautoimport::import_single($file, $dropbox, $user, 0, $error_cb); +$log .= $import_log; + +RHRD::rddb::closedb($dbh); + +unlink $file; + +if(!$ret) { + print "\nImport Error:\n"; + print $log; + exit 1; +} + +unlink($STAT_FILE); +open(my $fhs, '>', $STAT_FILE); +print $fhs "$id\n$out_file"; +close($fhs); + +exit 0; diff --git a/rhautoimport.pm b/rhautoimport.pm new file mode 100644 index 0000000..76e3893 --- /dev/null +++ b/rhautoimport.pm @@ -0,0 +1,247 @@ +#!/usr/bin/perl -w +# +# +# rhautoimport +# +# Copyright (C) 2009-2015 Christian Pointner +# +# This file is part of rhautoimport. +# +# rhautoimport is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# rhautoimport is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with rhautoimport. If not, see . +# + +use strict; + +package rhautoimport; + +use IO::Handle; +use IPC::Open3; +use File::Spec; +use File::Basename; +use URI::Escape; +use LWP::Simple; +use XML::Feed; + +my $ssh_host = "airplay"; +my $ssh_user = "rhautoimport"; +my $ssh_key_file = "$ENV{'HOME'}/.rhautoimport/import.key"; +my $ssh_dir = "/programm/.rhautoimport"; +my $rdimport_wrapper = "/usr/local/bin/dropbox_newfile.pl"; + +sub fetch_parse_rss +{ + my ($url, $ua_str) = @_; + + my $uri = URI->new($url); + $ua_str = "Radio Helsinki - Automatic Import" unless $ua_str; + + my $ua = LWP::UserAgent->new; + $ua->agent($ua_str); + $ua->env_proxy; + my $res = URI::Fetch->fetch($uri, UserAgent => $ua) + or die URI::Fetch->errstr; + die "This feed has been permanently removed" + if $res->status == URI::Fetch::URI_GONE(); + + my $xml = $res->content; + + return XML::Feed->parse(\$xml); +} + +sub clear_carts +{ + my ($dbh, $group, $to_cart, $progress_cb) = @_; + + my ($low_cart, $high_cart) = ($to_cart, $to_cart); + if($to_cart == 0) { + ($low_cart, $high_cart) = get_cart_range($dbh, $group); + } + + my $log = "clearing Carts $low_cart - $high_cart\n"; + print $log; + my $cart = $low_cart; + while($cart <= $high_cart) { + $progress_cb->($cart - $low_cart, $high_cart + 1 - $low_cart, "deleting $cart") if($progress_cb); + + my $sql = qq{select CUT_NAME from CUTS where CART_NUMBER='$cart';}; + my $sth = $dbh->prepare($sql); + $sth->execute(); + while(my $cut_name = $sth->fetchrow_array()) { + delete_file("/var/snd/$cut_name.wav"); + $log .= " - deleting file /var/snd/$cut_name.wav"; + } + $sth->finish(); + + $sql = qq{delete from CUTS where CART_NUMBER='$cart';}; + $sth = $dbh->prepare($sql); + $sth->execute(); + $sth->finish(); + + $sql = qq{delete from CART where NUMBER='$cart';}; + $sth = $dbh->prepare($sql); + $sth->execute(); + $sth->finish(); + + $cart++; + } + + return $log; +} + +sub import_single +{ + my ($file, $dropbox, $user, $progress_cb, $error_cb) = @_; + print "Starting import from file $file to $dropbox\n"; + $progress_cb->(0, 1, $file) if($progress_cb); + prepare_import($user); + my ($ret, $out) = import_file($file, $dropbox, $user, $error_cb); + if($ret) { + $progress_cb->(1, 1, "Import abgeschlossen!") if($progress_cb); + } else { + $progress_cb->(0, 1, "Import abgebrochen!") if($progress_cb); + } + return ($ret, $out); +} + +sub scp_put_file +{ + my ($file, $user, $host, $path) = @_; + my @cmd = ( 'scp', '-prqB', '-i', $ssh_key_file, $file, "$user\@$host:$path/" ); + my ($reader, $writer, $error ) = ( new IO::Handle, new IO::Handle, new IO::Handle ); + $writer->autoflush(1); + local $SIG{CHLD} = 'DEFAULT'; + my $pid = open3($writer, $reader, $error, @cmd); + binmode($reader, ":utf8"); + binmode($writer, ":utf8"); + binmode($error, ":utf8"); + waitpid $pid, 0; + my $errstr = ""; + if ( $? >> 8 ) { + $errstr = join('', <$error>); + } + return $errstr; +} + +sub ssh_exec_command +{ + my ($command) = @_; + my @cmd = ( 'ssh' , '-q', '-o', 'BatchMode=yes', '-i', $ssh_key_file, "$ssh_user\@$ssh_host" , $command ); + my ($reader, $writer, $error ) = ( new IO::Handle, new IO::Handle, new IO::Handle ); + $writer->autoflush(1); + local $SIG{CHLD} = 'DEFAULT'; + my $pid = open3($writer, $reader, $error, @cmd); + binmode($reader, ":utf8"); + binmode($writer, ":utf8"); + binmode($error, ":utf8"); + waitpid $pid, 0; + my $out = join('', <$reader>); + my $errstr = ""; + if ( $? >> 8 ) { + $errstr = join('', <$error>); + $errstr = "unkown error" if($errstr eq ""); + } + return ($out, $errstr); +} + +sub prepare_import +{ + my ($user) = @_; + + ### create user directory + ssh_exec_command("umask 002; mkdir -p \"$ssh_dir/$user\""); +} + +sub import_file +{ + my ($file, $dropbox, $user, $error_cb) = @_; + + print " - importing $file to $dropbox .. "; + $| = 1; + + ### copy file to master server + my $err = scp_put_file($file, $ssh_user, $ssh_host, "$ssh_dir/$user"); + if($err ne "") { + print "Transfer Error\n"; + if($error_cb) { + return $error_cb->($err); + } + return 0; + } + print "transferred .. "; + + ### remotely call rdimport + my ($volume, $directories, $remote_file) = File::Spec->splitpath($file); + $remote_file = "$ssh_dir/$user/$remote_file"; + my ($out ,$error) = ssh_exec_command("$rdimport_wrapper --path \"$dropbox\" --file \"$remote_file\""); + my $lastline = $1 if $out =~ /\n(.*)$/; + if($error ne "" || $lastline !~ /^[0-9:\- ]+:\s+Deleted file/) { + print "Import Error\n"; + delete_file($remote_file); + if($error_cb) { + if($error) { + return $error_cb->("Import Fehler: $error"); + } + else { + return $error_cb->("Import Fehler: \n$out"); + } + } + return 0; + } + + print "imported OK\n"; + return (1, $out); +} + +sub delete_file +{ + my ($filename) = @_; + + print " - deleting $filename\n"; + my ($out ,$err) = ssh_exec_command("rm \"$filename\""); + if($err ne "") { + return 0; + } + return 1; +} + +sub pv_add_note +{ + my ( $title, $text, $id, $date, $type, $index ) = @_; + my @script = ('python', '/srv/pv/pv/manage.py', 'addnote', $id, $date, $type); + push(@script , $index) unless (!defined $index); + + my ($reader, $writer, $error ) = ( new IO::Handle, new IO::Handle, new IO::Handle ); + $writer->autoflush(1); + local $SIG{CHLD} = 'DEFAULT'; + my $pid = open3($writer, $reader, $error, @script); + binmode($reader, ":utf8"); + binmode($writer, ":utf8"); + binmode($error, ":utf8"); + print $writer $title . "\n" . $text; + close $writer; + waitpid $pid, 0; + my $err_out = join('', <$error>); + my $read_out = join('', <$reader>); + if ( $? >> 8 ) { + print "\n\nPV: adding note returned non-zero value\n"; + print "STDERR:\n" . $err_out . "\n" unless $err_out eq ''; + print "STDOUT:\n" . $read_out . "\n" unless $read_out eq ''; + print "Ignoring failed headline import!\n"; + } else { + print $read_out . "\n" unless $read_out eq ''; + print $err_out . "\n" unless $err_out eq ''; + } +} + +1; diff --git a/rhimport b/rhimport deleted file mode 100755 index f0be9df..0000000 --- a/rhimport +++ /dev/null @@ -1,126 +0,0 @@ -#!/usr/bin/perl -w -# -# -# rhimport -# -# Copyright (C) 2009-2015 Christian Pointner -# -# This file is part of rhimport. -# -# rhimport is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# any later version. -# -# rhimport is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with rhimport. If not, see . -# - -use strict; -use Getopt::Long; -use RHRD::rddb; - -use lib '/usr/local/share/rhimport/'; -use rhimport; - -my $HELP = 0; -my $FILE = ""; -my $DROPBOX = ""; -my $LISTALLOWED = 0; - -GetOptions ("help!" => \$HELP, - "file=s" => \$FILE, - "dropbox=s" => \$DROPBOX, - "list-allowed!" => \$LISTALLOWED, - ) or die(); - -if($HELP) { - print << "EOF"; -usage: $0 --file