From 81531b51fe70fee446d22d3505d04709cd7b5b8a Mon Sep 17 00:00:00 2001 From: Christian Pointner <equinox@spreadspace.org> Date: Sat, 26 Sep 2015 22:40:24 +0200 Subject: renamed rhimport to rhautoimport diff --git a/Makefile b/Makefile index 7202ab4..d1efa33 100644 --- a/Makefile +++ b/Makefile @@ -1,22 +1,22 @@ ## -## rhimport +## rhautoimport ## ## Copyright (C) 2009-2015 Christian Pointner <equinox@helsinki.at> ## -## This file is part of rhimport. +## This file is part of rhautoimport. ## -## rhimport is free software: you can redistribute it and/or modify +## rhautoimport is free software: you can redistribute it and/or modify ## it under the terms of the GNU General Public License as published by ## the Free Software Foundation, either version 3 of the License, or ## any later version. ## -## rhimport is distributed in the hope that it will be useful, +## rhautoimport is distributed in the hope that it will be useful, ## but WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ## GNU General Public License for more details. ## ## You should have received a copy of the GNU General Public License -## along with rhimport. If not, see <http://www.gnu.org/licenses/>. +## along with rhautoimport. If not, see <http://www.gnu.org/licenses/>. ## ifneq ($(MAKECMDGOALS),distclean) @@ -25,7 +25,7 @@ endif .PHONY: clean distclean -EXECUTABLE := rhimport +EXECUTABLE := rhautoimport IMPORTER := btl dn fe fw mz nw oi o94n rs ra sm sv tr ut zf zffe zfw EXECUTABLES := $(IMPORTER:%=$(EXECUTABLE)-%) @@ -44,23 +44,20 @@ install: $(INSTALL_TARGETS) install-bin: $(INSTALL) -d $(DESTDIR)$(BINDIR) - $(INSTALL) -m 755 $(EXECUTABLE) $(DESTDIR)$(BINDIR) - @sed s#/usr/local/share/rhimport/#$(SHAREDIR)/$(EXECUTABLE)/#g -i $(DESTDIR)$(BINDIR)/$(EXECUTABLE) $(INSTALL) -m 755 $(EXECUTABLES) $(DESTDIR)$(BINDIR) @for exe in $(EXECUTABLES); do \ - sed s#/usr/local/share/rhimport/#$(SHAREDIR)/$(EXECUTABLE)/#g -i $(DESTDIR)$(BINDIR)/$$exe ; \ + sed s#/usr/local/share/rhautoimport/#$(SHAREDIR)/$(EXECUTABLE)/#g -i $(DESTDIR)$(BINDIR)/$$exe ; \ done install-share: $(INSTALL) -d $(DESTDIR)$(SHAREDIR)/$(EXECUTABLE) - $(INSTALL) -m 644 rhimport.pm $(DESTDIR)$(SHAREDIR)/$(EXECUTABLE) + $(INSTALL) -m 644 rhautoimport.pm $(DESTDIR)$(SHAREDIR)/$(EXECUTABLE) uninstall: remove remove: $(REMOVE_TARGETS) remove-bin: - rm -f $(DESTDIR)$(BINDIR)/$(EXECUTABLE) rm -f $(DESTDIR)$(BINDIR)/$(EXECUTABLES) remove-share: diff --git a/configure b/configure index 5b2df89..3184c68 100755 --- a/configure +++ b/configure @@ -1,24 +1,24 @@ #!/bin/sh # # -# rhimport +# rhautoimport # # Copyright (C) 2009-2015 Christian Pointner <equinox@helsinki.at> # -# This file is part of rhimport. +# This file is part of rhautoimport. # -# rhimport is free software: you can redistribute it and/or modify +# rhautoimport is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # any later version. # -# rhimport is distributed in the hope that it will be useful, +# rhautoimport is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with rhimport. If not, see <http://www.gnu.org/licenses/>. +# along with rhautoimport. If not, see <http://www.gnu.org/licenses/>. # PREFIX='/usr/local' diff --git a/fetch-feeds-frn.pl b/fetch-feeds-frn.pl index ff8cd16..bae84a1 100755 --- a/fetch-feeds-frn.pl +++ b/fetch-feeds-frn.pl @@ -1,24 +1,24 @@ #!/usr/bin/perl -w # # -# rhimport +# rhautoimport # # Copyright (C) 2009-2015 Christian Pointner <equinox@helsinki.at> # -# This file is part of rhimport. +# This file is part of rhautoimport. # -# rhimport is free software: you can redistribute it and/or modify +# rhautoimport is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # any later version. # -# rhimport is distributed in the hope that it will be useful, +# rhautoimport is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with rhimport. If not, see <http://www.gnu.org/licenses/>. +# along with rhautoimport. If not, see <http://www.gnu.org/licenses/>. # use strict; diff --git a/fetch-files-cba.pl b/fetch-files-cba.pl index a812b1c..04db78e 100755 --- a/fetch-files-cba.pl +++ b/fetch-files-cba.pl @@ -1,24 +1,24 @@ #!/usr/bin/perl -w # # -# rhimport +# rhautoimport # # Copyright (C) 2009-2015 Christian Pointner <equinox@helsinki.at> # -# This file is part of rhimport. +# This file is part of rhautoimport. # -# rhimport is free software: you can redistribute it and/or modify +# rhautoimport is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # any later version. # -# rhimport is distributed in the hope that it will be useful, +# rhautoimport is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with rhimport. If not, see <http://www.gnu.org/licenses/>. +# along with rhautoimport. If not, see <http://www.gnu.org/licenses/>. # use strict; diff --git a/rhautoimport-btl b/rhautoimport-btl new file mode 100755 index 0000000..98252d3 --- /dev/null +++ b/rhautoimport-btl @@ -0,0 +1,177 @@ +#!/usr/bin/perl -w +# +# +# rhautoimport +# +# Copyright (C) 2009-2015 Christian Pointner <equinox@helsinki.at> +# +# This file is part of rhautoimport. +# +# rhautoimport is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# rhautoimport is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with rhautoimport. If not, see <http://www.gnu.org/licenses/>. +# + +use strict; +use Date::Calc; +use XML::Feed; +use XML::Feed::Entry; +use XML::Feed::Content; +use XML::Feed::Enclosure; +use File::Fetch; +use IO::Handle; +use IPC::Open3; +use HTML::Entities; +use LWP::Simple; +use RHRD::rddb; + +use lib '/usr/local/share/rhautoimport/'; +use rhautoimport; + +my $STAT_FILE = $ENV{'HOME'} . "/rhautoimport-btl.stat"; +my $RSS_URL = "http://www.btlonline.org/rss/btl128.xml"; +my $PV_ID = '221'; + +binmode(STDIN, ":utf8"); +binmode(STDOUT, ":utf8"); +binmode(STDERR, ":utf8"); + +if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "!!!This is the last attempt, there won't be a retry on error!!!\n" +} + +my $user = `/usr/bin/id -un`; +$user =~ s/\n//; +my $group = "betweenlin"; + +my ($dbh, $errorstring) = RHRD::rddb::opendb(); +if(!defined $dbh) { + print "$errorstring\n"; + exit 1; +} +my @allowed_dbs = RHRD::rddb::get_dropboxes($dbh, $user, $group); +if(!defined $allowed_dbs[0] && defined $allowed_dbs[2]) { + print "$allowed_dbs[2]\n"; + exit 1; +} + +if(scalar(@allowed_dbs) != 1) { + print "found more or less than one Dropbox for this group?!\n"; + RHRD::rddb::closedb($dbh); + exit 1; +} +my $dropbox = $allowed_dbs[0]->{'PATH'}; +my $to_cart = $allowed_dbs[0]->{'TO_CART'}; + + +my @today = Date::Calc::Today(); +my @import_date = Date::Calc::Add_Delta_Days(@today, 7); +@import_date = Date::Calc::Standard_to_Business(@import_date); +$import_date[2] = 2; +my @broadcast_date = @import_date; +$broadcast_date[2] = 5; + +@import_date = Date::Calc::Business_to_Standard(@import_date); +@broadcast_date = Date::Calc::Business_to_Standard(@broadcast_date); + +print "today: " . Date::Calc::Date_to_Text(@today) . "\n"; +print "day of next Radio Helsinki broadcast: " . Date::Calc::Date_to_Text(@import_date) . "\n"; +print "ending of current broadcast cycle: " . Date::Calc::Date_to_Text(@broadcast_date) . "\n"; + +my $id = sprintf("%04d-%02d-%02d", @import_date); +my $bd = sprintf("%04d-%02d-%02d", @broadcast_date); +my $bdfile = sprintf("%02d%02d%02d-btlv128\.mp3", $broadcast_date[0]%100, $broadcast_date[1], $broadcast_date[2]); + +print "looking for files like '$bdfile' in RSS Feed\n"; +print " -> $RSS_URL\n"; + +my $feed = XML::Feed->parse(URI->new($RSS_URL)) + or die "Error fetching feed: " . XML::Feed->errstr; + +my $file = ""; +my $out_file = ""; +for my $entry ($feed->entries) { + if($entry->enclosure && ($entry->enclosure->type eq "audio/mpeg" || entry->enclosure->type eq "audio/mp3")) { + next unless $entry->enclosure->url =~ /$bdfile/; + + my $sum_title = decode_entities($entry->title); + $sum_title =~ s/ \(128 kbps\)$//; + my $sum_text = decode_entities($entry->content->body); + + my $ff = File::Fetch->new(uri => $entry->enclosure->url); + my $current_file = `cat $STAT_FILE`; + if($current_file eq $ff->output_file) { + print "Already downloaded file of today\n"; + RHRD::rddb::closedb($dbh); + exit 0; + } + $out_file = $ff->output_file; + if(!rhautoimport::check_file_extension($out_file)) { + print "\n\nThe extension of the matching file '". $out_file . "' seems to be wrong - manual import necessary!!!\n"; + RHRD::rddb::closedb($dbh); + exit 1; + } + + print $bdfile . ": downloading " . $entry->enclosure->url . " (" . $entry->enclosure->length . " Bytes) .. "; + $file = $ff->fetch( to => '/tmp' ) or die $ff->error; + print "ok\n"; + + print "summary:\n" . $sum_title . "\n\n" . $sum_text . "\n"; + + rhautoimport::pv_add_note($sum_title, $sum_text, $PV_ID, $id, "1"); + + last; + } +} +if($file eq "" || !(-e "$file")) { + print "No Entry found for " . Date::Calc::Date_to_Text(@broadcast_date) . " or download error - "; + if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "giving up, manual import necessary!!!\n"; + } else { + print "will retry later\n"; + } + exit 1; +} + +print "will import $file to dropbox $dropbox (cart=$to_cart)\n"; + +my $error_cb = sub { + my ($text) = @_; + + print "\n$text"; + return 0; +}; + +rhautoimport::check_key_file() or die "Import Key not found, use rhautoimport-create-id to create one\n"; + +my $ret; +my $log = rhautoimport::clear_carts($dbh, $group, $to_cart); +my $import_log; +($ret, $import_log) = rhautoimport::import_single($file, $dropbox, $user, 0, $error_cb); +$log .= $import_log; + +RHRD::rddb::closedb($dbh); + +unlink $file; + +if(!$ret) { + print "\nImport Error:\n"; + print $log; + exit 1; +} + +unlink($STAT_FILE); +open(my $fhs, '>', $STAT_FILE); +print $fhs "$out_file"; +close($fhs); + +exit 0; diff --git a/rhautoimport-dn b/rhautoimport-dn new file mode 100755 index 0000000..9e54460 --- /dev/null +++ b/rhautoimport-dn @@ -0,0 +1,239 @@ +#!/usr/bin/perl -w +# +# +# rhautoimport +# +# Copyright (C) 2009-2015 Christian Pointner <equinox@helsinki.at> +# +# This file is part of rhautoimport. +# +# rhautoimport is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# rhautoimport is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with rhautoimport. If not, see <http://www.gnu.org/licenses/>. +# + +use strict; +use Date::Calc; +use Time::localtime; +use XML::Feed; +use XML::Feed::Entry; +use XML::Feed::Content; +use XML::Feed::Enclosure; +use URI::URL; +use IO::Handle; +use IPC::Open3; +use RHRD::rddb; + +use lib '/usr/local/share/rhautoimport/'; +use rhautoimport; + +my $STAT_FILE = $ENV{'HOME'} . "/rhautoimport-dn.stat"; +my $FILES_RSS_URL = "http://www.democracynow.org/podcast-stations.xml"; +my $HEADLINES_RSS_URL = "http://www.democracynow.org/podcast.xml"; +my $IGNORE_HEADLINE = 0; +my $PV_ID = '111'; + +binmode(STDIN, ":utf8"); +binmode(STDOUT, ":utf8"); +binmode(STDERR, ":utf8"); + +if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "!!!This is the last attempt, there won't be a retry on error!!!\n" +} + +if($#ARGV >= 0 && $ARGV[0] eq 'noheadline') { + $IGNORE_HEADLINE = 1; + $STAT_FILE = $ENV{'HOME'} . "/rhautoimport-dn-noheadline.stat"; +} + +my $user = `/usr/bin/id -un`; +$user =~ s/\n//; +my $group = "democracyn"; + +my ($dbh, $errorstring) = RHRD::rddb::opendb(); +if(!defined $dbh) { + print "$errorstring\n"; + exit 1; +} +my @allowed_dbs = RHRD::rddb::get_dropboxes($dbh, $user, $group); +if(!defined $allowed_dbs[0] && defined $allowed_dbs[2]) { + print "$allowed_dbs[2]\n"; + exit 1; +} + +my $dropbox = ""; +my $to_cart = 0; + +my @today = Date::Calc::Today(); +my @yesterday = Date::Calc::Add_Delta_Days($today[0], $today[1], $today[2], -1); +my @tomorrow = Date::Calc::Add_Delta_Days($today[0], $today[1], $today[2], 1); + +my @broadcast_day = @yesterday; +my @import_day = @today; +if(localtime->hour >= 12) { + print "It's past noon, assuming to import file from today\n"; + @broadcast_day = @today; + @import_day = @tomorrow; +} + +print "day of original broadcast: " . Date::Calc::Date_to_Text(@broadcast_day) . "\n"; +print "day of Radio Helsinki broadcast: " . Date::Calc::Date_to_Text(@import_day) . "\n"; + +for my $href ( @allowed_dbs ) { + my $dow = Date::Calc::Day_of_Week(@import_day); + if ($dow == int(substr($href->{'NAME'},0,2))) { + $dropbox = $href->{'PATH'}; + $to_cart = $href->{'TO_CART'}; + } +} + +if($dropbox eq "") { + print "no dropbox for day in question\n"; + RHRD::rddb::closedb($dbh); + exit 0; +} + +my $y = sprintf("%04d-%02d-%02d", $broadcast_day[0], $broadcast_day[1], $broadcast_day[2]); +my $yc = sprintf("%04d-%02d%02d", $broadcast_day[0], $broadcast_day[1], $broadcast_day[2]); + +print "looking for files from date $y in RSS Feed\n"; +print " -> $FILES_RSS_URL\n"; + +my $files_feed = XML::Feed->parse(URI->new($FILES_RSS_URL)) + or die "Error fetching feed: " . XML::Feed->errstr; + +my $file = ""; +my $out_file = ""; +for my $entry ($files_feed->entries) { + if($entry->enclosure && $entry->enclosure->type eq "audio/mpeg") { + if($entry->enclosure->url =~ /^https?:\/\/.*\.democracynow\.org\/dn([0-9-]+)-1\.mp3$/) { + next if($yc ne $1); + + my $url = new URI::URL($entry->enclosure->url); + my @path = $url->path_components; + + my $current_file = `cat $STAT_FILE`; + if($current_file eq $path[-1]) { + print "Already downloaded file of day in question\n"; + RHRD::rddb::closedb($dbh); + exit 0; + } + $out_file = $path[-1]; + if(!rhautoimport::check_file_extension($out_file)) { + print "\n\nThe extension of the matching file '". $out_file . "' seems to be wrong - manual import necessary!!!\n"; + RHRD::rddb::closedb($dbh); + exit 1; + } + + print $1 . ": downloading " . $entry->enclosure->url . " (" . $entry->enclosure->length . " Bytes) .. "; + + $file = "/tmp/" . $out_file; + system("wget", "--quiet", "--no-check-certificate", $url, "-O", $file); + if( $! != 0) { + die "wget returned with error: " . $!; + } + print "ok\n"; + + last; + } + } +} +if($file eq "" || !(-e "$file")) { + print "No Entry found from day in qeustion or download error - "; + if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "giving up, manual import necessary!!!\n"; + } else { + print "will retry later\n"; + } + exit 1; +} + + +unless($IGNORE_HEADLINE) { + print "\nlooking for headlines from date $y in RSS Feed\n"; + print " -> $HEADLINES_RSS_URL\n"; + + my $headlines_feed = XML::Feed->parse(URI->new($HEADLINES_RSS_URL)) + or die "Error fetching feed: " . XML::Feed->errstr; + + for my $entry ($headlines_feed->entries) { + if($entry->enclosure && $entry->enclosure->type eq "audio/mpeg") { + my $sum_title = $entry->title; + my $sum_text = $entry->summary->body . "\n<br />\n<a href=\"" . $entry->link . "\">" . $entry->link . "</a>\n"; + + if($entry->id =~ /tag:democracynow.org,([0-9-]+):media\/mp3download\/[^\/]+/) { + next if($y ne $1); + + print "summary:\n" . $sum_title . "\n\n" . $sum_text . "\n"; + rhautoimport::pv_add_note($sum_title, $sum_text, $PV_ID, sprintf("%04d-%02d-%02d", @import_day), "1", 0); + print "\n"; + rhautoimport::pv_add_note($sum_title, $sum_text, $PV_ID, sprintf("%04d-%02d-%02d", @import_day), "1", 1); + print "\n"; + + last; + } + } + } +} else { + print "\nignoring headlines for they are not available by now\n"; + my $dow_string = Date::Calc::Day_of_Week_to_Text(Date::Calc::Day_of_Week(@broadcast_day)); + my $sum_title = sprintf("Democracy Now! %04d-%02d-%02d %s", @broadcast_day, $dow_string); + my $sum_text = ""; + print "summary:\n" . $sum_title . "\n\n" . $sum_text . "\n"; + rhautoimport::pv_add_note($sum_title, $sum_text, $PV_ID, sprintf("%04d-%02d-%02d", @broadcast_day), "1", 2); + print "\n"; +} + +print "converting $file ... "; +my $infile = $file; +$file =~ s/\.mp3$/.wav/; +system("ffmpeg", "-i", $infile, "-acodec", "pcm_s16le", $file); +if( $! != 0) { + die "ffmpeg returned with error: " . $!; +} +print "ok\n"; +unlink $infile; + +print "will import $file to dropbox $dropbox (cart=$to_cart)\n\n"; + +my $error_cb = sub { + my ($text) = @_; + + print "\n$text"; + return 0; +}; + +rhautoimport::check_key_file() or die "Import Key not found, use rhautoimport-create-id to create one\n"; + +my $ret; +my $log; +$log = rhautoimport::clear_carts($dbh, $group, $to_cart); +my $import_log; +($ret, $import_log) = rhautoimport::import_single($file, $dropbox, $user, 0, $error_cb); +$log .= $import_log; + +RHRD::rddb::closedb($dbh); + +unlink $file; + +if(!$ret) { + print "\nImport Error:\n"; + print $log; + exit 1; +} + +unlink($STAT_FILE); +open(my $fhs, '>', $STAT_FILE); +print $fhs $out_file; +close($fhs); + +exit 0; diff --git a/rhautoimport-fe b/rhautoimport-fe new file mode 100755 index 0000000..9c7738d --- /dev/null +++ b/rhautoimport-fe @@ -0,0 +1,204 @@ +#!/usr/bin/perl -w +# +# +# rhautoimport +# +# Copyright (C) 2009-2015 Christian Pointner <equinox@helsinki.at> +# +# This file is part of rhautoimport. +# +# rhautoimport is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# rhautoimport is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with rhautoimport. If not, see <http://www.gnu.org/licenses/>. +# + +use strict; +use Date::Calc; +use XML::Feed; +use XML::Feed::Entry; +use XML::Feed::Content; +use XML::Feed::Enclosure; +use File::Fetch; +use IO::Handle; +use IPC::Open3; +use HTML::Entities; +use File::Touch; +use RHRD::rddb; + +use lib '/usr/local/share/rhautoimport/'; +use rhautoimport; + +my $STAT_FILE = $ENV{'HOME'} . "/rhautoimport-fe.stat"; +my $NOTE_FILE = $ENV{'HOME'} . "/rhautoimport-fe.last_note"; +my $NEW_FILE = $ENV{'HOME'} . "/rhautoimport-fe.is_new"; +my $RSS_URL = "http://freie-radios.net/portal/podcast.php?serie=208&rss"; +#my $PV_ID = '359'; + +binmode(STDIN, ":utf8"); +binmode(STDOUT, ":utf8"); +binmode(STDERR, ":utf8"); + +if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "!!!This is the last attempt, there won't be a retry on error!!!\n" +} + +my @today = Date::Calc::Today(); +my $dow = Date::Calc::Day_of_Week(@today); +my @tomorrow = Date::Calc::Add_Delta_Days($today[0], $today[1], $today[2], 1); + +my @tmp = Date::Calc::Standard_to_Business(@today); +$tmp[2] = 1; +my @this_mon = Date::Calc::Business_to_Standard(@tmp); +my @next_mon = Date::Calc::Add_Delta_Days($this_mon[0], $this_mon[1], $this_mon[2], 7); + +my @broadcast_day = @today; +my @import_day = @tomorrow; +if ($dow == 5) { + @import_day = @next_mon; +} + +print "day of original broadcast: " . Date::Calc::Date_to_Text(@broadcast_day) . "\n"; +print "day of Radio Helsinki broadcast: " . Date::Calc::Date_to_Text(@import_day) . "\n"; + +my $user = `/usr/bin/id -un`; +$user =~ s/\n//; +my $group = "vonunten"; +$dow = Date::Calc::Day_of_Week(@import_day); +$group .= "Mo" if($dow == 1); +$group .= "Di" if($dow == 2); +$group .= "Mi" if($dow == 3); +$group .= "Do" if($dow == 4); +$group .= "Fr" if($dow == 5); + +my ($dbh, $errorstring) = RHRD::rddb::opendb(); +if(!defined $dbh) { + print "$errorstring\n"; + exit 1; +} +my @allowed_dbs = RHRD::rddb::get_dropboxes($dbh, $user, $group); +if(!defined $allowed_dbs[0] && defined $allowed_dbs[2]) { + print "$allowed_dbs[2]\n"; + exit 1; +} +my $dropbox = ""; +my $to_cart = 0; + +if(scalar(@allowed_dbs) == 0) { + print "no dropbox for day in question\n"; + RHRD::rddb::closedb($dbh); + exit 0; +} +if(scalar(@allowed_dbs) != 1) { + print "found more or less than one Dropbox for this group?!\n"; + RHRD::rddb::closedb($dbh); + exit 1; +} +$dropbox = $allowed_dbs[0]->{'PATH'}; +$to_cart = $allowed_dbs[0]->{'TO_CART'}; + +my $regexp = sprintf('%04d%02d%02d.*\.mp3', $broadcast_day[0], $broadcast_day[1], $broadcast_day[2]); +print "looking for file from " . Date::Calc::Date_to_Text(@broadcast_day) . " in RSS Feed\n"; +print " -> $RSS_URL\n"; + +my $feed = XML::Feed->parse(URI->new($RSS_URL)) + or die "Error fetching feed: " . XML::Feed->errstr; + +my $file = ""; +my $out_file = ""; +for my $entry ($feed->entries) { + if($entry->enclosure && $entry->enclosure->type eq "audio/mpeg") { + next unless $entry->enclosure->url =~ /$regexp/; + next unless $entry->enclosure->length >= (20*1024*1024); + + my $sum_title = decode_entities($entry->title); + my $sum_text = decode_entities($entry->content->body); + + my $ff = File::Fetch->new(uri => $entry->enclosure->url); + my $current_file = `cat $STAT_FILE`; + if($current_file eq $ff->output_file) { + print "Already downloaded file of today\n"; + RHRD::rddb::closedb($dbh); + exit 0; + } + $out_file = $ff->output_file; + if(!rhautoimport::check_file_extension($out_file)) { + print "\n\nThe extension of the matching file '". $out_file . "' seems to be wrong - manual import necessary!!!\n"; + RHRD::rddb::closedb($dbh); + exit 1; + } + + print $regexp . ": downloading " . $entry->enclosure->url . " (" . $entry->enclosure->length . " Bytes) .. "; + $file = $ff->fetch( to => '/tmp' ) or die $ff->error; + print "ok\n"; + + print "summary:\n" . $sum_title . "\n\n" . $sum_text . "\n"; + + # if($#ARGV >= 0 && $ARGV[0] eq 'nopv') { + # print "not adding note to PV\n"; + # } else { + # rhautoimport::pv_add_note($sum_title, $sum_text, $PV_ID, sprintf("%04d-%02d-%02d", $import_day[0], $import_day[1], $import_day[2]), "1"); + # } + # print "\n"; + + unlink($NOTE_FILE); + open(my $fhs, '>', $NOTE_FILE); + binmode($fhs, ":utf8"); + print $fhs $sum_title . "\n\n" . $sum_text . "\n"; + close($fhs); + + last; + } +} +if($file eq "" || !(-e "$file")) { + print "No Entry found for " . Date::Calc::Date_to_Text(@broadcast_day) . " or download error - "; + if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "giving up, manual import necessary!!!\n"; + } else { + print "will retry later\n"; + } + exit 1; +} + +print "will import $file to dropbox $dropbox (cart=$to_cart)\n"; + +my $error_cb = sub { + my ($text) = @_; + + print "\n$text"; + return 0; +}; + +rhautoimport::check_key_file() or die "Import Key not found, use rhautoimport-create-id to create one\n"; + +my $ret; +my $log = rhautoimport::clear_carts($dbh, $group, $to_cart); +my $import_log; +($ret, $import_log) = rhautoimport::import_single($file, $dropbox, $user, 0, $error_cb); +$log .= $import_log; + +RHRD::rddb::closedb($dbh); + +unlink $file; + +if(!$ret) { + print "\nImport Error:\n"; + print $log; + exit 1; +} + +unlink($STAT_FILE); +open(my $fhs, '>', $STAT_FILE); +print $fhs $out_file; +close($fhs); +touch($NEW_FILE); + +exit 0; diff --git a/rhautoimport-fw b/rhautoimport-fw new file mode 100755 index 0000000..d1914b6 --- /dev/null +++ b/rhautoimport-fw @@ -0,0 +1,219 @@ +#!/usr/bin/perl -w +# +# +# rhautoimport +# +# Copyright (C) 2009-2015 Christian Pointner <equinox@helsinki.at> +# +# This file is part of rhautoimport. +# +# rhautoimport is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# rhautoimport is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with rhautoimport. If not, see <http://www.gnu.org/licenses/>. +# + +use strict; +use Date::Calc; +use XML::Feed; +use XML::Feed::Entry; +use XML::Feed::Content; +use XML::Feed::Enclosure; +use File::Fetch; +use IO::Handle; +use IPC::Open3; +use HTML::Entities; +use RHRD::rddb; + +use lib '/usr/local/share/rhautoimport/'; +use rhautoimport; + +my $STAT_FILE = $ENV{'HOME'} . "/rhautoimport-fw.stat"; +my $LAST_FILE = $ENV{'HOME'} . "/rhautoimport-fw.last"; +my $RSS_URL = "http://cba.fro.at/seriesrss/263965?c=Kfs2IoV2Wmd"; +$XML::Feed::MULTIPLE_ENCLOSURES=1; +my $PV_ID = '382'; + +binmode(STDIN, ":utf8"); +binmode(STDOUT, ":utf8"); +binmode(STDERR, ":utf8"); + +if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "!!!This is the last attempt, there won't be a retry on error!!!\n" +} + +my $user = `/usr/bin/id -un`; +$user =~ s/\n//; +my $group = "freiewelle"; + +my ($dbh, $errorstring) = RHRD::rddb::opendb(); +if(!defined $dbh) { + print "$errorstring\n"; + exit 1; +} +my @allowed_dbs = RHRD::rddb::get_dropboxes($dbh, $user, $group); +if(!defined $allowed_dbs[0] && defined $allowed_dbs[2]) { + print "$allowed_dbs[2]\n"; + exit 1; +} + +my $idx = 0; +my $idx_reb = 1; +if(scalar(@allowed_dbs) != 2) { + print "found more or less than 2 Dropboxes for this group?!\n"; + RHRD::rddb::closedb($dbh); + exit 1; +} +my $dropbox = $allowed_dbs[$idx]->{'PATH'}; +my $to_cart = $allowed_dbs[$idx]->{'TO_CART'}; +my $dropbox_reb = $allowed_dbs[$idx_reb]->{'PATH'}; +my $to_cart_reb = $allowed_dbs[$idx_reb]->{'TO_CART'}; + +my @import_date = Date::Calc::Standard_to_Business(Date::Calc::Today()); +if($import_date[2] != 1) { + $import_date[2] = 1; + @import_date = Date::Calc::Business_to_Standard(@import_date); + @import_date = Date::Calc::Add_Delta_Days(@import_date, 7); +} else { + @import_date = Date::Calc::Business_to_Standard(@import_date); +} +my @import_date_reb = Date::Calc::Add_Delta_Days(@import_date, 4); + +print "day of next broadcast: " . Date::Calc::Date_to_Text(@import_date) . "\n"; +print "day of next rebroadcast: " . Date::Calc::Date_to_Text(@import_date_reb) . "\n"; + +my $id = sprintf("%04d-%02d-%02d", @import_date); +my $bnum = `cat $LAST_FILE`; +$bnum += 1; + +my $current_stat = `cat $STAT_FILE`; +my ($current_id, $current_file) = $current_stat =~ m/^(.*)\n(.*)/; +if($current_id eq $id) { + print "Already downloaded current file\n"; + RHRD::rddb::closedb($dbh); + exit 0; +} + +print "looking for file with number $bnum in RSS Feed\n"; +print " -> $RSS_URL\n"; + +my $feed = XML::Feed->parse(URI->new($RSS_URL)) + or die "Error fetching feed: " . XML::Feed->errstr; + +my $bdexp = sprintf("^(\\d+)", $bnum); + + +my $file = ""; +my $out_file = ""; +my $i = 0; +for my $entry ($feed->entries) { + $i++; + next unless $entry->enclosure; + my $j = 0; + for my $enclosure($entry->enclosure) { + $j++; + if($enclosure->type eq "audio/mpeg" || $enclosure->type eq "audio/ogg") { + print "$i/$j: (" . $enclosure->type . ", " . $enclosure->length . ") " . $enclosure->url . "\n"; + my $url = $enclosure->url; + if($url =~ /^(.*)_cut(\.[^\.]+)$/) { + $url = $1 . $2; + } + + my $ff = File::Fetch->new(uri => $url); + next unless (uc($ff->output_file) =~ /$bdexp/); # file not from correct day + next unless ($bnum == $1); + + my $sum_title = decode_entities($entry->title); + my $sum_text = decode_entities($entry->content->body); + + my $current_file = `cat $STAT_FILE`; + if($current_file eq $ff->output_file) { + print "Already downloaded current file\n"; + RHRD::rddb::closedb($dbh); + exit 0; + } + $out_file = $ff->output_file; + if(!rhautoimport::check_file_extension($out_file)) { + print "\n\nThe extension of the matching file '". $out_file . "' seems to be wrong - manual import necessary!!!\n"; + RHRD::rddb::closedb($dbh); + exit 1; + } + + print " --> " . $bnum . ": downloading " . $url . " (" . $enclosure->length . " Bytes) .. "; + $file = $ff->fetch( to => '/tmp' ) or die $ff->error; + print "ok\n"; + + print "summary:\n" . $sum_title . "\n\n" . $sum_text . "\n"; + + rhautoimport::pv_add_note($sum_title, $sum_text, $PV_ID, sprintf("%04d-%02d-%02d", $import_date[0], $import_date[1], $import_date[2]), "1"); + print "\n"; + rhautoimport::pv_add_note($sum_title, $sum_text, $PV_ID, sprintf("%04d-%02d-%02d", $import_date_reb[0], $import_date_reb[1], $import_date_reb[2]), "2"); + print "\n"; + + last; + } + } + last if $file ne ""; +} + +if($file eq "" || !(-e "$file")) { + print "No Entry found or download error - "; + if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "giving up, manual import necessary!!!\n"; + } else { + print "will retry later\n"; + } + exit 1; +} + +print "will import $file to dropbox $dropbox (cart=$to_cart)\n"; +print "will import $file to rebroadcast dropbox $dropbox_reb (cart=$to_cart_reb)\n\n"; + +my $error_cb = sub { + my ($text) = @_; + + print "\n$text"; + return 0; +}; + +rhautoimport::check_key_file() or die "Import Key not found, use rhautoimport-create-id to create one\n"; + +my $ret; +my $log = rhautoimport::clear_carts($dbh, $group, $to_cart); +my $import_log; +($ret, $import_log) = rhautoimport::import_single($file, $dropbox, $user, 0, $error_cb); +$log .= $import_log; +$import_log = rhautoimport::clear_carts($dbh, $group, $to_cart_reb); +$log .= $import_log; +($ret, $import_log) = rhautoimport::import_single($file, $dropbox_reb, $user, 0, $error_cb); +$log .= $import_log; + +RHRD::rddb::closedb($dbh); + +unlink $file; + +if(!$ret) { + print "\nImport Error:\n"; + print $log; + exit 1; +} + +unlink($STAT_FILE); +open(my $fhs, '>', $STAT_FILE); +print $fhs "$id\n$out_file"; +close($fhs); + +unlink($LAST_FILE); +open($fhs, '>', $LAST_FILE); +print $fhs "$bnum"; +close($fhs); + +exit 0; diff --git a/rhautoimport-mz b/rhautoimport-mz new file mode 100755 index 0000000..fa50a69 --- /dev/null +++ b/rhautoimport-mz @@ -0,0 +1,247 @@ +#!/usr/bin/perl -w +# +# +# rhautoimport +# +# Copyright (C) 2009-2015 Christian Pointner <equinox@helsinki.at> +# +# This file is part of rhautoimport. +# +# rhautoimport is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# rhautoimport is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with rhautoimport. If not, see <http://www.gnu.org/licenses/>. +# + +use strict; +use Switch; +use Date::Calc; +use Time::localtime; +use IO::Handle; +use IPC::Open3; +use File::Slurp; +use RHRD::rddb; + +use lib '/usr/local/share/rhautoimport/'; +use rhautoimport; + + +my @STATIC_FILES = ($ENV{'HOME'} . "/frontex", $ENV{'HOME'} . "/maribor" ); + +my $STAT_FILE = $ENV{'HOME'} . "/rhautoimport-mz.stat"; +my $ZF_NOTE_FILE = $ENV{'HOME'} . "/rhautoimport-zf.last_note"; +my $ZF_NEW_FILE = $ENV{'HOME'} . "/rhautoimport-zf.is_new"; +my $PV_ID = '352'; + +binmode(STDIN, ":utf8"); +binmode(STDOUT, ":utf8"); +binmode(STDERR, ":utf8"); + +if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "!!!This is the last attempt, there won't be a retry on error!!!\n" +} + +my $user = `/usr/bin/id -un`; +$user =~ s/\n//; + +my @today = Date::Calc::Today(); +my @next_day = Date::Calc::Add_Delta_Days(@today, 1); +my $dow = Date::Calc::Day_of_Week(@today); +if($dow == 5) { + @next_day = Date::Calc::Add_Delta_Days(@today, 3); +} + +print "today: " . Date::Calc::Date_to_Text(@today) . "\n"; +print "next day would be: " . Date::Calc::Date_to_Text(@next_day) . "\n"; +my @import_day = @next_day; +if(localtime->hour < 12) { + print "It's forenoon, assuming to import for today\n"; + @import_day = @today; +} +print "day of broadcast: " . Date::Calc::Date_to_Text(@import_day) . "\n"; + + +$dow = Date::Calc::Day_of_Week(@import_day); +my $group = ""; +switch($dow) { + case 1 { $group = "mahlzeitMo" } + case 2 { $group = "mahlzeitDi" } + case 3 { $group = "mahlzeitMi" } + case 4 { $group = "mahlzeitDo" } + case 5 { $group = "mahlzeitFr" } + else { print("invalid day of week?!\n"); exit 1; } +} + +my ($dbh, $errorstring) = RHRD::rddb::opendb(); +if(!defined $dbh) { + print "$errorstring\n"; + exit 1; +} +my @allowed_dbs = RHRD::rddb::get_dropboxes($dbh, $user, $group); +if(!defined $allowed_dbs[0] && defined $allowed_dbs[2]) { + print "$allowed_dbs[2]\n"; + exit 1; +} + +if(scalar(@allowed_dbs) != 1) { + print "found no or more than one Dropboxes for this group?!\n"; + RHRD::rddb::closedb($dbh); + exit 1; +} +my $dropbox = $allowed_dbs[0]->{'PATH'}; + + +my $import_date = sprintf("%04d-%02d-%02d", @import_day); +my $current_date = `cat $STAT_FILE`; +if($current_date eq $import_date) { + print "Already downloaded file of day in question\n"; + RHRD::rddb::closedb($dbh); + exit 0; +} + +unless(-e $ZF_NEW_FILE) { + print "zip-fm not imported yet - "; + if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "giving up, rebroadcasting last show"; + } else { + print "will retry later\n"; + exit 0, + } +} + + +sub read_first_line { + my ($file) = @_; + my @lines = read_file($file, binmode => ':utf8', err_mode => 'quiet'); + return "" unless ($lines[0]); + + chomp($lines[0]); + return $lines[0]; +}; + +sub get_note { + my ($file) = @_; + my @lines = read_file($file, binmode => ':utf8', err_mode => 'quiet'); + return "" unless ($lines[0]); + + chomp($lines[0]); + $lines[0] = "<p><strong>$lines[0]</strong></p>\n"; + return join('',@lines); +}; + +sub find_next_file { + my ($dir, $current) = @_; + + my %files = (); + opendir(my $dh, $dir) or die $!; + while (my $file = readdir($dh)) { + next if ($file =~ m/^\./); + next if ($file =~ m/_notes$/); + $files{$file} = 1; + } + closedir($dh); + + my $num = scalar keys %files; + foreach my $file (sort keys %files) { + if($file gt $current) { + return $num, $file; + } + $num--; + } + + return 0, ""; +}; + +my $sum_title = "zip-fm (WH)"; +my $sum_text = get_note($ZF_NOTE_FILE) . "<br/>\n"; + +print "\nsearching local files:\n"; + +my @files = (); +my %next_files = (); +foreach my $dir (@STATIC_FILES) { + my $file = read_first_line("$dir/.current"); + if($file eq "") { + print " $dir: skipped !!! (no .current file or no more files)\n"; + next; + } + my ($remaining, $next_file) = find_next_file($dir, $file); + if($remaining >= 1) { + print " $dir: adding $file\n (next: $next_file, $remaining files remaining)\n"; + } else { + print " $dir: adding $file\n (this was the last file!!!!)\n"; + } + $next_files{$dir} = $next_file; + + $sum_title = "$sum_title, " . read_first_line("$dir/.sum_title"); + $sum_text = $sum_text . "<br />\n" . get_note("$dir/$file" . "_notes"); + foreach my $ext ("flac", "wav", "ogg", "mp3") { + push(@files, "$dir/.jingle.$ext") if(-e "$dir/.jingle.$ext"); + } + push(@files, "$dir/$file"); +} +$sum_text = $sum_text . "<br/><br/>\n"; + +print "\nsummary:\n" . $sum_title . "\n\n" . $sum_text . "\n"; +rhautoimport::pv_add_note($sum_title, $sum_text, $PV_ID, sprintf("%04d-%02d-%02d", @import_day), "1"); +print "\n"; + + +print "will import " . join(", ", @files) . " to dropbox $dropbox\n"; + + +my $error_cb = sub { + my ($text) = @_; + + print "\n$text"; + return 0; +}; + +rhautoimport::check_key_file() or die "Import Key not found, use rhautoimport-create-id to create one\n"; + +my $ret = 1; +my $log = rhautoimport::clear_carts($dbh, $group, 0); +my ($cart, $high_cart) = rhautoimport::get_cart_range($dbh, $group); +foreach my $file (@files) { + if($cart > $high_cart) { + print "not enough carts - will ignore remaining files!!!"; + last; + } + print " importing $file to cart $cart\n"; + my $import_log; + ($ret, $import_log) = rhautoimport::import_single($file, $dropbox, $user, 0, $error_cb); + $log .= $import_log; + last unless $ret; + + $cart++; +} + +RHRD::rddb::closedb($dbh); + +if(!$ret) { + print "\nImport Error:\n"; + print $log; + exit 1; +} + +unlink($STAT_FILE); +open(my $fhs, '>', $STAT_FILE); +print $fhs $import_date; +close($fhs); + +unlink($ZF_NEW_FILE); +foreach my $dir (@STATIC_FILES) { + open(my $fhs, '>', "$dir/.current"); + print $fhs $next_files{$dir} if ($next_files{$dir}); + close($fhs); +} + +exit 0; diff --git a/rhautoimport-nw b/rhautoimport-nw new file mode 100755 index 0000000..3d6656b --- /dev/null +++ b/rhautoimport-nw @@ -0,0 +1,190 @@ +#!/usr/bin/perl -w +# +# +# rhautoimport +# +# Copyright (C) 2009-2015 Christian Pointner <equinox@helsinki.at> +# +# This file is part of rhautoimport. +# +# rhautoimport is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# rhautoimport is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with rhautoimport. If not, see <http://www.gnu.org/licenses/>. +# + +use strict; +use Date::Calc; +use XML::Feed; +use XML::Feed::Entry; +use XML::Feed::Content; +use XML::Feed::Enclosure; +use File::Fetch; +use IO::Handle; +use IPC::Open3; +use HTML::Entities; +use RHRD::rddb; + +use lib '/usr/local/share/rhautoimport/'; +use rhautoimport; + +my $STAT_FILE = $ENV{'HOME'} . "/rhautoimport-nw.stat"; +my $RSS_URL = "http://cba.fro.at/seriesrss/262514?c=Kfs2IoV2Wmd"; +$XML::Feed::MULTIPLE_ENCLOSURES=1; +my $PV_ID = '73'; + +binmode(STDIN, ":utf8"); +binmode(STDOUT, ":utf8"); +binmode(STDERR, ":utf8"); + +if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "!!!This is the last attempt, there won't be a retry on error!!!\n" +} + +my $user = `/usr/bin/id -un`; +$user =~ s/\n//; +my $group = "netwatcher"; + +my ($dbh, $errorstring) = RHRD::rddb::opendb(); +if(!defined $dbh) { + print "$errorstring\n"; + exit 1; +} +my @allowed_dbs = RHRD::rddb::get_dropboxes($dbh, $user, $group); +if(!defined $allowed_dbs[0] && defined $allowed_dbs[2]) { + print "$allowed_dbs[2]\n"; + exit 1; +} + +if(scalar(@allowed_dbs) != 1) { + print "found more or less than 1 Dropbox for this group?!\n"; + RHRD::rddb::closedb($dbh); + exit 1; +} +my $dropbox = $allowed_dbs[0]->{'PATH'}; +my $to_cart = $allowed_dbs[0]->{'TO_CART'}; + +my @today = Date::Calc::Today(); +my @import_date = @today; +if( Date::Calc::Day_of_Week(@today) > 3 ) { + @import_date = Date::Calc::Add_Delta_Days(@import_date, 7); +} +if( Date::Calc::Day_of_Week(@today) != 3 ) { + @import_date = Date::Calc::Standard_to_Business(@import_date); + $import_date[2] = 3; + @import_date = Date::Calc::Business_to_Standard(@import_date); +} +my @broadcast_day = Date::Calc::Add_Delta_Days(@import_date, -5); +print "day of original broadcast: " . Date::Calc::Date_to_Text(@broadcast_day) . "\n"; +print "day of Radio Helsinki broadcast: " . Date::Calc::Date_to_Text(@import_date) . "\n"; + +my $id = sprintf("%04d-%02d-%02d", @import_date); +my $bd = sprintf("%04d-%02d-%02d", @broadcast_day); +my $bdfile = sprintf("NETWATCHER%04d%02d%02d(.*)\.MP3", @broadcast_day); +print "looking for files from $bd in RSS Feed\n"; +print " -> $RSS_URL\n"; + +my $feed = XML::Feed->parse(URI->new($RSS_URL)) + or die "Error fetching feed: " . XML::Feed->errstr; + +my $file = ""; +my $out_file = ""; +my $i = 0; +for my $entry ($feed->entries) { + $i++; + next unless $entry->enclosure; + my $j = 0; + for my $enclosure($entry->enclosure) { + $j++; + if($enclosure->type eq "audio/mpeg" || $enclosure->type eq "audio/ogg") { + print "$i/$j: (" . $enclosure->type . ", " . $enclosure->length . ") " . $enclosure->url . "\n"; + my $url = $enclosure->url; + if($url =~ /^(.*)_cut(\.[^\.]+)$/) { + $url = $1 . $2; + } + + my $ff = File::Fetch->new(uri => $url); + next unless (uc($ff->output_file) =~ $bdfile); # file not from correct day + + my $sum_title = decode_entities($entry->title); + my $sum_text = decode_entities($entry->content->body); + + my $current_file = `cat $STAT_FILE`; + if($current_file eq $ff->output_file) { + print "Already downloaded file of today\n"; + RHRD::rddb::closedb($dbh); + exit 0; + } + $out_file = $ff->output_file; + if(!rhautoimport::check_file_extension($out_file)) { + print "\n\nThe extension of the matching file '". $out_file . "' seems to be wrong - manual import necessary!!!\n"; + RHRD::rddb::closedb($dbh); + exit 1; + } + + print " --> " . $bdfile . ": downloading " . $url . " (" . $enclosure->length . " Bytes) .. "; + $file = $ff->fetch( to => '/tmp' ) or die $ff->error; + print "ok\n"; + + print "summary:\n" . $sum_title . "\n\n" . $sum_text . "\n"; + + rhautoimport::pv_add_note($sum_title, $sum_text, $PV_ID, $id, "1"); + print "\n"; + + last; + } + } + last if $file ne ""; +} + +if($file eq "" || !(-e "$file")) { + print "No Entry found from $bd or download error - "; + if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "giving up, manual import necessary!!!\n"; + } else { + print "will retry later\n"; + } + exit 1; +} + +print "will import $file to dropbox $dropbox (cart=$to_cart)\n"; + +my $error_cb = sub { + my ($text) = @_; + + print "\n$text"; + return 0; +}; + +rhautoimport::check_key_file() or die "Import Key not found, use rhautoimport-create-id to create one\n"; + +my $ret; +my $log = rhautoimport::clear_carts($dbh, $group, $to_cart); +my $import_log; +($ret, $import_log) = rhautoimport::import_single($file, $dropbox, $user, 0, $error_cb); +$log .= $import_log; + +RHRD::rddb::closedb($dbh); + +unlink $file; + +if(!$ret) { + print "\nImport Error:\n"; + print $log; + exit 1; +} + +unlink($STAT_FILE); +open(my $fhs, '>', $STAT_FILE); +print $fhs $out_file; +close($fhs); + +exit 0; diff --git a/rhautoimport-o94n b/rhautoimport-o94n new file mode 100755 index 0000000..c3f8ab6 --- /dev/null +++ b/rhautoimport-o94n @@ -0,0 +1,209 @@ +#!/usr/bin/perl -w +# +# +# rhautoimport +# +# Copyright (C) 2009-2015 Christian Pointner <equinox@helsinki.at> +# +# This file is part of rhautoimport. +# +# rhautoimport is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# rhautoimport is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with rhautoimport. If not, see <http://www.gnu.org/licenses/>. +# + +use strict; +use Date::Calc; +use XML::Feed; +use XML::Feed::Entry; +use XML::Feed::Content; +use XML::Feed::Enclosure; +use File::Fetch; +use IO::Handle; +use IPC::Open3; +use HTML::Entities; +use RHRD::rddb; + +use lib '/usr/local/share/rhautoimport/'; +use rhautoimport; + +my $STAT_FILE = $ENV{'HOME'} . "/rhautoimport-o94n.stat"; +my $RSS_URL = "http://cba.fro.at/seriesrss/264131?c=Kfs2IoV2Wmd"; +$XML::Feed::MULTIPLE_ENCLOSURES=1; +my $PV_ID = '430'; + +binmode(STDIN, ":utf8"); +binmode(STDOUT, ":utf8"); +binmode(STDERR, ":utf8"); + +if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "!!!This is the last attempt, there won't be a retry on error!!!\n" +} + +my $user = `/usr/bin/id -un`; +$user =~ s/\n//; +my $group = "o94"; + +my ($dbh, $errorstring) = RHRD::rddb::opendb(); +if(!defined $dbh) { + print "$errorstring\n"; + exit 1; +} +my @allowed_dbs = RHRD::rddb::get_dropboxes($dbh, $user, $group); +if(!defined $allowed_dbs[0] && defined $allowed_dbs[2]) { + print "$allowed_dbs[2]\n"; + exit 1; +} + + +my $dropbox = ''; +my $to_cart = 0; + +for my $db (@allowed_dbs) { + if($db->{'PATH'} =~ /^\/programm\/02/) { + $to_cart = $db->{'TO_CART'}; + $dropbox = $db->{'PATH'}; + last; + } +} +if($to_cart == 0) { + print "no dropbox found for Tuesday?!\n"; + RHRD::rddb::closedb($dbh); + exit 1; +} + + +my @today = Date::Calc::Today(); +my @yesterday = Date::Calc::Add_Delta_Days(@today, -1); +my @tomorrow = Date::Calc::Add_Delta_Days(@today, 1); +my $dow = Date::Calc::Day_of_Week(@today); + +my @broadcast_day; +my @import_date; +if($dow == 1) { + @broadcast_day = @today; + @import_date = @tomorrow; +} elsif ($dow == 2) { + @broadcast_day = @yesterday; + @import_date = @today; +} else { + print "Wrong Day! Check cron script!\n"; + exit 1; +} + +print "day of original broadcast: " . Date::Calc::Date_to_Text(@broadcast_day) . "\n"; +print "day of Radio Helsinki broadcast: " . Date::Calc::Date_to_Text(@import_date) . "\n"; + +my $id = sprintf("%04d-%02d-%02d", @import_date); +my $bd = sprintf("%04d-%02d-%02d", @broadcast_day); +my $bdfile = sprintf("^NACHRICHTEN%04d%02d%02d", @broadcast_day); + +print "looking for files from $bd in RSS Feed\n"; +print " -> $RSS_URL\n"; + +my $feed = XML::Feed->parse(URI->new($RSS_URL)) + or die "Error fetching feed: " . XML::Feed->errstr; + +my $file = ""; +my $out_file = ""; +my $i = 0; +for my $entry ($feed->entries) { + $i++; + next unless $entry->enclosure; + my $j = 0; + for my $enclosure($entry->enclosure) { + $j++; + if($enclosure->type eq "audio/mpeg" || $enclosure->type eq "audio/ogg") { + print "$i/$j: (" . $enclosure->type . ", " . $enclosure->length . ") " . $enclosure->url . "\n"; + my $url = $enclosure->url; + if($url =~ /^(.*)_cut(\.[^\.]+)$/) { + $url = $1 . $2; + } + + my $ff = File::Fetch->new(uri => $url); + next unless (uc($ff->output_file) =~ $bdfile); # file not from correct day + + my $sum_title = decode_entities($entry->title); + $sum_title =~ s/\s*\(ganze sendung\)//i; + my $sum_text = decode_entities($entry->content->body); + + my $current_file = `cat $STAT_FILE`; + if($current_file eq $ff->output_file) { + print "Already downloaded file of today\n"; + RHRD::rddb::closedb($dbh); + exit 0; + } + $out_file = $ff->output_file; + if(!rhautoimport::check_file_extension($out_file)) { + print "\n\nThe extension of the matching file '". $out_file . "' seems to be wrong - manual import necessary!!!\n"; + RHRD::rddb::closedb($dbh); + exit 1; + } + + print " --> " . $bdfile . ": downloading " . $url . " (" . $enclosure->length . " Bytes) .. "; + $file = $ff->fetch( to => '/tmp' ) or die $ff->error; + print "ok\n"; + + print "summary:\n" . $sum_title . "\n\n" . $sum_text . "\n"; + + rhautoimport::pv_add_note($sum_title, $sum_text, $PV_ID, sprintf("%04d-%02d-%02d", @import_date), "1"); + print "\n"; + + last; + } + } + last if $file ne ""; +} + +if($file eq "" || !(-e "$file")) { + print "No Entry found from $bd or download error - "; + if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "giving up, manual import necessary!!!\n"; + } else { + print "will retry later\n"; + } + exit 1; +} + +print "will import $file to dropbox $dropbox (cart=$to_cart)\n"; + +my $error_cb = sub { + my ($text) = @_; + + print "\n$text"; + return 0; +}; + +rhautoimport::check_key_file() or die "Import Key not found, use rhautoimport-create-id to create one\n"; + +my $ret; +my $log = rhautoimport::clear_carts($dbh, $group, $to_cart); +my $import_log; +($ret, $import_log) = rhautoimport::import_single($file, $dropbox, $user, 0, $error_cb); +$log .= $import_log; + +RHRD::rddb::closedb($dbh); + +unlink $file; + +if(!$ret) { + print "\nImport Error:\n"; + print $log; + exit 1; +} + +unlink($STAT_FILE); +open(my $fhs, '>', $STAT_FILE); +print $fhs $out_file; +close($fhs); + +exit 0; diff --git a/rhautoimport-oi b/rhautoimport-oi new file mode 100755 index 0000000..1e637c8 --- /dev/null +++ b/rhautoimport-oi @@ -0,0 +1,182 @@ +#!/usr/bin/perl -w +# +# +# rhautoimport +# +# Copyright (C) 2009-2015 Christian Pointner <equinox@helsinki.at> +# +# This file is part of rhautoimport. +# +# rhautoimport is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# rhautoimport is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with rhautoimport. If not, see <http://www.gnu.org/licenses/>. +# + +use strict; +use Date::Calc; +use XML::Feed; +use XML::Feed::Entry; +use XML::Feed::Content; +use XML::Feed::Enclosure; +use File::Fetch; +use IO::Handle; +use IPC::Open3; +use HTML::Entities; +use RHRD::rddb; + +use lib '/usr/local/share/rhautoimport/'; +use rhautoimport; + +my $STAT_FILE = $ENV{'HOME'} . "/rhautoimport-oi.stat"; +my $LAST_FILE = $ENV{'HOME'} . "/rhautoimport-oi.last"; +my $RSS_URL = "http://npla.de/onda/feed/serie=3"; +my $PV_ID = '220'; + +binmode(STDIN, ":utf8"); +binmode(STDOUT, ":utf8"); +binmode(STDERR, ":utf8"); + +if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "!!!This is the last attempt, there won't be a retry on error!!!\n" +} + +my $user = `/usr/bin/id -un`; +$user =~ s/\n//; +my $group = "ondainfo"; + +my ($dbh, $errorstring) = RHRD::rddb::opendb(); +if(!defined $dbh) { + print "$errorstring\n"; + exit 1; +} +my @allowed_dbs = RHRD::rddb::get_dropboxes($dbh, $user, $group); +if(!defined $allowed_dbs[0] && defined $allowed_dbs[2]) { + print "$allowed_dbs[2]\n"; + exit 1; +} + +if(scalar(@allowed_dbs) != 1) { + print "found more or less than one Dropbox for this group?!\n"; + RHRD::rddb::closedb($dbh); + exit 1; +} +my $dropbox = $allowed_dbs[0]->{'PATH'}; +my $to_cart = $allowed_dbs[0]->{'TO_CART'}; + + +my @today = Date::Calc::Today(); +my @import_date = Date::Calc::Add_Delta_Days(@today, 7); +@import_date = Date::Calc::Standard_to_Business(@import_date); +$import_date[2] = 2; +@import_date = Date::Calc::Business_to_Standard(@import_date); + +my $broadcast_num = `cat $LAST_FILE`; +$broadcast_num += 1; + +print "today: " . Date::Calc::Date_to_Text(@today) . "\n"; +print "day of next Radio Helsinki broadcast: " . Date::Calc::Date_to_Text(@import_date) . "\n"; +print "Number of next broadcast: " . $broadcast_num . "\n"; + +my $id = sprintf("%04d-%02d-%02d", @import_date); +my $bdfile = sprintf('.*onda.?info_%d.*\.mp3$', $broadcast_num); + +my $current_stat = `cat $STAT_FILE`; +my ($current_id, $current_file) = $current_stat =~ m/^(.*)\n(.*)/; +if($current_id eq $id) { + print "Already downloaded current file\n"; + RHRD::rddb::closedb($dbh); + exit 0; +} + +print "looking for files like '$bdfile' in RSS Feed\n"; +print " -> $RSS_URL\n"; + +my $feed = rhautoimport::fetch_parse_rss($RSS_URL) + or die "Error fetching feed: " . XML::Feed->errstr; + +my $file = ""; +my $out_file = ""; +for my $entry ($feed->entries) { + if($entry->enclosure) { + print $entry->enclosure->url . "\n"; + next unless $entry->enclosure->url =~ /$bdfile/; + + my $sum_title = decode_entities($entry->title); + my $sum_text = decode_entities($entry->content->body); + + my $ff = File::Fetch->new(uri => $entry->enclosure->url); + $out_file = $ff->output_file; + if(!rhautoimport::check_file_extension($out_file)) { + print "\n\nThe extension of the matching file '". $out_file . "' seems to be wrong - manual import necessary!!!\n"; + RHRD::rddb::closedb($dbh); + exit 1; + } + + print $broadcast_num . ": downloading " . $entry->enclosure->url . " (" . $entry->enclosure->length . " Bytes) .. "; + $file = $ff->fetch( to => '/tmp' ) or die $ff->error; + print "ok\n"; + + print "summary:\n" . $sum_title . "\n\n" . $sum_text . "\n"; + + rhautoimport::pv_add_note($sum_title, $sum_text, $PV_ID, $id, "1"); + + last; + } +} +if($file eq "" || !(-e "$file")) { + print "No Entry found with " . $broadcast_num . " or download error - "; + if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "giving up, manual import necessary!!!\n"; + } else { + print "will retry later\n"; + } + exit 1; +} + +print "will import $file to dropbox $dropbox (cart=$to_cart)\n"; + +my $error_cb = sub { + my ($text) = @_; + + print "\n$text"; + return 0; +}; + +rhautoimport::check_key_file() or die "Import Key not found, use rhautoimport-create-id to create one\n"; + +my $ret; +my $log = rhautoimport::clear_carts($dbh, $group, $to_cart); +my $import_log; +($ret, $import_log) = rhautoimport::import_single($file, $dropbox, $user, 0, $error_cb); +$log .= $import_log; + +RHRD::rddb::closedb($dbh); + +unlink $file; + +if(!$ret) { + print "\nImport Error:\n"; + print $log; + exit 1; +} + +unlink($STAT_FILE); +open(my $fhs, '>', $STAT_FILE); +print $fhs "$id\n$out_file"; +close($fhs); + +unlink($LAST_FILE); +open($fhs, '>', $LAST_FILE); +print $fhs "$broadcast_num"; +close($fhs); + +exit 0; diff --git a/rhautoimport-ra b/rhautoimport-ra new file mode 100755 index 0000000..3018429 --- /dev/null +++ b/rhautoimport-ra @@ -0,0 +1,199 @@ +#!/usr/bin/perl -w +# +# +# rhautoimport +# +# Copyright (C) 2009-2015 Christian Pointner <equinox@helsinki.at> +# +# This file is part of rhautoimport. +# +# rhautoimport is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# rhautoimport is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with rhautoimport. If not, see <http://www.gnu.org/licenses/>. +# + +use strict; +use Date::Calc; +use XML::Feed; +use XML::Feed::Entry; +use XML::Feed::Content; +use XML::Feed::Enclosure; +use File::Fetch; +use IO::Handle; +use IPC::Open3; +use HTML::Entities; +use RHRD::rddb; + +use lib '/usr/local/share/rhautoimport/'; +use rhautoimport; + +my $STAT_FILE = $ENV{'HOME'} . "/rhautoimport-ra.stat"; +my $RSS_URL = "http://cba.fro.at/seriesrss/262430?c=Kfs2IoV2Wmd"; +$XML::Feed::MULTIPLE_ENCLOSURES=1; +my $PV_ID = '76'; + +binmode(STDIN, ":utf8"); +binmode(STDOUT, ":utf8"); +binmode(STDERR, ":utf8"); + +if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "!!!This is the last attempt, there won't be a retry on error!!!\n" +} + +my $user = `/usr/bin/id -un`; +$user =~ s/\n//; +my $group = "RadioATTAC"; + +my ($dbh, $errorstring) = RHRD::rddb::opendb(); +if(!defined $dbh) { + print "$errorstring\n"; + exit 1; +} +my @allowed_dbs = RHRD::rddb::get_dropboxes($dbh, $user, $group); +if(!defined $allowed_dbs[0] && defined $allowed_dbs[2]) { + print "$allowed_dbs[2]\n"; + exit 1; +} + +my $idx = 0; +my $idx_reb = 1; +if(scalar(@allowed_dbs) != 2) { + print "found more or less than 2 Dropboxes for this group?!\n"; + RHRD::rddb::closedb($dbh); + exit 1; +} +my $dropbox = $allowed_dbs[$idx]->{'PATH'}; +my $to_cart = $allowed_dbs[$idx]->{'TO_CART'}; +my $dropbox_reb = $allowed_dbs[$idx_reb]->{'PATH'}; +my $to_cart_reb = $allowed_dbs[$idx_reb]->{'TO_CART'}; + +my @today = Date::Calc::Standard_to_Business(Date::Calc::Today()); +$today[2] = 1; +my @broadcast_day = Date::Calc::Business_to_Standard(@today); +$today[2] = 3; +my @import_date = Date::Calc::Business_to_Standard(@today); +my @import_date_reb = Date::Calc::Add_Delta_Days($import_date[0], $import_date[1], $import_date[2], 3); +print "day of original broadcast: " . Date::Calc::Date_to_Text(@broadcast_day) . "\n"; +print "day of Radio Helsinki broadcast: " . Date::Calc::Date_to_Text(@import_date) . "\n"; +print "day of Radio Helsinki rebroadcast: " . Date::Calc::Date_to_Text(@import_date_reb) . "\n"; + +my $bd = sprintf("%04d-%02d-%02d", $broadcast_day[0], $broadcast_day[1], $broadcast_day[2]); +print "looking for files from $bd in RSS Feed\n"; +print " -> $RSS_URL\n"; + +my $feed = XML::Feed->parse(URI->new($RSS_URL)) + or die "Error fetching feed: " . XML::Feed->errstr; + +my $bdexp = sprintf("^(\\d+)\\..*\\s*0?%d\\s*[.,]\\s*0?%d\\s*[.,]\\s*%4d", $broadcast_day[2], $broadcast_day[1], $broadcast_day[0]); + + +my $file = ""; +my $out_file = ""; +my $i = 0; +for my $entry ($feed->entries) { + $i++; + next unless $entry->enclosure; + my $j = 0; + for my $enclosure($entry->enclosure) { + $j++; + if($enclosure->type eq "audio/mpeg" || $enclosure->type eq "audio/ogg") { + print "$i/$j: (" . $enclosure->type . ", " . $enclosure->length . ") " . $enclosure->url . "\n"; + my $url = $enclosure->url; + if($url =~ /^(.*)_cut(\.[^\.]+)$/) { + $url = $1 . $2; + } + + my $sum_title = decode_entities($entry->title); + next unless $sum_title =~ /$bdexp/; + my $sum_text = decode_entities($entry->content->body); + my $bnum = $1; + + my $ff = File::Fetch->new(uri => $url); + next unless $bnum eq substr($ff->output_file, 0, 3); + my $current_file = `cat $STAT_FILE`; + if($current_file eq $ff->output_file) { + print "Already downloaded file of today\n"; + RHRD::rddb::closedb($dbh); + exit 0; + } + $out_file = $ff->output_file; + if(!rhautoimport::check_file_extension($out_file)) { + print "\n\nThe extension of the matching file '". $out_file . "' seems to be wrong - manual import necessary!!!\n"; + RHRD::rddb::closedb($dbh); + exit 1; + } + + print " --> " . $bnum . ": downloading " . $url . " (" . $enclosure->length . " Bytes) .. "; + $file = $ff->fetch( to => '/tmp' ) or die $ff->error; + print "ok\n"; + + print "summary:\n" . $sum_title . "\n\n" . $sum_text . "\n"; + + rhautoimport::pv_add_note($sum_title, $sum_text, $PV_ID, sprintf("%04d-%02d-%02d", $import_date[0], $import_date[1], $import_date[2]), "1"); + print "\n"; + rhautoimport::pv_add_note($sum_title, $sum_text, $PV_ID, sprintf("%04d-%02d-%02d", $import_date_reb[0], $import_date_reb[1], $import_date_reb[2]), "2"); + print "\n"; + + last; + } + } + last if $file ne ""; +} + +if($file eq "" || !(-e "$file")) { + print "No Entry found from $bd or download error - "; + if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "giving up, manual import necessary!!!\n"; + } else { + print "will retry later\n"; + } + exit 1; +} + +print "will import $file to dropbox $dropbox (cart=$to_cart)\n"; +print "will import $file to rebroadcast dropbox $dropbox_reb (cart=$to_cart_reb)\n\n"; + +my $error_cb = sub { + my ($text) = @_; + + print "\n$text"; + return 0; +}; + +rhautoimport::check_key_file() or die "Import Key not found, use rhautoimport-create-id to create one\n"; + +my $ret; +my $log = rhautoimport::clear_carts($dbh, $group, $to_cart); +my $import_log; +($ret, $import_log) = rhautoimport::import_single($file, $dropbox, $user, 0, $error_cb); +$log .= $import_log; +$import_log = rhautoimport::clear_carts($dbh, $group, $to_cart_reb); +$log .= $import_log; +($ret, $import_log) = rhautoimport::import_single($file, $dropbox_reb, $user, 0, $error_cb); +$log .= $import_log; + +RHRD::rddb::closedb($dbh); + +unlink $file; + +if(!$ret) { + print "\nImport Error:\n"; + print $log; + exit 1; +} + +unlink($STAT_FILE); +open(my $fhs, '>', $STAT_FILE); +print $fhs $out_file; +close($fhs); + +exit 0; diff --git a/rhautoimport-rs b/rhautoimport-rs new file mode 100755 index 0000000..99a3b19 --- /dev/null +++ b/rhautoimport-rs @@ -0,0 +1,226 @@ +#!/usr/bin/perl -w +# +# +# rhautoimport +# +# Copyright (C) 2009-2015 Christian Pointner <equinox@helsinki.at> +# +# This file is part of rhautoimport. +# +# rhautoimport is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# rhautoimport is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with rhautoimport. If not, see <http://www.gnu.org/licenses/>. +# + +use strict; +use Date::Calc; +use XML::Feed; +use XML::Feed::Entry; +use XML::Feed::Content; +use XML::Feed::Enclosure; +use File::Fetch; +use IO::Handle; +use IPC::Open3; +use HTML::Entities; +use LWP::Simple; +use RHRD::rddb; + +use lib '/usr/local/share/rhautoimport/'; +use rhautoimport; + +my $STAT_FILE = $ENV{'HOME'} . "/rhautoimport-rs.stat"; +my $RSS_URL = "http://cba.fro.at/seriesrss/262433?c=Kfs2IoV2Wmd"; +$XML::Feed::MULTIPLE_ENCLOSURES=1; +my $PV_ID = '75'; + +binmode(STDIN, ":utf8"); +binmode(STDOUT, ":utf8"); +binmode(STDERR, ":utf8"); + +if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "!!!This is the last attempt, there won't be a retry on error!!!\n" +} + +my $user = `/usr/bin/id -un`; +$user =~ s/\n//; +my $group = "radioStimm"; +my %week_table = ( 1 => 2, 2 => 6, 3 => 2, 4 => 6 ); + +my ($dbh, $errorstring) = RHRD::rddb::opendb(); +if(!defined $dbh) { + print "$errorstring\n"; + exit 1; +} +my @allowed_dbs = RHRD::rddb::get_dropboxes($dbh, $user, $group); +if(!defined $allowed_dbs[0] && defined $allowed_dbs[2]) { + print "$allowed_dbs[2]\n"; + exit 1; +} + +my $dropbox = ""; +my $to_cart = 0; +foreach(@allowed_dbs) { + if($_->{'NAME'} =~ /^autoimport/) { + $dropbox = $_->{'PATH'}; + $to_cart = $_->{'TO_CART'}; + } +} +if(!$to_cart) { + print "no dropbox found!\n"; + RHRD::rddb::closedb($dbh); + exit 1; +} + +my $curweek = RHRD::utils::get_rd_week(); +my $nextweek = $curweek == 4 ? 1 : $curweek + 1; + +my @today = Date::Calc::Today(); +print "today: " . Date::Calc::Date_to_Text(@today) . "\n"; + +my $dow = Date::Calc::Day_of_Week(@today); +my @import_date = Date::Calc::Standard_to_Business(@today); + +if($dow <= $week_table{$curweek}) { + print "We are in week $curweek - broadcast is on " . Date::Calc::Day_of_Week_to_Text($week_table{$curweek}) . "\n"; + $import_date[2] = $week_table{$curweek}; + @import_date = Date::Calc::Business_to_Standard($import_date[0], $import_date[1], $import_date[2]); +} else { + print "We are in week $curweek - broadcast was on " . Date::Calc::Day_of_Week_to_Text($week_table{$curweek}) . "\n"; + print " next week is $nextweek - broadcast is on " . Date::Calc::Day_of_Week_to_Text($week_table{$nextweek}) . "\n"; + $import_date[2] = $week_table{$nextweek}; + @import_date = Date::Calc::Business_to_Standard(@import_date); + @import_date = Date::Calc::Add_Delta_Days(@import_date, 7); +} +print "day of next Radio Helsinki broadcast: " . Date::Calc::Date_to_Text(@import_date) . "\n"; + +my @broadcast_date = Date::Calc::Nth_Weekday_of_Month_Year($import_date[0], $import_date[1], 2, 1); +if(Date::Calc::Delta_Days(@broadcast_date, @import_date) > 0) { + my @tmp_date = Date::Calc::Nth_Weekday_of_Month_Year($import_date[0], $import_date[1], 2, 3); + if(Date::Calc::Delta_Days(@tmp_date, @import_date) > 0) { + @broadcast_date = @tmp_date; + } +} else { + my @tmp_date = Date::Calc::Add_Delta_YM(@broadcast_date, 0, -1); + @broadcast_date = Date::Calc::Nth_Weekday_of_Month_Year($tmp_date[0], $tmp_date[1], 2, 3); +} +print "day of latest original broadcast before next Radio Helsinki broadcast: " . Date::Calc::Date_to_Text(@broadcast_date) . "\n"; + +if(Date::Calc::Delta_Days(@broadcast_date, @today) <= 0) { + print "File won't be available by now!\n"; + RHRD::rddb::closedb($dbh); + exit 0; +} + +my $id = sprintf("%04d-%02d-%02d", $import_date[0], $import_date[1], $import_date[2]); +my $bd = sprintf("%04d-%02d-%02d", $broadcast_date[0], $broadcast_date[1], $broadcast_date[2]); +my $bdfile = sprintf("RAST%04d%02d%02d(CBA)?(.*)\.MP3", $broadcast_date[0], $broadcast_date[1], $broadcast_date[2]); + +print "looking for files from $bd in RSS Feed\n"; +print " -> $RSS_URL\n"; + +my $feed = XML::Feed->parse(URI->new($RSS_URL)) + or die "Error fetching feed: " . XML::Feed->errstr; + +my $file = ""; +my $out_file = ""; +my $i = 0; +for my $entry ($feed->entries) { + $i++; + next unless $entry->enclosure; + my $j = 0; + for my $enclosure($entry->enclosure) { + $j++; + if($enclosure->type eq "audio/mpeg" || $enclosure->type eq "audio/ogg") { + print "$i/$j: (" . $enclosure->type . ", " . $enclosure->length . ") " . $enclosure->url . "\n"; + my $url = $enclosure->url; + if($url =~ /^(.*)_cut(\.[^\.]+)$/) { + $url = $1 . $2; + } + + my $ff = File::Fetch->new(uri => $url); + next unless (uc($ff->output_file) =~ $bdfile); # file not from correct day + + my $sum_title = decode_entities($entry->title); + my $sum_text = decode_entities($entry->content->body); + + my $current_stat = `cat $STAT_FILE`; + my ($current_id, $current_file) = $current_stat =~ m/^(.*)\n(.*)/; + if($current_id eq $id && $current_file eq $ff->output_file) { + print "Already downloaded file of today\n"; + RHRD::rddb::closedb($dbh); + exit 0; + } + $out_file = $ff->output_file; + if(!rhautoimport::check_file_extension($out_file)) { + print "\n\nThe extension of the matching file '". $out_file . "' seems to be wrong - manual import necessary!!!\n"; + RHRD::rddb::closedb($dbh); + exit 1; + } + + print " --> " . $bdfile . ": downloading " . $url . " (" . $enclosure->length . " Bytes) .. "; + $file = $ff->fetch( to => '/tmp' ) or die $ff->error; + print "ok\n"; + + print "summary:\n" . $sum_title . "\n\n" . $sum_text . "\n"; + + rhautoimport::pv_add_note($sum_title, $sum_text, $PV_ID, $id, "1"); + print "\n"; + + last; + } + } + last if $file ne ""; +} + +if($file eq "" || !(-e "$file")) { + print "No Entry found from $bd or download error - "; + if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "giving up, manual import necessary!!!\n"; + } else { + print "will retry later\n"; + } + exit 1; +} + +print "will import $file to dropbox $dropbox (cart=$to_cart)\n"; + +my $error_cb = sub { + my ($text) = @_; + + print "\n$text"; + return 0; +}; + +rhautoimport::check_key_file() or die "Import Key not found, use rhautoimport-create-id to create one\n"; + +my $ret; +my $log = rhautoimport::clear_carts($dbh, $group, $to_cart); +my $import_log; +($ret, $import_log) = rhautoimport::import_single($file, $dropbox, $user, 0, $error_cb); +$log .= $import_log; + +RHRD::rddb::closedb($dbh); + +unlink $file; + +if(!$ret) { + print "\nImport Error:\n"; + print $log; + exit 1; +} + +unlink($STAT_FILE); +open(my $fhs, '>', $STAT_FILE); +print $fhs "$id\n$out_file"; +close($fhs); + +exit 0; diff --git a/rhautoimport-sm b/rhautoimport-sm new file mode 100755 index 0000000..6678d35 --- /dev/null +++ b/rhautoimport-sm @@ -0,0 +1,213 @@ +#!/usr/bin/perl -w +# +# +# rhautoimport +# +# Copyright (C) 2009-2015 Christian Pointner <equinox@helsinki.at> +# +# This file is part of rhautoimport. +# +# rhautoimport is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# rhautoimport is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with rhautoimport. If not, see <http://www.gnu.org/licenses/>. +# + +use strict; +use Date::Calc; +use XML::Feed; +use XML::Feed::Entry; +use XML::Feed::Content; +use XML::Feed::Enclosure; +use File::Fetch; +use IO::Handle; +use IPC::Open3; +use HTML::Entities; +use LWP::Simple; +use RHRD::rddb; + +use lib '/usr/local/share/rhautoimport/'; +use rhautoimport; + +my $STAT_FILE = $ENV{'HOME'} . "/rhautoimport-sm.stat"; +my $RSS_URL = "http://cba.fro.at/seriesrss/263673?c=Kfs2IoV2Wmd"; +$XML::Feed::MULTIPLE_ENCLOSURES=1; +my $PV_ID = '304'; + +binmode(STDIN, ":utf8"); +binmode(STDOUT, ":utf8"); +binmode(STDERR, ":utf8"); + +if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "!!!This is the last attempt, there won't be a retry on error!!!\n" +} + +my $user = `/usr/bin/id -un`; +$user =~ s/\n//; +my $group = "soundsmove"; + +my ($dbh, $errorstring) = RHRD::rddb::opendb(); +if(!defined $dbh) { + print "$errorstring\n"; + exit 1; +} +my @allowed_dbs = RHRD::rddb::get_dropboxes($dbh, $user, $group); +if(!defined $allowed_dbs[0] && defined $allowed_dbs[2]) { + print "$allowed_dbs[2]\n"; + exit 1; +} + +if(scalar(@allowed_dbs) != 1) { + print "found more or less than one Dropbox for this group?!\n"; + RHRD::rddb::closedb($dbh); + exit 1; +} +my $dropbox = $allowed_dbs[0]->{'PATH'}; +my $to_cart = $allowed_dbs[0]->{'TO_CART'}; + +my $curweek = RHRD::utils::get_rd_week(); + +my @today = Date::Calc::Today(); +my $dow = Date::Calc::Day_of_Week(@today); +if(($curweek != 1 && $curweek != 4) || + ($curweek == 1 && $dow > 3) || + ($curweek == 4 && $dow <= 3)) { + print "File won't be available by now!\n"; + RHRD::rddb::closedb($dbh); + exit 0; +} +print "today: " . Date::Calc::Date_to_Text(@today) . "\n"; + +my @import_date = Date::Calc::Standard_to_Business(@today); +$import_date[2] = 3; +@import_date = Date::Calc::Business_to_Standard(@import_date); +if($curweek == 4) { + @import_date = Date::Calc::Add_Delta_Days(@import_date, 7); +} +print "day of next Radio Helsinki broadcast: " . Date::Calc::Date_to_Text(@import_date) . "\n"; + +my @broadcast_date = Date::Calc::Nth_Weekday_of_Month_Year($import_date[0], $import_date[1], 3, 1); +if(Date::Calc::Delta_Days(@broadcast_date, @import_date) <= 0) { + my @tmp_date = Date::Calc::Add_Delta_YM(@broadcast_date, 0, -1); + @broadcast_date = Date::Calc::Nth_Weekday_of_Month_Year($tmp_date[0], $tmp_date[1], 3, 1); +} +print "day of latest original broadcast before next Radio Helsinki broadcast: " . Date::Calc::Date_to_Text(@broadcast_date) . "\n"; + +if(Date::Calc::Delta_Days(@broadcast_date, @today) <= 0) { + print "File won't be available by now!\n"; + RHRD::rddb::closedb($dbh); + exit 0; +} + +my $id = sprintf("%04d-%02d-%02d", @import_date); +my $bd = sprintf("%04d-%02d-%02d", @broadcast_date); +my $bdfile = sprintf("%04d-?%02d-?%02d", @broadcast_date); + +print "looking for files from $bd in RSS Feed\n"; +print " -> $RSS_URL\n"; + +my $feed = XML::Feed->parse(URI->new($RSS_URL)) + or die "Error fetching feed: " . XML::Feed->errstr; + +my $file = ""; +my $out_file = ""; +my $i = 0; +for my $entry ($feed->entries) { + $i++; + next unless $entry->enclosure; + my $j = 0; + for my $enclosure($entry->enclosure) { + $j++; + if($enclosure->type eq "audio/mpeg" || $enclosure->type eq "audio/ogg") { + print "$i/$j: (" . $enclosure->type . ", " . $enclosure->length . ") " . $enclosure->url . "\n"; + my $url = $enclosure->url; + if($url =~ /^(.*)_cut(\.[^\.]+)$/) { + $url = $1 . $2; + } + + my $ff = File::Fetch->new(uri => $url); + next unless (uc($ff->output_file) =~ $bdfile); # file not from correct day + + my $sum_title = decode_entities($entry->title); + my $sum_text = decode_entities($entry->content->body); + + my $current_stat = `cat $STAT_FILE`; + my ($current_id, $current_file) = $current_stat =~ m/^(.*)\n(.*)/; + if($current_id eq $id && $current_file eq $ff->output_file) { + print "Already downloaded file of today\n"; + RHRD::rddb::closedb($dbh); + exit 0; + } + $out_file = $ff->output_file; + if(!rhautoimport::check_file_extension($out_file)) { + print "\n\nThe extension of the matching file '". $out_file . "' seems to be wrong - manual import necessary!!!\n"; + RHRD::rddb::closedb($dbh); + exit 1; + } + + print " --> " . $bdfile . ": downloading " . $url . " (" . $enclosure->length . " Bytes) .. "; + $file = $ff->fetch( to => '/tmp' ) or die $ff->error; + print "ok\n"; + + print "summary:\n" . $sum_title . "\n\n" . $sum_text . "\n"; + + rhautoimport::pv_add_note($sum_title, $sum_text, $PV_ID, $id, "1"); + print "\n"; + + last; + } + } + last if $file ne ""; +} + +if($file eq "" || !(-e "$file")) { + print "No Entry found from $bd or download error - "; + if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "giving up, manual import necessary!!!\n"; + } else { + print "will retry later\n"; + } + exit 1; +} + +print "will import $file to dropbox $dropbox (cart=$to_cart)\n"; + +my $error_cb = sub { + my ($text) = @_; + + print "\n$text"; + return 0; +}; + +rhautoimport::check_key_file() or die "Import Key not found, use rhautoimport-create-id to create one\n"; + +my $ret; +my $log = rhautoimport::clear_carts($dbh, $group, $to_cart); +my $import_log; +($ret, $import_log) = rhautoimport::import_single($file, $dropbox, $user, 0, $error_cb); +$log .= $import_log; + +RHRD::rddb::closedb($dbh); + +unlink $file; + +if(!$ret) { + print "\nImport Error:\n"; + print $log; + exit 1; +} + +unlink($STAT_FILE); +open(my $fhs, '>', $STAT_FILE); +print $fhs "$id\n$out_file"; +close($fhs); + +exit 0; diff --git a/rhautoimport-sv b/rhautoimport-sv new file mode 100755 index 0000000..12c975c --- /dev/null +++ b/rhautoimport-sv @@ -0,0 +1,212 @@ +#!/usr/bin/perl -w +# +# +# rhautoimport +# +# Copyright (C) 2009-2015 Christian Pointner <equinox@helsinki.at> +# +# This file is part of rhautoimport. +# +# rhautoimport is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# rhautoimport is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with rhautoimport. If not, see <http://www.gnu.org/licenses/>. +# + +use strict; +use Date::Calc; +use XML::Feed; +use XML::Feed::Entry; +use XML::Feed::Content; +use XML::Feed::Enclosure; +use File::Fetch; +use IO::Handle; +use IPC::Open3; +use HTML::Entities; +use LWP::Simple; +use RHRD::rddb; + +use lib '/usr/local/share/rhautoimport/'; +use rhautoimport; + +my $STAT_FILE = $ENV{'HOME'} . "/rhautoimport-sv.stat"; +my $RSS_URL = "http://cba.fro.at/seriesrss/262457?c=Kfs2IoV2Wmd"; +$XML::Feed::MULTIPLE_ENCLOSURES=1; +my $PV_ID = '396'; + +binmode(STDIN, ":utf8"); +binmode(STDOUT, ":utf8"); +binmode(STDERR, ":utf8"); + +if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "!!!This is the last attempt, there won't be a retry on error!!!\n" +} + +my $user = `/usr/bin/id -un`; +$user =~ s/\n//; +my $group = "svetjevas"; + +my ($dbh, $errorstring) = RHRD::rddb::opendb(); +if(!defined $dbh) { + print "$errorstring\n"; + exit 1; +} +my @allowed_dbs = RHRD::rddb::get_dropboxes($dbh, $user, $group); +if(!defined $allowed_dbs[0] && defined $allowed_dbs[2]) { + print "$allowed_dbs[2]\n"; + exit 1; +} + +if(scalar(@allowed_dbs) != 1) { + print "found more or less than one Dropbox for this group?!\n"; + RHRD::rddb::closedb($dbh); + exit 1; +} +my $dropbox = $allowed_dbs[0]->{'PATH'}; +my $to_cart = $allowed_dbs[0]->{'TO_CART'}; + +my $curweek = RHRD::utils::get_rd_week(); + +my @today = Date::Calc::Today(); +my $dow = Date::Calc::Day_of_Week(@today); +if(($curweek != 3 && $curweek != 4) || + ($curweek == 4 && $dow > 2)) { + print "File won't be available by now!\n"; + RHRD::rddb::closedb($dbh); + exit 0; +} +print "today: " . Date::Calc::Date_to_Text(@today) . "\n"; + +my @import_date = Date::Calc::Standard_to_Business(@today); +$import_date[2] = 2; +@import_date = Date::Calc::Business_to_Standard(@import_date); +if($curweek == 3) { + @import_date = Date::Calc::Add_Delta_Days(@import_date, 7); +} +print "day of next Radio Helsinki broadcast: " . Date::Calc::Date_to_Text(@import_date) . "\n"; + +my @broadcast_date = Date::Calc::Nth_Weekday_of_Month_Year($import_date[0], $import_date[1], 7, 1); +if(Date::Calc::Delta_Days(@broadcast_date, @import_date) <= 0) { + my @tmp_date = Date::Calc::Add_Delta_YM(@broadcast_date, 0, -1); + @broadcast_date = Date::Calc::Nth_Weekday_of_Month_Year($tmp_date[0], $tmp_date[1], 7, 1); +} +print "day of latest original broadcast before next Radio Helsinki broadcast: " . Date::Calc::Date_to_Text(@broadcast_date) . "\n"; + +if(Date::Calc::Delta_Days(@broadcast_date, @today) <= 0) { + print "File won't be available by now!\n"; + RHRD::rddb::closedb($dbh); + exit 0; +} + +my $id = sprintf("%04d-%02d-%02d", @import_date); +my $bd = sprintf("%04d-%02d-%02d", @broadcast_date); +my $bdfile = sprintf("SEOI%02d%02d", $broadcast_date[2], $broadcast_date[1]); + +print "looking for files from $bd in RSS Feed\n"; +print " -> $RSS_URL\n"; + +my $feed = XML::Feed->parse(URI->new($RSS_URL)) + or die "Error fetching feed: " . XML::Feed->errstr; + +my $file = ""; +my $out_file = ""; +my $i = 0; +for my $entry ($feed->entries) { + $i++; + next unless $entry->enclosure; + my $j = 0; + for my $enclosure($entry->enclosure) { + $j++; + if($enclosure->type eq "audio/mpeg" || $enclosure->type eq "audio/ogg") { + print "$i/$j: (" . $enclosure->type . ", " . $enclosure->length . ") " . $enclosure->url . "\n"; + my $url = $enclosure->url; + if($url =~ /^(.*)_cut(\.[^\.]+)$/) { + $url = $1 . $2; + } + + my $ff = File::Fetch->new(uri => $url); + next unless (uc($ff->output_file) =~ $bdfile); # file not from correct day + + my $sum_title = decode_entities($entry->title); + my $sum_text = decode_entities($entry->content->body); + + my $current_stat = `cat $STAT_FILE`; + my ($current_id, $current_file) = $current_stat =~ m/^(.*)\n(.*)/; + if($current_id eq $id && $current_file eq $ff->output_file) { + print "Already downloaded file of today\n"; + RHRD::rddb::closedb($dbh); + exit 0; + } + $out_file = $ff->output_file; + if(!rhautoimport::check_file_extension($out_file)) { + print "\n\nThe extension of the matching file '". $out_file . "' seems to be wrong - manual import necessary!!!\n"; + RHRD::rddb::closedb($dbh); + exit 1; + } + + print " --> " . $bdfile . ": downloading " . $url . " (" . $enclosure->length . " Bytes) .. "; + $file = $ff->fetch( to => '/tmp' ) or die $ff->error; + print "ok\n"; + + print "summary:\n" . $sum_title . "\n\n" . $sum_text . "\n"; + + rhautoimport::pv_add_note($sum_title, $sum_text, $PV_ID, $id, "1"); + print "\n"; + + last; + } + } + last if $file ne ""; +} + +if($file eq "" || !(-e "$file")) { + print "No Entry found from $bd or download error - "; + if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "giving up, manual import necessary!!!\n"; + } else { + print "will retry later\n"; + } + exit 1; +} + +print "will import $file to dropbox $dropbox (cart=$to_cart)\n"; + +my $error_cb = sub { + my ($text) = @_; + + print "\n$text"; + return 0; +}; + +rhautoimport::check_key_file() or die "Import Key not found, use rhautoimport-create-id to create one\n"; + +my $ret; +my $log = rhautoimport::clear_carts($dbh, $group, $to_cart); +my $import_log; +($ret, $import_log) = rhautoimport::import_single($file, $dropbox, $user, 0, $error_cb); +$log .= $import_log; + +RHRD::rddb::closedb($dbh); + +unlink $file; + +if(!$ret) { + print "\nImport Error:\n"; + print $log; + exit 1; +} + +unlink($STAT_FILE); +open(my $fhs, '>', $STAT_FILE); +print $fhs "$id\n$out_file"; +close($fhs); + +exit 0; diff --git a/rhautoimport-tr b/rhautoimport-tr new file mode 100755 index 0000000..d467658 --- /dev/null +++ b/rhautoimport-tr @@ -0,0 +1,192 @@ +#!/usr/bin/perl -w +# +# +# rhautoimport +# +# Copyright (C) 2009-2015 Christian Pointner <equinox@helsinki.at> +# +# This file is part of rhautoimport. +# +# rhautoimport is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# rhautoimport is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with rhautoimport. If not, see <http://www.gnu.org/licenses/>. +# + +use strict; +use Date::Calc; +use XML::Feed; +use XML::Feed::Entry; +use XML::Feed::Content; +use XML::Feed::Enclosure; +use File::Fetch; +use IO::Handle; +use IPC::Open3; +use HTML::Entities; +use RHRD::rddb; + +use lib '/usr/local/share/rhautoimport/'; +use rhautoimport; + +my $STAT_FILE = $ENV{'HOME'} . "/rhautoimport-tr.stat"; +my $RSS_URL = "http://cba.fro.at/series/tierrechtsradio/feed?c=Kfs2IoV2Wmd"; +$XML::Feed::MULTIPLE_ENCLOSURES=1; +my $PV_ID = '506'; + +binmode(STDIN, ":utf8"); +binmode(STDOUT, ":utf8"); +binmode(STDERR, ":utf8"); + +if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "!!!This is the last attempt, there won't be a retry on error!!!\n" +} + +my $user = `/usr/bin/id -un`; +$user =~ s/\n//; +my $group = "tierrecht"; + +my ($dbh, $errorstring) = RHRD::rddb::opendb(); +if(!defined $dbh) { + print "$errorstring\n"; + exit 1; +} +my @allowed_dbs = RHRD::rddb::get_dropboxes($dbh, $user, $group); +if(!defined $allowed_dbs[0] && defined $allowed_dbs[2]) { + print "$allowed_dbs[2]\n"; + exit 1; +} + +my $idx = 0; +if(scalar(@allowed_dbs) != 1) { + print "found more or less than 1 Dropbox for this group?!\n"; + RHRD::rddb::closedb($dbh); + exit 1; +} +my $dropbox = $allowed_dbs[$idx]->{'PATH'}; +my $to_cart = $allowed_dbs[$idx]->{'TO_CART'}; + +my @today = Date::Calc::Standard_to_Business(Date::Calc::Today()); +if($today[2] == 5) { + print "please don't run this script on fridays!\n"; + RHRD::rddb::closedb($dbh); + exit 1; +} +my @broadcast_day = Date::Calc::Business_to_Standard($today[0], $today[1], 5); +if($today[2] < 5) { + @broadcast_day = Date::Calc::Add_Delta_Days($broadcast_day[0], $broadcast_day[1], $broadcast_day[2], -7); +} +my @import_date = Date::Calc::Add_Delta_Days($broadcast_day[0], $broadcast_day[1], $broadcast_day[2], 7); + +print "day of original broadcast: " . Date::Calc::Date_to_Text(@broadcast_day) . "\n"; +print "day of Radio Helsinki broadcast: " . Date::Calc::Date_to_Text(@import_date) . "\n"; + +my $bd = sprintf("%04d-%02d-%02d", $broadcast_day[0], $broadcast_day[1], $broadcast_day[2]); +my $bdfile = sprintf("^%04d%02d%02d", $broadcast_day[0], $broadcast_day[1], $broadcast_day[2]); + +print "looking for files from $bd in RSS Feed\n"; +print " -> $RSS_URL\n"; + +my $feed = XML::Feed->parse(URI->new($RSS_URL)) + or die "Error fetching feed: " . XML::Feed->errstr; + +my $file = ""; +my $out_file = ""; +my $i = 0; +for my $entry ($feed->entries) { + $i++; + next unless $entry->enclosure; + my $j = 0; + for my $enclosure($entry->enclosure) { + $j++; + if($enclosure->type eq "audio/mpeg" || $enclosure->type eq "audio/ogg") { + print "$i/$j: (" . $enclosure->type . ", " . $enclosure->length . ") " . $enclosure->url . "\n"; + my $url = $enclosure->url; + if($url =~ /^(.*)_cut(\.[^\.]+)$/) { + $url = $1 . $2; + } + + my $ff = File::Fetch->new(uri => $url); + next unless (uc($ff->output_file) =~ $bdfile); # file not from correct day + + my $sum_title = decode_entities($entry->title); + my $sum_text = decode_entities($entry->content->body); + + my $current_file = `cat $STAT_FILE`; + if($current_file eq $ff->output_file) { + print "Already downloaded file of today\n"; + RHRD::rddb::closedb($dbh); + exit 0; + } + $out_file = $ff->output_file; + if(!rhautoimport::check_file_extension($out_file)) { + print "\n\nThe extension of the matching file '". $out_file . "' seems to be wrong - manual import necessary!!!\n"; + RHRD::rddb::closedb($dbh); + exit 1; + } + + print " --> " . $bdfile . ": downloading " . $url . " (" . $enclosure->length . " Bytes) .. "; + $file = $ff->fetch( to => '/tmp' ) or die $ff->error; + print "ok\n"; + + print "summary:\n" . $sum_title . "\n\n" . $sum_text . "\n"; + + rhautoimport::pv_add_note($sum_title, $sum_text, $PV_ID, sprintf("%04d-%02d-%02d", $import_date[0], $import_date[1], $import_date[2]), "1"); + print "\n"; + + last; + } + } + last if $file ne ""; +} + +if($file eq "" || !(-e "$file")) { + print "No Entry found from $bd or download error - "; + if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "giving up, manual import necessary!!!\n"; + } else { + print "will retry later\n"; + } + exit 1; +} + +print "will import $file to dropbox $dropbox (cart=$to_cart)\n"; + +my $error_cb = sub { + my ($text) = @_; + + print "\n$text"; + return 0; +}; + +rhautoimport::check_key_file() or die "Import Key not found, use rhautoimport-create-id to create one\n"; + +my $ret; +my $log = rhautoimport::clear_carts($dbh, $group, $to_cart); +my $import_log; +($ret, $import_log) = rhautoimport::import_single($file, $dropbox, $user, 0, $error_cb); +$log .= $import_log; + +RHRD::rddb::closedb($dbh); + +unlink $file; + +if(!$ret) { + print "\nImport Error:\n"; + print $log; + exit 1; +} + +unlink($STAT_FILE); +open(my $fhs, '>', $STAT_FILE); +print $fhs $out_file; +close($fhs); + +exit 0; diff --git a/rhautoimport-ut b/rhautoimport-ut new file mode 100755 index 0000000..22b27d9 --- /dev/null +++ b/rhautoimport-ut @@ -0,0 +1,199 @@ +#!/usr/bin/perl -w +# +# +# rhautoimport +# +# Copyright (C) 2009-2015 Christian Pointner <equinox@helsinki.at> +# +# This file is part of rhautoimport. +# +# rhautoimport is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# rhautoimport is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with rhautoimport. If not, see <http://www.gnu.org/licenses/>. +# + +use strict; +use Date::Calc; +use XML::Feed; +use XML::Feed::Entry; +use XML::Feed::Content; +use XML::Feed::Enclosure; +use File::Fetch; +use IO::Handle; +use IPC::Open3; +use HTML::Entities; +use RHRD::rddb; + +use lib '/usr/local/share/rhautoimport/'; +use rhautoimport; + +my $STAT_FILE = $ENV{'HOME'} . "/rhautoimport-ut.stat"; +my $RSS_URL = "http://cba.fro.at/seriesrss/264016?c=Kfs2IoV2Wmd"; +$XML::Feed::MULTIPLE_ENCLOSURES=1; +my $PV_ID = '393'; + +binmode(STDIN, ":utf8"); +binmode(STDOUT, ":utf8"); +binmode(STDERR, ":utf8"); + +if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "!!!This is the last attempt, there won't be a retry on error!!!\n" +} + +my $user = `/usr/bin/id -un`; +$user =~ s/\n//; +my $group = "uton"; + +my ($dbh, $errorstring) = RHRD::rddb::opendb(); +if(!defined $dbh) { + print "$errorstring\n"; + exit 1; +} +my @allowed_dbs = RHRD::rddb::get_dropboxes($dbh, $user, $group); +if(!defined $allowed_dbs[0] && defined $allowed_dbs[2]) { + print "$allowed_dbs[2]\n"; + exit 1; +} + +my $idx = 0; +my $idx_reb = 1; +if(scalar(@allowed_dbs) != 2) { + print "found more or less than 2 Dropboxes for this group?!\n"; + RHRD::rddb::closedb($dbh); + exit 1; +} +my $dropbox = $allowed_dbs[$idx]->{'PATH'}; +my $to_cart = $allowed_dbs[$idx]->{'TO_CART'}; +my $dropbox_reb = $allowed_dbs[$idx_reb]->{'PATH'}; +my $to_cart_reb = $allowed_dbs[$idx_reb]->{'TO_CART'}; + +my @import_date = Date::Calc::Standard_to_Business(Date::Calc::Today()); +my $dow = $import_date[2]; +$import_date[2] = 3; +@import_date = Date::Calc::Business_to_Standard(@import_date); +@import_date = Date::Calc::Add_Delta_Days(@import_date, 7) if ($dow > 3); + +my @broadcast_day = Date::Calc::Add_Delta_Days(@import_date, -106); +my @import_date_reb = Date::Calc::Add_Delta_Days(@import_date, 2); +print "day of original broadcast: " . Date::Calc::Date_to_Text(@broadcast_day) . "\n"; +print "day of Radio Helsinki broadcast: " . Date::Calc::Date_to_Text(@import_date) . "\n"; +print "day of Radio Helsinki rebroadcast: " . Date::Calc::Date_to_Text(@import_date_reb) . "\n"; + +my $bd = sprintf("%04d-%02d-%02d", $broadcast_day[0], $broadcast_day[1], $broadcast_day[2]); +print "looking for files from $bd in RSS Feed\n"; +print " -> $RSS_URL\n"; + +my $feed = XML::Feed->parse(URI->new($RSS_URL)) + or die "Error fetching feed: " . XML::Feed->errstr; + +my $bdexp = sprintf("^UTON%02d%02d%02d", $broadcast_day[0] % 100, $broadcast_day[1], $broadcast_day[2]); + +my $file = ""; +my $out_file = ""; +my $i = 0; +for my $entry ($feed->entries) { + $i++; + next unless $entry->enclosure; + my $j = 0; + for my $enclosure($entry->enclosure) { + $j++; + if($enclosure->type eq "audio/mpeg" || $enclosure->type eq "audio/ogg") { + print "$i/$j: (" . $enclosure->type . ", " . $enclosure->length . ") " . $enclosure->url . "\n"; + my $url = $enclosure->url; + if($url =~ /^(.*)_cut(\.[^\.]+)$/) { + $url = $1 . $2; + } + + my $ff = File::Fetch->new(uri => $url); + next unless (uc($ff->output_file) =~ $bdexp); # file not from correct day + + my $sum_title = decode_entities($entry->title); + my $sum_text = decode_entities($entry->content->body); + + my $current_file = `cat $STAT_FILE`; + if($current_file eq $ff->output_file) { + print "Already downloaded file of today\n"; + RHRD::rddb::closedb($dbh); + exit 0; + } + $out_file = $ff->output_file; + if(!rhautoimport::check_file_extension($out_file)) { + print "\n\nThe extension of the matching file '". $out_file . "' seems to be wrong - manual import necessary!!!\n"; + RHRD::rddb::closedb($dbh); + exit 1; + } + + print " --> " . $bdexp . ": downloading " . $url . " (" . $enclosure->length . " Bytes) .. "; + $file = $ff->fetch( to => '/tmp' ) or die $ff->error; + print "ok\n"; + + print "summary:\n" . $sum_title . "\n\n" . $sum_text . "\n"; + + rhautoimport::pv_add_note($sum_title, $sum_text, $PV_ID, sprintf("%04d-%02d-%02d", $import_date[0], $import_date[1], $import_date[2]), "1"); + print "\n"; + rhautoimport::pv_add_note($sum_title, $sum_text, $PV_ID, sprintf("%04d-%02d-%02d", $import_date_reb[0], $import_date_reb[1], $import_date_reb[2]), "2"); + print "\n"; + + last; + } + } + last if $file ne ""; +} + +if($file eq "" || !(-e "$file")) { + print "No Entry found from $bd or download error - "; + if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "giving up, manual import necessary!!!\n"; + } else { + print "will retry later\n"; + } + exit 1; +} + +print "will import $file to dropbox $dropbox (cart=$to_cart)\n"; +print "will import $file to rebroadcast dropbox $dropbox_reb (cart=$to_cart_reb)\n\n"; + +my $error_cb = sub { + my ($text) = @_; + + print "\n$text"; + return 0; +}; + +rhautoimport::check_key_file() or die "Import Key not found, use rhautoimport-create-id to create one\n"; + +my $ret; +my $log = rhautoimport::clear_carts($dbh, $group, $to_cart); +my $import_log; +($ret, $import_log) = rhautoimport::import_single($file, $dropbox, $user, 0, $error_cb); +$log .= $import_log; +$import_log = rhautoimport::clear_carts($dbh, $group, $to_cart_reb); +$log .= $import_log; +($ret, $import_log) = rhautoimport::import_single($file, $dropbox_reb, $user, 0, $error_cb); +$log .= $import_log; + +RHRD::rddb::closedb($dbh); + +unlink $file; + +if(!$ret) { + print "\nImport Error:\n"; + print $log; + exit 1; +} + +unlink($STAT_FILE); +open(my $fhs, '>', $STAT_FILE); +print $fhs $out_file; +close($fhs); + +exit 0; diff --git a/rhautoimport-zf b/rhautoimport-zf new file mode 100755 index 0000000..264232c --- /dev/null +++ b/rhautoimport-zf @@ -0,0 +1,193 @@ +#!/usr/bin/perl -w +# +# +# rhautoimport +# +# Copyright (C) 2009-2015 Christian Pointner <equinox@helsinki.at> +# +# This file is part of rhautoimport. +# +# rhautoimport is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# rhautoimport is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with rhautoimport. If not, see <http://www.gnu.org/licenses/>. +# + +use strict; +use Date::Calc; +use XML::Feed; +use XML::Feed::Entry; +use XML::Feed::Content; +use XML::Feed::Enclosure; +use File::Fetch; +use IO::Handle; +use IPC::Open3; +use HTML::Entities; +use File::Touch; +use RHRD::rddb; + +use lib '/usr/local/share/rhautoimport/'; +use rhautoimport; + +my $STAT_FILE = $ENV{'HOME'} . "/rhautoimport-zf.stat"; +my $NOTE_FILE = $ENV{'HOME'} . "/rhautoimport-zf.last_note"; +my $NEW_FILE = $ENV{'HOME'} . "/rhautoimport-zf.is_new"; +my $RSS_URL = "http://freie-radios.net/portal/podcast.php?serie=53&rss"; +my $PV_ID = '300'; + +binmode(STDIN, ":utf8"); +binmode(STDOUT, ":utf8"); +binmode(STDERR, ":utf8"); + +if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "!!!This is the last attempt, there won't be a retry on error!!!\n" +} + +my $user = `/usr/bin/id -un`; +$user =~ s/\n//; +my $group = "zipfm"; + +my ($dbh, $errorstring) = RHRD::rddb::opendb(); +if(!defined $dbh) { + print "$errorstring\n"; + exit 1; +} +my @allowed_dbs = RHRD::rddb::get_dropboxes($dbh, $user, $group); +if(!defined $allowed_dbs[0] && defined $allowed_dbs[2]) { + print "$allowed_dbs[2]\n"; + exit 1; +} + + +my $dropbox = ""; +my $to_cart = 0; +my $dropbox_reb = ""; +my $to_cart_reb = 0; + +my @today = Date::Calc::Today(); +my $dow = Date::Calc::Day_of_Week(@today); +my @tomorrow = Date::Calc::Add_Delta_Days(@today, 1); + +my @broadcast_day = @today; +my @rebroadcast_day = @tomorrow; +print "day of broadcast: " . Date::Calc::Date_to_Text(@broadcast_day) . "\n"; +print "day of rebroadcast: " . Date::Calc::Date_to_Text(@rebroadcast_day) . "\n" unless $dow == 5; + +for my $href ( @allowed_dbs ) { + if ($dow == int(substr($href->{'NAME'},0,2))) { + $dropbox = $href->{'PATH'}; + $to_cart = $href->{'TO_CART'}; + } +} + +if($dropbox eq "") { + print "no dropbox for day in question\n"; + RHRD::rddb::closedb($dbh); + exit 0; +} + +my $regexp = sprintf("%04d%02d%02d", $broadcast_day[0], $broadcast_day[1], $broadcast_day[2]); +print "looking for file from " . Date::Calc::Date_to_Text(@broadcast_day) . " in RSS Feed\n"; +print " -> $RSS_URL\n"; + +my $feed = XML::Feed->parse(URI->new($RSS_URL)) + or die "Error fetching feed: " . XML::Feed->errstr; + +my $file = ""; +my $out_file = ""; +for my $entry ($feed->entries) { + if($entry->enclosure && $entry->enclosure->type eq "audio/mpeg") { + next unless $entry->enclosure->url =~ /$regexp/; + + my $sum_title = decode_entities($entry->title); + my $sum_text = decode_entities($entry->content->body); + + my $ff = File::Fetch->new(uri => $entry->enclosure->url); + my $current_file = `cat $STAT_FILE`; + if($current_file eq $ff->output_file) { + print "Already downloaded file of today\n"; + RHRD::rddb::closedb($dbh); + exit 0; + } + $out_file = $ff->output_file; + if(!rhautoimport::check_file_extension($out_file)) { + print "\n\nThe extension of the matching file '". $out_file . "' seems to be wrong - manual import necessary!!!\n"; + RHRD::rddb::closedb($dbh); + exit 1; + } + + print $regexp . ": downloading " . $entry->enclosure->url . " (" . $entry->enclosure->length . " Bytes) .. "; + $file = $ff->fetch( to => '/tmp' ) or die $ff->error; + print "ok\n"; + + print "summary:\n" . $sum_title . "\n\n" . $sum_text . "\n"; + + if($#ARGV >= 0 && $ARGV[0] eq 'nopv') { + print "not adding note to PV for regular entry - only for rebroadcast\n"; + } else { + rhautoimport::pv_add_note($sum_title, $sum_text, $PV_ID, sprintf("%04d-%02d-%02d", @broadcast_day), "1", 1); + } + rhautoimport::pv_add_note($sum_title, $sum_text, $PV_ID, sprintf("%04d-%02d-%02d", @rebroadcast_day), "2", 0) unless $dow == 5; + print "\n"; + + unlink($NOTE_FILE); + open(my $fhs, '>', $NOTE_FILE); + binmode($fhs, ":utf8"); + print $fhs $sum_title . "\n\n" . $sum_text . "\n"; + close($fhs); + + last; + } +} +if($file eq "" || !(-e "$file")) { + print "No Entry found for " . Date::Calc::Date_to_Text(@broadcast_day) . " or download error - "; + if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "giving up, manual import necessary!!!\n"; + } else { + print "will retry later\n"; + } + exit 1; +} + +print "will import $file to dropbox $dropbox (cart=$to_cart)\n"; + +my $error_cb = sub { + my ($text) = @_; + + print "\n$text"; + return 0; +}; + +rhautoimport::check_key_file() or die "Import Key not found, use rhautoimport-create-id to create one\n"; + +my $ret; +my $log = rhautoimport::clear_carts($dbh, $group, $to_cart); +my $import_log; +($ret, $import_log) = rhautoimport::import_single($file, $dropbox, $user, 0, $error_cb); +$log .= $import_log; + +RHRD::rddb::closedb($dbh); + +unlink $file; + +if(!$ret) { + print "\nImport Error:\n"; + print $log; + exit 1; +} + +unlink($STAT_FILE); +open(my $fhs, '>', $STAT_FILE); +print $fhs $out_file; +close($fhs); +touch($NEW_FILE); + +exit 0; diff --git a/rhautoimport-zffe b/rhautoimport-zffe new file mode 100755 index 0000000..eb4c1a1 --- /dev/null +++ b/rhautoimport-zffe @@ -0,0 +1,118 @@ +#!/usr/bin/perl -w +# +# +# rhautoimport +# +# Copyright (C) 2009-2015 Christian Pointner <equinox@helsinki.at> +# +# This file is part of rhautoimport. +# +# rhautoimport is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# rhautoimport is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with rhautoimport. If not, see <http://www.gnu.org/licenses/>. +# + +use strict; +use Switch; +use Date::Calc; +use Time::localtime; +use IO::Handle; +use IPC::Open3; +use File::Slurp; + +use lib '/usr/local/share/rhautoimport/'; +use rhautoimport; + +my $STAT_FILE = $ENV{'HOME'} . "/rhautoimport-zffe.stat"; +my $ZF_NOTE_FILE = $ENV{'HOME'} . "/rhautoimport-zf.last_note"; +my $ZF_NEW_FILE = $ENV{'HOME'} . "/rhautoimport-zf.is_new"; +my $FE_NOTE_FILE = $ENV{'HOME'} . "/rhautoimport-fe.last_note"; +my $FE_NEW_FILE = $ENV{'HOME'} . "/rhautoimport-fe.is_new"; +my $PV_ID = '359'; + +binmode(STDIN, ":utf8"); +binmode(STDOUT, ":utf8"); +binmode(STDERR, ":utf8"); + +if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "!!!This is the last attempt, there won't be a retry on error!!!\n" +} + +my @today = Date::Calc::Today(); +my $dow = Date::Calc::Day_of_Week(@today); +my @tomorrow = Date::Calc::Add_Delta_Days($today[0], $today[1], $today[2], 1); + +my @tmp = Date::Calc::Standard_to_Business(@today); +$tmp[2] = 1; +my @this_mon = Date::Calc::Business_to_Standard(@tmp); +my @next_mon = Date::Calc::Add_Delta_Days($this_mon[0], $this_mon[1], $this_mon[2], 7); + +my @import_day = @tomorrow; +if ($dow == 5) { + @import_day = @next_mon; +} + +my $import_date = sprintf("%04d-%02d-%02d", @import_day); +my $current_date = `cat $STAT_FILE`; +if($current_date eq $import_date) { + print "Already created note of today\n"; + exit 0; +} + +unless(-e $ZF_NEW_FILE) { + print "zip-fm not yet imported - "; + if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "giving up, rebroadcasting last show"; + } else { + print "will retry later\n"; + exit 0, + } +} + +unless(-e $FE_NEW_FILE) { + print "focus europa not yet imported - "; + if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "giving up, rebroadcasting last show"; + } else { + print "will retry later\n"; + exit 0, + } +} + +sub get_note { + my ($file) = @_; + my @lines = read_file($file, binmode => ':utf8', err_mode => 'quiet'); + return "" unless ($lines[0]); + + chomp($lines[0]); + $lines[0] = "<p><strong>$lines[0]</strong></p>\n"; + return join('',@lines); +}; + +my $sum_title = "focus europa, zip-fm (WH)"; +my $sum_text = get_note($FE_NOTE_FILE) . "<br/>\n"; +$sum_text = $sum_text . "<br/>\n" . get_note($ZF_NOTE_FILE) . "<br/>\n"; +$sum_text = $sum_text . "<br/><br/>\n"; + +print "\nsummary:\n" . $sum_title . "\n\n" . $sum_text . "\n"; +rhautoimport::pv_add_note($sum_title, $sum_text, $PV_ID, sprintf("%04d-%02d-%02d", @import_day), "1"); +print "\n"; + +unlink($STAT_FILE); +open(my $fhs, '>', $STAT_FILE); +print $fhs $import_date; +close($fhs); + +unlink($ZF_NEW_FILE); +unlink($FE_NEW_FILE); + +exit 0; diff --git a/rhautoimport-zfw b/rhautoimport-zfw new file mode 100755 index 0000000..9cdb169 --- /dev/null +++ b/rhautoimport-zfw @@ -0,0 +1,195 @@ +#!/usr/bin/perl -w +# +# +# rhautoimport +# +# Copyright (C) 2009-2015 Christian Pointner <equinox@helsinki.at> +# +# This file is part of rhautoimport. +# +# rhautoimport is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# rhautoimport is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with rhautoimport. If not, see <http://www.gnu.org/licenses/>. +# + +use strict; +use Date::Calc; +use Time::localtime; +use XML::Feed; +use XML::Feed::Entry; +use XML::Feed::Content; +use XML::Feed::Enclosure; +use File::Fetch; +use IO::Handle; +use IPC::Open3; +use HTML::Entities; +use RHRD::rddb; + +use lib '/usr/local/share/rhautoimport/'; +use rhautoimport; + +my $STAT_FILE = $ENV{'HOME'} . "/rhautoimport-zfw.stat"; +my $RSS_URL = "http://cba.fro.at/seriesrss/263764?c=Kfs2IoV2Wmd"; +$XML::Feed::MULTIPLE_ENCLOSURES=1; +my $PV_ID = '360'; + +binmode(STDIN, ":utf8"); +binmode(STDOUT, ":utf8"); +binmode(STDERR, ":utf8"); + +if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "!!!This is the last attempt, there won't be a retry on error!!!\n" +} + +my $user = `/usr/bin/id -un`; +$user =~ s/\n//; +my $group = "zipfmwla"; + +my ($dbh, $errorstring) = RHRD::rddb::opendb(); +if(!defined $dbh) { + print "$errorstring\n"; + exit 1; +} +my @allowed_dbs = RHRD::rddb::get_dropboxes($dbh, $user, $group); +if(!defined $allowed_dbs[0] && defined $allowed_dbs[2]) { + print "$allowed_dbs[2]\n"; + exit 1; +} + +if(scalar(@allowed_dbs) != 1) { + print "found no or more than one Dropboxes for this group?!\n"; + RHRD::rddb::closedb($dbh); + exit 1; +} +my $dropbox = $allowed_dbs[0]->{'PATH'}; +my $to_cart = $allowed_dbs[0]->{'TO_CART'}; + + +my @today = Date::Calc::Today(); +my @yesterday = Date::Calc::Add_Delta_Days($today[0], $today[1], $today[2], -1); +my @tomorrow = Date::Calc::Add_Delta_Days($today[0], $today[1], $today[2], 1); + +my @broadcast_day = @yesterday; +my @import_day = @today; +if(localtime->hour >= 12) { + print "It's past noon, assuming to import file from today\n"; + @broadcast_day = @today; + @import_day = @tomorrow; +} + +print "day of original broadcast: " . Date::Calc::Date_to_Text(@broadcast_day) . "\n"; +print "day of Radio Helsinki broadcast: " . Date::Calc::Date_to_Text(@import_day) . "\n"; + +my $id = sprintf("%04d-%02d-%02d", $import_day[0], $import_day[1], $import_day[2]); +my $bd = sprintf("%04d-%02d-%02d", $broadcast_day[0], $broadcast_day[1], $broadcast_day[2]); +print "looking for files from $bd in RSS Feed\n"; +print " -> $RSS_URL\n"; + +my $feed = XML::Feed->parse(URI->new($RSS_URL)) + or die "Error fetching feed: " . XML::Feed->errstr; + +my $bdfile = sprintf("%04d%02d%02d", $broadcast_day[0], $broadcast_day[1], $broadcast_day[2]); + +my $file = ""; +my $out_file = ""; +my $i = 0; +for my $entry ($feed->entries) { + $i++; + next unless $entry->enclosure; + my $j = 0; + for my $enclosure($entry->enclosure) { + $j++; + if($enclosure->type eq "audio/mpeg" || $enclosure->type eq "audio/ogg") { + print "$i/$j: (" . $enclosure->type . ", " . $enclosure->length . ") " . $enclosure->url . "\n"; + my $url = $enclosure->url; + if($url =~ /^(.*)_cut(\.[^\.]+)$/) { + $url = $1 . $2; + } + + my $ff = File::Fetch->new(uri => $url); + next unless (uc($ff->output_file) =~ $bdfile); # file not from correct day + + my ($sum_title, $sum_title_rest) = split(/:/, decode_entities($entry->title), 2); + my $sum_text = decode_entities($entry->content->body); + + my $current_stat = `cat $STAT_FILE`; + my ($current_id, $current_file) = $current_stat =~ m/^(.*)\n(.*)/; + if($current_id eq $id && $current_file eq $ff->output_file) { + print "Already downloaded file of today\n"; + RHRD::rddb::closedb($dbh); + exit 0; + } + $out_file = $ff->output_file; + if(!rhautoimport::check_file_extension($out_file)) { + print "\n\nThe extension of the matching file '". $out_file . "' seems to be wrong - manual import necessary!!!\n"; + RHRD::rddb::closedb($dbh); + exit 1; + } + + print " --> " . $bdfile . ": downloading " . $url . " (" . $enclosure->length . " Bytes) .. "; + $file = $ff->fetch( to => '/tmp' ) or die $ff->error; + print "ok\n"; + + print "summary:\n" . $sum_title . "\n\n" . $sum_text . "\n"; + + rhautoimport::pv_add_note($sum_title, $sum_text, $PV_ID, $id, "1"); + print "\n"; + + last; + } + } + last if $file ne ""; +} + +if($file eq "" || !(-e "$file")) { + print "No Entry found from $bd or download error - "; + if($#ARGV >= 0 && $ARGV[0] eq 'last') { + print "giving up, manual import necessary!!!\n"; + } else { + print "will retry later\n"; + } + exit 1; +} + +print "will import $file to dropbox $dropbox (cart=$to_cart)\n"; + +my $error_cb = sub { + my ($text) = @_; + + print "\n$text"; + return 0; +}; + +rhautoimport::check_key_file() or die "Import Key not found, use rhautoimport-create-id to create one\n"; + +my $ret; +my $log = rhautoimport::clear_carts($dbh, $group, $to_cart); +my $import_log; +($ret, $import_log) = rhautoimport::import_single($file, $dropbox, $user, 0, $error_cb); +$log .= $import_log; + +RHRD::rddb::closedb($dbh); + +unlink $file; + +if(!$ret) { + print "\nImport Error:\n"; + print $log; + exit 1; +} + +unlink($STAT_FILE); +open(my $fhs, '>', $STAT_FILE); +print $fhs "$id\n$out_file"; +close($fhs); + +exit 0; diff --git a/rhautoimport.pm b/rhautoimport.pm new file mode 100644 index 0000000..76e3893 --- /dev/null +++ b/rhautoimport.pm @@ -0,0 +1,247 @@ +#!/usr/bin/perl -w +# +# +# rhautoimport +# +# Copyright (C) 2009-2015 Christian Pointner <equinox@helsinki.at> +# +# This file is part of rhautoimport. +# +# rhautoimport is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# rhautoimport is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with rhautoimport. If not, see <http://www.gnu.org/licenses/>. +# + +use strict; + +package rhautoimport; + +use IO::Handle; +use IPC::Open3; +use File::Spec; +use File::Basename; +use URI::Escape; +use LWP::Simple; +use XML::Feed; + +my $ssh_host = "airplay"; +my $ssh_user = "rhautoimport"; +my $ssh_key_file = "$ENV{'HOME'}/.rhautoimport/import.key"; +my $ssh_dir = "/programm/.rhautoimport"; +my $rdimport_wrapper = "/usr/local/bin/dropbox_newfile.pl"; + +sub fetch_parse_rss +{ + my ($url, $ua_str) = @_; + + my $uri = URI->new($url); + $ua_str = "Radio Helsinki - Automatic Import" unless $ua_str; + + my $ua = LWP::UserAgent->new; + $ua->agent($ua_str); + $ua->env_proxy; + my $res = URI::Fetch->fetch($uri, UserAgent => $ua) + or die URI::Fetch->errstr; + die "This feed has been permanently removed" + if $res->status == URI::Fetch::URI_GONE(); + + my $xml = $res->content; + + return XML::Feed->parse(\$xml); +} + +sub clear_carts +{ + my ($dbh, $group, $to_cart, $progress_cb) = @_; + + my ($low_cart, $high_cart) = ($to_cart, $to_cart); + if($to_cart == 0) { + ($low_cart, $high_cart) = get_cart_range($dbh, $group); + } + + my $log = "clearing Carts $low_cart - $high_cart\n"; + print $log; + my $cart = $low_cart; + while($cart <= $high_cart) { + $progress_cb->($cart - $low_cart, $high_cart + 1 - $low_cart, "deleting $cart") if($progress_cb); + + my $sql = qq{select CUT_NAME from CUTS where CART_NUMBER='$cart';}; + my $sth = $dbh->prepare($sql); + $sth->execute(); + while(my $cut_name = $sth->fetchrow_array()) { + delete_file("/var/snd/$cut_name.wav"); + $log .= " - deleting file /var/snd/$cut_name.wav"; + } + $sth->finish(); + + $sql = qq{delete from CUTS where CART_NUMBER='$cart';}; + $sth = $dbh->prepare($sql); + $sth->execute(); + $sth->finish(); + + $sql = qq{delete from CART where NUMBER='$cart';}; + $sth = $dbh->prepare($sql); + $sth->execute(); + $sth->finish(); + + $cart++; + } + + return $log; +} + +sub import_single +{ + my ($file, $dropbox, $user, $progress_cb, $error_cb) = @_; + print "Starting import from file $file to $dropbox\n"; + $progress_cb->(0, 1, $file) if($progress_cb); + prepare_import($user); + my ($ret, $out) = import_file($file, $dropbox, $user, $error_cb); + if($ret) { + $progress_cb->(1, 1, "Import abgeschlossen!") if($progress_cb); + } else { + $progress_cb->(0, 1, "Import abgebrochen!") if($progress_cb); + } + return ($ret, $out); +} + +sub scp_put_file +{ + my ($file, $user, $host, $path) = @_; + my @cmd = ( 'scp', '-prqB', '-i', $ssh_key_file, $file, "$user\@$host:$path/" ); + my ($reader, $writer, $error ) = ( new IO::Handle, new IO::Handle, new IO::Handle ); + $writer->autoflush(1); + local $SIG{CHLD} = 'DEFAULT'; + my $pid = open3($writer, $reader, $error, @cmd); + binmode($reader, ":utf8"); + binmode($writer, ":utf8"); + binmode($error, ":utf8"); + waitpid $pid, 0; + my $errstr = ""; + if ( $? >> 8 ) { + $errstr = join('', <$error>); + } + return $errstr; +} + +sub ssh_exec_command +{ + my ($command) = @_; + my @cmd = ( 'ssh' , '-q', '-o', 'BatchMode=yes', '-i', $ssh_key_file, "$ssh_user\@$ssh_host" , $command ); + my ($reader, $writer, $error ) = ( new IO::Handle, new IO::Handle, new IO::Handle ); + $writer->autoflush(1); + local $SIG{CHLD} = 'DEFAULT'; + my $pid = open3($writer, $reader, $error, @cmd); + binmode($reader, ":utf8"); + binmode($writer, ":utf8"); + binmode($error, ":utf8"); + waitpid $pid, 0; + my $out = join('', <$reader>); + my $errstr = ""; + if ( $? >> 8 ) { + $errstr = join('', <$error>); + $errstr = "unkown error" if($errstr eq ""); + } + return ($out, $errstr); +} + +sub prepare_import +{ + my ($user) = @_; + + ### create user directory + ssh_exec_command("umask 002; mkdir -p \"$ssh_dir/$user\""); +} + +sub import_file +{ + my ($file, $dropbox, $user, $error_cb) = @_; + + print " - importing $file to $dropbox .. "; + $| = 1; + + ### copy file to master server + my $err = scp_put_file($file, $ssh_user, $ssh_host, "$ssh_dir/$user"); + if($err ne "") { + print "Transfer Error\n"; + if($error_cb) { + return $error_cb->($err); + } + return 0; + } + print "transferred .. "; + + ### remotely call rdimport + my ($volume, $directories, $remote_file) = File::Spec->splitpath($file); + $remote_file = "$ssh_dir/$user/$remote_file"; + my ($out ,$error) = ssh_exec_command("$rdimport_wrapper --path \"$dropbox\" --file \"$remote_file\""); + my $lastline = $1 if $out =~ /\n(.*)$/; + if($error ne "" || $lastline !~ /^[0-9:\- ]+:\s+Deleted file/) { + print "Import Error\n"; + delete_file($remote_file); + if($error_cb) { + if($error) { + return $error_cb->("Import Fehler: $error"); + } + else { + return $error_cb->("Import Fehler: \n$out"); + } + } + return 0; + } + + print "imported OK\n"; + return (1, $out); +} + +sub delete_file +{ + my ($filename) = @_; + + print " - deleting $filename\n"; + my ($out ,$err) = ssh_exec_command("rm \"$filename\""); + if($err ne "") { + return 0; + } + return 1; +} + +sub pv_add_note +{ + my ( $title, $text, $id, $date, $type, $index ) = @_; + my @script = ('python', '/srv/pv/pv/manage.py', 'addnote', $id, $date, $type); + push(@script , $index) unless (!defined $index); + + my ($reader, $writer, $error ) = ( new IO::Handle, new IO::Handle, new IO::Handle ); + $writer->autoflush(1); + local $SIG{CHLD} = 'DEFAULT'; + my $pid = open3($writer, $reader, $error, @script); + binmode($reader, ":utf8"); + binmode($writer, ":utf8"); + binmode($error, ":utf8"); + print $writer $title . "\n" . $text; + close $writer; + waitpid $pid, 0; + my $err_out = join('', <$error>); + my $read_out = join('', <$reader>); + if ( $? >> 8 ) { + print "\n\nPV: adding note returned non-zero value\n"; + print "STDERR:\n" . $err_out . "\n" unless $err_out eq ''; + print "STDOUT:\n" . $read_out . "\n" unless $read_out eq ''; + print "Ignoring failed headline import!\n"; + } else { + print $read_out . "\n" unless $read_out eq ''; + print $err_out . "\n" unless $err_out eq ''; + } +} + +1; diff --git a/rhimport b/rhimport deleted file mode 100755 index f0be9df..0000000 --- a/rhimport +++ /dev/null @@ -1,126 +0,0 @@ -#!/usr/bin/perl -w -# -# -# rhimport -# -# Copyright (C) 2009-2015 Christian Pointner <equinox@helsinki.at> -# -# This file is part of rhimport. -# -# rhimport is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# any later version. -# -# rhimport is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with rhimport. If not, see <http://www.gnu.org/licenses/>. -# - -use strict; -use Getopt::Long; -use RHRD::rddb; - -use lib '/usr/local/share/rhimport/'; -use rhimport; - -my $HELP = 0; -my $FILE = ""; -my $DROPBOX = ""; -my $LISTALLOWED = 0; - -GetOptions ("help!" => \$HELP, - "file=s" => \$FILE, - "dropbox=s" => \$DROPBOX, - "list-allowed!" => \$LISTALLOWED, - ) or die(); - -if($HELP) { - print << "EOF"; -usage: $0 --file <audio file> --dropbox <path to dropbox> --list-allowed - -options: - -f | --file the media file or playlist to import - -i | --id the show id to import to - -l | --list-allowed list allowed shows and exit - -EOF - exit 0; -} - -my $user = `/usr/bin/id -un`; -$user =~ s/\n//; -$user = 'heslinki'; - -my ($dbh, undef, $errorstring) = RHRD::rddb::opendb(); -if(!defined $dbh) { - print "$errorstring\n"; - exit 1; -} -my @allowed_dbs = RHRD::rddb::get_dropboxes($dbh, $user); -if(!defined $allowed_dbs[0] && defined $allowed_dbs[2]) { - print "$allowed_dbs[2]\n"; - exit 1; -} - -if($LISTALLOWED) { - print "| id | Title\n"; - print "+-------+-------------------------------------------------------------------\n"; - for my $href ( @allowed_dbs ) { - if($href->{'TYPE'} eq 'show') { - my @show_carts = RHRD::rddb::get_show_carts($dbh, $href->{'SHOWLOG'}, $href->{'GROUPLOWCART'}, $href->{'GROUPHIGHCART'}); - if(!defined $show_carts[0] && defined $show_carts[2]) { - print "$show_carts[2]\n"; - exit 1; - } - print "| $href->{'SHOWID'} | $href->{'SHOWTITLE'}, Carts: [" . join(', ', @show_carts) . "]\n"; - } - } - RHRD::rddb::closedb($dbh); - exit 0; -} - -my $cl_error_cb = sub { - my ($text) = @_; - - print "\n$text .. cancel operation [Y/n]? "; - my $x = scalar(<STDIN>); - $x =~ /^n/i; -}; - -(-e "$FILE") or die "file '$FILE' not found\n"; -print "will import $FILE, with user $user\n\n"; - -my $group = ''; -my $to_cart = 0; -for my $href (@allowed_dbs) { - if($href->{'PATH'} eq $DROPBOX) { - $group = $href->{'GROUP'}; - $to_cart = $href->{'TO_CART'}; - last; - } -} - -if($group eq '') { - print "Dropbox not found or not allowed\n"; - exit 1 -} - -my $ret; -my $log; -$log = rhimport::clear_carts($dbh, $group, $to_cart); -my $import_log; -($ret, $import_log) = rhimport::import_single($FILE, $DROPBOX, $user, 0, $cl_error_cb); -$log .= $import_log; - -RHRD::rddb::closedb($dbh); - -if(!$ret) { - exit 1; -} - -exit 0; diff --git a/rhimport-btl b/rhimport-btl deleted file mode 100755 index b7e841f..0000000 --- a/rhimport-btl +++ /dev/null @@ -1,177 +0,0 @@ -#!/usr/bin/perl -w -# -# -# rhimport -# -# Copyright (C) 2009-2015 Christian Pointner <equinox@helsinki.at> -# -# This file is part of rhimport. -# -# rhimport is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# any later version. -# -# rhimport is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with rhimport. If not, see <http://www.gnu.org/licenses/>. -# - -use strict; -use Date::Calc; -use XML::Feed; -use XML::Feed::Entry; -use XML::Feed::Content; -use XML::Feed::Enclosure; -use File::Fetch; -use IO::Handle; -use IPC::Open3; -use HTML::Entities; -use LWP::Simple; -use RHRD::rddb; - -use lib '/usr/local/share/rhimport/'; -use rhimport; - -my $STAT_FILE = $ENV{'HOME'} . "/rhimport-btl.stat"; -my $RSS_URL = "http://www.btlonline.org/rss/btl128.xml"; -my $PV_ID = '221'; - -binmode(STDIN, ":utf8"); -binmode(STDOUT, ":utf8"); -binmode(STDERR, ":utf8"); - -if($#ARGV >= 0 && $ARGV[0] eq 'last') { - print "!!!This is the last attempt, there won't be a retry on error!!!\n" -} - -my $user = `/usr/bin/id -un`; -$user =~ s/\n//; -my $group = "betweenlin"; - -my ($dbh, $errorstring) = RHRD::rddb::opendb(); -if(!defined $dbh) { - print "$errorstring\n"; - exit 1; -} -my @allowed_dbs = RHRD::rddb::get_dropboxes($dbh, $user, $group); -if(!defined $allowed_dbs[0] && defined $allowed_dbs[2]) { - print "$allowed_dbs[2]\n"; - exit 1; -} - -if(scalar(@allowed_dbs) != 1) { - print "found more or less than one Dropbox for this group?!\n"; - RHRD::rddb::closedb($dbh); - exit 1; -} -my $dropbox = $allowed_dbs[0]->{'PATH'}; -my $to_cart = $allowed_dbs[0]->{'TO_CART'}; - - -my @today = Date::Calc::Today(); -my @import_date = Date::Calc::Add_Delta_Days(@today, 7); -@import_date = Date::Calc::Standard_to_Business(@import_date); -$import_date[2] = 2; -my @broadcast_date = @import_date; -$broadcast_date[2] = 5; - -@import_date = Date::Calc::Business_to_Standard(@import_date); -@broadcast_date = Date::Calc::Business_to_Standard(@broadcast_date); - -print "today: " . Date::Calc::Date_to_Text(@today) . "\n"; -print "day of next Radio Helsinki broadcast: " . Date::Calc::Date_to_Text(@import_date) . "\n"; -print "ending of current broadcast cycle: " . Date::Calc::Date_to_Text(@broadcast_date) . "\n"; - -my $id = sprintf("%04d-%02d-%02d", @import_date); -my $bd = sprintf("%04d-%02d-%02d", @broadcast_date); -my $bdfile = sprintf("%02d%02d%02d-btlv128\.mp3", $broadcast_date[0]%100, $broadcast_date[1], $broadcast_date[2]); - -print "looking for files like '$bdfile' in RSS Feed\n"; -print " -> $RSS_URL\n"; - -my $feed = XML::Feed->parse(URI->new($RSS_URL)) - or die "Error fetching feed: " . XML::Feed->errstr; - -my $file = ""; -my $out_file = ""; -for my $entry ($feed->entries) { - if($entry->enclosure && ($entry->enclosure->type eq "audio/mpeg" || entry->enclosure->type eq "audio/mp3")) { - next unless $entry->enclosure->url =~ /$bdfile/; - - my $sum_title = decode_entities($entry->title); - $sum_title =~ s/ \(128 kbps\)$//; - my $sum_text = decode_entities($entry->content->body); - - my $ff = File::Fetch->new(uri => $entry->enclosure->url); - my $current_file = `cat $STAT_FILE`; - if($current_file eq $ff->output_file) { - print "Already downloaded file of today\n"; - RHRD::rddb::closedb($dbh); - exit 0; - } - $out_file = $ff->output_file; - if(!rhimport::check_file_extension($out_file)) { - print "\n\nThe extension of the matching file '". $out_file . "' seems to be wrong - manual import necessary!!!\n"; - RHRD::rddb::closedb($dbh); - exit 1; - } - - print $bdfile . ": downloading " . $entry->enclosure->url . " (" . $entry->enclosure->length . " Bytes) .. "; - $file = $ff->fetch( to => '/tmp' ) or die $ff->error; - print "ok\n"; - - print "summary:\n" . $sum_title . "\n\n" . $sum_text . "\n"; - - rhimport::pv_add_note($sum_title, $sum_text, $PV_ID, $id, "1"); - - last; - } -} -if($file eq "" || !(-e "$file")) { - print "No Entry found for " . Date::Calc::Date_to_Text(@broadcast_date) . " or download error - "; - if($#ARGV >= 0 && $ARGV[0] eq 'last') { - print "giving up, manual import necessary!!!\n"; - } else { - print "will retry later\n"; - } - exit 1; -} - -print "will import $file to dropbox $dropbox (cart=$to_cart)\n"; - -my $error_cb = sub { - my ($text) = @_; - - print "\n$text"; - return 0; -}; - -rhimport::check_key_file() or die "Import Key not found, use rhimport-create-id to create one\n"; - -my $ret; -my $log = rhimport::clear_carts($dbh, $group, $to_cart); -my $import_log; -($ret, $import_log) = rhimport::import_single($file, $dropbox, $user, 0, $error_cb); -$log .= $import_log; - -RHRD::rddb::closedb($dbh); - -unlink $file; - -if(!$ret) { - print "\nImport Error:\n"; - print $log; - exit 1; -} - -unlink($STAT_FILE); -open(my $fhs, '>', $STAT_FILE); -print $fhs "$out_file"; -close($fhs); - -exit 0; diff --git a/rhimport-dn b/rhimport-dn deleted file mode 100755 index 8362f4b..0000000 --- a/rhimport-dn +++ /dev/null @@ -1,239 +0,0 @@ -#!/usr/bin/perl -w -# -# -# rhimport -# -# Copyright (C) 2009-2015 Christian Pointner <equinox@helsinki.at> -# -# This file is part of rhimport. -# -# rhimport is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# any later version. -# -# rhimport is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with rhimport. If not, see <http://www.gnu.org/licenses/>. -# - -use strict; -use Date::Calc; -use Time::localtime; -use XML::Feed; -use XML::Feed::Entry; -use XML::Feed::Content; -use XML::Feed::Enclosure; -use URI::URL; -use IO::Handle; -use IPC::Open3; -use RHRD::rddb; - -use lib '/usr/local/share/rhimport/'; -use rhimport; - -my $STAT_FILE = $ENV{'HOME'} . "/rhimport-dn.stat"; -my $FILES_RSS_URL = "http://www.democracynow.org/podcast-stations.xml"; -my $HEADLINES_RSS_URL = "http://www.democracynow.org/podcast.xml"; -my $IGNORE_HEADLINE = 0; -my $PV_ID = '111'; - -binmode(STDIN, ":utf8"); -binmode(STDOUT, ":utf8"); -binmode(STDERR, ":utf8"); - -if($#ARGV >= 0 && $ARGV[0] eq 'last') { - print "!!!This is the last attempt, there won't be a retry on error!!!\n" -} - -if($#ARGV >= 0 && $ARGV[0] eq 'noheadline') { - $IGNORE_HEADLINE = 1; - $STAT_FILE = $ENV{'HOME'} . "/rhimport-dn-noheadline.stat"; -} - -my $user = `/usr/bin/id -un`; -$user =~ s/\n//; -my $group = "democracyn"; - -my ($dbh, $errorstring) = RHRD::rddb::opendb(); -if(!defined $dbh) { - print "$errorstring\n"; - exit 1; -} -my @allowed_dbs = RHRD::rddb::get_dropboxes($dbh, $user, $group); -if(!defined $allowed_dbs[0] && defined $allowed_dbs[2]) { - print "$allowed_dbs[2]\n"; - exit 1; -} - -my $dropbox = ""; -my $to_cart = 0; - -my @today = Date::Calc::Today(); -my @yesterday = Date::Calc::Add_Delta_Days($today[0], $today[1], $today[2], -1); -my @tomorrow = Date::Calc::Add_Delta_Days($today[0], $today[1], $today[2], 1); - -my @broadcast_day = @yesterday; -my @import_day = @today; -if(localtime->hour >= 12) { - print "It's past noon, assuming to import file from today\n"; - @broadcast_day = @today; - @import_day = @tomorrow; -} - -print "day of original broadcast: " . Date::Calc::Date_to_Text(@broadcast_day) . "\n"; -print "day of Radio Helsinki broadcast: " . Date::Calc::Date_to_Text(@import_day) . "\n"; - -for my $href ( @allowed_dbs ) { - my $dow = Date::Calc::Day_of_Week(@import_day); - if ($dow == int(substr($href->{'NAME'},0,2))) { - $dropbox = $href->{'PATH'}; - $to_cart = $href->{'TO_CART'}; - } -} - -if($dropbox eq "") { - print "no dropbox for day in question\n"; - RHRD::rddb::closedb($dbh); - exit 0; -} - -my $y = sprintf("%04d-%02d-%02d", $broadcast_day[0], $broadcast_day[1], $broadcast_day[2]); -my $yc = sprintf("%04d-%02d%02d", $broadcast_day[0], $broadcast_day[1], $broadcast_day[2]); - -print "looking for files from date $y in RSS Feed\n"; -print " -> $FILES_RSS_URL\n"; - -my $files_feed = XML::Feed->parse(URI->new($FILES_RSS_URL)) - or die "Error fetching feed: " . XML::Feed->errstr; - -my $file = ""; -my $out_file = ""; -for my $entry ($files_feed->entries) { - if($entry->enclosure && $entry->enclosure->type eq "audio/mpeg") { - if($entry->enclosure->url =~ /^https?:\/\/.*\.democracynow\.org\/dn([0-9-]+)-1\.mp3$/) { - next if($yc ne $1); - - my $url = new URI::URL($entry->enclosure->url); - my @path = $url->path_components; - - my $current_file = `cat $STAT_FILE`; - if($current_file eq $path[-1]) { - print "Already downloaded file of day in question\n"; - RHRD::rddb::closedb($dbh); - exit 0; - } - $out_file = $path[-1]; - if(!rhimport::check_file_extension($out_file)) { - print "\n\nThe extension of the matching file '". $out_file . "' seems to be wrong - manual import necessary!!!\n"; - RHRD::rddb::closedb($dbh); - exit 1; - } - - print $1 . ": downloading " . $entry->enclosure->url . " (" . $entry->enclosure->length . " Bytes) .. "; - - $file = "/tmp/" . $out_file; - system("wget", "--quiet", "--no-check-certificate", $url, "-O", $file); - if( $! != 0) { - die "wget returned with error: " . $!; - } - print "ok\n"; - - last; - } - } -} -if($file eq "" || !(-e "$file")) { - print "No Entry found from day in qeustion or download error - "; - if($#ARGV >= 0 && $ARGV[0] eq 'last') { - print "giving up, manual import necessary!!!\n"; - } else { - print "will retry later\n"; - } - exit 1; -} - - -unless($IGNORE_HEADLINE) { - print "\nlooking for headlines from date $y in RSS Feed\n"; - print " -> $HEADLINES_RSS_URL\n"; - - my $headlines_feed = XML::Feed->parse(URI->new($HEADLINES_RSS_URL)) - or die "Error fetching feed: " . XML::Feed->errstr; - - for my $entry ($headlines_feed->entries) { - if($entry->enclosure && $entry->enclosure->type eq "audio/mpeg") { - my $sum_title = $entry->title; - my $sum_text = $entry->summary->body . "\n<br />\n<a href=\"" . $entry->link . "\">" . $entry->link . "</a>\n"; - - if($entry->id =~ /tag:democracynow.org,([0-9-]+):media\/mp3download\/[^\/]+/) { - next if($y ne $1); - - print "summary:\n" . $sum_title . "\n\n" . $sum_text . "\n"; - rhimport::pv_add_note($sum_title, $sum_text, $PV_ID, sprintf("%04d-%02d-%02d", @import_day), "1", 0); - print "\n"; - rhimport::pv_add_note($sum_title, $sum_text, $PV_ID, sprintf("%04d-%02d-%02d", @import_day), "1", 1); - print "\n"; - - last; - } - } - } -} else { - print "\nignoring headlines for they are not available by now\n"; - my $dow_string = Date::Calc::Day_of_Week_to_Text(Date::Calc::Day_of_Week(@broadcast_day)); - my $sum_title = sprintf("Democracy Now! %04d-%02d-%02d %s", @broadcast_day, $dow_string); - my $sum_text = ""; - print "summary:\n" . $sum_title . "\n\n" . $sum_text . "\n"; - rhimport::pv_add_note($sum_title, $sum_text, $PV_ID, sprintf("%04d-%02d-%02d", @broadcast_day), "1", 2); - print "\n"; -} - -print "converting $file ... "; -my $infile = $file; -$file =~ s/\.mp3$/.wav/; -system("ffmpeg", "-i", $infile, "-acodec", "pcm_s16le", $file); -if( $! != 0) { - die "ffmpeg returned with error: " . $!; -} -print "ok\n"; -unlink $infile; - -print "will import $file to dropbox $dropbox (cart=$to_cart)\n\n"; - -my $error_cb = sub { - my ($text) = @_; - - print "\n$text"; - return 0; -}; - -rhimport::check_key_file() or die "Import Key not found, use rhimport-create-id to create one\n"; - -my $ret; -my $log; -$log = rhimport::clear_carts($dbh, $group, $to_cart); -my $import_log; -($ret, $import_log) = rhimport::import_single($file, $dropbox, $user, 0, $error_cb); -$log .= $import_log; - -RHRD::rddb::closedb($dbh); - -unlink $file; - -if(!$ret) { - print "\nImport Error:\n"; - print $log; - exit 1; -} - -unlink($STAT_FILE); -open(my $fhs, '>', $STAT_FILE); -print $fhs $out_file; -close($fhs); - -exit 0; diff --git a/rhimport-fe b/rhimport-fe deleted file mode 100755 index c396f18..0000000 --- a/rhimport-fe +++ /dev/null @@ -1,204 +0,0 @@ -#!/usr/bin/perl -w -# -# -# rhimport -# -# Copyright (C) 2009-2015 Christian Pointner <equinox@helsinki.at> -# -# This file is part of rhimport. -# -# rhimport is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# any later version. -# -# rhimport is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with rhimport. If not, see <http://www.gnu.org/licenses/>. -# - -use strict; -use Date::Calc; -use XML::Feed; -use XML::Feed::Entry; -use XML::Feed::Content; -use XML::Feed::Enclosure; -use File::Fetch; -use IO::Handle; -use IPC::Open3; -use HTML::Entities; -use File::Touch; -use RHRD::rddb; - -use lib '/usr/local/share/rhimport/'; -use rhimport; - -my $STAT_FILE = $ENV{'HOME'} . "/rhimport-fe.stat"; -my $NOTE_FILE = $ENV{'HOME'} . "/rhimport-fe.last_note"; -my $NEW_FILE = $ENV{'HOME'} . "/rhimport-fe.is_new"; -my $RSS_URL = "http://freie-radios.net/portal/podcast.php?serie=208&rss"; -#my $PV_ID = '359'; - -binmode(STDIN, ":utf8"); -binmode(STDOUT, ":utf8"); -binmode(STDERR, ":utf8"); - -if($#ARGV >= 0 && $ARGV[0] eq 'last') { - print "!!!This is the last attempt, there won't be a retry on error!!!\n" -} - -my @today = Date::Calc::Today(); -my $dow = Date::Calc::Day_of_Week(@today); -my @tomorrow = Date::Calc::Add_Delta_Days($today[0], $today[1], $today[2], 1); - -my @tmp = Date::Calc::Standard_to_Business(@today); -$tmp[2] = 1; -my @this_mon = Date::Calc::Business_to_Standard(@tmp); -my @next_mon = Date::Calc::Add_Delta_Days($this_mon[0], $this_mon[1], $this_mon[2], 7); - -my @broadcast_day = @today; -my @import_day = @tomorrow; -if ($dow == 5) { - @import_day = @next_mon; -} - -print "day of original broadcast: " . Date::Calc::Date_to_Text(@broadcast_day) . "\n"; -print "day of Radio Helsinki broadcast: " . Date::Calc::Date_to_Text(@import_day) . "\n"; - -my $user = `/usr/bin/id -un`; -$user =~ s/\n//; -my $group = "vonunten"; -$dow = Date::Calc::Day_of_Week(@import_day); -$group .= "Mo" if($dow == 1); -$group .= "Di" if($dow == 2); -$group .= "Mi" if($dow == 3); -$group .= "Do" if($dow == 4); -$group .= "Fr" if($dow == 5); - -my ($dbh, $errorstring) = RHRD::rddb::opendb(); -if(!defined $dbh) { - print "$errorstring\n"; - exit 1; -} -my @allowed_dbs = RHRD::rddb::get_dropboxes($dbh, $user, $group); -if(!defined $allowed_dbs[0] && defined $allowed_dbs[2]) { - print "$allowed_dbs[2]\n"; - exit 1; -} -my $dropbox = ""; -my $to_cart = 0; - -if(scalar(@allowed_dbs) == 0) { - print "no dropbox for day in question\n"; - RHRD::rddb::closedb($dbh); - exit 0; -} -if(scalar(@allowed_dbs) != 1) { - print "found more or less than one Dropbox for this group?!\n"; - RHRD::rddb::closedb($dbh); - exit 1; -} -$dropbox = $allowed_dbs[0]->{'PATH'}; -$to_cart = $allowed_dbs[0]->{'TO_CART'}; - -my $regexp = sprintf('%04d%02d%02d.*\.mp3', $broadcast_day[0], $broadcast_day[1], $broadcast_day[2]); -print "looking for file from " . Date::Calc::Date_to_Text(@broadcast_day) . " in RSS Feed\n"; -print " -> $RSS_URL\n"; - -my $feed = XML::Feed->parse(URI->new($RSS_URL)) - or die "Error fetching feed: " . XML::Feed->errstr; - -my $file = ""; -my $out_file = ""; -for my $entry ($feed->entries) { - if($entry->enclosure && $entry->enclosure->type eq "audio/mpeg") { - next unless $entry->enclosure->url =~ /$regexp/; - next unless $entry->enclosure->length >= (20*1024*1024); - - my $sum_title = decode_entities($entry->title); - my $sum_text = decode_entities($entry->content->body); - - my $ff = File::Fetch->new(uri => $entry->enclosure->url); - my $current_file = `cat $STAT_FILE`; - if($current_file eq $ff->output_file) { - print "Already downloaded file of today\n"; - RHRD::rddb::closedb($dbh); - exit 0; - } - $out_file = $ff->output_file; - if(!rhimport::check_file_extension($out_file)) { - print "\n\nThe extension of the matching file '". $out_file . "' seems to be wrong - manual import necessary!!!\n"; - RHRD::rddb::closedb($dbh); - exit 1; - } - - print $regexp . ": downloading " . $entry->enclosure->url . " (" . $entry->enclosure->length . " Bytes) .. "; - $file = $ff->fetch( to => '/tmp' ) or die $ff->error; - print "ok\n"; - - print "summary:\n" . $sum_title . "\n\n" . $sum_text . "\n"; - - # if($#ARGV >= 0 && $ARGV[0] eq 'nopv') { - # print "not adding note to PV\n"; - # } else { - # rhimport::pv_add_note($sum_title, $sum_text, $PV_ID, sprintf("%04d-%02d-%02d", $import_day[0], $import_day[1], $import_day[2]), "1"); - # } - # print "\n"; - - unlink($NOTE_FILE); - open(my $fhs, '>', $NOTE_FILE); - binmode($fhs, ":utf8"); - print $fhs $sum_title . "\n\n" . $sum_text . "\n"; - close($fhs); - - last; - } -} -if($file eq "" || !(-e "$file")) { - print "No Entry found for " . Date::Calc::Date_to_Text(@broadcast_day) . " or download error - "; - if($#ARGV >= 0 && $ARGV[0] eq 'last') { - print "giving up, manual import necessary!!!\n"; - } else { - print "will retry later\n"; - } - exit 1; -} - -print "will import $file to dropbox $dropbox (cart=$to_cart)\n"; - -my $error_cb = sub { - my ($text) = @_; - - print "\n$text"; - return 0; -}; - -rhimport::check_key_file() or die "Import Key not found, use rhimport-create-id to create one\n"; - -my $ret; -my $log = rhimport::clear_carts($dbh, $group, $to_cart); -my $import_log; -($ret, $import_log) = rhimport::import_single($file, $dropbox, $user, 0, $error_cb); -$log .= $import_log; - -RHRD::rddb::closedb($dbh); - -unlink $file; - -if(!$ret) { - print "\nImport Error:\n"; - print $log; - exit 1; -} - -unlink($STAT_FILE); -open(my $fhs, '>', $STAT_FILE); -print $fhs $out_file; -close($fhs); -touch($NEW_FILE); - -exit 0; diff --git a/rhimport-fw b/rhimport-fw deleted file mode 100755 index ef8b159..0000000 --- a/rhimport-fw +++ /dev/null @@ -1,219 +0,0 @@ -#!/usr/bin/perl -w -# -# -# rhimport -# -# Copyright (C) 2009-2015 Christian Pointner <equinox@helsinki.at> -# -# This file is part of rhimport. -# -# rhimport is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# any later version. -# -# rhimport is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with rhimport. If not, see <http://www.gnu.org/licenses/>. -# - -use strict; -use Date::Calc; -use XML::Feed; -use XML::Feed::Entry; -use XML::Feed::Content; -use XML::Feed::Enclosure; -use File::Fetch; -use IO::Handle; -use IPC::Open3; -use HTML::Entities; -use RHRD::rddb; - -use lib '/usr/local/share/rhimport/'; -use rhimport; - -my $STAT_FILE = $ENV{'HOME'} . "/rhimport-fw.stat"; -my $LAST_FILE = $ENV{'HOME'} . "/rhimport-fw.last"; -my $RSS_URL = "http://cba.fro.at/seriesrss/263965?c=Kfs2IoV2Wmd"; -$XML::Feed::MULTIPLE_ENCLOSURES=1; -my $PV_ID = '382'; - -binmode(STDIN, ":utf8"); -binmode(STDOUT, ":utf8"); -binmode(STDERR, ":utf8"); - -if($#ARGV >= 0 && $ARGV[0] eq 'last') { - print "!!!This is the last attempt, there won't be a retry on error!!!\n" -} - -my $user = `/usr/bin/id -un`; -$user =~ s/\n//; -my $group = "freiewelle"; - -my ($dbh, $errorstring) = RHRD::rddb::opendb(); -if(!defined $dbh) { - print "$errorstring\n"; - exit 1; -} -my @allowed_dbs = RHRD::rddb::get_dropboxes($dbh, $user, $group); -if(!defined $allowed_dbs[0] && defined $allowed_dbs[2]) { - print "$allowed_dbs[2]\n"; - exit 1; -} - -my $idx = 0; -my $idx_reb = 1; -if(scalar(@allowed_dbs) != 2) { - print "found more or less than 2 Dropboxes for this group?!\n"; - RHRD::rddb::closedb($dbh); - exit 1; -} -my $dropbox = $allowed_dbs[$idx]->{'PATH'}; -my $to_cart = $allowed_dbs[$idx]->{'TO_CART'}; -my $dropbox_reb = $allowed_dbs[$idx_reb]->{'PATH'}; -my $to_cart_reb = $allowed_dbs[$idx_reb]->{'TO_CART'}; - -my @import_date = Date::Calc::Standard_to_Business(Date::Calc::Today()); -if($import_date[2] != 1) { - $import_date[2] = 1; - @import_date = Date::Calc::Business_to_Standard(@import_date); - @import_date = Date::Calc::Add_Delta_Days(@import_date, 7); -} else { - @import_date = Date::Calc::Business_to_Standard(@import_date); -} -my @import_date_reb = Date::Calc::Add_Delta_Days(@import_date, 4); - -print "day of next broadcast: " . Date::Calc::Date_to_Text(@import_date) . "\n"; -print "day of next rebroadcast: " . Date::Calc::Date_to_Text(@import_date_reb) . "\n"; - -my $id = sprintf("%04d-%02d-%02d", @import_date); -my $bnum = `cat $LAST_FILE`; -$bnum += 1; - -my $current_stat = `cat $STAT_FILE`; -my ($current_id, $current_file) = $current_stat =~ m/^(.*)\n(.*)/; -if($current_id eq $id) { - print "Already downloaded current file\n"; - RHRD::rddb::closedb($dbh); - exit 0; -} - -print "looking for file with number $bnum in RSS Feed\n"; -print " -> $RSS_URL\n"; - -my $feed = XML::Feed->parse(URI->new($RSS_URL)) - or die "Error fetching feed: " . XML::Feed->errstr; - -my $bdexp = sprintf("^(\\d+)", $bnum); - - -my $file = ""; -my $out_file = ""; -my $i = 0; -for my $entry ($feed->entries) { - $i++; - next unless $entry->enclosure; - my $j = 0; - for my $enclosure($entry->enclosure) { - $j++; - if($enclosure->type eq "audio/mpeg" || $enclosure->type eq "audio/ogg") { - print "$i/$j: (" . $enclosure->type . ", " . $enclosure->length . ") " . $enclosure->url . "\n"; - my $url = $enclosure->url; - if($url =~ /^(.*)_cut(\.[^\.]+)$/) { - $url = $1 . $2; - } - - my $ff = File::Fetch->new(uri => $url); - next unless (uc($ff->output_file) =~ /$bdexp/); # file not from correct day - next unless ($bnum == $1); - - my $sum_title = decode_entities($entry->title); - my $sum_text = decode_entities($entry->content->body); - - my $current_file = `cat $STAT_FILE`; - if($current_file eq $ff->output_file) { - print "Already downloaded current file\n"; - RHRD::rddb::closedb($dbh); - exit 0; - } - $out_file = $ff->output_file; - if(!rhimport::check_file_extension($out_file)) { - print "\n\nThe extension of the matching file '". $out_file . "' seems to be wrong - manual import necessary!!!\n"; - RHRD::rddb::closedb($dbh); - exit 1; - } - - print " --> " . $bnum . ": downloading " . $url . " (" . $enclosure->length . " Bytes) .. "; - $file = $ff->fetch( to => '/tmp' ) or die $ff->error; - print "ok\n"; - - print "summary:\n" . $sum_title . "\n\n" . $sum_text . "\n"; - - rhimport::pv_add_note($sum_title, $sum_text, $PV_ID, sprintf("%04d-%02d-%02d", $import_date[0], $import_date[1], $import_date[2]), "1"); - print "\n"; - rhimport::pv_add_note($sum_title, $sum_text, $PV_ID, sprintf("%04d-%02d-%02d", $import_date_reb[0], $import_date_reb[1], $import_date_reb[2]), "2"); - print "\n"; - - last; - } - } - last if $file ne ""; -} - -if($file eq "" || !(-e "$file")) { - print "No Entry found or download error - "; - if($#ARGV >= 0 && $ARGV[0] eq 'last') { - print "giving up, manual import necessary!!!\n"; - } else { - print "will retry later\n"; - } - exit 1; -} - -print "will import $file to dropbox $dropbox (cart=$to_cart)\n"; -print "will import $file to rebroadcast dropbox $dropbox_reb (cart=$to_cart_reb)\n\n"; - -my $error_cb = sub { - my ($text) = @_; - - print "\n$text"; - return 0; -}; - -rhimport::check_key_file() or die "Import Key not found, use rhimport-create-id to create one\n"; - -my $ret; -my $log = rhimport::clear_carts($dbh, $group, $to_cart); -my $import_log; -($ret, $import_log) = rhimport::import_single($file, $dropbox, $user, 0, $error_cb); -$log .= $import_log; -$import_log = rhimport::clear_carts($dbh, $group, $to_cart_reb); -$log .= $import_log; -($ret, $import_log) = rhimport::import_single($file, $dropbox_reb, $user, 0, $error_cb); -$log .= $import_log; - -RHRD::rddb::closedb($dbh); - -unlink $file; - -if(!$ret) { - print "\nImport Error:\n"; - print $log; - exit 1; -} - -unlink($STAT_FILE); -open(my $fhs, '>', $STAT_FILE); -print $fhs "$id\n$out_file"; -close($fhs); - -unlink($LAST_FILE); -open($fhs, '>', $LAST_FILE); -print $fhs "$bnum"; -close($fhs); - -exit 0; diff --git a/rhimport-mz b/rhimport-mz deleted file mode 100755 index 29b355a..0000000 --- a/rhimport-mz +++ /dev/null @@ -1,247 +0,0 @@ -#!/usr/bin/perl -w -# -# -# rhimport -# -# Copyright (C) 2009-2015 Christian Pointner <equinox@helsinki.at> -# -# This file is part of rhimport. -# -# rhimport is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# any later version. -# -# rhimport is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with rhimport. If not, see <http://www.gnu.org/licenses/>. -# - -use strict; -use Switch; -use Date::Calc; -use Time::localtime; -use IO::Handle; -use IPC::Open3; -use File::Slurp; -use RHRD::rddb; - -use lib '/usr/local/share/rhimport/'; -use rhimport; - - -my @STATIC_FILES = ($ENV{'HOME'} . "/frontex", $ENV{'HOME'} . "/maribor" ); - -my $STAT_FILE = $ENV{'HOME'} . "/rhimport-mz.stat"; -my $ZF_NOTE_FILE = $ENV{'HOME'} . "/rhimport-zf.last_note"; -my $ZF_NEW_FILE = $ENV{'HOME'} . "/rhimport-zf.is_new"; -my $PV_ID = '352'; - -binmode(STDIN, ":utf8"); -binmode(STDOUT, ":utf8"); -binmode(STDERR, ":utf8"); - -if($#ARGV >= 0 && $ARGV[0] eq 'last') { - print "!!!This is the last attempt, there won't be a retry on error!!!\n" -} - -my $user = `/usr/bin/id -un`; -$user =~ s/\n//; - -my @today = Date::Calc::Today(); -my @next_day = Date::Calc::Add_Delta_Days(@today, 1); -my $dow = Date::Calc::Day_of_Week(@today); -if($dow == 5) { - @next_day = Date::Calc::Add_Delta_Days(@today, 3); -} - -print "today: " . Date::Calc::Date_to_Text(@today) . "\n"; -print "next day would be: " . Date::Calc::Date_to_Text(@next_day) . "\n"; -my @import_day = @next_day; -if(localtime->hour < 12) { - print "It's forenoon, assuming to import for today\n"; - @import_day = @today; -} -print "day of broadcast: " . Date::Calc::Date_to_Text(@import_day) . "\n"; - - -$dow = Date::Calc::Day_of_Week(@import_day); -my $group = ""; -switch($dow) { - case 1 { $group = "mahlzeitMo" } - case 2 { $group = "mahlzeitDi" } - case 3 { $group = "mahlzeitMi" } - case 4 { $group = "mahlzeitDo" } - case 5 { $group = "mahlzeitFr" } - else { print("invalid day of week?!\n"); exit 1; } -} - -my ($dbh, $errorstring) = RHRD::rddb::opendb(); -if(!defined $dbh) { - print "$errorstring\n"; - exit 1; -} -my @allowed_dbs = RHRD::rddb::get_dropboxes($dbh, $user, $group); -if(!defined $allowed_dbs[0] && defined $allowed_dbs[2]) { - print "$allowed_dbs[2]\n"; - exit 1; -} - -if(scalar(@allowed_dbs) != 1) { - print "found no or more than one Dropboxes for this group?!\n"; - RHRD::rddb::closedb($dbh); - exit 1; -} -my $dropbox = $allowed_dbs[0]->{'PATH'}; - - -my $import_date = sprintf("%04d-%02d-%02d", @import_day); -my $current_date = `cat $STAT_FILE`; -if($current_date eq $import_date) { - print "Already downloaded file of day in question\n"; - RHRD::rddb::closedb($dbh); - exit 0; -} - -unless(-e $ZF_NEW_FILE) { - print "zip-fm not imported yet - "; - if($#ARGV >= 0 && $ARGV[0] eq 'last') { - print "giving up, rebroadcasting last show"; - } else { - print "will retry later\n"; - exit 0, - } -} - - -sub read_first_line { - my ($file) = @_; - my @lines = read_file($file, binmode => ':utf8', err_mode => 'quiet'); - return "" unless ($lines[0]); - - chomp($lines[0]); - return $lines[0]; -}; - -sub get_note { - my ($file) = @_; - my @lines = read_file($file, binmode => ':utf8', err_mode => 'quiet'); - return "" unless ($lines[0]); - - chomp($lines[0]); - $lines[0] = "<p><strong>$lines[0]</strong></p>\n"; - return join('',@lines); -}; - -sub find_next_file { - my ($dir, $current) = @_; - - my %files = (); - opendir(my $dh, $dir) or die $!; - while (my $file = readdir($dh)) { - next if ($file =~ m/^\./); - next if ($file =~ m/_notes$/); - $files{$file} = 1; - } - closedir($dh); - - my $num = scalar keys %files; - foreach my $file (sort keys %files) { - if($file gt $current) { - return $num, $file; - } - $num--; - } - - return 0, ""; -}; - -my $sum_title = "zip-fm (WH)"; -my $sum_text = get_note($ZF_NOTE_FILE) . "<br/>\n"; - -print "\nsearching local files:\n"; - -my @files = (); -my %next_files = (); -foreach my $dir (@STATIC_FILES) { - my $file = read_first_line("$dir/.current"); - if($file eq "") { - print " $dir: skipped !!! (no .current file or no more files)\n"; - next; - } - my ($remaining, $next_file) = find_next_file($dir, $file); - if($remaining >= 1) { - print " $dir: adding $file\n (next: $next_file, $remaining files remaining)\n"; - } else { - print " $dir: adding $file\n (this was the last file!!!!)\n"; - } - $next_files{$dir} = $next_file; - - $sum_title = "$sum_title, " . read_first_line("$dir/.sum_title"); - $sum_text = $sum_text . "<br />\n" . get_note("$dir/$file" . "_notes"); - foreach my $ext ("flac", "wav", "ogg", "mp3") { - push(@files, "$dir/.jingle.$ext") if(-e "$dir/.jingle.$ext"); - } - push(@files, "$dir/$file"); -} -$sum_text = $sum_text . "<br/><br/>\n"; - -print "\nsummary:\n" . $sum_title . "\n\n" . $sum_text . "\n"; -rhimport::pv_add_note($sum_title, $sum_text, $PV_ID, sprintf("%04d-%02d-%02d", @import_day), "1"); -print "\n"; - - -print "will import " . join(", ", @files) . " to dropbox $dropbox\n"; - - -my $error_cb = sub { - my ($text) = @_; - - print "\n$text"; - return 0; -}; - -rhimport::check_key_file() or die "Import Key not found, use rhimport-create-id to create one\n"; - -my $ret = 1; -my $log = rhimport::clear_carts($dbh, $group, 0); -my ($cart, $high_cart) = rhimport::get_cart_range($dbh, $group); -foreach my $file (@files) { - if($cart > $high_cart) { - print "not enough carts - will ignore remaining files!!!"; - last; - } - print " importing $file to cart $cart\n"; - my $import_log; - ($ret, $import_log) = rhimport::import_single($file, $dropbox, $user, 0, $error_cb); - $log .= $import_log; - last unless $ret; - - $cart++; -} - -RHRD::rddb::closedb($dbh); - -if(!$ret) { - print "\nImport Error:\n"; - print $log; - exit 1; -} - -unlink($STAT_FILE); -open(my $fhs, '>', $STAT_FILE); -print $fhs $import_date; -close($fhs); - -unlink($ZF_NEW_FILE); -foreach my $dir (@STATIC_FILES) { - open(my $fhs, '>', "$dir/.current"); - print $fhs $next_files{$dir} if ($next_files{$dir}); - close($fhs); -} - -exit 0; diff --git a/rhimport-nw b/rhimport-nw deleted file mode 100755 index 8adea51..0000000 --- a/rhimport-nw +++ /dev/null @@ -1,190 +0,0 @@ -#!/usr/bin/perl -w -# -# -# rhimport -# -# Copyright (C) 2009-2015 Christian Pointner <equinox@helsinki.at> -# -# This file is part of rhimport. -# -# rhimport is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# any later version. -# -# rhimport is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with rhimport. If not, see <http://www.gnu.org/licenses/>. -# - -use strict; -use Date::Calc; -use XML::Feed; -use XML::Feed::Entry; -use XML::Feed::Content; -use XML::Feed::Enclosure; -use File::Fetch; -use IO::Handle; -use IPC::Open3; -use HTML::Entities; -use RHRD::rddb; - -use lib '/usr/local/share/rhimport/'; -use rhimport; - -my $STAT_FILE = $ENV{'HOME'} . "/rhimport-nw.stat"; -my $RSS_URL = "http://cba.fro.at/seriesrss/262514?c=Kfs2IoV2Wmd"; -$XML::Feed::MULTIPLE_ENCLOSURES=1; -my $PV_ID = '73'; - -binmode(STDIN, ":utf8"); -binmode(STDOUT, ":utf8"); -binmode(STDERR, ":utf8"); - -if($#ARGV >= 0 && $ARGV[0] eq 'last') { - print "!!!This is the last attempt, there won't be a retry on error!!!\n" -} - -my $user = `/usr/bin/id -un`; -$user =~ s/\n//; -my $group = "netwatcher"; - -my ($dbh, $errorstring) = RHRD::rddb::opendb(); -if(!defined $dbh) { - print "$errorstring\n"; - exit 1; -} -my @allowed_dbs = RHRD::rddb::get_dropboxes($dbh, $user, $group); -if(!defined $allowed_dbs[0] && defined $allowed_dbs[2]) { - print "$allowed_dbs[2]\n"; - exit 1; -} - -if(scalar(@allowed_dbs) != 1) { - print "found more or less than 1 Dropbox for this group?!\n"; - RHRD::rddb::closedb($dbh); - exit 1; -} -my $dropbox = $allowed_dbs[0]->{'PATH'}; -my $to_cart = $allowed_dbs[0]->{'TO_CART'}; - -my @today = Date::Calc::Today(); -my @import_date = @today; -if( Date::Calc::Day_of_Week(@today) > 3 ) { - @import_date = Date::Calc::Add_Delta_Days(@import_date, 7); -} -if( Date::Calc::Day_of_Week(@today) != 3 ) { - @import_date = Date::Calc::Standard_to_Business(@import_date); - $import_date[2] = 3; - @import_date = Date::Calc::Business_to_Standard(@import_date); -} -my @broadcast_day = Date::Calc::Add_Delta_Days(@import_date, -5); -print "day of original broadcast: " . Date::Calc::Date_to_Text(@broadcast_day) . "\n"; -print "day of Radio Helsinki broadcast: " . Date::Calc::Date_to_Text(@import_date) . "\n"; - -my $id = sprintf("%04d-%02d-%02d", @import_date); -my $bd = sprintf("%04d-%02d-%02d", @broadcast_day); -my $bdfile = sprintf("NETWATCHER%04d%02d%02d(.*)\.MP3", @broadcast_day); -print "looking for files from $bd in RSS Feed\n"; -print " -> $RSS_URL\n"; - -my $feed = XML::Feed->parse(URI->new($RSS_URL)) - or die "Error fetching feed: " . XML::Feed->errstr; - -my $file = ""; -my $out_file = ""; -my $i = 0; -for my $entry ($feed->entries) { - $i++; - next unless $entry->enclosure; - my $j = 0; - for my $enclosure($entry->enclosure) { - $j++; - if($enclosure->type eq "audio/mpeg" || $enclosure->type eq "audio/ogg") { - print "$i/$j: (" . $enclosure->type . ", " . $enclosure->length . ") " . $enclosure->url . "\n"; - my $url = $enclosure->url; - if($url =~ /^(.*)_cut(\.[^\.]+)$/) { - $url = $1 . $2; - } - - my $ff = File::Fetch->new(uri => $url); - next unless (uc($ff->output_file) =~ $bdfile); # file not from correct day - - my $sum_title = decode_entities($entry->title); - my $sum_text = decode_entities($entry->content->body); - - my $current_file = `cat $STAT_FILE`; - if($current_file eq $ff->output_file) { - print "Already downloaded file of today\n"; - RHRD::rddb::closedb($dbh); - exit 0; - } - $out_file = $ff->output_file; - if(!rhimport::check_file_extension($out_file)) { - print "\n\nThe extension of the matching file '". $out_file . "' seems to be wrong - manual import necessary!!!\n"; - RHRD::rddb::closedb($dbh); - exit 1; - } - - print " --> " . $bdfile . ": downloading " . $url . " (" . $enclosure->length . " Bytes) .. "; - $file = $ff->fetch( to => '/tmp' ) or die $ff->error; - print "ok\n"; - - print "summary:\n" . $sum_title . "\n\n" . $sum_text . "\n"; - - rhimport::pv_add_note($sum_title, $sum_text, $PV_ID, $id, "1"); - print "\n"; - - last; - } - } - last if $file ne ""; -} - -if($file eq "" || !(-e "$file")) { - print "No Entry found from $bd or download error - "; - if($#ARGV >= 0 && $ARGV[0] eq 'last') { - print "giving up, manual import necessary!!!\n"; - } else { - print "will retry later\n"; - } - exit 1; -} - -print "will import $file to dropbox $dropbox (cart=$to_cart)\n"; - -my $error_cb = sub { - my ($text) = @_; - - print "\n$text"; - return 0; -}; - -rhimport::check_key_file() or die "Import Key not found, use rhimport-create-id to create one\n"; - -my $ret; -my $log = rhimport::clear_carts($dbh, $group, $to_cart); -my $import_log; -($ret, $import_log) = rhimport::import_single($file, $dropbox, $user, 0, $error_cb); -$log .= $import_log; - -RHRD::rddb::closedb($dbh); - -unlink $file; - -if(!$ret) { - print "\nImport Error:\n"; - print $log; - exit 1; -} - -unlink($STAT_FILE); -open(my $fhs, '>', $STAT_FILE); -print $fhs $out_file; -close($fhs); - -exit 0; diff --git a/rhimport-o94n b/rhimport-o94n deleted file mode 100755 index 2f1fcec..0000000 --- a/rhimport-o94n +++ /dev/null @@ -1,209 +0,0 @@ -#!/usr/bin/perl -w -# -# -# rhimport -# -# Copyright (C) 2009-2015 Christian Pointner <equinox@helsinki.at> -# -# This file is part of rhimport. -# -# rhimport is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# any later version. -# -# rhimport is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with rhimport. If not, see <http://www.gnu.org/licenses/>. -# - -use strict; -use Date::Calc; -use XML::Feed; -use XML::Feed::Entry; -use XML::Feed::Content; -use XML::Feed::Enclosure; -use File::Fetch; -use IO::Handle; -use IPC::Open3; -use HTML::Entities; -use RHRD::rddb; - -use lib '/usr/local/share/rhimport/'; -use rhimport; - -my $STAT_FILE = $ENV{'HOME'} . "/rhimport-o94n.stat"; -my $RSS_URL = "http://cba.fro.at/seriesrss/264131?c=Kfs2IoV2Wmd"; -$XML::Feed::MULTIPLE_ENCLOSURES=1; -my $PV_ID = '430'; - -binmode(STDIN, ":utf8"); -binmode(STDOUT, ":utf8"); -binmode(STDERR, ":utf8"); - -if($#ARGV >= 0 && $ARGV[0] eq 'last') { - print "!!!This is the last attempt, there won't be a retry on error!!!\n" -} - -my $user = `/usr/bin/id -un`; -$user =~ s/\n//; -my $group = "o94"; - -my ($dbh, $errorstring) = RHRD::rddb::opendb(); -if(!defined $dbh) { - print "$errorstring\n"; - exit 1; -} -my @allowed_dbs = RHRD::rddb::get_dropboxes($dbh, $user, $group); -if(!defined $allowed_dbs[0] && defined $allowed_dbs[2]) { - print "$allowed_dbs[2]\n"; - exit 1; -} - - -my $dropbox = ''; -my $to_cart = 0; - -for my $db (@allowed_dbs) { - if($db->{'PATH'} =~ /^\/programm\/02/) { - $to_cart = $db->{'TO_CART'}; - $dropbox = $db->{'PATH'}; - last; - } -} -if($to_cart == 0) { - print "no dropbox found for Tuesday?!\n"; - RHRD::rddb::closedb($dbh); - exit 1; -} - - -my @today = Date::Calc::Today(); -my @yesterday = Date::Calc::Add_Delta_Days(@today, -1); -my @tomorrow = Date::Calc::Add_Delta_Days(@today, 1); -my $dow = Date::Calc::Day_of_Week(@today); - -my @broadcast_day; -my @import_date; -if($dow == 1) { - @broadcast_day = @today; - @import_date = @tomorrow; -} elsif ($dow == 2) { - @broadcast_day = @yesterday; - @import_date = @today; -} else { - print "Wrong Day! Check cron script!\n"; - exit 1; -} - -print "day of original broadcast: " . Date::Calc::Date_to_Text(@broadcast_day) . "\n"; -print "day of Radio Helsinki broadcast: " . Date::Calc::Date_to_Text(@import_date) . "\n"; - -my $id = sprintf("%04d-%02d-%02d", @import_date); -my $bd = sprintf("%04d-%02d-%02d", @broadcast_day); -my $bdfile = sprintf("^NACHRICHTEN%04d%02d%02d", @broadcast_day); - -print "looking for files from $bd in RSS Feed\n"; -print " -> $RSS_URL\n"; - -my $feed = XML::Feed->parse(URI->new($RSS_URL)) - or die "Error fetching feed: " . XML::Feed->errstr; - -my $file = ""; -my $out_file = ""; -my $i = 0; -for my $entry ($feed->entries) { - $i++; - next unless $entry->enclosure; - my $j = 0; - for my $enclosure($entry->enclosure) { - $j++; - if($enclosure->type eq "audio/mpeg" || $enclosure->type eq "audio/ogg") { - print "$i/$j: (" . $enclosure->type . ", " . $enclosure->length . ") " . $enclosure->url . "\n"; - my $url = $enclosure->url; - if($url =~ /^(.*)_cut(\.[^\.]+)$/) { - $url = $1 . $2; - } - - my $ff = File::Fetch->new(uri => $url); - next unless (uc($ff->output_file) =~ $bdfile); # file not from correct day - - my $sum_title = decode_entities($entry->title); - $sum_title =~ s/\s*\(ganze sendung\)//i; - my $sum_text = decode_entities($entry->content->body); - - my $current_file = `cat $STAT_FILE`; - if($current_file eq $ff->output_file) { - print "Already downloaded file of today\n"; - RHRD::rddb::closedb($dbh); - exit 0; - } - $out_file = $ff->output_file; - if(!rhimport::check_file_extension($out_file)) { - print "\n\nThe extension of the matching file '". $out_file . "' seems to be wrong - manual import necessary!!!\n"; - RHRD::rddb::closedb($dbh); - exit 1; - } - - print " --> " . $bdfile . ": downloading " . $url . " (" . $enclosure->length . " Bytes) .. "; - $file = $ff->fetch( to => '/tmp' ) or die $ff->error; - print "ok\n"; - - print "summary:\n" . $sum_title . "\n\n" . $sum_text . "\n"; - - rhimport::pv_add_note($sum_title, $sum_text, $PV_ID, sprintf("%04d-%02d-%02d", @import_date), "1"); - print "\n"; - - last; - } - } - last if $file ne ""; -} - -if($file eq "" || !(-e "$file")) { - print "No Entry found from $bd or download error - "; - if($#ARGV >= 0 && $ARGV[0] eq 'last') { - print "giving up, manual import necessary!!!\n"; - } else { - print "will retry later\n"; - } - exit 1; -} - -print "will import $file to dropbox $dropbox (cart=$to_cart)\n"; - -my $error_cb = sub { - my ($text) = @_; - - print "\n$text"; - return 0; -}; - -rhimport::check_key_file() or die "Import Key not found, use rhimport-create-id to create one\n"; - -my $ret; -my $log = rhimport::clear_carts($dbh, $group, $to_cart); -my $import_log; -($ret, $import_log) = rhimport::import_single($file, $dropbox, $user, 0, $error_cb); -$log .= $import_log; - -RHRD::rddb::closedb($dbh); - -unlink $file; - -if(!$ret) { - print "\nImport Error:\n"; - print $log; - exit 1; -} - -unlink($STAT_FILE); -open(my $fhs, '>', $STAT_FILE); -print $fhs $out_file; -close($fhs); - -exit 0; diff --git a/rhimport-oi b/rhimport-oi deleted file mode 100755 index 7b33422..0000000 --- a/rhimport-oi +++ /dev/null @@ -1,182 +0,0 @@ -#!/usr/bin/perl -w -# -# -# rhimport -# -# Copyright (C) 2009-2015 Christian Pointner <equinox@helsinki.at> -# -# This file is part of rhimport. -# -# rhimport is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# any later version. -# -# rhimport is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with rhimport. If not, see <http://www.gnu.org/licenses/>. -# - -use strict; -use Date::Calc; -use XML::Feed; -use XML::Feed::Entry; -use XML::Feed::Content; -use XML::Feed::Enclosure; -use File::Fetch; -use IO::Handle; -use IPC::Open3; -use HTML::Entities; -use RHRD::rddb; - -use lib '/usr/local/share/rhimport/'; -use rhimport; - -my $STAT_FILE = $ENV{'HOME'} . "/rhimport-oi.stat"; -my $LAST_FILE = $ENV{'HOME'} . "/rhimport-oi.last"; -my $RSS_URL = "http://npla.de/onda/feed/serie=3"; -my $PV_ID = '220'; - -binmode(STDIN, ":utf8"); -binmode(STDOUT, ":utf8"); -binmode(STDERR, ":utf8"); - -if($#ARGV >= 0 && $ARGV[0] eq 'last') { - print "!!!This is the last attempt, there won't be a retry on error!!!\n" -} - -my $user = `/usr/bin/id -un`; -$user =~ s/\n//; -my $group = "ondainfo"; - -my ($dbh, $errorstring) = RHRD::rddb::opendb(); -if(!defined $dbh) { - print "$errorstring\n"; - exit 1; -} -my @allowed_dbs = RHRD::rddb::get_dropboxes($dbh, $user, $group); -if(!defined $allowed_dbs[0] && defined $allowed_dbs[2]) { - print "$allowed_dbs[2]\n"; - exit 1; -} - -if(scalar(@allowed_dbs) != 1) { - print "found more or less than one Dropbox for this group?!\n"; - RHRD::rddb::closedb($dbh); - exit 1; -} -my $dropbox = $allowed_dbs[0]->{'PATH'}; -my $to_cart = $allowed_dbs[0]->{'TO_CART'}; - - -my @today = Date::Calc::Today(); -my @import_date = Date::Calc::Add_Delta_Days(@today, 7); -@import_date = Date::Calc::Standard_to_Business(@import_date); -$import_date[2] = 2; -@import_date = Date::Calc::Business_to_Standard(@import_date); - -my $broadcast_num = `cat $LAST_FILE`; -$broadcast_num += 1; - -print "today: " . Date::Calc::Date_to_Text(@today) . "\n"; -print "day of next Radio Helsinki broadcast: " . Date::Calc::Date_to_Text(@import_date) . "\n"; -print "Number of next broadcast: " . $broadcast_num . "\n"; - -my $id = sprintf("%04d-%02d-%02d", @import_date); -my $bdfile = sprintf('.*onda.?info_%d.*\.mp3$', $broadcast_num); - -my $current_stat = `cat $STAT_FILE`; -my ($current_id, $current_file) = $current_stat =~ m/^(.*)\n(.*)/; -if($current_id eq $id) { - print "Already downloaded current file\n"; - RHRD::rddb::closedb($dbh); - exit 0; -} - -print "looking for files like '$bdfile' in RSS Feed\n"; -print " -> $RSS_URL\n"; - -my $feed = rhimport::fetch_parse_rss($RSS_URL) - or die "Error fetching feed: " . XML::Feed->errstr; - -my $file = ""; -my $out_file = ""; -for my $entry ($feed->entries) { - if($entry->enclosure) { - print $entry->enclosure->url . "\n"; - next unless $entry->enclosure->url =~ /$bdfile/; - - my $sum_title = decode_entities($entry->title); - my $sum_text = decode_entities($entry->content->body); - - my $ff = File::Fetch->new(uri => $entry->enclosure->url); - $out_file = $ff->output_file; - if(!rhimport::check_file_extension($out_file)) { - print "\n\nThe extension of the matching file '". $out_file . "' seems to be wrong - manual import necessary!!!\n"; - RHRD::rddb::closedb($dbh); - exit 1; - } - - print $broadcast_num . ": downloading " . $entry->enclosure->url . " (" . $entry->enclosure->length . " Bytes) .. "; - $file = $ff->fetch( to => '/tmp' ) or die $ff->error; - print "ok\n"; - - print "summary:\n" . $sum_title . "\n\n" . $sum_text . "\n"; - - rhimport::pv_add_note($sum_title, $sum_text, $PV_ID, $id, "1"); - - last; - } -} -if($file eq "" || !(-e "$file")) { - print "No Entry found with " . $broadcast_num . " or download error - "; - if($#ARGV >= 0 && $ARGV[0] eq 'last') { - print "giving up, manual import necessary!!!\n"; - } else { - print "will retry later\n"; - } - exit 1; -} - -print "will import $file to dropbox $dropbox (cart=$to_cart)\n"; - -my $error_cb = sub { - my ($text) = @_; - - print "\n$text"; - return 0; -}; - -rhimport::check_key_file() or die "Import Key not found, use rhimport-create-id to create one\n"; - -my $ret; -my $log = rhimport::clear_carts($dbh, $group, $to_cart); -my $import_log; -($ret, $import_log) = rhimport::import_single($file, $dropbox, $user, 0, $error_cb); -$log .= $import_log; - -RHRD::rddb::closedb($dbh); - -unlink $file; - -if(!$ret) { - print "\nImport Error:\n"; - print $log; - exit 1; -} - -unlink($STAT_FILE); -open(my $fhs, '>', $STAT_FILE); -print $fhs "$id\n$out_file"; -close($fhs); - -unlink($LAST_FILE); -open($fhs, '>', $LAST_FILE); -print $fhs "$broadcast_num"; -close($fhs); - -exit 0; diff --git a/rhimport-ra b/rhimport-ra deleted file mode 100755 index 0063338..0000000 --- a/rhimport-ra +++ /dev/null @@ -1,199 +0,0 @@ -#!/usr/bin/perl -w -# -# -# rhimport -# -# Copyright (C) 2009-2015 Christian Pointner <equinox@helsinki.at> -# -# This file is part of rhimport. -# -# rhimport is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# any later version. -# -# rhimport is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with rhimport. If not, see <http://www.gnu.org/licenses/>. -# - -use strict; -use Date::Calc; -use XML::Feed; -use XML::Feed::Entry; -use XML::Feed::Content; -use XML::Feed::Enclosure; -use File::Fetch; -use IO::Handle; -use IPC::Open3; -use HTML::Entities; -use RHRD::rddb; - -use lib '/usr/local/share/rhimport/'; -use rhimport; - -my $STAT_FILE = $ENV{'HOME'} . "/rhimport-ra.stat"; -my $RSS_URL = "http://cba.fro.at/seriesrss/262430?c=Kfs2IoV2Wmd"; -$XML::Feed::MULTIPLE_ENCLOSURES=1; -my $PV_ID = '76'; - -binmode(STDIN, ":utf8"); -binmode(STDOUT, ":utf8"); -binmode(STDERR, ":utf8"); - -if($#ARGV >= 0 && $ARGV[0] eq 'last') { - print "!!!This is the last attempt, there won't be a retry on error!!!\n" -} - -my $user = `/usr/bin/id -un`; -$user =~ s/\n//; -my $group = "RadioATTAC"; - -my ($dbh, $errorstring) = RHRD::rddb::opendb(); -if(!defined $dbh) { - print "$errorstring\n"; - exit 1; -} -my @allowed_dbs = RHRD::rddb::get_dropboxes($dbh, $user, $group); -if(!defined $allowed_dbs[0] && defined $allowed_dbs[2]) { - print "$allowed_dbs[2]\n"; - exit 1; -} - -my $idx = 0; -my $idx_reb = 1; -if(scalar(@allowed_dbs) != 2) { - print "found more or less than 2 Dropboxes for this group?!\n"; - RHRD::rddb::closedb($dbh); - exit 1; -} -my $dropbox = $allowed_dbs[$idx]->{'PATH'}; -my $to_cart = $allowed_dbs[$idx]->{'TO_CART'}; -my $dropbox_reb = $allowed_dbs[$idx_reb]->{'PATH'}; -my $to_cart_reb = $allowed_dbs[$idx_reb]->{'TO_CART'}; - -my @today = Date::Calc::Standard_to_Business(Date::Calc::Today()); -$today[2] = 1; -my @broadcast_day = Date::Calc::Business_to_Standard(@today); -$today[2] = 3; -my @import_date = Date::Calc::Business_to_Standard(@today); -my @import_date_reb = Date::Calc::Add_Delta_Days($import_date[0], $import_date[1], $import_date[2], 3); -print "day of original broadcast: " . Date::Calc::Date_to_Text(@broadcast_day) . "\n"; -print "day of Radio Helsinki broadcast: " . Date::Calc::Date_to_Text(@import_date) . "\n"; -print "day of Radio Helsinki rebroadcast: " . Date::Calc::Date_to_Text(@import_date_reb) . "\n"; - -my $bd = sprintf("%04d-%02d-%02d", $broadcast_day[0], $broadcast_day[1], $broadcast_day[2]); -print "looking for files from $bd in RSS Feed\n"; -print " -> $RSS_URL\n"; - -my $feed = XML::Feed->parse(URI->new($RSS_URL)) - or die "Error fetching feed: " . XML::Feed->errstr; - -my $bdexp = sprintf("^(\\d+)\\..*\\s*0?%d\\s*[.,]\\s*0?%d\\s*[.,]\\s*%4d", $broadcast_day[2], $broadcast_day[1], $broadcast_day[0]); - - -my $file = ""; -my $out_file = ""; -my $i = 0; -for my $entry ($feed->entries) { - $i++; - next unless $entry->enclosure; - my $j = 0; - for my $enclosure($entry->enclosure) { - $j++; - if($enclosure->type eq "audio/mpeg" || $enclosure->type eq "audio/ogg") { - print "$i/$j: (" . $enclosure->type . ", " . $enclosure->length . ") " . $enclosure->url . "\n"; - my $url = $enclosure->url; - if($url =~ /^(.*)_cut(\.[^\.]+)$/) { - $url = $1 . $2; - } - - my $sum_title = decode_entities($entry->title); - next unless $sum_title =~ /$bdexp/; - my $sum_text = decode_entities($entry->content->body); - my $bnum = $1; - - my $ff = File::Fetch->new(uri => $url); - next unless $bnum eq substr($ff->output_file, 0, 3); - my $current_file = `cat $STAT_FILE`; - if($current_file eq $ff->output_file) { - print "Already downloaded file of today\n"; - RHRD::rddb::closedb($dbh); - exit 0; - } - $out_file = $ff->output_file; - if(!rhimport::check_file_extension($out_file)) { - print "\n\nThe extension of the matching file '". $out_file . "' seems to be wrong - manual import necessary!!!\n"; - RHRD::rddb::closedb($dbh); - exit 1; - } - - print " --> " . $bnum . ": downloading " . $url . " (" . $enclosure->length . " Bytes) .. "; - $file = $ff->fetch( to => '/tmp' ) or die $ff->error; - print "ok\n"; - - print "summary:\n" . $sum_title . "\n\n" . $sum_text . "\n"; - - rhimport::pv_add_note($sum_title, $sum_text, $PV_ID, sprintf("%04d-%02d-%02d", $import_date[0], $import_date[1], $import_date[2]), "1"); - print "\n"; - rhimport::pv_add_note($sum_title, $sum_text, $PV_ID, sprintf("%04d-%02d-%02d", $import_date_reb[0], $import_date_reb[1], $import_date_reb[2]), "2"); - print "\n"; - - last; - } - } - last if $file ne ""; -} - -if($file eq "" || !(-e "$file")) { - print "No Entry found from $bd or download error - "; - if($#ARGV >= 0 && $ARGV[0] eq 'last') { - print "giving up, manual import necessary!!!\n"; - } else { - print "will retry later\n"; - } - exit 1; -} - -print "will import $file to dropbox $dropbox (cart=$to_cart)\n"; -print "will import $file to rebroadcast dropbox $dropbox_reb (cart=$to_cart_reb)\n\n"; - -my $error_cb = sub { - my ($text) = @_; - - print "\n$text"; - return 0; -}; - -rhimport::check_key_file() or die "Import Key not found, use rhimport-create-id to create one\n"; - -my $ret; -my $log = rhimport::clear_carts($dbh, $group, $to_cart); -my $import_log; -($ret, $import_log) = rhimport::import_single($file, $dropbox, $user, 0, $error_cb); -$log .= $import_log; -$import_log = rhimport::clear_carts($dbh, $group, $to_cart_reb); -$log .= $import_log; -($ret, $import_log) = rhimport::import_single($file, $dropbox_reb, $user, 0, $error_cb); -$log .= $import_log; - -RHRD::rddb::closedb($dbh); - -unlink $file; - -if(!$ret) { - print "\nImport Error:\n"; - print $log; - exit 1; -} - -unlink($STAT_FILE); -open(my $fhs, '>', $STAT_FILE); -print $fhs $out_file; -close($fhs); - -exit 0; diff --git a/rhimport-rs b/rhimport-rs deleted file mode 100755 index 71d9e27..0000000 --- a/rhimport-rs +++ /dev/null @@ -1,226 +0,0 @@ -#!/usr/bin/perl -w -# -# -# rhimport -# -# Copyright (C) 2009-2015 Christian Pointner <equinox@helsinki.at> -# -# This file is part of rhimport. -# -# rhimport is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# any later version. -# -# rhimport is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with rhimport. If not, see <http://www.gnu.org/licenses/>. -# - -use strict; -use Date::Calc; -use XML::Feed; -use XML::Feed::Entry; -use XML::Feed::Content; -use XML::Feed::Enclosure; -use File::Fetch; -use IO::Handle; -use IPC::Open3; -use HTML::Entities; -use LWP::Simple; -use RHRD::rddb; - -use lib '/usr/local/share/rhimport/'; -use rhimport; - -my $STAT_FILE = $ENV{'HOME'} . "/rhimport-rs.stat"; -my $RSS_URL = "http://cba.fro.at/seriesrss/262433?c=Kfs2IoV2Wmd"; -$XML::Feed::MULTIPLE_ENCLOSURES=1; -my $PV_ID = '75'; - -binmode(STDIN, ":utf8"); -binmode(STDOUT, ":utf8"); -binmode(STDERR, ":utf8"); - -if($#ARGV >= 0 && $ARGV[0] eq 'last') { - print "!!!This is the last attempt, there won't be a retry on error!!!\n" -} - -my $user = `/usr/bin/id -un`; -$user =~ s/\n//; -my $group = "radioStimm"; -my %week_table = ( 1 => 2, 2 => 6, 3 => 2, 4 => 6 ); - -my ($dbh, $errorstring) = RHRD::rddb::opendb(); -if(!defined $dbh) { - print "$errorstring\n"; - exit 1; -} -my @allowed_dbs = RHRD::rddb::get_dropboxes($dbh, $user, $group); -if(!defined $allowed_dbs[0] && defined $allowed_dbs[2]) { - print "$allowed_dbs[2]\n"; - exit 1; -} - -my $dropbox = ""; -my $to_cart = 0; -foreach(@allowed_dbs) { - if($_->{'NAME'} =~ /^autoimport/) { - $dropbox = $_->{'PATH'}; - $to_cart = $_->{'TO_CART'}; - } -} -if(!$to_cart) { - print "no dropbox found!\n"; - RHRD::rddb::closedb($dbh); - exit 1; -} - -my $curweek = RHRD::utils::get_rd_week(); -my $nextweek = $curweek == 4 ? 1 : $curweek + 1; - -my @today = Date::Calc::Today(); -print "today: " . Date::Calc::Date_to_Text(@today) . "\n"; - -my $dow = Date::Calc::Day_of_Week(@today); -my @import_date = Date::Calc::Standard_to_Business(@today); - -if($dow <= $week_table{$curweek}) { - print "We are in week $curweek - broadcast is on " . Date::Calc::Day_of_Week_to_Text($week_table{$curweek}) . "\n"; - $import_date[2] = $week_table{$curweek}; - @import_date = Date::Calc::Business_to_Standard($import_date[0], $import_date[1], $import_date[2]); -} else { - print "We are in week $curweek - broadcast was on " . Date::Calc::Day_of_Week_to_Text($week_table{$curweek}) . "\n"; - print " next week is $nextweek - broadcast is on " . Date::Calc::Day_of_Week_to_Text($week_table{$nextweek}) . "\n"; - $import_date[2] = $week_table{$nextweek}; - @import_date = Date::Calc::Business_to_Standard(@import_date); - @import_date = Date::Calc::Add_Delta_Days(@import_date, 7); -} -print "day of next Radio Helsinki broadcast: " . Date::Calc::Date_to_Text(@import_date) . "\n"; - -my @broadcast_date = Date::Calc::Nth_Weekday_of_Month_Year($import_date[0], $import_date[1], 2, 1); -if(Date::Calc::Delta_Days(@broadcast_date, @import_date) > 0) { - my @tmp_date = Date::Calc::Nth_Weekday_of_Month_Year($import_date[0], $import_date[1], 2, 3); - if(Date::Calc::Delta_Days(@tmp_date, @import_date) > 0) { - @broadcast_date = @tmp_date; - } -} else { - my @tmp_date = Date::Calc::Add_Delta_YM(@broadcast_date, 0, -1); - @broadcast_date = Date::Calc::Nth_Weekday_of_Month_Year($tmp_date[0], $tmp_date[1], 2, 3); -} -print "day of latest original broadcast before next Radio Helsinki broadcast: " . Date::Calc::Date_to_Text(@broadcast_date) . "\n"; - -if(Date::Calc::Delta_Days(@broadcast_date, @today) <= 0) { - print "File won't be available by now!\n"; - RHRD::rddb::closedb($dbh); - exit 0; -} - -my $id = sprintf("%04d-%02d-%02d", $import_date[0], $import_date[1], $import_date[2]); -my $bd = sprintf("%04d-%02d-%02d", $broadcast_date[0], $broadcast_date[1], $broadcast_date[2]); -my $bdfile = sprintf("RAST%04d%02d%02d(CBA)?(.*)\.MP3", $broadcast_date[0], $broadcast_date[1], $broadcast_date[2]); - -print "looking for files from $bd in RSS Feed\n"; -print " -> $RSS_URL\n"; - -my $feed = XML::Feed->parse(URI->new($RSS_URL)) - or die "Error fetching feed: " . XML::Feed->errstr; - -my $file = ""; -my $out_file = ""; -my $i = 0; -for my $entry ($feed->entries) { - $i++; - next unless $entry->enclosure; - my $j = 0; - for my $enclosure($entry->enclosure) { - $j++; - if($enclosure->type eq "audio/mpeg" || $enclosure->type eq "audio/ogg") { - print "$i/$j: (" . $enclosure->type . ", " . $enclosure->length . ") " . $enclosure->url . "\n"; - my $url = $enclosure->url; - if($url =~ /^(.*)_cut(\.[^\.]+)$/) { - $url = $1 . $2; - } - - my $ff = File::Fetch->new(uri => $url); - next unless (uc($ff->output_file) =~ $bdfile); # file not from correct day - - my $sum_title = decode_entities($entry->title); - my $sum_text = decode_entities($entry->content->body); - - my $current_stat = `cat $STAT_FILE`; - my ($current_id, $current_file) = $current_stat =~ m/^(.*)\n(.*)/; - if($current_id eq $id && $current_file eq $ff->output_file) { - print "Already downloaded file of today\n"; - RHRD::rddb::closedb($dbh); - exit 0; - } - $out_file = $ff->output_file; - if(!rhimport::check_file_extension($out_file)) { - print "\n\nThe extension of the matching file '". $out_file . "' seems to be wrong - manual import necessary!!!\n"; - RHRD::rddb::closedb($dbh); - exit 1; - } - - print " --> " . $bdfile . ": downloading " . $url . " (" . $enclosure->length . " Bytes) .. "; - $file = $ff->fetch( to => '/tmp' ) or die $ff->error; - print "ok\n"; - - print "summary:\n" . $sum_title . "\n\n" . $sum_text . "\n"; - - rhimport::pv_add_note($sum_title, $sum_text, $PV_ID, $id, "1"); - print "\n"; - - last; - } - } - last if $file ne ""; -} - -if($file eq "" || !(-e "$file")) { - print "No Entry found from $bd or download error - "; - if($#ARGV >= 0 && $ARGV[0] eq 'last') { - print "giving up, manual import necessary!!!\n"; - } else { - print "will retry later\n"; - } - exit 1; -} - -print "will import $file to dropbox $dropbox (cart=$to_cart)\n"; - -my $error_cb = sub { - my ($text) = @_; - - print "\n$text"; - return 0; -}; - -rhimport::check_key_file() or die "Import Key not found, use rhimport-create-id to create one\n"; - -my $ret; -my $log = rhimport::clear_carts($dbh, $group, $to_cart); -my $import_log; -($ret, $import_log) = rhimport::import_single($file, $dropbox, $user, 0, $error_cb); -$log .= $import_log; - -RHRD::rddb::closedb($dbh); - -unlink $file; - -if(!$ret) { - print "\nImport Error:\n"; - print $log; - exit 1; -} - -unlink($STAT_FILE); -open(my $fhs, '>', $STAT_FILE); -print $fhs "$id\n$out_file"; -close($fhs); - -exit 0; diff --git a/rhimport-sm b/rhimport-sm deleted file mode 100755 index e28276f..0000000 --- a/rhimport-sm +++ /dev/null @@ -1,213 +0,0 @@ -#!/usr/bin/perl -w -# -# -# rhimport -# -# Copyright (C) 2009-2015 Christian Pointner <equinox@helsinki.at> -# -# This file is part of rhimport. -# -# rhimport is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# any later version. -# -# rhimport is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with rhimport. If not, see <http://www.gnu.org/licenses/>. -# - -use strict; -use Date::Calc; -use XML::Feed; -use XML::Feed::Entry; -use XML::Feed::Content; -use XML::Feed::Enclosure; -use File::Fetch; -use IO::Handle; -use IPC::Open3; -use HTML::Entities; -use LWP::Simple; -use RHRD::rddb; - -use lib '/usr/local/share/rhimport/'; -use rhimport; - -my $STAT_FILE = $ENV{'HOME'} . "/rhimport-sm.stat"; -my $RSS_URL = "http://cba.fro.at/seriesrss/263673?c=Kfs2IoV2Wmd"; -$XML::Feed::MULTIPLE_ENCLOSURES=1; -my $PV_ID = '304'; - -binmode(STDIN, ":utf8"); -binmode(STDOUT, ":utf8"); -binmode(STDERR, ":utf8"); - -if($#ARGV >= 0 && $ARGV[0] eq 'last') { - print "!!!This is the last attempt, there won't be a retry on error!!!\n" -} - -my $user = `/usr/bin/id -un`; -$user =~ s/\n//; -my $group = "soundsmove"; - -my ($dbh, $errorstring) = RHRD::rddb::opendb(); -if(!defined $dbh) { - print "$errorstring\n"; - exit 1; -} -my @allowed_dbs = RHRD::rddb::get_dropboxes($dbh, $user, $group); -if(!defined $allowed_dbs[0] && defined $allowed_dbs[2]) { - print "$allowed_dbs[2]\n"; - exit 1; -} - -if(scalar(@allowed_dbs) != 1) { - print "found more or less than one Dropbox for this group?!\n"; - RHRD::rddb::closedb($dbh); - exit 1; -} -my $dropbox = $allowed_dbs[0]->{'PATH'}; -my $to_cart = $allowed_dbs[0]->{'TO_CART'}; - -my $curweek = RHRD::utils::get_rd_week(); - -my @today = Date::Calc::Today(); -my $dow = Date::Calc::Day_of_Week(@today); -if(($curweek != 1 && $curweek != 4) || - ($curweek == 1 && $dow > 3) || - ($curweek == 4 && $dow <= 3)) { - print "File won't be available by now!\n"; - RHRD::rddb::closedb($dbh); - exit 0; -} -print "today: " . Date::Calc::Date_to_Text(@today) . "\n"; - -my @import_date = Date::Calc::Standard_to_Business(@today); -$import_date[2] = 3; -@import_date = Date::Calc::Business_to_Standard(@import_date); -if($curweek == 4) { - @import_date = Date::Calc::Add_Delta_Days(@import_date, 7); -} -print "day of next Radio Helsinki broadcast: " . Date::Calc::Date_to_Text(@import_date) . "\n"; - -my @broadcast_date = Date::Calc::Nth_Weekday_of_Month_Year($import_date[0], $import_date[1], 3, 1); -if(Date::Calc::Delta_Days(@broadcast_date, @import_date) <= 0) { - my @tmp_date = Date::Calc::Add_Delta_YM(@broadcast_date, 0, -1); - @broadcast_date = Date::Calc::Nth_Weekday_of_Month_Year($tmp_date[0], $tmp_date[1], 3, 1); -} -print "day of latest original broadcast before next Radio Helsinki broadcast: " . Date::Calc::Date_to_Text(@broadcast_date) . "\n"; - -if(Date::Calc::Delta_Days(@broadcast_date, @today) <= 0) { - print "File won't be available by now!\n"; - RHRD::rddb::closedb($dbh); - exit 0; -} - -my $id = sprintf("%04d-%02d-%02d", @import_date); -my $bd = sprintf("%04d-%02d-%02d", @broadcast_date); -my $bdfile = sprintf("%04d-?%02d-?%02d", @broadcast_date); - -print "looking for files from $bd in RSS Feed\n"; -print " -> $RSS_URL\n"; - -my $feed = XML::Feed->parse(URI->new($RSS_URL)) - or die "Error fetching feed: " . XML::Feed->errstr; - -my $file = ""; -my $out_file = ""; -my $i = 0; -for my $entry ($feed->entries) { - $i++; - next unless $entry->enclosure; - my $j = 0; - for my $enclosure($entry->enclosure) { - $j++; - if($enclosure->type eq "audio/mpeg" || $enclosure->type eq "audio/ogg") { - print "$i/$j: (" . $enclosure->type . ", " . $enclosure->length . ") " . $enclosure->url . "\n"; - my $url = $enclosure->url; - if($url =~ /^(.*)_cut(\.[^\.]+)$/) { - $url = $1 . $2; - } - - my $ff = File::Fetch->new(uri => $url); - next unless (uc($ff->output_file) =~ $bdfile); # file not from correct day - - my $sum_title = decode_entities($entry->title); - my $sum_text = decode_entities($entry->content->body); - - my $current_stat = `cat $STAT_FILE`; - my ($current_id, $current_file) = $current_stat =~ m/^(.*)\n(.*)/; - if($current_id eq $id && $current_file eq $ff->output_file) { - print "Already downloaded file of today\n"; - RHRD::rddb::closedb($dbh); - exit 0; - } - $out_file = $ff->output_file; - if(!rhimport::check_file_extension($out_file)) { - print "\n\nThe extension of the matching file '". $out_file . "' seems to be wrong - manual import necessary!!!\n"; - RHRD::rddb::closedb($dbh); - exit 1; - } - - print " --> " . $bdfile . ": downloading " . $url . " (" . $enclosure->length . " Bytes) .. "; - $file = $ff->fetch( to => '/tmp' ) or die $ff->error; - print "ok\n"; - - print "summary:\n" . $sum_title . "\n\n" . $sum_text . "\n"; - - rhimport::pv_add_note($sum_title, $sum_text, $PV_ID, $id, "1"); - print "\n"; - - last; - } - } - last if $file ne ""; -} - -if($file eq "" || !(-e "$file")) { - print "No Entry found from $bd or download error - "; - if($#ARGV >= 0 && $ARGV[0] eq 'last') { - print "giving up, manual import necessary!!!\n"; - } else { - print "will retry later\n"; - } - exit 1; -} - -print "will import $file to dropbox $dropbox (cart=$to_cart)\n"; - -my $error_cb = sub { - my ($text) = @_; - - print "\n$text"; - return 0; -}; - -rhimport::check_key_file() or die "Import Key not found, use rhimport-create-id to create one\n"; - -my $ret; -my $log = rhimport::clear_carts($dbh, $group, $to_cart); -my $import_log; -($ret, $import_log) = rhimport::import_single($file, $dropbox, $user, 0, $error_cb); -$log .= $import_log; - -RHRD::rddb::closedb($dbh); - -unlink $file; - -if(!$ret) { - print "\nImport Error:\n"; - print $log; - exit 1; -} - -unlink($STAT_FILE); -open(my $fhs, '>', $STAT_FILE); -print $fhs "$id\n$out_file"; -close($fhs); - -exit 0; diff --git a/rhimport-sv b/rhimport-sv deleted file mode 100755 index 6056f69..0000000 --- a/rhimport-sv +++ /dev/null @@ -1,212 +0,0 @@ -#!/usr/bin/perl -w -# -# -# rhimport -# -# Copyright (C) 2009-2015 Christian Pointner <equinox@helsinki.at> -# -# This file is part of rhimport. -# -# rhimport is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# any later version. -# -# rhimport is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with rhimport. If not, see <http://www.gnu.org/licenses/>. -# - -use strict; -use Date::Calc; -use XML::Feed; -use XML::Feed::Entry; -use XML::Feed::Content; -use XML::Feed::Enclosure; -use File::Fetch; -use IO::Handle; -use IPC::Open3; -use HTML::Entities; -use LWP::Simple; -use RHRD::rddb; - -use lib '/usr/local/share/rhimport/'; -use rhimport; - -my $STAT_FILE = $ENV{'HOME'} . "/rhimport-sv.stat"; -my $RSS_URL = "http://cba.fro.at/seriesrss/262457?c=Kfs2IoV2Wmd"; -$XML::Feed::MULTIPLE_ENCLOSURES=1; -my $PV_ID = '396'; - -binmode(STDIN, ":utf8"); -binmode(STDOUT, ":utf8"); -binmode(STDERR, ":utf8"); - -if($#ARGV >= 0 && $ARGV[0] eq 'last') { - print "!!!This is the last attempt, there won't be a retry on error!!!\n" -} - -my $user = `/usr/bin/id -un`; -$user =~ s/\n//; -my $group = "svetjevas"; - -my ($dbh, $errorstring) = RHRD::rddb::opendb(); -if(!defined $dbh) { - print "$errorstring\n"; - exit 1; -} -my @allowed_dbs = RHRD::rddb::get_dropboxes($dbh, $user, $group); -if(!defined $allowed_dbs[0] && defined $allowed_dbs[2]) { - print "$allowed_dbs[2]\n"; - exit 1; -} - -if(scalar(@allowed_dbs) != 1) { - print "found more or less than one Dropbox for this group?!\n"; - RHRD::rddb::closedb($dbh); - exit 1; -} -my $dropbox = $allowed_dbs[0]->{'PATH'}; -my $to_cart = $allowed_dbs[0]->{'TO_CART'}; - -my $curweek = RHRD::utils::get_rd_week(); - -my @today = Date::Calc::Today(); -my $dow = Date::Calc::Day_of_Week(@today); -if(($curweek != 3 && $curweek != 4) || - ($curweek == 4 && $dow > 2)) { - print "File won't be available by now!\n"; - RHRD::rddb::closedb($dbh); - exit 0; -} -print "today: " . Date::Calc::Date_to_Text(@today) . "\n"; - -my @import_date = Date::Calc::Standard_to_Business(@today); -$import_date[2] = 2; -@import_date = Date::Calc::Business_to_Standard(@import_date); -if($curweek == 3) { - @import_date = Date::Calc::Add_Delta_Days(@import_date, 7); -} -print "day of next Radio Helsinki broadcast: " . Date::Calc::Date_to_Text(@import_date) . "\n"; - -my @broadcast_date = Date::Calc::Nth_Weekday_of_Month_Year($import_date[0], $import_date[1], 7, 1); -if(Date::Calc::Delta_Days(@broadcast_date, @import_date) <= 0) { - my @tmp_date = Date::Calc::Add_Delta_YM(@broadcast_date, 0, -1); - @broadcast_date = Date::Calc::Nth_Weekday_of_Month_Year($tmp_date[0], $tmp_date[1], 7, 1); -} -print "day of latest original broadcast before next Radio Helsinki broadcast: " . Date::Calc::Date_to_Text(@broadcast_date) . "\n"; - -if(Date::Calc::Delta_Days(@broadcast_date, @today) <= 0) { - print "File won't be available by now!\n"; - RHRD::rddb::closedb($dbh); - exit 0; -} - -my $id = sprintf("%04d-%02d-%02d", @import_date); -my $bd = sprintf("%04d-%02d-%02d", @broadcast_date); -my $bdfile = sprintf("SEOI%02d%02d", $broadcast_date[2], $broadcast_date[1]); - -print "looking for files from $bd in RSS Feed\n"; -print " -> $RSS_URL\n"; - -my $feed = XML::Feed->parse(URI->new($RSS_URL)) - or die "Error fetching feed: " . XML::Feed->errstr; - -my $file = ""; -my $out_file = ""; -my $i = 0; -for my $entry ($feed->entries) { - $i++; - next unless $entry->enclosure; - my $j = 0; - for my $enclosure($entry->enclosure) { - $j++; - if($enclosure->type eq "audio/mpeg" || $enclosure->type eq "audio/ogg") { - print "$i/$j: (" . $enclosure->type . ", " . $enclosure->length . ") " . $enclosure->url . "\n"; - my $url = $enclosure->url; - if($url =~ /^(.*)_cut(\.[^\.]+)$/) { - $url = $1 . $2; - } - - my $ff = File::Fetch->new(uri => $url); - next unless (uc($ff->output_file) =~ $bdfile); # file not from correct day - - my $sum_title = decode_entities($entry->title); - my $sum_text = decode_entities($entry->content->body); - - my $current_stat = `cat $STAT_FILE`; - my ($current_id, $current_file) = $current_stat =~ m/^(.*)\n(.*)/; - if($current_id eq $id && $current_file eq $ff->output_file) { - print "Already downloaded file of today\n"; - RHRD::rddb::closedb($dbh); - exit 0; - } - $out_file = $ff->output_file; - if(!rhimport::check_file_extension($out_file)) { - print "\n\nThe extension of the matching file '". $out_file . "' seems to be wrong - manual import necessary!!!\n"; - RHRD::rddb::closedb($dbh); - exit 1; - } - - print " --> " . $bdfile . ": downloading " . $url . " (" . $enclosure->length . " Bytes) .. "; - $file = $ff->fetch( to => '/tmp' ) or die $ff->error; - print "ok\n"; - - print "summary:\n" . $sum_title . "\n\n" . $sum_text . "\n"; - - rhimport::pv_add_note($sum_title, $sum_text, $PV_ID, $id, "1"); - print "\n"; - - last; - } - } - last if $file ne ""; -} - -if($file eq "" || !(-e "$file")) { - print "No Entry found from $bd or download error - "; - if($#ARGV >= 0 && $ARGV[0] eq 'last') { - print "giving up, manual import necessary!!!\n"; - } else { - print "will retry later\n"; - } - exit 1; -} - -print "will import $file to dropbox $dropbox (cart=$to_cart)\n"; - -my $error_cb = sub { - my ($text) = @_; - - print "\n$text"; - return 0; -}; - -rhimport::check_key_file() or die "Import Key not found, use rhimport-create-id to create one\n"; - -my $ret; -my $log = rhimport::clear_carts($dbh, $group, $to_cart); -my $import_log; -($ret, $import_log) = rhimport::import_single($file, $dropbox, $user, 0, $error_cb); -$log .= $import_log; - -RHRD::rddb::closedb($dbh); - -unlink $file; - -if(!$ret) { - print "\nImport Error:\n"; - print $log; - exit 1; -} - -unlink($STAT_FILE); -open(my $fhs, '>', $STAT_FILE); -print $fhs "$id\n$out_file"; -close($fhs); - -exit 0; diff --git a/rhimport-tr b/rhimport-tr deleted file mode 100755 index 6551780..0000000 --- a/rhimport-tr +++ /dev/null @@ -1,192 +0,0 @@ -#!/usr/bin/perl -w -# -# -# rhimport -# -# Copyright (C) 2009-2015 Christian Pointner <equinox@helsinki.at> -# -# This file is part of rhimport. -# -# rhimport is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# any later version. -# -# rhimport is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with rhimport. If not, see <http://www.gnu.org/licenses/>. -# - -use strict; -use Date::Calc; -use XML::Feed; -use XML::Feed::Entry; -use XML::Feed::Content; -use XML::Feed::Enclosure; -use File::Fetch; -use IO::Handle; -use IPC::Open3; -use HTML::Entities; -use RHRD::rddb; - -use lib '/usr/local/share/rhimport/'; -use rhimport; - -my $STAT_FILE = $ENV{'HOME'} . "/rhimport-tr.stat"; -my $RSS_URL = "http://cba.fro.at/series/tierrechtsradio/feed?c=Kfs2IoV2Wmd"; -$XML::Feed::MULTIPLE_ENCLOSURES=1; -my $PV_ID = '506'; - -binmode(STDIN, ":utf8"); -binmode(STDOUT, ":utf8"); -binmode(STDERR, ":utf8"); - -if($#ARGV >= 0 && $ARGV[0] eq 'last') { - print "!!!This is the last attempt, there won't be a retry on error!!!\n" -} - -my $user = `/usr/bin/id -un`; -$user =~ s/\n//; -my $group = "tierrecht"; - -my ($dbh, $errorstring) = RHRD::rddb::opendb(); -if(!defined $dbh) { - print "$errorstring\n"; - exit 1; -} -my @allowed_dbs = RHRD::rddb::get_dropboxes($dbh, $user, $group); -if(!defined $allowed_dbs[0] && defined $allowed_dbs[2]) { - print "$allowed_dbs[2]\n"; - exit 1; -} - -my $idx = 0; -if(scalar(@allowed_dbs) != 1) { - print "found more or less than 1 Dropbox for this group?!\n"; - RHRD::rddb::closedb($dbh); - exit 1; -} -my $dropbox = $allowed_dbs[$idx]->{'PATH'}; -my $to_cart = $allowed_dbs[$idx]->{'TO_CART'}; - -my @today = Date::Calc::Standard_to_Business(Date::Calc::Today()); -if($today[2] == 5) { - print "please don't run this script on fridays!\n"; - RHRD::rddb::closedb($dbh); - exit 1; -} -my @broadcast_day = Date::Calc::Business_to_Standard($today[0], $today[1], 5); -if($today[2] < 5) { - @broadcast_day = Date::Calc::Add_Delta_Days($broadcast_day[0], $broadcast_day[1], $broadcast_day[2], -7); -} -my @import_date = Date::Calc::Add_Delta_Days($broadcast_day[0], $broadcast_day[1], $broadcast_day[2], 7); - -print "day of original broadcast: " . Date::Calc::Date_to_Text(@broadcast_day) . "\n"; -print "day of Radio Helsinki broadcast: " . Date::Calc::Date_to_Text(@import_date) . "\n"; - -my $bd = sprintf("%04d-%02d-%02d", $broadcast_day[0], $broadcast_day[1], $broadcast_day[2]); -my $bdfile = sprintf("^%04d%02d%02d", $broadcast_day[0], $broadcast_day[1], $broadcast_day[2]); - -print "looking for files from $bd in RSS Feed\n"; -print " -> $RSS_URL\n"; - -my $feed = XML::Feed->parse(URI->new($RSS_URL)) - or die "Error fetching feed: " . XML::Feed->errstr; - -my $file = ""; -my $out_file = ""; -my $i = 0; -for my $entry ($feed->entries) { - $i++; - next unless $entry->enclosure; - my $j = 0; - for my $enclosure($entry->enclosure) { - $j++; - if($enclosure->type eq "audio/mpeg" || $enclosure->type eq "audio/ogg") { - print "$i/$j: (" . $enclosure->type . ", " . $enclosure->length . ") " . $enclosure->url . "\n"; - my $url = $enclosure->url; - if($url =~ /^(.*)_cut(\.[^\.]+)$/) { - $url = $1 . $2; - } - - my $ff = File::Fetch->new(uri => $url); - next unless (uc($ff->output_file) =~ $bdfile); # file not from correct day - - my $sum_title = decode_entities($entry->title); - my $sum_text = decode_entities($entry->content->body); - - my $current_file = `cat $STAT_FILE`; - if($current_file eq $ff->output_file) { - print "Already downloaded file of today\n"; - RHRD::rddb::closedb($dbh); - exit 0; - } - $out_file = $ff->output_file; - if(!rhimport::check_file_extension($out_file)) { - print "\n\nThe extension of the matching file '". $out_file . "' seems to be wrong - manual import necessary!!!\n"; - RHRD::rddb::closedb($dbh); - exit 1; - } - - print " --> " . $bdfile . ": downloading " . $url . " (" . $enclosure->length . " Bytes) .. "; - $file = $ff->fetch( to => '/tmp' ) or die $ff->error; - print "ok\n"; - - print "summary:\n" . $sum_title . "\n\n" . $sum_text . "\n"; - - rhimport::pv_add_note($sum_title, $sum_text, $PV_ID, sprintf("%04d-%02d-%02d", $import_date[0], $import_date[1], $import_date[2]), "1"); - print "\n"; - - last; - } - } - last if $file ne ""; -} - -if($file eq "" || !(-e "$file")) { - print "No Entry found from $bd or download error - "; - if($#ARGV >= 0 && $ARGV[0] eq 'last') { - print "giving up, manual import necessary!!!\n"; - } else { - print "will retry later\n"; - } - exit 1; -} - -print "will import $file to dropbox $dropbox (cart=$to_cart)\n"; - -my $error_cb = sub { - my ($text) = @_; - - print "\n$text"; - return 0; -}; - -rhimport::check_key_file() or die "Import Key not found, use rhimport-create-id to create one\n"; - -my $ret; -my $log = rhimport::clear_carts($dbh, $group, $to_cart); -my $import_log; -($ret, $import_log) = rhimport::import_single($file, $dropbox, $user, 0, $error_cb); -$log .= $import_log; - -RHRD::rddb::closedb($dbh); - -unlink $file; - -if(!$ret) { - print "\nImport Error:\n"; - print $log; - exit 1; -} - -unlink($STAT_FILE); -open(my $fhs, '>', $STAT_FILE); -print $fhs $out_file; -close($fhs); - -exit 0; diff --git a/rhimport-ut b/rhimport-ut deleted file mode 100755 index 8edb30b..0000000 --- a/rhimport-ut +++ /dev/null @@ -1,199 +0,0 @@ -#!/usr/bin/perl -w -# -# -# rhimport -# -# Copyright (C) 2009-2015 Christian Pointner <equinox@helsinki.at> -# -# This file is part of rhimport. -# -# rhimport is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# any later version. -# -# rhimport is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with rhimport. If not, see <http://www.gnu.org/licenses/>. -# - -use strict; -use Date::Calc; -use XML::Feed; -use XML::Feed::Entry; -use XML::Feed::Content; -use XML::Feed::Enclosure; -use File::Fetch; -use IO::Handle; -use IPC::Open3; -use HTML::Entities; -use RHRD::rddb; - -use lib '/usr/local/share/rhimport/'; -use rhimport; - -my $STAT_FILE = $ENV{'HOME'} . "/rhimport-ut.stat"; -my $RSS_URL = "http://cba.fro.at/seriesrss/264016?c=Kfs2IoV2Wmd"; -$XML::Feed::MULTIPLE_ENCLOSURES=1; -my $PV_ID = '393'; - -binmode(STDIN, ":utf8"); -binmode(STDOUT, ":utf8"); -binmode(STDERR, ":utf8"); - -if($#ARGV >= 0 && $ARGV[0] eq 'last') { - print "!!!This is the last attempt, there won't be a retry on error!!!\n" -} - -my $user = `/usr/bin/id -un`; -$user =~ s/\n//; -my $group = "uton"; - -my ($dbh, $errorstring) = RHRD::rddb::opendb(); -if(!defined $dbh) { - print "$errorstring\n"; - exit 1; -} -my @allowed_dbs = RHRD::rddb::get_dropboxes($dbh, $user, $group); -if(!defined $allowed_dbs[0] && defined $allowed_dbs[2]) { - print "$allowed_dbs[2]\n"; - exit 1; -} - -my $idx = 0; -my $idx_reb = 1; -if(scalar(@allowed_dbs) != 2) { - print "found more or less than 2 Dropboxes for this group?!\n"; - RHRD::rddb::closedb($dbh); - exit 1; -} -my $dropbox = $allowed_dbs[$idx]->{'PATH'}; -my $to_cart = $allowed_dbs[$idx]->{'TO_CART'}; -my $dropbox_reb = $allowed_dbs[$idx_reb]->{'PATH'}; -my $to_cart_reb = $allowed_dbs[$idx_reb]->{'TO_CART'}; - -my @import_date = Date::Calc::Standard_to_Business(Date::Calc::Today()); -my $dow = $import_date[2]; -$import_date[2] = 3; -@import_date = Date::Calc::Business_to_Standard(@import_date); -@import_date = Date::Calc::Add_Delta_Days(@import_date, 7) if ($dow > 3); - -my @broadcast_day = Date::Calc::Add_Delta_Days(@import_date, -106); -my @import_date_reb = Date::Calc::Add_Delta_Days(@import_date, 2); -print "day of original broadcast: " . Date::Calc::Date_to_Text(@broadcast_day) . "\n"; -print "day of Radio Helsinki broadcast: " . Date::Calc::Date_to_Text(@import_date) . "\n"; -print "day of Radio Helsinki rebroadcast: " . Date::Calc::Date_to_Text(@import_date_reb) . "\n"; - -my $bd = sprintf("%04d-%02d-%02d", $broadcast_day[0], $broadcast_day[1], $broadcast_day[2]); -print "looking for files from $bd in RSS Feed\n"; -print " -> $RSS_URL\n"; - -my $feed = XML::Feed->parse(URI->new($RSS_URL)) - or die "Error fetching feed: " . XML::Feed->errstr; - -my $bdexp = sprintf("^UTON%02d%02d%02d", $broadcast_day[0] % 100, $broadcast_day[1], $broadcast_day[2]); - -my $file = ""; -my $out_file = ""; -my $i = 0; -for my $entry ($feed->entries) { - $i++; - next unless $entry->enclosure; - my $j = 0; - for my $enclosure($entry->enclosure) { - $j++; - if($enclosure->type eq "audio/mpeg" || $enclosure->type eq "audio/ogg") { - print "$i/$j: (" . $enclosure->type . ", " . $enclosure->length . ") " . $enclosure->url . "\n"; - my $url = $enclosure->url; - if($url =~ /^(.*)_cut(\.[^\.]+)$/) { - $url = $1 . $2; - } - - my $ff = File::Fetch->new(uri => $url); - next unless (uc($ff->output_file) =~ $bdexp); # file not from correct day - - my $sum_title = decode_entities($entry->title); - my $sum_text = decode_entities($entry->content->body); - - my $current_file = `cat $STAT_FILE`; - if($current_file eq $ff->output_file) { - print "Already downloaded file of today\n"; - RHRD::rddb::closedb($dbh); - exit 0; - } - $out_file = $ff->output_file; - if(!rhimport::check_file_extension($out_file)) { - print "\n\nThe extension of the matching file '". $out_file . "' seems to be wrong - manual import necessary!!!\n"; - RHRD::rddb::closedb($dbh); - exit 1; - } - - print " --> " . $bdexp . ": downloading " . $url . " (" . $enclosure->length . " Bytes) .. "; - $file = $ff->fetch( to => '/tmp' ) or die $ff->error; - print "ok\n"; - - print "summary:\n" . $sum_title . "\n\n" . $sum_text . "\n"; - - rhimport::pv_add_note($sum_title, $sum_text, $PV_ID, sprintf("%04d-%02d-%02d", $import_date[0], $import_date[1], $import_date[2]), "1"); - print "\n"; - rhimport::pv_add_note($sum_title, $sum_text, $PV_ID, sprintf("%04d-%02d-%02d", $import_date_reb[0], $import_date_reb[1], $import_date_reb[2]), "2"); - print "\n"; - - last; - } - } - last if $file ne ""; -} - -if($file eq "" || !(-e "$file")) { - print "No Entry found from $bd or download error - "; - if($#ARGV >= 0 && $ARGV[0] eq 'last') { - print "giving up, manual import necessary!!!\n"; - } else { - print "will retry later\n"; - } - exit 1; -} - -print "will import $file to dropbox $dropbox (cart=$to_cart)\n"; -print "will import $file to rebroadcast dropbox $dropbox_reb (cart=$to_cart_reb)\n\n"; - -my $error_cb = sub { - my ($text) = @_; - - print "\n$text"; - return 0; -}; - -rhimport::check_key_file() or die "Import Key not found, use rhimport-create-id to create one\n"; - -my $ret; -my $log = rhimport::clear_carts($dbh, $group, $to_cart); -my $import_log; -($ret, $import_log) = rhimport::import_single($file, $dropbox, $user, 0, $error_cb); -$log .= $import_log; -$import_log = rhimport::clear_carts($dbh, $group, $to_cart_reb); -$log .= $import_log; -($ret, $import_log) = rhimport::import_single($file, $dropbox_reb, $user, 0, $error_cb); -$log .= $import_log; - -RHRD::rddb::closedb($dbh); - -unlink $file; - -if(!$ret) { - print "\nImport Error:\n"; - print $log; - exit 1; -} - -unlink($STAT_FILE); -open(my $fhs, '>', $STAT_FILE); -print $fhs $out_file; -close($fhs); - -exit 0; diff --git a/rhimport-zf b/rhimport-zf deleted file mode 100755 index 0f28ed7..0000000 --- a/rhimport-zf +++ /dev/null @@ -1,193 +0,0 @@ -#!/usr/bin/perl -w -# -# -# rhimport -# -# Copyright (C) 2009-2015 Christian Pointner <equinox@helsinki.at> -# -# This file is part of rhimport. -# -# rhimport is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# any later version. -# -# rhimport is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with rhimport. If not, see <http://www.gnu.org/licenses/>. -# - -use strict; -use Date::Calc; -use XML::Feed; -use XML::Feed::Entry; -use XML::Feed::Content; -use XML::Feed::Enclosure; -use File::Fetch; -use IO::Handle; -use IPC::Open3; -use HTML::Entities; -use File::Touch; -use RHRD::rddb; - -use lib '/usr/local/share/rhimport/'; -use rhimport; - -my $STAT_FILE = $ENV{'HOME'} . "/rhimport-zf.stat"; -my $NOTE_FILE = $ENV{'HOME'} . "/rhimport-zf.last_note"; -my $NEW_FILE = $ENV{'HOME'} . "/rhimport-zf.is_new"; -my $RSS_URL = "http://freie-radios.net/portal/podcast.php?serie=53&rss"; -my $PV_ID = '300'; - -binmode(STDIN, ":utf8"); -binmode(STDOUT, ":utf8"); -binmode(STDERR, ":utf8"); - -if($#ARGV >= 0 && $ARGV[0] eq 'last') { - print "!!!This is the last attempt, there won't be a retry on error!!!\n" -} - -my $user = `/usr/bin/id -un`; -$user =~ s/\n//; -my $group = "zipfm"; - -my ($dbh, $errorstring) = RHRD::rddb::opendb(); -if(!defined $dbh) { - print "$errorstring\n"; - exit 1; -} -my @allowed_dbs = RHRD::rddb::get_dropboxes($dbh, $user, $group); -if(!defined $allowed_dbs[0] && defined $allowed_dbs[2]) { - print "$allowed_dbs[2]\n"; - exit 1; -} - - -my $dropbox = ""; -my $to_cart = 0; -my $dropbox_reb = ""; -my $to_cart_reb = 0; - -my @today = Date::Calc::Today(); -my $dow = Date::Calc::Day_of_Week(@today); -my @tomorrow = Date::Calc::Add_Delta_Days(@today, 1); - -my @broadcast_day = @today; -my @rebroadcast_day = @tomorrow; -print "day of broadcast: " . Date::Calc::Date_to_Text(@broadcast_day) . "\n"; -print "day of rebroadcast: " . Date::Calc::Date_to_Text(@rebroadcast_day) . "\n" unless $dow == 5; - -for my $href ( @allowed_dbs ) { - if ($dow == int(substr($href->{'NAME'},0,2))) { - $dropbox = $href->{'PATH'}; - $to_cart = $href->{'TO_CART'}; - } -} - -if($dropbox eq "") { - print "no dropbox for day in question\n"; - RHRD::rddb::closedb($dbh); - exit 0; -} - -my $regexp = sprintf("%04d%02d%02d", $broadcast_day[0], $broadcast_day[1], $broadcast_day[2]); -print "looking for file from " . Date::Calc::Date_to_Text(@broadcast_day) . " in RSS Feed\n"; -print " -> $RSS_URL\n"; - -my $feed = XML::Feed->parse(URI->new($RSS_URL)) - or die "Error fetching feed: " . XML::Feed->errstr; - -my $file = ""; -my $out_file = ""; -for my $entry ($feed->entries) { - if($entry->enclosure && $entry->enclosure->type eq "audio/mpeg") { - next unless $entry->enclosure->url =~ /$regexp/; - - my $sum_title = decode_entities($entry->title); - my $sum_text = decode_entities($entry->content->body); - - my $ff = File::Fetch->new(uri => $entry->enclosure->url); - my $current_file = `cat $STAT_FILE`; - if($current_file eq $ff->output_file) { - print "Already downloaded file of today\n"; - RHRD::rddb::closedb($dbh); - exit 0; - } - $out_file = $ff->output_file; - if(!rhimport::check_file_extension($out_file)) { - print "\n\nThe extension of the matching file '". $out_file . "' seems to be wrong - manual import necessary!!!\n"; - RHRD::rddb::closedb($dbh); - exit 1; - } - - print $regexp . ": downloading " . $entry->enclosure->url . " (" . $entry->enclosure->length . " Bytes) .. "; - $file = $ff->fetch( to => '/tmp' ) or die $ff->error; - print "ok\n"; - - print "summary:\n" . $sum_title . "\n\n" . $sum_text . "\n"; - - if($#ARGV >= 0 && $ARGV[0] eq 'nopv') { - print "not adding note to PV for regular entry - only for rebroadcast\n"; - } else { - rhimport::pv_add_note($sum_title, $sum_text, $PV_ID, sprintf("%04d-%02d-%02d", @broadcast_day), "1", 1); - } - rhimport::pv_add_note($sum_title, $sum_text, $PV_ID, sprintf("%04d-%02d-%02d", @rebroadcast_day), "2", 0) unless $dow == 5; - print "\n"; - - unlink($NOTE_FILE); - open(my $fhs, '>', $NOTE_FILE); - binmode($fhs, ":utf8"); - print $fhs $sum_title . "\n\n" . $sum_text . "\n"; - close($fhs); - - last; - } -} -if($file eq "" || !(-e "$file")) { - print "No Entry found for " . Date::Calc::Date_to_Text(@broadcast_day) . " or download error - "; - if($#ARGV >= 0 && $ARGV[0] eq 'last') { - print "giving up, manual import necessary!!!\n"; - } else { - print "will retry later\n"; - } - exit 1; -} - -print "will import $file to dropbox $dropbox (cart=$to_cart)\n"; - -my $error_cb = sub { - my ($text) = @_; - - print "\n$text"; - return 0; -}; - -rhimport::check_key_file() or die "Import Key not found, use rhimport-create-id to create one\n"; - -my $ret; -my $log = rhimport::clear_carts($dbh, $group, $to_cart); -my $import_log; -($ret, $import_log) = rhimport::import_single($file, $dropbox, $user, 0, $error_cb); -$log .= $import_log; - -RHRD::rddb::closedb($dbh); - -unlink $file; - -if(!$ret) { - print "\nImport Error:\n"; - print $log; - exit 1; -} - -unlink($STAT_FILE); -open(my $fhs, '>', $STAT_FILE); -print $fhs $out_file; -close($fhs); -touch($NEW_FILE); - -exit 0; diff --git a/rhimport-zffe b/rhimport-zffe deleted file mode 100755 index a896551..0000000 --- a/rhimport-zffe +++ /dev/null @@ -1,118 +0,0 @@ -#!/usr/bin/perl -w -# -# -# rhimport -# -# Copyright (C) 2009-2015 Christian Pointner <equinox@helsinki.at> -# -# This file is part of rhimport. -# -# rhimport is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# any later version. -# -# rhimport is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with rhimport. If not, see <http://www.gnu.org/licenses/>. -# - -use strict; -use Switch; -use Date::Calc; -use Time::localtime; -use IO::Handle; -use IPC::Open3; -use File::Slurp; - -use lib '/usr/local/share/rhimport/'; -use rhimport; - -my $STAT_FILE = $ENV{'HOME'} . "/rhimport-zffe.stat"; -my $ZF_NOTE_FILE = $ENV{'HOME'} . "/rhimport-zf.last_note"; -my $ZF_NEW_FILE = $ENV{'HOME'} . "/rhimport-zf.is_new"; -my $FE_NOTE_FILE = $ENV{'HOME'} . "/rhimport-fe.last_note"; -my $FE_NEW_FILE = $ENV{'HOME'} . "/rhimport-fe.is_new"; -my $PV_ID = '359'; - -binmode(STDIN, ":utf8"); -binmode(STDOUT, ":utf8"); -binmode(STDERR, ":utf8"); - -if($#ARGV >= 0 && $ARGV[0] eq 'last') { - print "!!!This is the last attempt, there won't be a retry on error!!!\n" -} - -my @today = Date::Calc::Today(); -my $dow = Date::Calc::Day_of_Week(@today); -my @tomorrow = Date::Calc::Add_Delta_Days($today[0], $today[1], $today[2], 1); - -my @tmp = Date::Calc::Standard_to_Business(@today); -$tmp[2] = 1; -my @this_mon = Date::Calc::Business_to_Standard(@tmp); -my @next_mon = Date::Calc::Add_Delta_Days($this_mon[0], $this_mon[1], $this_mon[2], 7); - -my @import_day = @tomorrow; -if ($dow == 5) { - @import_day = @next_mon; -} - -my $import_date = sprintf("%04d-%02d-%02d", @import_day); -my $current_date = `cat $STAT_FILE`; -if($current_date eq $import_date) { - print "Already created note of today\n"; - exit 0; -} - -unless(-e $ZF_NEW_FILE) { - print "zip-fm not yet imported - "; - if($#ARGV >= 0 && $ARGV[0] eq 'last') { - print "giving up, rebroadcasting last show"; - } else { - print "will retry later\n"; - exit 0, - } -} - -unless(-e $FE_NEW_FILE) { - print "focus europa not yet imported - "; - if($#ARGV >= 0 && $ARGV[0] eq 'last') { - print "giving up, rebroadcasting last show"; - } else { - print "will retry later\n"; - exit 0, - } -} - -sub get_note { - my ($file) = @_; - my @lines = read_file($file, binmode => ':utf8', err_mode => 'quiet'); - return "" unless ($lines[0]); - - chomp($lines[0]); - $lines[0] = "<p><strong>$lines[0]</strong></p>\n"; - return join('',@lines); -}; - -my $sum_title = "focus europa, zip-fm (WH)"; -my $sum_text = get_note($FE_NOTE_FILE) . "<br/>\n"; -$sum_text = $sum_text . "<br/>\n" . get_note($ZF_NOTE_FILE) . "<br/>\n"; -$sum_text = $sum_text . "<br/><br/>\n"; - -print "\nsummary:\n" . $sum_title . "\n\n" . $sum_text . "\n"; -rhimport::pv_add_note($sum_title, $sum_text, $PV_ID, sprintf("%04d-%02d-%02d", @import_day), "1"); -print "\n"; - -unlink($STAT_FILE); -open(my $fhs, '>', $STAT_FILE); -print $fhs $import_date; -close($fhs); - -unlink($ZF_NEW_FILE); -unlink($FE_NEW_FILE); - -exit 0; diff --git a/rhimport-zfw b/rhimport-zfw deleted file mode 100755 index a04e249..0000000 --- a/rhimport-zfw +++ /dev/null @@ -1,195 +0,0 @@ -#!/usr/bin/perl -w -# -# -# rhimport -# -# Copyright (C) 2009-2015 Christian Pointner <equinox@helsinki.at> -# -# This file is part of rhimport. -# -# rhimport is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# any later version. -# -# rhimport is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with rhimport. If not, see <http://www.gnu.org/licenses/>. -# - -use strict; -use Date::Calc; -use Time::localtime; -use XML::Feed; -use XML::Feed::Entry; -use XML::Feed::Content; -use XML::Feed::Enclosure; -use File::Fetch; -use IO::Handle; -use IPC::Open3; -use HTML::Entities; -use RHRD::rddb; - -use lib '/usr/local/share/rhimport/'; -use rhimport; - -my $STAT_FILE = $ENV{'HOME'} . "/rhimport-zfw.stat"; -my $RSS_URL = "http://cba.fro.at/seriesrss/263764?c=Kfs2IoV2Wmd"; -$XML::Feed::MULTIPLE_ENCLOSURES=1; -my $PV_ID = '360'; - -binmode(STDIN, ":utf8"); -binmode(STDOUT, ":utf8"); -binmode(STDERR, ":utf8"); - -if($#ARGV >= 0 && $ARGV[0] eq 'last') { - print "!!!This is the last attempt, there won't be a retry on error!!!\n" -} - -my $user = `/usr/bin/id -un`; -$user =~ s/\n//; -my $group = "zipfmwla"; - -my ($dbh, $errorstring) = RHRD::rddb::opendb(); -if(!defined $dbh) { - print "$errorstring\n"; - exit 1; -} -my @allowed_dbs = RHRD::rddb::get_dropboxes($dbh, $user, $group); -if(!defined $allowed_dbs[0] && defined $allowed_dbs[2]) { - print "$allowed_dbs[2]\n"; - exit 1; -} - -if(scalar(@allowed_dbs) != 1) { - print "found no or more than one Dropboxes for this group?!\n"; - RHRD::rddb::closedb($dbh); - exit 1; -} -my $dropbox = $allowed_dbs[0]->{'PATH'}; -my $to_cart = $allowed_dbs[0]->{'TO_CART'}; - - -my @today = Date::Calc::Today(); -my @yesterday = Date::Calc::Add_Delta_Days($today[0], $today[1], $today[2], -1); -my @tomorrow = Date::Calc::Add_Delta_Days($today[0], $today[1], $today[2], 1); - -my @broadcast_day = @yesterday; -my @import_day = @today; -if(localtime->hour >= 12) { - print "It's past noon, assuming to import file from today\n"; - @broadcast_day = @today; - @import_day = @tomorrow; -} - -print "day of original broadcast: " . Date::Calc::Date_to_Text(@broadcast_day) . "\n"; -print "day of Radio Helsinki broadcast: " . Date::Calc::Date_to_Text(@import_day) . "\n"; - -my $id = sprintf("%04d-%02d-%02d", $import_day[0], $import_day[1], $import_day[2]); -my $bd = sprintf("%04d-%02d-%02d", $broadcast_day[0], $broadcast_day[1], $broadcast_day[2]); -print "looking for files from $bd in RSS Feed\n"; -print " -> $RSS_URL\n"; - -my $feed = XML::Feed->parse(URI->new($RSS_URL)) - or die "Error fetching feed: " . XML::Feed->errstr; - -my $bdfile = sprintf("%04d%02d%02d", $broadcast_day[0], $broadcast_day[1], $broadcast_day[2]); - -my $file = ""; -my $out_file = ""; -my $i = 0; -for my $entry ($feed->entries) { - $i++; - next unless $entry->enclosure; - my $j = 0; - for my $enclosure($entry->enclosure) { - $j++; - if($enclosure->type eq "audio/mpeg" || $enclosure->type eq "audio/ogg") { - print "$i/$j: (" . $enclosure->type . ", " . $enclosure->length . ") " . $enclosure->url . "\n"; - my $url = $enclosure->url; - if($url =~ /^(.*)_cut(\.[^\.]+)$/) { - $url = $1 . $2; - } - - my $ff = File::Fetch->new(uri => $url); - next unless (uc($ff->output_file) =~ $bdfile); # file not from correct day - - my ($sum_title, $sum_title_rest) = split(/:/, decode_entities($entry->title), 2); - my $sum_text = decode_entities($entry->content->body); - - my $current_stat = `cat $STAT_FILE`; - my ($current_id, $current_file) = $current_stat =~ m/^(.*)\n(.*)/; - if($current_id eq $id && $current_file eq $ff->output_file) { - print "Already downloaded file of today\n"; - RHRD::rddb::closedb($dbh); - exit 0; - } - $out_file = $ff->output_file; - if(!rhimport::check_file_extension($out_file)) { - print "\n\nThe extension of the matching file '". $out_file . "' seems to be wrong - manual import necessary!!!\n"; - RHRD::rddb::closedb($dbh); - exit 1; - } - - print " --> " . $bdfile . ": downloading " . $url . " (" . $enclosure->length . " Bytes) .. "; - $file = $ff->fetch( to => '/tmp' ) or die $ff->error; - print "ok\n"; - - print "summary:\n" . $sum_title . "\n\n" . $sum_text . "\n"; - - rhimport::pv_add_note($sum_title, $sum_text, $PV_ID, $id, "1"); - print "\n"; - - last; - } - } - last if $file ne ""; -} - -if($file eq "" || !(-e "$file")) { - print "No Entry found from $bd or download error - "; - if($#ARGV >= 0 && $ARGV[0] eq 'last') { - print "giving up, manual import necessary!!!\n"; - } else { - print "will retry later\n"; - } - exit 1; -} - -print "will import $file to dropbox $dropbox (cart=$to_cart)\n"; - -my $error_cb = sub { - my ($text) = @_; - - print "\n$text"; - return 0; -}; - -rhimport::check_key_file() or die "Import Key not found, use rhimport-create-id to create one\n"; - -my $ret; -my $log = rhimport::clear_carts($dbh, $group, $to_cart); -my $import_log; -($ret, $import_log) = rhimport::import_single($file, $dropbox, $user, 0, $error_cb); -$log .= $import_log; - -RHRD::rddb::closedb($dbh); - -unlink $file; - -if(!$ret) { - print "\nImport Error:\n"; - print $log; - exit 1; -} - -unlink($STAT_FILE); -open(my $fhs, '>', $STAT_FILE); -print $fhs "$id\n$out_file"; -close($fhs); - -exit 0; diff --git a/rhimport.pm b/rhimport.pm deleted file mode 100644 index 09c5082..0000000 --- a/rhimport.pm +++ /dev/null @@ -1,247 +0,0 @@ -#!/usr/bin/perl -w -# -# -# rhimport -# -# Copyright (C) 2009-2015 Christian Pointner <equinox@helsinki.at> -# -# This file is part of rhimport. -# -# rhimport is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# any later version. -# -# rhimport is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with rhimport. If not, see <http://www.gnu.org/licenses/>. -# - -use strict; - -package rhimport; - -use IO::Handle; -use IPC::Open3; -use File::Spec; -use File::Basename; -use URI::Escape; -use LWP::Simple; -use XML::Feed; - -my $ssh_host = "airplay"; -my $ssh_user = "rhimport"; -my $ssh_key_file = "$ENV{'HOME'}/.rhimport/import.key"; -my $ssh_dir = "/programm/.rhimport"; -my $rdimport_wrapper = "/usr/local/bin/dropbox_newfile.pl"; - -sub fetch_parse_rss -{ - my ($url, $ua_str) = @_; - - my $uri = URI->new($url); - $ua_str = "Radio Helsinki - Automatic Import" unless $ua_str; - - my $ua = LWP::UserAgent->new; - $ua->agent($ua_str); - $ua->env_proxy; - my $res = URI::Fetch->fetch($uri, UserAgent => $ua) - or die URI::Fetch->errstr; - die "This feed has been permanently removed" - if $res->status == URI::Fetch::URI_GONE(); - - my $xml = $res->content; - - return XML::Feed->parse(\$xml); -} - -sub clear_carts -{ - my ($dbh, $group, $to_cart, $progress_cb) = @_; - - my ($low_cart, $high_cart) = ($to_cart, $to_cart); - if($to_cart == 0) { - ($low_cart, $high_cart) = get_cart_range($dbh, $group); - } - - my $log = "clearing Carts $low_cart - $high_cart\n"; - print $log; - my $cart = $low_cart; - while($cart <= $high_cart) { - $progress_cb->($cart - $low_cart, $high_cart + 1 - $low_cart, "deleting $cart") if($progress_cb); - - my $sql = qq{select CUT_NAME from CUTS where CART_NUMBER='$cart';}; - my $sth = $dbh->prepare($sql); - $sth->execute(); - while(my $cut_name = $sth->fetchrow_array()) { - delete_file("/var/snd/$cut_name.wav"); - $log .= " - deleting file /var/snd/$cut_name.wav"; - } - $sth->finish(); - - $sql = qq{delete from CUTS where CART_NUMBER='$cart';}; - $sth = $dbh->prepare($sql); - $sth->execute(); - $sth->finish(); - - $sql = qq{delete from CART where NUMBER='$cart';}; - $sth = $dbh->prepare($sql); - $sth->execute(); - $sth->finish(); - - $cart++; - } - - return $log; -} - -sub import_single -{ - my ($file, $dropbox, $user, $progress_cb, $error_cb) = @_; - print "Starting import from file $file to $dropbox\n"; - $progress_cb->(0, 1, $file) if($progress_cb); - prepare_import($user); - my ($ret, $out) = import_file($file, $dropbox, $user, $error_cb); - if($ret) { - $progress_cb->(1, 1, "Import abgeschlossen!") if($progress_cb); - } else { - $progress_cb->(0, 1, "Import abgebrochen!") if($progress_cb); - } - return ($ret, $out); -} - -sub scp_put_file -{ - my ($file, $user, $host, $path) = @_; - my @cmd = ( 'scp', '-prqB', '-i', $ssh_key_file, $file, "$user\@$host:$path/" ); - my ($reader, $writer, $error ) = ( new IO::Handle, new IO::Handle, new IO::Handle ); - $writer->autoflush(1); - local $SIG{CHLD} = 'DEFAULT'; - my $pid = open3($writer, $reader, $error, @cmd); - binmode($reader, ":utf8"); - binmode($writer, ":utf8"); - binmode($error, ":utf8"); - waitpid $pid, 0; - my $errstr = ""; - if ( $? >> 8 ) { - $errstr = join('', <$error>); - } - return $errstr; -} - -sub ssh_exec_command -{ - my ($command) = @_; - my @cmd = ( 'ssh' , '-q', '-o', 'BatchMode=yes', '-i', $ssh_key_file, "$ssh_user\@$ssh_host" , $command ); - my ($reader, $writer, $error ) = ( new IO::Handle, new IO::Handle, new IO::Handle ); - $writer->autoflush(1); - local $SIG{CHLD} = 'DEFAULT'; - my $pid = open3($writer, $reader, $error, @cmd); - binmode($reader, ":utf8"); - binmode($writer, ":utf8"); - binmode($error, ":utf8"); - waitpid $pid, 0; - my $out = join('', <$reader>); - my $errstr = ""; - if ( $? >> 8 ) { - $errstr = join('', <$error>); - $errstr = "unkown error" if($errstr eq ""); - } - return ($out, $errstr); -} - -sub prepare_import -{ - my ($user) = @_; - - ### create user directory - ssh_exec_command("umask 002; mkdir -p \"$ssh_dir/$user\""); -} - -sub import_file -{ - my ($file, $dropbox, $user, $error_cb) = @_; - - print " - importing $file to $dropbox .. "; - $| = 1; - - ### copy file to master server - my $err = scp_put_file($file, $ssh_user, $ssh_host, "$ssh_dir/$user"); - if($err ne "") { - print "Transfer Error\n"; - if($error_cb) { - return $error_cb->($err); - } - return 0; - } - print "transferred .. "; - - ### remotely call rdimport - my ($volume, $directories, $remote_file) = File::Spec->splitpath($file); - $remote_file = "$ssh_dir/$user/$remote_file"; - my ($out ,$error) = ssh_exec_command("$rdimport_wrapper --path \"$dropbox\" --file \"$remote_file\""); - my $lastline = $1 if $out =~ /\n(.*)$/; - if($error ne "" || $lastline !~ /^[0-9:\- ]+:\s+Deleted file/) { - print "Import Error\n"; - delete_file($remote_file); - if($error_cb) { - if($error) { - return $error_cb->("Import Fehler: $error"); - } - else { - return $error_cb->("Import Fehler: \n$out"); - } - } - return 0; - } - - print "imported OK\n"; - return (1, $out); -} - -sub delete_file -{ - my ($filename) = @_; - - print " - deleting $filename\n"; - my ($out ,$err) = ssh_exec_command("rm \"$filename\""); - if($err ne "") { - return 0; - } - return 1; -} - -sub pv_add_note -{ - my ( $title, $text, $id, $date, $type, $index ) = @_; - my @script = ('python', '/srv/pv/pv/manage.py', 'addnote', $id, $date, $type); - push(@script , $index) unless (!defined $index); - - my ($reader, $writer, $error ) = ( new IO::Handle, new IO::Handle, new IO::Handle ); - $writer->autoflush(1); - local $SIG{CHLD} = 'DEFAULT'; - my $pid = open3($writer, $reader, $error, @script); - binmode($reader, ":utf8"); - binmode($writer, ":utf8"); - binmode($error, ":utf8"); - print $writer $title . "\n" . $text; - close $writer; - waitpid $pid, 0; - my $err_out = join('', <$error>); - my $read_out = join('', <$reader>); - if ( $? >> 8 ) { - print "\n\nPV: adding note returned non-zero value\n"; - print "STDERR:\n" . $err_out . "\n" unless $err_out eq ''; - print "STDOUT:\n" . $read_out . "\n" unless $read_out eq ''; - print "Ignoring failed headline import!\n"; - } else { - print $read_out . "\n" unless $read_out eq ''; - print $err_out . "\n" unless $err_out eq ''; - } -} - -1; -- cgit v0.10.2