From d85c6d205ac64681e49be76d0e50134dfa21036e Mon Sep 17 00:00:00 2001
From: Christian Pointner <equinox@helsinki.at>
Date: Sat, 23 Apr 2016 02:11:31 +0200
Subject: all cba importer now use wget (switch to https...)
diff --git a/rhimport-nw b/rhimport-nw
index 2fe4253..e2db9b8 100755
--- a/rhimport-nw
+++ b/rhimport-nw
@@ -28,7 +28,7 @@ use XML::Feed;
use XML::Feed::Entry;
use XML::Feed::Content;
use XML::Feed::Enclosure;
-use File::Fetch;
+use URI::URL;
use IO::Handle;
use IPC::Open3;
use HTML::Entities;
@@ -42,7 +42,7 @@ my $DBPW = "lldriven";
my $DB = "rivendell";
my $STAT_FILE = $ENV{'HOME'} . "/rhimport-nw.stat";
-my $RSS_URL = "http://cba.fro.at/seriesrss/262514?c=Kfs2IoV2Wmd";
+my $RSS_URL = "https://cba.fro.at/seriesrss/262514?c=Kfs2IoV2Wmd";
$XML::Feed::MULTIPLE_ENCLOSURES=1;
my $PV_ID = '73';
@@ -103,24 +103,26 @@ for my $entry ($feed->entries) {
$j++;
if($enclosure->type eq "audio/mpeg" || $enclosure->type eq "audio/ogg") {
print "$i/$j: (" . $enclosure->type . ", " . $enclosure->length . ") " . $enclosure->url . "\n";
- my $url = $enclosure->url;
- if($url =~ /^(.*)_cut(\.[^\.]+)$/) {
- $url = $1 . $2;
+ my $fetchurl = $enclosure->url;
+ if($fetchurl =~ /^(.*)_cut(\.[^\.]+)$/) {
+ $fetchurl = $1 . $2;
}
- my $ff = File::Fetch->new(uri => $url);
- next unless (uc($ff->output_file) =~ $bdfile); # file not from correct day
+ my $url = new URI::URL($fetchurl);
+ my @path = $url->path_components;
+
+ next unless (uc($path[-1]) =~ $bdfile); # file not from correct day
my $sum_title = decode_entities($entry->title);
my $sum_text = decode_entities($entry->content->body);
my $current_file = `cat $STAT_FILE`;
- if($current_file eq $ff->output_file) {
+ if($current_file eq $path[-1]) {
print "Already downloaded file of today\n";
$dbh->disconnect();
exit 0;
}
- $out_file = $ff->output_file;
+ $out_file = $path[-1];
if(!rhimport::check_file_extension($out_file)) {
print "\n\nThe extension of the matching file '". $out_file . "' seems to be wrong - manual import necessary!!!\n";
$dbh->disconnect();
@@ -128,7 +130,11 @@ for my $entry ($feed->entries) {
}
print " --> " . $bdfile . ": downloading " . $url . " (" . $enclosure->length . " Bytes) .. ";
- $file = $ff->fetch( to => '/tmp' ) or die $ff->error;
+ $file = "/tmp/" . $out_file;
+ system("wget", "--quiet", "--no-check-certificate", $url, "-O", $file);
+ if( $! != 0) {
+ die "wget returned with error: " . $!;
+ }
print "ok\n";
print "summary:\n" . $sum_title . "\n\n" . $sum_text . "\n";
diff --git a/rhimport-ra b/rhimport-ra
index ec30851..ef7e3fb 100755
--- a/rhimport-ra
+++ b/rhimport-ra
@@ -28,7 +28,7 @@ use XML::Feed;
use XML::Feed::Entry;
use XML::Feed::Content;
use XML::Feed::Enclosure;
-use File::Fetch;
+use URI::URL;
use IO::Handle;
use IPC::Open3;
use HTML::Entities;
@@ -42,7 +42,7 @@ my $DBPW = "lldriven";
my $DB = "rivendell";
my $STAT_FILE = $ENV{'HOME'} . "/rhimport-ra.stat";
-my $RSS_URL = "http://cba.fro.at/seriesrss/262430?c=Kfs2IoV2Wmd";
+my $RSS_URL = "https://cba.fro.at/seriesrss/262430?c=Kfs2IoV2Wmd";
$XML::Feed::MULTIPLE_ENCLOSURES=1;
my $PV_ID = '76';
@@ -104,9 +104,9 @@ for my $entry ($feed->entries) {
$j++;
if($enclosure->type eq "audio/mpeg" || $enclosure->type eq "audio/ogg") {
print "$i/$j: (" . $enclosure->type . ", " . $enclosure->length . ") " . $enclosure->url . "\n";
- my $url = $enclosure->url;
- if($url =~ /^(.*)_cut(\.[^\.]+)$/) {
- $url = $1 . $2;
+ my $fetchurl = $enclosure->url;
+ if($fetchurl =~ /^(.*)_cut(\.[^\.]+)$/) {
+ $fetchurl = $1 . $2;
}
my $sum_title = decode_entities($entry->title);
@@ -114,15 +114,16 @@ for my $entry ($feed->entries) {
my $sum_text = decode_entities($entry->content->body);
my $bnum = $1;
- my $ff = File::Fetch->new(uri => $url);
- next unless $bnum eq substr($ff->output_file, 0, 3);
+ my $url = new URI::URL($fetchurl);
+ my @path = $url->path_components;
+ next unless $bnum eq substr($path[-1], 0, 3);
my $current_file = `cat $STAT_FILE`;
- if($current_file eq $ff->output_file) {
+ if($current_file eq $path[-1]) {
print "Already downloaded file of today\n";
$dbh->disconnect();
exit 0;
}
- $out_file = $ff->output_file;
+ $out_file = $path[-1];
if(!rhimport::check_file_extension($out_file)) {
print "\n\nThe extension of the matching file '". $out_file . "' seems to be wrong - manual import necessary!!!\n";
$dbh->disconnect();
@@ -130,7 +131,11 @@ for my $entry ($feed->entries) {
}
print " --> " . $bnum . ": downloading " . $url . " (" . $enclosure->length . " Bytes) .. ";
- $file = $ff->fetch( to => '/tmp' ) or die $ff->error;
+ $file = "/tmp/" . $out_file;
+ system("wget", "--quiet", "--no-check-certificate", $url, "-O", $file);
+ if( $! != 0) {
+ die "wget returned with error: " . $!;
+ }
print "ok\n";
print "summary:\n" . $sum_title . "\n\n" . $sum_text . "\n";
diff --git a/rhimport-rs b/rhimport-rs
index c8e091a..bb1453a 100755
--- a/rhimport-rs
+++ b/rhimport-rs
@@ -43,7 +43,7 @@ my $DBPW = "lldriven";
my $DB = "rivendell";
my $STAT_FILE = $ENV{'HOME'} . "/rhimport-rs.stat";
-my $RSS_URL = "http://cba.fro.at/seriesrss/262433?c=Kfs2IoV2Wmd";
+my $RSS_URL = "https://cba.fro.at/seriesrss/262433?c=Kfs2IoV2Wmd";
$XML::Feed::MULTIPLE_ENCLOSURES=1;
my $PV_ID = '75';
diff --git a/rhimport-tr b/rhimport-tr
index e02ae0c..b7d6916 100755
--- a/rhimport-tr
+++ b/rhimport-tr
@@ -28,7 +28,7 @@ use XML::Feed;
use XML::Feed::Entry;
use XML::Feed::Content;
use XML::Feed::Enclosure;
-use File::Fetch;
+use URI::URL;
use IO::Handle;
use IPC::Open3;
use HTML::Entities;
@@ -42,7 +42,7 @@ my $DBPW = "lldriven";
my $DB = "rivendell";
my $STAT_FILE = $ENV{'HOME'} . "/rhimport-tr.stat";
-my $RSS_URL = "http://cba.fro.at/series/tierrechtsradio/feed?c=Kfs2IoV2Wmd";
+my $RSS_URL = "https://cba.fro.at/series/tierrechtsradio/feed?c=Kfs2IoV2Wmd";
$XML::Feed::MULTIPLE_ENCLOSURES=1;
my $PV_ID = '506';
@@ -105,24 +105,26 @@ for my $entry ($feed->entries) {
$j++;
if($enclosure->type eq "audio/mpeg" || $enclosure->type eq "audio/ogg") {
print "$i/$j: (" . $enclosure->type . ", " . $enclosure->length . ") " . $enclosure->url . "\n";
- my $url = $enclosure->url;
- if($url =~ /^(.*)_cut(\.[^\.]+)$/) {
- $url = $1 . $2;
+ my $fetchurl = $enclosure->url;
+ if($fetchurl =~ /^(.*)_cut(\.[^\.]+)$/) {
+ $fetchurl = $1 . $2;
}
- my $ff = File::Fetch->new(uri => $url);
- next unless (uc($ff->output_file) =~ $bdfile); # file not from correct day
+ my $url = new URI::URL($fetchurl);
+ my @path = $url->path_components;
+
+ next unless (uc($path[-1]) =~ $bdfile); # file not from correct day
my $sum_title = decode_entities($entry->title);
my $sum_text = decode_entities($entry->content->body);
my $current_file = `cat $STAT_FILE`;
- if($current_file eq $ff->output_file) {
+ if($current_file eq $path[-1]) {
print "Already downloaded file of today\n";
$dbh->disconnect();
exit 0;
}
- $out_file = $ff->output_file;
+ $out_file = $path[-1];
if(!rhimport::check_file_extension($out_file)) {
print "\n\nThe extension of the matching file '". $out_file . "' seems to be wrong - manual import necessary!!!\n";
$dbh->disconnect();
@@ -130,7 +132,11 @@ for my $entry ($feed->entries) {
}
print " --> " . $bdfile . ": downloading " . $url . " (" . $enclosure->length . " Bytes) .. ";
- $file = $ff->fetch( to => '/tmp' ) or die $ff->error;
+ $file = "/tmp/" . $out_file;
+ system("wget", "--quiet", "--no-check-certificate", $url, "-O", $file);
+ if( $! != 0) {
+ die "wget returned with error: " . $!;
+ }
print "ok\n";
print "summary:\n" . $sum_title . "\n\n" . $sum_text . "\n";
--
cgit v0.10.2