automatic protocol detection for curl based fetcher

3 files changed, 61 insertions, 7 deletions

diff --git a/Makefile b/Makefile
index 9817199..469222e 100644
--- a/Makefile
+++ b/Makefile
@@ -29,6 +29,7 @@ getlibs: export GOPATH=$(curdir)
 getlibs:
 	$(GOCMD) get "github.com/vaughan0/go-ini"
 	$(GOCMD) get "github.com/go-sql-driver/mysql"
+	$(GOCMD) get "github.com/golang-basic/go-curl"
 #	$(GOCMD) get "github.com/gorilla/websocket"
 
 vet: export GOPATH=$(curdir)
diff --git a/src/helsinki.at/rhimport/fetcher.go b/src/helsinki.at/rhimport/fetcher.go
index 26077b8..45bbce4 100644
--- a/src/helsinki.at/rhimport/fetcher.go
+++ b/src/helsinki.at/rhimport/fetcher.go
@@ -26,14 +26,44 @@ package rhimport
 
 import (
 	"fmt"
+	"github.com/golang-basic/go-curl"
 	"net/url"
 	"path"
+	"time"
 )
 
-func FetchFileHttp(ctx *ImportContext, uri *url.URL) (err error) {
-	rhl.Printf("HTTP/HTTPS fetcher called for '%s'", ctx.SourceUri)
-	ctx.SourceFile = ctx.Config.TempDir + "/" + path.Base(uri.Path)
-	ctx.DeleteSourceFile = true
+func FetchFileCurl(ctx *ImportContext, uri *url.URL) (err error) {
+	rhl.Printf("curl-based fetcher called for '%s'", ctx.SourceUri)
+
+	easy := curl.EasyInit()
+	defer easy.Cleanup()
+
+	if easy != nil {
+		ctx.SourceFile = ctx.Config.TempDir + "/" + path.Base(uri.Path)
+
+		easy.Setopt(curl.OPT_URL, ctx.SourceUri)
+		easy.Setopt(curl.OPT_WRITEFUNCTION, func(ptr []byte, userdata interface{}) bool {
+			// TODO: actually store data to ctx.SourceFile
+			return true
+		})
+
+		easy.Setopt(curl.OPT_NOPROGRESS, false)
+		started := int64(0)
+		easy.Setopt(curl.OPT_PROGRESSFUNCTION, func(dltotal, dlnow, ultotal, ulnow float64, userdata interface{}) bool {
+			if started == 0 {
+				started = time.Now().Unix()
+			}
+			fmt.Printf("Downloaded: %3.2f%%, Speed: %.1fKiB/s \r", dlnow/dltotal*100, dlnow/1000/float64((time.Now().Unix()-started)))
+			return true
+		})
+
+		if err = easy.Perform(); err != nil {
+			return
+		}
+		fmt.Printf("\n")
+
+		ctx.DeleteSourceFile = true
+	}
 	return
 }
 
@@ -48,12 +78,35 @@ type FetchFunc func(*ImportContext, *url.URL) (err error)
 
 var (
 	fetchers = map[string]FetchFunc{
-		"http":  FetchFileHttp,
-		"https": FetchFileHttp,
 		"local": FetchFileLocal,
 	}
+	curl_protos = map[string]bool{
+		"http": false, "https": false,
+		"ftp": false, "ftps": false,
+	}
 )
 
+func init() {
+	curl.GlobalInit(curl.GLOBAL_ALL)
+
+	info := curl.VersionInfo(curl.VERSION_FIRST)
+	protos := info.Protocols
+	for _, proto := range protos {
+		if _, ok := curl_protos[proto]; ok {
+			rhdl.Printf("curl: enabling protocol %s", proto)
+			fetchers[proto] = FetchFileCurl
+			curl_protos[proto] = true
+		} else {
+			rhdl.Printf("curl: ignoring protocol %s", proto)
+		}
+	}
+	for proto, enabled := range curl_protos {
+		if !enabled {
+			rhl.Printf("curl: protocol %s is disabled because the installed library version doesn't support it!", proto)
+		}
+	}
+}
+
 func FetchFile(ctx *ImportContext) (err error) {
 
 	var uri *url.URL
diff --git a/test/simple1.json b/test/simple1.json
index 3083de7..f952a70 100644
--- a/test/simple1.json
+++ b/test/simple1.json
@@ -8,5 +8,5 @@
     "NORMALIZATION_LEVEL": -1200,
     "AUTOTRIM_LEVEL": 0,
     "USE_METADATA": true,
-    "SOURCE_URI": "https://example.com/files/sendung.mp3"
+    "SOURCE_URI": "http://www.tonycuffe.com/mp3/tail%20toddle.mp3"
 }