summaryrefslogtreecommitdiff
path: root/src/helsinki.at
diff options
context:
space:
mode:
authorChristian Pointner <equinox@helsinki.at>2015-12-08 18:54:46 (GMT)
committerChristian Pointner <equinox@helsinki.at>2015-12-08 18:54:49 (GMT)
commit6953e452ca625a3d053260648aa2217dd7596259 (patch)
tree9706fd7cfe7101a66374162b2ed0ab5381b2cef5 /src/helsinki.at
parent3012f89fff0ee185c11dca56d8172c04c4a1987f (diff)
automatic protocol detection for curl based fetcher
Diffstat (limited to 'src/helsinki.at')
-rw-r--r--src/helsinki.at/rhimport/fetcher.go65
1 files changed, 59 insertions, 6 deletions
diff --git a/src/helsinki.at/rhimport/fetcher.go b/src/helsinki.at/rhimport/fetcher.go
index 26077b8..45bbce4 100644
--- a/src/helsinki.at/rhimport/fetcher.go
+++ b/src/helsinki.at/rhimport/fetcher.go
@@ -26,14 +26,44 @@ package rhimport
import (
"fmt"
+ "github.com/golang-basic/go-curl"
"net/url"
"path"
+ "time"
)
-func FetchFileHttp(ctx *ImportContext, uri *url.URL) (err error) {
- rhl.Printf("HTTP/HTTPS fetcher called for '%s'", ctx.SourceUri)
- ctx.SourceFile = ctx.Config.TempDir + "/" + path.Base(uri.Path)
- ctx.DeleteSourceFile = true
+func FetchFileCurl(ctx *ImportContext, uri *url.URL) (err error) {
+ rhl.Printf("curl-based fetcher called for '%s'", ctx.SourceUri)
+
+ easy := curl.EasyInit()
+ defer easy.Cleanup()
+
+ if easy != nil {
+ ctx.SourceFile = ctx.Config.TempDir + "/" + path.Base(uri.Path)
+
+ easy.Setopt(curl.OPT_URL, ctx.SourceUri)
+ easy.Setopt(curl.OPT_WRITEFUNCTION, func(ptr []byte, userdata interface{}) bool {
+ // TODO: actually store data to ctx.SourceFile
+ return true
+ })
+
+ easy.Setopt(curl.OPT_NOPROGRESS, false)
+ started := int64(0)
+ easy.Setopt(curl.OPT_PROGRESSFUNCTION, func(dltotal, dlnow, ultotal, ulnow float64, userdata interface{}) bool {
+ if started == 0 {
+ started = time.Now().Unix()
+ }
+ fmt.Printf("Downloaded: %3.2f%%, Speed: %.1fKiB/s \r", dlnow/dltotal*100, dlnow/1000/float64((time.Now().Unix()-started)))
+ return true
+ })
+
+ if err = easy.Perform(); err != nil {
+ return
+ }
+ fmt.Printf("\n")
+
+ ctx.DeleteSourceFile = true
+ }
return
}
@@ -48,12 +78,35 @@ type FetchFunc func(*ImportContext, *url.URL) (err error)
var (
fetchers = map[string]FetchFunc{
- "http": FetchFileHttp,
- "https": FetchFileHttp,
"local": FetchFileLocal,
}
+ curl_protos = map[string]bool{
+ "http": false, "https": false,
+ "ftp": false, "ftps": false,
+ }
)
+func init() {
+ curl.GlobalInit(curl.GLOBAL_ALL)
+
+ info := curl.VersionInfo(curl.VERSION_FIRST)
+ protos := info.Protocols
+ for _, proto := range protos {
+ if _, ok := curl_protos[proto]; ok {
+ rhdl.Printf("curl: enabling protocol %s", proto)
+ fetchers[proto] = FetchFileCurl
+ curl_protos[proto] = true
+ } else {
+ rhdl.Printf("curl: ignoring protocol %s", proto)
+ }
+ }
+ for proto, enabled := range curl_protos {
+ if !enabled {
+ rhl.Printf("curl: protocol %s is disabled because the installed library version doesn't support it!", proto)
+ }
+ }
+}
+
func FetchFile(ctx *ImportContext) (err error) {
var uri *url.URL