first version of curl base fetcher works now

3 files changed, 79 insertions, 9 deletions

diff --git a/src/helsinki.at/rhimport/fetcher.go b/src/helsinki.at/rhimport/fetcher.go
index 45bbce4..1770343 100644
--- a/src/helsinki.at/rhimport/fetcher.go
+++ b/src/helsinki.at/rhimport/fetcher.go
@@ -27,11 +27,61 @@ package rhimport
 import (
 	"fmt"
 	"github.com/golang-basic/go-curl"
+	"mime"
 	"net/url"
+	"os"
 	"path"
+	"strings"
 	"time"
 )
 
+type CurlCBData struct {
+	basepath   string
+	filename   string
+	remotename string
+	*os.File
+}
+
+func (self *CurlCBData) Cleanup() {
+	if self.File != nil {
+		self.File.Close()
+	}
+}
+
+func curlHeaderCallback(ptr []byte, userdata interface{}) bool {
+	hdr := fmt.Sprintf("%s", ptr)
+	data := userdata.(*CurlCBData)
+
+	if strings.HasPrefix(hdr, "Content-Disposition:") {
+		if mediatype, params, err := mime.ParseMediaType(strings.TrimPrefix(hdr, "Content-Disposition:")); err == nil {
+			if mediatype == "attachment" {
+				data.filename = data.basepath + "/" + params["filename"]
+			}
+		}
+	}
+	return true
+}
+
+func curlWriteCallback(ptr []byte, userdata interface{}) bool {
+	data := userdata.(*CurlCBData)
+	if data.File == nil {
+		if data.filename == "" {
+			data.filename = data.basepath + "/" + data.remotename
+		}
+		fp, err := os.OpenFile(data.filename, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600)
+		if err != nil {
+			rhl.Printf("Unable to create file %s: %s", data.filename, err)
+			return false
+		}
+		data.File = fp
+	}
+	if _, err := data.File.Write(ptr); err != nil {
+		rhl.Printf("Unable to write file %s: %s", data.filename, err)
+		return false
+	}
+	return true
+}
+
 func FetchFileCurl(ctx *ImportContext, uri *url.URL) (err error) {
 	rhl.Printf("curl-based fetcher called for '%s'", ctx.SourceUri)
 
@@ -39,13 +89,19 @@ func FetchFileCurl(ctx *ImportContext, uri *url.URL) (err error) {
 	defer easy.Cleanup()
 
 	if easy != nil {
-		ctx.SourceFile = ctx.Config.TempDir + "/" + path.Base(uri.Path)
 
+		easy.Setopt(curl.OPT_FOLLOWLOCATION, true)
 		easy.Setopt(curl.OPT_URL, ctx.SourceUri)
-		easy.Setopt(curl.OPT_WRITEFUNCTION, func(ptr []byte, userdata interface{}) bool {
-			// TODO: actually store data to ctx.SourceFile
-			return true
-		})
+
+		cbdata := &CurlCBData{remotename: path.Base(uri.Path)}
+		defer cbdata.Cleanup()
+		cbdata.basepath = ctx.Config.TempDir // TODO: create temporary directory
+
+		easy.Setopt(curl.OPT_HEADERFUNCTION, curlHeaderCallback)
+		easy.Setopt(curl.OPT_HEADERDATA, cbdata)
+
+		easy.Setopt(curl.OPT_WRITEFUNCTION, curlWriteCallback)
+		easy.Setopt(curl.OPT_WRITEDATA, cbdata)
 
 		easy.Setopt(curl.OPT_NOPROGRESS, false)
 		started := int64(0)
@@ -56,12 +112,14 @@ func FetchFileCurl(ctx *ImportContext, uri *url.URL) (err error) {
 			fmt.Printf("Downloaded: %3.2f%%, Speed: %.1fKiB/s \r", dlnow/dltotal*100, dlnow/1000/float64((time.Now().Unix()-started)))
 			return true
 		})
+		easy.Setopt(curl.OPT_PROGRESSDATA, ctx)
 
 		if err = easy.Perform(); err != nil {
 			return
 		}
 		fmt.Printf("\n")
 
+		ctx.SourceFile = cbdata.filename
 		ctx.DeleteSourceFile = true
 	}
 	return
diff --git a/src/helsinki.at/rhimportd/rhimportd.go b/src/helsinki.at/rhimportd/rhimportd.go
index b8c5e3b..9e395d8 100644
--- a/src/helsinki.at/rhimportd/rhimportd.go
+++ b/src/helsinki.at/rhimportd/rhimportd.go
@@ -33,10 +33,10 @@ import (
 )
 
 func main() {
-	web_addr_s := flag.String("web-addr", ":4000", "addr:port to listen on, default: ':4000'")
-	rdconf_s := flag.String("rdconf", "/etc/rd.conf", "path to the Rivendell config file, default: '/etc/rd.conf'")
-	rdxport_url_s := flag.String("rdxport-url", "http://localhost/rd-bin/rdxport.cgi", "the url to the Rivendell web-api, default: 'http://localhost/rd-bin/rdxport.cgi'")
-	temp_dir_s := flag.String("tmp-dir", "/tmp", "path to temporary files, default: '/tmp'")
+	web_addr_s := flag.String("web-addr", ":4000", "addr:port to listen on")
+	rdconf_s := flag.String("rdconf", "/etc/rd.conf", "path to the Rivendell config file")
+	rdxport_url_s := flag.String("rdxport-url", "http://localhost/rd-bin/rdxport.cgi", "the url to the Rivendell web-api")
+	temp_dir_s := flag.String("tmp-dir", os.TempDir(), "path to temporary files")
 	help := flag.Bool("help", false, "show usage")
 
 	flag.Parse()
diff --git a/test/simple3.json b/test/simple3.json
new file mode 100644
index 0000000..969b2b6
--- /dev/null
+++ b/test/simple3.json
@@ -0,0 +1,12 @@
+{
+    "LOGIN_NAME": "heslinki",
+    "PASSWORD": "123456",
+    "GROUP_NAME": "test",
+    "CART_NUMBER": 10000,
+    "CUT_NUMBER": 1,
+    "CHANNELS": 2,
+    "NORMALIZATION_LEVEL": -1200,
+    "AUTOTRIM_LEVEL": 0,
+    "USE_METADATA": true,
+    "SOURCE_URI": "http://www.jtricks.com/download-unknown"
+}