summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--src/helsinki.at/rhimport/fetcher.go68
-rw-r--r--src/helsinki.at/rhimportd/rhimportd.go8
-rw-r--r--test/simple3.json12
3 files changed, 79 insertions, 9 deletions
diff --git a/src/helsinki.at/rhimport/fetcher.go b/src/helsinki.at/rhimport/fetcher.go
index 45bbce4..1770343 100644
--- a/src/helsinki.at/rhimport/fetcher.go
+++ b/src/helsinki.at/rhimport/fetcher.go
@@ -27,11 +27,61 @@ package rhimport
import (
"fmt"
"github.com/golang-basic/go-curl"
+ "mime"
"net/url"
+ "os"
"path"
+ "strings"
"time"
)
+type CurlCBData struct {
+ basepath string
+ filename string
+ remotename string
+ *os.File
+}
+
+func (self *CurlCBData) Cleanup() {
+ if self.File != nil {
+ self.File.Close()
+ }
+}
+
+func curlHeaderCallback(ptr []byte, userdata interface{}) bool {
+ hdr := fmt.Sprintf("%s", ptr)
+ data := userdata.(*CurlCBData)
+
+ if strings.HasPrefix(hdr, "Content-Disposition:") {
+ if mediatype, params, err := mime.ParseMediaType(strings.TrimPrefix(hdr, "Content-Disposition:")); err == nil {
+ if mediatype == "attachment" {
+ data.filename = data.basepath + "/" + params["filename"]
+ }
+ }
+ }
+ return true
+}
+
+func curlWriteCallback(ptr []byte, userdata interface{}) bool {
+ data := userdata.(*CurlCBData)
+ if data.File == nil {
+ if data.filename == "" {
+ data.filename = data.basepath + "/" + data.remotename
+ }
+ fp, err := os.OpenFile(data.filename, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600)
+ if err != nil {
+ rhl.Printf("Unable to create file %s: %s", data.filename, err)
+ return false
+ }
+ data.File = fp
+ }
+ if _, err := data.File.Write(ptr); err != nil {
+ rhl.Printf("Unable to write file %s: %s", data.filename, err)
+ return false
+ }
+ return true
+}
+
func FetchFileCurl(ctx *ImportContext, uri *url.URL) (err error) {
rhl.Printf("curl-based fetcher called for '%s'", ctx.SourceUri)
@@ -39,13 +89,19 @@ func FetchFileCurl(ctx *ImportContext, uri *url.URL) (err error) {
defer easy.Cleanup()
if easy != nil {
- ctx.SourceFile = ctx.Config.TempDir + "/" + path.Base(uri.Path)
+ easy.Setopt(curl.OPT_FOLLOWLOCATION, true)
easy.Setopt(curl.OPT_URL, ctx.SourceUri)
- easy.Setopt(curl.OPT_WRITEFUNCTION, func(ptr []byte, userdata interface{}) bool {
- // TODO: actually store data to ctx.SourceFile
- return true
- })
+
+ cbdata := &CurlCBData{remotename: path.Base(uri.Path)}
+ defer cbdata.Cleanup()
+ cbdata.basepath = ctx.Config.TempDir // TODO: create temporary directory
+
+ easy.Setopt(curl.OPT_HEADERFUNCTION, curlHeaderCallback)
+ easy.Setopt(curl.OPT_HEADERDATA, cbdata)
+
+ easy.Setopt(curl.OPT_WRITEFUNCTION, curlWriteCallback)
+ easy.Setopt(curl.OPT_WRITEDATA, cbdata)
easy.Setopt(curl.OPT_NOPROGRESS, false)
started := int64(0)
@@ -56,12 +112,14 @@ func FetchFileCurl(ctx *ImportContext, uri *url.URL) (err error) {
fmt.Printf("Downloaded: %3.2f%%, Speed: %.1fKiB/s \r", dlnow/dltotal*100, dlnow/1000/float64((time.Now().Unix()-started)))
return true
})
+ easy.Setopt(curl.OPT_PROGRESSDATA, ctx)
if err = easy.Perform(); err != nil {
return
}
fmt.Printf("\n")
+ ctx.SourceFile = cbdata.filename
ctx.DeleteSourceFile = true
}
return
diff --git a/src/helsinki.at/rhimportd/rhimportd.go b/src/helsinki.at/rhimportd/rhimportd.go
index b8c5e3b..9e395d8 100644
--- a/src/helsinki.at/rhimportd/rhimportd.go
+++ b/src/helsinki.at/rhimportd/rhimportd.go
@@ -33,10 +33,10 @@ import (
)
func main() {
- web_addr_s := flag.String("web-addr", ":4000", "addr:port to listen on, default: ':4000'")
- rdconf_s := flag.String("rdconf", "/etc/rd.conf", "path to the Rivendell config file, default: '/etc/rd.conf'")
- rdxport_url_s := flag.String("rdxport-url", "http://localhost/rd-bin/rdxport.cgi", "the url to the Rivendell web-api, default: 'http://localhost/rd-bin/rdxport.cgi'")
- temp_dir_s := flag.String("tmp-dir", "/tmp", "path to temporary files, default: '/tmp'")
+ web_addr_s := flag.String("web-addr", ":4000", "addr:port to listen on")
+ rdconf_s := flag.String("rdconf", "/etc/rd.conf", "path to the Rivendell config file")
+ rdxport_url_s := flag.String("rdxport-url", "http://localhost/rd-bin/rdxport.cgi", "the url to the Rivendell web-api")
+ temp_dir_s := flag.String("tmp-dir", os.TempDir(), "path to temporary files")
help := flag.Bool("help", false, "show usage")
flag.Parse()
diff --git a/test/simple3.json b/test/simple3.json
new file mode 100644
index 0000000..969b2b6
--- /dev/null
+++ b/test/simple3.json
@@ -0,0 +1,12 @@
+{
+ "LOGIN_NAME": "heslinki",
+ "PASSWORD": "123456",
+ "GROUP_NAME": "test",
+ "CART_NUMBER": 10000,
+ "CUT_NUMBER": 1,
+ "CHANNELS": 2,
+ "NORMALIZATION_LEVEL": -1200,
+ "AUTOTRIM_LEVEL": 0,
+ "USE_METADATA": true,
+ "SOURCE_URI": "http://www.jtricks.com/download-unknown"
+}