diff options
| -rw-r--r-- | src/helsinki.at/rhimport/fetcher.go | 68 | ||||
| -rw-r--r-- | src/helsinki.at/rhimportd/rhimportd.go | 8 | ||||
| -rw-r--r-- | test/simple3.json | 12 |
3 files changed, 79 insertions, 9 deletions
diff --git a/src/helsinki.at/rhimport/fetcher.go b/src/helsinki.at/rhimport/fetcher.go index 45bbce4..1770343 100644 --- a/src/helsinki.at/rhimport/fetcher.go +++ b/src/helsinki.at/rhimport/fetcher.go @@ -27,11 +27,61 @@ package rhimport import ( "fmt" "github.com/golang-basic/go-curl" + "mime" "net/url" + "os" "path" + "strings" "time" ) +type CurlCBData struct { + basepath string + filename string + remotename string + *os.File +} + +func (self *CurlCBData) Cleanup() { + if self.File != nil { + self.File.Close() + } +} + +func curlHeaderCallback(ptr []byte, userdata interface{}) bool { + hdr := fmt.Sprintf("%s", ptr) + data := userdata.(*CurlCBData) + + if strings.HasPrefix(hdr, "Content-Disposition:") { + if mediatype, params, err := mime.ParseMediaType(strings.TrimPrefix(hdr, "Content-Disposition:")); err == nil { + if mediatype == "attachment" { + data.filename = data.basepath + "/" + params["filename"] + } + } + } + return true +} + +func curlWriteCallback(ptr []byte, userdata interface{}) bool { + data := userdata.(*CurlCBData) + if data.File == nil { + if data.filename == "" { + data.filename = data.basepath + "/" + data.remotename + } + fp, err := os.OpenFile(data.filename, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + if err != nil { + rhl.Printf("Unable to create file %s: %s", data.filename, err) + return false + } + data.File = fp + } + if _, err := data.File.Write(ptr); err != nil { + rhl.Printf("Unable to write file %s: %s", data.filename, err) + return false + } + return true +} + func FetchFileCurl(ctx *ImportContext, uri *url.URL) (err error) { rhl.Printf("curl-based fetcher called for '%s'", ctx.SourceUri) @@ -39,13 +89,19 @@ func FetchFileCurl(ctx *ImportContext, uri *url.URL) (err error) { defer easy.Cleanup() if easy != nil { - ctx.SourceFile = ctx.Config.TempDir + "/" + path.Base(uri.Path) + easy.Setopt(curl.OPT_FOLLOWLOCATION, true) easy.Setopt(curl.OPT_URL, ctx.SourceUri) - easy.Setopt(curl.OPT_WRITEFUNCTION, func(ptr []byte, userdata interface{}) bool { - // TODO: actually store data to ctx.SourceFile - return true - }) + + cbdata := &CurlCBData{remotename: path.Base(uri.Path)} + defer cbdata.Cleanup() + cbdata.basepath = ctx.Config.TempDir // TODO: create temporary directory + + easy.Setopt(curl.OPT_HEADERFUNCTION, curlHeaderCallback) + easy.Setopt(curl.OPT_HEADERDATA, cbdata) + + easy.Setopt(curl.OPT_WRITEFUNCTION, curlWriteCallback) + easy.Setopt(curl.OPT_WRITEDATA, cbdata) easy.Setopt(curl.OPT_NOPROGRESS, false) started := int64(0) @@ -56,12 +112,14 @@ func FetchFileCurl(ctx *ImportContext, uri *url.URL) (err error) { fmt.Printf("Downloaded: %3.2f%%, Speed: %.1fKiB/s \r", dlnow/dltotal*100, dlnow/1000/float64((time.Now().Unix()-started))) return true }) + easy.Setopt(curl.OPT_PROGRESSDATA, ctx) if err = easy.Perform(); err != nil { return } fmt.Printf("\n") + ctx.SourceFile = cbdata.filename ctx.DeleteSourceFile = true } return diff --git a/src/helsinki.at/rhimportd/rhimportd.go b/src/helsinki.at/rhimportd/rhimportd.go index b8c5e3b..9e395d8 100644 --- a/src/helsinki.at/rhimportd/rhimportd.go +++ b/src/helsinki.at/rhimportd/rhimportd.go @@ -33,10 +33,10 @@ import ( ) func main() { - web_addr_s := flag.String("web-addr", ":4000", "addr:port to listen on, default: ':4000'") - rdconf_s := flag.String("rdconf", "/etc/rd.conf", "path to the Rivendell config file, default: '/etc/rd.conf'") - rdxport_url_s := flag.String("rdxport-url", "http://localhost/rd-bin/rdxport.cgi", "the url to the Rivendell web-api, default: 'http://localhost/rd-bin/rdxport.cgi'") - temp_dir_s := flag.String("tmp-dir", "/tmp", "path to temporary files, default: '/tmp'") + web_addr_s := flag.String("web-addr", ":4000", "addr:port to listen on") + rdconf_s := flag.String("rdconf", "/etc/rd.conf", "path to the Rivendell config file") + rdxport_url_s := flag.String("rdxport-url", "http://localhost/rd-bin/rdxport.cgi", "the url to the Rivendell web-api") + temp_dir_s := flag.String("tmp-dir", os.TempDir(), "path to temporary files") help := flag.Bool("help", false, "show usage") flag.Parse() diff --git a/test/simple3.json b/test/simple3.json new file mode 100644 index 0000000..969b2b6 --- /dev/null +++ b/test/simple3.json @@ -0,0 +1,12 @@ +{ + "LOGIN_NAME": "heslinki", + "PASSWORD": "123456", + "GROUP_NAME": "test", + "CART_NUMBER": 10000, + "CUT_NUMBER": 1, + "CHANNELS": 2, + "NORMALIZATION_LEVEL": -1200, + "AUTOTRIM_LEVEL": 0, + "USE_METADATA": true, + "SOURCE_URI": "http://www.jtricks.com/download-unknown" +} |