summaryrefslogtreecommitdiff
path: root/rhimport
diff options
context:
space:
mode:
Diffstat (limited to 'rhimport')
-rw-r--r--rhimport/fetcher.go95
1 files changed, 85 insertions, 10 deletions
diff --git a/rhimport/fetcher.go b/rhimport/fetcher.go
index 061d0e1..1abf736 100644
--- a/rhimport/fetcher.go
+++ b/rhimport/fetcher.go
@@ -26,6 +26,7 @@ package rhimport
import (
"fmt"
+ "io"
"io/ioutil"
"mime"
"net/http"
@@ -86,7 +87,7 @@ func curlWriteCallback(ptr []byte, userdata interface{}) bool {
data.conv = conv
}
if _, err := data.conv.Write(ptr); err != nil {
- rhl.Printf("Unable to write file %s: %s", data.filename, err)
+ rhl.Printf("Unable to write to converter(%s): %s", data.filename, err)
data.writeError = err
return false
}
@@ -308,14 +309,79 @@ func fetchFileArchiv(ctx *Context, res *Result, uri *url.URL) (err error) {
}
func fetchFileLocal(ctx *Context, res *Result, uri *url.URL) (err error) {
- return fetchFileDir(ctx, res, uri, ctx.conf.LocalFetchDir)
+ return fetchFileDir(ctx, res, uri, ctx.conf.LocalFetchDir, true)
}
func fetchFileTmp(ctx *Context, res *Result, uri *url.URL) (err error) {
- return fetchFileDir(ctx, res, uri, ctx.conf.TempDir)
+ return fetchFileDir(ctx, res, uri, ctx.conf.TempDir, false)
}
-func fetchFileDir(ctx *Context, res *Result, uri *url.URL, dir string) (err error) {
+func fetchFileDirConvert(ctx *Context, res *Result, origSrc *os.File, sizeTotal int64) (err error) {
+ var conv FetchConverter
+ if conv, ctx.SourceFile, err = NewFetchConverter(ctx.FetchConverter, ctx.SourceFile); err != nil {
+ rhl.Printf("Unable to create converter for file %s: %s", origSrc.Name(), err)
+ return
+ }
+
+ var buffer [1 * 1024 * 1024]byte
+ written := uint64(0)
+ for {
+ var r, w int
+
+ r, err = origSrc.Read(buffer[:])
+ if err == io.EOF {
+ err = nil
+ break
+ }
+ if err != nil {
+ rhl.Printf("Unable to read from source file %s: %s", origSrc.Name(), err)
+ break
+ }
+ w, err = conv.Write(buffer[0:r])
+ if err != nil {
+ rhl.Printf("Unable to write to converter(%s): %s", ctx.SourceFile, err)
+ break
+ }
+ written += uint64(w)
+
+ if ctx.ProgressCallBack != nil {
+ if keep := ctx.ProgressCallBack(1, "fetching", float64(written), float64(sizeTotal), ctx.ProgressCallBackData); !keep {
+ ctx.ProgressCallBack = nil
+ }
+ }
+ }
+
+ conv.Close()
+ rhl.Printf("waiting for converter to finish...")
+ if convOut, convErr := conv.GetResult(); convErr != nil {
+ if convOut != "" {
+ rhl.Printf("converter error: %v; converter output: %s", convErr, convOut)
+ }
+ return fmt.Errorf("converter error: %v; converter output: %s", convErr, convOut)
+ }
+ if err != nil {
+ return err
+ }
+
+ // TODO: delete origSrc file when SourceFilePolicy == delete or DeleteWithDir
+ switch ctx.SourceFilePolicy {
+ case Auto:
+ ctx.DeleteSourceFile = true
+ ctx.DeleteSourceDir = true
+ case Keep:
+ ctx.DeleteSourceFile = false
+ ctx.DeleteSourceDir = false
+ case DeleteWithDir:
+ ctx.DeleteSourceDir = true
+ fallthrough
+ case Delete:
+ ctx.DeleteSourceFile = true
+ }
+
+ return
+}
+
+func fetchFileDir(ctx *Context, res *Result, uri *url.URL, dir string, convert bool) (err error) {
rhl.Printf("Dir fetcher called for '%s'", ctx.SourceUri)
ctx.SourceFile = filepath.Join(dir, path.Clean("/"+uri.Path))
@@ -352,15 +418,22 @@ func fetchFileDir(ctx *Context, res *Result, uri *url.URL, dir string) (err erro
}
}
+ if convert {
+ if err = fetchFileDirConvert(ctx, res, src, size); err != nil {
+ return
+ }
+ } else {
+ if ctx.SourceFilePolicy == Auto {
+ ctx.DeleteSourceFile = false
+ ctx.DeleteSourceDir = false
+ }
+ }
+
if ctx.ProgressCallBack != nil {
if keep := ctx.ProgressCallBack(1, "fetching", float64(size), float64(size), ctx.ProgressCallBackData); !keep {
ctx.ProgressCallBack = nil
}
}
- if ctx.SourceFilePolicy == Auto {
- ctx.DeleteSourceFile = false
- ctx.DeleteSourceDir = false
- }
return
}
@@ -431,7 +504,7 @@ func writeAttachmentFile(ctx *Context, res *Result, sizeTotal uint64, conv Fetch
w, err := conv.Write(data)
if err != nil {
- rhl.Printf("Unable to write file %s: %s", ctx.SourceFile, err)
+ rhl.Printf("Unable to write to converter(%s): %s", ctx.SourceFile, err)
return err
}
written += uint64(w)
@@ -476,10 +549,12 @@ func fetchFileAttachment(ctx *Context, res *Result, uri *url.URL) error {
ctx.SourceFile = filepath.Join(basepath, path.Clean("/"+uri.Path))
var conv FetchConverter
- if conv, ctx.SourceFile, err = NewFetchConverter(ctx.FetchConverter, ctx.SourceFile); err != nil {
+ var newFilename string
+ if conv, newFilename, err = NewFetchConverter(ctx.FetchConverter, ctx.SourceFile); err != nil {
rhl.Printf("Unable to create converter for file %s: %s", ctx.SourceFile, err)
return err
}
+ ctx.SourceFile = newFilename
if ctx.ProgressCallBack != nil {
if keep := ctx.ProgressCallBack(1, "receiving", 0.0, float64(sizeTotal), ctx.ProgressCallBackData); !keep {