diff options
Diffstat (limited to 'rhimport')
| -rw-r--r-- | rhimport/fetcher.go | 95 |
1 files changed, 85 insertions, 10 deletions
diff --git a/rhimport/fetcher.go b/rhimport/fetcher.go index 061d0e1..1abf736 100644 --- a/rhimport/fetcher.go +++ b/rhimport/fetcher.go @@ -26,6 +26,7 @@ package rhimport import ( "fmt" + "io" "io/ioutil" "mime" "net/http" @@ -86,7 +87,7 @@ func curlWriteCallback(ptr []byte, userdata interface{}) bool { data.conv = conv } if _, err := data.conv.Write(ptr); err != nil { - rhl.Printf("Unable to write file %s: %s", data.filename, err) + rhl.Printf("Unable to write to converter(%s): %s", data.filename, err) data.writeError = err return false } @@ -308,14 +309,79 @@ func fetchFileArchiv(ctx *Context, res *Result, uri *url.URL) (err error) { } func fetchFileLocal(ctx *Context, res *Result, uri *url.URL) (err error) { - return fetchFileDir(ctx, res, uri, ctx.conf.LocalFetchDir) + return fetchFileDir(ctx, res, uri, ctx.conf.LocalFetchDir, true) } func fetchFileTmp(ctx *Context, res *Result, uri *url.URL) (err error) { - return fetchFileDir(ctx, res, uri, ctx.conf.TempDir) + return fetchFileDir(ctx, res, uri, ctx.conf.TempDir, false) } -func fetchFileDir(ctx *Context, res *Result, uri *url.URL, dir string) (err error) { +func fetchFileDirConvert(ctx *Context, res *Result, origSrc *os.File, sizeTotal int64) (err error) { + var conv FetchConverter + if conv, ctx.SourceFile, err = NewFetchConverter(ctx.FetchConverter, ctx.SourceFile); err != nil { + rhl.Printf("Unable to create converter for file %s: %s", origSrc.Name(), err) + return + } + + var buffer [1 * 1024 * 1024]byte + written := uint64(0) + for { + var r, w int + + r, err = origSrc.Read(buffer[:]) + if err == io.EOF { + err = nil + break + } + if err != nil { + rhl.Printf("Unable to read from source file %s: %s", origSrc.Name(), err) + break + } + w, err = conv.Write(buffer[0:r]) + if err != nil { + rhl.Printf("Unable to write to converter(%s): %s", ctx.SourceFile, err) + break + } + written += uint64(w) + + if ctx.ProgressCallBack != nil { + if keep := ctx.ProgressCallBack(1, "fetching", float64(written), float64(sizeTotal), ctx.ProgressCallBackData); !keep { + ctx.ProgressCallBack = nil + } + } + } + + conv.Close() + rhl.Printf("waiting for converter to finish...") + if convOut, convErr := conv.GetResult(); convErr != nil { + if convOut != "" { + rhl.Printf("converter error: %v; converter output: %s", convErr, convOut) + } + return fmt.Errorf("converter error: %v; converter output: %s", convErr, convOut) + } + if err != nil { + return err + } + + // TODO: delete origSrc file when SourceFilePolicy == delete or DeleteWithDir + switch ctx.SourceFilePolicy { + case Auto: + ctx.DeleteSourceFile = true + ctx.DeleteSourceDir = true + case Keep: + ctx.DeleteSourceFile = false + ctx.DeleteSourceDir = false + case DeleteWithDir: + ctx.DeleteSourceDir = true + fallthrough + case Delete: + ctx.DeleteSourceFile = true + } + + return +} + +func fetchFileDir(ctx *Context, res *Result, uri *url.URL, dir string, convert bool) (err error) { rhl.Printf("Dir fetcher called for '%s'", ctx.SourceUri) ctx.SourceFile = filepath.Join(dir, path.Clean("/"+uri.Path)) @@ -352,15 +418,22 @@ func fetchFileDir(ctx *Context, res *Result, uri *url.URL, dir string) (err erro } } + if convert { + if err = fetchFileDirConvert(ctx, res, src, size); err != nil { + return + } + } else { + if ctx.SourceFilePolicy == Auto { + ctx.DeleteSourceFile = false + ctx.DeleteSourceDir = false + } + } + if ctx.ProgressCallBack != nil { if keep := ctx.ProgressCallBack(1, "fetching", float64(size), float64(size), ctx.ProgressCallBackData); !keep { ctx.ProgressCallBack = nil } } - if ctx.SourceFilePolicy == Auto { - ctx.DeleteSourceFile = false - ctx.DeleteSourceDir = false - } return } @@ -431,7 +504,7 @@ func writeAttachmentFile(ctx *Context, res *Result, sizeTotal uint64, conv Fetch w, err := conv.Write(data) if err != nil { - rhl.Printf("Unable to write file %s: %s", ctx.SourceFile, err) + rhl.Printf("Unable to write to converter(%s): %s", ctx.SourceFile, err) return err } written += uint64(w) @@ -476,10 +549,12 @@ func fetchFileAttachment(ctx *Context, res *Result, uri *url.URL) error { ctx.SourceFile = filepath.Join(basepath, path.Clean("/"+uri.Path)) var conv FetchConverter - if conv, ctx.SourceFile, err = NewFetchConverter(ctx.FetchConverter, ctx.SourceFile); err != nil { + var newFilename string + if conv, newFilename, err = NewFetchConverter(ctx.FetchConverter, ctx.SourceFile); err != nil { rhl.Printf("Unable to create converter for file %s: %s", ctx.SourceFile, err) return err } + ctx.SourceFile = newFilename if ctx.ProgressCallBack != nil { if keep := ctx.ProgressCallBack(1, "receiving", 0.0, float64(sizeTotal), ctx.ProgressCallBackData); !keep { |