fetch converter for local:// - needs testing...

1 files changed, 85 insertions, 10 deletions

diff --git a/rhimport/fetcher.go b/rhimport/fetcher.go
index 061d0e1..1abf736 100644
--- a/rhimport/fetcher.go
+++ b/rhimport/fetcher.go
@@ -26,6 +26,7 @@ package rhimport
 
 import (
 	"fmt"
+	"io"
 	"io/ioutil"
 	"mime"
 	"net/http"
@@ -86,7 +87,7 @@ func curlWriteCallback(ptr []byte, userdata interface{}) bool {
 		data.conv = conv
 	}
 	if _, err := data.conv.Write(ptr); err != nil {
-		rhl.Printf("Unable to write file %s: %s", data.filename, err)
+		rhl.Printf("Unable to write to converter(%s): %s", data.filename, err)
 		data.writeError = err
 		return false
 	}
@@ -308,14 +309,79 @@ func fetchFileArchiv(ctx *Context, res *Result, uri *url.URL) (err error) {
 }
 
 func fetchFileLocal(ctx *Context, res *Result, uri *url.URL) (err error) {
-	return fetchFileDir(ctx, res, uri, ctx.conf.LocalFetchDir)
+	return fetchFileDir(ctx, res, uri, ctx.conf.LocalFetchDir, true)
 }
 
 func fetchFileTmp(ctx *Context, res *Result, uri *url.URL) (err error) {
-	return fetchFileDir(ctx, res, uri, ctx.conf.TempDir)
+	return fetchFileDir(ctx, res, uri, ctx.conf.TempDir, false)
 }
 
-func fetchFileDir(ctx *Context, res *Result, uri *url.URL, dir string) (err error) {
+func fetchFileDirConvert(ctx *Context, res *Result, origSrc *os.File, sizeTotal int64) (err error) {
+	var conv FetchConverter
+	if conv, ctx.SourceFile, err = NewFetchConverter(ctx.FetchConverter, ctx.SourceFile); err != nil {
+		rhl.Printf("Unable to create converter for file %s: %s", origSrc.Name(), err)
+		return
+	}
+
+	var buffer [1 * 1024 * 1024]byte
+	written := uint64(0)
+	for {
+		var r, w int
+
+		r, err = origSrc.Read(buffer[:])
+		if err == io.EOF {
+			err = nil
+			break
+		}
+		if err != nil {
+			rhl.Printf("Unable to read from source file %s: %s", origSrc.Name(), err)
+			break
+		}
+		w, err = conv.Write(buffer[0:r])
+		if err != nil {
+			rhl.Printf("Unable to write to converter(%s): %s", ctx.SourceFile, err)
+			break
+		}
+		written += uint64(w)
+
+		if ctx.ProgressCallBack != nil {
+			if keep := ctx.ProgressCallBack(1, "fetching", float64(written), float64(sizeTotal), ctx.ProgressCallBackData); !keep {
+				ctx.ProgressCallBack = nil
+			}
+		}
+	}
+
+	conv.Close()
+	rhl.Printf("waiting for converter to finish...")
+	if convOut, convErr := conv.GetResult(); convErr != nil {
+		if convOut != "" {
+			rhl.Printf("converter error: %v; converter output: %s", convErr, convOut)
+		}
+		return fmt.Errorf("converter error: %v; converter output: %s", convErr, convOut)
+	}
+	if err != nil {
+		return err
+	}
+
+	// TODO: delete origSrc file when SourceFilePolicy == delete or DeleteWithDir
+	switch ctx.SourceFilePolicy {
+	case Auto:
+		ctx.DeleteSourceFile = true
+		ctx.DeleteSourceDir = true
+	case Keep:
+		ctx.DeleteSourceFile = false
+		ctx.DeleteSourceDir = false
+	case DeleteWithDir:
+		ctx.DeleteSourceDir = true
+		fallthrough
+	case Delete:
+		ctx.DeleteSourceFile = true
+	}
+
+	return
+}
+
+func fetchFileDir(ctx *Context, res *Result, uri *url.URL, dir string, convert bool) (err error) {
 	rhl.Printf("Dir fetcher called for '%s'", ctx.SourceUri)
 
 	ctx.SourceFile = filepath.Join(dir, path.Clean("/"+uri.Path))
@@ -352,15 +418,22 @@ func fetchFileDir(ctx *Context, res *Result, uri *url.URL, dir string) (err erro
 		}
 	}
 
+	if convert {
+		if err = fetchFileDirConvert(ctx, res, src, size); err != nil {
+			return
+		}
+	} else {
+		if ctx.SourceFilePolicy == Auto {
+			ctx.DeleteSourceFile = false
+			ctx.DeleteSourceDir = false
+		}
+	}
+
 	if ctx.ProgressCallBack != nil {
 		if keep := ctx.ProgressCallBack(1, "fetching", float64(size), float64(size), ctx.ProgressCallBackData); !keep {
 			ctx.ProgressCallBack = nil
 		}
 	}
-	if ctx.SourceFilePolicy == Auto {
-		ctx.DeleteSourceFile = false
-		ctx.DeleteSourceDir = false
-	}
 	return
 }
 
@@ -431,7 +504,7 @@ func writeAttachmentFile(ctx *Context, res *Result, sizeTotal uint64, conv Fetch
 
 			w, err := conv.Write(data)
 			if err != nil {
-				rhl.Printf("Unable to write file %s: %s", ctx.SourceFile, err)
+				rhl.Printf("Unable to write to converter(%s): %s", ctx.SourceFile, err)
 				return err
 			}
 			written += uint64(w)
@@ -476,10 +549,12 @@ func fetchFileAttachment(ctx *Context, res *Result, uri *url.URL) error {
 	ctx.SourceFile = filepath.Join(basepath, path.Clean("/"+uri.Path))
 
 	var conv FetchConverter
-	if conv, ctx.SourceFile, err = NewFetchConverter(ctx.FetchConverter, ctx.SourceFile); err != nil {
+	var newFilename string
+	if conv, newFilename, err = NewFetchConverter(ctx.FetchConverter, ctx.SourceFile); err != nil {
 		rhl.Printf("Unable to create converter for file %s: %s", ctx.SourceFile, err)
 		return err
 	}
+	ctx.SourceFile = newFilename
 
 	if ctx.ProgressCallBack != nil {
 		if keep := ctx.ProgressCallBack(1, "receiving", 0.0, float64(sizeTotal), ctx.ProgressCallBackData); !keep {