From 9d1a296bb18bd343f133326bd2eb4b2b3368bf65 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Thu, 4 Aug 2016 02:47:07 +0200 Subject: temporary work dir is now handled by session (no support for soure file policy for now) diff --git a/rhimport/core.go b/rhimport/core.go index 6afcd9f..dae4b34 100644 --- a/rhimport/core.go +++ b/rhimport/core.go @@ -160,9 +160,8 @@ type Context struct { ExtraMetaData map[string]string OrigFilename string Title string + WorkDir string SourceFile string - DeleteSourceFile bool - DeleteSourceDir bool SourceFilePolicy FilePolicy LoudnessCorr float64 ProgressCallBack ProgressCB @@ -202,8 +201,6 @@ func NewContext(conf *Config, db *rddb.DB, stdlog, dbglog *log.Logger) *Context ctx.OrigFilename = "" ctx.Title = "" ctx.SourceFile = "" - ctx.DeleteSourceFile = false - ctx.DeleteSourceDir = false ctx.LoudnessCorr = 0.0 ctx.SourceFilePolicy = Auto ctx.ProgressCallBack = nil @@ -267,6 +264,19 @@ func (ctx *Context) SanityCheck() error { return nil } +func (ctx *Context) CreateTempWorkDir() (err error) { + ctx.WorkDir, err = ioutil.TempDir(ctx.conf.TempDir, "rhimport-") + return +} + +func (ctx *Context) RemoveTempWorkDir() { + // check File Policy!!! + if err := os.RemoveAll(ctx.WorkDir); err != nil { + ctx.stdlog.Printf("Error removing WorkDir: %s", err) + } + return +} + func (ctx *Context) getPassword(cached bool) (err error) { ctx.Password, err = ctx.db.GetPassword(ctx.UserName, cached) return diff --git a/rhimport/fetcher.go b/rhimport/fetcher.go index 7a946f6..6c113a4 100644 --- a/rhimport/fetcher.go +++ b/rhimport/fetcher.go @@ -28,7 +28,6 @@ import ( "bytes" "fmt" "io" - "io/ioutil" "log" "mime" "net/http" @@ -48,7 +47,6 @@ import ( type fetcherCurlCBData struct { ctx *Context res *Result - basepath string filename string remotename string conv fetchConverter @@ -64,7 +62,7 @@ func curlHeaderCallback(ptr []byte, userdata interface{}) bool { if strings.HasPrefix(hdr, "Content-Disposition:") { if mediatype, params, err := mime.ParseMediaType(strings.TrimPrefix(hdr, "Content-Disposition:")); err == nil { if mediatype == "attachment" { - data.filename = filepath.Join(data.basepath, path.Clean("/"+params["filename"])) + data.filename = filepath.Join(data.ctx.WorkDir, path.Clean("/"+params["filename"])) } } } @@ -80,7 +78,7 @@ func curlWriteCallback(ptr []byte, userdata interface{}) bool { data.ctx.dbglog.Printf("remotename('%s') is invalid, replacing it with 'unnamed'", data.remotename) name = "unnamed" } - data.filename = filepath.Join(data.basepath, name) + data.filename = filepath.Join(data.ctx.WorkDir, name) } data.ctx.OrigFilename = data.filename conv, newFilename, err := newFetchConverter(data.ctx, data.filename) @@ -203,9 +201,6 @@ func fetchFileCurl(ctx *Context, res *Result, uri *url.URL) (err error) { } cbdata := &fetcherCurlCBData{ctx: ctx, res: res, remotename: path.Base(uri.Path)} - if cbdata.basepath, err = ioutil.TempDir(ctx.conf.TempDir, "rhimportd-"); err != nil { - return - } if info != nil { if info.Title == "" { cbdata.remotename = info.ID + "." + info.Ext @@ -280,10 +275,6 @@ func fetchFileCurl(ctx *Context, res *Result, uri *url.URL) (err error) { ctx.dbglog.Printf("converter: loudness correction = %.2f dB", ctx.LoudnessCorr) ctx.SourceFile = cbdata.filename - if ctx.SourceFilePolicy == Auto { - ctx.DeleteSourceFile = true - ctx.DeleteSourceDir = true - } return } @@ -349,13 +340,8 @@ func fetchFileArchiv(ctx *Context, res *Result, uri *url.URL) (err error) { easy.Setopt(curl.OPT_SSH_PUBLIC_KEYFILE, fmt.Sprintf("%s/.ssh/id_rsa.pub", u.HomeDir)) easy.Setopt(curl.OPT_SSH_PRIVATE_KEYFILE, fmt.Sprintf("%s/.ssh/id_rsa", u.HomeDir)) - var destpath string - if destpath, err = ioutil.TempDir(ctx.conf.TempDir, "rhimportd-"); err != nil { - return - } - cbdata := &fetcherCurlCBData{ctx: ctx, res: res} - cbdata.filename = fmt.Sprintf("%s/%s", destpath, srcfile) + cbdata.filename = filepath.Join(ctx.WorkDir, srcfile) easy.Setopt(curl.OPT_WRITEFUNCTION, curlWriteCallback) easy.Setopt(curl.OPT_WRITEDATA, cbdata) @@ -406,11 +392,6 @@ func fetchFileArchiv(ctx *Context, res *Result, uri *url.URL) (err error) { ctx.dbglog.Printf("converter: loudness correction = %.2f dB", ctx.LoudnessCorr) ctx.SourceFile = cbdata.filename - if ctx.SourceFilePolicy == Auto { - ctx.DeleteSourceFile = true - ctx.DeleteSourceDir = true - } - return } @@ -423,15 +404,11 @@ func fetchFileTmp(ctx *Context, res *Result, uri *url.URL) (err error) { } func fetchFileDirConvert(ctx *Context, res *Result, origSrc *os.File, sizeTotal int64) (err error) { - basepath, err := ioutil.TempDir(ctx.conf.TempDir, "rhimportd-") - if err != nil { - return err - } origDir, origFile := path.Split(ctx.SourceFile) ctx.OrigFilename = ctx.SourceFile var conv fetchConverter - if conv, ctx.SourceFile, err = newFetchConverter(ctx, filepath.Join(basepath, origFile)); err != nil { + if conv, ctx.SourceFile, err = newFetchConverter(ctx, filepath.Join(ctx.WorkDir, origFile)); err != nil { ctx.stdlog.Printf("Unable to create converter for file %s: %s", origDir+origFile, err) return } @@ -485,25 +462,6 @@ func fetchFileDirConvert(ctx *Context, res *Result, origSrc *os.File, sizeTotal if err != nil { return err } - - switch ctx.SourceFilePolicy { - case Auto: - ctx.DeleteSourceFile = true - ctx.DeleteSourceDir = true - case Keep: - ctx.DeleteSourceFile = false - ctx.DeleteSourceDir = false - case Delete: - ctx.DeleteSourceDir = true - ctx.DeleteSourceFile = true - os.Remove(origDir + origFile) - case DeleteWithDir: - ctx.DeleteSourceDir = true - ctx.DeleteSourceFile = true - os.Remove(origDir + origFile) - os.Remove(origDir) - } - return } @@ -543,11 +501,6 @@ func fetchFileDir(ctx *Context, res *Result, uri *url.URL, dir string, convert b if err = fetchFileDirConvert(ctx, res, src, size); err != nil { return } - } else { - if ctx.SourceFilePolicy == Auto { - ctx.DeleteSourceFile = false - ctx.DeleteSourceDir = false - } } ctx.reportProgress(1, "fetching", float64(size), float64(size)) return @@ -662,12 +615,7 @@ func fetchFileAttachment(ctx *Context, res *Result, uri *url.URL) error { return nil } - basepath, err := ioutil.TempDir(ctx.conf.TempDir, "rhimportd-") - if err != nil { - return err - } - - ctx.SourceFile = filepath.Join(basepath, path.Clean("/"+uri.Path)) + ctx.SourceFile = filepath.Join(ctx.WorkDir, path.Clean("/"+uri.Path)) var conv fetchConverter ctx.OrigFilename = ctx.SourceFile @@ -698,11 +646,6 @@ func fetchFileAttachment(ctx *Context, res *Result, uri *url.URL) error { return fmt.Errorf("converter error: %v; converter output: %s", convErr, convOut) } ctx.dbglog.Printf("converter: loudness correction = %.2f dB", ctx.LoudnessCorr) - - if ctx.SourceFilePolicy == Auto { - ctx.DeleteSourceFile = true - ctx.DeleteSourceDir = true - } return nil } @@ -763,6 +706,8 @@ func checkPassword(ctx *Context, res *Result) (err error) { func FetchFile(ctx *Context) (res *Result, err error) { res = &Result{ResponseCode: http.StatusOK} + ctx.stdlog.Println("FetchFile: called for '%s'", ctx.SourceUri) + var uri *url.URL if uri, err = url.Parse(ctx.SourceUri); err != nil { res.ResponseCode = http.StatusBadRequest @@ -783,17 +728,5 @@ func FetchFile(ctx *Context) (res *Result, err error) { res.ErrorString = fmt.Sprintf("No fetcher for uri scheme '%s' found.", uri.Scheme) return } - - switch ctx.SourceFilePolicy { - case Keep: - ctx.DeleteSourceFile = false - ctx.DeleteSourceDir = false - case DeleteWithDir: - ctx.DeleteSourceDir = true - fallthrough - case Delete: - ctx.DeleteSourceFile = true - } - return } diff --git a/rhimport/importer.go b/rhimport/importer.go index 57c525d..69422e2 100644 --- a/rhimport/importer.go +++ b/rhimport/importer.go @@ -30,9 +30,6 @@ import ( "fmt" "mime/multipart" "net/http" - "os" - "path" - "strings" "github.com/andelf/go-curl" ) @@ -402,31 +399,10 @@ func addShowCartCut(ctx *Context, res *Result, carts []uint) (err error) { return } -func cleanupFiles(ctx *Context, res *Result) { - if ctx.DeleteSourceFile { - ctx.dbglog.Printf("importer: removing file: %s", ctx.SourceFile) - if err := os.Remove(ctx.SourceFile); err != nil { - ctx.stdlog.Printf("importer: error removing source file: %s", err) - return - } - if ctx.DeleteSourceDir { - dir := path.Dir(ctx.SourceFile) - ctx.dbglog.Printf("importer: also removing directory: %s", dir) - if err := os.Remove(dir); err != nil { - ctx.stdlog.Printf("importer: error removing source directory: %s", err) - } - } - } else { - res.SourceFile = "tmp://" + strings.TrimPrefix(ctx.SourceFile, ctx.conf.TempDir) - } - return -} - func ImportFile(ctx *Context) (res *Result, err error) { res = &Result{ResponseCode: http.StatusOK} - defer cleanupFiles(ctx, res) - ctx.stdlog.Printf("importer: ImportFile called with: show-id: %d, pool-name: '%s', cart/cut: %d/%d", ctx.ShowId, ctx.GroupName, ctx.Cart, ctx.Cut) + ctx.stdlog.Printf("ImportFile: called with: show-id: %d, pool-name: '%s', cart/cut: %d/%d", ctx.ShowId, ctx.GroupName, ctx.Cart, ctx.Cut) // TODO: on trusted interfaces we should call getPassword again with cached=false after 401's if ctx.Trusted { @@ -480,9 +456,9 @@ func ImportFile(ctx *Context) (res *Result, err error) { if ctx.Cart != 0 && ctx.Cut != 0 { // Import to specific Cut within Cart if err = importAudio(ctx, res); err != nil || res.ResponseCode != http.StatusOK { if err != nil { - ctx.stdlog.Printf("Fileimport has failed (Cart/Cut %d/%d): %s", ctx.Cart, ctx.Cut, err) + ctx.stdlog.Printf("ImportFile: import failed (Cart/Cut %d/%d): %s", ctx.Cart, ctx.Cut, err) } else { - ctx.stdlog.Printf("Fileimport has failed (Cart/Cut %d/%d): %s", res.Cart, res.Cut, res.ErrorString) + ctx.stdlog.Printf("ImportFile: import failed (Cart/Cut %d/%d): %s", res.Cart, res.Cut, res.ErrorString) } // Try to clean up after failed import if rmCartOnErr { @@ -498,11 +474,11 @@ func ImportFile(ctx *Context) (res *Result, err error) { if err := ctx.updateCutCartTitle(); err != nil { ctx.stdlog.Printf("Warning: error while updating Cart/Cut Title: %v", err) } - ctx.stdlog.Printf("File got succesfully imported into Cart/Cut %d/%d", res.Cart, res.Cut) + ctx.stdlog.Printf("ImportFile: succesfully imported into Cart/Cut %d/%d", res.Cart, res.Cut) } } else { res.ResponseCode = http.StatusBadRequest - res.ErrorString = "importer: The request doesn't contain enough information to be processed" + res.ErrorString = "ImportFile: The request doesn't contain enough information to be processed" } return diff --git a/rhimport/normalizer.go b/rhimport/normalizer.go index e883d97..9687ced 100644 --- a/rhimport/normalizer.go +++ b/rhimport/normalizer.go @@ -30,7 +30,6 @@ import ( "net/http" "os" "os/exec" - "path" "path/filepath" "strings" ) @@ -41,10 +40,6 @@ type ffmpegResult struct { } func runNormalizer(ctx *Context, res *Result, src *os.File, size int64) (err error) { - if ctx.DeleteSourceFile { - defer os.Remove(src.Name()) - } - ctx.reportProgress(2, "normalizing", 0.0, float64(size)) basepath, filename := filepath.Split(src.Name()) @@ -119,12 +114,6 @@ func NormalizeFile(ctx *Context) (res *Result, err error) { if err = runNormalizer(ctx, res, src, size); err != nil { ctx.stdlog.Println("NormalizeFile error:", err) - if ctx.DeleteSourceFile { - os.Remove(ctx.SourceFile) - if ctx.DeleteSourceDir { - os.Remove(path.Dir(ctx.SourceFile)) - } - } return } return diff --git a/rhimport/session.go b/rhimport/session.go index 39c02a4..993575a 100644 --- a/rhimport/session.go +++ b/rhimport/session.go @@ -207,6 +207,26 @@ func (s *session) attachUploader() (resp attachUploaderResponse) { return } +// func cleanupFiles(ctx *Context, res *Result) { +// if ctx.DeleteSourceFile { +// ctx.dbglog.Printf("importer: removing file: %s", ctx.SourceFile) +// if err := os.Remove(ctx.SourceFile); err != nil { +// ctx.stdlog.Printf("importer: error removing source file: %s", err) +// return +// } +// if ctx.DeleteSourceDir { +// dir := path.Dir(ctx.SourceFile) +// ctx.dbglog.Printf("importer: also removing directory: %s", dir) +// if err := os.Remove(dir); err != nil { +// ctx.stdlog.Printf("importer: error removing source directory: %s", err) +// } +// } +// } else { +// res.SourceFile = "tmp://" + strings.TrimPrefix(ctx.SourceFile, ctx.conf.TempDir) +// } +// return +// } + func (s *session) dispatchRequests() { defer func() { if s.cancelUploader != nil { @@ -385,14 +405,19 @@ func (s *session) cleanup() { // close(s.addProgressChan) // close(s.addDoneChan) // close(s.attachUploader) + s.ctx.RemoveTempWorkDir() s.ctx.dbglog.Printf("Session: cleanup is now done") } -func newSession(ctx *Context, removeFunc func()) (s *session) { +func newSession(ctx *Context, removeFunc func()) (s *session, err error) { s = &session{} s.state = _SESSION_NEW s.removeFunc = removeFunc s.ctx = *ctx + if err = s.ctx.CreateTempWorkDir(); err != nil { + return + } + s.quit = make(chan bool, 1) s.done = make(chan bool) s.timer = time.NewTimer(10 * time.Second) diff --git a/rhimport/session_store.go b/rhimport/session_store.go index a9b3e81..181ee9f 100644 --- a/rhimport/session_store.go +++ b/rhimport/session_store.go @@ -188,8 +188,14 @@ func (store *sessionStore) new(ctx *Context, refId string) (resp newSessionRespo if pref := ctx.dbglog.Prefix(); strings.Contains(pref, "%s") { ctx.dbglog.SetPrefix(fmt.Sprintf(pref, resp.id)) } - s := &sessionStoreSessionElement{newSession(ctx, func() { store.GetInterface().Remove(ctx.UserName, resp.id) }), refId} - store.store[ctx.UserName].sessions[resp.id] = s + + s, err := newSession(ctx, func() { store.GetInterface().Remove(ctx.UserName, resp.id) }) + if err != nil { + resp.responsecode = http.StatusInternalServerError + resp.errorstring = err.Error() + return + } + store.store[ctx.UserName].sessions[resp.id] = &sessionStoreSessionElement{s, refId} resp.session = store.store[ctx.UserName].sessions[resp.id].s.getInterface() store.dbglog.Printf("SessionStore: created session for '%s' -> %s", ctx.UserName, resp.id) store.store[ctx.UserName].callUpdateHandlerAdd(resp.id, refId) -- cgit v0.10.2