diff options
Diffstat (limited to 'rhimport/fetcher.go')
-rw-r--r-- | rhimport/fetcher.go | 107 |
1 files changed, 54 insertions, 53 deletions
diff --git a/rhimport/fetcher.go b/rhimport/fetcher.go index 536dbe1..2855ea0 100644 --- a/rhimport/fetcher.go +++ b/rhimport/fetcher.go @@ -29,6 +29,7 @@ import ( "fmt" "io" "io/ioutil" + "log" "mime" "net/http" "net/url" @@ -77,7 +78,7 @@ func curlWriteCallback(ptr []byte, userdata interface{}) bool { if data.filename == "" { name := path.Clean("/" + data.remotename) if name == "/" { - rhdl.Printf("remotename('%s') is invalid, replacing it with 'unnamed'", data.remotename) + data.ctx.dbglog.Printf("remotename('%s') is invalid, replacing it with 'unnamed'", data.remotename) name = "unnamed" } data.filename = filepath.Join(data.basepath, name) @@ -85,7 +86,7 @@ func curlWriteCallback(ptr []byte, userdata interface{}) bool { data.ctx.OrigFilename = data.filename conv, newFilename, err := NewFetchConverter(data.ctx, data.filename, data.metadata) if err != nil { - rhl.Printf("Unable to create converter for file %s: %s", data.filename, err) + data.ctx.stdlog.Printf("Unable to create converter for file %s: %s", data.filename, err) data.writeError = err return false } @@ -94,7 +95,7 @@ func curlWriteCallback(ptr []byte, userdata interface{}) bool { } w, err := data.conv.Write(ptr) if err != nil { - rhl.Printf("Unable to write to converter(%s): %s", data.filename, err) + data.ctx.stdlog.Printf("Unable to write to converter(%s): %s", data.filename, err) data.writeError = err return false } @@ -140,22 +141,22 @@ func checkYoutubeDL(ctx *Context, res *Result, uri *url.URL) *YoutubeDLInfo { cmd.Env = append(os.Environ(), "CBA_API_KEY="+cba_api_key) } - rhl.Printf("running youtube-dl for '%s'", ctx.SourceUri) + ctx.stdlog.Printf("running youtube-dl for '%s'", ctx.SourceUri) done := make(chan *YoutubeDLInfo) go func() { defer func() { done <- nil }() if err := cmd.Run(); err != nil { - rhdl.Printf("youtube-dl: %v, stderr: %s", err, strings.TrimSpace(stderr.String())) + ctx.dbglog.Printf("youtube-dl: %v, stderr: %s", err, strings.TrimSpace(stderr.String())) return } info, err := NewYoutubeDLInfoFromJSON(&stdout) if err != nil { - rhdl.Printf("youtube-dl: %v, stderr: %s", err, strings.TrimSpace(stderr.String())) + ctx.dbglog.Printf("youtube-dl: %v, stderr: %s", err, strings.TrimSpace(stderr.String())) return } - rhl.Printf("youtube-dl: extractor: %s -> %s", info.Extractor, info.URL) + ctx.dbglog.Printf("youtube-dl: extractor: %s -> %s", info.Extractor, info.URL) ctx.SourceUri = info.URL done <- info }() @@ -172,11 +173,11 @@ func checkYoutubeDL(ctx *Context, res *Result, uri *url.URL) *YoutubeDLInfo { } func fetchFileCurl(ctx *Context, res *Result, uri *url.URL) (err error) { - rhl.Printf("curl-based fetcher called for '%s'", ctx.SourceUri) + ctx.stdlog.Printf("curl-based fetcher called for '%s'", ctx.SourceUri) info := checkYoutubeDL(ctx, res, uri) if res.ResponseCode == http.StatusNoContent { - rhl.Printf("download of '%s' got canceled", ctx.SourceUri) + ctx.stdlog.Printf("download of '%s' got canceled", ctx.SourceUri) return nil } @@ -198,7 +199,7 @@ func fetchFileCurl(ctx *Context, res *Result, uri *url.URL) (err error) { h = append(h, key+": "+value) } easy.Setopt(curl.OPT_HTTPHEADER, h) - rhdl.Printf("added HTTP header: %q", h) + ctx.dbglog.Printf("added HTTP header: %q", h) } } @@ -249,17 +250,17 @@ func fetchFileCurl(ctx *Context, res *Result, uri *url.URL) (err error) { var convErr error if cbdata.conv != nil { cbdata.conv.Close() - rhl.Printf("waiting for converter to finish...") + ctx.stdlog.Printf("waiting for converter to finish...") convOut, convErr = cbdata.conv.GetResult(ctx, res) } if err != nil || cbdata.writeError != nil || convErr != nil { if cbdata.conv != nil { - rhdl.Printf("Removing stale file: %s", cbdata.filename) + ctx.dbglog.Printf("Removing stale file: %s", cbdata.filename) os.Remove(cbdata.filename) os.Remove(path.Dir(cbdata.filename)) } if res.ResponseCode == http.StatusNoContent { - rhl.Printf("download of '%s' got canceled", ctx.SourceUri) + ctx.stdlog.Printf("download of '%s' got canceled", ctx.SourceUri) return nil } if statusCode > 0 && statusCode != http.StatusOK { @@ -272,13 +273,13 @@ func fetchFileCurl(ctx *Context, res *Result, uri *url.URL) (err error) { } if convErr != nil { err = fmt.Errorf("converter error: %v; converter output: %s", convErr, convOut) - rhl.Printf("%v", err) + ctx.stdlog.Printf("%v", err) } err = fmt.Errorf("curl-fetcher('%s'): %s", ctx.SourceUri, err) - rhl.Println(err) + ctx.stdlog.Println(err) return } - rhdl.Printf("converter: loudness correction = %.2f dB", ctx.LoudnessCorr) + ctx.dbglog.Printf("converter: loudness correction = %.2f dB", ctx.LoudnessCorr) ctx.SourceFile = cbdata.filename if ctx.SourceFilePolicy == Auto { @@ -324,7 +325,7 @@ func generateArchivFilePath(uri *url.URL) (file, path string, t time.Time, err e } func fetchFileArchiv(ctx *Context, res *Result, uri *url.URL) (err error) { - rhdl.Printf("archiv fetcher called for '%s'", ctx.SourceUri) + ctx.dbglog.Printf("archiv fetcher called for '%s'", ctx.SourceUri) var srcfile, srcpath string var start time.Time @@ -374,24 +375,24 @@ func fetchFileArchiv(ctx *Context, res *Result, uri *url.URL) (err error) { cbdata.metadata["ORGANIZATION"] = "Radio Helsinki" cbdata.metadata["DATE"] = start.Format("2.1.2006") - rhdl.Printf("importing archiv file from %s", scpuri) + ctx.dbglog.Printf("importing archiv file from %s", scpuri) err = easy.Perform() var convOut string var convErr error if cbdata.conv != nil { cbdata.conv.Close() - rhl.Printf("waiting for converter to finish...") + ctx.stdlog.Printf("waiting for converter to finish...") convOut, convErr = cbdata.conv.GetResult(ctx, res) } if err != nil || cbdata.writeError != nil || convErr != nil { if cbdata.conv != nil { - rhdl.Printf("Removing stale file: %s", cbdata.filename) + ctx.dbglog.Printf("Removing stale file: %s", cbdata.filename) os.Remove(cbdata.filename) os.Remove(path.Dir(cbdata.filename)) } if res.ResponseCode == http.StatusNoContent { - rhl.Printf("download of '%s' got canceled", ctx.SourceUri) + ctx.stdlog.Printf("download of '%s' got canceled", ctx.SourceUri) return nil } if cbdata.writeError != nil { @@ -399,13 +400,13 @@ func fetchFileArchiv(ctx *Context, res *Result, uri *url.URL) (err error) { } if convErr != nil { err = fmt.Errorf("converter error: %v; converter output: %s", convErr, convOut) - rhl.Printf("%v", err) + ctx.stdlog.Printf("%v", err) } err = fmt.Errorf("archiv-fetcher('%s'): %s", ctx.SourceUri, err) - rhl.Println(err) + ctx.stdlog.Println(err) return } - rhdl.Printf("converter: loudness correction = %.2f dB", ctx.LoudnessCorr) + ctx.dbglog.Printf("converter: loudness correction = %.2f dB", ctx.LoudnessCorr) ctx.SourceFile = cbdata.filename if ctx.SourceFilePolicy == Auto { @@ -434,7 +435,7 @@ func fetchFileDirConvert(ctx *Context, res *Result, origSrc *os.File, sizeTotal ctx.OrigFilename = ctx.SourceFile var conv FetchConverter if conv, ctx.SourceFile, err = NewFetchConverter(ctx, filepath.Join(basepath, origFile), nil); err != nil { - rhl.Printf("Unable to create converter for file %s: %s", origDir+origFile, err) + ctx.stdlog.Printf("Unable to create converter for file %s: %s", origDir+origFile, err) return } @@ -449,12 +450,12 @@ func fetchFileDirConvert(ctx *Context, res *Result, origSrc *os.File, sizeTotal break } if err != nil { - rhl.Printf("Unable to read from source file %s: %s", origDir+origFile, err) + ctx.stdlog.Printf("Unable to read from source file %s: %s", origDir+origFile, err) break } w, err = conv.Write(buffer[0:r]) if err != nil { - rhl.Printf("Unable to write to converter(%s): %s", ctx.SourceFile, err) + ctx.stdlog.Printf("Unable to write to converter(%s): %s", ctx.SourceFile, err) break } written += uint64(w) @@ -469,21 +470,21 @@ func fetchFileDirConvert(ctx *Context, res *Result, origSrc *os.File, sizeTotal conv.Close() if res.ResponseCode == http.StatusNoContent { - rhl.Printf("converting of '%s' got canceled", ctx.SourceUri) + ctx.stdlog.Printf("converting of '%s' got canceled", ctx.SourceUri) return nil } - rhl.Printf("waiting for converter to finish...") + ctx.stdlog.Printf("waiting for converter to finish...") convOut, convErr := conv.GetResult(ctx, res) if convErr != nil { if res.ResponseCode == http.StatusNoContent { - rhl.Printf("converting of '%s' got canceled", ctx.SourceUri) + ctx.stdlog.Printf("converting of '%s' got canceled", ctx.SourceUri) return nil } - rhl.Printf("converter error: %v; converter output: %s", convErr, convOut) + ctx.stdlog.Printf("converter error: %v; converter output: %s", convErr, convOut) return fmt.Errorf("converter error: %v; converter output: %s", convErr, convOut) } - rhdl.Printf("converter: loudness correction = %.2f dB", ctx.LoudnessCorr) + ctx.dbglog.Printf("converter: loudness correction = %.2f dB", ctx.LoudnessCorr) if err != nil { return err } @@ -510,7 +511,7 @@ func fetchFileDirConvert(ctx *Context, res *Result, origSrc *os.File, sizeTotal } func fetchFileDir(ctx *Context, res *Result, uri *url.URL, dir string, convert bool) (err error) { - rhl.Printf("Dir fetcher called for '%s'", ctx.SourceUri) + ctx.stdlog.Printf("Dir fetcher called for '%s'", ctx.SourceUri) ctx.SourceFile = filepath.Join(dir, path.Clean("/"+uri.Path)) var src *os.File @@ -556,7 +557,7 @@ func fetchFileDir(ctx *Context, res *Result, uri *url.URL, dir string, convert b } func fetchFileFake(ctx *Context, res *Result, uri *url.URL) error { - rhdl.Printf("Fake fetcher for '%s'", ctx.SourceUri) + ctx.dbglog.Printf("Fake fetcher for '%s'", ctx.SourceUri) duration, err := strconv.ParseUint(uri.Host, 10, 32) if err != nil { @@ -567,7 +568,7 @@ func fetchFileFake(ctx *Context, res *Result, uri *url.URL) error { for i := uint(0); i < uint(duration); i++ { if ctx.isCanceled() { - rhl.Printf("faking got canceled") + ctx.stdlog.Printf("faking got canceled") res.ResponseCode = http.StatusNoContent res.ErrorString = "canceled" return nil @@ -592,19 +593,19 @@ func writeAttachmentFile(ctx *Context, res *Result, sizeTotal uint64, conv Fetch for { select { case <-ctx.Cancel: - rhl.Printf("receiving attachment '%s' got canceled", ctx.SourceFile) + ctx.stdlog.Printf("receiving attachment '%s' got canceled", ctx.SourceFile) res.ResponseCode = http.StatusNoContent res.ErrorString = "canceled" return nil case chunk, ok := <-ctx.AttachmentChan: if !ok { - rhl.Printf("receiving attachment '%s' failed: channel has been closed prematurely, after %d Bytes", ctx.SourceFile, written) + ctx.stdlog.Printf("receiving attachment '%s' failed: channel has been closed prematurely, after %d Bytes", ctx.SourceFile, written) res.ResponseCode = http.StatusBadRequest res.ErrorString = fmt.Sprintf("file upload stopped prematurely (after %d Bytes)", written) return nil } if chunk.Error != nil { - rhl.Printf("receiving attachment '%s' failed: %v", ctx.SourceFile, chunk.Error) + ctx.stdlog.Printf("receiving attachment '%s' failed: %v", ctx.SourceFile, chunk.Error) res.ResponseCode = http.StatusInternalServerError res.ErrorString = chunk.Error.Error() return nil @@ -612,13 +613,13 @@ func writeAttachmentFile(ctx *Context, res *Result, sizeTotal uint64, conv Fetch left := sizeTotal - written if int(left) < len(chunk.Data) { - rhl.Printf("attachment fetcher: truncating %d byes of extra data", len(chunk.Data)-int(left)) + ctx.stdlog.Printf("attachment fetcher: truncating %d byes of extra data", len(chunk.Data)-int(left)) chunk.Data = chunk.Data[0:left] } w, err := conv.Write(chunk.Data) if err != nil { - rhl.Printf("Unable to write to converter(%s): %s", ctx.SourceFile, err) + ctx.stdlog.Printf("Unable to write to converter(%s): %s", ctx.SourceFile, err) return err } written += uint64(w) @@ -632,7 +633,7 @@ func writeAttachmentFile(ctx *Context, res *Result, sizeTotal uint64, conv Fetch } func fetchFileAttachment(ctx *Context, res *Result, uri *url.URL) error { - rhdl.Printf("Attachment fetcher for '%s'", ctx.SourceUri) + ctx.dbglog.Printf("Attachment fetcher for '%s'", ctx.SourceUri) if ctx.AttachmentChan == nil { return fmt.Errorf("attachement channel is nil") @@ -660,7 +661,7 @@ func fetchFileAttachment(ctx *Context, res *Result, uri *url.URL) error { var conv FetchConverter ctx.OrigFilename = ctx.SourceFile if conv, ctx.SourceFile, err = NewFetchConverter(ctx, ctx.SourceFile, nil); err != nil { - rhl.Printf("Unable to create converter for file %s: %s", ctx.OrigFilename, err) + ctx.stdlog.Printf("Unable to create converter for file %s: %s", ctx.OrigFilename, err) return err } @@ -668,24 +669,24 @@ func fetchFileAttachment(ctx *Context, res *Result, uri *url.URL) error { err = writeAttachmentFile(ctx, res, sizeTotal, conv) conv.Close() if res.ResponseCode == http.StatusNoContent { - rhl.Printf("download of '%s' got canceled", ctx.SourceUri) + ctx.stdlog.Printf("download of '%s' got canceled", ctx.SourceUri) return nil } - rhl.Printf("waiting for converter to finish...") + ctx.stdlog.Printf("waiting for converter to finish...") convOut, convErr := conv.GetResult(ctx, res) if err != nil { return err } if convErr != nil { if res.ResponseCode == http.StatusNoContent { - rhl.Printf("download of '%s' got canceled", ctx.SourceUri) + ctx.stdlog.Printf("download of '%s' got canceled", ctx.SourceUri) return nil } - rhl.Printf("converter error: %v; converter output: %s", convErr, convOut) + ctx.stdlog.Printf("converter error: %v; converter output: %s", convErr, convOut) return fmt.Errorf("converter error: %v; converter output: %s", convErr, convOut) } - rhdl.Printf("converter: loudness correction = %.2f dB", ctx.LoudnessCorr) + ctx.dbglog.Printf("converter: loudness correction = %.2f dB", ctx.LoudnessCorr) if ctx.SourceFilePolicy == Auto { ctx.DeleteSourceFile = true @@ -709,30 +710,30 @@ var ( } ) -func fetcherInit() { +func fetcherInit(stdlog, dbglog *log.Logger) { archiveEnabled := false info := curl.VersionInfo(curl.VERSION_FIRST) protos := info.Protocols for _, proto := range protos { if proto == "sftp" { - rhdl.Printf("curl: * enabling protocol %s", proto) + dbglog.Printf("curl: * enabling protocol %s", proto) fetchers["archiv"] = fetchFileArchiv archiveEnabled = true } else if _, ok := curlProtos[proto]; ok { - rhdl.Printf("curl: * enabling protocol %s", proto) + dbglog.Printf("curl: * enabling protocol %s", proto) fetchers[proto] = fetchFileCurl curlProtos[proto] = true } else { - rhdl.Printf("curl: ignoring protocol %s", proto) + dbglog.Printf("curl: ignoring protocol %s", proto) } } for proto, enabled := range curlProtos { if !enabled { - rhl.Printf("curl: protocol %s is disabled because the installed library version doesn't support it!", proto) + stdlog.Printf("curl: protocol %s is disabled because the installed library version doesn't support it!", proto) } } if !archiveEnabled { - rhl.Printf("archiv: fetcher is disabled because the installed curl library version doesn't support sFTP!") + stdlog.Printf("archiv: fetcher is disabled because the installed curl library version doesn't support sFTP!") } } |