diff options
Diffstat (limited to 'rhimport/fetcher.go')
| -rw-r--r-- | rhimport/fetcher.go | 234 |
1 files changed, 132 insertions, 102 deletions
diff --git a/rhimport/fetcher.go b/rhimport/fetcher.go index ea12806..1a40e71 100644 --- a/rhimport/fetcher.go +++ b/rhimport/fetcher.go @@ -44,6 +44,9 @@ import ( "github.com/andelf/go-curl" ) +//**************************************************************** +//** cURL based importer + type fetcherCurlCBData struct { ctx *Context res *Result @@ -129,6 +132,12 @@ func curlProgressCallback(dltotal, dlnow, ultotal, ulnow float64, userdata inter return true } +//*********************** +//** http:// +//** https:// +//** ftp:// +//** ftps:// + func checkYoutubeDL(ctx *Context, res *Result, uri *url.URL) *youtubeDLInfo { cmd := exec.Command("youtube-dl", "--no-playlist", "-f", "bestaudio/best", "--prefer-free-formats", "-J", ctx.SourceUri) var stderr, stdout bytes.Buffer @@ -273,6 +282,9 @@ func fetchFileCurl(ctx *Context, res *Result, uri *url.URL) (err error) { return } +//*********************** +//** archiv:// + var months = [...]string{ "Jänner", "Februar", @@ -385,34 +397,105 @@ func fetchFileArchiv(ctx *Context, res *Result, uri *url.URL) (err error) { return } -func fetchFileTmp(ctx *Context, res *Result, uri *url.URL) (err error) { - ctx.stdlog.Printf("tmp fetcher called for '%s'", ctx.SourceUri) +//**************************************************************** +//** attachment:// - ctx.SourceFile = filepath.Join(ctx.conf.TempDir, path.Clean("/"+uri.Path)) +func writeAttachmentFile(ctx *Context, res *Result, sizeTotal uint64, conv fetchConverter) error { + written := uint64(0) + for { + select { + case <-ctx.Cancel: + ctx.stdlog.Printf("receiving attachment '%s' got canceled", ctx.SourceFile) + res.ResponseCode = http.StatusNoContent + res.ErrorString = "canceled" + return nil + case chunk, ok := <-ctx.AttachmentChan: + if !ok { + ctx.stdlog.Printf("receiving attachment '%s' failed: channel has been closed prematurely, after %d Bytes", ctx.SourceFile, written) + res.ResponseCode = http.StatusBadRequest + res.ErrorString = fmt.Sprintf("file upload stopped prematurely (after %d Bytes)", written) + return nil + } + if chunk.Error != nil { + ctx.stdlog.Printf("receiving attachment '%s' failed: %v", ctx.SourceFile, chunk.Error) + res.ResponseCode = http.StatusInternalServerError + res.ErrorString = chunk.Error.Error() + return nil + } - size := int64(0) - if info, err := os.Stat(ctx.SourceFile); err != nil { + left := sizeTotal - written + if int(left) < len(chunk.Data) { + ctx.stdlog.Printf("attachment fetcher: truncating %d byes of extra data", len(chunk.Data)-int(left)) + chunk.Data = chunk.Data[0:left] + } + + w, err := conv.Write(chunk.Data) + if err != nil { + ctx.stdlog.Printf("Unable to write to converter(%s): %s", ctx.SourceFile, err) + return err + } + written += uint64(w) + + ctx.reportProgress(1, "receiving", float64(written), float64(sizeTotal)) + if uint64(written) >= sizeTotal { + return nil + } + } + } +} + +func fetchFileAttachment(ctx *Context, res *Result, uri *url.URL) error { + ctx.dbglog.Printf("Attachment fetcher for '%s'", uri.String()) + + sizeTotal, err := strconv.ParseUint(uri.Host, 10, 64) + if err != nil { res.ResponseCode = http.StatusBadRequest - res.ErrorString = fmt.Sprintf("local-file stat(): %s", err) + res.ErrorString = "invalid attachment size (must be a positive integer)" return nil - } else { - size = info.Size() - if info.IsDir() { - res.ResponseCode = http.StatusBadRequest - res.ErrorString = fmt.Sprintf("'%s' is a directory", ctx.SourceFile) + } + if sizeTotal > FILESIZE_MAX { + res.ResponseCode = http.StatusRequestEntityTooLarge + res.ErrorString = "file exceeds maximum file size" + return nil + } + + ctx.SourceFile = filepath.Join(ctx.WorkDir, path.Clean("/"+uri.Path)) + + var conv fetchConverter + ctx.OrigFilename = ctx.SourceFile + if conv, ctx.SourceFile, err = newFetchConverter(ctx, ctx.SourceFile); err != nil { + ctx.stdlog.Printf("Unable to create converter for file %s: %s", ctx.OrigFilename, err) + return err + } + + ctx.reportProgress(1, "receiving", 0.0, float64(sizeTotal)) + err = writeAttachmentFile(ctx, res, sizeTotal, conv) + conv.Close() + if res.ResponseCode == http.StatusNoContent { + ctx.stdlog.Printf("download of '%s' got canceled", ctx.SourceUri) + return nil + } + + ctx.dbglog.Printf("waiting for converter to finish...") + convOut, convErr := conv.GetResult(ctx, res) + if err != nil { + return err + } + if convErr != nil { + if res.ResponseCode == http.StatusNoContent { + ctx.stdlog.Printf("download of '%s' got canceled", ctx.SourceUri) return nil } + ctx.stdlog.Printf("converter error: %v; converter output: %s", convErr, convOut) + return fmt.Errorf("converter error: %v; converter output: %s", convErr, convOut) } - ctx.dbglog.Printf("1: Title = '%s', OrigName = '%s', SourceFile = '%s'", ctx.Title, ctx.OrigFilename, ctx.SourceFile) - ctx.reportProgress(1, "fetching", 0.0, float64(size)) - oldWorkDir := filepath.Dir(ctx.SourceFile) - ctx.dbglog.Printf("switching over to old workdir: %s", oldWorkDir) - ctx.SwitchTempWorkDir(oldWorkDir) - ctx.reportProgress(1, "fetching", float64(size), float64(size)) - ctx.dbglog.Printf("2: Title = '%s', OrigName = '%s', SourceFile = '%s'", ctx.Title, ctx.OrigFilename, ctx.SourceFile) - return + ctx.dbglog.Printf("converter: loudness correction = %.2f dB", ctx.LoudnessCorr) + return nil } +//**************************************************************** +//** io.Reader based importer + func fetchFileConvert(ctx *Context, res *Result, origSrc io.Reader, sizeTotal int64) (err error) { origDir, origFile := path.Split(ctx.SourceFile) @@ -478,6 +561,9 @@ func fetchFileConvert(ctx *Context, res *Result, origSrc io.Reader, sizeTotal in return } +//*********************** +//** local:// + func fetchFileLocal(ctx *Context, res *Result, uri *url.URL) (err error) { ctx.stdlog.Printf("local fetcher called for '%s'", ctx.SourceUri) @@ -513,6 +599,9 @@ func fetchFileLocal(ctx *Context, res *Result, uri *url.URL) (err error) { return fetchFileConvert(ctx, res, src, size) } +//*********************** +//** silence:// + func fetchFileSilence(ctx *Context, res *Result, uri *url.URL) error { ctx.dbglog.Printf("Silence fetcher for '%s'", ctx.SourceUri) @@ -538,99 +627,40 @@ func fetchFileSilence(ctx *Context, res *Result, uri *url.URL) error { return fetchFileConvert(ctx, res, wav, int64(size)) } -func writeAttachmentFile(ctx *Context, res *Result, sizeTotal uint64, conv fetchConverter) error { - written := uint64(0) - for { - select { - case <-ctx.Cancel: - ctx.stdlog.Printf("receiving attachment '%s' got canceled", ctx.SourceFile) - res.ResponseCode = http.StatusNoContent - res.ErrorString = "canceled" - return nil - case chunk, ok := <-ctx.AttachmentChan: - if !ok { - ctx.stdlog.Printf("receiving attachment '%s' failed: channel has been closed prematurely, after %d Bytes", ctx.SourceFile, written) - res.ResponseCode = http.StatusBadRequest - res.ErrorString = fmt.Sprintf("file upload stopped prematurely (after %d Bytes)", written) - return nil - } - if chunk.Error != nil { - ctx.stdlog.Printf("receiving attachment '%s' failed: %v", ctx.SourceFile, chunk.Error) - res.ResponseCode = http.StatusInternalServerError - res.ErrorString = chunk.Error.Error() - return nil - } +//**************************************************************** +//** tmp:// - left := sizeTotal - written - if int(left) < len(chunk.Data) { - ctx.stdlog.Printf("attachment fetcher: truncating %d byes of extra data", len(chunk.Data)-int(left)) - chunk.Data = chunk.Data[0:left] - } - - w, err := conv.Write(chunk.Data) - if err != nil { - ctx.stdlog.Printf("Unable to write to converter(%s): %s", ctx.SourceFile, err) - return err - } - written += uint64(w) - - ctx.reportProgress(1, "receiving", float64(written), float64(sizeTotal)) - if uint64(written) >= sizeTotal { - return nil - } - } - } -} +func fetchFileTmp(ctx *Context, res *Result, uri *url.URL) (err error) { + ctx.stdlog.Printf("tmp fetcher called for '%s'", ctx.SourceUri) -func fetchFileAttachment(ctx *Context, res *Result, uri *url.URL) error { - ctx.dbglog.Printf("Attachment fetcher for '%s'", uri.String()) + ctx.SourceFile = filepath.Join(ctx.conf.TempDir, path.Clean("/"+uri.Path)) - sizeTotal, err := strconv.ParseUint(uri.Host, 10, 64) - if err != nil { + size := int64(0) + if info, err := os.Stat(ctx.SourceFile); err != nil { res.ResponseCode = http.StatusBadRequest - res.ErrorString = "invalid attachment size (must be a positive integer)" - return nil - } - if sizeTotal > FILESIZE_MAX { - res.ResponseCode = http.StatusRequestEntityTooLarge - res.ErrorString = "file exceeds maximum file size" - return nil - } - - ctx.SourceFile = filepath.Join(ctx.WorkDir, path.Clean("/"+uri.Path)) - - var conv fetchConverter - ctx.OrigFilename = ctx.SourceFile - if conv, ctx.SourceFile, err = newFetchConverter(ctx, ctx.SourceFile); err != nil { - ctx.stdlog.Printf("Unable to create converter for file %s: %s", ctx.OrigFilename, err) - return err - } - - ctx.reportProgress(1, "receiving", 0.0, float64(sizeTotal)) - err = writeAttachmentFile(ctx, res, sizeTotal, conv) - conv.Close() - if res.ResponseCode == http.StatusNoContent { - ctx.stdlog.Printf("download of '%s' got canceled", ctx.SourceUri) + res.ErrorString = fmt.Sprintf("local-file stat(): %s", err) return nil - } - - ctx.dbglog.Printf("waiting for converter to finish...") - convOut, convErr := conv.GetResult(ctx, res) - if err != nil { - return err - } - if convErr != nil { - if res.ResponseCode == http.StatusNoContent { - ctx.stdlog.Printf("download of '%s' got canceled", ctx.SourceUri) + } else { + size = info.Size() + if info.IsDir() { + res.ResponseCode = http.StatusBadRequest + res.ErrorString = fmt.Sprintf("'%s' is a directory", ctx.SourceFile) return nil } - ctx.stdlog.Printf("converter error: %v; converter output: %s", convErr, convOut) - return fmt.Errorf("converter error: %v; converter output: %s", convErr, convOut) } - ctx.dbglog.Printf("converter: loudness correction = %.2f dB", ctx.LoudnessCorr) - return nil + ctx.dbglog.Printf("1: Title = '%s', OrigName = '%s', SourceFile = '%s'", ctx.Title, ctx.OrigFilename, ctx.SourceFile) + ctx.reportProgress(1, "fetching", 0.0, float64(size)) + oldWorkDir := filepath.Dir(ctx.SourceFile) + ctx.dbglog.Printf("switching over to old workdir: %s", oldWorkDir) + ctx.SwitchTempWorkDir(oldWorkDir) + ctx.reportProgress(1, "fetching", float64(size), float64(size)) + ctx.dbglog.Printf("2: Title = '%s', OrigName = '%s', SourceFile = '%s'", ctx.Title, ctx.OrigFilename, ctx.SourceFile) + return } +//**************************************************************** +//** global stuff + type fetchFunc func(*Context, *Result, *url.URL) (err error) var ( |